def _getOutputs():
repoLocation = ''
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine( ignoreErrors = False )
repoLocation = clip.repo
if not repoLocation:
Script.showHelp()
dexit(1)
from DIRAC import gLogger
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac(True, repoLocation)
exitCode = 0
res = dirac.monitorRepository(False)
if not res['OK']:
gLogger.error("Failed because %s" % res['Message'])
dexit(1)
res = dirac.retrieveRepositorySandboxes()
if not res['OK']:
gLogger.error("Failed because %s" % res['Message'])
dexit(1)
if clip.outputdata:
res = dirac.retrieveRepositoryData()
if not res['OK']:
gLogger.error("Failed because %s" % res['Message'])
exit(1)
dexit(exitCode)
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if args:
subLogger.error( "Found the following positional args '%s', but we only accept switches" % args )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
switches = dict( Script.getUnprocessedSwitches() )
for key in ( 'status', 'se','limit'):
if not key in switches:
print "You're not using switch --%s, query may take long!" % key
if 'status' in switches.keys():
if not switches[ 'status' ] in ( 'New', 'Offline', 'Waiting','Failed','StageSubmitted','Staged' ):
subLogger.error( "Found \"%s\" as Status value. Incorrect value used!" % switches[ 'status' ] )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return switches
def getBoolean( value ):
if value.lower() == 'true':
return True
elif value.lower() == 'false':
return False
else:
Script.showHelp()
def _runFSTAgent():
""" read commands line params and run FST agent for a given transformation ID """
params = _Params()
params.registerSwitches()
Script.parseCommandLine()
if not params.checkSettings()['OK']:
Script.showHelp()
dexit(1)
from ILCDIRAC.ILCTransformationSystem.Agent.FileStatusTransformationAgent import FileStatusTransformationAgent
fstAgent = FileStatusTransformationAgent('ILCTransformation/FileStatusTransformationAgent',
'ILCTransformation/FileStatusTransformationAgent',
'dirac-ilc-filestatus-transformation')
fstAgent.log = gLogger
fstAgent.enabled = params.enabled
res = fstAgent.getTransformations(transID=params.transID)
if not res['OK']:
dexit(1)
if not res['Value']:
print("Transformation Not Found")
dexit(1)
trans = res['Value'][0]
res = fstAgent.processTransformation(
int(params.transID), trans['SourceSE'], trans['TargetSE'], trans['DataTransType'])
if not res["OK"]:
dexit(1)
dexit(0)
def addProperty( arg ):
global groupProperties
if not arg:
Script.showHelp()
DIRAC.exit( -1 )
if not arg in groupProperties:
groupProperties.append( arg )
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if args:
subLogger.error( "Found the following positional args '%s', but we only accept switches" % args )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
switches = dict( Script.getUnprocessedSwitches() )
# Default values
switches.setdefault( 'element', None )
if not switches[ 'element' ] in ( 'all', 'Site', 'Resource', 'Node', None ):
subLogger.error( "Found %s as element switch" % switches[ 'element' ] )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return switches
def main():
if len(args) < 1:
Script.showHelp()
return
fixedStatuses = ['Total', 'Done', 'Failed', 'Running', 'Waiting', 'Deleted']
for taskID in args:
taskID = int(taskID)
progress = getTaskEventProgress(taskID)
if not progress:
continue
print '='*80
print 'Task %s progress:' % taskID
print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12
print '%-12s %-12s %-12s' % ('Status', 'Job Number', 'Event Number')
print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12
printALine(progress, 'Total')
print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12
for fixedStatus in fixedStatuses:
if fixedStatus != 'Total':
printALine(progress, fixedStatus)
for p in progress:
if p not in fixedStatuses:
printALine(progress, p)
print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12
print ''
def addUserName( arg ):
global userNames
if not arg:
Script.showHelp()
DIRAC.exit( -1 )
if not arg in userNames:
userNames.append( arg )
def addProperty( arg ):
global hostProperties
if not arg:
Script.showHelp()
DIRAC.exit( -1 )
if not arg in hostProperties:
hostProperties.append( arg )
def addUserGroup(arg):
global userGroups
if not arg:
Script.showHelp()
DIRAC.exit(-1)
if not arg in userGroups:
userGroups.append(arg)
def main():
if len(args) < 1:
Script.showHelp()
return
for taskID in args:
taskID = int(taskID)
deleteTask(taskID)
print ""
def setUserMail(arg):
global userMail
if userMail or not arg:
Script.showHelp()
DIRAC.exit(-1)
if not arg.find("@") > 0:
Script.gLogger.error("Not a valid mail address", arg)
DIRAC.exit(-1)
userMail = arg
def error( msg ):
'''
Format error messages
'''
subLogger.error( "\nERROR:" )
subLogger.error( "\t" + msg )
subLogger.error( "\tPlease, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
def main():
if len(args) < 1:
Script.showHelp()
return
jobIDs = []
for jobID in args:
jobIDs.append(int(jobID))
showJobs(jobIDs)
print 'Totally %s job(s) displayed' % len(jobIDs)
def isOK(self):
if not self.cliParams.releasesToBuild:
gLogger.error("Missing releases to build!")
Script.showHelp()
return False
if not self.cliParams.destination:
self.cliParams.destination = tempfile.mkdtemp('DiracDist')
else:
mkDir(self.cliParams.destination)
gLogger.notice("Will generate tarballs in %s" % self.cliParams.destination)
return True
def _findInFC():
"""Find something in the FileCatalog"""
from DIRAC import exit as dexit
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len(args)<2:
Script.showHelp('ERROR: Not enough arguments')
gLogger.error("Run %s --help" % SCRIPTNAME )
dexit(1)
path = args[0]
if path == '.':
path = '/'
## Check that the first argument is not a MetaQuery
if any( op in path for op in OPLIST ):
gLogger.error("ERROR: Path '%s' is not a valid path! The first argument must be a path" % path)
gLogger.error("Run %s --help" % SCRIPTNAME )
dexit(1)
gLogger.verbose("Path:", path)
metaQuery = args[1:]
metaDataDict = _createQueryDict(metaQuery)
gLogger.verbose("Query:",str(metaDataDict))
if not metaDataDict:
gLogger.info("No query")
dexit(1)
fc = FileCatalogClient()
res = fc.findFilesByMetadata(metaDataDict, path)
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
if not res['Value']:
gLogger.notice("No files found")
listToPrint = None
if clip.printOnlyDirectories:
listToPrint = set( "/".join(fullpath.split("/")[:-1]) for fullpath in res['Value'] )
else:
listToPrint = res['Value']
for entry in listToPrint:
print entry
dexit(0)
def main():
if len(args) < 1:
Script.showHelp()
return
status = ['Failed']
for option in options:
(switch, val) = option
if switch == 'a' or switch == 'all':
status = []
for taskID in args:
taskID = int(taskID)
rescheduleTask(taskID, status)
print ''
def _setSwitches(self):
Script.registerSwitch("S:", "system=", "Systems to check, by default all of them are checked", self._setSystems)
Script.registerSwitch("M", "modified", "Show entries which differ from the default", self._setShowModified)
Script.registerSwitch("A", "added", "Show entries which do not exist in ConfigTemplate", self._setShowAdded)
Script.registerSwitch("U", "missingSection", "Show sections which do not exist in the current configuration",
self._setShowMissingSections)
Script.registerSwitch("O", "missingOption", "Show options which do not exist in the current configuration",
self._setShowMissingOptions)
Script.setUsageMessage('\n'.join([self.__doc__,
'Usage:',
' %s [option|cfgfile] -[MAUO] [-S <system]' % Script.scriptName]))
Script.parseCommandLine(ignoreErrors=True)
if not any([self.showModified, self.showAdded, self.showMissingSections, self.showMissingOptions]):
LOG.error("\nERROR: Set at least one of the flags M A U O")
Script.showHelp()
def _getOutputData():
cliParams = _Params()
cliParams.registerSwitches()
Script.parseCommandLine( ignoreErrors = False )
if not cliParams.repo:
Script.showHelp()
dexit(2)
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac(True, cliParams.repo)
exitCode = 0
dirac.monitorRepository(False)
dirac.retrieveRepositoryData()
dexit(exitCode)
def addUser():
"""Add user to configuration service and other things"""
clip = Params()
clip.registerSwitches()
Script.parseCommandLine()
if not ( clip.certCN and clip.groups and clip.certDN and clip.uname):
gLogger.error("Username, DN, CN, and groups have to be given")
Script.showHelp()
gLogger.notice("Add User to Egroup")
addUserToEgroup(clip)
if not clip.email:
gLogger.fatal("No email defined and not found in phonebook, you have to provide it: -E<email>")
dexit(1)
userProps = {'DN': clip.certDN, 'Email': clip.email, 'CN': clip.certCN, 'Groups': clip.groups}
gLogger.notice("Add User to CS")
addUserToCS(clip, userProps)
gLogger.notice("Add User to FC")
addUserToFC(clip)
gLogger.notice("Done")
def checkSettings(self):
"""check if all required parameters are set, print error message and return S_ERROR if not"""
args = Script.getPositionalArgs()
if len(args) < 4:
self.errorMessages.append("ERROR: Not enough arguments")
else:
self.setProdID( args[0] )
self.setTargetSE( args[1] )
self.setSourceSE( args[2] )
self.setDatatype( args[3] )
self.checkProxy()
if not self.errorMessages:
return S_OK()
gLogger.error("\n".join(self.errorMessages))
Script.showHelp()
return S_ERROR()
def addSoftware():
"""uploads, registers, and sends email about new software package"""
cliParams = Params()
cliParams.registerSwitches()
Script.parseCommandLine( ignoreErrors = True )
platform = cliParams.platform
appName = cliParams.name
appVersion = cliParams.version
comment = cliParams.comment
tarball_loc = cliParams.tarball
if not platform or not appName or not comment:
Script.showHelp()
dexit(2)
softAdder = SoftwareAdder(platform, appName, tarball_loc, appVersion, comment)
softAdder.addSoftware()
gLogger.notice("All done!")
dexit(0)
def _getProdLogs():
"""get production log files from LogSE"""
clip = _Params()
clip.registerSwitch()
Script.parseCommandLine()
if not ( clip.logF or clip.logD or clip.prodid ):
Script.showHelp()
dexit(1)
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
ops = Operations()
storageElementName = ops.getValue('/LogStorage/LogSE', 'LogSE')
from DIRAC.Resources.Storage.StorageElement import StorageElementItem as StorageElement
logSE = StorageElement(storageElementName)
if clip.prodid and not ( clip.logD or clip.logF ):
result = _getLogFolderFromID( clip )
if not result['OK']:
gLogger.error( result['Message'] )
dexit(1)
if clip.logD:
if not clip.noPromptBeforeDL:
res = promptUser('Are you sure you want to get ALL the files in this directory?')
if not res['OK']:
dexit()
choice = res['Value']
if choice.lower()=='n':
dexit(0)
if isinstance(clip.logD, str):
res = logSE.getDirectory(clip.logD, localPath=clip.outputdir)
_printErrorReport(res)
elif isinstance(clip.logD, list):
for logdir in clip.logD:
gLogger.notice('Getting log files from '+str(logdir))
res = logSE.getDirectory(logdir, localPath=clip.outputdir)
_printErrorReport(res)
if clip.logF:
res = logSE.getFile(clip.logF, localPath = clip.outputdir)
_printErrorReport(res)
def _createLFNList():
"""create the LFnList"""
cliparams = _Params()
cliparams.registerSwitches()
Script.parseCommandLine( ignoreErrors = False )
repoLocation = cliparams.repo
if not repoLocation:
Script.showHelp()
dexit(2)
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
dirac = DiracILC(True, repoLocation)
dirac.monitorRepository(False)
lfns = []
lfns = dirac.retrieveRepositoryOutputDataLFNs()
LOG.notice("lfnlist=[")
for lfn in lfns :
LOG.notice('"LFN:%s",' % lfn)
LOG.notice("]")
dexit(0)
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if args:
subLogger.error( "Found the following positional args '%s', but we only accept switches" % args )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
switches = dict( Script.getUnprocessedSwitches() )
switches.setdefault( 'statusType' , None )
switches.setdefault( 'releaseToken', False )
for key in ( 'element', 'name', 'reason' ):
if not key in switches:
subLogger.error( "%s Switch missing" % key )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
if not switches[ 'element' ] in ( 'Site', 'Resource', 'Node' ):
subLogger.error( "Found %s as element switch" % switches[ 'element' ] )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return switches
def addSoftware():
"""uploads, registers, and sends email about new software package"""
cliParams = Params()
cliParams.registerSwitches()
Script.parseCommandLine( ignoreErrors = True )
consistent = cliParams.checkConsistency()
if not consistent['OK']:
gLogger.error("Error checking consistency:", consistent['Message'])
Script.showHelp()
dexit(2)
softAdder = CVMFSAdder(cliParams)
resCheck = softAdder.checkConsistency()
if not resCheck['OK']:
Script.showHelp()
dexit(2)
softAdder.addSoftware()
gLogger.notice("All done!")
dexit(0)
def main():
if len(args) < 1:
Script.showHelp()
return
showJobNumber = False
for option in options:
(switch, val) = option
if switch == 'j' or switch == 'job':
showJobNumber = True
for taskID in args:
print '='*80
taskID = int(taskID)
showTask(taskID)
print ''
showTaskHistories(taskID)
print ''
if showJobNumber:
showTaskJobs(taskID)
print ''
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if not len(args) == 2:
Script.showHelp()
params = {}
params['ce'] = None
params['site'] = None
params['host'] = None
params['vo'] = None
params['info'] = args[0]
ret = getProxyInfo(disableVOMS=True)
if ret['OK'] and 'group' in ret['Value']:
params['vo'] = getVOForGroup(ret['Value']['group'])
else:
Script.gLogger.error('Could not determine VO')
Script.showHelp()
if params['info'] in ['ce', 'ce-state', 'ce-cluster', 'ce-vo']:
params['ce'] = args[1]
elif params['info']in ['site', 'site-se']:
params['site'] = args[1]
else:
Script.gLogger.error('Wrong argument value')
Script.showHelp()
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("H", "host"):
params['host'] = unprocSw[1]
if unprocSw[0] in ("V", "vo"):
params['vo'] = unprocSw[1]
return params
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if args:
subLogger.error("Found the following positional args '%s', but we only accept switches" % args)
subLogger.error("Please, check documentation below")
Script.showHelp()
DIRACExit(1)
switches = dict(Script.getUnprocessedSwitches())
# Default values
switches.setdefault("elementType", None)
switches.setdefault("name", None)
switches.setdefault("tokenOwner", None)
switches.setdefault("statusType", None)
switches.setdefault("status", None)
if not "element" in switches:
subLogger.error("element Switch missing")
subLogger.error("Please, check documentation below")
Script.showHelp()
DIRACExit(1)
if not switches["element"] in ("Site", "Resource", "Node"):
subLogger.error("Found %s as element switch" % switches["element"])
subLogger.error("Please, check documentation below")
Script.showHelp()
DIRACExit(1)
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.iteritems())
return switches
from DIRAC.Core.Base import Script
Script.setUsageMessage("""
Clean the given directory or a list of directories by removing it and all the
contained files and subdirectories from the physical storage and from the
file catalogs.
Usage:
%s <LFN_Directory | fileContainingLFN_Directories>
""" % Script.scriptName)
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp(exitCode=1)
inputFileName = args[0]
if os.path.exists(inputFileName):
lfns = [
lfn.strip().split()[0]
for lfn in sorted(open(inputFileName, 'r').read().splitlines())
]
else:
lfns = [inputFileName]
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
retVal = 0
for lfn in [lfn for lfn in lfns if lfn]:
def main():
Script.parseCommandLine()
from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp(exitCode=1)
# get arguments
prodID = args[0]
prodClient = ProductionClient()
transClient = TransformationClient()
res = prodClient.getProductionTransformations(prodID)
transIDs = []
if res['OK']:
transList = res['Value']
if not transList:
DIRAC.gLogger.notice(
'No transformation associated with production %s' % prodID)
DIRAC.exit(-1)
for trans in transList:
transIDs.append(trans['TransformationID'])
else:
DIRAC.gLogger.error(res['Message'])
DIRAC.exit(-1)
fields = [
'TransformationName', 'Status', 'F_Proc.', 'F_Proc.(%)',
'TransformationID', 'ProductionID', 'Prod_LastUpdate',
'Prod_InsertedTime'
]
records = []
paramShowNames = [
'TransformationID', 'TransformationName', 'Type', 'Status',
'Files_Total', 'Files_PercentProcessed', 'Files_Processed',
'Files_Unused', 'Jobs_TotalCreated', 'Jobs_Waiting', 'Jobs_Running',
'Jobs_Done', 'Jobs_Failed', 'Jobs_Stalled'
]
resList = []
res = transClient.getTransformationSummaryWeb(
{'TransformationID': transIDs}, [], 0, len(transIDs))
if not res['OK']:
DIRAC.gLogger.error(res['Message'])
DIRAC.exit(-1)
if res['Value']['TotalRecords'] > 0:
paramNames = res['Value']['ParameterNames']
for paramValues in res['Value']['Records']:
paramShowValues = map(
lambda pname: paramValues[paramNames.index(pname)],
paramShowNames)
showDict = dict(zip(paramShowNames, paramShowValues))
resList.append(showDict)
for res in resList:
files_Processed = res['Files_Processed']
files_PercentProcessed = res['Files_PercentProcessed']
status = res['Status']
type = res['Type']
transName = res['TransformationName']
transID = res['TransformationID']
records.append([
transName, status,
str(files_Processed),
str(files_PercentProcessed),
str(transID),
str(prodID),
str(trans['LastUpdate']),
str(trans['InsertedTime'])
])
printTable(fields, records)
DIRAC.exit(0)
def addUserName(arg):
global userNames
if not arg:
Script.showHelp(exitCode=1)
if arg not in userNames:
userNames.append(arg)
def main():
parameters = ['OwnerDN', 'StartExecTime', 'EndExecTime']
Script.registerSwitch(
'', 'Parameters=',
' List of strings to be matched by job parameters or attributes')
Script.parseCommandLine(ignoreErrors=True)
for switch in Script.getUnprocessedSwitches():
if switch[0] == 'Parameters':
parameters += [par for par in switch[1].split(',')]
parameters = [(i, par.lower()) for i, par in enumerate(parameters) if par]
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.Interfaces.API.Dirac import Dirac
diracAdmin = DiracAdmin()
dirac = Dirac()
errorList = []
for gridID in args:
result = {}
res = diracAdmin.getPilotInfo(gridID)
if not res['OK']:
errorList.append((gridID, res['Message']))
else:
jobIDs = set(
[int(jobID) for jobID in res['Value'][gridID]['Jobs']])
totCPU = 0
totWall = 0
effRequested = False
for jobID in sorted(jobIDs):
result.setdefault(jobID, {})
for func in (dirac.getJobParameters, dirac.getJobAttributes):
res = func(jobID)
if not res['OK']:
errorList.append(('Job %d' % jobID, res['Message']))
else:
params = res['Value']
if 'TotalCPUTime(s)' in params:
totCPU += float(params['TotalCPUTime(s)'])
totWall += float(params['WallClockTime(s)'])
params['CPUEfficiency'] = '%s %%' % (
100. * params['TotalCPUTime(s)'] /
params['WallClockTime(s)'])
for i, par in parameters:
for param in [
p for p in _stringInList(
str(par), str(params))
if not _stringInList(
str(p), str(result[jobID]))
]:
if param == 'CPUEfficiency':
effRequested = True
result[jobID]['%d.%s' %
(i, param)] = params[param]
if effRequested:
result['CPUEfficiency'] = '%s %%' % (100. * totCPU /
totWall)
print(diracAdmin.pPrint.pformat({gridID: result}))
for error in errorList:
print("ERROR %s: %s" % error)
def TrainRF(args=None):
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.Interfaces.API.Job import Job
user_script = './trainRF.sh'
if (len(args) != 3):
Script.showHelp()
direction = args[0]
zenith = args[1]
diffuse = args[2]
site = "PARANAL"
if diffuse == "0":
diffName = "point"
elif diffuse == "1":
diffName = "diff"
else:
print "Invalid extension definition!"
Script.showHelp()
return 1
if zenith == "40":
zenName = "40deg"
elif zenith == "20":
zenName = "20deg"
else:
print "Invalid zenith definition!"
Script.showHelp()
return 1
if direction == "N":
directionName = "north"
# deg = "180"
elif direction == "S":
directionName = "south"
# deg = "0"
else:
print 'Wrong direction. It can only be "N" or "S".'
Script.showHelp()
return 1
# Macro fixing the file check before continues.
ROOTmacro = "CTAtrain.C"
# List of files over which the training should be done
LFN_file_gammas = './training/gamma_ghtrain_%s_%s_%s.lfns' % (
zenName, diffName, direction)
LFN_file_protons = './training/proton_ghtrain_%s_%s_%s.lfns' % (
zenName, diffName, direction)
StatFile = './Statistic_train.txt'
for telType in range(0, 6):
jobName = "%s_%s_%s_%s_%s" % (user_script, directionName, diffName,
telType, zenName)
jobOut = "%s_%s_%s_%s.out" % (user_script, directionName, diffName,
telType)
script_args = "%s %s %s %s %s" % (direction, site, diffName, telType,
zenName)
j = Job()
j.setInputSandbox([
user_script, "setupPackageMARS.sh", LFN_file_gammas,
LFN_file_protons, ROOTmacro, StatFile
])
j.setExecutable(user_script, script_args)
j.setOutputSandbox([jobOut, "applicationLog.txt"])
j.setName(jobName)
Script.gLogger.info(j._toJDL())
print "Launching %s %s" % (user_script, script_args)
Dirac().submit(j)
def main():
Script.parseCommandLine(ignoreErrors=False)
args = Script.getPositionalArgs()
if len(args) < 2:
Script.showHelp()
targetSE = args.pop(0)
lfns = []
for inputFileName in args:
if os.path.exists(inputFileName):
inputFile = open(inputFileName, 'r')
string = inputFile.read()
inputFile.close()
lfns.extend([lfn.strip() for lfn in string.splitlines()])
else:
lfns.append(inputFileName)
from DIRAC.Resources.Storage.StorageElement import StorageElement
import DIRAC
# Check is provided SE is OK
if targetSE != 'All':
se = StorageElement(targetSE)
if not se.valid:
print(se.errorReason)
print()
Script.showHelp()
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
reqClient = ReqClient()
fc = FileCatalog()
requestOperation = 'RemoveReplica'
if targetSE == 'All':
requestOperation = 'RemoveFile'
for lfnList in breakListIntoChunks(lfns, 100):
oRequest = Request()
requestName = "%s_%s" % (
md5(repr(time.time()).encode()).hexdigest()[:16],
md5(repr(time.time()).encode()).hexdigest()[:16],
)
oRequest.RequestName = requestName
oOperation = Operation()
oOperation.Type = requestOperation
oOperation.TargetSE = targetSE
res = fc.getFileMetadata(lfnList)
if not res['OK']:
print("Can't get file metadata: %s" % res['Message'])
DIRAC.exit(1)
if res['Value']['Failed']:
print(
"Could not get the file metadata of the following, so skipping them:"
)
for fFile in res['Value']['Failed']:
print(fFile)
lfnMetadata = res['Value']['Successful']
for lfn in lfnMetadata:
rarFile = File()
rarFile.LFN = lfn
rarFile.Size = lfnMetadata[lfn]['Size']
rarFile.Checksum = lfnMetadata[lfn]['Checksum']
rarFile.GUID = lfnMetadata[lfn]['GUID']
rarFile.ChecksumType = 'ADLER32'
oOperation.addFile(rarFile)
oRequest.addOperation(oOperation)
isValid = RequestValidator().validate(oRequest)
if not isValid['OK']:
print("Request is not valid: ", isValid['Message'])
DIRAC.exit(1)
result = reqClient.putRequest(oRequest)
if result['OK']:
print('Request %d Submitted' % result['Value'])
else:
print('Failed to submit Request: ', result['Message'])
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch("T:", "template=", "Corsika Template")
Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig")
Script.registerSwitch("p:", "run_number=",
"Do not use: Run Number automatically set")
Script.registerSwitch("E:", "executable=",
"Executable (Use SetExecutable)")
Script.registerSwitch("V:", "version=", "Version (Use setVersion)")
Script.registerSwitch("M:", "mode=",
"Mode (corsika_standalone/corsika_simtelarray)")
Script.parseCommandLine(ignoreErrors=True)
## default values ##############
run_number = None
corsikaTemplate = None
simtelConfig = None
executable = None
version = None
mode = 'corsika_simtelarray'
### set switch values ###
for switch in Script.getUnprocessedSwitches():
if switch[0] == "run_number" or switch[0] == "p":
run_number = switch[1].split('ParametricParameters=')[1]
elif switch[0] == "template" or switch[0] == "T":
corsikaTemplate = switch[1]
elif switch[0] == "simtelConfig" or switch[0] == "S":
simtelConfig = switch[1]
elif switch[0] == "executable" or switch[0] == "E":
executable = switch[1]
elif switch[0] == "version" or switch[0] == "V":
version = switch[1]
elif switch[0] == "mode" or switch[0] == "M":
mode = switch[1]
if version == None or executable == None or run_number == None or corsikaTemplate == None:
Script.showHelp()
jobReport.setApplicationStatus('Missing options')
DIRAC.exit(-1)
from CTADIRAC.Core.Workflow.Modules.CorsikaApp import CorsikaApp
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
createGlobalsFromConfigFiles(corsikaTemplate, version)
############ Producing Corsika File
install_CorsikaSimtelPack(version)
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
cs = CorsikaApp()
cs.setSoftwarePackage(CorsikaSimtelPack)
cs.csExe = executable
cs.csArguments = [
'--run-number', run_number, '--run', 'corsika', corsikaTemplate
]
corsikaReturnCode = cs.execute()
if corsikaReturnCode != 0:
DIRAC.gLogger.error('Corsika Application: Failed')
jobReport.setApplicationStatus('Corsika Application: Failed')
DIRAC.exit(-1)
###################### rename of corsika output file #######################
rundir = 'run' + run_number
filein = rundir + '/' + corsikaOutputFileName
corsikaFileName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.gz'
mv_cmd = 'mv ' + filein + ' ' + corsikaFileName
if (os.system(mv_cmd)):
DIRAC.exit(-1)
### create corsika tar luisa ####################
corsikaTarName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.tar.gz'
filetar1 = rundir + '/' + 'input'
filetar2 = rundir + '/' + 'DAT' + run_number + '.dbase'
filetar3 = rundir + '/run' + str(int(run_number)) + '.log'
cmdTuple = [
'/bin/tar', 'zcf', corsikaTarName, filetar1, filetar2, filetar3
]
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute tar')
DIRAC.exit(-1)
if (mode == 'corsika_standalone'):
DIRAC.exit()
############ Producing SimTel File
######################Building simtel Directory Metadata #######################
cfg_dict = {
"4MSST": 'cta-prod2-4m-dc',
"SCSST": 'cta-prod2-sc-sst',
"STD": 'cta-prod2',
"NSBX3": 'cta-prod2',
"ASTRI": 'cta-prod2-astri',
"NORTH": 'cta-prod2n',
"SCMST": 'cta-prod2-sc3'
}
#if simtelConfig=="6INROW":
# all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD","SCMST"]
#elif simtelConfig=="5INROW":
# all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD"]
#else:
# all_configs=[simtelConfig]
all_configs = [simtelConfig]
for current_conf in all_configs:
DIRAC.gLogger.notice('current conf is', current_conf)
current_version = version
DIRAC.gLogger.notice('current version is', current_version)
#if current_conf == "SCMST":
# current_version = version + '_sc3'
# DIRAC.gLogger.notice('current version is', current_version)
# if os.path.isdir('sim_telarray'):
# DIRAC.gLogger.notice('Package found in the local area. Removing package...')
# cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run'
# if(os.system(cmd)):
# DIRAC.exit( -1 )
# install_CorsikaSimtelPack(current_version)
#else:
# current_version = version
# DIRAC.gLogger.notice('current version is', current_version)
#### execute simtelarray ################
fd = open('run_sim.sh', 'w')
fd.write("""#! /bin/sh
export SVNPROD2=$PWD
export SVNTAG=SVN-PROD2
export CORSIKA_IO_BUFFER=800MB
./grid_prod2-repro.sh %s %s""" % (corsikaFileName, current_conf))
fd.close()
####################################
os.system('chmod u+x run_sim.sh')
os.system('chmod u+x grid_prod2-repro.sh')
cmdTuple = ['./run_sim.sh']
ret = systemCall(0, cmdTuple, sendSimtelOutput)
simtelReturnCode, stdout, stderr = ret['Value']
if (os.system('grep Broken simtel.log') == 0):
DIRAC.gLogger.error('Broken string found in simtel.log')
jobReport.setApplicationStatus('Broken pipe')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute run_sim.sh')
DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode)
DIRAC.exit(-1)
## check simtel data/log/histo Output File exist
cfg = cfg_dict[current_conf]
simtelFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.simtel.gz'
cmd = 'mv Data/sim_telarray/' + cfg + '/*/Data/*.simtel.gz ' + simtelFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelLogFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz'
cmd = 'mv Data/sim_telarray/' + cfg + '/*/Log/*.log.gz ' + simtelLogFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelHistFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.hdata.gz'
cmd = 'mv Data/sim_telarray/' + cfg + '/*/Histograms/*.hdata.gz ' + simtelHistFileName
if (os.system(cmd)):
DIRAC.exit(-1)
################################################################
DIRAC.exit()
' or Upload',
' %s Path LFN SE' % Script.scriptName,
'Arguments:',
' LFN: Logical File Name (Path to directory)',
' Path: Local path to the file (Path to directory)',
' SE: DIRAC Storage Element']
)
)
Script.registerSwitch( "D" , "sync" , "Make target directory identical to source" )
Script.registerSwitch( "j:" , "parallel=" , "Multithreaded download and upload" )
Script.parseCommandLine( ignoreErrors = False )
args = Script.getPositionalArgs()
if len( args ) < 1 or len( args ) > 3:
Script.showHelp()
sync = False
parallel = 1
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "s" or switch[0].lower() == "sync":
sync = True
if switch[0].lower() == "j" or switch[0].lower() == "parallel":
parallel = int(switch[1])
from DIRAC import S_OK, S_ERROR
from DIRAC import gConfig, gLogger
from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
from DIRAC.Core.Utilities.File import mkDir
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
def setHostDN(arg):
global hostDN
if hostDN or not arg:
Script.showHelp()
DIRAC.exit(-1)
hostDN = arg
def main():
from DIRAC.Core.Base import Script
### DoCtaIrf options ##########################################################
Script.registerSwitch("A:", "analysis=", "Analysis Type", setAnalysisType)
Script.registerSwitch("C:", "cuts=", "Cuts Config", setCutsConfig)
Script.registerSwitch("R:", "runlist=", "Runlist", setRunlist)
Script.registerSwitch("Z:", "zenith=", "Zenith", setZenith)
Script.registerSwitch("O:", "offset=", "Offset", setOffset)
Script.registerSwitch("M:", "energy=", "Energy Method", setEnergyMethod)
Script.registerSwitch("T:", "arrayconfig=", "Array Configuration",
setArrayConfig)
Script.registerSwitch("P:", "particle=", "Particle Type", setParticleType)
## other options
Script.registerSwitch("V:", "version=", "HAP version", setVersion)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
from CTADIRAC.Core.Workflow.Modules.HapApplication import HapApplication
from CTADIRAC.Core.Workflow.Modules.HapRootMacro import HapRootMacro
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
ha = HapApplication()
HapPack = 'HAP/' + version + '/HAP'
packs = ['HESS/v0.2/lib', 'HESS/v0.3/root', HapPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
continue
if localArea:
if checkSoftwarePackage(package, localArea())['OK']:
DIRAC.gLogger.notice('Package found in Local Area:', package)
continue
if installSoftwarePackage(package, localArea())['OK']:
continue
DIRAC.gLogger.error('Check Failed for software package:', package)
DIRAC.gLogger.error('Software package not available')
DIRAC.exit(-1)
ha.setSoftwarePackage(HapPack)
ha.hapExecutable = 'DoCtaIrf'
runlistdir = os.environ['PWD']
build_infile(runlist)
ha.hapArguments = [
analysistype, cutsconfig, runlistdir, runlist, zenith, offset,
arrayconfig, energymethod, particle
]
DIRAC.gLogger.notice('Executing Hap Application')
res = ha.execute()
if not res['OK']:
DIRAC.gLogger.error('Failed to execute Hap Application')
jobReport.setApplicationStatus('Hap Application: Failed')
DIRAC.exit(-1)
###################### Check TTree Output File #######################
outfile = 'MVAFile_' + runlist + ".root"
if not os.path.isfile(outfile):
error = 'TTree file was not created:'
DIRAC.gLogger.error(error, outfile)
jobReport.setApplicationStatus('DoCtaIrf: TTree file not created')
DIRAC.exit(-1)
###################### Quality Check for TTree Output File: step0######################
hr = HapRootMacro()
hr.setSoftwarePackage(HapPack)
DIRAC.gLogger.notice('Executing TTree check step0')
hr.rootMacro = '/hapscripts/mva/Open_TT.C+'
outfilestr = '"' + outfile + '"'
args = [outfilestr]
DIRAC.gLogger.notice('Open_TT macro Arguments:', args)
hr.rootArguments = args
DIRAC.gLogger.notice('Executing Hap Open_TT macro')
res = hr.execute()
if not res['OK']:
DIRAC.gLogger.error('Open_TT: Failed')
DIRAC.exit(-1)
#########################Quality Check for TTree Output File: step1####################
DIRAC.gLogger.notice('Executing TTree check step1')
ret = getSoftwareEnviron(HapPack)
if not ret['OK']:
error = ret['Message']
DIRAC.gLogger.error(error, HapPack)
DIRAC.exit(-1)
hapEnviron = ret['Value']
hessroot = hapEnviron['HESSROOT']
check_script = hessroot + '/hapscripts/mva/check_TT.csh'
cmdTuple = [check_script]
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute TTree Check step1')
jobReport.setApplicationStatus('Check_TTree: Failed')
DIRAC.exit(-1)
#############################################
DIRAC.exit()
def setHostName(arg):
global hostName
if hostName or not arg:
Script.showHelp()
DIRAC.exit(-1)
hostName = arg
def main():
Script.registerSwitch(
"v:", "vo=",
"Location of pilot version in CS /Operations/<vo>/Pilot/Version"
" (default value specified in CS under /DIRAC/DefaultSetup)")
Script.parseCommandLine(ignoreErrors=False)
args = Script.getPositionalArgs()
if len(args) < 1 or len(args) > 2:
Script.showHelp()
version = args[0]
vo = None
for switch in Script.getUnprocessedSwitches():
if switch[0] == "v" or switch[0] == "vo":
vo = switch[1]
from DIRAC import S_OK, S_ERROR
from DIRAC import gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
def updatePilot(version, vo):
"""
Update in the CS the pilot version used,
If only one version present in CS it's overwritten.
If two versions present, the new one is added and the last removed
:param version: version vArBpC of pilot you want to use
:param vo: Location of pilot version in CS /Operations/<vo>/Pilot/Version
"""
setup = vo
if not vo:
setup = gConfig.getValue('/DIRAC/DefaultSetup')
if not setup:
return S_ERROR("No value set for /DIRAC/DefaultSetup in CS")
pilotVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup,
[])
if not pilotVersion:
return S_ERROR(
"No pilot version set under Operations/%s/Pilot/Version in CS"
% setup)
pilotVersion.pop()
pilotVersion.insert(0, version)
api = CSAPI()
api.setOption('Operations/%s/Pilot/Version' % setup,
", ".join(pilotVersion))
result = api.commit()
if not result['OK']:
gLogger.fatal('Could not commit new version of pilot!')
return result
newVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup)
return S_OK("New version of pilot set to %s" % newVersion)
result = updatePilot(version, vo)
if not result['OK']:
gLogger.fatal(result['Message'])
DIRAC.exit(1)
gLogger.notice(result['Value'])
DIRAC.exit(0)
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch("p:", "inputfile=", "Input File")
Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig")
Script.registerSwitch("V:", "version=", "Version (Use setVersion)")
Script.registerSwitch("C:", "comp=", "Compile (True/False)")
Script.parseCommandLine(ignoreErrors=True)
## default values ##############
simtelConfig = None
version = None
comp = True
### set switch values ###
for switch in Script.getUnprocessedSwitches():
if switch[0] == "inputfile" or switch[0] == "p":
corsikaFileLFN = switch[1].split('ParametricInputData=LFN:')[1]
elif switch[0] == "simtelConfig" or switch[0] == "S":
simtelConfig = switch[1]
elif switch[0] == "version" or switch[0] == "V":
version = switch[1]
elif switch[0] == "comp" or switch[0] == "C":
comp = switch[1]
if version == None:
Script.showHelp()
jobReport.setApplicationStatus('Missing options')
DIRAC.exit(-1)
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
############ Producing SimTel File
if comp == True:
install_CorsikaSimtelPack(version)
else:
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
res = installSoftwarePackage(CorsikaSimtelPack, workingArea())
if not res['OK']:
DIRAC.gLogger.error('Failed to execute installSoftwarePackage',
res)
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
cfg_dict = {
"4MSST": 'cta-prod2-4m-dc',
"SCSST": 'cta-prod2-sc-sst',
"STD": 'cta-prod2',
"NSBX3": 'cta-prod2',
"ASTRI": 'cta-prod2-astri',
"NORTH": 'cta-prod2n',
"ASTRISLN": 'cta-astri-sln'
}
#if simtelConfig=="6INROW":
# all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD","SCMST"]
#elif simtelConfig=="5INROW":
# all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD"]
#else:
# all_configs=[simtelConfig]
all_configs = [simtelConfig]
for current_conf in all_configs:
DIRAC.gLogger.notice('current conf is', current_conf)
current_version = version
DIRAC.gLogger.notice('current version is', current_version)
#if current_conf == "SCMST":
# current_version = version + '_sc3'
# DIRAC.gLogger.notice('current version is', current_version)
# if os.path.isdir('sim_telarray'):
# DIRAC.gLogger.notice('Package found in the local area. Removing package...')
# cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run'
# if(os.system(cmd)):
# DIRAC.exit( -1 )
# install_CorsikaSimtelPack(current_version)
#else:
# current_version = version
# DIRAC.gLogger.notice('current version is', current_version)
#### execute simtelarray ################
fd = open('run_sim.sh', 'w')
fd.write("""#! /bin/sh
export SVNPROD2=$PWD
export SVNTAG=SVN-PROD2
export CORSIKA_IO_BUFFER=800MB
./grid_prod2-repro.sh %s %s""" %
(os.path.basename(corsikaFileLFN), current_conf))
fd.close()
####################################
os.system('chmod u+x run_sim.sh')
os.system('chmod u+x grid_prod2-repro.sh')
cmdTuple = ['./run_sim.sh']
ret = systemCall(0, cmdTuple, sendSimtelOutput)
simtelReturnCode, stdout, stderr = ret['Value']
if (os.system('grep Broken simtel.log') == 0):
DIRAC.gLogger.error('Broken string found in simtel.log')
jobReport.setApplicationStatus('Broken pipe')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute run_sim.sh')
DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode)
DIRAC.exit(-1)
## check simtel data/log/histo Output File exist
cfg = cfg_dict[current_conf]
simtelFileName = os.path.basename(corsikaFileLFN).replace(
'corsika.gz', 'simtel.gz')
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Data/*.simtel.gz ' + simtelFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelLogFileName = simtelFileName.replace('simtel.gz', 'log.gz')
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Log/*.log.gz ' + simtelLogFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelHistFileName = simtelFileName.replace('simtel.gz', 'hdata.gz')
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName
if (os.system(cmd)):
DIRAC.exit(-1)
################################################################
DIRAC.exit()
def main():
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 3:
Script.showHelp(exitCode=1)
diracSiteName = args[0]
gridSiteName = args[1]
ces = args[2:]
try:
diracGridType, place, country = diracSiteName.split('.')
except ValueError:
gLogger.error(
"The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch"
)
DIRACExit(-1)
result = getDIRACSiteName(gridSiteName)
newSite = True
if result['OK'] and result['Value']:
if len(result['Value']) > 1:
gLogger.notice(
'%s GOC site name is associated with several DIRAC sites:' %
gridSiteName)
for i, dSite in enumerate(result['Value']):
gLogger.notice('%d: %s' % (i, dSite))
inp = six.moves.input('Enter your choice number: ')
try:
inp = int(inp)
except ValueError:
gLogger.error('You should enter an integer number')
DIRACExit(-1)
if 0 <= inp < len(result['Value']):
diracCSSite = result['Value'][inp]
else:
gLogger.error('Number out of range: %d' % inp)
DIRACExit(-1)
else:
diracCSSite = result['Value'][0]
if diracCSSite == diracSiteName:
gLogger.notice('Site with GOC name %s is already defined as %s' %
(gridSiteName, diracSiteName))
newSite = False
else:
gLogger.error(
'ERROR: Site with GOC name %s is already defined as %s' %
(gridSiteName, diracCSSite))
DIRACExit(-1)
else:
gLogger.error("ERROR getting DIRAC site name of %s" % gridSiteName,
result.get('Message'))
csAPI = CSAPI()
if newSite:
gLogger.notice("Site to CS: %s" % diracSiteName)
res = csAPI.addSite(diracSiteName, {"Name": gridSiteName})
if not res['OK']:
gLogger.error("Failed adding site to CS", res['Message'])
DIRACExit(1)
res = csAPI.commit()
if not res['OK']:
gLogger.error("Failure committing to CS", res['Message'])
DIRACExit(3)
for ce in ces:
gLogger.notice("Adding CE %s" % ce)
res = csAPI.addCEtoSite(diracSiteName, ce)
if not res['OK']:
gLogger.error("Failed adding CE %s to CS" % ce, res['Message'])
DIRACExit(2)
res = csAPI.commit()
if not res['OK']:
gLogger.error("Failure committing to CS", res['Message'])
DIRACExit(3)
def setGroupName(arg):
global groupName
if groupName or not arg:
Script.showHelp(exitCode=1)
groupName = arg
def main():
Script.registerSwitch(
"", "hosts=",
"Comma separated list of hosts or file containing row wise list of hosts"
" targeted for update (leave empty for all)")
Script.registerSwitch(
"", "retry=",
"Number of retry attempts on hosts that have failed to update")
Script.parseCommandLine(ignoreErrors=False)
args = Script.getPositionalArgs()
if len(args) < 1 or len(args) > 2:
Script.showHelp()
version = args[0]
retry = 0
hosts = []
for switch in Script.getUnprocessedSwitches():
if switch[0] == "hosts":
hosts = switch[1]
if switch[0] == "retry":
retry = int(switch[1])
try:
with open(hosts, 'r') as f:
hosts = f.read().splitlines()
hosts = [str(host) for host in hosts]
except Exception:
pass
if not isinstance(hosts, list):
hosts = hosts.split(',')
from concurrent.futures import ThreadPoolExecutor, as_completed
from DIRAC import S_OK, S_ERROR
from DIRAC import gLogger
from DIRAC.FrameworkSystem.Client.SystemAdministratorClient import SystemAdministratorClient
from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient
def getListOfHosts():
"""
Return the list of all hosts that constitute a DIRAC instance
"""
client = ComponentMonitoringClient()
result = client.getHosts({}, False, False)
if result['OK']:
hosts = [host['HostName'] for host in result['Value']]
return S_OK(hosts)
return S_ERROR('Cannot get list of hosts: %s' % result['Message'])
def parseHostname(hostName):
"""
Separate the hostname from the port
:param str hostName: hostname you want to parse
"""
hostList = hostName.split(':')
host = hostList[0]
if len(hostList) == 2:
port = hostList[1]
else:
port = None
return host, port
def updateHost(hostName, version):
"""
Apply update to specific host
:param str hostName: name of the host you want to update
:param str version: version vArBpC you want to update to
"""
host, port = parseHostname(hostName)
client = SystemAdministratorClient(host, port)
result = client.ping()
if not result['OK']:
gLogger.error("Cannot connect to %s" % host)
return result
gLogger.notice(
"Initiating software update of %s, this can take a while, please be patient ..."
% host)
result = client.updateSoftware(version, '', '', timeout=600)
if not result['OK']:
return result
return S_OK()
def updateHosts(version, hosts=[]):
"""
Apply update to all hosts
:param str version: version vArBpC you want to update to
:param list[str] hosts: list of hosts to be updated, defaults to []
"""
if not hosts:
result = getListOfHosts()
if not result['OK']:
return result
hosts = result['Value']
updateSuccess = []
updateFail = []
executor = ThreadPoolExecutor(max_workers=len(hosts))
futureUpdate = {
executor.submit(updateHost, host, version): host
for host in hosts
}
for future in as_completed(futureUpdate):
host = futureUpdate[future]
result = future.result()
if result['OK']:
updateSuccess.append(host)
else:
updateFail.append(host)
if not updateFail:
gLogger.notice("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
gLogger.notice("!!! Successfully updated all hosts !!!")
gLogger.notice("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
return S_OK([updateSuccess, updateFail])
if not updateSuccess:
gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
gLogger.notice("XXXXX Failed to update all hosts XXXXX")
gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
return S_OK([updateSuccess, updateFail])
gLogger.notice("X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!")
gLogger.notice("X!X!X Partially updated hosts X!X!X!")
gLogger.notice("Succeeded to update:")
for host in updateSuccess:
gLogger.notice(" + %s" % host)
gLogger.notice("Failed to update:")
for host in updateFail:
gLogger.notice(" - %s" % host)
gLogger.notice("X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!")
return S_OK([updateSuccess, updateFail])
def restartHost(hostName):
"""
Restart all systems and components of a host
:param str hostName: name of the host you want to restart
"""
host, port = parseHostname(hostName)
gLogger.notice("Pinging %s ..." % host)
client = SystemAdministratorClient(host, port)
result = client.ping()
if not result['OK']:
gLogger.error("Could not connect to %s: %s" %
(host, result['Message']))
return result
gLogger.notice("Host %s is active" % host)
gLogger.notice("Initiating restart of all systems and components")
# This restart call will always return S_ERROR because of SystemAdministrator restart
# Connection will be lost to the host
result = client.restartComponent('*', '*')
if result['Message'] == "Peer closed connection":
gLogger.notice(
"Restarted all systems on %s : connection to SystemAdministrator lost"
% host)
return S_OK(result['Message'])
gLogger.error("Received unxpected message: %s" % result['Message'])
return result
def updateInstance(version, hosts, retry):
"""
Update each server of an instance and restart them
:param str version: version vArBpC you want to update to
:param list[str] hosts: list of hosts to be updated
:param int retry: number of retry attempts on hosts that have failed to update
"""
result = updateHosts(version, hosts)
if not result['OK']:
return result
updateSuccess = result['Value'][0]
updateFail = result['Value'][1]
restartSuccess = []
restartFail = []
for host in updateSuccess:
result = restartHost(host)
if result['OK']:
restartSuccess.append(host)
else:
restartFail.append(host)
if not restartFail and not updateFail:
return S_OK("Successfully updated and restarted all hosts")
gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
gLogger.notice("XXXXX There were problems in the update process XXXXX")
gLogger.notice("Succeeded to update:")
for host in updateSuccess:
gLogger.notice(" + %s" % host)
gLogger.notice("Succeeded to restart:")
for host in restartSuccess:
gLogger.notice(" + %s" % host)
gLogger.notice("Failed to update:")
for host in updateFail:
gLogger.notice(" - %s" % host)
gLogger.notice("Failed to restart:")
for host in restartFail:
gLogger.notice(" - %s" % host)
gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
if retry > 0:
retryHosts = list(set(updateFail + restartFail))
gLogger.notice("Retrying update on (%s attempts remaining):" %
retry)
for host in retryHosts:
gLogger.notice(" - %s" % host)
gLogger.notice(
"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
return updateInstance(version, retryHosts, retry - 1)
return S_ERROR("Update failed!")
result = updateInstance(version, hosts, retry)
if not result['OK']:
gLogger.fatal(result['Message'])
DIRAC.exit(1)
gLogger.notice(result['Value'])
DIRAC.exit(0)
def main():
Script.registerSwitch("D", "sync",
"Make target directory identical to source")
Script.registerSwitch("j:", "parallel=",
"Multithreaded download and upload")
Script.parseCommandLine(ignoreErrors=False)
args = Script.getPositionalArgs()
if len(args) < 1 or len(args) > 3:
Script.showHelp()
sync = False
parallel = 1
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "s" or switch[0].lower() == "sync":
sync = True
if switch[0].lower() == "j" or switch[0].lower() == "parallel":
parallel = int(switch[1])
from DIRAC import S_OK, S_ERROR
from DIRAC import gConfig, gLogger
from DIRAC.Core.Utilities.List import breakListIntoChunks
from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
from DIRAC.Core.Utilities.File import mkDir
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC.Resources.Storage.StorageElement import StorageElement
def getSetOfLocalDirectoriesAndFiles(path):
"""Return a set of all directories and subdirectories and a set of
files contained therein for a given local path
"""
fullPath = os.path.abspath(path)
if not os.path.isdir(fullPath):
return S_ERROR("The path: " + fullPath + " does not exist!")
directories = set()
files = set()
for dirname, dirnames, filenames in os.walk(path):
# add path to all subdirectories first.
for subdirname in dirnames:
fullSubdirname = os.path.join(dirname, subdirname)
fullSubdirname = os.path.abspath(fullSubdirname)
fullSubdirname = fullSubdirname.replace(fullPath,
'').lstrip('/')
directories.add(fullSubdirname)
# add path to all filenames.
for filename in filenames:
fullFilename = os.path.join(dirname, filename)
fullFilename = os.path.abspath(fullFilename)
fullFilename = fullFilename.replace(fullPath, '').lstrip('/')
fileSize = os.path.getsize(fullPath + "/" + fullFilename)
if fileSize > 0:
files.add((fullFilename, int(fileSize)))
tree = {}
tree["Directories"] = directories
tree["Files"] = files
return S_OK(tree)
def getSetOfRemoteSubDirectoriesAndFiles(path, fc, directories, files):
"""
Recursively traverses all the subdirectories of a directory and returns a set of directories and files
"""
result = fc.listDirectory(path)
if result['OK']:
if result['Value']['Successful']:
for entry in result['Value']['Successful'][path]['Files']:
size = result['Value']['Successful'][path]['Files'][entry][
'MetaData']['Size']
files.add((entry, size))
for entry in result['Value']['Successful'][path]['SubDirs']:
directories.add(entry)
res = getSetOfRemoteSubDirectoriesAndFiles(
entry, fc, directories, files)
if not res['OK']:
return S_ERROR('Error: ' + res['Message'])
return S_OK()
else:
return S_ERROR("Error: %s" % result['Value'])
else:
return S_ERROR("Error:" + result['Message'])
def getSetOfRemoteDirectoriesAndFiles(fc, path):
"""
Return a set of all directories and subdirectories and the therein contained files for a given LFN
"""
directories = set()
files = set()
res = getSetOfRemoteSubDirectoriesAndFiles(path, fc, directories,
files)
if not res['OK']:
return S_ERROR('Could not list remote directory: ' +
res['Message'])
return_directories = set()
return_files = set()
for myfile in files:
return_files.add((myfile[0].replace(path,
'').lstrip('/'), myfile[1]))
for mydirectory in directories:
return_directories.add(mydirectory.replace(path, '').lstrip('/'))
tree = {}
tree["Directories"] = return_directories
tree["Files"] = return_files
return S_OK(tree)
def isInFileCatalog(fc, path):
"""
Check if the file is in the File Catalog
"""
result = fc.listDirectory(path)
if result['OK']:
if result['Value']['Successful']:
return S_OK()
else:
return S_ERROR()
else:
return S_ERROR()
def getContentToSync(upload, fc, source_dir, dest_dir):
"""
Return list of files and directories to be create and deleted
"""
if upload:
res = getSetOfRemoteDirectoriesAndFiles(fc, dest_dir)
if not res['OK']:
return S_ERROR(res['Message'])
to_dirs = res['Value']['Directories']
to_files = res['Value']['Files']
res = getSetOfLocalDirectoriesAndFiles(source_dir)
if not res['OK']:
return S_ERROR(res['Message'])
from_dirs = res['Value']['Directories']
from_files = res['Value']['Files']
else:
res = getSetOfLocalDirectoriesAndFiles(dest_dir)
if not res['OK']:
return S_ERROR(res['Message'])
to_dirs = res['Value']['Directories']
to_files = res['Value']['Files']
res = getSetOfRemoteDirectoriesAndFiles(fc, source_dir)
if not res['OK']:
return S_ERROR(res['Message'])
from_dirs = res['Value']['Directories']
from_files = res['Value']['Files']
# Create list of directories to delete
dirs_delete = list(to_dirs - from_dirs)
# Sort the list by depth of directory tree
dirs_delete.sort(key=lambda s: -s.count('/'))
# Create list of directories to create
dirs_create = list(from_dirs - to_dirs)
# Sort the list by depth of directory tree
dirs_create.sort(key=lambda s: s.count('/'))
# Flatten the list of pairs (filename, size) to list of filename
files_delete = [pair[0] for pair in list(to_files - from_files)]
files_create = [pair[0] for pair in list(from_files - to_files)]
create = {}
create["Directories"] = dirs_create
create["Files"] = files_create
delete = {}
delete["Directories"] = dirs_delete
delete["Files"] = files_delete
tree = {}
tree["Create"] = create
tree["Delete"] = delete
return S_OK(tree)
def removeRemoteFiles(dm, lfns):
"""
Remove file from the catalog
"""
for lfnList in breakListIntoChunks(lfns, 100):
res = dm.removeFile(lfnList)
if not res['OK']:
return S_ERROR("Failed to remove files:" + lfnList +
res['Message'])
else:
return S_OK()
def uploadLocalFile(dm, lfn, localfile, storage):
"""
Upload a local file to a storage element
"""
res = dm.putAndRegister(lfn, localfile, storage, None)
if not res['OK']:
return S_ERROR('Error: failed to upload %s to %s' % (lfn, storage))
else:
return S_OK('Successfully uploaded file to %s' % storage)
def downloadRemoteFile(dm, lfn, destination):
"""
Download a file from the system
"""
res = dm.getFile(lfn, destination)
if not res['OK']:
return S_ERROR('Error: failed to download %s ' % lfn)
else:
return S_OK('Successfully uploaded file %s' % lfn)
def removeStorageDirectoryFromSE(directory, storageElement):
"""
Delete directory on selected storage element
"""
se = StorageElement(storageElement, False)
res = returnSingleResult(se.exists(directory))
if not res['OK']:
return S_ERROR("Failed to obtain existence of directory" +
res['Message'])
exists = res['Value']
if not exists:
return S_OK("The directory %s does not exist at %s " %
(directory, storageElement))
res = returnSingleResult(se.removeDirectory(directory, recursive=True))
if not res['OK']:
return S_ERROR("Failed to remove storage directory" +
res['Message'])
return S_OK()
def removeRemoteDirectory(fc, lfn):
"""
Remove file from the catalog
"""
storageElements = gConfig.getValue(
'Resources/StorageElementGroups/SE_Cleaning_List', [])
for storageElement in sorted(storageElements):
res = removeStorageDirectoryFromSE(lfn, storageElement)
if not res['OK']:
return S_ERROR("Failed to clean storage directory at all SE:" +
res['Message'])
res = returnSingleResult(fc.removeDirectory(lfn, recursive=True))
if not res['OK']:
return S_ERROR("Failed to clean storage directory at all SE:" +
res['Message'])
return S_OK("Successfully removed directory")
def createRemoteDirectory(fc, newdir):
"""
Create directory in file catalog
"""
result = fc.createDirectory(newdir)
if result['OK']:
if result['Value']['Successful'] and newdir in result['Value'][
'Successful']:
return S_OK("Successfully created directory:" + newdir)
elif result['Value']['Failed'] and newdir in result['Value'][
'Failed']:
return S_ERROR('Failed to create directory: ' +
result['Value']['Failed'][newdir])
else:
return S_ERROR('Failed to create directory:' + result['Message'])
def createLocalDirectory(directory):
"""
Create local directory
"""
mkDir(directory)
if not os.path.exists(directory):
return S_ERROR('Directory creation failed')
return S_OK('Created directory successfully')
def removeLocalFile(path):
"""
Remove local file
"""
try:
os.remove(path)
except OSError as e:
return S_ERROR('Directory creation failed:' + e.strerror)
if os.path.isfile(path):
return S_ERROR('File deleting failed')
return S_OK('Removed file successfully')
def removeLocaDirectory(path):
"""
Remove local directory
"""
try:
os.rmdir(path)
except OSError as e:
return S_ERROR('Deleting directory failed: ' + e.strerror)
if os.path.isdir(path):
return S_ERROR('Directory deleting failed')
return S_OK('Removed directory successfully')
def doUpload(fc, dm, result, source_dir, dest_dir, storage, delete,
nthreads):
"""
Wrapper for uploading files
"""
if delete:
lfns = [
dest_dir + "/" + filename
for filename in result['Value']['Delete']['Files']
]
if len(lfns) > 0:
res = removeRemoteFiles(dm, lfns)
if not res['OK']:
gLogger.fatal('Deleting of files: ' + lfns +
" -X- [FAILED]" + res['Message'])
DIRAC.exit(1)
else:
gLogger.notice("Deleting " + ', '.join(lfns) +
" -> [DONE]")
for directoryname in result['Value']['Delete']['Directories']:
res = removeRemoteDirectory(fc, dest_dir + "/" + directoryname)
if not res['OK']:
gLogger.fatal('Deleting of directory: ' + directoryname +
" -X- [FAILED] " + res['Message'])
DIRAC.exit(1)
else:
gLogger.notice("Deleting " + directoryname + " -> [DONE]")
for directoryname in result['Value']['Create']['Directories']:
res = createRemoteDirectory(fc, dest_dir + "/" + directoryname)
if not res['OK']:
gLogger.fatal('Creation of directory: ' + directoryname +
" -X- [FAILED] " + res['Message'])
DIRAC.exit(1)
else:
gLogger.notice("Creating " + directoryname + " -> [DONE]")
listOfFiles = result['Value']['Create']['Files']
# Chech that we do not have to many threads
if nthreads > len(listOfFiles):
nthreads = len(listOfFiles)
if nthreads == 0:
return S_OK('Upload finished successfully')
listOfListOfFiles = chunkList(listOfFiles, nthreads)
res = runInParallel(arguments=[dm, source_dir, dest_dir, storage],
listOfLists=listOfListOfFiles,
function=uploadListOfFiles)
if not res['OK']:
return S_ERROR("Upload of files failed")
return S_OK('Upload finished successfully')
def uploadListOfFiles(dm, source_dir, dest_dir, storage, listOfFiles, tID):
"""
Wrapper for multithreaded uploading of a list of files
"""
log = gLogger.getSubLogger("[Thread %s] " % tID)
threadLine = "[Thread %s]" % tID
for filename in listOfFiles:
res = uploadLocalFile(dm, dest_dir + "/" + filename,
source_dir + "/" + filename, storage)
if not res['OK']:
log.fatal(threadLine + ' Uploading ' + filename +
' -X- [FAILED] ' + res['Message'])
DIRAC.exit(1)
else:
log.notice(threadLine + " Uploading " + filename +
" -> [DONE]")
def doDownload(dm, result, source_dir, dest_dir, delete, nthreads):
"""
Wrapper for downloading files
"""
if delete:
for filename in result['Value']['Delete']['Files']:
res = removeLocalFile(dest_dir + "/" + filename)
if not res['OK']:
gLogger.fatal('Deleting of file: ' + filename +
' -X- [FAILED] ' + res['Message'])
DIRAC.exit(1)
else:
gLogger.notice("Deleting " + filename + " -> [DONE]")
for directoryname in result['Value']['Delete']['Directories']:
res = removeLocaDirectory(dest_dir + "/" + directoryname)
if not res['OK']:
gLogger.fatal('Deleting of directory: ' + directoryname +
' -X- [FAILED] ' + res['Message'])
DIRAC.exit(1)
else:
gLogger.notice("Deleting " + directoryname + " -> [DONE]")
for directoryname in result['Value']['Create']['Directories']:
res = createLocalDirectory(dest_dir + "/" + directoryname)
if not res['OK']:
gLogger.fatal('Creation of directory: ' + directoryname +
' -X- [FAILED] ' + res['Message'])
DIRAC.exit(1)
else:
gLogger.notice("Creating " + directoryname + " -> [DONE]")
listOfFiles = result['Value']['Create']['Files']
# Chech that we do not have to many threads
if nthreads > len(listOfFiles):
nthreads = len(listOfFiles)
if nthreads == 0:
return S_OK('Upload finished successfully')
listOfListOfFiles = chunkList(listOfFiles, nthreads)
res = runInParallel(
arguments=[dm, source_dir, dest_dir],
listOfLists=listOfListOfFiles,
function=downloadListOfFiles,
)
if not res['OK']:
return S_ERROR("Download of files failed")
return S_OK('Upload finished successfully')
def chunkList(alist, nchunks):
"""
Split a list into a list of equaliy sized lists
"""
avg = len(alist) / float(nchunks)
out = []
last = 0.0
while last < len(alist):
out.append(alist[int(last):int(last + avg)])
last += avg
return out
def downloadListOfFiles(dm, source_dir, dest_dir, listOfFiles, tID):
"""
Wrapper for multithreaded downloading of a list of files
"""
log = gLogger.getSubLogger("[Thread %s] " % tID)
threadLine = "[Thread %s]" % tID
for filename in listOfFiles:
res = downloadRemoteFile(
dm, source_dir + "/" + filename,
dest_dir + ("/" + filename).rsplit("/", 1)[0])
if not res['OK']:
log.fatal(threadLine + ' Downloading ' + filename +
' -X- [FAILED] ' + res['Message'])
DIRAC.exit(1)
else:
log.notice(threadLine + " Downloading " + filename +
" -> [DONE]")
def runInParallel(arguments, listOfLists, function):
"""
Helper for execution of uploads and downloads in parallel
"""
from multiprocessing import Process
processes = []
for tID, alist in enumerate(listOfLists):
argums = arguments + [alist] + [tID]
pro = Process(target=function, args=argums)
pro.start()
processes.append(pro)
for process in processes:
process.join()
for process in processes:
if process.exitcode == 1:
return S_ERROR()
return S_OK()
def syncDestinations(upload, source_dir, dest_dir, storage, delete,
nthreads):
"""
Top level wrapper to execute functions
"""
fc = FileCatalog()
dm = DataManager()
result = getContentToSync(upload, fc, source_dir, dest_dir)
if not result['OK']:
return S_ERROR(result['Message'])
if upload:
res = doUpload(fc, dm, result, source_dir, dest_dir, storage,
delete, nthreads)
if not res['OK']:
return S_ERROR('Upload failed: ' + res['Message'])
else:
res = doDownload(dm, result, source_dir, dest_dir, delete,
nthreads)
if not res['OK']:
return S_ERROR('Download failed: ' + res['Message'])
return S_OK('Mirroring successfully finished')
def run(parameters, delete, nthreads):
"""
The main user interface
"""
source_dir = parameters[0]
dest_dir = parameters[1]
upload = False
storage = None
if len(parameters) == 3:
storage = parameters[2]
source_dir = os.path.abspath(source_dir)
dest_dir = dest_dir.rstrip('/')
upload = True
if not os.path.isdir(source_dir):
gLogger.fatal("Source directory does not exist")
DIRAC.exit(1)
if len(parameters) == 2:
dest_dir = os.path.abspath(dest_dir)
source_dir = source_dir.rstrip('/')
if not os.path.isdir(dest_dir):
gLogger.fatal("Destination directory does not exist")
DIRAC.exit(1)
res = syncDestinations(upload, source_dir, dest_dir, storage, delete,
nthreads)
if not res['OK']:
return S_ERROR(res['Message'])
return S_OK("Successfully mirrored " + source_dir + " into " +
dest_dir)
returnValue = run(args, sync, parallel)
if not returnValue['OK']:
gLogger.fatal(returnValue['Message'])
DIRAC.exit(1)
else:
gLogger.notice(returnValue['Value'])
DIRAC.exit(0)
def setUserName(arg):
global userName
if userName or not arg:
Script.showHelp()
DIRAC.exit(-1)
userName = arg
def setUserDN(arg):
global userDN
if userDN or not arg:
Script.showHelp()
DIRAC.exit(-1)
userDN = arg
def addProperty(arg):
global groupProperties
if not arg:
Script.showHelp(exitCode=1)
if arg not in groupProperties:
groupProperties.append(arg)
def MCProdTSExample(args=None):
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.Core.Workflow.Parameter import Parameter
from DIRAC.Interfaces.API.Dirac import Dirac
from CTADIRAC.Interfaces.API.ProdTSJob import ProdTSJob
if (len(args) != 2):
Script.gLogger.notice('Wrong number of arguments')
Script.showHelp()
cfgfile = args[0]
simtelArrayConfig = args[1]
j = ProdTSJob()
j.setVersion('prod-2_13112014b')
j.setApplication('corsika_autoinputs')
j.setProdName('ConfigxxxTS')
#j.setPathRoot('/vo.cta.in2p3.fr/MC/PROD2/') # official
j.setPathRoot('/vo.cta.in2p3.fr/user/a/arrabito/MC/PROD2/') # for test
#mode = 'corsika_standalone'
#mode = 'corsika_simtel'
mode = 'corsika_simtel_dst'
start_run_number = 0
j.setParameters([
'fileCatalog.cfg', '--template', cfgfile, '--mode', mode, '-i',
str(start_run_number), '--run_number', '@{JOB_ID}', '-N', '25000',
'-S', simtelArrayConfig, '--savecorsika', 'False'
])
j.setInputSandbox(
['LFN:/vo.cta.in2p3.fr/MC/PROD2/Armazones2K/prod2_cfg.tar.gz'])
j.setOutputSandbox(['*.log'])
j.setCPUTime(200000)
### Temporary fix #######
j.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
j.workflow.addParameter(
Parameter("PRODUCTION_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
t = Transformation()
t.setTransformationName("ProdExemple") # This must be unique
#t.setTransformationGroup("Group1")
t.setType("MCSimulation")
t.setDescription("MC prod example")
t.setLongDescription("corsika-simtel production") #mandatory
t.setBody(j.workflow.toXML())
res = t.addTransformation() # Transformation is created here
if not res['OK']:
print res['Message']
DIRAC.exit(-1)
t.setStatus("Active")
t.setAgentType("Automatic")
def setGroupName(arg):
global groupName
if groupName or not arg:
Script.showHelp()
DIRAC.exit(-1)
groupName = arg
def addUserGroup(arg):
global userGroups
if not arg:
Script.showHelp(exitCode=1)
if arg not in userGroups:
userGroups.append(arg)
def main():
Script.registerSwitch("E:", "email=",
"Boolean True/False (True by default)")
Script.parseCommandLine(ignoreErrors=True)
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC import exit as DIRACExit, gConfig, gLogger
def getBoolean(value):
if value.lower() == 'true':
return True
elif value.lower() == 'false':
return False
else:
Script.showHelp()
email = True
for switch in Script.getUnprocessedSwitches():
if switch[0] == "email":
email = getBoolean(switch[1])
args = Script.getPositionalArgs()
if len(args) < 2:
Script.showHelp()
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
setup = gConfig.getValue('/DIRAC/Setup', '')
if not setup:
print('ERROR: Could not contact Configuration Service')
exitCode = 2
DIRACExit(exitCode)
# result = promptUser(
# 'All the elements that are associated with this site will be active, '
# 'are you sure about this action?'
# )
# if not result['OK'] or result['Value'] is 'n':
# print 'Script stopped'
# DIRACExit( 0 )
site = args[0]
comment = args[1]
result = diracAdmin.allowSite(site, comment, printOutput=True)
if not result['OK']:
errorList.append((site, result['Message']))
exitCode = 2
else:
if email:
userName = diracAdmin._getCurrentUser()
if not userName['OK']:
print('ERROR: Could not obtain current username from proxy')
exitCode = 2
DIRACExit(exitCode)
userName = userName['Value']
subject = '%s is added in site mask for %s setup' % (site, setup)
body = 'Site %s is added to the site mask for %s setup by %s on %s.\n\n' % (
site, setup, userName, time.asctime())
body += 'Comment:\n%s' % comment
addressPath = 'EMail/Production'
address = Operations().getValue(addressPath, '')
if not address:
gLogger.notice(
"'%s' not defined in Operations, can not send Mail\n" %
addressPath, body)
else:
result = diracAdmin.sendMail(address, subject, body)
else:
print('Automatic email disabled by flag.')
for error in errorList:
print("ERROR %s: %s" % error)
DIRACExit(exitCode)
def CorsikaSimtelExample(args=None):
from CTADIRAC.Interfaces.API.CorsikaSimtelJob import CorsikaSimtelJob
from DIRAC.Interfaces.API.Dirac import Dirac
j = CorsikaSimtelJob()
j.setVersion('prod-2_15122013')
j.setExecutable('corsika_autoinputs')
if len(args) != 4:
Script.showHelp()
if args[3] not in [
'STD', '4MSST', 'SCSST', 'ASTRI', 'NSBX3', 'NORTH', 'SCMST'
]:
Script.gLogger.notice("reprocessing configuration is incorrect:",
args[3])
DIRAC.exit(-1)
runMin = int(args[0])
runMax = int(args[1])
cfgfile = args[2]
simtelArrayConfig = args[3]
ilist = []
for i in range(runMin, runMax + 1):
run_number = '%06d' % i
ilist.append(run_number)
j.setGenericParametricInput(ilist)
j.setName('run%s')
j.setInputSandbox([
cfgfile, 'grid_prod2-repro.sh',
'LFN:/vo.cta.in2p3.fr/MC/PROD2/SVN-PROD2.tar.gz'
])
j.setParameters([
'--template', cfgfile, '--mode', 'corsika_simtel', '-S',
simtelArrayConfig
])
j.setOutputSandbox(
['corsika_autoinputs.log', 'simtel.log', 'applicationLog.txt'])
# Retrieve your Output Data
corsika_out = '*.corsika.gz'
corsikatar_out = '*.corsika.tar.gz'
sim_out = '*.simtel.gz'
log_out = '*.log.gz'
hist_out = '*.hdata.gz'
j.setOutputData([corsika_out, corsikatar_out, sim_out, log_out, hist_out])
j.setCPUTime(200000)
Script.gLogger.info(j._toJDL())
res = Dirac().submit(j)
print res
def main():
from DIRAC.Core.Base import Script
### DoCtaIrf options ##########################################################
Script.registerSwitch("A:", "analysis=", "Analysis Type", setAnalysisType)
Script.registerSwitch("C:", "cuts=", "Cuts Config", setCutsConfig)
Script.registerSwitch("R:", "runlist=", "Runlist", setRunlist)
Script.registerSwitch("Z:", "zenith=", "Zenith", setZenith)
Script.registerSwitch("O:", "offset=", "Offset", setOffset)
Script.registerSwitch("M:", "energy=", "Energy Method", setEnergyMethod)
Script.registerSwitch("T:", "arrayconfig=", "Array Configuration",
setArrayConfig)
Script.registerSwitch("P:", "particle=", "Particle Type", setParticleType)
## other options
Script.registerSwitch("V:", "version=", "HAP version", setVersion)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
from CTADIRAC.Core.Workflow.Modules.HapApplication import HapApplication
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
ha = HapApplication()
HapPack = 'HAP/' + version + '/HAP'
packs = ['HESS/v0.2/lib', 'HESS/v0.3/root', HapPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
continue
if localArea:
if checkSoftwarePackage(package, localArea())['OK']:
DIRAC.gLogger.notice('Package found in Local Area:', package)
continue
if installSoftwarePackage(package, localArea())['OK']:
continue
DIRAC.gLogger.error('Check Failed for software package:', package)
DIRAC.gLogger.error('Software package not available')
DIRAC.exit(-1)
ha.setSoftwarePackage(HapPack)
ha.hapExecutable = 'DoCtaIrf'
runlistdir = os.environ['PWD']
build_infile(runlist)
ha.hapArguments = [
analysistype, cutsconfig, runlistdir, runlist, zenith, offset,
arrayconfig, energymethod, particle
]
DIRAC.gLogger.notice('Executing Hap Application')
res = ha.execute()
if not res['OK']:
DIRAC.gLogger.error('Failed to execute Hap Application')
jobReport.setApplicationStatus('Hap Application: Failed')
DIRAC.exit(-1)
DIRAC.exit()
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch('', 'Job=', ' JobID[,jobID2,...]')
Script.registerSwitch('', 'Transformation=', ' transformation ID')
Script.registerSwitch(
'', 'Tasks=', ' Associated to --Transformation, list of taskIDs')
Script.registerSwitch('', 'Verbose', ' Print more information')
Script.registerSwitch('', 'Terse', ' Only print request status')
Script.registerSwitch('', 'Full', ' Print full request content')
Script.registerSwitch('', 'Status=',
' Select all requests in a given status')
Script.registerSwitch(
'', 'Since=',
' Associated to --Status, start date yyyy-mm-dd or nb of days (default= -one day'
)
Script.registerSwitch(
'', 'Until=', ' Associated to --Status, end date (default= now')
Script.registerSwitch(
'', 'Maximum=',
' Associated to --Status, max number of requests ')
Script.registerSwitch('', 'Reset',
' Reset Failed files to Waiting if any')
Script.registerSwitch('', 'Force', ' Force reset even if not Failed')
Script.registerSwitch(
'', 'All',
' (if --Status Failed) all requests, otherwise exclude irrecoverable failures'
)
Script.registerSwitch('', 'FixJob',
' Set job Done if the request is Done')
Script.registerSwitch('', 'Cancel', ' Cancel the request')
Script.registerSwitch('', 'ListJobs', ' List the corresponding jobs')
Script.registerSwitch(
'', 'TargetSE=', ' Select request only if that SE is in the targetSEs')
from DIRAC.Core.Base.Script import parseCommandLine
parseCommandLine()
import DIRAC
from DIRAC import gLogger
jobs = []
requestID = 0
transID = None
taskIDs = None
tasks = None
requests = []
full = False
verbose = False
status = None
until = None
since = None
terse = False
allR = False
reset = False
fixJob = False
maxRequests = 999999999999
cancel = False
listJobs = False
force = False
targetSE = set()
for switch in Script.getUnprocessedSwitches():
if switch[0] == 'Job':
jobs = []
job = "Unknown"
try:
for arg in switch[1].split(','):
if os.path.exists(arg):
with open(arg, 'r') as fp:
lines = fp.readlines()
for line in lines:
for job in line.split(','):
jobs += [int(job.strip())]
gLogger.notice("Found %d jobs in file %s" %
(len(jobs), arg))
else:
jobs.append(int(arg))
except TypeError:
gLogger.fatal("Invalid jobID", job)
elif switch[0] == 'Transformation':
try:
transID = int(switch[1])
except Exception:
gLogger.fatal('Invalid transID', switch[1])
elif switch[0] == 'Tasks':
try:
taskIDs = [int(task) for task in switch[1].split(',')]
except Exception:
gLogger.fatal('Invalid tasks', switch[1])
elif switch[0] == 'Full':
full = True
elif switch[0] == 'Verbose':
verbose = True
elif switch[0] == 'Terse':
terse = True
elif switch[0] == 'All':
allR = True
elif switch[0] == 'Reset':
reset = True
elif switch[0] == 'Force':
force = True
elif switch[0] == 'Status':
status = switch[1].capitalize()
elif switch[0] == 'Since':
since = convertDate(switch[1])
elif switch[0] == 'Until':
until = convertDate(switch[1])
elif switch[0] == 'FixJob':
fixJob = True
elif switch[0] == 'Cancel':
cancel = True
elif switch[0] == 'ListJobs':
listJobs = True
elif switch[0] == 'Maximum':
try:
maxRequests = int(switch[1])
except Exception:
pass
elif switch[0] == 'TargetSE':
targetSE = set(switch[1].split(','))
if reset and not force:
status = 'Failed'
if fixJob:
status = 'Done'
if terse:
verbose = True
if status:
if not until:
until = datetime.datetime.utcnow()
if not since:
since = until - datetime.timedelta(hours=24)
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.RequestManagementSystem.Client.ReqClient import printRequest, recoverableRequest
reqClient = ReqClient()
if transID:
if not taskIDs:
gLogger.fatal(
"If Transformation is set, a list of Tasks should also be set")
Script.showHelp(exitCode=2)
# In principle, the task name is unique, so the request name should be unique as well
# If ever this would not work anymore, we would need to use the transformationClient
# to fetch the ExternalID
requests = ['%08d_%08d' % (transID, task) for task in taskIDs]
allR = True
elif not jobs:
requests = []
# Get full list of arguments, with and without comma
for arg in [
x.strip() for arg in Script.getPositionalArgs()
for x in arg.split(',')
]:
if os.path.exists(arg):
lines = open(arg, 'r').readlines()
requests += [
reqID.strip() for line in lines
for reqID in line.split(',')
]
gLogger.notice("Found %d requests in file" % len(requests))
else:
requests.append(arg)
allR = True
else:
res = reqClient.getRequestIDsForJobs(jobs)
if not res['OK']:
gLogger.fatal("Error getting request for jobs", res['Message'])
DIRAC.exit(2)
if res['Value']['Failed']:
gLogger.error(
"No request found for jobs %s" %
','.join(sorted(str(job) for job in res['Value']['Failed'])))
requests = sorted(res['Value']['Successful'].values())
if requests:
allR = True
else:
DIRAC.exit(0)
if status and not requests:
allR = allR or status != 'Failed'
res = reqClient.getRequestIDsList([status],
limit=maxRequests,
since=since,
until=until)
if not res['OK']:
gLogger.error("Error getting requests:", res['Message'])
DIRAC.exit(2)
requests = [
reqID for reqID, _st, updTime in res['Value']
if updTime > since and updTime <= until and reqID
]
gLogger.notice('Obtained %d requests %s between %s and %s' %
(len(requests), status, since, until))
if not requests:
gLogger.notice('No request selected....')
Script.showHelp(exitCode=2)
okRequests = []
warningPrinted = False
jobIDList = []
for reqID in requests:
# We allow reqID to be the requestName if it is unique
try:
requestID = int(reqID)
except ValueError:
requestID = reqClient.getRequestIDForName(reqID)
if not requestID['OK']:
gLogger.notice(requestID['Message'])
continue
requestID = requestID['Value']
request = reqClient.peekRequest(requestID)
if not request["OK"]:
gLogger.error(request["Message"])
DIRAC.exit(-1)
request = request["Value"]
if not request:
gLogger.error("no such request %s" % requestID)
continue
# If no operation as the targetSE, skip
if targetSE:
found = False
for op in request:
if op.TargetSE and targetSE.intersection(
op.TargetSE.split(',')):
found = True
break
if not found:
continue
# keep a list of jobIDs if requested
if request.JobID and listJobs:
jobIDList.append(request.JobID)
if status and request.Status != status:
gLogger.notice(
"Request %s is not in requested status %s%s" %
(reqID, status, ' (cannot be reset)' if reset else ''))
continue
if fixJob and request.Status == 'Done' and request.JobID:
# The request is for a job and is Done, verify that the job is in the proper status
result = reqClient.finalizeRequest(request.RequestID,
request.JobID,
useCertificates=False)
if not result['OK']:
gLogger.error("Error finalizing job", result['Message'])
else:
gLogger.notice("Job %d updated to %s" %
(request.JobID, result['Value']))
continue
if cancel:
if request.Status not in ('Done', 'Failed'):
ret = reqClient.cancelRequest(requestID)
if not ret['OK']:
gLogger.error("Error canceling request %s" % reqID,
ret['Message'])
else:
gLogger.notice("Request %s cancelled" % reqID)
else:
gLogger.notice("Request %s is in status %s, not cancelled" %
(reqID, request.Status))
elif allR or recoverableRequest(request):
okRequests.append(str(requestID))
if reset:
gLogger.notice('============ Request %s =============' %
requestID)
ret = reqClient.resetFailedRequest(requestID, allR=allR)
if not ret['OK']:
gLogger.error("Error resetting request %s" % requestID,
ret['Message'])
else:
if len(requests) > 1:
gLogger.notice('\n===================================')
dbStatus = reqClient.getRequestStatus(requestID).get(
'Value', 'Unknown')
printRequest(request,
status=dbStatus,
full=full,
verbose=verbose,
terse=terse)
if listJobs:
gLogger.notice("List of %d jobs:\n" % len(jobIDList),
','.join(str(jobID) for jobID in jobIDList))
if status and okRequests:
from DIRAC.Core.Utilities.List import breakListIntoChunks
gLogger.notice('\nList of %d selected requests:' % len(okRequests))
for reqs in breakListIntoChunks(okRequests, 100):
gLogger.notice(','.join(reqs))
def setUserName(arg):
global userName
if userName or not arg:
Script.showHelp(exitCode=1)
userName = arg
def setUserDN(arg):
global userDN
if userDN or not arg:
Script.showHelp(exitCode=1)
userDN = arg
Remove the given file replica or a list of file replicas from the File Catalog and from the storage. Usage: %s <LFN | fileContainingLFNs> SE [SE] """ % Script.scriptName ) Script.parseCommandLine() from DIRAC.Core.Utilities.List import sortList, breakListIntoChunks from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager rm = ReplicaManager() import os, sys if len( sys.argv ) < 3: Script.showHelp() DIRAC.exit( -1 ) else: inputFileName = sys.argv[1] storageElementNames = sys.argv[2:] if os.path.exists( inputFileName ): inputFile = open( inputFileName, 'r' ) string = inputFile.read() lfns = [ lfn.strip() for lfn in string.splitlines() ] inputFile.close() else: lfns = [inputFileName] for lfnList in breakListIntoChunks( sortList( lfns, True ), 500 ): for storageElementName in storageElementNames:
def dirLUT(args=None):
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.Interfaces.API.Job import Job
user_script = './dirLUT.sh'
if (len(args) != 3):
Script.showHelp()
direction = args[0]
zenith = args[1]
diffuse = args[2]
if diffuse == "0":
diffName = "point"
elif diffuse == "1":
diffName = "diff"
else:
print "Invalid extension definition!"
Script.showHelp()
return 1
if zenith == "40":
zenName = "40deg"
elif zenith == "20":
zenName = "20deg"
else:
print "Invalid zenith definition!"
Script.showHelp()
return 1
if direction == "N":
directionName = "north"
# deg = "180"
elif direction == "S":
directionName = "south"
# deg = "0"
else:
print 'Wrong direction. It can only be "N" or "S".'
Script.showHelp()
return 1
listname = './training/gamma_trainLUT_%s_%s_%s.lfns' % (zenName, diffName,
direction)
with open(listname) as f:
totalEntries = sum(1 for _ in f)
# Number of files used per job
runN = 20
runMin = 0
runMax = totalEntries / runN
for i in range(runMin, runMax):
jobName = "%s_%s_%s_%s_%s" % (user_script, direction, zenName,
diffName, i)
jobOut = "%s_%s_%s%s.out" % (user_script, directionName, diffName, i)
script_args = "%s %s %s %s %s" % (direction, zenName, diffName, i,
runN)
j = Job()
j.setInputSandbox([
user_script, listname, "setupPackageMARS.sh", "CheckFileZombie.C"
])
j.setExecutable(user_script, script_args)
j.setOutputSandbox([jobOut, "applicationLog.txt"])
j.setName(jobName)
j.setBannedSites([
'LCG.MSFG.fr', 'LCG.M3PEC.fr', 'LCG.OBSPM.fr',
'LCG.UNI-DORTMUND.de', 'LCG.UNIV-LILLE.fr'
])
Script.gLogger.info(j._toJDL())
print "Submitting job %s" % (script_args)
Dirac().submit(j)
