def _getOutputs(): repoLocation = '' clip = _Params() clip.registerSwitches() Script.parseCommandLine( ignoreErrors = False ) repoLocation = clip.repo if not repoLocation: Script.showHelp() dexit(1) from DIRAC import gLogger from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac(True, repoLocation) exitCode = 0 res = dirac.monitorRepository(False) if not res['OK']: gLogger.error("Failed because %s" % res['Message']) dexit(1) res = dirac.retrieveRepositorySandboxes() if not res['OK']: gLogger.error("Failed because %s" % res['Message']) dexit(1) if clip.outputdata: res = dirac.retrieveRepositoryData() if not res['OK']: gLogger.error("Failed because %s" % res['Message']) exit(1) dexit(exitCode)
def parseSwitches(): ''' Parses the arguments passed by the user ''' Script.parseCommandLine( ignoreErrors = True ) args = Script.getPositionalArgs() if args: subLogger.error( "Found the following positional args '%s', but we only accept switches" % args ) subLogger.error( "Please, check documentation below" ) Script.showHelp() DIRACExit( 1 ) switches = dict( Script.getUnprocessedSwitches() ) for key in ( 'status', 'se','limit'): if not key in switches: print "You're not using switch --%s, query may take long!" % key if 'status' in switches.keys(): if not switches[ 'status' ] in ( 'New', 'Offline', 'Waiting','Failed','StageSubmitted','Staged' ): subLogger.error( "Found \"%s\" as Status value. Incorrect value used!" % switches[ 'status' ] ) subLogger.error( "Please, check documentation below" ) Script.showHelp() DIRACExit( 1 ) subLogger.debug( "The switches used are:" ) map( subLogger.debug, switches.iteritems() ) return switches
def getBoolean( value ): if value.lower() == 'true': return True elif value.lower() == 'false': return False else: Script.showHelp()
def _runFSTAgent(): """ read commands line params and run FST agent for a given transformation ID """ params = _Params() params.registerSwitches() Script.parseCommandLine() if not params.checkSettings()['OK']: Script.showHelp() dexit(1) from ILCDIRAC.ILCTransformationSystem.Agent.FileStatusTransformationAgent import FileStatusTransformationAgent fstAgent = FileStatusTransformationAgent('ILCTransformation/FileStatusTransformationAgent', 'ILCTransformation/FileStatusTransformationAgent', 'dirac-ilc-filestatus-transformation') fstAgent.log = gLogger fstAgent.enabled = params.enabled res = fstAgent.getTransformations(transID=params.transID) if not res['OK']: dexit(1) if not res['Value']: print("Transformation Not Found") dexit(1) trans = res['Value'][0] res = fstAgent.processTransformation( int(params.transID), trans['SourceSE'], trans['TargetSE'], trans['DataTransType']) if not res["OK"]: dexit(1) dexit(0)
def addProperty( arg ): global groupProperties if not arg: Script.showHelp() DIRAC.exit( -1 ) if not arg in groupProperties: groupProperties.append( arg )
def parseSwitches(): ''' Parses the arguments passed by the user ''' Script.parseCommandLine( ignoreErrors = True ) args = Script.getPositionalArgs() if args: subLogger.error( "Found the following positional args '%s', but we only accept switches" % args ) subLogger.error( "Please, check documentation below" ) Script.showHelp() DIRACExit( 1 ) switches = dict( Script.getUnprocessedSwitches() ) # Default values switches.setdefault( 'element', None ) if not switches[ 'element' ] in ( 'all', 'Site', 'Resource', 'Node', None ): subLogger.error( "Found %s as element switch" % switches[ 'element' ] ) subLogger.error( "Please, check documentation below" ) Script.showHelp() DIRACExit( 1 ) subLogger.debug( "The switches used are:" ) map( subLogger.debug, switches.iteritems() ) return switches
def main(): if len(args) < 1: Script.showHelp() return fixedStatuses = ['Total', 'Done', 'Failed', 'Running', 'Waiting', 'Deleted'] for taskID in args: taskID = int(taskID) progress = getTaskEventProgress(taskID) if not progress: continue print '='*80 print 'Task %s progress:' % taskID print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12 print '%-12s %-12s %-12s' % ('Status', 'Job Number', 'Event Number') print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12 printALine(progress, 'Total') print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12 for fixedStatus in fixedStatuses: if fixedStatus != 'Total': printALine(progress, fixedStatus) for p in progress: if p not in fixedStatuses: printALine(progress, p) print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12 print ''
def addUserName( arg ): global userNames if not arg: Script.showHelp() DIRAC.exit( -1 ) if not arg in userNames: userNames.append( arg )
def addProperty( arg ): global hostProperties if not arg: Script.showHelp() DIRAC.exit( -1 ) if not arg in hostProperties: hostProperties.append( arg )
def addUserGroup(arg): global userGroups if not arg: Script.showHelp() DIRAC.exit(-1) if not arg in userGroups: userGroups.append(arg)
def main(): if len(args) < 1: Script.showHelp() return for taskID in args: taskID = int(taskID) deleteTask(taskID) print ""
def setUserMail(arg): global userMail if userMail or not arg: Script.showHelp() DIRAC.exit(-1) if not arg.find("@") > 0: Script.gLogger.error("Not a valid mail address", arg) DIRAC.exit(-1) userMail = arg
def error( msg ): ''' Format error messages ''' subLogger.error( "\nERROR:" ) subLogger.error( "\t" + msg ) subLogger.error( "\tPlease, check documentation below" ) Script.showHelp() DIRACExit( 1 )
def main(): if len(args) < 1: Script.showHelp() return jobIDs = [] for jobID in args: jobIDs.append(int(jobID)) showJobs(jobIDs) print 'Totally %s job(s) displayed' % len(jobIDs)
def isOK(self): if not self.cliParams.releasesToBuild: gLogger.error("Missing releases to build!") Script.showHelp() return False if not self.cliParams.destination: self.cliParams.destination = tempfile.mkdtemp('DiracDist') else: mkDir(self.cliParams.destination) gLogger.notice("Will generate tarballs in %s" % self.cliParams.destination) return True
def _findInFC(): """Find something in the FileCatalog""" from DIRAC import exit as dexit clip = _Params() clip.registerSwitches() Script.parseCommandLine() args = Script.getPositionalArgs() if len(args)<2: Script.showHelp('ERROR: Not enough arguments') gLogger.error("Run %s --help" % SCRIPTNAME ) dexit(1) path = args[0] if path == '.': path = '/' ## Check that the first argument is not a MetaQuery if any( op in path for op in OPLIST ): gLogger.error("ERROR: Path '%s' is not a valid path! The first argument must be a path" % path) gLogger.error("Run %s --help" % SCRIPTNAME ) dexit(1) gLogger.verbose("Path:", path) metaQuery = args[1:] metaDataDict = _createQueryDict(metaQuery) gLogger.verbose("Query:",str(metaDataDict)) if not metaDataDict: gLogger.info("No query") dexit(1) fc = FileCatalogClient() res = fc.findFilesByMetadata(metaDataDict, path) if not res['OK']: gLogger.error(res['Message']) dexit(1) if not res['Value']: gLogger.notice("No files found") listToPrint = None if clip.printOnlyDirectories: listToPrint = set( "/".join(fullpath.split("/")[:-1]) for fullpath in res['Value'] ) else: listToPrint = res['Value'] for entry in listToPrint: print entry dexit(0)
def main(): if len(args) < 1: Script.showHelp() return status = ['Failed'] for option in options: (switch, val) = option if switch == 'a' or switch == 'all': status = [] for taskID in args: taskID = int(taskID) rescheduleTask(taskID, status) print ''
def _setSwitches(self): Script.registerSwitch("S:", "system=", "Systems to check, by default all of them are checked", self._setSystems) Script.registerSwitch("M", "modified", "Show entries which differ from the default", self._setShowModified) Script.registerSwitch("A", "added", "Show entries which do not exist in ConfigTemplate", self._setShowAdded) Script.registerSwitch("U", "missingSection", "Show sections which do not exist in the current configuration", self._setShowMissingSections) Script.registerSwitch("O", "missingOption", "Show options which do not exist in the current configuration", self._setShowMissingOptions) Script.setUsageMessage('\n'.join([self.__doc__, 'Usage:', ' %s [option|cfgfile] -[MAUO] [-S <system]' % Script.scriptName])) Script.parseCommandLine(ignoreErrors=True) if not any([self.showModified, self.showAdded, self.showMissingSections, self.showMissingOptions]): LOG.error("\nERROR: Set at least one of the flags M A U O") Script.showHelp()
def _getOutputData(): cliParams = _Params() cliParams.registerSwitches() Script.parseCommandLine( ignoreErrors = False ) if not cliParams.repo: Script.showHelp() dexit(2) from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac(True, cliParams.repo) exitCode = 0 dirac.monitorRepository(False) dirac.retrieveRepositoryData() dexit(exitCode)
def addUser(): """Add user to configuration service and other things""" clip = Params() clip.registerSwitches() Script.parseCommandLine() if not ( clip.certCN and clip.groups and clip.certDN and clip.uname): gLogger.error("Username, DN, CN, and groups have to be given") Script.showHelp() gLogger.notice("Add User to Egroup") addUserToEgroup(clip) if not clip.email: gLogger.fatal("No email defined and not found in phonebook, you have to provide it: -E<email>") dexit(1) userProps = {'DN': clip.certDN, 'Email': clip.email, 'CN': clip.certCN, 'Groups': clip.groups} gLogger.notice("Add User to CS") addUserToCS(clip, userProps) gLogger.notice("Add User to FC") addUserToFC(clip) gLogger.notice("Done")
def checkSettings(self): """check if all required parameters are set, print error message and return S_ERROR if not""" args = Script.getPositionalArgs() if len(args) < 4: self.errorMessages.append("ERROR: Not enough arguments") else: self.setProdID( args[0] ) self.setTargetSE( args[1] ) self.setSourceSE( args[2] ) self.setDatatype( args[3] ) self.checkProxy() if not self.errorMessages: return S_OK() gLogger.error("\n".join(self.errorMessages)) Script.showHelp() return S_ERROR()
def addSoftware(): """uploads, registers, and sends email about new software package""" cliParams = Params() cliParams.registerSwitches() Script.parseCommandLine( ignoreErrors = True ) platform = cliParams.platform appName = cliParams.name appVersion = cliParams.version comment = cliParams.comment tarball_loc = cliParams.tarball if not platform or not appName or not comment: Script.showHelp() dexit(2) softAdder = SoftwareAdder(platform, appName, tarball_loc, appVersion, comment) softAdder.addSoftware() gLogger.notice("All done!") dexit(0)
def _getProdLogs(): """get production log files from LogSE""" clip = _Params() clip.registerSwitch() Script.parseCommandLine() if not ( clip.logF or clip.logD or clip.prodid ): Script.showHelp() dexit(1) from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations ops = Operations() storageElementName = ops.getValue('/LogStorage/LogSE', 'LogSE') from DIRAC.Resources.Storage.StorageElement import StorageElementItem as StorageElement logSE = StorageElement(storageElementName) if clip.prodid and not ( clip.logD or clip.logF ): result = _getLogFolderFromID( clip ) if not result['OK']: gLogger.error( result['Message'] ) dexit(1) if clip.logD: if not clip.noPromptBeforeDL: res = promptUser('Are you sure you want to get ALL the files in this directory?') if not res['OK']: dexit() choice = res['Value'] if choice.lower()=='n': dexit(0) if isinstance(clip.logD, str): res = logSE.getDirectory(clip.logD, localPath=clip.outputdir) _printErrorReport(res) elif isinstance(clip.logD, list): for logdir in clip.logD: gLogger.notice('Getting log files from '+str(logdir)) res = logSE.getDirectory(logdir, localPath=clip.outputdir) _printErrorReport(res) if clip.logF: res = logSE.getFile(clip.logF, localPath = clip.outputdir) _printErrorReport(res)
def _createLFNList(): """create the LFnList""" cliparams = _Params() cliparams.registerSwitches() Script.parseCommandLine( ignoreErrors = False ) repoLocation = cliparams.repo if not repoLocation: Script.showHelp() dexit(2) from ILCDIRAC.Interfaces.API.DiracILC import DiracILC dirac = DiracILC(True, repoLocation) dirac.monitorRepository(False) lfns = [] lfns = dirac.retrieveRepositoryOutputDataLFNs() LOG.notice("lfnlist=[") for lfn in lfns : LOG.notice('"LFN:%s",' % lfn) LOG.notice("]") dexit(0)
def parseSwitches(): """ Parses the arguments passed by the user """ Script.parseCommandLine( ignoreErrors = True ) args = Script.getPositionalArgs() if args: subLogger.error( "Found the following positional args '%s', but we only accept switches" % args ) subLogger.error( "Please, check documentation below" ) Script.showHelp() DIRACExit( 1 ) switches = dict( Script.getUnprocessedSwitches() ) switches.setdefault( 'statusType' , None ) switches.setdefault( 'releaseToken', False ) for key in ( 'element', 'name', 'reason' ): if not key in switches: subLogger.error( "%s Switch missing" % key ) subLogger.error( "Please, check documentation below" ) Script.showHelp() DIRACExit( 1 ) if not switches[ 'element' ] in ( 'Site', 'Resource', 'Node' ): subLogger.error( "Found %s as element switch" % switches[ 'element' ] ) subLogger.error( "Please, check documentation below" ) Script.showHelp() DIRACExit( 1 ) subLogger.debug( "The switches used are:" ) map( subLogger.debug, switches.iteritems() ) return switches
def addSoftware(): """uploads, registers, and sends email about new software package""" cliParams = Params() cliParams.registerSwitches() Script.parseCommandLine( ignoreErrors = True ) consistent = cliParams.checkConsistency() if not consistent['OK']: gLogger.error("Error checking consistency:", consistent['Message']) Script.showHelp() dexit(2) softAdder = CVMFSAdder(cliParams) resCheck = softAdder.checkConsistency() if not resCheck['OK']: Script.showHelp() dexit(2) softAdder.addSoftware() gLogger.notice("All done!") dexit(0)
def main(): if len(args) < 1: Script.showHelp() return showJobNumber = False for option in options: (switch, val) = option if switch == 'j' or switch == 'job': showJobNumber = True for taskID in args: print '='*80 taskID = int(taskID) showTask(taskID) print '' showTaskHistories(taskID) print '' if showJobNumber: showTaskJobs(taskID) print ''
def parseSwitches(): ''' Parses the arguments passed by the user ''' Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if not len(args) == 2: Script.showHelp() params = {} params['ce'] = None params['site'] = None params['host'] = None params['vo'] = None params['info'] = args[0] ret = getProxyInfo(disableVOMS=True) if ret['OK'] and 'group' in ret['Value']: params['vo'] = getVOForGroup(ret['Value']['group']) else: Script.gLogger.error('Could not determine VO') Script.showHelp() if params['info'] in ['ce', 'ce-state', 'ce-cluster', 'ce-vo']: params['ce'] = args[1] elif params['info']in ['site', 'site-se']: params['site'] = args[1] else: Script.gLogger.error('Wrong argument value') Script.showHelp() for unprocSw in Script.getUnprocessedSwitches(): if unprocSw[0] in ("H", "host"): params['host'] = unprocSw[1] if unprocSw[0] in ("V", "vo"): params['vo'] = unprocSw[1] return params
def parseSwitches(): """ Parses the arguments passed by the user """ Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if args: subLogger.error("Found the following positional args '%s', but we only accept switches" % args) subLogger.error("Please, check documentation below") Script.showHelp() DIRACExit(1) switches = dict(Script.getUnprocessedSwitches()) # Default values switches.setdefault("elementType", None) switches.setdefault("name", None) switches.setdefault("tokenOwner", None) switches.setdefault("statusType", None) switches.setdefault("status", None) if not "element" in switches: subLogger.error("element Switch missing") subLogger.error("Please, check documentation below") Script.showHelp() DIRACExit(1) if not switches["element"] in ("Site", "Resource", "Node"): subLogger.error("Found %s as element switch" % switches["element"]) subLogger.error("Please, check documentation below") Script.showHelp() DIRACExit(1) subLogger.debug("The switches used are:") map(subLogger.debug, switches.iteritems()) return switches
from DIRAC.Core.Base import Script Script.setUsageMessage(""" Clean the given directory or a list of directories by removing it and all the contained files and subdirectories from the physical storage and from the file catalogs. Usage: %s <LFN_Directory | fileContainingLFN_Directories> """ % Script.scriptName) Script.parseCommandLine() args = Script.getPositionalArgs() if len(args) != 1: Script.showHelp(exitCode=1) inputFileName = args[0] if os.path.exists(inputFileName): lfns = [ lfn.strip().split()[0] for lfn in sorted(open(inputFileName, 'r').read().splitlines()) ] else: lfns = [inputFileName] from DIRAC.DataManagementSystem.Client.DataManager import DataManager dm = DataManager() retVal = 0 for lfn in [lfn for lfn in lfns if lfn]:
def main(): Script.parseCommandLine() from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient args = Script.getPositionalArgs() if len(args) < 1: Script.showHelp(exitCode=1) # get arguments prodID = args[0] prodClient = ProductionClient() transClient = TransformationClient() res = prodClient.getProductionTransformations(prodID) transIDs = [] if res['OK']: transList = res['Value'] if not transList: DIRAC.gLogger.notice( 'No transformation associated with production %s' % prodID) DIRAC.exit(-1) for trans in transList: transIDs.append(trans['TransformationID']) else: DIRAC.gLogger.error(res['Message']) DIRAC.exit(-1) fields = [ 'TransformationName', 'Status', 'F_Proc.', 'F_Proc.(%)', 'TransformationID', 'ProductionID', 'Prod_LastUpdate', 'Prod_InsertedTime' ] records = [] paramShowNames = [ 'TransformationID', 'TransformationName', 'Type', 'Status', 'Files_Total', 'Files_PercentProcessed', 'Files_Processed', 'Files_Unused', 'Jobs_TotalCreated', 'Jobs_Waiting', 'Jobs_Running', 'Jobs_Done', 'Jobs_Failed', 'Jobs_Stalled' ] resList = [] res = transClient.getTransformationSummaryWeb( {'TransformationID': transIDs}, [], 0, len(transIDs)) if not res['OK']: DIRAC.gLogger.error(res['Message']) DIRAC.exit(-1) if res['Value']['TotalRecords'] > 0: paramNames = res['Value']['ParameterNames'] for paramValues in res['Value']['Records']: paramShowValues = map( lambda pname: paramValues[paramNames.index(pname)], paramShowNames) showDict = dict(zip(paramShowNames, paramShowValues)) resList.append(showDict) for res in resList: files_Processed = res['Files_Processed'] files_PercentProcessed = res['Files_PercentProcessed'] status = res['Status'] type = res['Type'] transName = res['TransformationName'] transID = res['TransformationID'] records.append([ transName, status, str(files_Processed), str(files_PercentProcessed), str(transID), str(prodID), str(trans['LastUpdate']), str(trans['InsertedTime']) ]) printTable(fields, records) DIRAC.exit(0)
def addUserName(arg): global userNames if not arg: Script.showHelp(exitCode=1) if arg not in userNames: userNames.append(arg)
def main(): parameters = ['OwnerDN', 'StartExecTime', 'EndExecTime'] Script.registerSwitch( '', 'Parameters=', ' List of strings to be matched by job parameters or attributes') Script.parseCommandLine(ignoreErrors=True) for switch in Script.getUnprocessedSwitches(): if switch[0] == 'Parameters': parameters += [par for par in switch[1].split(',')] parameters = [(i, par.lower()) for i, par in enumerate(parameters) if par] args = Script.getPositionalArgs() if len(args) < 1: Script.showHelp() from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from DIRAC.Interfaces.API.Dirac import Dirac diracAdmin = DiracAdmin() dirac = Dirac() errorList = [] for gridID in args: result = {} res = diracAdmin.getPilotInfo(gridID) if not res['OK']: errorList.append((gridID, res['Message'])) else: jobIDs = set( [int(jobID) for jobID in res['Value'][gridID]['Jobs']]) totCPU = 0 totWall = 0 effRequested = False for jobID in sorted(jobIDs): result.setdefault(jobID, {}) for func in (dirac.getJobParameters, dirac.getJobAttributes): res = func(jobID) if not res['OK']: errorList.append(('Job %d' % jobID, res['Message'])) else: params = res['Value'] if 'TotalCPUTime(s)' in params: totCPU += float(params['TotalCPUTime(s)']) totWall += float(params['WallClockTime(s)']) params['CPUEfficiency'] = '%s %%' % ( 100. * params['TotalCPUTime(s)'] / params['WallClockTime(s)']) for i, par in parameters: for param in [ p for p in _stringInList( str(par), str(params)) if not _stringInList( str(p), str(result[jobID])) ]: if param == 'CPUEfficiency': effRequested = True result[jobID]['%d.%s' % (i, param)] = params[param] if effRequested: result['CPUEfficiency'] = '%s %%' % (100. * totCPU / totWall) print(diracAdmin.pPrint.pformat({gridID: result})) for error in errorList: print("ERROR %s: %s" % error)
def TrainRF(args=None): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job user_script = './trainRF.sh' if (len(args) != 3): Script.showHelp() direction = args[0] zenith = args[1] diffuse = args[2] site = "PARANAL" if diffuse == "0": diffName = "point" elif diffuse == "1": diffName = "diff" else: print "Invalid extension definition!" Script.showHelp() return 1 if zenith == "40": zenName = "40deg" elif zenith == "20": zenName = "20deg" else: print "Invalid zenith definition!" Script.showHelp() return 1 if direction == "N": directionName = "north" # deg = "180" elif direction == "S": directionName = "south" # deg = "0" else: print 'Wrong direction. It can only be "N" or "S".' Script.showHelp() return 1 # Macro fixing the file check before continues. ROOTmacro = "CTAtrain.C" # List of files over which the training should be done LFN_file_gammas = './training/gamma_ghtrain_%s_%s_%s.lfns' % ( zenName, diffName, direction) LFN_file_protons = './training/proton_ghtrain_%s_%s_%s.lfns' % ( zenName, diffName, direction) StatFile = './Statistic_train.txt' for telType in range(0, 6): jobName = "%s_%s_%s_%s_%s" % (user_script, directionName, diffName, telType, zenName) jobOut = "%s_%s_%s_%s.out" % (user_script, directionName, diffName, telType) script_args = "%s %s %s %s %s" % (direction, site, diffName, telType, zenName) j = Job() j.setInputSandbox([ user_script, "setupPackageMARS.sh", LFN_file_gammas, LFN_file_protons, ROOTmacro, StatFile ]) j.setExecutable(user_script, script_args) j.setOutputSandbox([jobOut, "applicationLog.txt"]) j.setName(jobName) Script.gLogger.info(j._toJDL()) print "Launching %s %s" % (user_script, script_args) Dirac().submit(j)
def main(): Script.parseCommandLine(ignoreErrors=False) args = Script.getPositionalArgs() if len(args) < 2: Script.showHelp() targetSE = args.pop(0) lfns = [] for inputFileName in args: if os.path.exists(inputFileName): inputFile = open(inputFileName, 'r') string = inputFile.read() inputFile.close() lfns.extend([lfn.strip() for lfn in string.splitlines()]) else: lfns.append(inputFileName) from DIRAC.Resources.Storage.StorageElement import StorageElement import DIRAC # Check is provided SE is OK if targetSE != 'All': se = StorageElement(targetSE) if not se.valid: print(se.errorReason) print() Script.showHelp() from DIRAC.RequestManagementSystem.Client.Request import Request from DIRAC.RequestManagementSystem.Client.Operation import Operation from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator from DIRAC.Resources.Catalog.FileCatalog import FileCatalog reqClient = ReqClient() fc = FileCatalog() requestOperation = 'RemoveReplica' if targetSE == 'All': requestOperation = 'RemoveFile' for lfnList in breakListIntoChunks(lfns, 100): oRequest = Request() requestName = "%s_%s" % ( md5(repr(time.time()).encode()).hexdigest()[:16], md5(repr(time.time()).encode()).hexdigest()[:16], ) oRequest.RequestName = requestName oOperation = Operation() oOperation.Type = requestOperation oOperation.TargetSE = targetSE res = fc.getFileMetadata(lfnList) if not res['OK']: print("Can't get file metadata: %s" % res['Message']) DIRAC.exit(1) if res['Value']['Failed']: print( "Could not get the file metadata of the following, so skipping them:" ) for fFile in res['Value']['Failed']: print(fFile) lfnMetadata = res['Value']['Successful'] for lfn in lfnMetadata: rarFile = File() rarFile.LFN = lfn rarFile.Size = lfnMetadata[lfn]['Size'] rarFile.Checksum = lfnMetadata[lfn]['Checksum'] rarFile.GUID = lfnMetadata[lfn]['GUID'] rarFile.ChecksumType = 'ADLER32' oOperation.addFile(rarFile) oRequest.addOperation(oOperation) isValid = RequestValidator().validate(oRequest) if not isValid['OK']: print("Request is not valid: ", isValid['Message']) DIRAC.exit(1) result = reqClient.putRequest(oRequest) if result['OK']: print('Request %d Submitted' % result['Value']) else: print('Failed to submit Request: ', result['Message'])
def main(): from DIRAC.Core.Base import Script Script.registerSwitch("T:", "template=", "Corsika Template") Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig") Script.registerSwitch("p:", "run_number=", "Do not use: Run Number automatically set") Script.registerSwitch("E:", "executable=", "Executable (Use SetExecutable)") Script.registerSwitch("V:", "version=", "Version (Use setVersion)") Script.registerSwitch("M:", "mode=", "Mode (corsika_standalone/corsika_simtelarray)") Script.parseCommandLine(ignoreErrors=True) ## default values ############## run_number = None corsikaTemplate = None simtelConfig = None executable = None version = None mode = 'corsika_simtelarray' ### set switch values ### for switch in Script.getUnprocessedSwitches(): if switch[0] == "run_number" or switch[0] == "p": run_number = switch[1].split('ParametricParameters=')[1] elif switch[0] == "template" or switch[0] == "T": corsikaTemplate = switch[1] elif switch[0] == "simtelConfig" or switch[0] == "S": simtelConfig = switch[1] elif switch[0] == "executable" or switch[0] == "E": executable = switch[1] elif switch[0] == "version" or switch[0] == "V": version = switch[1] elif switch[0] == "mode" or switch[0] == "M": mode = switch[1] if version == None or executable == None or run_number == None or corsikaTemplate == None: Script.showHelp() jobReport.setApplicationStatus('Missing options') DIRAC.exit(-1) from CTADIRAC.Core.Workflow.Modules.CorsikaApp import CorsikaApp from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea from DIRAC.Core.Utilities.Subprocess import systemCall from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport jobID = os.environ['JOBID'] jobID = int(jobID) jobReport = JobReport(jobID) createGlobalsFromConfigFiles(corsikaTemplate, version) ############ Producing Corsika File install_CorsikaSimtelPack(version) CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray' cs = CorsikaApp() cs.setSoftwarePackage(CorsikaSimtelPack) cs.csExe = executable cs.csArguments = [ '--run-number', run_number, '--run', 'corsika', corsikaTemplate ] corsikaReturnCode = cs.execute() if corsikaReturnCode != 0: DIRAC.gLogger.error('Corsika Application: Failed') jobReport.setApplicationStatus('Corsika Application: Failed') DIRAC.exit(-1) ###################### rename of corsika output file ####################### rundir = 'run' + run_number filein = rundir + '/' + corsikaOutputFileName corsikaFileName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.gz' mv_cmd = 'mv ' + filein + ' ' + corsikaFileName if (os.system(mv_cmd)): DIRAC.exit(-1) ### create corsika tar luisa #################### corsikaTarName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.tar.gz' filetar1 = rundir + '/' + 'input' filetar2 = rundir + '/' + 'DAT' + run_number + '.dbase' filetar3 = rundir + '/run' + str(int(run_number)) + '.log' cmdTuple = [ '/bin/tar', 'zcf', corsikaTarName, filetar1, filetar2, filetar3 ] DIRAC.gLogger.notice('Executing command tuple:', cmdTuple) ret = systemCall(0, cmdTuple, sendOutput) if not ret['OK']: DIRAC.gLogger.error('Failed to execute tar') DIRAC.exit(-1) if (mode == 'corsika_standalone'): DIRAC.exit() ############ Producing SimTel File ######################Building simtel Directory Metadata ####################### cfg_dict = { "4MSST": 'cta-prod2-4m-dc', "SCSST": 'cta-prod2-sc-sst', "STD": 'cta-prod2', "NSBX3": 'cta-prod2', "ASTRI": 'cta-prod2-astri', "NORTH": 'cta-prod2n', "SCMST": 'cta-prod2-sc3' } #if simtelConfig=="6INROW": # all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD","SCMST"] #elif simtelConfig=="5INROW": # all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD"] #else: # all_configs=[simtelConfig] all_configs = [simtelConfig] for current_conf in all_configs: DIRAC.gLogger.notice('current conf is', current_conf) current_version = version DIRAC.gLogger.notice('current version is', current_version) #if current_conf == "SCMST": # current_version = version + '_sc3' # DIRAC.gLogger.notice('current version is', current_version) # if os.path.isdir('sim_telarray'): # DIRAC.gLogger.notice('Package found in the local area. Removing package...') # cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run' # if(os.system(cmd)): # DIRAC.exit( -1 ) # install_CorsikaSimtelPack(current_version) #else: # current_version = version # DIRAC.gLogger.notice('current version is', current_version) #### execute simtelarray ################ fd = open('run_sim.sh', 'w') fd.write("""#! /bin/sh export SVNPROD2=$PWD export SVNTAG=SVN-PROD2 export CORSIKA_IO_BUFFER=800MB ./grid_prod2-repro.sh %s %s""" % (corsikaFileName, current_conf)) fd.close() #################################### os.system('chmod u+x run_sim.sh') os.system('chmod u+x grid_prod2-repro.sh') cmdTuple = ['./run_sim.sh'] ret = systemCall(0, cmdTuple, sendSimtelOutput) simtelReturnCode, stdout, stderr = ret['Value'] if (os.system('grep Broken simtel.log') == 0): DIRAC.gLogger.error('Broken string found in simtel.log') jobReport.setApplicationStatus('Broken pipe') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Failed to execute run_sim.sh') DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode) DIRAC.exit(-1) ## check simtel data/log/histo Output File exist cfg = cfg_dict[current_conf] simtelFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str( obslev) + '_' + 'run' + run_number + '.simtel.gz' cmd = 'mv Data/sim_telarray/' + cfg + '/*/Data/*.simtel.gz ' + simtelFileName if (os.system(cmd)): DIRAC.exit(-1) simtelLogFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz' cmd = 'mv Data/sim_telarray/' + cfg + '/*/Log/*.log.gz ' + simtelLogFileName if (os.system(cmd)): DIRAC.exit(-1) simtelHistFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str( obslev) + '_' + 'run' + run_number + '.hdata.gz' cmd = 'mv Data/sim_telarray/' + cfg + '/*/Histograms/*.hdata.gz ' + simtelHistFileName if (os.system(cmd)): DIRAC.exit(-1) ################################################################ DIRAC.exit()
' or Upload', ' %s Path LFN SE' % Script.scriptName, 'Arguments:', ' LFN: Logical File Name (Path to directory)', ' Path: Local path to the file (Path to directory)', ' SE: DIRAC Storage Element'] ) ) Script.registerSwitch( "D" , "sync" , "Make target directory identical to source" ) Script.registerSwitch( "j:" , "parallel=" , "Multithreaded download and upload" ) Script.parseCommandLine( ignoreErrors = False ) args = Script.getPositionalArgs() if len( args ) < 1 or len( args ) > 3: Script.showHelp() sync = False parallel = 1 for switch in Script.getUnprocessedSwitches(): if switch[0].lower() == "s" or switch[0].lower() == "sync": sync = True if switch[0].lower() == "j" or switch[0].lower() == "parallel": parallel = int(switch[1]) from DIRAC import S_OK, S_ERROR from DIRAC import gConfig, gLogger from DIRAC.Core.Utilities.ReturnValues import returnSingleResult from DIRAC.Core.Utilities.File import mkDir from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
def setHostDN(arg): global hostDN if hostDN or not arg: Script.showHelp() DIRAC.exit(-1) hostDN = arg
def main(): from DIRAC.Core.Base import Script ### DoCtaIrf options ########################################################## Script.registerSwitch("A:", "analysis=", "Analysis Type", setAnalysisType) Script.registerSwitch("C:", "cuts=", "Cuts Config", setCutsConfig) Script.registerSwitch("R:", "runlist=", "Runlist", setRunlist) Script.registerSwitch("Z:", "zenith=", "Zenith", setZenith) Script.registerSwitch("O:", "offset=", "Offset", setOffset) Script.registerSwitch("M:", "energy=", "Energy Method", setEnergyMethod) Script.registerSwitch("T:", "arrayconfig=", "Array Configuration", setArrayConfig) Script.registerSwitch("P:", "particle=", "Particle Type", setParticleType) ## other options Script.registerSwitch("V:", "version=", "HAP version", setVersion) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) < 1: Script.showHelp() from CTADIRAC.Core.Workflow.Modules.HapApplication import HapApplication from CTADIRAC.Core.Workflow.Modules.HapRootMacro import HapRootMacro from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from DIRAC.Core.Utilities.Subprocess import systemCall from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport jobID = os.environ['JOBID'] jobID = int(jobID) jobReport = JobReport(jobID) ha = HapApplication() HapPack = 'HAP/' + version + '/HAP' packs = ['HESS/v0.2/lib', 'HESS/v0.3/root', HapPack] for package in packs: DIRAC.gLogger.notice('Checking:', package) if sharedArea: if checkSoftwarePackage(package, sharedArea())['OK']: DIRAC.gLogger.notice('Package found in Shared Area:', package) continue if localArea: if checkSoftwarePackage(package, localArea())['OK']: DIRAC.gLogger.notice('Package found in Local Area:', package) continue if installSoftwarePackage(package, localArea())['OK']: continue DIRAC.gLogger.error('Check Failed for software package:', package) DIRAC.gLogger.error('Software package not available') DIRAC.exit(-1) ha.setSoftwarePackage(HapPack) ha.hapExecutable = 'DoCtaIrf' runlistdir = os.environ['PWD'] build_infile(runlist) ha.hapArguments = [ analysistype, cutsconfig, runlistdir, runlist, zenith, offset, arrayconfig, energymethod, particle ] DIRAC.gLogger.notice('Executing Hap Application') res = ha.execute() if not res['OK']: DIRAC.gLogger.error('Failed to execute Hap Application') jobReport.setApplicationStatus('Hap Application: Failed') DIRAC.exit(-1) ###################### Check TTree Output File ####################### outfile = 'MVAFile_' + runlist + ".root" if not os.path.isfile(outfile): error = 'TTree file was not created:' DIRAC.gLogger.error(error, outfile) jobReport.setApplicationStatus('DoCtaIrf: TTree file not created') DIRAC.exit(-1) ###################### Quality Check for TTree Output File: step0###################### hr = HapRootMacro() hr.setSoftwarePackage(HapPack) DIRAC.gLogger.notice('Executing TTree check step0') hr.rootMacro = '/hapscripts/mva/Open_TT.C+' outfilestr = '"' + outfile + '"' args = [outfilestr] DIRAC.gLogger.notice('Open_TT macro Arguments:', args) hr.rootArguments = args DIRAC.gLogger.notice('Executing Hap Open_TT macro') res = hr.execute() if not res['OK']: DIRAC.gLogger.error('Open_TT: Failed') DIRAC.exit(-1) #########################Quality Check for TTree Output File: step1#################### DIRAC.gLogger.notice('Executing TTree check step1') ret = getSoftwareEnviron(HapPack) if not ret['OK']: error = ret['Message'] DIRAC.gLogger.error(error, HapPack) DIRAC.exit(-1) hapEnviron = ret['Value'] hessroot = hapEnviron['HESSROOT'] check_script = hessroot + '/hapscripts/mva/check_TT.csh' cmdTuple = [check_script] ret = systemCall(0, cmdTuple, sendOutput) if not ret['OK']: DIRAC.gLogger.error('Failed to execute TTree Check step1') jobReport.setApplicationStatus('Check_TTree: Failed') DIRAC.exit(-1) ############################################# DIRAC.exit()
def setHostName(arg): global hostName if hostName or not arg: Script.showHelp() DIRAC.exit(-1) hostName = arg
def main(): Script.registerSwitch( "v:", "vo=", "Location of pilot version in CS /Operations/<vo>/Pilot/Version" " (default value specified in CS under /DIRAC/DefaultSetup)") Script.parseCommandLine(ignoreErrors=False) args = Script.getPositionalArgs() if len(args) < 1 or len(args) > 2: Script.showHelp() version = args[0] vo = None for switch in Script.getUnprocessedSwitches(): if switch[0] == "v" or switch[0] == "vo": vo = switch[1] from DIRAC import S_OK, S_ERROR from DIRAC import gConfig, gLogger from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI def updatePilot(version, vo): """ Update in the CS the pilot version used, If only one version present in CS it's overwritten. If two versions present, the new one is added and the last removed :param version: version vArBpC of pilot you want to use :param vo: Location of pilot version in CS /Operations/<vo>/Pilot/Version """ setup = vo if not vo: setup = gConfig.getValue('/DIRAC/DefaultSetup') if not setup: return S_ERROR("No value set for /DIRAC/DefaultSetup in CS") pilotVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup, []) if not pilotVersion: return S_ERROR( "No pilot version set under Operations/%s/Pilot/Version in CS" % setup) pilotVersion.pop() pilotVersion.insert(0, version) api = CSAPI() api.setOption('Operations/%s/Pilot/Version' % setup, ", ".join(pilotVersion)) result = api.commit() if not result['OK']: gLogger.fatal('Could not commit new version of pilot!') return result newVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup) return S_OK("New version of pilot set to %s" % newVersion) result = updatePilot(version, vo) if not result['OK']: gLogger.fatal(result['Message']) DIRAC.exit(1) gLogger.notice(result['Value']) DIRAC.exit(0)
def main(): from DIRAC.Core.Base import Script Script.registerSwitch("p:", "inputfile=", "Input File") Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig") Script.registerSwitch("V:", "version=", "Version (Use setVersion)") Script.registerSwitch("C:", "comp=", "Compile (True/False)") Script.parseCommandLine(ignoreErrors=True) ## default values ############## simtelConfig = None version = None comp = True ### set switch values ### for switch in Script.getUnprocessedSwitches(): if switch[0] == "inputfile" or switch[0] == "p": corsikaFileLFN = switch[1].split('ParametricInputData=LFN:')[1] elif switch[0] == "simtelConfig" or switch[0] == "S": simtelConfig = switch[1] elif switch[0] == "version" or switch[0] == "V": version = switch[1] elif switch[0] == "comp" or switch[0] == "C": comp = switch[1] if version == None: Script.showHelp() jobReport.setApplicationStatus('Missing options') DIRAC.exit(-1) from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea from DIRAC.Core.Utilities.Subprocess import systemCall from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport jobID = os.environ['JOBID'] jobID = int(jobID) jobReport = JobReport(jobID) ############ Producing SimTel File if comp == True: install_CorsikaSimtelPack(version) else: CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray' res = installSoftwarePackage(CorsikaSimtelPack, workingArea()) if not res['OK']: DIRAC.gLogger.error('Failed to execute installSoftwarePackage', res) CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray' cfg_dict = { "4MSST": 'cta-prod2-4m-dc', "SCSST": 'cta-prod2-sc-sst', "STD": 'cta-prod2', "NSBX3": 'cta-prod2', "ASTRI": 'cta-prod2-astri', "NORTH": 'cta-prod2n', "ASTRISLN": 'cta-astri-sln' } #if simtelConfig=="6INROW": # all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD","SCMST"] #elif simtelConfig=="5INROW": # all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD"] #else: # all_configs=[simtelConfig] all_configs = [simtelConfig] for current_conf in all_configs: DIRAC.gLogger.notice('current conf is', current_conf) current_version = version DIRAC.gLogger.notice('current version is', current_version) #if current_conf == "SCMST": # current_version = version + '_sc3' # DIRAC.gLogger.notice('current version is', current_version) # if os.path.isdir('sim_telarray'): # DIRAC.gLogger.notice('Package found in the local area. Removing package...') # cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run' # if(os.system(cmd)): # DIRAC.exit( -1 ) # install_CorsikaSimtelPack(current_version) #else: # current_version = version # DIRAC.gLogger.notice('current version is', current_version) #### execute simtelarray ################ fd = open('run_sim.sh', 'w') fd.write("""#! /bin/sh export SVNPROD2=$PWD export SVNTAG=SVN-PROD2 export CORSIKA_IO_BUFFER=800MB ./grid_prod2-repro.sh %s %s""" % (os.path.basename(corsikaFileLFN), current_conf)) fd.close() #################################### os.system('chmod u+x run_sim.sh') os.system('chmod u+x grid_prod2-repro.sh') cmdTuple = ['./run_sim.sh'] ret = systemCall(0, cmdTuple, sendSimtelOutput) simtelReturnCode, stdout, stderr = ret['Value'] if (os.system('grep Broken simtel.log') == 0): DIRAC.gLogger.error('Broken string found in simtel.log') jobReport.setApplicationStatus('Broken pipe') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Failed to execute run_sim.sh') DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode) DIRAC.exit(-1) ## check simtel data/log/histo Output File exist cfg = cfg_dict[current_conf] simtelFileName = os.path.basename(corsikaFileLFN).replace( 'corsika.gz', 'simtel.gz') cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Data/*.simtel.gz ' + simtelFileName if (os.system(cmd)): DIRAC.exit(-1) simtelLogFileName = simtelFileName.replace('simtel.gz', 'log.gz') cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Log/*.log.gz ' + simtelLogFileName if (os.system(cmd)): DIRAC.exit(-1) simtelHistFileName = simtelFileName.replace('simtel.gz', 'hdata.gz') cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName if (os.system(cmd)): DIRAC.exit(-1) ################################################################ DIRAC.exit()
def main(): Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) < 3: Script.showHelp(exitCode=1) diracSiteName = args[0] gridSiteName = args[1] ces = args[2:] try: diracGridType, place, country = diracSiteName.split('.') except ValueError: gLogger.error( "The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch" ) DIRACExit(-1) result = getDIRACSiteName(gridSiteName) newSite = True if result['OK'] and result['Value']: if len(result['Value']) > 1: gLogger.notice( '%s GOC site name is associated with several DIRAC sites:' % gridSiteName) for i, dSite in enumerate(result['Value']): gLogger.notice('%d: %s' % (i, dSite)) inp = six.moves.input('Enter your choice number: ') try: inp = int(inp) except ValueError: gLogger.error('You should enter an integer number') DIRACExit(-1) if 0 <= inp < len(result['Value']): diracCSSite = result['Value'][inp] else: gLogger.error('Number out of range: %d' % inp) DIRACExit(-1) else: diracCSSite = result['Value'][0] if diracCSSite == diracSiteName: gLogger.notice('Site with GOC name %s is already defined as %s' % (gridSiteName, diracSiteName)) newSite = False else: gLogger.error( 'ERROR: Site with GOC name %s is already defined as %s' % (gridSiteName, diracCSSite)) DIRACExit(-1) else: gLogger.error("ERROR getting DIRAC site name of %s" % gridSiteName, result.get('Message')) csAPI = CSAPI() if newSite: gLogger.notice("Site to CS: %s" % diracSiteName) res = csAPI.addSite(diracSiteName, {"Name": gridSiteName}) if not res['OK']: gLogger.error("Failed adding site to CS", res['Message']) DIRACExit(1) res = csAPI.commit() if not res['OK']: gLogger.error("Failure committing to CS", res['Message']) DIRACExit(3) for ce in ces: gLogger.notice("Adding CE %s" % ce) res = csAPI.addCEtoSite(diracSiteName, ce) if not res['OK']: gLogger.error("Failed adding CE %s to CS" % ce, res['Message']) DIRACExit(2) res = csAPI.commit() if not res['OK']: gLogger.error("Failure committing to CS", res['Message']) DIRACExit(3)
def setGroupName(arg): global groupName if groupName or not arg: Script.showHelp(exitCode=1) groupName = arg
def main(): Script.registerSwitch( "", "hosts=", "Comma separated list of hosts or file containing row wise list of hosts" " targeted for update (leave empty for all)") Script.registerSwitch( "", "retry=", "Number of retry attempts on hosts that have failed to update") Script.parseCommandLine(ignoreErrors=False) args = Script.getPositionalArgs() if len(args) < 1 or len(args) > 2: Script.showHelp() version = args[0] retry = 0 hosts = [] for switch in Script.getUnprocessedSwitches(): if switch[0] == "hosts": hosts = switch[1] if switch[0] == "retry": retry = int(switch[1]) try: with open(hosts, 'r') as f: hosts = f.read().splitlines() hosts = [str(host) for host in hosts] except Exception: pass if not isinstance(hosts, list): hosts = hosts.split(',') from concurrent.futures import ThreadPoolExecutor, as_completed from DIRAC import S_OK, S_ERROR from DIRAC import gLogger from DIRAC.FrameworkSystem.Client.SystemAdministratorClient import SystemAdministratorClient from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient def getListOfHosts(): """ Return the list of all hosts that constitute a DIRAC instance """ client = ComponentMonitoringClient() result = client.getHosts({}, False, False) if result['OK']: hosts = [host['HostName'] for host in result['Value']] return S_OK(hosts) return S_ERROR('Cannot get list of hosts: %s' % result['Message']) def parseHostname(hostName): """ Separate the hostname from the port :param str hostName: hostname you want to parse """ hostList = hostName.split(':') host = hostList[0] if len(hostList) == 2: port = hostList[1] else: port = None return host, port def updateHost(hostName, version): """ Apply update to specific host :param str hostName: name of the host you want to update :param str version: version vArBpC you want to update to """ host, port = parseHostname(hostName) client = SystemAdministratorClient(host, port) result = client.ping() if not result['OK']: gLogger.error("Cannot connect to %s" % host) return result gLogger.notice( "Initiating software update of %s, this can take a while, please be patient ..." % host) result = client.updateSoftware(version, '', '', timeout=600) if not result['OK']: return result return S_OK() def updateHosts(version, hosts=[]): """ Apply update to all hosts :param str version: version vArBpC you want to update to :param list[str] hosts: list of hosts to be updated, defaults to [] """ if not hosts: result = getListOfHosts() if not result['OK']: return result hosts = result['Value'] updateSuccess = [] updateFail = [] executor = ThreadPoolExecutor(max_workers=len(hosts)) futureUpdate = { executor.submit(updateHost, host, version): host for host in hosts } for future in as_completed(futureUpdate): host = futureUpdate[future] result = future.result() if result['OK']: updateSuccess.append(host) else: updateFail.append(host) if not updateFail: gLogger.notice("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") gLogger.notice("!!! Successfully updated all hosts !!!") gLogger.notice("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") return S_OK([updateSuccess, updateFail]) if not updateSuccess: gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") gLogger.notice("XXXXX Failed to update all hosts XXXXX") gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") return S_OK([updateSuccess, updateFail]) gLogger.notice("X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!") gLogger.notice("X!X!X Partially updated hosts X!X!X!") gLogger.notice("Succeeded to update:") for host in updateSuccess: gLogger.notice(" + %s" % host) gLogger.notice("Failed to update:") for host in updateFail: gLogger.notice(" - %s" % host) gLogger.notice("X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!") return S_OK([updateSuccess, updateFail]) def restartHost(hostName): """ Restart all systems and components of a host :param str hostName: name of the host you want to restart """ host, port = parseHostname(hostName) gLogger.notice("Pinging %s ..." % host) client = SystemAdministratorClient(host, port) result = client.ping() if not result['OK']: gLogger.error("Could not connect to %s: %s" % (host, result['Message'])) return result gLogger.notice("Host %s is active" % host) gLogger.notice("Initiating restart of all systems and components") # This restart call will always return S_ERROR because of SystemAdministrator restart # Connection will be lost to the host result = client.restartComponent('*', '*') if result['Message'] == "Peer closed connection": gLogger.notice( "Restarted all systems on %s : connection to SystemAdministrator lost" % host) return S_OK(result['Message']) gLogger.error("Received unxpected message: %s" % result['Message']) return result def updateInstance(version, hosts, retry): """ Update each server of an instance and restart them :param str version: version vArBpC you want to update to :param list[str] hosts: list of hosts to be updated :param int retry: number of retry attempts on hosts that have failed to update """ result = updateHosts(version, hosts) if not result['OK']: return result updateSuccess = result['Value'][0] updateFail = result['Value'][1] restartSuccess = [] restartFail = [] for host in updateSuccess: result = restartHost(host) if result['OK']: restartSuccess.append(host) else: restartFail.append(host) if not restartFail and not updateFail: return S_OK("Successfully updated and restarted all hosts") gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") gLogger.notice("XXXXX There were problems in the update process XXXXX") gLogger.notice("Succeeded to update:") for host in updateSuccess: gLogger.notice(" + %s" % host) gLogger.notice("Succeeded to restart:") for host in restartSuccess: gLogger.notice(" + %s" % host) gLogger.notice("Failed to update:") for host in updateFail: gLogger.notice(" - %s" % host) gLogger.notice("Failed to restart:") for host in restartFail: gLogger.notice(" - %s" % host) gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") if retry > 0: retryHosts = list(set(updateFail + restartFail)) gLogger.notice("Retrying update on (%s attempts remaining):" % retry) for host in retryHosts: gLogger.notice(" - %s" % host) gLogger.notice( "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") return updateInstance(version, retryHosts, retry - 1) return S_ERROR("Update failed!") result = updateInstance(version, hosts, retry) if not result['OK']: gLogger.fatal(result['Message']) DIRAC.exit(1) gLogger.notice(result['Value']) DIRAC.exit(0)
def main(): Script.registerSwitch("D", "sync", "Make target directory identical to source") Script.registerSwitch("j:", "parallel=", "Multithreaded download and upload") Script.parseCommandLine(ignoreErrors=False) args = Script.getPositionalArgs() if len(args) < 1 or len(args) > 3: Script.showHelp() sync = False parallel = 1 for switch in Script.getUnprocessedSwitches(): if switch[0].lower() == "s" or switch[0].lower() == "sync": sync = True if switch[0].lower() == "j" or switch[0].lower() == "parallel": parallel = int(switch[1]) from DIRAC import S_OK, S_ERROR from DIRAC import gConfig, gLogger from DIRAC.Core.Utilities.List import breakListIntoChunks from DIRAC.Core.Utilities.ReturnValues import returnSingleResult from DIRAC.Core.Utilities.File import mkDir from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC.Resources.Storage.StorageElement import StorageElement def getSetOfLocalDirectoriesAndFiles(path): """Return a set of all directories and subdirectories and a set of files contained therein for a given local path """ fullPath = os.path.abspath(path) if not os.path.isdir(fullPath): return S_ERROR("The path: " + fullPath + " does not exist!") directories = set() files = set() for dirname, dirnames, filenames in os.walk(path): # add path to all subdirectories first. for subdirname in dirnames: fullSubdirname = os.path.join(dirname, subdirname) fullSubdirname = os.path.abspath(fullSubdirname) fullSubdirname = fullSubdirname.replace(fullPath, '').lstrip('/') directories.add(fullSubdirname) # add path to all filenames. for filename in filenames: fullFilename = os.path.join(dirname, filename) fullFilename = os.path.abspath(fullFilename) fullFilename = fullFilename.replace(fullPath, '').lstrip('/') fileSize = os.path.getsize(fullPath + "/" + fullFilename) if fileSize > 0: files.add((fullFilename, int(fileSize))) tree = {} tree["Directories"] = directories tree["Files"] = files return S_OK(tree) def getSetOfRemoteSubDirectoriesAndFiles(path, fc, directories, files): """ Recursively traverses all the subdirectories of a directory and returns a set of directories and files """ result = fc.listDirectory(path) if result['OK']: if result['Value']['Successful']: for entry in result['Value']['Successful'][path]['Files']: size = result['Value']['Successful'][path]['Files'][entry][ 'MetaData']['Size'] files.add((entry, size)) for entry in result['Value']['Successful'][path]['SubDirs']: directories.add(entry) res = getSetOfRemoteSubDirectoriesAndFiles( entry, fc, directories, files) if not res['OK']: return S_ERROR('Error: ' + res['Message']) return S_OK() else: return S_ERROR("Error: %s" % result['Value']) else: return S_ERROR("Error:" + result['Message']) def getSetOfRemoteDirectoriesAndFiles(fc, path): """ Return a set of all directories and subdirectories and the therein contained files for a given LFN """ directories = set() files = set() res = getSetOfRemoteSubDirectoriesAndFiles(path, fc, directories, files) if not res['OK']: return S_ERROR('Could not list remote directory: ' + res['Message']) return_directories = set() return_files = set() for myfile in files: return_files.add((myfile[0].replace(path, '').lstrip('/'), myfile[1])) for mydirectory in directories: return_directories.add(mydirectory.replace(path, '').lstrip('/')) tree = {} tree["Directories"] = return_directories tree["Files"] = return_files return S_OK(tree) def isInFileCatalog(fc, path): """ Check if the file is in the File Catalog """ result = fc.listDirectory(path) if result['OK']: if result['Value']['Successful']: return S_OK() else: return S_ERROR() else: return S_ERROR() def getContentToSync(upload, fc, source_dir, dest_dir): """ Return list of files and directories to be create and deleted """ if upload: res = getSetOfRemoteDirectoriesAndFiles(fc, dest_dir) if not res['OK']: return S_ERROR(res['Message']) to_dirs = res['Value']['Directories'] to_files = res['Value']['Files'] res = getSetOfLocalDirectoriesAndFiles(source_dir) if not res['OK']: return S_ERROR(res['Message']) from_dirs = res['Value']['Directories'] from_files = res['Value']['Files'] else: res = getSetOfLocalDirectoriesAndFiles(dest_dir) if not res['OK']: return S_ERROR(res['Message']) to_dirs = res['Value']['Directories'] to_files = res['Value']['Files'] res = getSetOfRemoteDirectoriesAndFiles(fc, source_dir) if not res['OK']: return S_ERROR(res['Message']) from_dirs = res['Value']['Directories'] from_files = res['Value']['Files'] # Create list of directories to delete dirs_delete = list(to_dirs - from_dirs) # Sort the list by depth of directory tree dirs_delete.sort(key=lambda s: -s.count('/')) # Create list of directories to create dirs_create = list(from_dirs - to_dirs) # Sort the list by depth of directory tree dirs_create.sort(key=lambda s: s.count('/')) # Flatten the list of pairs (filename, size) to list of filename files_delete = [pair[0] for pair in list(to_files - from_files)] files_create = [pair[0] for pair in list(from_files - to_files)] create = {} create["Directories"] = dirs_create create["Files"] = files_create delete = {} delete["Directories"] = dirs_delete delete["Files"] = files_delete tree = {} tree["Create"] = create tree["Delete"] = delete return S_OK(tree) def removeRemoteFiles(dm, lfns): """ Remove file from the catalog """ for lfnList in breakListIntoChunks(lfns, 100): res = dm.removeFile(lfnList) if not res['OK']: return S_ERROR("Failed to remove files:" + lfnList + res['Message']) else: return S_OK() def uploadLocalFile(dm, lfn, localfile, storage): """ Upload a local file to a storage element """ res = dm.putAndRegister(lfn, localfile, storage, None) if not res['OK']: return S_ERROR('Error: failed to upload %s to %s' % (lfn, storage)) else: return S_OK('Successfully uploaded file to %s' % storage) def downloadRemoteFile(dm, lfn, destination): """ Download a file from the system """ res = dm.getFile(lfn, destination) if not res['OK']: return S_ERROR('Error: failed to download %s ' % lfn) else: return S_OK('Successfully uploaded file %s' % lfn) def removeStorageDirectoryFromSE(directory, storageElement): """ Delete directory on selected storage element """ se = StorageElement(storageElement, False) res = returnSingleResult(se.exists(directory)) if not res['OK']: return S_ERROR("Failed to obtain existence of directory" + res['Message']) exists = res['Value'] if not exists: return S_OK("The directory %s does not exist at %s " % (directory, storageElement)) res = returnSingleResult(se.removeDirectory(directory, recursive=True)) if not res['OK']: return S_ERROR("Failed to remove storage directory" + res['Message']) return S_OK() def removeRemoteDirectory(fc, lfn): """ Remove file from the catalog """ storageElements = gConfig.getValue( 'Resources/StorageElementGroups/SE_Cleaning_List', []) for storageElement in sorted(storageElements): res = removeStorageDirectoryFromSE(lfn, storageElement) if not res['OK']: return S_ERROR("Failed to clean storage directory at all SE:" + res['Message']) res = returnSingleResult(fc.removeDirectory(lfn, recursive=True)) if not res['OK']: return S_ERROR("Failed to clean storage directory at all SE:" + res['Message']) return S_OK("Successfully removed directory") def createRemoteDirectory(fc, newdir): """ Create directory in file catalog """ result = fc.createDirectory(newdir) if result['OK']: if result['Value']['Successful'] and newdir in result['Value'][ 'Successful']: return S_OK("Successfully created directory:" + newdir) elif result['Value']['Failed'] and newdir in result['Value'][ 'Failed']: return S_ERROR('Failed to create directory: ' + result['Value']['Failed'][newdir]) else: return S_ERROR('Failed to create directory:' + result['Message']) def createLocalDirectory(directory): """ Create local directory """ mkDir(directory) if not os.path.exists(directory): return S_ERROR('Directory creation failed') return S_OK('Created directory successfully') def removeLocalFile(path): """ Remove local file """ try: os.remove(path) except OSError as e: return S_ERROR('Directory creation failed:' + e.strerror) if os.path.isfile(path): return S_ERROR('File deleting failed') return S_OK('Removed file successfully') def removeLocaDirectory(path): """ Remove local directory """ try: os.rmdir(path) except OSError as e: return S_ERROR('Deleting directory failed: ' + e.strerror) if os.path.isdir(path): return S_ERROR('Directory deleting failed') return S_OK('Removed directory successfully') def doUpload(fc, dm, result, source_dir, dest_dir, storage, delete, nthreads): """ Wrapper for uploading files """ if delete: lfns = [ dest_dir + "/" + filename for filename in result['Value']['Delete']['Files'] ] if len(lfns) > 0: res = removeRemoteFiles(dm, lfns) if not res['OK']: gLogger.fatal('Deleting of files: ' + lfns + " -X- [FAILED]" + res['Message']) DIRAC.exit(1) else: gLogger.notice("Deleting " + ', '.join(lfns) + " -> [DONE]") for directoryname in result['Value']['Delete']['Directories']: res = removeRemoteDirectory(fc, dest_dir + "/" + directoryname) if not res['OK']: gLogger.fatal('Deleting of directory: ' + directoryname + " -X- [FAILED] " + res['Message']) DIRAC.exit(1) else: gLogger.notice("Deleting " + directoryname + " -> [DONE]") for directoryname in result['Value']['Create']['Directories']: res = createRemoteDirectory(fc, dest_dir + "/" + directoryname) if not res['OK']: gLogger.fatal('Creation of directory: ' + directoryname + " -X- [FAILED] " + res['Message']) DIRAC.exit(1) else: gLogger.notice("Creating " + directoryname + " -> [DONE]") listOfFiles = result['Value']['Create']['Files'] # Chech that we do not have to many threads if nthreads > len(listOfFiles): nthreads = len(listOfFiles) if nthreads == 0: return S_OK('Upload finished successfully') listOfListOfFiles = chunkList(listOfFiles, nthreads) res = runInParallel(arguments=[dm, source_dir, dest_dir, storage], listOfLists=listOfListOfFiles, function=uploadListOfFiles) if not res['OK']: return S_ERROR("Upload of files failed") return S_OK('Upload finished successfully') def uploadListOfFiles(dm, source_dir, dest_dir, storage, listOfFiles, tID): """ Wrapper for multithreaded uploading of a list of files """ log = gLogger.getSubLogger("[Thread %s] " % tID) threadLine = "[Thread %s]" % tID for filename in listOfFiles: res = uploadLocalFile(dm, dest_dir + "/" + filename, source_dir + "/" + filename, storage) if not res['OK']: log.fatal(threadLine + ' Uploading ' + filename + ' -X- [FAILED] ' + res['Message']) DIRAC.exit(1) else: log.notice(threadLine + " Uploading " + filename + " -> [DONE]") def doDownload(dm, result, source_dir, dest_dir, delete, nthreads): """ Wrapper for downloading files """ if delete: for filename in result['Value']['Delete']['Files']: res = removeLocalFile(dest_dir + "/" + filename) if not res['OK']: gLogger.fatal('Deleting of file: ' + filename + ' -X- [FAILED] ' + res['Message']) DIRAC.exit(1) else: gLogger.notice("Deleting " + filename + " -> [DONE]") for directoryname in result['Value']['Delete']['Directories']: res = removeLocaDirectory(dest_dir + "/" + directoryname) if not res['OK']: gLogger.fatal('Deleting of directory: ' + directoryname + ' -X- [FAILED] ' + res['Message']) DIRAC.exit(1) else: gLogger.notice("Deleting " + directoryname + " -> [DONE]") for directoryname in result['Value']['Create']['Directories']: res = createLocalDirectory(dest_dir + "/" + directoryname) if not res['OK']: gLogger.fatal('Creation of directory: ' + directoryname + ' -X- [FAILED] ' + res['Message']) DIRAC.exit(1) else: gLogger.notice("Creating " + directoryname + " -> [DONE]") listOfFiles = result['Value']['Create']['Files'] # Chech that we do not have to many threads if nthreads > len(listOfFiles): nthreads = len(listOfFiles) if nthreads == 0: return S_OK('Upload finished successfully') listOfListOfFiles = chunkList(listOfFiles, nthreads) res = runInParallel( arguments=[dm, source_dir, dest_dir], listOfLists=listOfListOfFiles, function=downloadListOfFiles, ) if not res['OK']: return S_ERROR("Download of files failed") return S_OK('Upload finished successfully') def chunkList(alist, nchunks): """ Split a list into a list of equaliy sized lists """ avg = len(alist) / float(nchunks) out = [] last = 0.0 while last < len(alist): out.append(alist[int(last):int(last + avg)]) last += avg return out def downloadListOfFiles(dm, source_dir, dest_dir, listOfFiles, tID): """ Wrapper for multithreaded downloading of a list of files """ log = gLogger.getSubLogger("[Thread %s] " % tID) threadLine = "[Thread %s]" % tID for filename in listOfFiles: res = downloadRemoteFile( dm, source_dir + "/" + filename, dest_dir + ("/" + filename).rsplit("/", 1)[0]) if not res['OK']: log.fatal(threadLine + ' Downloading ' + filename + ' -X- [FAILED] ' + res['Message']) DIRAC.exit(1) else: log.notice(threadLine + " Downloading " + filename + " -> [DONE]") def runInParallel(arguments, listOfLists, function): """ Helper for execution of uploads and downloads in parallel """ from multiprocessing import Process processes = [] for tID, alist in enumerate(listOfLists): argums = arguments + [alist] + [tID] pro = Process(target=function, args=argums) pro.start() processes.append(pro) for process in processes: process.join() for process in processes: if process.exitcode == 1: return S_ERROR() return S_OK() def syncDestinations(upload, source_dir, dest_dir, storage, delete, nthreads): """ Top level wrapper to execute functions """ fc = FileCatalog() dm = DataManager() result = getContentToSync(upload, fc, source_dir, dest_dir) if not result['OK']: return S_ERROR(result['Message']) if upload: res = doUpload(fc, dm, result, source_dir, dest_dir, storage, delete, nthreads) if not res['OK']: return S_ERROR('Upload failed: ' + res['Message']) else: res = doDownload(dm, result, source_dir, dest_dir, delete, nthreads) if not res['OK']: return S_ERROR('Download failed: ' + res['Message']) return S_OK('Mirroring successfully finished') def run(parameters, delete, nthreads): """ The main user interface """ source_dir = parameters[0] dest_dir = parameters[1] upload = False storage = None if len(parameters) == 3: storage = parameters[2] source_dir = os.path.abspath(source_dir) dest_dir = dest_dir.rstrip('/') upload = True if not os.path.isdir(source_dir): gLogger.fatal("Source directory does not exist") DIRAC.exit(1) if len(parameters) == 2: dest_dir = os.path.abspath(dest_dir) source_dir = source_dir.rstrip('/') if not os.path.isdir(dest_dir): gLogger.fatal("Destination directory does not exist") DIRAC.exit(1) res = syncDestinations(upload, source_dir, dest_dir, storage, delete, nthreads) if not res['OK']: return S_ERROR(res['Message']) return S_OK("Successfully mirrored " + source_dir + " into " + dest_dir) returnValue = run(args, sync, parallel) if not returnValue['OK']: gLogger.fatal(returnValue['Message']) DIRAC.exit(1) else: gLogger.notice(returnValue['Value']) DIRAC.exit(0)
def setUserName(arg): global userName if userName or not arg: Script.showHelp() DIRAC.exit(-1) userName = arg
def setUserDN(arg): global userDN if userDN or not arg: Script.showHelp() DIRAC.exit(-1) userDN = arg
def addProperty(arg): global groupProperties if not arg: Script.showHelp(exitCode=1) if arg not in groupProperties: groupProperties.append(arg)
def MCProdTSExample(args=None): from DIRAC.TransformationSystem.Client.Transformation import Transformation from DIRAC.Core.Workflow.Parameter import Parameter from DIRAC.Interfaces.API.Dirac import Dirac from CTADIRAC.Interfaces.API.ProdTSJob import ProdTSJob if (len(args) != 2): Script.gLogger.notice('Wrong number of arguments') Script.showHelp() cfgfile = args[0] simtelArrayConfig = args[1] j = ProdTSJob() j.setVersion('prod-2_13112014b') j.setApplication('corsika_autoinputs') j.setProdName('ConfigxxxTS') #j.setPathRoot('/vo.cta.in2p3.fr/MC/PROD2/') # official j.setPathRoot('/vo.cta.in2p3.fr/user/a/arrabito/MC/PROD2/') # for test #mode = 'corsika_standalone' #mode = 'corsika_simtel' mode = 'corsika_simtel_dst' start_run_number = 0 j.setParameters([ 'fileCatalog.cfg', '--template', cfgfile, '--mode', mode, '-i', str(start_run_number), '--run_number', '@{JOB_ID}', '-N', '25000', '-S', simtelArrayConfig, '--savecorsika', 'False' ]) j.setInputSandbox( ['LFN:/vo.cta.in2p3.fr/MC/PROD2/Armazones2K/prod2_cfg.tar.gz']) j.setOutputSandbox(['*.log']) j.setCPUTime(200000) ### Temporary fix ####### j.workflow.addParameter( Parameter("JOB_ID", "000000", "string", "", "", True, False, "Temporary fix")) j.workflow.addParameter( Parameter("PRODUCTION_ID", "000000", "string", "", "", True, False, "Temporary fix")) t = Transformation() t.setTransformationName("ProdExemple") # This must be unique #t.setTransformationGroup("Group1") t.setType("MCSimulation") t.setDescription("MC prod example") t.setLongDescription("corsika-simtel production") #mandatory t.setBody(j.workflow.toXML()) res = t.addTransformation() # Transformation is created here if not res['OK']: print res['Message'] DIRAC.exit(-1) t.setStatus("Active") t.setAgentType("Automatic")
def setGroupName(arg): global groupName if groupName or not arg: Script.showHelp() DIRAC.exit(-1) groupName = arg
def addUserGroup(arg): global userGroups if not arg: Script.showHelp(exitCode=1) if arg not in userGroups: userGroups.append(arg)
def main(): Script.registerSwitch("E:", "email=", "Boolean True/False (True by default)") Script.parseCommandLine(ignoreErrors=True) from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC import exit as DIRACExit, gConfig, gLogger def getBoolean(value): if value.lower() == 'true': return True elif value.lower() == 'false': return False else: Script.showHelp() email = True for switch in Script.getUnprocessedSwitches(): if switch[0] == "email": email = getBoolean(switch[1]) args = Script.getPositionalArgs() if len(args) < 2: Script.showHelp() diracAdmin = DiracAdmin() exitCode = 0 errorList = [] setup = gConfig.getValue('/DIRAC/Setup', '') if not setup: print('ERROR: Could not contact Configuration Service') exitCode = 2 DIRACExit(exitCode) # result = promptUser( # 'All the elements that are associated with this site will be active, ' # 'are you sure about this action?' # ) # if not result['OK'] or result['Value'] is 'n': # print 'Script stopped' # DIRACExit( 0 ) site = args[0] comment = args[1] result = diracAdmin.allowSite(site, comment, printOutput=True) if not result['OK']: errorList.append((site, result['Message'])) exitCode = 2 else: if email: userName = diracAdmin._getCurrentUser() if not userName['OK']: print('ERROR: Could not obtain current username from proxy') exitCode = 2 DIRACExit(exitCode) userName = userName['Value'] subject = '%s is added in site mask for %s setup' % (site, setup) body = 'Site %s is added to the site mask for %s setup by %s on %s.\n\n' % ( site, setup, userName, time.asctime()) body += 'Comment:\n%s' % comment addressPath = 'EMail/Production' address = Operations().getValue(addressPath, '') if not address: gLogger.notice( "'%s' not defined in Operations, can not send Mail\n" % addressPath, body) else: result = diracAdmin.sendMail(address, subject, body) else: print('Automatic email disabled by flag.') for error in errorList: print("ERROR %s: %s" % error) DIRACExit(exitCode)
def CorsikaSimtelExample(args=None): from CTADIRAC.Interfaces.API.CorsikaSimtelJob import CorsikaSimtelJob from DIRAC.Interfaces.API.Dirac import Dirac j = CorsikaSimtelJob() j.setVersion('prod-2_15122013') j.setExecutable('corsika_autoinputs') if len(args) != 4: Script.showHelp() if args[3] not in [ 'STD', '4MSST', 'SCSST', 'ASTRI', 'NSBX3', 'NORTH', 'SCMST' ]: Script.gLogger.notice("reprocessing configuration is incorrect:", args[3]) DIRAC.exit(-1) runMin = int(args[0]) runMax = int(args[1]) cfgfile = args[2] simtelArrayConfig = args[3] ilist = [] for i in range(runMin, runMax + 1): run_number = '%06d' % i ilist.append(run_number) j.setGenericParametricInput(ilist) j.setName('run%s') j.setInputSandbox([ cfgfile, 'grid_prod2-repro.sh', 'LFN:/vo.cta.in2p3.fr/MC/PROD2/SVN-PROD2.tar.gz' ]) j.setParameters([ '--template', cfgfile, '--mode', 'corsika_simtel', '-S', simtelArrayConfig ]) j.setOutputSandbox( ['corsika_autoinputs.log', 'simtel.log', 'applicationLog.txt']) # Retrieve your Output Data corsika_out = '*.corsika.gz' corsikatar_out = '*.corsika.tar.gz' sim_out = '*.simtel.gz' log_out = '*.log.gz' hist_out = '*.hdata.gz' j.setOutputData([corsika_out, corsikatar_out, sim_out, log_out, hist_out]) j.setCPUTime(200000) Script.gLogger.info(j._toJDL()) res = Dirac().submit(j) print res
def main(): from DIRAC.Core.Base import Script ### DoCtaIrf options ########################################################## Script.registerSwitch("A:", "analysis=", "Analysis Type", setAnalysisType) Script.registerSwitch("C:", "cuts=", "Cuts Config", setCutsConfig) Script.registerSwitch("R:", "runlist=", "Runlist", setRunlist) Script.registerSwitch("Z:", "zenith=", "Zenith", setZenith) Script.registerSwitch("O:", "offset=", "Offset", setOffset) Script.registerSwitch("M:", "energy=", "Energy Method", setEnergyMethod) Script.registerSwitch("T:", "arrayconfig=", "Array Configuration", setArrayConfig) Script.registerSwitch("P:", "particle=", "Particle Type", setParticleType) ## other options Script.registerSwitch("V:", "version=", "HAP version", setVersion) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) < 1: Script.showHelp() from CTADIRAC.Core.Workflow.Modules.HapApplication import HapApplication from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from DIRAC.Core.Utilities.Subprocess import systemCall from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport jobID = os.environ['JOBID'] jobID = int(jobID) jobReport = JobReport(jobID) ha = HapApplication() HapPack = 'HAP/' + version + '/HAP' packs = ['HESS/v0.2/lib', 'HESS/v0.3/root', HapPack] for package in packs: DIRAC.gLogger.notice('Checking:', package) if sharedArea: if checkSoftwarePackage(package, sharedArea())['OK']: DIRAC.gLogger.notice('Package found in Shared Area:', package) continue if localArea: if checkSoftwarePackage(package, localArea())['OK']: DIRAC.gLogger.notice('Package found in Local Area:', package) continue if installSoftwarePackage(package, localArea())['OK']: continue DIRAC.gLogger.error('Check Failed for software package:', package) DIRAC.gLogger.error('Software package not available') DIRAC.exit(-1) ha.setSoftwarePackage(HapPack) ha.hapExecutable = 'DoCtaIrf' runlistdir = os.environ['PWD'] build_infile(runlist) ha.hapArguments = [ analysistype, cutsconfig, runlistdir, runlist, zenith, offset, arrayconfig, energymethod, particle ] DIRAC.gLogger.notice('Executing Hap Application') res = ha.execute() if not res['OK']: DIRAC.gLogger.error('Failed to execute Hap Application') jobReport.setApplicationStatus('Hap Application: Failed') DIRAC.exit(-1) DIRAC.exit()
def main(): from DIRAC.Core.Base import Script Script.registerSwitch('', 'Job=', ' JobID[,jobID2,...]') Script.registerSwitch('', 'Transformation=', ' transformation ID') Script.registerSwitch( '', 'Tasks=', ' Associated to --Transformation, list of taskIDs') Script.registerSwitch('', 'Verbose', ' Print more information') Script.registerSwitch('', 'Terse', ' Only print request status') Script.registerSwitch('', 'Full', ' Print full request content') Script.registerSwitch('', 'Status=', ' Select all requests in a given status') Script.registerSwitch( '', 'Since=', ' Associated to --Status, start date yyyy-mm-dd or nb of days (default= -one day' ) Script.registerSwitch( '', 'Until=', ' Associated to --Status, end date (default= now') Script.registerSwitch( '', 'Maximum=', ' Associated to --Status, max number of requests ') Script.registerSwitch('', 'Reset', ' Reset Failed files to Waiting if any') Script.registerSwitch('', 'Force', ' Force reset even if not Failed') Script.registerSwitch( '', 'All', ' (if --Status Failed) all requests, otherwise exclude irrecoverable failures' ) Script.registerSwitch('', 'FixJob', ' Set job Done if the request is Done') Script.registerSwitch('', 'Cancel', ' Cancel the request') Script.registerSwitch('', 'ListJobs', ' List the corresponding jobs') Script.registerSwitch( '', 'TargetSE=', ' Select request only if that SE is in the targetSEs') from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() import DIRAC from DIRAC import gLogger jobs = [] requestID = 0 transID = None taskIDs = None tasks = None requests = [] full = False verbose = False status = None until = None since = None terse = False allR = False reset = False fixJob = False maxRequests = 999999999999 cancel = False listJobs = False force = False targetSE = set() for switch in Script.getUnprocessedSwitches(): if switch[0] == 'Job': jobs = [] job = "Unknown" try: for arg in switch[1].split(','): if os.path.exists(arg): with open(arg, 'r') as fp: lines = fp.readlines() for line in lines: for job in line.split(','): jobs += [int(job.strip())] gLogger.notice("Found %d jobs in file %s" % (len(jobs), arg)) else: jobs.append(int(arg)) except TypeError: gLogger.fatal("Invalid jobID", job) elif switch[0] == 'Transformation': try: transID = int(switch[1]) except Exception: gLogger.fatal('Invalid transID', switch[1]) elif switch[0] == 'Tasks': try: taskIDs = [int(task) for task in switch[1].split(',')] except Exception: gLogger.fatal('Invalid tasks', switch[1]) elif switch[0] == 'Full': full = True elif switch[0] == 'Verbose': verbose = True elif switch[0] == 'Terse': terse = True elif switch[0] == 'All': allR = True elif switch[0] == 'Reset': reset = True elif switch[0] == 'Force': force = True elif switch[0] == 'Status': status = switch[1].capitalize() elif switch[0] == 'Since': since = convertDate(switch[1]) elif switch[0] == 'Until': until = convertDate(switch[1]) elif switch[0] == 'FixJob': fixJob = True elif switch[0] == 'Cancel': cancel = True elif switch[0] == 'ListJobs': listJobs = True elif switch[0] == 'Maximum': try: maxRequests = int(switch[1]) except Exception: pass elif switch[0] == 'TargetSE': targetSE = set(switch[1].split(',')) if reset and not force: status = 'Failed' if fixJob: status = 'Done' if terse: verbose = True if status: if not until: until = datetime.datetime.utcnow() if not since: since = until - datetime.timedelta(hours=24) from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.RequestManagementSystem.Client.ReqClient import printRequest, recoverableRequest reqClient = ReqClient() if transID: if not taskIDs: gLogger.fatal( "If Transformation is set, a list of Tasks should also be set") Script.showHelp(exitCode=2) # In principle, the task name is unique, so the request name should be unique as well # If ever this would not work anymore, we would need to use the transformationClient # to fetch the ExternalID requests = ['%08d_%08d' % (transID, task) for task in taskIDs] allR = True elif not jobs: requests = [] # Get full list of arguments, with and without comma for arg in [ x.strip() for arg in Script.getPositionalArgs() for x in arg.split(',') ]: if os.path.exists(arg): lines = open(arg, 'r').readlines() requests += [ reqID.strip() for line in lines for reqID in line.split(',') ] gLogger.notice("Found %d requests in file" % len(requests)) else: requests.append(arg) allR = True else: res = reqClient.getRequestIDsForJobs(jobs) if not res['OK']: gLogger.fatal("Error getting request for jobs", res['Message']) DIRAC.exit(2) if res['Value']['Failed']: gLogger.error( "No request found for jobs %s" % ','.join(sorted(str(job) for job in res['Value']['Failed']))) requests = sorted(res['Value']['Successful'].values()) if requests: allR = True else: DIRAC.exit(0) if status and not requests: allR = allR or status != 'Failed' res = reqClient.getRequestIDsList([status], limit=maxRequests, since=since, until=until) if not res['OK']: gLogger.error("Error getting requests:", res['Message']) DIRAC.exit(2) requests = [ reqID for reqID, _st, updTime in res['Value'] if updTime > since and updTime <= until and reqID ] gLogger.notice('Obtained %d requests %s between %s and %s' % (len(requests), status, since, until)) if not requests: gLogger.notice('No request selected....') Script.showHelp(exitCode=2) okRequests = [] warningPrinted = False jobIDList = [] for reqID in requests: # We allow reqID to be the requestName if it is unique try: requestID = int(reqID) except ValueError: requestID = reqClient.getRequestIDForName(reqID) if not requestID['OK']: gLogger.notice(requestID['Message']) continue requestID = requestID['Value'] request = reqClient.peekRequest(requestID) if not request["OK"]: gLogger.error(request["Message"]) DIRAC.exit(-1) request = request["Value"] if not request: gLogger.error("no such request %s" % requestID) continue # If no operation as the targetSE, skip if targetSE: found = False for op in request: if op.TargetSE and targetSE.intersection( op.TargetSE.split(',')): found = True break if not found: continue # keep a list of jobIDs if requested if request.JobID and listJobs: jobIDList.append(request.JobID) if status and request.Status != status: gLogger.notice( "Request %s is not in requested status %s%s" % (reqID, status, ' (cannot be reset)' if reset else '')) continue if fixJob and request.Status == 'Done' and request.JobID: # The request is for a job and is Done, verify that the job is in the proper status result = reqClient.finalizeRequest(request.RequestID, request.JobID, useCertificates=False) if not result['OK']: gLogger.error("Error finalizing job", result['Message']) else: gLogger.notice("Job %d updated to %s" % (request.JobID, result['Value'])) continue if cancel: if request.Status not in ('Done', 'Failed'): ret = reqClient.cancelRequest(requestID) if not ret['OK']: gLogger.error("Error canceling request %s" % reqID, ret['Message']) else: gLogger.notice("Request %s cancelled" % reqID) else: gLogger.notice("Request %s is in status %s, not cancelled" % (reqID, request.Status)) elif allR or recoverableRequest(request): okRequests.append(str(requestID)) if reset: gLogger.notice('============ Request %s =============' % requestID) ret = reqClient.resetFailedRequest(requestID, allR=allR) if not ret['OK']: gLogger.error("Error resetting request %s" % requestID, ret['Message']) else: if len(requests) > 1: gLogger.notice('\n===================================') dbStatus = reqClient.getRequestStatus(requestID).get( 'Value', 'Unknown') printRequest(request, status=dbStatus, full=full, verbose=verbose, terse=terse) if listJobs: gLogger.notice("List of %d jobs:\n" % len(jobIDList), ','.join(str(jobID) for jobID in jobIDList)) if status and okRequests: from DIRAC.Core.Utilities.List import breakListIntoChunks gLogger.notice('\nList of %d selected requests:' % len(okRequests)) for reqs in breakListIntoChunks(okRequests, 100): gLogger.notice(','.join(reqs))
def setUserName(arg): global userName if userName or not arg: Script.showHelp(exitCode=1) userName = arg
def setUserDN(arg): global userDN if userDN or not arg: Script.showHelp(exitCode=1) userDN = arg
Remove the given file replica or a list of file replicas from the File Catalog and from the storage. Usage: %s <LFN | fileContainingLFNs> SE [SE] """ % Script.scriptName ) Script.parseCommandLine() from DIRAC.Core.Utilities.List import sortList, breakListIntoChunks from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager rm = ReplicaManager() import os, sys if len( sys.argv ) < 3: Script.showHelp() DIRAC.exit( -1 ) else: inputFileName = sys.argv[1] storageElementNames = sys.argv[2:] if os.path.exists( inputFileName ): inputFile = open( inputFileName, 'r' ) string = inputFile.read() lfns = [ lfn.strip() for lfn in string.splitlines() ] inputFile.close() else: lfns = [inputFileName] for lfnList in breakListIntoChunks( sortList( lfns, True ), 500 ): for storageElementName in storageElementNames:
def dirLUT(args=None): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job user_script = './dirLUT.sh' if (len(args) != 3): Script.showHelp() direction = args[0] zenith = args[1] diffuse = args[2] if diffuse == "0": diffName = "point" elif diffuse == "1": diffName = "diff" else: print "Invalid extension definition!" Script.showHelp() return 1 if zenith == "40": zenName = "40deg" elif zenith == "20": zenName = "20deg" else: print "Invalid zenith definition!" Script.showHelp() return 1 if direction == "N": directionName = "north" # deg = "180" elif direction == "S": directionName = "south" # deg = "0" else: print 'Wrong direction. It can only be "N" or "S".' Script.showHelp() return 1 listname = './training/gamma_trainLUT_%s_%s_%s.lfns' % (zenName, diffName, direction) with open(listname) as f: totalEntries = sum(1 for _ in f) # Number of files used per job runN = 20 runMin = 0 runMax = totalEntries / runN for i in range(runMin, runMax): jobName = "%s_%s_%s_%s_%s" % (user_script, direction, zenName, diffName, i) jobOut = "%s_%s_%s%s.out" % (user_script, directionName, diffName, i) script_args = "%s %s %s %s %s" % (direction, zenName, diffName, i, runN) j = Job() j.setInputSandbox([ user_script, listname, "setupPackageMARS.sh", "CheckFileZombie.C" ]) j.setExecutable(user_script, script_args) j.setOutputSandbox([jobOut, "applicationLog.txt"]) j.setName(jobName) j.setBannedSites([ 'LCG.MSFG.fr', 'LCG.M3PEC.fr', 'LCG.OBSPM.fr', 'LCG.UNI-DORTMUND.de', 'LCG.UNIV-LILLE.fr' ]) Script.gLogger.info(j._toJDL()) print "Submitting job %s" % (script_args) Dirac().submit(j)