def addUserName( arg ): global userNames if not arg: Script.showHelp() DIRAC.exit( -1 ) if not arg in userNames: userNames.append( arg )
def showHelp( self, dummy = False ): """ Printout help message including a Usage message if defined via setUsageMessage method """ if self.__usageMessage: gLogger.notice( self.__usageMessage ) else: gLogger.notice( "Usage:" ) gLogger.notice( " %s (<options>|<cfgFile>)*" % os.path.basename( sys.argv[0] ) ) if dummy: gLogger.notice( dummy ) gLogger.notice( "General options:" ) iLastOpt = 0 for iPos in range( len( self.commandOptionList ) ): optionTuple = self.commandOptionList[ iPos ] gLogger.notice( " %s --%s : %s" % ( ( "-" + optionTuple[0].ljust( 3 ) if optionTuple[0] else " " * 4 ), optionTuple[1].ljust( 15 ), optionTuple[2] ) ) iLastOpt = iPos if optionTuple[0] == 'h': #Last general opt is always help break if iLastOpt + 1 < len( self.commandOptionList ): gLogger.notice( " \nOptions:" ) for iPos in range( iLastOpt + 1, len( self.commandOptionList ) ): optionTuple = self.commandOptionList[ iPos ] if optionTuple[0]: gLogger.notice( " -%s --%s : %s" % ( optionTuple[0].ljust( 3 ), optionTuple[1].ljust( 15 ), optionTuple[2] ) ) else: gLogger.notice( " %s --%s : %s" % ( ' ', optionTuple[1].ljust( 15 ), optionTuple[2] ) ) DIRAC.exit( 0 )
def addProperty( arg ): global groupProperties if not arg: Script.showHelp() DIRAC.exit( -1 ) if not arg in groupProperties: groupProperties.append( arg )
def addProperty( arg ): global hostProperties if not arg: Script.showHelp() DIRAC.exit( -1 ) if not arg in hostProperties: hostProperties.append( arg )
def getInfo(params): ''' Retrieve information from BDII ''' from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin diracAdmin = DiracAdmin() if params['info'] == 'ce': result = diracAdmin.getBDIICE(params['ce'], host=params['host']) if params['info'] == 'ce-state': result = diracAdmin.getBDIICEState(params['ce'], useVO=params['vo'], host=params['host']) if params['info'] == 'ce-cluster': result = diracAdmin.getBDIICluster(params['ce'], host=params['host']) if params['info'] == 'ce-vo': result = diracAdmin.getBDIICEVOView(params['ce'], useVO=params['vo'], host=params['host']) if params['info'] == 'site': result = diracAdmin.getBDIISite(params['site'], host=params['host']) if params['info'] == 'site-se': result = diracAdmin.getBDIISE(params['site'], useVO=params['vo'], host=params['host']) if not result['OK']: print result['Message'] DIRAC.exit(2) return result
def run( parameters , delete, nthreads ): """ The main user interface """ source_dir = parameters[0] dest_dir = parameters[1] upload = False storage = None if len( parameters ) == 3: storage = parameters[2] source_dir = os.path.abspath(source_dir) dest_dir = dest_dir.rstrip('/') upload = True if not os.path.isdir(source_dir): gLogger.fatal("Source directory does not exist") DIRAC.exit( 1 ) if len (parameters ) == 2: dest_dir = os.path.abspath(dest_dir) source_dir = source_dir.rstrip('/') if not os.path.isdir(dest_dir): gLogger.fatal("Destination directory does not exist") DIRAC.exit( 1 ) res = syncDestinations( upload, source_dir, dest_dir, storage, delete, nthreads ) if not res['OK']: return S_ERROR(res['Message']) return S_OK("Successfully mirrored " + source_dir + " into " + dest_dir)
def addUserGroup(arg): global userGroups if not arg: Script.showHelp() DIRAC.exit(-1) if not arg in userGroups: userGroups.append(arg)
def setUserMail(arg): global userMail if userMail or not arg: Script.showHelp() DIRAC.exit(-1) if not arg.find("@") > 0: Script.gLogger.error("Not a valid mail address", arg) DIRAC.exit(-1) userMail = arg
def showLicense( self, dummy = False ): """ Print license """ lpath = os.path.join( DIRAC.rootPath, "DIRAC", "LICENSE" ) sys.stdout.write( " - DIRAC is GPLv3 licensed\n\n" ) try: with open( lpath ) as fd: sys.stdout.write( fd.read() ) except IOError: sys.stdout.write( "Can't find GPLv3 license at %s. Somebody stole it!\n" % lpath ) sys.stdout.write( "Please check out http://www.gnu.org/licenses/gpl-3.0.html for more info\n" ) DIRAC.exit(0)
def uploadListOfFiles(dm, source_dir, dest_dir, storage, listOfFiles, tID): """ Wrapper for multithreaded uploading of a list of files """ log = gLogger.getSubLogger("[Thread %s] " % tID) threadLine = "[Thread %s]" % tID for filename in listOfFiles: res = uploadLocalFile(dm, dest_dir+"/"+filename, source_dir+"/"+filename, storage) if not res['OK']: log.fatal(threadLine + ' Uploading ' + filename + ' -X- [FAILED] ' + res['Message']) DIRAC.exit( 1 ) else: log.notice(threadLine+ " Uploading " + filename + " -> [DONE]")
def downloadListOfFiles(dm, source_dir, dest_dir, listOfFiles, tID): """ Wrapper for multithreaded downloading of a list of files """ log = gLogger.getSubLogger("[Thread %s] " % tID) threadLine = "[Thread %s]" % tID for filename in listOfFiles: res = downloadRemoteFile(dm, source_dir + "/" + filename, dest_dir + ("/" + filename).rsplit("/", 1)[0]) if not res['OK']: log.fatal(threadLine + ' Downloading ' + filename + ' -X- [FAILED] ' + res['Message']) DIRAC.exit( 1 ) else: log.notice(threadLine+ " Downloading " + filename + " -> [DONE]")
def _getTSFiles(self): """ Helper function - get files from the TS """ selectDict = {'TransformationID': self.prod} if self._lfns: selectDict['LFN'] = self._lfns elif self.runStatus and self.fromProd: res = self.transClient.getTransformationRuns( {'TransformationID': self.fromProd, 'Status': self.runStatus}) if not res['OK']: gLogger.error("Failed to get runs for transformation %d" % self.prod) else: if res['Value']: self.runsList.extend( [run['RunNumber'] for run in res['Value'] if run['RunNumber'] not in self.runsList]) gLogger.notice("%d runs selected" % len(res['Value'])) elif not self.runsList: gLogger.notice("No runs selected, check completed") DIRAC.exit(0) if not self._lfns and self.runsList: selectDict['RunNumber'] = self.runsList res = self.transClient.getTransformation(self.prod) if not res['OK']: gLogger.error("Failed to find transformation %s" % self.prod) return [], [], [] status = res['Value']['Status'] if status not in ('Active', 'Stopped', 'Completed', 'Idle'): gLogger.notice("Transformation %s in status %s, will not check if files are processed" % ( self.prod, status)) processedLFNs = [] nonProcessedLFNs = [] nonProcessedStatuses = [] if self._lfns: processedLFNs = self._lfns else: res = self.transClient.getTransformationFiles(selectDict) if not res['OK']: gLogger.error("Failed to get files for transformation %d" % self.prod, res['Message']) return [], [], [] else: processedLFNs = [item['LFN'] for item in res['Value'] if item['Status'] == 'Processed'] nonProcessedLFNs = [item['LFN'] for item in res['Value'] if item['Status'] != 'Processed'] nonProcessedStatuses = list( set(item['Status'] for item in res['Value'] if item['Status'] != 'Processed')) return processedLFNs, nonProcessedLFNs, nonProcessedStatuses
def run( self, lfn, metas ): retVal = self.catalog.getMeta( lfn ) if not retVal[ "OK" ]: print "Error:", retVal[ "Message" ] DIRAC.exit( -1 ) metadict = retVal[ "Value" ] if not metas: for k, v in metadict.items( ): print k+"="+str( v ) else: for meta in metas: if meta in metadict.keys( ): print meta+"="+metadict[ meta ]
def checkFunction(): """ gets CPU normalisation from MFJ or calculate itself """ from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import getPowerFromMJF from ILCDIRAC.Core.Utilities.CPUNormalization import getCPUNormalization from DIRAC import gLogger, gConfig result = getCPUNormalization() if not result['OK']: gLogger.error( result['Message'] ) norm = round( result['Value']['NORM'], 1 ) gLogger.notice( 'Estimated CPU power is %.1f %s' % ( norm, result['Value']['UNIT'] ) ) mjfPower = getPowerFromMJF() if mjfPower: gLogger.notice( 'CPU power from MJF is %.1f HS06' % mjfPower ) else: gLogger.notice( 'MJF not available on this node' ) if update and not configFile: gConfig.setOptionValue( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm ) gConfig.setOptionValue( '/LocalSite/CPUNormalizationFactor', norm ) gConfig.dumpLocalCFGToFile( gConfig.diracConfigFilePath ) if configFile: from DIRAC.Core.Utilities.CFG import CFG cfg = CFG() try: # Attempt to open the given file cfg.loadFromFile( configFile ) except: pass # Create the section if it does not exist if not cfg.existsKey( 'LocalSite' ): cfg.createNewSection( 'LocalSite' ) cfg.setOption( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm ) cfg.setOption( '/LocalSite/CPUNormalizationFactor', norm ) cfg.writeToFile( configFile ) DIRAC.exit()
def getSEInfo(lfns) : result = fileCatalog.getReplicas( lfns ) if not result['OK']: print 'ERROR: ', result['Message'] exitCode = 2 DIRAC.exit( exitCode ) replicas=result['Value']['Successful'][lfns] nreplicas = len(replicas) ralsrm='' selist='' for se in replicas.keys() : selist=selist+se+',' # if se == target : # ralsrm=replicas[se] print str(nreplicas)+' '+lfns+' '+selist[:-1]
def export_insertOpBD( self ): print "server, insertOpBD" db = OperationFileDB() res = db.createTables() if not res["OK"]: gLogger.error( ' error' ) DIRAC.exit( -1 ) gLogger.verbose( 'ok creation table' ) args = dict() args['LFN'] = "testLFN" args['SESource'] = "testSESource" args['SEDestination'] = "testSEDestination" args['blob'] = "testBlob" operationFile = OperationFile( args ) res = db.putOperationFile( operationFile ) return res
def showHelp( self, dummy = False, exitCode = 0 ): """ Printout help message including a Usage message if defined via setUsageMessage method """ if self.__usageMessage: gLogger.notice( '\n'+self.__usageMessage.lstrip() ) else: gLogger.notice( "\nUsage:" ) gLogger.notice( "\n %s (<options>|<cfgFile>)*" % os.path.basename( sys.argv[0] ) ) if dummy: gLogger.notice( dummy ) gLogger.notice( "\nGeneral options:" ) iLastOpt = 0 for iPos in range( len( self.commandOptionList ) ): optionTuple = self.commandOptionList[ iPos ] if optionTuple[0].endswith( ':' ): line = " -%s --%s : %s" % ( optionTuple[0][:-1].ljust( 2 ), (optionTuple[1][:-1] + ' <value> ').ljust( 22 ), optionTuple[2] ) gLogger.notice( line ) else: gLogger.notice( " -%s --%s : %s" % ( optionTuple[0].ljust( 2 ), optionTuple[1].ljust( 22 ), optionTuple[2] ) ) iLastOpt = iPos if optionTuple[0] == 'h': #Last general opt is always help break if iLastOpt + 1 < len( self.commandOptionList ): gLogger.notice( " \nOptions:" ) for iPos in range( iLastOpt + 1, len( self.commandOptionList ) ): optionTuple = self.commandOptionList[ iPos ] if optionTuple[0].endswith( ':' ): line = " -%s --%s : %s" % ( optionTuple[0][:-1].ljust( 2 ), (optionTuple[1][:-1] + ' <value> ').ljust( 22 ), optionTuple[2] ) gLogger.notice( line ) else: gLogger.notice( " -%s --%s : %s" % ( optionTuple[0].ljust( 2 ), optionTuple[1].ljust( 22 ), optionTuple[2] ) ) gLogger.notice( "" ) DIRAC.exit( exitCode )
def syncStorageElementsInit(): ses = gConfig.getSections( '/Resources/StorageElements' ) if not ses[ 'OK' ]: gLogger.error( ses[ 'Message' ] ) DIRAC.exit( 2 ) statuses = rsc.getValidStatuses()[ 'Value' ] for se in ses[ 'Value' ]: opts = gConfig.getOptions( '/Resources/StorageElements/%s' % se )[ 'Value' ] statusTypes = [ 'Read', 'Write', 'Check', 'Remove' ] for opt in opts: if not opt.endswith( 'Access' ): continue status = gConfig.getValue( '/Resources/StorageElements/%s/%s' % ( se, opt ) ) if status in [ 'NotAllowed', 'InActive' ]: status = 'Banned' if not status in statuses: gLogger.error( '%s not a valid status for %s - %s' % ( status, se, statusType ) ) continue statusType = opt.replace( 'Access', '' ) statusTypes.remove( statusType ) rsc.modifyElementStatus( 'StorageElement', se, statusType, status = status, reason = 'Init sync' ) for sType in statusTypes: # If there is nothing on the CS, we set the statusType to DEFAULT_STATUS DEFAULT_STATUS = 'Active' rsc.modifyElementStatus( 'StorageElement', se, sType, status = DEFAULT_STATUS, reason = 'Default status' )
def doUpload(fc, dm, result, source_dir, dest_dir, storage, delete, nthreads): """ Wrapper for uploading files """ if delete: lfns = [dest_dir+"/"+filename for filename in result['Value']['Delete']['Files']] if len(lfns)>0: res = removeRemoteFiles(dm,lfns) if not res['OK']: gLogger.fatal('Deleting of files: ' + lfns + " -X- [FAILED]" +res['Message']) DIRAC.exit( 1 ) else: gLogger.notice("Deleting "+ ', '.join(lfns) + " -> [DONE]") for directoryname in result['Value']['Delete']['Directories']: res = removeRemoteDirectory(fc, dest_dir + "/" + directoryname) if not res['OK']: gLogger.fatal('Deleting of directory: ' + directoryname + " -X- [FAILED] " + res['Message']) DIRAC.exit( 1 ) else: gLogger.notice("Deleting "+ directoryname + " -> [DONE]") for directoryname in result['Value']['Create']['Directories']: res = createRemoteDirectory(fc, dest_dir+"/"+ directoryname) if not res['OK']: gLogger.fatal('Creation of directory: ' + directoryname + " -X- [FAILED] " + res['Message']) DIRAC.exit( 1 ) else: gLogger.notice("Creating " + directoryname + " -> [DONE]") listOfFiles = result['Value']['Create']['Files'] #Chech that we do not have to many threads if nthreads > len(listOfFiles): nthreads = len(listOfFiles) if nthreads == 0: return S_OK('Upload finished successfully') listOfListOfFiles = chunkList( listOfFiles, nthreads ) res = runInParallel(arguments=[dm, source_dir, dest_dir, storage], listOfLists=listOfListOfFiles, function=uploadListOfFiles ) if not res['OK']: return S_ERROR("Upload of files failed") return S_OK('Upload finished successfully')
def doDownload(dm, result, source_dir, dest_dir, delete, nthreads): """ Wrapper for downloading files """ if delete: for filename in result['Value']['Delete']['Files']: res = removeLocalFile(dest_dir+"/"+ filename) if not res['OK']: gLogger.fatal('Deleting of file: ' + filename + ' -X- [FAILED] ' + res['Message']) DIRAC.exit( 1 ) else: gLogger.notice("Deleting "+ filename + " -> [DONE]") for directoryname in result['Value']['Delete']['Directories']: res = removeLocaDirectory( dest_dir + "/" + directoryname ) if not res['OK']: gLogger.fatal('Deleting of directory: ' + directoryname + ' -X- [FAILED] ' + res['Message']) DIRAC.exit( 1 ) else: gLogger.notice("Deleting "+ directoryname + " -> [DONE]") for directoryname in result['Value']['Create']['Directories']: res = createLocalDirectory( dest_dir+"/"+ directoryname ) if not res['OK']: gLogger.fatal('Creation of directory: ' + directoryname + ' -X- [FAILED] ' + res['Message']) DIRAC.exit( 1 ) else: gLogger.notice("Creating " + directoryname + " -> [DONE]") listOfFiles = result['Value']['Create']['Files'] #Chech that we do not have to many threads if nthreads > len(listOfFiles): nthreads = len(listOfFiles) if nthreads==0: return S_OK('Upload finished successfully') listOfListOfFiles = chunkList( listOfFiles, nthreads ) res = runInParallel(arguments=[dm, source_dir, dest_dir], listOfLists=listOfListOfFiles, function=downloadListOfFiles ) if not res['OK']: return S_ERROR("Download of files failed") return S_OK('Upload finished successfully')
def generateProxy(params): if params.checkClock: result = getClockDeviation() if result['OK']: deviation = result['Value'] if deviation > 600: print "Error: Your host clock seems to be off by more than TEN MINUTES! Thats really bad." print "We're cowardly refusing to generate a proxy. Please fix your system time" DIRAC.exit(1) elif deviation > 180: print "Error: Your host clock seems to be off by more than THREE minutes! Thats bad." print "Warn : We'll generate the proxy but please fix your system time" elif deviation > 60: print "Error: Your host clock seems to be off by more than a minute! Thats not good." print "Warn : We'll generate the proxy but please fix your system time" certLoc = params.certLoc keyLoc = params.keyLoc if not certLoc or not keyLoc: cakLoc = Locations.getCertificateAndKeyLocation() if not cakLoc: return S_ERROR("Can't find user certificate and key") if not certLoc: certLoc = cakLoc[0] if not keyLoc: keyLoc = cakLoc[1] testChain = X509Chain() retVal = testChain.loadKeyFromFile(keyLoc, password=params.userPasswd) if not retVal['OK']: passwdPrompt = "Enter Certificate password:"******"\n") else: userPasswd = getpass.getpass(passwdPrompt) params.userPasswd = userPasswd proxyLoc = params.proxyLoc if not proxyLoc: proxyLoc = Locations.getDefaultProxyLocation() if params.debug: h = int(params.proxyLifeTime / 3600) m = int(params.proxyLifeTime / 60) - h * 60 print "Proxy lifetime will be %02d:%02d" % (h, m) print "User cert is %s" % certLoc print "User key is %s" % keyLoc print "Proxy will be written to %s" % proxyLoc if params.diracGroup: print "DIRAC Group will be set to %s" % params.diracGroup else: print "No DIRAC Group will be set" print "Proxy strength will be %s" % params.proxyStrength if params.limitedProxy: print "Proxy will be limited" chain = X509Chain() #Load user cert and key retVal = chain.loadChainFromFile(certLoc) if not retVal['OK']: params.debugMsg("ERROR: %s" % retVal['Message']) return S_ERROR("Can't load %s" % certLoc) retVal = chain.loadKeyFromFile(keyLoc, password=params.userPasswd) if not retVal['OK']: params.debugMsg("ERROR: %s" % retVal['Message']) return S_ERROR("Can't load %s" % keyLoc) if params.checkWithCS and params.diracGroup: retVal = chain.generateProxyToFile(proxyLoc, params.proxyLifeTime, strength=params.proxyStrength, limited=params.limitedProxy) params.debugMsg("Contacting CS...") retVal = Script.enableCS() if not retVal['OK']: params.debugMsg("ERROR: %s" % retVal['Message']) return S_ERROR("Can't contact DIRAC CS: %s" % retVal['Message']) if not params.diracGroup: params.diracGroup = CS.getDefaultUserGroup() userDN = chain.getCertInChain(-1)['Value'].getSubjectDN()['Value'] params.debugMsg("Checking DN %s" % userDN) retVal = CS.getUsernameForDN(userDN) if not retVal['OK']: params.debugMsg("ERROR: %s" % retVal['Message']) return S_ERROR("DN %s is not registered" % userDN) username = retVal['Value'] params.debugMsg("Username is %s" % username) retVal = CS.getGroupsForUser(username) if not retVal['OK']: params.debugMsg("ERROR: %s" % retVal['Message']) return S_ERROR("User %s has no groups defined" % username) groups = retVal['Value'] if params.diracGroup not in groups: return S_ERROR("Requested group %s is not valid for user %s" % (params.diracGroup, username)) params.debugMsg("Creating proxy for %s@%s (%s)" % (username, params.diracGroup, userDN)) retVal = chain.generateProxyToFile(proxyLoc, params.proxyLifeTime, params.diracGroup, strength=params.proxyStrength, limited=params.limitedProxy) if not retVal['OK']: params.debugMsg("ERROR: %s" % retVal['Message']) return S_ERROR("Couldn't generate proxy: %s" % retVal['Message']) return S_OK(proxyLoc)
import os import DIRAC from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient stageLfns = {} if os.path.exists( fileName ): try: lfnFile = open( fileName ) lfns = [ k.strip() for k in lfnFile.readlines() ] lfnFile.close() except Exception: DIRAC.gLogger.exception( 'Can not open file', fileName ) DIRAC.exit( -1 ) else: lfns = args[1:] stageLfns[seName] = lfns stagerClient = StorageManagerClient() res = stagerClient.setRequest( stageLfns, 'WorkloadManagement', 'updateJobFromStager@WorkloadManagement/JobStateUpdate', 0 ) # fake JobID = 0 if not res['OK']: DIRAC.gLogger.error( res['Message'] ) DIRAC.exit( -1 ) else: print "Stage request submitted for LFNs:\n %s" %lfns print "SE= %s" %seName
def main(): """ Main executive code """ Script.registerSwitch("", "Job=", " JobID[,jobID2,...]") Script.registerSwitch("", "Transformation=", " transformation ID") Script.registerSwitch("", "Tasks=", " Associated to --Transformation, list of taskIDs") Script.registerSwitch("", "Verbose", " Print more information") Script.registerSwitch("", "Terse", " Only print request status") Script.registerSwitch("", "Full", " Print full request content") Script.registerSwitch("", "Status=", " Select all requests in a given status") Script.registerSwitch( "", "Since=", " Associated to --Status, start date yyyy-mm-dd or nb of days (default= -one day" ) Script.registerSwitch("", "Until=", " Associated to --Status, end date (default= now") Script.registerSwitch("", "Maximum=", " Associated to --Status, max number of requests ") Script.registerSwitch("", "Reset", " Reset Failed files to Waiting if any") Script.registerSwitch("", "Force", " Force reset even if not Failed") Script.registerSwitch( "", "All", " (if --Status Failed) all requests, otherwise exclude irrecoverable failures" ) Script.registerSwitch("", "FixJob", " Set job Done if the request is Done") Script.registerSwitch("", "Cancel", " Cancel the request") Script.registerSwitch("", "ListJobs", " List the corresponding jobs") Script.registerSwitch("", "TargetSE=", " Select request only if that SE is in the targetSEs") # Registering arguments will automatically add their description to the help menu Script.registerArgument( ( "file: a file containing a list of requests (Comma-separated on each line)", "request: a request ID or a unique request name", ), mandatory=False, ) Script.registerArgument(["request: a request ID or a unique request name"], mandatory=False) Script.parseCommandLine() import DIRAC from DIRAC import gLogger jobs = [] requestID = 0 transID = None taskIDs = None tasks = None requests = [] full = False verbose = False status = None until = None since = None terse = False allR = False reset = False fixJob = False maxRequests = 999999999999 cancel = False listJobs = False force = False targetSE = set() for switch in Script.getUnprocessedSwitches(): if switch[0] == "Job": jobs = [] job = "Unknown" try: for arg in switch[1].split(","): if os.path.exists(arg): with open(arg, "r") as fp: lines = fp.readlines() for line in lines: for job in line.split(","): jobs += [int(job.strip())] gLogger.notice("Found %d jobs in file %s" % (len(jobs), arg)) else: jobs.append(int(arg)) except TypeError: gLogger.fatal("Invalid jobID", job) elif switch[0] == "Transformation": try: transID = int(switch[1]) except Exception: gLogger.fatal("Invalid transID", switch[1]) elif switch[0] == "Tasks": try: taskIDs = [int(task) for task in switch[1].split(",")] except Exception: gLogger.fatal("Invalid tasks", switch[1]) elif switch[0] == "Full": full = True elif switch[0] == "Verbose": verbose = True elif switch[0] == "Terse": terse = True elif switch[0] == "All": allR = True elif switch[0] == "Reset": reset = True elif switch[0] == "Force": force = True elif switch[0] == "Status": status = switch[1].capitalize() elif switch[0] == "Since": since = convertDate(switch[1]) elif switch[0] == "Until": until = convertDate(switch[1]) elif switch[0] == "FixJob": fixJob = True elif switch[0] == "Cancel": cancel = True elif switch[0] == "ListJobs": listJobs = True elif switch[0] == "Maximum": try: maxRequests = int(switch[1]) except Exception: pass elif switch[0] == "TargetSE": targetSE = set(switch[1].split(",")) if reset and not force: status = "Failed" if fixJob: status = "Done" if terse: verbose = True if status: if not until: until = datetime.datetime.utcnow() if not since: since = until - datetime.timedelta(hours=24) from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.RequestManagementSystem.Client.ReqClient import printRequest, recoverableRequest reqClient = ReqClient() if transID: if not taskIDs: gLogger.fatal("If Transformation is set, a list of Tasks should also be set") Script.showHelp(exitCode=2) # In principle, the task name is unique, so the request name should be unique as well # If ever this would not work anymore, we would need to use the transformationClient # to fetch the ExternalID requests = ["%08d_%08d" % (transID, task) for task in taskIDs] allR = True elif not jobs: requests = [] # Get full list of arguments, with and without comma for arg in [x.strip() for ar in Script.getPositionalArgs() for x in ar.split(",")]: if os.path.exists(arg): lines = open(arg, "r").readlines() requests += [reqID.strip() for line in lines for reqID in line.split(",")] gLogger.notice("Found %d requests in file" % len(requests)) else: requests.append(arg) allR = True else: res = reqClient.getRequestIDsForJobs(jobs) if not res["OK"]: gLogger.fatal("Error getting request for jobs", res["Message"]) DIRAC.exit(2) if res["Value"]["Failed"]: gLogger.error("No request found for jobs %s" % ",".join(sorted(str(job) for job in res["Value"]["Failed"]))) requests = sorted(res["Value"]["Successful"].values()) if requests: allR = True else: DIRAC.exit(0) if status and not requests: allR = allR or status != "Failed" res = reqClient.getRequestIDsList([status], limit=maxRequests, since=since, until=until) if not res["OK"]: gLogger.error("Error getting requests:", res["Message"]) DIRAC.exit(2) requests = [reqID for reqID, _st, updTime in res["Value"] if updTime > since and updTime <= until and reqID] gLogger.notice("Obtained %d requests %s between %s and %s" % (len(requests), status, since, until)) if not requests: gLogger.notice("No request selected....") Script.showHelp(exitCode=2) okRequests = [] jobIDList = [] for reqID in requests: # We allow reqID to be the requestName if it is unique try: # PEP-515 allows for underscore in numerical literals # So a request name 00123_00456 # is interpreted as a requestID 12300456 # Using an exception here for non-string is not an option if isinstance(reqID, str) and not reqID.isdigit(): raise ValueError() requestID = int(reqID) except (ValueError, TypeError): requestID = reqClient.getRequestIDForName(reqID) if not requestID["OK"]: gLogger.notice(requestID["Message"]) continue requestID = requestID["Value"] request = reqClient.peekRequest(requestID) if not request["OK"]: gLogger.error(request["Message"]) DIRAC.exit(-1) request = request["Value"] if not request: gLogger.error("no such request %s" % requestID) continue # If no operation as the targetSE, skip if targetSE: found = False for op in request: if op.TargetSE and targetSE.intersection(op.TargetSE.split(",")): found = True break if not found: continue # keep a list of jobIDs if requested if request.JobID and listJobs: jobIDList.append(request.JobID) if status and request.Status != status: gLogger.notice( "Request %s is not in requested status %s%s" % (reqID, status, " (cannot be reset)" if reset else "") ) continue if fixJob and request.Status == "Done" and request.JobID: # The request is for a job and is Done, verify that the job is in the proper status result = reqClient.finalizeRequest(request.RequestID, request.JobID, useCertificates=False) if not result["OK"]: gLogger.error("Error finalizing job", result["Message"]) else: gLogger.notice("Job %d updated to %s" % (request.JobID, result["Value"])) continue if cancel: if request.Status not in ("Done", "Failed"): ret = reqClient.cancelRequest(requestID) if not ret["OK"]: gLogger.error("Error canceling request %s" % reqID, ret["Message"]) else: gLogger.notice("Request %s cancelled" % reqID) else: gLogger.notice("Request %s is in status %s, not cancelled" % (reqID, request.Status)) elif allR or recoverableRequest(request): okRequests.append(str(requestID)) if reset: gLogger.notice("============ Request %s =============" % requestID) ret = reqClient.resetFailedRequest(requestID, allR=allR) if not ret["OK"]: gLogger.error("Error resetting request %s" % requestID, ret["Message"]) else: if len(requests) > 1: gLogger.notice("\n===================================") dbStatus = reqClient.getRequestStatus(requestID).get("Value", "Unknown") printRequest(request, status=dbStatus, full=full, verbose=verbose, terse=terse) if listJobs: gLogger.notice("List of %d jobs:\n" % len(jobIDList), ",".join(str(jobID) for jobID in jobIDList)) if status and okRequests: from DIRAC.Core.Utilities.List import breakListIntoChunks gLogger.notice("\nList of %d selected requests:" % len(okRequests)) for reqs in breakListIntoChunks(okRequests, 100): gLogger.notice(",".join(reqs))
def main(): from DIRAC.Core.Base import Script ### DoCtaIrf options ########################################################## Script.registerSwitch("A:", "analysis=", "Analysis Type", setAnalysisType) Script.registerSwitch("C:", "cuts=", "Cuts Config", setCutsConfig) Script.registerSwitch("R:", "runlist=", "Runlist", setRunlist) Script.registerSwitch("Z:", "zenith=", "Zenith", setZenith) Script.registerSwitch("O:", "offset=", "Offset", setOffset) Script.registerSwitch("M:", "energy=", "Energy Method", setEnergyMethod) Script.registerSwitch("T:", "arrayconfig=", "Array Configuration", setArrayConfig) Script.registerSwitch("P:", "particle=", "Particle Type", setParticleType) ## other options Script.registerSwitch("V:", "version=", "HAP version", setVersion) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) < 1: Script.showHelp() from CTADIRAC.Core.Workflow.Modules.HapApplication import HapApplication from CTADIRAC.Core.Workflow.Modules.HapRootMacro import HapRootMacro from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from DIRAC.Core.Utilities.Subprocess import systemCall from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport jobID = os.environ['JOBID'] jobID = int(jobID) jobReport = JobReport(jobID) ha = HapApplication() HapPack = 'HAP/' + version + '/HAP' packs = ['HESS/v0.2/lib', 'HESS/v0.3/root', HapPack] for package in packs: DIRAC.gLogger.notice('Checking:', package) if sharedArea: if checkSoftwarePackage(package, sharedArea())['OK']: DIRAC.gLogger.notice('Package found in Shared Area:', package) continue if localArea: if checkSoftwarePackage(package, localArea())['OK']: DIRAC.gLogger.notice('Package found in Local Area:', package) continue if installSoftwarePackage(package, localArea())['OK']: continue DIRAC.gLogger.error('Check Failed for software package:', package) DIRAC.gLogger.error('Software package not available') DIRAC.exit(-1) ha.setSoftwarePackage(HapPack) ha.hapExecutable = 'DoCtaIrf' runlistdir = os.environ['PWD'] build_infile(runlist) ha.hapArguments = [ analysistype, cutsconfig, runlistdir, runlist, zenith, offset, arrayconfig, energymethod, particle ] DIRAC.gLogger.notice('Executing Hap Application') res = ha.execute() if not res['OK']: DIRAC.gLogger.error('Failed to execute Hap Application') jobReport.setApplicationStatus('Hap Application: Failed') DIRAC.exit(-1) ###################### Check TTree Output File ####################### outfile = 'MVAFile_' + runlist + ".root" if not os.path.isfile(outfile): error = 'TTree file was not created:' DIRAC.gLogger.error(error, outfile) jobReport.setApplicationStatus('DoCtaIrf: TTree file not created') DIRAC.exit(-1) ###################### Quality Check for TTree Output File: step0###################### hr = HapRootMacro() hr.setSoftwarePackage(HapPack) DIRAC.gLogger.notice('Executing TTree check step0') hr.rootMacro = '/hapscripts/mva/Open_TT.C+' outfilestr = '"' + outfile + '"' args = [outfilestr] DIRAC.gLogger.notice('Open_TT macro Arguments:', args) hr.rootArguments = args DIRAC.gLogger.notice('Executing Hap Open_TT macro') res = hr.execute() if not res['OK']: DIRAC.gLogger.error('Open_TT: Failed') DIRAC.exit(-1) #########################Quality Check for TTree Output File: step1#################### DIRAC.gLogger.notice('Executing TTree check step1') ret = getSoftwareEnviron(HapPack) if not ret['OK']: error = ret['Message'] DIRAC.gLogger.error(error, HapPack) DIRAC.exit(-1) hapEnviron = ret['Value'] hessroot = hapEnviron['HESSROOT'] check_script = hessroot + '/hapscripts/mva/check_TT.csh' cmdTuple = [check_script] ret = systemCall(0, cmdTuple, sendOutput) if not ret['OK']: DIRAC.gLogger.error('Failed to execute TTree Check step1') jobReport.setApplicationStatus('Check_TTree: Failed') DIRAC.exit(-1) ############################################# DIRAC.exit()
def main(): from DIRAC.Core.Base import Script Script.registerSwitch("T:", "template=", "Corsika Template") Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig") Script.registerSwitch("p:", "run_number=", "Do not use: Run Number automatically set") Script.registerSwitch("E:", "executable=", "Executable (Use SetExecutable)") Script.registerSwitch("V:", "version=", "Version (Use setVersion)") Script.registerSwitch("M:", "mode=", "Mode (corsika_standalone/corsika_simtelarray)") Script.parseCommandLine(ignoreErrors=True) ## default values ############## run_number = None corsikaTemplate = None simtelConfig = None executable = None version = None mode = 'corsika_simtelarray' ### set switch values ### for switch in Script.getUnprocessedSwitches(): if switch[0] == "run_number" or switch[0] == "p": run_number = switch[1].split('ParametricParameters=')[1] elif switch[0] == "template" or switch[0] == "T": corsikaTemplate = switch[1] elif switch[0] == "simtelConfig" or switch[0] == "S": simtelConfig = switch[1] elif switch[0] == "executable" or switch[0] == "E": executable = switch[1] elif switch[0] == "version" or switch[0] == "V": version = switch[1] elif switch[0] == "mode" or switch[0] == "M": mode = switch[1] if version == None or executable == None or run_number == None or corsikaTemplate == None: Script.showHelp() jobReport.setApplicationStatus('Missing options') DIRAC.exit(-1) from CTADIRAC.Core.Workflow.Modules.CorsikaApp import CorsikaApp from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea from DIRAC.Core.Utilities.Subprocess import systemCall from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport jobID = os.environ['JOBID'] jobID = int(jobID) jobReport = JobReport(jobID) createGlobalsFromConfigFiles(corsikaTemplate, version) ############ Producing Corsika File install_CorsikaSimtelPack(version) CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray' cs = CorsikaApp() cs.setSoftwarePackage(CorsikaSimtelPack) cs.csExe = executable cs.csArguments = [ '--run-number', run_number, '--run', 'corsika', corsikaTemplate ] corsikaReturnCode = cs.execute() if corsikaReturnCode != 0: DIRAC.gLogger.error('Corsika Application: Failed') jobReport.setApplicationStatus('Corsika Application: Failed') DIRAC.exit(-1) ###################### rename of corsika output file ####################### rundir = 'run' + run_number filein = rundir + '/' + corsikaOutputFileName corsikaFileName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.gz' mv_cmd = 'mv ' + filein + ' ' + corsikaFileName if (os.system(mv_cmd)): DIRAC.exit(-1) ### create corsika tar luisa #################### corsikaTarName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.tar.gz' filetar1 = rundir + '/' + 'input' filetar2 = rundir + '/' + 'DAT' + run_number + '.dbase' filetar3 = rundir + '/run' + str(int(run_number)) + '.log' cmdTuple = [ '/bin/tar', 'zcf', corsikaTarName, filetar1, filetar2, filetar3 ] DIRAC.gLogger.notice('Executing command tuple:', cmdTuple) ret = systemCall(0, cmdTuple, sendOutput) if not ret['OK']: DIRAC.gLogger.error('Failed to execute tar') DIRAC.exit(-1) if (mode == 'corsika_standalone'): DIRAC.exit() ############ Producing SimTel File ######################Building simtel Directory Metadata ####################### cfg_dict = { "4MSST": 'cta-prod2-4m-dc', "SCSST": 'cta-prod2-sc-sst', "STD": 'cta-prod2', "NSBX3": 'cta-prod2', "ASTRI": 'cta-prod2-astri', "NORTH": 'cta-prod2n', "SCMST": 'cta-prod2-sc3' } #if simtelConfig=="6INROW": # all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD","SCMST"] #elif simtelConfig=="5INROW": # all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD"] #else: # all_configs=[simtelConfig] all_configs = [simtelConfig] for current_conf in all_configs: DIRAC.gLogger.notice('current conf is', current_conf) current_version = version DIRAC.gLogger.notice('current version is', current_version) #if current_conf == "SCMST": # current_version = version + '_sc3' # DIRAC.gLogger.notice('current version is', current_version) # if os.path.isdir('sim_telarray'): # DIRAC.gLogger.notice('Package found in the local area. Removing package...') # cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run' # if(os.system(cmd)): # DIRAC.exit( -1 ) # install_CorsikaSimtelPack(current_version) #else: # current_version = version # DIRAC.gLogger.notice('current version is', current_version) #### execute simtelarray ################ fd = open('run_sim.sh', 'w') fd.write("""#! /bin/sh export SVNPROD2=$PWD export SVNTAG=SVN-PROD2 export CORSIKA_IO_BUFFER=800MB ./grid_prod2-repro.sh %s %s""" % (corsikaFileName, current_conf)) fd.close() #################################### os.system('chmod u+x run_sim.sh') os.system('chmod u+x grid_prod2-repro.sh') cmdTuple = ['./run_sim.sh'] ret = systemCall(0, cmdTuple, sendSimtelOutput) simtelReturnCode, stdout, stderr = ret['Value'] if (os.system('grep Broken simtel.log') == 0): DIRAC.gLogger.error('Broken string found in simtel.log') jobReport.setApplicationStatus('Broken pipe') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Failed to execute run_sim.sh') DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode) DIRAC.exit(-1) ## check simtel data/log/histo Output File exist cfg = cfg_dict[current_conf] simtelFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str( obslev) + '_' + 'run' + run_number + '.simtel.gz' cmd = 'mv Data/sim_telarray/' + cfg + '/*/Data/*.simtel.gz ' + simtelFileName if (os.system(cmd)): DIRAC.exit(-1) simtelLogFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz' cmd = 'mv Data/sim_telarray/' + cfg + '/*/Log/*.log.gz ' + simtelLogFileName if (os.system(cmd)): DIRAC.exit(-1) simtelHistFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str( obslev) + '_' + 'run' + run_number + '.hdata.gz' cmd = 'mv Data/sim_telarray/' + cfg + '/*/Histograms/*.hdata.gz ' + simtelHistFileName if (os.system(cmd)): DIRAC.exit(-1) ################################################################ DIRAC.exit()
def main(): from DIRAC.Core.Base import Script Script.registerSwitch("p:", "inputfile=", "Input File") Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig") Script.registerSwitch("V:", "version=", "Version (Use setVersion)") Script.registerSwitch("C:", "comp=", "Compile (True/False)") Script.parseCommandLine(ignoreErrors=True) ## default values ############## simtelConfig = None version = None comp = True ### set switch values ### for switch in Script.getUnprocessedSwitches(): if switch[0] == "inputfile" or switch[0] == "p": corsikaFileLFN = switch[1].split('ParametricInputData=LFN:')[1] elif switch[0] == "simtelConfig" or switch[0] == "S": simtelConfig = switch[1] elif switch[0] == "version" or switch[0] == "V": version = switch[1] elif switch[0] == "comp" or switch[0] == "C": comp = switch[1] if version == None: Script.showHelp() jobReport.setApplicationStatus('Missing options') DIRAC.exit(-1) from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea from DIRAC.Core.Utilities.Subprocess import systemCall from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport jobID = os.environ['JOBID'] jobID = int(jobID) jobReport = JobReport(jobID) ############ Producing SimTel File if comp == True: install_CorsikaSimtelPack(version) else: CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray' res = installSoftwarePackage(CorsikaSimtelPack, workingArea()) if not res['OK']: DIRAC.gLogger.error('Failed to execute installSoftwarePackage', res) CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray' cfg_dict = { "4MSST": 'cta-prod2-4m-dc', "SCSST": 'cta-prod2-sc-sst', "STD": 'cta-prod2', "NSBX3": 'cta-prod2', "ASTRI": 'cta-prod2-astri', "NORTH": 'cta-prod2n', "ASTRISLN": 'cta-astri-sln' } #if simtelConfig=="6INROW": # all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD","SCMST"] #elif simtelConfig=="5INROW": # all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD"] #else: # all_configs=[simtelConfig] all_configs = [simtelConfig] for current_conf in all_configs: DIRAC.gLogger.notice('current conf is', current_conf) current_version = version DIRAC.gLogger.notice('current version is', current_version) #if current_conf == "SCMST": # current_version = version + '_sc3' # DIRAC.gLogger.notice('current version is', current_version) # if os.path.isdir('sim_telarray'): # DIRAC.gLogger.notice('Package found in the local area. Removing package...') # cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run' # if(os.system(cmd)): # DIRAC.exit( -1 ) # install_CorsikaSimtelPack(current_version) #else: # current_version = version # DIRAC.gLogger.notice('current version is', current_version) #### execute simtelarray ################ fd = open('run_sim.sh', 'w') fd.write("""#! /bin/sh export SVNPROD2=$PWD export SVNTAG=SVN-PROD2 export CORSIKA_IO_BUFFER=800MB ./grid_prod2-repro.sh %s %s""" % (os.path.basename(corsikaFileLFN), current_conf)) fd.close() #################################### os.system('chmod u+x run_sim.sh') os.system('chmod u+x grid_prod2-repro.sh') cmdTuple = ['./run_sim.sh'] ret = systemCall(0, cmdTuple, sendSimtelOutput) simtelReturnCode, stdout, stderr = ret['Value'] if (os.system('grep Broken simtel.log') == 0): DIRAC.gLogger.error('Broken string found in simtel.log') jobReport.setApplicationStatus('Broken pipe') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Failed to execute run_sim.sh') DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode) DIRAC.exit(-1) ## check simtel data/log/histo Output File exist cfg = cfg_dict[current_conf] simtelFileName = os.path.basename(corsikaFileLFN).replace( 'corsika.gz', 'simtel.gz') cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Data/*.simtel.gz ' + simtelFileName if (os.system(cmd)): DIRAC.exit(-1) simtelLogFileName = simtelFileName.replace('simtel.gz', 'log.gz') cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Log/*.log.gz ' + simtelLogFileName if (os.system(cmd)): DIRAC.exit(-1) simtelHistFileName = simtelFileName.replace('simtel.gz', 'hdata.gz') cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName if (os.system(cmd)): DIRAC.exit(-1) ################################################################ DIRAC.exit()
import DIRAC from DIRAC.Core.Base import Script Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, 'Arguments:', ' PilotID: Grid ID of the pilot' ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) < 1: Script.showHelp() from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin diracAdmin = DiracAdmin() exitCode = 0 errorList = [] for gridID in args: result = diracAdmin.getPilotOutput(gridID) if not result['OK']: errorList.append((gridID, result['Message'])) exitCode = 2 for error in errorList: print "ERROR %s: %s" % error DIRAC.exit(exitCode)
#from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from DIRAC import gConfig, gLogger from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.Security.ProxyInfo import getProxyInfo #csAPI = CSAPI() diracAdmin = DiracAdmin() exitCode = 0 errorList = [] setup = gConfig.getValue('/DIRAC/Setup', '') if not setup: print 'ERROR: Could not contact Configuration Service' exitCode = 2 DIRAC.exit(exitCode) res = getProxyInfo() if not res['OK']: gLogger.error('Failed to get proxy information', res['Message']) DIRAC.exit(2) userName = res['Value'].get('username') if not userName: gLogger.error('Failed to get username for proxy') DIRAC.exit(2) if site: res = gConfig.getOptionsDict('/Resources/Sites/LCG/%s' % site) if not res['OK']: gLogger.error('The provided site (%s) is not known.' % site)
def main(): from DIRAC.Core.Base import Script Script.registerSwitch("p:", "inputfile=", "Input File", setInputFile) Script.registerSwitch("E:", "simtelExecName=", "SimtelExecName", setExecutable) Script.registerSwitch("V:", "version=", "Version", setVersion) Script.registerSwitch("D:", "storage_element=", "Storage Element", setStorageElement) from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient from DIRAC.Resources.Catalog.FileCatalog import FileCatalog Script.parseCommandLine() global fcc, fcL, storage_element from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea from DIRAC.Core.Utilities.Subprocess import systemCall from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport jobID = os.environ['JOBID'] jobID = int(jobID) jobReport = JobReport(jobID) CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray' packs = [CorsikaSimtelPack] for package in packs: DIRAC.gLogger.notice('Checking:', package) if sharedArea: if checkSoftwarePackage(package, sharedArea())['OK']: DIRAC.gLogger.notice('Package found in Shared Area:', package) installSoftwareEnviron(package, workingArea()) packageTuple = package.split('/') corsika_subdir = sharedArea( ) + '/' + packageTuple[0] + '/' + version cmd = 'cp -u -r ' + corsika_subdir + '/* .' os.system(cmd) continue if workingArea: if checkSoftwarePackage(package, workingArea())['OK']: DIRAC.gLogger.notice('Package found in Local Area:', package) continue if installSoftwarePackage(package, workingArea())['OK']: ############## compile ############################# if version == 'prod-2_21122012': cmdTuple = ['./build_all', 'prod2', 'qgs2'] else: cmdTuple = ['./build_all', 'ultra', 'qgs2'] ret = systemCall(0, cmdTuple, sendOutput) if not ret['OK']: DIRAC.gLogger.error('Failed to compile') DIRAC.exit(-1) continue DIRAC.gLogger.error('Check Failed for software package:', package) DIRAC.gLogger.error('Software package not available') DIRAC.exit(-1) ########### ## Checking MD coherence fc = FileCatalog('LcgFileCatalog') res = fc._getCatalogConfigDetails('DIRACFileCatalog') print 'DFC CatalogConfigDetails:', res res = fc._getCatalogConfigDetails('LcgFileCatalog') print 'LCG CatalogConfigDetails:', res fcc = FileCatalogClient() fcL = FileCatalog('LcgFileCatalog') from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() ############################ ############# # CLAUDIA: simtelConfigFile should be built from ??? simtelConfigFilesPath = 'sim_telarray/multi' simtelConfigFile = simtelConfigFilesPath + '/multi_cta-ultra5.cfg' createGlobalsFromConfigFiles(simtelConfigFile) ####################### ## files spread in 1000-runs subDirectories corsikaFileName = os.path.basename(corsikaFileLFN) run_number = corsikaFileName.split('run')[1].split('.corsika.gz')[ 0] # run001412.corsika.gz runNum = int(run_number) subRunNumber = '%03d' % runNum runNumModMille = runNum % 1000 runNumTrunc = (runNum - runNumModMille) / 1000 runNumSeriesDir = '%03dxxx' % runNumTrunc print 'runNumSeriesDir=', runNumSeriesDir f = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w') f.close() ##### If storage element is IN2P3-tape save simtel file on disk ############### if storage_element == 'CC-IN2P3-Tape': storage_element = 'CC-IN2P3-Disk' ############ Producing SimTel File ######################Building simtel Directory Metadata ####################### resultCreateSimtelDirMD = createSimtelFileSystAndMD() if not resultCreateSimtelDirMD['OK']: DIRAC.gLogger.error('Failed to create simtelArray Directory MD') jobReport.setApplicationStatus( 'Failed to create simtelArray Directory MD') DIRAC.gLogger.error( 'Metadata coherence problem, no simtelArray File produced') DIRAC.exit(-1) else: print 'simtel Directory MD successfully created' #### execute simtelarray ################ fd = open('run_sim.sh', 'w') fd.write( """#! /bin/sh echo "go for sim_telarray" . ./examples_common.sh export CORSIKA_IO_BUFFER=800MB zcat %s | $SIM_TELARRAY_PATH/run_sim_%s""" % (corsikaFileName, simtelExecName)) fd.close() os.system('chmod u+x run_sim.sh') cmdTuple = ['./run_sim.sh'] ret = systemCall(0, cmdTuple, sendOutputSimTel) simtelReturnCode, stdout, stderr = ret['Value'] if (os.system('grep Broken simtel.log')): print 'not broken' else: print 'broken' # Tag corsika File if Broken Pipe corsikaTagMD = {} corsikaTagMD['CorsikaToReprocess'] = 'CorsikaToReprocess' result = fcc.setMetadata(corsikaFileLFN, corsikaTagMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] jobReport.setApplicationStatus('Broken pipe') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Failed to execute run_sim.sh') DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode) DIRAC.exit(-1) ## putAndRegister simtel data/log/histo Output File: simtelFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.simtel.gz' cmd = 'mv Data/sim_telarray/' + simtelExecName + '/0.0deg/Data/*.simtel.gz ' + simtelFileName if (os.system(cmd)): DIRAC.exit(-1) simtelOutFileDir = os.path.join(simtelDirPath, 'Data', runNumSeriesDir) simtelOutFileLFN = os.path.join(simtelOutFileDir, simtelFileName) simtelRunNumberSeriesDirExist = fcc.isDirectory( simtelOutFileDir)['Value']['Successful'][simtelOutFileDir] newSimtelRunFileSeriesDir = ( simtelRunNumberSeriesDirExist != True ) # if new runFileSeries, will need to add new MD simtelLogFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz' cmd = 'mv Data/sim_telarray/' + simtelExecName + '/0.0deg/Log/*.log.gz ' + simtelLogFileName if (os.system(cmd)): DIRAC.exit(-1) simtelOutLogFileDir = os.path.join(simtelDirPath, 'Log', runNumSeriesDir) simtelOutLogFileLFN = os.path.join(simtelOutLogFileDir, simtelLogFileName) simtelHistFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.hdata.gz' cmd = 'mv Data/sim_telarray/' + simtelExecName + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName if (os.system(cmd)): DIRAC.exit(-1) simtelOutHistFileDir = os.path.join(simtelDirPath, 'Histograms', runNumSeriesDir) simtelOutHistFileLFN = os.path.join(simtelOutHistFileDir, simtelHistFileName) ################################################ DIRAC.gLogger.notice('Put and register simtel File in LFC and DFC:', simtelOutFileLFN) ret = dirac.addFile(simtelOutFileLFN, simtelFileName, storage_element) res = CheckCatalogCoherence(simtelOutFileLFN) if res != DIRAC.S_OK: DIRAC.gLogger.error('Job failed: Catalog Coherence problem found') jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Error during addFile call:', ret['Message']) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) ###################################################################### DIRAC.gLogger.notice('Put and register simtel Log File in LFC and DFC:', simtelOutLogFileLFN) ret = dirac.addFile(simtelOutLogFileLFN, simtelLogFileName, storage_element) res = CheckCatalogCoherence(simtelOutLogFileLFN) if res != DIRAC.S_OK: DIRAC.gLogger.error('Job failed: Catalog Coherence problem found') jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Error during addFile call:', ret['Message']) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) ###################################################################### DIRAC.gLogger.notice('Put and register simtel Histo File in LFC and DFC:', simtelOutHistFileLFN) ret = dirac.addFile(simtelOutHistFileLFN, simtelHistFileName, storage_element) res = CheckCatalogCoherence(simtelOutHistFileLFN) if res != DIRAC.S_OK: DIRAC.gLogger.error('Job failed: Catalog Coherence problem found') jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Error during addFile call:', ret['Message']) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) ###################################################################### if newSimtelRunFileSeriesDir: insertRunFileSeriesMD(simtelOutFileDir, runNumTrunc) insertRunFileSeriesMD(simtelOutLogFileDir, runNumTrunc) insertRunFileSeriesMD(simtelOutHistFileDir, runNumTrunc) ###### simtel File level metadata ############################################ simtelFileMD = {} simtelFileMD['runNumber'] = int(run_number) simtelFileMD['jobID'] = jobID simtelFileMD['simtelReturnCode'] = simtelReturnCode result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.setMetadata(simtelOutLogFileLFN, simtelFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.setMetadata(simtelOutHistFileLFN, simtelFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.addFileAncestors( {simtelOutFileLFN: { 'Ancestors': [corsikaFileLFN] }}) print 'result addFileAncestor:', result result = fcc.addFileAncestors( {simtelOutLogFileLFN: { 'Ancestors': [corsikaFileLFN] }}) print 'result addFileAncestor:', result result = fcc.addFileAncestors( {simtelOutHistFileLFN: { 'Ancestors': [corsikaFileLFN] }}) print 'result addFileAncestor:', result result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD) if not result['OK']: print 'ResultSetMetadata:', result['Message'] DIRAC.exit()
def setHostDN(arg): global hostDN if hostDN or not arg: Script.showHelp() DIRAC.exit(-1) hostDN = arg
def setHostName(arg): global hostName if hostName or not arg: Script.showHelp() DIRAC.exit(-1) hostName = arg
def main(): Script.registerSwitch( "v:", "vo=", "Location of pilot version in CS /Operations/<vo>/Pilot/Version" " (default value specified in CS under /DIRAC/DefaultSetup)") Script.parseCommandLine(ignoreErrors=False) args = Script.getPositionalArgs() if len(args) < 1 or len(args) > 2: Script.showHelp() version = args[0] vo = None for switch in Script.getUnprocessedSwitches(): if switch[0] == "v" or switch[0] == "vo": vo = switch[1] from DIRAC import S_OK, S_ERROR from DIRAC import gConfig, gLogger from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI def updatePilot(version, vo): """ Update in the CS the pilot version used, If only one version present in CS it's overwritten. If two versions present, the new one is added and the last removed :param version: version vArBpC of pilot you want to use :param vo: Location of pilot version in CS /Operations/<vo>/Pilot/Version """ setup = vo if not vo: setup = gConfig.getValue('/DIRAC/DefaultSetup') if not setup: return S_ERROR("No value set for /DIRAC/DefaultSetup in CS") pilotVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup, []) if not pilotVersion: return S_ERROR( "No pilot version set under Operations/%s/Pilot/Version in CS" % setup) pilotVersion.pop() pilotVersion.insert(0, version) api = CSAPI() api.setOption('Operations/%s/Pilot/Version' % setup, ", ".join(pilotVersion)) result = api.commit() if not result['OK']: gLogger.fatal('Could not commit new version of pilot!') return result newVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup) return S_OK("New version of pilot set to %s" % newVersion) result = updatePilot(version, vo) if not result['OK']: gLogger.fatal(result['Message']) DIRAC.exit(1) gLogger.notice(result['Value']) DIRAC.exit(0)
def main(): from DIRAC import gLogger from DIRAC.Core.Base import Script Script.parseCommandLine() from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea from CTADIRAC.Core.Utilities.SoftwareInstallation import createSharedArea from DIRAC.Core.Utilities.Subprocess import systemCall DIRAC.gLogger.notice('Platform is:') os.system('dirac-platform') args = Script.getPositionalArgs() version = args[0] area = sharedArea() if area: gLogger.notice('Using Shared Area at:', area) # if defined, check that it really exists if not os.path.isdir(area): gLogger.error('Missing Shared Area Directory:', area) if createSharedArea() == True: gLogger.notice('Shared Area created') else: gLogger.error('Failed to create Shared Area Directory:', area) DIRAC.exit(-1) else: if createSharedArea() == True: gLogger.notice('Shared Area created') else: gLogger.error('Failed to create Shared Area Directory:', area) DIRAC.exit(-1) EvnDispSimtelPack = os.path.join('evndisplay', version, 'evndisplay') packs = [EvnDispSimtelPack] for package in packs: DIRAC.gLogger.notice('Checking:', package) packageTuple = package.split('/') if sharedArea: if checkSoftwarePackage(package, sharedArea())['OK']: DIRAC.gLogger.notice('Package found in Shared Area:', package) continue if installSoftwarePackage(package, sharedArea())['OK']: if not os.path.isdir( os.path.join(sharedArea(), packageTuple[0], packageTuple[1])): DIRAC.gLogger.error( 'Software package missing in the shared area') DIRAC.exit(-1) continue DIRAC.gLogger.error('Software package not correctly installed') DIRAC.exit(-1) DIRAC.exit()
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(" sourceSE: source SE") Script.registerArgument(" LFN: LFN or file containing a List of LFNs") Script.registerArgument(["targetSE: target SEs"]) Script.parseCommandLine() import DIRAC from DIRAC import gLogger # parseCommandLine show help when mandatory arguments are not specified or incorrect argument args = Script.getPositionalArgs() sourceSE = args[0] lfnList = getLFNList(args[1]) targetSEs = list(set([se for targetSE in args[2:] for se in targetSE.split(",")])) gLogger.info( "Will create request with 'MoveReplica' " "operation using %s lfns and %s target SEs" % (len(lfnList), len(targetSEs)) ) from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.RequestManagementSystem.Client.Request import Request from DIRAC.RequestManagementSystem.Client.Operation import Operation from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.Core.Utilities.List import breakListIntoChunks lfnChunks = breakListIntoChunks(lfnList, 100) multiRequests = len(lfnChunks) > 1 error = 0 count = 0 reqClient = ReqClient() fc = FileCatalog() for lfnChunk in lfnChunks: metaDatas = fc.getFileMetadata(lfnChunk) if not metaDatas["OK"]: gLogger.error("unable to read metadata for lfns: %s" % metaDatas["Message"]) error = -1 continue metaDatas = metaDatas["Value"] for failedLFN, reason in metaDatas["Failed"].items(): gLogger.error("skipping %s: %s" % (failedLFN, reason)) lfnChunk = set(metaDatas["Successful"]) if not lfnChunk: gLogger.error("LFN list is empty!!!") error = -1 continue if len(lfnChunk) > Operation.MAX_FILES: gLogger.error("too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES) error = -1 continue count += 1 request = Request() request.RequestName = "%s_%s" % ( md5(repr(time.time()).encode()).hexdigest()[:16], md5(repr(time.time()).encode()).hexdigest()[:16], ) moveReplica = Operation() moveReplica.Type = "MoveReplica" moveReplica.SourceSE = sourceSE moveReplica.TargetSE = ",".join(targetSEs) for lfn in lfnChunk: metaDict = metaDatas["Successful"][lfn] opFile = File() opFile.LFN = lfn opFile.Size = metaDict["Size"] if "Checksum" in metaDict: # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD' opFile.Checksum = metaDict["Checksum"] opFile.ChecksumType = "ADLER32" moveReplica.addFile(opFile) request.addOperation(moveReplica) result = reqClient.putRequest(request) if not result["OK"]: gLogger.error("Failed to submit Request: %s" % (result["Message"])) error = -1 continue if not multiRequests: gLogger.always("Request %d submitted successfully" % result["Value"]) if multiRequests: gLogger.always("%d requests have been submitted" % (count)) DIRAC.exit(error)
def main(): read = True write = True check = True remove = True sites = [] mute = False Script.registerSwitch("r", "BanRead", " Ban only reading from the storage element") Script.registerSwitch("w", "BanWrite", " Ban writing to the storage element") Script.registerSwitch("k", "BanCheck", " Ban check access to the storage element") Script.registerSwitch("v", "BanRemove", " Ban remove access to the storage element") Script.registerSwitch("a", "All", " Ban all access to the storage element") Script.registerSwitch("m", "Mute", " Do not send email") Script.registerSwitch( "S:", "Site=", " Ban all SEs associate to site (note that if writing is allowed, check is always allowed)" ) # Registering arguments will automatically add their description to the help menu Script.registerArgument( ["seGroupList: list of SEs or comma-separated SEs"]) switches, ses = Script.parseCommandLine(ignoreErrors=True) for switch in switches: if switch[0].lower() in ("r", "banread"): write = False check = False remove = False if switch[0].lower() in ("w", "banwrite"): read = False check = False remove = False if switch[0].lower() in ("k", "bancheck"): read = False write = False remove = False if switch[0].lower() in ("v", "banremove"): read = False write = False check = False if switch[0].lower() in ("a", "all"): pass if switch[0].lower() in ("m", "mute"): mute = True if switch[0].lower() in ("s", "site"): sites = switch[1].split(",") # from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC import gConfig, gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup, DMSHelpers ses = resolveSEGroup(ses) diracAdmin = DiracAdmin() setup = gConfig.getValue("/DIRAC/Setup", "") if not setup: print("ERROR: Could not contact Configuration Service") DIRAC.exit(2) res = getProxyInfo() if not res["OK"]: gLogger.error("Failed to get proxy information", res["Message"]) DIRAC.exit(2) userName = res["Value"].get("username") if not userName: gLogger.error("Failed to get username for proxy") DIRAC.exit(2) for site in sites: res = DMSHelpers().getSEsForSite(site) if not res["OK"]: gLogger.error(res["Message"], site) DIRAC.exit(-1) ses.extend(res["Value"]) if not ses: gLogger.error("There were no SEs provided") DIRAC.exit(-1) readBanned = [] writeBanned = [] checkBanned = [] removeBanned = [] resourceStatus = ResourceStatus() res = resourceStatus.getElementStatus(ses, "StorageElement") if not res["OK"]: gLogger.error("Storage Element %s does not exist" % ses) DIRAC.exit(-1) reason = "Forced with dirac-admin-ban-se by %s" % userName for se, seOptions in res["Value"].items(): resW = resC = resR = {"OK": False} # Eventually, we will get rid of the notion of InActive, as we always write Banned. if read and "ReadAccess" in seOptions: if seOptions["ReadAccess"] == "Banned": gLogger.notice("Read access already banned", se) resR["OK"] = True elif not seOptions["ReadAccess"] in [ "Active", "Degraded", "Probing" ]: gLogger.notice("Read option for %s is %s, instead of %s" % (se, seOptions["ReadAccess"], ["Active", "Degraded", "Probing"])) gLogger.notice("Try specifying the command switches") else: resR = resourceStatus.setElementStatus(se, "StorageElement", "ReadAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" ) if not resR["OK"]: gLogger.error("Failed to update %s read access to Banned" % se) else: gLogger.notice( "Successfully updated %s read access to Banned" % se) readBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if write and "WriteAccess" in seOptions: if seOptions["WriteAccess"] == "Banned": gLogger.notice("Write access already banned", se) resW["OK"] = True elif not seOptions["WriteAccess"] in [ "Active", "Degraded", "Probing" ]: gLogger.notice("Write option for %s is %s, instead of %s" % (se, seOptions["WriteAccess"], ["Active", "Degraded", "Probing"])) gLogger.notice("Try specifying the command switches") else: resW = resourceStatus.setElementStatus(se, "StorageElement", "WriteAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" ) if not resW["OK"]: gLogger.error( "Failed to update %s write access to Banned" % se) else: gLogger.notice( "Successfully updated %s write access to Banned" % se) writeBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if check and "CheckAccess" in seOptions: if seOptions["CheckAccess"] == "Banned": gLogger.notice("Check access already banned", se) resC["OK"] = True elif not seOptions["CheckAccess"] in [ "Active", "Degraded", "Probing" ]: gLogger.notice("Check option for %s is %s, instead of %s" % (se, seOptions["CheckAccess"], ["Active", "Degraded", "Probing"])) gLogger.notice("Try specifying the command switches") else: resC = resourceStatus.setElementStatus(se, "StorageElement", "CheckAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) if not resC["OK"]: gLogger.error( "Failed to update %s check access to Banned" % se) else: gLogger.notice( "Successfully updated %s check access to Banned" % se) checkBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if remove and "RemoveAccess" in seOptions: if seOptions["RemoveAccess"] == "Banned": gLogger.notice("Remove access already banned", se) resC["OK"] = True elif not seOptions["RemoveAccess"] in [ "Active", "Degraded", "Probing" ]: gLogger.notice("Remove option for %s is %s, instead of %s" % (se, seOptions["RemoveAccess"], ["Active", "Degraded", "Probing"])) gLogger.notice("Try specifying the command switches") else: resC = resourceStatus.setElementStatus(se, "StorageElement", "RemoveAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) if not resC["OK"]: gLogger.error( "Failed to update %s remove access to Banned" % se) else: gLogger.notice( "Successfully updated %s remove access to Banned" % se) removeBanned.append(se) if not (resR["OK"] or resW["OK"] or resC["OK"]): DIRAC.exit(-1) if not (writeBanned or readBanned or checkBanned or removeBanned): gLogger.notice("No storage elements were banned") DIRAC.exit(-1) if mute: gLogger.notice("Email is muted by script switch") DIRAC.exit(0) subject = "%s storage elements banned for use" % len( writeBanned + readBanned + checkBanned + removeBanned) addressPath = "EMail/Production" address = Operations().getValue(addressPath, "") body = "" if read: body = "%s\n\nThe following storage elements were banned for reading:" % body for se in readBanned: body = "%s\n%s" % (body, se) if write: body = "%s\n\nThe following storage elements were banned for writing:" % body for se in writeBanned: body = "%s\n%s" % (body, se) if check: body = "%s\n\nThe following storage elements were banned for check access:" % body for se in checkBanned: body = "%s\n%s" % (body, se) if remove: body = "%s\n\nThe following storage elements were banned for remove access:" % body for se in removeBanned: body = "%s\n%s" % (body, se) if not address: gLogger.notice( "'%s' not defined in Operations, can not send Mail\n" % addressPath, body) DIRAC.exit(0) res = diracAdmin.sendMail(address, subject, body) gLogger.notice("Notifying %s" % address) if res["OK"]: gLogger.notice(res["Value"]) else: gLogger.notice(res["Message"]) DIRAC.exit(0)
def setupWorkflow(self, debug=False): """ Override the base class job workflow to adapt to NSB test simulations All parameters shall have been defined before that method is called. """ # step 1 - debug only iStep = 1 if debug: lsStep = self.setExecutable('/bin/ls -alhtr', logFile='LS_Init_Log.txt') lsStep['Value']['name'] = 'Step%i_LS_Init' % iStep lsStep['Value']['descr_short'] = 'list files in working directory' iStep += 1 # step 2 swStep = self.setExecutable( 'cta-prod3-setupsw', arguments='%s %s simulations sl6-gcc44'% (self.package, self.version),\ logFile='SetupSoftware_Log.txt') swStep['Value']['name'] = 'Step%i_SetupSoftware' % iStep swStep['Value']['descr_short'] = 'Setup software' iStep += 1 # step 3 if self.cta_site == 'Paranal': prod_script = './dirac_prod3_paranal_baseline' elif self.cta_site == 'LaPalma': prod_script = './dirac_prod3_lapalma_baseline' else: DIRAC.gLogger.error('Unknown site: %s' % self.cta_site) DIRAC.gLogger.error('No shell script associated') DIRAC.exit(-1) csStep = self.setExecutable( prod_script, \ arguments = '--start_run %s --run %s %s %s %s %s %s' % \ ( self.start_run_number, self.run_number, \ self.extra_tag, self.cta_site,\ self.particle, self.pointing_dir, self.zenith_angle ), \ logFile='CorsikaSimtel_Log.txt') csStep['Value']['name'] = 'Step%i_CorsikaSimtel' % iStep csStep['Value']['descr_short'] = 'Run Corsika piped into simtel' iStep += 1 # step 4 verify merged data mgvStep = self.setExecutable( 'cta-prod3-verifysteps', \ arguments = "generic %0d 1000 '%s/Data/*.zst'"%\ (self.N_output_files, self.inputpath),\ logFile='Verify_Simtel_Log.txt') mgvStep['Value']['name'] = 'Step%i_VerifySimtel' % iStep mgvStep['Value']['descr_short'] = 'Verify simtel files' iStep += 1 # step 5 - debug only if debug: lsStep = self.setExecutable('/bin/ls -Ralhtr', logFile='LS_End_Log.txt') lsStep['Value']['name'] = 'Step%i_LS_End' % iStep lsStep['Value'][ 'descr_short'] = 'list files in working directory and sub-directory' iStep += 1 # step 6 # meta data # ## the order of the metadata dictionary is important, since it's used to build the directory structure metadata = collections.OrderedDict() metadata['array_layout'] = self.array_layout metadata['site'] = self.cta_site metadata['particle'] = self.particle if self.pointing_dir == 'North': metadata['phiP'] = 180 if self.pointing_dir == 'South': metadata['phiP'] = 0 metadata['thetaP'] = float(self.zenith_angle) metadata[self.program_category + '_prog'] = 'simtel' metadata[self.program_category + '_prog_version'] = self.version metadata['data_level'] = self.output_data_level metadata['configuration_id'] = self.configuration_id mdjson = json.dumps(metadata) # meta data field metadatafield = { 'array_layout': 'VARCHAR(128)', 'site': 'VARCHAR(128)', 'particle': 'VARCHAR(128)', 'phiP': 'float', 'thetaP': 'float', self.program_category + '_prog': 'VARCHAR(128)', self.program_category + '_prog_version': 'VARCHAR(128)', 'data_level': 'int', 'configuration_id': 'int' } mdfieldjson = json.dumps(metadatafield) # register Data ## file meta data #filemetadata = {'runNumber': self.run_number, 'nsb':1} filemetadata = {'runNumber': self.run_number} file_md_json = json.dumps(filemetadata) outputpattern = './Data/sim_telarray/*/*/Data/*baseline.simtel.zst' dmStep = self.setExecutable('../CTADIRAC/Core/scripts/cta-analysis-managedata.py', arguments = "'%s' '%s' '%s' %s '%s' %s %s '%s'" %\ (mdjson, mdfieldjson, file_md_json, self.basepath, outputpattern, self.package, self.program_category, self.catalogs), logFile = 'Data_DataManagement_Log.txt') dmStep['Value']['name'] = 'Step%i_DataManagement_1' % iStep dmStep['Value'][ 'descr_short'] = 'Save data files to SE and register them in DFC' iStep += 1 ## log file outputpattern = './Data/sim_telarray/*/*/Log/*baseline.log.gz' dmStep = self.setExecutable('../CTADIRAC/Core/scripts/cta-analysis-managedata.py', arguments = "'%s' '%s' '%s' %s '%s' %s %s '%s' Log" % \ (mdjson, mdfieldjson, file_md_json, self.basepath, outputpattern, self.package, self.program_category, self.catalogs), logFile = 'Log_DataManagement_Log.txt') dmStep['Value']['name'] = 'Step%i_Log_DataManagement' % iStep dmStep['Value'][ 'descr_short'] = 'Save log files to SE and register them in DFC' iStep += 1 ## histogram # outputpattern = './Data/sim_telarray/*/*/Histograms/*baseline.hdata.gz' # dmStep = self.setExecutable('../CTADIRAC/Core/scripts/cta-analysis-managedata.py', # arguments = "'%s' '%s' '%s' %s '%s' %s %s '%s' Histograms" % \ # (mdjson, mdfieldjson, file_md_json, self.basepath, # outputpattern, self.package, self.program_category, self.catalogs), # logFile = 'Histo_DataManagement_Log.txt') # dmStep['Value']['name'] = 'Step%i_Histo_DataManagement' % iStep # dmStep['Value']['descr_short'] = 'Save hitograms files to SE and register them in DFC' # iStep += 1 # Number of showers is passed via an environment variable self.setExecutionEnv({'NSHOW': '%s' % self.nShower})
def install_CorsikaSimtelPack(version): from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea from DIRAC.Core.Utilities.Subprocess import systemCall CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray' packs = [CorsikaSimtelPack] for package in packs: DIRAC.gLogger.notice('Checking:', package) if sharedArea: if checkSoftwarePackage(package, sharedArea())['OK']: DIRAC.gLogger.notice('Package found in Shared Area:', package) installSoftwareEnviron(package, workingArea()) packageTuple = package.split('/') corsika_subdir = sharedArea( ) + '/' + packageTuple[0] + '/' + version cmd = 'cp -u -r ' + corsika_subdir + '/* .' os.system(cmd) continue if workingArea: if checkSoftwarePackage(package, workingArea())['OK']: DIRAC.gLogger.notice('Package found in Local Area:', package) continue if installSoftwarePackage(package, workingArea())['OK']: ############## compile ############################# cmdTuple = ['./build_all', 'prod2', 'qgs2'] ######### special case for Astri ############################ if version == 'prod-2_08072014_to': ############## compile ############################# fd = open('run_compile.sh', 'w') fd.write("""#! /bin/sh source ./build_all prod2 qgs2 # echo " Let's check that build_all did its work " ls -alFsh echo "+++++++++++++++++++++++++++++++++++++++++++++++++" echo " Let's see what files are in the corsika-run directory " ls -alFsh ./corsika-run # if [ ! -x ./corsika-run/corsika ] then echo " ERROR: Corsika executable found. Exit " exit 1 fi echo "+++++++++++++++++++++++++++++++++++++++++++++++++" # echo " Now let's try to compile hessio according to Federico's recipe " cd ./hessioxxx make clean make EXTRA_DEFINES="-DCTA_PROD2 -DWITH_LOW_GAIN_CHANNEL" # echo " Let's see what files are in the lib directory " ls -alFsh ./lib # if [ ! -f ./lib/libhessio.so ] then echo " ERROR: libhessio library not found. Exit " exit 1 fi echo "+++++++++++++++++++++++++++++++++++++++++++++++++" # cd .. # come back to original dir # echo " Now let's try to compile simtel according to Federico's recipe " cd ./sim_telarray make clean make EXTRA_DEFINES="-DCTA_PROD2 -DWITH_LOW_GAIN_CHANNEL" make install # echo " Let's see what files are in the bin directory " ls -alFsh ./bin # if [ ! -x ./bin/sim_telarray ] then echo " ERROR: sim_telarray excutable not found. Exit " exit 1 fi echo "+++++++++++++++++++++++++++++++++++++++++++++++++" # echo " Everything was compiled and linked properly" """) fd.close() os.system('chmod u+x run_compile.sh') cmdTuple = ['./run_compile.sh'] ########################################################################## ret = systemCall(0, cmdTuple, sendOutput) if not ret['OK']: DIRAC.gLogger.error('Failed to execute build') DIRAC.exit(-1) continue DIRAC.gLogger.error('Check Failed for software package:', package) DIRAC.gLogger.error('Software package not available') DIRAC.exit(-1)
mjfPower = getPowerFromMJF() if mjfPower: gLogger.notice( 'CPU power from MJF is %.1f HS06' % mjfPower ) else: gLogger.notice( 'MJF not available on this node' ) if update and not configFile: gConfig.setOptionValue( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm ) gConfig.setOptionValue( '/LocalSite/CPUNormalizationFactor', norm ) gConfig.dumpLocalCFGToFile( gConfig.diracConfigFilePath ) if configFile: from DIRAC.Core.Utilities.CFG import CFG cfg = CFG() try: # Attempt to open the given file cfg.loadFromFile( configFile ) except: pass # Create the section if it does not exist if not cfg.existsKey( 'LocalSite' ): cfg.createNewSection( 'LocalSite' ) cfg.setOption( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm ) cfg.setOption( '/LocalSite/CPUNormalizationFactor', norm ) cfg.writeToFile( configFile ) DIRAC.exit()
def platform(arg): Script.disableCS() print(DIRAC.getPlatform()) DIRAC.exit(0)
reqClient = ReqClient() fc = FileCatalog() for lfnList in breakListIntoChunks( lfns, 100 ): oRequest = Request() oRequest.RequestName = "%s_%s" % ( md5( repr( time.time() ) ).hexdigest()[:16], md5( repr( time.time() ) ).hexdigest()[:16] ) replicateAndRegister = Operation() replicateAndRegister.Type = 'ReplicateAndRegister' replicateAndRegister.TargetSE = targetSE res = fc.getFileMetadata( lfnList ) if not res['OK']: print "Can't get file metadata: %s" % res['Message'] DIRAC.exit( 1 ) if res['Value']['Failed']: print "Could not get the file metadata of the following, so skipping them:" for fFile in res['Value']['Failed']: print fFile lfnMetadata = res['Value']['Successful'] for lfn in lfnMetadata: rarFile = File() rarFile.LFN = lfn rarFile.Size = lfnMetadata[lfn]['Size'] rarFile.Checksum = lfnMetadata[lfn]['Checksum'] rarFile.GUID = lfnMetadata[lfn]['GUID'] rarFile.ChecksumType = 'ADLER32' replicateAndRegister.addFile( rarFile )
def version(arg): Script.disableCS() print(DIRAC.version) DIRAC.exit(0)
Script.parseCommandLine( ignoreErrors = True ) args = Script.getPositionalArgs() if len( args ) < 2: Script.showHelp() if len( args ) > 2: print 'Only one LFN SE pair will be considered' from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() exitCode = 0 lfn = args[0] seName = args[1] try: f = open( lfn, 'r' ) lfns = f.read().splitlines() f.close() except: lfns = [lfn] for lfn in lfns: result = dirac.removeReplica( lfn, seName, printOutput = True ) if not result['OK']: print 'ERROR: ', result['Message'] exitCode = 2 DIRAC.exit( exitCode )
outStr = "%s\n------SRM staging request info--------------" % outStr for srid in stageRequests.keys(): outStr = "%s\n%s: %s" % ( outStr, 'SRM RequestID'.ljust( 8 ), stageRequests[srid]['RequestID'].ljust( 100 ) ) outStr = "%s\n%s: %s" % ( outStr, 'SRM StageStatus'.ljust( 8 ), stageRequests[srid]['StageStatus'].ljust( 100 ) ) outStr = "%s\n%s: %s" % ( outStr, 'SRM StageRequestSubmitTime'.ljust( 8 ), str(stageRequests[srid]['StageRequestSubmitTime']).ljust( 100 ) ) outStr = "%s\n%s: %s" % ( outStr, 'SRM StageRequestCompletedTime'.ljust( 8 ), str(stageRequests[srid]['StageRequestCompletedTime']).ljust( 100 ) ) outStr = "%s\n%s: %s" % ( outStr, 'SRM PinExpiryTime'.ljust( 8 ), str(stageRequests[srid]['PinExpiryTime']).ljust( 100 ) ) outStr = "%s\n%s: %s sec" % ( outStr, 'SRM PinLength'.ljust( 8 ), str(stageRequests[srid]['PinLength']).ljust( 100 ) ) else: outStr = '%s\nThere are no staging requests submitted to the site yet.'.ljust( 8) % outStr else: outStr = "\nThere is no such file requested for staging. Check for typo's!" #Script.showHelp() print outStr DIRAC.exit( 0 ) ''' Example: dirac-stager-monitor-file.py /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056603_1.full.dst GRIDKA-RDST -------------------- LFN : /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056603_1.full.dst SE : GRIDKA-RDST PFN : srm://gridka-dCache.fzk.de/pnfs/gridka.de/lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056603_1.full.dst Status : StageSubmitted LastUpdate: 2013-06-11 18:13:40 Reason : None Jobs requesting this file to be staged: 48518896 ------SRM staging request info-------------- SRM RequestID: -1768636375 SRM StageStatus: StageSubmitted SRM StageRequestSubmitTime: 2013-06-11 18:13:38
def CorsikaSimtelExample(args=None): from CTADIRAC.Interfaces.API.CorsikaSimtelJob import CorsikaSimtelJob from DIRAC.Interfaces.API.Dirac import Dirac j = CorsikaSimtelJob() j.setVersion('prod-2_15122013') j.setExecutable('corsika_autoinputs') if len(args) != 4: Script.showHelp() if args[3] not in [ 'STD', '4MSST', 'SCSST', 'ASTRI', 'NSBX3', 'NORTH', 'SCMST' ]: Script.gLogger.notice("reprocessing configuration is incorrect:", args[3]) DIRAC.exit(-1) runMin = int(args[0]) runMax = int(args[1]) cfgfile = args[2] simtelArrayConfig = args[3] ilist = [] for i in range(runMin, runMax + 1): run_number = '%06d' % i ilist.append(run_number) j.setGenericParametricInput(ilist) j.setName('run%s') j.setInputSandbox([ cfgfile, 'grid_prod2-repro.sh', 'LFN:/vo.cta.in2p3.fr/MC/PROD2/SVN-PROD2.tar.gz' ]) j.setParameters([ '--template', cfgfile, '--mode', 'corsika_simtel', '-S', simtelArrayConfig ]) j.setOutputSandbox( ['corsika_autoinputs.log', 'simtel.log', 'applicationLog.txt']) # Retrieve your Output Data corsika_out = '*.corsika.gz' corsikatar_out = '*.corsika.tar.gz' sim_out = '*.simtel.gz' log_out = '*.log.gz' hist_out = '*.hdata.gz' j.setOutputData([corsika_out, corsikatar_out, sim_out, log_out, hist_out]) j.setCPUTime(200000) Script.gLogger.info(j._toJDL()) res = Dirac().submit(j) print res
def setUserDN(arg): global userDN if userDN or not arg: Script.showHelp() DIRAC.exit(-1) userDN = arg
gLogger.notice(" + %s" % host) gLogger.notice("Succeeded to restart:") for host in restartSuccess: gLogger.notice(" + %s" % host) gLogger.notice("Failed to update:") for host in updateFail: gLogger.notice(" - %s" % host) gLogger.notice("Failed to restart:") for host in restartFail: gLogger.notice(" - %s" % host) gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") if retry > 0: retryHosts = list(set(updateFail + restartFail)) gLogger.notice("Retrying update on (%s attempts remaining):" % retry) for host in retryHosts: gLogger.notice(" - %s" % host) gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") return updateInstance(version, retryHosts, retry - 1) return S_ERROR("Update failed!") if __name__ == "__main__": result = updateInstance(version, hosts, retry) if not result['OK']: gLogger.fatal(result['Message']) DIRAC.exit(1) gLogger.notice(result['Value']) DIRAC.exit(0)
def setUserName(arg): global userName if userName or not arg: Script.showHelp() DIRAC.exit(-1) userName = arg
DIRAC.gLogger.notice('Checking:', package) packageTuple = package.split('/') if sharedArea: if checkSoftwarePackage(package, sharedArea())['OK']: DIRAC.gLogger.notice('Package found in Shared Area:', package) continue if installSoftwarePackage(package, sharedArea())['OK']: if not os.path.isdir( os.path.join(sharedArea(), packageTuple[0], packageTuple[1])): DIRAC.gLogger.error( 'Software package missing in the shared area') DIRAC.exit(-1) continue DIRAC.gLogger.error('Software package not correctly installed') DIRAC.exit(-1) DIRAC.exit() if __name__ == '__main__': try: main() except Exception: DIRAC.gLogger.exception() DIRAC.exit(-1)
mjfPower = getPowerFromMJF() if mjfPower: gLogger.notice('CPU power from MJF is %.1f HS06' % mjfPower) else: gLogger.notice('MJF not available on this node') if update and not configFile: gConfig.setOptionValue('/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm) gConfig.setOptionValue('/LocalSite/CPUNormalizationFactor', norm) gConfig.dumpLocalCFGToFile(gConfig.diracConfigFilePath) if configFile: from DIRAC.Core.Utilities.CFG import CFG cfg = CFG() try: # Attempt to open the given file cfg.loadFromFile(configFile) except: pass # Create the section if it does not exist if not cfg.existsKey('LocalSite'): cfg.createNewSection('LocalSite') cfg.setOption('/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm) cfg.setOption('/LocalSite/CPUNormalizationFactor', norm) cfg.writeToFile(configFile) DIRAC.exit()
def install_CorsikaSimtelPack(version): from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea from DIRAC.Core.Utilities.Subprocess import systemCall CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray' packs = [CorsikaSimtelPack] for package in packs: DIRAC.gLogger.notice('Checking:', package) if sharedArea: if checkSoftwarePackage(package, sharedArea())['OK']: DIRAC.gLogger.notice('Package found in Shared Area:', package) installSoftwareEnviron(package, workingArea()) packageTuple = package.split('/') corsika_subdir = sharedArea( ) + '/' + packageTuple[0] + '/' + version cmd = 'cp -u -r ' + corsika_subdir + '/* .' os.system(cmd) continue if workingArea: if checkSoftwarePackage(package, workingArea())['OK']: DIRAC.gLogger.notice('Package found in Local Area:', package) continue if installSoftwarePackage(package, workingArea())['OK']: ############## compile ############################# if version == 'prod-2_22072013_tox': ############### compile tox ################ fd = open('run_compile.sh', 'w') fd.write("""#! /bin/sh ./build_all prod2 qgs2 # If the code was already there, we just clean but do not remove it. if [ -d "hessioxxx" ]; then (cd hessioxxx && make clean && make EXTRA_DEFINES='-DH_MAX_TEL=55 -DH_MAX_PIX=1984 -DH_MAX_SECTORS=13100 -DNO_LOW_GAIN') fi # If the code was already there, we just clean but do not remove it. if [ -d "sim_telarray" ]; then (cd sim_telarray && make clean && make EXTRA_DEFINES='-DH_MAX_TEL=55 -DH_MAX_PIX=1984 -DH_MAX_SECTORS=13100 -DNO_LOW_GAIN' install) fi""") fd.close() os.system('chmod u+x run_compile.sh') cmdTuple = ['./run_compile.sh'] ########################################## else: cmdTuple = ['./build_all', 'prod2', 'qgs2'] ret = systemCall(0, cmdTuple, sendOutput) if not ret['OK']: DIRAC.gLogger.error('Failed to execute build') DIRAC.exit(-1) continue DIRAC.gLogger.error('Check Failed for software package:', package) DIRAC.gLogger.error('Software package not available') DIRAC.exit(-1)
outputDir = value elif sw.lower() in ( 'f', 'file' ): if os.path.exists( value ): jFile = open( value ) jobs += jFile.read().split() jFile.close() elif sw.lower() in ( 'g', 'jobgroup' ): group = value jobDate = toString( date() - 30 * day ) # Choose jobs in final state, no more than 30 days old result = dirac.selectJobs( jobGroup = value, date = jobDate, status = 'Done' ) if not result['OK']: if not "No jobs selected" in result['Message']: print "Error:", result['Message'] DIRAC.exit( -1 ) else: jobs += result['Value'] result = dirac.selectJobs( jobGroup = value, date = jobDate, status = 'Failed' ) if not result['OK']: if not "No jobs selected" in result['Message']: print "Error:", result['Message'] DIRAC.exit( -1 ) else: jobs += result['Value'] for arg in parseArguments( args ): if os.path.isdir( arg ): print "Output for job %s already retrieved, remove the output directory to redownload" % arg else: jobs.append( arg )
def main(): days = 0 months = 0 years = 0 wildcard = None baseDir = "" emptyDirsFlag = False Script.registerSwitch("D:", "Days=", "Match files older than number of days [%s]" % days) Script.registerSwitch( "M:", "Months=", "Match files older than number of months [%s]" % months) Script.registerSwitch( "Y:", "Years=", "Match files older than number of years [%s]" % years) Script.registerSwitch("w:", "Wildcard=", "Wildcard for matching filenames [All]") Script.registerSwitch( "b:", "BaseDir=", "Base directory to begin search (default /[vo]/user/[initial]/[username])" ) Script.registerSwitch("e", "EmptyDirs", "Create a list of empty directories") Script.parseCommandLine(ignoreErrors=False) for switch in Script.getUnprocessedSwitches(): if switch[0] == "D" or switch[0].lower() == "days": days = int(switch[1]) if switch[0] == "M" or switch[0].lower() == "months": months = int(switch[1]) if switch[0] == "Y" or switch[0].lower() == "years": years = int(switch[1]) if switch[0].lower() == "w" or switch[0].lower() == "wildcard": wildcard = "*" + switch[1] if switch[0].lower() == "b" or switch[0].lower() == "basedir": baseDir = switch[1] if switch[0].lower() == "e" or switch[0].lower() == "emptydirs": emptyDirsFlag = True import DIRAC from DIRAC import gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from datetime import datetime, timedelta import sys import os import time import fnmatch fc = FileCatalog() def isOlderThan(cTimeStruct, days): timeDelta = timedelta(days=days) maxCTime = datetime.utcnow() - timeDelta if cTimeStruct < maxCTime: return True return False withMetadata = False if days or months or years: withMetadata = True totalDays = 0 if years: totalDays += 365 * years if months: totalDays += 30 * months if days: totalDays += days res = getProxyInfo(False, False) if not res["OK"]: gLogger.error("Failed to get client proxy information.", res["Message"]) DIRAC.exit(2) proxyInfo = res["Value"] if proxyInfo["secondsLeft"] == 0: gLogger.error("Proxy expired") DIRAC.exit(2) username = proxyInfo["username"] vo = "" if "group" in proxyInfo: vo = getVOForGroup(proxyInfo["group"]) if not baseDir: if not vo: gLogger.error("Could not determine VO") Script.showHelp() baseDir = "/%s/user/%s/%s" % (vo, username[0], username) baseDir = baseDir.rstrip("/") gLogger.notice("Will search for files in %s%s" % (baseDir, (" matching %s" % wildcard) if wildcard else "")) activeDirs = [baseDir] allFiles = [] emptyDirs = [] while len(activeDirs) > 0: currentDir = activeDirs.pop() res = fc.listDirectory(currentDir, withMetadata, timeout=360) if not res["OK"]: gLogger.error("Error retrieving directory contents", "%s %s" % (currentDir, res["Message"])) elif currentDir in res["Value"]["Failed"]: gLogger.error( "Error retrieving directory contents", "%s %s" % (currentDir, res["Value"]["Failed"][currentDir])) else: dirContents = res["Value"]["Successful"][currentDir] subdirs = dirContents["SubDirs"] files = dirContents["Files"] if not subdirs and not files: emptyDirs.append(currentDir) gLogger.notice("%s: empty directory" % currentDir) else: for subdir in sorted(subdirs, reverse=True): if (not withMetadata) or isOlderThan( subdirs[subdir]["CreationDate"], totalDays): activeDirs.append(subdir) for filename in sorted(files): fileOK = False if (not withMetadata) or isOlderThan( files[filename]["MetaData"]["CreationDate"], totalDays): if wildcard is None or fnmatch.fnmatch( filename, wildcard): fileOK = True if not fileOK: files.pop(filename) allFiles += sorted(files) if len(files) or len(subdirs): gLogger.notice( "%s: %d files%s, %d sub-directories" % (currentDir, len(files), " matching" if withMetadata or wildcard else "", len(subdirs))) outputFileName = "%s.lfns" % baseDir.replace("/%s" % vo, "%s" % vo).replace("/", "-") outputFile = open(outputFileName, "w") for lfn in sorted(allFiles): outputFile.write(lfn + "\n") outputFile.close() gLogger.notice("%d matched files have been put in %s" % (len(allFiles), outputFileName)) if emptyDirsFlag: outputFileName = "%s.emptydirs" % baseDir.replace( "/%s" % vo, "%s" % vo).replace("/", "-") outputFile = open(outputFileName, "w") for dir in sorted(emptyDirs): outputFile.write(dir + "\n") outputFile.close() gLogger.notice("%d empty directories have been put in %s" % (len(emptyDirs), outputFileName)) DIRAC.exit(0)
# # execution if __name__ == "__main__": from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() import DIRAC from DIRAC import gLogger args = Script.getPositionalArgs() requestName = None targetSEs = None if len( args ) < 3: Script.showHelp() DIRAC.exit( 1 ) requestName = args[0] lfnList = getLFNList( args[1] ) targetSEs = list( set( [ se for targetSE in args[2:] for se in targetSE.split( ',' ) ] ) ) gLogger.info( "Will create request '%s' with 'ReplicateAndRegister' "\ "operation using %s lfns and %s target SEs" % ( requestName, len( lfnList ), len( targetSEs ) ) ) from DIRAC.RequestManagementSystem.Client.Request import Request from DIRAC.RequestManagementSystem.Client.Operation import Operation from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.Core.Utilities.List import breakListIntoChunks
def main(): from DIRAC.Core.Base import Script Script.registerSwitch('', 'Job=', ' JobID[,jobID2,...]') Script.registerSwitch('', 'Transformation=', ' transformation ID') Script.registerSwitch( '', 'Tasks=', ' Associated to --Transformation, list of taskIDs') Script.registerSwitch('', 'Verbose', ' Print more information') Script.registerSwitch('', 'Terse', ' Only print request status') Script.registerSwitch('', 'Full', ' Print full request content') Script.registerSwitch('', 'Status=', ' Select all requests in a given status') Script.registerSwitch( '', 'Since=', ' Associated to --Status, start date yyyy-mm-dd or nb of days (default= -one day' ) Script.registerSwitch( '', 'Until=', ' Associated to --Status, end date (default= now') Script.registerSwitch( '', 'Maximum=', ' Associated to --Status, max number of requests ') Script.registerSwitch('', 'Reset', ' Reset Failed files to Waiting if any') Script.registerSwitch('', 'Force', ' Force reset even if not Failed') Script.registerSwitch( '', 'All', ' (if --Status Failed) all requests, otherwise exclude irrecoverable failures' ) Script.registerSwitch('', 'FixJob', ' Set job Done if the request is Done') Script.registerSwitch('', 'Cancel', ' Cancel the request') Script.registerSwitch('', 'ListJobs', ' List the corresponding jobs') Script.registerSwitch( '', 'TargetSE=', ' Select request only if that SE is in the targetSEs') from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() import DIRAC from DIRAC import gLogger jobs = [] requestID = 0 transID = None taskIDs = None tasks = None requests = [] full = False verbose = False status = None until = None since = None terse = False allR = False reset = False fixJob = False maxRequests = 999999999999 cancel = False listJobs = False force = False targetSE = set() for switch in Script.getUnprocessedSwitches(): if switch[0] == 'Job': jobs = [] job = "Unknown" try: for arg in switch[1].split(','): if os.path.exists(arg): with open(arg, 'r') as fp: lines = fp.readlines() for line in lines: for job in line.split(','): jobs += [int(job.strip())] gLogger.notice("Found %d jobs in file %s" % (len(jobs), arg)) else: jobs.append(int(arg)) except TypeError: gLogger.fatal("Invalid jobID", job) elif switch[0] == 'Transformation': try: transID = int(switch[1]) except Exception: gLogger.fatal('Invalid transID', switch[1]) elif switch[0] == 'Tasks': try: taskIDs = [int(task) for task in switch[1].split(',')] except Exception: gLogger.fatal('Invalid tasks', switch[1]) elif switch[0] == 'Full': full = True elif switch[0] == 'Verbose': verbose = True elif switch[0] == 'Terse': terse = True elif switch[0] == 'All': allR = True elif switch[0] == 'Reset': reset = True elif switch[0] == 'Force': force = True elif switch[0] == 'Status': status = switch[1].capitalize() elif switch[0] == 'Since': since = convertDate(switch[1]) elif switch[0] == 'Until': until = convertDate(switch[1]) elif switch[0] == 'FixJob': fixJob = True elif switch[0] == 'Cancel': cancel = True elif switch[0] == 'ListJobs': listJobs = True elif switch[0] == 'Maximum': try: maxRequests = int(switch[1]) except Exception: pass elif switch[0] == 'TargetSE': targetSE = set(switch[1].split(',')) if reset and not force: status = 'Failed' if fixJob: status = 'Done' if terse: verbose = True if status: if not until: until = datetime.datetime.utcnow() if not since: since = until - datetime.timedelta(hours=24) from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.RequestManagementSystem.Client.ReqClient import printRequest, recoverableRequest reqClient = ReqClient() if transID: if not taskIDs: gLogger.fatal( "If Transformation is set, a list of Tasks should also be set") Script.showHelp(exitCode=2) # In principle, the task name is unique, so the request name should be unique as well # If ever this would not work anymore, we would need to use the transformationClient # to fetch the ExternalID requests = ['%08d_%08d' % (transID, task) for task in taskIDs] allR = True elif not jobs: requests = [] # Get full list of arguments, with and without comma for arg in [ x.strip() for arg in Script.getPositionalArgs() for x in arg.split(',') ]: if os.path.exists(arg): lines = open(arg, 'r').readlines() requests += [ reqID.strip() for line in lines for reqID in line.split(',') ] gLogger.notice("Found %d requests in file" % len(requests)) else: requests.append(arg) allR = True else: res = reqClient.getRequestIDsForJobs(jobs) if not res['OK']: gLogger.fatal("Error getting request for jobs", res['Message']) DIRAC.exit(2) if res['Value']['Failed']: gLogger.error( "No request found for jobs %s" % ','.join(sorted(str(job) for job in res['Value']['Failed']))) requests = sorted(res['Value']['Successful'].values()) if requests: allR = True else: DIRAC.exit(0) if status and not requests: allR = allR or status != 'Failed' res = reqClient.getRequestIDsList([status], limit=maxRequests, since=since, until=until) if not res['OK']: gLogger.error("Error getting requests:", res['Message']) DIRAC.exit(2) requests = [ reqID for reqID, _st, updTime in res['Value'] if updTime > since and updTime <= until and reqID ] gLogger.notice('Obtained %d requests %s between %s and %s' % (len(requests), status, since, until)) if not requests: gLogger.notice('No request selected....') Script.showHelp(exitCode=2) okRequests = [] warningPrinted = False jobIDList = [] for reqID in requests: # We allow reqID to be the requestName if it is unique try: requestID = int(reqID) except ValueError: requestID = reqClient.getRequestIDForName(reqID) if not requestID['OK']: gLogger.notice(requestID['Message']) continue requestID = requestID['Value'] request = reqClient.peekRequest(requestID) if not request["OK"]: gLogger.error(request["Message"]) DIRAC.exit(-1) request = request["Value"] if not request: gLogger.error("no such request %s" % requestID) continue # If no operation as the targetSE, skip if targetSE: found = False for op in request: if op.TargetSE and targetSE.intersection( op.TargetSE.split(',')): found = True break if not found: continue # keep a list of jobIDs if requested if request.JobID and listJobs: jobIDList.append(request.JobID) if status and request.Status != status: gLogger.notice( "Request %s is not in requested status %s%s" % (reqID, status, ' (cannot be reset)' if reset else '')) continue if fixJob and request.Status == 'Done' and request.JobID: # The request is for a job and is Done, verify that the job is in the proper status result = reqClient.finalizeRequest(request.RequestID, request.JobID, useCertificates=False) if not result['OK']: gLogger.error("Error finalizing job", result['Message']) else: gLogger.notice("Job %d updated to %s" % (request.JobID, result['Value'])) continue if cancel: if request.Status not in ('Done', 'Failed'): ret = reqClient.cancelRequest(requestID) if not ret['OK']: gLogger.error("Error canceling request %s" % reqID, ret['Message']) else: gLogger.notice("Request %s cancelled" % reqID) else: gLogger.notice("Request %s is in status %s, not cancelled" % (reqID, request.Status)) elif allR or recoverableRequest(request): okRequests.append(str(requestID)) if reset: gLogger.notice('============ Request %s =============' % requestID) ret = reqClient.resetFailedRequest(requestID, allR=allR) if not ret['OK']: gLogger.error("Error resetting request %s" % requestID, ret['Message']) else: if len(requests) > 1: gLogger.notice('\n===================================') dbStatus = reqClient.getRequestStatus(requestID).get( 'Value', 'Unknown') printRequest(request, status=dbStatus, full=full, verbose=verbose, terse=terse) if listJobs: gLogger.notice("List of %d jobs:\n" % len(jobIDList), ','.join(str(jobID) for jobID in jobIDList)) if status and okRequests: from DIRAC.Core.Utilities.List import breakListIntoChunks gLogger.notice('\nList of %d selected requests:' % len(okRequests)) for reqs in breakListIntoChunks(okRequests, 100): gLogger.notice(','.join(reqs))
""" % Script.scriptName) Script.parseCommandLine() import DIRAC from DIRAC import gConfig,gLogger from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.Core.Utilities.List import sortList storageCFGBase = "/Resources/StorageElements" res = gConfig.getSections( storageCFGBase, True ) if not res[ 'OK' ]: gLogger.error( 'Failed to get storage element info' ) gLogger.error( res[ 'Message' ] ) DIRAC.exit( -1 ) gLogger.info( "%s %s %s" % ( 'Storage Element'.ljust( 25 ), 'Read Status'.rjust( 15 ), 'Write Status'.rjust( 15 ) ) ) seList = sortList( res[ 'Value' ] ) resourceStatus = ResourceStatus() res = resourceStatus.getStorageElementStatus( seList ) if not res[ 'OK' ]: gLogger.error( "Failed to get StorageElement status for %s" % str( seList ) ) for k,v in res[ 'Value' ].items(): readState, writeState = 'Active', 'Active'
Script.parseCommandLine(ignoreErrors=True) runRanges = [] for arg in Script.getPositionalArgs(): runRanges += arg.split(',') runSet = set() for run in runRanges: try: if ':' in arg: run1, run2 = run.split(':') runSet.update(range(int(run1), int(run2) + 1)) else: runSet.add(int(run)) except (ValueError, IndexError) as e: gLogger.exception("Invalid run number", arg, lException=e) DIRAC.exit(1) production = None item = None byValue = False active = False for switch in Script.getUnprocessedSwitches(): if switch[0] == 'Production': try: production = [int(prod) for prod in switch[1].split(',')] except ValueError as e: gLogger.exception('Bad production ID', lException=e) DIRAC.exit(1) elif switch[0] == 'Information': item = switch[1] elif switch[0] == 'ByValue':
if not len( args ) == 1: Script.showHelp() ce = args[0] host = None vo = getVO( 'lhcb' ) for unprocSw in Script.getUnprocessedSwitches(): if unprocSw[0] in ( "H", "host" ): host = unprocSw[1] if unprocSw[0] in ( "V", "vo" ): vo = unprocSw[1] diracAdmin = DiracAdmin() result = diracAdmin.getBDIICEState( ce, useVO = vo, host = host ) if not result['OK']: print result['Message'] DIRAC.exit( 2 ) ces = result['Value'] for ce in ces: print "CE: %s {" % ce.get( 'GlueCEUniqueID', 'Unknown' ) for item in ce.iteritems(): print "%s: %s" % item print "}"
verbose = False if days or months or years: verbose = True totalDays = 0 if years: totalDays += 365 * years if months: totalDays += 30 * months if days: totalDays += days res = getProxyInfo(False, False) if not res['OK']: gLogger.error("Failed to get client proxy information.", res['Message']) DIRAC.exit(2) proxyInfo = res['Value'] username = proxyInfo['username'] vo = '' if 'group' in proxyInfo: vo = getVOForGroup(proxyInfo['group']) if not baseDir: if not vo: gLogger.error('Could not determine VO') Script.showHelp() baseDir = '/%s/user/%s/%s' % (vo, username[0], username) baseDir = baseDir.rstrip('/') gLogger.info('Will search for files in %s' % baseDir) activeDirs = [baseDir]
Usage: %s <lfn | fileContainingLfns> <SE> <status> """ % Script.scriptName ) Script.parseCommandLine( ignoreErrors = False ) for switch in Script.getUnprocessedSwitches(): if switch[0].lower() == "u" or switch[0].lower() == "unit": unit = switch[1] scaleDict = { 'MB' : 1000 * 1000.0, 'GB' : 1000 * 1000 * 1000.0, 'TB' : 1000 * 1000 * 1000 * 1000.0, 'PB' : 1000 * 1000 * 1000 * 1000 * 1000.0} if not unit in scaleDict.keys(): gLogger.error( "Unit must be one of MB,GB,TB,PB" ) DIRAC.exit( 2 ) scaleFactor = scaleDict[unit] args = Script.getPositionalArgs() lfns = [] for inputFileName in args: if os.path.exists( inputFileName ): inputFile = open( inputFileName, 'r' ) string = inputFile.read() inputFile.close() lfns.extend( [ lfn.strip() for lfn in string.splitlines() ] ) else: lfns.append( inputFileName ) from DIRAC.Resources.Catalog.FileCatalog import FileCatalog res = FileCatalog().getFileSize( lfns )
string = inputFile.read() inputFile.close() lfns.extend( [ lfn.strip() for lfn in string.splitlines() ] ) else: lfns.append( inputFileName ) from DIRAC.Core.Utilities.List import breakListIntoChunks from DIRAC.DataManagementSystem.Client.DataManager import DataManager dm = DataManager() errorReasons = {} successfullyRemoved = 0 for lfnList in breakListIntoChunks( lfns, 100 ): res = dm.removeFile( lfnList ) if not res['OK']: gLogger.error( "Failed to remove data", res['Message'] ) DIRAC.exit( -2 ) for lfn, r in res['Value']['Failed'].items(): reason = str( r ) if not reason in errorReasons.keys(): errorReasons[reason] = [] errorReasons[reason].append( lfn ) successfullyRemoved += len( res['Value']['Successful'].keys() ) for reason, lfns in errorReasons.items(): gLogger.notice( "Failed to remove %d files with error: %s" % ( len( lfns ), reason ) ) if successfullyRemoved > 0: gLogger.notice( "Successfully removed %d files" % successfullyRemoved ) DIRAC.exit( 0 )