def submit_TS_no_MQ(job, infileList): """ Create a transformation executing the job workflow """ t = Transformation() tc = TransformationClient() t.setType("DataReprocessing") t.setDescription("Runs ImageExtractor analysis for array HB9 SCT") t.setLongDescription( "merge_simtel, ImageExtractor DL0->DL1 conversion for HB9 SCT") t.setGroupSize(5) t.setBody(job.workflow.toXML()) res = t.addTransformation() # Transformation is created here if not res['OK']: Script.gLogger.error(res['Message']) DIRAC.exit(-1) t.setStatus("Active") t.setAgentType("Automatic") transID = t.getTransformationID() Script.gLogger.notice('Adding %s files to transformation' % len(infileList)) tc.addFilesToTransformation(transID['Value'], infileList) return res
def OldSubmitTS(job, infileList): """ Create a transformation executing the job workflow """ t = Transformation() tc = TransformationClient() t.setType("DataReprocessing") t.setDescription("Runs EvnDisp analysis for array HB9 SCT") t.setLongDescription( "merge_simtel, evndisp converter and evndisp analysis for HB9 SCT" ) # mandatory t.setGroupSize(5) t.setBody(job.workflow.toXML()) res = t.addTransformation() # Transformation is created here if not res['OK']: print res['Message'] DIRAC.exit(-1) t.setStatus("Active") t.setAgentType("Automatic") transID = t.getTransformationID() print('Adding %s files to transformation' % len(infileList)) tc.addFilesToTransformation(transID['Value'], infileList) # Files added here return res
def ReproTSExample( args = None ) : from DIRAC.TransformationSystem.Client.Transformation import Transformation from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient from CTADIRAC.Interfaces.API.ReproTSJob import ReproTSJob if (len(args) != 2): Script.gLogger.notice('Wrong number of arguments') Script.showHelp() infile = args[0] f = open(infile,'r') infileList = [] for line in f: infile = line.strip() if line!="\n": infileList.append(infile) if args[1] not in ['STD','NSBX3','SCMST','SCSST','4MSST','ASTRI','NORTH']: Script.gLogger.notice('reprocessing configuration incorrect:',args[1]) Script.showHelp() simtelArrayConfig = args[1] j = ReproTSJob() j.setVersion('prod-2_15122013') j.setParameters(['fileCatalog.cfg','-S',simtelArrayConfig]) j.setInputSandbox( ['LFN:/vo.cta.in2p3.fr/MC/PROD2/CFG_rev6956/prod2_cfg.tar.gz']) j.setOutputSandbox( ['*.log','applicationLog.txt']) j.setCPUTime(200000) t = Transformation( ) tc = TransformationClient( ) t.setTransformationName("Reprotest1") # This must vary #t.setTransformationGroup("Group1") t.setType("DataReprocessing") t.setDescription("simtel repro example") t.setLongDescription( "simtel reprocessing" ) #mandatory t.setBody ( j.workflow.toXML() ) t.addTransformation() #transformation is created here t.setStatus("Active") t.setAgentType("Automatic") transID = t.getTransformationID() tc.addFilesToTransformation(transID['Value'],infileList) # Files added here
def submit_trans(job, infileList, trans_name, group_size): """ Create a transformation executing the job workflow """ DIRAC.gLogger.notice('submit_trans : %s' % trans_name) # Initialize JOB_ID job.workflow.addParameter( Parameter("JOB_ID", "000000", "string", "", "", True, False, "Temporary fix")) trans = Transformation() trans.setTransformationName(trans_name) # this must be unique trans.setType("DataReprocessing") trans.setDescription("Simtel TS example") trans.setLongDescription("Simtel tel_sim") # mandatory trans.setBody(job.workflow.toXML()) trans.setGroupSize(group_size) res = trans.addTransformation() # transformation is created here if not res['OK']: return res trans.setStatus("Active") trans.setAgentType("Automatic") # add 10*group_size files to transformation (to have the first 10 jobs) trans_id = trans.getTransformationID() trans_client = TransformationClient() res = trans_client.addFilesToTransformation(trans_id['Value'], infileList[:10 * group_size]) return res
def DataReplicaRemovalTSExample(args=None): from DIRAC.TransformationSystem.Client.Transformation import Transformation from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient if (len(args) != 1): Script.gLogger.notice('Wrong number of arguments') Script.showHelp() infile = args[0] f = open(infile, 'r') infileList = [] for line in f: infile = line.strip() if line != "\n": infileList.append(infile) t = Transformation() tc = TransformationClient() t.setType("Removal") t.setPlugin("Broadcast") # Mandatory for ReplicaRemoval t.setSourceSE(['CC-IN2P3-Tape' ]) # A list of SE where at least 1 SE is the valid one t.setTargetSE([ 'CEA-Disk', 'LAPP-Disk', 'DESY-ZN-Disk', 'CC-IN2P3-Disk', 'CYF-STORM-Disk' ]) # A list of SE where at least 1 SE is the valid one t.setDescription("AarProd2 ReplicaRemoval") t.setLongDescription("AarProd2 ReplicaRemoval") # Mandatory t.setGroupSize( 100 ) # Here you specify how many files should be grouped within the same request, e.g. 100 t.setBody("Removal;RemoveReplica" ) # Mandatory (the default is a ReplicateAndRegister operation) t.addTransformation() # Transformation is created here t.setStatus("Active") t.setAgentType("Automatic") transID = t.getTransformationID() tc.addFilesToTransformation(transID['Value'], infileList) # Files are added here
def DataReplicationTSExample(args=None): from DIRAC.TransformationSystem.Client.Transformation import Transformation from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient if (len(args) != 1): Script.gLogger.notice('Wrong number of arguments') Script.showHelp() infile = args[0] f = open(infile, 'r') infileList = [] for line in f: infile = line.strip() if line != "\n": infileList.append(infile) t = Transformation() tc = TransformationClient() t.setTransformationName("DM_Replication") # This must be unique #t.setTransformationGroup("Group1") t.setType("Replication") t.setSourceSE(['CYF-STORM-Disk', 'DESY-ZN-Disk' ]) # A list of SE where at least 1 SE is the valid one t.setTargetSE(['CEA-Disk']) t.setDescription("corsika Replication") t.setLongDescription("corsika Replication") # Mandatory t.setGroupSize( 1 ) # Here you specify how many files should be grouped within the same request, e.g. 100 t.setPlugin("Broadcast") # Mandatory for replication t.addTransformation() # Transformation is created here t.setStatus("Active") t.setAgentType("Automatic") transID = t.getTransformationID() tc.addFilesToTransformation(transID['Value'], infileList) # Files are added here
def DataRemovalTSExample(args=None): from DIRAC.TransformationSystem.Client.Transformation import Transformation from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient if (len(args) != 1): Script.gLogger.notice('Wrong number of arguments') Script.showHelp() infile = args[0] f = open(infile, 'r') infileList = [] for line in f: infile = line.strip() if line != "\n": infileList.append(infile) t = Transformation() tc = TransformationClient() t.setTransformationName("DM_Removal26") # Must be unique #t.setTransformationGroup("Group1") t.setType("Removal") t.setPlugin("Standard") # Not needed. The default is 'Standard' t.setDescription("corsika Removal") t.setLongDescription("corsika Removal") # Mandatory t.setGroupSize( 1 ) # Here you specify how many files should be grouped within the same request, e.g. 100 t.setBody("Removal;RemoveFile" ) # Mandatory (the default is a ReplicateAndRegister operation) t.addTransformation() # Transformation is created here t.setStatus("Active") t.setAgentType("Automatic") transID = t.getTransformationID() tc.addFilesToTransformation(transID['Value'], infileList) # Files are added here
def submitTS( job, infileList ): """ Create a transformation executing the job workflow """ t = Transformation() tc = TransformationClient() t.setType( "DataReprocessing" ) t.setDescription( "Chimp example" ) t.setLongDescription( "Chimp analysis" ) # mandatory t.setGroupSize(1) t.setBody ( job.workflow.toXML() ) res = t.addTransformation() # Transformation is created here if not res['OK']: print res['Message'] DIRAC.exit( -1 ) t.setStatus( "Active" ) t.setAgentType( "Automatic" ) transID = t.getTransformationID() tc.addFilesToTransformation( transID['Value'], infileList ) # Files added here return res
def submitTS( job, infileList ): """ Create a transformation executing the job workflow """ t = Transformation() tc = TransformationClient() t.setType( "SimtelMerging" ) t.setDescription( "Runs merge_simtel for array 3HB89" ) t.setLongDescription( "Merging array 3HB89 analysis" ) # mandatory t.setGroupSize(5) t.setBody ( job.workflow.toXML() ) res = t.addTransformation() # Transformation is created here if not res['OK']: print res['Message'] DIRAC.exit( -1 ) t.setStatus( "Active" ) t.setAgentType( "Automatic" ) transID = t.getTransformationID() print('Adding %s files to transformation'%len(infileList)) tc.addFilesToTransformation( transID['Value'], infileList ) # Files added here return res
def main(): Script.parseCommandLine() from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient args = Script.getPositionalArgs() if len(args) < 2: Script.showHelp(exitCode=1) # get arguments inputFileName = args[1] lfns = [] if os.path.exists(inputFileName): inputFile = open(inputFileName, 'r') string = inputFile.read() inputFile.close() lfns.extend([lfn.strip() for lfn in string.splitlines()]) else: lfns.append(inputFileName) tc = TransformationClient() res = tc.addFilesToTransformation(args[0], lfns) # Files added here if not res['OK']: DIRAC.gLogger.error(res['Message']) DIRAC.exit(2) successfullyAdded = 0 alreadyPresent = 0 for lfn, message in res['Value']['Successful'].items(): if message == 'Added': successfullyAdded += 1 elif message == 'Present': alreadyPresent += 1 if successfullyAdded > 0: DIRAC.gLogger.notice("Successfully added %d files" % successfullyAdded) if alreadyPresent > 0: DIRAC.gLogger.notice("Already present %d files" % alreadyPresent) DIRAC.exit(0)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument("TransID: transformation ID") Script.registerArgument(("LFN: LFN", "FileName: file containing LFNs")) Script.parseCommandLine() from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient tID, inputFileName = Script.getPositionalArgs(group=True) # get arguments lfns = [] if os.path.exists(inputFileName): inputFile = open(inputFileName, "r") string = inputFile.read() inputFile.close() lfns.extend([lfn.strip() for lfn in string.splitlines()]) else: lfns.append(inputFileName) tc = TransformationClient() res = tc.addFilesToTransformation(tID, lfns) # Files added here if not res["OK"]: DIRAC.gLogger.error(res["Message"]) DIRAC.exit(2) successfullyAdded = 0 alreadyPresent = 0 for lfn, message in res["Value"]["Successful"].items(): if message == "Added": successfullyAdded += 1 elif message == "Present": alreadyPresent += 1 if successfullyAdded > 0: DIRAC.gLogger.notice("Successfully added %d files" % successfullyAdded) if alreadyPresent > 0: DIRAC.gLogger.notice("Already present %d files" % alreadyPresent) DIRAC.exit(0)
def submit_trans(job, trans_name, infileList, group_size): """ Create a transformation executing the job workflow """ DIRAC.gLogger.notice('submit_trans : %s' % trans_name) # Initialize JOB_ID job.workflow.addParameter( Parameter("JOB_ID", "000000", "string", "", "", True, False, "Temporary fix")) trans = Transformation() trans.setTransformationName(trans_name) # this must be unique trans.setType("DataReprocessing") trans.setDescription("SimpleCtapipe example") trans.setLongDescription( "ctapipe classify, reconstruct and merge: calib_imgreco") # mandatory trans.setBody(job.workflow.toXML()) trans.setGroupSize(group_size) res = trans.addTransformation() # transformation is created here if not res['OK']: DIRAC.gLogger.error('Error creating transformation: ', res['Message']) return res transID = res['Value'] # Activate the transformation trans.setStatus("Active") trans.setAgentType("Automatic") # Add files to the transformation tc = TransformationClient() res = tc.addFilesToTransformation(transID, infileList) if not res['OK']: DIRAC.gLogger.error('Error adding files to the transformation: ', res['Message']) return res
class InputDataAgent(AgentModule): ############################################################################# def initialize(self): """ Make the necessary initializations """ self.fileLog = {} self.timeLog = {} self.fullTimeLog = {} self.pollingTime = self.am_getOption('PollingTime', 120) self.fullUpdatePeriod = self.am_getOption('FullUpdatePeriod', 86400) gMonitor.registerActivity("Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM) self.transClient = TransformationClient('TransformationDB') self.metadataClient = FileCatalogClient() return S_OK() ############################################################################## def execute(self): """ Main execution method """ gMonitor.addMark('Iteration', 1) # Get all the transformations result = self.transClient.getTransformations( condDict={'Status': 'Active'}) activeTransforms = [] if not result['OK']: gLogger.error( "InputDataAgent.execute: Failed to get transformations.", result['Message']) return S_OK() # Process each transformation for transDict in result['Value']: transID = long(transDict['TransformationID']) res = self.transClient.getTransformationInputDataQuery(transID) if not res['OK']: if res['Message'] == 'No InputDataQuery found for transformation': gLogger.info( "InputDataAgent.execute: No input data query found for transformation %d" % transID) else: gLogger.error( "InputDataAgent.execute: Failed to get input data query for %d" % transID, res['Message']) continue inputDataQuery = res['Value'] # Determine the correct time stamp to use for this transformation if self.timeLog.has_key(transID): if self.fullTimeLog.has_key(transID): # If it is more than a day since the last reduced query, make a full query just in case if (datetime.datetime.utcnow() - self.fullTimeLog[transID]) < datetime.timedelta( seconds=self.fullUpdatePeriod): timeStamp = self.timeLog[transID] inputDataQuery['StartDate'] = ( timeStamp - datetime.timedelta(seconds=10) ).strftime('%Y-%m-%d %H:%M:%S') else: self.fullTimeLog[transID] = datetime.datetime.utcnow() self.timeLog[transID] = datetime.datetime.utcnow() if not self.fullTimeLog.has_key(transID): self.fullTimeLog[transID] = datetime.datetime.utcnow() # Perform the query to the metadata catalog gLogger.verbose( "Using input data query for transformation %d: %s" % (transID, str(inputDataQuery))) start = time.time() result = self.metadataClient.findFilesByMetadata(inputDataQuery) rtime = time.time() - start gLogger.verbose("Metadata catalog query time: %.2f seconds." % (rtime)) if not result['OK']: gLogger.error( "InputDataAgent.execute: Failed to get response from the metadata catalog", result['Message']) continue lfnList = result['Value'] # Check if the number of files has changed since the last cycle nlfns = len(lfnList) gLogger.info( "%d files returned for transformation %d from the metadata catalog" % (nlfns, int(transID))) if self.fileLog.has_key(transID): if nlfns == self.fileLog[transID]: gLogger.verbose( 'No new files in metadata catalog since last check') self.fileLog[transID] = nlfns # Add any new files to the transformation addedLfns = [] if lfnList: gLogger.verbose('Processing %d lfns for transformation %d' % (len(lfnList), transID)) # Add the files to the transformation gLogger.verbose('Adding %d lfns for transformation %d' % (len(lfnList), transID)) result = self.transClient.addFilesToTransformation( transID, sortList(lfnList)) if not result['OK']: gLogger.warn( "InputDataAgent.execute: failed to add lfns to transformation", result['Message']) self.fileLog[transID] = 0 else: if result['Value']['Failed']: for lfn, error in res['Value']['Failed'].items(): gLogger.warn( "InputDataAgent.execute: Failed to add %s to transformation" % lfn, error) if result['Value']['Successful']: for lfn, status in result['Value']['Successful'].items( ): if status == 'Added': addedLfns.append(lfn) gLogger.info( "InputDataAgent.execute: Added %d files to transformation" % len(addedLfns)) return S_OK()
DIRAC.exit( 1 ) # get arguments inputFileName = args[1] lfns = [] if os.path.exists( inputFileName ): inputFile = open( inputFileName, 'r' ) string = inputFile.read() inputFile.close() lfns.extend( [ lfn.strip() for lfn in string.splitlines() ] ) else: lfns.append( inputFileName ) tc = TransformationClient() res = tc.addFilesToTransformation( args[0] , lfns ) # Files added here if not res['OK']: DIRAC.gLogger.error ( res['Message'] ) DIRAC.exit( 2 ) successfullyAdded = 0 alreadyPresent = 0 for lfn, message in res['Value']['Successful'].items(): if message == 'Added': successfullyAdded += 1 elif message == 'Present': alreadyPresent += 1 if successfullyAdded > 0: DIRAC.gLogger.notice( "Successfully added %d files" % successfullyAdded )
'\ne.g: %s 381 Paranal_gamma_North.list' % Script.scriptName, ])) Script.parseCommandLine() from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient args = Script.getPositionalArgs() if (len(args) != 2): Script.showHelp() # get arguments TransID = args[0] infile = args[1] f = open(infile, 'r') infileList = [] for line in f: infile = line.strip() if line != "\n": infileList.append(infile) tc = TransformationClient() res = tc.addFilesToTransformation(TransID, infileList) # Files added here if not res['OK']: DIRAC.gLogger.error(res['Message']) DIRAC.exit(-1) else: DIRAC.exit(0)
class InputDataAgent(AgentModule): def __init__(self, *args, **kwargs): """c'tor""" AgentModule.__init__(self, *args, **kwargs) self.fileLog = {} self.timeLog = {} self.fullTimeLog = {} self.pollingTime = self.am_getOption("PollingTime", 120) self.fullUpdatePeriod = self.am_getOption("FullUpdatePeriod", 86400) self.refreshonly = self.am_getOption("RefreshOnly", False) self.dateKey = self.am_getOption("DateKey", None) self.transClient = TransformationClient() self.metadataClient = FileCatalogClient() self.transformationTypes = None ############################################################################# def initialize(self): """Make the necessary initializations""" agentTSTypes = self.am_getOption("TransformationTypes", []) if agentTSTypes: self.transformationTypes = sorted(agentTSTypes) else: dataProc = Operations().getValue("Transformations/DataProcessing", ["MCSimulation", "Merge"]) dataManip = Operations().getValue( "Transformations/DataManipulation", ["Replication", "Removal"]) self.transformationTypes = sorted(dataProc + dataManip) extendables = Operations().getValue( "Transformations/ExtendableTransfTypes", []) if extendables: for extendable in extendables: if extendable in self.transformationTypes: self.transformationTypes.remove(extendable) # This is because the Extendables do not use this Agent (have no Input data query) return S_OK() ############################################################################## def execute(self): """Main execution method""" # Get all the transformations result = self.transClient.getTransformations({ "Status": "Active", "Type": self.transformationTypes }) if not result["OK"]: self.log.error( "InputDataAgent.execute: Failed to get transformations.", result["Message"]) return S_OK() # Process each transformation for transDict in result["Value"]: transID = int(transDict["TransformationID"]) # res = self.transClient.getTransformationInputDataQuery( transID ) res = self.transClient.getTransformationMetaQuery(transID, "Input") if not res["OK"]: if cmpError(res, ENOENT): self.log.info( "InputDataAgent.execute: No input data query found for transformation", transID) else: self.log.error( "InputDataAgent.execute: Failed to get input data query", "for %d: %s" % (transID, res["Message"]), ) continue inputDataQuery = res["Value"] if self.refreshonly: # Determine the correct time stamp to use for this transformation if transID in self.timeLog: if transID in self.fullTimeLog: # If it is more than a day since the last reduced query, make a full query just in case if (datetime.datetime.utcnow() - self.fullTimeLog[transID] ) < datetime.timedelta( seconds=self.fullUpdatePeriod): timeStamp = self.timeLog[transID] if self.dateKey: inputDataQuery[self.dateKey] = ( timeStamp - datetime.timedelta(seconds=10) ).strftime("%Y-%m-%d %H:%M:%S") else: self.log.error( "DateKey was not set in the CS, cannot use the RefreshOnly" ) else: self.fullTimeLog[ transID] = datetime.datetime.utcnow() self.timeLog[transID] = datetime.datetime.utcnow() if transID not in self.fullTimeLog: self.fullTimeLog[transID] = datetime.datetime.utcnow() # Perform the query to the metadata catalog self.log.verbose("Using input data query for transformation", "%d: %s" % (transID, str(inputDataQuery))) start = time.time() result = self.metadataClient.findFilesByMetadata(inputDataQuery) rtime = time.time() - start self.log.verbose("Metadata catalog query time", ": %.2f seconds." % (rtime)) if not result["OK"]: self.log.error( "InputDataAgent.execute: Failed to get response from the metadata catalog", result["Message"]) continue lfnList = result["Value"] # Check if the number of files has changed since the last cycle nlfns = len(lfnList) self.log.info( "files returned for transformation from the metadata catalog: ", "%d -> %d" % (int(transID), nlfns)) if nlfns == self.fileLog.get(transID): self.log.verbose( "No new files in metadata catalog since last check") self.fileLog[transID] = nlfns # Add any new files to the transformation addedLfns = [] if lfnList: self.log.verbose("Processing lfns for transformation:", "%d -> %d" % (transID, len(lfnList))) # Add the files to the transformation self.log.verbose("Adding lfns for transformation:", "%d -> %d" % (transID, len(lfnList))) result = self.transClient.addFilesToTransformation( transID, sorted(lfnList)) if not result["OK"]: self.log.warn( "InputDataAgent.execute: failed to add lfns to transformation", result["Message"]) self.fileLog[transID] = 0 else: if result["Value"]["Failed"]: for lfn, error in res["Value"]["Failed"].items(): self.log.warn( "InputDataAgent.execute: Failed to add to transformation:", "%s: %s" % (lfn, error)) if result["Value"]["Successful"]: for lfn, status in result["Value"]["Successful"].items( ): if status == "Added": addedLfns.append(lfn) self.log.info( "InputDataAgent.execute: Added files to transformation", "(%d)" % len(addedLfns)) return S_OK()
Script.showHelp(exitCode=1) # get arguments inputFileName = args[1] lfns = [] if os.path.exists(inputFileName): inputFile = open(inputFileName, 'r') string = inputFile.read() inputFile.close() lfns.extend([lfn.strip() for lfn in string.splitlines()]) else: lfns.append(inputFileName) tc = TransformationClient() res = tc.addFilesToTransformation(args[0], lfns) # Files added here if not res['OK']: DIRAC.gLogger.error(res['Message']) DIRAC.exit(2) successfullyAdded = 0 alreadyPresent = 0 for lfn, message in res['Value']['Successful'].items(): if message == 'Added': successfullyAdded += 1 elif message == 'Present': alreadyPresent += 1 if successfullyAdded > 0: DIRAC.gLogger.notice("Successfully added %d files" % successfullyAdded)
class InputDataAgent( AgentModule ): def __init__( self, *args, **kwargs ): ''' c'tor ''' AgentModule.__init__( self, *args, **kwargs ) self.fileLog = {} self.timeLog = {} self.fullTimeLog = {} self.pollingTime = self.am_getOption( 'PollingTime', 120 ) self.fullUpdatePeriod = self.am_getOption( 'FullUpdatePeriod', 86400 ) self.refreshonly = self.am_getOption( 'RefreshOnly', False ) self.dateKey = self.am_getOption( 'DateKey', None ) self.transClient = TransformationClient() self.metadataClient = FileCatalogClient() self.transformationTypes = None ############################################################################# def initialize( self ): ''' Make the necessary initializations ''' gMonitor.registerActivity( "Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM ) agentTSTypes = self.am_getOption( 'TransformationTypes', [] ) if agentTSTypes: self.transformationTypes = sorted( agentTSTypes ) else: dataProc = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] ) dataManip = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] ) self.transformationTypes = sorted( dataProc + dataManip ) extendables = Operations().getValue( 'Transformations/ExtendableTransfTypes', []) if extendables: for extendable in extendables: if extendable in self.transformationTypes: self.transformationTypes.remove(extendable) #This is because the Extendables do not use this Agent (have no Input data query) return S_OK() ############################################################################## def execute( self ): ''' Main execution method ''' gMonitor.addMark( 'Iteration', 1 ) # Get all the transformations result = self.transClient.getTransformations( {'Status' : 'Active', 'Type' : self.transformationTypes } ) if not result['OK']: gLogger.error( "InputDataAgent.execute: Failed to get transformations.", result['Message'] ) return S_OK() # Process each transformation for transDict in result['Value']: transID = long( transDict['TransformationID'] ) res = self.transClient.getTransformationInputDataQuery( transID ) if not res['OK']: if res['Message'] == 'No InputDataQuery found for transformation': gLogger.info( "InputDataAgent.execute: No input data query found for transformation %d" % transID ) else: gLogger.error( "InputDataAgent.execute: Failed to get input data query for %d" % transID, res['Message'] ) continue inputDataQuery = res['Value'] if self.refreshonly: # Determine the correct time stamp to use for this transformation if self.timeLog.has_key( transID ): if self.fullTimeLog.has_key( transID ): # If it is more than a day since the last reduced query, make a full query just in case if ( datetime.datetime.utcnow() - self.fullTimeLog[transID] ) < datetime.timedelta( seconds = self.fullUpdatePeriod ): timeStamp = self.timeLog[transID] if self.dateKey: inputDataQuery[self.dateKey] = ( timeStamp - datetime.timedelta( seconds = 10 ) ).strftime( '%Y-%m-%d %H:%M:%S' ) else: gLogger.error( "DateKey was not set in the CS, cannot use the RefreshOnly" ) else: self.fullTimeLog[transID] = datetime.datetime.utcnow() self.timeLog[transID] = datetime.datetime.utcnow() if not self.fullTimeLog.has_key( transID ): self.fullTimeLog[transID] = datetime.datetime.utcnow() # Perform the query to the metadata catalog gLogger.verbose( "Using input data query for transformation %d: %s" % ( transID, str( inputDataQuery ) ) ) start = time.time() result = self.metadataClient.findFilesByMetadata( inputDataQuery ) rtime = time.time() - start gLogger.verbose( "Metadata catalog query time: %.2f seconds." % ( rtime ) ) if not result['OK']: gLogger.error( "InputDataAgent.execute: Failed to get response from the metadata catalog", result['Message'] ) continue lfnList = result['Value'] # Check if the number of files has changed since the last cycle nlfns = len( lfnList ) gLogger.info( "%d files returned for transformation %d from the metadata catalog" % ( nlfns, int( transID ) ) ) if self.fileLog.has_key( transID ): if nlfns == self.fileLog[transID]: gLogger.verbose( 'No new files in metadata catalog since last check' ) self.fileLog[transID] = nlfns # Add any new files to the transformation addedLfns = [] if lfnList: gLogger.verbose( 'Processing %d lfns for transformation %d' % ( len( lfnList ), transID ) ) # Add the files to the transformation gLogger.verbose( 'Adding %d lfns for transformation %d' % ( len( lfnList ), transID ) ) result = self.transClient.addFilesToTransformation( transID, sorted( lfnList ) ) if not result['OK']: gLogger.warn( "InputDataAgent.execute: failed to add lfns to transformation", result['Message'] ) self.fileLog[transID] = 0 else: if result['Value']['Failed']: for lfn, error in res['Value']['Failed'].items(): gLogger.warn( "InputDataAgent.execute: Failed to add %s to transformation" % lfn, error ) if result['Value']['Successful']: for lfn, status in result['Value']['Successful'].items(): if status == 'Added': addedLfns.append( lfn ) gLogger.info( "InputDataAgent.execute: Added %d files to transformation" % len( addedLfns ) ) return S_OK()
dl_count_step2 = [None] * N_TH_STEP_2 dl_count_step3 = [None] * N_TH_STEP_3 f = [] for r in range(0, N_TH_STEP_1): min_tmp = min[r] if ((min_tmp + N_TH_STEP_3) >= n_file[r]): # Si verifica se (e quanti) File bisogna creare per aggiungere un Task per una specifica Trasformazione diff_file = ((min_tmp + N_TH_STEP_3) - n_file[r]) + 1 # Si crea il numero giusto di File guardando se il numero di file da creare e' positivo if (diff_file > 0): for i in range(0, diff_file): min_tmp += 1 #lfns_tmp = "/aa/lfn." + str(min_tmp) + "." + str(r) + ".txt" lfns_tmp = "/aa/lfn." + str(uuid_str) + "." + str( i) + ".txt" res = transClient.addFilesToTransformation( transID[r], [lfns_tmp]) if res['OK'] == True: print "**** Creato un File per creare un nuovo Task (per il thread ", r + 1, ") ****" lfns[r].append(lfns_tmp) else: print "\033[1;31m#### ERROR: File per aggiungere i Task non creati ####\033[1;m" print res['Message'] min_tmp = min[r] f_tmp = [] for i in range(0, N_TH_STEP_3): min_tmp = min_tmp + 1 f_tmp.append([lfns[r][min_tmp]]) f.append(f_tmp) for i in range(1, N_TH_STEP_2 + 1): r = random.randint(0, N_TH_STEP_1 - 1)
t.setTransformationName(transferName) # Must be unique t.setTransformationGroup("Transfer") t.setType("Transfer-JUNO") #t.setPlugin("Standard") # Not needed. The default is 'Standard' t.setDescription("Test Data Transfer") t.setLongDescription( "Long description of Data Transfer" ) # Mandatory t.setGroupSize(3) # Here you specify how many files should be grouped within he same request, e.g. 100 transBody = [ ( "ReplicateAndRegister", { "SourceSE": fromSE, "TargetSE": toSE }) ] t.setBody ( transBody ) # Mandatory result = t.addTransformation() # Transformation is created here if not result['OK']: gLogger.error('Can not add transformation: %s' % result['Message']) exit(2) t.setStatus("Active") t.setAgentType("Automatic") transID = t.getTransformationID() result = tc.addFilesToTransformation(transID['Value'], infileList) # Files are added here if not result['OK']: gLogger.error('Can not add files to transformation: %s' % result['Message']) exit(2) result = tc.setTransformationParameter( transID['Value'], 'Status', 'Flush' ) if not result['OK']: gLogger.error('Can not flush transformation: %s' % result['Message']) exit(2)
times_s3 = [None]*N_TH_STEP_3 dl_count_step2 = [None]*N_TH_STEP_2 dl_count_step3 = [None]*N_TH_STEP_3 f = [] for r in range(0,N_TH_STEP_1): min_tmp = min[r] if ((min_tmp + N_TH_STEP_3) >= n_file[r]): # Si verifica se (e quanti) File bisogna creare per aggiungere un Task per una specifica Trasformazione diff_file = ((min_tmp + N_TH_STEP_3) - n_file[r]) +1 # Si crea il numero giusto di File guardando se il numero di file da creare e' positivo if (diff_file > 0): for i in range(0,diff_file): min_tmp += 1 #lfns_tmp = "/aa/lfn." + str(min_tmp) + "." + str(r) + ".txt" lfns_tmp = "/aa/lfn." + str(uuid_str) + "." + str(i) + ".txt" res = transClient.addFilesToTransformation( transID[r], [lfns_tmp] ) if res['OK']==True: print "**** Creato un File per creare un nuovo Task (per il thread ",r+1,") ****" lfns[r].append(lfns_tmp) else: print "\033[1;31m#### ERROR: File per aggiungere i Task non creati ####\033[1;m" print res['Message'] min_tmp = min[r] f_tmp = [] for i in range(0,N_TH_STEP_3): min_tmp = min_tmp + 1 f_tmp.append([lfns[r][min_tmp]]) f.append(f_tmp) for i in range(1, N_TH_STEP_2 + 1): r = random.randint(0, N_TH_STEP_1 - 1)
class InputDataAgent(AgentModule): ############################################################################# def initialize(self): """ Make the necessary initializations """ self.fileLog = {} self.timeLog = {} self.fullTimeLog = {} self.pollingTime = self.am_getOption('PollingTime',120) self.fullUpdatePeriod = self.am_getOption('FullUpdatePeriod',86400) gMonitor.registerActivity("Iteration","Agent Loops",AGENT_NAME,"Loops/min",gMonitor.OP_SUM) self.transClient = TransformationClient('TransformationDB') self.metadataClient = FileCatalogClient() return S_OK() ############################################################################## def execute(self): """ Main execution method """ gMonitor.addMark('Iteration',1) # Get all the transformations result = self.transClient.getTransformations(condDict={'Status':'Active'}) activeTransforms = [] if not result['OK']: gLogger.error("InputDataAgent.execute: Failed to get transformations.", result['Message']) return S_OK() # Process each transformation for transDict in result['Value']: transID = long(transDict['TransformationID']) res = self.transClient.getTransformationInputDataQuery(transID) if not res['OK']: if res['Message'] == 'No InputDataQuery found for transformation': gLogger.info("InputDataAgent.execute: No input data query found for transformation %d" % transID) else: gLogger.error("InputDataAgent.execute: Failed to get input data query for %d" % transID, res['Message']) continue inputDataQuery = res['Value'] # Determine the correct time stamp to use for this transformation if self.timeLog.has_key(transID): if self.fullTimeLog.has_key(transID): # If it is more than a day since the last reduced query, make a full query just in case if (datetime.datetime.utcnow() - self.fullTimeLog[transID]) < datetime.timedelta(seconds=self.fullUpdatePeriod): timeStamp = self.timeLog[transID] inputDataQuery['StartDate'] = (timeStamp - datetime.timedelta(seconds=10)).strftime('%Y-%m-%d %H:%M:%S') else: self.fullTimeLog[transID] = datetime.datetime.utcnow() self.timeLog[transID] = datetime.datetime.utcnow() if not self.fullTimeLog.has_key(transID): self.fullTimeLog[transID] = datetime.datetime.utcnow() # Perform the query to the metadata catalog gLogger.verbose("Using input data query for transformation %d: %s" % (transID,str(inputDataQuery))) start = time.time() result = self.metadataClient.findFilesByMetadata(inputDataQuery) rtime = time.time()-start gLogger.verbose("Metadata catalog query time: %.2f seconds." % (rtime)) if not result['OK']: gLogger.error("InputDataAgent.execute: Failed to get response from the metadata catalog", result['Message']) continue lfnList = result['Value'] # Check if the number of files has changed since the last cycle nlfns = len(lfnList) gLogger.info("%d files returned for transformation %d from the metadata catalog" % (nlfns,int(transID)) ) if self.fileLog.has_key(transID): if nlfns == self.fileLog[transID]: gLogger.verbose('No new files in metadata catalog since last check') self.fileLog[transID] = nlfns # Add any new files to the transformation addedLfns = [] if lfnList: gLogger.verbose('Processing %d lfns for transformation %d' % (len(lfnList),transID) ) # Add the files to the transformation gLogger.verbose('Adding %d lfns for transformation %d' % (len(lfnList),transID) ) result = self.transClient.addFilesToTransformation(transID,sortList(lfnList)) if not result['OK']: gLogger.warn("InputDataAgent.execute: failed to add lfns to transformation", result['Message']) self.fileLog[transID] = 0 else: if result['Value']['Failed']: for lfn,error in res['Value']['Failed'].items(): gLogger.warn("InputDataAgent.execute: Failed to add %s to transformation" % lfn,error) if result['Value']['Successful']: for lfn,status in result['Value']['Successful'].items(): if status == 'Added': addedLfns.append(lfn) gLogger.info("InputDataAgent.execute: Added %d files to transformation" % len(addedLfns)) return S_OK()