def test_getdestinationse_no_assoc_alias(self): options_dict = { '/Resources/StorageElements/myOutputSE': S_ERROR('bla') } option_dict = { '/Resources/Countries/country/AssignedTo': S_OK('MyTestCountry.Assigned'), '/Resources/Countries/MyTestCountry.Assigned/AssociatedSEs': S_OK('something') } values_dict = { '/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE': [], '/Resources/StorageElementGroups/myOutputSE': ['some_elements', 'ignore_me'], '/Resources/Countries/MyTestCountry.Assigned/AssociatedSEs/myOutputSE': [], '/Resources/Countries/country/AssociatedSEs/myOutputSE': '' } conf_mock = Mock() conf_mock.getOptions.side_effect = lambda path: options_dict[path] conf_mock.getOption.side_effect = lambda path: option_dict[path] conf_mock.getValue.side_effect = lambda path, _: values_dict[path] with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'myTestStorageElement1', 'otherStorageElem2' ]))), \ patch('%s.gConfig' % MODULE_NAME, conf_mock): result = getDestinationSEList('myOutputSE', 'pre.myTestSite.country', 'Local') assertDiracFailsWith(result, 'failed to resolve se', self)
def test_getdestinationse_alias_found( self ): options_dict = { '/Resources/StorageElements/myOutputSE' : S_ERROR( 'bla' ) } values_dict = { '/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE' : 'myTest_Alias_value' } conf_mock = Mock() conf_mock.getOptions.side_effect = lambda path: options_dict[path] conf_mock.getValue.side_effect = lambda path, _: values_dict[path] with patch('%s.gConfig' % MODULE_NAME, conf_mock): result = getDestinationSEList( 'myOutputSE', 'pre.myTestSite.country', 'AnyOutputmodeTest' ) assertDiracSucceedsWith( result, 'myTest_Alias_value', self )
def test_getdestinationse( self ): options_dict = { '/Resources/StorageElements/myOutputSE' : S_ERROR( 'bla' ) } values_dict = { '/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE' : [], '/Resources/StorageElementGroups/myOutputSE' : [ 'some_grp_member', 'GroupElement', 'StorEl', 'LocalElement' ] } conf_mock = Mock() conf_mock.getOptions.side_effect = lambda path: options_dict[path] conf_mock.getValue.side_effect = lambda path, _: values_dict[path] with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'StorEl', 'LocalElement' ]))), \ patch('%s.gConfig' % MODULE_NAME, conf_mock): result = getDestinationSEList( 'myOutputSE', 'pre.myTestSite.country', 'AnyOutputmodeTest' ) assertDiracSucceedsWith_equals( result, [ 'StorEl', 'LocalElement', 'some_grp_member', 'GroupElement' ], self )
def test_getdestinationse_associatedse( self ): options_dict = { '/Resources/StorageElements/myOutputSE' : S_ERROR( 'bla' ) } values_dict = { '/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE' : [], '/Resources/StorageElementGroups/myOutputSE' : [ 'some_elements', 'ignore_me' ], '/Resources/Countries/country/AssociatedSEs/myOutputSE' : 'myassociated_se' } conf_mock = Mock() conf_mock.getOptions.side_effect = lambda path: options_dict[path] conf_mock.getValue.side_effect = lambda path, _: values_dict[path] with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'myTestStorageElement1', 'otherStorageElem2' ]))), \ patch('%s.gConfig' % MODULE_NAME, conf_mock): result = getDestinationSEList( 'myOutputSE', 'pre.myTestSite.country', 'Local' ) assertDiracSucceedsWith_equals( result, [ 'myassociated_se' ], self )
def test_getdestinationse_nogroupse( self ): options_dict = { '/Resources/StorageElements/myOutputSE' : S_ERROR( 'bla' ) } values_dict = { '/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE' : [], '/Resources/StorageElementGroups/myOutputSE' : [] } conf_mock = Mock() conf_mock.getOptions.side_effect = lambda path: options_dict[path] conf_mock.getValue.side_effect = lambda path, _: values_dict[path] #DIRAC.gConfig = conf_mock with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'myTestStorageElement1', 'otherStorageElem2' ]))), \ patch('%s.gConfig' % MODULE_NAME, conf_mock): result = getDestinationSEList( 'myOutputSE', 'pre.myTestSite.country', 'AnyOutputmodeTest' ) assertDiracFailsWith( result, 'failed to resolve se myoutputse', self )
def test_getdestinationse_no_assoc_list( self ): options_dict = { '/Resources/StorageElements/myOutputSE' : S_ERROR( 'bla' ) } option_dict = { '/Resources/Countries//AssignedTo' : S_OK( '' ) } values_dict = { '/Resources/Sites/pre/pre.myTestSite./AssociatedSEs/myOutputSE' : [], '/Resources/StorageElementGroups/myOutputSE' : [ 'some_elements', 'ignore_me' ], '/Resources/Countries/MyTestCountry.Assigned/AssociatedSEs/myOutputSE' : [ 'myassociated_alias_se' ], '/Resources/Countries//AssociatedSEs/myOutputSE' : '' } conf_mock = Mock() conf_mock.getOptions.side_effect = lambda path: options_dict[path] conf_mock.getOption.side_effect = lambda path: option_dict[path] conf_mock.getValue.side_effect = lambda path, _: values_dict[path] with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'myTestStorageElement1', 'otherStorageElem2' ]))), \ patch('%s.gConfig' % MODULE_NAME, conf_mock): result = getDestinationSEList( 'myOutputSE', 'pre.myTestSite.', 'Local' ) assertDiracFailsWith( result, 'could not determine associated se list', self )
def test_getdestinationse_alias_found(self): options_dict = { '/Resources/StorageElements/myOutputSE': S_ERROR('bla') } values_dict = { '/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE': 'myTest_Alias_value' } conf_mock = Mock() conf_mock.getOptions.side_effect = lambda path: options_dict[path] conf_mock.getValue.side_effect = lambda path, _: values_dict[path] with patch('%s.gConfig' % MODULE_NAME, conf_mock): result = getDestinationSEList('myOutputSE', 'pre.myTestSite.country', 'AnyOutputmodeTest') assertDiracSucceedsWith(result, 'myTest_Alias_value', self)
def test_getdestinationse_localgroupse(self): options_dict = { '/Resources/StorageElements/myOutputSE': S_ERROR('bla') } values_dict = { '/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE': [], '/Resources/StorageElementGroups/myOutputSE': ['some_elements', 'ignore_me', 'otherStorageElem2'] } conf_mock = Mock() conf_mock.getOptions.side_effect = lambda path: options_dict[path] conf_mock.getValue.side_effect = lambda path, _: values_dict[path] with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'myTestStorageElement1', 'otherStorageElem2' ]))), \ patch('%s.gConfig' % MODULE_NAME, conf_mock): result = getDestinationSEList('myOutputSE', 'pre.myTestSite.country', 'Local') assertDiracSucceedsWith_equals(result, ['otherStorageElem2'], self)
def test_getdestinationse_nogroupse(self): options_dict = { '/Resources/StorageElements/myOutputSE': S_ERROR('bla') } values_dict = { '/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE': [], '/Resources/StorageElementGroups/myOutputSE': [] } conf_mock = Mock() conf_mock.getOptions.side_effect = lambda path: options_dict[path] conf_mock.getValue.side_effect = lambda path, _: values_dict[path] #DIRAC.gConfig = conf_mock with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'myTestStorageElement1', 'otherStorageElem2' ]))), \ patch('%s.gConfig' % MODULE_NAME, conf_mock): result = getDestinationSEList('myOutputSE', 'pre.myTestSite.country', 'AnyOutputmodeTest') assertDiracFailsWith(result, 'failed to resolve se myoutputse', self)
def execute(self): """ Main execution function. """ #Have to work out if the module is part of the last step i.e. #user jobs can have any number of steps and we only want #to run the finalization once. currentStep = int(self.step_commons['STEP_NUMBER']) totalSteps = int(self.workflow_commons['TotalSteps']) if currentStep == totalSteps: self.lastStep = True else: self.log.verbose('Current step = %s, total steps of workflow = %s, UserJobFinalization will enable itself only \ at the last workflow step.' % (currentStep, totalSteps)) if not self.lastStep: return S_OK() result = self.resolveInputVariables() if not result['OK']: self.log.error(result['Message']) return result self.log.info('Initializing %s' % self.version) if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('No output data upload attempted') if not self.userOutputData: self.log.info('No user output data is specified for this job, nothing to do') return S_OK('No output data to upload') #Determine the final list of possible output files for the #workflow and all the parameters needed to upload them. outputList = [] for i in self.userOutputData: outputList.append({'outputPath' : string.upper(string.split(i, '.')[-1]), 'outputDataSE' : self.userOutputSE, 'outputFile' : os.path.basename(i)}) userOutputLFNs = [] if self.userOutputData: self.log.info('Constructing user output LFN(s) for %s' % (string.join(self.userOutputData, ', '))) if not self.jobID: self.jobID = 12345 owner = '' if self.workflow_commons.has_key('Owner'): owner = self.workflow_commons['Owner'] else: res = self.getCurrentOwner() if not res['OK']: return S_ERROR('Could not obtain owner from proxy') owner = res['Value'] vo = '' if self.workflow_commons.has_key('VO'): vo = self.workflow_commons['VO'] else: res = self.getCurrentVO() if not res['OK']: return S_ERROR('Could not obtain VO from proxy') vo = res['Value'] result = constructUserLFNs(int(self.jobID), vo, owner, self.userOutputData, self.userOutputPath) if not result['OK']: self.log.error('Could not create user LFNs', result['Message']) return result userOutputLFNs = result['Value'] self.log.verbose('Calling getCandidateFiles( %s, %s, %s)' % (outputList, userOutputLFNs, self.outputDataFileMask)) result = self.getCandidateFiles(outputList, userOutputLFNs, self.outputDataFileMask) if not result['OK']: if not self.ignoreapperrors: self.setApplicationStatus(result['Message']) return S_OK() fileDict = result['Value'] result = self.getFileMetadata(fileDict) if not result['OK']: if not self.ignoreapperrors: self.setApplicationStatus(result['Message']) return S_OK() if not result['Value']: if not self.ignoreapperrors: self.log.info('No output data files were determined to be uploaded for this workflow') self.setApplicationStatus('No Output Data Files To Upload') return S_OK() fileMetadata = result['Value'] #First get the local (or assigned) SE to try first for upload and others in random fashion result = getDestinationSEList('Tier1-USER', DIRAC.siteName(), outputmode='local') if not result['OK']: self.log.error('Could not resolve output data SE', result['Message']) self.setApplicationStatus('Failed To Resolve OutputSE') return result localSE = result['Value'] self.log.verbose('Site Local SE for user outputs is: %s' % (localSE)) orderedSEs = self.defaultOutputSE for se in localSE: if se in orderedSEs: orderedSEs.remove(se) for se in self.userOutputSE: if se in orderedSEs: orderedSEs.remove(se) orderedSEs = localSE + List.randomize(orderedSEs) if self.userOutputSE: prependSEs = [] for userSE in self.userOutputSE: if not userSE in orderedSEs: prependSEs.append(userSE) orderedSEs = prependSEs + orderedSEs self.log.info('Ordered list of output SEs is: %s' % (string.join(orderedSEs, ', '))) final = {} for fileName, metadata in fileMetadata.items(): final[fileName] = metadata final[fileName]['resolvedSE'] = orderedSEs #At this point can exit and see exactly what the module will upload if not self.enable: self.log.info('Module is disabled by control flag, would have attempted \ to upload the following files %s' % string.join(final.keys(), ', ')) for fileName, metadata in final.items(): self.log.info('--------%s--------' % fileName) for n, v in metadata.items(): self.log.info('%s = %s' %(n, v)) return S_OK('Module is disabled by control flag') #Instantiate the failover transfer client with the global request object failoverTransfer = FailoverTransfer(self.request) #One by one upload the files with failover if necessary replication = {} failover = {} uploaded = [] if not self.failoverTest: for fileName, metadata in final.items(): self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName, string.join(metadata['resolvedSE'], ', '))) result = failoverTransfer.transferAndRegisterFile(fileName, metadata['localpath'], metadata['lfn'], metadata['resolvedSE'], fileGUID = metadata['guid'], fileCatalog = self.userFileCatalog) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata)) failover[fileName] = metadata else: #Only attempt replication after successful upload lfn = metadata['lfn'] uploaded.append(lfn) seList = metadata['resolvedSE'] replicateSE = '' if result['Value'].has_key('uploadedSE'): uploadedSE = result['Value']['uploadedSE'] for se in seList: if not se == uploadedSE: replicateSE = se break if replicateSE and lfn: self.log.info('Will attempt to replicate %s to %s' % (lfn, replicateSE)) replication[lfn] = replicateSE else: failover = final cleanUp = False for fileName, metadata in failover.items(): random.shuffle(self.failoverSEs) targetSE = metadata['resolvedSE'][0] metadata['resolvedSE'] = self.failoverSEs result = failoverTransfer.transferAndRegisterFileFailover(fileName, metadata['localpath'], metadata['lfn'], targetSE, metadata['resolvedSE'], fileGUID = metadata['guid'], fileCatalog = self.userFileCatalog) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata)) cleanUp = True continue #for users can continue even if one completely fails else: lfn = metadata['lfn'] uploaded.append(lfn) #For files correctly uploaded must report LFNs to job parameters if uploaded: report = string.join( uploaded, ', ' ) self.jobReport.setJobParameter( 'UploadedOutputData', report ) #Now after all operations, retrieve potentially modified request object result = failoverTransfer.getRequestObject() if not result['OK']: self.log.error(result) return S_ERROR('Could Not Retrieve Modified Request') self.request = result['Value'] #If some or all of the files failed to be saved to failover if cleanUp: self.workflow_commons['Request'] = self.request #Leave any uploaded files just in case it is useful for the user #do not try to replicate any files. return S_ERROR('Failed To Upload Output Data') #If there is now at least one replica for uploaded files can trigger replication rm = ReplicaManager() self.log.info('Sleeping for 10 seconds before attempting replication of recently uploaded files') time.sleep(10) for lfn, repSE in replication.items(): result = rm.replicateAndRegister(lfn, repSE, catalog = self.userFileCatalog) if not result['OK']: self.log.info('Replication failed with below error but file already exists in Grid storage with \ at least one replica:\n%s' % (result)) self.workflow_commons['Request'] = self.request #Now must ensure if any pending requests are generated that these are propagated to the job wrapper reportRequest = None if self.jobReport: result = self.jobReport.generateRequest() if not result['OK']: self.log.warn('Could not generate request for job report with result:\n%s' % (result)) else: reportRequest = result['Value'] if reportRequest: self.log.info('Populating request with job report information') self.request.update(reportRequest) if not self.request.isEmpty()['Value']: request_string = self.request.toXML()['Value'] # Write out the request string fname = 'user_job_%s_request.xml' % (self.jobID) xmlfile = open(fname, 'w') xmlfile.write(request_string) xmlfile.close() self.log.info('Creating failover request for deferred operations for job %s:' % self.jobID) result = self.request.getDigest() if result['OK']: digest = result['Value'] self.log.info(digest) self.setApplicationStatus('Job Finished Successfully') return S_OK('Output data uploaded')
def execute(self): """ Main execution function. """ #Have to work out if the module is part of the last step i.e. #user jobs can have any number of steps and we only want #to run the finalization once. Not a problem if this is not the last step so return S_OK() resultLS = self.isLastStep() if not resultLS['OK']: return S_OK() self.logWorkingDirectory() resultIV = self.resolveInputVariables() if not resultIV['OK']: self.log.error("Failed to resolve input parameters:", resultIV['Message']) return resultIV self.log.info('Initializing %s' % self.version) if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('No output data upload attempted') if not self.userOutputData: self.log.info('No user output data is specified for this job, nothing to do') return S_OK('No output data to upload') #Determine the final list of possible output files for the #workflow and all the parameters needed to upload them. outputList = self.getOutputList() userOutputLFNs = [] if self.userOutputData: resultOLfn = self.constructOutputLFNs() if not resultOLfn['OK']: self.log.error('Could not create user LFNs', resultOLfn['Message']) return resultOLfn userOutputLFNs = resultOLfn['Value'] self.log.verbose('Calling getCandidateFiles( %s, %s, %s)' % (outputList, userOutputLFNs, self.outputDataFileMask)) self.log.debug("IgnoreAppErrors? '%s' " % self.ignoreapperrors) resultCF = self.getCandidateFiles(outputList, userOutputLFNs, self.outputDataFileMask) if not resultCF['OK']: if not self.ignoreapperrors: self.log.error(resultCF['Message']) self.setApplicationStatus(resultCF['Message']) return S_OK() fileDict = resultCF['Value'] resultFMD = self.getFileMetadata(fileDict) if not resultFMD['OK']: if not self.ignoreapperrors: self.log.error(resultFMD['Message']) self.setApplicationStatus(resultFMD['Message']) return S_OK() if not resultFMD['Value']: if not self.ignoreapperrors: self.log.info('No output data files were determined to be uploaded for this workflow') self.setApplicationStatus('No Output Data Files To Upload') return S_OK() fileMetadata = resultFMD['Value'] #First get the local (or assigned) SE to try first for upload and others in random fashion resultSEL = getDestinationSEList('Tier1-USER', DIRAC.siteName(), outputmode='local') if not resultSEL['OK']: self.log.error('Could not resolve output data SE', resultSEL['Message']) self.setApplicationStatus('Failed To Resolve OutputSE') return resultSEL localSE = resultSEL['Value'] orderedSEs = [ se for se in self.defaultOutputSE if se not in localSE and se not in self.userOutputSE] orderedSEs = localSE + List.randomize(orderedSEs) if self.userOutputSE: prependSEs = [] for userSE in self.userOutputSE: if not userSE in orderedSEs: prependSEs.append(userSE) orderedSEs = prependSEs + orderedSEs self.log.info('Ordered list of output SEs is: %s' % (', '.join(orderedSEs))) final = {} for fileName, metadata in fileMetadata.iteritems(): final[fileName] = metadata final[fileName]['resolvedSE'] = orderedSEs #At this point can exit and see exactly what the module will upload self.printOutputInfo(final) if not self.enable: return S_OK('Module is disabled by control flag') #Instantiate the failover transfer client with the global request object failoverTransfer = FailoverTransfer(self._getRequestContainer()) #One by one upload the files with failover if necessary filesToReplicate = {} filesToFailover = {} filesUploaded = [] if not self.failoverTest: self.transferAndRegisterFiles(final, failoverTransfer, filesToFailover, filesUploaded, filesToReplicate) else: filesToFailover = final ##if there are files to be failovered, we do it now resultTRFF = self.transferRegisterAndFailoverFiles(failoverTransfer, filesToFailover, filesUploaded) cleanUp = resultTRFF['Value']['cleanUp'] #For files correctly uploaded must report LFNs to job parameters if filesUploaded: report = ', '.join( filesUploaded ) self.jobReport.setJobParameter( 'UploadedOutputData', report ) self.workflow_commons['Request'] = failoverTransfer.request #If some or all of the files failed to be saved to failover if cleanUp: #Leave any uploaded files just in case it is useful for the user #do not try to replicate any files. return S_ERROR('Failed To Upload Output Data') #If there is now at least one replica for uploaded files can trigger replication datMan = DataManager( catalogs = self.userFileCatalog ) self.log.info('Sleeping for 10 seconds before attempting replication of recently uploaded files') time.sleep(10) for lfn, repSE in filesToReplicate.items(): resultRAR = datMan.replicateAndRegister(lfn, repSE) if not resultRAR['OK']: self.log.info('Replication failed with below error but file already exists in Grid storage with \ at least one replica:\n%s' % (resultRAR)) self.generateFailoverFile() self.setApplicationStatus('Job Finished Successfully') return S_OK('Output data uploaded')
def execute(self): """ Main execution function. """ self.log.info('Initializing %s' % self.version) result = self.resolveInputVariables() if not result['OK']: self.log.error("Failed to resolve input parameters:", result['Message']) return result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('No output data upload attempted') ##determine the experiment example_file = self.prodOutputLFNs[0] if "/ilc/prod/clic" in example_file: self.experiment = "CLIC" elif "/ilc/prod/ilc/sid" in example_file: self.experiment = 'ILC_SID' elif "/ilc/prod/ilc/mc-dbd" in example_file: self.experiment = 'ILC_ILD' else: self.log.warn("Failed to determine experiment, reverting to default") #Determine the final list of possible output files for the #workflow and all the parameters needed to upload them. result = self.getCandidateFiles(self.outputList, self.prodOutputLFNs, self.outputDataFileMask) if not result['OK']: self.log.error(result['Message']) self.setApplicationStatus(result['Message']) return result fileDict = result['Value'] result = self.getFileMetadata(fileDict) if not result['OK']: self.log.error(result['Message']) self.setApplicationStatus(result['Message']) return result if not result['Value']: self.log.info('No output data files were determined to be uploaded for this workflow') return S_OK() fileMetadata = result['Value'] #Get final, resolved SE list for files final = {} for fileName, metadata in fileMetadata.items(): result = getDestinationSEList(metadata['workflowSE'], DIRAC.siteName(), self.outputMode) if not result['OK']: self.log.error('Could not resolve output data SE', result['Message']) self.setApplicationStatus('Failed To Resolve OutputSE') return result resolvedSE = result['Value'] final[fileName] = metadata final[fileName]['resolvedSE'] = resolvedSE self.log.info('The following files will be uploaded: %s' % (', '.join(final.keys() ))) for fileName, metadata in final.items(): self.log.info('--------%s--------' % fileName) for metaName, metaValue in metadata.items(): self.log.info('%s = %s' % (metaName, metaValue)) #At this point can exit and see exactly what the module would have uploaded if not self.enable: self.log.info('Module is disabled by control flag, would have attempted to upload the \ following files %s' % ', '.join(final.keys())) return S_OK('Module is disabled by control flag') #Disable the watchdog check in case the file uploading takes a long time self.log.info('Creating DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK in order to disable the Watchdog prior to upload') fopen = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK','w') fopen.write('%s' % time.asctime()) fopen.close() #Instantiate the failover transfer client with the global request object failoverTransfer = FailoverTransfer(self._getRequestContainer()) catalogs = self.ops.getValue('Production/%s/Catalogs' % self.experiment, ['FileCatalog', 'LcgFileCatalog']) #One by one upload the files with failover if necessary failover = {} if not self.failoverTest: for fileName, metadata in final.iteritems(): self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName, ', '.join(metadata['resolvedSE']))) result = failoverTransfer.transferAndRegisterFile(fileName, metadata['localpath'], metadata['lfn'], metadata['resolvedSE'], fileMetaDict = metadata['filedict'], fileCatalog = catalogs) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata['filedict'])) failover[fileName] = metadata else: #lfn = metadata['lfn'] pass else: failover = final self.failoverSEs = self.ops.getValue("Production/%s/FailOverSE" % self.experiment, self.failoverSEs) cleanUp = False for fileName, metadata in failover.iteritems(): self.log.info('Setting default catalog for failover transfer to FileCatalog') failovers = self.failoverSEs targetSE = metadata['resolvedSE'][0] try:#remove duplicate site, otherwise it will do nasty things where processing the request failovers.remove(targetSE) except ValueError: pass random.shuffle(failovers) metadata['resolvedSE'] = failovers result = failoverTransfer.transferAndRegisterFileFailover(fileName, metadata['localpath'], metadata['lfn'], targetSE, metadata['resolvedSE'], fileMetaDict = metadata['filedict'], fileCatalog = catalogs) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata['filedict'])) cleanUp = True break #no point continuing if one completely fails os.remove("DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK") #cleanup the mess self.workflow_commons['Request'] = failoverTransfer.request #If some or all of the files failed to be saved to failover if cleanUp: lfns = [] for fileName, metadata in final.items(): lfns.append(metadata['lfn']) result = self._cleanUp(lfns) return S_ERROR('Failed to upload output data') return S_OK('Output data uploaded')
def execute(self): """ Main execution function. """ self.log.info('Initializing %s' % self.version) result = self.resolveInputVariables() if not result['OK']: self.log.error(result['Message']) return result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('No output data upload attempted') ##determine the experiment example_file = self.prodOutputLFNs[0] if "/ilc/prod/clic" in example_file: self.experiment = "CLIC" elif "/ilc/prod/ilc/sid" in example_file: self.experiment = 'ILC_SID' elif "/ilc/prod/ilc/mc-dbd" in example_file: self.experiment = 'ILC_ILD' else: self.log.warn("Failed to determine experiment, reverting to default") #Determine the final list of possible output files for the #workflow and all the parameters needed to upload them. result = self.getCandidateFiles(self.outputList, self.prodOutputLFNs, self.outputDataFileMask) if not result['OK']: self.setApplicationStatus(result['Message']) return result fileDict = result['Value'] result = self.getFileMetadata(fileDict) if not result['OK']: self.setApplicationStatus(result['Message']) return result if not result['Value']: self.log.info('No output data files were determined to be uploaded for this workflow') return S_OK() fileMetadata = result['Value'] #Get final, resolved SE list for files final = {} for fileName, metadata in fileMetadata.items(): result = getDestinationSEList(metadata['workflowSE'], DIRAC.siteName(), self.outputMode) if not result['OK']: self.log.error('Could not resolve output data SE', result['Message']) self.setApplicationStatus('Failed To Resolve OutputSE') return result resolvedSE = result['Value'] final[fileName] = metadata final[fileName]['resolvedSE'] = resolvedSE self.log.info('The following files will be uploaded: %s' % (string.join(final.keys(), ', '))) for fileName, metadata in final.items(): self.log.info('--------%s--------' % fileName) for n, v in metadata.items(): self.log.info('%s = %s' % (n, v)) #At this point can exit and see exactly what the module would have uploaded if not self.enable: self.log.info('Module is disabled by control flag, would have attempted to upload the \ following files %s' % string.join(final.keys(), ', ')) return S_OK('Module is disabled by control flag') #Disable the watchdog check in case the file uploading takes a long time self.log.info('Creating DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK in order to disable the Watchdog prior to upload') fopen = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK','w') fopen.write('%s' % time.asctime()) fopen.close() #Instantiate the failover transfer client with the global request object failoverTransfer = FailoverTransfer(self.request) catalogs = ['FileCatalog', 'LcgFileCatalog'] #One by one upload the files with failover if necessary failover = {} if not self.failoverTest: for fileName, metadata in final.items(): self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName, string.join(metadata['resolvedSE'], ', '))) result = failoverTransfer.transferAndRegisterFile(fileName, metadata['localpath'], metadata['lfn'], metadata['resolvedSE'], fileGUID = metadata['guid'], fileCatalog = catalogs) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata)) failover[fileName] = metadata else: lfn = metadata['lfn'] else: failover = final self.failoverSEs = self.ops.getValue("Production/%s/FailOverSE" % self.experiment, self.failoverSEs) cleanUp = False for fileName, metadata in failover.items(): self.log.info('Setting default catalog for failover transfer to FileCatalog') random.shuffle(self.failoverSEs) targetSE = metadata['resolvedSE'][0] metadata['resolvedSE'] = self.failoverSEs result = failoverTransfer.transferAndRegisterFileFailover(fileName, metadata['localpath'], metadata['lfn'], targetSE, metadata['resolvedSE'], fileGUID = metadata['guid'], fileCatalog = catalogs) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata)) cleanUp = True break #no point continuing if one completely fails os.remove("DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK") #cleanup the mess #Now after all operations, retrieve potentially modified request object result = failoverTransfer.getRequestObject() if not result['OK']: self.log.error(result) return S_ERROR('Could not retrieve modified request') self.request = result['Value'] #If some or all of the files failed to be saved to failover if cleanUp: lfns = [] for fileName, metadata in final.items(): lfns.append(metadata['lfn']) result = self.__cleanUp(lfns) self.workflow_commons['Request'] = self.request return S_ERROR('Failed to upload output data') # #Can now register the successfully uploaded files in the BK # if not performBKRegistration: # self.log.info('There are no files to perform the BK registration for, all could be saved to failover') # else: # rm = ReplicaManager() # result = rm.addCatalogFile(performBKRegistration,catalogs=['BookkeepingDB']) # self.log.verbose(result) # if not result['OK']: # self.log.error(result) # return S_ERROR('Could Not Perform BK Registration') # if result['Value']['Failed']: # for lfn,error in result['Value']['Failed'].items(): # self.log.info('BK registration for %s failed with message: "%s" setting failover request' %(lfn,error)) # result = self.request.addSubRequest({'Attributes':{'Operation':'registerFile','ExecutionOrder':0, 'Catalogue':'BookkeepingDB'}},'register') # if not result['OK']: # self.log.error('Could not set registerFile request:\n%s' %result) # return S_ERROR('Could Not Set BK Registration Request') # fileDict = {'LFN':lfn,'Status':'Waiting'} # index = result['Value'] # self.request.setSubRequestFiles(index,'register',[fileDict]) self.workflow_commons['Request'] = self.request return S_OK('Output data uploaded')
def execute(self): """ Main execution function. """ #Have to work out if the module is part of the last step i.e. #user jobs can have any number of steps and we only want #to run the finalization once. Not a problem if this is not the last step so return S_OK() resultLS = self.isLastStep() if not resultLS['OK']: return S_OK() self.logWorkingDirectory() resultIV = self.resolveInputVariables() if not resultIV['OK']: self.log.error("Failed to resolve input parameters:", resultIV['Message']) return resultIV self.log.info('Initializing %s' % self.version) if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('No output data upload attempted') if not self.userOutputData: self.log.info('No user output data is specified for this job, nothing to do') return S_OK('No output data to upload') #Determine the final list of possible output files for the #workflow and all the parameters needed to upload them. outputList = self.getOutputList() userOutputLFNs = [] if self.userOutputData: resultOLfn = self.constructOutputLFNs() if not resultOLfn['OK']: self.log.error('Could not create user LFNs', resultOLfn['Message']) return resultOLfn userOutputLFNs = resultOLfn['Value'] self.log.verbose('Calling getCandidateFiles( %s, %s, %s)' % (outputList, userOutputLFNs, self.outputDataFileMask)) self.log.debug("IgnoreAppErrors? '%s' " % self.ignoreapperrors) resultCF = self.getCandidateFiles(outputList, userOutputLFNs, self.outputDataFileMask) if not resultCF['OK']: if not self.ignoreapperrors: self.log.error(resultCF['Message']) self.setApplicationStatus(resultCF['Message']) return S_OK() fileDict = resultCF['Value'] resultFMD = self.getFileMetadata(fileDict) if not resultFMD['OK']: if not self.ignoreapperrors: self.log.error(resultFMD['Message']) self.setApplicationStatus(resultFMD['Message']) return S_OK() if not resultFMD['Value']: if not self.ignoreapperrors: self.log.info('No output data files were determined to be uploaded for this workflow') self.setApplicationStatus('No Output Data Files To Upload') return S_OK() fileMetadata = resultFMD['Value'] #First get the local (or assigned) SE to try first for upload and others in random fashion resultSEL = getDestinationSEList('Tier1-USER', DIRAC.siteName(), outputmode='local') if not resultSEL['OK']: self.log.error('Could not resolve output data SE', resultSEL['Message']) self.setApplicationStatus('Failed To Resolve OutputSE') return resultSEL localSE = resultSEL['Value'] orderedSEs = [ se for se in self.defaultOutputSE if se not in localSE and se not in self.userOutputSE] orderedSEs = localSE + List.randomize(orderedSEs) if self.userOutputSE: prependSEs = [] for userSE in self.userOutputSE: if userSE not in orderedSEs: prependSEs.append(userSE) orderedSEs = prependSEs + orderedSEs self.log.info('Ordered list of output SEs is: %s' % (', '.join(orderedSEs))) final = {} for fileName, metadata in fileMetadata.iteritems(): final[fileName] = metadata final[fileName]['resolvedSE'] = orderedSEs #At this point can exit and see exactly what the module will upload self.printOutputInfo(final) if not self.enable: return S_OK('Module is disabled by control flag') self.injectJobIndex( final ) #Instantiate the failover transfer client with the global request object failoverTransfer = FailoverTransfer(self._getRequestContainer()) #One by one upload the files with failover if necessary filesToReplicate = {} filesToFailover = {} filesUploaded = [] if not self.failoverTest: self.transferAndRegisterFiles(final, failoverTransfer, filesToFailover, filesUploaded, filesToReplicate) else: filesToFailover = final ##if there are files to be failovered, we do it now resultTRFF = self.transferRegisterAndFailoverFiles(failoverTransfer, filesToFailover, filesUploaded) cleanUp = resultTRFF['Value']['cleanUp'] #For files correctly uploaded must report LFNs to job parameters if filesUploaded: report = ', '.join( filesUploaded ) self.jobReport.setJobParameter( 'UploadedOutputData', report ) self.workflow_commons['Request'] = failoverTransfer.request #If some or all of the files failed to be saved to failover if cleanUp: #Leave any uploaded files just in case it is useful for the user #do not try to replicate any files. return S_ERROR('Failed To Upload Output Data') #If there is now at least one replica for uploaded files can trigger replication datMan = DataManager( catalogs = self.userFileCatalog ) self.log.info('Sleeping for 10 seconds before attempting replication of recently uploaded files') time.sleep(10) for lfn, repSE in filesToReplicate.items(): resultRAR = datMan.replicateAndRegister(lfn, repSE) if not resultRAR['OK']: self.log.info('Replication failed with below error but file already exists in Grid storage with \ at least one replica:\n%s' % (resultRAR)) self.generateFailoverFile() self.setApplicationStatus('Job Finished Successfully') return S_OK('Output data uploaded')