def __call__( self ): """ call me maybe """ # # counter for failed files failedFiles = 0 # # catalog(s) to use catalogs = self.operation.Catalog if catalogs: catalogs = [ cat.strip() for cat in catalogs.split( ',' ) ] dm = DataManager( catalogs = catalogs ) # # get waiting files waitingFiles = self.getWaitingFilesList() # # loop over files for opFile in waitingFiles: gMonitor.addMark( "RegisterAtt", 1 ) # # get LFN lfn = opFile.LFN # # and others fileTuple = ( lfn , opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum ) # # call DataManager registerFile = dm.registerFile( fileTuple ) # # check results if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]: gMonitor.addMark( "RegisterFail", 1 ) # self.dataLoggingClient().addFileRecord( lfn, "RegisterFail", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" ) reason = str( registerFile.get( "Message", registerFile.get( "Value", {} ).get( "Failed", {} ).get( lfn, 'Unknown' ) ) ) errorStr = "failed to register LFN" opFile.Error = "%s: %s" % ( errorStr, reason ) if 'GUID already registered' in reason: opFile.Status = 'Failed' self.log.error( errorStr, "%s: %s" % ( lfn, reason ) ) elif 'File already registered with no replicas' in reason: self.log.warn( errorStr, "%s: %s, will remove it and retry" % ( lfn, reason ) ) dm.removeFile( lfn ) else: self.log.warn( errorStr, "%s: %s" % ( lfn, reason ) ) failedFiles += 1 else: gMonitor.addMark( "RegisterOK", 1 ) # self.dataLoggingClient().addFileRecord( lfn, "Register", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" ) self.log.verbose( "file %s has been registered at %s" % ( lfn, ','.join( catalogs ) if catalogs else "all catalogs" ) ) opFile.Status = "Done" # # final check if failedFiles: self.log.warn( "all files processed, %s files failed to register" % failedFiles ) self.operation.Error = "some files failed to register" return S_ERROR( self.operation.Error ) return S_OK()
def uploadProcessListToFileCatalog(self, path_to_process_list, appVersion): """Upload the new processList to the FileCatalog """ from ILCDIRAC.Core.Utilities.FileUtils import upload from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC import gConfig, exit as dexit datMan = DataManager() LOG.notice("Removing process list from file catalog" + path_to_process_list) res = datMan.removeFile(path_to_process_list) if not res['OK']: LOG.error("Could not remove process list from file catalog, do it by hand") dexit(2) LOG.notice("Done removing process list from file catalog") res = upload(os.path.dirname(path_to_process_list) + "/", self.location ) if not res['OK']: LOG.error("something went wrong in the copy") dexit(2) LOG.notice("Putting process list to local processlist directory") localprocesslistpath = gConfig.getOption("/LocalSite/ProcessListPath", "") if localprocesslistpath['Value']: try: localSvnRepo = "/afs/cern.ch/eng/clic/software/whizard/whizard_195/" shutil.copy(self.location, localSvnRepo) ## because it does not make a difference if we hardcode it here or in ${DIRAC}/etc/dirac.cfg, yours truly APS, JFS except OSError, err: LOG.error("Copy of process list to %s failed with error %s!" % (localSvnRepo, str(err))) try: subprocess.call( ["svn","ci", os.path.join( localSvnRepo, os.path.basename(localprocesslistpath['Value'] )), "-m'Process list for whizard version %s'" % appVersion ], shell=False ) except OSError, err: LOG.error("Commit failed! Error: %s" % str(err))
def uploadProcessListToFileCatalog(self, path_to_process_list, appVersion): """Upload the new processList to the FileCatalog """ from ILCDIRAC.Core.Utilities.FileUtils import upload from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC import gConfig, exit as dexit datMan = DataManager() gLogger.notice("Removing process list from file catalog" + path_to_process_list) res = datMan.removeFile(path_to_process_list) if not res['OK']: gLogger.error("Could not remove process list from file catalog, do it by hand") dexit(2) gLogger.notice("Done removing process list from file catalog") res = upload(os.path.dirname(path_to_process_list) + "/", self.location ) if not res['OK']: gLogger.error("something went wrong in the copy") dexit(2) gLogger.notice("Putting process list to local processlist directory") localprocesslistpath = gConfig.getOption("/LocalSite/ProcessListPath", "") if localprocesslistpath['Value']: try: localSvnRepo = "/afs/cern.ch/eng/clic/software/whizard/whizard_195/" shutil.copy(self.location, localSvnRepo) ## because it does not make a difference if we hardcode it here or in ${DIRAC}/etc/dirac.cfg, yours truly APS, JFS except OSError, err: gLogger.error("Copy of process list to %s failed with error %s!" % (localSvnRepo, str(err))) try: subprocess.call( ["svn","ci", os.path.join( localSvnRepo, os.path.basename(localprocesslistpath['Value'] )), "-m'Process list for whizard version %s'" % appVersion ], shell=False ) except OSError, err: gLogger.error("Commit failed! Error: %s" % str(err))
def _remove_file(self, lfn): dm = DataManager() res = dm.removeFile([lfn]) if not res['OK']: gLogger.error("Failed to remove data", res['Message']) return res if lfn in res['Value']['Successful']: return S_OK(res['Value']['Successful']) return S_ERROR(res['Value']['Failed'])
def setUp( self ): super( IntegrationTest, self ).setUp() dm = DataManager() res = dm.removeFile( ['/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar', '/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim'], force = True ) if not res['OK']: print "Could not remove files", res['Message'] exit( 1 )
def setUp( self ): super( FailingUserJobTestCase, self ).setUp() dm = DataManager() res = dm.removeFile( ['/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar', '/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim'], force = True ) if not res['OK']: print("Could not remove files", res['Message']) exit( 1 )
def setUp(self): super(IntegrationTest, self).setUp() dm = DataManager() res = dm.removeFile([ '/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar', '/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim' ], force=True) if not res['OK']: print "Could not remove files", res['Message'] exit(1)
def setUp(self): super(FailingUserJobTestCase, self).setUp() dm = DataManager() res = dm.removeFile( [ "/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar", "/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim", ], force=True, ) if not res["OK"]: print("Could not remove files", res["Message"]) exit(1)
def setUp(self): super(FailingUserJobTestCase, self).setUp() dm = DataManager() res = dm.removeFile( [ "/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar", "/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim", ], force=True, ) if not res["OK"]: print "Could not remove files", res["Message"] exit(1)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(("LocalFile: Path to local file containing LFNs", "LFN: Logical File Names")) Script.registerArgument(["LFN: Logical File Names"], mandatory=False) Script.parseCommandLine() import os import DIRAC from DIRAC import gLogger first, lfns = Script.getPositionalArgs(group=True) if os.path.exists(first): with open(first, "r") as inputFile: string = inputFile.read() lfns.extend([lfn.strip() for lfn in string.splitlines()]) else: lfns.insert(0, first) from DIRAC.Core.Utilities.List import breakListIntoChunks from DIRAC.DataManagementSystem.Client.DataManager import DataManager dm = DataManager() errorReasons = {} successfullyRemoved = 0 for lfnList in breakListIntoChunks(lfns, 100): res = dm.removeFile(lfnList) if not res["OK"]: gLogger.error("Failed to remove data", res["Message"]) DIRAC.exit(-2) for lfn, r in res["Value"]["Failed"].items(): reason = str(r) if reason not in errorReasons: errorReasons[reason] = [] errorReasons[reason].append(lfn) successfullyRemoved += len(res["Value"]["Successful"]) for reason, lfns in errorReasons.items(): gLogger.notice("Failed to remove %d files with error: %s" % (len(lfns), reason)) if successfullyRemoved > 0: gLogger.notice("Successfully removed %d files" % successfullyRemoved) DIRAC.exit(0)
def main(): Script.parseCommandLine() import os import DIRAC from DIRAC import gLogger args = Script.getPositionalArgs() lfns = [] for inputFileName in args: if os.path.exists(inputFileName): inputFile = open(inputFileName, 'r') string = inputFile.read() inputFile.close() lfns.extend([lfn.strip() for lfn in string.splitlines()]) else: lfns.append(inputFileName) from DIRAC.Core.Utilities.List import breakListIntoChunks from DIRAC.DataManagementSystem.Client.DataManager import DataManager dm = DataManager() errorReasons = {} successfullyRemoved = 0 for lfnList in breakListIntoChunks(lfns, 100): res = dm.removeFile(lfnList) if not res['OK']: gLogger.error("Failed to remove data", res['Message']) DIRAC.exit(-2) for lfn, r in res['Value']['Failed'].items(): reason = str(r) if reason not in errorReasons: errorReasons[reason] = [] errorReasons[reason].append(lfn) successfullyRemoved += len(res['Value']['Successful']) for reason, lfns in errorReasons.items(): gLogger.notice("Failed to remove %d files with error: %s" % (len(lfns), reason)) if successfullyRemoved > 0: gLogger.notice("Successfully removed %d files" % successfullyRemoved) DIRAC.exit(0)
class RemoveInputData(ModuleBase): """ Remove the input data: to be used when Merging things """ def __init__(self): super(RemoveInputData, self).__init__() self.datMan = DataManager() self.log = gLogger.getSubLogger("RemoveInputData") self.enable = True def applicationSpecificInputs(self): self.enable = self.step_commons.get('Enable', self.enable) if not isinstance(self.enable, bool): self.log.warn( 'Enable flag set to non-boolean value %s, setting to False' % self.enable) self.enable = False return S_OK('Parameters resolved') def execute(self): """ Remove the input data, and pass by failover in case of failure """ self.result = self.resolveInputVariables() if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('No removal of input data attempted') if not self.enable: self.log.info("Would have tried to remove %s" % self.InputData) return S_OK('Input Data Removed') try: #Try to remove the file list with failover if necessary failover = [] self.log.info('Attempting rm.removeFile("%s")' % (self.InputData)) result = self.datMan.removeFile(self.InputData) self.log.verbose(result) if not result['OK']: self.log.error('Could not remove files with message:\n"%s"\n\ Will set removal requests just in case.' % (result['Message'])) failover = self.InputData try: if result['Value']['Failed']: failureDict = result['Value']['Failed'] if failureDict: self.log.info( 'Not all files were successfully removed, see "LFN : reason" below\n%s' % (failureDict)) failover = failureDict.keys() except KeyError: self.log.error( 'Setting files for removal request to be the input data: %s' % self.InputData) failover = self.InputData for lfn in failover: self.__setFileRemovalRequest(lfn) return S_OK('Input Data Removed') except OSError as e: self.log.exception(e) return S_ERROR(e) return S_OK() def __setFileRemovalRequest(self, lfn): """ Sets a removal request for a file including all replicas. """ self.log.info('Setting file removal request for %s' % lfn) self.addRemovalRequests([lfn])
from DIRAC import S_OK, S_ERROR, gLogger, exit from DIRAC.DataManagementSystem.Client.DataManager import DataManager lfn = args[0] pfn = args[1] se = args[2] exit_code = 0 log = '' dm = DataManager() start = time.time() result = dm.removeFile(lfn) result = dm.putAndRegister(lfn, pfn, se) uploadTime = time.time() - start if result['OK']: log += 'Succeed to upload file to SE %s.\n' % se log += 'Upload Time : %ss\n' % uploadTime start = time.time() result = dm.getReplica(lfn, se, tempfile.gettempdir()) downloadTime = time.time() - start if result['OK']: log += 'Succeed to download file from SE %s.\n' % se log += 'Download Time : %ss\n' % downloadTime else: exit_code = 1 log += 'Failed to download file from SE %s : %s\n' % (
class SETest( TestBase ): """ SETest is used to test the availability of SE. """ def __init__( self, args = None, apis = None ): super( SETest, self ).__init__( args, apis ) self.__lfnPath = '/bes/user/z/zhaoxh/' self.__testFile = 'test.dat' self.__localPath = '/tmp/' if 'DataManager' in self.apis: self.dm = self.apis[ 'DataManager' ] else: self.dm = DataManager() def doTest( self, elementDict ): """ Test upload and download for specified SE. """ elementName = elementDict[ 'ElementName' ] testFilePath = self.__localPath + self.__testFile if not os.path.exists( testFilePath ) or not os.path.isfile( testFilePath ): f = open( testFilePath, 'w' ) f.write( 'hello' ) f.close() status = 'OK' log = '' lfnPath = self.__lfnPath + elementName + '-' + self.__testFile submissionTime = datetime.utcnow().replace( microsecond = 0 ) LOCK.acquire() start = time.time() result = self.dm.putAndRegister( lfnPath, testFilePath, elementName ) uploadTime = time.time() - start if result[ 'OK' ]: log += 'Succeed to upload file to SE %s.\n' % elementName log += 'Upload Time : %ss\n' % uploadTime start = time.time() result = self.dm.getReplica( lfnPath, elementName, self.__localPath ) downloadTime = time.time() - start if result[ 'OK' ]: log += 'Succeed to download file from SE %s.\n' % elementName log += 'Download Time : %ss\n' % downloadTime else: status = 'Bad' log += 'Failed to download file from SE %s : %s\n' % ( elementName, result[ 'Message' ] ) result = self.dm.removeFile( lfnPath ) if result[ 'OK' ]: log += 'Succeed to delete file from SE %s.\n' % elementName else: log += 'Faile to delete file from SE %s : %s\n' % ( elementName, result[ 'Message' ] ) else: status = 'Bad' log += 'Failed to upload file to SE %s : %s\n' % ( elementName, result[ 'Message' ] ) LOCK.release() completionTime = datetime.utcnow().replace( microsecond = 0 ) applicationTime = ( completionTime - submissionTime ).total_seconds() result = { 'Result' : { 'Status' : status, 'Log' : log, 'SubmissionTime' : submissionTime, 'CompletionTime' : completionTime, 'ApplicationTime' : applicationTime }, 'Finish' : True } localFile = self.__localPath + elementName +'-' + self.__testFile if os.path.exists( localFile ) and os.path.isfile( localFile ): os.remove( localFile ) return S_OK( result )
class MoveInFC(ModuleBase): ''' classdocs ''' def __init__(self): ''' Constructor ''' super(MoveInFC, self).__init__() self.enable = False self.STEP_NUMBER = '' self.applicationName = 'MoveInFC' self.repMan = DataManager() self.listoutput = {} self.outputpath = '' def applicationSpecificInputs(self): """ Resolve all input variables for the module here. :return: S_OK() """ if not len(self.InputFile) and len(self.InputData): for files in self.InputData: self.InputFile.append(files) if 'listoutput' in self.step_commons: self.listoutput = self.step_commons['listoutput'][0] return S_OK() def execute(self): """ Run the module """ result = self.resolveInputVariables() if not result['OK']: return result self.result = S_OK() if not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('%s should not proceed as previous step did not end properly'% self.applicationName) ### Now remove the files in the FC lfns = self.InputFile ##Check that all the files are here: res = resolveIFpaths(lfns) if not res['OK']: LOG.error(res['Message']) return S_ERROR("Failed to find a file locally") #All files are here and available paths = res['Value'] localpaths = [] for inputfile in paths: basename = os.path.basename(inputfile) locname = os.path.join(os.getcwd(),basename) if not locname == inputfile: try: shutil.copy(inputfile, locname) except shutil.Error: LOG.error("Failed to copy file locally, will have to stop") return S_ERROR("Failed copy to local directory") localpaths.append(locname) try: os.unlink(inputfile) except OSError: LOG.warn("Failed to remove initial file, increased \ disk space usage") #all the files are in the run directory #get all metadata, ancestor/daughter relations, etc. for all the files #Update the listoutput if self.listoutput: outputlist = [] for localFile in localpaths: item = {} item['outputFile'] = localFile item['outputPath'] = self.listoutput['outputPath'] item['outputDataSE'] = self.listoutput['outputDataSE'] outputlist.append(item) if self.enable: self.step_commons['listoutput'] = outputlist else: LOG.info("listoutput would have been ", outputlist) ## Make sure the path contains / at the end as we are going to ## concatenate final path and local files if not self.outputpath[-1]=='/': self.outputpath += "/" if 'ProductionOutputData' in self.workflow_commons: file_list = ";".join([self.outputpath+name for name in [os.path.basename(fin) for fin in localpaths]]) if self.enable: self.workflow_commons['ProductionOutputData'] = file_list else: LOG.info("ProductionOutputData would have been", file_list) #Now remove them if self.enable: res = self.repMan.removeFile(lfns, force=True) if not res['OK']: LOG.error("Failed to remove the files") self.setApplicationStatus("Failed to remove the file") #return S_ERROR("Failed to remove the files") else: LOG.info("Would have removed: ", "%s" % str(lfns)) ## Now the files are not on the storage anymore, they exist only locally. We can hope ## that the job will not be killed between now and the time the UploadOutputData module ## is called return self.finalStatusReport(0)
class Prod3DataManager(object) : """ Manage data and meta-data """ def __init__(self,catalogs=['DIRACFileCatalog']): """ Constructor """ self.setupCatalogClient( catalogs ) self.printCatalogConfig( catalogs ) self.setupDataManagerClient( catalogs ) def setupCatalogClient( self, catalogs ): """ Init FileCatalog client Ideally we would like to use only FileCatalog but it doesn't work for setMetadata because the returned value has a wrong format. So use of FileCatalogClient instead """ self.fc = FileCatalog( catalogs ) self.fcc = FileCatalogClient() def printCatalogConfig( self, catalogs ): """ Dumps catalog configuration """ for catalog in catalogs: res = self.fc._getCatalogConfigDetails( catalog ) DIRAC.gLogger.info( 'CatalogConfigDetails:', res ) def setupDataManagerClient( self, catalogs ): """ Init DataManager client """ self.dm = DataManager( catalogs ) def _getSEList( self, SEType = 'ProductionOutputs', DataType = 'SimtelProd' ): """ get from CS the list of available SE for data upload """ opsHelper = Operations() optionName = os.path.join( SEType, DataType ) SEList = opsHelper.getValue( optionName , [] ) SEList = List.randomize( SEList ) DIRAC.gLogger.notice( 'List of %s SE: %s ' % ( SEType, SEList ) ) # # Check if the local SE is in the list. If yes try it first by reversing list order localSEList = [] res = getSEsForSite( DIRAC.siteName() ) if res['OK']: localSEList = res['Value'] retainedlocalSEList = [] for localSE in localSEList: if localSE in SEList: DIRAC.gLogger.notice( 'The local Storage Element is an available SE: ', localSE ) retainedlocalSEList.append( localSE ) SEList.remove( localSE ) SEList = retainedlocalSEList + SEList if len( SEList ) == 0: return DIRAC.S_ERROR( 'Error in building SEList' ) return DIRAC.S_OK( SEList ) def _checkemptydir( self, path ): """ check that the directory is not empty """ if len ( glob.glob( path ) ) == 0: error = 'Empty directory: %s' % path return DIRAC.S_ERROR( error ) else: return DIRAC.S_OK() def _getRunPath( self, filemetadata ): """ format path to string with 1 digit precision run_number is taken from filemetadata filemetadata can be a dict or the run_number itself """ if type( filemetadata ) == type( dict() ): run_number = int( filemetadata['runNumber'] ) else: run_number = int( filemetadata ) run_numberMod = run_number % 1000 runpath = '%03dxxx' % ( ( run_number - run_numberMod ) / 1000 ) return runpath def _formatPath( self, path ): """ format path to string with 1 digit precision """ if type( path ) == float or type( path ) == int: path = '%.1f' % path return str( path ) def createTarLogFiles ( self, inputpath, tarname ): """ create tar of log and histogram files """ tarmode = 'w:gz' tar = tarfile.open( tarname, tarmode ) for subdir in ['Log/*', 'Histograms/*']: logdir = os.path.join( inputpath, subdir ) res = self._checkemptydir( logdir ) if not res['OK']: return res for localfile in glob.glob( logdir ): tar.add( localfile, arcname = localfile.split( '/' )[-1] ) tar.close() return DIRAC.S_OK() def createMDStructure( self, metadata, metadatafield, basepath, program_category ): """ create meta data structure """ # ## Add metadata fields to the DFC mdfield = json.loads( metadatafield ) for key, value in mdfield.items(): res = self.fc.addMetadataField( key, value ) if not res['OK']: return res # ## Create the directory structure md = json.loads( metadata , object_pairs_hook = collections.OrderedDict ) path = basepath process_program = program_category + '_prog' for key, value in collections.OrderedDict( ( k, md[k] ) for k in ( 'site', 'particle', process_program ) if k in md ).items(): path = os.path.join( path, self._formatPath( value ) ) res = self.fc.createDirectory( path ) if not res['OK']: return res # Set directory metadata for each subdir: 'site', 'particle', 'process_program' res = self.fcc.setMetadata( path, {key:value} ) if not res['OK']: return res # Create the TransformationID subdir and set MD # ## Get the TransformationID if os.environ.has_key( 'JOBID' ): jobID = os.environ['JOBID'] dirac = Dirac() res = dirac.getJobJDL( jobID ) TransformationID = '0000' if res['Value'].has_key( 'TransformationID' ): TransformationID = res['Value']['TransformationID'] path = os.path.join( path, TransformationID ) res = self.fc.createDirectory( path ) if not res['OK']: return res process_program_version = process_program + '_version' res = self.fcc.setMetadata( path, dict( ( k, md[k] ) for k in ( 'phiP', 'thetaP', 'array_layout', process_program_version ) if k in md ) ) if not res['OK']: return res # Create the Data and Log subdirs and set MD Transformation_path = path for subdir in ['Data', 'Log']: path = os.path.join( Transformation_path, subdir ) res = self.fc.createDirectory( path ) if not res['OK']: return res # Set metadata if not already defined res = self.fcc.getDirectoryUserMetadata( path ) if not res['OK']: return res if 'outputType' not in res['Value']: res = self.fcc.setMetadata( path, {'outputType':subdir} ) if not res['OK']: return res # MD for the Data directory - data_level and configuration_id path = os.path.join(Transformation_path, 'Data') # Set metadata if not already defined res = self.fcc.getDirectoryUserMetadata( path ) if not res['OK']: return res if 'data_level' and 'configuration_id' not in res['Value']: res = self.fcc.setMetadata(path, {'data_level': md['data_level'], 'configuration_id': md['configuration_id'] }) if not res['OK']: return res return DIRAC.S_OK( Transformation_path ) def putAndRegister( self, lfn, localfile, filemetadata, DataType = 'SimtelProd' ): """ put and register one file and set metadata """ # ## Get the list of Production SE res = self._getSEList( 'ProductionOutputs', DataType ) if res['OK']: ProductionSEList = res['Value'] else: return res # ## Get the list of Failover SE res = self._getSEList( 'ProductionOutputsFailover', DataType ) if res['OK']: FailoverSEList = res['Value'] else: return res # ## Upload file to a Production SE res = self._putAndRegisterToSEList( lfn, localfile, ProductionSEList ) if not res['OK']: DIRAC.gLogger.error( 'Failed to upload file to any Production SE: %s' % ProductionSEList ) # ## Upload file to a Failover SE res = self._putAndRegisterToSEList( lfn, localfile, FailoverSEList ) if not res['OK']: return DIRAC.S_ERROR( 'Failed to upload file to any Failover SE: %s' % FailoverSEList ) # ## Set file metadata: jobID, subarray, sct if res['OK']: fmd = json.loads( filemetadata ) if os.environ.has_key( 'JOBID' ): fmd.update( {'jobID':os.environ['JOBID']} ) filename = os.path.basename( localfile ) # set subarray and sct md from filename p = re.compile( 'subarray-\d+' ) if p.search( filename ) != None: subarray = p.search( filename ).group() fmd.update( {'subarray':subarray} ) sct = 'False' p = re.compile( 'nosct' ) psct = re.compile( 'sct' ) if p.search( filename ) == None and psct.search( filename ) != None: sct = 'True' # ## Set sct flag only for production data res = self.fcc.getFileUserMetadata( lfn ) if DataType == 'SimtelProd' and res['Value']['outputType'] == 'Data': fmd.update( {'sct':sct} ) res = self.fc.setMetadata( lfn, fmd ) if not res['OK']: return res return DIRAC.S_OK() def _putAndRegisterToSEList( self, lfn, localfile, SEList ): """ put and register one file to one SE in the SEList """ # ## Try to upload file to a SE in the list for SE in SEList: msg = 'Try to upload local file: %s \nwith LFN: %s \nto %s' % ( localfile, lfn, SE ) DIRAC.gLogger.notice( msg ) res = self.dm.putAndRegister( lfn, localfile, SE ) DIRAC.gLogger.notice(res) # ## check if failed if not res['OK']: DIRAC.gLogger.error( 'Failed to putAndRegister %s \nto %s \nwith message: %s' % ( lfn, SE, res['Message'] ) ) DIRAC.gLogger.notice('Trying to clean up %s' % lfn) res = self.dm.removeFile(lfn) if res['OK']: DIRAC.gLogger.notice('Successfully removed %s \n that was not supposed to have been uploaded successfully' % lfn) continue elif res['Value']['Failed'].has_key( lfn ): DIRAC.gLogger.error( 'Failed to putAndRegister %s to %s' % ( lfn, SE ) ) DIRAC.gLogger.notice('Trying to clean up %s' % lfn) res = self.dm.removeFile(lfn) if res['OK']: DIRAC.gLogger.notice('Successfully removed %s \n that was not supposed to have been uploaded successfully' % lfn) continue else: return DIRAC.S_OK() return DIRAC.S_ERROR() def cleanLocalFiles ( self, datadir, pattern = '*' ): """ remove files matching pattern in datadir """ for localfile in glob.glob( os.path.join( datadir, pattern ) ): DIRAC.gLogger.notice( 'Removing local file: ', localfile ) os.remove( localfile ) return DIRAC.S_OK()
class MoveInFC(ModuleBase): ''' classdocs ''' def __init__(self): ''' Constructor ''' super(MoveInFC, self).__init__() self.enable = False self.STEP_NUMBER = '' self.log = gLogger.getSubLogger("MoveInFC") self.applicationName = 'MoveInFC' self.repMan = DataManager() self.listoutput = {} self.outputpath = '' def applicationSpecificInputs(self): """ Resolve all input variables for the module here. :return: S_OK() """ if not len(self.InputFile) and len(self.InputData): for files in self.InputData: self.InputFile.append(files) if 'listoutput' in self.step_commons: self.listoutput = self.step_commons['listoutput'][0] return S_OK() def execute(self): """ Run the module """ result = self.resolveInputVariables() if not result['OK']: return result self.result = S_OK() if not self.applicationLog: self.result = S_ERROR('No Log file provided') if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK( '%s should not proceed as previous step did not end properly' % self.applicationName) ### Now remove the files in the FC lfns = self.InputFile ##Check that all the files are here: res = resolveIFpaths(lfns) if not res['OK']: self.log.error(res['Message']) return S_ERROR("Failed to find a file locally") #All files are here and available paths = res['Value'] localpaths = [] for inputfile in paths: basename = os.path.basename(inputfile) locname = os.path.join(os.getcwd(), basename) if not locname == inputfile: try: shutil.copy(inputfile, locname) except shutil.Error: self.log.error( "Failed to copy file locally, will have to stop") return S_ERROR("Failed copy to local directory") localpaths.append(locname) try: os.unlink(inputfile) except OSError: self.log.warn("Failed to remove initial file, increased \ disk space usage") #all the files are in the run directory #get all metadata, ancestor/daughter relations, etc. for all the files #Update the listoutput if self.listoutput: outputlist = [] for localFile in localpaths: item = {} item['outputFile'] = localFile item['outputPath'] = self.listoutput['outputPath'] item['outputDataSE'] = self.listoutput['outputDataSE'] outputlist.append(item) if self.enable: self.step_commons['listoutput'] = outputlist else: self.log.info("listoutput would have been ", outputlist) ## Make sure the path contains / at the end as we are going to ## concatenate final path and local files if not self.outputpath[-1] == '/': self.outputpath += "/" if 'ProductionOutputData' in self.workflow_commons: file_list = ";".join([ self.outputpath + name for name in [os.path.basename(fin) for fin in localpaths] ]) if self.enable: self.workflow_commons['ProductionOutputData'] = file_list else: self.log.info("ProductionOutputData would have been", file_list) #Now remove them if self.enable: res = self.repMan.removeFile(lfns, force=True) if not res['OK']: self.log.error("Failed to remove the files") self.setApplicationStatus("Failed to remove the file") #return S_ERROR("Failed to remove the files") else: self.log.info("Would have removed: ", "%s" % str(lfns)) ## Now the files are not on the storage anymore, they exist only locally. We can hope ## that the job will not be killed between now and the time the UploadOutputData module ## is called return self.finalStatusReport(0)
if os.path.exists( inputFileName ): inputFile = open( inputFileName, 'r' ) string = inputFile.read() inputFile.close() lfns.extend( [ lfn.strip() for lfn in string.splitlines() ] ) else: lfns.append( inputFileName ) from DIRAC.Core.Utilities.List import breakListIntoChunks from DIRAC.DataManagementSystem.Client.DataManager import DataManager dm = DataManager() errorReasons = {} successfullyRemoved = 0 for lfnList in breakListIntoChunks( lfns, 100 ): res = dm.removeFile( lfnList ) if not res['OK']: gLogger.error( "Failed to remove data", res['Message'] ) DIRAC.exit( -2 ) for lfn, r in res['Value']['Failed'].items(): reason = str( r ) if not reason in errorReasons.keys(): errorReasons[reason] = [] errorReasons[reason].append( lfn ) successfullyRemoved += len( res['Value']['Successful'].keys() ) for reason, lfns in errorReasons.items(): gLogger.notice( "Failed to remove %d files with error: %s" % ( len( lfns ), reason ) ) if successfullyRemoved > 0: gLogger.notice( "Successfully removed %d files" % successfullyRemoved ) DIRAC.exit( 0 )
def __call__(self): """ call me maybe """ # The flag 'rmsMonitoring' is set by the RequestTask and is False by default. # Here we use 'createRMSRecord' to create the ES record which is defined inside OperationHandlerBase. if self.rmsMonitoring: self.rmsMonitoringReporter = MonitoringReporter( monitoringType="RMSMonitoring") else: # # RegisterFile specific monitor info gMonitor.registerActivity("RegisterAtt", "Attempted file registrations", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM) gMonitor.registerActivity("RegisterOK", "Successful file registrations", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM) gMonitor.registerActivity("RegisterFail", "Failed file registrations", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM) # # counter for failed files failedFiles = 0 # # catalog(s) to use catalogs = self.operation.Catalog if catalogs: catalogs = [cat.strip() for cat in catalogs.split(',')] dm = DataManager(catalogs=catalogs) # # get waiting files waitingFiles = self.getWaitingFilesList() if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord( self.createRMSRecord("Attempted", len(waitingFiles))) # # loop over files for opFile in waitingFiles: if not self.rmsMonitoring: gMonitor.addMark("RegisterAtt", 1) # # get LFN lfn = opFile.LFN # # and others fileTuple = (lfn, opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum) # # call DataManager registerFile = dm.registerFile(fileTuple) # # check results if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]: if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord( self.createRMSRecord("Failed", 1)) else: gMonitor.addMark("RegisterFail", 1) # self.dataLoggingClient().addFileRecord( # lfn, "RegisterFail", ','.join(catalogs) if catalogs else "all catalogs", "", "RegisterFile") reason = str( registerFile.get( "Message", registerFile.get("Value", {}).get("Failed", {}).get(lfn, 'Unknown'))) errorStr = "failed to register LFN" opFile.Error = "%s: %s" % (errorStr, reason) if 'GUID already registered' in reason: opFile.Status = 'Failed' self.log.error(errorStr, "%s: %s" % (lfn, reason)) elif 'File already registered with no replicas' in reason: self.log.warn( errorStr, "%s: %s, will remove it and retry" % (lfn, reason)) dm.removeFile(lfn) else: self.log.warn(errorStr, "%s: %s" % (lfn, reason)) failedFiles += 1 else: if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord( self.createRMSRecord("Successful", 1)) else: gMonitor.addMark("RegisterOK", 1) # self.dataLoggingClient().addFileRecord( # lfn, "Register", ','.join(catalogs) if catalogs else "all catalogs", "", "RegisterFile") self.log.verbose( "file %s has been registered at %s" % (lfn, ','.join(catalogs) if catalogs else "all catalogs")) opFile.Status = "Done" if self.rmsMonitoring: self.rmsMonitoringReporter.commit() # # final check if failedFiles: self.log.warn("all files processed, %s files failed to register" % failedFiles) self.operation.Error = "some files failed to register" return S_ERROR(self.operation.Error) return S_OK()
class ReplicaManagerTestCase(unittest.TestCase): """ Base class for the Replica Manager test cases """ def setUp(self): self.dataManager = DataManager() self.fileName = '/tmp/temporaryLocalFile' file = open(self.fileName, 'w') file.write("%s" % time.time()) file.close() def test_putAndRegister(self): print '\n\n#########################################################################\n\n\t\t\tPut and register test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegister/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterReplicate(self): print '\n\n#########################################################################\n\n\t\t\tReplication test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterReplicate/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) replicateRes = self.dataManager.replicateAndRegister( lfn, 'CNAF-DST') #,sourceSE='',destPath='',localCache='') removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replicate was successful self.assert_(replicateRes['OK']) self.assert_(replicateRes['Value'].has_key('Successful')) self.assert_(replicateRes['Value']['Successful'].has_key(lfn)) self.assert_(replicateRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterGetReplicaMetadata(self): print '\n\n#########################################################################\n\n\t\t\tGet metadata test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetReplicaMetadata/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) metadataRes = self.dataManager.getReplicaMetadata(lfn, diracSE) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the metadata query was successful self.assert_(metadataRes['OK']) self.assert_(metadataRes['Value'].has_key('Successful')) self.assert_(metadataRes['Value']['Successful'].has_key(lfn)) self.assert_(metadataRes['Value']['Successful'][lfn]) metadataDict = metadataRes['Value']['Successful'][lfn] self.assert_(metadataDict.has_key('Cached')) self.assert_(metadataDict.has_key('Migrated')) self.assert_(metadataDict.has_key('Size')) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegsiterGetAccessUrl(self): print '\n\n#########################################################################\n\n\t\t\tGet Access Url test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetAccessUrl/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) getAccessUrlRes = self.dataManager.getReplicaAccessUrl(lfn, diracSE) print getAccessUrlRes removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the access url was successful self.assert_(getAccessUrlRes['OK']) self.assert_(getAccessUrlRes['Value'].has_key('Successful')) self.assert_(getAccessUrlRes['Value']['Successful'].has_key(lfn)) self.assert_(getAccessUrlRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterRemoveReplica(self): print '\n\n#########################################################################\n\n\t\t\tRemove replica test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterRemoveReplica/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) removeReplicaRes = self.dataManager.removeReplica(diracSE, lfn) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeReplicaRes['OK']) self.assert_(removeReplicaRes['Value'].has_key('Successful')) self.assert_(removeReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(removeReplicaRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_registerFile(self): lfn = '/lhcb/test/unit-test/ReplicaManager/registerFile/testFile.%s' % time.time( ) physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn fileSize = 10000 storageElementName = 'CERN-RAW' fileGuid = makeGuid() fileTuple = (lfn, physicalFile, fileSize, storageElementName, fileGuid) registerRes = self.dataManager.registerFile(fileTuple) removeCatalogReplicaRes = self.dataManager.removeCatalogReplica( storageElementName, lfn) removeFileRes = self.dataManager.removeFile(lfn) # Check that the file registration was done correctly self.assert_(registerRes['OK']) self.assert_(registerRes['Value'].has_key('Successful')) self.assert_(registerRes['Value']['Successful'].has_key(lfn)) self.assert_(registerRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes['OK']) self.assert_(removeCatalogReplicaRes['Value'].has_key('Successful')) self.assert_( removeCatalogReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value'].has_key('Successful')) self.assert_(removeFileRes['Value']['Successful'].has_key(lfn)) self.assert_(removeFileRes['Value']['Successful'][lfn]) def test_registerReplica(self): print '\n\n#########################################################################\n\n\t\t\tRegister replica test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/registerReplica/testFile.%s' % time.time( ) physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn fileSize = 10000 storageElementName = 'CERN-RAW' fileGuid = makeGuid() fileTuple = (lfn, physicalFile, fileSize, storageElementName, fileGuid) registerRes = self.dataManager.registerFile(fileTuple) seName = 'GRIDKA-RAW' replicaTuple = (lfn, physicalFile, seName) registerReplicaRes = self.dataManager.registerReplica(replicaTuple) removeCatalogReplicaRes1 = self.dataManager.removeCatalogReplica( storageElementName, lfn) removeCatalogReplicaRes2 = self.dataManager.removeCatalogReplica( seName, lfn) removeFileRes = self.dataManager.removeFile(lfn) # Check that the file registration was done correctly self.assert_(registerRes['OK']) self.assert_(registerRes['Value'].has_key('Successful')) self.assert_(registerRes['Value']['Successful'].has_key(lfn)) self.assert_(registerRes['Value']['Successful'][lfn]) # Check that the replica registration was successful self.assert_(registerReplicaRes['OK']) self.assert_(registerReplicaRes['Value'].has_key('Successful')) self.assert_(registerReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(registerReplicaRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes1['OK']) self.assert_(removeCatalogReplicaRes1['Value'].has_key('Successful')) self.assert_( removeCatalogReplicaRes1['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes1['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes2['OK']) self.assert_(removeCatalogReplicaRes2['Value'].has_key('Successful')) self.assert_( removeCatalogReplicaRes2['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes2['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value'].has_key('Successful')) self.assert_(removeFileRes['Value']['Successful'].has_key(lfn)) self.assert_(removeFileRes['Value']['Successful'][lfn]) def test_putAndRegisterGet(self): print '\n\n#########################################################################\n\n\t\t\tGet file test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGet/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) getRes = self.dataManager.getFile(lfn) removeRes = self.dataManager.removeFile(lfn) localFilePath = "%s/%s" % (os.getcwd(), os.path.basename(lfn)) if os.path.exists(localFilePath): os.remove(localFilePath) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(getRes['OK']) self.assert_(getRes['Value'].has_key('Successful')) self.assert_(getRes['Value']['Successful'].has_key(lfn)) self.assertEqual(getRes['Value']['Successful'][lfn], localFilePath) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn])
class ReplicaManagerTestCase(unittest.TestCase): """ Base class for the Replica Manager test cases """ def setUp(self): self.dataManager = DataManager() self.fileName = '/tmp/temporaryLocalFile' file = open(self.fileName,'w') file.write("%s" % time.time()) file.close() def test_putAndRegister(self): print '\n\n#########################################################################\n\n\t\t\tPut and register test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegister/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterReplicate(self): print '\n\n#########################################################################\n\n\t\t\tReplication test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterReplicate/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) replicateRes = self.dataManager.replicateAndRegister(lfn,'CNAF-DST') #,sourceSE='',destPath='',localCache='') removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replicate was successful self.assert_(replicateRes['OK']) self.assert_(replicateRes['Value'].has_key('Successful')) self.assert_(replicateRes['Value']['Successful'].has_key(lfn)) self.assert_(replicateRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterGetReplicaMetadata(self): print '\n\n#########################################################################\n\n\t\t\tGet metadata test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetReplicaMetadata/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) metadataRes = self.dataManager.getReplicaMetadata(lfn,diracSE) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the metadata query was successful self.assert_(metadataRes['OK']) self.assert_(metadataRes['Value'].has_key('Successful')) self.assert_(metadataRes['Value']['Successful'].has_key(lfn)) self.assert_(metadataRes['Value']['Successful'][lfn]) metadataDict = metadataRes['Value']['Successful'][lfn] self.assert_(metadataDict.has_key('Cached')) self.assert_(metadataDict.has_key('Migrated')) self.assert_(metadataDict.has_key('Size')) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegsiterGetAccessUrl(self): print '\n\n#########################################################################\n\n\t\t\tGet Access Url test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetAccessUrl/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) getAccessUrlRes = self.dataManager.getReplicaAccessUrl(lfn,diracSE) print getAccessUrlRes removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the access url was successful self.assert_(getAccessUrlRes['OK']) self.assert_(getAccessUrlRes['Value'].has_key('Successful')) self.assert_(getAccessUrlRes['Value']['Successful'].has_key(lfn)) self.assert_(getAccessUrlRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterRemoveReplica(self): print '\n\n#########################################################################\n\n\t\t\tRemove replica test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterRemoveReplica/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) removeReplicaRes = self.dataManager.removeReplica(diracSE,lfn) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeReplicaRes['OK']) self.assert_(removeReplicaRes['Value'].has_key('Successful')) self.assert_(removeReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(removeReplicaRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_registerFile(self): lfn = '/lhcb/test/unit-test/ReplicaManager/registerFile/testFile.%s' % time.time() physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn fileSize = 10000 storageElementName = 'CERN-RAW' fileGuid = makeGuid() fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid) registerRes = self.dataManager.registerFile(fileTuple) removeCatalogReplicaRes = self.dataManager.removeCatalogReplica(storageElementName,lfn) removeFileRes = self.dataManager.removeFile(lfn) # Check that the file registration was done correctly self.assert_(registerRes['OK']) self.assert_(registerRes['Value'].has_key('Successful')) self.assert_(registerRes['Value']['Successful'].has_key(lfn)) self.assert_(registerRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes['OK']) self.assert_(removeCatalogReplicaRes['Value'].has_key('Successful')) self.assert_(removeCatalogReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value'].has_key('Successful')) self.assert_(removeFileRes['Value']['Successful'].has_key(lfn)) self.assert_(removeFileRes['Value']['Successful'][lfn]) def test_registerReplica(self): print '\n\n#########################################################################\n\n\t\t\tRegister replica test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/registerReplica/testFile.%s' % time.time() physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn fileSize = 10000 storageElementName = 'CERN-RAW' fileGuid = makeGuid() fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid) registerRes = self.dataManager.registerFile(fileTuple) seName = 'GRIDKA-RAW' replicaTuple = (lfn,physicalFile,seName) registerReplicaRes = self.dataManager.registerReplica(replicaTuple) removeCatalogReplicaRes1 = self.dataManager.removeCatalogReplica(storageElementName,lfn) removeCatalogReplicaRes2 = self.dataManager.removeCatalogReplica(seName,lfn) removeFileRes = self.dataManager.removeFile(lfn) # Check that the file registration was done correctly self.assert_(registerRes['OK']) self.assert_(registerRes['Value'].has_key('Successful')) self.assert_(registerRes['Value']['Successful'].has_key(lfn)) self.assert_(registerRes['Value']['Successful'][lfn]) # Check that the replica registration was successful self.assert_(registerReplicaRes['OK']) self.assert_(registerReplicaRes['Value'].has_key('Successful')) self.assert_(registerReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(registerReplicaRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes1['OK']) self.assert_(removeCatalogReplicaRes1['Value'].has_key('Successful')) self.assert_(removeCatalogReplicaRes1['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes1['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes2['OK']) self.assert_(removeCatalogReplicaRes2['Value'].has_key('Successful')) self.assert_(removeCatalogReplicaRes2['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes2['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value'].has_key('Successful')) self.assert_(removeFileRes['Value']['Successful'].has_key(lfn)) self.assert_(removeFileRes['Value']['Successful'][lfn]) def test_putAndRegisterGet(self): print '\n\n#########################################################################\n\n\t\t\tGet file test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGet/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) getRes = self.dataManager.getFile(lfn) removeRes = self.dataManager.removeFile(lfn) localFilePath = "%s/%s" % (os.getcwd(),os.path.basename(lfn)) if os.path.exists(localFilePath): os.remove(localFilePath) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(getRes['OK']) self.assert_(getRes['Value'].has_key('Successful')) self.assert_(getRes['Value']['Successful'].has_key(lfn)) self.assertEqual(getRes['Value']['Successful'][lfn],localFilePath) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn])
class RecursiveRm(object): def __init__(self): self.__rpcclient = RPCClient("DataManagement/FileCatalog") self.__dm = DataManager() self.__n_dirs = 0 self.__n_files = 0 self.__space_freed = 0L def clear_directory(self, directory_path, se_name, dry_run=True): print "Clearing directory: %s" % directory_path dir_content = self.__rpcclient.listDirectory(directory_path, False) if not dir_content["OK"]: print "Failed to contact DIRAC server for %s" % directory_path return if directory_path in dir_content['Value']['Failed']: print "Could not access %s, maybe it doesn't exist?" % directory_path return subdirs = dir_content['Value']['Successful'][directory_path]['SubDirs'] for subdir in subdirs.keys(): self.clear_directory(subdir, se_name, dry_run) # Now do files... files = dir_content['Value']['Successful'][directory_path]['Files'] for filename in files.keys(): fullpath = os.path.join(directory_path, filename) if self.clear_file(fullpath, se_name, dry_run): self.__n_files += 1 self.__space_freed += files[filename]['MetaData']['Size'] if self.remove_empty_dir(directory_path, dry_run): self.__n_dirs += 1 def remove_empty_dir(self, directory_path, dry_run=True): # check if directory is now empty and the remove the directory dir_content = self.__rpcclient.listDirectory(directory_path, False) if not dir_content["OK"]: print "Could not access %s" % directory_path return False subdirs = dir_content['Value']['Successful'][directory_path]['SubDirs'] files = dir_content['Value']['Successful'][directory_path]['Files'] if not subdirs and not files: if not dry_run: self.__dm.fileCatalog.removeDirectory(directory_path, recursive=False) return True return False def clear_file(self, filename, se_name, dry_run=True): res = self.__rpcclient.getReplicas(filename, False) if not res["OK"]: print "Could not get replica status for %s" % filename return False ses = res['Value']['Successful'][filename].keys() # remove file regardless of number of replicas if se_name == "Any": print "%s" % filename if not dry_run: deleted = self.__dm.removeFile(filename) if not deleted["OK"]: print "Function call to removeFile failed, file not deleted: %s" \ % str(deleted) return False elif not deleted["Value"]["Successful"]: print "Failed to delete file: %s" % str(deleted) return False return True # file exists only at the chosen SE # -> delete file and remove from file catalogue if len(ses) == 1 and se_name in ses: print "%s" % filename if not dry_run: deleted = self.__dm.removeFile(filename) if not deleted["OK"]: print "Function call to removeFile failed, file not deleted: %s" \ % str(deleted) return False elif not deleted["Value"]["Successful"]: print "Failed to delete file: %s" % str(deleted) return False return True # file exists at the chosen SE and elswhere -> delete replica at chosen SE if len(ses) > 1 and se_name in ses: print "%s" % filename if not dry_run: deleted = self.__dm.removeReplica(se_name, filename) if not deleted["OK"]: print "Function call to removeReplica failed, replica not deleted: %s" \ % str(deleted) return False elif not deleted["Value"]["Successful"]: print "Failed to delete replica: %s" % str(deleted) return False return True return False def print_stats(self): print "" print "Number of files deleted: %s" % self.__n_files print "NUmber of (sub)directories deleted: %s" % self.__n_dirs space = self.__space_freed / (1024.0 * 1024.0 * 1024.0) print "Space freed: %0.3f GB" % space
class DIRACBackend(GridBackend): """Grid backend using the GFAL command line tools `gfal-*`.""" def __init__(self, **kwargs): GridBackend.__init__(self, catalogue_prefix='', **kwargs) from DIRAC.Core.Base import Script Script.initialize() from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient self.pm = ProxyManagerClient() proxy = self.pm.getUserProxiesInfo() if not proxy['OK']: raise BackendException("Proxy error.") from DIRAC.Interfaces.API.Dirac import Dirac self.dirac = Dirac() from DIRAC.Resources.Catalog.FileCatalog import FileCatalog self.fc = FileCatalog() from DIRAC.DataManagementSystem.Client.DataManager import DataManager self.dm = DataManager() self._xattr_cmd = sh.Command('gfal-xattr').bake(_tty_out=False) self._replica_checksum_cmd = sh.Command('gfal-sum').bake(_tty_out=False) self._bringonline_cmd = sh.Command('gfal-legacy-bringonline').bake(_tty_out=False) self._cp_cmd = sh.Command('gfal-copy').bake(_tty_out=False) self._ls_se_cmd = sh.Command('gfal-ls').bake(color='never', _tty_out=False) self._move_cmd = sh.Command('gfal-rename').bake(_tty_out=False) self._mkdir_cmd = sh.Command('gfal-mkdir').bake(_tty_out=False) self._replicate_cmd = sh.Command('dirac-dms-replicate-lfn').bake(_tty_out=False) self._add_cmd = sh.Command('dirac-dms-add-file').bake(_tty_out=False) @staticmethod def _check_return_value(ret): if not ret['OK']: raise BackendException("Failed: %s", ret['Message']) for path, error in ret['Value']['Failed'].items(): if ('No such' in error) or ('Directory does not' in error): raise DoesNotExistException("No such file or directory.") else: raise BackendException(error) def _is_dir(self, lurl): isdir = self.fc.isDirectory(lurl) self._check_return_value(isdir) return isdir['Value']['Successful'][lurl] def _is_file(self, lurl): isfile = self.fc.isFile(lurl) self._check_return_value(isfile) return isfile['Value']['Successful'][lurl] def _get_dir_entry(self, lurl, infodict=None): """Take a lurl and return a DirEntry.""" # If no dctionary with the information is specified, get it from the catalogue try: md = infodict['MetaData'] except TypeError: md = self.fc.getFileMetadata(lurl) if not md['OK']: raise BackendException("Failed to list path '%s': %s", lurl, md['Message']) for path, error in md['Value']['Failed'].items(): if 'No such file' in error: # File does not exist, maybe a directory? md = self.fc.getDirectoryMetadata(lurl) for path, error in md['Value']['Failed'].items(): raise DoesNotExistException("No such file or directory.") else: raise BackendException(md['Value']['Failed'][lurl]) md = md['Value']['Successful'][lurl] return DirEntry(posixpath.basename(lurl), mode=oct(md.get('Mode', -1)), links=md.get('links', -1), gid=md['OwnerGroup'], uid=md['Owner'], size=md.get('Size', -1), modified=str(md.get('ModificationDate', '?'))) def _iter_directory(self, lurl): """Iterate over entries in a directory.""" ret = self.fc.listDirectory(lurl) if not ret['OK']: raise BackendException("Failed to list path '%s': %s", lurl, ret['Message']) for path, error in ret['Value']['Failed'].items(): if 'Directory does not' in error: # Dir does not exist, maybe a File? if self.fc.isFile(lurl): lst = [(lurl, None)] break else: raise DoesNotExistException("No such file or Directory.") else: raise BackendException(ret['Value']['Failed'][lurl]) else: # Sort items by keys, i.e. paths lst = sorted(ret['Value']['Successful'][lurl]['Files'].items() + ret['Value']['Successful'][lurl]['SubDirs'].items()) for item in lst: yield item # = path, dict def _ls(self, lurl, **kwargs): # Translate keyword arguments d = kwargs.pop('directory', False) if d: # Just the requested entry itself yield self._get_dir_entry(lurl) return for path, info in self._iter_directory(lurl): yield self._get_dir_entry(path, info) def _ls_se(self, surl, **kwargs): # Translate keyword arguments d = kwargs.pop('directory', False) args = [] if -d: args.append('-d') args.append('-l') args.append(surl) try: output = self._ls_se_cmd(*args, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or Directory.") else: raise BackendException(e.stderr) for line in output: fields = line.split() mode, links, gid, uid, size = fields[:5] name = fields[-1] modified = ' '.join(fields[5:-1]) yield DirEntry(name, mode=mode, links=int(links), gid=gid, uid=uid, size=int(size), modified=modified) def _replicas(self, lurl, **kwargs): # Check the lurl actually exists self._ls(lurl, directory=True) rep = self.dirac.getReplicas(lurl) self._check_return_value(rep) rep = rep['Value']['Successful'][lurl] return rep.values() def _exists(self, surl, **kwargs): try: ret = self._ls_se_cmd(surl, '-d', '-l', **kwargs).strip() except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: return False else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) else: return ret[0] != 'd' # Return `False` for directories def _register(self, surl, lurl, verbose=False, **kwargs): # Register an existing physical copy in the file catalogue se = storage.get_SE(surl).name # See if file already exists in DFC ret = self.fc.getFileMetadata(lurl) try: self._check_return_value(ret) except DoesNotExistException: # Add new file size = next(self._ls_se(surl, directory=True)).size checksum = self.checksum(surl) guid = str(uuid.uuid4()) # The guid does not seem to be important. Make it unique if possible. ret = self.dm.registerFile((lurl, surl, size, se, guid, checksum)) else: # Add new replica ret = self.dm.registerReplica((lurl, surl, se)) self._check_return_value(ret) if verbose: print_("Successfully registered replica %s of %s from %s."%(surl, lurl, se)) return True def _deregister(self, surl, lurl, verbose=False, **kwargs): # DIRAC only needs to know the SE name to deregister a replica se = storage.get_SE(surl).name ret = self.dm.removeReplicaFromCatalog(se, [lurl]) self._check_return_value(ret) if verbose: print_("Successfully deregistered replica of %s from %s."%(lurl, se)) return True def _state(self, surl, **kwargs): try: state = self._xattr_cmd(surl, 'user.status', **kwargs).strip() except sh.ErrorReturnCode as e: if "No such file" in e.stderr: raise DoesNotExistException("No such file or Directory.") state = '?' except sh.SignalException_SIGSEGV: state = '?' return state def _checksum(self, surl, **kwargs): try: checksum = self._replica_checksum_cmd(surl, 'ADLER32', **kwargs).split()[1] except sh.ErrorReturnCode: checksum = '?' except sh.SignalException_SIGSEGV: checksum = '?' except IndexError: checksum = '?' return checksum def _bringonline(self, surl, timeout, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None # gfal does not notice when files come online, it seems # Just send a single short request, then check regularly if verbose: out = sys.stdout else: out = None end = time.time() + timeout try: self._bringonline_cmd('-t', 10, surl, _out=out, **kwargs) except sh.ErrorReturnCode as e: # The command fails if the file is not online # To be expected after 10 seconds if "No such file" in e.stderr: # Except when the file does not actually exist on the tape storage raise DoesNotExistException("No such file or Directory.") wait = 5 while(True): if verbose: print_("Checking replica state...") if self.is_online(surl): if verbose: print_("Replica brought online.") return True time_left = end - time.time() if time_left <= 0: if verbose: print_("Could not bring replica online.") return False wait *= 2 if time_left < wait: wait = time_left if verbose: print_("Timeout remaining: %d s"%(time_left)) print_("Checking again in: %d s"%(wait)) time.sleep(wait) def _replicate(self, source_surl, destination_surl, lurl, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None source = storage.get_SE(source_surl).name destination = storage.get_SE(destination_surl).name try: self._replicate_cmd(lurl, destination, source, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return True def _get(self, surl, localpath, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None try: self._cp_cmd('-f', '--checksum', 'ADLER32', surl, localpath, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return os.path.isfile(localpath) def _put(self, localpath, surl, lurl, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None se = storage.get_SE(surl).name try: self._add_cmd(lurl, localpath, se, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return True def _remove(self, surl, lurl, last=False, verbose=False, **kwargs): se = storage.get_SE(surl).name if last: # Delete lfn if verbose: print_("Removing all replicas of %s."%(lurl,)) ret = self.dm.removeFile([lurl]) else: if verbose: print_("Removing replica of %s from %s."%(lurl, se)) ret = self.dm.removeReplica(se, [lurl]) if not ret['OK']: raise BackendException('Failed: %s'%(ret['Message'])) for lurl, error in ret['Value']['Failed'].items(): if 'No such file' in error: raise DoesNotExistException("No such file or directory.") else: raise BackendException(error) return True def _rmdir(self, lurl, verbose=False): """Remove the an empty directory from the catalogue.""" rep = self.fc.removeDirectory(lurl) self._check_return_value(rep) return True def _move_replica(self, surl, new_surl, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None try: folder = posixpath.dirname(new_surl) self._mkdir_cmd(folder, '-p', _out=out, **kwargs) self._move_cmd(surl, new_surl, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return True
class RecursiveRm(object): def __init__(self): self.__rpcclient = RPCClient( "DataManagement/FileCatalog" ) self.__dm = DataManager() self.__n_dirs = 0 self.__n_files = 0 self.__space_freed = 0L def clear_directory(self, directory_path, se_name, dry_run=True): print "Clearing directory: %s" % directory_path dir_content = self.__rpcclient.listDirectory(directory_path, False) if not dir_content["OK"]: print "Failed to contact DIRAC server for %s" % directory_path return if directory_path in dir_content['Value']['Failed']: print "Could not access %s, maybe it doesn't exist?" % directory_path return subdirs = dir_content['Value']['Successful'][directory_path]['SubDirs'] for subdir in subdirs.keys(): self.clear_directory(subdir, se_name, dry_run) # Now do files... files = dir_content['Value']['Successful'][directory_path]['Files'] for filename in files.keys(): fullpath = os.path.join(directory_path, filename) if self.clear_file(fullpath, se_name, dry_run): self.__n_files += 1 self.__space_freed += files[filename]['MetaData']['Size'] if self.remove_empty_dir(directory_path, dry_run): self.__n_dirs += 1 def remove_empty_dir(self, directory_path, dry_run=True): # check if directory is now empty and the remove the directory dir_content = self.__rpcclient.listDirectory(directory_path, False) if not dir_content["OK"]: print "Could not access %s" % directory_path return False subdirs = dir_content['Value']['Successful'][directory_path]['SubDirs'] files = dir_content['Value']['Successful'][directory_path]['Files'] if not subdirs and not files: if not dry_run: self.__dm.fileCatalog.removeDirectory(directory_path, recursive=False) return True return False def clear_file(self, filename, se_name, dry_run=True): res = self.__rpcclient.getReplicas(filename, False) if not res["OK"]: print "Could not get replica status for %s" % filename return False ses = res['Value']['Successful'][filename].keys() # remove file regardless of number of replicas if se_name == "Any": print "%s" % filename if not dry_run: deleted = self.__dm.removeFile(filename) if not deleted["OK"]: print "Function call to removeFile failed, file not deleted: %s" \ % str(deleted) return False elif not deleted["Value"]["Successful"]: print "Failed to delete file: %s" % str(deleted) return False return True # file exists only at the chosen SE # -> delete file and remove from file catalogue if len(ses) == 1 and se_name in ses: print "%s" % filename if not dry_run: deleted = self.__dm.removeFile(filename) if not deleted["OK"]: print "Function call to removeFile failed, file not deleted: %s" \ % str(deleted) return False elif not deleted["Value"]["Successful"]: print "Failed to delete file: %s" % str(deleted) return False return True # file exists at the chosen SE and elswhere -> delete replica at chosen SE if len(ses) > 1 and se_name in ses: print "%s" % filename if not dry_run: deleted = self.__dm.removeReplica(se_name, filename) if not deleted["OK"]: print "Function call to removeReplica failed, replica not deleted: %s" \ % str(deleted) return False elif not deleted["Value"]["Successful"]: print "Failed to delete replica: %s" % str(deleted) return False return True return False def print_stats(self): print "" print "Number of files deleted: %s" % self.__n_files print "NUmber of (sub)directories deleted: %s" % self.__n_dirs space = self.__space_freed/(1024.0 * 1024.0 * 1024.0) print "Space freed: %0.3f GB" % space
class TransformationCleaningAgent( AgentModule ): """ .. class:: TransformationCleaningAgent :param DataManger dm: DataManager instance :param TransfromationClient transClient: TransfromationClient instance :param FileCatalogClient metadataClient: FileCatalogClient instance """ def __init__( self, *args, **kwargs ): """ c'tor """ AgentModule.__init__( self, *args, **kwargs ) # # data manager self.dm = None # # transformation client self.transClient = None # # wms client self.wmsClient = None # # request client self.reqClient = None # # file catalog client self.metadataClient = None # # transformations types self.transformationTypes = None # # directory locations self.directoryLocations = None # # transformation metadata self.transfidmeta = None # # archive periof in days self.archiveAfter = None # # active SEs self.activeStorages = None # # transformation log SEs self.logSE = None # # enable/disable execution self.enableFlag = None def initialize( self ): """ agent initialisation reading and setting confing opts :param self: self reference """ # # shifter proxy self.am_setOption( 'shifterProxy', 'DataManager' ) # # transformations types self.dataProcTTypes = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] ) self.dataManipTTypes = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] ) agentTSTypes = self.am_getOption( 'TransformationTypes', [] ) if agentTSTypes: self.transformationTypes = sorted( agentTSTypes ) else: self.transformationTypes = sorted( self.dataProcTTypes + self.dataManipTTypes ) self.log.info( "Will consider the following transformation types: %s" % str( self.transformationTypes ) ) # # directory locations self.directoryLocations = sorted( self.am_getOption( 'DirectoryLocations', [ 'TransformationDB', 'MetadataCatalog' ] ) ) self.log.info( "Will search for directories in the following locations: %s" % str( self.directoryLocations ) ) # # transformation metadata self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" ) self.log.info( "Will use %s as metadata tag name for TransformationID" % self.transfidmeta ) # # archive periof in days self.archiveAfter = self.am_getOption( 'ArchiveAfter', 7 ) # days self.log.info( "Will archive Completed transformations after %d days" % self.archiveAfter ) # # active SEs self.activeStorages = sorted( self.am_getOption( 'ActiveSEs', [] ) ) self.log.info( "Will check the following storage elements: %s" % str( self.activeStorages ) ) # # transformation log SEs self.logSE = self.am_getOption( 'TransformationLogSE', 'LogSE' ) self.log.info( "Will remove logs found on storage element: %s" % self.logSE ) # # enable/disable execution, should be using CS option Status?? with default value as 'Active'?? self.enableFlag = self.am_getOption( 'EnableFlag', 'True' ) # # data manager self.dm = DataManager() # # transformation client self.transClient = TransformationClient() # # wms client self.wmsClient = WMSClient() # # request client self.reqClient = ReqClient() # # file catalog client self.metadataClient = FileCatalogClient() return S_OK() ############################################################################# def execute( self ): """ execution in one agent's cycle :param self: self reference """ self.enableFlag = self.am_getOption( 'EnableFlag', 'True' ) if not self.enableFlag == 'True': self.log.info( 'TransformationCleaningAgent is disabled by configuration option EnableFlag' ) return S_OK( 'Disabled via CS flag' ) # # Obtain the transformations in Cleaning status and remove any mention of the jobs/files res = self.transClient.getTransformations( { 'Status' : 'Cleaning', 'Type' : self.transformationTypes } ) if res['OK']: for transDict in res['Value']: # # if transformation is of type `Replication` or `Removal`, there is nothing to clean. # # We just archive if transDict[ 'Type' ] in self.dataManipTTypes: res = self.archiveTransformation( transDict['TransformationID'] ) if not res['OK']: self.log.error( "Problems archiving transformation %s: %s" % ( transDict['TransformationID'], res['Message'] ) ) else: res = self.cleanTransformation( transDict['TransformationID'] ) if not res['OK']: self.log.error( "Problems cleaning transformation %s: %s" % ( transDict['TransformationID'], res['Message'] ) ) # # Obtain the transformations in RemovingFiles status and (wait for it) removes the output files res = self.transClient.getTransformations( { 'Status' : 'RemovingFiles', 'Type' : self.transformationTypes} ) if res['OK']: for transDict in res['Value']: res = self.removeTransformationOutput( transDict['TransformationID'] ) if not res['OK']: self.log.error( "Problems removing transformation %s: %s" % ( transDict['TransformationID'], res['Message'] ) ) # # Obtain the transformations in Completed status and archive if inactive for X days olderThanTime = datetime.utcnow() - timedelta( days = self.archiveAfter ) res = self.transClient.getTransformations( { 'Status' : 'Completed', 'Type' : self.transformationTypes }, older = olderThanTime, timeStamp = 'LastUpdate' ) if res['OK']: for transDict in res['Value']: res = self.archiveTransformation( transDict['TransformationID'] ) if not res['OK']: self.log.error( "Problems archiving transformation %s: %s" % ( transDict['TransformationID'], res['Message'] ) ) else: self.log.error( "Could not get the transformations" ) return S_OK() ############################################################################# # # Get the transformation directories for checking # def getTransformationDirectories( self, transID ): """ get the directories for the supplied transformation from the transformation system :param self: self reference :param int transID: transformation ID """ directories = [] if 'TransformationDB' in self.directoryLocations: res = self.transClient.getTransformationParameters( transID, ['OutputDirectories'] ) if not res['OK']: self.log.error( "Failed to obtain transformation directories", res['Message'] ) return res transDirectories = res['Value'].splitlines() directories = self._addDirs( transID, transDirectories, directories ) if 'MetadataCatalog' in self.directoryLocations: res = self.metadataClient.findDirectoriesByMetadata( {self.transfidmeta:transID} ) if not res['OK']: self.log.error( "Failed to obtain metadata catalog directories", res['Message'] ) return res transDirectories = res['Value'] directories = self._addDirs( transID, transDirectories, directories ) if not directories: self.log.info( "No output directories found" ) directories = sorted( directories ) return S_OK( directories ) # FIXME If a classmethod, should it not have cls instead of self? @classmethod def _addDirs( self, transID, newDirs, existingDirs ): """ append uniqe :newDirs: list to :existingDirs: list :param self: self reference :param int transID: transformationID :param list newDirs: src list of paths :param list existingDirs: dest list of paths """ for folder in newDirs: transStr = str( transID ).zfill( 8 ) if re.search( transStr, str( folder ) ): if not folder in existingDirs: existingDirs.append( folder ) return existingDirs ############################################################################# # # These are the methods for performing the cleaning of catalogs and storage # def cleanStorageContents( self, directory ): """ delete lfn dir from all active SE :param self: self reference :param sre directory: folder name """ for storageElement in self.activeStorages: res = self.__removeStorageDirectory( directory, storageElement ) if not res['OK']: return res return S_OK() def __removeStorageDirectory( self, directory, storageElement ): """ wipe out all contents from :directory: at :storageElement: :param self: self reference :param str directory: path :param str storageElement: SE name """ self.log.info( 'Removing the contents of %s at %s' % ( directory, storageElement ) ) se = StorageElement( storageElement ) res = se.getPfnForLfn( [directory] ) if not res['OK']: self.log.error( "Failed to get PFN for directory", res['Message'] ) return res if directory in res['Value']['Failed']: self.log.verbose( 'Failed to obtain directory PFN from LFN', '%s %s' % ( directory, res['Value']['Failed'][directory] ) ) return S_ERROR( 'Failed to obtain directory PFN from LFNs' ) storageDirectory = res['Value']['Successful'][directory] res = returnSingleResult( se.exists( storageDirectory ) ) if not res['OK']: self.log.error( "Failed to obtain existance of directory", res['Message'] ) return res exists = res['Value'] if not exists: self.log.info( "The directory %s does not exist at %s " % ( directory, storageElement ) ) return S_OK() res = returnSingleResult( se.removeDirectory( storageDirectory, recursive = True ) ) if not res['OK']: self.log.error( "Failed to remove storage directory", res['Message'] ) return res self.log.info( "Successfully removed %d files from %s at %s" % ( res['Value']['FilesRemoved'], directory, storageElement ) ) return S_OK() def cleanCatalogContents( self, directory ): """ wipe out everything from catalog under folder :directory: :param self: self reference :params str directory: folder name """ res = self.__getCatalogDirectoryContents( [directory] ) if not res['OK']: return res filesFound = res['Value'] if not filesFound: self.log.info( "No files are registered in the catalog directory %s" % directory ) return S_OK() self.log.info( "Attempting to remove %d possible remnants from the catalog and storage" % len( filesFound ) ) res = self.dm.removeFile( filesFound, force = True ) if not res['OK']: return res realFailure = False for lfn, reason in res['Value']['Failed'].items(): if "File does not exist" in str( reason ): self.log.warn( "File %s not found in some catalog: " % ( lfn ) ) else: self.log.error( "Failed to remove file found in the catalog", "%s %s" % ( lfn, reason ) ) realFailure = True if realFailure: return S_ERROR( "Failed to remove all files found in the catalog" ) return S_OK() def __getCatalogDirectoryContents( self, directories ): """ get catalog contents under paths :directories: :param self: self reference :param list directories: list of paths in catalog """ self.log.info( 'Obtaining the catalog contents for %d directories:' % len( directories ) ) for directory in directories: self.log.info( directory ) activeDirs = directories allFiles = {} fc = FileCatalog() while len( activeDirs ) > 0: currentDir = activeDirs[0] res = returnSingleResult( fc.listDirectory( currentDir ) ) activeDirs.remove( currentDir ) if not res['OK'] and res['Message'].endswith( 'The supplied path does not exist' ): self.log.info( "The supplied directory %s does not exist" % currentDir ) elif not res['OK']: if "No such file or directory" in res['Message']: self.log.info( "%s: %s" % ( currentDir, res['Message'] ) ) else: self.log.error( "Failed to get directory %s content: %s" % ( currentDir, res['Message'] ) ) else: dirContents = res['Value'] activeDirs.extend( dirContents['SubDirs'] ) allFiles.update( dirContents['Files'] ) self.log.info( "Found %d files" % len( allFiles ) ) return S_OK( allFiles.keys() ) def cleanTransformationLogFiles( self, directory ): """ clean up transformation logs from directory :directory: :param self: self reference :param str directory: folder name """ self.log.info( "Removing log files found in the directory %s" % directory ) res = returnSingleResult( StorageElement( self.logSE ).removeDirectory( directory ) ) if not res['OK']: self.log.error( "Failed to remove log files", res['Message'] ) return res self.log.info( "Successfully removed transformation log directory" ) return S_OK() ############################################################################# # # These are the functional methods for archiving and cleaning transformations # def removeTransformationOutput( self, transID ): """ This just removes any mention of the output data from the catalog and storage """ self.log.info( "Removing output data for transformation %s" % transID ) res = self.getTransformationDirectories( transID ) if not res['OK']: self.log.error( 'Problem obtaining directories for transformation %s with result "%s"' % ( transID, res ) ) return S_OK() directories = res['Value'] for directory in directories: if not re.search( '/LOG/', directory ): res = self.cleanCatalogContents( directory ) if not res['OK']: return res res = self.cleanStorageContents( directory ) if not res['OK']: return res self.log.info( "Removed directories in the catalog and storage for transformation" ) # Clean ALL the possible remnants found in the metadata catalog res = self.cleanMetadataCatalogFiles( transID ) if not res['OK']: return res self.log.info( "Successfully removed output of transformation %d" % transID ) # Change the status of the transformation to RemovedFiles res = self.transClient.setTransformationParameter( transID, 'Status', 'RemovedFiles' ) if not res['OK']: self.log.error( "Failed to update status of transformation %s to RemovedFiles" % ( transID ), res['Message'] ) return res self.log.info( "Updated status of transformation %s to RemovedFiles" % ( transID ) ) return S_OK() def archiveTransformation( self, transID ): """ This just removes job from the jobDB and the transformation DB :param self: self reference :param int transID: transformation ID """ self.log.info( "Archiving transformation %s" % transID ) # Clean the jobs in the WMS and any failover requests found res = self.cleanTransformationTasks( transID ) if not res['OK']: return res # Clean the transformation DB of the files and job information res = self.transClient.cleanTransformation( transID ) if not res['OK']: return res self.log.info( "Successfully archived transformation %d" % transID ) # Change the status of the transformation to archived res = self.transClient.setTransformationParameter( transID, 'Status', 'Archived' ) if not res['OK']: self.log.error( "Failed to update status of transformation %s to Archived" % ( transID ), res['Message'] ) return res self.log.info( "Updated status of transformation %s to Archived" % ( transID ) ) return S_OK() def cleanTransformation( self, transID ): """ This removes what was produced by the supplied transformation, leaving only some info and log in the transformation DB. """ self.log.info( "Cleaning transformation %s" % transID ) res = self.getTransformationDirectories( transID ) if not res['OK']: self.log.error( 'Problem obtaining directories for transformation %s with result "%s"' % ( transID, res ) ) return S_OK() directories = res['Value'] # Clean the jobs in the WMS and any failover requests found res = self.cleanTransformationTasks( transID ) if not res['OK']: return res # Clean the log files for the jobs for directory in directories: if re.search( '/LOG/', directory ): res = self.cleanTransformationLogFiles( directory ) if not res['OK']: return res res = self.cleanCatalogContents( directory ) if not res['OK']: return res res = self.cleanStorageContents( directory ) if not res['OK']: return res # Clean ALL the possible remnants found in the BK res = self.cleanMetadataCatalogFiles( transID ) if not res['OK']: return res # Clean the transformation DB of the files and job information res = self.transClient.cleanTransformation( transID ) if not res['OK']: return res self.log.info( "Successfully cleaned transformation %d" % transID ) res = self.transClient.setTransformationParameter( transID, 'Status', 'Cleaned' ) if not res['OK']: self.log.error( "Failed to update status of transformation %s to Cleaned" % ( transID ), res['Message'] ) return res self.log.info( "Updated status of transformation %s to Cleaned" % ( transID ) ) return S_OK() def cleanMetadataCatalogFiles( self, transID ): """ wipe out files from catalog """ res = self.metadataClient.findFilesByMetadata( { self.transfidmeta : transID } ) if not res['OK']: return res fileToRemove = res['Value'] if not fileToRemove: self.log.info( 'No files found for transID %s' % transID ) return S_OK() res = self.dm.removeFile( fileToRemove, force = True ) if not res['OK']: return res for lfn, reason in res['Value']['Failed'].items(): self.log.error( "Failed to remove file found in metadata catalog", "%s %s" % ( lfn, reason ) ) if res['Value']['Failed']: return S_ERROR( "Failed to remove all files found in the metadata catalog" ) self.log.info( "Successfully removed all files found in the BK" ) return S_OK() ############################################################################# # # These are the methods for removing the jobs from the WMS and transformation DB # def cleanTransformationTasks( self, transID ): """ clean tasks from WMS, or from the RMS if it is a DataManipulation transformation """ res = self.__getTransformationExternalIDs( transID ) if not res['OK']: return res externalIDs = res['Value'] if externalIDs: res = self.transClient.getTransformationParameters( transID, ['Type'] ) if not res['OK']: self.log.error( "Failed to determine transformation type" ) return res transType = res['Value'] if transType in self.dataProcTTypes: res = self.__removeWMSTasks( externalIDs ) else: res = self.__removeRequests( externalIDs ) if not res['OK']: return res return S_OK() def __getTransformationExternalIDs( self, transID ): """ collect all ExternalIDs for transformation :transID: :param self: self reference :param int transID: transforamtion ID """ res = self.transClient.getTransformationTasks( condDict = { 'TransformationID' : transID } ) if not res['OK']: self.log.error( "Failed to get externalIDs for transformation %d" % transID, res['Message'] ) return res externalIDs = [ taskDict['ExternalID'] for taskDict in res["Value"] ] self.log.info( "Found %d tasks for transformation" % len( externalIDs ) ) return S_OK( externalIDs ) def __removeRequests( self, requestIDs ): """ This will remove requests from the (new) RMS system - #FIXME: if the old system is still installed, it won't remove anything!!! (we don't want to risk removing from the new RMS what is instead in the old) """ # FIXME: checking if the old system is still installed! from DIRAC.ConfigurationSystem.Client import PathFinder if PathFinder.getServiceURL( "RequestManagement/RequestManager" ): self.log.warn( "NOT removing requests!!" ) return S_OK() rIDs = [ int( long( j ) ) for j in requestIDs if long( j ) ] for requestName in rIDs: self.reqClient.deleteRequest( requestName ) return S_OK() def __removeWMSTasks( self, transJobIDs ): """ wipe out jobs and their requests from the system TODO: should check request status, maybe FTS files as well ??? :param self: self reference :param list trasnJobIDs: job IDs """ # Prevent 0 job IDs jobIDs = [ int( j ) for j in transJobIDs if int( j ) ] allRemove = True for jobList in breakListIntoChunks( jobIDs, 500 ): res = self.wmsClient.killJob( jobList ) if res['OK']: self.log.info( "Successfully killed %d jobs from WMS" % len( jobList ) ) elif ( "InvalidJobIDs" in res ) and ( "NonauthorizedJobIDs" not in res ) and ( "FailedJobIDs" not in res ): self.log.info( "Found %s jobs which did not exist in the WMS" % len( res['InvalidJobIDs'] ) ) elif "NonauthorizedJobIDs" in res: self.log.error( "Failed to kill %s jobs because not authorized" % len( res['NonauthorizedJobIDs'] ) ) allRemove = False elif "FailedJobIDs" in res: self.log.error( "Failed to kill %s jobs" % len( res['FailedJobIDs'] ) ) allRemove = False res = self.wmsClient.deleteJob( jobList ) if res['OK']: self.log.info( "Successfully removed %d jobs from WMS" % len( jobList ) ) elif ( "InvalidJobIDs" in res ) and ( "NonauthorizedJobIDs" not in res ) and ( "FailedJobIDs" not in res ): self.log.info( "Found %s jobs which did not exist in the WMS" % len( res['InvalidJobIDs'] ) ) elif "NonauthorizedJobIDs" in res: self.log.error( "Failed to remove %s jobs because not authorized" % len( res['NonauthorizedJobIDs'] ) ) allRemove = False elif "FailedJobIDs" in res: self.log.error( "Failed to remove %s jobs" % len( res['FailedJobIDs'] ) ) allRemove = False if not allRemove: return S_ERROR( "Failed to remove all remnants from WMS" ) self.log.info( "Successfully removed all tasks from the WMS" ) if not jobIDs: self.log.info( "JobIDs not present, unable to remove asociated requests." ) return S_OK() failed = 0 # FIXME: double request client: old/new -> only the new will survive sooner or later # this is the old try: res = RequestClient().getRequestForJobs( jobIDs ) if not res['OK']: self.log.error( "Failed to get requestID for jobs.", res['Message'] ) return res failoverRequests = res['Value'] self.log.info( "Found %d jobs with associated failover requests (in the old RMS)" % len( failoverRequests ) ) if not failoverRequests: return S_OK() for jobID, requestName in failoverRequests.items(): # Put this check just in case, tasks must have associated jobs if jobID == 0 or jobID == '0': continue res = RequestClient().deleteRequest( requestName ) if not res['OK']: self.log.error( "Failed to remove request from RequestDB", res['Message'] ) failed += 1 else: self.log.verbose( "Removed request %s associated to job %d." % ( requestName, jobID ) ) except RuntimeError: failoverRequests = {} pass # FIXME: and this is the new res = self.reqClient.getRequestNamesForJobs( jobIDs ) if not res['OK']: self.log.error( "Failed to get requestID for jobs.", res['Message'] ) return res failoverRequests.update( res['Value']['Successful'] ) if not failoverRequests: return S_OK() for jobID, requestName in res['Value']['Successful'].items(): # Put this check just in case, tasks must have associated jobs if jobID == 0 or jobID == '0': continue res = self.reqClient.deleteRequest( requestName ) if not res['OK']: self.log.error( "Failed to remove request from RequestDB", res['Message'] ) failed += 1 else: self.log.verbose( "Removed request %s associated to job %d." % ( requestName, jobID ) ) if failed: self.log.info( "Successfully removed %s requests" % ( len( failoverRequests ) - failed ) ) self.log.info( "Failed to remove %s requests" % failed ) return S_ERROR( "Failed to remove all the request from RequestDB" ) self.log.info( "Successfully removed all the associated failover requests" ) return S_OK()
from DIRAC import S_OK, S_ERROR, gLogger, exit from DIRAC.DataManagementSystem.Client.DataManager import DataManager lfn = args[0] pfn = args[1] se = args[2] exit_code = 0 log = '' dm = DataManager() start = time.time() result = dm.removeFile( lfn ) result = dm.putAndRegister( lfn, pfn, se ) uploadTime = time.time() - start if result[ 'OK' ]: log += 'Succeed to upload file to SE %s.\n' % se log += 'Upload Time : %ss\n' % uploadTime start = time.time() result = dm.getReplica( lfn, se, tempfile.gettempdir() ) downloadTime = time.time() - start if result[ 'OK' ]: log += 'Succeed to download file from SE %s.\n' % se log += 'Download Time : %ss\n' % downloadTime else: exit_code = 1 log += 'Failed to download file from SE %s : %s\n' % ( se, result[ 'Message' ] )