def main(): dm = DataManager() fileTupleBuffer = [] counter = 0 for f in files: counter += 1 if not f.startswith('/cefs'): gLogger.error('File must be under "/cefs"') return 1 lfn = '/cepc/lustre-ro' + f result = fcc.isFile(lfn) if result['OK'] and lfn in result['Value']['Successful'] and result[ 'Value']['Successful'][lfn]: continue size = os.path.getsize(f) adler32 = fileAdler(f) guid = makeGuid() fileTuple = (lfn, f, size, _se, guid, adler32) fileTupleBuffer.append(fileTuple) gLogger.debug('Register to lfn: %s' % lfn) gLogger.debug('fileTuple: %s' % (fileTuple, )) if len(fileTupleBuffer) >= _bufferSize: result = dm.registerFile(fileTupleBuffer) print('register result', result) if not result['OK']: gLogger.error('Register file failed') return 1 del fileTupleBuffer[:] gLogger.debug('%s files registered' % counter) if fileTupleBuffer: result = dm.registerFile(fileTupleBuffer) print('register result', result) if not result['OK']: gLogger.error('Register file failed') return 1 del fileTupleBuffer[:] gLogger.info('Totally %s files registered' % counter) return 0
def __call__(self): """ call me maybe """ # # counter for failed files failedFiles = 0 # # catalog to use catalog = self.operation.Catalog dm = DataManager(catalogs=catalog) # # get waiting files waitingFiles = self.getWaitingFilesList() # # loop over files for opFile in waitingFiles: gMonitor.addMark("RegisterAtt", 1) # # get LFN lfn = opFile.LFN # # and others fileTuple = (lfn, opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum) # # call DataManager registerFile = dm.registerFile(fileTuple) # # check results if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]: gMonitor.addMark("RegisterFail", 1) self.dataLoggingClient().addFileRecord(lfn, "RegisterFail", catalog, "", "RegisterFile") reason = registerFile.get( "Message", registerFile.get("Value", {}).get("Failed", {}).get(lfn, 'Unknown')) errorStr = "failed to register LFN %s: %s" % (lfn, reason) opFile.Error = errorStr self.log.warn(errorStr) failedFiles += 1 else: gMonitor.addMark("RegisterOK", 1) self.dataLoggingClient().addFileRecord(lfn, "Register", catalog, "", "RegisterFile") self.log.info("file %s has been registered at %s" % (lfn, catalog)) opFile.Status = "Done" # # final check if failedFiles: self.log.info("all files processed, %s files failed to register" % failedFiles) self.operation.Error = "some files failed to register" return S_ERROR(self.operation.Error) return S_OK()
def __call__( self ): """ call me maybe """ # # counter for failed files failedFiles = 0 # # catalog(s) to use catalogs = self.operation.Catalog if catalogs: catalogs = [ cat.strip() for cat in catalogs.split( ',' ) ] dm = DataManager( catalogs = catalogs ) # # get waiting files waitingFiles = self.getWaitingFilesList() # # loop over files for opFile in waitingFiles: gMonitor.addMark( "RegisterAtt", 1 ) # # get LFN lfn = opFile.LFN # # and others fileTuple = ( lfn , opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum ) # # call DataManager registerFile = dm.registerFile( fileTuple ) # # check results if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]: gMonitor.addMark( "RegisterFail", 1 ) # self.dataLoggingClient().addFileRecord( lfn, "RegisterFail", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" ) reason = str( registerFile.get( "Message", registerFile.get( "Value", {} ).get( "Failed", {} ).get( lfn, 'Unknown' ) ) ) errorStr = "failed to register LFN" opFile.Error = "%s: %s" % ( errorStr, reason ) if 'GUID already registered' in reason: opFile.Status = 'Failed' self.log.error( errorStr, "%s: %s" % ( lfn, reason ) ) elif 'File already registered with no replicas' in reason: self.log.warn( errorStr, "%s: %s, will remove it and retry" % ( lfn, reason ) ) dm.removeFile( lfn ) else: self.log.warn( errorStr, "%s: %s" % ( lfn, reason ) ) failedFiles += 1 else: gMonitor.addMark( "RegisterOK", 1 ) # self.dataLoggingClient().addFileRecord( lfn, "Register", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" ) self.log.verbose( "file %s has been registered at %s" % ( lfn, ','.join( catalogs ) if catalogs else "all catalogs" ) ) opFile.Status = "Done" # # final check if failedFiles: self.log.warn( "all files processed, %s files failed to register" % failedFiles ) self.operation.Error = "some files failed to register" return S_ERROR( self.operation.Error ) return S_OK()
def __call__( self ): """ call me maybe """ # # counter for failed files failedFiles = 0 # # catalog to use catalog = self.operation.Catalog dm = DataManager( catalogs = catalog ) # # get waiting files waitingFiles = self.getWaitingFilesList() # # loop over files for opFile in waitingFiles: gMonitor.addMark( "RegisterAtt", 1 ) # # get LFN lfn = opFile.LFN # # and others fileTuple = ( lfn , opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum ) # # call DataManager registerFile = dm.registerFile( fileTuple ) # # check results if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]: gMonitor.addMark( "RegisterFail", 1 ) self.dataLoggingClient().addFileRecord( lfn, "RegisterFail", catalog, "", "RegisterFile" ) reason = registerFile.get( "Message", registerFile.get( "Value", {} ).get( "Failed", {} ).get( lfn, 'Unknown' ) ) errorStr = "failed to register LFN %s: %s" % ( lfn, reason ) opFile.Error = errorStr self.log.warn( errorStr ) failedFiles += 1 else: gMonitor.addMark( "RegisterOK", 1 ) self.dataLoggingClient().addFileRecord( lfn, "Register", catalog, "", "RegisterFile" ) self.log.info( "file %s has been registered at %s" % ( lfn, catalog ) ) opFile.Status = "Done" # # final check if failedFiles: self.log.info( "all files processed, %s files failed to register" % failedFiles ) self.operation.Error = "some files failed to register" return S_ERROR( self.operation.Error ) return S_OK()
if result['OK'] and lfn in result['Value'][ 'Successful'] and result['Value']['Successful'][lfn]: if counter % 1000 == 0: gLogger.notice('Skip file existed counter: %s' % counter) continue size = os.path.getsize(fullFn) adler32 = fileAdler(fullFn) guid = makeGuid() fileTuple = (lfn, fullFn, size, toSE, guid, adler32) fileTupleBuffer.append(fileTuple) gLogger.debug('Register to lfn: %s' % lfn) gLogger.debug('fileTuple: %s' % (fileTuple, )) if len(fileTupleBuffer) >= bufferSize: result = dm.registerFile(fileTupleBuffer) if not result['OK']: gLogger.error('Can not register %s' % fullFn) exit(1) del fileTupleBuffer[:] gLogger.notice('%s files registered' % counter) if fileTupleBuffer: result = dm.registerFile(fileTupleBuffer) if not result['OK']: gLogger.error('Can not register %s' % fullFn) exit(1) del fileTupleBuffer[:] gLogger.notice('Total %s files registered' % counter)
class ReplicaManagerTestCase(unittest.TestCase): """ Base class for the Replica Manager test cases """ def setUp(self): self.dataManager = DataManager() self.fileName = '/tmp/temporaryLocalFile' file = open(self.fileName,'w') file.write("%s" % time.time()) file.close() def test_putAndRegister(self): print '\n\n#########################################################################\n\n\t\t\tPut and register test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegister/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterReplicate(self): print '\n\n#########################################################################\n\n\t\t\tReplication test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterReplicate/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) replicateRes = self.dataManager.replicateAndRegister(lfn,'CNAF-DST') #,sourceSE='',destPath='',localCache='') removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replicate was successful self.assert_(replicateRes['OK']) self.assert_(replicateRes['Value'].has_key('Successful')) self.assert_(replicateRes['Value']['Successful'].has_key(lfn)) self.assert_(replicateRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterGetReplicaMetadata(self): print '\n\n#########################################################################\n\n\t\t\tGet metadata test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetReplicaMetadata/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) metadataRes = self.dataManager.getReplicaMetadata(lfn,diracSE) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the metadata query was successful self.assert_(metadataRes['OK']) self.assert_(metadataRes['Value'].has_key('Successful')) self.assert_(metadataRes['Value']['Successful'].has_key(lfn)) self.assert_(metadataRes['Value']['Successful'][lfn]) metadataDict = metadataRes['Value']['Successful'][lfn] self.assert_(metadataDict.has_key('Cached')) self.assert_(metadataDict.has_key('Migrated')) self.assert_(metadataDict.has_key('Size')) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegsiterGetAccessUrl(self): print '\n\n#########################################################################\n\n\t\t\tGet Access Url test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetAccessUrl/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) getAccessUrlRes = self.dataManager.getReplicaAccessUrl(lfn,diracSE) print getAccessUrlRes removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the access url was successful self.assert_(getAccessUrlRes['OK']) self.assert_(getAccessUrlRes['Value'].has_key('Successful')) self.assert_(getAccessUrlRes['Value']['Successful'].has_key(lfn)) self.assert_(getAccessUrlRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterRemoveReplica(self): print '\n\n#########################################################################\n\n\t\t\tRemove replica test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterRemoveReplica/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) removeReplicaRes = self.dataManager.removeReplica(diracSE,lfn) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeReplicaRes['OK']) self.assert_(removeReplicaRes['Value'].has_key('Successful')) self.assert_(removeReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(removeReplicaRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_registerFile(self): lfn = '/lhcb/test/unit-test/ReplicaManager/registerFile/testFile.%s' % time.time() physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn fileSize = 10000 storageElementName = 'CERN-RAW' fileGuid = makeGuid() fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid) registerRes = self.dataManager.registerFile(fileTuple) removeCatalogReplicaRes = self.dataManager.removeCatalogReplica(storageElementName,lfn) removeFileRes = self.dataManager.removeFile(lfn) # Check that the file registration was done correctly self.assert_(registerRes['OK']) self.assert_(registerRes['Value'].has_key('Successful')) self.assert_(registerRes['Value']['Successful'].has_key(lfn)) self.assert_(registerRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes['OK']) self.assert_(removeCatalogReplicaRes['Value'].has_key('Successful')) self.assert_(removeCatalogReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value'].has_key('Successful')) self.assert_(removeFileRes['Value']['Successful'].has_key(lfn)) self.assert_(removeFileRes['Value']['Successful'][lfn]) def test_registerReplica(self): print '\n\n#########################################################################\n\n\t\t\tRegister replica test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/registerReplica/testFile.%s' % time.time() physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn fileSize = 10000 storageElementName = 'CERN-RAW' fileGuid = makeGuid() fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid) registerRes = self.dataManager.registerFile(fileTuple) seName = 'GRIDKA-RAW' replicaTuple = (lfn,physicalFile,seName) registerReplicaRes = self.dataManager.registerReplica(replicaTuple) removeCatalogReplicaRes1 = self.dataManager.removeCatalogReplica(storageElementName,lfn) removeCatalogReplicaRes2 = self.dataManager.removeCatalogReplica(seName,lfn) removeFileRes = self.dataManager.removeFile(lfn) # Check that the file registration was done correctly self.assert_(registerRes['OK']) self.assert_(registerRes['Value'].has_key('Successful')) self.assert_(registerRes['Value']['Successful'].has_key(lfn)) self.assert_(registerRes['Value']['Successful'][lfn]) # Check that the replica registration was successful self.assert_(registerReplicaRes['OK']) self.assert_(registerReplicaRes['Value'].has_key('Successful')) self.assert_(registerReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(registerReplicaRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes1['OK']) self.assert_(removeCatalogReplicaRes1['Value'].has_key('Successful')) self.assert_(removeCatalogReplicaRes1['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes1['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes2['OK']) self.assert_(removeCatalogReplicaRes2['Value'].has_key('Successful')) self.assert_(removeCatalogReplicaRes2['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes2['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value'].has_key('Successful')) self.assert_(removeFileRes['Value']['Successful'].has_key(lfn)) self.assert_(removeFileRes['Value']['Successful'][lfn]) def test_putAndRegisterGet(self): print '\n\n#########################################################################\n\n\t\t\tGet file test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGet/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) getRes = self.dataManager.getFile(lfn) removeRes = self.dataManager.removeFile(lfn) localFilePath = "%s/%s" % (os.getcwd(),os.path.basename(lfn)) if os.path.exists(localFilePath): os.remove(localFilePath) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(getRes['OK']) self.assert_(getRes['Value'].has_key('Successful')) self.assert_(getRes['Value']['Successful'].has_key(lfn)) self.assertEqual(getRes['Value']['Successful'][lfn],localFilePath) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn])
) for i in fileUnlinkedList: gLogger.notice(i) gLogger.notice("---------- Total: %d unlinked files. ----------\n" % len(fileUnlinkedList)) if deletionSwitch: for i in fileDarkList: g.unlink(HEAD0 + '/' + i) gLogger.notice("%s deleted!" % (HEAD0 + '/' + i)) gLogger.notice("---------- All dark files deleted! ----------\n") if registerOption != 'none': if registerOption in ['dark', 'both']: gLogger.notice("---------- Register dark files: ----------") for i in fileDarkList: fileStat = g.stat(HEAD0 + '/' + i) fileSize = fileStat.st_size fileGuid = makeGuid() fileSum = g.checksum(HEAD0 + '/' + i, "ADLER32") fileTuple = (i, HEAD0 + '/' + i, fileSize, SITE, fileGuid, fileSum) dm.registerFile(fileTuple) gLogger.notice(i + ' registered successfully!') if registerOption in ['unlinked', 'both']: gLogger.notice("---------- Register unlinked files: ----------") for i in fileUnlinkedList: fileTuple = (i, HEAD0 + '/' + i, SITE) dm.registerReplica(fileTuple) gLogger.notice(i + ' registered successfully!') gLogger.notice("---------- Register finished ----------\n")
def __call__(self): """ call me maybe """ # The flag 'rmsMonitoring' is set by the RequestTask and is False by default. # Here we use 'createRMSRecord' to create the ES record which is defined inside OperationHandlerBase. if self.rmsMonitoring: self.rmsMonitoringReporter = MonitoringReporter( monitoringType="RMSMonitoring") else: # # RegisterFile specific monitor info gMonitor.registerActivity("RegisterAtt", "Attempted file registrations", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM) gMonitor.registerActivity("RegisterOK", "Successful file registrations", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM) gMonitor.registerActivity("RegisterFail", "Failed file registrations", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM) # # counter for failed files failedFiles = 0 # # catalog(s) to use catalogs = self.operation.Catalog if catalogs: catalogs = [cat.strip() for cat in catalogs.split(',')] dm = DataManager(catalogs=catalogs) # # get waiting files waitingFiles = self.getWaitingFilesList() if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord( self.createRMSRecord("Attempted", len(waitingFiles))) # # loop over files for opFile in waitingFiles: if not self.rmsMonitoring: gMonitor.addMark("RegisterAtt", 1) # # get LFN lfn = opFile.LFN # # and others fileTuple = (lfn, opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum) # # call DataManager registerFile = dm.registerFile(fileTuple) # # check results if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]: if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord( self.createRMSRecord("Failed", 1)) else: gMonitor.addMark("RegisterFail", 1) # self.dataLoggingClient().addFileRecord( # lfn, "RegisterFail", ','.join(catalogs) if catalogs else "all catalogs", "", "RegisterFile") reason = str( registerFile.get( "Message", registerFile.get("Value", {}).get("Failed", {}).get(lfn, 'Unknown'))) errorStr = "failed to register LFN" opFile.Error = "%s: %s" % (errorStr, reason) if 'GUID already registered' in reason: opFile.Status = 'Failed' self.log.error(errorStr, "%s: %s" % (lfn, reason)) elif 'File already registered with no replicas' in reason: self.log.warn( errorStr, "%s: %s, will remove it and retry" % (lfn, reason)) dm.removeFile(lfn) else: self.log.warn(errorStr, "%s: %s" % (lfn, reason)) failedFiles += 1 else: if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord( self.createRMSRecord("Successful", 1)) else: gMonitor.addMark("RegisterOK", 1) # self.dataLoggingClient().addFileRecord( # lfn, "Register", ','.join(catalogs) if catalogs else "all catalogs", "", "RegisterFile") self.log.verbose( "file %s has been registered at %s" % (lfn, ','.join(catalogs) if catalogs else "all catalogs")) opFile.Status = "Done" if self.rmsMonitoring: self.rmsMonitoringReporter.commit() # # final check if failedFiles: self.log.warn("all files processed, %s files failed to register" % failedFiles) self.operation.Error = "some files failed to register" return S_ERROR(self.operation.Error) return S_OK()
def main(): dm = DataManager() fileTupleBuffer = [] res = getProxyInfo( False, False ) if not res['OK']: gLogger.error( "Failed to get client proxy information.", res['Message'] ) DIRAC.exit( 2 ) proxyInfo = res['Value'] if proxyInfo['secondsLeft'] == 0: gLogger.error( "Proxy expired" ) DIRAC.exit( 2 ) username = proxyInfo['username'] vo = '' if 'group' in proxyInfo: vo = getVOForGroup( proxyInfo['group'] ) counter = 0 for f in files: counter += 1 local_f=f if not f.startswith('/cefs'): # gLogger.error('File must be under "/cefs"') # continue #if the file to reg is not under /cefs, use put and register folder_name=os.path.basename(os.path.dirname(f)) # lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f) lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f) # dirname = os.path.dirname(local_f) # os.system('mkdir -p %s'%(dirname)) # os.system('cp %s %s' %(f,local_f)) do_put_and_register=True else: lfn = '/cepc/lustre-ro' + os.path.abspath(f) do_put_and_register=False result = fcc.isFile(lfn) if result['OK'] and lfn in result['Value']['Successful'] and result['Value']['Successful'][lfn]: continue size = os.path.getsize(f) adler32 = fileAdler(f) guid = makeGuid() fileTuple = (lfn, local_f, size, _se, guid, adler32) fileTupleBuffer.append(fileTuple) gLogger.debug('Register to lfn: %s' % lfn) gLogger.debug('fileTuple: %s' % (fileTuple,)) if len(fileTupleBuffer) >= _bufferSize: if do_put_and_register: result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite) else: result = dm.registerFile(fileTupleBuffer) print('register result', result) # if not result['OK']: # gLogger.error('Register file failed') # return 1 del fileTupleBuffer[:] gLogger.debug('%s files registered' % counter) if fileTupleBuffer: if do_put_and_register: result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite) else: result = dm.registerFile(fileTupleBuffer) print('register result', result) # if not result['OK']: # gLogger.error('Register file failed') # return 1 del fileTupleBuffer[:] gLogger.info('Totally %s files registered' % counter) return 0
class ReplicaManagerTestCase(unittest.TestCase): """ Base class for the Replica Manager test cases """ def setUp(self): self.dataManager = DataManager() self.fileName = '/tmp/temporaryLocalFile' file = open(self.fileName, 'w') file.write("%s" % time.time()) file.close() def test_putAndRegister(self): print '\n\n#########################################################################\n\n\t\t\tPut and register test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegister/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterReplicate(self): print '\n\n#########################################################################\n\n\t\t\tReplication test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterReplicate/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) replicateRes = self.dataManager.replicateAndRegister( lfn, 'CNAF-DST') #,sourceSE='',destPath='',localCache='') removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replicate was successful self.assert_(replicateRes['OK']) self.assert_(replicateRes['Value'].has_key('Successful')) self.assert_(replicateRes['Value']['Successful'].has_key(lfn)) self.assert_(replicateRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterGetReplicaMetadata(self): print '\n\n#########################################################################\n\n\t\t\tGet metadata test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetReplicaMetadata/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) metadataRes = self.dataManager.getReplicaMetadata(lfn, diracSE) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the metadata query was successful self.assert_(metadataRes['OK']) self.assert_(metadataRes['Value'].has_key('Successful')) self.assert_(metadataRes['Value']['Successful'].has_key(lfn)) self.assert_(metadataRes['Value']['Successful'][lfn]) metadataDict = metadataRes['Value']['Successful'][lfn] self.assert_(metadataDict.has_key('Cached')) self.assert_(metadataDict.has_key('Migrated')) self.assert_(metadataDict.has_key('Size')) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegsiterGetAccessUrl(self): print '\n\n#########################################################################\n\n\t\t\tGet Access Url test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetAccessUrl/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) getAccessUrlRes = self.dataManager.getReplicaAccessUrl(lfn, diracSE) print getAccessUrlRes removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the access url was successful self.assert_(getAccessUrlRes['OK']) self.assert_(getAccessUrlRes['Value'].has_key('Successful')) self.assert_(getAccessUrlRes['Value']['Successful'].has_key(lfn)) self.assert_(getAccessUrlRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterRemoveReplica(self): print '\n\n#########################################################################\n\n\t\t\tRemove replica test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterRemoveReplica/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) removeReplicaRes = self.dataManager.removeReplica(diracSE, lfn) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeReplicaRes['OK']) self.assert_(removeReplicaRes['Value'].has_key('Successful')) self.assert_(removeReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(removeReplicaRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_registerFile(self): lfn = '/lhcb/test/unit-test/ReplicaManager/registerFile/testFile.%s' % time.time( ) physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn fileSize = 10000 storageElementName = 'CERN-RAW' fileGuid = makeGuid() fileTuple = (lfn, physicalFile, fileSize, storageElementName, fileGuid) registerRes = self.dataManager.registerFile(fileTuple) removeCatalogReplicaRes = self.dataManager.removeCatalogReplica( storageElementName, lfn) removeFileRes = self.dataManager.removeFile(lfn) # Check that the file registration was done correctly self.assert_(registerRes['OK']) self.assert_(registerRes['Value'].has_key('Successful')) self.assert_(registerRes['Value']['Successful'].has_key(lfn)) self.assert_(registerRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes['OK']) self.assert_(removeCatalogReplicaRes['Value'].has_key('Successful')) self.assert_( removeCatalogReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value'].has_key('Successful')) self.assert_(removeFileRes['Value']['Successful'].has_key(lfn)) self.assert_(removeFileRes['Value']['Successful'][lfn]) def test_registerReplica(self): print '\n\n#########################################################################\n\n\t\t\tRegister replica test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/registerReplica/testFile.%s' % time.time( ) physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn fileSize = 10000 storageElementName = 'CERN-RAW' fileGuid = makeGuid() fileTuple = (lfn, physicalFile, fileSize, storageElementName, fileGuid) registerRes = self.dataManager.registerFile(fileTuple) seName = 'GRIDKA-RAW' replicaTuple = (lfn, physicalFile, seName) registerReplicaRes = self.dataManager.registerReplica(replicaTuple) removeCatalogReplicaRes1 = self.dataManager.removeCatalogReplica( storageElementName, lfn) removeCatalogReplicaRes2 = self.dataManager.removeCatalogReplica( seName, lfn) removeFileRes = self.dataManager.removeFile(lfn) # Check that the file registration was done correctly self.assert_(registerRes['OK']) self.assert_(registerRes['Value'].has_key('Successful')) self.assert_(registerRes['Value']['Successful'].has_key(lfn)) self.assert_(registerRes['Value']['Successful'][lfn]) # Check that the replica registration was successful self.assert_(registerReplicaRes['OK']) self.assert_(registerReplicaRes['Value'].has_key('Successful')) self.assert_(registerReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(registerReplicaRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes1['OK']) self.assert_(removeCatalogReplicaRes1['Value'].has_key('Successful')) self.assert_( removeCatalogReplicaRes1['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes1['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes2['OK']) self.assert_(removeCatalogReplicaRes2['Value'].has_key('Successful')) self.assert_( removeCatalogReplicaRes2['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes2['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value'].has_key('Successful')) self.assert_(removeFileRes['Value']['Successful'].has_key(lfn)) self.assert_(removeFileRes['Value']['Successful'][lfn]) def test_putAndRegisterGet(self): print '\n\n#########################################################################\n\n\t\t\tGet file test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGet/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) getRes = self.dataManager.getFile(lfn) removeRes = self.dataManager.removeFile(lfn) localFilePath = "%s/%s" % (os.getcwd(), os.path.basename(lfn)) if os.path.exists(localFilePath): os.remove(localFilePath) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(getRes['OK']) self.assert_(getRes['Value'].has_key('Successful')) self.assert_(getRes['Value']['Successful'].has_key(lfn)) self.assertEqual(getRes['Value']['Successful'][lfn], localFilePath) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn])
class DIRACBackend(GridBackend): """Grid backend using the GFAL command line tools `gfal-*`.""" def __init__(self, **kwargs): GridBackend.__init__(self, catalogue_prefix='', **kwargs) from DIRAC.Core.Base import Script Script.initialize() from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient self.pm = ProxyManagerClient() proxy = self.pm.getUserProxiesInfo() if not proxy['OK']: raise BackendException("Proxy error.") from DIRAC.Interfaces.API.Dirac import Dirac self.dirac = Dirac() from DIRAC.Resources.Catalog.FileCatalog import FileCatalog self.fc = FileCatalog() from DIRAC.DataManagementSystem.Client.DataManager import DataManager self.dm = DataManager() self._xattr_cmd = sh.Command('gfal-xattr').bake(_tty_out=False) self._replica_checksum_cmd = sh.Command('gfal-sum').bake(_tty_out=False) self._bringonline_cmd = sh.Command('gfal-legacy-bringonline').bake(_tty_out=False) self._cp_cmd = sh.Command('gfal-copy').bake(_tty_out=False) self._ls_se_cmd = sh.Command('gfal-ls').bake(color='never', _tty_out=False) self._move_cmd = sh.Command('gfal-rename').bake(_tty_out=False) self._mkdir_cmd = sh.Command('gfal-mkdir').bake(_tty_out=False) self._replicate_cmd = sh.Command('dirac-dms-replicate-lfn').bake(_tty_out=False) self._add_cmd = sh.Command('dirac-dms-add-file').bake(_tty_out=False) @staticmethod def _check_return_value(ret): if not ret['OK']: raise BackendException("Failed: %s", ret['Message']) for path, error in ret['Value']['Failed'].items(): if ('No such' in error) or ('Directory does not' in error): raise DoesNotExistException("No such file or directory.") else: raise BackendException(error) def _is_dir(self, lurl): isdir = self.fc.isDirectory(lurl) self._check_return_value(isdir) return isdir['Value']['Successful'][lurl] def _is_file(self, lurl): isfile = self.fc.isFile(lurl) self._check_return_value(isfile) return isfile['Value']['Successful'][lurl] def _get_dir_entry(self, lurl, infodict=None): """Take a lurl and return a DirEntry.""" # If no dctionary with the information is specified, get it from the catalogue try: md = infodict['MetaData'] except TypeError: md = self.fc.getFileMetadata(lurl) if not md['OK']: raise BackendException("Failed to list path '%s': %s", lurl, md['Message']) for path, error in md['Value']['Failed'].items(): if 'No such file' in error: # File does not exist, maybe a directory? md = self.fc.getDirectoryMetadata(lurl) for path, error in md['Value']['Failed'].items(): raise DoesNotExistException("No such file or directory.") else: raise BackendException(md['Value']['Failed'][lurl]) md = md['Value']['Successful'][lurl] return DirEntry(posixpath.basename(lurl), mode=oct(md.get('Mode', -1)), links=md.get('links', -1), gid=md['OwnerGroup'], uid=md['Owner'], size=md.get('Size', -1), modified=str(md.get('ModificationDate', '?'))) def _iter_directory(self, lurl): """Iterate over entries in a directory.""" ret = self.fc.listDirectory(lurl) if not ret['OK']: raise BackendException("Failed to list path '%s': %s", lurl, ret['Message']) for path, error in ret['Value']['Failed'].items(): if 'Directory does not' in error: # Dir does not exist, maybe a File? if self.fc.isFile(lurl): lst = [(lurl, None)] break else: raise DoesNotExistException("No such file or Directory.") else: raise BackendException(ret['Value']['Failed'][lurl]) else: # Sort items by keys, i.e. paths lst = sorted(ret['Value']['Successful'][lurl]['Files'].items() + ret['Value']['Successful'][lurl]['SubDirs'].items()) for item in lst: yield item # = path, dict def _ls(self, lurl, **kwargs): # Translate keyword arguments d = kwargs.pop('directory', False) if d: # Just the requested entry itself yield self._get_dir_entry(lurl) return for path, info in self._iter_directory(lurl): yield self._get_dir_entry(path, info) def _ls_se(self, surl, **kwargs): # Translate keyword arguments d = kwargs.pop('directory', False) args = [] if -d: args.append('-d') args.append('-l') args.append(surl) try: output = self._ls_se_cmd(*args, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or Directory.") else: raise BackendException(e.stderr) for line in output: fields = line.split() mode, links, gid, uid, size = fields[:5] name = fields[-1] modified = ' '.join(fields[5:-1]) yield DirEntry(name, mode=mode, links=int(links), gid=gid, uid=uid, size=int(size), modified=modified) def _replicas(self, lurl, **kwargs): # Check the lurl actually exists self._ls(lurl, directory=True) rep = self.dirac.getReplicas(lurl) self._check_return_value(rep) rep = rep['Value']['Successful'][lurl] return rep.values() def _exists(self, surl, **kwargs): try: ret = self._ls_se_cmd(surl, '-d', '-l', **kwargs).strip() except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: return False else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) else: return ret[0] != 'd' # Return `False` for directories def _register(self, surl, lurl, verbose=False, **kwargs): # Register an existing physical copy in the file catalogue se = storage.get_SE(surl).name # See if file already exists in DFC ret = self.fc.getFileMetadata(lurl) try: self._check_return_value(ret) except DoesNotExistException: # Add new file size = next(self._ls_se(surl, directory=True)).size checksum = self.checksum(surl) guid = str(uuid.uuid4()) # The guid does not seem to be important. Make it unique if possible. ret = self.dm.registerFile((lurl, surl, size, se, guid, checksum)) else: # Add new replica ret = self.dm.registerReplica((lurl, surl, se)) self._check_return_value(ret) if verbose: print_("Successfully registered replica %s of %s from %s."%(surl, lurl, se)) return True def _deregister(self, surl, lurl, verbose=False, **kwargs): # DIRAC only needs to know the SE name to deregister a replica se = storage.get_SE(surl).name ret = self.dm.removeReplicaFromCatalog(se, [lurl]) self._check_return_value(ret) if verbose: print_("Successfully deregistered replica of %s from %s."%(lurl, se)) return True def _state(self, surl, **kwargs): try: state = self._xattr_cmd(surl, 'user.status', **kwargs).strip() except sh.ErrorReturnCode as e: if "No such file" in e.stderr: raise DoesNotExistException("No such file or Directory.") state = '?' except sh.SignalException_SIGSEGV: state = '?' return state def _checksum(self, surl, **kwargs): try: checksum = self._replica_checksum_cmd(surl, 'ADLER32', **kwargs).split()[1] except sh.ErrorReturnCode: checksum = '?' except sh.SignalException_SIGSEGV: checksum = '?' except IndexError: checksum = '?' return checksum def _bringonline(self, surl, timeout, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None # gfal does not notice when files come online, it seems # Just send a single short request, then check regularly if verbose: out = sys.stdout else: out = None end = time.time() + timeout try: self._bringonline_cmd('-t', 10, surl, _out=out, **kwargs) except sh.ErrorReturnCode as e: # The command fails if the file is not online # To be expected after 10 seconds if "No such file" in e.stderr: # Except when the file does not actually exist on the tape storage raise DoesNotExistException("No such file or Directory.") wait = 5 while(True): if verbose: print_("Checking replica state...") if self.is_online(surl): if verbose: print_("Replica brought online.") return True time_left = end - time.time() if time_left <= 0: if verbose: print_("Could not bring replica online.") return False wait *= 2 if time_left < wait: wait = time_left if verbose: print_("Timeout remaining: %d s"%(time_left)) print_("Checking again in: %d s"%(wait)) time.sleep(wait) def _replicate(self, source_surl, destination_surl, lurl, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None source = storage.get_SE(source_surl).name destination = storage.get_SE(destination_surl).name try: self._replicate_cmd(lurl, destination, source, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return True def _get(self, surl, localpath, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None try: self._cp_cmd('-f', '--checksum', 'ADLER32', surl, localpath, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return os.path.isfile(localpath) def _put(self, localpath, surl, lurl, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None se = storage.get_SE(surl).name try: self._add_cmd(lurl, localpath, se, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return True def _remove(self, surl, lurl, last=False, verbose=False, **kwargs): se = storage.get_SE(surl).name if last: # Delete lfn if verbose: print_("Removing all replicas of %s."%(lurl,)) ret = self.dm.removeFile([lurl]) else: if verbose: print_("Removing replica of %s from %s."%(lurl, se)) ret = self.dm.removeReplica(se, [lurl]) if not ret['OK']: raise BackendException('Failed: %s'%(ret['Message'])) for lurl, error in ret['Value']['Failed'].items(): if 'No such file' in error: raise DoesNotExistException("No such file or directory.") else: raise BackendException(error) return True def _rmdir(self, lurl, verbose=False): """Remove the an empty directory from the catalogue.""" rep = self.fc.removeDirectory(lurl) self._check_return_value(rep) return True def _move_replica(self, surl, new_surl, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None try: folder = posixpath.dirname(new_surl) self._mkdir_cmd(folder, '-p', _out=out, **kwargs) self._move_cmd(surl, new_surl, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return True