def main():
dm = DataManager()
fileTupleBuffer = []
counter = 0
for f in files:
counter += 1
if not f.startswith('/cefs'):
gLogger.error('File must be under "/cefs"')
return 1
lfn = '/cepc/lustre-ro' + f
result = fcc.isFile(lfn)
if result['OK'] and lfn in result['Value']['Successful'] and result[
'Value']['Successful'][lfn]:
continue
size = os.path.getsize(f)
adler32 = fileAdler(f)
guid = makeGuid()
fileTuple = (lfn, f, size, _se, guid, adler32)
fileTupleBuffer.append(fileTuple)
gLogger.debug('Register to lfn: %s' % lfn)
gLogger.debug('fileTuple: %s' % (fileTuple, ))
if len(fileTupleBuffer) >= _bufferSize:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
if not result['OK']:
gLogger.error('Register file failed')
return 1
del fileTupleBuffer[:]
gLogger.debug('%s files registered' % counter)
if fileTupleBuffer:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
if not result['OK']:
gLogger.error('Register file failed')
return 1
del fileTupleBuffer[:]
gLogger.info('Totally %s files registered' % counter)
return 0
def __call__(self):
""" call me maybe """
# # counter for failed files
failedFiles = 0
# # catalog to use
catalog = self.operation.Catalog
dm = DataManager(catalogs=catalog)
# # get waiting files
waitingFiles = self.getWaitingFilesList()
# # loop over files
for opFile in waitingFiles:
gMonitor.addMark("RegisterAtt", 1)
# # get LFN
lfn = opFile.LFN
# # and others
fileTuple = (lfn, opFile.PFN, opFile.Size,
self.operation.targetSEList[0], opFile.GUID,
opFile.Checksum)
# # call DataManager
registerFile = dm.registerFile(fileTuple)
# # check results
if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]:
gMonitor.addMark("RegisterFail", 1)
self.dataLoggingClient().addFileRecord(lfn, "RegisterFail",
catalog, "",
"RegisterFile")
reason = registerFile.get(
"Message",
registerFile.get("Value", {}).get("Failed",
{}).get(lfn, 'Unknown'))
errorStr = "failed to register LFN %s: %s" % (lfn, reason)
opFile.Error = errorStr
self.log.warn(errorStr)
failedFiles += 1
else:
gMonitor.addMark("RegisterOK", 1)
self.dataLoggingClient().addFileRecord(lfn, "Register",
catalog, "",
"RegisterFile")
self.log.info("file %s has been registered at %s" %
(lfn, catalog))
opFile.Status = "Done"
# # final check
if failedFiles:
self.log.info("all files processed, %s files failed to register" %
failedFiles)
self.operation.Error = "some files failed to register"
return S_ERROR(self.operation.Error)
return S_OK()
def __call__( self ):
""" call me maybe """
# # counter for failed files
failedFiles = 0
# # catalog(s) to use
catalogs = self.operation.Catalog
if catalogs:
catalogs = [ cat.strip() for cat in catalogs.split( ',' ) ]
dm = DataManager( catalogs = catalogs )
# # get waiting files
waitingFiles = self.getWaitingFilesList()
# # loop over files
for opFile in waitingFiles:
gMonitor.addMark( "RegisterAtt", 1 )
# # get LFN
lfn = opFile.LFN
# # and others
fileTuple = ( lfn , opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum )
# # call DataManager
registerFile = dm.registerFile( fileTuple )
# # check results
if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]:
gMonitor.addMark( "RegisterFail", 1 )
# self.dataLoggingClient().addFileRecord( lfn, "RegisterFail", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )
reason = str( registerFile.get( "Message", registerFile.get( "Value", {} ).get( "Failed", {} ).get( lfn, 'Unknown' ) ) )
errorStr = "failed to register LFN"
opFile.Error = "%s: %s" % ( errorStr, reason )
if 'GUID already registered' in reason:
opFile.Status = 'Failed'
self.log.error( errorStr, "%s: %s" % ( lfn, reason ) )
elif 'File already registered with no replicas' in reason:
self.log.warn( errorStr, "%s: %s, will remove it and retry" % ( lfn, reason ) )
dm.removeFile( lfn )
else:
self.log.warn( errorStr, "%s: %s" % ( lfn, reason ) )
failedFiles += 1
else:
gMonitor.addMark( "RegisterOK", 1 )
# self.dataLoggingClient().addFileRecord( lfn, "Register", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )
self.log.verbose( "file %s has been registered at %s" % ( lfn, ','.join( catalogs ) if catalogs else "all catalogs" ) )
opFile.Status = "Done"
# # final check
if failedFiles:
self.log.warn( "all files processed, %s files failed to register" % failedFiles )
self.operation.Error = "some files failed to register"
return S_ERROR( self.operation.Error )
return S_OK()
def __call__( self ):
""" call me maybe """
# # counter for failed files
failedFiles = 0
# # catalog(s) to use
catalogs = self.operation.Catalog
if catalogs:
catalogs = [ cat.strip() for cat in catalogs.split( ',' ) ]
dm = DataManager( catalogs = catalogs )
# # get waiting files
waitingFiles = self.getWaitingFilesList()
# # loop over files
for opFile in waitingFiles:
gMonitor.addMark( "RegisterAtt", 1 )
# # get LFN
lfn = opFile.LFN
# # and others
fileTuple = ( lfn , opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum )
# # call DataManager
registerFile = dm.registerFile( fileTuple )
# # check results
if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]:
gMonitor.addMark( "RegisterFail", 1 )
# self.dataLoggingClient().addFileRecord( lfn, "RegisterFail", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )
reason = str( registerFile.get( "Message", registerFile.get( "Value", {} ).get( "Failed", {} ).get( lfn, 'Unknown' ) ) )
errorStr = "failed to register LFN"
opFile.Error = "%s: %s" % ( errorStr, reason )
if 'GUID already registered' in reason:
opFile.Status = 'Failed'
self.log.error( errorStr, "%s: %s" % ( lfn, reason ) )
elif 'File already registered with no replicas' in reason:
self.log.warn( errorStr, "%s: %s, will remove it and retry" % ( lfn, reason ) )
dm.removeFile( lfn )
else:
self.log.warn( errorStr, "%s: %s" % ( lfn, reason ) )
failedFiles += 1
else:
gMonitor.addMark( "RegisterOK", 1 )
# self.dataLoggingClient().addFileRecord( lfn, "Register", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )
self.log.verbose( "file %s has been registered at %s" % ( lfn, ','.join( catalogs ) if catalogs else "all catalogs" ) )
opFile.Status = "Done"
# # final check
if failedFiles:
self.log.warn( "all files processed, %s files failed to register" % failedFiles )
self.operation.Error = "some files failed to register"
return S_ERROR( self.operation.Error )
return S_OK()
def __call__( self ):
""" call me maybe """
# # counter for failed files
failedFiles = 0
# # catalog to use
catalog = self.operation.Catalog
dm = DataManager( catalogs = catalog )
# # get waiting files
waitingFiles = self.getWaitingFilesList()
# # loop over files
for opFile in waitingFiles:
gMonitor.addMark( "RegisterAtt", 1 )
# # get LFN
lfn = opFile.LFN
# # and others
fileTuple = ( lfn , opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum )
# # call DataManager
registerFile = dm.registerFile( fileTuple )
# # check results
if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]:
gMonitor.addMark( "RegisterFail", 1 )
self.dataLoggingClient().addFileRecord( lfn, "RegisterFail", catalog, "", "RegisterFile" )
reason = registerFile.get( "Message", registerFile.get( "Value", {} ).get( "Failed", {} ).get( lfn, 'Unknown' ) )
errorStr = "failed to register LFN %s: %s" % ( lfn, reason )
opFile.Error = errorStr
self.log.warn( errorStr )
failedFiles += 1
else:
gMonitor.addMark( "RegisterOK", 1 )
self.dataLoggingClient().addFileRecord( lfn, "Register", catalog, "", "RegisterFile" )
self.log.info( "file %s has been registered at %s" % ( lfn, catalog ) )
opFile.Status = "Done"
# # final check
if failedFiles:
self.log.info( "all files processed, %s files failed to register" % failedFiles )
self.operation.Error = "some files failed to register"
return S_ERROR( self.operation.Error )
return S_OK()
if result['OK'] and lfn in result['Value'][
'Successful'] and result['Value']['Successful'][lfn]:
if counter % 1000 == 0:
gLogger.notice('Skip file existed counter: %s' % counter)
continue
size = os.path.getsize(fullFn)
adler32 = fileAdler(fullFn)
guid = makeGuid()
fileTuple = (lfn, fullFn, size, toSE, guid, adler32)
fileTupleBuffer.append(fileTuple)
gLogger.debug('Register to lfn: %s' % lfn)
gLogger.debug('fileTuple: %s' % (fileTuple, ))
if len(fileTupleBuffer) >= bufferSize:
result = dm.registerFile(fileTupleBuffer)
if not result['OK']:
gLogger.error('Can not register %s' % fullFn)
exit(1)
del fileTupleBuffer[:]
gLogger.notice('%s files registered' % counter)
if fileTupleBuffer:
result = dm.registerFile(fileTupleBuffer)
if not result['OK']:
gLogger.error('Can not register %s' % fullFn)
exit(1)
del fileTupleBuffer[:]
gLogger.notice('Total %s files registered' % counter)
class ReplicaManagerTestCase(unittest.TestCase):
""" Base class for the Replica Manager test cases
"""
def setUp(self):
self.dataManager = DataManager()
self.fileName = '/tmp/temporaryLocalFile'
file = open(self.fileName,'w')
file.write("%s" % time.time())
file.close()
def test_putAndRegister(self):
print '\n\n#########################################################################\n\n\t\t\tPut and register test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegister/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterReplicate(self):
print '\n\n#########################################################################\n\n\t\t\tReplication test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterReplicate/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
replicateRes = self.dataManager.replicateAndRegister(lfn,'CNAF-DST') #,sourceSE='',destPath='',localCache='')
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replicate was successful
self.assert_(replicateRes['OK'])
self.assert_(replicateRes['Value'].has_key('Successful'))
self.assert_(replicateRes['Value']['Successful'].has_key(lfn))
self.assert_(replicateRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterGetReplicaMetadata(self):
print '\n\n#########################################################################\n\n\t\t\tGet metadata test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetReplicaMetadata/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
metadataRes = self.dataManager.getReplicaMetadata(lfn,diracSE)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the metadata query was successful
self.assert_(metadataRes['OK'])
self.assert_(metadataRes['Value'].has_key('Successful'))
self.assert_(metadataRes['Value']['Successful'].has_key(lfn))
self.assert_(metadataRes['Value']['Successful'][lfn])
metadataDict = metadataRes['Value']['Successful'][lfn]
self.assert_(metadataDict.has_key('Cached'))
self.assert_(metadataDict.has_key('Migrated'))
self.assert_(metadataDict.has_key('Size'))
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegsiterGetAccessUrl(self):
print '\n\n#########################################################################\n\n\t\t\tGet Access Url test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetAccessUrl/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
getAccessUrlRes = self.dataManager.getReplicaAccessUrl(lfn,diracSE)
print getAccessUrlRes
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the access url was successful
self.assert_(getAccessUrlRes['OK'])
self.assert_(getAccessUrlRes['Value'].has_key('Successful'))
self.assert_(getAccessUrlRes['Value']['Successful'].has_key(lfn))
self.assert_(getAccessUrlRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterRemoveReplica(self):
print '\n\n#########################################################################\n\n\t\t\tRemove replica test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterRemoveReplica/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
removeReplicaRes = self.dataManager.removeReplica(diracSE,lfn)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeReplicaRes['OK'])
self.assert_(removeReplicaRes['Value'].has_key('Successful'))
self.assert_(removeReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(removeReplicaRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_registerFile(self):
lfn = '/lhcb/test/unit-test/ReplicaManager/registerFile/testFile.%s' % time.time()
physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn
fileSize = 10000
storageElementName = 'CERN-RAW'
fileGuid = makeGuid()
fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid)
registerRes = self.dataManager.registerFile(fileTuple)
removeCatalogReplicaRes = self.dataManager.removeCatalogReplica(storageElementName,lfn)
removeFileRes = self.dataManager.removeFile(lfn)
# Check that the file registration was done correctly
self.assert_(registerRes['OK'])
self.assert_(registerRes['Value'].has_key('Successful'))
self.assert_(registerRes['Value']['Successful'].has_key(lfn))
self.assert_(registerRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes['OK'])
self.assert_(removeCatalogReplicaRes['Value'].has_key('Successful'))
self.assert_(removeCatalogReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value'].has_key('Successful'))
self.assert_(removeFileRes['Value']['Successful'].has_key(lfn))
self.assert_(removeFileRes['Value']['Successful'][lfn])
def test_registerReplica(self):
print '\n\n#########################################################################\n\n\t\t\tRegister replica test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/registerReplica/testFile.%s' % time.time()
physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn
fileSize = 10000
storageElementName = 'CERN-RAW'
fileGuid = makeGuid()
fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid)
registerRes = self.dataManager.registerFile(fileTuple)
seName = 'GRIDKA-RAW'
replicaTuple = (lfn,physicalFile,seName)
registerReplicaRes = self.dataManager.registerReplica(replicaTuple)
removeCatalogReplicaRes1 = self.dataManager.removeCatalogReplica(storageElementName,lfn)
removeCatalogReplicaRes2 = self.dataManager.removeCatalogReplica(seName,lfn)
removeFileRes = self.dataManager.removeFile(lfn)
# Check that the file registration was done correctly
self.assert_(registerRes['OK'])
self.assert_(registerRes['Value'].has_key('Successful'))
self.assert_(registerRes['Value']['Successful'].has_key(lfn))
self.assert_(registerRes['Value']['Successful'][lfn])
# Check that the replica registration was successful
self.assert_(registerReplicaRes['OK'])
self.assert_(registerReplicaRes['Value'].has_key('Successful'))
self.assert_(registerReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(registerReplicaRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes1['OK'])
self.assert_(removeCatalogReplicaRes1['Value'].has_key('Successful'))
self.assert_(removeCatalogReplicaRes1['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes1['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes2['OK'])
self.assert_(removeCatalogReplicaRes2['Value'].has_key('Successful'))
self.assert_(removeCatalogReplicaRes2['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes2['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value'].has_key('Successful'))
self.assert_(removeFileRes['Value']['Successful'].has_key(lfn))
self.assert_(removeFileRes['Value']['Successful'][lfn])
def test_putAndRegisterGet(self):
print '\n\n#########################################################################\n\n\t\t\tGet file test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGet/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
getRes = self.dataManager.getFile(lfn)
removeRes = self.dataManager.removeFile(lfn)
localFilePath = "%s/%s" % (os.getcwd(),os.path.basename(lfn))
if os.path.exists(localFilePath):
os.remove(localFilePath)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(getRes['OK'])
self.assert_(getRes['Value'].has_key('Successful'))
self.assert_(getRes['Value']['Successful'].has_key(lfn))
self.assertEqual(getRes['Value']['Successful'][lfn],localFilePath)
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
)
for i in fileUnlinkedList:
gLogger.notice(i)
gLogger.notice("---------- Total: %d unlinked files. ----------\n" %
len(fileUnlinkedList))
if deletionSwitch:
for i in fileDarkList:
g.unlink(HEAD0 + '/' + i)
gLogger.notice("%s deleted!" % (HEAD0 + '/' + i))
gLogger.notice("---------- All dark files deleted! ----------\n")
if registerOption != 'none':
if registerOption in ['dark', 'both']:
gLogger.notice("---------- Register dark files: ----------")
for i in fileDarkList:
fileStat = g.stat(HEAD0 + '/' + i)
fileSize = fileStat.st_size
fileGuid = makeGuid()
fileSum = g.checksum(HEAD0 + '/' + i, "ADLER32")
fileTuple = (i, HEAD0 + '/' + i, fileSize, SITE, fileGuid, fileSum)
dm.registerFile(fileTuple)
gLogger.notice(i + ' registered successfully!')
if registerOption in ['unlinked', 'both']:
gLogger.notice("---------- Register unlinked files: ----------")
for i in fileUnlinkedList:
fileTuple = (i, HEAD0 + '/' + i, SITE)
dm.registerReplica(fileTuple)
gLogger.notice(i + ' registered successfully!')
gLogger.notice("---------- Register finished ----------\n")
def __call__(self):
""" call me maybe """
# The flag 'rmsMonitoring' is set by the RequestTask and is False by default.
# Here we use 'createRMSRecord' to create the ES record which is defined inside OperationHandlerBase.
if self.rmsMonitoring:
self.rmsMonitoringReporter = MonitoringReporter(
monitoringType="RMSMonitoring")
else:
# # RegisterFile specific monitor info
gMonitor.registerActivity("RegisterAtt",
"Attempted file registrations",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterOK",
"Successful file registrations",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterFail",
"Failed file registrations",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
# # counter for failed files
failedFiles = 0
# # catalog(s) to use
catalogs = self.operation.Catalog
if catalogs:
catalogs = [cat.strip() for cat in catalogs.split(',')]
dm = DataManager(catalogs=catalogs)
# # get waiting files
waitingFiles = self.getWaitingFilesList()
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord(
self.createRMSRecord("Attempted", len(waitingFiles)))
# # loop over files
for opFile in waitingFiles:
if not self.rmsMonitoring:
gMonitor.addMark("RegisterAtt", 1)
# # get LFN
lfn = opFile.LFN
# # and others
fileTuple = (lfn, opFile.PFN, opFile.Size,
self.operation.targetSEList[0], opFile.GUID,
opFile.Checksum)
# # call DataManager
registerFile = dm.registerFile(fileTuple)
# # check results
if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]:
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord(
self.createRMSRecord("Failed", 1))
else:
gMonitor.addMark("RegisterFail", 1)
# self.dataLoggingClient().addFileRecord(
# lfn, "RegisterFail", ','.join(catalogs) if catalogs else "all catalogs", "", "RegisterFile")
reason = str(
registerFile.get(
"Message",
registerFile.get("Value",
{}).get("Failed",
{}).get(lfn, 'Unknown')))
errorStr = "failed to register LFN"
opFile.Error = "%s: %s" % (errorStr, reason)
if 'GUID already registered' in reason:
opFile.Status = 'Failed'
self.log.error(errorStr, "%s: %s" % (lfn, reason))
elif 'File already registered with no replicas' in reason:
self.log.warn(
errorStr,
"%s: %s, will remove it and retry" % (lfn, reason))
dm.removeFile(lfn)
else:
self.log.warn(errorStr, "%s: %s" % (lfn, reason))
failedFiles += 1
else:
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord(
self.createRMSRecord("Successful", 1))
else:
gMonitor.addMark("RegisterOK", 1)
# self.dataLoggingClient().addFileRecord(
# lfn, "Register", ','.join(catalogs) if catalogs else "all catalogs", "", "RegisterFile")
self.log.verbose(
"file %s has been registered at %s" %
(lfn, ','.join(catalogs) if catalogs else "all catalogs"))
opFile.Status = "Done"
if self.rmsMonitoring:
self.rmsMonitoringReporter.commit()
# # final check
if failedFiles:
self.log.warn("all files processed, %s files failed to register" %
failedFiles)
self.operation.Error = "some files failed to register"
return S_ERROR(self.operation.Error)
return S_OK()
def main():
dm = DataManager()
fileTupleBuffer = []
res = getProxyInfo( False, False )
if not res['OK']:
gLogger.error( "Failed to get client proxy information.", res['Message'] )
DIRAC.exit( 2 )
proxyInfo = res['Value']
if proxyInfo['secondsLeft'] == 0:
gLogger.error( "Proxy expired" )
DIRAC.exit( 2 )
username = proxyInfo['username']
vo = ''
if 'group' in proxyInfo:
vo = getVOForGroup( proxyInfo['group'] )
counter = 0
for f in files:
counter += 1
local_f=f
if not f.startswith('/cefs'):
# gLogger.error('File must be under "/cefs"')
# continue
#if the file to reg is not under /cefs, use put and register
folder_name=os.path.basename(os.path.dirname(f))
# lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f)
lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f)
# dirname = os.path.dirname(local_f)
# os.system('mkdir -p %s'%(dirname))
# os.system('cp %s %s' %(f,local_f))
do_put_and_register=True
else:
lfn = '/cepc/lustre-ro' + os.path.abspath(f)
do_put_and_register=False
result = fcc.isFile(lfn)
if result['OK'] and lfn in result['Value']['Successful'] and result['Value']['Successful'][lfn]:
continue
size = os.path.getsize(f)
adler32 = fileAdler(f)
guid = makeGuid()
fileTuple = (lfn, local_f, size, _se, guid, adler32)
fileTupleBuffer.append(fileTuple)
gLogger.debug('Register to lfn: %s' % lfn)
gLogger.debug('fileTuple: %s' % (fileTuple,))
if len(fileTupleBuffer) >= _bufferSize:
if do_put_and_register:
result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite)
else:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
# if not result['OK']:
# gLogger.error('Register file failed')
# return 1
del fileTupleBuffer[:]
gLogger.debug('%s files registered' % counter)
if fileTupleBuffer:
if do_put_and_register:
result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite)
else:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
# if not result['OK']:
# gLogger.error('Register file failed')
# return 1
del fileTupleBuffer[:]
gLogger.info('Totally %s files registered' % counter)
return 0
class ReplicaManagerTestCase(unittest.TestCase):
""" Base class for the Replica Manager test cases
"""
def setUp(self):
self.dataManager = DataManager()
self.fileName = '/tmp/temporaryLocalFile'
file = open(self.fileName, 'w')
file.write("%s" % time.time())
file.close()
def test_putAndRegister(self):
print '\n\n#########################################################################\n\n\t\t\tPut and register test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegister/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterReplicate(self):
print '\n\n#########################################################################\n\n\t\t\tReplication test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterReplicate/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
replicateRes = self.dataManager.replicateAndRegister(
lfn, 'CNAF-DST') #,sourceSE='',destPath='',localCache='')
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replicate was successful
self.assert_(replicateRes['OK'])
self.assert_(replicateRes['Value'].has_key('Successful'))
self.assert_(replicateRes['Value']['Successful'].has_key(lfn))
self.assert_(replicateRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterGetReplicaMetadata(self):
print '\n\n#########################################################################\n\n\t\t\tGet metadata test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetReplicaMetadata/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
metadataRes = self.dataManager.getReplicaMetadata(lfn, diracSE)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the metadata query was successful
self.assert_(metadataRes['OK'])
self.assert_(metadataRes['Value'].has_key('Successful'))
self.assert_(metadataRes['Value']['Successful'].has_key(lfn))
self.assert_(metadataRes['Value']['Successful'][lfn])
metadataDict = metadataRes['Value']['Successful'][lfn]
self.assert_(metadataDict.has_key('Cached'))
self.assert_(metadataDict.has_key('Migrated'))
self.assert_(metadataDict.has_key('Size'))
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegsiterGetAccessUrl(self):
print '\n\n#########################################################################\n\n\t\t\tGet Access Url test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetAccessUrl/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
getAccessUrlRes = self.dataManager.getReplicaAccessUrl(lfn, diracSE)
print getAccessUrlRes
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the access url was successful
self.assert_(getAccessUrlRes['OK'])
self.assert_(getAccessUrlRes['Value'].has_key('Successful'))
self.assert_(getAccessUrlRes['Value']['Successful'].has_key(lfn))
self.assert_(getAccessUrlRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterRemoveReplica(self):
print '\n\n#########################################################################\n\n\t\t\tRemove replica test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterRemoveReplica/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
removeReplicaRes = self.dataManager.removeReplica(diracSE, lfn)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeReplicaRes['OK'])
self.assert_(removeReplicaRes['Value'].has_key('Successful'))
self.assert_(removeReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(removeReplicaRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_registerFile(self):
lfn = '/lhcb/test/unit-test/ReplicaManager/registerFile/testFile.%s' % time.time(
)
physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn
fileSize = 10000
storageElementName = 'CERN-RAW'
fileGuid = makeGuid()
fileTuple = (lfn, physicalFile, fileSize, storageElementName, fileGuid)
registerRes = self.dataManager.registerFile(fileTuple)
removeCatalogReplicaRes = self.dataManager.removeCatalogReplica(
storageElementName, lfn)
removeFileRes = self.dataManager.removeFile(lfn)
# Check that the file registration was done correctly
self.assert_(registerRes['OK'])
self.assert_(registerRes['Value'].has_key('Successful'))
self.assert_(registerRes['Value']['Successful'].has_key(lfn))
self.assert_(registerRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes['OK'])
self.assert_(removeCatalogReplicaRes['Value'].has_key('Successful'))
self.assert_(
removeCatalogReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value'].has_key('Successful'))
self.assert_(removeFileRes['Value']['Successful'].has_key(lfn))
self.assert_(removeFileRes['Value']['Successful'][lfn])
def test_registerReplica(self):
print '\n\n#########################################################################\n\n\t\t\tRegister replica test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/registerReplica/testFile.%s' % time.time(
)
physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn
fileSize = 10000
storageElementName = 'CERN-RAW'
fileGuid = makeGuid()
fileTuple = (lfn, physicalFile, fileSize, storageElementName, fileGuid)
registerRes = self.dataManager.registerFile(fileTuple)
seName = 'GRIDKA-RAW'
replicaTuple = (lfn, physicalFile, seName)
registerReplicaRes = self.dataManager.registerReplica(replicaTuple)
removeCatalogReplicaRes1 = self.dataManager.removeCatalogReplica(
storageElementName, lfn)
removeCatalogReplicaRes2 = self.dataManager.removeCatalogReplica(
seName, lfn)
removeFileRes = self.dataManager.removeFile(lfn)
# Check that the file registration was done correctly
self.assert_(registerRes['OK'])
self.assert_(registerRes['Value'].has_key('Successful'))
self.assert_(registerRes['Value']['Successful'].has_key(lfn))
self.assert_(registerRes['Value']['Successful'][lfn])
# Check that the replica registration was successful
self.assert_(registerReplicaRes['OK'])
self.assert_(registerReplicaRes['Value'].has_key('Successful'))
self.assert_(registerReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(registerReplicaRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes1['OK'])
self.assert_(removeCatalogReplicaRes1['Value'].has_key('Successful'))
self.assert_(
removeCatalogReplicaRes1['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes1['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes2['OK'])
self.assert_(removeCatalogReplicaRes2['Value'].has_key('Successful'))
self.assert_(
removeCatalogReplicaRes2['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes2['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value'].has_key('Successful'))
self.assert_(removeFileRes['Value']['Successful'].has_key(lfn))
self.assert_(removeFileRes['Value']['Successful'][lfn])
def test_putAndRegisterGet(self):
print '\n\n#########################################################################\n\n\t\t\tGet file test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGet/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
getRes = self.dataManager.getFile(lfn)
removeRes = self.dataManager.removeFile(lfn)
localFilePath = "%s/%s" % (os.getcwd(), os.path.basename(lfn))
if os.path.exists(localFilePath):
os.remove(localFilePath)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(getRes['OK'])
self.assert_(getRes['Value'].has_key('Successful'))
self.assert_(getRes['Value']['Successful'].has_key(lfn))
self.assertEqual(getRes['Value']['Successful'][lfn], localFilePath)
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
class DIRACBackend(GridBackend):
"""Grid backend using the GFAL command line tools `gfal-*`."""
def __init__(self, **kwargs):
GridBackend.__init__(self, catalogue_prefix='', **kwargs)
from DIRAC.Core.Base import Script
Script.initialize()
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient
self.pm = ProxyManagerClient()
proxy = self.pm.getUserProxiesInfo()
if not proxy['OK']:
raise BackendException("Proxy error.")
from DIRAC.Interfaces.API.Dirac import Dirac
self.dirac = Dirac()
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
self.fc = FileCatalog()
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
self.dm = DataManager()
self._xattr_cmd = sh.Command('gfal-xattr').bake(_tty_out=False)
self._replica_checksum_cmd = sh.Command('gfal-sum').bake(_tty_out=False)
self._bringonline_cmd = sh.Command('gfal-legacy-bringonline').bake(_tty_out=False)
self._cp_cmd = sh.Command('gfal-copy').bake(_tty_out=False)
self._ls_se_cmd = sh.Command('gfal-ls').bake(color='never', _tty_out=False)
self._move_cmd = sh.Command('gfal-rename').bake(_tty_out=False)
self._mkdir_cmd = sh.Command('gfal-mkdir').bake(_tty_out=False)
self._replicate_cmd = sh.Command('dirac-dms-replicate-lfn').bake(_tty_out=False)
self._add_cmd = sh.Command('dirac-dms-add-file').bake(_tty_out=False)
@staticmethod
def _check_return_value(ret):
if not ret['OK']:
raise BackendException("Failed: %s", ret['Message'])
for path, error in ret['Value']['Failed'].items():
if ('No such' in error) or ('Directory does not' in error):
raise DoesNotExistException("No such file or directory.")
else:
raise BackendException(error)
def _is_dir(self, lurl):
isdir = self.fc.isDirectory(lurl)
self._check_return_value(isdir)
return isdir['Value']['Successful'][lurl]
def _is_file(self, lurl):
isfile = self.fc.isFile(lurl)
self._check_return_value(isfile)
return isfile['Value']['Successful'][lurl]
def _get_dir_entry(self, lurl, infodict=None):
"""Take a lurl and return a DirEntry."""
# If no dctionary with the information is specified, get it from the catalogue
try:
md = infodict['MetaData']
except TypeError:
md = self.fc.getFileMetadata(lurl)
if not md['OK']:
raise BackendException("Failed to list path '%s': %s", lurl, md['Message'])
for path, error in md['Value']['Failed'].items():
if 'No such file' in error:
# File does not exist, maybe a directory?
md = self.fc.getDirectoryMetadata(lurl)
for path, error in md['Value']['Failed'].items():
raise DoesNotExistException("No such file or directory.")
else:
raise BackendException(md['Value']['Failed'][lurl])
md = md['Value']['Successful'][lurl]
return DirEntry(posixpath.basename(lurl), mode=oct(md.get('Mode', -1)), links=md.get('links', -1), gid=md['OwnerGroup'], uid=md['Owner'], size=md.get('Size', -1), modified=str(md.get('ModificationDate', '?')))
def _iter_directory(self, lurl):
"""Iterate over entries in a directory."""
ret = self.fc.listDirectory(lurl)
if not ret['OK']:
raise BackendException("Failed to list path '%s': %s", lurl, ret['Message'])
for path, error in ret['Value']['Failed'].items():
if 'Directory does not' in error:
# Dir does not exist, maybe a File?
if self.fc.isFile(lurl):
lst = [(lurl, None)]
break
else:
raise DoesNotExistException("No such file or Directory.")
else:
raise BackendException(ret['Value']['Failed'][lurl])
else:
# Sort items by keys, i.e. paths
lst = sorted(ret['Value']['Successful'][lurl]['Files'].items() + ret['Value']['Successful'][lurl]['SubDirs'].items())
for item in lst:
yield item # = path, dict
def _ls(self, lurl, **kwargs):
# Translate keyword arguments
d = kwargs.pop('directory', False)
if d:
# Just the requested entry itself
yield self._get_dir_entry(lurl)
return
for path, info in self._iter_directory(lurl):
yield self._get_dir_entry(path, info)
def _ls_se(self, surl, **kwargs):
# Translate keyword arguments
d = kwargs.pop('directory', False)
args = []
if -d:
args.append('-d')
args.append('-l')
args.append(surl)
try:
output = self._ls_se_cmd(*args, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or Directory.")
else:
raise BackendException(e.stderr)
for line in output:
fields = line.split()
mode, links, gid, uid, size = fields[:5]
name = fields[-1]
modified = ' '.join(fields[5:-1])
yield DirEntry(name, mode=mode, links=int(links), gid=gid, uid=uid, size=int(size), modified=modified)
def _replicas(self, lurl, **kwargs):
# Check the lurl actually exists
self._ls(lurl, directory=True)
rep = self.dirac.getReplicas(lurl)
self._check_return_value(rep)
rep = rep['Value']['Successful'][lurl]
return rep.values()
def _exists(self, surl, **kwargs):
try:
ret = self._ls_se_cmd(surl, '-d', '-l', **kwargs).strip()
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
return False
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
else:
return ret[0] != 'd' # Return `False` for directories
def _register(self, surl, lurl, verbose=False, **kwargs):
# Register an existing physical copy in the file catalogue
se = storage.get_SE(surl).name
# See if file already exists in DFC
ret = self.fc.getFileMetadata(lurl)
try:
self._check_return_value(ret)
except DoesNotExistException:
# Add new file
size = next(self._ls_se(surl, directory=True)).size
checksum = self.checksum(surl)
guid = str(uuid.uuid4()) # The guid does not seem to be important. Make it unique if possible.
ret = self.dm.registerFile((lurl, surl, size, se, guid, checksum))
else:
# Add new replica
ret = self.dm.registerReplica((lurl, surl, se))
self._check_return_value(ret)
if verbose:
print_("Successfully registered replica %s of %s from %s."%(surl, lurl, se))
return True
def _deregister(self, surl, lurl, verbose=False, **kwargs):
# DIRAC only needs to know the SE name to deregister a replica
se = storage.get_SE(surl).name
ret = self.dm.removeReplicaFromCatalog(se, [lurl])
self._check_return_value(ret)
if verbose:
print_("Successfully deregistered replica of %s from %s."%(lurl, se))
return True
def _state(self, surl, **kwargs):
try:
state = self._xattr_cmd(surl, 'user.status', **kwargs).strip()
except sh.ErrorReturnCode as e:
if "No such file" in e.stderr:
raise DoesNotExistException("No such file or Directory.")
state = '?'
except sh.SignalException_SIGSEGV:
state = '?'
return state
def _checksum(self, surl, **kwargs):
try:
checksum = self._replica_checksum_cmd(surl, 'ADLER32', **kwargs).split()[1]
except sh.ErrorReturnCode:
checksum = '?'
except sh.SignalException_SIGSEGV:
checksum = '?'
except IndexError:
checksum = '?'
return checksum
def _bringonline(self, surl, timeout, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
# gfal does not notice when files come online, it seems
# Just send a single short request, then check regularly
if verbose:
out = sys.stdout
else:
out = None
end = time.time() + timeout
try:
self._bringonline_cmd('-t', 10, surl, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
# The command fails if the file is not online
# To be expected after 10 seconds
if "No such file" in e.stderr:
# Except when the file does not actually exist on the tape storage
raise DoesNotExistException("No such file or Directory.")
wait = 5
while(True):
if verbose:
print_("Checking replica state...")
if self.is_online(surl):
if verbose:
print_("Replica brought online.")
return True
time_left = end - time.time()
if time_left <= 0:
if verbose:
print_("Could not bring replica online.")
return False
wait *= 2
if time_left < wait:
wait = time_left
if verbose:
print_("Timeout remaining: %d s"%(time_left))
print_("Checking again in: %d s"%(wait))
time.sleep(wait)
def _replicate(self, source_surl, destination_surl, lurl, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
source = storage.get_SE(source_surl).name
destination = storage.get_SE(destination_surl).name
try:
self._replicate_cmd(lurl, destination, source, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return True
def _get(self, surl, localpath, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
try:
self._cp_cmd('-f', '--checksum', 'ADLER32', surl, localpath, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return os.path.isfile(localpath)
def _put(self, localpath, surl, lurl, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
se = storage.get_SE(surl).name
try:
self._add_cmd(lurl, localpath, se, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return True
def _remove(self, surl, lurl, last=False, verbose=False, **kwargs):
se = storage.get_SE(surl).name
if last:
# Delete lfn
if verbose:
print_("Removing all replicas of %s."%(lurl,))
ret = self.dm.removeFile([lurl])
else:
if verbose:
print_("Removing replica of %s from %s."%(lurl, se))
ret = self.dm.removeReplica(se, [lurl])
if not ret['OK']:
raise BackendException('Failed: %s'%(ret['Message']))
for lurl, error in ret['Value']['Failed'].items():
if 'No such file' in error:
raise DoesNotExistException("No such file or directory.")
else:
raise BackendException(error)
return True
def _rmdir(self, lurl, verbose=False):
"""Remove the an empty directory from the catalogue."""
rep = self.fc.removeDirectory(lurl)
self._check_return_value(rep)
return True
def _move_replica(self, surl, new_surl, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
try:
folder = posixpath.dirname(new_surl)
self._mkdir_cmd(folder, '-p', _out=out, **kwargs)
self._move_cmd(surl, new_surl, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return True
