def __setFileReplicationRequest( self, lfn, targetSE, fileMetaDict, sourceSE = '' ):
""" Sets a registration request.
"""
self.log.info( 'Setting replication request for %s to %s' % ( lfn, targetSE ) )
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.TargetSE = targetSE
if sourceSE:
transfer.SourceSE = sourceSE
trFile = File()
trFile.LFN = lfn
cksm = fileMetaDict.get( "Checksum", None )
cksmType = fileMetaDict.get( "ChecksumType", None )
if cksm and cksmType:
trFile.Checksum = cksm
trFile.ChecksumType = cksmType
size = fileMetaDict.get( "Size", 0 )
if size:
trFile.Size = size
guid = fileMetaDict.get( "GUID", "" )
if guid:
trFile.GUID = guid
transfer.addFile( trFile )
self.request.addOperation( transfer )
return S_OK()
def getRegisterOperation(self,
opFile,
targetSE,
type='RegisterFile',
catalog=None):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = StorageElement(targetSE).getPfnForLfn(
opFile.LFN).get('Value', {}).get('Successful', {}).get(opFile.LFN)
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile(registerFile)
return registerOperation
def __setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE
:param list catalog: list of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get( "Checksum", "" )
regFile.ChecksumType = fileDict.get( "ChecksumType", "" )
regFile.Size = fileDict.get( "Size", 0 )
regFile.GUID = fileDict.get( "GUID", "" )
se = StorageElement( targetSE )
pfn = se.getPfnForLfn( lfn )
if not pfn["OK"]:
self.log.error( "unable to get PFN for LFN: %s" % pfn["Message"] )
return pfn
regFile.PFN = pfn["Value"]
register.addFile( regFile )
self.request.addOperation( register )
return S_OK()
def setBKRegistrationRequest(self, lfn, error='',
metaData={'Checksum': 'justSomething',
'ChecksumType': 'ADLER32',
'GUID': 'aGUID'}):
""" Set a BK registration request for changing the replica flag.
Uses the global request object (self.request).
"""
if error:
self.log.info('BK registration for %s failed with message: "%s" setting failover request' % (lfn, error))
else:
self.log.info('Setting BK registration request for %s' % (lfn))
regFile = Operation()
regFile.Type = 'RegisterFile'
regFile.Catalog = 'BookkeepingDB'
bkFile = File()
bkFile.LFN = lfn
# this should NOT be needed... but RMS complains!
bkFile.PFN = lfn
bkFile.GUID = metaData['GUID']
bkFile.Checksum = metaData['Checksum']
bkFile.ChecksumType = metaData['ChecksumType']
regFile.addFile(bkFile)
res = self.request.addOperation(regFile)
if not res['OK']:
raise RuntimeError(res['Message'])
def getRegisterOperation(self, opFile, targetSE, type="RegisterFile", catalog=None):
"""add RegisterReplica operation for file
:param ~DIRAC.RequestManagementSystem.Client.File.File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = (
StorageElement(targetSE)
.getURL(opFile.LFN, protocol=self.registrationProtocols)
.get("Value", {})
.get("Successful", {})
.get(opFile.LFN)
)
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile(registerFile)
return registerOperation
def _setFileReplicationRequest(self,
lfn,
targetSE,
fileMetaDict,
sourceSE=""):
"""Sets a registration request."""
self.log.info("Setting ReplicateAndRegister request",
"for %s to %s" % (lfn, targetSE))
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.TargetSE = targetSE
if sourceSE:
transfer.SourceSE = sourceSE
trFile = File()
trFile.LFN = lfn
cksm = fileMetaDict.get("Checksum", None)
cksmType = fileMetaDict.get("ChecksumType", self.defaultChecksumType)
if cksm and cksmType:
trFile.Checksum = cksm
trFile.ChecksumType = cksmType
size = fileMetaDict.get("Size", 0)
if size:
trFile.Size = size
guid = fileMetaDict.get("GUID", "")
if guid:
trFile.GUID = guid
transfer.addFile(trFile)
self.request.addOperation(transfer)
return S_OK()
def buildRequest( self, owner, group, sourceSE, targetSE1, targetSE2 ):
files = self.files( owner, group )
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
putFile = File()
putFile.LFN = lfn
putFile.PFN = fname
putFile.Checksum = checksum
putFile.ChecksumType = "adler32"
putFile.Size = size
putFile.GUID = guid
putAndRegister.addFile( putFile )
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = "%s,%s" % ( targetSE1, targetSE2 )
for fname, lfn, size, checksum, guid in files:
repFile = File()
repFile.LFN = lfn
repFile.Size = size
repFile.Checksum = checksum
repFile.ChecksumType = "adler32"
replicateAndRegister.addFile( repFile )
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
removeReplica.addFile( File( {"LFN": lfn } ) )
removeFile = Operation()
removeFile.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFile.addFile( File( {"LFN": lfn } ) )
removeFileInit = Operation()
removeFileInit.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFileInit.addFile( File( {"LFN": lfn } ) )
req = Request()
req.addOperation( removeFileInit )
req.addOperation( putAndRegister )
req.addOperation( replicateAndRegister )
req.addOperation( removeReplica )
req.addOperation( removeFile )
return req
def buildRequest(self, owner, group, sourceSE, targetSE1, targetSE2):
files = self.files(owner, group)
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
putFile = File()
putFile.LFN = lfn
putFile.PFN = fname
putFile.Checksum = checksum
putFile.ChecksumType = "adler32"
putFile.Size = size
putFile.GUID = guid
putAndRegister.addFile(putFile)
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = "%s,%s" % (targetSE1, targetSE2)
for fname, lfn, size, checksum, guid in files:
repFile = File()
repFile.LFN = lfn
repFile.Size = size
repFile.Checksum = checksum
repFile.ChecksumType = "adler32"
replicateAndRegister.addFile(repFile)
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
removeReplica.addFile(File({"LFN": lfn}))
removeFile = Operation()
removeFile.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFile.addFile(File({"LFN": lfn}))
removeFileInit = Operation()
removeFileInit.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFileInit.addFile(File({"LFN": lfn}))
req = Request()
req.addOperation(removeFileInit)
req.addOperation(putAndRegister)
req.addOperation(replicateAndRegister)
req.addOperation(removeReplica)
req.addOperation(removeFile)
return req
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info('Setting registration request for %s at %s.' %
(lfn, targetSE))
if not type(catalog) == type([]):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get("Checksum", "")
regFile.ChecksumType = fileDict.get("ChecksumType",
self.defaultChecksumType)
regFile.Size = fileDict.get("Size", 0)
regFile.GUID = fileDict.get("GUID", "")
se = StorageElement(targetSE)
pfn = se.getPfnForLfn(lfn)
if not pfn["OK"] or lfn not in pfn["Value"]['Successful']:
self.log.error(
"unable to get PFN for LFN: %s" %
pfn.get('Message',
pfn.get('Value', {}).get('Failed', {}).get(lfn)))
return pfn
regFile.PFN = pfn["Value"]['Successful'][lfn]
register.addFile(regFile)
self.request.addOperation(register)
return S_OK()
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog):
"""Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info("Setting registration request",
"for %s at %s." % (lfn, targetSE))
if not isinstance(catalog, list):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get("Checksum", "")
regFile.ChecksumType = fileDict.get("ChecksumType",
self.defaultChecksumType)
regFile.Size = fileDict.get("Size", 0)
regFile.GUID = fileDict.get("GUID", "")
se = StorageElement(targetSE)
res = returnSingleResult(se.getURL(lfn,
self.registrationProtocols))
if not res["OK"]:
self.log.error("Unable to get PFN for LFN", res["Message"])
return res
regFile.PFN = res["Value"]
register.addFile(regFile)
self.request.addOperation(register)
return S_OK()
def test_valid_properties():
theFile = File()
theFile.FileID = 1
assert theFile.FileID == 1
theFile.Status = "Done"
assert theFile.Status == "Done"
theFile.LFN = "/some/path/somewhere"
assert theFile.LFN == "/some/path/somewhere"
theFile.PFN = "/some/path/somewhere"
assert theFile.PFN == "/some/path/somewhere"
theFile.Attempt = 1
assert theFile.Attempt == 1
theFile.Size = 1
assert theFile.Size == 1
theFile.GUID = "2bbabe80-e2f1-11e1-9b23-0800200c9a66"
assert theFile.GUID == "2bbabe80-e2f1-11e1-9b23-0800200c9a66"
theFile.ChecksumType = "adler32"
assert theFile.ChecksumType == "ADLER32"
theFile.Checksum = "123456"
assert theFile.Checksum == "123456"
def getRegisterOperation(self, opFile, targetSE):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = "RegisterFile"
registerOperation.TargetSE = targetSE
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = opFile.PFN
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile(registerFile)
return registerOperation
def getRegisterOperation( self, opFile, targetSE ):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = "RegisterFile"
registerOperation.TargetSE = targetSE
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = opFile.PFN
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile( registerFile )
return registerOperation
def addRegisterReplica( self, opFile, targetSE ):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = "RegisterFile"
registerOperation.TargetSE = targetSE
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = opFile.PFN
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile( registerFile )
self.request.insertAfter( registerOperation, self.operation )
return S_OK()
def addRegisterReplica(self, opFile, targetSE):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = "RegisterFile"
registerOperation.TargetSE = targetSE
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = opFile.PFN
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile(registerFile)
self.request.insertAfter(registerOperation, self.operation)
return S_OK()
def _setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )
if not isinstance( catalog, list ):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get( "Checksum", "" )
regFile.ChecksumType = fileDict.get( "ChecksumType", self.defaultChecksumType )
regFile.Size = fileDict.get( "Size", 0 )
regFile.GUID = fileDict.get( "GUID", "" )
se = StorageElement( targetSE )
pfn = se.getURL( lfn, self.registrationProtocols )
if not pfn["OK"] or lfn not in pfn["Value"]['Successful']:
self.log.error( "Unable to get PFN for LFN", "%s" % pfn.get( 'Message', pfn.get( 'Value', {} ).get( 'Failed', {} ).get( lfn ) ) )
return pfn
regFile.PFN = pfn["Value"]['Successful'][lfn]
register.addFile( regFile )
self.request.addOperation( register )
return S_OK()
def getRegisterOperation( self, opFile, targetSE, type = 'RegisterFile', catalog = None ):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = StorageElement( targetSE ).getURL( opFile.LFN, protocol = self.registrationProtocols ).get( 'Value', {} ).get( 'Successful', {} ).get( opFile.LFN )
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile( registerFile )
return registerOperation
def test02props( self ):
""" test props and attributes """
theFile = File()
# valid props
theFile.FileID = 1
self.assertEqual( theFile.FileID, 1 )
theFile.Status = "Done"
self.assertEqual( theFile.Status, "Done" )
theFile.LFN = "/some/path/somewhere"
self.assertEqual( theFile.LFN, "/some/path/somewhere" )
theFile.PFN = "/some/path/somewhere"
self.assertEqual( theFile.PFN, "/some/path/somewhere" )
theFile.Attempt = 1
self.assertEqual( theFile.Attempt, 1 )
theFile.Size = 1
self.assertEqual( theFile.Size, 1 )
theFile.GUID = "2bbabe80-e2f1-11e1-9b23-0800200c9a66"
self.assertEqual( theFile.GUID, "2bbabe80-e2f1-11e1-9b23-0800200c9a66" )
theFile.ChecksumType = "adler32"
self.assertEqual( theFile.ChecksumType, "ADLER32" )
theFile.Checksum = "123456"
self.assertEqual( theFile.Checksum, "123456" )
# #
theFile.Checksum = None
theFile.ChecksumType = None
self.assertEqual( theFile.Checksum, "" )
self.assertEqual( theFile.ChecksumType, "" )
# # invalid props
# FileID
try:
theFile.FileID = "foo"
except Exception, error:
self.assertEqual( isinstance( error, ValueError ), True )
def test02props(self):
""" test props and attributes """
theFile = File()
# valid props
theFile.FileID = 1
self.assertEqual(theFile.FileID, 1)
theFile.Status = "Done"
self.assertEqual(theFile.Status, "Done")
theFile.LFN = "/some/path/somewhere"
self.assertEqual(theFile.LFN, "/some/path/somewhere")
theFile.PFN = "/some/path/somewhere"
self.assertEqual(theFile.PFN, "/some/path/somewhere")
theFile.Attempt = 1
self.assertEqual(theFile.Attempt, 1)
theFile.Size = 1
self.assertEqual(theFile.Size, 1)
theFile.GUID = "2bbabe80-e2f1-11e1-9b23-0800200c9a66"
self.assertEqual(theFile.GUID, "2bbabe80-e2f1-11e1-9b23-0800200c9a66")
theFile.ChecksumType = "adler32"
self.assertEqual(theFile.ChecksumType, "ADLER32")
theFile.Checksum = "123456"
self.assertEqual(theFile.Checksum, "123456")
# #
theFile.Checksum = None
theFile.ChecksumType = None
self.assertEqual(theFile.Checksum, "")
self.assertEqual(theFile.ChecksumType, "")
# # invalid props
# FileID
try:
theFile.FileID = "foo"
except Exception, error:
self.assertEqual(isinstance(error, ValueError), True)
if not res['OK']:
print "Can't get file metadata: %s" % res['Message']
DIRAC.exit( 1 )
if res['Value']['Failed']:
print "Could not get the file metadata of the following, so skipping them:"
for fFile in res['Value']['Failed']:
print fFile
lfnMetadata = res['Value']['Successful']
for lfn in lfnMetadata:
rarFile = File()
rarFile.LFN = lfn
rarFile.Size = lfnMetadata[lfn]['Size']
rarFile.Checksum = lfnMetadata[lfn]['Checksum']
rarFile.GUID = lfnMetadata[lfn]['GUID']
rarFile.ChecksumType = 'ADLER32'
replicateAndRegister.addFile( rarFile )
oRequest.addOperation( replicateAndRegister )
isValid = RequestValidator().validate( oRequest )
if not isValid['OK']:
print "Request is not valid: ", isValid['Message']
DIRAC.exit( 1 )
result = reqClient.putRequest( oRequest )
if result['OK']:
print "Request %d submitted successfully" % result['Value']
else:
print "Failed to submit Request: ", result['Message']
def main():
Script.parseCommandLine(ignoreErrors=False)
args = Script.getPositionalArgs()
if len(args) < 2:
Script.showHelp()
targetSE = args.pop(0)
lfns = []
for inputFileName in args:
if os.path.exists(inputFileName):
inputFile = open(inputFileName, 'r')
string = inputFile.read()
inputFile.close()
lfns.extend([lfn.strip() for lfn in string.splitlines()])
else:
lfns.append(inputFileName)
from DIRAC.Resources.Storage.StorageElement import StorageElement
import DIRAC
# Check is provided SE is OK
if targetSE != 'All':
se = StorageElement(targetSE)
if not se.valid:
print(se.errorReason)
print()
Script.showHelp()
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
reqClient = ReqClient()
fc = FileCatalog()
requestOperation = 'RemoveReplica'
if targetSE == 'All':
requestOperation = 'RemoveFile'
for lfnList in breakListIntoChunks(lfns, 100):
oRequest = Request()
requestName = "%s_%s" % (
md5(repr(time.time()).encode()).hexdigest()[:16],
md5(repr(time.time()).encode()).hexdigest()[:16],
)
oRequest.RequestName = requestName
oOperation = Operation()
oOperation.Type = requestOperation
oOperation.TargetSE = targetSE
res = fc.getFileMetadata(lfnList)
if not res['OK']:
print("Can't get file metadata: %s" % res['Message'])
DIRAC.exit(1)
if res['Value']['Failed']:
print(
"Could not get the file metadata of the following, so skipping them:"
)
for fFile in res['Value']['Failed']:
print(fFile)
lfnMetadata = res['Value']['Successful']
for lfn in lfnMetadata:
rarFile = File()
rarFile.LFN = lfn
rarFile.Size = lfnMetadata[lfn]['Size']
rarFile.Checksum = lfnMetadata[lfn]['Checksum']
rarFile.GUID = lfnMetadata[lfn]['GUID']
rarFile.ChecksumType = 'ADLER32'
oOperation.addFile(rarFile)
oRequest.addOperation(oOperation)
isValid = RequestValidator().validate(oRequest)
if not isValid['OK']:
print("Request is not valid: ", isValid['Message'])
DIRAC.exit(1)
result = reqClient.putRequest(oRequest)
if result['OK']:
print('Request %d Submitted' % result['Value'])
else:
print('Failed to submit Request: ', result['Message'])
def test02props(self):
""" test props and attributes """
theFile = File()
# valid props
theFile.FileID = 1
self.assertEqual(theFile.FileID, 1)
theFile.Status = "Done"
self.assertEqual(theFile.Status, "Done")
theFile.LFN = "/some/path/somewhere"
self.assertEqual(theFile.LFN, "/some/path/somewhere")
theFile.PFN = "/some/path/somewhere"
self.assertEqual(theFile.PFN, "/some/path/somewhere")
theFile.Attempt = 1
self.assertEqual(theFile.Attempt, 1)
theFile.Size = 1
self.assertEqual(theFile.Size, 1)
theFile.GUID = "2bbabe80-e2f1-11e1-9b23-0800200c9a66"
self.assertEqual(theFile.GUID, "2bbabe80-e2f1-11e1-9b23-0800200c9a66")
theFile.ChecksumType = "adler32"
self.assertEqual(theFile.ChecksumType, "ADLER32")
theFile.Checksum = "123456"
self.assertEqual(theFile.Checksum, "123456")
# #
theFile.Checksum = None
theFile.ChecksumType = None
# self.assertEqual( theFile.Checksum, "" )
# self.assertEqual( theFile.ChecksumType, "" )
# # invalid props
# FileID
try:
theFile.FileID = "foo"
except Exception as error:
self.assertEqual(isinstance(error, ValueError), True)
# parent
parent = Operation({"OperationID": 99999})
parent += theFile
theFile.FileID = 0
# self.assertEqual( parent.OperationID, theFile.OperationID )
try:
theFile.OperationID = 111111
except Exception as error:
self.assertEqual(isinstance(error, AttributeError), True)
self.assertEqual(str(error), "can't set attribute")
# LFN
try:
theFile.LFN = 1
except Exception as error:
self.assertEqual(isinstance(error, TypeError), True)
self.assertEqual(str(error), "LFN has to be a string!")
try:
theFile.LFN = "../some/path"
except Exception as error:
self.assertEqual(isinstance(error, ValueError), True)
self.assertEqual(str(error), "LFN should be an absolute path!")
# PFN
try:
theFile.PFN = 1
except Exception as error:
self.assertEqual(isinstance(error, TypeError), True)
self.assertEqual(str(error), "PFN has to be a string!")
try:
theFile.PFN = "snafu"
except Exception as error:
self.assertEqual(isinstance(error, ValueError), True)
self.assertEqual(str(error), "Wrongly formatted PFN!")
# Size
try:
theFile.Size = "snafu"
except Exception as error:
self.assertEqual(isinstance(error, ValueError), True)
try:
theFile.Size = -1
except Exception as error:
self.assertEqual(isinstance(error, ValueError), True)
self.assertEqual(str(error), "Size should be a positive integer!")
# GUID
try:
theFile.GUID = "snafuu-uuu-uuu-uuu-uuu-u"
except Exception as error:
self.assertEqual(isinstance(error, ValueError), True)
self.assertEqual(str(error), "'snafuu-uuu-uuu-uuu-uuu-u' is not a valid GUID!")
try:
theFile.GUID = 2233345
except Exception as error:
self.assertEqual(isinstance(error, TypeError), True)
self.assertEqual(str(error), "GUID should be a string!")
# Attempt
try:
theFile.Attempt = "snafu"
except Exception as error:
self.assertEqual(isinstance(error, ValueError), True)
try:
theFile.Attempt = -1
except Exception as error:
self.assertEqual(isinstance(error, ValueError), True)
self.assertEqual(str(error), "Attempt should be a positive integer!")
# Status
try:
theFile.Status = None
except Exception as error:
self.assertEqual(isinstance(error, ValueError), True)
self.assertEqual(str(error), "Unknown Status: None!")
# Error
try:
theFile.Error = Exception("test")
except Exception as error:
self.assertEqual(isinstance(error, TypeError), True)
self.assertEqual(str(error), "Error has to be a string!")
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(" SE: StorageElement|All")
Script.registerArgument(["LFN: LFN or file containing a List of LFNs"])
Script.parseCommandLine(ignoreErrors=False)
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
args = Script.getPositionalArgs()
targetSE = args.pop(0)
lfns = []
for inputFileName in args:
if os.path.exists(inputFileName):
with open(inputFileName, "r") as inputFile:
string = inputFile.read()
lfns.extend([lfn.strip() for lfn in string.splitlines()])
else:
lfns.append(inputFileName)
from DIRAC.Resources.Storage.StorageElement import StorageElement
import DIRAC
# Check is provided SE is OK
if targetSE != "All":
se = StorageElement(targetSE)
if not se.valid:
print(se.errorReason)
print()
Script.showHelp()
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
reqClient = ReqClient()
fc = FileCatalog()
requestOperation = "RemoveReplica"
if targetSE == "All":
requestOperation = "RemoveFile"
for lfnList in breakListIntoChunks(lfns, 100):
oRequest = Request()
requestName = "%s_%s" % (
md5(repr(time.time()).encode()).hexdigest()[:16],
md5(repr(time.time()).encode()).hexdigest()[:16],
)
oRequest.RequestName = requestName
oOperation = Operation()
oOperation.Type = requestOperation
oOperation.TargetSE = targetSE
res = fc.getFileMetadata(lfnList)
if not res["OK"]:
print("Can't get file metadata: %s" % res["Message"])
DIRAC.exit(1)
if res["Value"]["Failed"]:
print(
"Could not get the file metadata of the following, so skipping them:"
)
for fFile in res["Value"]["Failed"]:
print(fFile)
lfnMetadata = res["Value"]["Successful"]
for lfn in lfnMetadata:
rarFile = File()
rarFile.LFN = lfn
rarFile.Size = lfnMetadata[lfn]["Size"]
rarFile.Checksum = lfnMetadata[lfn]["Checksum"]
rarFile.GUID = lfnMetadata[lfn]["GUID"]
rarFile.ChecksumType = "ADLER32"
oOperation.addFile(rarFile)
oRequest.addOperation(oOperation)
isValid = RequestValidator().validate(oRequest)
if not isValid["OK"]:
print("Request is not valid: ", isValid["Message"])
DIRAC.exit(1)
result = reqClient.putRequest(oRequest)
if result["OK"]:
print("Request %d Submitted" % result["Value"])
else:
print("Failed to submit Request: ", result["Message"])
if not res['OK']:
print "Can't get file metadata: %s" % res['Message']
DIRAC.exit(1)
if res['Value']['Failed']:
print "Could not get the file metadata of the following, so skipping them:"
for fFile in res['Value']['Failed']:
print fFile
lfnMetadata = res['Value']['Successful']
for lfn in lfnMetadata:
rarFile = File()
rarFile.LFN = lfn
rarFile.Size = lfnMetadata[lfn]['Size']
rarFile.Checksum = lfnMetadata[lfn]['Checksum']
rarFile.GUID = lfnMetadata[lfn]['GUID']
rarFile.ChecksumType = 'ADLER32'
replicateAndRegister.addFile(rarFile)
oRequest.addOperation(replicateAndRegister)
isValid = gRequestValidator.validate(oRequest)
if not isValid['OK']:
print "Request is not valid: ", isValid['Message']
DIRAC.exit(1)
result = reqClient.putRequest(oRequest)
if result['OK']:
print "Request %d submitted successfully" % result['Value']
else:
print "Failed to submit Request: ", result['Message']
