class CatalogPlugInTestCase(unittest.TestCase):
"""Base class for the CatalogPlugin test case"""
def setUp(self):
self.fullMetadata = [
"Status",
"ChecksumType",
"OwnerRole",
"CreationDate",
"Checksum",
"ModificationDate",
"OwnerDN",
"Mode",
"GUID",
"Size",
]
self.dirMetadata = self.fullMetadata + ["NumberOfSubPaths"]
self.fileMetadata = self.fullMetadata + ["NumberOfLinks"]
self.catalog = FileCatalog(catalogs=[catalogClientToTest])
valid = self.catalog.isOK()
self.assertTrue(valid)
self.destDir = "/lhcb/test/unit-test/TestCatalogPlugin"
self.link = "%s/link" % self.destDir
# Clean the existing directory
self.cleanDirectory()
res = self.catalog.createDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
# Register some files to work with
self.numberOfFiles = 2
self.files = []
for i in range(self.numberOfFiles):
lfn = "%s/testFile_%d" % (self.destDir, i)
res = self.registerFile(lfn)
self.assertTrue(res)
self.files.append(lfn)
def registerFile(self, lfn):
pfn = "protocol://host:port/storage/path%s" % lfn
size = 10000000
se = "DIRAC-storage"
guid = makeGuid()
adler = stringAdler(guid)
fileDict = {}
fileDict[lfn] = {
"PFN": pfn,
"Size": size,
"SE": se,
"GUID": guid,
"Checksum": adler
}
res = self.catalog.addFile(fileDict)
return self.parseResult(res, lfn)
def parseResult(self, res, path):
self.assertTrue(res["OK"])
self.assertTrue(res["Value"])
self.assertTrue(res["Value"]["Successful"])
self.assertTrue(path in res["Value"]["Successful"])
return res["Value"]["Successful"][path]
def parseError(self, res, path):
self.assertTrue(res["OK"])
self.assertTrue(res["Value"])
self.assertTrue(res["Value"]["Failed"])
self.assertTrue(path in res["Value"]["Failed"])
return res["Value"]["Failed"][path]
def cleanDirectory(self):
res = self.catalog.exists(self.destDir)
returnValue = self.parseResult(res, self.destDir)
if not returnValue:
return
res = self.catalog.listDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
toRemove = list(returnValue["Files"])
if toRemove:
self.purgeFiles(toRemove)
res = self.catalog.removeDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
self.assertTrue(returnValue)
def purgeFiles(self, lfns):
for lfn in lfns:
res = self.catalog.getReplicas(lfn, True)
replicas = self.parseResult(res, lfn)
for se, pfn in replicas.items():
repDict = {}
repDict[lfn] = {"PFN": pfn, "SE": se}
res = self.catalog.removeReplica(repDict)
self.parseResult(res, lfn)
res = self.catalog.removeFile(lfn)
self.parseResult(res, lfn)
def tearDown(self):
self.cleanDirectory()
class FileUploader(object):
def __init__(self):
# set up gfal
self.__gfalctxt = gfal2.creat_context()
# setup DIRAC file catalogue
self.__fc = FileCatalog()
def safe_upload(self, source_file, base_pfn):
"""uploads a file using gfal2"""
# TODO: This function needs a retry loop, catching any of the gfal exceptions
full_source = "file://%s" % os.path.abspath(source_file)
gfalparams = self.__gfalctxt.transfer_parameters()
# this hopefully enables checksum verification (hard to test ...)
gfalparams.checksum_check = True
fileinfo = self.__gfalctxt.stat(full_source)
filesize = fileinfo.st_size
filesum = self.__gfalctxt.checksum(full_source, "adler32")
gfalparams.set_user_defined_checksum("adler32", filesum)
full_pfn = "%s/%s" % (base_pfn, source_file)
self.__gfalctxt.filecopy(gfalparams, full_source, full_pfn)
return (filesize, filesum)
def upload_files(self, source, base_pfn, base_lfn, target_se):
"""for directories: recursively searches the source directory for files and uploads them to SE
otherwise, just uploads file"""
if os.path.isfile(source):
# TODO: this and the same block below should be a function
filesize, filesum = self.safe_upload(source, base_pfn)
full_lfn = "%s/%s" % (base_lfn, source)
full_pfn = "%s/%s" % (base_pfn, source)
self.registerfile(full_pfn, full_lfn, target_se, filesum, filesize)
return
for dirpath, subdirs, files in os.walk(source):
for filename in files:
source_file = os.path.join(dirpath, filename)
print "Trying %s" % filename
filesize, filesum = self.safe_upload(source_file, base_pfn)
print "Uploaded: %s (%s %s)" % (filename, filesize, filesum)
full_lfn = "%s/%s" % (base_lfn, source_file)
full_pfn = "%s/%s" % (base_pfn, source_file)
self.registerfile(full_pfn, full_lfn, target_se, filesum,
filesize)
# TODO: if the registering fails, the file should be deleted to avoid 'dark data' that is on disk, but not in the catalogue
def registerfile(self, full_pfn, full_lfn, target_se, checksum, size):
infoDict = {}
infoDict['PFN'] = full_pfn
infoDict['Size'] = int(size)
infoDict['SE'] = target_se
infoDict['GUID'] = str(uuid.uuid4())
infoDict['Checksum'] = checksum
fileDict = {}
fileDict[full_lfn] = infoDict
result = self.__fc.addFile(fileDict)
# TODO: handle the return code as an exception
if not result["OK"]:
print result
return
if result["Value"]["Failed"]:
print result["Value"]
return
class CatalogPlugInTestCase(unittest.TestCase):
""" Base class for the CatalogPlugin test case """
def setUp(self):
self.fullMetadata = [
'Status', 'ChecksumType', 'OwnerRole', 'CreationDate', 'Checksum',
'ModificationDate', 'OwnerDN', 'Mode', 'GUID', 'Size'
]
self.dirMetadata = self.fullMetadata + ['NumberOfSubPaths']
self.fileMetadata = self.fullMetadata + ['NumberOfLinks']
self.catalog = FileCatalog(catalogs=[catalogClientToTest])
valid = self.catalog.isOK()
self.assertTrue(valid)
self.destDir = '/lhcb/test/unit-test/TestCatalogPlugin'
self.link = "%s/link" % self.destDir
# Clean the existing directory
self.cleanDirectory()
res = self.catalog.createDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
# Register some files to work with
self.numberOfFiles = 2
self.files = []
for i in xrange(self.numberOfFiles):
lfn = "%s/testFile_%d" % (self.destDir, i)
res = self.registerFile(lfn)
self.assertTrue(res)
self.files.append(lfn)
def registerFile(self, lfn):
pfn = 'protocol://host:port/storage/path%s' % lfn
size = 10000000
se = 'DIRAC-storage'
guid = makeGuid()
adler = stringAdler(guid)
fileDict = {}
fileDict[lfn] = {
'PFN': pfn,
'Size': size,
'SE': se,
'GUID': guid,
'Checksum': adler
}
res = self.catalog.addFile(fileDict)
return self.parseResult(res, lfn)
def parseResult(self, res, path):
self.assertTrue(res['OK'])
self.assertTrue(res['Value'])
self.assertTrue(res['Value']['Successful'])
self.assertTrue(path in res['Value']['Successful'])
return res['Value']['Successful'][path]
def parseError(self, res, path):
self.assertTrue(res['OK'])
self.assertTrue(res['Value'])
self.assertTrue(res['Value']['Failed'])
self.assertTrue(path in res['Value']['Failed'])
return res['Value']['Failed'][path]
def cleanDirectory(self):
res = self.catalog.exists(self.destDir)
returnValue = self.parseResult(res, self.destDir)
if not returnValue:
return
res = self.catalog.listDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
toRemove = returnValue['Files'].keys()
if toRemove:
self.purgeFiles(toRemove)
res = self.catalog.removeDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
self.assertTrue(returnValue)
def purgeFiles(self, lfns):
for lfn in lfns:
res = self.catalog.getReplicas(lfn, True)
replicas = self.parseResult(res, lfn)
for se, pfn in replicas.items():
repDict = {}
repDict[lfn] = {'PFN': pfn, 'SE': se}
res = self.catalog.removeReplica(repDict)
self.parseResult(res, lfn)
res = self.catalog.removeFile(lfn)
self.parseResult(res, lfn)
def tearDown(self):
self.cleanDirectory()
class FileUploader(object):
def __init__(self):
# set up gfal
self.__gfalctxt = gfal2.creat_context()
# setup DIRAC file catalogue
self.__fc = FileCatalog()
def safe_upload(self, source_file, base_pfn):
"""uploads a file using gfal2"""
# TODO: This function needs a retry loop, catching any of the gfal exceptions
full_source = "file://%s" % os.path.abspath(source_file)
gfalparams = self.__gfalctxt.transfer_parameters()
# this hopefully enables checksum verification (hard to test ...)
gfalparams.checksum_check = True
fileinfo = self.__gfalctxt.stat(full_source)
filesize = fileinfo.st_size
filesum = self.__gfalctxt.checksum(full_source, "adler32")
gfalparams.set_user_defined_checksum("adler32", filesum)
full_pfn = "%s/%s" % (base_pfn, source_file)
self.__gfalctxt.filecopy(gfalparams, full_source, full_pfn)
return (filesize, filesum)
def upload_files(self, source, base_pfn, base_lfn, target_se):
"""for directories: recursively searches the source directory for files and uploads them to SE
otherwise, just uploads file"""
if os.path.isfile(source):
# TODO: this and the same block below should be a function
filesize, filesum = self.safe_upload(source, base_pfn)
full_lfn = "%s/%s" %(base_lfn, source)
full_pfn = "%s/%s" %(base_pfn, source)
self.registerfile(full_pfn, full_lfn, target_se, filesum, filesize)
return
for dirpath, subdirs, files in os.walk(source):
for filename in files:
source_file = os.path.join(dirpath, filename)
print "Trying %s" % filename
filesize, filesum = self.safe_upload(source_file, base_pfn)
print "Uploaded: %s (%s %s)" % (filename, filesize, filesum)
full_lfn = "%s/%s" %(base_lfn, source_file)
full_pfn = "%s/%s" %(base_pfn, source_file)
self.registerfile(full_pfn, full_lfn, target_se, filesum, filesize)
# TODO: if the registering fails, the file should be deleted to avoid 'dark data' that is on disk, but not in the catalogue
def registerfile(self, full_pfn, full_lfn, target_se, checksum, size):
infoDict = {}
infoDict['PFN'] = full_pfn
infoDict['Size'] = int(size)
infoDict['SE'] = target_se
infoDict['GUID'] = str(uuid.uuid4())
infoDict['Checksum'] = checksum
fileDict = {}
fileDict[full_lfn] = infoDict
result = self.__fc.addFile(fileDict)
# TODO: handle the return code as an exception
if not result["OK"]:
print result
return
if result["Value"]["Failed"]:
print result["Value"]
return
for line in lines:
line = line.strip()
bits = line.split(' ')
filename = bits[0]
if not registering:
if filename == firstfile:
registering = True
if registering:
if bits[1] == "ERROR!":
print("Skipping file", filename, "due to upload error")
else:
size = int(bits[1])
print("Registering file", filename, "with size", size)
lfn = LFNBASE + filename
infoDict = {}
infoDict['PFN'] = PFNBASE + filename
infoDict['Size'] = size
infoDict['SE'] = SE
infoDict['Checksum'] = ''
infoDict['GUID'] = str(uuid.uuid4())
fileDict = {}
fileDict[lfn] = infoDict
result = fc.addFile(fileDict)
print("Result:", result)
class CatalogPlugInTestCase(unittest.TestCase):
""" Base class for the CatalogPlugin test case """
def setUp(self):
self.fullMetadata = ['Status', 'CheckSumType', 'OwnerRole', 'CreationDate', 'Checksum', 'ModificationDate', 'OwnerDN', 'Mode', 'GUID', 'Size']
self.dirMetadata = self.fullMetadata + ['NumberOfSubPaths']
self.fileMetadata = self.fullMetadata + ['NumberOfLinks']
self.catalog = FileCatalog(catalogs=[catalogClientToTest])
valid = self.catalog.isOK()
self.assert_(valid)
self.destDir = '/lhcb/test/unit-test/TestCatalogPlugin'
self.link = "%s/link" % self.destDir
# Clean the existing directory
self.cleanDirectory()
res = self.catalog.createDirectory(self.destDir)
returnValue = self.parseResult(res,self.destDir)
# Register some files to work with
self.numberOfFiles = 2
self.files = []
for i in range(self.numberOfFiles):
lfn = "%s/testFile_%d" % (self.destDir,i)
res = self.registerFile(lfn)
self.assert_(res)
self.files.append(lfn)
def registerFile(self,lfn):
pfn = 'protocol://host:port/storage/path%s' % lfn
size = 10000000
se = 'DIRAC-storage'
guid = makeGuid()
adler = stringAdler(guid)
fileDict = {}
fileDict[lfn] = {'PFN':pfn,'Size':size,'SE':se,'GUID':guid,'Checksum':adler}
res = self.catalog.addFile(fileDict)
return self.parseResult(res,lfn)
def parseResult(self,res,path):
self.assert_(res['OK'])
self.assert_(res['Value'])
self.assert_(res['Value']['Successful'])
self.assert_(res['Value']['Successful'].has_key(path))
return res['Value']['Successful'][path]
def parseError(self,res,path):
self.assert_(res['OK'])
self.assert_(res['Value'])
self.assert_(res['Value']['Failed'])
self.assert_(res['Value']['Failed'].has_key(path))
return res['Value']['Failed'][path]
def cleanDirectory(self):
res = self.catalog.exists(self.destDir)
returnValue = self.parseResult(res,self.destDir)
if not returnValue:
return
res = self.catalog.listDirectory(self.destDir)
returnValue = self.parseResult(res,self.destDir)
toRemove = returnValue['Files'].keys()
if toRemove:
self.purgeFiles(toRemove)
res = self.catalog.removeDirectory(self.destDir)
returnValue = self.parseResult(res,self.destDir)
self.assert_(returnValue)
def purgeFiles(self,lfns):
for lfn in lfns:
res = self.catalog.getReplicas(lfn,True)
replicas = self.parseResult(res,lfn)
for se,pfn in replicas.items():
repDict = {}
repDict[lfn] = {'PFN':pfn,'SE':se}
res = self.catalog.removeReplica(repDict)
self.parseResult(res,lfn)
res = self.catalog.removeFile(lfn)
self.parseResult(res,lfn)
def tearDown(self):
self.cleanDirectory()
class RAWIntegrityAgent(AgentModule):
"""
.. class:: RAWIntegirtyAgent
:param RAWIntegrityDB rawIntegrityDB: RAWIntegrityDB instance
:param str gatewayUrl: URL to online RequestClient
"""
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.rawIntegrityDB = None
self.fileCatalog = None
self.onlineRequestMgr = None
def initialize(self):
""" agent initialisation """
self.rawIntegrityDB = RAWIntegrityDB()
# The file catalog is used to register file once it has been transfered
# But we want to register it in all the catalogs except the RAWIntegrityDB
# otherwise it is register twice
# We also remove the BK catalog because some files are not registered there
# (detector calibration files for example). The real data are registered in
# the bookeeping by the DataMover
self.fileCatalog = FileCatalog(catalogs = 'FileCatalog')
gMonitor.registerActivity("Iteration", "Agent Loops/min", "RAWIntegriryAgent", "Loops",
gMonitor.OP_SUM)
gMonitor.registerActivity("WaitingFiles", "Files waiting for migration", "RAWIntegriryAgent",
"Files", gMonitor.OP_MEAN)
gMonitor.registerActivity("WaitSize", "Size of migration buffer", "RAWIntegrityAgent", "GB",
gMonitor.OP_MEAN)
gMonitor.registerActivity("NewlyMigrated", "Newly migrated files", "RAWIntegriryAgent", "Files",
gMonitor.OP_SUM)
gMonitor.registerActivity("TotMigrated", "Total migrated files", "RAWIntegriryAgent", "Files",
gMonitor.OP_ACUM)
gMonitor.registerActivity("TotMigratedSize", "Total migrated file size", "RAWIntegriryAgent",
"GB", gMonitor.OP_ACUM)
gMonitor.registerActivity("BadChecksum", "Checksums mismatch", "RAWIntegriryAgent", "Files",
gMonitor.OP_SUM)
gMonitor.registerActivity("ErrorMetadata", "Error when getting files metadata",
"RAWIntegriryAgent", "Files", gMonitor.OP_SUM)
gMonitor.registerActivity("ErrorRegister", "Error when registering files", "RAWIntegriryAgent",
"Files", gMonitor.OP_SUM)
gMonitor.registerActivity("ErrorRemove", "Error when removing files", "RAWIntegriryAgent",
"Files", gMonitor.OP_SUM)
gMonitor.registerActivity("FailedMigrated",
"Number of files encountering error during migration",
"RAWIntegriryAgent", "Files", gMonitor.OP_SUM)
gMonitor.registerActivity("TotFailMigrated",
"Total number of files encountering error during migration",
"RAWIntegriryAgent", "Files", gMonitor.OP_ACUM)
gMonitor.registerActivity("MigrationTime", "Average migration time", "RAWIntegriryAgent",
"Seconds", gMonitor.OP_MEAN)
# gMonitor.registerActivity("TimeInQueue", "Average current wait for migration",
# "RAWIntegriryAgent", "Minutes", gMonitor.OP_MEAN)
gMonitor.registerActivity("MigrationRate", "Observed migration rate", "RAWIntegriryAgent",
"MB/s", gMonitor.OP_MEAN)
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataProcessing')
return S_OK()
def _checkMigrationStatus(self, filesMetadata, lfnsMetadata):
""" Check that the lfns in parameters are properly migrated,
and compares the checksum between castor and the RAWIntegrityDB
:param filesMetadata: dict {lfn: se metadata}
:param lfnsMetadata: dict {lfn: metadata in RAWIntegrityDB}
:returns: True/False in successful, Failed in case of problem
"""
############################################################
#
# Determine the files that have been newly migrated and their success
#
successful = {}
failed = {}
for lfn, seMetadata in filesMetadata.iteritems():
isMigrated = seMetadata.get('Migrated', False)
# If it is not migrated, go to the next one
if not isMigrated:
successful[lfn] = False
continue
else:
self.log.info("%s is copied." % lfn)
castorChecksum = seMetadata['Checksum']
onlineChecksum = lfnsMetadata[lfn]['Checksum']
if castorChecksum.lower().lstrip('0') == onlineChecksum.lower().lstrip('0').lstrip('x'):
self.log.info("%s migrated checksum match." % lfn)
successful[lfn] = True
else:
errStr = "Migrated checksum mis-match.", "%s %s %s" % (
lfn, castorChecksum.lstrip('0'), onlineChecksum.lstrip('0').lstrip('x'))
self.log.error(errStr)
failed[lfn] = errStr
return S_OK({'Successful': successful, 'Failed': failed})
def getNewlyCopiedFiles(self, activeFiles):
"""
Get the list of files newly copied and those not yet copied.
:param activeFiles: dict {lfn:RawIntegrityDB metadata} for files in Active status
:return: tuple filesNewlyCopied, filesNotYetCopied (lfns lists)
"""
# This is a list of all the lfns that we will have newly copied
filesNewlyCopied = []
# This are the lfns that are not yet copied
filesNotYetCopied = []
self.log.info("Obtaining physical file metadata.")
# Group the lfns by SEs
seLfns = {}
for lfn, metadataDict in activeFiles.iteritems():
se = metadataDict['SE']
seLfns.setdefault(se, []).append(lfn)
for se in sorted(seLfns):
lfnList = seLfns[se]
failedMetadata = {}
successfulMetadata = {}
res = StorageElement(se).getFileMetadata(lfnList)
if not res['OK']:
errStr = "Failed to obtain physical file metadata."
self.log.error(errStr, res['Message'])
failedMetadata = dict((lfn, errStr) for lfn in lfnList)
else:
successfulMetadata = res['Value']['Successful']
failedMetadata = res['Value']['Failed']
if failedMetadata:
self.log.info("Failed to obtain physical file metadata for %s files." % len(failedMetadata))
gMonitor.addMark('ErrorMetadata', len(failedMetadata))
if successfulMetadata:
self.log.info("Obtained physical file metadata for %s files." % len(successfulMetadata))
############################################################
#
# Determine the files that have been newly migrated and their success
#
res = self._checkMigrationStatus(successfulMetadata, activeFiles)
if not res['OK']:
self.log.error("Error when checking migration status", res)
gMonitor.addMark("BadChecksum", len(successfulMetadata))
else:
succCompare = res['Value']['Successful']
failedCompare = res['Value']['Failed']
seFilesCopied = []
seFilesNotCopied = []
# The copied files are those in True in the successful dictionary
for lfn, isCopied in succCompare.iteritems():
if isCopied:
seFilesCopied.append(lfn)
else:
seFilesNotCopied.append(lfn)
filesNewlyCopied.extend(seFilesCopied)
filesNotYetCopied.extend(seFilesNotCopied)
gMonitor.addMark("BadChecksum", len(failedCompare))
self.log.info("%s files newly copied at %s." % (len(seFilesCopied), se))
self.log.info("Found %s checksum mis-matches at %s." % (len(failedCompare), se))
return filesNewlyCopied, filesNotYetCopied
def registerCopiedFiles(self, filesNewlyCopied, copiedFiles, allUnmigratedFilesMeta):
"""
Register successfuly copied files (newly, or in Copied status in the DB) in the DFC.
:param filesNewlyCopied: [lfns] of files newly copied
:param copiedFiles: {lfn:RIDb metadata} of files that were in Copied state.
:param allUnmigratedFilesMeta: {lfn:RI Db metadata} for all lfns non migrated at
the beginning of the loop.
:return: {lfn:True} for successfuly registered lfns
"""
if filesNewlyCopied or copiedFiles:
self.log.info("Attempting to register %s newly copied and %s previously copied files" %
(len(filesNewlyCopied), len(copiedFiles)))
else:
self.log.info("No files to be registered")
# Update copiedFiles to also contain the newly copied files
copiedFiles.update(dict((lfn, allUnmigratedFilesMeta[lfn]) for lfn in filesNewlyCopied))
successfulRegister = {}
failedRegister = {}
# Try to register them by batch
for lfnChunk in breakListIntoChunks(copiedFiles, 100):
# Add the metadata
lfnDictChuck = dict((lfn, copiedFiles[lfn]) for lfn in lfnChunk)
res = self.fileCatalog.addFile(lfnDictChuck)
if not res['OK']:
self.log.error("Completely failed to register some successfully copied file.",
res['Message'])
failedRegister.update(dict((lfn, res['Message']) for lfn in lfnDictChuck))
else:
successfulRegister.update(res['Value']['Successful'])
failedRegister.update(res['Value']['Failed'])
gMonitor.addMark("ErrorRegister", len(failedRegister))
for lfn, reason in failedRegister.iteritems():
self.log.error("Failed to register lfn. Setting to Copied", "%s: %s" % (lfn, reason))
res = self.rawIntegrityDB.setFileStatus(lfn, 'Copied')
if not res['OK']:
self.log.error("Error setting file status to Copied", "%s: %s" % (lfn, res['Message']))
for lfn in successfulRegister:
self.log.info("Successfully registered %s in the File Catalog." % lfn)
return successfulRegister
def removeRegisteredFiles(self, filesNewlyRegistered, registeredFiles, allUnmigratedFilesMeta):
"""
Remove successfuly registered files (newly, or in Registered status in the DB)
from the OnlineStorage
:param filesNewlyCopied: [lfns] of files newly copied
:param copiedFiles: {lfn:RIDb metadata} of files that were in Copied state.
:param allUnmigratedFilesMeta: {lfn:RI Db metadata} for all lfns non migrated at
the beginning of the loop.
:return: {lfn:True} for successfuly registered lfns
"""
if filesNewlyRegistered or registeredFiles:
self.log.info("Attempting to remove %s newly registered and %s previously registered files" %
(len(filesNewlyRegistered), len(registeredFiles)))
else:
self.log.info("No files to be removed")
# Update registeredFiles to also contain the newly registered files
registeredFiles.update(dict((lfn, allUnmigratedFilesMeta[lfn]) for lfn in filesNewlyRegistered))
onlineSE = StorageElement('OnlineRunDB')
# Try to them them all
res = onlineSE.removeFile(registeredFiles)
filesNewlyRemoved = {}
failedRemove = {}
if not res['OK']:
self.log.error("Completely failed to remove successfully registered files.", res['Message'])
failedRemove = dict((lfn, res['Message']) for lfn in registeredFiles)
else:
filesNewlyRemoved = res['Value']['Successful']
failedRemove = res['Value']['Failed']
gMonitor.addMark("ErrorRemove", len(failedRemove))
for lfn, reason in failedRemove.iteritems():
self.log.error("Failed to remove lfn. Setting to Registered", "%s: %s" % (lfn, reason))
res = self.rawIntegrityDB.setFileStatus(lfn, 'Registered')
if not res['OK']:
self.log.error("Error setting file status to Registered", "%s: %s" % (lfn, res['Message']))
now = datetime.datetime.utcnow()
for lfn in filesNewlyRemoved:
self.log.info("Successfully removed %s from the Online storage. Setting it to Done" % lfn)
res = self.rawIntegrityDB.setFileStatus(lfn, 'Done')
if not res['OK']:
self.log.error("Error setting file status to Done", "%s: %s" % (lfn, res['Message']))
else:
# SubmitTime is ALREADY a datetime since it is declared as such in the DB.
submitTime = allUnmigratedFilesMeta[lfn]['SubmitTime']
migrationTime = (now - submitTime).total_seconds()
gMonitor.addMark("MigrationTime", migrationTime)
fileSizeMB = allUnmigratedFilesMeta[lfn]['Size'] / (1024 * 1024.0)
gMonitor.addMark("MigrationRate", fileSizeMB / migrationTime)
return filesNewlyRemoved
def execute(self):
""" execution in one cycle
"""
# Don't use the server certificate otherwise the DFC wont let us write
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false')
gMonitor.addMark("Iteration", 1)
############################################################
#
# Obtain the files which have not yet been migrated
#
self.log.info("Obtaining un-migrated files.")
res = self.rawIntegrityDB.getUnmigratedFiles()
if not res['OK']:
errStr = "Failed to obtain un-migrated files."
self.log.error(errStr, res['Message'])
return S_OK()
# allUnmigratedFilesMeta contain all the files that are not yet
# migrated (not copied, not registered, not removed), as well as their metadata
allUnmigratedFilesMeta = res['Value']
self.log.info("Obtained %s un-migrated files." % len(allUnmigratedFilesMeta))
if not allUnmigratedFilesMeta:
return S_OK()
# activeFiles contains the files that are not yet copied
activeFiles = {}
# copiedFiles contains files that are copied but not yet registered
copiedFiles = {}
# registeredFiles contains files that are copied, registered, but not removed from source
registeredFiles = {}
# Assign them
for lfn, lfnMetadata in allUnmigratedFilesMeta.iteritems():
status = lfnMetadata.pop('Status')
if status == 'Active':
activeFiles[lfn] = lfnMetadata
elif status == 'Copied':
copiedFiles[lfn] = lfnMetadata
elif status == 'Registered':
registeredFiles[lfn] = lfnMetadata
gMonitor.addMark("WaitingFiles", len(activeFiles))
totalSize = 0
for lfn, fileDict in activeFiles.iteritems():
totalSize += int(fileDict['Size'])
# gMonitor.addMark("TimeInQueue", (fileDict['WaitTime'] / 60))
gMonitor.addMark("WaitSize", (totalSize / (1024 * 1024 * 1024.0)))
############################################################
#
# Checking newly copied files
#
# Get the list of lfns properly copied and not copied
filesNewlyCopied, filesNotYetCopied = self.getNewlyCopiedFiles(activeFiles)
####################################################
#
# Registering copied files
#
####################################################
filesNewlyRegistered = self.registerCopiedFiles(filesNewlyCopied, copiedFiles,
allUnmigratedFilesMeta)
####################################################
#
# Performing the removal from the online storage
#
####################################################
filesNewlyRemoved = self.removeRegisteredFiles(filesNewlyRegistered, registeredFiles,
allUnmigratedFilesMeta)
# Doing some accounting
migratedSize = sum(allUnmigratedFilesMeta[lfn]['Size'] for lfn in filesNewlyRemoved)
# The number of files that we failed at migrating
# is the number of files at the beginning, minus the one we processed completely
# minus those that are not yet copied
failedMigrated = len(allUnmigratedFilesMeta) - len(filesNewlyRemoved) - len(filesNotYetCopied)
res = self.rawIntegrityDB.setLastMonitorTime()
migratedSizeGB = migratedSize / (1024 * 1024 * 1024.0)
gMonitor.addMark("TotMigratedSize", migratedSizeGB)
gMonitor.addMark("NewlyMigrated", len(filesNewlyRemoved))
gMonitor.addMark("TotMigrated", len(filesNewlyRemoved))
gMonitor.addMark("FailedMigrated", failedMigrated)
gMonitor.addMark("TotFailMigrated", failedMigrated)
return S_OK()
class CatalogPlugInTestCase(unittest.TestCase):
""" Base class for the CatalogPlugin test case """
def setUp(self):
self.fullMetadata = [
"Status",
"ChecksumType",
"OwnerRole",
"CreationDate",
"Checksum",
"ModificationDate",
"OwnerDN",
"Mode",
"GUID",
"Size",
]
self.dirMetadata = self.fullMetadata + ["NumberOfSubPaths"]
self.fileMetadata = self.fullMetadata + ["NumberOfLinks"]
self.catalog = FileCatalog(catalogs=[catalogClientToTest])
valid = self.catalog.isOK()
self.assert_(valid)
self.destDir = "/lhcb/test/unit-test/TestCatalogPlugin"
self.link = "%s/link" % self.destDir
# Clean the existing directory
self.cleanDirectory()
res = self.catalog.createDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
# Register some files to work with
self.numberOfFiles = 2
self.files = []
for i in range(self.numberOfFiles):
lfn = "%s/testFile_%d" % (self.destDir, i)
res = self.registerFile(lfn)
self.assert_(res)
self.files.append(lfn)
def registerFile(self, lfn):
pfn = "protocol://host:port/storage/path%s" % lfn
size = 10000000
se = "DIRAC-storage"
guid = makeGuid()
adler = stringAdler(guid)
fileDict = {}
fileDict[lfn] = {"PFN": pfn, "Size": size, "SE": se, "GUID": guid, "Checksum": adler}
res = self.catalog.addFile(fileDict)
return self.parseResult(res, lfn)
def parseResult(self, res, path):
self.assert_(res["OK"])
self.assert_(res["Value"])
self.assert_(res["Value"]["Successful"])
self.assert_(res["Value"]["Successful"].has_key(path))
return res["Value"]["Successful"][path]
def parseError(self, res, path):
self.assert_(res["OK"])
self.assert_(res["Value"])
self.assert_(res["Value"]["Failed"])
self.assert_(res["Value"]["Failed"].has_key(path))
return res["Value"]["Failed"][path]
def cleanDirectory(self):
res = self.catalog.exists(self.destDir)
returnValue = self.parseResult(res, self.destDir)
if not returnValue:
return
res = self.catalog.listDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
toRemove = returnValue["Files"].keys()
if toRemove:
self.purgeFiles(toRemove)
res = self.catalog.removeDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
self.assert_(returnValue)
def purgeFiles(self, lfns):
for lfn in lfns:
res = self.catalog.getReplicas(lfn, True)
replicas = self.parseResult(res, lfn)
for se, pfn in replicas.items():
repDict = {}
repDict[lfn] = {"PFN": pfn, "SE": se}
res = self.catalog.removeReplica(repDict)
self.parseResult(res, lfn)
res = self.catalog.removeFile(lfn)
self.parseResult(res, lfn)
def tearDown(self):
self.cleanDirectory()
