class RAWIntegrityAgent(AgentModule):
"""
.. class:: RAWIntegirtyAgent
:param RAWIntegrityDB rawIntegrityDB: RAWIntegrityDB instance
:param str gatewayUrl: URL to online RequestClient
"""
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.rawIntegrityDB = None
self.fileCatalog = None
self.onlineRequestMgr = None
def initialize(self):
""" agent initialisation """
self.rawIntegrityDB = RAWIntegrityDB()
# The file catalog is used to register file once it has been transfered
# But we want to register it in all the catalogs except the RAWIntegrityDB
# otherwise it is register twice
# We also remove the BK catalog because some files are not registered there
# (detector calibration files for example). The real data are registered in
# the bookeeping by the DataMover
self.fileCatalog = FileCatalog(catalogs = 'FileCatalog')
gMonitor.registerActivity("Iteration", "Agent Loops/min", "RAWIntegriryAgent", "Loops",
gMonitor.OP_SUM)
gMonitor.registerActivity("WaitingFiles", "Files waiting for migration", "RAWIntegriryAgent",
"Files", gMonitor.OP_MEAN)
gMonitor.registerActivity("WaitSize", "Size of migration buffer", "RAWIntegrityAgent", "GB",
gMonitor.OP_MEAN)
gMonitor.registerActivity("NewlyMigrated", "Newly migrated files", "RAWIntegriryAgent", "Files",
gMonitor.OP_SUM)
gMonitor.registerActivity("TotMigrated", "Total migrated files", "RAWIntegriryAgent", "Files",
gMonitor.OP_ACUM)
gMonitor.registerActivity("TotMigratedSize", "Total migrated file size", "RAWIntegriryAgent",
"GB", gMonitor.OP_ACUM)
gMonitor.registerActivity("BadChecksum", "Checksums mismatch", "RAWIntegriryAgent", "Files",
gMonitor.OP_SUM)
gMonitor.registerActivity("ErrorMetadata", "Error when getting files metadata",
"RAWIntegriryAgent", "Files", gMonitor.OP_SUM)
gMonitor.registerActivity("ErrorRegister", "Error when registering files", "RAWIntegriryAgent",
"Files", gMonitor.OP_SUM)
gMonitor.registerActivity("ErrorRemove", "Error when removing files", "RAWIntegriryAgent",
"Files", gMonitor.OP_SUM)
gMonitor.registerActivity("FailedMigrated",
"Number of files encountering error during migration",
"RAWIntegriryAgent", "Files", gMonitor.OP_SUM)
gMonitor.registerActivity("TotFailMigrated",
"Total number of files encountering error during migration",
"RAWIntegriryAgent", "Files", gMonitor.OP_ACUM)
gMonitor.registerActivity("MigrationTime", "Average migration time", "RAWIntegriryAgent",
"Seconds", gMonitor.OP_MEAN)
# gMonitor.registerActivity("TimeInQueue", "Average current wait for migration",
# "RAWIntegriryAgent", "Minutes", gMonitor.OP_MEAN)
gMonitor.registerActivity("MigrationRate", "Observed migration rate", "RAWIntegriryAgent",
"MB/s", gMonitor.OP_MEAN)
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataProcessing')
return S_OK()
def _checkMigrationStatus(self, filesMetadata, lfnsMetadata):
""" Check that the lfns in parameters are properly migrated,
and compares the checksum between castor and the RAWIntegrityDB
:param filesMetadata: dict {lfn: se metadata}
:param lfnsMetadata: dict {lfn: metadata in RAWIntegrityDB}
:returns: True/False in successful, Failed in case of problem
"""
############################################################
#
# Determine the files that have been newly migrated and their success
#
successful = {}
failed = {}
for lfn, seMetadata in filesMetadata.iteritems():
isMigrated = seMetadata.get('Migrated', False)
# If it is not migrated, go to the next one
if not isMigrated:
successful[lfn] = False
continue
else:
self.log.info("%s is copied." % lfn)
castorChecksum = seMetadata['Checksum']
onlineChecksum = lfnsMetadata[lfn]['Checksum']
if castorChecksum.lower().lstrip('0') == onlineChecksum.lower().lstrip('0').lstrip('x'):
self.log.info("%s migrated checksum match." % lfn)
successful[lfn] = True
else:
errStr = "Migrated checksum mis-match.", "%s %s %s" % (
lfn, castorChecksum.lstrip('0'), onlineChecksum.lstrip('0').lstrip('x'))
self.log.error(errStr)
failed[lfn] = errStr
return S_OK({'Successful': successful, 'Failed': failed})
def getNewlyCopiedFiles(self, activeFiles):
"""
Get the list of files newly copied and those not yet copied.
:param activeFiles: dict {lfn:RawIntegrityDB metadata} for files in Active status
:return: tuple filesNewlyCopied, filesNotYetCopied (lfns lists)
"""
# This is a list of all the lfns that we will have newly copied
filesNewlyCopied = []
# This are the lfns that are not yet copied
filesNotYetCopied = []
self.log.info("Obtaining physical file metadata.")
# Group the lfns by SEs
seLfns = {}
for lfn, metadataDict in activeFiles.iteritems():
se = metadataDict['SE']
seLfns.setdefault(se, []).append(lfn)
for se in sorted(seLfns):
lfnList = seLfns[se]
failedMetadata = {}
successfulMetadata = {}
res = StorageElement(se).getFileMetadata(lfnList)
if not res['OK']:
errStr = "Failed to obtain physical file metadata."
self.log.error(errStr, res['Message'])
failedMetadata = dict((lfn, errStr) for lfn in lfnList)
else:
successfulMetadata = res['Value']['Successful']
failedMetadata = res['Value']['Failed']
if failedMetadata:
self.log.info("Failed to obtain physical file metadata for %s files." % len(failedMetadata))
gMonitor.addMark('ErrorMetadata', len(failedMetadata))
if successfulMetadata:
self.log.info("Obtained physical file metadata for %s files." % len(successfulMetadata))
############################################################
#
# Determine the files that have been newly migrated and their success
#
res = self._checkMigrationStatus(successfulMetadata, activeFiles)
if not res['OK']:
self.log.error("Error when checking migration status", res)
gMonitor.addMark("BadChecksum", len(successfulMetadata))
else:
succCompare = res['Value']['Successful']
failedCompare = res['Value']['Failed']
seFilesCopied = []
seFilesNotCopied = []
# The copied files are those in True in the successful dictionary
for lfn, isCopied in succCompare.iteritems():
if isCopied:
seFilesCopied.append(lfn)
else:
seFilesNotCopied.append(lfn)
filesNewlyCopied.extend(seFilesCopied)
filesNotYetCopied.extend(seFilesNotCopied)
gMonitor.addMark("BadChecksum", len(failedCompare))
self.log.info("%s files newly copied at %s." % (len(seFilesCopied), se))
self.log.info("Found %s checksum mis-matches at %s." % (len(failedCompare), se))
return filesNewlyCopied, filesNotYetCopied
def registerCopiedFiles(self, filesNewlyCopied, copiedFiles, allUnmigratedFilesMeta):
"""
Register successfuly copied files (newly, or in Copied status in the DB) in the DFC.
:param filesNewlyCopied: [lfns] of files newly copied
:param copiedFiles: {lfn:RIDb metadata} of files that were in Copied state.
:param allUnmigratedFilesMeta: {lfn:RI Db metadata} for all lfns non migrated at
the beginning of the loop.
:return: {lfn:True} for successfuly registered lfns
"""
if filesNewlyCopied or copiedFiles:
self.log.info("Attempting to register %s newly copied and %s previously copied files" %
(len(filesNewlyCopied), len(copiedFiles)))
else:
self.log.info("No files to be registered")
# Update copiedFiles to also contain the newly copied files
copiedFiles.update(dict((lfn, allUnmigratedFilesMeta[lfn]) for lfn in filesNewlyCopied))
successfulRegister = {}
failedRegister = {}
# Try to register them by batch
for lfnChunk in breakListIntoChunks(copiedFiles, 100):
# Add the metadata
lfnDictChuck = dict((lfn, copiedFiles[lfn]) for lfn in lfnChunk)
res = self.fileCatalog.addFile(lfnDictChuck)
if not res['OK']:
self.log.error("Completely failed to register some successfully copied file.",
res['Message'])
failedRegister.update(dict((lfn, res['Message']) for lfn in lfnDictChuck))
else:
successfulRegister.update(res['Value']['Successful'])
failedRegister.update(res['Value']['Failed'])
gMonitor.addMark("ErrorRegister", len(failedRegister))
for lfn, reason in failedRegister.iteritems():
self.log.error("Failed to register lfn. Setting to Copied", "%s: %s" % (lfn, reason))
res = self.rawIntegrityDB.setFileStatus(lfn, 'Copied')
if not res['OK']:
self.log.error("Error setting file status to Copied", "%s: %s" % (lfn, res['Message']))
for lfn in successfulRegister:
self.log.info("Successfully registered %s in the File Catalog." % lfn)
return successfulRegister
def removeRegisteredFiles(self, filesNewlyRegistered, registeredFiles, allUnmigratedFilesMeta):
"""
Remove successfuly registered files (newly, or in Registered status in the DB)
from the OnlineStorage
:param filesNewlyCopied: [lfns] of files newly copied
:param copiedFiles: {lfn:RIDb metadata} of files that were in Copied state.
:param allUnmigratedFilesMeta: {lfn:RI Db metadata} for all lfns non migrated at
the beginning of the loop.
:return: {lfn:True} for successfuly registered lfns
"""
if filesNewlyRegistered or registeredFiles:
self.log.info("Attempting to remove %s newly registered and %s previously registered files" %
(len(filesNewlyRegistered), len(registeredFiles)))
else:
self.log.info("No files to be removed")
# Update registeredFiles to also contain the newly registered files
registeredFiles.update(dict((lfn, allUnmigratedFilesMeta[lfn]) for lfn in filesNewlyRegistered))
onlineSE = StorageElement('OnlineRunDB')
# Try to them them all
res = onlineSE.removeFile(registeredFiles)
filesNewlyRemoved = {}
failedRemove = {}
if not res['OK']:
self.log.error("Completely failed to remove successfully registered files.", res['Message'])
failedRemove = dict((lfn, res['Message']) for lfn in registeredFiles)
else:
filesNewlyRemoved = res['Value']['Successful']
failedRemove = res['Value']['Failed']
gMonitor.addMark("ErrorRemove", len(failedRemove))
for lfn, reason in failedRemove.iteritems():
self.log.error("Failed to remove lfn. Setting to Registered", "%s: %s" % (lfn, reason))
res = self.rawIntegrityDB.setFileStatus(lfn, 'Registered')
if not res['OK']:
self.log.error("Error setting file status to Registered", "%s: %s" % (lfn, res['Message']))
now = datetime.datetime.utcnow()
for lfn in filesNewlyRemoved:
self.log.info("Successfully removed %s from the Online storage. Setting it to Done" % lfn)
res = self.rawIntegrityDB.setFileStatus(lfn, 'Done')
if not res['OK']:
self.log.error("Error setting file status to Done", "%s: %s" % (lfn, res['Message']))
else:
# SubmitTime is ALREADY a datetime since it is declared as such in the DB.
submitTime = allUnmigratedFilesMeta[lfn]['SubmitTime']
migrationTime = (now - submitTime).total_seconds()
gMonitor.addMark("MigrationTime", migrationTime)
fileSizeMB = allUnmigratedFilesMeta[lfn]['Size'] / (1024 * 1024.0)
gMonitor.addMark("MigrationRate", fileSizeMB / migrationTime)
return filesNewlyRemoved
def execute(self):
""" execution in one cycle
"""
# Don't use the server certificate otherwise the DFC wont let us write
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false')
gMonitor.addMark("Iteration", 1)
############################################################
#
# Obtain the files which have not yet been migrated
#
self.log.info("Obtaining un-migrated files.")
res = self.rawIntegrityDB.getUnmigratedFiles()
if not res['OK']:
errStr = "Failed to obtain un-migrated files."
self.log.error(errStr, res['Message'])
return S_OK()
# allUnmigratedFilesMeta contain all the files that are not yet
# migrated (not copied, not registered, not removed), as well as their metadata
allUnmigratedFilesMeta = res['Value']
self.log.info("Obtained %s un-migrated files." % len(allUnmigratedFilesMeta))
if not allUnmigratedFilesMeta:
return S_OK()
# activeFiles contains the files that are not yet copied
activeFiles = {}
# copiedFiles contains files that are copied but not yet registered
copiedFiles = {}
# registeredFiles contains files that are copied, registered, but not removed from source
registeredFiles = {}
# Assign them
for lfn, lfnMetadata in allUnmigratedFilesMeta.iteritems():
status = lfnMetadata.pop('Status')
if status == 'Active':
activeFiles[lfn] = lfnMetadata
elif status == 'Copied':
copiedFiles[lfn] = lfnMetadata
elif status == 'Registered':
registeredFiles[lfn] = lfnMetadata
gMonitor.addMark("WaitingFiles", len(activeFiles))
totalSize = 0
for lfn, fileDict in activeFiles.iteritems():
totalSize += int(fileDict['Size'])
# gMonitor.addMark("TimeInQueue", (fileDict['WaitTime'] / 60))
gMonitor.addMark("WaitSize", (totalSize / (1024 * 1024 * 1024.0)))
############################################################
#
# Checking newly copied files
#
# Get the list of lfns properly copied and not copied
filesNewlyCopied, filesNotYetCopied = self.getNewlyCopiedFiles(activeFiles)
####################################################
#
# Registering copied files
#
####################################################
filesNewlyRegistered = self.registerCopiedFiles(filesNewlyCopied, copiedFiles,
allUnmigratedFilesMeta)
####################################################
#
# Performing the removal from the online storage
#
####################################################
filesNewlyRemoved = self.removeRegisteredFiles(filesNewlyRegistered, registeredFiles,
allUnmigratedFilesMeta)
# Doing some accounting
migratedSize = sum(allUnmigratedFilesMeta[lfn]['Size'] for lfn in filesNewlyRemoved)
# The number of files that we failed at migrating
# is the number of files at the beginning, minus the one we processed completely
# minus those that are not yet copied
failedMigrated = len(allUnmigratedFilesMeta) - len(filesNewlyRemoved) - len(filesNotYetCopied)
res = self.rawIntegrityDB.setLastMonitorTime()
migratedSizeGB = migratedSize / (1024 * 1024 * 1024.0)
gMonitor.addMark("TotMigratedSize", migratedSizeGB)
gMonitor.addMark("NewlyMigrated", len(filesNewlyRemoved))
gMonitor.addMark("TotMigrated", len(filesNewlyRemoved))
gMonitor.addMark("FailedMigrated", failedMigrated)
gMonitor.addMark("TotFailMigrated", failedMigrated)
return S_OK()
class RAWIntegrityDBTest(unittest.TestCase):
""" Tests for the DB part of the RAWIntegrity system
"""
def setUp(self):
super(RAWIntegrityDBTest, self).setUp()
self.db = RAWIntegrityDB()
def tearDown(self):
# Only one file is before 'now' and 'after'
res = self.db.selectFiles({})
lfns = [fTuple[0] for fTuple in res['Value']]
# clean after us
for lfn in lfns:
self.db.removeFile(lfn)
def test_01_setupDB(self):
""" Test table creations"""
# At first, there should be no table
res = self.db.showTables()
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], [])
# Lets create them
res = self.db._checkTable()
self.assertTrue(res['OK'], res)
# and check they are now here
res = self.db.showTables()
self.assertTrue(res['OK'], res)
self.assertEqual(sorted(res['Value']), sorted(['Files', 'LastMonitor']))
# Lets create them again, there should be no error
res = self.db._checkTable()
self.assertTrue(res['OK'], res)
def test_02_lastMonitorTime(self):
""" Test the last monitor time function"""
# Just after creation, we insert initial timestamp so no error
res = self.db.getLastMonitorTimeDiff()
self.assertTrue(res['OK'], res)
# set the monitor time
res = self.db.setLastMonitorTime()
self.assertTrue(res['OK'], res)
# we wait a bit, and check that the difference is correct
# we expect not more than 1 second delay
sleepTime = 3
time.sleep(sleepTime)
res = self.db.getLastMonitorTimeDiff()
self.assertTrue(res['OK'], res)
self.assertTrue(sleepTime <= res['Value'] <= sleepTime + 1, res['Value'])
def test_03_fileManipulation(self):
""" Testing all the file manipulation operations"""
# There should be no new files so far
res = self.db.getFiles('New')
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], {})
testFile = {
'LFN': 'lfn',
'PFN': 'pfn',
'Size': 123,
'SE': 'se',
'GUID': 'guid',
'Checksum': 'checksum'
}
# adding a file
res = self.db.addFile(testFile['LFN'], testFile['PFN'], testFile['Size'], testFile['SE'],
testFile['GUID'], testFile['Checksum'])
self.assertTrue(res['OK'], res)
sleepTime = 2
time.sleep(sleepTime)
# There should be now one active file
res = self.db.getFiles('Active')
self.assertTrue(res['OK'], res)
self.assertEqual(len(res['Value']), 1)
self.assertTrue(testFile['LFN'] in res['Value'], res)
activeFile = res['Value'][testFile['LFN']]
for attribute in ['PFN', 'Size', 'SE', 'GUID', 'Checksum']:
self.assertEqual(testFile[attribute], activeFile[attribute])
self.assertTrue(sleepTime <= activeFile['WaitTime'] <= sleepTime + 1)
# Change the file status to Done
res = self.db.setFileStatus(testFile['LFN'], 'Done')
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], 1)
# The file should not be returned when asking for Active files anymore
res = self.db.getFiles('Active')
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], {})
# Change the file status back to Active
# It should not work, Done files cannot change
res = self.db.setFileStatus(testFile['LFN'], 'Active')
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], 0)
# The file should not be back
res = self.db.getFiles('Active')
self.assertEqual(res['Value'], {})
# Remove the file
res = self.db.removeFile(testFile['LFN'])
self.assertTrue(res['OK'], res)
# adding the file back
# (no need to test that its visible, we just did it)
res = self.db.addFile(testFile['LFN'], testFile['PFN'], testFile['Size'], testFile['SE'],
testFile['GUID'], testFile['Checksum'])
self.assertTrue(res['OK'], res)
# remove the file
res = self.db.removeFile(testFile['LFN'])
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], 1)
# There should be no file
res = self.db.getFiles('Active')
self.assertEqual(res['Value'], {})
# Adding two time the same files
# It should work
res = self.db.addFile(testFile['LFN'], testFile['PFN'], testFile['Size'], testFile['SE'],
testFile['GUID'], testFile['Checksum'])
self.assertTrue(res['OK'], res)
res = self.db.addFile(testFile['LFN'], testFile['PFN'], testFile['Size'], testFile['SE'],
testFile['GUID'], testFile['Checksum'])
self.assertTrue(res['OK'], res)
# We should get only one file, so processed only once
res = self.db.getFiles('Active')
self.assertTrue(res['OK'], res)
self.assertEqual(len(res['Value']), 1)
res = self.db.removeFile(testFile['LFN'])
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], 2)
# Setting status of a non existing file
res = self.db.setFileStatus('fake', 'Done')
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], 0)
# Removing a non existing file
res = self.db.removeFile('fake')
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], 0)
def test_03_testUnmigratedFiles(self):
""" Test that getUnmigratedFiles returns only the files
in the appropriate status
"""
unmigratedStatus = ['Active', 'Copied', 'Registered']
for st in ['Active', 'Copied', 'Registered', 'Done', 'Fake']:
self.db.addFile("lfn%s" % st, 'PFN', 1, 'SE', 'GUID', 'Checksum')
self.db.setFileStatus('lfn%s' % st, st)
res = self.db.getUnmigratedFiles()
self.assertTrue(res['OK'], res)
self.assertEqual(sorted(res['Value']), sorted(['lfn%s' % st for st in unmigratedStatus]))
def test_04_webQueries(self):
""" Test all the web related methods"""
# The DB should be empty now
res = self.db.getGlobalStatistics()
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], {})
res = self.db.getFileSelections()
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], {'StorageElement': [], 'Status': []})
# Adding two files Assigned, 1 Failed and 1 done
for i in xrange(1, 5):
res = self.db.addFile('lfn%s' % i, 'pfn%s' % i, 5 - i, 'se%s' % (i % 2), 'GUID%s' % i,
'Checksum%s' % i)
self.assertTrue(res['OK'], res)
res = self.db.setFileStatus('lfn3', 'Done')
self.assertTrue(res['OK'], res)
res = self.db.setFileStatus('lfn4', 'Failed')
self.assertTrue(res['OK'], res)
res = self.db.getGlobalStatistics()
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], {'Active': 2, 'Done': 1, 'Failed': 1})
res = self.db.getFileSelections()
self.assertTrue(res['OK'], res)
self.assertEqual(sorted(res['Value']['StorageElement']), sorted(['se0', 'se1']))
self.assertEqual(sorted(res['Value']['Status']), sorted(['Done', 'Failed', 'Active']))
# Do some selection test
res = self.db.selectFiles({'StorageElement': 'se0'})
self.assertTrue(res['OK'], res)
# they should be sorted by LFN by default
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn2', 'lfn4'])
res = self.db.selectFiles({'StorageElement': 'se0', 'PFN': 'pfn2'})
self.assertTrue(res['OK'], res)
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn2'])
# Impossible condition
res = self.db.selectFiles({'StorageElement': 'se1', 'PFN': 'pfn2'})
self.assertTrue(res['OK'], res)
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, [])
# limit to 1
res = self.db.selectFiles({'StorageElement': 'se0'}, limit=1)
self.assertTrue(res['OK'], res)
# they should be sorted by LFN by default
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn2'])
# Same selection, sorted by size
res = self.db.selectFiles({'StorageElement': 'se0'}, orderAttribute='Size')
self.assertTrue(res['OK'], res)
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn4', 'lfn2'])
# Test the time based selections
time.sleep(1)
now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
sleepTime = 2
time.sleep(sleepTime)
res = self.db.addFile('lfn6', 'pfn6', 6, 'se1', 'GUID6', 'Checksum6')
self.assertTrue(res['OK'], res)
# select the old files with no conditions
res = self.db.selectFiles({}, older=now)
self.assertTrue(res['OK'], res)
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn%i' % i for i in xrange(1, 5)])
# select the new file
res = self.db.selectFiles({}, newer=now)
self.assertTrue(res['OK'], res)
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn6'])
# add some more
time.sleep(1)
after = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
time.sleep(sleepTime)
res = self.db.addFile('lfn7', 'pfn7', 7, 'se1', 'GUID7', 'Checksum7')
self.assertTrue(res['OK'], res)
# We should now have two files after 'now'
res = self.db.selectFiles({}, newer=now)
self.assertTrue(res['OK'], res)
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn6', 'lfn7'])
# Only one file is before 'now' and 'after'
res = self.db.selectFiles({}, newer=now, older=after)
self.assertTrue(res['OK'], res)
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn6'])
# clean after us
for i in xrange(1, 8):
res = self.db.removeFile('lfn%s' % i)
self.assertTrue(res['OK'], res)
def test_05_perf(self):
""" Performance tests
For reminder, this is more ore less
the timing that were obtained on a super crapy
virtualMachine. Let's hope we never go above them..
criticalInsertTime = 34
criticalRetrieveTime = 1
criticalUpdateTime = 5
criticalRemoveTime = 30
"""
nbFiles = 5000
# Inserting files
startTime = time.time()
for i in xrange(nbFiles):
res = self.db.addFile('lfn%s' % i, 'pfn%s' % i, i, 'se%s' % (i % 2), 'GUID%s' % i,
'Checksum%s' % i)
self.assertTrue(res['OK'], res)
insertTime = time.time() - startTime
# Sleep 2 seconds so that the DB has
# a consistant commited state
time.sleep(2)
# getting all of them
startTime = time.time()
res = self.db.getFiles('Active')
self.assertTrue(res['OK'], res)
self.assertEqual(len(res['Value']), nbFiles)
getFileTime = time.time() - startTime
# Setting some of them
startTime = time.time()
rndIds = set()
for _ in xrange(nbFiles / 10):
rndId = random.randint(1, nbFiles)
rndIds.add(rndId)
self.db.setFileStatus('lfn%s' % rndId, 'Done')
self.assertTrue(res['OK'], res)
updateStatusTime = time.time() - startTime
# getting less of them
startTime = time.time()
res = self.db.getFiles('Active')
self.assertTrue(res['OK'], res)
self.assertEqual(len(res['Value']), nbFiles - len(rndIds))
getFileTime2 = time.time() - startTime
# deleting all of them
startTime = time.time()
for i in xrange(1, nbFiles):
res = self.db.removeFile('lfn%s' % i)
self.assertTrue(res['OK'], res)
removeTime = time.time() - startTime
print "Performance result"
print "Inserting %s files: %s" % (nbFiles, insertTime)
print "Getting all active files: %s" % getFileTime
print "Updating %s status: %s" % (nbFiles / 10, updateStatusTime)
print "Getting again active files: %s" % getFileTime2
print "Removing all files: %s" % removeTime