def initializeRAWIntegrityHandler(_serviceInfo):
""" handler initialisation """
global gRAWIntegrityDB
gRAWIntegrityDB = RAWIntegrityDB()
# Create DB tables if needed
return gRAWIntegrityDB._checkTable()
class RAWIntegrityDBTest(unittest.TestCase):
""" Tests for the DB part of the RAWIntegrity system
"""
def setUp(self):
super(RAWIntegrityDBTest, self).setUp()
self.db = RAWIntegrityDB()
def tearDown(self):
# Only one file is before 'now' and 'after'
res = self.db.selectFiles({})
lfns = [fTuple[0] for fTuple in res['Value']]
# clean after us
for lfn in lfns:
self.db.removeFile(lfn)
def test_01_setupDB(self):
""" Test table creations"""
# At first, there should be no table
res = self.db.showTables()
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], [])
# Lets create them
res = self.db._checkTable()
self.assertTrue(res['OK'], res)
# and check they are now here
res = self.db.showTables()
self.assertTrue(res['OK'], res)
self.assertEqual(sorted(res['Value']), sorted(['Files', 'LastMonitor']))
# Lets create them again, there should be no error
res = self.db._checkTable()
self.assertTrue(res['OK'], res)
def test_02_lastMonitorTime(self):
""" Test the last monitor time function"""
# Just after creation, we insert initial timestamp so no error
res = self.db.getLastMonitorTimeDiff()
self.assertTrue(res['OK'], res)
# set the monitor time
res = self.db.setLastMonitorTime()
self.assertTrue(res['OK'], res)
# we wait a bit, and check that the difference is correct
# we expect not more than 1 second delay
sleepTime = 3
time.sleep(sleepTime)
res = self.db.getLastMonitorTimeDiff()
self.assertTrue(res['OK'], res)
self.assertTrue(sleepTime <= res['Value'] <= sleepTime + 1, res['Value'])
def test_03_fileManipulation(self):
""" Testing all the file manipulation operations"""
# There should be no new files so far
res = self.db.getFiles('New')
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], {})
testFile = {
'LFN': 'lfn',
'PFN': 'pfn',
'Size': 123,
'SE': 'se',
'GUID': 'guid',
'Checksum': 'checksum'
}
# adding a file
res = self.db.addFile(testFile['LFN'], testFile['PFN'], testFile['Size'], testFile['SE'],
testFile['GUID'], testFile['Checksum'])
self.assertTrue(res['OK'], res)
sleepTime = 2
time.sleep(sleepTime)
# There should be now one active file
res = self.db.getFiles('Active')
self.assertTrue(res['OK'], res)
self.assertEqual(len(res['Value']), 1)
self.assertTrue(testFile['LFN'] in res['Value'], res)
activeFile = res['Value'][testFile['LFN']]
for attribute in ['PFN', 'Size', 'SE', 'GUID', 'Checksum']:
self.assertEqual(testFile[attribute], activeFile[attribute])
self.assertTrue(sleepTime <= activeFile['WaitTime'] <= sleepTime + 1)
# Change the file status to Done
res = self.db.setFileStatus(testFile['LFN'], 'Done')
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], 1)
# The file should not be returned when asking for Active files anymore
res = self.db.getFiles('Active')
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], {})
# Change the file status back to Active
# It should not work, Done files cannot change
res = self.db.setFileStatus(testFile['LFN'], 'Active')
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], 0)
# The file should not be back
res = self.db.getFiles('Active')
self.assertEqual(res['Value'], {})
# Remove the file
res = self.db.removeFile(testFile['LFN'])
self.assertTrue(res['OK'], res)
# adding the file back
# (no need to test that its visible, we just did it)
res = self.db.addFile(testFile['LFN'], testFile['PFN'], testFile['Size'], testFile['SE'],
testFile['GUID'], testFile['Checksum'])
self.assertTrue(res['OK'], res)
# remove the file
res = self.db.removeFile(testFile['LFN'])
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], 1)
# There should be no file
res = self.db.getFiles('Active')
self.assertEqual(res['Value'], {})
# Adding two time the same files
# It should work
res = self.db.addFile(testFile['LFN'], testFile['PFN'], testFile['Size'], testFile['SE'],
testFile['GUID'], testFile['Checksum'])
self.assertTrue(res['OK'], res)
res = self.db.addFile(testFile['LFN'], testFile['PFN'], testFile['Size'], testFile['SE'],
testFile['GUID'], testFile['Checksum'])
self.assertTrue(res['OK'], res)
# We should get only one file, so processed only once
res = self.db.getFiles('Active')
self.assertTrue(res['OK'], res)
self.assertEqual(len(res['Value']), 1)
res = self.db.removeFile(testFile['LFN'])
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], 2)
# Setting status of a non existing file
res = self.db.setFileStatus('fake', 'Done')
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], 0)
# Removing a non existing file
res = self.db.removeFile('fake')
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], 0)
def test_03_testUnmigratedFiles(self):
""" Test that getUnmigratedFiles returns only the files
in the appropriate status
"""
unmigratedStatus = ['Active', 'Copied', 'Registered']
for st in ['Active', 'Copied', 'Registered', 'Done', 'Fake']:
self.db.addFile("lfn%s" % st, 'PFN', 1, 'SE', 'GUID', 'Checksum')
self.db.setFileStatus('lfn%s' % st, st)
res = self.db.getUnmigratedFiles()
self.assertTrue(res['OK'], res)
self.assertEqual(sorted(res['Value']), sorted(['lfn%s' % st for st in unmigratedStatus]))
def test_04_webQueries(self):
""" Test all the web related methods"""
# The DB should be empty now
res = self.db.getGlobalStatistics()
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], {})
res = self.db.getFileSelections()
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], {'StorageElement': [], 'Status': []})
# Adding two files Assigned, 1 Failed and 1 done
for i in xrange(1, 5):
res = self.db.addFile('lfn%s' % i, 'pfn%s' % i, 5 - i, 'se%s' % (i % 2), 'GUID%s' % i,
'Checksum%s' % i)
self.assertTrue(res['OK'], res)
res = self.db.setFileStatus('lfn3', 'Done')
self.assertTrue(res['OK'], res)
res = self.db.setFileStatus('lfn4', 'Failed')
self.assertTrue(res['OK'], res)
res = self.db.getGlobalStatistics()
self.assertTrue(res['OK'], res)
self.assertEqual(res['Value'], {'Active': 2, 'Done': 1, 'Failed': 1})
res = self.db.getFileSelections()
self.assertTrue(res['OK'], res)
self.assertEqual(sorted(res['Value']['StorageElement']), sorted(['se0', 'se1']))
self.assertEqual(sorted(res['Value']['Status']), sorted(['Done', 'Failed', 'Active']))
# Do some selection test
res = self.db.selectFiles({'StorageElement': 'se0'})
self.assertTrue(res['OK'], res)
# they should be sorted by LFN by default
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn2', 'lfn4'])
res = self.db.selectFiles({'StorageElement': 'se0', 'PFN': 'pfn2'})
self.assertTrue(res['OK'], res)
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn2'])
# Impossible condition
res = self.db.selectFiles({'StorageElement': 'se1', 'PFN': 'pfn2'})
self.assertTrue(res['OK'], res)
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, [])
# limit to 1
res = self.db.selectFiles({'StorageElement': 'se0'}, limit=1)
self.assertTrue(res['OK'], res)
# they should be sorted by LFN by default
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn2'])
# Same selection, sorted by size
res = self.db.selectFiles({'StorageElement': 'se0'}, orderAttribute='Size')
self.assertTrue(res['OK'], res)
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn4', 'lfn2'])
# Test the time based selections
time.sleep(1)
now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
sleepTime = 2
time.sleep(sleepTime)
res = self.db.addFile('lfn6', 'pfn6', 6, 'se1', 'GUID6', 'Checksum6')
self.assertTrue(res['OK'], res)
# select the old files with no conditions
res = self.db.selectFiles({}, older=now)
self.assertTrue(res['OK'], res)
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn%i' % i for i in xrange(1, 5)])
# select the new file
res = self.db.selectFiles({}, newer=now)
self.assertTrue(res['OK'], res)
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn6'])
# add some more
time.sleep(1)
after = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
time.sleep(sleepTime)
res = self.db.addFile('lfn7', 'pfn7', 7, 'se1', 'GUID7', 'Checksum7')
self.assertTrue(res['OK'], res)
# We should now have two files after 'now'
res = self.db.selectFiles({}, newer=now)
self.assertTrue(res['OK'], res)
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn6', 'lfn7'])
# Only one file is before 'now' and 'after'
res = self.db.selectFiles({}, newer=now, older=after)
self.assertTrue(res['OK'], res)
returnedLfns = [ret[0] for ret in res['Value']]
self.assertEqual(returnedLfns, ['lfn6'])
# clean after us
for i in xrange(1, 8):
res = self.db.removeFile('lfn%s' % i)
self.assertTrue(res['OK'], res)
def test_05_perf(self):
""" Performance tests
For reminder, this is more ore less
the timing that were obtained on a super crapy
virtualMachine. Let's hope we never go above them..
criticalInsertTime = 34
criticalRetrieveTime = 1
criticalUpdateTime = 5
criticalRemoveTime = 30
"""
nbFiles = 5000
# Inserting files
startTime = time.time()
for i in xrange(nbFiles):
res = self.db.addFile('lfn%s' % i, 'pfn%s' % i, i, 'se%s' % (i % 2), 'GUID%s' % i,
'Checksum%s' % i)
self.assertTrue(res['OK'], res)
insertTime = time.time() - startTime
# Sleep 2 seconds so that the DB has
# a consistant commited state
time.sleep(2)
# getting all of them
startTime = time.time()
res = self.db.getFiles('Active')
self.assertTrue(res['OK'], res)
self.assertEqual(len(res['Value']), nbFiles)
getFileTime = time.time() - startTime
# Setting some of them
startTime = time.time()
rndIds = set()
for _ in xrange(nbFiles / 10):
rndId = random.randint(1, nbFiles)
rndIds.add(rndId)
self.db.setFileStatus('lfn%s' % rndId, 'Done')
self.assertTrue(res['OK'], res)
updateStatusTime = time.time() - startTime
# getting less of them
startTime = time.time()
res = self.db.getFiles('Active')
self.assertTrue(res['OK'], res)
self.assertEqual(len(res['Value']), nbFiles - len(rndIds))
getFileTime2 = time.time() - startTime
# deleting all of them
startTime = time.time()
for i in xrange(1, nbFiles):
res = self.db.removeFile('lfn%s' % i)
self.assertTrue(res['OK'], res)
removeTime = time.time() - startTime
print "Performance result"
print "Inserting %s files: %s" % (nbFiles, insertTime)
print "Getting all active files: %s" % getFileTime
print "Updating %s status: %s" % (nbFiles / 10, updateStatusTime)
print "Getting again active files: %s" % getFileTime2
print "Removing all files: %s" % removeTime