def testDataStructsFile(self):
"""
_testDataStructsFile_
Tests our ability to create a WMBS file from a DataStructs File and vice versa
"""
myThread = threading.currentThread()
testLFN = "lfn1"
testSize = 1024
testEvents = 100
testCksum = {"cksum": '1'}
testParents = set(["lfn2"])
testRun = Run(1, *[45])
testSE = "se1.cern.ch"
parentFile = File(lfn="lfn2")
parentFile.create()
testFile = File()
inputFile = WMFile(lfn=testLFN,
size=testSize,
events=testEvents,
checksums=testCksum,
parents=testParents)
inputFile.addRun(testRun)
inputFile.setLocation(se=testSE)
testFile.loadFromDataStructsFile(file=inputFile)
testFile.create()
testFile.save()
loadFile = File(lfn="lfn1")
loadFile.loadData(parentage=1)
self.assertEqual(loadFile['size'], testSize)
self.assertEqual(loadFile['events'], testEvents)
self.assertEqual(loadFile['checksums'], testCksum)
self.assertEqual(loadFile['locations'], set([testSE]))
#self.assertEqual(loadFile['parents'].pop()['lfn'], 'lfn2')
wmFile = loadFile.returnDataStructsFile()
self.assertEqual(wmFile == inputFile, True)
return
def testDataStructsFile(self):
"""
_testDataStructsFile_
Tests our ability to create a WMBS file from a DataStructs File and vice versa
"""
myThread = threading.currentThread()
testLFN = "lfn1"
testSize = 1024
testEvents = 100
testCksum = {"cksum": '1'}
testParents = set(["lfn2"])
testRun = Run( 1, *[45])
testSE = "se1.cern.ch"
parentFile = File(lfn= "lfn2")
parentFile.create()
testFile = File()
inputFile = WMFile(lfn = testLFN, size = testSize, events = testEvents, checksums = testCksum, parents = testParents)
inputFile.addRun(testRun)
inputFile.setLocation(se = testSE)
testFile.loadFromDataStructsFile(file = inputFile)
testFile.create()
testFile.save()
loadFile = File(lfn = "lfn1")
loadFile.loadData(parentage = 1)
self.assertEqual(loadFile['size'], testSize)
self.assertEqual(loadFile['events'], testEvents)
self.assertEqual(loadFile['checksums'], testCksum)
self.assertEqual(loadFile['locations'], set([testSE]))
#self.assertEqual(loadFile['parents'].pop()['lfn'], 'lfn2')
wmFile = loadFile.returnDataStructsFile()
self.assertEqual(wmFile == inputFile, True)
return
def __call__(self, filesets):
"""
The algorithm itself
"""
# Update run list
self.getNewRuns()
# Do per fileset work, abandon fileset processing on exception
for fileset in filesets:
ds = fileset.name
try:
# Do per run work
watchCompleteFiles = []
for watch in self.watchedRuns:
# Ensure watcher has dataset listed
watch.addDatasetOfInterest(ds)
# Query DBS to find all blocks for this run / dataset
(files, blocks, fileInfoMap) = \
self.dbsHelper.getFileInfo(watch.run, ds)
# Now determine all required parent blocks
parentBlocks = set()
if fileset.requireParents:
parentDs = self.dbsHelper.getParentDataset(ds)
parentBlocks = self.dbsHelper.getBlockInfo(
watch.run, parentDs)
# Final list of all required blocks
allBlocks = blocks[:]
allBlocks.update(parentBlocks)
# Find all sites where all blocks are complete
sites = self.phedexHelper.getCompleteSites(blocks)
# Get sites with newly completed transfers
newSites = watch.getNewSites(ds, sites)
if len(newSites) > 0:
# Add the files for these blocks to the fileset
for file in fileInfoMap:
fi = fileInfoMap[file]
# First add parent file
if fileset.requireParents:
parentFile = File(lfn=fi["file.parent"])
parentFile.save()
parentFile.setLocation(newSites)
# Add actual file
fileToAdd = File(lfn=file,
size=fi["file.size"],
events=fi["file.events"],
run=watch.run,
umi=fi["file.lumi"])
if not fileToAdd.exists(
) and fileset.requireParents:
fileToAdd.addParent(fi["file.parent"])
# Add new locations but don't persist immediately
fileToAdd.setLocations(newSites,
immediateSave=False)
# Add the file to the new file list
fileset.addFile(fileToAdd)
# Add the site info to the watcher list
watchCompleteFiles.append([watch, ds, newSites])
# Commit the fileset
fileset.commit()
# Add the watched runs
for a in watchCompleteFiles:
a[0].addCompletedNodes(a[1], a[2])
except:
# Reset the watch list so we re-evaluate next call
watchCompleteFiles = []
# Purge old runs
self.purgeWatchedRuns()
def __call__(self, filesets):
"""
The algorithm itself
"""
# Update run list
self.getNewRuns()
# Do per fileset work, abandon fileset processing on exception
for fileset in filesets:
ds = fileset.name
try:
# Do per run work
watchCompleteFiles = []
for watch in self.watchedRuns:
# Ensure watcher has dataset listed
watch.addDatasetOfInterest(ds)
# Query DBS to find all blocks for this run / dataset
(files, blocks, fileInfoMap) = \
self.dbsHelper.getFileInfo(watch.run, ds)
# Now determine all required parent blocks
parentBlocks = set()
if fileset.requireParents:
parentDs = self.dbsHelper.getParentDataset(ds)
parentBlocks = self.dbsHelper.getBlockInfo(watch.run,
parentDs)
# Final list of all required blocks
allBlocks = blocks[:]
allBlocks.update(parentBlocks)
# Find all sites where all blocks are complete
sites = self.phedexHelper.getCompleteSites(blocks)
# Get sites with newly completed transfers
newSites = watch.getNewSites(ds, sites)
if len(newSites) > 0:
# Add the files for these blocks to the fileset
for file in fileInfoMap:
fi = fileInfoMap[file]
# First add parent file
if fileset.requireParents:
parentFile = File(lfn=fi["file.parent"])
parentFile.save()
parentFile.setLocation(newSites)
# Add actual file
fileToAdd = File(lfn=file, size=fi["file.size"],
events=fi["file.events"],
run=watch.run,
umi=fi["file.lumi"])
if not fileToAdd.exists() and fileset.requireParents:
fileToAdd.addParent(fi["file.parent"])
# Add new locations but don't persist immediately
fileToAdd.setLocations(newSites, immediateSave=False)
# Add the file to the new file list
fileset.addFile(fileToAdd)
# Add the site info to the watcher list
watchCompleteFiles.append([watch, ds, newSites])
# Commit the fileset
fileset.commit()
# Add the watched runs
for a in watchCompleteFiles:
a[0].addCompletedNodes(a[1], a[2])
except:
# Reset the watch list so we re-evaluate next call
watchCompleteFiles = []
# Purge old runs
self.purgeWatchedRuns()