def testDeleteTransaction(self):
"""
_testDeleteTransaction_
Create a file and commit it to the database. Start a new transaction
and delete the file. Rollback the transaction after the file has been
deleted. Use the file class's exists() method to verify that the file
does not exist after it has been deleted but does exist after the
transaction is rolled back.
"""
testFile = File(lfn = "/this/is/a/lfn", size = 1024, events = 10,
checksums={'cksum': 1111})
assert testFile.exists() == False, \
"ERROR: File exists before it was created"
testFile.addRun(Run(1, *[45]))
testFile.create()
assert testFile.exists() > 0, \
"ERROR: File does not exist after it was created"
myThread = threading.currentThread()
myThread.transaction.begin()
testFile.delete()
assert testFile.exists() == False, \
"ERROR: File exists after it has been deleted"
myThread.transaction.rollback()
assert testFile.exists() > 0, \
"ERROR: File does not exist after transaction was rolled back."
return
def testCreateTransaction(self):
"""
_testCreateTransaction_
Begin a transaction and then create a file in the database. Afterwards,
rollback the transaction. Use the File class's exists() method to
to verify that the file doesn't exist before it was created, exists
after it was created and doesn't exist after the transaction was rolled
back.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
testFile = File(lfn = "/this/is/a/lfn", size = 1024, events = 10, checksums={'cksum':1111})
assert testFile.exists() == False, \
"ERROR: File exists before it was created"
testFile.addRun(Run(1, *[45]))
testFile.create()
assert testFile.exists() > 0, \
"ERROR: File does not exist after it was created"
myThread.transaction.rollback()
assert testFile.exists() == False, \
"ERROR: File exists after transaction was rolled back."
return
def testCreateTransaction(self):
"""
_testCreateTransaction_
Begin a transaction and then create a file in the database. Afterwards,
rollback the transaction. Use the File class's exists() method to
to verify that the file doesn't exist before it was created, exists
after it was created and doesn't exist after the transaction was rolled
back.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
testFile = File(lfn="/this/is/a/lfn",
size=1024,
events=10,
checksums={'cksum': 1111})
assert testFile.exists() == False, \
"ERROR: File exists before it was created"
testFile.addRun(Run(1, *[45]))
testFile.create()
assert testFile.exists() > 0, \
"ERROR: File does not exist after it was created"
myThread.transaction.rollback()
assert testFile.exists() == False, \
"ERROR: File exists after transaction was rolled back."
return
def testCreateDeleteExists(self):
"""
_testCreateDeleteExists_
Test the create(), delete() and exists() methods of the file class
by creating and deleting a file. The exists() method will be
called before and after creation and after deletion.
"""
testFile = File(lfn="/this/is/a/lfn",
size=1024,
events=10,
checksums={'cksum': 1111})
assert testFile.exists() == False, \
"ERROR: File exists before it was created"
testFile.addRun(Run(1, *[45]))
testFile.create()
assert testFile.exists() > 0, \
"ERROR: File does not exist after it was created"
testFile.delete()
assert testFile.exists() == False, \
"ERROR: File exists after it has been deleted"
return
def testCreateDeleteExists(self):
"""
_testCreateDeleteExists_
Test the create(), delete() and exists() methods of the file class
by creating and deleting a file. The exists() method will be
called before and after creation and after deletion.
"""
testFile = File(lfn="/this/is/a/lfn", size=1024, events=10, checksums={"cksum": 1111})
assert testFile.exists() == False, "ERROR: File exists before it was created"
testFile.addRun(Run(1, *[45]))
testFile.create()
assert testFile.exists() > 0, "ERROR: File does not exist after it was created"
testFile.delete()
assert testFile.exists() == False, "ERROR: File exists after it has been deleted"
return
def testDeleteTransaction(self):
"""
_testDeleteTransaction_
Create a file and commit it to the database. Start a new transaction
and delete the file. Rollback the transaction after the file has been
deleted. Use the file class's exists() method to verify that the file
does not exist after it has been deleted but does exist after the
transaction is rolled back.
"""
testFile = File(lfn="/this/is/a/lfn",
size=1024,
events=10,
checksums={'cksum': 1111})
assert testFile.exists() == False, \
"ERROR: File exists before it was created"
testFile.addRun(Run(1, *[45]))
testFile.create()
assert testFile.exists() > 0, \
"ERROR: File does not exist after it was created"
myThread = threading.currentThread()
myThread.transaction.begin()
testFile.delete()
assert testFile.exists() == False, \
"ERROR: File exists after it has been deleted"
myThread.transaction.rollback()
assert testFile.exists() > 0, \
"ERROR: File does not exist after transaction was rolled back."
return
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
global LOCK
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), os.getenv("DIALECT"), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)
locationNew = daofactory(classname="Locations.New")
getFileLoc = daofactory(classname="Files.GetLocation")
fileInFileset = daofactory(classname="Files.InFileset")
logging.debug("the T0Feeder is processing %s" % filesetToProcess.name)
logging.debug("the fileset name %s" % (filesetToProcess.name).split(":")[0])
# Get the start Run if asked
startRun = (filesetToProcess.name).split(":")[3]
fileType = (filesetToProcess.name).split(":")[2]
LASTIME = filesetToProcess.lastUpdate
# url builder
primaryDataset = ((filesetToProcess.name).split(":")[0]).split("/")[1]
processedDataset = ((filesetToProcess.name).split(":")[0]).split("/")[2]
dataTier = ((filesetToProcess.name).split(":")[0]).split("/")[3]
url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % (
fileType,
LASTIME,
primaryDataset,
processedDataset,
dataTier,
)
tries = 1
while True:
try:
myRequester = JSONRequests(url="vocms52.cern.ch:8889")
requestResult = myRequester.get(url + "/" + "?return_type=text/json%2Bdas")
newFilesList = requestResult[0]["results"]
except:
logging.debug("T0Reader call error...")
if tries == self.maxRetries:
return
else:
tries += 1
continue
logging.debug("T0 queries done ...")
now = time.time()
LASTIME = int(newFilesList["end_time"]) + 1
break
# process all files
if len(newFilesList["files"]):
try:
locationNew.execute(siteName="caf.cern.ch", seName="caf.cern.ch")
except Exception, e:
logging.debug("Error when adding new location...")
logging.debug(e)
logging.debug(format_exc())
for files in newFilesList["files"]:
# Assume parents aren't asked
newfile = File(str(files["lfn"]), size=files["file_size"], events=files["events"])
try:
LOCK.acquire()
if newfile.exists() == False:
newfile.create()
for run in files["runs"]:
runSet = set()
runSet.add(Run(run, *files["runs"][run]))
newfile.addRunSet(runSet)
else:
newfile.loadData()
fileLoc = getFileLoc.execute(file=files["lfn"])
if "caf.cern.ch" not in fileLoc:
newfile.setLocation("caf.cern.ch")
# else:
# logging.debug("File already associated to %s" %fileLoc)
LOCK.release()
if len(newfile["runs"]):
val = 0
for run in newfile["runs"]:
if run.run < int(startRun):
val = 1
break
if not val:
listFile = fileInFileset.execute(filesetToProcess.id)
if {"fileid": newfile["id"]} not in listFile:
filesetToProcess.addFile(newfile)
filesetToProcess.setLastUpdate(int(newFilesList["end_time"]) + 1)
filesetToProcess.commit()
logging.debug("new file created/loaded added by T0AST...")
except Exception, e:
logging.debug("Error when adding new files in T0AST...")
logging.debug(e)
logging.debug(format_exc())
LOCK.release()
if startRun != 'None':
if len(files['LumiList']):
for lumi in files['LumiList']:
if int(startRun) <= int(lumi['RunNumber']):
newfile = File(files['LogicalFileName'], \
size=files['FileSize'], events=files\
['NumberOfEvents'])
LOCK.acquire()
if newfile.exists() == False:
newfile.create()
filesetToProcess.addFile(newfile)
filesetToProcess.setLastUpdate(\
int(time.time()))
filesetToProcess.commit()
runSet = set()
runSet.add(Run( lumi\
['RunNumber' ], *[lumi['LumiSectionNumber']] ))
newfile.addRunSet(runSet)
else:
newfile.loadData()
if startRun != 'None':
if len(files['LumiList']):
for lumi in files['LumiList']:
if int(startRun) <= int(lumi['RunNumber' ]):
newfile = File(files['LogicalFileName'], \
size=files['FileSize'], events=files\
['NumberOfEvents'])
LOCK.acquire()
if newfile.exists() == False :
newfile.create()
filesetToProcess.addFile(newfile)
filesetToProcess.setLastUpdate(\
int(time.time()))
filesetToProcess.commit()
runSet = set()
runSet.add(Run( lumi\
['RunNumber' ], *[lumi['LumiSectionNumber']] ))
newfile.addRunSet(runSet)
else:
newfile.loadData()
def __call__(self, filesets):
"""
The algorithm itself
"""
# Update run list
self.getNewRuns()
# Do per fileset work, abandon fileset processing on exception
for fileset in filesets:
ds = fileset.name
try:
# Do per run work
watchCompleteFiles = []
for watch in self.watchedRuns:
# Ensure watcher has dataset listed
watch.addDatasetOfInterest(ds)
# Query DBS to find all blocks for this run / dataset
(files, blocks, fileInfoMap) = \
self.dbsHelper.getFileInfo(watch.run, ds)
# Now determine all required parent blocks
parentBlocks = set()
if fileset.requireParents:
parentDs = self.dbsHelper.getParentDataset(ds)
parentBlocks = self.dbsHelper.getBlockInfo(
watch.run, parentDs)
# Final list of all required blocks
allBlocks = blocks[:]
allBlocks.update(parentBlocks)
# Find all sites where all blocks are complete
sites = self.phedexHelper.getCompleteSites(blocks)
# Get sites with newly completed transfers
newSites = watch.getNewSites(ds, sites)
if len(newSites) > 0:
# Add the files for these blocks to the fileset
for file in fileInfoMap:
fi = fileInfoMap[file]
# First add parent file
if fileset.requireParents:
parentFile = File(lfn=fi["file.parent"])
parentFile.save()
parentFile.setLocation(newSites)
# Add actual file
fileToAdd = File(lfn=file,
size=fi["file.size"],
events=fi["file.events"],
run=watch.run,
umi=fi["file.lumi"])
if not fileToAdd.exists(
) and fileset.requireParents:
fileToAdd.addParent(fi["file.parent"])
# Add new locations but don't persist immediately
fileToAdd.setLocations(newSites,
immediateSave=False)
# Add the file to the new file list
fileset.addFile(fileToAdd)
# Add the site info to the watcher list
watchCompleteFiles.append([watch, ds, newSites])
# Commit the fileset
fileset.commit()
# Add the watched runs
for a in watchCompleteFiles:
a[0].addCompletedNodes(a[1], a[2])
except:
# Reset the watch list so we re-evaluate next call
watchCompleteFiles = []
# Purge old runs
self.purgeWatchedRuns()
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
global LOCK
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
locationNew = daofactory(classname="Locations.New")
getFileLoc = daofactory(classname="Files.GetLocation")
logging.debug("the T0Feeder is processing %s" % \
filesetToProcess.name)
logging.debug("the fileset name %s" % \
(filesetToProcess.name).split(":")[0])
startRun = (filesetToProcess.name).split(":")[3]
fileType = (filesetToProcess.name).split(":")[2]
# url builder
primaryDataset = ((filesetToProcess.name).split(":")[0]).split('/')[1]
processedDataset = ((
filesetToProcess.name).split(":")[0]).split('/')[2]
dataTier = (((filesetToProcess.name\
).split(":")[0]).split('/')[3]).split('-')[0]
# Fisrt call to T0 db for this fileset
# Here add test for the closed fileset
LASTIME = filesetToProcess.lastUpdate
url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % \
(fileType, LASTIME, primaryDataset,processedDataset, dataTier)
tries = 1
while True:
try:
myRequester = JSONRequests(url="vocms52.cern.ch:8889")
requestResult = myRequester.get(\
url+"/"+"?return_type=text/json%2Bdas")
newFilesList = requestResult[0]["results"]
except:
logging.debug("T0Reader call error...")
if tries == self.maxRetries:
return
else:
tries += 1
continue
logging.debug("T0ASTRun queries done ...")
now = time.time()
filesetToProcess.last_update = now
LASTIME = int(newFilesList['end_time']) + 1
break
# process all files
if len(newFilesList['files']):
LOCK.acquire()
try:
locationNew.execute(siteName="caf.cern.ch",
seName="caf.cern.ch")
except Exception as e:
logging.debug("Error when adding new location...")
logging.debug(e)
logging.debug(format_exc())
for files in newFilesList['files']:
# Assume parents aren't asked
newfile = File(str(files['lfn']), \
size = files['file_size'], events = files['events'])
try:
if newfile.exists() == False:
newfile.create()
else:
newfile.loadData()
#Add run test if already exist
for run in files['runs']:
if startRun != 'None' and int(startRun) <= int(run):
# ToDo: Distinguish between
# filestA-RunX and filesetA-Run[0-9]*
filesetRun = Fileset( name = (((\
filesetToProcess.name).split(':')[0]).split('/')[0]\
)+'/'+(((filesetToProcess.name).split(':')[0]).split\
('/')[1])+'/'+(((filesetToProcess.name).split(':')[0]\
).split('/')[2])+'/'+((((filesetToProcess.name).split\
(':')[0]).split('/')[3]).split('-')[0])+'-'+'Run'+str\
(run)+":"+":".join((filesetToProcess.name).split(':')[1:] \
) )
if filesetRun.exists() == False:
filesetRun.create()
else:
filesetRun.loadData()
# Add test runs already there
# (for growing dataset) -
# to support file with different runs and lumi
if not newfile['runs']:
runSet = set()
runSet.add(Run(run, *files['runs'][run]))
newfile.addRunSet(runSet)
fileLoc = getFileLoc.execute(file=files['lfn'])
if 'caf.cern.ch' not in fileLoc:
newfile.setLocation("caf.cern.ch")
filesetRun.addFile(newfile)
logging.debug(
"new file created/loaded added by T0ASTRun...")
filesetRun.commit()
except Exception as e:
logging.debug("Error when adding new files in T0ASTRun...")
logging.debug(e)
logging.debug(format_exc())
filesetToProcess.setLastUpdate\
(int(newFilesList['end_time']) + 1)
filesetToProcess.commit()
LOCK.release()
else:
logging.debug("nothing to do...")
# For re-opned fileset or empty, try until the purge time
if (int(now) / 3600 - LASTIME / 3600) > self.reopenTime:
filesetToProcess.setLastUpdate(time.time())
filesetToProcess.commit()
if LASTIME:
myRequester = JSONRequests(url="vocms52.cern.ch:8889")
requestResult = myRequester.get("/tier0/runs")
for listRun in requestResult[0]:
if int(startRun) <= int(listRun['run']):
if listRun['status'] =='CloseOutExport' or \
listRun['status'] =='Complete' or listRun['status'] ==\
'CloseOutT1Skimming':
closeFileset = Fileset( name = (((\
filesetToProcess.name).split(':')[0]).split('/')[0])+'/'+\
(((filesetToProcess.name).split(':')[0]).split('/')[1]\
)+'/'+(((filesetToProcess.name).split(':')[0]).split('/')\
[2])+'/'+((((filesetToProcess.name).split(':')[0]).split\
('/')[3]).split('-')[0])+'-'+'Run'+str(listRun['run'])\
+":"+":".join((filesetToProcess.name).split(':')[1:] ) )
if closeFileset.exists() != False:
closeFileset = Fileset(id=closeFileset.exists())
closeFileset.loadData()
if closeFileset.open == True:
closeFileset.markOpen(False)
# Commit the fileset
filesetToProcess.commit()
# Commit the fileset
logging.debug("Test purge in T0ASTRun ...")
filesetToProcess.load()
LASTIME = filesetToProcess.lastUpdate
if (int(now) / 3600 - LASTIME / 3600) > self.purgeTime:
filesetToProcess.markOpen(False)
logging.debug("Purge Done...")
filesetToProcess.commit()
def __call__(self, filesets):
"""
The algorithm itself
"""
# Update run list
self.getNewRuns()
# Do per fileset work, abandon fileset processing on exception
for fileset in filesets:
ds = fileset.name
try:
# Do per run work
watchCompleteFiles = []
for watch in self.watchedRuns:
# Ensure watcher has dataset listed
watch.addDatasetOfInterest(ds)
# Query DBS to find all blocks for this run / dataset
(files, blocks, fileInfoMap) = \
self.dbsHelper.getFileInfo(watch.run, ds)
# Now determine all required parent blocks
parentBlocks = set()
if fileset.requireParents:
parentDs = self.dbsHelper.getParentDataset(ds)
parentBlocks = self.dbsHelper.getBlockInfo(watch.run,
parentDs)
# Final list of all required blocks
allBlocks = blocks[:]
allBlocks.update(parentBlocks)
# Find all sites where all blocks are complete
sites = self.phedexHelper.getCompleteSites(blocks)
# Get sites with newly completed transfers
newSites = watch.getNewSites(ds, sites)
if len(newSites) > 0:
# Add the files for these blocks to the fileset
for file in fileInfoMap:
fi = fileInfoMap[file]
# First add parent file
if fileset.requireParents:
parentFile = File(lfn=fi["file.parent"])
parentFile.save()
parentFile.setLocation(newSites)
# Add actual file
fileToAdd = File(lfn=file, size=fi["file.size"],
events=fi["file.events"],
run=watch.run,
umi=fi["file.lumi"])
if not fileToAdd.exists() and fileset.requireParents:
fileToAdd.addParent(fi["file.parent"])
# Add new locations but don't persist immediately
fileToAdd.setLocations(newSites, immediateSave=False)
# Add the file to the new file list
fileset.addFile(fileToAdd)
# Add the site info to the watcher list
watchCompleteFiles.append([watch, ds, newSites])
# Commit the fileset
fileset.commit()
# Add the watched runs
for a in watchCompleteFiles:
a[0].addCompletedNodes(a[1], a[2])
except:
# Reset the watch list so we re-evaluate next call
watchCompleteFiles = []
# Purge old runs
self.purgeWatchedRuns()
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
global LOCK
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
locationNew = daofactory(classname="Locations.New")
getFileLoc = daofactory(classname="Files.GetLocation")
fileInFileset = daofactory(classname="Files.InFileset")
logging.debug("the T0Feeder is processing %s" % \
filesetToProcess.name)
logging.debug("the fileset name %s" % \
(filesetToProcess.name).split(":")[0])
# Get the start Run if asked
startRun = (filesetToProcess.name).split(":")[3]
fileType = (filesetToProcess.name).split(":")[2]
LASTIME = filesetToProcess.lastUpdate
# url builder
primaryDataset = ((filesetToProcess.name).split(":")[0]).split('/')[1]
processedDataset = ((
filesetToProcess.name).split(":")[0]).split('/')[2]
dataTier = ((filesetToProcess.name\
).split(":")[0]).split('/')[3]
url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % \
(fileType, LASTIME, primaryDataset,processedDataset, dataTier)
tries = 1
while True:
try:
myRequester = JSONRequests(url="vocms52.cern.ch:8889")
requestResult = myRequester.get(\
url+"/"+"?return_type=text/json%2Bdas")
newFilesList = requestResult[0]["results"]
except:
logging.debug("T0Reader call error...")
if tries == self.maxRetries:
return
else:
tries += 1
continue
logging.debug("T0 queries done ...")
now = time.time()
LASTIME = int(newFilesList['end_time']) + 1
break
# process all files
if len(newFilesList['files']):
try:
locationNew.execute(siteName="caf.cern.ch",
seName="caf.cern.ch")
except Exception as e:
logging.debug("Error when adding new location...")
logging.debug(e)
logging.debug(format_exc())
for files in newFilesList['files']:
# Assume parents aren't asked
newfile = File(str(files['lfn']), \
size = files['file_size'], events = files['events'])
try:
LOCK.acquire()
if newfile.exists() == False:
newfile.create()
for run in files['runs']:
runSet = set()
runSet.add(Run(run, *files['runs'][run]))
newfile.addRunSet(runSet)
else:
newfile.loadData()
fileLoc = getFileLoc.execute(file=files['lfn'])
if 'caf.cern.ch' not in fileLoc:
newfile.setLocation("caf.cern.ch")
# else:
# logging.debug("File already associated to %s" %fileLoc)
LOCK.release()
if len(newfile["runs"]):
val = 0
for run in newfile['runs']:
if run.run < int(startRun):
val = 1
break
if not val:
listFile = fileInFileset.execute\
(filesetToProcess.id)
if {'fileid': newfile['id']} not in listFile:
filesetToProcess.addFile(newfile)
filesetToProcess.setLastUpdate\
(int(newFilesList['end_time']) + 1)
filesetToProcess.commit()
logging.debug(
"new file created/loaded added by T0AST..."
)
except Exception as e:
logging.debug("Error when adding new files in T0AST...")
logging.debug(e)
logging.debug(format_exc())
LOCK.release()
filesetToProcess.commit()
else:
logging.debug("nothing to do in T0AST...")
# For reopned fileset or empty
# try until the purge time is reached
if (int(now) / 3600 - LASTIME / 3600) > self.reopenTime:
filesetToProcess.setLastUpdate(time.time())
filesetToProcess.commit()
# Commit the fileset
logging.debug("Test purge in T0AST ...")
filesetToProcess.load()
LASTIME = filesetToProcess.lastUpdate
if (int(now) / 3600 - LASTIME / 3600) > self.purgeTime:
filesetToProcess.markOpen(False)
logging.debug("Purge Done...")
filesetToProcess.commit()
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
global LOCK
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
locationNew = daofactory(classname = "Locations.New")
getFileLoc = daofactory(classname = "Files.GetLocation")
logging.debug("the T0Feeder is processing %s" % \
filesetToProcess.name)
logging.debug("the fileset name %s" % \
(filesetToProcess.name).split(":")[0])
startRun = (filesetToProcess.name).split(":")[3]
fileType = (filesetToProcess.name).split(":")[2]
# url builder
primaryDataset = ((filesetToProcess.name).split(":")[0]).split('/')[1]
processedDataset = ((filesetToProcess.name).split(":")[0]).split('/')[2]
dataTier = (((filesetToProcess.name\
).split(":")[0]).split('/')[3]).split('-')[0]
# Fisrt call to T0 db for this fileset
# Here add test for the closed fileset
LASTIME = filesetToProcess.lastUpdate
url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % \
(fileType, LASTIME, primaryDataset,processedDataset, dataTier)
tries = 1
while True:
try:
myRequester = JSONRequests(url = "vocms52.cern.ch:8889")
requestResult = myRequester.get(\
url+"/"+"?return_type=text/json%2Bdas")
newFilesList = requestResult[0]["results"]
except:
logging.debug("T0Reader call error...")
if tries == self.maxRetries:
return
else:
tries += 1
continue
logging.debug("T0ASTRun queries done ...")
now = time.time()
filesetToProcess.last_update = now
LASTIME = int(newFilesList['end_time']) + 1
break
# process all files
if len(newFilesList['files']):
LOCK.acquire()
try:
locationNew.execute(siteName = "caf.cern.ch", seName = "caf.cern.ch")
except Exception as e:
logging.debug("Error when adding new location...")
logging.debug(e)
logging.debug( format_exc() )
for files in newFilesList['files']:
# Assume parents aren't asked
newfile = File(str(files['lfn']), \
size = files['file_size'], events = files['events'])
try:
if newfile.exists() == False :
newfile.create()
else:
newfile.loadData()
#Add run test if already exist
for run in files['runs']:
if startRun != 'None' and int(startRun) <= int(run):
# ToDo: Distinguish between
# filestA-RunX and filesetA-Run[0-9]*
filesetRun = Fileset( name = (((\
filesetToProcess.name).split(':')[0]).split('/')[0]\
)+'/'+(((filesetToProcess.name).split(':')[0]).split\
('/')[1])+'/'+(((filesetToProcess.name).split(':')[0]\
).split('/')[2])+'/'+((((filesetToProcess.name).split\
(':')[0]).split('/')[3]).split('-')[0])+'-'+'Run'+str\
(run)+":"+":".join((filesetToProcess.name).split(':')[1:] \
) )
if filesetRun.exists() == False :
filesetRun.create()
else:
filesetRun.loadData()
# Add test runs already there
# (for growing dataset) -
# to support file with different runs and lumi
if not newfile['runs']:
runSet = set()
runSet.add(Run( run, *files['runs'][run]))
newfile.addRunSet(runSet)
fileLoc = getFileLoc.execute(file = files['lfn'])
if 'caf.cern.ch' not in fileLoc:
newfile.setLocation("caf.cern.ch")
filesetRun.addFile(newfile)
logging.debug("new file created/loaded added by T0ASTRun...")
filesetRun.commit()
except Exception as e:
logging.debug("Error when adding new files in T0ASTRun...")
logging.debug(e)
logging.debug( format_exc() )
filesetToProcess.setLastUpdate\
(int(newFilesList['end_time']) + 1)
filesetToProcess.commit()
LOCK.release()
else:
logging.debug("nothing to do...")
# For re-opned fileset or empty, try until the purge time
if (int(now)/3600 - LASTIME/3600) > self.reopenTime:
filesetToProcess.setLastUpdate(time.time())
filesetToProcess.commit()
if LASTIME:
myRequester = JSONRequests(url = "vocms52.cern.ch:8889")
requestResult = myRequester.get("/tier0/runs")
for listRun in requestResult[0]:
if int(startRun) <= int(listRun['run']):
if listRun['status'] =='CloseOutExport' or \
listRun['status'] =='Complete' or listRun['status'] ==\
'CloseOutT1Skimming':
closeFileset = Fileset( name = (((\
filesetToProcess.name).split(':')[0]).split('/')[0])+'/'+\
(((filesetToProcess.name).split(':')[0]).split('/')[1]\
)+'/'+(((filesetToProcess.name).split(':')[0]).split('/')\
[2])+'/'+((((filesetToProcess.name).split(':')[0]).split\
('/')[3]).split('-')[0])+'-'+'Run'+str(listRun['run'])\
+":"+":".join((filesetToProcess.name).split(':')[1:] ) )
if closeFileset.exists() != False :
closeFileset = Fileset( id = closeFileset.exists())
closeFileset.loadData()
if closeFileset.open == True:
closeFileset.markOpen(False)
# Commit the fileset
filesetToProcess.commit()
# Commit the fileset
logging.debug("Test purge in T0ASTRun ...")
filesetToProcess.load()
LASTIME = filesetToProcess.lastUpdate
if (int(now)/3600 - LASTIME/3600) > self.purgeTime:
filesetToProcess.markOpen(False)
logging.debug("Purge Done...")
filesetToProcess.commit()
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
locationNew = daofactory(classname = "Locations.New")
getFileLoc = daofactory(classname = "Files.GetLocation")
fileInFileset = daofactory(classname = "Files.InFileset")
logging.debug("DBSFeeder is processing %s" % \
filesetToProcess.name)
logging.debug("the filesetBase name %s" \
% (filesetToProcess.name).split(":")[0])
LASTIME = filesetToProcess.lastUpdate
# Get the start Run if asked
startRun = (filesetToProcess.name).split(":")[3]
# get list of files
tries = 1
while True:
try:
blocks = self.dbsReader.getFiles(\
(filesetToProcess.name).split(":")[0])
now = time.time()
logging.debug("DBS queries done ...")
break
except DBSReaderError as ex:
logging.error("DBS error: %s, cannot get files for %s" % \
(str(ex), filesetToProcess.name))
# Close fileset
filesetToProcess.markOpen(False)
return
# connection error, retry
except DbsConnectionError as ex:
logging.error("Unable to connect to DBS, retrying: " + \
str(ex))
if tries > self.connectionAttempts: #too many errors - bail out
return
tries = tries + 1
# check for empty datasets
if blocks == {}:
logging.debug("DBS: Empty blocks - %s" %filesetToProcess.name)
return filesetToProcess
# get all file blocks
blockList = blocks.keys()
# process all file blocks
for fileBlock in blockList:
seList = blocks[fileBlock]['StorageElements']
# add files for non blocked SE
if seList is None or seList == []:
logging.info("fileblock %s - no SE's associated" % \
fileBlock)
continue
else:
for loc in seList:
locationNew.execute(siteName = loc, seName = loc)
for files in blocks[fileBlock]['Files']:
if startRun != 'None':
if len(files['LumiList']):
for lumi in files['LumiList']:
if int(startRun) <= int(lumi['RunNumber' ]):
newfile = File(files['LogicalFileName'], \
size=files['FileSize'], events=files\
['NumberOfEvents'])
LOCK.acquire()
if newfile.exists() == False :
newfile.create()
filesetToProcess.addFile(newfile)
filesetToProcess.setLastUpdate(\
int(time.time()))
filesetToProcess.commit()
runSet = set()
runSet.add(Run( lumi\
['RunNumber' ], *[lumi['LumiSectionNumber']] ))
newfile.addRunSet(runSet)
else:
newfile.loadData()
listFile = fileInFileset.execute\
(filesetToProcess.id)
if {'fileid': newfile[\
'id']} not in listFile:
filesetToProcess.addFile(newfile)
filesetToProcess.setLastUpdate\
(int(time.time()))
filesetToProcess.commit()
val = 0
for run in newfile['runs']:
if lumi['RunNumber' ] == run.run:
val = 1
break
if not val:
runSet = set()
runSet.add(Run(\
lumi['RunNumber' ], *[lumi['LumiSectionNumber']]))
newfile.addRunSet(runSet)
fileLoc = getFileLoc.execute(\
file = files['LogicalFileName'])
if fileLoc:
for loc in seList:
if loc not in fileLoc:
newfile.setLocation(\
loc)
else:
newfile.setLocation(seList)
LOCK.release()
else:
# Assume parents and LumiSection aren't asked
newfile = File(files['LogicalFileName'], \
size=files['FileSize'], events=files['NumberOfEvents'])
LOCK.acquire()
if newfile.exists() == False :
newfile.create()
# Update fileset last update parameter
filesetToProcess.addFile(newfile)
logging.debug("new file created and added by DBS")
filesetToProcess.setLastUpdate(int(time.time()))
filesetToProcess.commit()
else:
newfile.loadData()
listFile = fileInFileset.execute(filesetToProcess.id)
if {'fileid': newfile['id']} not in listFile:
filesetToProcess.addFile(newfile)
logging.debug("new file loaded and added by DBS")
filesetToProcess.setLastUpdate(int(time.time()))
filesetToProcess.commit()
fileLoc = getFileLoc.execute(\
file = files['LogicalFileName'])
if fileLoc:
for loc in seList:
if loc not in fileLoc:
newfile.setLocation(loc)
else:
newfile.setLocation(seList)
LOCK.release()
filesetToProcess.load()
LASTIME = filesetToProcess.lastUpdate
# For re-opned fileset or empty, try until the purge time
if (int(now)/3600 - LASTIME/3600) > self.reopenTime:
filesetToProcess.setLastUpdate(int(time.time()))
filesetToProcess.commit()
if (int(now)/3600 - LASTIME/3600) > self.purgeTime:
filesetToProcess.markOpen(False)
logging.debug("Purge Done...")
filesetToProcess.commit()
logging.debug("DBS feeder work done...")