def loadFilesByBlock(self, blockname):
"""
_loadFilesByBlock_
Get all files associated with a block
"""
dbsFiles = []
findFiles = self.daoFactory(classname="LoadFilesByBlock")
results = findFiles.execute(blockname=blockname, transaction=False)
for entry in results:
# Add loaded information
dbsfile = DBSBufferFile(id=entry['id'])
dbsfile.update(entry)
dbsFiles.append(dbsfile)
for dbsfile in dbsFiles:
if 'runInfo' in dbsfile.keys():
# Then we have to replace it with a real run
for r in dbsfile['runInfo'].keys():
run = Run(runNumber=r)
run.extend(dbsfile['runInfo'][r])
dbsfile.addRun(run)
del dbsfile['runInfo']
if 'parentLFNs' in dbsfile.keys():
# Then we have some parents
for lfn in dbsfile['parentLFNs']:
newFile = DBSBufferFile(lfn=lfn)
dbsfile['parents'].add(newFile)
del dbsfile['parentLFNs']
return dbsFiles
def loadFilesByBlock(self, blockname):
"""
_loadFilesByBlock_
Get all files associated with a block
"""
dbsFiles = []
findFiles = self.daoFactory(classname = "LoadFilesByBlock")
results = findFiles.execute(blockname = blockname, transaction = False)
for entry in results:
# Add loaded information
dbsfile = DBSBufferFile(id=entry['id'])
dbsfile.update(entry)
dbsFiles.append(dbsfile)
for dbsfile in dbsFiles:
if 'runInfo' in dbsfile.keys():
# Then we have to replace it with a real run
for r in dbsfile['runInfo'].keys():
run = Run(runNumber = r)
run.extend(dbsfile['runInfo'][r])
dbsfile.addRun(run)
del dbsfile['runInfo']
if 'parentLFNs' in dbsfile.keys():
# Then we have some parents
for lfn in dbsfile['parentLFNs']:
newFile = DBSBufferFile(lfn = lfn)
dbsfile['parents'].add(newFile)
del dbsfile['parentLFNs']
return dbsFiles
def findUploadableFilesByDAS(self, datasetpath):
"""
_findUploadableDAS_
Find all the uploadable files for a given DatasetPath.
"""
dbsFiles = []
findFiles = self.daoFactory(classname="LoadDBSFilesByDAS")
results = findFiles.execute(datasetpath=datasetpath, transaction=False)
for entry in results:
# Add loaded information
dbsfile = DBSBufferFile(id=entry['id'])
dbsfile.update(entry)
dbsFiles.append(dbsfile)
for dbsfile in dbsFiles:
if 'runInfo' in dbsfile.keys():
# Then we have to replace it with a real run
for r in dbsfile['runInfo'].keys():
run = Run(runNumber=r)
run.extend(dbsfile['runInfo'][r])
dbsfile.addRun(run)
del dbsfile['runInfo']
if 'parentLFNs' in dbsfile.keys():
# Then we have some parents
for lfn in dbsfile['parentLFNs']:
newFile = DBSBufferFile(lfn=lfn)
dbsfile['parents'].add(newFile)
del dbsfile['parentLFNs']
return dbsFiles
def findUploadableFilesByDAS(self, datasetpath):
"""
_findUploadableDAS_
Find all the uploadable files for a given DatasetPath.
"""
dbsFiles = []
findFiles = self.daoFactory(classname = "LoadDBSFilesByDAS")
results = findFiles.execute(datasetpath = datasetpath, transaction = False)
for entry in results:
# Add loaded information
dbsfile = DBSBufferFile(id=entry['id'])
dbsfile.update(entry)
dbsFiles.append(dbsfile)
for dbsfile in dbsFiles:
if 'runInfo' in dbsfile.keys():
# Then we have to replace it with a real run
for r in dbsfile['runInfo'].keys():
run = Run(runNumber = r)
run.extend(dbsfile['runInfo'][r])
dbsfile.addRun(run)
del dbsfile['runInfo']
if 'parentLFNs' in dbsfile.keys():
# Then we have some parents
for lfn in dbsfile['parentLFNs']:
newFile = DBSBufferFile(lfn = lfn)
dbsfile['parents'].add(newFile)
del dbsfile['parentLFNs']
return dbsFiles
def loadDBSBufferFilesBulk(self, fileObjs):
"""
_loadDBSBufferFilesBulk_
Yes, this is a stupid place to put it.
No, there's not better place.
"""
myThread = threading.currentThread()
dbsFiles = []
existingTransaction = self.beginTransaction()
binds = []
for f in fileObjs:
binds.append(f["id"])
loadFiles = self.daoFactory(classname = "DBSBufferFiles.LoadBulkFilesByID")
results = loadFiles.execute(files = binds, conn = self.getDBConn(),
transaction = self.existingTransaction())
for entry in results:
# Add loaded information
dbsfile = DBSBufferFile(id=entry['id'])
dbsfile.update(entry)
dbsFiles.append(dbsfile)
for dbsfile in dbsFiles:
if 'runInfo' in dbsfile.keys():
# Then we have to replace it with a real run
for r in dbsfile['runInfo'].keys():
run = Run(runNumber = r)
run.extend(dbsfile['runInfo'][r])
dbsfile.addRun(run)
del dbsfile['runInfo']
if 'parentLFNs' in dbsfile.keys():
# Then we have some parents
for lfn in dbsfile['parentLFNs']:
newFile = DBSBufferFile(lfn = lfn)
dbsfile['parents'].add(newFile)
del dbsfile['parentLFNs']
self.commitTransaction(existingTransaction)
return dbsFiles
def loadDBSBufferFilesBulk(self, fileObjs):
"""
_loadDBSBufferFilesBulk_
Yes, this is a stupid place to put it.
No, there's not better place.
"""
myThread = threading.currentThread()
dbsFiles = []
existingTransaction = self.beginTransaction()
binds = []
for f in fileObjs:
binds.append(f["id"])
loadFiles = self.daoFactory(classname = "DBSBufferFiles.LoadBulkFilesByID")
results = loadFiles.execute(files = binds, conn = self.getDBConn(),
transaction = self.existingTransaction())
for entry in results:
# Add loaded information
dbsfile = DBSBufferFile(id=entry['id'])
dbsfile.update(entry)
dbsFiles.append(dbsfile)
for dbsfile in dbsFiles:
if 'runInfo' in dbsfile.keys():
# Then we have to replace it with a real run
for r in dbsfile['runInfo'].keys():
run = Run(runNumber = r)
run.extend(dbsfile['runInfo'][r])
dbsfile.addRun(run)
del dbsfile['runInfo']
if 'parentLFNs' in dbsfile.keys():
# Then we have some parents
for lfn in dbsfile['parentLFNs']:
newFile = DBSBufferFile(lfn = lfn)
dbsfile['parents'].add(newFile)
del dbsfile['parentLFNs']
self.commitTransaction(existingTransaction)
return dbsFiles
def findUploadableFilesByDAS(self, das):
"""
_findUploadableDAS_
Find all the Dataset-Algo files available
with uploadable files.
"""
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
dbsFiles = []
factory = DAOFactory(package = "WMComponent.DBSUpload.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
findFiles = factory(classname = "LoadDBSFilesByDAS")
results = findFiles.execute(das = das,
conn = self.getDBConn(),
transaction=self.existingTransaction())
for entry in results:
# Add loaded information
dbsfile = DBSBufferFile(id=entry['id'])
dbsfile.update(entry)
dbsFiles.append(dbsfile)
for dbsfile in dbsFiles:
if 'runInfo' in dbsfile.keys():
# Then we have to replace it with a real run
for r in dbsfile['runInfo'].keys():
run = Run(runNumber = r)
run.extend(dbsfile['runInfo'][r])
dbsfile.addRun(run)
del dbsfile['runInfo']
if 'parentLFNs' in dbsfile.keys():
# Then we have some parents
for lfn in dbsfile['parentLFNs']:
newFile = DBSBufferFile(lfn = lfn)
dbsfile['parents'].add(newFile)
del dbsfile['parentLFNs']
self.commitTransaction(existingTransaction)
return dbsFiles
def findUploadableFilesByDAS(self, das):
"""
_findUploadableDAS_
Find all the Dataset-Algo files available
with uploadable files.
"""
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
dbsFiles = []
factory = DAOFactory(package="WMComponent.DBSUpload.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
findFiles = factory(classname="LoadDBSFilesByDAS")
results = findFiles.execute(das=das,
conn=self.getDBConn(),
transaction=self.existingTransaction())
for entry in results:
# Add loaded information
dbsfile = DBSBufferFile(id=entry['id'])
dbsfile.update(entry)
dbsFiles.append(dbsfile)
for dbsfile in dbsFiles:
if 'runInfo' in dbsfile.keys():
# Then we have to replace it with a real run
for r in dbsfile['runInfo'].keys():
run = Run(runNumber=r)
run.extend(dbsfile['runInfo'][r])
dbsfile.addRun(run)
del dbsfile['runInfo']
if 'parentLFNs' in dbsfile.keys():
# Then we have some parents
for lfn in dbsfile['parentLFNs']:
newFile = DBSBufferFile(lfn=lfn)
dbsfile['parents'].add(newFile)
del dbsfile['parentLFNs']
self.commitTransaction(existingTransaction)
return dbsFiles
def loadFilesFromBlocks(self, blockID):
"""
_loadFilesFromBlocks_
Load the files from all active blocks
"""
findFiles = self.factory(classname = "LoadFilesFromBlocks")
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
dbsFiles = []
results = findFiles.execute(blockID = blockID,
conn = self.getDBConn(),
transaction=self.existingTransaction())
for entry in results:
# Add loaded information
dbsfile = DBSBufferFile(id=entry['id'])
dbsfile.update(entry)
dbsFiles.append(dbsfile)
for dbsfile in dbsFiles:
if 'runInfo' in dbsfile.keys():
# Then we have to replace it with a real run
for r in dbsfile['runInfo'].keys():
run = Run(runNumber = r)
run.extend(dbsfile['runInfo'][r])
dbsfile.addRun(run)
del dbsfile['runInfo']
if 'parentLFNs' in dbsfile.keys():
# Then we have some parents
for lfn in dbsfile['parentLFNs']:
newFile = DBSBufferFile(lfn = lfn)
dbsfile['parents'].add(newFile)
del dbsfile['parentLFNs']
self.commitTransaction(existingTransaction)
return dbsFiles
def loadFilesFromBlocks(self, blockID):
"""
_loadFilesFromBlocks_
Load the files from all active blocks
"""
findFiles = self.factory(classname="LoadFilesFromBlocks")
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
dbsFiles = []
results = findFiles.execute(blockID=blockID,
conn=self.getDBConn(),
transaction=self.existingTransaction())
for entry in results:
# Add loaded information
dbsfile = DBSBufferFile(id=entry['id'])
dbsfile.update(entry)
dbsFiles.append(dbsfile)
for dbsfile in dbsFiles:
if 'runInfo' in dbsfile.keys():
# Then we have to replace it with a real run
for r in dbsfile['runInfo'].keys():
run = Run(runNumber=r)
run.extend(dbsfile['runInfo'][r])
dbsfile.addRun(run)
del dbsfile['runInfo']
if 'parentLFNs' in dbsfile.keys():
# Then we have some parents
for lfn in dbsfile['parentLFNs']:
newFile = DBSBufferFile(lfn=lfn)
dbsfile['parents'].add(newFile)
del dbsfile['parentLFNs']
self.commitTransaction(existingTransaction)
return dbsFiles
def findUploadableFilesByDAS(self, das):
"""
_findUploadableDAS_
Find all the Dataset-Algo files available
with uploadable files.
"""
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
dbsFiles = []
findFiles = self.daoFactory(classname="LoadDBSFilesByDAS")
results = findFiles.execute(das=das, conn=self.getDBConn(), transaction=self.existingTransaction())
for entry in results:
# Add loaded information
dbsfile = DBSBufferFile(id=entry["id"])
dbsfile.update(entry)
dbsFiles.append(dbsfile)
for dbsfile in dbsFiles:
if "runInfo" in dbsfile.keys():
# Then we have to replace it with a real run
for r in dbsfile["runInfo"].keys():
run = Run(runNumber=r)
run.extend(dbsfile["runInfo"][r])
dbsfile.addRun(run)
del dbsfile["runInfo"]
if "parentLFNs" in dbsfile.keys():
# Then we have some parents
for lfn in dbsfile["parentLFNs"]:
newFile = DBSBufferFile(lfn=lfn)
dbsfile["parents"].add(newFile)
del dbsfile["parentLFNs"]
self.commitTransaction(existingTransaction)
return dbsFiles
def loadFilesByBlock(self, blockname):
"""
_loadFilesByBlock_
Get all files associated with a block
"""
dbsFiles = []
existingTransaction = self.beginTransaction()
findFiles = self.daoFactory(classname="LoadFilesByBlock")
results = findFiles.execute(blockname=blockname, conn=self.getDBConn(), transaction=self.existingTransaction())
for entry in results:
# Add loaded information
dbsfile = DBSBufferFile(id=entry["id"])
dbsfile.update(entry)
dbsFiles.append(dbsfile)
for dbsfile in dbsFiles:
if "runInfo" in dbsfile.keys():
# Then we have to replace it with a real run
for r in dbsfile["runInfo"].keys():
run = Run(runNumber=r)
run.extend(dbsfile["runInfo"][r])
dbsfile.addRun(run)
del dbsfile["runInfo"]
if "parentLFNs" in dbsfile.keys():
# Then we have some parents
for lfn in dbsfile["parentLFNs"]:
newFile = DBSBufferFile(lfn=lfn)
dbsfile["parents"].add(newFile)
del dbsfile["parentLFNs"]
self.commitTransaction(existingTransaction)
return dbsFiles
