def _addACDCFileToWMBSFile(self, acdcFile, inFileset=True):
"""
adds the ACDC files into WMBS database
"""
wmbsParents = []
# TODO: this check can be removed when ErrorHandler filters parents file for unmerged data
if acdcFile["parents"]:
firstParent = next(iter(acdcFile["parents"]))
# If files is merged and has unmerged parents skip the wmbs population
if acdcFile.get("merged", 0) and ("/store/unmerged/" in firstParent
or "MCFakeFile" in firstParent):
# don't set the parents
pass
else:
# set the parentage for all the unmerged parents
for parent in acdcFile["parents"]:
logging.debug("WMBS ACDC Parent File: %s", parent)
parent = self._addACDCFileToWMBSFile(DatastructFile(
lfn=parent,
locations=acdcFile["locations"],
merged=True),
inFileset=False)
wmbsParents.append(parent)
# pass empty check sum since it won't be updated to dbs anyway
checksums = {}
wmbsFile = File(lfn=str(acdcFile["lfn"]),
size=acdcFile["size"],
events=acdcFile["events"],
first_event=acdcFile.get('first_event', 0),
last_event=acdcFile.get('last_event', 0),
checksums=checksums,
parents=wmbsParents,
locations=acdcFile["locations"],
merged=acdcFile.get('merged', True))
## TODO need to get the lumi lists
for run in acdcFile['runs']:
wmbsFile.addRun(run)
if not acdcFile["lfn"].startswith("/store/unmerged") or wmbsParents:
# only add to DBSBuffer if is not unmerged file or it has parents.
dbsFile = self._convertACDCFileToDBSFile(acdcFile)
self._addToDBSBuffer(dbsFile, checksums, acdcFile["locations"])
logging.debug("WMBS ACDC File: %s on Location: %s", wmbsFile['lfn'],
wmbsFile['newlocations'])
wmbsFile['inFileset'] = bool(inFileset)
self.wmbsFilesToCreate.add(wmbsFile)
return wmbsFile
def _addACDCFileToWMBSFile(self, acdcFile, inFileset=True):
"""
adds the ACDC files into WMBS database
"""
wmbsParents = []
# If file is merged, then it will be the parent of whatever output that
# process this job (it twists my mind!). Meaning, block below can be skipped
if int(acdcFile.get("merged", 0)) == 0 and acdcFile["parents"]:
# set the parentage for all the unmerged parents
for parent in acdcFile["parents"]:
if parent.startswith("/store/unmerged/") or parent.startswith(
"MCFakeFile"):
logging.warning(
"WMBS ACDC skipped parent invalid file: %s", parent)
continue
logging.debug("WMBS ACDC Parent File: %s", parent)
parent = self._addACDCFileToWMBSFile(DatastructFile(
lfn=parent, locations=acdcFile["locations"], merged=True),
inFileset=False)
wmbsParents.append(parent)
# pass empty check sum since it won't be updated to dbs anyway
checksums = {}
wmbsFile = File(lfn=str(acdcFile["lfn"]),
size=acdcFile["size"],
events=acdcFile["events"],
first_event=acdcFile.get('first_event', 0),
last_event=acdcFile.get('last_event', 0),
checksums=checksums,
parents=wmbsParents,
locations=acdcFile["locations"],
merged=acdcFile.get('merged', True))
## TODO need to get the lumi lists
for run in acdcFile['runs']:
wmbsFile.addRun(run)
if not acdcFile["lfn"].startswith("/store/unmerged") or wmbsParents:
# only add to DBSBuffer if is not unmerged file or it has parents.
dbsFile = self._convertACDCFileToDBSFile(acdcFile)
self._addToDBSBuffer(dbsFile, checksums, acdcFile["locations"])
logging.debug("WMBS ACDC File: %s on Location: %s", wmbsFile['lfn'],
wmbsFile['newlocations'])
wmbsFile['inFileset'] = bool(inFileset)
self.wmbsFilesToCreate.add(wmbsFile)
return wmbsFile
def _addACDCFileToWMBSFile(self, acdcFile, inFileset=True):
"""
"""
wmbsParents = []
for parent in acdcFile["parents"]:
parent = self._addACDCFileToWMBSFile(DatastructFile(
lfn=parent, locations=acdcFile["locations"]),
inFileset=False)
wmbsParents.append(parent)
#pass empty check sum since it won't be updated to dbs anyway
checksums = {}
wmbsFile = File(lfn=str(acdcFile["lfn"]),
size=acdcFile["size"],
events=acdcFile["events"],
first_event=acdcFile.get('first_event', 0),
last_event=acdcFile.get('last_event', 0),
checksums=checksums,
parents=wmbsParents,
locations=acdcFile["locations"],
merged=acdcFile.get('merged', True))
## TODO need to get the lumi lists
for run in acdcFile['runs']:
wmbsFile.addRun(run)
dbsFile = self._convertACDCFileToDBSFile(acdcFile)
self._addToDBSBuffer(dbsFile, checksums, acdcFile["locations"])
logging.info("WMBS File: %s\n on Location: %s" %
(wmbsFile['lfn'], wmbsFile['newlocations']))
if inFileset:
wmbsFile['inFileset'] = True
else:
wmbsFile['inFileset'] = False
self.wmbsFilesToCreate.append(wmbsFile)
return wmbsFile