def testGetAncestorLFNs(self):
"""
_testGenAncestorLFNs_
Create a series of files that have several generations of parentage
information. Verify that the parentage information is reported
correctly.
"""
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10, checksums={"cksum": 1}, locations="se1.fnal.gov")
testFileA.create()
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10, checksums={"cksum": 1}, locations="se1.fnal.gov")
testFileB.create()
testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10, checksums={"cksum": 1}, locations="se1.fnal.gov")
testFileC.create()
testFileD = File(lfn="/this/is/a/lfnD", size=1024, events=10, checksums={"cksum": 1}, locations="se1.fnal.gov")
testFileD.create()
testFileE = File(lfn="/this/is/a/lfnE", size=1024, events=10, checksums={"cksum": 1}, locations="se1.fnal.gov")
testFileE.create()
testFileE = File(lfn="/this/is/a/lfnF", size=1024, events=10, checksums={"cksum": 1}, locations="se1.fnal.gov")
testFileE.create()
testFileA.addParent(lfn="/this/is/a/lfnB")
testFileA.addParent(lfn="/this/is/a/lfnC")
testFileB.addParent(lfn="/this/is/a/lfnD")
testFileC.addParent(lfn="/this/is/a/lfnD")
testFileD.addParent(lfn="/this/is/a/lfnE")
testFileD.addParent(lfn="/this/is/a/lfnF")
level1 = ["/this/is/a/lfnB", "/this/is/a/lfnC"]
level2 = ["/this/is/a/lfnD"]
level3 = ["/this/is/a/lfnE", "/this/is/a/lfnF"]
level4 = level5 = []
decs2 = ["/this/is/a/lfnA"]
assert testFileA.getAncestors(level=1, type="lfn") == level1, "ERROR: level 1 test failed"
assert testFileA.getAncestors(level=2, type="lfn") == level2, "ERROR: level 2 test failed"
assert testFileA.getAncestors(level=3, type="lfn") == level3, "ERROR: level 3 test failed"
assert testFileA.getAncestors(level=4, type="lfn") == level4, "ERROR: level 4 test failed"
assert testFileA.getAncestors(level=5, type="lfn") == level5, "ERROR: level 5 test failed"
assert testFileD.getDescendants(level=1, type="lfn") == level1, "ERROR: level 1 desc test failed"
assert testFileD.getDescendants(level=2, type="lfn") == decs2, "ERROR: level 2 desc test failed"
assert testFileD.getDescendants(level=3, type="lfn") == level4, "ERROR: level 3 desc test failed"
return
def testLotsOfAncestors(self):
"""
_testLotsOfAncestors_
Create a file with 15 parents with each parent having 100 parents to
verify that the query to return grandparents works correctly.
"""
raise nose.SkipTest
testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10,
checksums = {"cksum": "1"}, locations = "se1.fnal.gov")
testFileA.create()
for i in xrange(15):
testParent = File(lfn = makeUUID(), size = 1024, events = 10,
checksums = {"cksum": "1"}, locations = "se1.fnal.gov")
testParent.create()
testFileA.addParent(testParent["lfn"])
for i in xrange(100):
testGParent = File(lfn = makeUUID(), size = 1024, events = 10,
checksums = {"cksum": "1"}, locations = "se1.fnal.gov")
testGParent.create()
testParent.addParent(testGParent["lfn"])
assert len(testFileA.getAncestors(level = 2, type = "lfn")) == 1500, \
"ERROR: Incorrect grand parents returned"
return
def testLotsOfAncestors(self):
"""
_testLotsOfAncestors_
Create a file with 15 parents with each parent having 100 parents to
verify that the query to return grandparents works correctly.
"""
raise nose.SkipTest
testFileA = File(lfn="/this/is/a/lfnA",
size=1024,
events=10,
checksums={"cksum": "1"},
locations="se1.fnal.gov")
testFileA.create()
for i in xrange(15):
testParent = File(lfn=makeUUID(),
size=1024,
events=10,
checksums={"cksum": "1"},
locations="se1.fnal.gov")
testParent.create()
testFileA.addParent(testParent["lfn"])
for i in xrange(100):
testGParent = File(lfn=makeUUID(),
size=1024,
events=10,
checksums={"cksum": "1"},
locations="se1.fnal.gov")
testGParent.create()
testParent.addParent(testGParent["lfn"])
assert len(testFileA.getAncestors(level=2, type="lfn")) == 1500, \
"ERROR: Incorrect grand parents returned"
return
class T0ASTFile(dict):
"""
_T0ASTFile_
A dictionary based object meant to represent a file in T0AST.
It contains the following keys:
ID
STREAM
RUN_ID
FILESIZE
EVENTS
CKSUM
LFN
DATASET_ID
DATA_TIER
TYPE
STATUS
EXPORT_STATUS
STREAM
STREAM_ID
LUMI_ID
LUMI_LIST
PARENT_LIST
PSET_HASH
BRANCH_HASH
LOCATIONS
"""
def __init__(self, wmbsFile=None, **args):
"""
___init___
Initialize all attributes.
"""
dict.__init__(self)
self.runLumi = None
# place holder for list of parent(repacked) files names
# only used for reco type, for repacked this should be empty
self.parentList = None
self.parentIDList = None
# this is the dataset_path_ID in data table not primary dataset id
self.datasetPathID = None
# place holder for PSet hash, Gets from Job Report
self.setdefault("RUN_ID", None)
self.setdefault("STREAM", None)
self.setdefault("DATASET_ID", None)
self.setdefault("PRIMARY_DATASET", None)
self.setdefault("PROCESSED_DATASET", None)
self.setdefault("DATA_TIER", None)
self.setdefault("BLOCK_ID", None)
self.setdefault("STREAM", None)
self.setdefault("STREAM_ID", None)
# currently using WM Run structure here but need to use lumi structure
# place holder for list of lumi data struct
#self.setdefault("LUMI_LIST", [])
self.setdefault("PSET_HASH", None)
self.setdefault("BRANCH_HASH", None)
self.setdefault("LOCATIONS", None)
self.wmbsFile = wmbsFile
if wmbsFile == None:
self.setdefault("ID", None)
self.setdefault("FILESIZE", None)
self.setdefault("EVENTS", None)
self.setdefault("FIRST_EVENT", 0)
self.setdefault("LAST_EVENT", 0)
self.setdefault("CKSUM", None)
self.setdefault("LFN", None)
# this is primary dataset ID
self.update(args)
else:
self["ID"] = wmbsFile["id"]
self["FILESIZE"] = wmbsFile["size"]
self["CKSUM"] = wmbsFile["cksum"]
self["EVENTS"] = wmbsFile["events"]
self["FIRST_EVENT"] = wmbsFile["first_event"]
self["LAST_EVENT"] = wmbsFile["last_event"]
self["LFN"] = wmbsFile["lfn"]
def convertToWMBSFile(self):
"""
__getWMBSFile__
return WMBSFile instance converted from T0AST file.
"""
if self.wmbsFile == None:
# hack: make sure first_event/last_event is None
# if they are add 0 event
if self["FIRST_EVENT"] == None:
self["FIRST_EVENT"] = 0
if self["LAST_EVENT"] == None:
self["LAST_EVENT"] = 0
self.wmbsFile = WMBSFile(lfn=self["LFN"], size=self["FILESIZE"],
events=self["EVENTS"], cksum=self["CKSUM"],
first_event = self["FIRST_EVENT"],
last_event = self["LAST_EVENT"],
locations=self["LOCATIONS"])
self.wmbsFile.create()
self["ID"] = self.wmbsFile["id"]
return self.wmbsFile
def getParentList(self, dataType='lfn', stream="Normal"):
"""
_getParentList_
"""
if self["DATA_TIER"] == "RAW":
return []
#TODO: place holder for correct express stream handling
if stream == "Express":
# TODO get using the lumi comparason
# For now return empty list for parent
return []
self.convertToWMBSFile()
if dataType == "id":
parents = self.parentIDList
elif dataType == "lfn":
parents = self.parentList
else:
raise Exception, "Unknown Type"
if parents == None:
# this will retrun right parents for both RECO -> RAW and
# RAW -> [] files except the big files merged directly
if self["DATA_TIER"] == "ALCARECO" \
and alcaNewSelection():
parents = self.wmbsFile.getAncestors(
level=3, type=dataType)
else:
parents = self.wmbsFile.getAncestors(level=2, type=dataType)
#update the private variable to prevent expensive call over and over again
if dataType == "id":
self.parentIDList = parents
elif dataType == "lfn":
self.parentList = parents
return parents
def getLumiList(self):
"""
_getLumiList_
"""
self.convertToWMBSFile()
if self.runLumi == None:
if len(self.wmbsFile["runs"]) == 0:
self.wmbsFile.loadData()
self.runLumi = list(self.wmbsFile["runs"])[0]
self["RUN_ID"] = self.runLumi.run
return self.runLumi
def getRunID(self):
"""
_getRunID_
"""
self.convertToWMBSFile()
if self["RUN_ID"] == None:
if len(self.wmbsFile["runs"]) == 0:
self.wmbsFile.loadData()
self.runLumi = list(self.wmbsFile["runs"])[0]
self["RUN_ID"] = self.runLumi.run
return self["RUN_ID"]
def getDatasetPathID(self):
"""
_getDatasetPathID_
To do: this need database connection to get the value.
need to be modified like wmbs wrapper class
"""
return self.datasetPathID
def testGetAncestorLFNs(self):
"""
_testGenAncestorLFNs_
Create a series of files that have several generations of parentage
information. Verify that the parentage information is reported
correctly.
"""
testFileA = File(lfn="/this/is/a/lfnA",
size=1024,
events=10,
checksums={'cksum': 1},
locations="se1.fnal.gov")
testFileA.create()
testFileB = File(lfn="/this/is/a/lfnB",
size=1024,
events=10,
checksums={'cksum': 1},
locations="se1.fnal.gov")
testFileB.create()
testFileC = File(lfn="/this/is/a/lfnC",
size=1024,
events=10,
checksums={'cksum': 1},
locations="se1.fnal.gov")
testFileC.create()
testFileD = File(lfn="/this/is/a/lfnD",
size=1024,
events=10,
checksums={'cksum': 1},
locations="se1.fnal.gov")
testFileD.create()
testFileE = File(lfn="/this/is/a/lfnE",
size=1024,
events=10,
checksums={'cksum': 1},
locations="se1.fnal.gov")
testFileE.create()
testFileE = File(lfn="/this/is/a/lfnF",
size=1024,
events=10,
checksums={'cksum': 1},
locations="se1.fnal.gov")
testFileE.create()
testFileA.addParent(lfn="/this/is/a/lfnB")
testFileA.addParent(lfn="/this/is/a/lfnC")
testFileB.addParent(lfn="/this/is/a/lfnD")
testFileC.addParent(lfn="/this/is/a/lfnD")
testFileD.addParent(lfn="/this/is/a/lfnE")
testFileD.addParent(lfn="/this/is/a/lfnF")
level1 = ["/this/is/a/lfnB", "/this/is/a/lfnC"]
level2 = ["/this/is/a/lfnD"]
level3 = ["/this/is/a/lfnE", "/this/is/a/lfnF"]
level4 = level5 = []
decs2 = ["/this/is/a/lfnA"]
assert testFileA.getAncestors(level=1, type='lfn') == level1, \
"ERROR: level 1 test failed"
assert testFileA.getAncestors(level=2, type='lfn') == level2, \
"ERROR: level 2 test failed"
assert testFileA.getAncestors(level=3, type='lfn') == level3, \
"ERROR: level 3 test failed"
assert testFileA.getAncestors(level=4, type='lfn') == level4, \
"ERROR: level 4 test failed"
assert testFileA.getAncestors(level=5, type='lfn') == level5, \
"ERROR: level 5 test failed"
assert testFileD.getDescendants(level=1, type='lfn') == level1, \
"ERROR: level 1 desc test failed"
assert testFileD.getDescendants(level=2, type='lfn') == decs2, \
"ERROR: level 2 desc test failed"
assert testFileD.getDescendants(level=3, type='lfn') == level4, \
"ERROR: level 3 desc test failed"
return
