def setUp(self):
self.__log = DozerLogger("DozerGraph_unittest.log",
"test_DozerGraph",
toConsole=True)
self.package = "DummyTestPackage"
self.framework = "RTT"
self.nightly = "dev"
self.install = "build"
self.cmtconfig = "i686-slc4-gcc34-opt"
self.project = "AtlasProduction"
self.jobId = "job_1"
self.channel = DozerChannel("test channel", "test channel description")
self.channel += DozerData("double test data", "Double", "GeV")
self.channel += DozerData("double test data;error", "Double", "GeV")
self.channel += DozerData("int test data", "Int", "counts")
self.channel += DozerData("int test data;uperror", "Double", "counts")
self.channel += DozerData("int test data;downerror", "Double",
"counts")
today = DozerIOV()
self.pointsInTime = []
try:
import random
except ImportError:
exit(0)
for i in range(0, 30):
since = today.lastNDays(i)
until = today.lastNDays(i - 1)
if (i in (5, 10)): continue
chan = DozerChannel.fromXMLDoc(self.channel.xml())
if (i in (15, 20, 25)):
self.pointsInTime.append((since, until, chan))
else:
chan["double test data"].setValue(i + random.gauss(0.0, 3.0))
chan["double test data;error"].setValue(random.gauss(5.0, 5.0))
chan["int test data"].setValue(i + random.randrange(10, 20))
chan["int test data;uperror"].setValue(random.randrange(
10, 15))
chan["int test data;downerror"].setValue(
random.randrange(10, 15))
self.pointsInTime.append((since, until, chan))
print i
self.graph = DozerGraph(self.package, self.framework, self.nightly,
self.install, self.cmtconfig, self.project,
self.jobId, self.pointsInTime)
self.graph.style()
def __sinceUntil(self, since, until=None):
sinceIOV = untilIOV = None
if (isinstance(since, DozerIOV)):
sinceIOV = since
elif (type(since) in (LongType, IntType)):
sinceIOV = DozerIOV(long(since))
if (until == None):
untilIOV = DozerIOV(DozerIOV.max())
elif (isinstance(until, DozerIOV)):
untilIOV = until
elif (type(until) in (LongType, IntType)):
untilIOV = DozerIOV(long(until))
if (None not in (sinceIOV, untilIOV)):
if (sinceIOV < untilIOV):
self.debug("since %s" % sinceIOV)
self.debug("until %s" % untilIOV)
return (sinceIOV, untilIOV)
else:
self.error(
"wrong values for arguments 'since' and 'until', 'since' >= 'until'!"
)
self.debug("since %s" % sinceIOV)
self.debug("until %s" % untilIOV)
else:
self.error(
"wrong values for arguments 'since' or 'until', wrong types?")
return (None, None)
def __init__(self,
package,
framework,
nightly,
install,
cmtconfig,
project,
jobId,
since,
until,
log=None):
super(DozerRecordSet, self).__init__(log)
self.__package = package
self.__framework = framework
self.__nightly = nightly
self.__install = install
self.__cmtconfig = cmtconfig
self.__project = project
self.__jobId = jobId
if (type(since) not in (LongType, IntType)):
raise DozerError(
"error constructing DozerRecordSet instance, parameter since should be integer or long"
)
else:
self.__since = DozerIOV(long(since))
if (type(until) not in (LongType, IntType)):
raise DozerError(
"error constructing DozerRecordSet instance, parameter until should be integer or long"
)
else:
self.__until = DozerIOV(long(until))
self.info("constructing DozerRecordSet since='%s' until='%s'" %
(self.__since.AsISO(), self.__until.AsISO()))
self.__channel = None
self.__dataNames = {}
self.__style = "Default"
self.__stylist = DozerStyle(self.__style)
self.__graph = None
self.info(str(self))
def test_02_add(self):
channel = DozerChannel("test channel", "test channel description")
channel += DozerData("IntDataRTT", "Int", "counts")
channel += DozerData("IntDataRTT;error", "Int", "counts")
channel += DozerData("FloatDataRTT", "Float", "some unit")
channel += DozerData("FloatDataRTT;uperror", "Float", "some unit")
channel += DozerData("FloatDataRTT;downerror", "Float", "some unit")
print self.payload.since()
print self.payload.until()
pointsInTime = {}
today = DozerIOV()
for i in range(30):
since = today.lastNDays(i - 1)
until = today.lastNDays(i)
ichan = DozerChannel.fromXMLDoc(channel.xml())
ichan.get("IntDataRTT").setValue(i)
ichan.get("IntDataRTT;error").setValue(1)
pointsInTime[(since, until)] = ichan
for k in sorted(pointsInTime.keys()):
since, until = k
val = pointsInTime[k].get("IntDataRTT").value()
self.__log.info("since %s until %s value %s" %
(str(since), str(until), str(val)))
self.payload += (since, until, pointsInTime[k])
self.assertEqual(isinstance(self.payload, DozerRecordSet), True)
self.assertEqual(len(self.payload.dataNames()), 5)
def __DOMXML(self):
self.doc = xml.dom.minidom.Document()
recordSetNode = self.doc.createElement("recordset")
recordSetNode.setAttribute("since", self.__since.AsISO())
recordSetNode.setAttribute("until", self.__until.AsISO())
recordSetNode.setAttribute("framework", self.__framework)
recordSetNode.setAttribute("package", self.__package)
recordSetNode.setAttribute("nightly", self.__nightly)
recordSetNode.setAttribute("install", self.__install)
recordSetNode.setAttribute("cmtconfig", self.__cmtconfig)
recordSetNode.setAttribute("project", self.__project)
recordSetNode.setAttribute("jobId", self.__jobId)
recordSetNode.setAttribute("channel", self.__channel)
for key, value in self.__pointsInTime.iteritems():
recordNode = self.doc.createElement("record")
since, until = key
recordNode.setAttribute("since", DozerIOV(since).AsISO())
recordNode.setAttribute("until", DozerIOV(until).AsISO())
recordNode.appendChild(value.xml().documentElement)
recordSetNode.appendChild(recordNode)
self.doc.appendChild(recordSetNode)
return self.doc
def __prepareData(self, pointsInTime):
for pointInTime in pointsInTime:
if ((type(pointInTime) is TupleType) and (len(pointInTime) == 3)):
since = None
until = None
channel = None
if (isinstance(pointInTime[0], DozerIOV)):
since = pointInTime[0]
elif (type(pointInTime[0]) is LongType):
since = DozerIOV(pointInTime[0])
if (self.__since == None): self.__since = since
elif (self.__since and self.__since > since):
self.__since = since
if (isinstance(pointInTime[1], DozerIOV)):
until = pointInTime[1]
elif (type(pointInTime[1]) is LongType):
until = DozerIOV(pointInTime)
if (self.__until == None): self.__until = until
elif (self.__until and self.__until < until):
self.__until = until
dataDict = None
if (type(pointInTime[2]) is DozerChannel):
channel = pointInTime[2]
if (self.__channel == None):
self.__channel = channel.name()
dataDict = {}
for key, data in channel.iteritems():
if (data.type() in ("Int", "Long", "Float", "Double")):
dataDict[data.name()] = data
self.debug(
"adding data name='%s' type='%s' value='%s' since='%s' to plotter "
% (data.name(), data.type(), str(
data.value()), since.AsISO()))
if (data.name() not in self.__plotable):
self.__plotable.append(data.name())
else:
self.debug(
"skipping data name='%s' type='%s', unable to plot"
% (data.name(), data.type()))
if (None not in (since, until, channel, dataDict)):
self.pointsInTime[(since.AsCool(),
until.AsCool())] = dataDict
def setUp(self):
self.package = "Dummy"
self.framework = "RTT"
self.nightly = "dev"
self.install = "build"
self.cmtconfig = "i686-slc4-gcc34-opt"
self.project = "AtlasProduction"
self.jobId = "job_1"
globals()["test_DozerRecordSet_logger"] = DozerLogger(
"DozerRecordSet_unittest.log", toConsole=True)
self.__log = globals()["test_DozerRecordSet_logger"]
self.since = DozerIOV.fromRelease("rel_0").AsCool()
self.until = DozerIOV.fromRelease("rel_6").AsCool()
self.payload = DozerRecordSet(self.package, self.framework,
self.nightly, self.install,
self.cmtconfig, self.project, self.jobId,
self.since, self.until)
channel = DozerChannel("test channel", "test channel description")
channel += DozerData("IntDataRTT", "Int", "counts")
channel += DozerData("IntDataRTT;error", "Int", "counts")
channel += DozerData("FloatDataRTT", "Float", "some unit")
channel += DozerData("FloatDataRTT;uperror", "Float", "some unit")
channel += DozerData("FloatDataRTT;downerror", "Float", "some unit")
pointsInTime = {}
today = DozerIOV()
for i in range(30):
since = today.lastNDays(i - 1)
until = today.lastNDays(i)
ichan = DozerChannel.fromXMLDoc(channel.xml())
if (i != 10):
ichan.get("IntDataRTT").setValue(i)
ichan.get("IntDataRTT;error").setValue(5)
if (i % 3 != 0):
ichan.get("FloatDataRTT").setValue(i)
ichan.get("FloatDataRTT;uperror").setValue(i / 2.0)
ichan.get("FloatDataRTT;downerror").setValue(i / 4.0)
pointsInTime[(since, until)] = ichan
for k in sorted(pointsInTime.keys()):
since, until = k
val = pointsInTime[k].get("IntDataRTT").value()
self.__log.info("since %s until %s value %s" %
(str(since), str(until), str(val)))
self.payload += (since, until, pointsInTime[k])
def __init__(self, package, framework, dry=False, db="TEST", log=None):
super(DozerDB, self).__init__(log)
self.__log = log
self.__iov = DozerIOV(log=self.__log)
self.__dv = DozerVersion()
self.info(self.__dv)
self.__cv = DozerValid(log=self.__log)
self.info("setting package to '%s'" % package)
self.__package = package
self.info("setting test framework to '%s'" % framework)
self.__framework = framework
self.info("setting db to " + db)
self.__dry = False
if (type(dry) is BooleanType):
self.__dry = dry
if (dry == True):
db = "TEST"
if ((type(db) is StringType) and (db in self.__cv.connects().keys())):
if (db == "TEST"):
self.info("*** DRY RUN ***")
self.__dry = True
self.__connect = self.__cv.connects(
)["TEST"] % self.__framework
self.debug("will use 'TEST' db [DRY RUN]")
else:
self.__connect = self.__cv.connects()[db]
self.debug("will use " + db + " db")
else:
self.error("know nothing about '" + str(db) + "' db, exiting!")
raise DozerError("know nothing about '" + str(db) +
"' db, exiting!")
self.__dbSvc = self.__getDBSvc()
self.info("will use PyCool %s" % self.__dbSvc.serviceVersion())
self.openDB()
def test_03_dict(self):
for since, until in self.payload.keys():
value = self.payload[(since, until)]["IntDataRTT"].value()
print "since=%s until=%s value=%s" % (
DozerIOV(since).AsISO(), DozerIOV(until).AsISO(), str(value))
def AsHTMLTable(self, dataName=None, tableCSS=None):
out = "<table"
if (tableCSS != None):
out += " class='" + str(tableCSS) + "'"
out += ">"
if (not dataName):
out += "\n<caption>Payload for channel '%s' (%s/%s/%s/%s/%s).</caption>" % (
self.channel(), self.nightly(), self.install(),
self.cmtconfig(), self.project(), self.jobId())
else:
out += "\n<caption>Payload for data '%s' (%s/%s/%s/%s/%s/%s).</caption>" % (
str(dataName),
self.nightly(),
self.install(),
self.cmtconfig(),
self.project(),
self.jobId(),
self.channel(),
)
# head row
out += "\n<tr>"
out += "<th>date</th>" # <th>until</th>"
if (dataName):
out += "<th>"
if (dataName in self.__dataNames):
out += dataName
if (self.__dataNames[dataName] != ""):
out += " [" + self.__dataNames[dataName] + "]"
out += "</th>"
else:
for data in sorted(self.__dataNames):
self.debug("data = " + data)
if ((";error" not in data) and (";downerror" not in data)
and (";uperror" not in data)):
unit = ""
if (self.__dataNames[data] != ""):
unit = "[%s]" % self.__dataNames[data]
out += "<th> %s %s </th>" % (data, unit)
out += "</tr>"
# data points rows
for stamps, chan in sorted(self.__pointsInTime.iteritems(),
reverse=True):
since, until = DozerIOV(stamps[0]).AsISO().split()[0], DozerIOV(
stamps[1]).AsISO().split()[0]
out += "\n<tr>"
out += "<td>" + since + "</td>" # "<td>" + until + "</td>"
if (dataName):
val = "—"
err = ""
if (dataName in chan and str(chan[dataName].value()) != ""):
val = str(chan[dataName].value())
if ((dataName + ";error" in chan)
and (chan[dataName + ";error"].value() != "")):
err = " ± " + str(
chan[dataName + ";error"].value())
uerr = ""
## up asymm error
if ((dataName + ";uperror" in chan)
and (chan[dataName + ";uperror"].value() != "")):
uerr = " + " + str(chan[dataName + ";uperror"].value())
derr = ""
if ((dataName + ";downerror" in chan)
and (chan[dataName + ";downerror"].value() != "")):
derr = " - " + str(
chan[dataName + ";downerror"].value())
if ((uerr + derr) != ""):
err = uerr + derr
out += "<td>%s</td>" % (val + err)
else:
for data in sorted(self.__dataNames):
if ((";error" not in data) and (";downerror" not in data)
and (";uperror" not in data)):
val = "—"
err = ""
if (data in chan and str(chan[data].value()) != ""):
val = str(chan[data].value())
err = ""
## symm error
if ((data + ";error" in chan)
and (chan[data + ";error"].value() != "")):
err = " ± " + str(
chan[data + ";error"].value())
uerr = ""
## up asymm error
if ((data + ";uperror" in chan) and
(chan[data + ";uperror"].value() != "")):
uerr = " + " + str(
chan[data + ";uperror"].value())
derr = ""
if ((data + ";downerror" in chan) and
(chan[data + ";downerror"].value() != "")):
derr = " - " + str(
chan[data + ";downerror"].value())
if ((uerr + derr) != ""):
err = uerr + derr
out += "<td>%s</td>" % (val + err)
out += "</tr>"
out += "\n</table>"
return out
def __iadd__(self, pointInTime):
self.debug("add point in time")
if ((type(pointInTime) is TupleType) and (len(pointInTime) == 3)):
since = until = channel = None
if (type(pointInTime[0]) is LongType):
since = DozerIOV(pointInTime[0])
if (self.__since > since):
self.debug("extending 'since' timestamp to %s" %
since.AsISO())
self.__since = since
if (type(pointInTime[1]) is LongType):
until = DozerIOV(pointInTime[1])
if (self.__until < until):
self.debug("extending 'until' timestamp to %s" %
until.AsISO())
self.__until = until
if (type(pointInTime[2]) is DozerChannel):
channel = pointInTime[2]
if (None not in (since, until, channel)):
if (self.__channel == None): self.__channel = channel.name()
self.debug(
"adding point in time ['%s', '%s') => channel name='%s'" %
(since.AsISO(), until.AsISO(), channel.name()))
for data in channel:
if (data not in self.__dataNames.keys()):
self.__dataNames[data] = channel[data].unit()
if ((since.AsCool(), until.AsCool())
not in self.__pointsInTime.keys()):
self.__pointsInTime[(since.AsCool(),
until.AsCool())] = channel
else:
del self.__pointsInTime[(since.AsCool(), until.AsCool())]
self.__pointsInTime[(since.AsCool(),
until.AsCool())] = channel
else:
self.error(
"wrong argument to += operator for DozerRecordSet, should be a tuple ( (long), (long), (DozerChannel) )"
)
return self
class DozerRecordSet(DozerObject.DozerObject):
## DozerChannel name
__channelName = None
## DozerChannel UUID
__channelUUID = None
## begin DozerIOV
__since = None
## end DozerIOV
__until = None
## points in time dictionary
__pointsInTime = {}
## c'tor
# @param self "Me, myself and Irene"
# @param package a package name
# @param framework test framework name
# @param nightly a nighlty name
# @param install install name
# @param cmtconfig a CMTCONFIG value
# @param project an Atlas project name
# @param jobId job id
# @param since cool time stamp
# @param until cool time stamp
# @param log DozerLogger instance
def __init__(self,
package,
framework,
nightly,
install,
cmtconfig,
project,
jobId,
since,
until,
log=None):
super(DozerRecordSet, self).__init__(log)
self.__package = package
self.__framework = framework
self.__nightly = nightly
self.__install = install
self.__cmtconfig = cmtconfig
self.__project = project
self.__jobId = jobId
if (type(since) not in (LongType, IntType)):
raise DozerError(
"error constructing DozerRecordSet instance, parameter since should be integer or long"
)
else:
self.__since = DozerIOV(long(since))
if (type(until) not in (LongType, IntType)):
raise DozerError(
"error constructing DozerRecordSet instance, parameter until should be integer or long"
)
else:
self.__until = DozerIOV(long(until))
self.info("constructing DozerRecordSet since='%s' until='%s'" %
(self.__since.AsISO(), self.__until.AsISO()))
self.__channel = None
self.__dataNames = {}
self.__style = "Default"
self.__stylist = DozerStyle(self.__style)
self.__graph = None
self.info(str(self))
## DozerChannel name getter
# @param self "Me, myself and Irene"
def channel(self):
return self.__channel
## package name getter
# @param self "Me, myself and Irene"
def package(self):
return self.__package
## test framework name getter
# @param self "Me, myself and Irene"
def framework(self):
return self.__framework
## nightly name getter
# @param self "Me, myself and Irene"
def nightly(self):
return self.__nightly
## install name getter
# @param self "Me, myself and Irene"
def install(self):
return self.__install
## CMTCONFIG getter
# @param self "Me, myself and Irene"
def cmtconfig(self):
return self.__cmtconfig
## project name getter
# @param self "Me, myself and Irene"
def project(self):
return self.__project
## job Id getter
# @param self "Me, myself and Irene"
def jobId(self):
return self.__jobId
## DozerData names getter
# @param self "Me, myself and Irene"
# @return a list with DozerData names
def dataNames(self):
return self.__dataNames.keys()
## += pointInTime operator
# @param self "Me, myself and Irene"
# @param pointInTime tuple with two cool timestamps + DozerChannel instance
def __iadd__(self, pointInTime):
self.debug("add point in time")
if ((type(pointInTime) is TupleType) and (len(pointInTime) == 3)):
since = until = channel = None
if (type(pointInTime[0]) is LongType):
since = DozerIOV(pointInTime[0])
if (self.__since > since):
self.debug("extending 'since' timestamp to %s" %
since.AsISO())
self.__since = since
if (type(pointInTime[1]) is LongType):
until = DozerIOV(pointInTime[1])
if (self.__until < until):
self.debug("extending 'until' timestamp to %s" %
until.AsISO())
self.__until = until
if (type(pointInTime[2]) is DozerChannel):
channel = pointInTime[2]
if (None not in (since, until, channel)):
if (self.__channel == None): self.__channel = channel.name()
self.debug(
"adding point in time ['%s', '%s') => channel name='%s'" %
(since.AsISO(), until.AsISO(), channel.name()))
for data in channel:
if (data not in self.__dataNames.keys()):
self.__dataNames[data] = channel[data].unit()
if ((since.AsCool(), until.AsCool())
not in self.__pointsInTime.keys()):
self.__pointsInTime[(since.AsCool(),
until.AsCool())] = channel
else:
del self.__pointsInTime[(since.AsCool(), until.AsCool())]
self.__pointsInTime[(since.AsCool(),
until.AsCool())] = channel
else:
self.error(
"wrong argument to += operator for DozerRecordSet, should be a tuple ( (long), (long), (DozerChannel) )"
)
return self
## str( ) operator
# @param self "Me, myself and Irene"
def __str__(self):
out = "DozerRecordSet since=%s until=%s package=%s channel=%s " % (
self.__since.AsISO(), self.__until.AsISO(), self.__package,
self.__channel)
out += "framework=%s nightly=%s install=%s cmtconfig=%s project=%s jobId=%s" % (
self.__framework, self.__nightly, self.__install, self.__cmtconfig,
self.__project, self.__jobId)
return out
## since cool stamp gatter
# @param self "Me, myself and Irene"
# @return DozerIOV instance
def since(self):
return self.__since
## until cool stamp getter
# @param self "Me, myself and Irene"
# @return DozerIOV instance
def until(self):
return self.__until
## keys getter
# @param self "Me, myself and Irene"
def keys(self):
return sorted(self.__pointsInTime.keys())
## item getter
# @param self "Me, myself and Irene"
# @param key tuple ( long, long )
def __getitem__(self, key):
if (key in self.__pointsInTime.keys()):
return self.__pointsInTime[key]
## iterator
# @param self "Me, myself and Irene"
def __iter__(self):
return self.__pointsInTime.__iter__()
## in operator
# @param self "Me, myself and Irene"
# @param key a key to check
def __contains__(self, key):
if (key in self.__pointsInTime): return True
return False
## has_key()
# @param self "Me, myself and Irene"
# @param key a key to check
def has_key(self, key):
if (self.__pointsInTime.has_key(key)): return True
return False
## data items getter
# @param self "Me, myself and Irene"
def items(self):
return self.__pointsInTime.items()
## iteritems getter
# @param self "Me, myself and Irene"
def iteritems(self):
return self.__pointsInTime.iteritems()
## iterkeys getter
# @param self "Me, myself and Irene"
def iterkeys(self):
return self.__pointsInTime.iterkeys()
## number of points in time getter
# @param self "Me, myself and Irene"
def __len__(self):
return len(self.__pointsInTime.keys())
## transforms pointsInTime to DozerGraph format
# @param self "Me, myself and Irene"
def __transform(self):
forPlot = []
for key, value in self.__pointsInTime.iteritems():
since, until = key
forPlot.append((since, until, value))
return forPlot
## DozerGraph instance getter
# @param self "Me, myself and Irene"
def getDozerGraph(self):
try:
forPlot = self.__transform()
return DozerGraph(self.__package, self.__framework, self.__nightly,
self.__install, self.__cmtconfig, self.__project,
self.__jobId, forPlot)
except:
self.epanic("unknown excption when making DozerGraph instance")
return None
## XML DOM document builder
# @param self "Me, myself and Irene"
# @brief XML DOM document builder
def __DOMXML(self):
self.doc = xml.dom.minidom.Document()
recordSetNode = self.doc.createElement("recordset")
recordSetNode.setAttribute("since", self.__since.AsISO())
recordSetNode.setAttribute("until", self.__until.AsISO())
recordSetNode.setAttribute("framework", self.__framework)
recordSetNode.setAttribute("package", self.__package)
recordSetNode.setAttribute("nightly", self.__nightly)
recordSetNode.setAttribute("install", self.__install)
recordSetNode.setAttribute("cmtconfig", self.__cmtconfig)
recordSetNode.setAttribute("project", self.__project)
recordSetNode.setAttribute("jobId", self.__jobId)
recordSetNode.setAttribute("channel", self.__channel)
for key, value in self.__pointsInTime.iteritems():
recordNode = self.doc.createElement("record")
since, until = key
recordNode.setAttribute("since", DozerIOV(since).AsISO())
recordNode.setAttribute("until", DozerIOV(until).AsISO())
recordNode.appendChild(value.xml().documentElement)
recordSetNode.appendChild(recordNode)
self.doc.appendChild(recordSetNode)
return self.doc
## convert record set to comma separated value string
# @param self "Me, myself and Irene"
# @return string with CSV file content
def AsCSV(self):
out = ""
for key, channel in sorted(self.__pointsInTime.iteritems()):
if (out == ""):
out = "# %s\n" % str(self)
out += "since,until"
for data in channel.keys():
out += "," + data
if (channel[data].unit() != ""):
out += " [" + channel[data].unit() + "]"
out += "\n"
since, until = key
out += "%s,%s" % (since, until)
for data in channel:
value = channel[data].value()
if (value == None):
value + ""
out += "," + str(value)
out += "\n"
return out
## convert to DOM XML document
# @param self "Me, myself and Irene"
# @return DOM XML document instance
def AsXML(self):
return self.__DOMXML()
## convert record set to HTML table
# @param self "Me, myself and Irene"
# @param dataName DozerData to dump (default None means all)
# @param tableCSS css class for table
# @return HTML fragment with table
def AsHTMLTable(self, dataName=None, tableCSS=None):
out = "<table"
if (tableCSS != None):
out += " class='" + str(tableCSS) + "'"
out += ">"
if (not dataName):
out += "\n<caption>Payload for channel '%s' (%s/%s/%s/%s/%s).</caption>" % (
self.channel(), self.nightly(), self.install(),
self.cmtconfig(), self.project(), self.jobId())
else:
out += "\n<caption>Payload for data '%s' (%s/%s/%s/%s/%s/%s).</caption>" % (
str(dataName),
self.nightly(),
self.install(),
self.cmtconfig(),
self.project(),
self.jobId(),
self.channel(),
)
# head row
out += "\n<tr>"
out += "<th>date</th>" # <th>until</th>"
if (dataName):
out += "<th>"
if (dataName in self.__dataNames):
out += dataName
if (self.__dataNames[dataName] != ""):
out += " [" + self.__dataNames[dataName] + "]"
out += "</th>"
else:
for data in sorted(self.__dataNames):
self.debug("data = " + data)
if ((";error" not in data) and (";downerror" not in data)
and (";uperror" not in data)):
unit = ""
if (self.__dataNames[data] != ""):
unit = "[%s]" % self.__dataNames[data]
out += "<th> %s %s </th>" % (data, unit)
out += "</tr>"
# data points rows
for stamps, chan in sorted(self.__pointsInTime.iteritems(),
reverse=True):
since, until = DozerIOV(stamps[0]).AsISO().split()[0], DozerIOV(
stamps[1]).AsISO().split()[0]
out += "\n<tr>"
out += "<td>" + since + "</td>" # "<td>" + until + "</td>"
if (dataName):
val = "—"
err = ""
if (dataName in chan and str(chan[dataName].value()) != ""):
val = str(chan[dataName].value())
if ((dataName + ";error" in chan)
and (chan[dataName + ";error"].value() != "")):
err = " ± " + str(
chan[dataName + ";error"].value())
uerr = ""
## up asymm error
if ((dataName + ";uperror" in chan)
and (chan[dataName + ";uperror"].value() != "")):
uerr = " + " + str(chan[dataName + ";uperror"].value())
derr = ""
if ((dataName + ";downerror" in chan)
and (chan[dataName + ";downerror"].value() != "")):
derr = " - " + str(
chan[dataName + ";downerror"].value())
if ((uerr + derr) != ""):
err = uerr + derr
out += "<td>%s</td>" % (val + err)
else:
for data in sorted(self.__dataNames):
if ((";error" not in data) and (";downerror" not in data)
and (";uperror" not in data)):
val = "—"
err = ""
if (data in chan and str(chan[data].value()) != ""):
val = str(chan[data].value())
err = ""
## symm error
if ((data + ";error" in chan)
and (chan[data + ";error"].value() != "")):
err = " ± " + str(
chan[data + ";error"].value())
uerr = ""
## up asymm error
if ((data + ";uperror" in chan) and
(chan[data + ";uperror"].value() != "")):
uerr = " + " + str(
chan[data + ";uperror"].value())
derr = ""
if ((data + ";downerror" in chan) and
(chan[data + ";downerror"].value() != "")):
derr = " - " + str(
chan[data + ";downerror"].value())
if ((uerr + derr) != ""):
err = uerr + derr
out += "<td>%s</td>" % (val + err)
out += "</tr>"
out += "\n</table>"
return out
def __trend(self, name, fit, errors):
upErrorKey = errors[0]
downErrorKey = errors[1]
graph = ROOT.TGraphAsymmErrors()
xmax = float((self.until().AsUnix() - self.since().AsUnix()) / 84600)
days = int(xmax)
offset = self.since().AsUnix()
unit = ""
points = []
for (since, until), data in sorted(self.pointsInTime.iteritems()):
value = None
upError = None
downError = None
since = abs((DozerIOV(since).AsUnix() - offset) / 84600)
until = abs((DozerIOV(until).AsUnix() - offset) / 84600)
if name in data.keys():
unit = data[name].unit()
value = data[name].value()
if (upErrorKey in data.keys()):
upError = data[upErrorKey].value()
if (downErrorKey in data.keys()):
downError = data[downErrorKey].value()
points.append((since, value, upError, downError))
i = 0
ymin = None
ymax = None
for (x, y, ue, de) in points:
self.debug("point i=%s x=%s y=%s uerr=%s derr=%s" %
(str(i), str(x), str(y), str(ue), str(de)))
if (None not in (x, y) and y != ""):
graph.SetPoint(i, x + 0.5, y)
if (not ue): ue = 0.0
if (not de): de = 0.0
if (errors[0] == errors[1]):
de = ue
graph.SetPointError(i, 0.0, 0.0, ue, ue)
else:
graph.SetPointError(i, 0.0, 0.0, ue, de)
if (not ymin):
ymin = y - de
if (not ymax):
ymax = y + ue
if (ymax < (y + ue)):
ymax = y + ue
if (ymin > (y - de)):
ymin = y - de
i += 1
else:
self.error("skipping point, x=%s y=%s" % (str(x), str(y)))
self.nbPoints = i
#self.info("number of points %d" % self.nbPoints )
if (None not in (ymax, ymin)):
dy = 1.2 * abs(ymax - ymin)
graph.SetMinimum(ymin - dy)
graph.SetMaximum(ymax + dy)
timeAxis = graph.GetXaxis()
timeAxis.Set(days + 1, 0, days + 1)
timeAxis.SetNdivisions(days + 1, False)
for i in range(0, days + 1):
label = DozerIOV(self.since().lastNDays(days -
i)).AsISO().split(" ")[0]
timeAxis.SetBinLabel(i + 1, label)
timeAxis.SetLabelOffset(0.005)
timeAxis.SetTitleOffset(2.1)
timeAxis.LabelsOption("v")
timeAxis.SetTitle("days")
graph.GetYaxis().SetTitle(name + "[" + unit + "]")
if (fit and self.nbPoints > 1):
fitFcn = self.__linearFcn(0, days + 1)
graph.Fit(fitFcn, "R+")
return graph
def get(self, path=".", since=None, until=None):
if (until == None):
until = self.__iov.today().AsCool()
elif (isinstance(until, DozerIOV)):
until = until.AsCool()
elif (type(until) in (LongType, IntType)):
until = long(until)
else:
raise DozerError(
"wrong type for argument 'until' in DozerDB.get call, must be None, DozerIOV or integer"
)
if (since == None):
since = self.__iov.lastNDays(7)
elif (isinstance(since, DozerIOV)):
since = since.AsCool()
elif (type(since) in (LongType, IntType)):
since = long(since)
else:
raise DozerError(
"wrong type for argument 'since' in DozerDB.get call, must be None, DozerIOV or integer"
)
if (since > until):
raise DozerError(
"wrong values for parameters 'since' and 'until' in DozerDB.get, 'since' > 'until'!"
)
normpath = os.path.normpath(path)
if (not normpath.startswith(self.pathChunk())):
self.debug("path doesn't start with package name, extending... ")
normpath = os.path.normpath(self.pathChunk() + "/" + normpath)
if normpath.endswith("."): normpath = normpath.rstrip(".")
self.debug("path='%s' normpath='%s'" % (path, normpath))
if (not self.isOpened()):
raise DozerError("cannot get from db, db is not opened!")
words = str(normpath).split("/")
if (len(words) <= 7):
self.cd(normpath)
self.pwd()
elif (len(words) == 8):
path = "/".join(words[:7])
channel = words[7]
self.cd(path)
recordSet = None
if self.__folder.existsChannel(channel):
channelId = self.__folder.channelId(str(channel))
self.debug("channel '%s' found in path '%s'" % (channel, path))
package, nightly, install, cmtconfig, project, jobId = path.split(
"/")[1:]
self.debug("channelId=%d since=%s until=%s" %
(channelId, DozerIOV(since).AsISO(),
DozerIOV(until).AsISO()))
payloads = self.__folder.browseObjects(
since, until, cool.ChannelSelection(channelId))
empty = "empty" if payloads.isEmpty() == 1 else "not empty"
self.debug("payload is " + empty + ", records in payload = " +
str(payloads.size()))
#if empty == "empty":
# return None
recordSet = DozerRecordSet(package, self.__framework, nightly,
install, cmtconfig, project, jobId,
since, until)
for obj in payloads:
self.debug("adding payload since=" + str(obj.since()) +
" until=" + str(obj.until()))
recordSet += (obj.since(), obj.until(),
DozerChannel.fromXMLString(
obj.payload()["xml"]))
return recordSet
def addSchemaPayload(self, schema, since, until=None):
if (isinstance(schema, DozerSchema) == False):
raise DozerError(
"cannot add payload, schema parameter isn't an instnace of DozerSchema class"
)
if (until == None):
until = DozerIOV.max()
elif (isinstance(until, DozerIOV)):
until = until.AsCool()
elif (type(until) in (LongType, IntType)):
until = long(until)
else:
raise DozerError(
"parameter 'until' for DozerDB.addPayload isn't of type None, DozerIOV or (long) integer"
)
if (isinstance(since, DozerIOV)):
since = since.AsCool()
elif (type(since) in (LongType, IntType)):
since = long(since)
else:
raise DozerError(
"parameter 'since' for DozerDB.addPayload is not of type DozerIOV or (long) integer"
)
if (since > until):
raise DozerError(
"wrong values for parameters 'since' and 'until' in DozerDB.addPayload, 'since' > 'until'!"
)
if (schema.isSpecific()):
path = "/" + self.__package + "/" + schema.pathChunk()
if (not self.isOpened()):
raise DozerError("cannot add payload, db is nor opened!")
self += schema
self.cd(path)
self.pwd()
self.cd(path)
self.pwd()
for chKey in schema:
channel = schema[chKey]
chName = str(channel.name())
self.error(self.__folder.fullPath())
if (self.__folder.existsChannel(chName)):
channelId = self.__folder.channelId(chName)
data = cool.Record(self.__recordSpec())
data["xml"] = str(channel.xml().toxml())
self.debug(
"storing channel name='%s' since=%s until=%s data=%s" %
(chName, DozerIOV(since).AsISO(),
DozerIOV(until).AsISO(), data["xml"]))
self.__folder.storeObject(since, until, data, channelId)
else:
self.warn("channel name='%s' not found in db, skipping!" %
channel.name())
self.__folder.flushStorageBuffer()
def test_03_addPayload(self):
iov = DozerIOV()
schema = self.confRTT["bugfix"]["build"]["i686-slc4-gcc34-opt"][
"AtlasProduction"]["job_1"]
today = iov.today()
schema["testChannelRTT"]["testDataRTT"].setValue(7)
self.db.addSchemaPayload(schema, iov.lastNDays(6))
schema["testChannelRTT"]["testDataRTT"].setValue(6)
self.db.addSchemaPayload(schema, iov.lastNDays(5))
schema["testChannelRTT"]["testDataRTT"].setValue(5)
self.db.addSchemaPayload(schema, iov.lastNDays(4))
schema["testChannelRTT"]["testDataRTT"].setValue(4)
self.db.addSchemaPayload(schema, iov.lastNDays(3))
schema["testChannelRTT"]["testDataRTT"].setValue(3)
self.db.addSchemaPayload(schema, iov.lastNDays(2))
schema["testChannelRTT"]["testDataRTT"].setValue(2)
self.db.addSchemaPayload(schema, iov.lastNDays(1))
schema["testChannelRTT"]["testDataRTT"].setValue(1)
self.db.addSchemaPayload(schema, today)
channel = schema["testChannelRTT"]
channel["testDataRTT"].setValue(10)
since = DozerIOV(iov.lastNDays(7))
until = DozerIOV(iov.lastNDays(6))
self.assertEqual(self.db.addChannelPayload(channel, since, until),
True)
