def __init__(self, name, type, unit=None, mime=None, log=None):
super(DozerData, self).__init__(self)
self.__name = ""
self.__type = ""
self.__unit = ""
self.__mime = ""
self.__value = ""
self.debug(
"constructing DozerData name='%s' type='%s' unit='%s' mime='%s'" %
(str(name), str(type), str(unit), str(mime)))
self.__name = str(name)
aType = str(type).capitalize()
if (aType in self.knownTypes):
self.__type = self.knownTypes[aType]
else:
raise DozerError(
"not known or unsupported type=%s for DozerData name=%s" %
(self.__name, aType))
# set MIME for Blobs
if ("Blob" in aType):
if (not mime):
raise DozerError(
"MIME not set for DozerData name=%s, but type is Blob" %
self.__name)
else:
self.__mime = str(mime)
# set unit
if (unit):
self.__unit = str(unit)
def __init__(self,
package,
framework,
nightly,
install,
cmtconfig,
project,
jobId,
since,
until,
log=None):
super(DozerRecordSet, self).__init__(log)
self.__package = package
self.__framework = framework
self.__nightly = nightly
self.__install = install
self.__cmtconfig = cmtconfig
self.__project = project
self.__jobId = jobId
if (type(since) not in (LongType, IntType)):
raise DozerError(
"error constructing DozerRecordSet instance, parameter since should be integer or long"
)
else:
self.__since = DozerIOV(long(since))
if (type(until) not in (LongType, IntType)):
raise DozerError(
"error constructing DozerRecordSet instance, parameter until should be integer or long"
)
else:
self.__until = DozerIOV(long(until))
self.info("constructing DozerRecordSet since='%s' until='%s'" %
(self.__since.AsISO(), self.__until.AsISO()))
self.__channel = None
self.__dataNames = {}
self.__style = "Default"
self.__stylist = DozerStyle(self.__style)
self.__graph = None
self.info(str(self))
def fromISO(cls, iso, log=None):
words = re.findall('\d+', iso)
if (len(words) != 6):
raise DozerError(
"too few or too many value(=%d) to unpack from iso string='%s'"
% (len(words), iso))
else:
try:
words = [int(word) for word in words]
unix = time.mktime(
datetime.datetime(words[0], words[1], words[2], words[3],
words[4], words[5]).timetuple())
return cls.fromUnix(unix, log)
except TypeError, value:
raise DozerError(
"cannot convert string to int in DozerIOV.fromISO() call")
def AsTDatime(self):
unix = datetime.datetime.fromtimestamp(self.AsUnix())
if (unix.year < 1995):
raise DozerError(
"cannot construct ROOT.TDatime object, year must be >= 1995!")
return ROOT.TDatime(unix.year, unix.month, unix.day, unix.hour,
unix.minute, unix.second)
def getSchema(self, fromPath=None, since=None):
if (not self.isOpened()):
raise DozerError(
"cannot read back schema from db, db is not opened!")
if (not since):
since = self.__iov.today()
if (fromPath):
if (not fromPath.startswith(self.pathChunk())):
fromPath = self.pathChunk() + "/" + fromPath
self.cd(str(fromPath))
pwd = self.pwd()
try:
if (self.__folder.existsChannel(str("cooldozer metadata"))):
dbSchemaObj = self.__getChannel(str("cooldozer metadata"),
since)
if (dbSchemaObj):
schema = DozerSchema.fromXMLString(
dbSchemaObj.payload()["xml"])
return schema
except:
self.error("cannot find schema in cool Folder path '%s'" % pwd)
def ls(self):
out = {"channels": {}, "folders": {}}
if (not self.isOpened()):
raise DozerError("cannot list, db is not opened!")
if (self.__folder.isLeaf()):
try:
channels = self.__folder.listChannels()
for channel in channels:
chName = self.__folder.channelName(channel)
chUUID = self.__folder.channelName(channel)
self.debug("channel id='%d' name='%s'" % (channel, chName))
out["channels"][chName] = (chUUID, channel)
except:
self.epanic("can't read list fo channels for folder " +
self.__folder.fullPath() + ", skipping...")
pass
return "channels", out["channels"]
else:
for folder in self.__folder.listFolderSets():
out["folders"][folder] = folder
for folder in self.__folder.listFolders():
out["folders"][folder] = folder
return "folders", out["folders"]
def addChannelPayload(self, channel, since, until=None):
since, until = self.__sinceUntil(since, until)
if (None in (since, until)):
raise DozerError(
"wrong arguments for 'since' or 'until' passed to DozerDB.addChannelPayload"
)
if (isinstance(channel, DozerChannel) == False):
raise DozerError(
"wrong type for argument 'channel' in DozerDB.addChannelPayload, should be DozerChannel instance"
)
if (not self.isOpened()):
raise DozerError("cannot add payload, db is nor opened!")
channelName = str(channel.name())
try:
if (self.__folder.existsChannel(channelName)):
channelId = self.__folder.channelId(channelName)
data = cool.Record(self.__recordSpec())
data["xml"] = str(channel.xml().toxml())
self.debug(
"storing channel name='%s' since=%s until=%s data=%s" %
(channelName, since.AsISO(), until.AsISO(), data["xml"]))
self.__folder.storeObject(since.AsCool(), until.AsCool(), data,
channelId)
self.info("payload for channel name='%s' has been added" %
channelName)
self.__folder.flushStorageBuffer()
return True
else:
self.error("DozerChannel name='%s' not found in db folder %s" %
(channelName, self.__folder.fullPath()))
except:
self.epanic(
"unknow error when processing DozerDB.addChannelPayload")
return False
def fromRelease(cls, release, log=None):
if (type(release) in StringTypes and release.startswith("rel_")):
try:
release = int(release.lstrip("rel_"))
except ValueError, value:
raise DozerError(
"cannot convert release to the day of the week")
if (release not in range(7)):
raise DozerError(
"cannot construct DozerIOV, release day of week out of range [0,6]"
)
today = datetime.datetime.today()
weekday = today.weekday() + 1
if (weekday == 7): weekday = 0
if (weekday == release):
return cls(log=log)
elif (weekday > release):
return cls(cls.lastNDays(weekday - release), log=log)
else:
return cls(cls.lastNDays((6 - release) + weekday + 1), log=log)
def __init__(self, package, framework, dry=False, db="TEST", log=None):
super(DozerDB, self).__init__(log)
self.__log = log
self.__iov = DozerIOV(log=self.__log)
self.__dv = DozerVersion()
self.info(self.__dv)
self.__cv = DozerValid(log=self.__log)
self.info("setting package to '%s'" % package)
self.__package = package
self.info("setting test framework to '%s'" % framework)
self.__framework = framework
self.info("setting db to " + db)
self.__dry = False
if (type(dry) is BooleanType):
self.__dry = dry
if (dry == True):
db = "TEST"
if ((type(db) is StringType) and (db in self.__cv.connects().keys())):
if (db == "TEST"):
self.info("*** DRY RUN ***")
self.__dry = True
self.__connect = self.__cv.connects(
)["TEST"] % self.__framework
self.debug("will use 'TEST' db [DRY RUN]")
else:
self.__connect = self.__cv.connects()[db]
self.debug("will use " + db + " db")
else:
self.error("know nothing about '" + str(db) + "' db, exiting!")
raise DozerError("know nothing about '" + str(db) +
"' db, exiting!")
self.__dbSvc = self.__getDBSvc()
self.info("will use PyCool %s" % self.__dbSvc.serviceVersion())
self.openDB()
def get(self, path=".", since=None, until=None):
if (until == None):
until = self.__iov.today().AsCool()
elif (isinstance(until, DozerIOV)):
until = until.AsCool()
elif (type(until) in (LongType, IntType)):
until = long(until)
else:
raise DozerError(
"wrong type for argument 'until' in DozerDB.get call, must be None, DozerIOV or integer"
)
if (since == None):
since = self.__iov.lastNDays(7)
elif (isinstance(since, DozerIOV)):
since = since.AsCool()
elif (type(since) in (LongType, IntType)):
since = long(since)
else:
raise DozerError(
"wrong type for argument 'since' in DozerDB.get call, must be None, DozerIOV or integer"
)
if (since > until):
raise DozerError(
"wrong values for parameters 'since' and 'until' in DozerDB.get, 'since' > 'until'!"
)
normpath = os.path.normpath(path)
if (not normpath.startswith(self.pathChunk())):
self.debug("path doesn't start with package name, extending... ")
normpath = os.path.normpath(self.pathChunk() + "/" + normpath)
if normpath.endswith("."): normpath = normpath.rstrip(".")
self.debug("path='%s' normpath='%s'" % (path, normpath))
if (not self.isOpened()):
raise DozerError("cannot get from db, db is not opened!")
words = str(normpath).split("/")
if (len(words) <= 7):
self.cd(normpath)
self.pwd()
elif (len(words) == 8):
path = "/".join(words[:7])
channel = words[7]
self.cd(path)
recordSet = None
if self.__folder.existsChannel(channel):
channelId = self.__folder.channelId(str(channel))
self.debug("channel '%s' found in path '%s'" % (channel, path))
package, nightly, install, cmtconfig, project, jobId = path.split(
"/")[1:]
self.debug("channelId=%d since=%s until=%s" %
(channelId, DozerIOV(since).AsISO(),
DozerIOV(until).AsISO()))
payloads = self.__folder.browseObjects(
since, until, cool.ChannelSelection(channelId))
empty = "empty" if payloads.isEmpty() == 1 else "not empty"
self.debug("payload is " + empty + ", records in payload = " +
str(payloads.size()))
#if empty == "empty":
# return None
recordSet = DozerRecordSet(package, self.__framework, nightly,
install, cmtconfig, project, jobId,
since, until)
for obj in payloads:
self.debug("adding payload since=" + str(obj.since()) +
" until=" + str(obj.until()))
recordSet += (obj.since(), obj.until(),
DozerChannel.fromXMLString(
obj.payload()["xml"]))
return recordSet
def addSchemaPayload(self, schema, since, until=None):
if (isinstance(schema, DozerSchema) == False):
raise DozerError(
"cannot add payload, schema parameter isn't an instnace of DozerSchema class"
)
if (until == None):
until = DozerIOV.max()
elif (isinstance(until, DozerIOV)):
until = until.AsCool()
elif (type(until) in (LongType, IntType)):
until = long(until)
else:
raise DozerError(
"parameter 'until' for DozerDB.addPayload isn't of type None, DozerIOV or (long) integer"
)
if (isinstance(since, DozerIOV)):
since = since.AsCool()
elif (type(since) in (LongType, IntType)):
since = long(since)
else:
raise DozerError(
"parameter 'since' for DozerDB.addPayload is not of type DozerIOV or (long) integer"
)
if (since > until):
raise DozerError(
"wrong values for parameters 'since' and 'until' in DozerDB.addPayload, 'since' > 'until'!"
)
if (schema.isSpecific()):
path = "/" + self.__package + "/" + schema.pathChunk()
if (not self.isOpened()):
raise DozerError("cannot add payload, db is nor opened!")
self += schema
self.cd(path)
self.pwd()
self.cd(path)
self.pwd()
for chKey in schema:
channel = schema[chKey]
chName = str(channel.name())
self.error(self.__folder.fullPath())
if (self.__folder.existsChannel(chName)):
channelId = self.__folder.channelId(chName)
data = cool.Record(self.__recordSpec())
data["xml"] = str(channel.xml().toxml())
self.debug(
"storing channel name='%s' since=%s until=%s data=%s" %
(chName, DozerIOV(since).AsISO(),
DozerIOV(until).AsISO(), data["xml"]))
self.__folder.storeObject(since, until, data, channelId)
else:
self.warn("channel name='%s' not found in db, skipping!" %
channel.name())
self.__folder.flushStorageBuffer()
def __iadd__(self, schema):
if (isinstance(schema, DozerSchema)):
path = schema.pathChunk()
self.info("adding schema '%s'" % path)
if (not schema.isSpecific()):
self.error(
"cannot add generic schema '%s' to db, next try to add specific one!"
% path)
return self
else:
path = "/" + self.__package + "/" + path
description = "CoolDozer Folder for schema %s" % path
if (not self.isOpened()):
raise DozerError("cannot add schema, db is not opened!")
if (self.__db.existsFolder(str(path))):
self.info(
"db folder path='%s' already in database, will only update schema"
% path)
self.cd(path)
self.__createSchema(schema)
else:
self.info(
"will create db folder name='%s' description='%s'" %
(path, description))
try:
#self.error( self.__db )
#self.error( self.__connect )
#self.error( self.isOpened() )
#self.error( self.__db.databaseId() )
if self.isOpened():
self.closeDB()
## switch to OWNER for a while
self.info("switching to DB OWNER account...")
## save original db URI string
self.__connectSAVE = self.__connect
## get db URI for OWNER
self.__connect = self.__cv.connects()["PROD_OWNER"]
dbSvc = cool.DatabaseSvcFactory.databaseService()
dbOwner = dbSvc.openDatabase(str(self.__connect),
False)
if (dbOwner): self.info("db for OWNER opened")
else:
raise DozerError(
"can't open db for OWNER, no more sessions left on server?"
)
self.info("will create folder " + str(path))
recordSpec = cool.RecordSpecification()
recordSpec.extend("xml", cool.StorageType.String64k)
dbOwner.createFolder(path, recordSpec, description, 1,
True)
self.info("folder %s has been created" % path)
self.cd(path)
## create schema
self.__createSchema(schema)
## close db
self.closeBD()
## reopen for PROD (writer)
self.info("swiching back to original DB...")
self.__connect = self.__connectSAVE
self.openDB()
self.__folder = self.cd(str(path))
except Exception, value:
msg = "creation of folder %s falied - %s" % (
path, value.__str__())
self.epanic(msg)
raise DozerError(msg)
def __init__(self,
package,
testFramework,
nightly,
install,
cmtconfig,
project,
jobId,
log=None):
super(DozerRTConfig, self).__init__(log)
self.__log = log
self.cv = DozerValid(log=self.__log)
self.debug(
"contructing RuntimeConfig %s/%s/%s/%s/%s/%s" %
(package, testFramework, nightly, install, cmtconfig, project))
self.__package = package
if (testFramework in self.cv.frameworks()
and "*" not in testFramework):
self.__testFramework = testFramework
else:
raise DozerError(
"wrong runtime configuration, testFramework cannot be set to %s"
% str(testFramework))
if (nightly in self.cv.nightlies() and "*" not in nightly):
self.__nightly = str(nightly)
else:
raise DozerError(
"wrong runtime configuration, nightly cannot be set to %s" %
str(nighlty))
if (install in self.cv.installs() and "*" not in install):
self.__install = install
else:
raise DozerError(
"wrong runtime configuration, install cannot be set to %s" %
str(install))
if (cmtconfig in self.cv.cmtconfigs() and "*" not in cmtconfig):
self.__cmtconfig = cmtconfig
else:
raise DozerError(
"wrong runtime configuration, cmtconfig cannot be set to %s" %
str(cmtconfig))
if (project in self.cv.projects() and "*" not in project):
self.__project = project
else:
raise DozerError(
"wrong runtime configuration, project cannot be set to %s" %
str(project))
if ("*" not in jobId):
self.__jobId = jobId
else:
raise DozerError(
"wrong runtime configuration, jobId cannot be set to %s" %
str(project))