def injectSkimConfig(self):
"""
_injectSkimConfig_
Create a bogus config cache document for the skims and inject it into
couch. Return the ID of the document.
"""
newConfig = Document()
newConfig["info"] = None
newConfig["config"] = None
newConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e580f"
newConfig["pset_hash"] = "7c856ad35f9f544839d8525ca10259a7"
newConfig["owner"] = {"group": "cmsdataops", "user": "******"}
newConfig["pset_tweak_details"] = {"process": {"outputModules_": ["SkimA", "SkimB"],
"SkimA": {"dataset": {"filterName": "SkimAFilter",
"dataTier": "RAW-RECO"}},
"SkimB": {"dataset": {"filterName": "SkimBFilter",
"dataTier": "USER"}}}}
result = self.configDatabase.commitOne(newConfig)
return result[0]["id"]
def injectReRecoConfig(self):
"""
_injectReRecoConfig_
Inject a ReReco config document that we can use to set the outputModules
"""
newConfig = Document()
newConfig["info"] = None
newConfig["config"] = None
newConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e580f"
newConfig["pset_hash"] = "7c856ad35f9f544839d8525ca10259a7"
newConfig["owner"] = {"group": "cmsdataops", "user": "******"}
newConfig["pset_tweak_details"] = {"process": {"outputModules_": ['RECOoutput', 'DQMoutput'],
"RECOoutput": {'dataset': {'filterName': 'RECOoutputFilter',
'dataTier': 'RECO'}},
"DQMoutput": {'dataset': {'filterName': 'DQMoutputFilter',
'dataTier': 'DQM'}}}}
result = self.configDatabase.commitOne(newConfig)
return result[0]["id"]
def testA(self):
"""instantiate"""
document = Document()
document[u'pset_tweak_details'] = {}
document[u'pset_tweak_details'][u'process'] = {}
document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'] = {}
document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed1'] = {}
document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed2'] = {}
document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed3'] = {}
document = self.database.commitOne(document)[0]
seeder = ReproducibleSeeding(CouchUrl = self.testInit.couchUrl,
CouchDBName = self.testInit.couchDbName,
ConfigCacheDoc = document[u'id'])
job = Job("testjob")
seeder(job)
baggage = job.getBaggage()
seed1 = getattr(baggage.process.RandomNumberGeneratorService, "seed1", None)
self.failUnless(seed1 != None)
def _getConfigBase(self):
"""
The RelValMC workload is supposed to have the same set of config
values like MonteCarlo, ReReco and PromptSkim workloads combined
plus three additional config values:
- GenConfig - ConfigCacheID of the config for the generation task (MonteCarlo)
- RecoConfig - ConfigCacheID of the config for the reco step (ReReco)
- AlcaRecoConfig - ConfigCacheID of the config for the skim/alcareco step (PromptSkim)
this base config values are taken from MonteCarlo_t, ReReco_t, no test for PromptSkim_t
configurations will be similar, they'll only differ in the output modules they define.
"""
config = Document()
config["info"] = None
config["config"] = None
config["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e580f"
config["pset_hash"] = "7c856ad35f9f544839d8525ca10259a7"
config["owner"] = {"group": "cmsdataops", "user": "******"}
config["pset_tweak_details"] = None
return config
def __init__(self, dbURL, couchDBName = None, id = None, rev = None, usePYCurl = False,
ckey = None, cert = None, capath = None, detail = True):
self.dbname = couchDBName
self.dburl = dbURL
self.detail = detail
try:
self.couchdb = CouchServer(self.dburl, usePYCurl=usePYCurl, ckey=ckey, cert=cert, capath=capath)
if self.dbname not in self.couchdb.listDatabases():
self.createDatabase()
self.database = self.couchdb.connectDatabase(self.dbname)
except Exception as ex:
msg = "Error connecting to couch: %s\n" % str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise ConfigCacheException(message = msg)
# local cache
self.docs_cache = DocumentCache(self.database, self.detail)
# UserGroup variables
self.group = None
self.owner = None
# Internal data structure
self.document = Document()
self.attachments = {}
self.document['type'] = "config"
self.document['description'] = {}
self.document['description']['config_label'] = None
self.document['description']['config_desc'] = None
if id != None:
self.document['_id'] = id
self.document['pset_tweak_details'] = None
self.document['info'] = None
self.document['config'] = None
return
def testUpdateHandlerAndBulkUpdateProfile(self):
"""
Test that update function support works
"""
# for actual test increase the size value: For 10000 records, 96 sec vs 4 sec
size = 100
for i in range(size):
self.db.queue(Document(id="%s" % i, inputDict={'name':123, 'counter':0}))
update_doc = {
'_id':'_design/foo',
'language': 'javascript',
'updates':{
"change-counter" : """function(doc, req) { if (doc) { var data = JSON.parse(req.body);
for (var field in data) {doc.field = data.field;} return [doc, 'OK'];}}""",
}
}
self.db.commit(update_doc)
start = time.time()
for id in range(size):
doc_id = "%s" % id
self.db.updateDocument(doc_id, 'foo', 'change-counter', {'counter': 1}, useBody=True)
end = time.time()
print("update handler: %s sec" % (end - start))
start = time.time()
ids = []
for id in range(size):
doc_id = "%s" % id
ids.append(doc_id)
self.db.updateBulkDocumentsWithConflictHandle(ids, {'counter': 2}, 1000)
end = time.time()
print("bulk update: %s sec" % (end - start))
def injectStepChainConfigMC(couchDatabase):
"""
_injectStepChainConfigMC_
Create a few configs in couch for a 3 step MC workflow, basically:
GEN-SIM -> DIGI -> RECO
then return a map of config names to IDs
"""
genConfig = Document()
genConfig["info"] = None
genConfig["config"] = None
genConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e234f"
genConfig["pset_hash"] = "7c856ad35f9f544839d8525ca10876a7"
genConfig["owner"] = {"group": "DATAOPS", "user": "******"}
genConfig["pset_tweak_details"] = {
"process": {
"outputModules_": ["RAWSIMoutput"],
"RAWSIMoutput": {
"dataset": {
"filterName": "FilterA",
"dataTier": "GEN-SIM"
}
}
}
}
digiConfig = Document()
digiConfig["info"] = None
digiConfig["config"] = None
digiConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e736f"
digiConfig["pset_hash"] = "7c856ad35f9f544839d8525ca11765a7"
digiConfig["owner"] = {"group": "DATAOPS", "user": "******"}
digiConfig["pset_tweak_details"] = {
"process": {
"outputModules_": ["RAWSIMoutput"],
"RAWSIMoutput": {
"dataset": {
"filterName": "FilterB",
"dataTier": "GEN-SIM-RAW"
}
}
}
}
recoConfig = Document()
recoConfig["info"] = None
recoConfig["config"] = None
recoConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5cab278a"
recoConfig["pset_hash"] = "7c856ad35f9f544839d8524ca53728a6"
recoConfig["owner"] = {"group": "DATAOPS", "user": "******"}
recoConfig["pset_tweak_details"] = {
"process": {
"outputModules_": ["RECOSIMoutput", "AODSIMoutput"],
"RECOSIMoutput": {
"dataset": {
"filterName": "FilterC",
"dataTier": "GEN-SIM-RECO"
}
},
"AODSIMoutput": {
"dataset": {
"filterName": "FilterD",
"dataTier": "AODSIM"
}
}
}
}
digi2Config = Document()
digi2Config["info"] = None
digi2Config["config"] = None
digi2Config["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e736a"
digi2Config["pset_hash"] = "7c856ad35f9f544839d8525ca11765aa"
digi2Config["owner"] = {"group": "DATAOPS", "user": "******"}
digi2Config["pset_tweak_details"] = {
"process": {
"outputModules_": ["RAWSIMoutput"],
"RAWSIMoutput": {
"dataset": {
"filterName": "",
"dataTier": "GEN-SIM-RAW"
}
}
}
}
couchDatabase.queue(genConfig)
couchDatabase.queue(digiConfig)
couchDatabase.queue(recoConfig)
couchDatabase.queue(digi2Config)
result = couchDatabase.commit()
docMap = {
"Step1": result[0][u'id'],
"Step2": result[1][u'id'],
"Step3": result[2][u'id'],
"Step4": result[3][u'id']
}
return docMap
def insertGenericRequest(self, doc):
doc = Document(doc["RequestName"], doc)
result = self.couchDB.commitOne(doc)
self.updateRequestStatus(doc["RequestName"], "new")
return result
def makeProcessingConfigs(couchDatabase):
"""
_makeProcessingConfigs_
Make a bunch of processing configs in couch for a processing chain consisting of
DigiHLT - Reco - ALCAReco - Skims
returns a map of config names to IDs
"""
rawConfig = Document()
rawConfig["info"] = None
rawConfig["config"] = None
rawConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e234f"
rawConfig["pset_hash"] = "7c856ad35f9f544839d8525ca10876a7"
rawConfig["owner"] = {"group": "cmsdataops", "user": "******"}
rawConfig["pset_tweak_details"] = {
"process": {
"outputModules_": ["writeRAWDIGI", "writeRAWDEBUGDIGI"],
"writeRAWDIGI": {
"dataset": {
"filterName": "RawDigiFilter",
"dataTier": "RAW-DIGI"
}
},
"writeRAWDEBUGDIGI": {
"dataset": {
"filterName": "RawDebugDigiFilter",
"dataTier": "RAW-DEBUG-DIGI"
}
}
}
}
recoConfig = Document()
recoConfig["info"] = None
recoConfig["config"] = None
recoConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e736f"
recoConfig["pset_hash"] = "7c856ad35f9f544839d8525ca11765a7"
recoConfig["owner"] = {"group": "cmsdataops", "user": "******"}
recoConfig["pset_tweak_details"] = {
"process": {
"outputModules_": ["writeRECO", "writeAOD", "writeALCA"],
"writeRECO": {
"dataset": {
"dataTier": "RECO",
"filterName": "reco"
}
},
"writeAOD": {
"dataset": {
"dataTier": "AOD",
"filterName": "AOD"
}
},
"writeALCA": {
"dataset": {
"dataTier": "ALCARECO",
"filterName": "alca"
}
},
}
}
alcaConfig = Document()
alcaConfig["info"] = None
alcaConfig["config"] = None
alcaConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e826a"
alcaConfig["pset_hash"] = "7c856ad35f9f544839d8525ca53628a7"
alcaConfig["owner"] = {"group": "cmsdataops", "user": "******"}
alcaConfig["pset_tweak_details"] = {
"process": {
"outputModules_":
["writeALCA1", "writeALCA2", "writeALCA3", "writeALCA4"],
"writeALCA1": {
"dataset": {
"dataTier": "ALCARECO",
"filterName": "alca1"
}
},
"writeALCA2": {
"dataset": {
"dataTier": "ALCARECO",
"filterName": "alca2"
}
},
"writeALCA3": {
"dataset": {
"dataTier": "ALCARECO",
"filterName": "alca3"
}
},
"writeALCA4": {
"dataset": {
"dataTier": "ALCARECO",
"filterName": "alca4"
}
},
}
}
skimsConfig = Document()
skimsConfig["info"] = None
skimsConfig["config"] = None
skimsConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5cab278a"
skimsConfig["pset_hash"] = "7c856ad35f9f544839d8524ca53728a6"
skimsConfig["owner"] = {"group": "cmsdataops", "user": "******"}
skimsConfig["pset_tweak_details"] = {
"process": {
"outputModules_": [
"writeSkim1", "writeSkim2", "writeSkim3", "writeSkim4",
"writeSkim5"
],
"writeSkim1": {
"dataset": {
"dataTier": "RECO-AOD",
"filterName": "skim1"
}
},
"writeSkim2": {
"dataset": {
"dataTier": "RECO-AOD",
"filterName": "skim2"
}
},
"writeSkim3": {
"dataset": {
"dataTier": "RECO-AOD",
"filterName": "skim3"
}
},
"writeSkim4": {
"dataset": {
"dataTier": "RECO-AOD",
"filterName": "skim4"
}
},
"writeSkim5": {
"dataset": {
"dataTier": "RECO-AOD",
"filterName": "skim5"
}
},
}
}
couchDatabase.queue(rawConfig)
couchDatabase.queue(recoConfig)
couchDatabase.queue(alcaConfig)
couchDatabase.queue(skimsConfig)
result = couchDatabase.commit()
docMap = {
"DigiHLT": result[0][u'id'],
"Reco": result[1][u'id'],
"ALCAReco": result[2][u'id'],
"Skims": result[3][u'id'],
}
return docMap
