if "RunWhitelist" in schema:
helper.setRunWhitelist(schema["RunWhitelist"])
# can't save Request object directly, because it makes it hard to retrieve the _rev
metadata = {}
metadata.update(request)
# Add the output datasets if necessary
for ds in helper.listOutputDatasets():
if not ds in request['OutputDatasets']:
request['OutputDatasets'].append(ds)
# don't want to JSONify the whole workflow
del metadata['WorkloadSpec']
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
try:
wmstatSvc = WMStatsWriter(wmstatUrl)
CheckIn.checkIn(request, kwargs['RequestType'], wmstatSvc)
except CheckIn.RequestCheckInError, ex:
msg = ex._message
raise HTTPError(400, "Error in Request check-in: %s" % msg)
return request
def requestDetails(requestName):
""" Adds details from the Couch document as well as the database """
WMCore.Lexicon.identifier(requestName)
request = GetRequest.getRequestDetails(requestName)
helper = loadWorkload(request)
schema = helper.data.request.schema.dictionary_whole_tree_()
# take the stuff from the DB preferentially
schema.update(request)
task = helper.getTopLevelTask()[0]
schema['Site Whitelist'] = task.siteWhitelist()
# Add the output datasets if necessary
# for some bizarre reason OutpuDatasets is list of lists, when cloning
# [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
# #3743
if not clone:
for ds in helper.listOutputDatasets():
if ds not in request['OutputDatasets']:
request['OutputDatasets'].append(ds)
# don't want to JSONify the whole workflow
del metadata['WorkloadSpec']
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
try:
CheckIn.checkIn(request, reqSchema['RequestType'])
except CheckIn.RequestCheckInError, ex:
msg = ex._message
raise HTTPError(400, "Error in Request check-in: %s" % msg)
try:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.insertRequest(request)
except Exception as ex:
webApi.error("Could not update WMStats, reason: %s" % ex)
raise HTTPError(400,
"Creating request failed, could not update WMStats.")
return request
# Add the output datasets if necessary
# for some bizarre reason OutpuDatasets is list of lists, when cloning
# [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
# #3743
if not clone:
for ds in helper.listOutputDatasets():
if ds not in request['OutputDatasets']:
request['OutputDatasets'].append(ds)
# don't want to JSONify the whole workflow
del metadata['WorkloadSpec']
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
try:
CheckIn.checkIn(request, reqSchema['RequestType'])
except CheckIn.RequestCheckInError, ex:
raise HTTPError(400, "Error in Request check-in: %s" % str(ex))
# Inconsistent request parameters between Oracle and Couch (#4380, #4388)
# metadata above is what is saved into couch to represent a request document.
# Number of request arguments on a corresponding couch document
# is not set, has default null/None values, update those accordingly now.
# It's a mess to have two mutually inconsistent database backends.
# Not easy to handle this earlier since couch is stored first and
# some parameters are worked out later when storing into Oracle.
reqDetails = requestDetails(request["RequestName"])
# couchdb request parameters which are null at the injection time and remain so
paramsToUpdate = ["RequestStatus",
"RequestSizeFiles",
"AcquisitionEra",
def buildWorkloadAndCheckIn(webApi, reqSchema, couchUrl, couchDB, wmstatUrl, clone=False):
"""
If clone is True, the function is called on a cloned request in which
case no modification of the reqSchema shall happen and should be checked in
as is.
"""
try:
request = buildWorkloadForRequest(typename = reqSchema["RequestType"],
schema = reqSchema)
except WMSpecFactoryException as ex:
logging.error(traceback.format_exc())
raise HTTPError(400, "Error in Workload Validation: %s" % ex.message())
helper = WMWorkloadHelper(request['WorkloadSpec'])
# update request as well for wmstats update
# there is a better way to do this (passing helper to request but make sure all the information is there)
request["Campaign"] = helper.getCampaign()
# Add the output datasets if necessary
# for some bizarre reason OutpuDatasets is list of lists, when cloning
# [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
# #3743
if not clone:
for ds in helper.listOutputDatasets():
if ds not in request['OutputDatasets']:
request['OutputDatasets'].append(ds)
#TODO: need to update output dataset by Task for task chain requests
# can't save Request object directly, because it makes it hard to retrieve the _rev
metadata = {}
metadata.update(request)
# don't want to JSONify the whole workflow
del metadata['WorkloadSpec']
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
try:
CheckIn.checkIn(request, reqSchema['RequestType'])
except CheckIn.RequestCheckInError as ex:
raise HTTPError(400, "Error in Request check-in: %s" % str(ex))
# Inconsistent request parameters between Oracle and Couch (#4380, #4388)
# metadata above is what is saved into couch to represent a request document.
# Number of request arguments on a corresponding couch document
# is not set, has default null/None values, update those accordingly now.
# It's a mess to have two mutually inconsistent database backends.
# Not easy to handle this earlier since couch is stored first and
# some parameters are worked out later when storing into Oracle.
reqDetails = requestDetails(request["RequestName"])
# couchdb request parameters which are null at the injection time and remain so
paramsToUpdate = ["RequestStatus",
"RequestSizeFiles",
"AcquisitionEra",
"RequestWorkflow",
"RequestType",
"RequestStatus",
"RequestPriority",
"Requestor",
"Group",
"SizePerEvent",
"PrepID",
"RequestNumEvents",
"ProcessingString",
"ProcessingVersion",
]
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
fields = {}
for key in paramsToUpdate:
fields[key] = reqDetails[key]
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields, useBody=True)
try:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.insertRequest(request)
except Exception as ex:
webApi.error("Could not update WMStats, reason: %s" % ex)
raise HTTPError(400, "Creating request failed, could not update WMStats.")
return request
def makeRequest(kwargs, couchUrl, couchDB):
logging.info(kwargs)
""" Handles the submission of requests """
# make sure no extra spaces snuck in
for k, v in kwargs.iteritems():
if isinstance(v, str):
kwargs[k] = v.strip()
maker = retrieveRequestMaker(kwargs["RequestType"])
schema = maker.newSchema()
schema.update(kwargs)
currentTime = time.strftime('%y%m%d_%H%M%S',
time.localtime(time.time()))
requestString = schema.get('RequestString', "")
if requestString != "":
schema['RequestName'] = "%s_%s_%s" % (
schema['Requestor'], requestString, currentTime)
else:
schema['RequestName'] = "%s_%s" % (schema['Requestor'], currentTime)
schema["Campaign"] = kwargs.get("Campaign", "")
if 'Scenario' in kwargs and 'ProdConfigCacheID' in kwargs:
# Use input mode to delete the unused one
inputMode = kwargs['inputMode']
inputValues = {'scenario':'Scenario',
'couchDB':'ProdConfigCacheID'}
for n, v in inputValues.iteritems():
if n != inputMode:
schema[v] = ""
if kwargs.has_key("InputDataset"):
schema["InputDatasets"] = [kwargs["InputDataset"]]
if kwargs.has_key("FilterEfficiency"):
kwargs["FilterEfficiency"] = float(kwargs["FilterEfficiency"])
skimNumber = 1
# a list of dictionaries
schema["SkimConfigs"] = []
while kwargs.has_key("SkimName%s" % skimNumber):
d = {}
d["SkimName"] = kwargs["SkimName%s" % skimNumber]
d["SkimInput"] = kwargs["SkimInput%s" % skimNumber]
d["Scenario"] = kwargs["Scenario"]
if kwargs.get("Skim%sConfigCacheID" % skimNumber, None) != None:
d["ConfigCacheID"] = kwargs["Skim%sConfigCacheID" % skimNumber]
schema["SkimConfigs"].append(d)
skimNumber += 1
if kwargs.has_key("DataPileup") or kwargs.has_key("MCPileup"):
schema["PileupConfig"] = {}
if kwargs.has_key("DataPileup") and kwargs["DataPileup"] != "":
schema["PileupConfig"]["data"] = [kwargs["DataPileup"]]
if kwargs.has_key("MCPileup") and kwargs["MCPileup"] != "":
schema["PileupConfig"]["mc"] = [kwargs["MCPileup"]]
for runlist in ["RunWhitelist", "RunBlacklist"]:
if runlist in kwargs:
schema[runlist] = parseRunList(kwargs[runlist])
for blocklist in ["BlockWhitelist", "BlockBlacklist"]:
if blocklist in kwargs:
schema[blocklist] = parseBlockList(kwargs[blocklist])
validate(schema)
request = maker(schema)
helper = WMWorkloadHelper(request['WorkflowSpec'])
helper.setCampaign(schema["Campaign"])
helper.setRunWhitelist(schema["RunWhitelist"])
# can't save Request object directly, because it makes it hard to retrieve the _rev
metadata = {}
metadata.update(request)
# don't want to JSONify the whole workflow
del metadata['WorkflowSpec']
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
CheckIn.checkIn(request)
return request
helper.setCustodialSite(siteName = schema['SiteWhitelist'][0])
if "RunWhitelist" in schema:
helper.setRunWhitelist(schema["RunWhitelist"])
# can't save Request object directly, because it makes it hard to retrieve the _rev
metadata = {}
metadata.update(request)
# Add the output datasets if necessary
for ds in helper.listOutputDatasets():
if not ds in request['OutputDatasets']:
request['OutputDatasets'].append(ds)
# don't want to JSONify the whole workflow
del metadata['WorkloadSpec']
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
try:
CheckIn.checkIn(request)
except CheckIn.RequestCheckInError, ex:
msg = ex._message
raise HTTPError(400, "Error in Request check-in: %s" % msg)
return request
def requestDetails(requestName):
""" Adds details from the Couch document as well as the database """
WMCore.Lexicon.identifier(requestName)
request = GetRequest.getRequestDetails(requestName)
helper = loadWorkload(request)
schema = helper.data.request.schema.dictionary_()
# take the stuff from the DB preferentially
schema.update(request)
task = helper.getTopLevelTask()[0]
schema['Site Whitelist'] = task.siteWhitelist()
helper.setCustodialSite(siteName = schema['SiteWhitelist'][0])
if "RunWhitelist" in schema:
helper.setRunWhitelist(schema["RunWhitelist"])
# can't save Request object directly, because it makes it hard to retrieve the _rev
metadata = {}
metadata.update(request)
# Add the output datasets if necessary
for ds in helper.listOutputDatasets():
if not ds in request['OutputDatasets']:
request['OutputDatasets'].append(ds)
# don't want to JSONify the whole workflow
del metadata['WorkloadSpec']
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
try:
CheckIn.checkIn(request, requestType = kwargs['RequestType'])
except CheckIn.RequestCheckInError, ex:
msg = ex._message
raise HTTPError(400, "Error in Request check-in: %s" % msg)
return request
def requestDetails(requestName):
""" Adds details from the Couch document as well as the database """
WMCore.Lexicon.identifier(requestName)
request = GetRequest.getRequestDetails(requestName)
helper = loadWorkload(request)
schema = helper.data.request.schema.dictionary_()
# take the stuff from the DB preferentially
schema.update(request)
task = helper.getTopLevelTask()[0]
schema['Site Whitelist'] = task.siteWhitelist()
def buildWorkloadAndCheckIn(webApi,
reqSchema,
couchUrl,
couchDB,
wmstatUrl,
clone=False):
"""
If clone is True, the function is called on a cloned request in which
case no modification of the reqSchema shall happen and should be checked in
as is.
"""
try:
request = buildWorkloadForRequest(typename=reqSchema["RequestType"],
schema=reqSchema)
except WMSpecFactoryException as ex:
logging.error(traceback.format_exc())
raise HTTPError(400, "Error in Workload Validation: %s" % ex.message())
helper = WMWorkloadHelper(request['WorkloadSpec'])
# update request as well for wmstats update
# there is a better way to do this (passing helper to request but make sure all the information is there)
request["Campaign"] = helper.getCampaign()
# Add the output datasets if necessary
# for some bizarre reason OutpuDatasets is list of lists, when cloning
# [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
# #3743
if not clone:
for ds in helper.listOutputDatasets():
if ds not in request['OutputDatasets']:
request['OutputDatasets'].append(ds)
#TODO: need to update output dataset by Task for task chain requests
# can't save Request object directly, because it makes it hard to retrieve the _rev
metadata = {}
metadata.update(request)
# don't want to JSONify the whole workflow
del metadata['WorkloadSpec']
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
try:
CheckIn.checkIn(request, reqSchema['RequestType'])
except CheckIn.RequestCheckInError as ex:
raise HTTPError(400, "Error in Request check-in: %s" % str(ex))
# Inconsistent request parameters between Oracle and Couch (#4380, #4388)
# metadata above is what is saved into couch to represent a request document.
# Number of request arguments on a corresponding couch document
# is not set, has default null/None values, update those accordingly now.
# It's a mess to have two mutually inconsistent database backends.
# Not easy to handle this earlier since couch is stored first and
# some parameters are worked out later when storing into Oracle.
reqDetails = requestDetails(request["RequestName"])
# couchdb request parameters which are null at the injection time and remain so
paramsToUpdate = [
"RequestStatus",
"RequestSizeFiles",
"AcquisitionEra",
"RequestWorkflow",
"RequestType",
"RequestStatus",
"RequestPriority",
"Requestor",
"Group",
"SizePerEvent",
"PrepID",
"RequestNumEvents",
"ProcessingString",
"ProcessingVersion",
]
couchDb = Database(reqDetails["CouchWorkloadDBName"],
reqDetails["CouchURL"])
fields = {}
for key in paramsToUpdate:
fields[key] = reqDetails[key]
couchDb.updateDocument(request["RequestName"],
"ReqMgr",
"updaterequest",
fields=fields,
useBody=True)
try:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.insertRequest(request)
except Exception as ex:
webApi.error("Could not update WMStats, reason: %s" % ex)
raise HTTPError(400,
"Creating request failed, could not update WMStats.")
return request
# Add the output datasets if necessary
# for some bizarre reason OutpuDatasets is list of lists, when cloning
# [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
# #3743
if not clone:
for ds in helper.listOutputDatasets():
if ds not in request["OutputDatasets"]:
request["OutputDatasets"].append(ds)
# don't want to JSONify the whole workflow
del metadata["WorkloadSpec"]
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request["RequestWorkflow"] = removePasswordFromUrl(workloadUrl)
try:
CheckIn.checkIn(request, reqSchema["RequestType"])
except CheckIn.RequestCheckInError, ex:
msg = ex._message
raise HTTPError(400, "Error in Request check-in: %s" % msg)
# Inconsistent request parameters between Oracle and Couch (#4380, #4388)
# metadata above is what is saved into couch to represent a request document.
# Number of request arguments on a corresponding couch document
# is not set, has default null/None values, update those accordingly now.
# It's a mess to have two mutually inconsistent database backends.
# Not easy to handle this earlier since couch is stored first and
# some parameters are worked out later when storing into Oracle.
reqDetails = requestDetails(request["RequestName"])
# couchdb request parameters which are null at the injection time and remain so
paramsToUpdate = [
"RequestStatus",