def buildWorkload(self, originalRequestURL, arguments):
"""
_buildWorkload_
Build a resubmission workload from a previous
workload, it loads the workload and truncates it.
"""
helper = WMWorkloadHelper()
helper.loadSpecFromCouch(originalRequestURL, self.originalRequestName)
helper.truncate(self.workloadName, self.initialTaskPath,
self.acdcServer, self.acdcDatabase,
self.collectionName)
helper.ignoreOutputModules(self.ignoredOutputModules)
# override a couple of parameters, if provided by user
if 'Memory' in arguments:
helper.setMemory(arguments['Memory'])
if 'Campaign' in arguments and not helper.getCampaign():
helper.setCampaign(arguments["Campaign"])
if 'RequestPriority' in arguments:
helper.setPriority(arguments["RequestPriority"])
if 'TimePerEvent' in arguments:
for task in helper.taskIterator():
task.setJobResourceInformation(timePerEvent=arguments["TimePerEvent"])
return helper
def buildWorkload(self, arguments):
"""
_buildWorkload_
Build a resubmission workload from a previous
workload, it loads the workload and truncates it.
"""
helper = WMWorkloadHelper()
# where to find the original spec file
originalRequestURL = "%s/%s" % (arguments['CouchURL'],
arguments['CouchWorkloadDBName'])
helper.loadSpecFromCouch(originalRequestURL, self.originalRequestName)
helper.truncate(self.workloadName, self.initialTaskPath,
self.acdcServer, self.acdcDatabase,
self.collectionName)
helper.ignoreOutputModules(self.ignoredOutputModules)
# override a couple of parameters, if provided by user
if 'Memory' in arguments:
helper.setMemory(arguments['Memory'])
if 'Campaign' in arguments and not helper.getCampaign():
helper.setCampaign(arguments["Campaign"])
if 'RequestPriority' in arguments:
helper.setPriority(arguments["RequestPriority"])
if 'TimePerEvent' in arguments:
for task in helper.taskIterator():
task.setJobResourceInformation(
timePerEvent=arguments["TimePerEvent"])
return helper
def getDataFromSpecFile(specFile):
workload = WMWorkloadHelper()
workload.load(specFile)
campaign = workload.getCampaign()
result = {"Campaign": campaign}
for task in workload.taskIterator():
result[task.getPathName()] = task.getPrepID()
return result
def getDataFromSpecFile(specFile):
workload = WMWorkloadHelper()
workload.load(specFile)
campaign = workload.getCampaign()
result = {"Campaign": campaign}
for task in workload.taskIterator():
result[task.getPathName()] = task.getPrepID()
return result
def buildWorkload(self, originalRequestURL, arguments):
"""
_buildWorkload_
Build a resubmission workload from a previous
workload, it loads the workload and truncates it.
"""
#TODO remove the dependency on reqmgr1
if originalRequestURL == None:
# reqmgr1 call (Due to reqmgr2 dependency imports here
from WMCore.HTTPFrontEnd.RequestManager.ReqMgrWebTools import loadWorkload
from WMCore.RequestManager.RequestDB.Interface.Request.GetRequest import getRequestByName
originalRequest = getRequestByName(self.originalRequestName)
helper = loadWorkload(originalRequest)
else:
# reqmgr2 call
helper = WMWorkloadHelper()
helper.loadSpecFromCouch(originalRequestURL,
self.originalRequestName)
helper.truncate(self.workloadName, self.initialTaskPath,
self.acdcServer, self.acdcDatabase,
self.collectionName)
helper.ignoreOutputModules(self.ignoredOutputModules)
# override a couple of parameters, if provided by user
if 'Memory' in arguments:
helper.setMemory(arguments['Memory'])
if 'Campaign' in arguments and not helper.getCampaign():
helper.setCampaign(arguments["Campaign"])
if 'RequestPriority' in arguments:
helper.setPriority(arguments["RequestPriority"])
if 'TimePerEvent' in arguments:
for task in helper.taskIterator():
task.setJobResourceInformation(
timePerEvent=arguments["TimePerEvent"])
return helper
def buildWorkload(self, originalRequestURL, arguments):
"""
_buildWorkload_
Build a resubmission workload from a previous
workload, it loads the workload and truncates it.
"""
#TODO remove the dependency on reqmgr1
if originalRequestURL == None:
# reqmgr1 call (Due to reqmgr2 dependency imports here
from WMCore.HTTPFrontEnd.RequestManager.ReqMgrWebTools import loadWorkload
from WMCore.RequestManager.RequestDB.Interface.Request.GetRequest import getRequestByName
originalRequest = getRequestByName(self.originalRequestName)
helper = loadWorkload(originalRequest)
else:
# reqmgr2 call
helper = WMWorkloadHelper()
helper.loadSpecFromCouch(originalRequestURL, self.originalRequestName)
helper.truncate(self.workloadName, self.initialTaskPath,
self.acdcServer, self.acdcDatabase,
self.collectionName)
helper.ignoreOutputModules(self.ignoredOutputModules)
# override a couple of parameters, if provided by user
if 'Memory' in arguments:
helper.setMemory(arguments['Memory'])
if 'Campaign' in arguments and not helper.getCampaign():
helper.setCampaign(arguments["Campaign"])
if 'RequestPriority' in arguments:
helper.setPriority(arguments["RequestPriority"])
if 'TimePerEvent' in arguments:
for task in helper.taskIterator():
task.setJobResourceInformation(timePerEvent=arguments["TimePerEvent"])
return helper
def buildWorkloadAndCheckIn(webApi, reqSchema, couchUrl, couchDB, wmstatUrl, clone=False):
"""
If clone is True, the function is called on a cloned request in which
case no modification of the reqSchema shall happen and should be checked in
as is.
"""
try:
request = buildWorkloadForRequest(typename = reqSchema["RequestType"],
schema = reqSchema)
except WMSpecFactoryException as ex:
logging.error(traceback.format_exc())
raise HTTPError(400, "Error in Workload Validation: %s" % ex.message())
helper = WMWorkloadHelper(request['WorkloadSpec'])
# update request as well for wmstats update
# there is a better way to do this (passing helper to request but make sure all the information is there)
request["Campaign"] = helper.getCampaign()
# Add the output datasets if necessary
# for some bizarre reason OutpuDatasets is list of lists, when cloning
# [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
# #3743
if not clone:
for ds in helper.listOutputDatasets():
if ds not in request['OutputDatasets']:
request['OutputDatasets'].append(ds)
#TODO: need to update output dataset by Task for task chain requests
# can't save Request object directly, because it makes it hard to retrieve the _rev
metadata = {}
metadata.update(request)
# don't want to JSONify the whole workflow
del metadata['WorkloadSpec']
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
try:
CheckIn.checkIn(request, reqSchema['RequestType'])
except CheckIn.RequestCheckInError as ex:
raise HTTPError(400, "Error in Request check-in: %s" % str(ex))
# Inconsistent request parameters between Oracle and Couch (#4380, #4388)
# metadata above is what is saved into couch to represent a request document.
# Number of request arguments on a corresponding couch document
# is not set, has default null/None values, update those accordingly now.
# It's a mess to have two mutually inconsistent database backends.
# Not easy to handle this earlier since couch is stored first and
# some parameters are worked out later when storing into Oracle.
reqDetails = requestDetails(request["RequestName"])
# couchdb request parameters which are null at the injection time and remain so
paramsToUpdate = ["RequestStatus",
"RequestSizeFiles",
"AcquisitionEra",
"RequestWorkflow",
"RequestType",
"RequestStatus",
"RequestPriority",
"Requestor",
"Group",
"SizePerEvent",
"PrepID",
"RequestNumEvents",
"ProcessingString",
"ProcessingVersion",
]
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
fields = {}
for key in paramsToUpdate:
fields[key] = reqDetails[key]
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields, useBody=True)
try:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.insertRequest(request)
except Exception as ex:
webApi.error("Could not update WMStats, reason: %s" % ex)
raise HTTPError(400, "Creating request failed, could not update WMStats.")
return request
as is.
"""
try:
request = buildWorkloadForRequest(typename = reqSchema["RequestType"],
schema = reqSchema)
except WMSpecFactoryException, ex:
raise HTTPError(400, "Error in Workload Validation: %s" % str(ex))
helper = WMWorkloadHelper(request['WorkloadSpec'])
#4378 - ACDC (Resubmission) requests should inherit the Campaign ...
# for Resubmission request, there already is previous Campaign set
# this call would override it with initial request arguments where
# it is not specified, so would become ''
if not helper.getCampaign():
helper.setCampaign(reqSchema["Campaign"])
# update request as well for wmstats update
# there is a better way to do this (passing helper to request but make sure all the information is there)
request["Campaign"] = helper.getCampaign()
# can't save Request object directly, because it makes it hard to retrieve the _rev
metadata = {}
metadata.update(request)
# Add the output datasets if necessary
# for some bizarre reason OutpuDatasets is list of lists, when cloning
# [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
# #3743
if not clone:
# will be stored under request["WorkloadSpec"]
self.create_workload_attach_to_request(request, request_input_dict)
cherrypy.log("INFO: Request corresponding workload instantiated, storing ...")
helper = WMWorkloadHelper(request["WorkloadSpec"])
# TODO
# this should be revised for ReqMgr2
#4378 - ACDC (Resubmission) requests should inherit the Campaign ...
# for Resubmission request, there already is previous Campaign set
# this call would override it with initial request arguments where
# it is not specified, so would become ''
# TODO
# these kind of calls should go into some workload initialization
if not helper.getCampaign():
helper.setCampaign(request["Campaign"])
if request.has_key("RunWhitelist"):
helper.setRunWhitelist(request["RunWhitelist"])
# storing the request document into Couch
# can't save Request object directly, because it makes it hard to retrieve
# the _rev
# TODO
# don't understand this. may just be possible to keep dealing with
# 'request' and not create this metadata
metadata = {}
metadata.update(request)
# will be stored under request["WorkloadSpec"]
self.create_workload_attach_to_request(request, request_input_dict)
cherrypy.log(
"INFO: Request corresponding workload instantiated, storing ...")
helper = WMWorkloadHelper(request["WorkloadSpec"])
# TODO
# this should be revised for ReqMgr2
#4378 - ACDC (Resubmission) requests should inherit the Campaign ...
# for Resubmission request, there already is previous Campaign set
# this call would override it with initial request arguments where
# it is not specified, so would become ''
# TODO
# these kind of calls should go into some workload initialization
if not helper.getCampaign():
helper.setCampaign(request["Campaign"])
if request.has_key("RunWhitelist"):
helper.setRunWhitelist(request["RunWhitelist"])
# storing the request document into Couch
# can't save Request object directly, because it makes it hard to retrieve
# the _rev
# TODO
# don't understand this. may just be possible to keep dealing with
# 'request' and not create this metadata
metadata = {}
metadata.update(request)
as is.
"""
try:
request = buildWorkloadForRequest(typename = reqSchema["RequestType"],
schema = reqSchema)
except WMSpecFactoryException, ex:
raise HTTPError(400, "Error in Workload Validation: %s" % str(ex))
helper = WMWorkloadHelper(request['WorkloadSpec'])
#4378 - ACDC (Resubmission) requests should inherit the Campaign ...
# for Resubmission request, there already is previous Campaign set
# this call would override it with initial request arguments where
# it is not specified, so would become ''
if not helper.getCampaign():
helper.setCampaign(reqSchema["Campaign"])
# update request as well for wmstats update
# there is a better way to do this (passing helper to request but make sure all the information is there)
request["Campaign"] = helper.getCampaign()
# can't save Request object directly, because it makes it hard to retrieve the _rev
metadata = {}
metadata.update(request)
# Add the output datasets if necessary
# for some bizarre reason OutpuDatasets is list of lists, when cloning
# [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
# #3743
if not clone:
def buildWorkloadAndCheckIn(webApi,
reqSchema,
couchUrl,
couchDB,
wmstatUrl,
clone=False):
"""
If clone is True, the function is called on a cloned request in which
case no modification of the reqSchema shall happen and should be checked in
as is.
"""
try:
request = buildWorkloadForRequest(typename=reqSchema["RequestType"],
schema=reqSchema)
except WMSpecFactoryException as ex:
logging.error(traceback.format_exc())
raise HTTPError(400, "Error in Workload Validation: %s" % ex.message())
helper = WMWorkloadHelper(request['WorkloadSpec'])
# update request as well for wmstats update
# there is a better way to do this (passing helper to request but make sure all the information is there)
request["Campaign"] = helper.getCampaign()
# Add the output datasets if necessary
# for some bizarre reason OutpuDatasets is list of lists, when cloning
# [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
# #3743
if not clone:
for ds in helper.listOutputDatasets():
if ds not in request['OutputDatasets']:
request['OutputDatasets'].append(ds)
#TODO: need to update output dataset by Task for task chain requests
# can't save Request object directly, because it makes it hard to retrieve the _rev
metadata = {}
metadata.update(request)
# don't want to JSONify the whole workflow
del metadata['WorkloadSpec']
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
try:
CheckIn.checkIn(request, reqSchema['RequestType'])
except CheckIn.RequestCheckInError as ex:
raise HTTPError(400, "Error in Request check-in: %s" % str(ex))
# Inconsistent request parameters between Oracle and Couch (#4380, #4388)
# metadata above is what is saved into couch to represent a request document.
# Number of request arguments on a corresponding couch document
# is not set, has default null/None values, update those accordingly now.
# It's a mess to have two mutually inconsistent database backends.
# Not easy to handle this earlier since couch is stored first and
# some parameters are worked out later when storing into Oracle.
reqDetails = requestDetails(request["RequestName"])
# couchdb request parameters which are null at the injection time and remain so
paramsToUpdate = [
"RequestStatus",
"RequestSizeFiles",
"AcquisitionEra",
"RequestWorkflow",
"RequestType",
"RequestStatus",
"RequestPriority",
"Requestor",
"Group",
"SizePerEvent",
"PrepID",
"RequestNumEvents",
"ProcessingString",
"ProcessingVersion",
]
couchDb = Database(reqDetails["CouchWorkloadDBName"],
reqDetails["CouchURL"])
fields = {}
for key in paramsToUpdate:
fields[key] = reqDetails[key]
couchDb.updateDocument(request["RequestName"],
"ReqMgr",
"updaterequest",
fields=fields,
useBody=True)
try:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.insertRequest(request)
except Exception as ex:
webApi.error("Could not update WMStats, reason: %s" % ex)
raise HTTPError(400,
"Creating request failed, could not update WMStats.")
return request
