def loadWorkload(self, inputWorkload):
"""
If workload is sane, then use it
"""
if inputWorkload == None:
self.workload = None
if isinstance(inputWorkload, WMWorkload):
self.workload = WMWorkloadHelper(inputWorkload)
return
if isinstance(inputWorkload, WMWorkloadHelper):
self.workload = inputWorkload
return
if not os.path.exists(inputWorkload):
raise Exception('Could not find %s in local file system' %
(str(inputWorkload)))
testWorkload = WMWorkloadHelper(WMWorkload("workload"))
testWorkload.load(inputWorkload)
self.workload = testWorkload
return
def loadWorkload(request):
""" Returns a WMWorkloadHelper for the workload contained in the request """
url = request['RequestWorkflow']
helper = WMWorkloadHelper()
try:
WMCore.Lexicon.couchurl(url)
except Exception:
raise cherrypy.HTTPError(400, "Invalid workload "+urllib.quote(url))
helper = WMWorkloadHelper()
try:
helper.load(url)
except Exception:
raise cherrypy.HTTPError(404, "Cannot find workload "+removePasswordFromUrl(url))
return helper
def buildWorkload(self, arguments):
"""
_buildWorkload_
Build a resubmission workload from a previous
workload, it loads the workload and truncates it.
"""
helper = WMWorkloadHelper()
# where to find the original spec file
originalRequestURL = "%s/%s" % (arguments['CouchURL'],
arguments['CouchWorkloadDBName'])
helper.loadSpecFromCouch(originalRequestURL, self.originalRequestName)
helper.truncate(self.workloadName, self.initialTaskPath,
self.acdcServer, self.acdcDatabase,
self.collectionName)
helper.ignoreOutputModules(self.ignoredOutputModules)
# override a couple of parameters, if provided by user
if 'Memory' in arguments:
helper.setMemory(arguments['Memory'])
if 'Campaign' in arguments and not helper.getCampaign():
helper.setCampaign(arguments["Campaign"])
if 'RequestPriority' in arguments:
helper.setPriority(arguments["RequestPriority"])
if 'TimePerEvent' in arguments:
for task in helper.taskIterator():
task.setJobResourceInformation(
timePerEvent=arguments["TimePerEvent"])
return helper
def buildWorkload(self, arguments):
"""
_buildWorkload_
Build a resubmission workload from a previous
workload, it loads the workload and truncates it.
"""
helper = WMWorkloadHelper()
# where to find the original spec file
originalRequestURL = "%s/%s" % (arguments['CouchURL'],
arguments['CouchWorkloadDBName'])
helper.loadSpecFromCouch(originalRequestURL, self.originalRequestName)
helper.truncate(self.workloadName, self.initialTaskPath,
self.acdcServer, self.acdcDatabase,
self.collectionName)
helper.ignoreOutputModules(self.ignoredOutputModules)
# override a couple of parameters, if provided by user
# Note that if it was provided by the user, then it's already part of the arguments too
if "RequestPriority" in self.userArgs:
helper.setPriority(arguments["RequestPriority"])
if "Memory" in self.userArgs:
helper.setMemory(arguments["Memory"])
if "Multicore" in self.userArgs or "EventStreams" in self.userArgs:
self.setCoresAndStreams(helper, arguments)
if "TimePerEvent" in self.userArgs:
helper.setTimePerEvent(arguments.get("TimePerEvent"))
return helper
def retrieveSchema(workflowName):
specURL = os.path.join(reqmgrCouchURL, workflowName, "spec")
helper = WMWorkloadHelper()
helper.load(specURL)
schema = {}
# for (key, value) in helper.data.request.schema.dictionary_().iteritems():
for (key, value
) in helper.data.request.schema.dictionary_whole_tree_().iteritems():
if key == 'ProdConfigCacheID':
schema['ConfigCacheID'] = value
elif key == 'ProcConfigCacheID':
schema['ConfigCacheID'] = value
elif key == 'RequestSizeEvents':
schema['RequestNumEvents'] = value
elif value != None:
schema[key] = value
# print "Retrieved schema:\n", schema
schema['Requestor'] = 'anlevin'
schema['Group'] = 'DATAOPS'
# schema['Task1']['SplittingArguments'] = {'lumis_per_job': 5}
# schema['Memory'] = 1394
# schema['Task1']['KeepOutput'] = True
# schema['RequestString'] = 'TEST_DELETE_THIS_'
# schema['BlockWhitelist'] = ['/MinimumBias/Run2012D-HLTPhysics-Tier1PromptSkim-v1/RAW-RECO#ce668e80-26a2-11e2-80e7-00155dffff9d']
# schema['Task1']['BlockBlacklist'] = ['/DoubleMu/Run2011A-ZMu-08Nov2011-v1/RAW-RECO#93c53d22-25b2-11e1-8c62-003048f02c8a']
# schema['Task1']['RunWhitelist'] = [208307]
# del schema['Task2']['MCPileup']
return schema
def buildWorkload(self, originalRequestURL):
"""
_buildWorkload_
Build a resubmission workload from a previous
workload, it loads the workload and truncates it.
"""
#TODO remove the dependency on reqmgr1
if originalRequestURL == None:
# reqmgr1 call (Due to reqmgr2 dependency imports here
from WMCore.HTTPFrontEnd.RequestManager.ReqMgrWebTools import loadWorkload
from WMCore.RequestManager.RequestDB.Interface.Request.GetRequest import getRequestByName
originalRequest = getRequestByName(self.originalRequestName)
helper = loadWorkload(originalRequest)
else:
# reqmgr2 call
helper = WMWorkloadHelper()
helper.loadSpecFromCouch(originalRequestURL, self.originalRequestName)
helper.truncate(self.workloadName, self.initialTaskPath,
self.acdcServer, self.acdcDatabase,
self.collectionName)
helper.ignoreOutputModules(self.ignoredOutputModules)
return helper
def getWMWorkload(self):
"""
_getWMTask_
Get the WMTask for this space
TODO: Refactor to getWMWorkload method
"""
if self.workload != None:
return self.workload
try:
import WMSandbox
except ImportError as ex:
msg = "Error importing WMSandbox module"
msg += str(ex)
raise RuntimeError(msg)
wmsandboxLoc = inspect.getsourcefile(WMSandbox)
workloadPcl = wmsandboxLoc.replace("__init__.py","WMWorkload.pkl")
with open(workloadPcl, 'rb') as handle:
wmWorkload = pickle.load(handle)
self.workload = WMWorkloadHelper(wmWorkload)
return
def buildWorkload(self, arguments):
"""
_buildWorkload_
Build a resubmission workload from a previous
workload, it loads the workload and truncates it.
"""
helper = WMWorkloadHelper()
# where to find the original spec file
originalRequestURL = "%s/%s" % (arguments['CouchURL'], arguments['CouchWorkloadDBName'])
helper.loadSpecFromCouch(originalRequestURL, self.originalRequestName)
helper.truncate(self.workloadName, self.initialTaskPath,
self.acdcServer, self.acdcDatabase,
self.collectionName)
helper.ignoreOutputModules(self.ignoredOutputModules)
# override a couple of parameters, if provided by user
if 'RequestPriority' in arguments:
helper.setPriority(arguments["RequestPriority"])
if arguments['OriginalRequestType'] != 'TaskChain' or isinstance(arguments['Memory'], dict):
helper.setMemory(arguments['Memory'])
if arguments['OriginalRequestType'] != 'TaskChain' or isinstance(arguments.get('TimePerEvent'), dict):
helper.setTimePerEvent(arguments.get("TimePerEvent"))
return helper
def modifyWorkload(self, requestName, workload,
CMSSWVersion=None, GlobalTag=None,
runWhitelist=None, runBlacklist=None,
blockWhitelist=None, blockBlacklist=None,
ScramArch=None):
""" handles the "Modify" button of the details page """
self.validate(requestName)
helper = WMWorkloadHelper()
helper.load(workload)
schema = helper.data.request.schema
message = ""
if runWhitelist != "" and runWhitelist != None:
l = Utilities.parseRunList(runWhitelist)
helper.setRunWhitelist(l)
schema.RunWhitelist = l
message += 'Changed runWhiteList to %s<br>' % l
if runBlacklist != "" and runBlacklist != None:
l = Utilities.parseRunList(runBlacklist)
helper.setRunBlacklist(l)
schema.RunBlacklist = l
message += 'Changed runBlackList to %s<br>' % l
if blockWhitelist != "" and blockWhitelist != None:
l = Utilities.parseBlockList(blockWhitelist)
helper.setBlockWhitelist(l)
schema.BlockWhitelist = l
message += 'Changed blockWhiteList to %s<br>' % l
if blockBlacklist != "" and blockBlacklist != None:
l = Utilities.parseBlockList(blockBlacklist)
helper.setBlockBlacklist(l)
schema.BlockBlacklist = l
message += 'Changed blockBlackList to %s<br>' % l
Utilities.saveWorkload(helper, workload)
return message + detailsBackLink(requestName)
def get(self, name, web_form=False):
"""
getting job splitting algorithm.
:arg str name: name to appear in the result message.
:returns: row with response, here 1 item list with message.
"""
helper = WMWorkloadHelper()
try:
helper.loadSpecFromCouch(self.reqdb_url, name)
except Exception:
raise cherrypy.HTTPError(404, "Cannot find workload: %s" % name)
splittingDict = helper.listJobSplittingParametersByTask(
performance=False)
taskNames = sorted(splittingDict.keys())
splitInfo = []
for taskName in taskNames:
splitInfo.append({
"splitAlgo": splittingDict[taskName]["algorithm"],
"splitParams": splittingDict[taskName],
"taskType": splittingDict[taskName]["type"],
"taskName": taskName
})
if web_form:
splitInfo = create_web_splitting_format(splitInfo)
return splitInfo
def retrieveSchema(workflowName, user, group):
specURL = os.path.join(reqmgrCouchURL, workflowName, "spec")
helper = WMWorkloadHelper()
#print " retrieving original workflow...",
helper.load(specURL)
#print "done."
schema = {}
#for (key, value) in helper.data.request.schema.dictionary_whole_tree_().iteritems():
for (key, value) in helper.data.request.schema.dictionary_().iteritems():
#print key, value
if key == 'ProcConfigCacheID':
schema['ConfigCacheID'] = value
elif key == 'RequestSizeEvents':
schema['RequestSizeEvents'] = value
#schema['RequestNumEvents'] = int(value)
elif key == 'Requestor':
schema['Requestor'] = user
elif key == 'Group':
schema['Group'] = group
#elif key=='ProdJobSplitArgs':
# schema['ProdJobSplitArgs']={'events_per_job': 500000, 'events_per_lumi': 300, 'lheInputFiles': True}
elif value != None:
schema[key] = value
return schema
def queueWork(self, wmspecUrl, request=None, team=None):
"""
Take and queue work from a WMSpec.
If request name is provided but doesn't match WMSpec name
an error is raised.
If team is provided work will only be available to queue's
belonging to that team.
Duplicate specs will be ignored.
"""
self.logger.info('queueWork() begin queueing "%s"' % wmspecUrl)
wmspec = WMWorkloadHelper()
wmspec.load(wmspecUrl)
if request: # validate request name
try:
Lexicon.requestName(request)
except Exception, ex: # can throw many errors e.g. AttributeError, AssertionError etc.
error = WorkQueueWMSpecError(
wmspec, "Request name validation error: %s" % str(ex))
raise error
if request != wmspec.name():
raise WorkQueueWMSpecError(
wmspec, 'Request & workflow name mismatch %s vs %s' %
(request, wmspec.name()))
def retrieveSchema(url, workflowName, user, group):
specURL = os.path.join(reqmgrCouchURL, workflowName, "spec")
helper = WMWorkloadHelper()
#print " retrieving original workflow...",
helper.load(specURL)
#print "done."
schema = {}
#for (key, value) in helper.data.request.schema.dictionary_whole_tree_().iteritems():
for (key, value) in helper.data.request.schema.dictionary_().iteritems():
#print key, value
if key == 'ProcConfigCacheID':
schema['ConfigCacheID'] = value
elif key == 'RequestSizeEvents':
schema['RequestSizeEvents'] = value
#schema['RequestNumEvents'] = int(value)
elif key == 'Requestor':
schema['Requestor'] = user
elif key == 'Group':
schema['Group'] = group
elif key == 'RequestNumEvents':
schema['RequestNumEvents'] = getFinalRequestedNumEvents(
url, workflowName)
elif key == 'FirstLumi':
schema['FirstLumi'] = getMaxLumi(url, workflowName) * 2
elif key == 'FirstEvent':
schema['FirstEvent'] = deprecated.dbsTest.getInputEvents(
url, workflowName) * 2
elif key == 'RequestString':
schema['RequestString'] = 'ACDC_' + value
elif value != None:
schema[key] = value
return schema
def getMasterName(startDir, wmWorkload = None, workflow = None):
"""
Gets a universal name for the jobGroup directory
Return the uid as the name if none available (THIS SHOULD NEVER HAPPEN)
"""
if wmWorkload != None:
workload = wmWorkload.name()
elif not os.path.exists(workflow.spec):
msg = "Could not find Workflow spec %s: " % (workflow.spec)
msg += "Cannot create work area without spec!"
logging.error(msg)
raise CreateWorkAreaException(msg)
else:
wmWorkload = WMWorkloadHelper(WMWorkload("workload"))
wmWorkload.load(workflow.spec)
workload = wmWorkload.name()
task = workflow.task
if task.startswith("/" + workload + "/"):
task = task[len(workload) + 2:]
return (os.path.join(startDir, workload),
os.path.join(startDir, workload, task))
def getMasterName(self):
"""
Gets a universal name for the jobGroup directory
Return the uid as the name if none available (THIS SHOULD NEVER HAPPEN)
"""
if self.wmWorkload != None:
workload = self.wmWorkload.name()
elif not os.path.exists(self.workflow.spec):
logging.error(
"Could not find Workflow spec %s; labeling jobs by job ID only!"
% (self.workflow.spec))
return os.path.join(self.startDir,
self.jobGroup.uid), os.path.join(
self.startDir, self.jobGroup.uid)
else:
wmWorkload = WMWorkloadHelper(WMWorkload("workload"))
wmWorkload.load(self.workflow.spec)
workload = wmWorkload.name()
task = self.workflow.task
if task.startswith("/" + workload + "/"):
task = task[len(workload) + 2:]
return os.path.join(self.startDir,
workload), os.path.join(self.startDir, workload,
task)
def retrieveSchema(workflowName):
specURL = os.path.join(reqmgrCouchURL, workflowName, "spec")
helper = WMWorkloadHelper()
helper.load(specURL)
paramBlacklist = [
'AcquisitionEra', 'BlockCloseMaxEvents', 'BlockCloseMaxFiles',
'BlockCloseMaxSize', 'BlockCloseMaxWaitTime', 'CouchURL',
'CouchWorkloadDBName', 'CustodialGroup', 'CustodialSubType',
'Dashboard', 'GracePeriod', 'Group', 'HardTimeout', 'InitialPriority',
'inputMode', 'MaxMergeEvents', 'MaxMergeSize', 'MaxRSS', 'MaxVSize',
'MergedLFNBase', 'MinMergeSize', 'NonCustodialGroup',
'NonCustodialSubType', 'ProcessingString', 'OutputDatasets',
'ReqMgr2Only', 'Requestor', 'RequestDate'
'RequestorDN', 'RequestName', 'RequestStatus', 'RequestTransition',
'RequestWorkflow', 'SiteWhitelist', 'SoftTimeout', 'SoftwareVersions',
'SubscriptionPriority', 'Team', 'timeStamp'
]
schema = {}
for (key, value
) in helper.data.request.schema.dictionary_whole_tree_().iteritems():
if not value or key in paramBlacklist:
continue
else:
schema[key] = value
schema['Requestor'] = 'amaltaro'
schema['Group'] = 'DATAOPS'
return schema
def getDataFromSpecFile(specFile):
workload = WMWorkloadHelper()
workload.load(specFile)
campaign = workload.getCampaign()
result = {"Campaign": campaign}
for task in workload.taskIterator():
result[task.getPathName()] = task.getPrepID()
return result
def validate_request_update_args(request_args, config, reqmgr_db_service,
param):
"""
param and safe structure is RESTArgs structure: named tuple
RESTArgs(args=[], kwargs={})
validate post request
1. read data from body
2. validate the permission (authentication)
3. validate state transition (against previous state from couchdb)
2. validate using workload validation
3. convert data from body to arguments (spec instance, argument with default setting)
TODO: rasie right kind of error with clear message
"""
request_name = request_args["RequestName"]
# this need to be deleted for validation
del request_args["RequestName"]
couchurl = '%s/%s' % (config.couch_host, config.couch_reqmgr_db)
workload = WMWorkloadHelper()
# param structure is RESTArgs structure.
workload.loadSpecFromCouch(couchurl, request_name)
# first validate the permission by status and request type.
# if the status is not set only ReqMgr Admin can change the the values
# TODO for each step, assigned, approved, announce find out what other values
# can be set
request_args["RequestType"] = workload.requestType()
permission = getWritePermission(request_args)
authz_match(permission['role'], permission['group'])
del request_args["RequestType"]
#validate the status
if "RequestStatus" in request_args:
validate_state_transition(reqmgr_db_service, request_name,
request_args["RequestStatus"])
# delete request_args since it is not part of spec argument sand validation
args_without_status = {}
args_without_status.update(request_args)
del args_without_status["RequestStatus"]
else:
args_without_status = request_args
if len(args_without_status) > 0 and not workqueue_stat_validation(
args_without_status):
# validate the arguments against the spec argumentSpecdefinition
#TODO: currently only assigned status allows any update other then Status update
workload.validateArgumentForAssignment(args_without_status)
# to update request_args with type conversion
request_args.update(args_without_status)
return workload, request_args
def getSplitting(requestName):
reqmgrUrl = 'https://cmsweb.cern.ch/reqmgr/reqMgr/'
reqmgr = RequestManager(dict={'endpoint': reqmgrUrl})
result = reqmgr.getRequest(requestName)
workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
workloadPickle = workloadDB.getAttachment(requestName, 'spec')
spec = pickle.loads(workloadPickle)
workload = WMWorkloadHelper(spec)
params = workload.getTopLevelTask()[0].jobSplittingParameters()
algo = workload.getTopLevelTask()[0].jobSplittingAlgorithm()
return params['events_per_job']
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
def validate_request_update_args(request_args, config, reqmgr_db_service,
param):
"""
param and safe structure is RESTArgs structure: named tuple
RESTArgs(args=[], kwargs={})
validate post/put request
1. read data from body
2. validate the permission (authentication)
3. validate state transition (against previous state from couchdb)
2. validate using workload validation
3. convert data from body to arguments (spec instance, argument with default setting)
TODO: raise right kind of error with clear message
"""
# this needs to be deleted for validation
request_name = request_args.pop("RequestName")
couchurl = '%s/%s' % (config.couch_host, config.couch_reqmgr_db)
workload = WMWorkloadHelper()
workload.loadSpecFromCouch(couchurl, request_name)
# first validate the permission by status and request type.
# if the status is not set only ReqMgr Admin can change the values
# TODO for each step, assigned, approved, announce find out what other values
# can be set
request_args["RequestType"] = workload.requestType()
permission = getWritePermission(request_args)
authz_match(permission['role'], permission['group'])
del request_args["RequestType"]
# validate the status
if "RequestStatus" in request_args:
validate_state_transition(reqmgr_db_service, request_name,
request_args["RequestStatus"])
if request_args["RequestStatus"] in STATES_ALLOW_ONLY_STATE_TRANSITION:
# if state change doesn't allow other transition nothing else to validate
args_only_status = {}
args_only_status["RequestStatus"] = request_args["RequestStatus"]
args_only_status["cascade"] = request_args.get("cascade", False)
return workload, args_only_status
elif request_args["RequestStatus"] == 'assigned':
workload.validateArgumentForAssignment(request_args)
# TODO: fetch it from the assignment arg definition
if 'RequestPriority' in request_args:
request_args['RequestPriority'] = int(request_args['RequestPriority'])
if (lambda x:
(x >= 0 and x < 1e6))(request_args['RequestPriority']) is False:
raise InvalidSpecParameterValue(
"RequestPriority must be an integer between 0 and 1e6")
return workload, request_args
def loadOverride(self, specPath):
"""
_loadOverride_
Loads the spec (if not in the cache)
and gets the location where the files should be injected to.
If not possible to load the spec, raise an exception.
"""
helper = WMWorkloadHelper()
helper.load(specPath)
overrideSite = helper.getPhEDExInjectionOverride()
self.specCache[specPath] = overrideSite
return overrideSite
def loadWorkload():
"""
_loadWorkload_
Load the Workload from the WMSandbox Area
"""
sandboxLoc = locateWMSandbox()
workloadPcl = "%s/WMWorkload.pkl" % sandboxLoc
with open(workloadPcl, 'r') as handle:
wmWorkload = pickle.load(handle)
return WMWorkloadHelper(wmWorkload)
def getElementsForSplitting(self):
"""Returns the elements from the inbox that need to be split,
if WorkflowName specified only return elements to split for that workflow"""
elements = self.getInboxElements(status='Negotiating')
specs = {} # cache as may have multiple elements for same spec
for ele in elements:
if ele['RequestName'] not in specs:
wmspec = WMWorkloadHelper()
wmspec.load(self.parentCouchUrlWithAuth + "/%s/spec" % ele['RequestName'])
specs[ele['RequestName']] = wmspec
ele['WMSpec'] = specs[ele['RequestName']]
del specs
return elements
def retrieveSchema(
workflowName,
reqmgrCouchURL="https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache"):
"""
Creates the cloned specs for the original request
Updates parameters
"""
from WMCore.WMSpec.WMWorkload import WMWorkloadHelper
specURL = os.path.join(reqmgrCouchURL, workflowName, "spec")
helper = WMWorkloadHelper()
helper.load(specURL)
return helper
def main():
start = time.time()
# blockName = match['Inputs'].keys()[0]
blockName = "/acdc/vlimant_ACDC0_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_190218_145226_481/:pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222:SUS-RunIIFall18wmLHEGS-00025_0/0/31055"
# acdcInfo = match['ACDC']
acdcInfo = {"database": "acdcserver",
"fileset": "/pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222/SUS-RunIIFall18wmLHEGS-00025_0",
"collection": "pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222",
"server": "https://cmsweb.cern.ch/couchdb"}
acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"])
splitedBlockName = ACDCBlock.splitBlockName(blockName)
print("Splitted block name: %s" % splitedBlockName)
fileLists = acdc.getChunkFiles(acdcInfo['collection'],
acdcInfo['fileset'],
splitedBlockName['Offset'],
splitedBlockName['NumOfFiles'])
print("Retrieved %d unique files from the ACDCServer" % len(fileLists))
block = {}
block["Files"] = fileLists
wantedLumis = set([252052, 240646])
for f in fileLists:
for run in f['runs']:
maskDict = run.json()
lumisSet = set(maskDict['Lumis'].keys())
if wantedLumis.intersection(lumisSet):
print("File: %s with events: %s, contains these lumis: %s" % (f['lfn'], f['events'], wantedLumis.intersection(lumisSet)))
# with open("chunkfiles.json", 'w') as fo:
# json.dump(block, fo)
end = time.time()
print("Spent %s secs running so far" % (end - start))
sys.exit(1)
### Now doing the WMBSHelper stuff
reqUrl = "https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache"
requestName = "vlimant_ACDC0_task_HIG-RunIIFall17wmLHEGS-01122__v1_T_180808_130708_5376"
wmspec = WMWorkloadHelper()
wmspec.loadSpecFromCouch(reqUrl, requestName)
taskName = "HIG-RunIIFall17DRPremix-00788_0"
mask = None
cacheDir = "/data/srv/wmagent/v1.1.14.patch6/install/wmagent/WorkQueueManager/cache"
# wmbsHelper = WMBSHelper(wmspec, match['TaskName'], blockName, mask, self.params['CacheDir'])
wmbsHelper = WMBSHelper(wmspec, taskName, blockName, mask, cacheDir)
sub, numFilesAdded = wmbsHelper.createSubscriptionAndAddFiles(block=block)
def retrieveSchema(workflowName):
specURL = os.path.join(reqmgrCouchURL, workflowName, "spec")
helper = WMWorkloadHelper()
helper.load(specURL)
schema = {}
# for (key, value) in helper.data.request.schema.dictionary_().iteritems():
for (key, value
) in helper.data.request.schema.dictionary_whole_tree_().iteritems():
if key == 'ProdConfigCacheID':
schema['ConfigCacheID'] = value
elif key == 'ProcConfigCacheID':
schema['ConfigCacheID'] = value
elif key == 'RequestSizeEvents':
schema['RequestNumEvents'] = value
elif key == 'ProcessingString' and value == {}:
continue
elif key == 'AcquisitionEra' and value == {}:
continue
elif key == 'SkimConfigs' and not value:
continue
elif value != None:
schema[key] = value
# print "Retrieved schema:\n", schema ### FOR DEBUG
request = deepcopy(schema)
request['Requestor'] = 'anlevin'
request['Group'] = 'DATAOPS'
### Now changing the parameters according to HG1309
x = 1
while x <= schema['TaskChain']:
task = 'Task' + str(x)
for (key, value) in schema[task].iteritems():
if key == "SplittingAlgorithm":
request[task]['SplittingAlgo'] = value
del request[task]['SplittingAlgorithm']
elif key == "SplittingArguments":
for (k2, v2) in schema[task][key].iteritems():
if k2 == "lumis_per_job":
request[task]["LumisPerJob"] = v2
elif k2 == "events_per_job":
request[task]["EventsPerJob"] = v2
del request[task]['SplittingArguments']
x += 1
#request['Memory'] = 3900
#del request['SiteWhitelist']
#request['RequestString'] = 'TEST_ANDREW_TEST'
# request['Task1']['BlockBlacklist'] = ['/DoubleMu/Run2011A-ZMu-08Nov2011-v1/RAW-RECO#93c53d22-25b2-11e1-8c62-003048f02c8a']
#request['Task1']['BlockWhitelist'] = ['/RelValQCD_Pt_80_170_BCtoE_8TeV/CMSSW_6_2_0_pre8-PRE_ST62_V8-v3/GEN-SIM#d99587e0-625e-11e3-ad0f-00221959e7c0','/RelValQCD_Pt_80_170_BCtoE_8TeV/CMSSW_6_2_0_pre8-PRE_ST62_V8-v3/GEN-SIM#8689de68-606d-11e3-ad0f-00221959e7c0']
return request
def getRequestInformationAndWorkload(requestName, reqmgrUrl, centralRequestDBURL):
"""
_getRequestInformationAndWorkload_
Retrieve the request information for assignment
and the full pickled workload.
"""
wfDBReader = RequestDBReader(centralRequestDBURL, couchapp="ReqMgr")
result = wfDBReader.getRequestByNames(requestName, True)
workloadDB = Database(result[requestName]['CouchWorkloadDBName'], result[requestName]['CouchURL'])
workloadPickle = workloadDB.getAttachment(requestName, 'spec')
spec = pickle.loads(workloadPickle)
workload = WMWorkloadHelper(spec)
return workload, result[requestName]
def getRequestInformationAndWorkload(requestName, reqmgrUrl):
"""
_getRequestInformationAndWorkload_
Retrieve the request information for assignment
and the full pickled workload.
"""
reqmgr = RequestManager(dict={'endpoint': reqmgrUrl})
result = reqmgr.getRequest(requestName)
workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
workloadPickle = workloadDB.getAttachment(requestName, 'spec')
spec = pickle.loads(workloadPickle)
workload = WMWorkloadHelper(spec)
return workload, result
def retrieveSchema(workflowName, newBlockBlacklist):
specURL = os.path.join(reqmgrCouchURL, workflowName, "spec")
helper = WMWorkloadHelper()
helper.load(specURL)
schema = {}
for (key, value) in helper.data.request.schema.dictionary_().iteritems():
#print key
if key == 'ProdConfigCacheID':
schema['ProdConfigCacheID'] = value
elif value != None:
schema[key] = value
schema["BlockBlacklist"] = "[" + newBlockBlacklist + "]"
return schema
