def _handleAssignmentStateTransition(self, workload, request_args, dn):
if ('SoftTimeout' in request_args) and ('GracePeriod' in request_args):
request_args['HardTimeout'] = request_args['SoftTimeout'] + request_args['GracePeriod']
# Only allow extra value update for assigned status
cherrypy.log("Assign request %s, input args: %s ..." % (workload.name(), request_args))
try:
workload.updateArguments(request_args)
except Exception as ex:
msg = traceback.format_exc()
cherrypy.log("Error for request args %s: %s" % (request_args, msg))
raise InvalidSpecParameterValue(str(ex))
# validate/update OutputDatasets after ProcessingString and AcquisionEra is updated
request_args['OutputDatasets'] = workload.listOutputDatasets()
validateOutputDatasets(request_args['OutputDatasets'], workload.getDbsUrl())
# by default, it contains all unmerged LFNs (used by sites to protect the unmerged area)
request_args['OutputModulesLFNBases'] = workload.listAllOutputModulesLFNBases()
# Add parentage relation for step chain, task chain:
chainMap = workload.getChainParentageSimpleMapping()
if chainMap:
request_args["ChainParentageMap"] = chainMap
# save the spec first before update the reqmgr request status to prevent race condition
# when workflow is pulled to GQ before site white list is updated
workload.saveCouch(self.config.couch_host, self.config.couch_reqmgr_db)
report = self.reqmgr_db_service.updateRequestProperty(workload.name(), request_args, dn)
return report
def _handleAssignmentStateTransition(self, workload, request_args, dn):
req_status = request_args["RequestStatus"]
if req_status == "assigned" and not request_args.get('Team', '').strip():
raise InvalidSpecParameterValue("Team must be set during workflow assignment: %s" % request_args)
if ('SoftTimeout' in request_args) and ('GracePeriod' in request_args):
request_args['SoftTimeout'] = int(request_args['SoftTimeout'])
#TODO: not sure why GracePeriod when passed from web ingerface but convert here
request_args['GracePeriod'] = int(request_args['GracePeriod'])
request_args['HardTimeout'] = request_args['SoftTimeout'] + request_args['GracePeriod']
#Only allow extra value update for assigned status
cherrypy.log("INFO: Assign request, input args: %s ..." % request_args)
try:
workload.updateArguments(request_args)
except Exception as ex:
msg = traceback.format_exc()
cherrypy.log("Error for request args %s: %s" % (request_args, msg))
raise InvalidSpecParameterValue(str(ex))
# validate/update OutputDatasets after ProcessingString and AcquisionEra is updated
request_args['OutputDatasets'] = workload.listOutputDatasets()
validateOutputDatasets(request_args['OutputDatasets'], workload.getDbsUrl())
# legacy update schema to support ops script
loadRequestSchema(workload, request_args)
report = self.reqmgr_db_service.updateRequestProperty(workload.name(), request_args, dn)
workload.saveCouch(self.config.couch_host, self.config.couch_reqmgr_db)
return report
def _handleAssignmentStateTransition(self, workload, request_args, dn):
if ('SoftTimeout' in request_args) and ('GracePeriod' in request_args):
request_args['HardTimeout'] = request_args['SoftTimeout'] + request_args['GracePeriod']
# Only allow extra value update for assigned status
cherrypy.log("INFO: Assign request %s, input args: %s ..." % (workload.name(), request_args))
try:
workload.updateArguments(request_args)
except Exception as ex:
msg = traceback.format_exc()
cherrypy.log("Error for request args %s: %s" % (request_args, msg))
raise InvalidSpecParameterValue(str(ex))
# validate/update OutputDatasets after ProcessingString and AcquisionEra is updated
request_args['OutputDatasets'] = workload.listOutputDatasets()
validateOutputDatasets(request_args['OutputDatasets'], workload.getDbsUrl())
# by default, it contains all unmerged LFNs (used by sites to protect the unmerged area)
request_args['OutputModulesLFNBases'] = workload.listAllOutputModulesLFNBases()
# FIXME: remove it on HG1710, when this #7355 is complete fixed
if not request_args['Team'] or not isinstance(request_args['Team'], basestring):
raise InvalidSpecParameterValue("Team MUST be a non-empty string")
# legacy update schema to support ops script
loadRequestSchema(workload, request_args)
# save the spec first before update the reqmgr request status to prevent race condition
# when workflow is pulled to GQ before site white list is updated
workload.saveCouch(self.config.couch_host, self.config.couch_reqmgr_db)
report = self.reqmgr_db_service.updateRequestProperty(workload.name(), request_args, dn)
return report
def _handleAssignmentStateTransition(self, workload, request_args, dn):
if 'Team' not in request_args or not request_args['Team'].strip():
raise InvalidSpecParameterValue("A Team name must be set during workflow assignment")
if 'SiteWhitelist' not in request_args or not request_args['SiteWhitelist']:
raise InvalidSpecParameterValue("A non-empty SiteWhitelist must be set at assignment")
if ('SoftTimeout' in request_args) and ('GracePeriod' in request_args):
request_args['SoftTimeout'] = int(request_args['SoftTimeout'])
request_args['GracePeriod'] = int(request_args['GracePeriod'])
request_args['HardTimeout'] = request_args['SoftTimeout'] + request_args['GracePeriod']
# Only allow extra value update for assigned status
cherrypy.log("INFO: Assign request %s, input args: %s ..." % (workload.name(), request_args))
try:
workload.updateArguments(request_args)
except Exception as ex:
msg = traceback.format_exc()
cherrypy.log("Error for request args %s: %s" % (request_args, msg))
raise InvalidSpecParameterValue(str(ex))
# validate/update OutputDatasets after ProcessingString and AcquisionEra is updated
request_args['OutputDatasets'] = workload.listOutputDatasets()
validateOutputDatasets(request_args['OutputDatasets'], workload.getDbsUrl())
# legacy update schema to support ops script
loadRequestSchema(workload, request_args)
report = self.reqmgr_db_service.updateRequestProperty(workload.name(), request_args, dn)
workload.saveCouch(self.config.couch_host, self.config.couch_reqmgr_db)
return report
def _handleAssignmentStateTransition(self, workload, request_args, dn):
if request_args.get('Team', '').strip() == '':
raise InvalidSpecParameterValue(
"A Team name must be set during workflow assignment")
if ('SoftTimeout' in request_args) and ('GracePeriod' in request_args):
request_args['SoftTimeout'] = int(request_args['SoftTimeout'])
#TODO: not sure why GracePeriod when passed from web ingerface but convert here
request_args['GracePeriod'] = int(request_args['GracePeriod'])
request_args['HardTimeout'] = request_args[
'SoftTimeout'] + request_args['GracePeriod']
#Only allow extra value update for assigned status
cherrypy.log("INFO: Assign request, input args: %s ..." % request_args)
try:
workload.updateArguments(request_args)
except Exception as ex:
msg = traceback.format_exc()
cherrypy.log("Error for request args %s: %s" % (request_args, msg))
raise InvalidSpecParameterValue(str(ex))
# validate/update OutputDatasets after ProcessingString and AcquisionEra is updated
request_args['OutputDatasets'] = workload.listOutputDatasets()
validateOutputDatasets(request_args['OutputDatasets'],
workload.getDbsUrl())
# legacy update schema to support ops script
loadRequestSchema(workload, request_args)
report = self.reqmgr_db_service.updateRequestProperty(
workload.name(), request_args, dn)
workload.saveCouch(self.config.couch_host, self.config.couch_reqmgr_db)
return report
def testValidateOutputDatasets(self):
"""
Test the validateOutputDatasets function
"""
dbsUrl = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader/'
outputDsets = ['/PD1/AcqEra1-ProcStr1-v1/GEN']
self.assertIsNone(validateOutputDatasets(outputDsets, dbsUrl))
outputDsets.append('/PD1/AcqEra1-ProcStr1-v1/GEN-SIM')
self.assertIsNone(validateOutputDatasets(outputDsets, dbsUrl))
outputDsets.append('/PD1/AcqEra1-ProcStr1-v1/GEN-SIM-RAW')
self.assertIsNone(validateOutputDatasets(outputDsets, dbsUrl))
outputDsets.append('/PD1/AcqEra1-ProcStr1-v1/GEN')
with self.assertRaises(InvalidSpecParameterValue):
validateOutputDatasets(outputDsets, dbsUrl)
outputDsets.remove('/PD1/AcqEra1-ProcStr1-v1/GEN')
outputDsets.append('/PD1//AOD')
with self.assertRaises(InvalidSpecParameterValue):
validateOutputDatasets(outputDsets, dbsUrl)
outputDsets.remove('/PD1//AOD')
outputDsets.append('/PD1/None/AOD')
with self.assertRaises(InvalidSpecParameterValue):
validateOutputDatasets(outputDsets, dbsUrl)
outputDsets.remove('/PD1/None/AOD')
outputDsets.append('/PD1/AcqEra1-ProcStr1-v1/ALAN')
with self.assertRaises(InvalidSpecParameterValue):
validateOutputDatasets(outputDsets, dbsUrl)
def testValidateOutputDatasets(self):
"""
Test the validateOutputDatasets function
"""
dbsUrl = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader/'
outputDsets = ['/PD1/AcqEra1-ProcStr1-v1/GEN']
self.assertIsNone(validateOutputDatasets(outputDsets, dbsUrl))
outputDsets.append('/PD1/AcqEra1-ProcStr1-v1/GEN-SIM')
self.assertIsNone(validateOutputDatasets(outputDsets, dbsUrl))
outputDsets.append('/PD1/AcqEra1-ProcStr1-v1/GEN-SIM-RAW')
self.assertIsNone(validateOutputDatasets(outputDsets, dbsUrl))
outputDsets.append('/PD1/AcqEra1-ProcStr1-v1/GEN')
with self.assertRaises(InvalidSpecParameterValue):
validateOutputDatasets(outputDsets, dbsUrl)
outputDsets.remove('/PD1/AcqEra1-ProcStr1-v1/GEN')
outputDsets.append('/PD1//AOD')
with self.assertRaises(InvalidSpecParameterValue):
validateOutputDatasets(outputDsets, dbsUrl)
outputDsets.remove('/PD1//AOD')
outputDsets.append('/PD1/None/AOD')
with self.assertRaises(InvalidSpecParameterValue):
validateOutputDatasets(outputDsets, dbsUrl)
outputDsets.remove('/PD1/None/AOD')
outputDsets.append('/PD1/AcqEra1-ProcStr1-v1/ALAN')
with self.assertRaises(InvalidSpecParameterValue):
validateOutputDatasets(outputDsets, dbsUrl)