def _raiseCheckInError(request, ex, msg):
"""
Private function called only from this module.
Always from the except exception block - the traceback
is put in the local log file.
"""
requestName = request['RequestName']
msg += '\n' + str(ex)
msg += "\nUnable to check in new request %s" % requestName
reqId = GetRequest.requestID(requestName)
# make absolutely sure you're deleting the right one
oldReqId = request['RequestID']
if reqId:
# make absolutely sure you're deleting the right one
oldReqId = request['RequestID']
if oldReqId != reqId:
raise RequestCheckInError(
"Bad state deleting request %s/%s. Please contact a ReqMgr administrator"
% (oldReqId / reqId))
else:
RequestAdmin.deleteRequest(requestName)
# get information about the last exception
trace = traceback.format_exception(*sys.exc_info())
traceString = ''.join(trace)
logging.error("%s\n%s" % (msg, traceString))
raise RequestCheckInError(msg)
def _raiseCheckInError(request, ex, msg):
"""
Private function called only from this module.
Always from the except exception block - the traceback
is put in the local log file.
"""
requestName = request['RequestName']
msg +='\n' + str(ex)
msg += "\nUnable to check in new request %s" % requestName
reqId = GetRequest.requestID(requestName)
# make absolutely sure you're deleting the right one
oldReqId = request['RequestID']
if reqId:
# make absolutely sure you're deleting the right one
oldReqId = request['RequestID']
if oldReqId != reqId:
raise RequestCheckInError("Bad state deleting request %s/%s. Please contact a ReqMgr administrator" % (oldReqId/ reqId))
else:
RequestAdmin.deleteRequest(reqId)
# get information about the last exception
trace = traceback.format_exception(*sys.exc_info())
traceString = ''.join(trace)
logging.error("%s\n%s" % (msg, traceString))
raise RequestCheckInError(msg)
def putCampaign(self, campaign, request=None):
""" Adds a campaign if it doesn't already exist, and optionally
associates a request with it """
if request:
requestID = GetRequest.requestID(request)
if requestID:
return Campaign.associateCampaign(campaign, requestID)
else:
return False
else:
Campaign.addCampaign(campaign)
def putCampaign(self, campaign, request=None):
""" Adds a campaign if it doesn't already exist, and optionally
associates a request with it """
if request:
requestID = GetRequest.requestID(request)
if requestID:
result = Campaign.associateCampaign(campaign, requestID)
Utilities.associateCampaign(campaign = campaign,
requestName = request,
couchURL = self.couchUrl,
couchDBName = self.workloadDBName)
return result
else:
return False
else:
Campaign.addCampaign(campaign)
def putCampaign(self, campaign, request=None):
""" Adds a campaign if it doesn't already exist, and optionally
associates a request with it """
if request:
requestID = GetRequest.requestID(request)
if requestID:
result = Campaign.associateCampaign(campaign, requestID)
Utilities.associateCampaign(campaign=campaign,
requestName=request,
couchURL=self.couchUrl,
couchDBName=self.workloadDBName)
return result
else:
return False
else:
Campaign.addCampaign(campaign)
def raiseCheckInError(request, ex, msg):
requestName = request['RequestName']
msg +='\n' + str(ex)
msg += "\nUnable to check in new request %s" % requestName
reqId = GetRequest.requestID(requestName)
# make absolutely sure you're deleting the right one
oldReqId = request['RequestID']
if reqId:
# make absolutely sure you're deleting the right one
oldReqId = request['RequestID']
if oldReqId != reqId:
raise RuntimeError, "Bad state deleting request %s/%s. Please contact a ReqMgr administrator" % (oldReqId/ reqId)
else:
RequestAdmin.deleteRequest(reqId)
raise RuntimeError, msg
def testE_CheckStatusChanges(self):
"""
_CheckStatusChanges_
Check status changes for a single request. See whether
we can move the request through the proper chain. Figure
out what happens when we fail.
"""
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
# There should only be one request in the DB
result = GetRequest.requestID(requestName = requestName)
self.assertEqual(result, 1)
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['Group'], groupName)
self.assertEqual(result[0]['Requestor'], userName)
# Let's see what we can do in terms of setting status
self.changeStatusAndCheck(requestName = requestName,
statusName = 'new')
# Let's try an illegal status change, just for the hell of it
raises = False
try:
self.jsonSender.put('request/%s?status=negotiating' % requestName)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 403)
self.assertTrue('Failed to change status' in ex.result)
self.assertFalse(requestName in ex.result)
def testE_CheckStatusChanges(self):
"""
_CheckStatusChanges_
Check status changes for a single request. See whether
we can move the request through the proper chain. Figure
out what happens when we fail.
"""
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
# There should only be one request in the DB
result = GetRequest.requestID(requestName=requestName)
self.assertEqual(result, 1)
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['Group'], groupName)
self.assertEqual(result[0]['Requestor'], userName)
# Let's see what we can do in terms of setting status
self.changeStatusAndCheck(requestName=requestName, statusName='new')
# Let's try an illegal status change, just for the hell of it
raises = False
try:
self.jsonSender.put('request/%s?status=negotiating' % requestName)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 403)
self.assertTrue('Failed to change status' in ex.result)
self.assertFalse(requestName in ex.result)
def testE_CheckStatusChanges(self):
"""
_CheckStatusChanges_
Check status changes for a single request. See whether
we can move the request through the proper chain. Figure
out what happens when we fail.
"""
myThread = threading.currentThread()
userName = "******"
groupName = "Li"
teamName = "Tang"
CMSSWVersion = "CMSSW_3_5_8"
schema = self.setupSchema(userName=userName, groupName=groupName, teamName=teamName, CMSSWVersion=CMSSWVersion)
result = self.jsonSender.put("request/testRequest", schema)
self.assertEqual(result[1], 200)
requestName = result[0]["RequestName"]
# There should only be one request in the DB
result = GetRequest.requestID(requestName=requestName)
self.assertEqual(result, 1)
result = self.jsonSender.get("request/%s" % requestName)
self.assertEqual(result[0]["Group"], groupName)
self.assertEqual(result[0]["Requestor"], userName)
# Let's see what we can do in terms of setting status
self.changeStatusAndCheck(requestName=requestName, statusName="new")
# Let's try an illegal status change, just for the hell of it
raises = False
try:
self.jsonSender.put("request/%s?status=negotiating" % requestName)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 403)
self.assertTrue("Failed to change status" in ex.result)
self.assertFalse(requestName in ex.result)
try:
reqId = MakeRequest.createRequest(
request['Requestor'],
request['Group'],
requestName,
request['RequestType'],
request['RequestWorkflow'],
request.get('PrepID', None)
)
except Exception, ex:
msg = "Error creating new request:\n"
msg += str(ex)
raise RequestCheckInError( msg )
#FIXME LAST_INSERT_ID doesn't work on oracle
reqId = GetRequest.requestID(requestName)
request['RequestID'] = reqId
logging.info("Request %s created with request id %s" % (
requestName, request['RequestID'])
)
# //
# // add metadata about the request
#//
try:
if request['InputDatasetTypes'] != {}:
for ds, dsType in request['InputDatasetTypes'].items():
MakeRequest.associateInputDataset(
requestName, ds, dsType)
elif isinstance(request['InputDatasets'], list):
for ds in request['InputDatasets']:
def checkIn(request, requestType='None'):
"""
_CheckIn_
Check in of a request manager
Given a new request, check it in to the DB and add the
appropriate IDs.
"""
# //
# // First try and register the request in the DB
#//
requestName = request['RequestName']
# test if the software versions are registered first
versions = SoftwareManagement.listSoftware()
scramArch = request.get('ScramArch')
if requestType.lower() in ['resubmission']:
# Do nothing, as we have no valid software version yet
pass
else:
if not scramArch in versions.keys():
m = (
"Cannot find scramArch %s in ReqMgr (the one(s) available: %s)"
% (scramArch, versions))
raise RequestCheckInError(m)
for version in request.get('SoftwareVersions', []):
if not version in versions[scramArch]:
raise RequestCheckInError(
"Cannot find software version %s in ReqMgr for "
"scramArch %s. Supported versions: %s" %
(version, scramArch, versions[scramArch]))
try:
reqId = MakeRequest.createRequest(request['Requestor'],
request['Group'], requestName,
request['RequestType'],
request['RequestWorkflow'],
request.get('PrepID', None),
request.get('RequestPriority', None))
except Exception as ex:
msg = "Error creating new request:\n"
msg += str(ex)
raise RequestCheckInError(msg)
#FIXME LAST_INSERT_ID doesn't work on oracle
reqId = GetRequest.requestID(requestName)
request['RequestID'] = reqId
logging.info("Request %s created with request id %s" %
(requestName, request['RequestID']))
# //
# // add metadata about the request
#//
try:
if request['InputDatasetTypes'] != {}:
for ds, dsType in request['InputDatasetTypes'].items():
MakeRequest.associateInputDataset(requestName, ds, dsType)
elif isinstance(request['InputDatasets'], list):
for ds in request['InputDatasets']:
MakeRequest.associateInputDataset(requestName, ds)
else:
MakeRequest.associateInputDataset(requestName,
request['InputDatasets'])
except Exception as ex:
_raiseCheckInError(request, ex,
"Unable to Associate input datasets to request")
try:
for ds in request['OutputDatasets']:
# request['OutputDatasets'] may contain a list of lists (each sublist for a task)
# which is actually not understood why but seems to be correct (Steve)
# dirty
if isinstance(ds, list):
for dss in ds:
MakeRequest.associateOutputDataset(requestName, dss)
else:
MakeRequest.associateOutputDataset(requestName, ds)
except Exception as ex:
_raiseCheckInError(request, ex,
"Unable to Associate output datasets to request")
try:
for sw in request['SoftwareVersions']:
MakeRequest.associateSoftware(requestName, sw)
except Exception as ex:
_raiseCheckInError(request, ex,
"Unable to associate software for this request")
MakeRequest.updateRequestSize(requestName,
request.get("RequestNumEvents", 0),
request.get("RequestSizeFiles", 0),
request.get("SizePerEvent", 0))
campaign = request.get("Campaign", "")
if campaign != "" and campaign != None:
Campaign.associateCampaign(campaign, reqId)
logging.info("Request '%s' built with request id '%s"
'' % (requestName, request['RequestID']))
"Cannot find software version %s in ReqMgr for "
"scramArch %s. Supported versions: %s" %
(version, scramArch, versions[scramArch]))
try:
reqId = MakeRequest.createRequest(
request['Requestor'], request['Group'], requestName,
request['RequestType'], request['RequestWorkflow'],
request.get('PrepID', None),
request.get('ReqMgrRequestBasePriority', None))
except Exception, ex:
msg = "Error creating new request:\n"
msg += str(ex)
raise RequestCheckInError(msg)
#FIXME LAST_INSERT_ID doesn't work on oracle
reqId = GetRequest.requestID(requestName)
request['RequestID'] = reqId
logging.info("Request %s created with request id %s" %
(requestName, request['RequestID']))
# //
# // add metadata about the request
#//
try:
if request['InputDatasetTypes'] != {}:
for ds, dsType in request['InputDatasetTypes'].items():
MakeRequest.associateInputDataset(requestName, ds, dsType)
elif isinstance(request['InputDatasets'], list):
for ds in request['InputDatasets']:
MakeRequest.associateInputDataset(requestName, ds)
else:
def testE_CheckStatusChanges(self):
"""
_CheckStatusChanges_
Check status changes for a single request. See whether
we can move the request through the proper chain. Figure
out what happens when we fail.
"""
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
# There should only be one request in the DB
result = GetRequest.requestID(requestName = requestName)
self.assertEqual(result, 1)
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['Group'], groupName)
self.assertEqual(result[0]['Requestor'], userName)
# Let's see what we can do in terms of setting status
self.changeStatusAndCheck(requestName = requestName,
statusName = 'new')
# Let's try an illegal status change, just for the hell of it
raises = False
try:
self.jsonSender.put('request/%s?status=assigned' % requestName)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 403)
self.assertTrue('Failed to change status' in ex.result)
self.assertFalse(requestName in ex.result)
self.assertTrue(raises)
# Now, let's try a totally bogus status
raises = False
try:
self.jsonSender.put('request/%s?status=bogus' % requestName)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 403)
self.assertTrue('Failed to change status' in ex.result)
self.assertFalse(requestName in ex.result)
self.assertTrue(raises)
# We should still be in new
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['RequestStatus'], 'new')
# Let's go on in a full loop
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing-approved')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'tested')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'assignment-approved')
# This should fail, as you cannot assign a request without a team
raises = False
try:
self.changeStatusAndCheck(requestName = requestName,
statusName = 'assigned')
except HTTPException as ex:
raises = True
self.assertTrue('Cannot change status without a team' in ex.result)
self.assertTrue(raises)
self.jsonSender.put(urllib.quote('assignment/%s/%s' % (teamName, requestName)))
self.changeStatusAndCheck(requestName = requestName,
statusName = 'acquired')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'running-open')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'running-closed')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'completed')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'closed-out')
def testE_CheckStatusChanges(self):
"""
_CheckStatusChanges_
Check status changes for a single request. See whether
we can move the request through the proper chain. Figure
out what happens when we fail.
"""
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
# There should only be one request in the DB
result = GetRequest.requestID(requestName = requestName)
self.assertEqual(result, 1)
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['Group'], groupName)
self.assertEqual(result[0]['Requestor'], userName)
# Let's see what we can do in terms of setting status
self.changeStatusAndCheck(requestName = requestName,
statusName = 'new')
# Let's try an illegal status change, just for the hell of it
raises = False
try:
self.jsonSender.put('request/%s?status=negotiating' % requestName)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 403)
self.assertTrue('Failed to change status' in ex.result)
self.assertFalse(requestName in ex.result)
self.assertTrue(raises)
# Now, let's try a totally bogus status
raises = False
try:
self.jsonSender.put('request/%s?status=bogus' % requestName)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 403)
self.assertTrue('Failed to change status' in ex.result)
self.assertFalse(requestName in ex.result)
self.assertTrue(raises)
# We should still be in new
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['RequestStatus'], 'new')
# Let's go on in a full loop
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing-approved')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'tested')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'assignment-approved')
# This should fail, as you cannot assign a request without a team
raises = False
try:
self.changeStatusAndCheck(requestName = requestName,
statusName = 'assigned')
except HTTPException as ex:
raises = True
self.assertTrue('Cannot change status without a team' in ex.result)
self.assertTrue(raises)
self.jsonSender.put(urllib.quote('assignment/%s/%s' % (teamName, requestName)))
self.changeStatusAndCheck(requestName = requestName,
statusName = 'negotiating')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'acquired')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'running-open')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'running-closed')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'completed')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'closed-out')
def checkIn(request, requestType="None"):
"""
_CheckIn_
Check in of a request manager
Given a new request, check it in to the DB and add the
appropriate IDs.
"""
# //
# // First try and register the request in the DB
# //
requestName = request["RequestName"]
# test if the software versions are registered first
versions = SoftwareManagement.listSoftware()
scramArch = request.get("ScramArch")
if requestType.lower() in ["resubmission"]:
# Do nothing, as we have no valid software version yet
pass
else:
if not scramArch in versions.keys():
m = "Cannot find scramArch %s in ReqMgr (the one(s) available: %s)" % (scramArch, versions)
raise RequestCheckInError(m)
for version in request.get("SoftwareVersions", []):
if not version in versions[scramArch]:
raise RequestCheckInError(
"Cannot find software version %s in ReqMgr for "
"scramArch %s. Supported versions: %s" % (version, scramArch, versions[scramArch])
)
try:
reqId = MakeRequest.createRequest(
request["Requestor"],
request["Group"],
requestName,
request["RequestType"],
request["RequestWorkflow"],
request.get("PrepID", None),
request.get("RequestPriority", None),
)
except Exception as ex:
msg = "Error creating new request:\n"
msg += str(ex)
raise RequestCheckInError(msg)
# FIXME LAST_INSERT_ID doesn't work on oracle
reqId = GetRequest.requestID(requestName)
request["RequestID"] = reqId
logging.info("Request %s created with request id %s" % (requestName, request["RequestID"]))
# //
# // add metadata about the request
# //
try:
if request["InputDatasetTypes"] != {}:
for ds, dsType in request["InputDatasetTypes"].items():
MakeRequest.associateInputDataset(requestName, ds, dsType)
elif isinstance(request["InputDatasets"], list):
for ds in request["InputDatasets"]:
MakeRequest.associateInputDataset(requestName, ds)
else:
MakeRequest.associateInputDataset(requestName, request["InputDatasets"])
except Exception as ex:
_raiseCheckInError(request, ex, "Unable to Associate input datasets to request")
try:
for ds in request["OutputDatasets"]:
# request['OutputDatasets'] may contain a list of lists (each sublist for a task)
# which is actually not understood why but seems to be correct (Steve)
# dirty
if isinstance(ds, list):
for dss in ds:
MakeRequest.associateOutputDataset(requestName, dss)
else:
MakeRequest.associateOutputDataset(requestName, ds)
except Exception as ex:
_raiseCheckInError(request, ex, "Unable to Associate output datasets to request")
try:
for sw in request["SoftwareVersions"]:
MakeRequest.associateSoftware(requestName, sw)
except Exception as ex:
_raiseCheckInError(request, ex, "Unable to associate software for this request")
MakeRequest.updateRequestSize(
requestName,
request.get("RequestNumEvents", 0),
request.get("RequestSizeFiles", 0),
request.get("SizePerEvent", 0),
)
campaign = request.get("Campaign", "")
if campaign != "" and campaign != None:
Campaign.associateCampaign(campaign, reqId)
logging.info("Request '%s' built with request id '%s" "" % (requestName, request["RequestID"]))
