def testA_SoftwareManagement(self):
"""
_SoftwareManagement_
Test the SoftwareManagement code
"""
self.assertEqual(SoftwareManagement.listSoftware(), {})
softwareVersions = ReqMgrWebTools.allScramArchsAndVersions()
ReqMgrWebTools.updateScramArchsAndCMSSWVersions()
result = SoftwareManagement.listSoftware()
for scramArch in result.keys():
self.assertEqual(set(result[scramArch]),
set(softwareVersions[scramArch]))
# Now for each scramArch insert a blank set
# Because of the way that updateSoftware works, this interprets a blank list
# as telling it that no softwareVersions are available.
# It deletes every software version it is not handed, so it should give nothing out.
for scramArch in result.keys():
SoftwareManagement.updateSoftware(softwareNames=[],
scramArch=scramArch)
self.assertEqual(SoftwareManagement.listSoftware(), {})
# import has to be here, otherwise getting:
# AttributeError: 'Toolbox' object has no attribute 'secmodv2' from the Admin module
from WMCore.HTTPFrontEnd.RequestManager import Admin
setattr(self.config, 'database', self.testInit.coreConfig.CoreDatabase)
self.config.section_('templates')
self.config.section_('html')
admin = Admin.Admin(self.config)
ReqMgrWebTools.updateScramArchsAndCMSSWVersions()
self.assertTrue('slc5_amd64_gcc434' in admin.scramArchs())
def testH_RemoveSoftwareVersion(self):
"""
_RemoveSoftwareVersion_
Remove the software version after submitting the request. See what that does.
"""
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
# Delete software versions and make sure they're gone from the DB
SoftwareManagement.removeSoftware(softwareName = schema["CMSSWVersion"],
scramArch = schema["ScramArch"])
versions = myThread.dbi.processData("SELECT * FROM reqmgr_software")[0].fetchall()
self.assertEqual(versions, [])
assocs = myThread.dbi.processData("SELECT * FROM reqmgr_software_dependency")[0].fetchall()
self.assertEqual(assocs, [])
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
def index(self):
""" Main web page for creating requests """
# Get the scram Architecture from the keys and the
# CMSSW versions from the values
self.scramArchs = SoftwareAdmin.listSoftware().keys()
versionLists = SoftwareAdmin.listSoftware().values()
self.versions = []
for l in versionLists:
for v in l:
if not v in self.versions:
self.versions.append(v)
self.versions.sort()
# see if this was configured with a hardcoded user. If not, take from the request header
requestor = self.requestor
if not requestor:
requestor = cherrypy.request.user["login"]
if not requestor:
return "No username found in your certificate"
if not requestor in Registration.listUsers():
return "User %s is not registered. Contact a ReqMgr administrator." % requestor
groups = GroupInfo.groupsForUser(requestor).keys()
if groups == []:
return "User " + requestor + " is not in any groups. Contact a ReqMgr administrator."
campaigns = Campaign.listCampaigns()
return self.templatepage("WebRequestSchema", yuiroot=self.yuiroot,
requestor=requestor,
groups=groups,
versions=self.versions,
archs = self.scramArchs,
alldocs = Utilities.unidecode(self.allDocs()),
allcampaigns = campaigns,
defaultVersion=self.cmsswVersion,
defaultSkimConfig=self.defaultSkimConfig)
def testA_SoftwareManagement(self):
"""
_SoftwareManagement_
Test the SoftwareManagement code
"""
self.assertEqual(SoftwareManagement.listSoftware(), {})
softwareVersions = ReqMgrWebTools.allScramArchsAndVersions()
ReqMgrWebTools.updateScramArchsAndCMSSWVersions()
result = SoftwareManagement.listSoftware()
for scramArch in result.keys():
self.assertEqual(set(result[scramArch]), set(softwareVersions[scramArch]))
# Now for each scramArch insert a blank set
# Because of the way that updateSoftware works, this interprets a blank list
# as telling it that no softwareVersions are available.
# It deletes every software version it is not handed, so it should give nothing out.
for scramArch in result.keys():
SoftwareManagement.updateSoftware(softwareNames = [], scramArch = scramArch)
self.assertEqual(SoftwareManagement.listSoftware(), {})
from WMCore.HTTPFrontEnd.RequestManager import Admin
setattr(self.config, 'database', self.testInit.coreConfig.CoreDatabase)
self.config.section_('templates')
self.config.section_('html')
admin = Admin.Admin(self.config)
ReqMgrWebTools.updateScramArchsAndCMSSWVersions()
self.assertTrue('slc5_amd64_gcc434' in admin.scramArchs())
return
def index(self):
""" Main web page for creating requests """
# Get the scram Architecture from the keys and the
# CMSSW versions from the values
self.scramArchs = SoftwareAdmin.listSoftware().keys()
versionLists = SoftwareAdmin.listSoftware().values()
self.versions = []
for l in versionLists:
for v in l:
if not v in self.versions:
self.versions.append(v)
self.versions.sort()
# see if this was configured with a hardcoded user. If not, take from the request header
requestor = self.requestor
if not requestor:
requestor = cherrypy.request.user["login"]
if not requestor:
return "No username found in your certificate"
if not requestor in Registration.listUsers():
return "User %s is not registered. Contact a ReqMgr administrator." % requestor
groups = GroupInfo.groupsForUser(requestor).keys()
if groups == []:
return "User " + requestor + " is not in any groups. Contact a ReqMgr administrator."
campaigns = Campaign.listCampaigns()
return self.templatepage("WebRequestSchema",
yuiroot=self.yuiroot,
requestor=requestor,
groups=groups,
versions=self.versions,
archs=self.scramArchs,
alldocs=Utilities.unidecode(self.allDocs()),
allcampaigns=campaigns,
defaultVersion=self.cmsswVersion,
defaultArch=self.defaultArch,
defaultSkimConfig=self.defaultSkimConfig)
def testH_RemoveSoftwareVersion(self):
"""
_RemoveSoftwareVersion_
Remove the software version after submitting the request. See what that does.
"""
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
# Delete software versions and make sure they're gone from the DB
SoftwareManagement.removeSoftware(softwareName=schema["CMSSWVersion"],
scramArch=schema["ScramArch"])
versions = myThread.dbi.processData(
"SELECT * FROM reqmgr_software")[0].fetchall()
self.assertEqual(versions, [])
assocs = myThread.dbi.processData(
"SELECT * FROM reqmgr_software_dependency")[0].fetchall()
self.assertEqual(assocs, [])
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
def testH_RemoveSoftwareVersion(self):
"""
_RemoveSoftwareVersion_
Remove the software version after submitting the request. See what that does.
"""
myThread = threading.currentThread()
userName = "******"
groupName = "Li"
teamName = "Tang"
CMSSWVersion = "CMSSW_3_5_8"
scramArch = "slc5_ia32_gcc434"
schema = self.setupSchema(
userName=userName, groupName=groupName, teamName=teamName, CMSSWVersion=CMSSWVersion, scramArch=scramArch
)
result = self.jsonSender.put("request/testRequest", schema)
self.assertEqual(result[1], 200)
requestName = result[0]["RequestName"]
req = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(req["SoftwareVersions"], [CMSSWVersion])
# Delete software versions and make sure they're gone from the DB
SoftwareManagement.removeSoftware(softwareName=CMSSWVersion, scramArch=scramArch)
versions = myThread.dbi.processData("SELECT * FROM reqmgr_software")[0].fetchall()
self.assertEqual(versions, [])
assocs = myThread.dbi.processData("SELECT * FROM reqmgr_software_dependency")[0].fetchall()
self.assertEqual(assocs, [])
req = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(req["SoftwareVersions"], [CMSSWVersion])
return
def handleAllVersions(self):
""" Registers all versions in the TC """
currentVersions = SoftwareAdmin.listSoftware().keys()
allVersions = Utilities.allSoftwareVersions()
result = ""
for version in allVersions:
if not version in currentVersions:
WMCore.Lexicon.cmsswversion(version)
SoftwareAdmin.addSoftware(version)
result += "Added version %s<br/>" % version
if result == "":
result = "Version list is up to date"
return result
def checkIn(request):
"""
_CheckIn_
Check in of a request manager
Given a new request, check it in to the DB and add the
appropriate IDs.
"""
# //
# // First try and register the request in the DB
#//
requestName = request['RequestName']
# test if the software versions are registered first
versions = SoftwareManagement.listSoftware()
for version in request.get('SoftwareVersions', []):
if not version in versions:
raise RuntimeError, "Cannot find software version %s in ReqMgr" % version
try:
reqId = MakeRequest.createRequest(
request['Requestor'],
request['Group'],
requestName,
request['RequestType'],
request['RequestWorkflow'],
request.get('PrepID', None)
)
except Exception, ex:
msg = "Error creating new request:\n"
msg += str(ex)
raise RuntimeError, msg
def versions(self):
""" Lists all versions """
versions = SoftwareAdmin.listSoftware().keys()
versions.sort()
for version in versions:
WMCore.Lexicon.cmsswversion(version)
return self.templatepage("Versions", versions=versions)
def scramArchs(self):
"""
_scramArchs_
List all scramArchs in the DB
Prelim for putting this in the template pages
"""
return SoftwareAdmin.listSoftware().keys()
def updateScramArchsAndCMSSWVersions():
"""
_updateScramArchsAndCMSSWVersions_
Update both the scramArchs and their associated software versions to
the current tag collector standard.
"""
allArchsAndVersions = allScramArchsAndVersions()
if allArchsAndVersions == {}:
# The tag collector is probably down
# NO valid CMSSW Versions is not a valid use case!
# Do nothing.
logging.error("Handed blank list of scramArchs/versions. Ignoring for this cycle.")
return
for scramArch in allArchsAndVersions.keys():
SoftwareAdmin.updateSoftware(scramArch=scramArch, softwareNames=allArchsAndVersions[scramArch])
def updateScramArchsAndCMSSWVersions():
"""
_updateScramArchsAndCMSSWVersions_
Update both the scramArchs and their associated software versions to
the current tag collector standard.
"""
allArchsAndVersions = allScramArchsAndVersions()
if allArchsAndVersions == {}:
# The tag collector is probably down
# NO valid CMSSW Versions is not a valid use case!
# Do nothing.
logging.error("Handed blank list of scramArchs/versions. Ignoring for this cycle.")
return
for scramArch in allArchsAndVersions.keys():
SoftwareAdmin.updateSoftware(scramArch = scramArch,
softwareNames = allArchsAndVersions[scramArch])
def getVersion(self):
""" Returns a list of all CMSSW versions registered with ReqMgr """
archList = SoftwareAdmin.listSoftware()
result = []
for arch in archList.keys():
for version in archList[arch]:
if not version in result:
result.append(version)
return result
def getVersion(self):
""" Returns a list of all CMSSW versions registered with ReqMgr """
archList = SoftwareAdmin.listSoftware()
result = []
for arch in archList.keys():
for version in archList[arch]:
if not version in result:
result.append(version)
return result
def versions(self):
""" Lists all versions """
archList = SoftwareAdmin.listSoftware()
versions = []
for versionList in archList.values():
for version in versionList:
if not version in versions:
versions.append(version)
versions.sort()
for version in versions:
WMCore.Lexicon.cmsswversion(version)
return self.templatepage("Versions", versions=versions)
def checkIn(request, requestType = 'None'):
"""
_CheckIn_
Check in of a request manager
Given a new request, check it in to the DB and add the
appropriate IDs.
"""
# //
# // First try and register the request in the DB
#//
requestName = request['RequestName']
# test if the software versions are registered first
versions = SoftwareManagement.listSoftware()
scramArch = request.get('ScramArch')
if requestType.lower() in ['resubmission']:
# Do nothing, as we have no valid software version yet
pass
else:
if not scramArch in versions.keys():
m = ("Cannot find scramArch %s in ReqMgr (the one(s) available: %s)" %
(scramArch, versions))
raise RequestCheckInError(m)
for version in request.get('SoftwareVersions', []):
if not version in versions[scramArch]:
raise RequestCheckInError("Cannot find software version %s in ReqMgr for "
"scramArch %s. Supported versions: %s" %
(version, scramArch, versions[scramArch]))
try:
reqId = MakeRequest.createRequest(
request['Requestor'],
request['Group'],
requestName,
request['RequestType'],
request['RequestWorkflow'],
request.get('PrepID', None),
request.get('RequestPriority', None)
)
except Exception, ex:
msg = "Error creating new request:\n"
msg += str(ex)
raise RequestCheckInError( msg )
def checkIn(request, requestType = 'None'):
"""
_CheckIn_
Check in of a request manager
Given a new request, check it in to the DB and add the
appropriate IDs.
"""
# //
# // First try and register the request in the DB
#//
requestName = request['RequestName']
# test if the software versions are registered first
versions = SoftwareManagement.listSoftware()
scramArch = request.get('ScramArch')
if requestType.lower() in ['resubmission']:
# Do nothing, as we have no valid software version yet
pass
else:
if not scramArch in versions.keys():
m = ("Cannot find scramArch %s in ReqMgr (the one(s) available: %s)" %
(scramArch, versions))
raise RequestCheckInError(m)
for version in request.get('SoftwareVersions', []):
if not version in versions[scramArch]:
raise RequestCheckInError("Cannot find software version %s in ReqMgr for "
"scramArch %s. Supported versions: %s" %
(version, scramArch, versions[scramArch]))
try:
reqId = MakeRequest.createRequest(
request['Requestor'],
request['Group'],
requestName,
request['RequestType'],
request['RequestWorkflow'],
request.get('PrepID', None),
request.get('ReqMgrRequestBasePriority', None)
)
except Exception, ex:
msg = "Error creating new request:\n"
msg += str(ex)
raise RequestCheckInError( msg )
def checkIn(request, requestType="None", wmstatSvc=None):
"""
_CheckIn_
Check in of a request manager
Given a new request, check it in to the DB and add the
appropriate IDs.
"""
# //
# // First try and register the request in the DB
# //
requestName = request["RequestName"]
# test if the software versions are registered first
versions = SoftwareManagement.listSoftware()
scramArch = request.get("ScramArch")
if requestType.lower() in ["resubmission"]:
# Do nothing, as we have no valid software version yet
pass
else:
if not scramArch in versions.keys():
m = "Cannot find scramArch %s in ReqMgr (the one(s) available: %s)" % (scramArch, versions)
raise RequestCheckInError(m)
for version in request.get("SoftwareVersions", []):
if not version in versions[scramArch]:
raise RequestCheckInError(
"Cannot find software version %s in ReqMgr for scramArch %s" % (version, scramArch)
)
try:
reqId = MakeRequest.createRequest(
request["Requestor"],
request["Group"],
requestName,
request["RequestType"],
request["RequestWorkflow"],
request.get("PrepID", None),
)
except Exception, ex:
msg = "Error creating new request:\n"
msg += str(ex)
raise RequestCheckInError(msg)
def checkIn(request, requestType="None"):
"""
_CheckIn_
Check in of a request manager
Given a new request, check it in to the DB and add the
appropriate IDs.
"""
# //
# // First try and register the request in the DB
# //
requestName = request["RequestName"]
# test if the software versions are registered first
versions = SoftwareManagement.listSoftware()
scramArch = request.get("ScramArch")
if requestType.lower() in ["resubmission"]:
# Do nothing, as we have no valid software version yet
pass
else:
if not scramArch in versions.keys():
m = "Cannot find scramArch %s in ReqMgr (the one(s) available: %s)" % (scramArch, versions)
raise RequestCheckInError(m)
for version in request.get("SoftwareVersions", []):
if not version in versions[scramArch]:
raise RequestCheckInError(
"Cannot find software version %s in ReqMgr for "
"scramArch %s. Supported versions: %s" % (version, scramArch, versions[scramArch])
)
try:
reqId = MakeRequest.createRequest(
request["Requestor"],
request["Group"],
requestName,
request["RequestType"],
request["RequestWorkflow"],
request.get("PrepID", None),
request.get("RequestPriority", None),
)
except Exception as ex:
msg = "Error creating new request:\n"
msg += str(ex)
raise RequestCheckInError(msg)
# FIXME LAST_INSERT_ID doesn't work on oracle
reqId = GetRequest.requestID(requestName)
request["RequestID"] = reqId
logging.info("Request %s created with request id %s" % (requestName, request["RequestID"]))
# //
# // add metadata about the request
# //
try:
if request["InputDatasetTypes"] != {}:
for ds, dsType in request["InputDatasetTypes"].items():
MakeRequest.associateInputDataset(requestName, ds, dsType)
elif isinstance(request["InputDatasets"], list):
for ds in request["InputDatasets"]:
MakeRequest.associateInputDataset(requestName, ds)
else:
MakeRequest.associateInputDataset(requestName, request["InputDatasets"])
except Exception as ex:
_raiseCheckInError(request, ex, "Unable to Associate input datasets to request")
try:
for ds in request["OutputDatasets"]:
# request['OutputDatasets'] may contain a list of lists (each sublist for a task)
# which is actually not understood why but seems to be correct (Steve)
# dirty
if isinstance(ds, list):
for dss in ds:
MakeRequest.associateOutputDataset(requestName, dss)
else:
MakeRequest.associateOutputDataset(requestName, ds)
except Exception as ex:
_raiseCheckInError(request, ex, "Unable to Associate output datasets to request")
try:
for sw in request["SoftwareVersions"]:
MakeRequest.associateSoftware(requestName, sw)
except Exception as ex:
_raiseCheckInError(request, ex, "Unable to associate software for this request")
MakeRequest.updateRequestSize(
requestName,
request.get("RequestNumEvents", 0),
request.get("RequestSizeFiles", 0),
request.get("SizePerEvent", 0),
)
campaign = request.get("Campaign", "")
if campaign != "" and campaign != None:
Campaign.associateCampaign(campaign, reqId)
logging.info("Request '%s' built with request id '%s" "" % (requestName, request["RequestID"]))
def getVersion(self):
""" Returns a list of all CMSSW versions registered with ReqMgr """
return SoftwareAdmin.listSoftware().keys()
def putVersion(self, version):
""" Registers a new CMSSW version with ReqMgr """
return SoftwareAdmin.addSoftware(version)
def handleAddVersion(self, version):
""" Registers a version """
WMCore.Lexicon.cmsswversion(version)
SoftwareAdmin.updateSoftware(version)
return "Added version %s" % version
def checkIn(request, requestType='None'):
"""
_CheckIn_
Check in of a request manager
Given a new request, check it in to the DB and add the
appropriate IDs.
"""
# //
# // First try and register the request in the DB
#//
requestName = request['RequestName']
# test if the software versions are registered first
versions = SoftwareManagement.listSoftware()
scramArch = request.get('ScramArch')
if requestType.lower() in ['resubmission']:
# Do nothing, as we have no valid software version yet
pass
else:
if not scramArch in versions.keys():
m = (
"Cannot find scramArch %s in ReqMgr (the one(s) available: %s)"
% (scramArch, versions))
raise RequestCheckInError(m)
for version in request.get('SoftwareVersions', []):
if not version in versions[scramArch]:
raise RequestCheckInError(
"Cannot find software version %s in ReqMgr for "
"scramArch %s. Supported versions: %s" %
(version, scramArch, versions[scramArch]))
try:
reqId = MakeRequest.createRequest(request['Requestor'],
request['Group'], requestName,
request['RequestType'],
request['RequestWorkflow'],
request.get('PrepID', None),
request.get('RequestPriority', None))
except Exception as ex:
msg = "Error creating new request:\n"
msg += str(ex)
raise RequestCheckInError(msg)
#FIXME LAST_INSERT_ID doesn't work on oracle
reqId = GetRequest.requestID(requestName)
request['RequestID'] = reqId
logging.info("Request %s created with request id %s" %
(requestName, request['RequestID']))
# //
# // add metadata about the request
#//
try:
if request['InputDatasetTypes'] != {}:
for ds, dsType in request['InputDatasetTypes'].items():
MakeRequest.associateInputDataset(requestName, ds, dsType)
elif isinstance(request['InputDatasets'], list):
for ds in request['InputDatasets']:
MakeRequest.associateInputDataset(requestName, ds)
else:
MakeRequest.associateInputDataset(requestName,
request['InputDatasets'])
except Exception as ex:
_raiseCheckInError(request, ex,
"Unable to Associate input datasets to request")
try:
for ds in request['OutputDatasets']:
# request['OutputDatasets'] may contain a list of lists (each sublist for a task)
# which is actually not understood why but seems to be correct (Steve)
# dirty
if isinstance(ds, list):
for dss in ds:
MakeRequest.associateOutputDataset(requestName, dss)
else:
MakeRequest.associateOutputDataset(requestName, ds)
except Exception as ex:
_raiseCheckInError(request, ex,
"Unable to Associate output datasets to request")
try:
for sw in request['SoftwareVersions']:
MakeRequest.associateSoftware(requestName, sw)
except Exception as ex:
_raiseCheckInError(request, ex,
"Unable to associate software for this request")
MakeRequest.updateRequestSize(requestName,
request.get("RequestNumEvents", 0),
request.get("RequestSizeFiles", 0),
request.get("SizePerEvent", 0))
campaign = request.get("Campaign", "")
if campaign != "" and campaign != None:
Campaign.associateCampaign(campaign, reqId)
logging.info("Request '%s' built with request id '%s"
'' % (requestName, request['RequestID']))
def deleteVersion(self, version, scramArch):
""" Un-register this software version with ReqMgr """
SoftwareAdmin.removeSoftware(version, scramArch)
def putVersion(self, version, scramArch = None):
""" Registers a new CMSSW version with ReqMgr """
return SoftwareAdmin.addSoftware(version, scramArch = scramArch)
def deleteVersion(self, version, scramArch):
""" Un-register this software version with ReqMgr """
SoftwareAdmin.removeSoftware(version, scramArch)
def putVersion(self, version, scramArch=None):
""" Registers a new CMSSW version with ReqMgr """
return SoftwareAdmin.addSoftware(version, scramArch=scramArch)