class ReqMgrConfigData(RESTEntity):
def __init__(self, app, api, config, mount):
# CouchDB auxiliary database name
RESTEntity.__init__(self, app, api, config, mount)
self.reqmgr_aux_db = api.db_handler.get_db(config.couch_reqmgr_aux_db)
def _validate_args(self, param, safe):
# TODO: need proper validation but for now pass everything
args_length = len(param.args)
if args_length == 1:
safe.kwargs["doc_name"] = param.args[0]
param.args.pop()
else:
raise APINotSpecified()
return
def _validate_put_args(self, param, safe):
args_length = len(param.args)
if args_length == 1:
safe.kwargs["doc_name"] = param.args[0]
param.args.pop()
data = cherrypy.request.body.read()
if data:
config_args = json.loads(data)
# TODO need to validate the args depending on the config
safe.kwargs["config_dict"] = config_args
def validate(self, apiobj, method, api, param, safe):
"""
Validate request input data.
Has to be implemented, otherwise the service fails to start.
If it's not implemented correctly (e.g. just pass), the arguments
are not passed in the method at all.
"""
if method == "GET":
self._validate_args(param, safe)
elif method == "PUT":
self._validate_put_args(param, safe)
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
def get(self, doc_name):
"""
"""
config = ReqMgrConfigDataCache.getConfig(doc_name)
return rows([config])
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
def put(self, doc_name, config_dict=None):
"""
"""
if doc_name == "DEFAULT":
return ReqMgrConfigDataCache.putDefaultConfig()
return ReqMgrConfigDataCache.replaceConfig(doc_name, config_dict)
class ActiveRequestJobInfo(RESTEntity):
"""
This class need to move under WMStats server when wmstats server created
"""
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
wmstats_url = "%s/%s" % (self.config.couch_host,
self.config.couch_wmstats_db)
reqdb_url = "%s/%s" % (self.config.couch_host,
self.config.couch_reqmgr_db)
self.wmstats = WMStatsReader(wmstats_url,
reqdb_url,
reqdbCouchApp="ReqMgr")
def validate(self, apiobj, method, api, param, safe):
args_length = len(param.args)
return
@restcall(formats=[('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self):
results = DataCache.getlatestJobData()
if results == None or DataCache.islatestJobDataExpired():
results = self.wmstats.getActiveData(jobInfoFlag=True)
return rows([results])
class JobDetailInfo(RESTEntity):
"""
This class need to move under WMStats server when wmstats server created
"""
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
wmstats_url = "%s/%s" % (self.config.couch_host, self.config.couch_wmstats_db)
reqdb_url = "%s/%s" % (self.config.couch_host, self.config.couch_reqmgr_db)
self.wmstats = WMStatsReader(wmstats_url, reqdbURL=reqdb_url, reqdbCouchApp="ReqMgr")
def validate(self, apiobj, method, api, param, safe):
args_length = len(param.args)
if args_length == 1:
safe.args.append(param.args[0])
param.args.pop()
prop = 'sample_size'
safe.kwargs[prop] = int(param.kwargs.get(prop, 1))
if prop in param.kwargs:
del param.kwargs[prop]
return
@restcall(formats = [('text/plain', PrettyJSONFormat()), ('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self, request_name, sample_size):
result = self.wmstats.getTaskJobSummaryByRequest(request_name, sample_size)
return rows([result])
class RequestStatus(RESTEntity):
def __init__(self, app, api, config, mount):
RESTEntity.__init__(self, app, api, config, mount)
def validate(self, apiobj, method, api, param, safe):
validate_str("transition", param, safe, rx.RX_BOOL_FLAG, optional=True)
args_length = len(param.args)
if args_length == 1:
safe.kwargs["transiton"] = param.args[0]
param.args.pop()
return
@restcall(formats = [('application/json', JSONFormat())])
def get(self, transition):
"""
Return list of allowed request status.
If transition, return exhaustive list with all request status
and their defined transitions.
"""
if transition == "true":
return rows([REQUEST_STATE_TRANSITION])
else:
return rows(REQUEST_STATE_LIST)
class RequestSpec(RESTEntity):
def validate(self, apiobj, method, api, param, safe):
"""
Validate request input data.
Has to be implemented, otherwise the service fails to start.
If it's not implemented correctly (e.g. just pass), the arguments
are not passed in the method at all.
"""
validate_str("name", param, safe, rx.RX_REQUEST_NAME, optional=False)
@restcall(formats = [('text/plain', PrettyJSONFormat()), ('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self, name):
"""
Spec templete API call.
:arg str name: name to appear in the result message.
:returns: row with response, here 1 item list with message.
"""
result = get_request_template_from_type(name)
return [result]
class HelloWorld(RESTEntity):
def validate(self, apiobj, method, api, param, safe):
"""
Validate request input data.
Has to be implemented, otherwise the service fails to start.
If it's not implemented correctly (e.g. just pass), the arguments
are not passed in the method at all.
"""
validate_str("name", param, safe, rx.RX_REQUEST_NAME, optional=True)
@restcall(formats=[('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self, name):
"""
Hello world API call.
:arg str name: name to appear in the result message.
:returns: row with response, here 1 item list with message.
"""
msg = "Hello "
msg += name or "world"
#return rows(["Hello: world"]) returns the same as above
return msg
def __init__(self, app, config, mount):
"""
:arg app: reference to application object; passed to all entities.
:arg config: reference to configuration; passed to all entities.
:arg str mount: API URL mount point; passed to all entities."""
RESTApi.__init__(self, app, config, mount)
cherrypy.log("WMStats entire configuration:\n%s" %
Configuration.getInstance())
cherrypy.log("WMStats REST hub configuration subset:\n%s" % config)
# only allows json format for return value
self.formats = [('application/json', JSONFormat())]
self._add({
"info":
ServerInfo(app, self, config, mount),
"teams":
TeamInfo(app, self, config, mount),
"request":
RequestInfo(app, self, config, mount),
"isfinished":
FinishedStatusInfo(app, self, config, mount),
"jobdetail":
JobDetailInfo(app, self, config, mount),
"requestcache":
ActiveRequestJobInfo(app, self, config, mount)
})
class RequestInfo(RESTEntity):
"""
This class need to move under WMStats server when wmstats server created
"""
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
wmstats_url = "%s/%s" % (self.config.couch_host,
self.config.couch_wmstats_db)
reqdb_url = "%s/%s" % (self.config.couch_host,
self.config.couch_reqmgr_db)
self.wmstats = WMStatsReader(wmstats_url,
reqdbURL=reqdb_url,
reqdbCouchApp="ReqMgr")
def validate(self, apiobj, method, api, param, safe):
args_length = len(param.args)
if args_length == 1:
safe.args.append(param.args[0])
param.args.pop()
return
@restcall(formats=[('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self, request_name):
result = self.wmstats.getRequestSummaryWithJobInfo(request_name)
return rows([result])
def __init__(self, app, config, mount):
DatabaseRESTApi.__init__(self, app, config, mount)
self.formats = [ ('application/json', JSONFormat()) ]
status, serverdn = getstatusoutput('openssl x509 -noout -subject -in %s | cut -f2- -d\ ' % config.serverhostcert)
if status is not 0:
raise ExecutionError("Internal issue when retrieving crabserver service DN.")
extconfig = Utils.ConfigCache(centralconfig=Utils.getCentralConfig(extconfigurl=config.extconfigurl, mode=config.mode),
cachetime=mktime(gmtime()))
#Global initialization of Data objects. Parameters coming from the config should go here
DataUserWorkflow.globalinit(config)
DataWorkflow.globalinit(dbapi=self, phedexargs={'endpoint': config.phedexurl},\
credpath=config.credpath, centralcfg=extconfig, config=config)
DataFileMetadata.globalinit(dbapi=self, config=config)
RESTTask.globalinit(centralcfg=extconfig)
Utils.globalinit(config.serverhostkey, config.serverhostcert, serverdn, config.credpath)
## TODO need a check to verify the format depending on the resource
## the RESTFileMetadata has the specifc requirement of getting xml reports
self._add( {'workflow': RESTUserWorkflow(app, self, config, mount, extconfig),
'info': RESTServerInfo(app, self, config, mount, serverdn, extconfig),
'filemetadata': RESTFileMetadata(app, self, config, mount),
'workflowdb': RESTWorkerWorkflow(app, self, config, mount),
'task': RESTTask(app, self, config, mount),
'filetransfers': RESTFileTransfers(app, self, config, mount),
'fileusertransfers': RESTFileUserTransfers(app, self, config, mount),
})
self._initLogger( getattr(config, 'loggingFile', None), getattr(config, 'loggingLevel', None) )
class FinishedStatusInfo(RESTEntity):
"""
This class need to move under WMStats server when wmstats server created
"""
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
wmstats_url = "%s/%s" % (self.config.couch_host,
self.config.couch_wmstats_db)
reqdb_url = "%s/%s" % (self.config.couch_host,
self.config.couch_reqmgr_db)
self.wmstats = WMStatsReader(wmstats_url,
reqdbURL=reqdb_url,
reqdbCouchApp="ReqMgr")
def validate(self, apiobj, method, api, param, safe):
args_length = len(param.args)
if args_length == 1:
safe.args.append(param.args[0])
param.args.pop()
return
@restcall(formats=[('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self, request_name):
try:
result = self.wmstats.isWorkflowCompletedWithLogCollectAndCleanUp(
request_name)
except KeyError:
raise cherrypy.HTTPError(404,
"Cannot find request: %s" % request_name)
return rows([result])
class FilteredActiveRequestJobInfo(RESTEntity):
"""
get all the active requests with job information attatched
"""
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
def validate(self, apiobj, method, api, param, safe):
for prop in param.kwargs:
safe.kwargs[prop] = param.kwargs[prop]
for prop in safe.kwargs:
del param.kwargs[prop]
return
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self, mask=None, **input_condition):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows(DataCache.filterDataByRequest(input_condition, mask))
class RequestType(RESTEntity):
def __init__(self, app, api, config, mount):
RESTEntity.__init__(self, app, api, config, mount)
def validate(self, apiobj, method, api, param, safe):
pass
@restcall(formats=[('application/json', JSONFormat())])
def get(self):
return rows(REQUEST_TYPES)
def __init__(self, app, config, mount):
DatabaseRESTApi.__init__(self, app, config, mount)
self.formats = [('application/json', JSONFormat())]
status, serverdn = getstatusoutput(
'openssl x509 -noout -subject -in %s | cut -f2- -d\ ' %
config.serverhostcert)
if status is not 0:
raise ExecutionError(
"Internal issue when retrieving crabserver service DN.")
hbuf = StringIO.StringIO()
bbuf = StringIO.StringIO()
curl = pycurl.Curl()
curl.setopt(pycurl.URL, config.extconfigurl)
curl.setopt(pycurl.WRITEFUNCTION, bbuf.write)
curl.setopt(pycurl.HEADERFUNCTION, hbuf.write)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.perform()
curl.close()
header = ResponseHeader(hbuf.getvalue())
if header.status < 200 or header.status >= 300:
cherrypy.log("Problem %d reading from %s." %
(config.extconfigurl, header.status))
raise ExecutionError(
"Internal issue when retrieving external confifuration")
extconfig = json.decode(bbuf.getvalue())
#Global initialization of Data objects. Parameters coming from the config should go here
DataUserWorkflow.globalinit(config.workflowManager)
DataWorkflow.globalinit(dbapi=self, phedexargs={'endpoint': config.phedexurl}, dbsurl=config.dbsurl,\
credpath=config.credpath, transformation=config.transformation)
DataFileMetadata.globalinit(dbapi=self)
Utils.globalinit(config.serverhostkey, config.serverhostcert, serverdn,
config.credpath)
## TODO need a check to verify the format depending on the resource
## the RESTFileMetadata has the specifc requirement of getting xml reports
self._add({
'workflow':
RESTUserWorkflow(app, self, config, mount),
'campaign':
RESTCampaign(app, self, config, mount),
'info':
RESTServerInfo(app, self, config, mount, serverdn,
extconfig.get('delegate-dn', [])),
'filemetadata':
RESTFileMetadata(app, self, config, mount),
})
self._initLogger(getattr(config, 'loggingFile', None),
getattr(config, 'loggingLevel', None))
class Info(RESTEntity):
"""
general information about reqmgr2, i.e. version, etc
"""
def __init__(self, app, api, config, mount):
RESTEntity.__init__(self, app, api, config, mount)
self.reqmgr_db = api.db_handler.get_db(config.couch_reqmgr_db)
self.config = config
def validate(self, apiobj, method, api, param, safe):
pass
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self):
# authorization / access control:
# cherrypy (modified CMS web version here) can read this information
# from HTTP headers. CMS web frontend puts this information into
# headers as read from SiteDB (or on private VM from a fake
# SiteDB file)
# print "cherrypy.request", cherrypy.request
# print "DN: %s" % cherrypy.request.user['dn']
# print "Requestor/login: %s" % cherrypy.request.user['login']
# print "cherrypy.request: %s" % cherrypy.request
# print "cherrypy.request.user: %s" % cherrypy.request.user
# from WMCore.REST.Auth import authz_match
# authz_match(role=["Global Admin"], group=["global"])
# check SiteDB/DataWhoAmI.py
# implement as authentication decorator over modification calls
# check config.py main.authz_defaults and SiteDB
# (only Admin: ReqMgr to be able to modify stuff)
wmcore_reqmgr_version = WMCore.__version__
reqmgr_db_info = self.reqmgr_db.info()
reqmgr_db_info["reqmgr_couch_url"] = self.config.couch_host
# retrieve the last injected request in the system
# curl ... /reqmgr_workload_cache/_design/ReqMgr/_view/bydate?descending=true&limit=1
options = {"descending": True, "limit": 1}
reqmgr_last_injected_request = self.reqmgr_db.loadView("ReqMgr",
"bydate",
options=options)
result = {
"wmcore_reqmgr_version": wmcore_reqmgr_version,
"reqmgr_db_info": reqmgr_db_info,
"reqmgr_last_injected_request": reqmgr_last_injected_request
}
# NOTE:
# "return result" only would return dict with keys without (!!) values set
return rows([result])
class ServerInfo(RESTEntity):
def validate(self, apiobj, method, api, param, safe):
"""
"""
pass
@restcall(formats=[('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self):
"""
"""
wmstats_version = WMCore.__version__
return "version: %s" % wmstats_version
def __init__(self, app, config, mount):
RESTApi.__init__(self, app, config, mount)
self.formats = [ ('application/json', JSONFormat()) ]
if not os.path.exists(config.cachedir) or not os.path.isdir(config.cachedir):
raise Exception("Failing to start because of wrong cache directory '%s'" % config.cachedir)
if hasattr(config, 'powerusers'):
UserFileCache.RESTExtensions.POWER_USERS_LIST = config.powerusers
if hasattr(config, 'quota_user_limit'):
UserFileCache.RESTExtensions.QUOTA_USER_LIMIT = config.quota_user_limit * 1024 * 1024
self._add( {'logfile': RESTLogFile(app, self, config, mount),
'file': RESTFile(app, self, config, mount),
'info': RESTInfo(app, self, config, mount)} )
class AuxBaseAPI(RESTEntity):
"""
Base class for Aux db RESTEntry which contains get, post method
"""
def __init__(self, app, api, config, mount):
RESTEntity.__init__(self, app, api, config, mount)
# CouchDB auxiliary database name
self.reqmgr_aux_db = api.db_handler.get_db(config.couch_reqmgr_aux_db)
self.setName()
def setName(self):
"Sets the document name"
raise NotImplementedError(
"Couch document id(name) should be specified. i.e. self.name='software'"
)
def validate(self, apiobj, method, api, param, safe):
pass
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
def get(self):
"""
Return entire self.name document
"""
try:
sw = self.reqmgr_aux_db.document(self.name)
del sw["_id"]
del sw["_rev"]
except CouchNotFoundError:
raise NoSuchInstance
return rows([sw])
@restcall
def post(self):
"""
post sofware version doucment
"""
data = cherrypy.request.body.read()
if not data:
raise MissingPostData()
else:
doc = json.loads(data)
doc = Document(self.name, doc)
result = self.reqmgr_aux_db.commitOne(doc)
return result
class ActiveRequestJobInfo(RESTEntity):
"""
This class need to move under WMStats server when wmstats server created
"""
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
def validate(self, apiobj, method, api, param, safe):
return
@restcall(formats=[('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows([DataCache.getlatestJobData()])
class WorkloadConfig(RESTEntity):
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
self.reqdb_url = "%s/%s" % (config.couch_host, config.couch_reqmgr_db)
def _validate_args(self, param, safe):
# TODO: need proper validation but for now pass everything
args_length = len(param.args)
if args_length == 1:
safe.kwargs["name"] = param.args[0]
param.args.pop()
else:
raise MethodWithoutQueryString
return
def validate(self, apiobj, method, api, param, safe):
"""
Validate request input data.
Has to be implemented, otherwise the service fails to start.
If it's not implemented correctly (e.g. just pass), the arguments
are not passed in the method at all.
"""
self._validate_args(param, safe)
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self, name):
"""
Workload config world API call.
:arg str name: name to appear in the result message.
:returns: row with response.
"""
helper = WMWorkloadHelper()
try:
helper.loadSpecFromCouch(self.reqdb_url, name)
except Exception:
raise cherrypy.HTTPError(404, "Cannot find workload: %s" % name)
return str(helper.data)
class GlobalLockList(RESTEntity):
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
def validate(self, apiobj, method, api, param, safe):
return
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows(
DataCache.filterData(
ACTIVE_NO_CLOSEOUT_FILTER,
["InputDataset", "OutputDatasets", "MCPileup", "DataPileup"]))
class ProcessMatrix(RESTEntity):
"""
Process and thread matrix in reqmgr2
"""
def __init__(self, app, api, config, mount):
# CouchDB auxiliary database name
RESTEntity.__init__(self, app, api, config, mount)
self.time0 = time.time()
def validate(self, apiobj, method, api, param, safe):
pass
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
def get(self):
sdict = {'server': processStatusDict()}
sdict['server'].update({'uptime': time.time() - self.time0})
return rows([sdict])
class ProtectedLFNListOnlyFinalOutput(RESTEntity):
"""
Same as ProtectedLFNList API, however this one only provides LFNs that are not
transient, so only final output LFNs.
"""
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
def validate(self, apiobj, method, api, param, safe):
return
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows(DataCache.getProtectedLFNs())
class ParentLockList(RESTEntity):
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
def validate(self, apiobj, method, api, param, safe):
return
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self):
# This assumes both the DataCache and the parentage cache list
# get periodically updated.
# In case of problems, the WMStats cherrypy threads logs need to be checked
if DataCache.isEmpty():
raise DataCacheEmpty()
else:
return rows(DataCache.getParentDatasetList())
class ProtectedLFNList(RESTEntity):
"""
API which provides a list of ALL possible unmerged LFN bases (including
transient datasets/LFNs).
"""
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
def validate(self, apiobj, method, api, param, safe):
return
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows(
DataCache.filterData(ACTIVE_STATUS_FILTER,
["OutputModulesLFNBases"]))
def __init__(self, app, config, mount):
"""
:arg app: reference to application object; passed to all entities.
:arg config: reference to configuration; passed to all entities.
:arg str mount: API URL mount point; passed to all entities."""
RESTApi.__init__(self, app, config, mount)
cherrypy.log("WMStats entire configuration:\n%s" %
Configuration.getInstance())
cherrypy.log("WMStats REST hub configuration subset:\n%s" % config)
# only allows json format for return value
self.formats = [('application/json', JSONFormat())]
self._add({
"info":
ServerInfo(app, self, config, mount),
"teams":
TeamInfo(app, self, config, mount),
"request":
RequestInfo(app, self, config, mount),
"isfinished":
FinishedStatusInfo(app, self, config, mount),
"jobdetail":
JobDetailInfo(app, self, config, mount),
"requestcache":
ActiveRequestJobInfo(app, self, config, mount),
"filtered_requests":
FilteredActiveRequestJobInfo(app, self, config, mount),
"protectedlfns":
ProtectedLFNList(app, self, config, mount),
"protectedlfns_final":
ProtectedLFNListOnlyFinalOutput(app, self, config, mount),
"globallocks":
GlobalLockList(app, self, config, mount),
"parentlocks":
ParentLockList(app, self, config, mount),
"proc_status":
ProcessMatrix(app, self, config, mount)
})
class TeamInfo(RESTEntity):
"""
This class need to move under WMStats server when wmstats server created
"""
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
wmstats_url = "%s/%s" % (self.config.couch_host,
self.config.couch_wmstats_db)
self.wmstats = WMStatsReader(wmstats_url)
def validate(self, apiobj, method, api, param, safe):
args_length = len(param.args)
if args_length == 1:
safe.args.append(param.args[0])
param.args.pop()
return
@restcall(formats=[('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self):
result = self.wmstats.agentsByTeam(filterDrain=False)
return rows(result)
class Request(RESTEntity):
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
self.reqmgr_db = api.db_handler.get_db(config.couch_reqmgr_db)
self.reqmgr_db_service = RequestDBWriter(self.reqmgr_db, couchapp="ReqMgr")
# this need for the post validtiaon
self.gq_service = WorkQueue(config.couch_host, config.couch_workqueue_db)
def _validateGET(self, param, safe):
# TODO: need proper validation but for now pass everything
args_length = len(param.args)
if args_length == 1:
safe.kwargs["name"] = param.args[0]
param.args.pop()
return
no_multi_key = ["detail", "_nostale", "date_range", "common_dict"]
for key, value in param.kwargs.items():
# convert string to list
if key not in no_multi_key and isinstance(value, basestring):
param.kwargs[key] = [value]
detail = param.kwargs.get('detail', True)
if detail in (False, "false", "False", "FALSE"):
detail = False
if "status" in param.kwargs and detail:
for status in param.kwargs["status"]:
if status.endswith("-archived"):
raise InvalidSpecParameterValue(
"""Can't retrieve bulk archived status requests with detail option True,
set detail=false or use other search arguments""")
for prop in param.kwargs.keys():
safe.kwargs[prop] = param.kwargs.pop(prop)
return
def _validateRequestBase(self, param, safe, valFunc, requestName=None):
data = cherrypy.request.body.read()
if data:
request_args = json.loads(data)
else:
request_args = {}
cherrypy.log('Updating request "%s" with these user-provided args: %s' % (requestName, request_args))
# In case key args are also passed and request body also exists.
# If the request.body is dictionary update the key args value as well
if isinstance(request_args, dict):
for prop in param.kwargs.keys():
request_args[prop] = param.kwargs.pop(prop)
if requestName:
request_args["RequestName"] = requestName
request_args = [request_args]
safe.kwargs['workload_pair_list'] = []
for args in request_args:
workload, r_args = valFunc(args, self.config, self.reqmgr_db_service, param)
safe.kwargs['workload_pair_list'].append((workload, r_args))
def _get_request_names(self, ids):
"Extract request names from given documents"
# cherrypy.log("request names %s" % ids)
doc = {}
if isinstance(ids, list):
for rid in ids:
doc[rid] = 'on'
elif isinstance(ids, basestring):
doc[ids] = 'on'
docs = []
for key in doc.keys():
if key.startswith('request'):
rid = key.split('request-')[-1]
if rid != 'all':
docs.append(rid)
del doc[key]
return docs
def _getMultiRequestArgs(self, multiRequestForm):
request_args = {}
for prop in multiRequestForm:
if prop == "ids":
request_names = self._get_request_names(multiRequestForm["ids"])
elif prop == "new_status":
request_args["RequestStatus"] = multiRequestForm[prop]
# remove this
# elif prop in ["CustodialSites", "AutoApproveSubscriptionSites"]:
# request_args[prop] = [multiRequestForm[prop]]
else:
request_args[prop] = multiRequestForm[prop]
return request_names, request_args
def _validateMultiRequests(self, param, safe, valFunc):
data = cherrypy.request.body.read()
if data:
request_names, request_args = self._getMultiRequestArgs(json.loads(data))
else:
# actually this is error case
# cherrypy.log(str(param.kwargs))
request_names, request_args = self._getMultiRequestArgs(param.kwargs)
for prop in request_args:
if prop == "RequestStatus":
del param.kwargs["new_status"]
else:
del param.kwargs[prop]
del param.kwargs["ids"]
safe.kwargs['workload_pair_list'] = []
for request_name in request_names:
request_args["RequestName"] = request_name
workload, r_args = valFunc(request_args, self.config, self.reqmgr_db_service, param)
safe.kwargs['workload_pair_list'].append((workload, r_args))
safe.kwargs["multi_update_flag"] = True
def _getRequestNamesFromBody(self, safe):
request_names = json.loads(cherrypy.request.body.read())
safe.kwargs['workload_pair_list'] = request_names
safe.kwargs["multi_names_flag"] = True
def validate(self, apiobj, method, api, param, safe):
# to make validate successful
# move the validated argument to safe
# make param empty
# other wise raise the error
try:
if method == 'GET':
self._validateGET(param, safe)
elif method == 'PUT':
args_length = len(param.args)
if args_length == 1:
requestName = param.args[0]
param.args.pop()
else:
requestName = None
self._validateRequestBase(param, safe, validate_request_update_args, requestName)
elif method == 'POST':
args_length = len(param.args)
if args_length == 2 and param.args[0] == "clone":
# handles clone workflow.- don't validtate args here
param.kwargs['OriginalRequestName'] = param.args[1]
param.args.pop()
param.args.pop()
self._validateRequestBase(param, safe, validate_clone_create_args)
elif args_length == 1 and param.args[0] == "multi_update":
# special case for multi update from browser.
param.args.pop()
self._validateMultiRequests(param, safe, validate_request_update_args)
elif args_length == 1 and param.args[0] == "bynames":
# special case for multi update from browser.
param.args.pop()
self._getRequestNamesFromBody(safe)
else:
self._validateRequestBase(param, safe, validate_request_create_args)
except InvalidSpecParameterValue as ex:
raise ex
except Exception as ex:
# TODO add proper error message instead of trace back
msg = traceback.format_exc()
cherrypy.log("Error: %s" % msg)
if hasattr(ex, "message"):
if hasattr(ex.message, '__call__'):
msg = ex.message()
else:
msg = str(ex)
else:
msg = str(ex)
raise InvalidSpecParameterValue(msg)
def _maskResult(self, mask, result):
"""
If a mask of parameters was provided in the query string, then filter
the request key/values accordingly.
:param mask: a list of strings (keys of the request dictionary)
:param result: a dict key'ed by the request name, with the whole
request dictionary as a value
:return: updates the result object in place and returns it (dict)
"""
if len(mask) == 1 and mask[0] == "DAS":
mask = ReqMgrConfigDataCache.getConfig("DAS_RESULT_FILTER")["filter_list"]
if len(mask) > 0:
maskedResult = {}
for reqName, reqDict in result.items():
reqInfo = RequestInfo(reqDict)
maskedResult.setdefault(reqName, {})
for maskKey in mask:
foundValue = reqInfo.get(maskKey, None)
maskedResult[reqName].update({maskKey: foundValue})
return maskedResult
else:
return result
@restcall(formats=[('text/plain', PrettyJSONFormat()), ('application/json', JSONFormat())])
def get(self, **kwargs):
"""
Returns request info depending on the conditions set by kwargs
Currently defined kwargs are following.
statusList, requestNames, requestType, prepID, inputDataset, outputDataset, dateRange
If jobInfo is True, returns jobInfomation about the request as well.
TODO:
stuff like this has to masked out from result of this call:
_attachments: {u'spec': {u'stub': True, u'length': 51712, u'revpos': 2, u'content_type': u'application/json'}}
_id: maxa_RequestString-OVERRIDE-ME_130621_174227_9225
_rev: 4-c6ceb2737793aaeac3f1cdf591593da4
"""
### pop arguments unrelated to the user query
mask = kwargs.pop("mask", [])
detail = kwargs.pop("detail", True)
common_dict = int(kwargs.pop("common_dict", 0)) # modifies the response format
nostale = kwargs.pop("_nostale", False)
### these are the query strings supported by this API
status = kwargs.get("status", [])
name = kwargs.get("name", [])
request_type = kwargs.get("request_type", [])
prep_id = kwargs.get("prep_id", [])
inputdataset = kwargs.get("inputdataset", [])
outputdataset = kwargs.get("outputdataset", [])
date_range = kwargs.get("date_range", False)
campaign = kwargs.get("campaign", [])
team = kwargs.get("team", [])
mc_pileup = kwargs.get("mc_pileup", [])
data_pileup = kwargs.get("data_pileup", [])
requestor = kwargs.get("requestor", [])
# further tweaks to the couch queries
if len(status) == 1 and status[0] == "ACTIVE":
status = ACTIVE_STATUS
if detail in (False, "false", "False", "FALSE"):
option = {"include_docs": False}
else:
option = {"include_docs": True}
# everything should be stale view. this only needs for test
if nostale:
self.reqmgr_db_service._setNoStale()
request_info = []
queryMatched = False # flag to avoid calling the same view twice
if len(kwargs) == 2:
if status and team:
query_keys = [[t, s] for t in team for s in status]
request_info.append(self.reqmgr_db_service.getRequestByCouchView("byteamandstatus",
option, query_keys))
queryMatched = True
elif status and request_type:
query_keys = [[s, rt] for rt in request_type for s in status]
request_info.append(self.reqmgr_db_service.getRequestByCouchView("requestsbystatusandtype",
option, query_keys))
queryMatched = True
elif status and requestor:
query_keys = [[s, r] for r in requestor for s in status]
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bystatusandrequestor",
option, query_keys))
queryMatched = True
elif len(kwargs) == 3:
if status and request_type and requestor:
query_keys = [[s, rt, req] for s in status for rt in request_type for req in requestor]
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bystatusandtypeandrequestor",
option, query_keys))
queryMatched = True
# anything else that hasn't matched the query combination above
if not queryMatched:
if status:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bystatus",
option, status))
if name:
request_info.append(self.reqmgr_db_service.getRequestByNames(name))
if request_type:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bytype",
option, request_type))
if prep_id:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("byprepid",
option, prep_id))
if inputdataset:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("byinputdataset",
option, inputdataset))
if outputdataset:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("byoutputdataset",
option, outputdataset))
if date_range:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bydate",
option, date_range))
if campaign:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bycampaign",
option, campaign))
if mc_pileup:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bymcpileup",
option, mc_pileup))
if data_pileup:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bydatapileup",
option, data_pileup))
# get the intersection of the request data
result = self._intersection_of_request_info(request_info)
if not result:
return []
result = self._maskResult(mask, result)
if not option["include_docs"]:
return result.keys()
# set the return format. default format has request name as a key
# if is set to one it returns list of dictionary with RequestName field.
if common_dict == 1:
response_list = result.values()
else:
response_list = [result]
return rows(response_list)
def _intersection_of_request_info(self, request_info):
requests = {}
if len(request_info) < 1:
return requests
request_key_set = set(request_info[0].keys())
for info in request_info:
request_key_set = set(request_key_set) & set(info.keys())
# TODO: need to assume some data maight not contains include docs
for request_name in request_key_set:
requests[request_name] = request_info[0][request_name]
return requests
def _retrieveResubmissionChildren(self, request_name):
"""
Fetches all the direct children requests from CouchDB.
Response from CouchDB view is in the following format:
[{u'id': u'child_workflow_name',
u'key': u'parent_workflow_name',
u'value': 'current_request_status'}]
:param request_name: string with the parent workflow name
:return: a list of dictionaries with the parent and child workflow and the child status
"""
result = self.reqmgr_db.loadView('ReqMgr', 'childresubmissionrequests', keys=[request_name])['rows']
childrenRequestAndStatus = []
for childInfo in result:
childrenRequestAndStatus.append(childInfo)
childrenRequestAndStatus.extend(self._retrieveResubmissionChildren(childInfo['id']))
return childrenRequestAndStatus
def _handleNoStatusUpdate(self, workload, request_args, dn):
"""
For no-status update, we only support the following parameters:
1. RequestPriority
2. Global workqueue statistics, while acquiring a workflow
"""
if 'RequestPriority' in request_args:
# Yes, we completely ignore any other arguments posted by the user (web UI case)
request_args = {'RequestPriority': request_args['RequestPriority']}
validate_request_priority(request_args)
# must update three places: GQ elements, workload_cache and workload spec
self.gq_service.updatePriority(workload.name(), request_args['RequestPriority'])
report = self.reqmgr_db_service.updateRequestProperty(workload.name(), request_args, dn)
workload.setPriority(request_args['RequestPriority'])
workload.saveCouchUrl(workload.specUrl())
cherrypy.log('Updated priority of "{}" to: {}'.format(workload.name(), request_args['RequestPriority']))
elif workqueue_stat_validation(request_args):
report = self.reqmgr_db_service.updateRequestStats(workload.name(), request_args)
cherrypy.log('Updated workqueue statistics of "{}", with: {}'.format(workload.name(), request_args))
else:
msg = "There are invalid arguments for no-status update: %s" % request_args
raise InvalidSpecParameterValue(msg)
return report
def _handleAssignmentApprovedTransition(self, workload, request_args, dn):
"""
Allows only two arguments: RequestStatus and RequestPriority
"""
if "RequestPriority" not in request_args:
msg = "There are invalid arguments for assignment-approved transition: %s" % request_args
raise InvalidSpecParameterValue(msg)
validate_request_priority(request_args)
report = self.reqmgr_db_service.updateRequestProperty(workload.name(), request_args, dn)
return report
def _handleAssignmentStateTransition(self, workload, request_args, dn):
if ('SoftTimeout' in request_args) and ('GracePeriod' in request_args):
request_args['HardTimeout'] = request_args['SoftTimeout'] + request_args['GracePeriod']
# Only allow extra value update for assigned status
cherrypy.log("Assign request %s, input args: %s ..." % (workload.name(), request_args))
try:
workload.updateArguments(request_args)
except Exception as ex:
msg = traceback.format_exc()
cherrypy.log("Error for request args %s: %s" % (request_args, msg))
raise InvalidSpecParameterValue(str(ex))
# validate/update OutputDatasets after ProcessingString and AcquisionEra is updated
request_args['OutputDatasets'] = workload.listOutputDatasets()
validateOutputDatasets(request_args['OutputDatasets'], workload.getDbsUrl())
# by default, it contains all unmerged LFNs (used by sites to protect the unmerged area)
request_args['OutputModulesLFNBases'] = workload.listAllOutputModulesLFNBases()
# Add parentage relation for step chain, task chain:
chainMap = workload.getChainParentageSimpleMapping()
if chainMap:
request_args["ChainParentageMap"] = chainMap
# save the spec first before update the reqmgr request status to prevent race condition
# when workflow is pulled to GQ before site white list is updated
workload.saveCouch(self.config.couch_host, self.config.couch_reqmgr_db)
report = self.reqmgr_db_service.updateRequestProperty(workload.name(), request_args, dn)
return report
def _handleOnlyStateTransition(self, workload, request_args, dn):
"""
It handles only the state transition.
Special handling needed if a request is aborted or force completed.
"""
# if we got here, then the main workflow has been already validated
# and the status transition is allowed
req_status = request_args["RequestStatus"]
cascade = request_args.get("cascade", False)
if req_status in ["aborted", "force-complete"]:
# cancel the workflow first
self.gq_service.cancelWorkflow(workload.name())
# cascade option is only supported for these 3 statuses. If set, we need to
# find all the children requests and perform the same status transition
if req_status in ["rejected", "closed-out", "announced"] and cascade:
childrenNamesAndStatus = self._retrieveResubmissionChildren(workload.name())
msg = "Workflow {} has {} ".format(workload.name(), len(childrenNamesAndStatus))
msg += "children workflows to have a status transition to: {}".format(req_status)
cherrypy.log(msg)
for childInfo in childrenNamesAndStatus:
if check_allowed_transition(childInfo['value'], req_status):
cherrypy.log('Updating request status for {} to {}.'.format(childInfo['id'], req_status))
self.reqmgr_db_service.updateRequestStatus(childInfo['id'], req_status, dn)
else:
msg = "Status transition from {} to {} ".format(childInfo['value'], req_status)
msg += "not allowed for workflow: {}, skipping it!".format(childInfo['id'])
cherrypy.log(msg)
# then update the original/parent workflow status in couchdb
cherrypy.log('Updating request status for {} to {}.'.format(workload.name(), req_status))
report = self.reqmgr_db_service.updateRequestStatus(workload.name(), req_status, dn)
return report
def _updateRequest(self, workload, request_args):
dn = get_user_info().get("dn", "unknown")
if "RequestStatus" not in request_args:
report = self._handleNoStatusUpdate(workload, request_args, dn)
else:
req_status = request_args["RequestStatus"]
if len(request_args) == 2 and req_status == "assignment-approved":
report = self._handleAssignmentApprovedTransition(workload, request_args, dn)
elif len(request_args) > 1 and req_status == "assigned":
report = self._handleAssignmentStateTransition(workload, request_args, dn)
elif len(request_args) == 1 or (len(request_args) == 2 and "cascade" in request_args):
report = self._handleOnlyStateTransition(workload, request_args, dn)
else:
msg = "There are invalid arguments with this status transition: %s" % request_args
raise InvalidSpecParameterValue(msg)
if report == 'OK':
return {workload.name(): "OK"}
return {workload.name(): "ERROR"}
@restcall(formats=[('application/json', JSONFormat())])
def put(self, workload_pair_list):
"""workloadPairList is a list of tuple containing (workload, request_args)"""
report = []
for workload, request_args in workload_pair_list:
result = self._updateRequest(workload, request_args)
report.append(result)
return report
@restcall(formats=[('application/json', JSONFormat())])
def delete(self, request_name):
cherrypy.log("INFO: Deleting request document '%s' ..." % request_name)
try:
self.reqmgr_db.delete_doc(request_name)
except CouchError as ex:
msg = "ERROR: Delete failed."
cherrypy.log(msg + " Reason: %s" % ex)
raise cherrypy.HTTPError(404, msg)
# TODO
# delete should also happen on WMStats
cherrypy.log("INFO: Delete '%s' done." % request_name)
def _update_additional_request_args(self, workload, request_args):
"""
add to request_args properties which is not initially set from user.
This data will put in to couchdb.
Update request_args here if additional information need to be put in couchdb
"""
request_args['RequestWorkflow'] = sanitizeURL("%s/%s/%s/spec" % (request_args["CouchURL"],
request_args["CouchWorkloadDBName"],
workload.name()))['url']
# Add the output datasets if necessary
# for some bizarre reason OutpuDatasets is list of lists
request_args['OutputDatasets'] = workload.listOutputDatasets()
# Add initial priority only for the creation of the request
request_args['InitialPriority'] = request_args["RequestPriority"]
return
@restcall(formats=[('application/json', JSONFormat())])
def post(self, workload_pair_list, multi_update_flag=False, multi_names_flag=False):
"""
Create and update couchDB with a new request.
request argument is passed from validation
(validation convert cherrypy.request.body data to argument)
TODO:
this method will have some parts factored out so that e.g. clone call
can share functionality.
NOTES:
1) do not strip spaces, #4705 will fails upon injection with spaces;
currently the chain relies on a number of things coming in #4705
2) reqInputArgs = Utilities.unidecode(json.loads(body))
(from ReqMgrRESTModel.putRequest)
"""
# storing the request document into Couch
if multi_update_flag:
return self.put(workload_pair_list)
if multi_names_flag:
return self.get(name=workload_pair_list)
out = []
for workload, request_args in workload_pair_list:
self._update_additional_request_args(workload, request_args)
cherrypy.log("Create request, input args: %s ..." % request_args)
try:
workload.saveCouch(request_args["CouchURL"], request_args["CouchWorkloadDBName"],
metadata=request_args)
out.append({'request': workload.name()})
except Exception as ex:
# then it failed to add the spec file as attachment
# we better delete the original request to avoid confusion in wmstats
cherrypy.log("Error saving request spec to couch: %s " % str(ex))
self.delete(request_args['RequestName'])
return out
class WorkloadSplitting(RESTEntity):
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
self.reqdb_url = "%s/%s" % (config.couch_host, config.couch_reqmgr_db)
def _validate_args(self, param, safe):
# TODO: need proper validation but for now pass everything
args_length = len(param.args)
if args_length == 1:
safe.kwargs["name"] = param.args[0]
param.args.pop()
elif args_length == 2 and param.args[0] == "web_form":
safe.kwargs["web_form"] = True
safe.kwargs["name"] = param.args[1]
param.args.pop()
param.args.pop()
return
def validate(self, apiobj, method, api, param, safe):
"""
Validate request input data.
Has to be implemented, otherwise the service fails to start.
If it's not implemented correctly (e.g. just pass), the arguments
are not passed in the method at all.
"""
self._validate_args(param, safe)
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self, name, web_form=False):
"""
getting job splitting algorithm.
:arg str name: name to appear in the result message.
:returns: row with response, here 1 item list with message.
"""
helper = WMWorkloadHelper()
try:
helper.loadSpecFromCouch(self.reqdb_url, name)
except Exception:
raise cherrypy.HTTPError(404, "Cannot find workload: %s" % name)
splittingDict = helper.listJobSplittingParametersByTask(
performance=False)
taskNames = sorted(splittingDict.keys())
splitInfo = []
for taskName in taskNames:
splitInfo.append({
"splitAlgo": splittingDict[taskName]["algorithm"],
"splitParams": splittingDict[taskName],
"taskType": splittingDict[taskName]["type"],
"taskName": taskName
})
if web_form:
splitInfo = create_web_splitting_format(splitInfo)
return splitInfo
@restcall(formats=[('application/json', JSONFormat())])
@tools.expires(secs=-1)
def post(self, name):
"""
Parse job splitting parameters sent from the splitting parameter update
page. Pull down the request and modify the new spec applying the
updated splitting parameters.
"""
data = cherrypy.request.body.read()
splittingInfo = json.loads(data)
helper = WMWorkloadHelper()
try:
helper.loadSpecFromCouch(self.reqdb_url, name)
except Exception:
raise cherrypy.HTTPError(404,
"Cannot find workload for: %s" % name)
for taskInfo in splittingInfo:
splittingTask = taskInfo["taskName"]
splittingAlgo = taskInfo["splitAlgo"]
submittedParams = taskInfo["splitParams"]
splitParams = {}
for param in submittedParams:
validFlag, castType = _validate_split_param(
splittingAlgo, param)
if validFlag:
_assign_key_value(param, submittedParams[param],
splitParams, castType)
else:
msg = "Parameter '%s' is not supported in the algorithm '%s'" % (
param, splittingAlgo)
raise cherrypy.HTTPError(400, msg)
helper.setJobSplittingParameters(splittingTask, splittingAlgo,
splitParams)
# Now persist all these changes in the workload
url = "%s/%s" % (self.reqdb_url, name)
result = helper.saveCouchUrl(url)
return result
class WMAData(RESTEntity):
"REST interface for WMArchive"
def __init__(self, app, api, config, mount):
RESTEntity.__init__(self, app, api, config, mount)
self.config = config
self.mgr = WMArchiveManager(config)
def validate(self, apiobj, method, api, param, safe):
"""
Validate request input data.
Has to be implemented, otherwise the service fails to start.
If it's not implemented correctly (e.g. just pass), the arguments
are not passed in the method at all.
"""
if method == 'GET':
# Check for `performance` endpoint, documented in
# https://github.com/knly/WMArchiveAggregation
if len(param.args) == 1 and param.args[0] == 'performance':
safe.args.append(param.args[0])
param.args.remove(param.args[0])
# Validate arguments
validate_strlist('metrics[]', param, safe,
re.compile(r'^[a-zA-Z.]+'))
validate_strlist('axes[]', param, safe,
re.compile(r'^[a-zA-Z_]+'))
validate_strlist('suggestions[]', param, safe,
re.compile(r'^[a-zA-Z]+'))
date_pattern = PAT_YYYYMMDD
validate_str('start_date',
param,
safe,
date_pattern,
optional=True)
validate_str('end_date',
param,
safe,
date_pattern,
optional=True)
validate_rx('workflow', param, safe, optional=True)
validate_rx('task', param, safe, optional=True)
validate_rx('host', param, safe, optional=True)
validate_rx('site', param, safe, optional=True)
validate_rx('jobtype', param, safe, optional=True)
validate_rx('jobstate', param, safe, optional=True)
validate_rx('acquisitionEra', param, safe, optional=True)
validate_rx('exitCode', param, safe, optional=True)
validate_rx('exitStep', param, safe, optional=True)
validate_str('_', param, safe, PAT_INFO, optional=True)
return True
if 'query' in param.kwargs.keys():
validate_str('query', param, safe, PAT_QUERY, optional=True)
for key in ['status', 'jobs', '_']:
if key in param.kwargs.keys():
validate_str(key, param, safe, PAT_INFO, optional=True)
# test if user provided uid
if len(param.args) == 1 and PAT_UID.match(param.args[0]):
safe.args.append(param.args[0])
param.args.remove(param.args[0])
return True
elif method == 'POST':
if not param.args or not param.kwargs:
return False # this class does not need any parameters
return True
@restcall(formats=[('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self, *args, **kwds):
"""
Implement GET request with given uid or set of parameters
All work is done by WMArchiveManager
"""
if 'performance' in args:
# documented in https://github.com/knly/WMArchiveAggregation
kwds['metrics'] = kwds.pop('metrics[]', None)
kwds['axes'] = kwds.pop('axes[]', None)
kwds['suggestions'] = kwds.pop('suggestions[]', None)
return results(dict(performance=self.mgr.performance(**kwds)))
if kwds.get('status', ''):
return results(dict(status=self.mgr.status()))
if kwds.get('jobs', ''):
return results(dict(jobs=self.mgr.jobs()))
if args and len(args) == 1: # requested uid
return results(self.mgr.read(args[0], []))
return results({'request': kwds, 'results': 'Not available'})
@restcall(formats=[('application/json', JSONFormat())])
@tools.expires(secs=-1)
def post(self):
"""
Implement POST request API, all work is done by WMArchiveManager.
The request should either provide query to fetch results from back-end
or data to store to the back-end.
The input HTTP request should be either
{"data":some_data} for posting the data into WMArchive or
{"spec":some_query, "fields":return_fields} for querying the data in WMArchive.
The some_data should be proper JSON document(s).
The some_query should be use MongoDB QL.
"""
msg = 'expect "data", "query" attributes in your request'
result = {'status': 'Not supported, %s' % msg, 'data': []}
try:
request = json.load(cherrypy.request.body)
if 'spec' in request.keys() and 'fields' in request.keys():
result = self.mgr.read(request['spec'], request['fields'])
elif 'data' in request.keys():
result = self.mgr.write(request['data'])
elif 'job' in request.keys():
result = self.mgr.write(request['job'])
if isinstance(result, GeneratorType):
result = [r for r in result]
return results(result)
except cherrypy.HTTPError:
raise
except Exception as exp:
traceback.print_exc()
raise cherrypy.HTTPError()
@restcall(formats=[('application/json', JSONFormat())])
@tools.expires(secs=-1)
def put(self):
"""
Implement PUT request API, all work is done by WMArchiveManager.
The request should either provide query to fetch results from back-end
or data to store to the back-end.
The input HTTP request should be in a form
{"ids":[list_of_ids], "spec": update_spec}
"""
msg = 'expect "data", "query" attributes in your request'
result = {'status': 'Not supported, %s' % msg, 'data': []}
try:
request = json.load(cherrypy.request.body)
result = self.mgr.update(request['ids'], request['spec'])
if isinstance(result, GeneratorType):
result = [r for r in result]
return results(result)
except cherrypy.HTTPError:
raise
except Exception as exp:
traceback.print_exc()
raise cherrypy.HTTPError()
class AuxBaseAPI(RESTEntity):
"""
Base class for Aux db RESTEntry which contains get, post method
"""
def __init__(self, app, api, config, mount):
RESTEntity.__init__(self, app, api, config, mount)
# CouchDB auxiliary database name
self.reqmgr_aux_db = api.db_handler.get_db(config.couch_reqmgr_aux_db)
self.setName()
def setName(self):
"Sets the document name"
raise NotImplementedError(
"Couch document id(name) should be specified. i.e. self.name='software'"
)
def validate(self, apiobj, method, api, param, safe):
args_length = len(param.args)
if args_length == 1:
safe.kwargs["subName"] = param.args.pop(0)
return
return
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
def get(self, subName=None):
"""
Return entire self.name document
subName is subcategory of document which is added as postfix string
"""
try:
if subName:
docName = "%s_%s" % (self.name, subName)
else:
docName = self.name
sw = self.reqmgr_aux_db.document(docName)
del sw["_id"]
del sw["_rev"]
except CouchNotFoundError:
raise NoSuchInstance
return rows([sw])
@restcall(formats=[('application/json', JSONFormat())])
def post(self, subName=None):
"""
If the document already exists, replace it with a new one.
"""
data = cherrypy.request.body.read()
if not data:
raise MissingPostData()
else:
doc = json.loads(data)
if subName:
docName = "%s_%s" % (self.name, subName)
else:
docName = self.name
doc["ConfigType"] = self.name
doc = Document(docName, doc)
result = self.reqmgr_aux_db.commitOne(doc)
return result
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
def put(self, subName=None):
"""
update document for the given self.name and subName
"""
data = cherrypy.request.body.read()
if not data:
raise MissingPostData()
else:
propertyDict = json.loads(data)
try:
result = None
if subName:
docName = "%s_%s" % (self.name, subName)
else:
docName = self.name
if propertyDict:
existDoc = self.reqmgr_aux_db.document(docName)
existDoc.update(propertyDict)
result = self.reqmgr_aux_db.commitOne(existDoc)
except CouchNotFoundError:
raise NoSuchInstance
return result
@restcall(formats=[('application/json', JSONFormat())])
def delete(self, subName):
"""
Delete a document from ReqMgrAux
"""
docName = "%s_%s" % (self.name, subName)
try:
self.reqmgr_aux_db.delete_doc(docName)
except (CouchError, CouchNotFoundError) as ex:
msg = "ERROR: failed to delete document: %s\nReason: %s" % (
docName, str(ex))
cherrypy.log(msg)