def put(self, team_name):
"""
Adds team of team_name into the database.
Creates teams document if it doesn't exist.
"""
try:
teams = self.reqmgr_aux_db.document("teams")
except CouchNotFoundError as ex:
msg = ("ERROR: Retrieving teams document failed, reason: %s"
" Creating the document ..." % ex)
cherrypy.log(msg)
try:
doc = Document(id="teams", inputDict={team_name: None})
self.reqmgr_aux_db.commitOne(doc)
return
except CouchError as ex:
msg = "ERROR: Creating document teams failed, reason: %s" % ex
cherrypy.log(msg)
raise cherrypy.HTTPError(400, msg)
if team_name in teams:
return rows(["Already exists."])
else:
teams[team_name] = None
# TODO
# this should ideally also wrap try-except
self.reqmgr_aux_db.commitOne(teams)
return rows(["OK"])
def get(self, transition):
"""
Return list of allowed request status.
If transition, return exhaustive list with all request status
and their defined transitions.
"""
if transition == "true":
return rows([REQUEST_STATE_TRANSITION])
else:
return rows(REQUEST_STATE_LIST)
def delete(self, type, site_name, alias):
"""Delete site name associations. Only Global admins and SiteDB operators,
can delete names for the sites. When more than one
argument is given, there must be an equal number of arguments for all
the parameters. For input validation requirements, see the field
descriptions above. It is an error to attempt to delete a non-existent
site name association.
:arg list type: values to delete;
:arg list site_name: values to delete;
:arg list alias: values to delete;
:returns: a list with a dict in which *modified* gives the number of objects
deleted from the database, which is always *len(type).*"""
binds = self.api.bindmap(type = type, site_name = site_name, alias = alias)
lcg = filter(lambda b: b['type'] == 'lcg', binds)
cms = filter(lambda b: b['type'] == 'cms', binds)
phedex = filter(lambda b: b['type'] == 'phedex', binds)
for b in binds: del b['type']
updated = 0
if cms:
c, _ = self.api.executemany("""
delete from cms_name
where id in (select cmap.cms_name_id
from site s
join site_cms_name_map cmap on cmap.site_id = s.id
where s.name = :site_name)
and name = :alias
""", cms)
self.api.rowstatus(c, len(cms))
updated += c.rowcount
if lcg:
c, _ = self.api.executemany("""
delete from sam_name
where id in (select smap.sam_id
from site s
join site_cms_name_map cmap on cmap.site_id = s.id
join sam_cms_name_map smap on smap.cms_name_id = cmap.cms_name_id
where s.name = :site_name)
and name = :alias
""", lcg)
self.api.rowstatus(c, len(lcg))
updated += c.rowcount
if phedex:
c, _ = self.api.executemany("""
delete from phedex_node
where site = (select s.id from site s where s.name = :site_name)
and name = :alias
""", phedex)
self.api.rowstatus(c, len(phedex))
updated += c.rowcount
result = rows([{ "modified": updated }])
trace = request.db["handle"]["trace"]
trace and cherrypy.log("%s commit" % trace)
request.db["handle"]["connection"].commit()
return result
def get(self):
# authorization / access control:
# cherrypy (modified CMS web version here) can read this information
# from HTTP headers. CMS web frontend puts this information into
# headers as read from SiteDB (or on private VM from a fake
# SiteDB file)
# print "cherrypy.request", cherrypy.request
# print "DN: %s" % cherrypy.request.user['dn']
# print "Requestor/login: %s" % cherrypy.request.user['login']
# print "cherrypy.request: %s" % cherrypy.request
# print "cherrypy.request.user: %s" % cherrypy.request.user
# from WMCore.REST.Auth import authz_match
# authz_match(role=["Global Admin"], group=["global"])
# check SiteDB/DataWhoAmI.py
# implement as authentication decorator over modification calls
# check config.py main.authz_defaults and SiteDB
# (only Admin: ReqMgr to be able to modify stuff)
wmcore_reqmgr_version = WMCore.__version__
reqmgr_db_info = self.reqmgr_db.info()
reqmgr_db_info["reqmgr_couch_url"] = self.config.couch_host
# retrieve the last injected request in the system
# curl ... /reqmgr_workload_cache/_design/ReqMgr/_view/bydate?descending=true&limit=1
options = {"descending": True, "limit": 1}
reqmgr_last_injected_request = self.reqmgr_db.loadView("ReqMgr", "bydate", options=options)
result = {"wmcore_reqmgr_version": wmcore_reqmgr_version,
"reqmgr_db_info": reqmgr_db_info,
"reqmgr_last_injected_request": reqmgr_last_injected_request}
# NOTE:
# "return result" only would return dict with keys without (!!) values set
return rows([result])
def put(self, type, site_name, alias):
"""Insert new site names. Site executive can update their own site, global
admin can update names for all the sites. When more than one argument is
given, there must be an equal number of arguments for all the parameters.
For input validation requirements, see the field descriptions above.
It is an error to attempt to insert an existing name alias triplet.
:arg list type: new values;
:arg list site_name: new values;
:arg list alias: new values;
:returns: a list with a dict in which *modified* gives the number of objects
inserted into the database, which is always *len(type).*"""
self._authz(site_name)
binds = self.api.bindmap(type = type, site_name = site_name, alias = alias)
lcg = filter(lambda b: b['type'] == 'lcg', binds)
cms = filter(lambda b: b['type'] == 'cms', binds)
phedex = filter(lambda b: b['type'] == 'phedex', binds)
for b in binds: del b['type']
updated = 0
if cms:
c, _ = self.api.executemany("""
insert all
into cms_name (id, name) values (cms_name_sq.nextval, alias)
into site_cms_name_map (site_id, cms_name_id) values (site_id, cms_name_sq.nextval)
select s.id site_id, :alias alias
from site s where s.name = :site_name
""", cms)
self.api.rowstatus(c, 2*len(cms))
updated += c.rowcount / 2
if lcg:
c, _ = self.api.executemany("""
insert all
into sam_name (id, name) values (sam_name_sq.nextval, alias)
into sam_cms_name_map (cms_name_id, sam_id) values (cms_id, sam_name_sq.nextval)
select cmap.cms_name_id cms_id, :alias alias
from site s join site_cms_name_map cmap on cmap.site_id = s.id
where s.name = :site_name
""", lcg)
self.api.rowstatus(c, 2*len(lcg))
updated += c.rowcount / 2
if phedex:
c, _ = self.api.executemany("""
insert into phedex_node (id, site, name)
select phedex_node_sq.nextval, s.id, :alias
from site s where s.name = :site_name
""", phedex)
self.api.rowstatus(c, len(phedex))
updated += c.rowcount
result = rows([{ "modified": updated }])
trace = request.db["handle"]["trace"]
trace and cherrypy.log("%s commit" % trace)
request.db["handle"]["connection"].commit()
return result
def get(self, request_name):
try:
result = self.wmstats.isWorkflowCompletedWithLogCollectAndCleanUp(request_name)
except KeyError:
raise cherrypy.HTTPError(404, "Cannot find request: %s" % request_name)
return rows([result])
def get(self):
"""Hello world api call.
``paramname``
This is a paramater.
:returns: world"""
return rows(["world"])
def get(self):
"""
Return entire "software" document - all versions and scramarchs.
"""
sw = self.reqmgr_aux_db.document("software")
del sw["_id"]
del sw["_rev"]
return rows([sw])
def get(self):
"""
Return list of all groups.
"""
groups = self.db_handler.document(self.db_name, "groups")
del groups["_id"]
del groups["_rev"]
return rows(groups.keys())
def get(self):
"""
Return list of all teams.
"""
teams = self.db_handler.document(self.db_name, "teams")
del teams["_id"]
del teams["_rev"]
return rows(teams.keys())
def get(self):
"""
Return list of all groups.
"""
groups = self.reqmgr_aux_db.document("groups")
del groups["_id"]
del groups["_rev"]
return rows(groups.keys())
def get(self):
"""
Return entire "software" document - all versions and scramarchs.
"""
sw = self.db_handler.document(self.db_name, "software")
del sw["_id"]
del sw["_rev"]
return rows([sw])
def get(self):
"""
Return list of all teams.
"""
teams = self.reqmgr_aux_db.document("teams")
del teams["_id"]
del teams["_rev"]
return rows(teams.keys())
def get(self, **kwargs):
"""
Returns request info depending on the conditions set by kwargs
Currently defined kwargs are following.
statusList, requestNames, requestType, prepID, inputDataset, outputDataset, dateRange
If jobInfo is True, returns jobInfomation about the request as well.
TODO:
stuff like this has to filtered out from result of this call:
_attachments: {u'spec': {u'stub': True, u'length': 51712, u'revpos': 2, u'content_type': u'application/json'}}
_id: maxa_RequestString-OVERRIDE-ME_130621_174227_9225
_rev: 4-c6ceb2737793aaeac3f1cdf591593da4
"""
# list of status
status = kwargs.get("status", False)
# list of request names
name = kwargs.get("name", False)
type = kwargs.get("type", False)
prep_id = kwargs.get("prep_id", False)
inputdataset = kwargs.get("inputdataset", False)
outputdataset = kwargs.get("outputdataset", False)
date_range = kwargs.get("date_range", False)
campaign = kwargs.get("campaign", False)
workqueue = kwargs.get("workqueue", False)
team = kwargs.get("team", False)
# eventhing should be stale view. this only needs for test
_nostale = kwargs.get("_nostale", False)
option = {}
if not _nostale:
option['stale'] = "update_after"
request_info =[]
if status and not team:
request_info.append(self.get_reqmgr_view("bystatus" , option, status, "list"))
if status and team:
request_info.append(self.get_reqmgr_view("byteamandstatus", option, team, "list"))
if name:
request_doc = self.reqmgr_db.document(name)
request_info.append(rows([request_doc]))
if prep_id:
request_info.append(self.get_reqmgr_view("byprepid", option, prep_id, "list"))
if inputdataset:
request_info.append(self.get_reqmgr_view("byinputdataset", option, inputdataset, "list"))
if outputdataset:
request_info.append(self.get_reqmgr_view("byoutputdataset", option, outputdataset, "list"))
if date_range:
request_info.append(self.get_reqmgr_view("bydate", option, date_range, "list"))
if campaign:
request_info.append( self.get_reqmgr_view("bycampaign", option, campaign, "list"))
if workqueue:
request_info.append(self.get_reqmgr_view("byworkqueue", option, workqueue, "list"))
#get interaction of the request
return self._intersection_of_request_info(request_info);
def get(self, request_name, all):
"""
Returns most recent list of requests in the system.
Query particular request if request_name is specified.
Return complete list of all requests in the system if all is set.
If all is not set, check "default_view_requests_since_num_days"
config value and show only requests not older than this
number of days.
"""
if request_name:
request_doc = self.db_handler.document(self.db_name, request_name)
return rows([request_doc])
else:
options = {"descending": True}
if not all:
past_days = self.config.default_view_requests_since_num_days
current_date = list(time.gmtime()[:6])
from_date = datetime(*current_date) - timedelta(days=past_days)
options["endkey"] = list(from_date.timetuple()[:6])
request_docs = self.db_handler.view(self.db_name,
"ReqMgr", "bydate",
options=options)
return rows([request_docs])
def delete(self, group_name):
"""
Removes an existing group from the database, raises an error
if group_name doesn't exist.
"""
groups = self.reqmgr_aux_db.document("groups")
if group_name in groups:
del groups[group_name]
# TODO
# this should ideally also wrap try-except
self.reqmgr_aux_db.commitOne(groups)
return rows(["OK"])
else:
msg = "ERROR: Group '%s' not found in the database." % group_name
cherrypy.log(msg)
raise cherrypy.HTTPError(404, msg)
def delete(self, team_name):
"""
Removes an existing team from the database, raises an error
if team_name doesn't exist.
"""
teams = self.reqmgr_aux_db.document("teams")
if team_name in teams:
del teams[team_name]
# TODO
# this should ideally also wrap try-except
self.reqmgr_aux_db.commitOne(teams)
return rows(["OK"])
else:
msg = "ERROR: Team '%s' not found in the database." % team_name
cherrypy.log(msg)
raise cherrypy.HTTPError(404, msg)
def get(self, subName=None):
"""
Return entire self.name document
subName is subcategory of document which is added as postfix string
"""
try:
if subName:
docName = "%s_%s" % (self.name, subName)
else:
docName = self.name
sw = self.reqmgr_aux_db.document(docName)
del sw["_id"]
del sw["_rev"]
except CouchNotFoundError:
raise NoSuchInstance
return rows([sw])
def modifynocheck(self, sql, *binds, **kwbinds):
"""This is the same as `WMCore.REST.Server`:modify method but
not implementing any kind of checks on the number of modified
rows.
:arg str sql: SQL modify statement.
:arg list binds: Bind variables by position: list of dictionaries.
:arg dict kwbinds: Bind variables by keyword: dictionary of lists.
:result: See :meth:`rowstatus` and description in `WMCore.REST.Server`."""
if binds:
c, _ = self.executemany(sql, *binds, **kwbinds)
else:
kwbinds = self.bindmap(**kwbinds)
c, _ = self.executemany(sql, kwbinds, *binds)
trace = cherrypy.request.db["handle"]["trace"]
trace and cherrypy.log("%s commit" % trace)
cherrypy.request.db["handle"]["connection"].commit()
return rows([{ "modified": c.rowcount }])
def get(self):
"""Return information on the calling user.
The user description contains the following fields. All the information
is always present, regardless of authentication method. Data not known
will be null, for example if access is made using CMS VO X509 certificate
which is not registered in SiteDB, much of the information will be empty.
``name``
The full name.
``login``
CERN account.
``dn``
X509 certificate distinguished name.
``method``
Authentication method, one of ``X509Cert``, ``X509Proxy``, ``HNLogin``,
``HostIP``, ``AUCookie`` or ``None``. In practice for SiteDB it will
only be one of the first two since the latter three are not allowed
authentication methods for SiteDB and this REST entity.
``roles``
A dictionary of authorisation roles possessed by the user. For each
role the person has, there will be a key-value pair where the key is
the *canonical* role name, and the value is another dictionary with
keys ``group`` and ``site``, each of whose value is a list. The lists
will contain the *canonical* group and site names for which the role
applies, respectively. Canonical names are all lower-case, with all
word delimiters replaced with a single dash. For example the canonical
role title for "Global Admin" is "global-admin", and for the site
"T1\_CH\_CERN" it is "t1-ch-cern".
:returns: sequence of one dictionary which describes the user."""
user = cherrypy.request.user
for authz in user['roles'].values():
for k in ('site', 'group'):
authz[k] = [x for x in authz[k]]
return rows([cherrypy.request.user])
def put(self, site_name, type, fqdn, is_primary):
"""Insert new site resources. Site executive / admin can update their own
site, global admin can update resources for all the sites. When more than
one argument is given, there must be an equal number of arguments for all
the parameters. For input validation requirements, see the field
descriptions above. It is an error to attemp to insert an existing site
resource.
:arg list site_name: new values;
:arg list type: new values;
:arg list fqdn: new values;
:arg list is_primary: new values;
:returns: a list with a dict in which *modified* gives the number of objects
inserted into the database, which is always *len(site_name).*"""
self._authz(site_name)
# Update both the new and the old table to be compatible with v1.
# The old one could be withdraw once v1 gets fully deprecated.
binds = self.api.bindmap(site_name = site_name, type = type,
fqdn = fqdn, is_primary = is_primary)
c, _ = self.api.executemany("""
insert all
into resource_element (id, site, fqdn, type, is_primary)
values (resource_element_sq.nextval, site_id, :fqdn, :type, :is_primary)
into resource_cms_name_map (resource_id, cms_name_id)
values (resource_element_sq.nextval, cms_name_id)
select s.id site_id, ss.cms_name_id cms_name_id
from site s join site_cms_name_map ss on s.id = ss.site_id
where s.name = :site_name
""", binds)
self.api.rowstatus(c, 2*len(binds))
result = rows([{ "modified": c.rowcount / 2 }])
trace = request.db["handle"]["trace"]
trace and cherrypy.log("%s commit" % trace)
request.db["handle"]["connection"].commit()
return result
def get(self, **kwargs):
"""
Returns request info depending on the conditions set by kwargs
Currently defined kwargs are following.
statusList, requestNames, requestType, prepID, inputDataset, outputDataset, dateRange
If jobInfo is True, returns jobInfomation about the request as well.
TODO:
stuff like this has to filtered out from result of this call:
_attachments: {u'spec': {u'stub': True, u'length': 51712, u'revpos': 2, u'content_type': u'application/json'}}
_id: maxa_RequestString-OVERRIDE-ME_130621_174227_9225
_rev: 4-c6ceb2737793aaeac3f1cdf591593da4
"""
if len(kwargs) == 0:
kwargs['status'] = "running"
options = {"descending": True, 'include_docs': True, 'limit': 200}
request_docs = self.reqmgr_db.loadView("ReqMgr", "bystatus",
options)
return rows([request_docs])
# list of status
status = kwargs.get("status", False)
# list of request names
name = kwargs.get("name", False)
request_type = kwargs.get("request_type", False)
prep_id = kwargs.get("prep_id", False)
inputdataset = kwargs.get("inputdataset", False)
outputdataset = kwargs.get("outputdataset", False)
date_range = kwargs.get("date_range", False)
campaign = kwargs.get("campaign", False)
workqueue = kwargs.get("workqueue", False)
team = kwargs.get("team", False)
# eventhing should be stale view. this only needs for test
_nostale = kwargs.get("_nostale", False)
option = {}
if _nostale:
self.reqmgr_db_service._setNoStale()
request_info = []
if status and not team and not request_type:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"bystatus", option, status))
if status and team:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"byteamandstatus", option, [[team, status]]))
if status and request_type:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"byteamandstatus", option, [[team, status]]))
if name:
request_info.append(self.reqmgr_db_service.getRequestByNames(name))
if prep_id:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"byprepid", option, prep_id))
if inputdataset:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"byinputdataset", option, inputdataset))
if outputdataset:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"byoutputdataset", option, outputdataset))
if date_range:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"bydate", option, date_range))
if campaign:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"bycampaign", option, campaign))
if workqueue:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"byworkqueue", option, workqueue))
#get interaction of the request
result = self._intersection_of_request_info(request_info)
if len(result) == 0:
return []
return rows([result])
def get(self):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows(DataCache.filterData(ACTIVE_NO_CLOSEOUT_FILTER,
["InputDataset", "OutputDatasets", "MCPileup", "DataPileup"]))
def get(self, request_name):
result = self.wmstats.getRequestSummaryWithJobInfo(request_name)
return rows([result])
groups = self.reqmgr_aux_db.document("groups")
except CouchNotFoundError, ex:
msg = ("ERROR: Retrieving groups document failed, reason: %s"
" Creating the document ..." % ex)
cherrypy.log(msg)
try:
doc = Document(id="groups", inputDict={group_name: None})
self.reqmgr_aux_db.commitOne(doc)
return
except CouchError, ex:
msg = "ERROR: Creating document groups failed, reason: %s" % ex
cherrypy.log(msg)
raise cherrypy.HTTPError(400, msg)
if group_name in groups:
return rows(["Already exists."])
else:
groups[group_name] = None
# TODO
# this should ideally also wrap try-except
self.reqmgr_aux_db.commitOne(groups)
return rows(["OK"])
class Team(RESTEntity):
"""
Teams are stored in the ReqMgr reqmgr_auxiliary database.
"teams" is used as id of the document, but the document
itself has to be JSON: we use {"teams": [list, of, group, names]
When request goes through assignment status, it gets assigned
def get(self, doc_name):
"""
"""
config = ReqMgrConfigDataCache.getConfig(doc_name)
return rows([config])
def get(self, **kwargs):
"""
Returns request info depending on the conditions set by kwargs
Currently defined kwargs are following.
statusList, requestNames, requestType, prepID, inputDataset, outputDataset, dateRange
If jobInfo is True, returns jobInfomation about the request as well.
TODO:
stuff like this has to masked out from result of this call:
_attachments: {u'spec': {u'stub': True, u'length': 51712, u'revpos': 2, u'content_type': u'application/json'}}
_id: maxa_RequestString-OVERRIDE-ME_130621_174227_9225
_rev: 4-c6ceb2737793aaeac3f1cdf591593da4
"""
# list of status
status = kwargs.get("status", [])
# list of request names
name = kwargs.get("name", [])
request_type = kwargs.get("request_type", [])
prep_id = kwargs.get("prep_id", [])
inputdataset = kwargs.get("inputdataset", [])
outputdataset = kwargs.get("outputdataset",[])
date_range = kwargs.get("date_range", False)
campaign = kwargs.get("campaign", [])
workqueue = kwargs.get("workqueue", [])
team = kwargs.get("team", [])
mc_pileup = kwargs.get("mc_pileup", [])
data_pileup = kwargs.get("data_pileup", [])
requestor = kwargs.get("requestor", [])
mask = kwargs.get("mask", [])
detail = kwargs.get("detail", True)
# set the return format. default format has requset name as a key
# if is set to one it returns list of dictionary with RequestName field.
common_dict = int(kwargs.get("common_dict", 0))
if detail in (False, "false", "False", "FALSE"):
option = {"include_docs": False}
else:
option = {"include_docs": True}
# eventhing should be stale view. this only needs for test
_nostale = kwargs.get("_nostale", False)
if _nostale:
self.reqmgr_db_service._setNoStale()
request_info = []
if len(status) == 1 and status[0] == "ACTIVE":
status = ACTIVE_STATUS
if status and not team and not request_type and not requestor:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bystatus", option, status))
if status and team:
query_keys = [[t, s] for t in team for s in status]
request_info.append(
self.reqmgr_db_service.getRequestByCouchView("byteamandstatus", option, query_keys))
if status and request_type:
query_keys = [[s, rt] for rt in request_type for s in status]
request_info.append(self.reqmgr_db_service.getRequestByCouchView("requestsbystatusandtype",
option, query_keys))
if status and requestor:
query_keys = [[s, r] for r in requestor for s in status]
request_info.append(
self.reqmgr_db_service.getRequestByCouchView("bystatusandrequestor", option, query_keys))
if name:
request_info.append(self.reqmgr_db_service.getRequestByNames(name))
if prep_id:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("byprepid", option, prep_id))
if inputdataset:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("byinputdataset", option, inputdataset))
if outputdataset:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("byoutputdataset", option, outputdataset))
if date_range:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bydate", option, date_range))
if campaign:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bycampaign", option, campaign))
if workqueue:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("byworkqueue", option, workqueue))
if mc_pileup:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bymcpileup", option, mc_pileup))
if data_pileup:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bydatapileup", option, data_pileup))
# get interaction of the request
result = self._intersection_of_request_info(request_info)
if len(result) == 0:
return []
result = self._mask_result(mask, result)
# If detail is set to False return just list of request name
if not option["include_docs"]:
return result.keys()
if common_dict == 1:
response_list = result.values()
else:
response_list = [result]
return rows(response_list)
def get(self, **kwargs):
"""
Returns request info depending on the conditions set by kwargs
Currently defined kwargs are following.
statusList, requestNames, requestType, prepID, inputDataset, outputDataset, dateRange
If jobInfo is True, returns jobInfomation about the request as well.
TODO:
stuff like this has to masked out from result of this call:
_attachments: {u'spec': {u'stub': True, u'length': 51712, u'revpos': 2, u'content_type': u'application/json'}}
_id: maxa_RequestString-OVERRIDE-ME_130621_174227_9225
_rev: 4-c6ceb2737793aaeac3f1cdf591593da4
"""
### pop arguments unrelated to the user query
mask = kwargs.pop("mask", [])
detail = kwargs.pop("detail", True)
common_dict = int(kwargs.pop("common_dict", 0)) # modifies the response format
nostale = kwargs.pop("_nostale", False)
### these are the query strings supported by this API
status = kwargs.get("status", [])
name = kwargs.get("name", [])
request_type = kwargs.get("request_type", [])
prep_id = kwargs.get("prep_id", [])
inputdataset = kwargs.get("inputdataset", [])
outputdataset = kwargs.get("outputdataset", [])
date_range = kwargs.get("date_range", False)
campaign = kwargs.get("campaign", [])
team = kwargs.get("team", [])
mc_pileup = kwargs.get("mc_pileup", [])
data_pileup = kwargs.get("data_pileup", [])
requestor = kwargs.get("requestor", [])
# further tweaks to the couch queries
if len(status) == 1 and status[0] == "ACTIVE":
status = ACTIVE_STATUS
if detail in (False, "false", "False", "FALSE"):
option = {"include_docs": False}
else:
option = {"include_docs": True}
# everything should be stale view. this only needs for test
if nostale:
self.reqmgr_db_service._setNoStale()
request_info = []
queryMatched = False # flag to avoid calling the same view twice
if len(kwargs) == 2:
if status and team:
query_keys = [[t, s] for t in team for s in status]
request_info.append(self.reqmgr_db_service.getRequestByCouchView("byteamandstatus",
option, query_keys))
queryMatched = True
elif status and request_type:
query_keys = [[s, rt] for rt in request_type for s in status]
request_info.append(self.reqmgr_db_service.getRequestByCouchView("requestsbystatusandtype",
option, query_keys))
queryMatched = True
elif status and requestor:
query_keys = [[s, r] for r in requestor for s in status]
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bystatusandrequestor",
option, query_keys))
queryMatched = True
elif len(kwargs) == 3:
if status and request_type and requestor:
query_keys = [[s, rt, req] for s in status for rt in request_type for req in requestor]
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bystatusandtypeandrequestor",
option, query_keys))
queryMatched = True
# anything else that hasn't matched the query combination above
if not queryMatched:
if status:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bystatus",
option, status))
if name:
request_info.append(self.reqmgr_db_service.getRequestByNames(name))
if request_type:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bytype",
option, request_type))
if prep_id:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("byprepid",
option, prep_id))
if inputdataset:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("byinputdataset",
option, inputdataset))
if outputdataset:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("byoutputdataset",
option, outputdataset))
if date_range:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bydate",
option, date_range))
if campaign:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bycampaign",
option, campaign))
if mc_pileup:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bymcpileup",
option, mc_pileup))
if data_pileup:
request_info.append(self.reqmgr_db_service.getRequestByCouchView("bydatapileup",
option, data_pileup))
# get the intersection of the request data
result = self._intersection_of_request_info(request_info)
if not result:
return []
result = self._maskResult(mask, result)
if not option["include_docs"]:
return list(result)
# set the return format. default format has request name as a key
# if is set to one it returns list of dictionary with RequestName field.
if common_dict == 1:
response_list = listvalues(result)
else:
response_list = [result]
return rows(response_list)
def get(self):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows(DataCache.getProtectedLFNs())
def get(self):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows([DataCache.getlatestJobData()])
def get(self):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows(DataCache.getProtectedLFNs())
def get(self, mask=None, **input_condition):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows(DataCache.filterDataByRequest(input_condition, mask))
def get(self):
return rows(['foo'])
def get(self):
result = self.wmstats.agentsByTeam(filterDrain=False)
return rows(result.keys())
def get(self, mask=None, **input_condition):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows(DataCache.filterDataByRequest(input_condition, mask))
def delete(self, type, site_name, alias):
"""Delete site name associations. Only Global admins and SiteDB operators,
can delete names for the sites. When more than one
argument is given, there must be an equal number of arguments for all
the parameters. For input validation requirements, see the field
descriptions above. It is an error to attempt to delete a non-existent
site name association.
:arg list type: values to delete;
:arg list site_name: values to delete;
:arg list alias: values to delete;
:returns: a list with a dict in which *modified* gives the number of objects
deleted from the database, which is always *len(type).*"""
binds = self.api.bindmap(type=type, site_name=site_name, alias=alias)
lcg = filter(lambda b: b['type'] == 'lcg', binds)
cms = filter(lambda b: b['type'] == 'cms', binds)
phedex = filter(lambda b: b['type'] == 'phedex', binds)
psn = filter(lambda b: b['type'] == 'psn', binds)
for b in binds:
del b['type']
updated = 0
if cms:
c, _ = self.api.executemany(
"""
delete from cms_name
where id in (select cmap.cms_name_id
from site s
join site_cms_name_map cmap on cmap.site_id = s.id
where s.name = :site_name)
and name = :alias
""", cms)
self.api.rowstatus(c, len(cms))
updated += c.rowcount
if lcg:
c, _ = self.api.executemany(
"""
delete from sam_name
where id in (select smap.sam_id
from site s
join site_cms_name_map cmap on cmap.site_id = s.id
join sam_cms_name_map smap on smap.cms_name_id = cmap.cms_name_id
where s.name = :site_name)
and name = :alias
""", lcg)
self.api.rowstatus(c, len(lcg))
updated += c.rowcount
if phedex:
c, _ = self.api.executemany(
"""
delete from phedex_node
where site = (select s.id from site s where s.name = :site_name)
and name = :alias
""", phedex)
self.api.rowstatus(c, len(phedex))
updated += c.rowcount
if psn:
c, _ = self.api.executemany(
"""
delete from psn_node
where site = (select s.id from site s where s.name = :site_name)
and name = :alias
""", psn)
self.api.rowstatus(c, len(psn))
updated += c.rowcount
result = rows([{"modified": updated}])
trace = request.db["handle"]["trace"]
trace and cherrypy.log("%s commit" % trace)
request.db["handle"]["connection"].commit()
return result
def get(self):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows(DataCache.filterData(ACTIVE_STATUS_FILTER, ["OutputModulesLFNBases"]))
def put(self, type, site_name, alias):
"""Insert new site names. Only global admins and SiteDB operators can
update names for the sites. When more than one argument is
given, there must be an equal number of arguments for all the parameters.
For input validation requirements, see the field descriptions above.
It is an error to attempt to insert an existing name alias triplet.
:arg list type: new values;
:arg list site_name: new values;
:arg list alias: new values;
:returns: a list with a dict in which *modified* gives the number of objects
inserted into the database, which is always *len(type).*"""
binds = self.api.bindmap(type=type, site_name=site_name, alias=alias)
lcg = filter(lambda b: b['type'] == 'lcg', binds)
cms = filter(lambda b: b['type'] == 'cms', binds)
phedex = filter(lambda b: b['type'] == 'phedex', binds)
psn = filter(lambda b: b['type'] == 'psn', binds)
for b in binds:
del b['type']
updated = 0
if cms:
c, _ = self.api.executemany(
"""
insert all
into cms_name (id, name) values (cms_name_sq.nextval, alias)
into site_cms_name_map (site_id, cms_name_id) values (site_id, cms_name_sq.nextval)
select s.id site_id, :alias alias
from site s where s.name = :site_name
""", cms)
self.api.rowstatus(c, 2 * len(cms))
updated += c.rowcount / 2
if lcg:
c, _ = self.api.executemany(
"""
insert all
into sam_name (id, name) values (sam_name_sq.nextval, alias)
into sam_cms_name_map (cms_name_id, sam_id) values (cms_id, sam_name_sq.nextval)
select cmap.cms_name_id cms_id, :alias alias
from site s join site_cms_name_map cmap on cmap.site_id = s.id
where s.name = :site_name
""", lcg)
self.api.rowstatus(c, 2 * len(lcg))
updated += c.rowcount / 2
if phedex:
c, _ = self.api.executemany(
"""
insert into phedex_node (id, site, name)
select phedex_node_sq.nextval, s.id, :alias
from site s where s.name = :site_name
""", phedex)
self.api.rowstatus(c, len(phedex))
updated += c.rowcount
if psn:
c, _ = self.api.executemany(
"""
insert into psn_node (id, site, name)
select psn_node_sq.nextval, s.id, :alias
from site s where s.name = :site_name
""", psn)
self.api.rowstatus(c, len(psn))
updated += c.rowcount
result = rows([{"modified": updated}])
trace = request.db["handle"]["trace"]
trace and cherrypy.log("%s commit" % trace)
request.db["handle"]["connection"].commit()
return result
def get(self):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows([DataCache.getlatestJobData()])
def get(self):
return rows(REQUEST_TYPES)
def get(self, request_name):
result = self.wmstats.getRequestSummaryWithJobInfo(request_name)
return rows([result])
def get(self):
return rows(REQUEST_TYPES)
def get(self, request_name, sample_size):
result = self.wmstats.getTaskJobSummaryByRequest(request_name, sample_size)
return rows([result])
def get(self, **kwargs):
"""
Returns request info depending on the conditions set by kwargs
Currently defined kwargs are following.
statusList, requestNames, requestType, prepID, inputDataset, outputDataset, dateRange
If jobInfo is True, returns jobInfomation about the request as well.
TODO:
stuff like this has to masked out from result of this call:
_attachments: {u'spec': {u'stub': True, u'length': 51712, u'revpos': 2, u'content_type': u'application/json'}}
_id: maxa_RequestString-OVERRIDE-ME_130621_174227_9225
_rev: 4-c6ceb2737793aaeac3f1cdf591593da4
"""
# list of status
status = kwargs.get("status", [])
# list of request names
name = kwargs.get("name", [])
request_type = kwargs.get("request_type", [])
prep_id = kwargs.get("prep_id", [])
inputdataset = kwargs.get("inputdataset", [])
outputdataset = kwargs.get("outputdataset", [])
date_range = kwargs.get("date_range", False)
campaign = kwargs.get("campaign", [])
workqueue = kwargs.get("workqueue", [])
team = kwargs.get("team", [])
mc_pileup = kwargs.get("mc_pileup", [])
data_pileup = kwargs.get("data_pileup", [])
requestor = kwargs.get("requestor", [])
mask = kwargs.get("mask", [])
detail = kwargs.get("detail", True)
# set the return format. default format has requset name as a key
# if is set to one it returns list of dictionary with RequestName field.
common_dict = int(kwargs.get("common_dict", 0))
if detail in (False, "false", "False", "FALSE"):
option = {"include_docs": False}
else:
option = {"include_docs": True}
# eventhing should be stale view. this only needs for test
_nostale = kwargs.get("_nostale", False)
if _nostale:
self.reqmgr_db_service._setNoStale()
request_info = []
if len(status) == 1 and status[0] == "ACTIVE":
status = ACTIVE_STATUS
if status and not team and not request_type and not requestor:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"bystatus", option, status))
if status and team:
query_keys = [[t, s] for t in team for s in status]
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"byteamandstatus", option, query_keys))
if status and request_type:
query_keys = [[s, rt] for rt in request_type for s in status]
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"requestsbystatusandtype", option, query_keys))
if status and requestor:
query_keys = [[s, r] for r in requestor for s in status]
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"bystatusandrequestor", option, query_keys))
if name:
request_info.append(self.reqmgr_db_service.getRequestByNames(name))
if prep_id:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"byprepid", option, prep_id))
if inputdataset:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"byinputdataset", option, inputdataset))
if outputdataset:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"byoutputdataset", option, outputdataset))
if date_range:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"bydate", option, date_range))
if campaign:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"bycampaign", option, campaign))
if workqueue:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"byworkqueue", option, workqueue))
if mc_pileup:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"bymcpileup", option, mc_pileup))
if data_pileup:
request_info.append(
self.reqmgr_db_service.getRequestByCouchView(
"bydatapileup", option, data_pileup))
# get interaction of the request
result = self._intersection_of_request_info(request_info)
if len(result) == 0:
return []
result = self._mask_result(mask, result)
# If detail is set to False return just list of request name
if not option["include_docs"]:
return result.keys()
if common_dict == 1:
response_list = result.values()
else:
response_list = [result]
return rows(response_list)
def get(self):
results = DataCache.getlatestJobData()
if results == None or DataCache.islatestJobDataExpired():
results = self.wmstats.getActiveData(jobInfoFlag=True)
return rows([results])
def get(self, doc_name):
"""
"""
config = ReqMgrConfigDataCache.getConfig(doc_name)
return rows([config])
