def kill(self, workflow, force, jobids, killwarning, userdn, userproxy=None):
"""Request to Abort a workflow.
:arg str workflow: a workflow name"""
retmsg = "ok"
self.logger.info("About to kill workflow: %s. Getting status first.", workflow)
row = self.api.query(None, None, self.Task.ID_sql, taskname = workflow)
try:
#just one row is picked up by the previous query
row = self.Task.ID_tuple(*next(row))
except StopIteration:
raise ExecutionError("Impossible to find task %s in the database." % workflow)
warnings = literal_eval(row.task_warnings.read() if row.task_warnings else '[]') #there should actually is a default, but just in case
if killwarning:
warnings += [killwarning]
warnings = str(warnings)
args = {'ASOURL' : getattr(row, 'asourl', '')}
if row.task_status in ['SUBMITTED', 'KILLFAILED', 'RESUBMITFAILED', 'FAILED', 'KILLED', 'TAPERECALL']:
args.update({"killList": jobids})
#Set arguments first so in case of failure we don't do any "damage"
self.api.modify(self.Task.SetArgumentsTask_sql, taskname = [workflow], arguments = [str(args)])
self.api.modify(self.Task.SetStatusWarningTask_sql, status = ["NEW"], command = ["KILL"], taskname = [workflow], warnings = [str(warnings)])
elif row.task_status == 'NEW' and row.task_command == 'SUBMIT':
#if the task has just been submitted and not acquired by the TW
self.api.modify(self.Task.SetStatusWarningTask_sql, status = ["KILLED"], command = ["KILL"], taskname = [workflow], warnings = [str(warnings)])
else:
raise ExecutionError("You cannot kill a task if it is in the %s status" % row.task_status)
return [{"result":retmsg}]
def _getAsoConfig(self, asourl, asodb):
""" Figures out which asourl and asodb to use. Here some rules:
1) If ASOURL is set up in the client then use it. If ASODB is None then default to 'asynctransfer' (for old clients).
If ASODB is defined but ASOURL is not, then give an error (somebody is doing it wrong!).
2) If ASOURL is not defined in the client then look at the external configuration of the server which looks like:
...
"ASOURL" : "https://cmsweb.cern.ch/couchdb",
"asoConfig" : [
{"couchURL" : "https://cmsweb.cern.ch/couchdb", "couchDBName" : "asynctransfer"},
{"couchURL" : "https://cmsweb-testbed.cern.ch/couchdb2", "couchDBName" : "asynctransfer1"}
]
...
2.1) The external configuration contains asoConfig (the new and default thing now):
Pick up a random element from the list and figures out the asourl and asodb.
No validation of the configuration is done,
we assume asocConfig is a list that contains dicts (at least one),
and each dict has both couchURL and couchDBName.
Documentation about the external conf is here:
https://twiki.cern.ch/twiki/bin/view/CMSPublic/CMSCrabRESTInterface#External_dynamic_configuration
2.2) Else if the external configuration contains the old key ASOURL:
ASOURL could either be a string (the value for asourl that we need to use) or a list (in which case
we will use a random element)
3) If asourl is not defined in the client nor in the external config give an error.
"""
ASYNC_DEFAULT_DB = 'asynctransfer'
#1) The user is trying to pass something to us
if asourl:
self.logger.info("ASO url and database defined in the client configuration")
if not asodb:
asodb = ASYNC_DEFAULT_DB
if asodb and not asourl:
raise ExecutionError("You set up Debug.ASODB but you did not set up Debug.ASOURL. Please specify ASOURL as well or remove ASODB.")
#2) We need to figure out the values ourselves
if not asourl: #just checking asourl here because either both asourl and asodb are set, or neither are
extconf = self.centralcfg.centralconfig.get("backend-urls", {})
#2.1) Get asourl and asodb from the new extrnal configuration (that's what we usually do for most of the users)
if 'asoConfig' in extconf:
asoconf = random.choice(extconf['asoConfig'])
asourl = asoconf['couchURL']
asodb = asoconf['couchDBName']
#2.2) No asoConfig defined, let's look for the old ASOURL.
elif 'ASOURL' in extconf:
msg = "You are using the old ASOURL parameter in your external configuration, please use asoConfig instead."
msg += "\nSee https://twiki.cern.ch/twiki/bin/view/CMSPublic/CMSCrabRESTInterface#External_dynamic_configuration"
self.logger.warning(msg)
if isinstance(extconf['ASOURL'], list):
asourl = random.choice(asourl)
else:
asourl = extconf['ASOURL']
asodb = ASYNC_DEFAULT_DB
#3) Give an error if we cannot figure out aso configuration
if not (asourl and asodb):
raise ExecutionError("The server configuration is wrong (asoConfig missing): cannot figure out ASO urls.")
return asourl, asodb
def __init__(self, app, config, mount):
DatabaseRESTApi.__init__(self, app, config, mount)
self.formats = [('application/json', JSONFormat())]
status, serverdn = getstatusoutput(
'openssl x509 -noout -subject -in %s | cut -f2- -d\ ' %
config.serverhostcert)
if status is not 0:
raise ExecutionError(
"Internal issue when retrieving crabserver service DN.")
hbuf = StringIO.StringIO()
bbuf = StringIO.StringIO()
curl = pycurl.Curl()
curl.setopt(pycurl.URL, config.extconfigurl)
curl.setopt(pycurl.WRITEFUNCTION, bbuf.write)
curl.setopt(pycurl.HEADERFUNCTION, hbuf.write)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.perform()
curl.close()
header = ResponseHeader(hbuf.getvalue())
if header.status < 200 or header.status >= 300:
cherrypy.log("Problem %d reading from %s." %
(config.extconfigurl, header.status))
raise ExecutionError(
"Internal issue when retrieving external confifuration")
extconfig = json.decode(bbuf.getvalue())
#Global initialization of Data objects. Parameters coming from the config should go here
DataUserWorkflow.globalinit(config.workflowManager)
DataWorkflow.globalinit(dbapi=self, phedexargs={'endpoint': config.phedexurl}, dbsurl=config.dbsurl,\
credpath=config.credpath, transformation=config.transformation)
DataFileMetadata.globalinit(dbapi=self)
Utils.globalinit(config.serverhostkey, config.serverhostcert, serverdn,
config.credpath)
## TODO need a check to verify the format depending on the resource
## the RESTFileMetadata has the specifc requirement of getting xml reports
self._add({
'workflow':
RESTUserWorkflow(app, self, config, mount),
'campaign':
RESTCampaign(app, self, config, mount),
'info':
RESTServerInfo(app, self, config, mount, serverdn,
extconfig.get('delegate-dn', [])),
'filemetadata':
RESTFileMetadata(app, self, config, mount),
})
self._initLogger(getattr(config, 'loggingFile', None),
getattr(config, 'loggingLevel', None))
def webdirprx(self, **kwargs):
""" Returns the proxied url for the schedd if the schedd has any, returns an empty list instead. Raises in case of other errors.
To test it use:
curl -X GET 'https://mmascher-dev6.cern.ch/crabserver/dev/task?subresource=webdirprx&workflow=150224_230633:mmascher_crab_testecmmascher-dev6_3'\
-k --key /tmp/x509up_u8440 --cert /tmp/x509up_u8440 -v
"""
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter(
"Task name not found in the input parameters")
workflow = kwargs['workflow']
self.logger.info("Getting proxied url for %s" % workflow)
try:
row = self.Task.ID_tuple(*next(
self.api.query(None, None, self.Task.ID_sql,
taskname=workflow)))
except StopIteration:
raise ExecutionError(
"Impossible to find task %s in the database." %
kwargs["workflow"])
if row.user_webdir:
#extract /cms1425/taskname from the user webdir
suffix = re.search(r"(/[^/]+/[^/]+/?)$", row.user_webdir).group(0)
else:
raise ExecutionError(
"Webdir not set in the database. Cannot build proxied webdir")
#=============================================================================
# scheddObj is a dictionary composed like this (see the value of htcondorSchedds):
# "htcondorSchedds": {
# "*****@*****.**": {
# "proxiedurl": "https://cmsweb.cern.ch/scheddmon/5"
# },
# ...
# }
# so that they have a "proxied URL" to be used in case the schedd is
# behind a firewall.
#=============================================================================
scheddsObj = self.centralcfg.centralconfig['backend-urls'].get(
'htcondorSchedds', {})
self.logger.info(
"ScheddObj for task %s is: %s\nSchedd used for submission %s" %
(workflow, scheddsObj, row.schedd))
#be careful that htcondorSchedds could be a list (backward compatibility). We might want to remove this in the future
if row.schedd in list(scheddsObj) and isinstance(scheddsObj, dict):
self.logger.debug("Found schedd %s" % row.schedd)
proxiedurlbase = scheddsObj[row.schedd].get('proxiedurl')
self.logger.debug("Proxied url base is %s" % proxiedurlbase)
if proxiedurlbase:
yield proxiedurlbase + suffix
else:
self.logger.info("Could not determine proxied url for task %s" %
workflow)
def kill(self, workflow, force, jobids, userdn, userproxy=None):
"""Request to Abort a workflow.
:arg str workflow: a workflow name"""
retmsg = "ok"
self.logger.info("About to kill workflow: %s. Getting status first." %
workflow)
statusRes = self.status(workflow, userdn, userproxy)[0]
args = {'ASOURL': statusRes.get("ASOURL", "")}
# Hm...
dbSerializer = str
if statusRes['status'] in [
'SUBMITTED', 'KILLFAILED', 'RESUBMITFAILED', 'FAILED'
]:
killList = [
jobid for jobstatus, jobid in statusRes['jobList']
if jobstatus not in self.successList
]
if jobids:
#if the user wants to kill specific jobids make the intersection
killList = list(set(killList) & set(jobids))
#check if all the requested jobids can be resubmitted
if len(killList) != len(jobids):
retmsg = "Cannot request kill for %s" % (set(jobids) -
set(killList))
if not killList:
raise ExecutionError(
"There are no jobs to kill. Only jobs not in %s states can be killed"
% self.successList)
self.logger.info("Jobs to kill: %s" % killList)
args.update({"killList": killList, "killAll": jobids == []})
self.api.modify(self.Task.SetStatusTask_sql,
status=["KILL"],
taskname=[workflow])
self.api.modify(self.Task.SetArgumentsTask_sql,
taskname=[workflow],
arguments=[dbSerializer(args)])
elif statusRes['status'] == 'NEW':
self.api.modify(self.Task.SetStatusTask_sql,
status=["KILLED"],
taskname=[workflow])
else:
raise ExecutionError(
"You cannot kill a task if it is in the %s state" %
statusRes['status'])
return [{"result": retmsg}]
def resubmit(self, workflow, siteblacklist, sitewhitelist, jobids, userdn,
userproxy):
"""Request to reprocess what the workflow hasn't finished to reprocess.
This needs to create a new workflow in the same campaign
:arg str workflow: a valid workflow name
:arg str list siteblacklist: black list of sites, with CMS name;
:arg str list sitewhitelist: white list of sites, with CMS name."""
retmsg = "ok"
self.logger.info(
"About to resubmit workflow: %s. Getting status first." % workflow)
statusRes = self.status(workflow, userdn, userproxy)[0]
#if there are failed jobdef submission we fail
if statusRes['failedJobdefs']:
raise ExecutionError(
"You cannot resubmit a task if not all the jobs have been submitted. The feature will be available in the future"
)
if statusRes['status'] in ['SUBMITTED', 'KILLED', 'FAILED']:
resubmitList = [
jobid for jobstatus, jobid in statusRes['jobList']
if jobstatus in self.failedList
]
if jobids:
#if the user wants to kill specific jobids make the intersection
resubmitList = list(set(resubmitList) & set(jobids))
#check if all the requested jobids can be resubmitted
if len(resubmitList) != len(jobids):
retmsg = "Cannot request resubmission for %s" % (
set(jobids) - set(resubmitList))
if not resubmitList:
raise ExecutionError(
"There are no jobs to resubmit. Only jobs in %s states are resubmitted"
% self.failedList)
self.logger.info("Jobs to resubmit: %s" % resubmitList)
self.api.modify(SetStatusTask.sql,
status=["RESUBMIT"],
taskname=[workflow])
self.api.modify(SetArgumentsTask.sql, taskname = [workflow],\
arguments = [str({"siteBlackList":siteblacklist, "siteWhiteList":sitewhitelist, "resubmitList":resubmitList})])
else:
raise ExecutionError(
"You cannot resubmit a task if it is in the %s state" %
statusRes['status'])
return [{"result": retmsg}]
def kill(self, workflow, force, userdn, userproxy=None):
"""Request to Abort a workflow.
:arg str workflow: a workflow name"""
self.logger.info("About to kill workflow: %s. Getting status first." %
workflow)
statusRes = self.status(workflow, userdn, userproxy)[0]
if statusRes['status'] == 'SUBMITTED':
killList = [
jobid for jobstatus, jobid in statusRes['jobList']
if jobstatus not in self.successList
]
self.logger.info("Jobs to kill: %s" % killList)
self.api.modify(SetStatusTask.sql,
status=["KILL"],
taskname=[workflow])
self.api.modify(SetArgumentsTask.sql, taskname = [workflow],\
arguments = [str({"killList": killList, "killAll": True})])
elif statusRes['status'] == 'NEW':
self.api.modify(SetStatusTask.sql,
status=["KILLED"],
taskname=[workflow])
else:
raise ExecutionError(
"You cannot kill a task if it is in the %s state" %
statusRes['status'])
def resubmit(self, workflow, siteblacklist, sitewhitelist, userdn,
userproxy):
"""Request to reprocess what the workflow hasn't finished to reprocess.
This needs to create a new workflow in the same campaign
:arg str workflow: a valid workflow name
:arg str list siteblacklist: black list of sites, with CMS name;
:arg str list sitewhitelist: white list of sites, with CMS name."""
self.logger.info(
"About to resubmit workflow: %s. Getting status first." % workflow)
statusRes = self.status(workflow, userdn, userproxy)[0]
if statusRes['status'] in ['SUBMITTED', 'KILLED']:
resubmitList = [
jobid for jobstatus, jobid in statusRes['jobList']
if jobstatus in self.failedList
]
self.logger.info("Jobs to resubmit: %s" % resubmitList)
self.api.modify(SetStatusTask.sql,
status=["RESUBMIT"],
taskname=[workflow])
self.api.modify(SetArgumentsTask.sql, taskname = [workflow],\
arguments = [str({"siteBlackList":siteblacklist, "siteWhiteList":sitewhitelist, "resubmitList":resubmitList})])
else:
raise ExecutionError(
"You cannot resubmit a task if it is in the %s state" %
statusRes['status'])
def stream_chunked(self, stream, etag, preamble, trailer):
"""Generator for actually producing the output."""
comma = " "
try:
if preamble:
etag.update(preamble)
yield preamble
try:
for obj in stream:
chunk = comma + json.dumps(obj, indent=2)
etag.update(chunk)
yield chunk
comma = ","
except GeneratorExit:
etag.invalidate()
trailer = None
raise
finally:
if trailer:
etag.update(trailer)
yield trailer
cherrypy.response.headers["X-REST-Status"] = 100
except RESTError as e:
etag.invalidate()
report_rest_error(e, format_exc(), False)
except Exception as e:
etag.invalidate()
report_rest_error(ExecutionError(), format_exc(), False)
def format_obj(obj):
"""Render an object `obj` into HTML."""
if isinstance(obj, type(None)):
result = ""
elif isinstance(obj, (unicode, str)):
obj = xml.sax.saxutils.quoteattr(obj)
result = "<pre>%s</pre>" % obj if '\n' in obj else obj
elif isinstance(obj, (int, float, bool)):
result = "%s" % obj
elif isinstance(obj, dict):
result = "<ul>"
for k, v in obj.iteritems():
result += "<li><b>%s</b>: %s</li>" % (
k, PrettyJSONHTMLFormat.format_obj(v))
result += "</ul>"
elif is_iterable(obj):
empty = True
result = "<details open><ul>"
for v in obj:
empty = False
result += "<li>%s</li>" % PrettyJSONHTMLFormat.format_obj(v)
result += "</ul></details>"
if empty:
result = ""
else:
cherrypy.log("cannot represent object of type %s in xml (%s)" %
(type(obj).__class__.__name__, repr(obj)))
raise ExecutionError("cannot represent object in xml")
return result
def format_obj(obj):
"""Render an object `obj` into XML."""
if isinstance(obj, type(None)):
result = ""
elif isinstance(obj, (unicode, str)):
result = xml.sax.saxutils.escape(obj).encode("utf-8")
elif isinstance(obj, (int, float, bool)):
result = xml.sax.saxutils.escape(str(obj)).encode("utf-8")
elif isinstance(obj, dict):
result = "<dict>"
for k, v in obj.iteritems():
result += "<key>%s</key><value>%s</value>" % \
(xml.sax.saxutils.escape(k).encode("utf-8"),
XMLFormat.format_obj(v))
result += "</dict>"
elif is_iterable(obj):
result = "<array>"
for v in obj:
result += "<i>%s</i>" % XMLFormat.format_obj(v)
result += "</array>"
else:
cherrypy.log("cannot represent object of type %s in xml (%s)" %
(type(obj).__class__.__name__, repr(obj)))
raise ExecutionError("cannot represent object in xml")
return result
def __init__(self, app, config, mount):
DatabaseRESTApi.__init__(self, app, config, mount)
self.formats = [ ('application/json', JSONFormat()) ]
status, serverdn = getstatusoutput('openssl x509 -noout -subject -in %s | cut -f2- -d\ ' % config.serverhostcert)
if status is not 0:
raise ExecutionError("Internal issue when retrieving crabserver service DN.")
extconfig = Utils.ConfigCache(centralconfig=Utils.getCentralConfig(extconfigurl=config.extconfigurl, mode=config.mode),
cachetime=mktime(gmtime()))
#Global initialization of Data objects. Parameters coming from the config should go here
DataUserWorkflow.globalinit(config)
DataWorkflow.globalinit(dbapi=self, phedexargs={'endpoint': config.phedexurl},\
credpath=config.credpath, centralcfg=extconfig, config=config)
DataFileMetadata.globalinit(dbapi=self, config=config)
RESTTask.globalinit(centralcfg=extconfig)
Utils.globalinit(config.serverhostkey, config.serverhostcert, serverdn, config.credpath)
## TODO need a check to verify the format depending on the resource
## the RESTFileMetadata has the specifc requirement of getting xml reports
self._add( {'workflow': RESTUserWorkflow(app, self, config, mount, extconfig),
'info': RESTServerInfo(app, self, config, mount, serverdn, extconfig),
'filemetadata': RESTFileMetadata(app, self, config, mount),
'workflowdb': RESTWorkerWorkflow(app, self, config, mount),
'task': RESTTask(app, self, config, mount),
'filetransfers': RESTFileTransfers(app, self, config, mount),
'fileusertransfers': RESTFileUserTransfers(app, self, config, mount),
})
self._initLogger( getattr(config, 'loggingFile', None), getattr(config, 'loggingLevel', None) )
def deletewarnings(self, **kwargs):
""" Deleet warnings from the warning column in the database. Can be tested with:
curl -X POST https://mmascher-poc.cern.ch/crabserver/dev/task -k --key /tmp/x509up_u8440 --cert /tmp/x509up_u8440 \
-d 'subresource=deletewarnings&workflow=140710_233424_crab3test-5:mmascher_crab_HCprivate12' -v
"""
#check if the parameter is there
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter("Task name not found in the input parameters")
#decoding and setting the parameters
workflow = kwargs['workflow']
authz_owner_match(self.api, [workflow], self.Task) #check that I am modifying my own workflow
# rows = self.api.query(None, None, "SELECT tm_task_warnings FROM tasks WHERE tm_taskname = :workflow", workflow=workflow)#self.Task.TASKSUMMARY_sql)
rows = self.api.query(None, None, self.Task.ID_sql, taskname=workflow)#self.Task.TASKSUMMARY_sql)
rows = list(rows) #from generator to list
if len(rows)==0:
raise InvalidParameter("Task %s not found in the task database" % workflow)
row = self.Task.ID_tuple(*rows[0])
warnings = literal_eval(row.task_warnings.read() if row.task_warnings else '[]')
if len(warnings)<1:
raise ExecutionError("No warnings to remove.")
self.api.modify(self.Task.DeleteWarnings_sql, workflow=[workflow])
return []
def stream_chunked(self, stream, etag, preamble, trailer):
"""Generator for actually producing the output."""
comma = " "
try:
if preamble:
etag.update(preamble)
yield preamble
try:
for obj in stream:
chunk = comma + json.dumps(obj) + "\n"
etag.update(chunk)
yield chunk
comma = ","
except GeneratorExit:
etag.invalidate()
trailer = None
raise
except Exception as exp:
print("ERROR, json.dumps failed to serialize %s, type %s\nException: %s" \
% (obj, type(obj), str(exp)))
raise
finally:
if trailer:
etag.update(trailer)
yield trailer
cherrypy.response.headers["X-REST-Status"] = 100
except RESTError as e:
etag.invalidate()
report_rest_error(e, format_exc(), False)
except Exception as e:
etag.invalidate()
report_rest_error(ExecutionError(), format_exc(), False)
def retrieveConfig(externalLink):
hbuf = StringIO.StringIO()
bbuf = StringIO.StringIO()
curl = pycurl.Curl()
curl.setopt(pycurl.URL, externalLink)
curl.setopt(pycurl.WRITEFUNCTION, bbuf.write)
curl.setopt(pycurl.HEADERFUNCTION, hbuf.write)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.perform()
curl.close()
header = ResponseHeader(hbuf.getvalue())
if (header.status < 200 or header.status >= 300):
msg = "Reading %s returned %s." % (externalLink, header.status)
if centralCfgFallback:
msg += "\nUsing cached values for external configuration."
cherrypy.log(msg)
return centralCfgFallback
else:
cherrypy.log(msg)
raise ExecutionError(
"Internal issue when retrieving external configuration from %s"
% externalLink)
jsonConfig = bbuf.getvalue()
return jsonConfig
def getCentralConfig(extconfigurl, mode):
"""Utility to retrieve the central configuration to be used for dynamic variables
arg str extconfigurl: the url pointing to the exteranl configuration parameter
arg str mode: also known as the variant of the rest (prod, preprod, dev, private)
return: the dictionary containing the external configuration for the selected mode."""
hbuf = StringIO.StringIO()
bbuf = StringIO.StringIO()
curl = pycurl.Curl()
curl.setopt(pycurl.URL, extconfigurl)
curl.setopt(pycurl.WRITEFUNCTION, bbuf.write)
curl.setopt(pycurl.HEADERFUNCTION, hbuf.write)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.perform()
curl.close()
header = ResponseHeader(hbuf.getvalue())
if header.status < 200 or header.status >= 300:
cherrypy.log("Problem %d reading from %s." %
(extconfigurl, header.status))
raise ExecutionError(
"Internal issue when retrieving external confifuration")
return json.decode(bbuf.getvalue())[mode]
class RawFormat(RESTFormat):
"""Format an iterable of objects as raw data.
Generates raw data completely unmodified, for example image data or
streaming arbitrary external data files including even plain text.
Computes an ETag on the output in the process. The result is always
chunked, even simple strings on input. Usually small enough responses
will automatically be converted back to a single string response post
compression and ETag processing.
Any exceptions raised by input stream are reported to `report_rest_error`
and swallowed, as this is normally used to generate output for CherryPy
responses, which cannot handle exceptions reasonably after the output
generation begins; later processing may reconvert those back to exceptions
however (cf. stream_maybe_etag()). A X-REST-Status trailer header is added
if (and only if) an exception occurs; the client must inspect that to find
out if it got the complete output. There is normally 'X-REST-Status: 100'
in normal response headers, and it remains valid in case of success.
No ETag header is generated in case of an exception."""
def stream_chunked(self, stream, etag):
"""Generator for actually producing the output."""
try:
for chunk in stream:
etag.update(chunk)
yield chunk
except RESTError, e:
etag.invalidate()
report_rest_error(e, format_exc(), False)
except Exception, e:
etag.invalidate()
report_rest_error(ExecutionError(), format_exc(), False)
def addwarning(self, **kwargs):
""" Add a warning to the wraning column in the database. Can be tested with:
curl -X POST https://mmascher-poc.cern.ch/crabserver/dev/task -k --key /tmp/x509up_u8440 --cert /tmp/x509up_u8440 \
-d 'subresource=addwarning&workflow=140710_233424_crab3test-5:mmascher_crab_HCprivate12&warning=blahblah' -v
"""
#check if the parameters are there
if 'warning' not in kwargs or not kwargs['warning']:
raise InvalidParameter("Warning message not found in the input parameters")
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter("Task name not found in the input parameters")
#decoding and setting the parameters
workflow = kwargs['workflow']
authz_owner_match(self.api, [workflow], self.Task) #check that I am modifying my own workflow
try:
warning = b64decode(kwargs['warning'])
except TypeError:
raise InvalidParameter("Failure message is not in the accepted format")
# rows = self.api.query(None, None, "SELECT tm_task_warnings FROM tasks WHERE tm_taskname = :workflow", workflow=workflow)#self.Task.TASKSUMMARY_sql)
rows = self.api.query(None, None, self.Task.ID_sql, taskname=workflow)#self.Task.TASKSUMMARY_sql)
rows = list(rows) #from generator to list
if len(rows)==0:
raise InvalidParameter("Task %s not found in the task database" % workflow)
row = self.Task.ID_tuple(*rows[0])
warnings = literal_eval(row.task_warnings.read() if row.task_warnings else '[]')
if len(warnings)>10:
raise ExecutionError("You cannot add more than 10 warnings to a task")
warnings.append(warning)
self.api.modify(self.Task.SetWarnings_sql, warnings=[str(warnings)], workflow=[workflow])
return []
def stream_chunked(self, stream, etag, preamble, trailer):
"""Generator for actually producing the output."""
try:
etag.update(preamble)
yield preamble
try:
for obj in stream:
chunk = XMLFormat.format_obj(obj)
etag.update(chunk)
yield chunk
except GeneratorExit:
etag.invalidate()
trailer = None
raise
finally:
if trailer:
etag.update(trailer)
yield trailer
except RESTError as e:
etag.invalidate()
report_rest_error(e, format_exc(), False)
except Exception as e:
etag.invalidate()
report_rest_error(ExecutionError(), format_exc(), False)
def search(self, **kwargs):
"""Retrieves all the columns of a task in the task table (select * from task ...)
The API is (only?) used in the monitor for operator.
curl -X GET 'https://mmascher-dev6.cern.ch/crabserver/dev/task?subresource=search&workflow=150224_230633:mmascher_crab_testecmmascher-dev6_3' \
-k --key /tmp/x509up_u8440 --cert /tmp/x509up_u8440 -v"""
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter(
"Task name not found in the input parameters")
try:
row = next(
self.api.query(None,
None,
self.Task.QuickSearch_sql,
taskname=kwargs["workflow"]))
except StopIteration:
raise ExecutionError(
"Impossible to find task %s in the database." %
kwargs["workflow"])
def getval(col):
""" Some columns in oracle can be CLOB and we need to call read on them.
"""
#TODO move the function in ServerUtils and use it when required (e.g.: mysql LONGTEXT does not need read())
try:
return str(col)
except:
return col.read()
return [getval(col) for col in row]
def getpublishurl(self, **kwargs):
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter("Task name not found in the input parameters")
try:
row = next(self.api.query(None, None, self.Task.GetPublishUrl_sql, taskname=kwargs['workflow']))
except StopIteration:
raise ExecutionError("Impossible to find task %s in the database." % kwargs['workflow'])
yield row
def myPerform(cls, curl, url):
try:
curl.perform()
except pycurl.error as e:
raise ExecutionError(("Failed to contact Grid scheduler when getting URL %s. "
"This might be a temporary error, please retry later and "
"contact %s if the error persist. Error from curl: %s"
% (url, FEEDBACKMAIL, str(e))))
def __enter__(self):
ctr = self.throttle._incUser(self.user)
#self.throttle.logger.debug("Entering throttled function with counter %d for user %s" % (ctr, self.user))
if ctr >= self.throttle.getLimit():
self.throttle._decUser(self.user)
raise ExecutionError(
"The current number of active operations for this resource exceeds the limit of %d for user %s"
% (self.throttle.getLimit(), self.user))
def webdir(self, **kwargs):
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter("Task name not found in the input parameters")
workflow = kwargs['workflow']
try:
row = self.Task.ID_tuple(*next(self.api.query(None, None, self.Task.ID_sql, taskname=workflow)))
except StopIteration:
raise ExecutionError("Impossible to find task %s in the database." % kwargs["workflow"])
yield row.user_webdir
def scheddaddress(self, **kwargs):
backendurl=self.centralcfg.centralconfig['backend-urls']
workflow = kwargs['workflow']
try:
loc = HTCondorLocator.HTCondorLocator(backendurl)
schedd, address = loc.getScheddObj(workflow)
except Exception as ex:
self.logger.exception(ex)
raise ExecutionError("Unable to get schedd address for task %s" % (workflow))(ex)
yield loc.scheddAd['Machine']
def publicationStatus(self, workflow, asourl):
publication_info = {}
if not asourl:
raise ExecutionError("This CRAB server is not configured to publish; no publication status is available.")
server = CMSCouch.CouchServer(dburl=asourl, ckey=self.serverKey, cert=self.serverCert)
try:
db = server.connectDatabase('asynctransfer')
except Exception, ex:
msg = "Error while connecting to asynctransfer CouchDB"
self.logger.exception(msg)
publication_info = {'error' : msg}
return publication_info
def publicationStatusCouch(self, workflow, asourl, asodb):
publicationInfo = {'status': {}, 'failure_reasons': {}}
if not asourl:
raise ExecutionError("This CRAB server is not configured to publish; no publication status is available.")
server = CMSCouch.CouchServer(dburl=asourl, ckey=self.serverKey, cert=self.serverCert)
try:
db = server.connectDatabase(asodb)
except Exception:
msg = "Error while connecting to asynctransfer CouchDB for workflow %s " % workflow
msg += "\n asourl=%s asodb=%s" % (asourl, asodb)
self.logger.exception(msg)
publicationInfo['status'] = {'error': msg}
return publicationInfo
# Get the publication status for the given workflow. The next query to the
# CouchDB view returns a list of 1 dictionary (row) with:
# 'key' : workflow,
# 'value' : a dictionary with possible publication statuses as keys and the
# counts as values.
query = {'reduce': True, 'key': workflow, 'stale': 'update_after'}
try:
publicationList = db.loadView('AsyncTransfer', 'PublicationStateByWorkflow', query)['rows']
except Exception:
msg = "Error while querying CouchDB for publication status information for workflow %s " % workflow
self.logger.exception(msg)
publicationInfo['status'] = {'error': msg}
return publicationInfo
if publicationList:
publicationStatusDict = publicationList[0]['value']
publicationInfo['status'] = publicationStatusDict
# Get the publication failure reasons for the given workflow. The next query to
# the CouchDB view returns a list of N_different_publication_failures
# dictionaries (rows) with:
# 'key' : [workflow, publication failure],
# 'value' : count.
numFailedPublications = publicationStatusDict['publication_failed']
if numFailedPublications:
query = {'group': True, 'startkey': [workflow], 'endkey': [workflow, {}], 'stale': 'update_after'}
try:
publicationFailedList = db.loadView('DBSPublisher', 'PublicationFailedByWorkflow', query)['rows']
except Exception:
msg = "Error while querying CouchDB for publication failures information for workflow %s " % workflow
self.logger.exception(msg)
publicationInfo['failure_reasons']['error'] = msg
return publicationInfo
publicationInfo['failure_reasons']['result'] = []
for publicationFailed in publicationFailedList:
failureReason = publicationFailed['key'][1]
numFailedFiles = publicationFailed['value']
publicationInfo['failure_reasons']['result'].append((failureReason, numFailedFiles))
return publicationInfo
def get(self, workflow, subresource, username, limit, shortformat,
exitcode, jobids, verbose, timestamp):
"""Retrieves the workflow information, like a status summary, in case the workflow unique name is specified.
Otherwise returns all workflows since (now - age) for which the user is the owner.
The caller needs to be a valid CMS user.
:arg str workflow: unique name identifier of workflow;
:arg str timestamp: max workflow age in hours;
:arg str subresource: the specific workflow information to be accessed;
:arg int limit: limit of return entries for some specific subresource;
:arg int exitcode: exitcode for which the specific subresource is needed (eg log file of a job with that exitcode)
:retrun: workflow with the relative status summary in case of per user request; or
the requested subresource."""
result = []
if workflow:
userdn = cherrypy.request.headers['Cms-Authn-Dn']
# if have the wf then retrieve the wf status summary
if not subresource:
result = self.userworkflowmgr.status(workflow,
verbose=verbose,
userdn=userdn)
# if have a subresource then it should be one of these
elif subresource == 'logs':
result = self.userworkflowmgr.logs(workflow,
limit,
exitcode,
jobids,
userdn=userdn)
elif subresource == 'data':
result = self.userworkflowmgr.output(workflow,
limit,
jobids,
userdn=userdn)
elif subresource == 'errors':
result = self.userworkflowmgr.errors(workflow, shortformat)
elif subresource == 'report':
result = self.userworkflowmgr.report(workflow,
userdn=userdn,
usedbs=shortformat)
# if here means that no valid subresource has been requested
# flow should never pass through here since validation restrict this
else:
raise ExecutionError("Validation or method error")
else:
# retrieve the information about latest worfklows for that user
# age can have a default: 1 week ?
cherrypy.log("Found user '%s'" % cherrypy.request.user['login'])
result = self.userworkflowmgr.getLatests(
username or cherrypy.request.user['login'],
timestamp) #eric added timestamp to match username
return result
def proceed(self, workflow):
"""Continue a task which was initialized with 'crab submit --dryrun'.
:arg str workflow: a workflow name
"""
row = self.Task.ID_tuple(*next(self.api.query(None, None, self.Task.ID_sql, taskname=workflow)))
if row.task_status != 'UPLOADED':
msg = 'Can only proceed if task is in the UPLOADED status, but it is in the %s status.' % row.task_status
raise ExecutionError(msg)
else:
self.api.modify(self.Task.SetDryRun_sql, taskname=[workflow], dry_run=['F'])
self.api.modify(self.Task.SetStatusTask_sql, taskname=[workflow], status=['NEW'], command=['SUBMIT'])
return [{'result': 'ok'}]
def stream_chunked(self, stream, etag):
"""Generator for actually producing the output."""
try:
for chunk in stream:
etag.update(chunk)
yield chunk
except RESTError as e:
etag.invalidate()
report_rest_error(e, format_exc(), False)
except Exception as e:
etag.invalidate()
report_rest_error(ExecutionError(), format_exc(), False)
except BaseException:
etag.invalidate()
raise
