def __init__(self, request):
self.request = request
self.execution = None
self.service_name = None
self.processid = None
self.process = None
if 'job_id' in request.params:
job = request.db.jobs.find_one(
{'identifier': request.params['job_id']})
self.service_name = job.get('service_name')
self.execution = check_status(url=job.get('status_location'),
response=job.get('response'),
verify=False,
sleep_secs=0)
self.processid = self.execution.process.identifier
elif 'wps' in request.params:
self.service_name = request.params.get('wps')
self.processid = request.params.get('process')
if self.service_name:
# TODO: avoid getcaps
self.wps = WebProcessingService(url=request.route_url(
'owsproxy', service_name=self.service_name),
verify=False)
# TODO: need to fix owslib to handle special identifiers
self.process = self.wps.describeprocess(self.processid)
super(ExecuteProcess, self).__init__(request,
name='processes_execute',
title='')
def collect_outputs(status_location=None, response=None):
execution = check_status(url=status_location,
response=response,
sleep_secs=0)
outputs = {}
for output in execution.processOutputs:
outputs[output.identifier] = output
return outputs
def test_check_status():
doc = etree.parse(WPS_RESPONSE_XML)
execution = wps.check_status(response=etree.tostring(doc), sleep_secs=0)
assert execution.response is not None
assert isinstance(execution.response, etree._Element)
assert execution.isSucceded()
assert execution.statusLocation ==\
'https://localhost:28090/wpsoutputs/hummingbird/56cd4294-bd69-11e6-80fe-68f72837e1b4.xml'
def view(self):
status = 'ProcessAccepted'
log = None
# is job running?
if self.collection.find({"identifier": self.job_id}).count() == 1:
job = self.collection.find_one({"identifier": self.job_id})
progress = job.get('progress', 0)
status = job['status']
log = job.get('log', ['No status message'])
if status == 'ProcessSucceeded':
execution = check_status(job['status_location'], verify=False)
for output in execution.processOutputs:
if output.identifier == 'output':
break
details = output_details(self.request, output)
if details.get('reference'):
result = '<a href="{0}" class="btn btn-success btn-xs" target="_blank">Show Output</a>'.format(
details['reference'])
else:
result = '<strong>{0}</strong>'.format(', '.join(
details.get('data', '')))
msg = '<h4>Job Succeeded: {1} <a href="{0}" class="btn btn-info btn-xs"> Details</a></h4>'
url = self.request.route_path('job_details',
tab='outputs',
job_id=self.job_id)
self.session.flash(msg.format(url, result), queue="success")
elif status == 'ProcessFailed':
msg = '<h4>Job Failed [{0}/100]</h4>'
self.session.flash(msg.format(progress), queue="danger")
else:
msg = '<h4><i class="fa fa-cog fa-spin text-muted fa-lg"></i> Job Running [{0}/100]</h4>'
# msg = """
# <div class="row">
# <div class="col-md-3"
# <h3>
# <i class="fa fa-cog fa-spin text-muted fa-lg"></i>
# Job Running
# <div class="progress" data-toggle="tooltip" title="Job progress.">
# <div class="progress-bar progress-bar-striped active" role="progressbar"
# aria-valuenow="{0}"
# aria-valuemin="0" aria-valuemax="100" style="min-width: 2em; width: {0}%;">
# <span>{0}% Complete</span>
# </div>
# </h3>
# </div>
# </div>"""
self.session.flash(msg.format(progress), queue="warning")
return dict(status=status, log=log)
job['request'] = execution.request
job['response'] = etree.tostring(execution.response)
LOGGER.debug("job init done %s ...", self.request.id)
LOGGER.debug("status location={}".format(execution.statusLocation))
num_retries = 0
run_step = 0
while execution.isNotComplete() or run_step == 0:
if num_retries >= 5:
raise Exception(
"Could not read status document after 5 retries. Giving up."
)
try:
execution = check_status(url=execution.statusLocation,
verify=False,
sleep_secs=wait_secs(run_step))
job['response'] = etree.tostring(execution.response)
job['status'] = execution.getStatus()
job['status_message'] = execution.statusMessage
job['progress'] = execution.percentCompleted
duration = datetime.now() - job.get('created', datetime.now())
job['duration'] = str(duration).split('.')[0]
if execution.isComplete():
job['finished'] = datetime.now()
if execution.isSucceded():
LOGGER.debug("job succeded")
job['progress'] = 100
else:
LOGGER.debug("job failed.")
# job['title'] = getattr(execution.process, "title")
job['abstract'] = getattr(execution.process, "abstract")
job['status_location'] = execution.statusLocation
job['request'] = execution.request
job['response'] = etree.tostring(execution.response)
LOGGER.debug("job init done %s ...", self.request.id)
LOGGER.debug("status location={}".format(execution.statusLocation))
num_retries = 0
run_step = 0
while execution.isNotComplete() or run_step == 0:
if num_retries >= 5:
raise Exception("Could not read status document after 5 retries. Giving up.")
try:
execution = check_status(url=execution.statusLocation, verify=False,
sleep_secs=wait_secs(run_step))
job['response'] = etree.tostring(execution.response)
job['status'] = execution.getStatus()
job['status_message'] = execution.statusMessage
job['progress'] = execution.percentCompleted
duration = datetime.now() - job.get('created', datetime.now())
job['duration'] = str(duration).split('.')[0]
if execution.isComplete():
job['finished'] = datetime.now()
if execution.isSucceded():
LOGGER.debug("job succeded")
job['progress'] = 100
else:
LOGGER.debug("job failed.")
job['status_message'] = '\n'.join(error.text for error in execution.errors)
def execute_process(self, url, service_name, identifier, inputs, outputs, use_async=True, userid=None, caption=None):
registry = app.conf['PYRAMID_REGISTRY']
db = mongodb(registry)
job = add_job(
db,
userid=userid,
task_id=self.request.id,
service_name=service_name,
process_id=identifier,
use_async=use_async,
caption=caption)
try:
wps = WebProcessingService(url=url, skip_caps=False, verify=False, headers=wps_headers(userid))
LOGGER.debug('wps url={}, headers={}'.format(url, wps_headers(userid)))
# TODO: complex type detection is currently broken due to pywps bug.
outputs = [('output', True)]
try:
# TODO: sync is non-default
if use_async is False:
mode = SYNC
else:
mode = ASYNC
execution = wps.execute(
identifier=identifier,
inputs=inputs,
output=outputs,
mode=mode,
lineage=True)
except Exception:
LOGGER.warn("Setting execution mode is not supported. Using default async mode.")
execution = wps.execute(identifier,
inputs=inputs,
output=outputs
)
# job['service'] = wps.identification.title
# job['title'] = getattr(execution.process, "title")
job['abstract'] = getattr(execution.process, "abstract")
job['status_location'] = execution.statusLocation
job['request'] = execution.request
job['response'] = etree.tostring(execution.response)
LOGGER.debug("job init done %s ...", self.request.id)
LOGGER.debug("status location={}".format(execution.statusLocation))
num_retries = 0
run_step = 0
while execution.isNotComplete() or run_step == 0:
if num_retries >= 5:
raise Exception("Could not read status document after 5 retries. Giving up.")
try:
execution = check_status(url=execution.statusLocation, verify=False,
sleep_secs=wait_secs(run_step))
job['response'] = etree.tostring(execution.response)
job['status'] = execution.getStatus()
job['status_message'] = execution.statusMessage
job['progress'] = execution.percentCompleted
duration = datetime.now() - job.get('created', datetime.now())
job['duration'] = str(duration).split('.')[0]
if execution.isComplete():
job['finished'] = datetime.now()
if execution.isSucceded():
LOGGER.debug("job succeded")
job['progress'] = 100
else:
LOGGER.debug("job failed.")
job['status_message'] = '\n'.join(error.text for error in execution.errors)
for error in execution.errors:
save_log(job, error)
except Exception:
num_retries += 1
LOGGER.exception("Could not read status xml document for job %s. Trying again ...", self.request.id)
sleep(1)
else:
LOGGER.debug("update job %s ...", self.request.id)
num_retries = 0
run_step += 1
finally:
save_log(job)
db.jobs.update({'identifier': job['identifier']}, job)
except Exception as exc:
LOGGER.exception("Failed to run Job")
job['status'] = "ProcessFailed"
job['status_message'] = "Error: {0}".format(exc)
finally:
save_log(job)
db.jobs.update({'identifier': job['identifier']}, job)
registry.notify(JobFinished(job))
return job['status']
def collect_inputs(status_location=None, response=None):
execution = check_status(url=status_location,
response=response,
sleep_secs=0)
return execution.dataInputs
def execute_workflow(self, userid, url, service_name, workflow, caption=None):
registry = app.conf['PYRAMID_REGISTRY']
db = mongodb(registry)
job = add_job(db,
userid=userid,
task_id=self.request.id,
is_workflow=True,
service_name=service_name,
process_id=workflow['worker']['identifier'],
caption=caption)
try:
# generate and run dispel workflow
# TODO: fix owslib wps for unicode/yaml parameters
# logger.debug('workflow=%s', workflow)
headers = wps_headers(userid)
# TODO: handle access token in workflow
# workflow['worker']['url'] = build_get_url(
# workflow['worker']['url'],
# {'access_token': headers.get('Access-Token', '')})
logger.debug('workflow=%s', workflow)
inputs = [(
'workflow',
ComplexDataInput(
# TODO: pywps-4 expects base64 encoding when not set to ''
dump_json(workflow),
mimeType="text/yaml",
encoding=""))]
outputs = [('output', True), ('logfile', True)]
wps = WebProcessingService(url=url,
skip_caps=True,
verify=False,
headers=headers)
# worker_wps = WebProcessingService(url=workflow['worker']['url'],
# skip_caps=False, verify=False)
execution = wps.execute(identifier='workflow',
inputs=inputs,
output=outputs,
lineage=True)
# job['service'] = worker_wps.identification.title
# job['title'] = getattr(execution.process, "title")
# job['abstract'] = getattr(execution.process, "abstract")
job['status_location'] = execution.statusLocation
job['response'] = etree.tostring(execution.response)
logger.debug("job init done %s ...", self.request.id)
run_step = 0
num_retries = 0
while execution.isNotComplete():
if num_retries >= 5:
raise Exception(
"Could not read status document after 5 retries. Giving up."
)
try:
execution = check_status(url=execution.statusLocation,
verify=False,
sleep_secs=wait_secs(run_step))
job['response'] = etree.tostring(execution.response)
job['status'] = execution.getStatus()
job['status_message'] = execution.statusMessage
job['progress'] = execution.percentCompleted
duration = datetime.now() - job.get('created', datetime.now())
job['duration'] = str(duration).split('.')[0]
if execution.isComplete():
job['finished'] = datetime.now()
if execution.isSucceded():
for output in execution.processOutputs:
if 'output' == output.identifier:
result = yaml.load(
urllib.urlopen(output.reference))
job['worker_status_location'] = result[
'worker']['status_location']
job['progress'] = 100
save_log(job)
else:
job['status_message'] = '\n'.join(
error.text for error in execution.errors)
for error in execution.errors:
save_log(job, error)
else:
save_log(job)
except Exception:
num_retries += 1
logger.exception(
"Could not read status xml document for job %s. Trying again ...",
self.request.id)
sleep(1)
else:
logger.debug("update job %s ...", self.request.id)
num_retries = 0
run_step += 1
db.jobs.update({'identifier': job['identifier']}, job)
except Exception as exc:
logger.exception("Failed to run Job")
job['status'] = "ProcessFailed"
job['status_message'] = "Failed to run Job. %s" % exc.message
finally:
save_log(job)
db.jobs.update({'identifier': job['identifier']}, job)
registry.notify(JobFinished(job))
return job['status']
def collect_outputs(status_location=None, response=None):
execution = check_status(url=status_location, response=response, sleep_secs=0)
outputs = {}
for output in execution.processOutputs:
outputs[output.identifier] = output
return outputs
def collect_inputs(status_location=None, response=None):
execution = check_status(url=status_location, response=response, sleep_secs=0)
return execution.dataInputs
def job_to_state(request, job_id):
# TODO: quite dirty ... needs clean up
state = {}
job = request.db.jobs.find_one({'identifier': job_id})
execution = check_status(url=job.get('status_location'),
response=job.get('response'),
verify=False,
sleep_secs=0)
if len(execution.dataInputs) == 1:
if len(execution.dataInputs[0].data) == 1:
workflow = yaml.load(execution.dataInputs[0].data[0])
# TODO: avoid getcaps
logger.debug("worker url: %s", workflow['worker']['url'])
parsed_url = urlparse(workflow['worker']['url'])
url = "{0.scheme}://{0.netloc}{0.path}".format(parsed_url)
wps = WebProcessingService(url=url, verify=False, skip_caps=False)
logger.debug("wps url: %s", wps.url)
if '/ows/proxy' in parsed_url.path:
service_name = parsed_url.path.split('/')[-1]
else:
service = request.catalog.get_service_by_url(wps.url)
service_name = service['name']
logger.debug('service_name=%s', service_name)
state['wizard_wps'] = {'identifier': service_name}
state['wizard_process'] = {
'identifier': workflow['worker']['identifier']
}
inputs = {}
for inp in workflow['worker']['inputs']:
key, value = inp[0], inp[1]
if key in inputs:
inputs[key].extend(value)
else:
inputs[key] = [value]
process = wps.describeprocess(workflow['worker']['identifier'])
for inp in process.dataInputs:
if 'boolean' in inp.dataType and inp.identifier in inputs:
inputs[inp.identifier] = [
val.lower() == 'true' for val in inputs[inp.identifier]
]
if inp.maxOccurs < 2 and inp.identifier in inputs:
inputs[inp.identifier] = inputs[inp.identifier][0]
state['wizard_literal_inputs'] = inputs
state['wizard_complex_inputs'] = {
'identifier': workflow['worker']['resource']
}
if workflow['name'] == 'wizard_esgf_search':
state['wizard_source'] = {'source': 'wizard_esgf_search'}
import json
state['wizard_esgf_search'] = {
'selection': json.dumps(workflow['source']['esgf'])
}
elif workflow['name'] == 'wizard_solr':
state['wizard_source'] = {'source': 'wizard_solr'}
elif workflow['name'] == 'wizard_threddsservice':
state['wizard_source'] = {'source': 'wizard_threddsservice'}
state['wizard_threddsservice'] = {
'url': workflow['source']['thredds']['catalog_url']
}
state['wizard_done'] = {'caption': job.get('caption')}
return state