def __init__(self, request): self.request = request self.execution = None self.service_name = None self.processid = None self.process = None if 'job_id' in request.params: job = request.db.jobs.find_one( {'identifier': request.params['job_id']}) self.service_name = job.get('service_name') self.execution = check_status(url=job.get('status_location'), response=job.get('response'), verify=False, sleep_secs=0) self.processid = self.execution.process.identifier elif 'wps' in request.params: self.service_name = request.params.get('wps') self.processid = request.params.get('process') if self.service_name: # TODO: avoid getcaps self.wps = WebProcessingService(url=request.route_url( 'owsproxy', service_name=self.service_name), verify=False) # TODO: need to fix owslib to handle special identifiers self.process = self.wps.describeprocess(self.processid) super(ExecuteProcess, self).__init__(request, name='processes_execute', title='')
def collect_outputs(status_location=None, response=None): execution = check_status(url=status_location, response=response, sleep_secs=0) outputs = {} for output in execution.processOutputs: outputs[output.identifier] = output return outputs
def test_check_status(): doc = etree.parse(WPS_RESPONSE_XML) execution = wps.check_status(response=etree.tostring(doc), sleep_secs=0) assert execution.response is not None assert isinstance(execution.response, etree._Element) assert execution.isSucceded() assert execution.statusLocation ==\ 'https://localhost:28090/wpsoutputs/hummingbird/56cd4294-bd69-11e6-80fe-68f72837e1b4.xml'
def view(self): status = 'ProcessAccepted' log = None # is job running? if self.collection.find({"identifier": self.job_id}).count() == 1: job = self.collection.find_one({"identifier": self.job_id}) progress = job.get('progress', 0) status = job['status'] log = job.get('log', ['No status message']) if status == 'ProcessSucceeded': execution = check_status(job['status_location'], verify=False) for output in execution.processOutputs: if output.identifier == 'output': break details = output_details(self.request, output) if details.get('reference'): result = '<a href="{0}" class="btn btn-success btn-xs" target="_blank">Show Output</a>'.format( details['reference']) else: result = '<strong>{0}</strong>'.format(', '.join( details.get('data', ''))) msg = '<h4>Job Succeeded: {1} <a href="{0}" class="btn btn-info btn-xs"> Details</a></h4>' url = self.request.route_path('job_details', tab='outputs', job_id=self.job_id) self.session.flash(msg.format(url, result), queue="success") elif status == 'ProcessFailed': msg = '<h4>Job Failed [{0}/100]</h4>' self.session.flash(msg.format(progress), queue="danger") else: msg = '<h4><i class="fa fa-cog fa-spin text-muted fa-lg"></i> Job Running [{0}/100]</h4>' # msg = """ # <div class="row"> # <div class="col-md-3" # <h3> # <i class="fa fa-cog fa-spin text-muted fa-lg"></i> # Job Running # <div class="progress" data-toggle="tooltip" title="Job progress."> # <div class="progress-bar progress-bar-striped active" role="progressbar" # aria-valuenow="{0}" # aria-valuemin="0" aria-valuemax="100" style="min-width: 2em; width: {0}%;"> # <span>{0}% Complete</span> # </div> # </h3> # </div> # </div>""" self.session.flash(msg.format(progress), queue="warning") return dict(status=status, log=log)
job['request'] = execution.request job['response'] = etree.tostring(execution.response) LOGGER.debug("job init done %s ...", self.request.id) LOGGER.debug("status location={}".format(execution.statusLocation)) num_retries = 0 run_step = 0 while execution.isNotComplete() or run_step == 0: if num_retries >= 5: raise Exception( "Could not read status document after 5 retries. Giving up." ) try: execution = check_status(url=execution.statusLocation, verify=False, sleep_secs=wait_secs(run_step)) job['response'] = etree.tostring(execution.response) job['status'] = execution.getStatus() job['status_message'] = execution.statusMessage job['progress'] = execution.percentCompleted duration = datetime.now() - job.get('created', datetime.now()) job['duration'] = str(duration).split('.')[0] if execution.isComplete(): job['finished'] = datetime.now() if execution.isSucceded(): LOGGER.debug("job succeded") job['progress'] = 100 else: LOGGER.debug("job failed.")
# job['title'] = getattr(execution.process, "title") job['abstract'] = getattr(execution.process, "abstract") job['status_location'] = execution.statusLocation job['request'] = execution.request job['response'] = etree.tostring(execution.response) LOGGER.debug("job init done %s ...", self.request.id) LOGGER.debug("status location={}".format(execution.statusLocation)) num_retries = 0 run_step = 0 while execution.isNotComplete() or run_step == 0: if num_retries >= 5: raise Exception("Could not read status document after 5 retries. Giving up.") try: execution = check_status(url=execution.statusLocation, verify=False, sleep_secs=wait_secs(run_step)) job['response'] = etree.tostring(execution.response) job['status'] = execution.getStatus() job['status_message'] = execution.statusMessage job['progress'] = execution.percentCompleted duration = datetime.now() - job.get('created', datetime.now()) job['duration'] = str(duration).split('.')[0] if execution.isComplete(): job['finished'] = datetime.now() if execution.isSucceded(): LOGGER.debug("job succeded") job['progress'] = 100 else: LOGGER.debug("job failed.") job['status_message'] = '\n'.join(error.text for error in execution.errors)
def execute_process(self, url, service_name, identifier, inputs, outputs, use_async=True, userid=None, caption=None): registry = app.conf['PYRAMID_REGISTRY'] db = mongodb(registry) job = add_job( db, userid=userid, task_id=self.request.id, service_name=service_name, process_id=identifier, use_async=use_async, caption=caption) try: wps = WebProcessingService(url=url, skip_caps=False, verify=False, headers=wps_headers(userid)) LOGGER.debug('wps url={}, headers={}'.format(url, wps_headers(userid))) # TODO: complex type detection is currently broken due to pywps bug. outputs = [('output', True)] try: # TODO: sync is non-default if use_async is False: mode = SYNC else: mode = ASYNC execution = wps.execute( identifier=identifier, inputs=inputs, output=outputs, mode=mode, lineage=True) except Exception: LOGGER.warn("Setting execution mode is not supported. Using default async mode.") execution = wps.execute(identifier, inputs=inputs, output=outputs ) # job['service'] = wps.identification.title # job['title'] = getattr(execution.process, "title") job['abstract'] = getattr(execution.process, "abstract") job['status_location'] = execution.statusLocation job['request'] = execution.request job['response'] = etree.tostring(execution.response) LOGGER.debug("job init done %s ...", self.request.id) LOGGER.debug("status location={}".format(execution.statusLocation)) num_retries = 0 run_step = 0 while execution.isNotComplete() or run_step == 0: if num_retries >= 5: raise Exception("Could not read status document after 5 retries. Giving up.") try: execution = check_status(url=execution.statusLocation, verify=False, sleep_secs=wait_secs(run_step)) job['response'] = etree.tostring(execution.response) job['status'] = execution.getStatus() job['status_message'] = execution.statusMessage job['progress'] = execution.percentCompleted duration = datetime.now() - job.get('created', datetime.now()) job['duration'] = str(duration).split('.')[0] if execution.isComplete(): job['finished'] = datetime.now() if execution.isSucceded(): LOGGER.debug("job succeded") job['progress'] = 100 else: LOGGER.debug("job failed.") job['status_message'] = '\n'.join(error.text for error in execution.errors) for error in execution.errors: save_log(job, error) except Exception: num_retries += 1 LOGGER.exception("Could not read status xml document for job %s. Trying again ...", self.request.id) sleep(1) else: LOGGER.debug("update job %s ...", self.request.id) num_retries = 0 run_step += 1 finally: save_log(job) db.jobs.update({'identifier': job['identifier']}, job) except Exception as exc: LOGGER.exception("Failed to run Job") job['status'] = "ProcessFailed" job['status_message'] = "Error: {0}".format(exc) finally: save_log(job) db.jobs.update({'identifier': job['identifier']}, job) registry.notify(JobFinished(job)) return job['status']
def collect_inputs(status_location=None, response=None): execution = check_status(url=status_location, response=response, sleep_secs=0) return execution.dataInputs
def execute_workflow(self, userid, url, service_name, workflow, caption=None): registry = app.conf['PYRAMID_REGISTRY'] db = mongodb(registry) job = add_job(db, userid=userid, task_id=self.request.id, is_workflow=True, service_name=service_name, process_id=workflow['worker']['identifier'], caption=caption) try: # generate and run dispel workflow # TODO: fix owslib wps for unicode/yaml parameters # logger.debug('workflow=%s', workflow) headers = wps_headers(userid) # TODO: handle access token in workflow # workflow['worker']['url'] = build_get_url( # workflow['worker']['url'], # {'access_token': headers.get('Access-Token', '')}) logger.debug('workflow=%s', workflow) inputs = [( 'workflow', ComplexDataInput( # TODO: pywps-4 expects base64 encoding when not set to '' dump_json(workflow), mimeType="text/yaml", encoding=""))] outputs = [('output', True), ('logfile', True)] wps = WebProcessingService(url=url, skip_caps=True, verify=False, headers=headers) # worker_wps = WebProcessingService(url=workflow['worker']['url'], # skip_caps=False, verify=False) execution = wps.execute(identifier='workflow', inputs=inputs, output=outputs, lineage=True) # job['service'] = worker_wps.identification.title # job['title'] = getattr(execution.process, "title") # job['abstract'] = getattr(execution.process, "abstract") job['status_location'] = execution.statusLocation job['response'] = etree.tostring(execution.response) logger.debug("job init done %s ...", self.request.id) run_step = 0 num_retries = 0 while execution.isNotComplete(): if num_retries >= 5: raise Exception( "Could not read status document after 5 retries. Giving up." ) try: execution = check_status(url=execution.statusLocation, verify=False, sleep_secs=wait_secs(run_step)) job['response'] = etree.tostring(execution.response) job['status'] = execution.getStatus() job['status_message'] = execution.statusMessage job['progress'] = execution.percentCompleted duration = datetime.now() - job.get('created', datetime.now()) job['duration'] = str(duration).split('.')[0] if execution.isComplete(): job['finished'] = datetime.now() if execution.isSucceded(): for output in execution.processOutputs: if 'output' == output.identifier: result = yaml.load( urllib.urlopen(output.reference)) job['worker_status_location'] = result[ 'worker']['status_location'] job['progress'] = 100 save_log(job) else: job['status_message'] = '\n'.join( error.text for error in execution.errors) for error in execution.errors: save_log(job, error) else: save_log(job) except Exception: num_retries += 1 logger.exception( "Could not read status xml document for job %s. Trying again ...", self.request.id) sleep(1) else: logger.debug("update job %s ...", self.request.id) num_retries = 0 run_step += 1 db.jobs.update({'identifier': job['identifier']}, job) except Exception as exc: logger.exception("Failed to run Job") job['status'] = "ProcessFailed" job['status_message'] = "Failed to run Job. %s" % exc.message finally: save_log(job) db.jobs.update({'identifier': job['identifier']}, job) registry.notify(JobFinished(job)) return job['status']
def job_to_state(request, job_id): # TODO: quite dirty ... needs clean up state = {} job = request.db.jobs.find_one({'identifier': job_id}) execution = check_status(url=job.get('status_location'), response=job.get('response'), verify=False, sleep_secs=0) if len(execution.dataInputs) == 1: if len(execution.dataInputs[0].data) == 1: workflow = yaml.load(execution.dataInputs[0].data[0]) # TODO: avoid getcaps logger.debug("worker url: %s", workflow['worker']['url']) parsed_url = urlparse(workflow['worker']['url']) url = "{0.scheme}://{0.netloc}{0.path}".format(parsed_url) wps = WebProcessingService(url=url, verify=False, skip_caps=False) logger.debug("wps url: %s", wps.url) if '/ows/proxy' in parsed_url.path: service_name = parsed_url.path.split('/')[-1] else: service = request.catalog.get_service_by_url(wps.url) service_name = service['name'] logger.debug('service_name=%s', service_name) state['wizard_wps'] = {'identifier': service_name} state['wizard_process'] = { 'identifier': workflow['worker']['identifier'] } inputs = {} for inp in workflow['worker']['inputs']: key, value = inp[0], inp[1] if key in inputs: inputs[key].extend(value) else: inputs[key] = [value] process = wps.describeprocess(workflow['worker']['identifier']) for inp in process.dataInputs: if 'boolean' in inp.dataType and inp.identifier in inputs: inputs[inp.identifier] = [ val.lower() == 'true' for val in inputs[inp.identifier] ] if inp.maxOccurs < 2 and inp.identifier in inputs: inputs[inp.identifier] = inputs[inp.identifier][0] state['wizard_literal_inputs'] = inputs state['wizard_complex_inputs'] = { 'identifier': workflow['worker']['resource'] } if workflow['name'] == 'wizard_esgf_search': state['wizard_source'] = {'source': 'wizard_esgf_search'} import json state['wizard_esgf_search'] = { 'selection': json.dumps(workflow['source']['esgf']) } elif workflow['name'] == 'wizard_solr': state['wizard_source'] = {'source': 'wizard_solr'} elif workflow['name'] == 'wizard_threddsservice': state['wizard_source'] = {'source': 'wizard_threddsservice'} state['wizard_threddsservice'] = { 'url': workflow['source']['thredds']['catalog_url'] } state['wizard_done'] = {'caption': job.get('caption')} return state