def add_mongodb(event): settings = event.request.registry.settings if settings.get('db') is None: try: from phoenix.models import mongodb settings['db'] = mongodb(event.request.registry) except: logger.exception('Could not connect to mongodb') event.request.db = settings.get('db')
def index_thredds(self, url, maxrecords=-1, depth=2): registry = app.conf['PYRAMID_REGISTRY'] db = mongodb(registry) task = dict(task_id=self.request.id, url=url, status='started') db.tasks.save(task) service = registry.settings.get('solr.url') try: feed_from_thredds(service=service, catalog_url=url, depth=depth, maxrecords=maxrecords, batch_size=50000) task['status'] = 'success' except: task['status'] = 'failure' raise finally: db.tasks.update({'url': task['url']}, task)
def execute_workflow(self, userid, url, workflow): registry = app.conf['PYRAMID_REGISTRY'] # generate and run dispel workflow # TODO: fix owslib wps for unicode/yaml parameters inputs=[('workflow', json.dumps(workflow))] outputs=[('output', True), ('logfile', True)] wps = WebProcessingService(url=url, skip_caps=True) worker_wps = WebProcessingService(url=workflow['worker']['url'], skip_caps=False) execution = wps.execute(identifier='workflow', inputs=inputs, output=outputs) db = mongodb(registry) job = add_job(db, userid, task_id = self.request.id, is_workflow = True, service = worker_wps.identification.title, title = workflow['worker']['identifier'], abstract = '', status_location = execution.statusLocation) while execution.isNotComplete(): execution.checkStatus(sleepSecs=1) job['status'] = execution.getStatus() job['status_message'] = execution.statusMessage job['is_complete'] = execution.isComplete() job['is_succeded'] = execution.isSucceded() job['progress'] = execution.percentCompleted duration = datetime.now() - job.get('created', datetime.now()) job['duration'] = str(duration).split('.')[0] if execution.isComplete(): job['finished'] = datetime.now() if execution.isSucceded(): for output in execution.processOutputs: if 'output' == output.identifier: result = yaml.load(urllib.urlopen(output.reference)) job['worker_status_location'] = result['worker']['status_location'] job['progress'] = 100 log(job) else: job['status_message'] = '\n'.join(error.text for error in execution.errors) for error in execution.errors: log_error(job, error) else: log(job) db.jobs.update({'identifier': job['identifier']}, job) return execution.getStatus()
def esgf_logon(self, userid, url, openid, password): registry = app.conf['PYRAMID_REGISTRY'] inputs = [] inputs.append( ('openid', openid.encode('ascii', 'ignore')) ) inputs.append( ('password', password.encode('ascii', 'ignore')) ) outputs = [('output',True),('expires',False)] wps = WebProcessingService(url=url, skip_caps=True) execution = wps.execute(identifier="esgf_logon", inputs=inputs, output=outputs) monitorExecution(execution) if execution.isSucceded(): credentials = execution.processOutputs[0].reference cert_expires = execution.processOutputs[1].data[0] db = mongodb(registry) user = db.users.find_one({'identifier':userid}) user['credentials'] = credentials user['cert_expires'] = cert_expires db.users.update({'identifier':userid}, user) return execution.status
def execute_process(self, userid, url, identifier, inputs, outputs, keywords=None): registry = app.conf['PYRAMID_REGISTRY'] wps = WebProcessingService(url=url, skip_caps=False) execution = wps.execute(identifier, inputs=inputs, output=outputs) db = mongodb(registry) job = add_job(db, userid, task_id = self.request.id, is_workflow = False, service = wps.identification.title, title = execution.process.identifier, abstract = getattr(execution.process, "abstract", ""), status_location = execution.statusLocation) while execution.isNotComplete(): execution.checkStatus(sleepSecs=1) job['status'] = execution.getStatus() job['status_message'] = execution.statusMessage job['is_complete'] = execution.isComplete() job['is_succeded'] = execution.isSucceded() job['progress'] = execution.percentCompleted duration = datetime.now() - job.get('created', datetime.now()) job['duration'] = str(duration).split('.')[0] if execution.isComplete(): job['finished'] = datetime.now() if execution.isSucceded(): job['progress'] = 100 log(job) else: job['status_message'] = '\n'.join(error.text for error in execution.errors) for error in execution.errors: log_error(job, error) else: log(job) db.jobs.update({'identifier': job['identifier']}, job) return execution.getStatus()
def clear_index(self): registry = app.conf['PYRAMID_REGISTRY'] db = mongodb(registry) service = registry.settings.get('solr.url') clear(service=service) db.tasks.drop()