def add_mongodb(event):
settings = event.request.registry.settings
if settings.get('db') is None:
try:
from phoenix.models import mongodb
settings['db'] = mongodb(event.request.registry)
except:
logger.exception('Could not connect to mongodb')
event.request.db = settings.get('db')
def index_thredds(self, url, maxrecords=-1, depth=2):
registry = app.conf['PYRAMID_REGISTRY']
db = mongodb(registry)
task = dict(task_id=self.request.id, url=url, status='started')
db.tasks.save(task)
service = registry.settings.get('solr.url')
try:
feed_from_thredds(service=service, catalog_url=url, depth=depth, maxrecords=maxrecords, batch_size=50000)
task['status'] = 'success'
except:
task['status'] = 'failure'
raise
finally:
db.tasks.update({'url': task['url']}, task)
def execute_workflow(self, userid, url, workflow):
registry = app.conf['PYRAMID_REGISTRY']
# generate and run dispel workflow
# TODO: fix owslib wps for unicode/yaml parameters
inputs=[('workflow', json.dumps(workflow))]
outputs=[('output', True), ('logfile', True)]
wps = WebProcessingService(url=url, skip_caps=True)
worker_wps = WebProcessingService(url=workflow['worker']['url'], skip_caps=False)
execution = wps.execute(identifier='workflow', inputs=inputs, output=outputs)
db = mongodb(registry)
job = add_job(db, userid,
task_id = self.request.id,
is_workflow = True,
service = worker_wps.identification.title,
title = workflow['worker']['identifier'],
abstract = '',
status_location = execution.statusLocation)
while execution.isNotComplete():
execution.checkStatus(sleepSecs=1)
job['status'] = execution.getStatus()
job['status_message'] = execution.statusMessage
job['is_complete'] = execution.isComplete()
job['is_succeded'] = execution.isSucceded()
job['progress'] = execution.percentCompleted
duration = datetime.now() - job.get('created', datetime.now())
job['duration'] = str(duration).split('.')[0]
if execution.isComplete():
job['finished'] = datetime.now()
if execution.isSucceded():
for output in execution.processOutputs:
if 'output' == output.identifier:
result = yaml.load(urllib.urlopen(output.reference))
job['worker_status_location'] = result['worker']['status_location']
job['progress'] = 100
log(job)
else:
job['status_message'] = '\n'.join(error.text for error in execution.errors)
for error in execution.errors:
log_error(job, error)
else:
log(job)
db.jobs.update({'identifier': job['identifier']}, job)
return execution.getStatus()
def esgf_logon(self, userid, url, openid, password):
registry = app.conf['PYRAMID_REGISTRY']
inputs = []
inputs.append( ('openid', openid.encode('ascii', 'ignore')) )
inputs.append( ('password', password.encode('ascii', 'ignore')) )
outputs = [('output',True),('expires',False)]
wps = WebProcessingService(url=url, skip_caps=True)
execution = wps.execute(identifier="esgf_logon", inputs=inputs, output=outputs)
monitorExecution(execution)
if execution.isSucceded():
credentials = execution.processOutputs[0].reference
cert_expires = execution.processOutputs[1].data[0]
db = mongodb(registry)
user = db.users.find_one({'identifier':userid})
user['credentials'] = credentials
user['cert_expires'] = cert_expires
db.users.update({'identifier':userid}, user)
return execution.status
def execute_process(self, userid, url, identifier, inputs, outputs, keywords=None):
registry = app.conf['PYRAMID_REGISTRY']
wps = WebProcessingService(url=url, skip_caps=False)
execution = wps.execute(identifier, inputs=inputs, output=outputs)
db = mongodb(registry)
job = add_job(db, userid,
task_id = self.request.id,
is_workflow = False,
service = wps.identification.title,
title = execution.process.identifier,
abstract = getattr(execution.process, "abstract", ""),
status_location = execution.statusLocation)
while execution.isNotComplete():
execution.checkStatus(sleepSecs=1)
job['status'] = execution.getStatus()
job['status_message'] = execution.statusMessage
job['is_complete'] = execution.isComplete()
job['is_succeded'] = execution.isSucceded()
job['progress'] = execution.percentCompleted
duration = datetime.now() - job.get('created', datetime.now())
job['duration'] = str(duration).split('.')[0]
if execution.isComplete():
job['finished'] = datetime.now()
if execution.isSucceded():
job['progress'] = 100
log(job)
else:
job['status_message'] = '\n'.join(error.text for error in execution.errors)
for error in execution.errors:
log_error(job, error)
else:
log(job)
db.jobs.update({'identifier': job['identifier']}, job)
return execution.getStatus()
def clear_index(self):
registry = app.conf['PYRAMID_REGISTRY']
db = mongodb(registry)
service = registry.settings.get('solr.url')
clear(service=service)
db.tasks.drop()
