def save_credentials(registry, userid, file=None, filename=None):
# settings = registry.settings
db = mongodb(registry)
storage = registry.getUtility(IFileStorage)
# store credentials.pem in storage
storage_options = dict(
folder="esgf_certs",
extensions=('pem',),
randomize=True)
if file:
stored_credentials = storage.save_file(
file,
filename=ESGF_CREDENTIALS,
**storage_options)
elif filename:
stored_credentials = storage.save_filename(
filename,
**storage_options)
else:
raise Exception("No credentials to save. Use file or filename parameter.")
# get cert infos
infos = cert_infos(storage.path(stored_credentials))
# update database
user = db.users.find_one({'identifier': userid})
user['openid'] = infos.get('openid')
user['credentials'] = storage.url(stored_credentials)
user['cert_expires'] = infos.get('expires_at')
db.users.update({'identifier': userid}, user)
def generate_access_token(registry, userid, valid_in_hours=1):
service = twitcher_service_factory(registry)
db = mongodb(registry)
collection = db.users
data = {}
user = collection.find_one({'identifier': userid})
if user.get('esgf_token'):
try:
esgf_token = refresh_token(registry, token=user['esgf_token'], userid=userid)
except Exception as err:
LOGGER.warn("Could not refresh token: {}".format(err))
else:
data['esgf_access_token'] = esgf_token.get('access_token', '')
data['esgf_slcs_service_url'] = registry.settings.get('esgf.slcs.url', '')
LOGGER.debug("passing token with esgf slcs token")
elif user.get('credentials'):
data['esgf_credentials'] = user['credentials']
LOGGER.debug("passing token with esgf credentials")
# call to service
token = service.generate_token(valid_in_hours, data)
if userid and token:
collection.update_one(
{'identifier': userid},
{'$set': {'twitcher_token': token, }})
return token
def generate_access_token(registry, userid, valid_in_hours=1):
service = twitcher_service_factory(registry)
db = mongodb(registry)
collection = db.users
data = {}
user = collection.find_one({'identifier': userid})
if user.get('esgf_token'):
try:
esgf_token = refresh_token(registry, token=user['esgf_token'], userid=userid)
except Exception as err:
LOGGER.warn("Could not refresh token: {}".format(err))
else:
data['esgf_access_token'] = esgf_token.get('access_token', '')
data['esgf_slcs_service_url'] = registry.settings.get('esgf.slcs.url', '')
LOGGER.debug("passing token with esgf slcs token")
elif user.get('credentials'):
data['esgf_credentials'] = user['credentials']
LOGGER.debug("passing token with esgf credentials")
# call to service
token = service.generate_token(valid_in_hours)
if userid and token:
collection.update_one(
{'identifier': userid},
{'$set': {'twitcher_token': token, }})
return token
def save_credentials(registry, userid, file=None, filename=None):
settings = registry.settings
db = mongodb(registry)
storage = registry.getUtility(IFileStorage)
# store credentials.pem in storage
storage_options = dict(folder="esgf_certs",
extensions=('pem', ),
randomize=True)
if file:
stored_credentials = storage.save_file(file,
filename=ESGF_CREDENTIALS,
**storage_options)
elif filename:
stored_credentials = storage.save_filename(filename, **storage_options)
else:
raise Exception(
"No credentials to save. Use file or filename parameter.")
# get cert infos
infos = cert_infos(storage.path(stored_credentials))
# update database
user = db.users.find_one({'identifier': userid})
user['openid'] = infos.get('openid')
user['credentials'] = storage.url(stored_credentials)
user['cert_expires'] = infos.get('expires_at')
db.users.update({'identifier': userid}, user)
def clear_index(self):
from birdfeeder import clear
registry = app.conf['PYRAMID_REGISTRY']
db = mongodb(registry)
service = registry.settings.get('solr.url')
clear(service=service)
db.tasks.drop()
def get_access_token(userid):
registry = app.conf['PYRAMID_REGISTRY']
db = mongodb(registry)
# refresh access token
token = generate_access_token(registry, userid=userid)
return token.get('access_token')
def save_token(registry, token, userid):
"""
Store the token in the database.
"""
db = mongodb(registry)
db.users.update_one(
{'identifier': userid},
{'$set': {'esgf_token': token, }})
def save_token(registry, token, userid):
"""
Store the token in the database.
"""
db = mongodb(registry)
db.users.update_one(
{'identifier': userid},
{'$set': {'esgf_token': token, }})
def __init__(self, registry):
self.settings = registry.settings
self.db = mongodb(registry)
self.refresh_url = ''
self.introspect_url = ''
self.client_id = ''
self.client_secret = ''
self.verify = False
self.scope = 'compute'
def execute_workflow(self, userid, url, workflow):
registry = app.conf['PYRAMID_REGISTRY']
db = mongodb(registry)
# generate and run dispel workflow
# TODO: fix owslib wps for unicode/yaml parameters
logger.debug('workflow=%s', workflow)
# using secure url
workflow['worker']['url'] = secure_url(db, workflow['worker']['url'], userid)
inputs=[('workflow', json.dumps(workflow))]
logger.debug('inputs=%s', inputs)
outputs=[('output', True), ('logfile', True)]
wps = WebProcessingService(url=secure_url(db, url, userid), skip_caps=True, verify=False)
worker_wps = WebProcessingService(url=workflow['worker']['url'], skip_caps=False, verify=False)
execution = wps.execute(identifier='workflow', inputs=inputs, output=outputs)
job = add_job(db, userid,
task_id = self.request.id,
is_workflow = True,
service = worker_wps.identification.title,
title = workflow['worker']['identifier'],
abstract = '',
status_location = execution.statusLocation)
while execution.isNotComplete():
try:
execution.checkStatus(sleepSecs=3)
job['status'] = execution.getStatus()
job['status_message'] = execution.statusMessage
job['is_complete'] = execution.isComplete()
job['is_succeded'] = execution.isSucceded()
job['progress'] = execution.percentCompleted
duration = datetime.now() - job.get('created', datetime.now())
job['duration'] = str(duration).split('.')[0]
if execution.isComplete():
job['finished'] = datetime.now()
if execution.isSucceded():
for output in execution.processOutputs:
if 'output' == output.identifier:
result = yaml.load(urllib.urlopen(output.reference))
job['worker_status_location'] = result['worker']['status_location']
job['progress'] = 100
log(job)
else:
job['status_message'] = '\n'.join(error.text for error in execution.errors)
for error in execution.errors:
log_error(job, error)
else:
log(job)
except:
logger.exception("Could not read status xml document.")
else:
db.jobs.update({'identifier': job['identifier']}, job)
return execution.getStatus()
def index_thredds(self, url, maxrecords=-1, depth=2):
registry = app.conf['PYRAMID_REGISTRY']
db = mongodb(registry)
task = dict(task_id=self.request.id, url=url, status='started')
db.tasks.save(task)
service = registry.settings.get('solr.url')
try:
feed_from_thredds(service=service, catalog_url=url, depth=depth, maxrecords=maxrecords, batch_size=50000)
task['status'] = 'success'
except:
task['status'] = 'failure'
raise
finally:
db.tasks.update({'url': task['url']}, task)
def index_thredds(self, url, maxrecords=-1, depth=2):
registry = app.conf['PYRAMID_REGISTRY']
db = mongodb(registry)
task = dict(task_id=self.request.id, url=url, status='started')
db.tasks.save(task)
service = registry.settings.get('solr.url')
try:
feed_from_thredds(service=service, catalog_url=url, depth=depth, maxrecords=maxrecords, batch_size=50000)
task['status'] = 'success'
except:
task['status'] = 'failure'
raise
finally:
db.tasks.update({'url': task['url']}, task)
def esgf_logon(self, userid, url, openid, password):
registry = app.conf['PYRAMID_REGISTRY']
db = mongodb(registry)
inputs = []
inputs.append( ('openid', openid) )
inputs.append( ('password', password) )
outputs = [('output',True),('expires',False)]
wps = WebProcessingService(url=secure_url(db, url, userid), skip_caps=True, verify=False)
execution = wps.execute(identifier="esgf_logon", inputs=inputs, output=outputs)
monitorExecution(execution)
if execution.isSucceded():
credentials = execution.processOutputs[0].reference
cert_expires = execution.processOutputs[1].data[0]
user = db.users.find_one({'identifier':userid})
user['openid'] = openid
user['credentials'] = credentials
user['cert_expires'] = cert_expires
db.users.update({'identifier':userid}, user)
return execution.status
def execute_process(self, userid, url, identifier, inputs, outputs, keywords=None):
registry = app.conf['PYRAMID_REGISTRY']
db = mongodb(registry)
wps = WebProcessingService(url=secure_url(db, url, userid), skip_caps=False, verify=False)
execution = wps.execute(identifier, inputs=inputs, output=outputs)
job = add_job(db, userid,
task_id = self.request.id,
is_workflow = False,
service = wps.identification.title,
title = execution.process.identifier,
abstract = getattr(execution.process, "abstract", ""),
status_location = execution.statusLocation)
while execution.isNotComplete():
try:
execution.checkStatus(sleepSecs=3)
job['status'] = execution.getStatus()
job['status_message'] = execution.statusMessage
job['is_complete'] = execution.isComplete()
job['is_succeded'] = execution.isSucceded()
job['progress'] = execution.percentCompleted
duration = datetime.now() - job.get('created', datetime.now())
job['duration'] = str(duration).split('.')[0]
if execution.isComplete():
job['finished'] = datetime.now()
if execution.isSucceded():
job['progress'] = 100
log(job)
else:
job['status_message'] = '\n'.join(error.text for error in execution.errors)
for error in execution.errors:
log_error(job, error)
else:
log(job)
except:
logger.exception("Could not read status xml document.")
else:
db.jobs.update({'identifier': job['identifier']}, job)
return execution.getStatus()
from owslib.wps import WebProcessingService
from owslib.wps import SYNC, ASYNC
from phoenix.db import mongodb
from phoenix.events import JobFinished
from phoenix.tasks.utils import wps_headers, save_log, add_job, wait_secs
from phoenix.wps import check_status
from celery.utils.log import get_task_logger
LOGGER = get_task_logger(__name__)
@app.task(bind=True)
def execute_process(self, url, service_name, identifier, inputs, outputs, async=True, userid=None, caption=None):
registry = app.conf['PYRAMID_REGISTRY']
db = mongodb(registry)
job = add_job(
db,
userid=userid,
task_id=self.request.id,
service_name=service_name,
process_id=identifier,
async=async,
caption=caption)
try:
wps = WebProcessingService(url=url, skip_caps=False, verify=False, headers=wps_headers(userid))
# TODO: complex type detection is currently broken due to pywps bug.
outputs = [('output', True)]
try:
# TODO: sync is non-default
def get_c4i_access_token(userid):
registry = app.conf['PYRAMID_REGISTRY']
db = mongodb(registry)
user = db.users.find_one(dict(identifier=userid))
return user.get('c4i_token')
def catalog_factory(registry):
service_registry = twitcher_service_factory(registry)
db = mongodb(registry)
catalog = MongodbCatalog(db.catalog, service_registry)
return catalog
def execute_workflow(self, userid, url, service_name, workflow, caption=None):
registry = app.conf['PYRAMID_REGISTRY']
db = mongodb(registry)
job = add_job(db,
userid=userid,
task_id=self.request.id,
is_workflow=True,
service_name=service_name,
process_id=workflow['worker']['identifier'],
caption=caption)
try:
# generate and run dispel workflow
# TODO: fix owslib wps for unicode/yaml parameters
# logger.debug('workflow=%s', workflow)
headers = wps_headers(userid)
# TODO: handle access token in workflow
# workflow['worker']['url'] = build_get_url(
# workflow['worker']['url'],
# {'access_token': headers.get('Access-Token', '')})
logger.debug('workflow=%s', workflow)
inputs = [(
'workflow',
ComplexDataInput(
# TODO: pywps-4 expects base64 encoding when not set to ''
dump_json(workflow),
mimeType="text/yaml",
encoding=""))]
outputs = [('output', True), ('logfile', True)]
wps = WebProcessingService(url=url,
skip_caps=True,
verify=False,
headers=headers)
# worker_wps = WebProcessingService(url=workflow['worker']['url'],
# skip_caps=False, verify=False)
execution = wps.execute(identifier='workflow',
inputs=inputs,
output=outputs,
lineage=True)
# job['service'] = worker_wps.identification.title
# job['title'] = getattr(execution.process, "title")
# job['abstract'] = getattr(execution.process, "abstract")
job['status_location'] = execution.statusLocation
job['response'] = etree.tostring(execution.response)
logger.debug("job init done %s ...", self.request.id)
run_step = 0
num_retries = 0
while execution.isNotComplete():
if num_retries >= 5:
raise Exception(
"Could not read status document after 5 retries. Giving up."
)
try:
execution = check_status(url=execution.statusLocation,
verify=False,
sleep_secs=wait_secs(run_step))
job['response'] = etree.tostring(execution.response)
job['status'] = execution.getStatus()
job['status_message'] = execution.statusMessage
job['progress'] = execution.percentCompleted
duration = datetime.now() - job.get('created', datetime.now())
job['duration'] = str(duration).split('.')[0]
if execution.isComplete():
job['finished'] = datetime.now()
if execution.isSucceded():
for output in execution.processOutputs:
if 'output' == output.identifier:
result = yaml.load(
urllib.urlopen(output.reference))
job['worker_status_location'] = result[
'worker']['status_location']
job['progress'] = 100
save_log(job)
else:
job['status_message'] = '\n'.join(
error.text for error in execution.errors)
for error in execution.errors:
save_log(job, error)
else:
save_log(job)
except Exception:
num_retries += 1
logger.exception(
"Could not read status xml document for job %s. Trying again ...",
self.request.id)
sleep(1)
else:
logger.debug("update job %s ...", self.request.id)
num_retries = 0
run_step += 1
db.jobs.update({'identifier': job['identifier']}, job)
except Exception as exc:
logger.exception("Failed to run Job")
job['status'] = "ProcessFailed"
job['status_message'] = "Failed to run Job. %s" % exc.message
finally:
save_log(job)
db.jobs.update({'identifier': job['identifier']}, job)
registry.notify(JobFinished(job))
return job['status']
def clear_index(self):
registry = app.conf['PYRAMID_REGISTRY']
db = mongodb(registry)
service = registry.settings.get('solr.url')
clear(service=service)
db.tasks.drop()
