def save_credentials(registry, userid, file=None, filename=None): # settings = registry.settings db = mongodb(registry) storage = registry.getUtility(IFileStorage) # store credentials.pem in storage storage_options = dict( folder="esgf_certs", extensions=('pem',), randomize=True) if file: stored_credentials = storage.save_file( file, filename=ESGF_CREDENTIALS, **storage_options) elif filename: stored_credentials = storage.save_filename( filename, **storage_options) else: raise Exception("No credentials to save. Use file or filename parameter.") # get cert infos infos = cert_infos(storage.path(stored_credentials)) # update database user = db.users.find_one({'identifier': userid}) user['openid'] = infos.get('openid') user['credentials'] = storage.url(stored_credentials) user['cert_expires'] = infos.get('expires_at') db.users.update({'identifier': userid}, user)
def generate_access_token(registry, userid, valid_in_hours=1): service = twitcher_service_factory(registry) db = mongodb(registry) collection = db.users data = {} user = collection.find_one({'identifier': userid}) if user.get('esgf_token'): try: esgf_token = refresh_token(registry, token=user['esgf_token'], userid=userid) except Exception as err: LOGGER.warn("Could not refresh token: {}".format(err)) else: data['esgf_access_token'] = esgf_token.get('access_token', '') data['esgf_slcs_service_url'] = registry.settings.get('esgf.slcs.url', '') LOGGER.debug("passing token with esgf slcs token") elif user.get('credentials'): data['esgf_credentials'] = user['credentials'] LOGGER.debug("passing token with esgf credentials") # call to service token = service.generate_token(valid_in_hours, data) if userid and token: collection.update_one( {'identifier': userid}, {'$set': {'twitcher_token': token, }}) return token
def generate_access_token(registry, userid, valid_in_hours=1): service = twitcher_service_factory(registry) db = mongodb(registry) collection = db.users data = {} user = collection.find_one({'identifier': userid}) if user.get('esgf_token'): try: esgf_token = refresh_token(registry, token=user['esgf_token'], userid=userid) except Exception as err: LOGGER.warn("Could not refresh token: {}".format(err)) else: data['esgf_access_token'] = esgf_token.get('access_token', '') data['esgf_slcs_service_url'] = registry.settings.get('esgf.slcs.url', '') LOGGER.debug("passing token with esgf slcs token") elif user.get('credentials'): data['esgf_credentials'] = user['credentials'] LOGGER.debug("passing token with esgf credentials") # call to service token = service.generate_token(valid_in_hours) if userid and token: collection.update_one( {'identifier': userid}, {'$set': {'twitcher_token': token, }}) return token
def save_credentials(registry, userid, file=None, filename=None): settings = registry.settings db = mongodb(registry) storage = registry.getUtility(IFileStorage) # store credentials.pem in storage storage_options = dict(folder="esgf_certs", extensions=('pem', ), randomize=True) if file: stored_credentials = storage.save_file(file, filename=ESGF_CREDENTIALS, **storage_options) elif filename: stored_credentials = storage.save_filename(filename, **storage_options) else: raise Exception( "No credentials to save. Use file or filename parameter.") # get cert infos infos = cert_infos(storage.path(stored_credentials)) # update database user = db.users.find_one({'identifier': userid}) user['openid'] = infos.get('openid') user['credentials'] = storage.url(stored_credentials) user['cert_expires'] = infos.get('expires_at') db.users.update({'identifier': userid}, user)
def clear_index(self): from birdfeeder import clear registry = app.conf['PYRAMID_REGISTRY'] db = mongodb(registry) service = registry.settings.get('solr.url') clear(service=service) db.tasks.drop()
def get_access_token(userid): registry = app.conf['PYRAMID_REGISTRY'] db = mongodb(registry) # refresh access token token = generate_access_token(registry, userid=userid) return token.get('access_token')
def save_token(registry, token, userid): """ Store the token in the database. """ db = mongodb(registry) db.users.update_one( {'identifier': userid}, {'$set': {'esgf_token': token, }})
def __init__(self, registry): self.settings = registry.settings self.db = mongodb(registry) self.refresh_url = '' self.introspect_url = '' self.client_id = '' self.client_secret = '' self.verify = False self.scope = 'compute'
def execute_workflow(self, userid, url, workflow): registry = app.conf['PYRAMID_REGISTRY'] db = mongodb(registry) # generate and run dispel workflow # TODO: fix owslib wps for unicode/yaml parameters logger.debug('workflow=%s', workflow) # using secure url workflow['worker']['url'] = secure_url(db, workflow['worker']['url'], userid) inputs=[('workflow', json.dumps(workflow))] logger.debug('inputs=%s', inputs) outputs=[('output', True), ('logfile', True)] wps = WebProcessingService(url=secure_url(db, url, userid), skip_caps=True, verify=False) worker_wps = WebProcessingService(url=workflow['worker']['url'], skip_caps=False, verify=False) execution = wps.execute(identifier='workflow', inputs=inputs, output=outputs) job = add_job(db, userid, task_id = self.request.id, is_workflow = True, service = worker_wps.identification.title, title = workflow['worker']['identifier'], abstract = '', status_location = execution.statusLocation) while execution.isNotComplete(): try: execution.checkStatus(sleepSecs=3) job['status'] = execution.getStatus() job['status_message'] = execution.statusMessage job['is_complete'] = execution.isComplete() job['is_succeded'] = execution.isSucceded() job['progress'] = execution.percentCompleted duration = datetime.now() - job.get('created', datetime.now()) job['duration'] = str(duration).split('.')[0] if execution.isComplete(): job['finished'] = datetime.now() if execution.isSucceded(): for output in execution.processOutputs: if 'output' == output.identifier: result = yaml.load(urllib.urlopen(output.reference)) job['worker_status_location'] = result['worker']['status_location'] job['progress'] = 100 log(job) else: job['status_message'] = '\n'.join(error.text for error in execution.errors) for error in execution.errors: log_error(job, error) else: log(job) except: logger.exception("Could not read status xml document.") else: db.jobs.update({'identifier': job['identifier']}, job) return execution.getStatus()
def index_thredds(self, url, maxrecords=-1, depth=2): registry = app.conf['PYRAMID_REGISTRY'] db = mongodb(registry) task = dict(task_id=self.request.id, url=url, status='started') db.tasks.save(task) service = registry.settings.get('solr.url') try: feed_from_thredds(service=service, catalog_url=url, depth=depth, maxrecords=maxrecords, batch_size=50000) task['status'] = 'success' except: task['status'] = 'failure' raise finally: db.tasks.update({'url': task['url']}, task)
def esgf_logon(self, userid, url, openid, password): registry = app.conf['PYRAMID_REGISTRY'] db = mongodb(registry) inputs = [] inputs.append( ('openid', openid) ) inputs.append( ('password', password) ) outputs = [('output',True),('expires',False)] wps = WebProcessingService(url=secure_url(db, url, userid), skip_caps=True, verify=False) execution = wps.execute(identifier="esgf_logon", inputs=inputs, output=outputs) monitorExecution(execution) if execution.isSucceded(): credentials = execution.processOutputs[0].reference cert_expires = execution.processOutputs[1].data[0] user = db.users.find_one({'identifier':userid}) user['openid'] = openid user['credentials'] = credentials user['cert_expires'] = cert_expires db.users.update({'identifier':userid}, user) return execution.status
def execute_process(self, userid, url, identifier, inputs, outputs, keywords=None): registry = app.conf['PYRAMID_REGISTRY'] db = mongodb(registry) wps = WebProcessingService(url=secure_url(db, url, userid), skip_caps=False, verify=False) execution = wps.execute(identifier, inputs=inputs, output=outputs) job = add_job(db, userid, task_id = self.request.id, is_workflow = False, service = wps.identification.title, title = execution.process.identifier, abstract = getattr(execution.process, "abstract", ""), status_location = execution.statusLocation) while execution.isNotComplete(): try: execution.checkStatus(sleepSecs=3) job['status'] = execution.getStatus() job['status_message'] = execution.statusMessage job['is_complete'] = execution.isComplete() job['is_succeded'] = execution.isSucceded() job['progress'] = execution.percentCompleted duration = datetime.now() - job.get('created', datetime.now()) job['duration'] = str(duration).split('.')[0] if execution.isComplete(): job['finished'] = datetime.now() if execution.isSucceded(): job['progress'] = 100 log(job) else: job['status_message'] = '\n'.join(error.text for error in execution.errors) for error in execution.errors: log_error(job, error) else: log(job) except: logger.exception("Could not read status xml document.") else: db.jobs.update({'identifier': job['identifier']}, job) return execution.getStatus()
from owslib.wps import WebProcessingService from owslib.wps import SYNC, ASYNC from phoenix.db import mongodb from phoenix.events import JobFinished from phoenix.tasks.utils import wps_headers, save_log, add_job, wait_secs from phoenix.wps import check_status from celery.utils.log import get_task_logger LOGGER = get_task_logger(__name__) @app.task(bind=True) def execute_process(self, url, service_name, identifier, inputs, outputs, async=True, userid=None, caption=None): registry = app.conf['PYRAMID_REGISTRY'] db = mongodb(registry) job = add_job( db, userid=userid, task_id=self.request.id, service_name=service_name, process_id=identifier, async=async, caption=caption) try: wps = WebProcessingService(url=url, skip_caps=False, verify=False, headers=wps_headers(userid)) # TODO: complex type detection is currently broken due to pywps bug. outputs = [('output', True)] try: # TODO: sync is non-default
def get_c4i_access_token(userid): registry = app.conf['PYRAMID_REGISTRY'] db = mongodb(registry) user = db.users.find_one(dict(identifier=userid)) return user.get('c4i_token')
def catalog_factory(registry): service_registry = twitcher_service_factory(registry) db = mongodb(registry) catalog = MongodbCatalog(db.catalog, service_registry) return catalog
def execute_workflow(self, userid, url, service_name, workflow, caption=None): registry = app.conf['PYRAMID_REGISTRY'] db = mongodb(registry) job = add_job(db, userid=userid, task_id=self.request.id, is_workflow=True, service_name=service_name, process_id=workflow['worker']['identifier'], caption=caption) try: # generate and run dispel workflow # TODO: fix owslib wps for unicode/yaml parameters # logger.debug('workflow=%s', workflow) headers = wps_headers(userid) # TODO: handle access token in workflow # workflow['worker']['url'] = build_get_url( # workflow['worker']['url'], # {'access_token': headers.get('Access-Token', '')}) logger.debug('workflow=%s', workflow) inputs = [( 'workflow', ComplexDataInput( # TODO: pywps-4 expects base64 encoding when not set to '' dump_json(workflow), mimeType="text/yaml", encoding=""))] outputs = [('output', True), ('logfile', True)] wps = WebProcessingService(url=url, skip_caps=True, verify=False, headers=headers) # worker_wps = WebProcessingService(url=workflow['worker']['url'], # skip_caps=False, verify=False) execution = wps.execute(identifier='workflow', inputs=inputs, output=outputs, lineage=True) # job['service'] = worker_wps.identification.title # job['title'] = getattr(execution.process, "title") # job['abstract'] = getattr(execution.process, "abstract") job['status_location'] = execution.statusLocation job['response'] = etree.tostring(execution.response) logger.debug("job init done %s ...", self.request.id) run_step = 0 num_retries = 0 while execution.isNotComplete(): if num_retries >= 5: raise Exception( "Could not read status document after 5 retries. Giving up." ) try: execution = check_status(url=execution.statusLocation, verify=False, sleep_secs=wait_secs(run_step)) job['response'] = etree.tostring(execution.response) job['status'] = execution.getStatus() job['status_message'] = execution.statusMessage job['progress'] = execution.percentCompleted duration = datetime.now() - job.get('created', datetime.now()) job['duration'] = str(duration).split('.')[0] if execution.isComplete(): job['finished'] = datetime.now() if execution.isSucceded(): for output in execution.processOutputs: if 'output' == output.identifier: result = yaml.load( urllib.urlopen(output.reference)) job['worker_status_location'] = result[ 'worker']['status_location'] job['progress'] = 100 save_log(job) else: job['status_message'] = '\n'.join( error.text for error in execution.errors) for error in execution.errors: save_log(job, error) else: save_log(job) except Exception: num_retries += 1 logger.exception( "Could not read status xml document for job %s. Trying again ...", self.request.id) sleep(1) else: logger.debug("update job %s ...", self.request.id) num_retries = 0 run_step += 1 db.jobs.update({'identifier': job['identifier']}, job) except Exception as exc: logger.exception("Failed to run Job") job['status'] = "ProcessFailed" job['status_message'] = "Failed to run Job. %s" % exc.message finally: save_log(job) db.jobs.update({'identifier': job['identifier']}, job) registry.notify(JobFinished(job)) return job['status']
def clear_index(self): registry = app.conf['PYRAMID_REGISTRY'] db = mongodb(registry) service = registry.settings.get('solr.url') clear(service=service) db.tasks.drop()