def update_model_in_registry(pkg, loaded): try: registry = PackageRegistry(self.output_es) except Exception as exception: logger.info('STATUS: FAILED TO UPDATE MODEL') logger.exception(exception) return owner = self.owner_id dataset_name = '{}_{}'.format( self.config.get(CONFIG_TAXONOMY_ID), self.config.get(CONFIG_EXTRA_METADATA_DATASET_NAME), ) dataset_id = '{}:{}'.format( owner, dataset_name ) private = self.config.get(CONFIG_EXTRA_PRIVATE) # TODO: replace by real URL datapackage_url = 'datapackage-url' datapackage = copy.deepcopy(pkg.descriptor) datapackage.update(dict( private=private, owner=owner )) registry.save_model( dataset_id, datapackage_url, datapackage, datapackage.get('babbage_model'), dataset_name, 'openspending', 'done' if loaded else 'loading-data', loaded ) logger.info('STATUS: UPDATED MODEL')
def modify_datapackage(dp, parameters, *_): dataset_id = parameters['dataset-id'] loaded = parameters.get('loaded') private = parameters.get('private') owner = parameters.get('owner') datapackage_url = parameters.get('datapackage-url') if ES_ADDRESS: registry = PackageRegistry(ES_ADDRESS) datapackage = copy.deepcopy(dp) params = {} if 'babbageModel' in datapackage: model = datapackage['babbageModel'] del datapackage['babbageModel'] params['model'] = model if private is not None: datapackage['private'] = private if owner is not None: datapackage['owner'] = owner if datapackage_url: params['datapackage_url'] = datapackage_url params['datapackage'] = datapackage if loaded is not None: params['loaded'] = loaded params['loading_status'] = 'done' if loaded else 'loading-data' registry.update_model(dataset_id, **params) return dp
def setUp(self): # Clean index self.es = Elasticsearch(hosts=[LOCAL_ELASTICSEARCH]) try: self.es.indices.delete(index='packages') except NotFoundError: pass self.pr = PackageRegistry(es_instance=self.es)
def setUp(self): # Clean index self.es = Elasticsearch(hosts=[LOCAL_ELASTICSEARCH]) try: self.es.indices.delete(index=PACKAGES_INDEX_NAME) except NotFoundError: pass self.pr = PackageRegistry(es_instance=self.es, index_name=PACKAGES_INDEX_NAME)
def remove_package(id, index, es_host): """Remove package(s) with id.""" if es_host is None: raise click.UsageError('No es-host provided. See help for details.') es = Elasticsearch(hosts=[es_host], use_ssl='https' in es_host) pr = PackageRegistry(es_instance=es, index_name=index) for i in id: ret = pr.delete_model(i) if ret is False: click.echo('Package "{}" not found in index.'.format(i))
def setUp(self): # Clean index self.es = Elasticsearch(hosts=[LOCAL_ELASTICSEARCH]) try: self.es.indices.delete(index='users') self.es.indices.delete(index='packages') except NotFoundError: pass self.es.indices.create('users') time.sleep(1) self.pr = PackageRegistry(es_connection_string=LOCAL_ELASTICSEARCH) self.pr.save_model(self.DATASET_NAME, 'datapackage_url', {}, {}, 'dataset', 'author', '', True)
def create_app(): logging.info('OS-API create_app') _app = Flask('os_api') _app.wsgi_app = ProxyFix(_app.wsgi_app) registry = PackageRegistry(es_connection_string=os.environ.get( 'OS_ELASTICSEARCH_ADDRESS', 'localhost:9200')) manager = OSCubeManager(get_engine(), registry) logging.info('OS-API configuring query blueprints') _app.register_blueprint(configure_babbage_api(_app, manager), url_prefix='/api/3') _app.register_blueprint(infoAPI, url_prefix='/api/3') _app.extensions['model_registry'] = registry CORS(_app) Sentry(_app, dsn=os.environ.get('SENTRY_DSN', '')) logging.info('OS-API app created') return _app
if target_index is None: target_index = 'packages-1' assert source_index != target_index logging.info('SOURCE INDEX %s', source_index) logging.info('TARGET INDEX %s', target_index) logging.info('BACKUP INDEX %s', backup_index) try: logging.info('DELETING TARGET INDEX') es.indices.delete(target_index) except NotFoundError: logging.info('TARGET INDEX NOT FOUND') source_pr = PackageRegistry(es_instance=es, index_name=source_index) backup_pr = PackageRegistry(es_instance=es, index_name=backup_index) reindex(source_pr, backup_pr) if es.indices.exists_alias(source_index, 'packages'): es.indices.delete_alias(source_index, 'packages') target_pr = PackageRegistry(es_instance=es, index_name=target_index) reindex(source_pr, target_pr) es.indices.delete(source_index) es.indices.put_alias(target_index, 'packages') # Find orphan DB tables used_tables = set()
import os from elasticsearch import Elasticsearch from os_package_registry import PackageRegistry # ## ElasticSearch _es_engine = None def _get_es_engine(): global _es_engine if _es_engine is None: es_host = os.environ['OS_ELASTICSEARCH_ADDRESS'] _es_engine = Elasticsearch(hosts=[es_host], use_ssl='https' in es_host) return _es_engine # Package Registry package_registry = PackageRegistry(es_instance=_get_es_engine())
import os from elasticsearch import Elasticsearch from os_package_registry import PackageRegistry # ## ElasticSearch _es_engine = None PACKAGES_INDEX_NAME = os.environ.get('OS_ES_PACKAGES_INDEX_NAME', 'packages') def _get_es_engine(): global _es_engine if _es_engine is None: es_host = os.environ['OS_ELASTICSEARCH_ADDRESS'] _es_engine = Elasticsearch(hosts=[es_host], use_ssl='https' in es_host) return _es_engine package_registry = PackageRegistry(es_instance=_get_es_engine(), index_name=PACKAGES_INDEX_NAME)