def update_model_in_registry(pkg, loaded):
try:
registry = PackageRegistry(self.output_es)
except Exception as exception:
logger.info('STATUS: FAILED TO UPDATE MODEL')
logger.exception(exception)
return
owner = self.owner_id
dataset_name = '{}_{}'.format(
self.config.get(CONFIG_TAXONOMY_ID),
self.config.get(CONFIG_EXTRA_METADATA_DATASET_NAME),
)
dataset_id = '{}:{}'.format(
owner,
dataset_name
)
private = self.config.get(CONFIG_EXTRA_PRIVATE)
# TODO: replace by real URL
datapackage_url = 'datapackage-url'
datapackage = copy.deepcopy(pkg.descriptor)
datapackage.update(dict(
private=private,
owner=owner
))
registry.save_model(
dataset_id,
datapackage_url,
datapackage,
datapackage.get('babbage_model'),
dataset_name,
'openspending',
'done' if loaded else 'loading-data',
loaded
)
logger.info('STATUS: UPDATED MODEL')
def modify_datapackage(dp, parameters, *_):
dataset_id = parameters['dataset-id']
loaded = parameters.get('loaded')
private = parameters.get('private')
owner = parameters.get('owner')
datapackage_url = parameters.get('datapackage-url')
if ES_ADDRESS:
registry = PackageRegistry(ES_ADDRESS)
datapackage = copy.deepcopy(dp)
params = {}
if 'babbageModel' in datapackage:
model = datapackage['babbageModel']
del datapackage['babbageModel']
params['model'] = model
if private is not None:
datapackage['private'] = private
if owner is not None:
datapackage['owner'] = owner
if datapackage_url:
params['datapackage_url'] = datapackage_url
params['datapackage'] = datapackage
if loaded is not None:
params['loaded'] = loaded
params['loading_status'] = 'done' if loaded else 'loading-data'
registry.update_model(dataset_id, **params)
return dp
def setUp(self):
# Clean index
self.es = Elasticsearch(hosts=[LOCAL_ELASTICSEARCH])
try:
self.es.indices.delete(index='packages')
except NotFoundError:
pass
self.pr = PackageRegistry(es_instance=self.es)
def setUp(self):
# Clean index
self.es = Elasticsearch(hosts=[LOCAL_ELASTICSEARCH])
try:
self.es.indices.delete(index=PACKAGES_INDEX_NAME)
except NotFoundError:
pass
self.pr = PackageRegistry(es_instance=self.es,
index_name=PACKAGES_INDEX_NAME)
def remove_package(id, index, es_host):
"""Remove package(s) with id."""
if es_host is None:
raise click.UsageError('No es-host provided. See help for details.')
es = Elasticsearch(hosts=[es_host], use_ssl='https' in es_host)
pr = PackageRegistry(es_instance=es, index_name=index)
for i in id:
ret = pr.delete_model(i)
if ret is False:
click.echo('Package "{}" not found in index.'.format(i))
def setUp(self):
# Clean index
self.es = Elasticsearch(hosts=[LOCAL_ELASTICSEARCH])
try:
self.es.indices.delete(index='users')
self.es.indices.delete(index='packages')
except NotFoundError:
pass
self.es.indices.create('users')
time.sleep(1)
self.pr = PackageRegistry(es_connection_string=LOCAL_ELASTICSEARCH)
self.pr.save_model(self.DATASET_NAME, 'datapackage_url', {}, {},
'dataset', 'author', '', True)
def create_app():
logging.info('OS-API create_app')
_app = Flask('os_api')
_app.wsgi_app = ProxyFix(_app.wsgi_app)
registry = PackageRegistry(es_connection_string=os.environ.get(
'OS_ELASTICSEARCH_ADDRESS', 'localhost:9200'))
manager = OSCubeManager(get_engine(), registry)
logging.info('OS-API configuring query blueprints')
_app.register_blueprint(configure_babbage_api(_app, manager),
url_prefix='/api/3')
_app.register_blueprint(infoAPI, url_prefix='/api/3')
_app.extensions['model_registry'] = registry
CORS(_app)
Sentry(_app, dsn=os.environ.get('SENTRY_DSN', ''))
logging.info('OS-API app created')
return _app
if target_index is None:
target_index = 'packages-1'
assert source_index != target_index
logging.info('SOURCE INDEX %s', source_index)
logging.info('TARGET INDEX %s', target_index)
logging.info('BACKUP INDEX %s', backup_index)
try:
logging.info('DELETING TARGET INDEX')
es.indices.delete(target_index)
except NotFoundError:
logging.info('TARGET INDEX NOT FOUND')
source_pr = PackageRegistry(es_instance=es, index_name=source_index)
backup_pr = PackageRegistry(es_instance=es, index_name=backup_index)
reindex(source_pr, backup_pr)
if es.indices.exists_alias(source_index, 'packages'):
es.indices.delete_alias(source_index, 'packages')
target_pr = PackageRegistry(es_instance=es, index_name=target_index)
reindex(source_pr, target_pr)
es.indices.delete(source_index)
es.indices.put_alias(target_index, 'packages')
# Find orphan DB tables
used_tables = set()
import os
from elasticsearch import Elasticsearch
from os_package_registry import PackageRegistry
# ## ElasticSearch
_es_engine = None
def _get_es_engine():
global _es_engine
if _es_engine is None:
es_host = os.environ['OS_ELASTICSEARCH_ADDRESS']
_es_engine = Elasticsearch(hosts=[es_host], use_ssl='https' in es_host)
return _es_engine
# Package Registry
package_registry = PackageRegistry(es_instance=_get_es_engine())
import os
from elasticsearch import Elasticsearch
from os_package_registry import PackageRegistry
# ## ElasticSearch
_es_engine = None
PACKAGES_INDEX_NAME = os.environ.get('OS_ES_PACKAGES_INDEX_NAME', 'packages')
def _get_es_engine():
global _es_engine
if _es_engine is None:
es_host = os.environ['OS_ELASTICSEARCH_ADDRESS']
_es_engine = Elasticsearch(hosts=[es_host], use_ssl='https' in es_host)
return _es_engine
package_registry = PackageRegistry(es_instance=_get_es_engine(),
index_name=PACKAGES_INDEX_NAME)