def __init__(self, provider_def):
"""
Initialize object
:param provider_def: provider definition dict
:returns: `geomet_data_registry.layer.base.BaseLayer`
"""
# list of dictionaries
self.items = []
self.model_run_list = []
self.file_creation_datetime = None
self.receive_datetime = datetime.now()
self.identify_datetime = None
self.register_datetime = None
self.filepath = None
self.model = None
self.model_run = None
self.wx_variable = None
self.name = provider_def['name']
self.store = load_plugin('store', STORE_PROVIDER_DEF)
self.tileindex = load_plugin('tileindex', TILEINDEX_PROVIDER_DEF)
def handle(self):
"""
handle incoming file
:returns: `bool` of status result
"""
LOGGER.debug('Detecting filename pattern')
for key in DATASET_HANDLERS.keys():
if key in self.filepath:
plugin_def = {
'type': DATASET_HANDLERS[key],
}
LOGGER.debug('Loading plugin {}'.format(plugin_def))
self.layer_plugin = load_plugin('layer', plugin_def)
if self.layer_plugin is None:
msg = 'Plugin not found'
LOGGER.error(msg)
raise RuntimeError(msg)
LOGGER.debug('Identifying file')
identify_status = self.layer_plugin.identify(self.filepath)
if identify_status:
self.layer_plugin.identify_datetime = get_today_and_now()
LOGGER.debug('Registering file')
self.layer_plugin.register()
if self.layer_plugin.new_key_store:
self.layer_plugin.add_time_key()
return True
def set_key(ctx, key, config, raw):
"""populate store"""
if all([key is None, config is None]):
raise click.ClickException('Missing --key/-k or --config/-c option')
provider_def = {'type': STORE_TYPE, 'url': STORE_URL}
st = load_plugin('store', provider_def)
try:
with codecs.open(config) as ff:
yml_dict = load(ff, Loader=Loader)
string_ = json.dumps(yml_dict)
if raw:
click.echo('Setting {} key in store ({}).'.format(key, st.url))
st.set_key(key, string_, raw=True)
else:
click.echo(
'Setting geomet-data-registry_{} key in store ({}).'.
format(key, st.url))
st.set_key(key, string_)
except StoreError as err:
raise click.ClickException(err)
click.echo('Done')
def get_key(ctx, key, raw):
"""get key from store"""
if all([key is None]):
raise click.ClickException('Missing --key/-k')
provider_def = {'type': STORE_TYPE, 'url': STORE_URL}
st = load_plugin('store', provider_def)
try:
if raw:
click.echo('Getting {} key from store ({}).'.format(key, st.url))
retrieved_key = st.get_key(key, raw=True)
else:
click.echo(
'Getting geomet-data-registry_{} key from store ({}).'.format(
key, st.url))
retrieved_key = st.get_key(key)
if retrieved_key:
try:
click.echo('{}'.format(
json_pretty_print(json.loads(retrieved_key))))
except ValueError:
click.echo(retrieved_key)
except StoreError as err:
raise click.ClickException(err)
click.echo('Done')
def list_keys(ctx, pattern=None):
"""list all keys in store"""
provider_def = {'type': STORE_TYPE, 'url': STORE_URL}
st = load_plugin('store', provider_def)
try:
click.echo(json_pretty_print(st.list_keys(pattern)))
except StoreError as err:
raise click.ClickException(err)
click.echo('Done')
def teardown(ctx, group=None):
"""delete store"""
provider_def = {'type': STORE_TYPE, 'url': STORE_URL, 'group': group}
st = load_plugin('store', provider_def)
try:
click.echo('Deleting store {}'.format(st.url))
st.teardown()
except StoreError as err:
raise click.ClickException(err)
click.echo('Done')
def handle(self):
"""
handle incoming file
:returns: `bool` of status result
"""
LOGGER.debug('Detecting filename pattern')
for key in DATASET_HANDLERS.keys():
if key in self.filepath:
plugin_def = {
'type': DATASET_HANDLERS[key],
}
LOGGER.debug('Loading plugin {}'.format(plugin_def))
self.layer_plugin = load_plugin('layer', plugin_def)
if self.layer_plugin is None:
msg = 'Plugin not found'
LOGGER.error(msg)
raise RuntimeError(msg)
LOGGER.debug('Identifying file')
identify_status = self.layer_plugin.identify(self.filepath)
if identify_status:
self.layer_plugin.identify_datetime = datetime.now().isoformat()
LOGGER.debug('Registering file')
register_status = self.layer_plugin.register()
if register_status:
register_datetime_ = datetime.now()
self.layer_plugin.register_datetime = register_datetime_
query_dict = {
'filepath': '*{}*'.format(self.layer_plugin.filepath)
}
update_dict = {
'register_datetime': register_datetime_
}
self.layer_plugin.tileindex.update_by_query(
query_dict, update_dict)
return True
def teardown(ctx, group=None):
"""delete tileindex"""
provider_def = {
'type': TILEINDEX_TYPE,
'url': TILEINDEX_BASEURL,
'name': TILEINDEX_NAME,
'group': group
}
ti = load_plugin('tileindex', provider_def)
try:
click.echo('Deleting tileindex {}'.format(ti.fullpath))
ti.teardown()
except TileIndexError as err:
raise click.ClickException(err)
click.echo('Done')
def get_key(ctx, key):
"""get key from store"""
if all([key is None]):
raise click.ClickException('Missing --key/-k')
provider_def = {'type': STORE_TYPE, 'url': STORE_URL}
st = load_plugin('store', provider_def)
try:
click.echo('Getting {} key from store ({}).'.format(key, st.url))
retrived_key = json.loads(st.get_key(key))
if retrived_key:
click.echo('{}'.format(json_pretty_print(retrived_key)))
except StoreError as err:
raise click.ClickException(err)
click.echo('Done')
def list_keys(ctx, raw, pattern=None):
"""list all keys in store"""
provider_def = {'type': STORE_TYPE, 'url': STORE_URL}
st = load_plugin('store', provider_def)
try:
pattern = 'geomet-data-registry*{}'.format(pattern if pattern else '')
if raw:
keys = st.list_keys(pattern)
else:
keys = [
remove_prefix(key, 'geomet-data-registry_')
for key in st.list_keys(pattern)
]
click.echo(json_pretty_print(keys))
except StoreError as err:
raise click.ClickException(err)
click.echo('Done')
