def __init__(self, plugin_def):
"""initializer"""
BaseLoader.__init__(self)
self.ES = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH)
self.filepath = None
self.region_name_code = None
self.language = None
self.root = None
self.area = {}
self.items = []
# create marine weather indices if it don't exist
for item in MAPPINGS:
if not self.ES.indices.exists(INDEX_NAME.format(item)):
SETTINGS['mappings']['properties']['properties'][
'properties'
] = MAPPINGS[
item
] # noqa
self.ES.indices.create(
index=INDEX_NAME.format(item),
body=SETTINGS,
request_timeout=MSC_PYGEOAPI_ES_TIMEOUT,
)
def delete_index(ctx, index_name):
"""
Delete a particular ES index with a given name as argument or all if no
argument is passed
"""
es = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH)
if index_name:
if click.confirm(
'Are you sure you want to delete ES index named: {}?'.format(
click.style(index_name, fg='red')
),
abort=True,
):
LOGGER.info('Deleting ES index {}'.format(index_name))
es.indices.delete(index=index_name)
return True
else:
if click.confirm(
'Are you sure you want to delete {} marine forecast'
' indices ({})?'.format(
click.style('ALL', fg='red'),
click.style(", ".join(INDICES), fg='red'),
),
abort=True,
):
es.indices.delete(index=",".join(INDICES))
return True
def delete_index(ctx):
"""Delete bulletins index"""
es = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH)
if es.indices.exists(INDEX_NAME):
es.indices.delete(INDEX_NAME)
def clean_records(ctx, days):
"""Delete old documents"""
es = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH)
today = datetime.now().replace(hour=0, minute=0)
older_than = (today - timedelta(days=days)).strftime('%Y-%m-%dT%H:%M')
click.echo('Deleting documents older than {} ({} full days)'
.format(older_than.replace('T', ' '), days))
query = {
'query': {
'range': {
'properties.DATETIME': {
'lt': older_than,
'format': 'strict_date_hour_minute'
}
}
}
}
response = es.delete_by_query(index=INDEX_NAME, body=query,
request_timeout=90)
click.echo('Deleted {} documents'.format(response['deleted']))
if len(response['failures']) > 0:
click.echo('Failed to delete {} documents in time range'
.format(len(response['failures'])))
def delete_index(ctx):
"""Delete hydrometric realtime indexes"""
es = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH)
if es.indices.exists(INDEX_NAME):
es.indices.delete(INDEX_NAME)
def __init__(self, plugin_def):
"""initializer"""
BaseLoader.__init__(self)
self.ES = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH)
if not self.ES.indices.exists(INDEX_NAME):
self.ES.indices.create(index=INDEX_NAME,
body=SETTINGS,
request_timeout=MSC_PYGEOAPI_ES_TIMEOUT)
def __init__(self, filepath):
"""initializer"""
BaseLoader.__init__(self)
self.DD_URL = 'https://dd.weather.gc.ca/bulletins/alphanumeric'
self.ES = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH)
if not self.ES.indices.exists(INDEX_NAME):
self.ES.indices.create(index=INDEX_NAME,
body=SETTINGS,
request_timeout=MSC_PYGEOAPI_ES_TIMEOUT)
def clean_records(ctx, days):
"""Delete old documents"""
es = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH)
older_than = (datetime.now() -
timedelta(days=days)).strftime('%Y-%m-%d %H:%M')
click.echo('Deleting documents older than {} ({} days)'.format(
older_than, days))
query = {'query': {'range': {'properties.datetime': {'lte': older_than}}}}
es.delete_by_query(index=INDEX_NAME, body=query)
def deactivate(ctx, days):
"""deactivate hurricane forecasts older than N days"""
es = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH)
for index in INDICES:
query = {
"script": "ctx._source.properties.active=false",
"query": {
"range": {
"properties.filedate": {
"lte": "now-{}d".format(days)
}
}
}
}
es.update_by_query(index=index, body=query)
return True
def __init__(self, plugin_def):
"""initializer"""
BaseLoader.__init__(self)
self.ES = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH)
self.filepath = None
self.version = None
self.zone = None
self.items = []
# create storm variable indices if it don't exist
for item in FILE_PROPERTIES:
if not self.ES.indices.exists(INDEX_NAME.format(item)):
SETTINGS['mappings']['properties']['properties'][
'properties'] = FILE_PROPERTIES[item]
self.ES.indices.create(index=INDEX_NAME.format(item),
body=SETTINGS,
request_timeout=MSC_PYGEOAPI_ES_TIMEOUT)
def __init__(self, plugin_def):
"""initializer"""
BaseLoader.__init__(self)
self.ES = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH)
self.filepath = None
self.version = None
self.zone = None
self.items = []
# create forecast polygon indices if they don't exist
for index in INDICES:
zone = index.split('_')[2]
if not self.ES.indices.exists(index):
SETTINGS['mappings']['properties']['properties'][
'properties'] = FILE_PROPERTIES[zone]
self.ES.indices.create(index=index,
body=SETTINGS,
request_timeout=MSC_PYGEOAPI_ES_TIMEOUT)
def delete_references_alerts(self):
"""Delete old alerts documents"""
if self.references_arr and len(self.references_arr) != 0:
es = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH)
click.echo('Deleting old alerts')
query = {
'query': {
'terms': {
'properties.reference': self.references_arr
}
}
}
es.delete_by_query(index=INDEX_NAME, body=query)
return True
else:
return False
def ahccd(ctx, path, es, username, password, dataset):
"""
Loads AHCCD and CMIP5 data into Elasticsearch
Controls transformation from oracle to Elasticsearch.
The JSON locations file should be a JSON of the form:
{
"stations": "/path/to/stations.json",
"annual": "/path/to/annual.json",
"monthly": "/path/to/monthly.json",
"seasonal": "/path/to/seasonal.json",
"trends": "/path/to/trends.json"
}
:param path: path to file with raw JSON locations
:param es: path to Elasticsearch index.
:param username: username for HTTP authentication.
:param password: password for HTTP authentication.
:param dataset: name of dataset to load, or all for all datasets.
"""
auth = (username, password)
es_client = util.get_es(es, auth)
try:
with open(path, 'r') as f:
path_dict = json.loads(f.read())
except Exception as err:
LOGGER.error('Could not open JSON location file due to: {}.'.format(
str(err)))
if dataset == 'all':
try:
LOGGER.info('Populating stations...')
create_index(es_client, 'stations')
stations = generate_docs(path_dict['stations'], 'stations')
util.submit_elastic_package(es_client, stations)
LOGGER.info('Stations populated.')
except Exception as err:
LOGGER.error('Could not populate stations due to: {}.'.format(
str(err)))
try:
LOGGER.info('Populating trends...')
create_index(es_client, 'trends')
trends = generate_docs(path_dict['trends'], 'trends')
util.submit_elastic_package(es_client, trends)
LOGGER.info('Trends populated.')
except Exception as err:
LOGGER.error('Could not populate trends due to: {}.'.format(
str(err)))
try:
LOGGER.info('Populating annual...')
create_index(es_client, 'annual')
annuals = generate_docs(path_dict['annual'], 'annual')
util.submit_elastic_package(es_client, annuals)
LOGGER.info('Annual populated.')
except Exception as err:
LOGGER.error('Could not populate annual due to: {}.'.format(
str(err)))
try:
LOGGER.info('Populating seasonal...')
create_index(es_client, 'seasonal')
seasonals = generate_docs(path_dict['seasonal'], 'seasonal')
util.submit_elastic_package(es_client, seasonals)
LOGGER.info('Seasonal populated.')
except Exception as err:
LOGGER.error('Could not populate seasonal due to: {}.'.format(
str(err)))
try:
LOGGER.info('Populating monthly...')
create_index(es_client, 'monthly')
monthlies = generate_docs(path_dict['monthly'], 'monthly')
util.submit_elastic_package(es_client, monthlies)
LOGGER.info('Monthly populated.')
except Exception as err:
LOGGER.error('Could not populate monthly due to: {}.'.format(
str(err)))
elif dataset == 'stations':
try:
LOGGER.info('Populating stations...')
create_index(es_client, 'stations')
stations = generate_docs(path_dict['stations'], 'stations')
util.submit_elastic_package(es_client, stations)
LOGGER.info('Stations populated.')
except Exception as err:
LOGGER.error('Could not populate stations due to: {}.'.format(
str(err)))
elif dataset == 'trends':
try:
LOGGER.info('Populating trends...')
create_index(es_client, 'trends')
trends = generate_docs(path_dict['trends'], 'trends')
util.submit_elastic_package(es_client, trends)
LOGGER.info('Trends populated.')
except Exception as err:
LOGGER.error('Could not populate trends due to: {}.'.format(
str(err)))
elif dataset == 'annual':
try:
LOGGER.info('Populating annual...')
create_index(es_client, 'annual')
annuals = generate_docs(path_dict['annual'], 'annual')
util.submit_elastic_package(es_client, annuals)
LOGGER.info('Annual populated.')
except Exception as err:
LOGGER.error('Could not populate annual due to: {}.'.format(
str(err)))
elif dataset == 'seasonal':
try:
LOGGER.info('Populating seasonal...')
create_index(es_client, 'seasonal')
seasonals = generate_docs(path_dict['seasonal'], 'seasonal')
util.submit_elastic_package(es_client, seasonals)
LOGGER.info('Seasonal populated.')
except Exception as err:
LOGGER.error('Could not populate seasonal due to: {}.'.format(
str(err)))
elif dataset == 'monthly':
try:
LOGGER.info('Populating monthly...')
create_index(es_client, 'monthly')
monthlies = generate_docs(path_dict['monthly'], 'monthly')
util.submit_elastic_package(es_client, monthlies)
LOGGER.info('Monthly populated.')
except Exception as err:
LOGGER.error('Could not populate monthly due to: {}.'.format(
str(err)))
else:
LOGGER.critical(
'Unknown dataset parameter {}, skipping index population.'.format(
dataset)) # noqa
LOGGER.info('Finished populating indices.')
