def __init__(self, plugin_def): """initializer""" BaseLoader.__init__(self) self.ES = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH) self.filepath = None self.region_name_code = None self.language = None self.root = None self.area = {} self.items = [] # create marine weather indices if it don't exist for item in MAPPINGS: if not self.ES.indices.exists(INDEX_NAME.format(item)): SETTINGS['mappings']['properties']['properties'][ 'properties' ] = MAPPINGS[ item ] # noqa self.ES.indices.create( index=INDEX_NAME.format(item), body=SETTINGS, request_timeout=MSC_PYGEOAPI_ES_TIMEOUT, )
def delete_index(ctx, index_name): """ Delete a particular ES index with a given name as argument or all if no argument is passed """ es = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH) if index_name: if click.confirm( 'Are you sure you want to delete ES index named: {}?'.format( click.style(index_name, fg='red') ), abort=True, ): LOGGER.info('Deleting ES index {}'.format(index_name)) es.indices.delete(index=index_name) return True else: if click.confirm( 'Are you sure you want to delete {} marine forecast' ' indices ({})?'.format( click.style('ALL', fg='red'), click.style(", ".join(INDICES), fg='red'), ), abort=True, ): es.indices.delete(index=",".join(INDICES)) return True
def delete_index(ctx): """Delete bulletins index""" es = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH) if es.indices.exists(INDEX_NAME): es.indices.delete(INDEX_NAME)
def clean_records(ctx, days): """Delete old documents""" es = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH) today = datetime.now().replace(hour=0, minute=0) older_than = (today - timedelta(days=days)).strftime('%Y-%m-%dT%H:%M') click.echo('Deleting documents older than {} ({} full days)' .format(older_than.replace('T', ' '), days)) query = { 'query': { 'range': { 'properties.DATETIME': { 'lt': older_than, 'format': 'strict_date_hour_minute' } } } } response = es.delete_by_query(index=INDEX_NAME, body=query, request_timeout=90) click.echo('Deleted {} documents'.format(response['deleted'])) if len(response['failures']) > 0: click.echo('Failed to delete {} documents in time range' .format(len(response['failures'])))
def delete_index(ctx): """Delete hydrometric realtime indexes""" es = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH) if es.indices.exists(INDEX_NAME): es.indices.delete(INDEX_NAME)
def __init__(self, plugin_def): """initializer""" BaseLoader.__init__(self) self.ES = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH) if not self.ES.indices.exists(INDEX_NAME): self.ES.indices.create(index=INDEX_NAME, body=SETTINGS, request_timeout=MSC_PYGEOAPI_ES_TIMEOUT)
def __init__(self, filepath): """initializer""" BaseLoader.__init__(self) self.DD_URL = 'https://dd.weather.gc.ca/bulletins/alphanumeric' self.ES = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH) if not self.ES.indices.exists(INDEX_NAME): self.ES.indices.create(index=INDEX_NAME, body=SETTINGS, request_timeout=MSC_PYGEOAPI_ES_TIMEOUT)
def clean_records(ctx, days): """Delete old documents""" es = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH) older_than = (datetime.now() - timedelta(days=days)).strftime('%Y-%m-%d %H:%M') click.echo('Deleting documents older than {} ({} days)'.format( older_than, days)) query = {'query': {'range': {'properties.datetime': {'lte': older_than}}}} es.delete_by_query(index=INDEX_NAME, body=query)
def deactivate(ctx, days): """deactivate hurricane forecasts older than N days""" es = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH) for index in INDICES: query = { "script": "ctx._source.properties.active=false", "query": { "range": { "properties.filedate": { "lte": "now-{}d".format(days) } } } } es.update_by_query(index=index, body=query) return True
def __init__(self, plugin_def): """initializer""" BaseLoader.__init__(self) self.ES = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH) self.filepath = None self.version = None self.zone = None self.items = [] # create storm variable indices if it don't exist for item in FILE_PROPERTIES: if not self.ES.indices.exists(INDEX_NAME.format(item)): SETTINGS['mappings']['properties']['properties'][ 'properties'] = FILE_PROPERTIES[item] self.ES.indices.create(index=INDEX_NAME.format(item), body=SETTINGS, request_timeout=MSC_PYGEOAPI_ES_TIMEOUT)
def __init__(self, plugin_def): """initializer""" BaseLoader.__init__(self) self.ES = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH) self.filepath = None self.version = None self.zone = None self.items = [] # create forecast polygon indices if they don't exist for index in INDICES: zone = index.split('_')[2] if not self.ES.indices.exists(index): SETTINGS['mappings']['properties']['properties'][ 'properties'] = FILE_PROPERTIES[zone] self.ES.indices.create(index=index, body=SETTINGS, request_timeout=MSC_PYGEOAPI_ES_TIMEOUT)
def delete_references_alerts(self): """Delete old alerts documents""" if self.references_arr and len(self.references_arr) != 0: es = get_es(MSC_PYGEOAPI_ES_URL, MSC_PYGEOAPI_ES_AUTH) click.echo('Deleting old alerts') query = { 'query': { 'terms': { 'properties.reference': self.references_arr } } } es.delete_by_query(index=INDEX_NAME, body=query) return True else: return False
def ahccd(ctx, path, es, username, password, dataset): """ Loads AHCCD and CMIP5 data into Elasticsearch Controls transformation from oracle to Elasticsearch. The JSON locations file should be a JSON of the form: { "stations": "/path/to/stations.json", "annual": "/path/to/annual.json", "monthly": "/path/to/monthly.json", "seasonal": "/path/to/seasonal.json", "trends": "/path/to/trends.json" } :param path: path to file with raw JSON locations :param es: path to Elasticsearch index. :param username: username for HTTP authentication. :param password: password for HTTP authentication. :param dataset: name of dataset to load, or all for all datasets. """ auth = (username, password) es_client = util.get_es(es, auth) try: with open(path, 'r') as f: path_dict = json.loads(f.read()) except Exception as err: LOGGER.error('Could not open JSON location file due to: {}.'.format( str(err))) if dataset == 'all': try: LOGGER.info('Populating stations...') create_index(es_client, 'stations') stations = generate_docs(path_dict['stations'], 'stations') util.submit_elastic_package(es_client, stations) LOGGER.info('Stations populated.') except Exception as err: LOGGER.error('Could not populate stations due to: {}.'.format( str(err))) try: LOGGER.info('Populating trends...') create_index(es_client, 'trends') trends = generate_docs(path_dict['trends'], 'trends') util.submit_elastic_package(es_client, trends) LOGGER.info('Trends populated.') except Exception as err: LOGGER.error('Could not populate trends due to: {}.'.format( str(err))) try: LOGGER.info('Populating annual...') create_index(es_client, 'annual') annuals = generate_docs(path_dict['annual'], 'annual') util.submit_elastic_package(es_client, annuals) LOGGER.info('Annual populated.') except Exception as err: LOGGER.error('Could not populate annual due to: {}.'.format( str(err))) try: LOGGER.info('Populating seasonal...') create_index(es_client, 'seasonal') seasonals = generate_docs(path_dict['seasonal'], 'seasonal') util.submit_elastic_package(es_client, seasonals) LOGGER.info('Seasonal populated.') except Exception as err: LOGGER.error('Could not populate seasonal due to: {}.'.format( str(err))) try: LOGGER.info('Populating monthly...') create_index(es_client, 'monthly') monthlies = generate_docs(path_dict['monthly'], 'monthly') util.submit_elastic_package(es_client, monthlies) LOGGER.info('Monthly populated.') except Exception as err: LOGGER.error('Could not populate monthly due to: {}.'.format( str(err))) elif dataset == 'stations': try: LOGGER.info('Populating stations...') create_index(es_client, 'stations') stations = generate_docs(path_dict['stations'], 'stations') util.submit_elastic_package(es_client, stations) LOGGER.info('Stations populated.') except Exception as err: LOGGER.error('Could not populate stations due to: {}.'.format( str(err))) elif dataset == 'trends': try: LOGGER.info('Populating trends...') create_index(es_client, 'trends') trends = generate_docs(path_dict['trends'], 'trends') util.submit_elastic_package(es_client, trends) LOGGER.info('Trends populated.') except Exception as err: LOGGER.error('Could not populate trends due to: {}.'.format( str(err))) elif dataset == 'annual': try: LOGGER.info('Populating annual...') create_index(es_client, 'annual') annuals = generate_docs(path_dict['annual'], 'annual') util.submit_elastic_package(es_client, annuals) LOGGER.info('Annual populated.') except Exception as err: LOGGER.error('Could not populate annual due to: {}.'.format( str(err))) elif dataset == 'seasonal': try: LOGGER.info('Populating seasonal...') create_index(es_client, 'seasonal') seasonals = generate_docs(path_dict['seasonal'], 'seasonal') util.submit_elastic_package(es_client, seasonals) LOGGER.info('Seasonal populated.') except Exception as err: LOGGER.error('Could not populate seasonal due to: {}.'.format( str(err))) elif dataset == 'monthly': try: LOGGER.info('Populating monthly...') create_index(es_client, 'monthly') monthlies = generate_docs(path_dict['monthly'], 'monthly') util.submit_elastic_package(es_client, monthlies) LOGGER.info('Monthly populated.') except Exception as err: LOGGER.error('Could not populate monthly due to: {}.'.format( str(err))) else: LOGGER.critical( 'Unknown dataset parameter {}, skipping index population.'.format( dataset)) # noqa LOGGER.info('Finished populating indices.')