def _process_extra_facets(extra_facets):
if (not extra_facets) or extra_facets == ['']:
current_app.config['EXTRA_FACET_INFO'] = {}
return
es = Elasticsearch(current_app.config['ELASTICSEARCH_URL'])
facets = OrderedDict()
mapping = es.indices.get_mapping(index=current_app.config['INDEX_NAME'])
for es_base_field_name in extra_facets:
es_parent_field_name = es_base_field_name.rsplit('.', 1)[0]
is_time_series = elasticsearch_util.is_time_series(
es_base_field_name, mapping)
parent_is_time_series = elasticsearch_util.is_time_series(
es_parent_field_name, mapping)
if is_time_series:
time_series_vals = elasticsearch_util.get_time_series_vals(
es_base_field_name, mapping)
es_field_names = [
es_base_field_name + '.' + tsv for tsv in time_series_vals
]
for es_field_name in es_field_names:
separate_panel = (es_field_name in facets
and facets[es_field_name]['separate_panel'])
_add_facet(es_field_name, is_time_series,
parent_is_time_series, True, separate_panel,
time_series_vals, facets, es, mapping)
elif parent_is_time_series:
time_series_vals = elasticsearch_util.get_time_series_vals(
es_parent_field_name, mapping)
time_series_panel = (
es_base_field_name in facets
and facets[es_base_field_name]['time_series_panel'])
_add_facet(es_base_field_name, is_time_series,
parent_is_time_series, time_series_panel, True,
time_series_vals, facets, es, mapping)
else:
_add_facet(es_base_field_name, is_time_series,
parent_is_time_series, False, True, [], facets, es,
mapping)
# Map from Elasticsearch field name to dict with ui facet name,
# Elasticsearch field type, optional UI facet description and Elasticsearch
# facet. This map is for extra facets added from the field search dropdown
# on the UI.
# This must be stored separately from FACET_INFO. If this were added to
# FACET_INFO, then if user deletes extra facets chip, we wouldn't know which
# facet to remove from FACET_INFO.
current_app.config['EXTRA_FACET_INFO'] = facets
def _results_from_fields_index(fields, mapping):
results = []
for field in fields['hits']['hits']:
es_base_field_name = field["_id"]
if "description" in field["_source"]:
results.append(
SearchResult(facet_name=field["_source"]["name"],
facet_description=field["_source"]["description"],
elasticsearch_field_name=es_base_field_name,
facet_value="",
is_time_series=False))
if elasticsearch_util.is_time_series(es_base_field_name, mapping):
time_series_vals = elasticsearch_util.get_time_series_vals(
es_base_field_name, mapping)
for tsv in time_series_vals:
results.append(
SearchResult(
facet_name=field["_source"]["name"],
facet_description=field["_source"]["description"],
elasticsearch_field_name=es_base_field_name + '.' +
tsv,
facet_value="",
is_time_series=True))
else:
results.append(
SearchResult(facet_name=field["_source"]["name"],
elasticsearch_field_name=es_base_field_name,
facet_value="",
is_time_series=False))
if elasticsearch_util.is_time_series(es_base_field_name, mapping):
time_series_vals = elasticsearch_util.get_time_series_vals(
es_base_field_name, mapping)
for tsv in time_series_vals:
results.append(
SearchResult(
facet_name=field["_source"]["name"],
elasticsearch_field_name=es_base_field_name + '.' +
tsv,
facet_value="",
is_time_series=True))
return results
def _process_extra_facets(es, extra_facets):
"""Processes extra facets and sets current_app.config['EXTRA_FACET_INFO'].
Args:
es: Elasticsearch
extra_facets: List of es_field_name
Returns:
A list of es_field_names that don't exist in Elasticsearch index. The
invalid fields are not included in current_app.config['EXTRA_FACET_INFO']
"""
if (not extra_facets) or extra_facets == ['']:
current_app.config['EXTRA_FACET_INFO'] = {}
return
facets = OrderedDict()
mapping = es.indices.get_mapping(index=current_app.config['INDEX_NAME'])
invalid_extra_facets = []
for es_base_field_name in extra_facets:
if not elasticsearch_util.field_exists(es, es_base_field_name):
invalid_extra_facets.append(es_base_field_name)
continue
es_parent_field_name = es_base_field_name.rsplit('.', 1)[0]
is_time_series = elasticsearch_util.is_time_series(
es_base_field_name, mapping)
parent_is_time_series = elasticsearch_util.is_time_series(
es_parent_field_name, mapping)
if is_time_series:
time_series_vals = elasticsearch_util.get_time_series_vals(
es_base_field_name, mapping)
es_field_names = [
es_base_field_name + '.' + tsv for tsv in time_series_vals
]
for es_field_name in es_field_names:
separate_panel = (es_field_name in facets
and facets[es_field_name]['separate_panel'])
_add_facet(es_field_name, is_time_series,
parent_is_time_series, True, separate_panel,
time_series_vals, facets, es, mapping)
elif parent_is_time_series:
time_series_vals = elasticsearch_util.get_time_series_vals(
es_parent_field_name, mapping)
time_series_panel = (
es_base_field_name in facets
and facets[es_base_field_name]['time_series_panel'])
_add_facet(es_base_field_name, is_time_series,
parent_is_time_series, time_series_panel, True,
time_series_vals, facets, es, mapping)
else:
_add_facet(es_base_field_name, is_time_series,
parent_is_time_series, False, True, [], facets, es,
mapping)
# Map that follows same format as FACET_INFO in __main__.py.
# This map is for extra facets added from the search dropdown.
# This must be stored separately from FACET_INFO. If this were added to
# FACET_INFO, then if user deletes extra facets chip, we wouldn't know which
# facet to remove from FACET_INFO.
current_app.config['EXTRA_FACET_INFO'] = facets
return invalid_extra_facets
def _process_facets(es):
config_path = os.path.join(app.app.config['DATASET_CONFIG_DIR'], 'ui.json')
facets_config = _parse_json_file(config_path)['facets']
# Preserve order, so facets are returned in same order as the config file.
facets = OrderedDict()
# Add a 'Samples Overview' facet if sample_file_columns were specified in
# bigquery.json. This facet is mapped to multiple Elasticsearch facets, and
# has keys - 'elasticsearch_field_names', 'type', 'ui_facet_name' and 'es_facet'.
if app.app.config['SAMPLE_FILE_COLUMNS']:
# Construct Elasticsearch filters. See
# https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-filters-aggregation.html
es_field_names = {}
for name, field in app.app.config['SAMPLE_FILE_COLUMNS'].iteritems():
facet_name = 'Has %s' % name
es_field_name = 'samples._has_%s' % name.lower().replace(' ', '_')
es_field_names[facet_name] = es_field_name
facets['Samples Overview'] = {
'elasticsearch_field_names': es_field_names,
'type': 'samples_overview',
'ui_facet_name': 'Samples Overview'
}
facets['Samples Overview'][
'es_facet'] = elasticsearch_util.get_samples_overview_facet(
es_field_names)
app.app.config['NESTED_PATHS'] = elasticsearch_util.get_nested_paths(es)
# Precompute mapping for getting time series values later.
mapping = es.indices.get_mapping(index=app.app.config['INDEX_NAME'])
for facet_config in facets_config:
es_base_field_name = facet_config['elasticsearch_field_name']
es_parent_field_name = es_base_field_name.rsplit('.', 1)[0]
try:
is_time_series = elasticsearch_util.is_time_series(
es_base_field_name, mapping)
except KeyError:
raise EnvironmentError(
'Elasticsearch field name %s in ui.json not found in index' %
es_base_field_name)
parent_is_time_series = elasticsearch_util.is_time_series(
es_parent_field_name, mapping)
if is_time_series:
time_series_vals = elasticsearch_util.get_time_series_vals(
es_base_field_name, mapping)
es_field_names = [
es_base_field_name + '.' + tsv for tsv in time_series_vals
]
for es_field_name in es_field_names:
separate_panel = (es_field_name in facets
and facets[es_field_name]['separate_panel'])
_add_facet(es_field_name, is_time_series,
parent_is_time_series, True, separate_panel,
facet_config, time_series_vals, facets, es, mapping)
elif parent_is_time_series:
time_series_vals = elasticsearch_util.get_time_series_vals(
es_parent_field_name, mapping)
time_series_panel = (
es_base_field_name in facets
and facets[es_base_field_name]['time_series_panel'])
_add_facet(es_base_field_name, is_time_series,
parent_is_time_series, time_series_panel, True,
facet_config, time_series_vals, facets, es, mapping)
else:
_add_facet(es_base_field_name, is_time_series,
parent_is_time_series, False, True, facet_config, [],
facets, es, mapping)
# Map from Elasticsearch field name to dict with
# - ui_facet_name: name to display in UI
# - type: Elasticsearch field type
# - time_series_panel: If facet is for a time series field, and is part of a
# larger time series facet that contains a panel for each time
# (https://i.imgur.com/7txytrL.png)
# - separate_panel: If facet is either not for a time series field, or is
# for a time series field with its own separate panel
# (https://i.imgur.com/JsTb5r0.png)
# - time_series_field: If facet is for a time series field
# - description: optional UI facet description
# - es_facet: Elasticsearch facet
app.app.config['FACET_INFO'] = facets
app.app.config['EXTRA_FACET_INFO'] = {}
def _process_facets(es):
config_path = os.path.join(app.app.config['DATASET_CONFIG_DIR'], 'ui.json')
facets_config = _parse_json_file(config_path)['facets']
# Preserve order, so facets are returned in same order as the config file.
facets = OrderedDict()
app.app.config['NESTED_PATHS'] = elasticsearch_util.get_nested_paths(es)
# Precompute mapping for getting time series values later.
mapping = es.indices.get_mapping(index=app.app.config['INDEX_NAME'])
for facet_config in facets_config:
es_base_field_name = facet_config['elasticsearch_field_name']
es_parent_field_name = es_base_field_name.rsplit('.', 1)[0]
try:
is_time_series = elasticsearch_util.is_time_series(
es_base_field_name, mapping)
except KeyError:
raise EnvironmentError(
'Elasticsearch field name %s in ui.json not found in index' %
es_base_field_name)
parent_is_time_series = elasticsearch_util.is_time_series(
es_parent_field_name, mapping)
if is_time_series:
time_series_vals = elasticsearch_util.get_time_series_vals(
es_base_field_name, mapping)
es_field_names = [
es_base_field_name + '.' + tsv for tsv in time_series_vals
]
for es_field_name in es_field_names:
separate_panel = (es_field_name in facets
and facets[es_field_name]['separate_panel'])
_add_facet(es_field_name, is_time_series,
parent_is_time_series, True, separate_panel,
facet_config, time_series_vals, facets, es, mapping)
elif parent_is_time_series:
time_series_vals = elasticsearch_util.get_time_series_vals(
es_parent_field_name, mapping)
time_series_panel = (
es_base_field_name in facets
and facets[es_base_field_name]['time_series_panel'])
_add_facet(es_base_field_name, is_time_series,
parent_is_time_series, time_series_panel, True,
facet_config, time_series_vals, facets, es, mapping)
else:
_add_facet(es_base_field_name, is_time_series,
parent_is_time_series, False, True, facet_config, [],
facets, es, mapping)
# Map from Elasticsearch field name to dict with
# - ui_facet_name: name to display in UI
# - type: Elasticsearch field type
# - time_series_panel: If facet is for a time series field, and is part of a
# larger time series facet that contains a panel for each time
# (https://i.imgur.com/7txytrL.png)
# - separate_panel: If facet is either not for a time series field, or is
# for a time series field with its own separate panel
# (https://i.imgur.com/JsTb5r0.png)
# - time_series_field: If facet is for a time series field
# - description: optional UI facet description
# - es_facet: Elasticsearch facet
app.app.config['FACET_INFO'] = facets
app.app.config['EXTRA_FACET_INFO'] = {}