def get_collection(collection_id):
"""Fetch a collection from the index."""
if collection_id is None:
return
key = cache.object_key(Collection, collection_id)
data = cache.get_complex(key)
if data is not None:
return data
collection = Collection.by_id(collection_id)
if collection is None:
return
data = collection.to_dict()
stats = get_collection_stats(collection.id)
data['count'] = stats['count']
data['schemata'] = stats['schemata']
# if no countries or langs are given, take the most common from the data.
countries = ensure_list(collection.countries)
countries = countries or stats['countries'].keys()
data['countries'] = registry.country.normalize_set(countries)
languages = ensure_list(collection.languages)
languages = languages or stats['languages'].keys()
data['languages'] = registry.language.normalize_set(languages)
cache.set_complex(key, data, expire=cache.EXPIRE)
return data
def to_token(self):
if self.token_id is None:
self.token_id = "%s.%s" % (self.id, make_token())
key = cache.key(self.TOKENS, self.token_id)
state = {
"id": self.id,
"roles": list(self.roles),
"is_admin": self.is_admin,
}
cache.set_complex(key, state, expires=self.expire)
return self.token_id
def get_role(role_id):
if role_id is None:
return
key = cache.object_key(Role, role_id)
data = cache.get_complex(key)
if data is None:
role = Role.by_id(role_id)
if role is None:
return
data = role.to_dict()
cache.set_complex(key, data, expires=cache.EXPIRE)
return data
def metadata():
"""Get operational metadata for the frontend.
---
get:
summary: Retrieve system metadata from the application.
responses:
'200':
description: OK
content:
application/json:
schema:
type: object
tags:
- System
"""
locale = get_locale()
enable_cache(vary_user=False, vary=str(locale))
key = cache.key('metadata', settings.PROCESS_ID, locale)
data = cache.get_complex(key)
if data is not None:
return jsonify(data)
auth = {}
if settings.PASSWORD_LOGIN:
auth['password_login_uri'] = url_for('sessions_api.password_login')
auth['registration_uri'] = url_for('roles_api.create_code')
if settings.OAUTH:
auth['oauth_uri'] = url_for('sessions_api.oauth_init')
locales = settings.UI_LANGUAGES
locales = {l: Locale(l).get_language_name(l) for l in locales}
data = {
'status': 'ok',
'maintenance': request.authz.in_maintenance,
'app': {
'title': settings.APP_TITLE,
'description': settings.APP_DESCRIPTION,
'version': __version__,
'banner': settings.APP_BANNER,
'ui_uri': settings.APP_UI_URL,
'samples': settings.SAMPLE_SEARCHES,
'logo': settings.APP_LOGO,
'favicon': settings.APP_FAVICON,
'locale': str(locale),
'locales': locales
},
'categories': Collection.CATEGORIES,
'model': model,
'auth': auth
}
cache.set_complex(key, data, expires=120)
return jsonify(data)
def get_role(role_id):
if role_id is None:
return
key = cache.object_key(Role, role_id)
data = cache.get_complex(key)
if data is None:
log.debug("Role [%s]: object cache miss", role_id)
role = Role.by_id(role_id)
if role is None:
return
data = role.to_dict()
cache.set_complex(key, data, expire=cache.EXPIRE)
return data
def get_role(role_id):
if role_id is None:
return
key = cache.object_key(Role, role_id)
data = cache.get_complex(key)
if data is None:
log.debug("Role [%s]: object cache miss", role_id)
role = Role.by_id(role_id)
if role is None:
return
data = role.to_dict()
cache.set_complex(key, data, expire=cache.EXPIRE)
return data
def get_export(export_id):
if export_id is None:
return
key = cache.object_key(Export, export_id)
data = cache.get_complex(key)
if data is None:
export = Export.by_id(export_id)
if export is None:
return
log.debug("Export cache refresh: %r", export)
data = export.to_dict()
cache.set_complex(key, data, expires=cache.EXPIRE)
return data
def get_collection_stats(collection_id):
"""Compute some statistics on the content of a collection."""
key = cache.key('cstats', collection_id)
data = cache.get_complex(key)
if data is not None:
return data
log.info("Generating collection stats: %s", collection_id)
query = {
'size': 0,
'query': {
'bool': {
'filter': [{
'term': {
'collection_id': collection_id
}
}]
}
},
'aggs': {
'schemata': {
'terms': {
'field': 'schema',
'size': 1000
}
},
'countries': {
'terms': {
'field': 'countries',
'size': 500
}
},
'languages': {
'terms': {
'field': 'languages',
'size': 10
}
},
}
}
result = search_safe(index=entities_read_index(), body=query)
aggregations = result.get('aggregations', {})
data = {'count': result['hits']['total']}
for facet in ['schemata', 'countries', 'languages']:
data[facet] = {}
for bucket in aggregations[facet]['buckets']:
data[facet][bucket['key']] = bucket['doc_count']
expire = randint(3600 * 3, 3600 * 12)
cache.set_complex(key, data, expire=expire)
return data
def get_role(role_id):
key = cache.object_key(Role, role_id)
data = cache.get_complex(key)
if data is None:
role = Role.by_id(role_id)
if role is None:
return
data = {
'id': role.id,
'name': role.name,
'label': role.label,
'type': role.type
}
cache.set_complex(key, data, expire=cache.EXPIRE)
return data
def get_collection_facet(collection_id, facet, refresh=False):
"""Compute some statistics on the content of a collection."""
key = cache.object_key(Collection, collection_id, facet)
data = cache.get_complex(key)
if not refresh and data is not None:
return data
query = {'term': {'collection_id': collection_id}}
query = {
'size': 0,
'query': {
'bool': {
'filter': [query]
}
},
'aggs': {
'values': {
'terms': {
'field': facet,
'size': 300
}
},
'total': {
'cardinality': {
'field': facet
}
}
}
}
schemata = set()
facet_type = registry.groups.get(facet)
if facet_type is not None:
schemata = model.get_type_schemata(facet_type)
result = es.search(index=entities_read_index(schema=schemata),
body=query,
request_timeout=3600,
timeout='20m')
aggregations = result.get('aggregations')
values = {}
for bucket in aggregations.get('values').get('buckets', []):
values[bucket['key']] = bucket['doc_count']
data = {
'values': values,
'total': aggregations.get('total').get('value', 0)
}
cache.set_complex(key, data, expires=cache.EXPIRE)
return data
def get_alert(alert_id):
key = cache.object_key(Alert, alert_id)
data = cache.get_complex(key)
if data is None:
alert = Alert.by_id(alert_id)
if alert is None:
return
data = {
'id': alert.id,
'query': alert.query,
'role_id': alert.role_id,
'notified_at': alert.notified_at,
'created_at': alert.created_at,
'updated_at': alert.updated_at
}
cache.set_complex(key, data, expire=cache.EXPIRE)
return data
def get_collection(collection_id):
"""Fetch a collection from the index."""
if collection_id is None:
return
key = cache.object_key(Collection, collection_id)
data = cache.get_complex(key)
if data is not None:
return data
collection = Collection.by_id(collection_id)
if collection is None:
return
data = collection.to_dict()
things = get_collection_things(collection.id)
data['count'] = sum(things.values())
cache.set_complex(key, data, expires=cache.EXPIRE)
return data
def entities_by_ids(ids,
schemata=None,
cached=False,
includes=PROXY_INCLUDES,
excludes=None):
"""Iterate over unpacked entities based on a search for the given
entity IDs."""
ids = ensure_list(ids)
if not len(ids):
return
cached = cached and excludes is None and includes == PROXY_INCLUDES
entities = {}
if cached:
keys = [cache.object_key(Entity, i) for i in ids]
for _, entity in cache.get_many_complex(keys):
if entity is not None:
entities[entity.get("id")] = entity
missing = [i for i in ids if entities.get(id) is None]
index = entities_read_index(schema=schemata)
query = {
"query": {
"ids": {
"values": missing
}
},
"_source": _source_spec(includes, excludes),
"size": MAX_PAGE,
}
result = es.search(index=index, body=query)
for doc in result.get("hits", {}).get("hits", []):
entity = unpack_result(doc)
if entity is not None:
entity_id = entity.get("id")
entities[entity_id] = entity
if cached:
key = cache.object_key(Entity, entity_id)
cache.set_complex(key, entity, expires=60 * 60 * 2)
for i in ids:
entity = entities.get(i)
if entity is not None:
yield entity
def oauth_init():
"""Init OAuth auth flow.
---
get:
summary: Start OAuth authentication
description: Initiate a forward to the OAuth server.
responses:
'302':
description: Redirect
tags:
- Role
"""
require(settings.OAUTH)
url = url_for(".oauth_callback")
state = oauth.provider.create_authorization_url(url)
state["next_url"] = request.args.get("next", request.referrer)
state["redirect_uri"] = url
cache.set_complex(_oauth_session(state.get("state")), state, expires=3600)
return redirect(state["url"])
def get_collection(collection_id):
"""Fetch a collection from the index."""
if collection_id is None:
return
key = cache.object_key(Collection, collection_id)
data = cache.get_complex(key)
if data is not None:
return data
collection = Collection.by_id(collection_id)
if collection is None:
return
data = collection.to_dict()
schemata = get_facet_values(collection.id, 'schema')
schemata = schemata.get('values', {})
data['count'] = sum(schemata.values())
data['schemata'] = schemata
cache.set_complex(key, data, expires=cache.EXPIRE)
return data
def metadata():
locale = get_locale()
enable_cache(vary_user=False, vary=str(locale))
key = cache.key('metadata', settings.PROCESS_ID, locale)
data = cache.get_complex(key)
if data is not None:
return jsonify(data)
auth = {}
if settings.PASSWORD_LOGIN:
auth['password_login_uri'] = url_for('sessions_api.password_login')
auth['registration_uri'] = url_for('roles_api.create_code')
if settings.OAUTH:
auth['oauth_uri'] = url_for('sessions_api.oauth_init')
locales = settings.UI_LANGUAGES
locales = {l: Locale(l).get_language_name(l) for l in locales}
data = {
'status': 'ok',
'maintenance': request.authz.in_maintenance,
'app': {
'title': settings.APP_TITLE,
'description': settings.APP_DESCRIPTION,
'version': __version__,
'banner': settings.APP_BANNER,
'ui_uri': settings.APP_UI_URL,
'samples': settings.SAMPLE_SEARCHES,
'logo': settings.APP_LOGO,
'favicon': settings.APP_FAVICON,
'locale': str(locale),
'locales': locales
},
'categories': Collection.CATEGORIES,
'countries': registry.country.names,
'languages': registry.language.names,
'model': model,
'auth': auth
}
cache.set_complex(key, data, expire=120)
return jsonify(data)
def get_collection(collection_id):
"""Fetch a collection from the index."""
if collection_id is None:
return
key = cache.object_key(Collection, collection_id)
data = cache.get_complex(key)
if data is not None:
return data
collection = Collection.by_id(collection_id)
if collection is None:
return
data = collection.to_dict()
index = entities_read_index(schema=Entity.THING)
query = {"term": {"collection_id": collection_id}}
result = es.count(index=index, body={"query": query})
data["count"] = result.get("count", 0)
cache.set_complex(key, data, expires=cache.EXPIRE)
return data
def metadata():
locale = get_locale()
enable_cache(vary_user=False, vary=str(locale))
key = cache.key('metadata', locale)
data = cache.get_complex(key)
if data is not None:
return jsonify(data)
auth = {}
if settings.PASSWORD_LOGIN:
auth['password_login_uri'] = url_for('sessions_api.password_login')
auth['registration_uri'] = url_for('roles_api.create_code')
if settings.OAUTH:
auth['oauth_uri'] = url_for('sessions_api.oauth_init')
data = {
'status': 'ok',
'maintenance': request.authz.in_maintenance,
'app': {
'title': settings.APP_TITLE,
'description': settings.APP_DESCRIPTION,
'version': __version__,
'banner': settings.APP_BANNER,
'ui_uri': settings.APP_UI_URL,
'samples': settings.SAMPLE_SEARCHES,
'logo': settings.APP_LOGO,
'favicon': settings.APP_FAVICON,
'locale': str(locale),
'locales': settings.UI_LANGUAGES
},
'categories': Collection.CATEGORIES,
'countries': registry.country.names,
'languages': registry.language.names,
'model': model,
'auth': auth
}
cache.set_complex(key, data, expire=120)
return jsonify(data)
def update_collection_stats(collection_id, facets=STATS_FACETS):
"""Compute some statistics on the content of a collection."""
aggs = {}
for facet in facets:
# Regarding facet size, 300 would be optimal because it's
# guaranteed to capture all schemata and countries. But it
# adds a whole lot to the compute time, so let's see how
# this goes.
aggs[facet + ".values"] = {"terms": {"field": facet, "size": 100}}
aggs[facet + ".total"] = {"cardinality": {"field": facet}}
query = {"term": {"collection_id": collection_id}}
query = {"size": 0, "query": query, "aggs": aggs}
result = es.search(index=entities_read_index(),
body=query,
request_timeout=3600,
timeout="20m")
results = result.get("aggregations", {})
for facet in facets:
buckets = results.get(facet + ".values").get("buckets", [])
values = {b["key"]: b["doc_count"] for b in buckets}
total = results.get(facet + ".total", {}).get("value", 0)
data = {"values": values, "total": total}
cache.set_complex(_facet_key(collection_id, facet), data)
def metadata():
locale = get_locale()
enable_cache(vary_user=False, vary=str(locale))
key = cache.key('metadata', id(settings), locale)
data = cache.get_complex(key)
if data is not None:
return jsonify(data)
auth = {}
if settings.PASSWORD_LOGIN:
auth['password_login_uri'] = url_for('sessions_api.password_login')
auth['registration_uri'] = url_for('roles_api.create_code')
if settings.OAUTH:
auth['oauth_uri'] = url_for('sessions_api.oauth_init')
data = {
'status': 'ok',
'maintenance': request.authz.in_maintenance,
'app': {
'title': settings.APP_TITLE,
'version': __version__,
'ui_uri': settings.APP_UI_URL,
'samples': settings.SAMPLE_SEARCHES,
'logo': settings.APP_LOGO,
'favicon': settings.APP_FAVICON,
'locale': str(locale),
'locales': settings.UI_LANGUAGES
},
'categories': Collection.CATEGORIES,
'statistics': get_instance_stats(Authz.from_role(None)),
'countries': registry.country.names,
'languages': registry.language.names,
'schemata': model,
'auth': auth
}
cache.set_complex(key, data, expire=84600)
return jsonify(data)
def entities_by_ids(ids, schemata=None, cached=False,
includes=None, excludes=None):
"""Iterate over unpacked entities based on a search for the given
entity IDs."""
ids = ensure_list(ids)
if not len(ids):
return
index = entities_read_index(schema=schemata)
query = {'ids': {'values': ids}}
# query = {'bool': {'filter': query}}
query = {
'query': query,
'_source': _source_spec(includes, excludes),
'size': MAX_PAGE
}
result = es.search(index=index, body=query)
for doc in result.get('hits', {}).get('hits', []):
entity = unpack_result(doc)
if entity is not None:
# Cache entities only briefly to avoid filling up the cache:
if cached:
key = cache.object_key(Entity, entity.get('id'))
cache.set_complex(key, entity, expire=60 * 60)
yield entity
def entities_by_ids(ids, schemata=None, cached=False,
includes=None, excludes=None):
"""Iterate over unpacked entities based on a search for the given
entity IDs."""
ids = ensure_list(ids)
if not len(ids):
return
index = entities_read_index(schema=schemata)
query = {'ids': {'values': ids}}
# query = {'bool': {'filter': query}}
query = {
'query': query,
'_source': _source_spec(includes, excludes),
'size': MAX_PAGE
}
result = es.search(index=index, body=query)
for doc in result.get('hits', {}).get('hits', []):
entity = unpack_result(doc)
if entity is not None:
# Cache entities only briefly to avoid filling up the cache:
if cached:
key = cache.object_key(Entity, entity.get('id'))
cache.set_complex(key, entity, expire=60 * 60)
yield entity
def _cache_entity(entity):
# Cache entities only briefly to avoid filling up redis
key = cache.object_key(Entity, entity.get('id'))
cache.set_complex(key, entity, expires=60 * 60 * 2)