def get_collection(collection_id):
"""Fetch a collection from the index."""
if collection_id is None:
return
key = cache.object_key(Collection, collection_id)
data = cache.get_complex(key)
if data is not None:
return data
collection = Collection.by_id(collection_id)
if collection is None:
return
data = collection.to_dict()
stats = get_collection_stats(collection.id)
data['count'] = stats['count']
data['schemata'] = stats['schemata']
# if no countries or langs are given, take the most common from the data.
countries = ensure_list(collection.countries)
countries = countries or stats['countries'].keys()
data['countries'] = registry.country.normalize_set(countries)
languages = ensure_list(collection.languages)
languages = languages or stats['languages'].keys()
data['languages'] = registry.language.normalize_set(languages)
cache.set_complex(key, data, expire=cache.EXPIRE)
return data
def get_collection(collection_id):
"""Fetch a collection from the index."""
if collection_id is None:
return
key = cache.object_key(Collection, collection_id)
data = cache.get_complex(key)
if data is not None:
return data
collection = Collection.by_id(collection_id)
if collection is None:
return
data = collection.to_dict()
stats = get_collection_stats(collection.id)
data['count'] = stats['count']
data['schemata'] = stats['schemata']
# if no countries or langs are given, take the most common from the data.
countries = ensure_list(collection.countries)
countries = countries or stats['countries'].keys()
data['countries'] = registry.country.normalize_set(countries)
languages = ensure_list(collection.languages)
languages = languages or stats['languages'].keys()
data['languages'] = registry.language.normalize_set(languages)
cache.set_complex(key, data, expire=cache.EXPIRE)
return data
def get_collection_facet(collection_id, facet, refresh=False):
"""Compute some statistics on the content of a collection."""
key = cache.object_key(Collection, collection_id, facet)
data = cache.get_complex(key)
if not refresh and data is not None:
return data
query = {'term': {'collection_id': collection_id}}
query = {
'size': 0,
'query': {'bool': {'filter': [query]}},
'aggs': {
'values': {'terms': {'field': facet, 'size': 300}},
'total': {'cardinality': {'field': facet}}
}
}
schemata = set()
facet_type = registry.groups.get(facet)
if facet_type is not None:
schemata = model.get_type_schemata(facet_type)
result = es.search(index=entities_read_index(schema=schemata),
body=query,
request_timeout=3600,
timeout='20m')
aggregations = result.get('aggregations')
values = {}
for bucket in aggregations.get('values').get('buckets', []):
values[bucket['key']] = bucket['doc_count']
data = {
'values': values,
'total': aggregations.get('total').get('value', 0)
}
cache.set_complex(key, data, expires=cache.EXPIRE)
return data
def get_entity(entity_id):
key = cache.object_key(Entity, entity_id)
entity = cache.get_complex(key)
if entity is None:
entity = index.get_entity(entity_id)
cache.set_complex(key, entity, expire=cache.EXPIRE)
return entity
def get_collection(collection_id):
key = cache.object_key(Collection, collection_id)
data = cache.get_complex(key)
if data is None:
data = index.get_collection(collection_id)
cache.set_complex(key, data, expire=cache.EXPIRE)
return data
def oauth_callback():
require(settings.OAUTH)
err = Unauthorized(gettext("Authentication has failed."))
state = cache.get_complex(_oauth_session(request.args.get("state")))
if state is None:
raise err
try:
oauth.provider.framework.set_session_data(request, "state",
state.get("state"))
uri = state.get("redirect_uri")
token = oauth.provider.authorize_access_token(redirect_uri=uri)
except AuthlibBaseError as err:
log.warning("Failed OAuth: %r", err)
raise err
if token is None or isinstance(token, AuthlibBaseError):
log.warning("Failed OAuth: %r", token)
raise err
role = handle_oauth(oauth.provider, token)
if role is None:
raise err
db.session.commit()
update_role(role)
log.debug("Logged in: %r", role)
request.authz = Authz.from_role(role)
next_path = get_url_path(state.get("next_url"))
next_url = ui_url("oauth", next=next_path)
next_url = "%s#token=%s" % (next_url, request.authz.to_token())
session.clear()
return redirect(next_url)
def get_profile(entityset_id, authz=None):
"""A profile is an entityset having a party. The idea is to cache
profile metadata for the API, and to generate a merged view of all
the entities the current user has access to."""
if entityset_id is None:
return
key = cache.object_key(EntitySet, entityset_id)
data = cache.get_complex(key)
stub = Stub()
if data is None:
entityset = get_entityset(entityset_id)
if entityset is None:
return
data = entityset.to_dict()
data["items"] = []
for item in entityset.items():
data["items"].append(item.to_dict())
cache.set_complex(key, data, expires=cache.EXPIRE)
# Filter the subset of items the current user can access
if authz is not None:
items = [
i for i in data["items"]
if authz.can(i["collection_id"], authz.READ)
]
data["items"] = items
# Load the constituent entities for the profile and generate a
# combined proxy with all of the given properties.
for item in data["items"]:
if Judgement(item["judgement"]) == Judgement.POSITIVE:
resolver.queue(stub, Entity, item.get("entity_id"))
resolver.resolve(stub)
merged = None
data["proxies"] = []
for item in data["items"]:
item["entity"] = resolver.get(stub, Entity, item.get("entity_id"))
if item["entity"] is not None:
proxy = model.get_proxy(item["entity"])
proxy.context = {}
data["proxies"].append(proxy)
if merged is None:
merged = proxy.clone()
merged.context["entities"] = [proxy.id]
else:
merged.merge(proxy)
merged.context["entities"].append(proxy.id)
if merged is None:
merged = model.make_entity(Entity.LEGAL_ENTITY)
# Polish it a bit:
merged.id = data.get("id")
merged = name_entity(merged)
data["merged"] = merged
data["label"] = merged.caption
data["shallow"] = False
return data
def metadata():
"""Get operational metadata for the frontend.
---
get:
summary: Retrieve system metadata from the application.
responses:
'200':
description: OK
content:
application/json:
schema:
type: object
tags:
- System
"""
locale = get_locale()
enable_cache(vary_user=False, vary=str(locale))
key = cache.key('metadata', settings.PROCESS_ID, locale)
data = cache.get_complex(key)
if data is not None:
return jsonify(data)
auth = {}
if settings.PASSWORD_LOGIN:
auth['password_login_uri'] = url_for('sessions_api.password_login')
auth['registration_uri'] = url_for('roles_api.create_code')
if settings.OAUTH:
auth['oauth_uri'] = url_for('sessions_api.oauth_init')
locales = settings.UI_LANGUAGES
locales = {l: Locale(l).get_language_name(l) for l in locales}
data = {
'status': 'ok',
'maintenance': request.authz.in_maintenance,
'app': {
'title': settings.APP_TITLE,
'description': settings.APP_DESCRIPTION,
'version': __version__,
'banner': settings.APP_BANNER,
'ui_uri': settings.APP_UI_URL,
'samples': settings.SAMPLE_SEARCHES,
'logo': settings.APP_LOGO,
'favicon': settings.APP_FAVICON,
'locale': str(locale),
'locales': locales
},
'categories': Collection.CATEGORIES,
'countries': registry.country.names,
'languages': registry.language.names,
'model': model,
'auth': auth
}
cache.set_complex(key, data, expires=120)
return jsonify(data)
def get_entity(entity_id, **kwargs):
"""Fetch an entity from the index."""
if entity_id is None:
return
key = cache.object_key(Entity, entity_id)
entity = cache.get_complex(key)
if entity is not None:
return entity
log.debug("Entity [%s]: object cache miss", entity_id)
for entity in entities_by_ids(entity_id, cached=True):
return entity
def get_entity(entity_id, **kwargs):
"""Fetch an entity from the index."""
if entity_id is None:
return
key = cache.object_key(Entity, entity_id)
entity = cache.get_complex(key)
if entity is not None:
return entity
log.debug("Entity [%s]: object cache miss", entity_id)
for entity in entities_by_ids(entity_id, cached=True):
return entity
def from_token(cls, token_id):
state_key = cache.key(cls.TOKENS, token_id)
state = cache.get_complex(state_key)
if state is None:
raise Unauthorized()
return cls(
state.get("id"),
state.get("roles"),
is_admin=state.get("is_admin"),
token_id=token_id,
)
def get_collection_things(collection_id):
"""Showing the number of things in a collection is more indicative
of its size than the overall collection entity count."""
schemata = cache.get_complex(_facet_key(collection_id, "schema"))
if schemata is None:
return {}
things = {}
for schema, count in schemata.get("values", {}).items():
schema = model.get(schema)
if schema is not None and schema.is_a(Entity.THING):
things[schema.name] = count
return things
def get_role(role_id):
if role_id is None:
return
key = cache.object_key(Role, role_id)
data = cache.get_complex(key)
if data is None:
role = Role.by_id(role_id)
if role is None:
return
data = role.to_dict()
cache.set_complex(key, data, expires=cache.EXPIRE)
return data
def get_role(role_id):
if role_id is None:
return
key = cache.object_key(Role, role_id)
data = cache.get_complex(key)
if data is None:
log.debug("Role [%s]: object cache miss", role_id)
role = Role.by_id(role_id)
if role is None:
return
data = role.to_dict()
cache.set_complex(key, data, expire=cache.EXPIRE)
return data
def get_role(role_id):
if role_id is None:
return
key = cache.object_key(Role, role_id)
data = cache.get_complex(key)
if data is None:
log.debug("Role [%s]: object cache miss", role_id)
role = Role.by_id(role_id)
if role is None:
return
data = role.to_dict()
cache.set_complex(key, data, expire=cache.EXPIRE)
return data
def get_export(export_id):
if export_id is None:
return
key = cache.object_key(Export, export_id)
data = cache.get_complex(key)
if data is None:
export = Export.by_id(export_id)
if export is None:
return
log.debug("Export cache refresh: %r", export)
data = export.to_dict()
cache.set_complex(key, data, expires=cache.EXPIRE)
return data
def get_collection_stats(collection_id):
"""Compute some statistics on the content of a collection."""
key = cache.key('cstats', collection_id)
data = cache.get_complex(key)
if data is not None:
return data
log.info("Generating collection stats: %s", collection_id)
query = {
'size': 0,
'query': {
'bool': {
'filter': [{
'term': {
'collection_id': collection_id
}
}]
}
},
'aggs': {
'schemata': {
'terms': {
'field': 'schema',
'size': 1000
}
},
'countries': {
'terms': {
'field': 'countries',
'size': 500
}
},
'languages': {
'terms': {
'field': 'languages',
'size': 10
}
},
}
}
result = search_safe(index=entities_read_index(), body=query)
aggregations = result.get('aggregations', {})
data = {'count': result['hits']['total']}
for facet in ['schemata', 'countries', 'languages']:
data[facet] = {}
for bucket in aggregations[facet]['buckets']:
data[facet][bucket['key']] = bucket['doc_count']
expire = randint(3600 * 3, 3600 * 12)
cache.set_complex(key, data, expire=expire)
return data
def get_role(role_id):
key = cache.object_key(Role, role_id)
data = cache.get_complex(key)
if data is None:
role = Role.by_id(role_id)
if role is None:
return
data = {
'id': role.id,
'name': role.name,
'label': role.label,
'type': role.type
}
cache.set_complex(key, data, expire=cache.EXPIRE)
return data
def get_alert(alert_id):
key = cache.object_key(Alert, alert_id)
data = cache.get_complex(key)
if data is None:
alert = Alert.by_id(alert_id)
if alert is None:
return
data = {
'id': alert.id,
'query': alert.query,
'role_id': alert.role_id,
'notified_at': alert.notified_at,
'created_at': alert.created_at,
'updated_at': alert.updated_at
}
cache.set_complex(key, data, expire=cache.EXPIRE)
return data
def get_collection(collection_id):
"""Fetch a collection from the index."""
if collection_id is None:
return
key = cache.object_key(Collection, collection_id)
data = cache.get_complex(key)
if data is not None:
return data
collection = Collection.by_id(collection_id)
if collection is None:
return
data = collection.to_dict()
things = get_collection_things(collection.id)
data['count'] = sum(things.values())
cache.set_complex(key, data, expires=cache.EXPIRE)
return data
def oauth_callback():
require(settings.OAUTH)
err = Unauthorized(gettext("Authentication has failed."))
state = cache.get_complex(_oauth_session(request.args.get("state")))
if state is None:
raise err
try:
oauth.provider.framework.set_session_data(request, "state", state.get("state"))
uri = state.get("redirect_uri")
oauth_token = oauth.provider.authorize_access_token(redirect_uri=uri)
except AuthlibBaseError as err:
log.warning("Failed OAuth: %r", err)
raise err
if oauth_token is None or isinstance(oauth_token, AuthlibBaseError):
log.warning("Failed OAuth: %r", oauth_token)
raise err
role = handle_oauth(oauth.provider, oauth_token)
if role is None:
raise err
# Determine session duration based on OAuth settings
expire = oauth_token.get("expires_in", Authz.EXPIRE)
expire = oauth_token.get("refresh_expires_in", expire)
db.session.commit()
update_role(role)
log.debug("Logged in: %r", role)
request.authz = Authz.from_role(role, expire=expire)
token = request.authz.to_token()
# Store id_token to generate logout URL later
id_token = oauth_token.get("id_token")
if id_token is not None:
cache.set(_token_session(token), id_token, expires=expire)
next_path = get_url_path(state.get("next_url"))
next_url = ui_url("oauth", next=next_path)
next_url = "%s#token=%s" % (next_url, token)
session.clear()
return redirect(next_url)
def get_collection(collection_id):
"""Fetch a collection from the index."""
if collection_id is None:
return
key = cache.object_key(Collection, collection_id)
data = cache.get_complex(key)
if data is not None:
return data
collection = Collection.by_id(collection_id)
if collection is None:
return
data = collection.to_dict()
schemata = get_facet_values(collection.id, 'schema')
schemata = schemata.get('values', {})
data['count'] = sum(schemata.values())
data['schemata'] = schemata
cache.set_complex(key, data, expires=cache.EXPIRE)
return data
def get_collection(collection_id):
"""Fetch a collection from the index."""
if collection_id is None:
return
key = cache.object_key(Collection, collection_id)
data = cache.get_complex(key)
if data is not None:
return data
collection = Collection.by_id(collection_id)
if collection is None:
return
data = collection.to_dict()
index = entities_read_index(schema=Entity.THING)
query = {"term": {"collection_id": collection_id}}
result = es.count(index=index, body={"query": query})
data["count"] = result.get("count", 0)
cache.set_complex(key, data, expires=cache.EXPIRE)
return data
def metadata():
locale = get_locale()
enable_cache(vary_user=False, vary=str(locale))
key = cache.key('metadata', locale)
data = cache.get_complex(key)
if data is not None:
return jsonify(data)
auth = {}
if settings.PASSWORD_LOGIN:
auth['password_login_uri'] = url_for('sessions_api.password_login')
auth['registration_uri'] = url_for('roles_api.create_code')
if settings.OAUTH:
auth['oauth_uri'] = url_for('sessions_api.oauth_init')
data = {
'status': 'ok',
'maintenance': request.authz.in_maintenance,
'app': {
'title': settings.APP_TITLE,
'description': settings.APP_DESCRIPTION,
'version': __version__,
'banner': settings.APP_BANNER,
'ui_uri': settings.APP_UI_URL,
'samples': settings.SAMPLE_SEARCHES,
'logo': settings.APP_LOGO,
'favicon': settings.APP_FAVICON,
'locale': str(locale),
'locales': settings.UI_LANGUAGES
},
'categories': Collection.CATEGORIES,
'countries': registry.country.names,
'languages': registry.language.names,
'model': model,
'auth': auth
}
cache.set_complex(key, data, expire=120)
return jsonify(data)
def metadata():
locale = get_locale()
enable_cache(vary_user=False, vary=str(locale))
key = cache.key('metadata', locale)
data = cache.get_complex(key)
if data is not None:
return jsonify(data)
auth = {}
if settings.PASSWORD_LOGIN:
auth['password_login_uri'] = url_for('sessions_api.password_login')
auth['registration_uri'] = url_for('roles_api.create_code')
if settings.OAUTH:
auth['oauth_uri'] = url_for('sessions_api.oauth_init')
data = {
'status': 'ok',
'maintenance': request.authz.in_maintenance,
'app': {
'title': settings.APP_TITLE,
'description': settings.APP_DESCRIPTION,
'version': __version__,
'banner': settings.APP_BANNER,
'ui_uri': settings.APP_UI_URL,
'samples': settings.SAMPLE_SEARCHES,
'logo': settings.APP_LOGO,
'favicon': settings.APP_FAVICON,
'locale': str(locale),
'locales': settings.UI_LANGUAGES
},
'categories': Collection.CATEGORIES,
'countries': registry.country.names,
'languages': registry.language.names,
'model': model,
'auth': auth
}
cache.set_complex(key, data, expire=120)
return jsonify(data)
def statistics():
"""Get a summary of the data acessible to an anonymous user.
Changed [3.9]: Previously, this would return user-specific stats.
---
get:
summary: System-wide user statistics.
description: >
Get a summary of the data acessible to an anonymous user.
responses:
'200':
description: OK
content:
application/json:
schema:
type: object
tags:
- System
"""
enable_cache(vary_user=False)
key = cache.key(cache.STATISTICS)
data = {"countries": [], "schemata": [], "categories": []}
data = cache.get_complex(key) or data
return jsonify(data)
