def get_collection(collection_id): """Fetch a collection from the index.""" if collection_id is None: return key = cache.object_key(Collection, collection_id) data = cache.get_complex(key) if data is not None: return data collection = Collection.by_id(collection_id) if collection is None: return data = collection.to_dict() stats = get_collection_stats(collection.id) data['count'] = stats['count'] data['schemata'] = stats['schemata'] # if no countries or langs are given, take the most common from the data. countries = ensure_list(collection.countries) countries = countries or stats['countries'].keys() data['countries'] = registry.country.normalize_set(countries) languages = ensure_list(collection.languages) languages = languages or stats['languages'].keys() data['languages'] = registry.language.normalize_set(languages) cache.set_complex(key, data, expire=cache.EXPIRE) return data
def get_collection(collection_id): """Fetch a collection from the index.""" if collection_id is None: return key = cache.object_key(Collection, collection_id) data = cache.get_complex(key) if data is not None: return data collection = Collection.by_id(collection_id) if collection is None: return data = collection.to_dict() stats = get_collection_stats(collection.id) data['count'] = stats['count'] data['schemata'] = stats['schemata'] # if no countries or langs are given, take the most common from the data. countries = ensure_list(collection.countries) countries = countries or stats['countries'].keys() data['countries'] = registry.country.normalize_set(countries) languages = ensure_list(collection.languages) languages = languages or stats['languages'].keys() data['languages'] = registry.language.normalize_set(languages) cache.set_complex(key, data, expire=cache.EXPIRE) return data
def get_collection_facet(collection_id, facet, refresh=False): """Compute some statistics on the content of a collection.""" key = cache.object_key(Collection, collection_id, facet) data = cache.get_complex(key) if not refresh and data is not None: return data query = {'term': {'collection_id': collection_id}} query = { 'size': 0, 'query': {'bool': {'filter': [query]}}, 'aggs': { 'values': {'terms': {'field': facet, 'size': 300}}, 'total': {'cardinality': {'field': facet}} } } schemata = set() facet_type = registry.groups.get(facet) if facet_type is not None: schemata = model.get_type_schemata(facet_type) result = es.search(index=entities_read_index(schema=schemata), body=query, request_timeout=3600, timeout='20m') aggregations = result.get('aggregations') values = {} for bucket in aggregations.get('values').get('buckets', []): values[bucket['key']] = bucket['doc_count'] data = { 'values': values, 'total': aggregations.get('total').get('value', 0) } cache.set_complex(key, data, expires=cache.EXPIRE) return data
def get_entity(entity_id): key = cache.object_key(Entity, entity_id) entity = cache.get_complex(key) if entity is None: entity = index.get_entity(entity_id) cache.set_complex(key, entity, expire=cache.EXPIRE) return entity
def get_collection(collection_id): key = cache.object_key(Collection, collection_id) data = cache.get_complex(key) if data is None: data = index.get_collection(collection_id) cache.set_complex(key, data, expire=cache.EXPIRE) return data
def oauth_callback(): require(settings.OAUTH) err = Unauthorized(gettext("Authentication has failed.")) state = cache.get_complex(_oauth_session(request.args.get("state"))) if state is None: raise err try: oauth.provider.framework.set_session_data(request, "state", state.get("state")) uri = state.get("redirect_uri") token = oauth.provider.authorize_access_token(redirect_uri=uri) except AuthlibBaseError as err: log.warning("Failed OAuth: %r", err) raise err if token is None or isinstance(token, AuthlibBaseError): log.warning("Failed OAuth: %r", token) raise err role = handle_oauth(oauth.provider, token) if role is None: raise err db.session.commit() update_role(role) log.debug("Logged in: %r", role) request.authz = Authz.from_role(role) next_path = get_url_path(state.get("next_url")) next_url = ui_url("oauth", next=next_path) next_url = "%s#token=%s" % (next_url, request.authz.to_token()) session.clear() return redirect(next_url)
def get_profile(entityset_id, authz=None): """A profile is an entityset having a party. The idea is to cache profile metadata for the API, and to generate a merged view of all the entities the current user has access to.""" if entityset_id is None: return key = cache.object_key(EntitySet, entityset_id) data = cache.get_complex(key) stub = Stub() if data is None: entityset = get_entityset(entityset_id) if entityset is None: return data = entityset.to_dict() data["items"] = [] for item in entityset.items(): data["items"].append(item.to_dict()) cache.set_complex(key, data, expires=cache.EXPIRE) # Filter the subset of items the current user can access if authz is not None: items = [ i for i in data["items"] if authz.can(i["collection_id"], authz.READ) ] data["items"] = items # Load the constituent entities for the profile and generate a # combined proxy with all of the given properties. for item in data["items"]: if Judgement(item["judgement"]) == Judgement.POSITIVE: resolver.queue(stub, Entity, item.get("entity_id")) resolver.resolve(stub) merged = None data["proxies"] = [] for item in data["items"]: item["entity"] = resolver.get(stub, Entity, item.get("entity_id")) if item["entity"] is not None: proxy = model.get_proxy(item["entity"]) proxy.context = {} data["proxies"].append(proxy) if merged is None: merged = proxy.clone() merged.context["entities"] = [proxy.id] else: merged.merge(proxy) merged.context["entities"].append(proxy.id) if merged is None: merged = model.make_entity(Entity.LEGAL_ENTITY) # Polish it a bit: merged.id = data.get("id") merged = name_entity(merged) data["merged"] = merged data["label"] = merged.caption data["shallow"] = False return data
def metadata(): """Get operational metadata for the frontend. --- get: summary: Retrieve system metadata from the application. responses: '200': description: OK content: application/json: schema: type: object tags: - System """ locale = get_locale() enable_cache(vary_user=False, vary=str(locale)) key = cache.key('metadata', settings.PROCESS_ID, locale) data = cache.get_complex(key) if data is not None: return jsonify(data) auth = {} if settings.PASSWORD_LOGIN: auth['password_login_uri'] = url_for('sessions_api.password_login') auth['registration_uri'] = url_for('roles_api.create_code') if settings.OAUTH: auth['oauth_uri'] = url_for('sessions_api.oauth_init') locales = settings.UI_LANGUAGES locales = {l: Locale(l).get_language_name(l) for l in locales} data = { 'status': 'ok', 'maintenance': request.authz.in_maintenance, 'app': { 'title': settings.APP_TITLE, 'description': settings.APP_DESCRIPTION, 'version': __version__, 'banner': settings.APP_BANNER, 'ui_uri': settings.APP_UI_URL, 'samples': settings.SAMPLE_SEARCHES, 'logo': settings.APP_LOGO, 'favicon': settings.APP_FAVICON, 'locale': str(locale), 'locales': locales }, 'categories': Collection.CATEGORIES, 'countries': registry.country.names, 'languages': registry.language.names, 'model': model, 'auth': auth } cache.set_complex(key, data, expires=120) return jsonify(data)
def get_entity(entity_id, **kwargs): """Fetch an entity from the index.""" if entity_id is None: return key = cache.object_key(Entity, entity_id) entity = cache.get_complex(key) if entity is not None: return entity log.debug("Entity [%s]: object cache miss", entity_id) for entity in entities_by_ids(entity_id, cached=True): return entity
def get_entity(entity_id, **kwargs): """Fetch an entity from the index.""" if entity_id is None: return key = cache.object_key(Entity, entity_id) entity = cache.get_complex(key) if entity is not None: return entity log.debug("Entity [%s]: object cache miss", entity_id) for entity in entities_by_ids(entity_id, cached=True): return entity
def from_token(cls, token_id): state_key = cache.key(cls.TOKENS, token_id) state = cache.get_complex(state_key) if state is None: raise Unauthorized() return cls( state.get("id"), state.get("roles"), is_admin=state.get("is_admin"), token_id=token_id, )
def get_collection_things(collection_id): """Showing the number of things in a collection is more indicative of its size than the overall collection entity count.""" schemata = cache.get_complex(_facet_key(collection_id, "schema")) if schemata is None: return {} things = {} for schema, count in schemata.get("values", {}).items(): schema = model.get(schema) if schema is not None and schema.is_a(Entity.THING): things[schema.name] = count return things
def get_role(role_id): if role_id is None: return key = cache.object_key(Role, role_id) data = cache.get_complex(key) if data is None: role = Role.by_id(role_id) if role is None: return data = role.to_dict() cache.set_complex(key, data, expires=cache.EXPIRE) return data
def get_role(role_id): if role_id is None: return key = cache.object_key(Role, role_id) data = cache.get_complex(key) if data is None: log.debug("Role [%s]: object cache miss", role_id) role = Role.by_id(role_id) if role is None: return data = role.to_dict() cache.set_complex(key, data, expire=cache.EXPIRE) return data
def get_role(role_id): if role_id is None: return key = cache.object_key(Role, role_id) data = cache.get_complex(key) if data is None: log.debug("Role [%s]: object cache miss", role_id) role = Role.by_id(role_id) if role is None: return data = role.to_dict() cache.set_complex(key, data, expire=cache.EXPIRE) return data
def get_export(export_id): if export_id is None: return key = cache.object_key(Export, export_id) data = cache.get_complex(key) if data is None: export = Export.by_id(export_id) if export is None: return log.debug("Export cache refresh: %r", export) data = export.to_dict() cache.set_complex(key, data, expires=cache.EXPIRE) return data
def get_collection_stats(collection_id): """Compute some statistics on the content of a collection.""" key = cache.key('cstats', collection_id) data = cache.get_complex(key) if data is not None: return data log.info("Generating collection stats: %s", collection_id) query = { 'size': 0, 'query': { 'bool': { 'filter': [{ 'term': { 'collection_id': collection_id } }] } }, 'aggs': { 'schemata': { 'terms': { 'field': 'schema', 'size': 1000 } }, 'countries': { 'terms': { 'field': 'countries', 'size': 500 } }, 'languages': { 'terms': { 'field': 'languages', 'size': 10 } }, } } result = search_safe(index=entities_read_index(), body=query) aggregations = result.get('aggregations', {}) data = {'count': result['hits']['total']} for facet in ['schemata', 'countries', 'languages']: data[facet] = {} for bucket in aggregations[facet]['buckets']: data[facet][bucket['key']] = bucket['doc_count'] expire = randint(3600 * 3, 3600 * 12) cache.set_complex(key, data, expire=expire) return data
def get_role(role_id): key = cache.object_key(Role, role_id) data = cache.get_complex(key) if data is None: role = Role.by_id(role_id) if role is None: return data = { 'id': role.id, 'name': role.name, 'label': role.label, 'type': role.type } cache.set_complex(key, data, expire=cache.EXPIRE) return data
def get_alert(alert_id): key = cache.object_key(Alert, alert_id) data = cache.get_complex(key) if data is None: alert = Alert.by_id(alert_id) if alert is None: return data = { 'id': alert.id, 'query': alert.query, 'role_id': alert.role_id, 'notified_at': alert.notified_at, 'created_at': alert.created_at, 'updated_at': alert.updated_at } cache.set_complex(key, data, expire=cache.EXPIRE) return data
def get_collection(collection_id): """Fetch a collection from the index.""" if collection_id is None: return key = cache.object_key(Collection, collection_id) data = cache.get_complex(key) if data is not None: return data collection = Collection.by_id(collection_id) if collection is None: return data = collection.to_dict() things = get_collection_things(collection.id) data['count'] = sum(things.values()) cache.set_complex(key, data, expires=cache.EXPIRE) return data
def oauth_callback(): require(settings.OAUTH) err = Unauthorized(gettext("Authentication has failed.")) state = cache.get_complex(_oauth_session(request.args.get("state"))) if state is None: raise err try: oauth.provider.framework.set_session_data(request, "state", state.get("state")) uri = state.get("redirect_uri") oauth_token = oauth.provider.authorize_access_token(redirect_uri=uri) except AuthlibBaseError as err: log.warning("Failed OAuth: %r", err) raise err if oauth_token is None or isinstance(oauth_token, AuthlibBaseError): log.warning("Failed OAuth: %r", oauth_token) raise err role = handle_oauth(oauth.provider, oauth_token) if role is None: raise err # Determine session duration based on OAuth settings expire = oauth_token.get("expires_in", Authz.EXPIRE) expire = oauth_token.get("refresh_expires_in", expire) db.session.commit() update_role(role) log.debug("Logged in: %r", role) request.authz = Authz.from_role(role, expire=expire) token = request.authz.to_token() # Store id_token to generate logout URL later id_token = oauth_token.get("id_token") if id_token is not None: cache.set(_token_session(token), id_token, expires=expire) next_path = get_url_path(state.get("next_url")) next_url = ui_url("oauth", next=next_path) next_url = "%s#token=%s" % (next_url, token) session.clear() return redirect(next_url)
def get_collection(collection_id): """Fetch a collection from the index.""" if collection_id is None: return key = cache.object_key(Collection, collection_id) data = cache.get_complex(key) if data is not None: return data collection = Collection.by_id(collection_id) if collection is None: return data = collection.to_dict() schemata = get_facet_values(collection.id, 'schema') schemata = schemata.get('values', {}) data['count'] = sum(schemata.values()) data['schemata'] = schemata cache.set_complex(key, data, expires=cache.EXPIRE) return data
def get_collection(collection_id): """Fetch a collection from the index.""" if collection_id is None: return key = cache.object_key(Collection, collection_id) data = cache.get_complex(key) if data is not None: return data collection = Collection.by_id(collection_id) if collection is None: return data = collection.to_dict() index = entities_read_index(schema=Entity.THING) query = {"term": {"collection_id": collection_id}} result = es.count(index=index, body={"query": query}) data["count"] = result.get("count", 0) cache.set_complex(key, data, expires=cache.EXPIRE) return data
def metadata(): locale = get_locale() enable_cache(vary_user=False, vary=str(locale)) key = cache.key('metadata', locale) data = cache.get_complex(key) if data is not None: return jsonify(data) auth = {} if settings.PASSWORD_LOGIN: auth['password_login_uri'] = url_for('sessions_api.password_login') auth['registration_uri'] = url_for('roles_api.create_code') if settings.OAUTH: auth['oauth_uri'] = url_for('sessions_api.oauth_init') data = { 'status': 'ok', 'maintenance': request.authz.in_maintenance, 'app': { 'title': settings.APP_TITLE, 'description': settings.APP_DESCRIPTION, 'version': __version__, 'banner': settings.APP_BANNER, 'ui_uri': settings.APP_UI_URL, 'samples': settings.SAMPLE_SEARCHES, 'logo': settings.APP_LOGO, 'favicon': settings.APP_FAVICON, 'locale': str(locale), 'locales': settings.UI_LANGUAGES }, 'categories': Collection.CATEGORIES, 'countries': registry.country.names, 'languages': registry.language.names, 'model': model, 'auth': auth } cache.set_complex(key, data, expire=120) return jsonify(data)
def metadata(): locale = get_locale() enable_cache(vary_user=False, vary=str(locale)) key = cache.key('metadata', locale) data = cache.get_complex(key) if data is not None: return jsonify(data) auth = {} if settings.PASSWORD_LOGIN: auth['password_login_uri'] = url_for('sessions_api.password_login') auth['registration_uri'] = url_for('roles_api.create_code') if settings.OAUTH: auth['oauth_uri'] = url_for('sessions_api.oauth_init') data = { 'status': 'ok', 'maintenance': request.authz.in_maintenance, 'app': { 'title': settings.APP_TITLE, 'description': settings.APP_DESCRIPTION, 'version': __version__, 'banner': settings.APP_BANNER, 'ui_uri': settings.APP_UI_URL, 'samples': settings.SAMPLE_SEARCHES, 'logo': settings.APP_LOGO, 'favicon': settings.APP_FAVICON, 'locale': str(locale), 'locales': settings.UI_LANGUAGES }, 'categories': Collection.CATEGORIES, 'countries': registry.country.names, 'languages': registry.language.names, 'model': model, 'auth': auth } cache.set_complex(key, data, expire=120) return jsonify(data)
def statistics(): """Get a summary of the data acessible to an anonymous user. Changed [3.9]: Previously, this would return user-specific stats. --- get: summary: System-wide user statistics. description: > Get a summary of the data acessible to an anonymous user. responses: '200': description: OK content: application/json: schema: type: object tags: - System """ enable_cache(vary_user=False) key = cache.key(cache.STATISTICS) data = {"countries": [], "schemata": [], "categories": []} data = cache.get_complex(key) or data return jsonify(data)