def oauth_callback(): if not settings.OAUTH: abort(404) resp = oauth.provider.authorized_response() if resp is None or isinstance(resp, OAuthException): log.warning("Failed OAuth: %r", resp) return Unauthorized("Authentication has failed.") response = signals.handle_oauth_session.send(provider=oauth.provider, oauth=resp) for (_, role) in response: if role is None: continue db.session.commit() update_role(role) log.info("Logged in: %r", role) request.authz = Authz.from_role(role) record_audit(Audit.ACT_LOGIN) token = request.authz.to_token(role=role) token = token.decode('utf-8') state = request.args.get('state') next_url = get_best_next_url(state, request.referrer) next_url, _ = urldefrag(next_url) next_url = '%s#token=%s' % (next_url, token) return redirect(next_url) log.error("No OAuth handler for %r was installed.", oauth.provider.name) return Unauthorized("Authentication has failed.")
def test_bulk_entitysets_api(self): role, headers = self.login(is_admin=True) authz = Authz.from_role(role) data = {"type": EntitySet.LIST, "label": "Foo"} eset = EntitySet.create(data, self.col, authz) db.session.commit() eset_id = eset.id data = json.dumps([ { "id": "4345800498380953840", "schema": "Person", "properties": { "name": "Osama bin Laden" }, }, { "id": "7598743983789743598", "schema": "Person", "properties": { "name": "Osama bin Laden" }, }, ]) url = "/api/2/collections/%s/_bulk?entityset_id=%s" % (self.col.id, eset_id) res = self.client.post(url, headers=headers, data=data) assert res.status_code == 204, res query = "/api/2/entitysets/%s/entities?filter:schema=Person" % eset_id res = self.client.get(query, headers=headers) assert res.json["total"] == 2, res.json
def test_anonymous(self): authz = Authz.from_role(None) assert authz.logged_in is False, authz assert authz.is_admin is False, authz.is_admin assert authz.id is None, authz.id assert authz.role is None, authz.role assert len(authz.roles) == 1, authz.roles
def sample_entities(secret, properties, schematas, seed, sample_pct, limit, outfile): """Sample random entities""" random.seed(seed) authz = Authz.from_role(Role.load_cli_user()) collections = list(Collection.all_by_secret(secret, authz)) random.shuffle(collections) iter_proxies_kwargs = { "authz": authz, "schemata": schematas or None, "randomize": True, "random_seed": seed, } n_entities = 0 for collection in collections: for entity in iter_proxies(collection_id=collection.id, **iter_proxies_kwargs): if properties and not any( entity.properties.get(prop) for prop in properties): continue if not sample_pct or random.random() < sample_pct: write_object(outfile, entity) n_entities += 1 if limit and n_entities >= limit: return
def test_token(self): authz = Authz.from_role(self.admin) token = authz.to_token() with self.assertRaises(Unauthorized): Authz.from_token("banana") sauthz = Authz.from_token(token) assert sauthz.id == authz.id
def sitemap(): """ --- get: summary: Get a sitemap description: >- Returns a site map for search engine robots. This lists each published collection on the current instance. responses: '200': description: OK content: text/xml: schema: type: object tags: - System """ enable_cache(vary_user=False) request.rate_limit = None collections = [] for collection in Collection.all_authz(Authz.from_role(None)): updated_at = collection.updated_at.date().isoformat() updated_at = max(settings.SITEMAP_FLOOR, updated_at) collections.append({ 'url': collection_url(collection.id), 'updated_at': updated_at }) return render_xml('sitemap.xml', collections=collections)
def oauth_callback(): require(settings.OAUTH) try: token = oauth.provider.authorize_access_token() except AuthlibBaseError as err: log.warning("Failed OAuth: %r", err) raise Unauthorized(gettext("Authentication has failed.")) if token is None or isinstance(token, AuthlibBaseError): log.warning("Failed OAuth: %r", token) raise Unauthorized(gettext("Authentication has failed.")) role = handle_oauth(oauth.provider, token) if role is None: log.error("No OAuth handler was installed.") raise Unauthorized(gettext("Authentication has failed.")) if role.is_blocked: raise Unauthorized(gettext("Your account is blocked.")) db.session.commit() update_role(role) log.info("Logged in: %r", role) request.authz = Authz.from_role(role) token = request.authz.to_token(role=role) token = token.decode('utf-8') next_path = get_url_path(request.args.get('state')) next_url = ui_url(settings.OAUTH_UI_CALLBACK, next=next_path) next_url = '%s#token=%s' % (next_url, token) return redirect(next_url)
def check_alert(alert_id): alert = Alert.by_id(alert_id) if alert is None or alert.role is None: return log.info("Check alert [%s]: %s", alert.id, alert.query) authz = Authz.from_role(alert.role) query = alert_query(alert, authz) index = entities_read_index(schema=Entity.THING) result = es.search(index=index, body=query) for result in result.get('hits').get('hits', []): entity = unpack_result(result) if entity is None: continue log.info('Alert [%s]: %s', alert.query, entity.get('name')) params = { 'alert': alert, 'role': alert.role, 'entity': entity.get('id') } publish(Events.MATCH_ALERT, params=params, channels=[alert.role]) db.session.flush() alert.update() db.session.commit() db.session.close()
def check_alert(alert_id): alert = Alert.by_id(alert_id) if alert is None or alert.role is None: return log.info("Check alert [%s]: %s", alert.id, alert.query) authz = Authz.from_role(alert.role) try: query = alert_query(alert, authz) index = entities_read_index(schema=Entity.THING) result = es.search(index=index, body=query) except RequestError as re: log.error("Invalid query [%s]: %r", alert.query, re.error) alert.delete() db.session.commit() return for result in result.get("hits").get("hits", []): entity = unpack_result(result) if entity is None: continue log.info("Alert [%s]: %s", alert.query, entity.get("id")) params = { "alert": alert, "role": alert.role, "entity": entity.get("id"), "collection": entity.get("collection_id"), } channels = [alert.role] # channels.append(channel_tag(collection_id, Collection)) publish(Events.MATCH_ALERT, params=params, channels=channels) alert.update() db.session.commit()
def generate_sitemap(collection_id): """Generate entries for a collection-based sitemap.xml file.""" # cf. https://www.sitemaps.org/protocol.html query = { 'query': { 'bool': { 'filter': [ {'term': {'collection_id': collection_id}}, {'term': {'schemata': Entity.THING}}, authz_query(Authz.from_role(None)) ] } }, '_source': {'includes': ['schemata', 'updated_at']} } scanner = scan(es, index=entities_index(), query=query) # strictly, the limit for sitemap.xml is 50,000 for res in islice(scanner, 49500): source = res.get('_source', {}) updated_at = source.get('updated_at', '').split('T', 1)[0] if Document.SCHEMA in source.get('schemata', []): url = document_url(res.get('_id')) else: url = entity_url(res.get('_id')) yield (url, updated_at)
def oauth_callback(): require(settings.OAUTH) err = Unauthorized(gettext("Authentication has failed.")) state = cache.get_complex(_oauth_session(request.args.get("state"))) if state is None: raise err try: oauth.provider.framework.set_session_data(request, "state", state.get("state")) uri = state.get("redirect_uri") token = oauth.provider.authorize_access_token(redirect_uri=uri) except AuthlibBaseError as err: log.warning("Failed OAuth: %r", err) raise err if token is None or isinstance(token, AuthlibBaseError): log.warning("Failed OAuth: %r", token) raise err role = handle_oauth(oauth.provider, token) if role is None: raise err db.session.commit() update_role(role) log.debug("Logged in: %r", role) request.authz = Authz.from_role(role) next_path = get_url_path(state.get("next_url")) next_url = ui_url("oauth", next=next_path) next_url = "%s#token=%s" % (next_url, request.authz.to_token()) session.clear() return redirect(next_url)
def create_collection(self, creator=None, **kwargs): authz = Authz.from_role(creator) collection = Collection.create(kwargs, authz) db.session.add(collection) db.session.commit() update_collection(collection, sync=True) return collection
def check_alert(alert_id): alert = Alert.by_id(alert_id) if alert is None or alert.role is None: return if not alert.role.is_alertable: return authz = Authz.from_role(alert.role) query = alert_query(alert, authz) index = entities_read_index(schema=Entity.THING) result = es.search(index=index, body=query) for result in result.get('hits').get('hits', []): entity = unpack_result(result) if entity is None: continue log.info('Alert [%s]: %s', alert.query, entity.get('name')) params = { 'alert': alert, 'role': alert.role, 'entity': entity } publish(Events.MATCH_ALERT, actor_id=entity.get('uploader_id'), params=params) alert.update() db.session.commit() db.session.close()
def compute_collections(): """Update collection caches, including the global stats cache.""" authz = Authz.from_role(None) schemata = defaultdict(int) countries = defaultdict(int) categories = defaultdict(int) for collection in Collection.all(): compute_collection(collection) if authz.can(collection.id, authz.READ): categories[collection.category] += 1 things = index.get_collection_things(collection.id) for schema, count in things.items(): schemata[schema] += count for country in collection.countries: countries[country] += 1 log.info("Updating global statistics cache...") data = { "collections": sum(categories.values()), "schemata": dict(schemata), "countries": dict(countries), "categories": dict(categories), "things": sum(schemata.values()), } key = cache.key(cache.STATISTICS) cache.set_complex(key, data, expires=cache.EXPIRE)
def test_anon_claim(self): authz = Authz.from_role(None) claim_url = archive_url(authz, self.content_hash, file_name="foo") res = self.client.get(claim_url) assert res.status_code == 200, res.status_code disposition = res.headers.get("Content-Disposition") assert "foo" in disposition, disposition
def create(): require(settings.PASSWORD_LOGIN) require(not request.authz.in_maintenance) data = parse_request(RoleCreateSchema) try: email = Role.SIGNATURE.loads(data.get('code'), max_age=Role.SIGNATURE_MAX_AGE) except BadSignature: return jsonify({ 'status': 'error', 'message': gettext('Invalid code') }, status=400) role = Role.by_email(email) if role is not None: return jsonify( { 'status': 'error', 'message': gettext('Email is already registered') }, status=409) role = Role.load_or_create(foreign_id='password:{}'.format(email), type=Role.USER, name=data.get('name') or email, email=email) role.set_password(data.get('password')) db.session.add(role) db.session.commit() update_role(role) # Let the serializer return more info about this user request.authz = Authz.from_role(role) tag_request(role_id=role.id) return RoleSerializer.jsonify(role, status=201)
def oauth_callback(): require(settings.OAUTH) resp = oauth.provider.authorized_response() if resp is None or isinstance(resp, OAuthException): log.warning("Failed OAuth: %r", resp) raise Unauthorized(gettext("Authentication has failed.")) response = signals.handle_oauth_session.send(provider=oauth.provider, oauth=resp) for (_, role) in response: if role is None: continue db.session.commit() update_role(role) log.info("Logged in: %r", role) request.authz = Authz.from_role(role) token = request.authz.to_token(role=role) token = token.decode('utf-8') next_path = get_url_path(request.args.get('state')) next_url = ui_url(settings.OAUTH_UI_CALLBACK, next=next_path) next_url = '%s#token=%s' % (next_url, token) return redirect(next_url) log.error("No OAuth handler for %r was installed.", oauth.provider.name) raise Unauthorized(gettext("Authentication has failed."))
def test_maintenance(self): settings.MAINTENANCE = True authz = Authz.from_role(self.admin) assert authz.logged_in is True, authz assert authz.is_admin is True, authz.is_admin assert authz.can(self.public, authz.WRITE) is False, authz._collections settings.MAINTENANCE = False
def ensure_collection(foreign_id, label): authz = Authz.from_role(Role.load_cli_user()) config = { 'foreign_id': foreign_id, 'label': label, } create_collection(config, authz) return Collection.by_foreign_id(foreign_id)
def decode_authz(): authz = None if 'Authorization' in request.headers: credential = request.headers.get('Authorization') authz = _get_credential_authz(credential) if authz is None and 'api_key' in request.args: authz = _get_credential_authz(request.args.get('api_key')) request.authz = authz or Authz.from_role(role=None)
def sitemap(): enable_cache(vary_user=False) collections = [] for collection in Collection.all_authz(Authz.from_role(None)): updated_at = collection.updated_at.date().isoformat() updated_at = max(settings.SITEMAP_FLOOR, updated_at) collections.append({ 'url': collection_url(collection.id), 'updated_at': updated_at }) return render_xml('sitemap.xml', collections=collections)
def sitemap(): enable_cache(vary_user=False) collections = [] for collection in Collection.all_authz(Authz.from_role(None)): updated_at = collection.updated_at.date().isoformat() updated_at = max(settings.SITEMAP_FLOOR, updated_at) collections.append({ 'url': collection_url(collection.id), 'updated_at': updated_at }) return render_xml('sitemap.xml', collections=collections)
def _get_credential_authz(credential): if credential is None or not len(credential): return if " " in credential: method, credential = credential.split(" ", 1) if method == "Token": return Authz.from_token(credential) role = Role.by_api_key(credential) if role is not None: return Authz.from_role(role=role)
def test_admin(self): authz = Authz.from_role(self.admin) assert authz.logged_in is True, authz assert authz.is_admin is True, authz.is_admin assert authz.id == self.admin.id, authz.id assert len(authz.roles) == 3, authz.roles assert authz.can(self.public, authz.READ) is True, authz._collections assert authz.can(self.public, authz.WRITE) is True, authz._collections assert authz.can(self.private, authz.READ) is True, authz._collections assert authz.can(self.private, authz.WRITE) is True, authz._collections
def sitemap_index(): enable_cache(vary_user=False) collections = [] for collection in Collection.all_authz(Authz.from_role(None)): collections.append({ 'url': url_for('collections_api.sitemap', id=collection.id), 'updated_at': collection.updated_at.date().isoformat() }) return render_xml('sitemap_index.xml', collections=collections)
def test_user(self): authz = Authz.from_role(self.user) assert authz.logged_in is True, authz assert authz.is_admin is False, authz.is_admin # assert authz.id == self.user.id, authz.id # assert authz.role == self.user, authz.role # assert len(authz.roles) == 4, (authz.roles, self.user.roles) assert authz.can(self.public, authz.READ) is True, authz._collections assert authz.can(self.public, authz.WRITE) is False, authz._collections assert authz.can(self.private, authz.READ) is True, authz._collections assert authz.can(self.private, authz.WRITE) is False, authz._collections
def test_scope(self): authz = Authz.from_role(self.admin) token = authz.to_token(scope="/bla") with self.assertRaises(Unauthorized): Authz.from_token(token) with self.assertRaises(Unauthorized): Authz.from_token(token, scope="/blubb") sauthz = Authz.from_token(token, scope="/bla") assert sauthz.id == authz.id assert abs(sauthz.expire - authz.expire) < timedelta(seconds=2) assert sauthz.expire > datetime.utcnow()
def crawldir(path, language=None, foreign_id=None): """Crawl the given directory.""" path = Path(path) if foreign_id is None: foreign_id = 'directory:%s' % slugify(path) authz = Authz.from_role(Role.load_cli_user()) config = {'foreign_id': foreign_id, 'label': path.name, 'casefile': False} create_collection(config, authz) collection = Collection.by_foreign_id(foreign_id) log.info('Crawling %s to %s (%s)...', path, foreign_id, collection.id) crawl_directory(collection, path) log.info('Complete. Make sure a worker is running :)')
def decode_authz(): authz = None if 'Authorization' in request.headers: credential = request.headers.get('Authorization') authz = _get_credential_authz(credential) if authz is None and 'api_key' in request.args: authz = _get_credential_authz(request.args.get('api_key')) authz = authz or Authz.from_role(role=None) request.authz = authz
def _get_credential_authz(credential): if credential is None or not len(credential): return if ' ' in credential: mechanism, credential = credential.split(' ', 1) authz = Authz.from_token(credential, scope=request.path) if authz is not None: return authz role = Role.by_api_key(credential) if role is not None: return Authz.from_role(role=role)
def enable_authz(request): authz = None if "Authorization" in request.headers: credential = request.headers.get("Authorization") authz = _get_credential_authz(credential) if authz is None and "api_key" in request.args: authz = _get_credential_authz(request.args.get("api_key")) authz = authz or Authz.from_role(role=None) request.authz = authz
def metadata(): """Get operational metadata for the frontend. --- get: summary: Retrieve system metadata from the application. responses: '200': description: OK content: application/json: schema: type: object tags: - System """ locale = get_locale() auth = {} if settings.PASSWORD_LOGIN: auth['password_login_uri'] = url_for('sessions_api.password_login') auth['registration_uri'] = url_for('roles_api.create_code') if settings.OAUTH: auth['oauth_uri'] = url_for('sessions_api.oauth_init') locales = settings.UI_LANGUAGES locales = {l: Locale(l).get_language_name(l) for l in locales} data = { 'status': 'ok', 'maintenance': request.authz.in_maintenance, 'app': { 'title': settings.APP_TITLE, 'description': settings.APP_DESCRIPTION, 'version': __version__, 'banner': settings.APP_BANNER, 'ui_uri': settings.APP_UI_URL, 'samples': settings.SAMPLE_SEARCHES, 'logo': settings.APP_LOGO, 'favicon': settings.APP_FAVICON, 'locale': str(locale), 'locales': locales }, 'categories': Collection.CATEGORIES, 'model': model, 'token': None, 'auth': auth } if settings.SINGLE_USER: role = Role.load_cli_user() authz = Authz.from_role(role) data['token'] = authz.to_token(role=role) return jsonify(data)
def password_login(): """Provides email and password authentication.""" data = parse_request(LoginSchema) role = Role.by_email(data.get('email')) if role is None or not role.has_password: return Unauthorized("Authentication has failed.") if not role.check_password(data.get('password')): return Unauthorized("Authentication has failed.") db.session.commit() update_role(role) authz = Authz.from_role(role) request.authz = authz record_audit(Audit.ACT_LOGIN) return jsonify({ 'status': 'ok', 'token': authz.to_token(role=role) })