def delete_collection(collection, sync=False): flush_notifications(collection) collection.delete() db.session.commit() index.delete_collection(collection.id, sync=sync) delete_collection_content.apply_async([collection.id], priority=7) refresh_collection(collection.id) Authz.flush()
def create_collection(data, role=None, sync=False): role = role or Role.load_cli_user() created_at = datetime.utcnow() collection = Collection.create(data, role=role, created_at=created_at) if collection.created_at == created_at: publish(Events.CREATE_COLLECTION, actor_id=role.id, params={'collection': collection}) db.session.commit() Authz.flush() refresh_collection(collection.id) return index.index_collection(collection, sync=sync)
def _get_credential_authz(credential): if credential is None or not len(credential): return if ' ' in credential: mechanism, credential = credential.split(' ', 1) authz = Authz.from_token(credential, scope=request.path) if authz is not None: return authz role = Role.by_api_key(credential) if role is not None: return Authz.from_role(role=role)
def test_maintenance(self): settings.MAINTENANCE = True authz = Authz.from_role(self.admin) assert authz.logged_in is True, authz assert authz.is_admin is True, authz.is_admin assert authz.can(self.public, authz.WRITE) is False, authz._collections settings.MAINTENANCE = False
def oauth_callback(): if not settings.OAUTH: abort(404) resp = oauth.provider.authorized_response() if resp is None or isinstance(resp, OAuthException): log.warning("Failed OAuth: %r", resp) return Unauthorized("Authentication has failed.") response = signals.handle_oauth_session.send(provider=oauth.provider, oauth=resp) for (_, role) in response: if role is None: continue db.session.commit() update_role(role) log.info("Logged in: %r", role) request.authz = Authz.from_role(role) record_audit(Audit.ACT_LOGIN) token = request.authz.to_token(role=role) token = token.decode('utf-8') state = request.args.get('state') next_url = get_best_next_url(state, request.referrer) next_url, _ = urldefrag(next_url) next_url = '%s#token=%s' % (next_url, token) return redirect(next_url) log.error("No OAuth handler for %r was installed.", oauth.provider.name) return Unauthorized("Authentication has failed.")
def test_anonymous(self): authz = Authz.from_role(None) assert authz.logged_in is False, authz assert authz.is_admin is False, authz.is_admin assert authz.id is None, authz.id assert authz.role is None, authz.role assert len(authz.roles) == 1, authz.roles
def check_alert(alert_id): alert = Alert.by_id(alert_id) if alert is None or alert.role is None: return log.info("Check alert [%s]: %s", alert.id, alert.query) authz = Authz.from_role(alert.role) try: query = alert_query(alert, authz) index = entities_read_index(schema=Entity.THING) result = es.search(index=index, body=query) except RequestError as re: log.error("Invalid query [%s]: %r", alert.query, re.error) alert.delete() db.session.commit() return for result in result.get('hits').get('hits', []): entity = unpack_result(result) if entity is None: continue log.info('Alert [%s]: %s', alert.query, entity.get('id')) params = { 'alert': alert, 'role': alert.role, 'entity': entity.get('id'), 'collection': entity.get('collection_id') } channels = [alert.role] # channels.append(channel_tag(collection_id, Collection)) publish(Events.MATCH_ALERT, params=params, channels=channels) alert.update() db.session.commit()
def oauth_callback(): if not settings.OAUTH: abort(404) resp = oauth.provider.authorized_response() if resp is None or isinstance(resp, OAuthException): log.warning("Failed OAuth: %r", resp) return Unauthorized("Authentication has failed.") response = signals.handle_oauth_session.send(provider=oauth.provider, oauth=resp) for (_, role) in response: if role is None: continue update_role(role) db.session.commit() log.info("Logged in: %r", role) authz = Authz.from_role(role) token = authz.to_token(role=role) token = token.decode('utf-8') state = request.args.get('state') next_url = get_best_next_url(state, request.referrer) next_url, _ = urldefrag(next_url) next_url = '%s#token=%s' % (next_url, token) return redirect(next_url) log.error("No OAuth handler for %r was installed.", oauth.provider.name) return Unauthorized("Authentication has failed.")
def oauth_callback(): require(settings.OAUTH) token = oauth.provider.authorize_access_token() if token is None or isinstance(token, AuthlibBaseError): log.warning("Failed OAuth: %r", token) raise Unauthorized(gettext("Authentication has failed.")) response = signals.handle_oauth_session.send(provider=oauth.provider, oauth_token=token) for (_, role) in response: if role is None: continue db.session.commit() update_role(role) log.info("Logged in: %r", role) request.authz = Authz.from_role(role) token = request.authz.to_token(role=role) token = token.decode('utf-8') next_path = get_url_path(request.args.get('state')) next_url = ui_url(settings.OAUTH_UI_CALLBACK, next=next_path) next_url = '%s#token=%s' % (next_url, token) return redirect(next_url) log.error("No OAuth handler for %r was installed.", oauth.provider.name) raise Unauthorized(gettext("Authentication has failed."))
def ensure_collection(foreign_id, label): authz = Authz.from_role(Role.load_cli_user()) config = { 'foreign_id': foreign_id, 'label': label, } create_collection(config, authz) return Collection.by_foreign_id(foreign_id)
def update_permission(role, collection, read, write, editor_id=None): """Update a roles permission to access a given collection.""" pre = Permission.by_collection_role(collection, role) post = Permission.grant(collection, role, read, write) params = {'role': role, 'collection': collection} if (pre is None or not pre.read) and post.read: if role.is_public: publish(Events.PUBLISH_COLLECTION, actor_id=editor_id, params=params, channels=[Notification.GLOBAL]) else: publish(Events.GRANT_COLLECTION, actor_id=editor_id, params=params) elif pre is not None and pre.read and not post.read: publish(Events.REVOKE_COLLECTION, actor_id=editor_id, params=params) db.session.commit() Authz.flush() return post
def delete_collection(collection, keep_metadata=False, sync=False, reset_sync=False): reset_collection(collection, sync=reset_sync) deleted_at = collection.deleted_at or datetime.utcnow() Entity.delete_by_collection(collection.id, deleted_at=deleted_at) Mapping.delete_by_collection(collection.id, deleted_at=deleted_at) Diagram.delete_by_collection(collection.id, deleted_at=deleted_at) Document.delete_by_collection(collection.id) if not keep_metadata: # Considering this metadata for now, might be wrong: Linkage.delete_by_collection(collection.id) Permission.delete_by_collection(collection.id, deleted_at=deleted_at) collection.delete(deleted_at=deleted_at) db.session.commit() if not keep_metadata: index.delete_collection(collection.id, sync=sync) Authz.flush() refresh_collection(collection.id, sync=True)
def load_role(): role = None credential = request.headers.get('Authorization', '') if len(credential): if ' ' in credential: mechanism, credential = credential.split(' ', 1) role = _get_credential_role(credential) elif 'api_key' in request.args: role = _get_credential_role(request.args.get('api_key')) request.authz = Authz(role=role)
def sitemap(): enable_cache(vary_user=False) collections = [] for collection in Collection.all_authz(Authz.from_role(None)): updated_at = collection.updated_at.date().isoformat() updated_at = max(settings.SITEMAP_FLOOR, updated_at) collections.append({ 'url': collection_url(collection.id), 'updated_at': updated_at }) return render_xml('sitemap.xml', collections=collections)
def sitemap_index(): enable_cache(vary_user=False) collections = [] for collection in Collection.all_authz(Authz.from_role(None)): collections.append({ 'url': url_for('collections_api.sitemap', id=collection.id), 'updated_at': collection.updated_at.date().isoformat() }) return render_xml('sitemap_index.xml', collections=collections)
def delete_collection(collection, keep_metadata=False, sync=False): cancel_queue(collection) aggregator = get_aggregator(collection) aggregator.drop() flush_notifications(collection, sync=sync) index.delete_entities(collection.id, sync=sync) xref_index.delete_xref(collection, sync=sync) deleted_at = collection.deleted_at or datetime.utcnow() Mapping.delete_by_collection(collection.id) EntitySet.delete_by_collection(collection.id, deleted_at) Entity.delete_by_collection(collection.id) Document.delete_by_collection(collection.id) if not keep_metadata: Permission.delete_by_collection(collection.id) collection.delete(deleted_at=deleted_at) db.session.commit() if not keep_metadata: index.delete_collection(collection.id, sync=True) Authz.flush() refresh_collection(collection.id)
def test_user(self): authz = Authz.from_role(self.user) assert authz.logged_in is True, authz assert authz.is_admin is False, authz.is_admin # assert authz.id == self.user.id, authz.id # assert authz.role == self.user, authz.role # assert len(authz.roles) == 4, (authz.roles, self.user.roles) assert authz.can(self.public, authz.READ) is True, authz._collections assert authz.can(self.public, authz.WRITE) is False, authz._collections assert authz.can(self.private, authz.READ) is True, authz._collections assert authz.can(self.private, authz.WRITE) is False, authz._collections
def decode_authz(): authz = None if 'Authorization' in request.headers: credential = request.headers.get('Authorization') authz = _get_credential_authz(credential) if authz is None and 'api_key' in request.args: authz = _get_credential_authz(request.args.get('api_key')) request.authz = authz or Authz.from_role(role=None)
def test_admin(self): authz = Authz.from_role(self.admin) assert authz.logged_in is True, authz assert authz.is_admin is True, authz.is_admin assert authz.id == self.admin.id, authz.id assert len(authz.roles) == 3, authz.roles assert authz.can(self.public, authz.READ) is True, authz._collections assert authz.can(self.public, authz.WRITE) is True, authz._collections assert authz.can(self.private, authz.READ) is True, authz._collections assert authz.can(self.private, authz.WRITE) is True, authz._collections
def update_permission(role, collection, read, write, editor_id=None): """Update a roles permission to access a given collection.""" pre = Permission.by_collection_role(collection, role) post = Permission.grant(collection, role, read, write) params = {'role': role, 'collection': collection} if (pre is None or not pre.read) and post.read: if role.foreign_id == Role.SYSTEM_GUEST: publish(Events.PUBLISH_COLLECTION, actor_id=editor_id, params=params, channels=[Notification.GLOBAL]) else: publish(Events.GRANT_COLLECTION, actor_id=editor_id, params=params, channels=[role]) db.session.commit() Authz.flush() refresh_role(role) return post
def decode_authz(): authz = None if 'Authorization' in request.headers: credential = request.headers.get('Authorization') authz = _get_credential_authz(credential) if authz is None and 'api_key' in request.args: authz = _get_credential_authz(request.args.get('api_key')) authz = authz or Authz.from_role(role=None) request.authz = authz
def enable_authz(request): authz = None if "Authorization" in request.headers: credential = request.headers.get("Authorization") authz = _get_credential_authz(credential) if authz is None and "api_key" in request.args: authz = _get_credential_authz(request.args.get("api_key")) authz = authz or Authz.from_role(role=None) request.authz = authz
def crawldir(path, language=None, foreign_id=None): """Crawl the given directory.""" path = Path(path) if foreign_id is None: foreign_id = 'directory:%s' % slugify(path) authz = Authz.from_role(Role.load_cli_user()) config = {'foreign_id': foreign_id, 'label': path.name, 'casefile': False} create_collection(config, authz) collection = Collection.by_foreign_id(foreign_id) log.info('Crawling %s to %s (%s)...', path, foreign_id, collection.id) crawl_directory(collection, path) log.info('Complete. Make sure a worker is running :)')
def metadata(): """Get operational metadata for the frontend. --- get: summary: Retrieve system metadata from the application. responses: '200': description: OK content: application/json: schema: type: object tags: - System """ locale = get_locale() auth = {} if settings.PASSWORD_LOGIN: auth['password_login_uri'] = url_for('sessions_api.password_login') auth['registration_uri'] = url_for('roles_api.create_code') if settings.OAUTH: auth['oauth_uri'] = url_for('sessions_api.oauth_init') locales = settings.UI_LANGUAGES locales = {l: Locale(l).get_language_name(l) for l in locales} data = { 'status': 'ok', 'maintenance': request.authz.in_maintenance, 'app': { 'title': settings.APP_TITLE, 'description': settings.APP_DESCRIPTION, 'version': __version__, 'banner': settings.APP_BANNER, 'ui_uri': settings.APP_UI_URL, 'samples': settings.SAMPLE_SEARCHES, 'logo': settings.APP_LOGO, 'favicon': settings.APP_FAVICON, 'locale': str(locale), 'locales': locales }, 'categories': Collection.CATEGORIES, 'model': model, 'token': None, 'auth': auth } if settings.SINGLE_USER: role = Role.load_cli_user() authz = Authz.from_role(role) data['token'] = authz.to_token(role=role) return jsonify(data)
def load_role(): role = None if 'Authorization' in request.headers: credential = request.headers.get('Authorization') if ' ' in credential: mechanism, credential = credential.split(' ', 1) data = check_token(credential) if data is not None: role = Role.by_id(data.get('id')) else: role = Role.by_api_key(credential) elif 'api_key' in request.args: role = Role.by_api_key(request.args.get('api_key')) request.authz = Authz(role=role)
def create(): """Create a user role. --- post: summary: Create a user account description: > Create a user role by supplying the required account details. requestBody: content: application/json: schema: $ref: '#/components/schemas/RoleCreate' responses: '200': description: OK content: application/json: schema: $ref: '#/components/schemas/Role' tags: - Role """ require(settings.PASSWORD_LOGIN) require(not request.authz.in_maintenance) data = parse_request("RoleCreate") try: email = Role.SIGNATURE.loads(data.get("code"), max_age=Role.SIGNATURE_MAX_AGE) except BadSignature: return jsonify({ "status": "error", "message": gettext("Invalid code") }, status=400) role = Role.by_email(email) if role is not None: return jsonify( { "status": "error", "message": gettext("Email is already registered") }, status=409, ) role = create_user(email, data.get("name"), data.get("password")) # Let the serializer return more info about this user request.authz = Authz.from_role(role) tag_request(role_id=role.id) return RoleSerializer.jsonify(role, status=201)
def password_login(): """Provides email and password authentication.""" data = parse_request(LoginSchema) role = Role.by_email(data.get('email')) if role is None or not role.has_password: return Unauthorized("Authentication has failed.") if not role.check_password(data.get('password')): return Unauthorized("Authentication has failed.") update_role(role) db.session.commit() authz = Authz.from_role(role) return jsonify({'status': 'ok', 'token': authz.to_token(role=role)})
def get_deep_role(role): authz = Authz.from_role(role) alerts = Alert.by_role_id(role.id).count() exports = Export.by_role_id(role.id).count() casefiles = Collection.all_casefiles(authz=authz).count() entitysets = EntitySet.type_counts(authz=authz) return { "counts": { "alerts": alerts, "entitysets": entitysets, "casefiles": casefiles, "exports": exports, }, "shallow": False, }
def generate_sitemap(collection_id): """Generate entries for a collection-based sitemap.xml file.""" # cf. https://www.sitemaps.org/protocol.html entities = iter_entities(authz=Authz.from_role(None), collection_id=collection_id, schemata=[Entity.THING], includes=['schemata', 'updated_at']) # strictly, the limit for sitemap.xml is 50,000 for entity in islice(entities, 49500): updated_at = entity.get('updated_at', '').split('T', 1)[0] if Document.SCHEMA in entity.get('schemata', []): url = document_url(entity.get('id')) else: url = entity_url(entity.get('id')) yield (url, updated_at)
def update_permission(role, collection, read, write, editor_id=None): """Update a roles permission to access a given collection.""" pre = Permission.by_collection_role(collection, role) post = Permission.grant(collection, role, read, write) params = {'role': role, 'collection': collection} if (pre is None or not pre.read) and post.read: if role.is_public: publish(Events.PUBLISH_COLLECTION, actor_id=editor_id, params=params, channels=[Notification.GLOBAL]) else: publish(Events.GRANT_COLLECTION, actor_id=editor_id, params=params) elif pre is not None and pre.read and not post.read: publish(Events.REVOKE_COLLECTION, actor_id=editor_id, params=params) db.session.commit() Authz.flush() refresh_role(role) return post
def check_alert(alert): authz = Authz(role=alert.role) query = alert_query(alert, authz) found = 0 for result in scan(es, query=query, index=entities_index()): entity = unpack_result(result) found += 1 params = {'alert': alert, 'role': authz.role, 'entity': entity} publish(Events.MATCH_ALERT, actor_id=entity.get('uploader_id'), params=params) alert.update() log.info('Found %d new results for: %s', found, alert.label) db.session.commit()
def export_matches(export_id): """Export the top N matches of cross-referencing for the given collection to an Excel formatted export.""" export = Export.by_id(export_id) export_dir = ensure_path(mkdtemp(prefix="aleph.export.")) try: role = Role.by_id(export.creator_id) authz = Authz.from_role(role) collection = Collection.by_id(export.collection_id) file_name = "%s - Crossreference.xlsx" % collection.label file_path = export_dir.joinpath(file_name) excel = ExcelWriter() headers = [ "Score", "Entity Name", "Entity Date", "Entity Countries", "Candidate Collection", "Candidate Name", "Candidate Date", "Candidate Countries", "Entity Link", "Candidate Link", ] sheet = excel.make_sheet("Cross-reference", headers) batch = [] for match in iter_matches(collection, authz): batch.append(match) if len(batch) >= BULK_PAGE: _iter_match_batch(excel, sheet, batch) batch = [] if len(batch): _iter_match_batch(excel, sheet, batch) with open(file_path, "wb") as fp: buffer = excel.get_bytesio() for data in buffer: fp.write(data) complete_export(export_id, file_path) except Exception: log.exception("Failed to process export [%s]", export_id) export = Export.by_id(export_id) export.set_status(status=Status.FAILED) db.session.commit() finally: shutil.rmtree(export_dir)
def password_login(): """Provides email and password authentication.""" require(settings.PASSWORD_LOGIN) data = parse_request(LoginSchema) role = Role.by_email(data.get('email')) if role is None or not role.has_password: raise BadRequest(gettext("Invalid user or password.")) if not role.check_password(data.get('password')): raise BadRequest(gettext("Invalid user or password.")) db.session.commit() update_role(role) authz = Authz.from_role(role) request.authz = authz return jsonify({'status': 'ok', 'token': authz.to_token(role=role)})
def get_role_channels(role): """Generate the set of notification channels that the current user should listen to.""" key = cache.object_key(Role, role.id, 'channels') channels = cache.get_list(key) if len(channels): return channels channels = [Notification.GLOBAL] if role.deleted_at is None and role.type == Role.USER: authz = Authz.from_role(role) for role_id in authz.roles: channels.append(channel(role_id, Role)) for coll_id in authz.collections(authz.READ): channels.append(channel(coll_id, Collection)) cache.set_list(key, channels, expire=cache.EXPIRE) return channels
def test_scope(self): authz = Authz.from_role(self.admin) token = authz.to_token(scope="/bla") with self.assertRaises(Unauthorized): Authz.from_token(token) with self.assertRaises(Unauthorized): Authz.from_token(token, scope="/blubb") sauthz = Authz.from_token(token, scope="/bla") assert sauthz.id == authz.id assert abs(sauthz.expire - authz.expire) < timedelta(seconds=2) assert sauthz.expire > datetime.utcnow()
def password_login(): """Provides email and password authentication. --- post: summary: Log in as a user description: Create a session token using a username and password. requestBody: content: application/json: schema: $ref: '#/components/schemas/Login' responses: '200': description: OK content: application/json: schema: type: object properties: status: type: string token: type: string tags: - Role """ require(settings.PASSWORD_LOGIN) data = parse_request("Login") role = Role.by_email(data.get("email")) if role is None or not role.has_password: raise BadRequest(gettext("Invalid user or password.")) if not role.check_password(data.get("password")): raise BadRequest(gettext("Invalid user or password.")) if role.is_blocked: raise Unauthorized(gettext("Your account is blocked.")) role.touch() db.session.commit() update_role(role) authz = Authz.from_role(role) request.authz = authz return jsonify({"status": "ok", "token": authz.to_token(role=role)})
def password_login(): """Provides email and password authentication.""" data = parse_request(LoginSchema) role = Role.by_email(data.get('email')) if role is None or not role.has_password: return Unauthorized("Authentication has failed.") if not role.check_password(data.get('password')): return Unauthorized("Authentication has failed.") db.session.commit() update_role(role) authz = Authz.from_role(role) request.authz = authz record_audit(Audit.ACT_LOGIN) return jsonify({ 'status': 'ok', 'token': authz.to_token(role=role) })
def update_collection(collection, sync=False): """Create or update a collection.""" Authz.flush() refresh_collection(collection.id) return index.index_collection(collection, sync=sync)