def delete_collection(collection, sync=False):
flush_notifications(collection)
collection.delete()
db.session.commit()
index.delete_collection(collection.id, sync=sync)
delete_collection_content.apply_async([collection.id], priority=7)
refresh_collection(collection.id)
Authz.flush()
def create_collection(data, role=None, sync=False):
role = role or Role.load_cli_user()
created_at = datetime.utcnow()
collection = Collection.create(data, role=role, created_at=created_at)
if collection.created_at == created_at:
publish(Events.CREATE_COLLECTION,
actor_id=role.id,
params={'collection': collection})
db.session.commit()
Authz.flush()
refresh_collection(collection.id)
return index.index_collection(collection, sync=sync)
def _get_credential_authz(credential):
if credential is None or not len(credential):
return
if ' ' in credential:
mechanism, credential = credential.split(' ', 1)
authz = Authz.from_token(credential, scope=request.path)
if authz is not None:
return authz
role = Role.by_api_key(credential)
if role is not None:
return Authz.from_role(role=role)
def test_maintenance(self):
settings.MAINTENANCE = True
authz = Authz.from_role(self.admin)
assert authz.logged_in is True, authz
assert authz.is_admin is True, authz.is_admin
assert authz.can(self.public, authz.WRITE) is False, authz._collections
settings.MAINTENANCE = False
def oauth_callback():
if not settings.OAUTH:
abort(404)
resp = oauth.provider.authorized_response()
if resp is None or isinstance(resp, OAuthException):
log.warning("Failed OAuth: %r", resp)
return Unauthorized("Authentication has failed.")
response = signals.handle_oauth_session.send(provider=oauth.provider,
oauth=resp)
for (_, role) in response:
if role is None:
continue
db.session.commit()
update_role(role)
log.info("Logged in: %r", role)
request.authz = Authz.from_role(role)
record_audit(Audit.ACT_LOGIN)
token = request.authz.to_token(role=role)
token = token.decode('utf-8')
state = request.args.get('state')
next_url = get_best_next_url(state, request.referrer)
next_url, _ = urldefrag(next_url)
next_url = '%s#token=%s' % (next_url, token)
return redirect(next_url)
log.error("No OAuth handler for %r was installed.", oauth.provider.name)
return Unauthorized("Authentication has failed.")
def test_anonymous(self):
authz = Authz.from_role(None)
assert authz.logged_in is False, authz
assert authz.is_admin is False, authz.is_admin
assert authz.id is None, authz.id
assert authz.role is None, authz.role
assert len(authz.roles) == 1, authz.roles
def check_alert(alert_id):
alert = Alert.by_id(alert_id)
if alert is None or alert.role is None:
return
log.info("Check alert [%s]: %s", alert.id, alert.query)
authz = Authz.from_role(alert.role)
try:
query = alert_query(alert, authz)
index = entities_read_index(schema=Entity.THING)
result = es.search(index=index, body=query)
except RequestError as re:
log.error("Invalid query [%s]: %r", alert.query, re.error)
alert.delete()
db.session.commit()
return
for result in result.get('hits').get('hits', []):
entity = unpack_result(result)
if entity is None:
continue
log.info('Alert [%s]: %s', alert.query, entity.get('id'))
params = {
'alert': alert,
'role': alert.role,
'entity': entity.get('id'),
'collection': entity.get('collection_id')
}
channels = [alert.role]
# channels.append(channel_tag(collection_id, Collection))
publish(Events.MATCH_ALERT, params=params, channels=channels)
alert.update()
db.session.commit()
def oauth_callback():
if not settings.OAUTH:
abort(404)
resp = oauth.provider.authorized_response()
if resp is None or isinstance(resp, OAuthException):
log.warning("Failed OAuth: %r", resp)
return Unauthorized("Authentication has failed.")
response = signals.handle_oauth_session.send(provider=oauth.provider,
oauth=resp)
for (_, role) in response:
if role is None:
continue
update_role(role)
db.session.commit()
log.info("Logged in: %r", role)
authz = Authz.from_role(role)
token = authz.to_token(role=role)
token = token.decode('utf-8')
state = request.args.get('state')
next_url = get_best_next_url(state, request.referrer)
next_url, _ = urldefrag(next_url)
next_url = '%s#token=%s' % (next_url, token)
return redirect(next_url)
log.error("No OAuth handler for %r was installed.", oauth.provider.name)
return Unauthorized("Authentication has failed.")
def oauth_callback():
require(settings.OAUTH)
token = oauth.provider.authorize_access_token()
if token is None or isinstance(token, AuthlibBaseError):
log.warning("Failed OAuth: %r", token)
raise Unauthorized(gettext("Authentication has failed."))
response = signals.handle_oauth_session.send(provider=oauth.provider,
oauth_token=token)
for (_, role) in response:
if role is None:
continue
db.session.commit()
update_role(role)
log.info("Logged in: %r", role)
request.authz = Authz.from_role(role)
token = request.authz.to_token(role=role)
token = token.decode('utf-8')
next_path = get_url_path(request.args.get('state'))
next_url = ui_url(settings.OAUTH_UI_CALLBACK, next=next_path)
next_url = '%s#token=%s' % (next_url, token)
return redirect(next_url)
log.error("No OAuth handler for %r was installed.", oauth.provider.name)
raise Unauthorized(gettext("Authentication has failed."))
def ensure_collection(foreign_id, label):
authz = Authz.from_role(Role.load_cli_user())
config = {
'foreign_id': foreign_id,
'label': label,
}
create_collection(config, authz)
return Collection.by_foreign_id(foreign_id)
def update_permission(role, collection, read, write, editor_id=None):
"""Update a roles permission to access a given collection."""
pre = Permission.by_collection_role(collection, role)
post = Permission.grant(collection, role, read, write)
params = {'role': role, 'collection': collection}
if (pre is None or not pre.read) and post.read:
if role.is_public:
publish(Events.PUBLISH_COLLECTION,
actor_id=editor_id,
params=params,
channels=[Notification.GLOBAL])
else:
publish(Events.GRANT_COLLECTION, actor_id=editor_id, params=params)
elif pre is not None and pre.read and not post.read:
publish(Events.REVOKE_COLLECTION, actor_id=editor_id, params=params)
db.session.commit()
Authz.flush()
return post
def delete_collection(collection, keep_metadata=False,
sync=False, reset_sync=False):
reset_collection(collection, sync=reset_sync)
deleted_at = collection.deleted_at or datetime.utcnow()
Entity.delete_by_collection(collection.id, deleted_at=deleted_at)
Mapping.delete_by_collection(collection.id, deleted_at=deleted_at)
Diagram.delete_by_collection(collection.id, deleted_at=deleted_at)
Document.delete_by_collection(collection.id)
if not keep_metadata:
# Considering this metadata for now, might be wrong:
Linkage.delete_by_collection(collection.id)
Permission.delete_by_collection(collection.id, deleted_at=deleted_at)
collection.delete(deleted_at=deleted_at)
db.session.commit()
if not keep_metadata:
index.delete_collection(collection.id, sync=sync)
Authz.flush()
refresh_collection(collection.id, sync=True)
def load_role():
role = None
credential = request.headers.get('Authorization', '')
if len(credential):
if ' ' in credential:
mechanism, credential = credential.split(' ', 1)
role = _get_credential_role(credential)
elif 'api_key' in request.args:
role = _get_credential_role(request.args.get('api_key'))
request.authz = Authz(role=role)
def sitemap():
enable_cache(vary_user=False)
collections = []
for collection in Collection.all_authz(Authz.from_role(None)):
updated_at = collection.updated_at.date().isoformat()
updated_at = max(settings.SITEMAP_FLOOR, updated_at)
collections.append({
'url': collection_url(collection.id),
'updated_at': updated_at
})
return render_xml('sitemap.xml', collections=collections)
def sitemap():
enable_cache(vary_user=False)
collections = []
for collection in Collection.all_authz(Authz.from_role(None)):
updated_at = collection.updated_at.date().isoformat()
updated_at = max(settings.SITEMAP_FLOOR, updated_at)
collections.append({
'url': collection_url(collection.id),
'updated_at': updated_at
})
return render_xml('sitemap.xml', collections=collections)
def sitemap_index():
enable_cache(vary_user=False)
collections = []
for collection in Collection.all_authz(Authz.from_role(None)):
collections.append({
'url':
url_for('collections_api.sitemap', id=collection.id),
'updated_at':
collection.updated_at.date().isoformat()
})
return render_xml('sitemap_index.xml', collections=collections)
def delete_collection(collection, keep_metadata=False, sync=False):
cancel_queue(collection)
aggregator = get_aggregator(collection)
aggregator.drop()
flush_notifications(collection, sync=sync)
index.delete_entities(collection.id, sync=sync)
xref_index.delete_xref(collection, sync=sync)
deleted_at = collection.deleted_at or datetime.utcnow()
Mapping.delete_by_collection(collection.id)
EntitySet.delete_by_collection(collection.id, deleted_at)
Entity.delete_by_collection(collection.id)
Document.delete_by_collection(collection.id)
if not keep_metadata:
Permission.delete_by_collection(collection.id)
collection.delete(deleted_at=deleted_at)
db.session.commit()
if not keep_metadata:
index.delete_collection(collection.id, sync=True)
Authz.flush()
refresh_collection(collection.id)
def test_user(self):
authz = Authz.from_role(self.user)
assert authz.logged_in is True, authz
assert authz.is_admin is False, authz.is_admin
# assert authz.id == self.user.id, authz.id
# assert authz.role == self.user, authz.role
# assert len(authz.roles) == 4, (authz.roles, self.user.roles)
assert authz.can(self.public, authz.READ) is True, authz._collections
assert authz.can(self.public, authz.WRITE) is False, authz._collections
assert authz.can(self.private, authz.READ) is True, authz._collections
assert authz.can(self.private, authz.WRITE) is False, authz._collections
def decode_authz():
authz = None
if 'Authorization' in request.headers:
credential = request.headers.get('Authorization')
authz = _get_credential_authz(credential)
if authz is None and 'api_key' in request.args:
authz = _get_credential_authz(request.args.get('api_key'))
request.authz = authz or Authz.from_role(role=None)
def test_admin(self):
authz = Authz.from_role(self.admin)
assert authz.logged_in is True, authz
assert authz.is_admin is True, authz.is_admin
assert authz.id == self.admin.id, authz.id
assert len(authz.roles) == 3, authz.roles
assert authz.can(self.public, authz.READ) is True, authz._collections
assert authz.can(self.public, authz.WRITE) is True, authz._collections
assert authz.can(self.private, authz.READ) is True, authz._collections
assert authz.can(self.private, authz.WRITE) is True, authz._collections
def update_permission(role, collection, read, write, editor_id=None):
"""Update a roles permission to access a given collection."""
pre = Permission.by_collection_role(collection, role)
post = Permission.grant(collection, role, read, write)
params = {'role': role, 'collection': collection}
if (pre is None or not pre.read) and post.read:
if role.foreign_id == Role.SYSTEM_GUEST:
publish(Events.PUBLISH_COLLECTION,
actor_id=editor_id,
params=params,
channels=[Notification.GLOBAL])
else:
publish(Events.GRANT_COLLECTION,
actor_id=editor_id,
params=params,
channels=[role])
db.session.commit()
Authz.flush()
refresh_role(role)
return post
def decode_authz():
authz = None
if 'Authorization' in request.headers:
credential = request.headers.get('Authorization')
authz = _get_credential_authz(credential)
if authz is None and 'api_key' in request.args:
authz = _get_credential_authz(request.args.get('api_key'))
authz = authz or Authz.from_role(role=None)
request.authz = authz
def enable_authz(request):
authz = None
if "Authorization" in request.headers:
credential = request.headers.get("Authorization")
authz = _get_credential_authz(credential)
if authz is None and "api_key" in request.args:
authz = _get_credential_authz(request.args.get("api_key"))
authz = authz or Authz.from_role(role=None)
request.authz = authz
def crawldir(path, language=None, foreign_id=None):
"""Crawl the given directory."""
path = Path(path)
if foreign_id is None:
foreign_id = 'directory:%s' % slugify(path)
authz = Authz.from_role(Role.load_cli_user())
config = {'foreign_id': foreign_id, 'label': path.name, 'casefile': False}
create_collection(config, authz)
collection = Collection.by_foreign_id(foreign_id)
log.info('Crawling %s to %s (%s)...', path, foreign_id, collection.id)
crawl_directory(collection, path)
log.info('Complete. Make sure a worker is running :)')
def metadata():
"""Get operational metadata for the frontend.
---
get:
summary: Retrieve system metadata from the application.
responses:
'200':
description: OK
content:
application/json:
schema:
type: object
tags:
- System
"""
locale = get_locale()
auth = {}
if settings.PASSWORD_LOGIN:
auth['password_login_uri'] = url_for('sessions_api.password_login')
auth['registration_uri'] = url_for('roles_api.create_code')
if settings.OAUTH:
auth['oauth_uri'] = url_for('sessions_api.oauth_init')
locales = settings.UI_LANGUAGES
locales = {l: Locale(l).get_language_name(l) for l in locales}
data = {
'status': 'ok',
'maintenance': request.authz.in_maintenance,
'app': {
'title': settings.APP_TITLE,
'description': settings.APP_DESCRIPTION,
'version': __version__,
'banner': settings.APP_BANNER,
'ui_uri': settings.APP_UI_URL,
'samples': settings.SAMPLE_SEARCHES,
'logo': settings.APP_LOGO,
'favicon': settings.APP_FAVICON,
'locale': str(locale),
'locales': locales
},
'categories': Collection.CATEGORIES,
'model': model,
'token': None,
'auth': auth
}
if settings.SINGLE_USER:
role = Role.load_cli_user()
authz = Authz.from_role(role)
data['token'] = authz.to_token(role=role)
return jsonify(data)
def load_role():
role = None
if 'Authorization' in request.headers:
credential = request.headers.get('Authorization')
if ' ' in credential:
mechanism, credential = credential.split(' ', 1)
data = check_token(credential)
if data is not None:
role = Role.by_id(data.get('id'))
else:
role = Role.by_api_key(credential)
elif 'api_key' in request.args:
role = Role.by_api_key(request.args.get('api_key'))
request.authz = Authz(role=role)
def create():
"""Create a user role.
---
post:
summary: Create a user account
description: >
Create a user role by supplying the required account details.
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/RoleCreate'
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/Role'
tags:
- Role
"""
require(settings.PASSWORD_LOGIN)
require(not request.authz.in_maintenance)
data = parse_request("RoleCreate")
try:
email = Role.SIGNATURE.loads(data.get("code"),
max_age=Role.SIGNATURE_MAX_AGE)
except BadSignature:
return jsonify({
"status": "error",
"message": gettext("Invalid code")
},
status=400)
role = Role.by_email(email)
if role is not None:
return jsonify(
{
"status": "error",
"message": gettext("Email is already registered")
},
status=409,
)
role = create_user(email, data.get("name"), data.get("password"))
# Let the serializer return more info about this user
request.authz = Authz.from_role(role)
tag_request(role_id=role.id)
return RoleSerializer.jsonify(role, status=201)
def password_login():
"""Provides email and password authentication."""
data = parse_request(LoginSchema)
role = Role.by_email(data.get('email'))
if role is None or not role.has_password:
return Unauthorized("Authentication has failed.")
if not role.check_password(data.get('password')):
return Unauthorized("Authentication has failed.")
update_role(role)
db.session.commit()
authz = Authz.from_role(role)
return jsonify({'status': 'ok', 'token': authz.to_token(role=role)})
def get_deep_role(role):
authz = Authz.from_role(role)
alerts = Alert.by_role_id(role.id).count()
exports = Export.by_role_id(role.id).count()
casefiles = Collection.all_casefiles(authz=authz).count()
entitysets = EntitySet.type_counts(authz=authz)
return {
"counts": {
"alerts": alerts,
"entitysets": entitysets,
"casefiles": casefiles,
"exports": exports,
},
"shallow": False,
}
def generate_sitemap(collection_id):
"""Generate entries for a collection-based sitemap.xml file."""
# cf. https://www.sitemaps.org/protocol.html
entities = iter_entities(authz=Authz.from_role(None),
collection_id=collection_id,
schemata=[Entity.THING],
includes=['schemata', 'updated_at'])
# strictly, the limit for sitemap.xml is 50,000
for entity in islice(entities, 49500):
updated_at = entity.get('updated_at', '').split('T', 1)[0]
if Document.SCHEMA in entity.get('schemata', []):
url = document_url(entity.get('id'))
else:
url = entity_url(entity.get('id'))
yield (url, updated_at)
def update_permission(role, collection, read, write, editor_id=None):
"""Update a roles permission to access a given collection."""
pre = Permission.by_collection_role(collection, role)
post = Permission.grant(collection, role, read, write)
params = {'role': role, 'collection': collection}
if (pre is None or not pre.read) and post.read:
if role.is_public:
publish(Events.PUBLISH_COLLECTION,
actor_id=editor_id,
params=params,
channels=[Notification.GLOBAL])
else:
publish(Events.GRANT_COLLECTION,
actor_id=editor_id,
params=params)
elif pre is not None and pre.read and not post.read:
publish(Events.REVOKE_COLLECTION,
actor_id=editor_id,
params=params)
db.session.commit()
Authz.flush()
refresh_role(role)
return post
def check_alert(alert):
authz = Authz(role=alert.role)
query = alert_query(alert, authz)
found = 0
for result in scan(es, query=query, index=entities_index()):
entity = unpack_result(result)
found += 1
params = {'alert': alert, 'role': authz.role, 'entity': entity}
publish(Events.MATCH_ALERT,
actor_id=entity.get('uploader_id'),
params=params)
alert.update()
log.info('Found %d new results for: %s', found, alert.label)
db.session.commit()
def export_matches(export_id):
"""Export the top N matches of cross-referencing for the given collection
to an Excel formatted export."""
export = Export.by_id(export_id)
export_dir = ensure_path(mkdtemp(prefix="aleph.export."))
try:
role = Role.by_id(export.creator_id)
authz = Authz.from_role(role)
collection = Collection.by_id(export.collection_id)
file_name = "%s - Crossreference.xlsx" % collection.label
file_path = export_dir.joinpath(file_name)
excel = ExcelWriter()
headers = [
"Score",
"Entity Name",
"Entity Date",
"Entity Countries",
"Candidate Collection",
"Candidate Name",
"Candidate Date",
"Candidate Countries",
"Entity Link",
"Candidate Link",
]
sheet = excel.make_sheet("Cross-reference", headers)
batch = []
for match in iter_matches(collection, authz):
batch.append(match)
if len(batch) >= BULK_PAGE:
_iter_match_batch(excel, sheet, batch)
batch = []
if len(batch):
_iter_match_batch(excel, sheet, batch)
with open(file_path, "wb") as fp:
buffer = excel.get_bytesio()
for data in buffer:
fp.write(data)
complete_export(export_id, file_path)
except Exception:
log.exception("Failed to process export [%s]", export_id)
export = Export.by_id(export_id)
export.set_status(status=Status.FAILED)
db.session.commit()
finally:
shutil.rmtree(export_dir)
def password_login():
"""Provides email and password authentication."""
require(settings.PASSWORD_LOGIN)
data = parse_request(LoginSchema)
role = Role.by_email(data.get('email'))
if role is None or not role.has_password:
raise BadRequest(gettext("Invalid user or password."))
if not role.check_password(data.get('password')):
raise BadRequest(gettext("Invalid user or password."))
db.session.commit()
update_role(role)
authz = Authz.from_role(role)
request.authz = authz
return jsonify({'status': 'ok', 'token': authz.to_token(role=role)})
def get_role_channels(role):
"""Generate the set of notification channels that the current
user should listen to."""
key = cache.object_key(Role, role.id, 'channels')
channels = cache.get_list(key)
if len(channels):
return channels
channels = [Notification.GLOBAL]
if role.deleted_at is None and role.type == Role.USER:
authz = Authz.from_role(role)
for role_id in authz.roles:
channels.append(channel(role_id, Role))
for coll_id in authz.collections(authz.READ):
channels.append(channel(coll_id, Collection))
cache.set_list(key, channels, expire=cache.EXPIRE)
return channels
def test_scope(self):
authz = Authz.from_role(self.admin)
token = authz.to_token(scope="/bla")
with self.assertRaises(Unauthorized):
Authz.from_token(token)
with self.assertRaises(Unauthorized):
Authz.from_token(token, scope="/blubb")
sauthz = Authz.from_token(token, scope="/bla")
assert sauthz.id == authz.id
assert abs(sauthz.expire - authz.expire) < timedelta(seconds=2)
assert sauthz.expire > datetime.utcnow()
def password_login():
"""Provides email and password authentication.
---
post:
summary: Log in as a user
description: Create a session token using a username and password.
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/Login'
responses:
'200':
description: OK
content:
application/json:
schema:
type: object
properties:
status:
type: string
token:
type: string
tags:
- Role
"""
require(settings.PASSWORD_LOGIN)
data = parse_request("Login")
role = Role.by_email(data.get("email"))
if role is None or not role.has_password:
raise BadRequest(gettext("Invalid user or password."))
if not role.check_password(data.get("password")):
raise BadRequest(gettext("Invalid user or password."))
if role.is_blocked:
raise Unauthorized(gettext("Your account is blocked."))
role.touch()
db.session.commit()
update_role(role)
authz = Authz.from_role(role)
request.authz = authz
return jsonify({"status": "ok", "token": authz.to_token(role=role)})
def password_login():
"""Provides email and password authentication."""
data = parse_request(LoginSchema)
role = Role.by_email(data.get('email'))
if role is None or not role.has_password:
return Unauthorized("Authentication has failed.")
if not role.check_password(data.get('password')):
return Unauthorized("Authentication has failed.")
db.session.commit()
update_role(role)
authz = Authz.from_role(role)
request.authz = authz
record_audit(Audit.ACT_LOGIN)
return jsonify({
'status': 'ok',
'token': authz.to_token(role=role)
})
def update_collection(collection, sync=False):
"""Create or update a collection."""
Authz.flush()
refresh_collection(collection.id)
return index.index_collection(collection, sync=sync)
