def oauth_callback():
if not settings.OAUTH:
abort(404)
resp = oauth.provider.authorized_response()
if resp is None or isinstance(resp, OAuthException):
log.warning("Failed OAuth: %r", resp)
return Unauthorized("Authentication has failed.")
response = signals.handle_oauth_session.send(provider=oauth.provider,
oauth=resp)
for (_, role) in response:
if role is None:
continue
db.session.commit()
update_role(role)
log.info("Logged in: %r", role)
request.authz = Authz.from_role(role)
record_audit(Audit.ACT_LOGIN)
token = request.authz.to_token(role=role)
token = token.decode('utf-8')
state = request.args.get('state')
next_url = get_best_next_url(state, request.referrer)
next_url, _ = urldefrag(next_url)
next_url = '%s#token=%s' % (next_url, token)
return redirect(next_url)
log.error("No OAuth handler for %r was installed.", oauth.provider.name)
return Unauthorized("Authentication has failed.")
def test_bulk_entitysets_api(self):
role, headers = self.login(is_admin=True)
authz = Authz.from_role(role)
data = {"type": EntitySet.LIST, "label": "Foo"}
eset = EntitySet.create(data, self.col, authz)
db.session.commit()
eset_id = eset.id
data = json.dumps([
{
"id": "4345800498380953840",
"schema": "Person",
"properties": {
"name": "Osama bin Laden"
},
},
{
"id": "7598743983789743598",
"schema": "Person",
"properties": {
"name": "Osama bin Laden"
},
},
])
url = "/api/2/collections/%s/_bulk?entityset_id=%s" % (self.col.id,
eset_id)
res = self.client.post(url, headers=headers, data=data)
assert res.status_code == 204, res
query = "/api/2/entitysets/%s/entities?filter:schema=Person" % eset_id
res = self.client.get(query, headers=headers)
assert res.json["total"] == 2, res.json
def test_anonymous(self):
authz = Authz.from_role(None)
assert authz.logged_in is False, authz
assert authz.is_admin is False, authz.is_admin
assert authz.id is None, authz.id
assert authz.role is None, authz.role
assert len(authz.roles) == 1, authz.roles
def sample_entities(secret, properties, schematas, seed, sample_pct, limit,
outfile):
"""Sample random entities"""
random.seed(seed)
authz = Authz.from_role(Role.load_cli_user())
collections = list(Collection.all_by_secret(secret, authz))
random.shuffle(collections)
iter_proxies_kwargs = {
"authz": authz,
"schemata": schematas or None,
"randomize": True,
"random_seed": seed,
}
n_entities = 0
for collection in collections:
for entity in iter_proxies(collection_id=collection.id,
**iter_proxies_kwargs):
if properties and not any(
entity.properties.get(prop) for prop in properties):
continue
if not sample_pct or random.random() < sample_pct:
write_object(outfile, entity)
n_entities += 1
if limit and n_entities >= limit:
return
def test_token(self):
authz = Authz.from_role(self.admin)
token = authz.to_token()
with self.assertRaises(Unauthorized):
Authz.from_token("banana")
sauthz = Authz.from_token(token)
assert sauthz.id == authz.id
def sitemap():
"""
---
get:
summary: Get a sitemap
description: >-
Returns a site map for search engine robots. This lists each
published collection on the current instance.
responses:
'200':
description: OK
content:
text/xml:
schema:
type: object
tags:
- System
"""
enable_cache(vary_user=False)
request.rate_limit = None
collections = []
for collection in Collection.all_authz(Authz.from_role(None)):
updated_at = collection.updated_at.date().isoformat()
updated_at = max(settings.SITEMAP_FLOOR, updated_at)
collections.append({
'url': collection_url(collection.id),
'updated_at': updated_at
})
return render_xml('sitemap.xml', collections=collections)
def oauth_callback():
require(settings.OAUTH)
try:
token = oauth.provider.authorize_access_token()
except AuthlibBaseError as err:
log.warning("Failed OAuth: %r", err)
raise Unauthorized(gettext("Authentication has failed."))
if token is None or isinstance(token, AuthlibBaseError):
log.warning("Failed OAuth: %r", token)
raise Unauthorized(gettext("Authentication has failed."))
role = handle_oauth(oauth.provider, token)
if role is None:
log.error("No OAuth handler was installed.")
raise Unauthorized(gettext("Authentication has failed."))
if role.is_blocked:
raise Unauthorized(gettext("Your account is blocked."))
db.session.commit()
update_role(role)
log.info("Logged in: %r", role)
request.authz = Authz.from_role(role)
token = request.authz.to_token(role=role)
token = token.decode('utf-8')
next_path = get_url_path(request.args.get('state'))
next_url = ui_url(settings.OAUTH_UI_CALLBACK, next=next_path)
next_url = '%s#token=%s' % (next_url, token)
return redirect(next_url)
def check_alert(alert_id):
alert = Alert.by_id(alert_id)
if alert is None or alert.role is None:
return
log.info("Check alert [%s]: %s", alert.id, alert.query)
authz = Authz.from_role(alert.role)
query = alert_query(alert, authz)
index = entities_read_index(schema=Entity.THING)
result = es.search(index=index, body=query)
for result in result.get('hits').get('hits', []):
entity = unpack_result(result)
if entity is None:
continue
log.info('Alert [%s]: %s', alert.query, entity.get('name'))
params = {
'alert': alert,
'role': alert.role,
'entity': entity.get('id')
}
publish(Events.MATCH_ALERT, params=params, channels=[alert.role])
db.session.flush()
alert.update()
db.session.commit()
db.session.close()
def check_alert(alert_id):
alert = Alert.by_id(alert_id)
if alert is None or alert.role is None:
return
log.info("Check alert [%s]: %s", alert.id, alert.query)
authz = Authz.from_role(alert.role)
try:
query = alert_query(alert, authz)
index = entities_read_index(schema=Entity.THING)
result = es.search(index=index, body=query)
except RequestError as re:
log.error("Invalid query [%s]: %r", alert.query, re.error)
alert.delete()
db.session.commit()
return
for result in result.get("hits").get("hits", []):
entity = unpack_result(result)
if entity is None:
continue
log.info("Alert [%s]: %s", alert.query, entity.get("id"))
params = {
"alert": alert,
"role": alert.role,
"entity": entity.get("id"),
"collection": entity.get("collection_id"),
}
channels = [alert.role]
# channels.append(channel_tag(collection_id, Collection))
publish(Events.MATCH_ALERT, params=params, channels=channels)
alert.update()
db.session.commit()
def generate_sitemap(collection_id):
"""Generate entries for a collection-based sitemap.xml file."""
# cf. https://www.sitemaps.org/protocol.html
query = {
'query': {
'bool': {
'filter': [
{'term': {'collection_id': collection_id}},
{'term': {'schemata': Entity.THING}},
authz_query(Authz.from_role(None))
]
}
},
'_source': {'includes': ['schemata', 'updated_at']}
}
scanner = scan(es, index=entities_index(), query=query)
# strictly, the limit for sitemap.xml is 50,000
for res in islice(scanner, 49500):
source = res.get('_source', {})
updated_at = source.get('updated_at', '').split('T', 1)[0]
if Document.SCHEMA in source.get('schemata', []):
url = document_url(res.get('_id'))
else:
url = entity_url(res.get('_id'))
yield (url, updated_at)
def oauth_callback():
require(settings.OAUTH)
err = Unauthorized(gettext("Authentication has failed."))
state = cache.get_complex(_oauth_session(request.args.get("state")))
if state is None:
raise err
try:
oauth.provider.framework.set_session_data(request, "state",
state.get("state"))
uri = state.get("redirect_uri")
token = oauth.provider.authorize_access_token(redirect_uri=uri)
except AuthlibBaseError as err:
log.warning("Failed OAuth: %r", err)
raise err
if token is None or isinstance(token, AuthlibBaseError):
log.warning("Failed OAuth: %r", token)
raise err
role = handle_oauth(oauth.provider, token)
if role is None:
raise err
db.session.commit()
update_role(role)
log.debug("Logged in: %r", role)
request.authz = Authz.from_role(role)
next_path = get_url_path(state.get("next_url"))
next_url = ui_url("oauth", next=next_path)
next_url = "%s#token=%s" % (next_url, request.authz.to_token())
session.clear()
return redirect(next_url)
def create_collection(self, creator=None, **kwargs):
authz = Authz.from_role(creator)
collection = Collection.create(kwargs, authz)
db.session.add(collection)
db.session.commit()
update_collection(collection, sync=True)
return collection
def check_alert(alert_id):
alert = Alert.by_id(alert_id)
if alert is None or alert.role is None:
return
if not alert.role.is_alertable:
return
authz = Authz.from_role(alert.role)
query = alert_query(alert, authz)
index = entities_read_index(schema=Entity.THING)
result = es.search(index=index, body=query)
for result in result.get('hits').get('hits', []):
entity = unpack_result(result)
if entity is None:
continue
log.info('Alert [%s]: %s', alert.query, entity.get('name'))
params = {
'alert': alert,
'role': alert.role,
'entity': entity
}
publish(Events.MATCH_ALERT,
actor_id=entity.get('uploader_id'),
params=params)
alert.update()
db.session.commit()
db.session.close()
def compute_collections():
"""Update collection caches, including the global stats cache."""
authz = Authz.from_role(None)
schemata = defaultdict(int)
countries = defaultdict(int)
categories = defaultdict(int)
for collection in Collection.all():
compute_collection(collection)
if authz.can(collection.id, authz.READ):
categories[collection.category] += 1
things = index.get_collection_things(collection.id)
for schema, count in things.items():
schemata[schema] += count
for country in collection.countries:
countries[country] += 1
log.info("Updating global statistics cache...")
data = {
"collections": sum(categories.values()),
"schemata": dict(schemata),
"countries": dict(countries),
"categories": dict(categories),
"things": sum(schemata.values()),
}
key = cache.key(cache.STATISTICS)
cache.set_complex(key, data, expires=cache.EXPIRE)
def test_anon_claim(self):
authz = Authz.from_role(None)
claim_url = archive_url(authz, self.content_hash, file_name="foo")
res = self.client.get(claim_url)
assert res.status_code == 200, res.status_code
disposition = res.headers.get("Content-Disposition")
assert "foo" in disposition, disposition
def create():
require(settings.PASSWORD_LOGIN)
require(not request.authz.in_maintenance)
data = parse_request(RoleCreateSchema)
try:
email = Role.SIGNATURE.loads(data.get('code'),
max_age=Role.SIGNATURE_MAX_AGE)
except BadSignature:
return jsonify({
'status': 'error',
'message': gettext('Invalid code')
},
status=400)
role = Role.by_email(email)
if role is not None:
return jsonify(
{
'status': 'error',
'message': gettext('Email is already registered')
},
status=409)
role = Role.load_or_create(foreign_id='password:{}'.format(email),
type=Role.USER,
name=data.get('name') or email,
email=email)
role.set_password(data.get('password'))
db.session.add(role)
db.session.commit()
update_role(role)
# Let the serializer return more info about this user
request.authz = Authz.from_role(role)
tag_request(role_id=role.id)
return RoleSerializer.jsonify(role, status=201)
def oauth_callback():
require(settings.OAUTH)
resp = oauth.provider.authorized_response()
if resp is None or isinstance(resp, OAuthException):
log.warning("Failed OAuth: %r", resp)
raise Unauthorized(gettext("Authentication has failed."))
response = signals.handle_oauth_session.send(provider=oauth.provider,
oauth=resp)
for (_, role) in response:
if role is None:
continue
db.session.commit()
update_role(role)
log.info("Logged in: %r", role)
request.authz = Authz.from_role(role)
token = request.authz.to_token(role=role)
token = token.decode('utf-8')
next_path = get_url_path(request.args.get('state'))
next_url = ui_url(settings.OAUTH_UI_CALLBACK, next=next_path)
next_url = '%s#token=%s' % (next_url, token)
return redirect(next_url)
log.error("No OAuth handler for %r was installed.", oauth.provider.name)
raise Unauthorized(gettext("Authentication has failed."))
def test_maintenance(self):
settings.MAINTENANCE = True
authz = Authz.from_role(self.admin)
assert authz.logged_in is True, authz
assert authz.is_admin is True, authz.is_admin
assert authz.can(self.public, authz.WRITE) is False, authz._collections
settings.MAINTENANCE = False
def ensure_collection(foreign_id, label):
authz = Authz.from_role(Role.load_cli_user())
config = {
'foreign_id': foreign_id,
'label': label,
}
create_collection(config, authz)
return Collection.by_foreign_id(foreign_id)
def decode_authz():
authz = None
if 'Authorization' in request.headers:
credential = request.headers.get('Authorization')
authz = _get_credential_authz(credential)
if authz is None and 'api_key' in request.args:
authz = _get_credential_authz(request.args.get('api_key'))
request.authz = authz or Authz.from_role(role=None)
def sitemap():
enable_cache(vary_user=False)
collections = []
for collection in Collection.all_authz(Authz.from_role(None)):
updated_at = collection.updated_at.date().isoformat()
updated_at = max(settings.SITEMAP_FLOOR, updated_at)
collections.append({
'url': collection_url(collection.id),
'updated_at': updated_at
})
return render_xml('sitemap.xml', collections=collections)
def sitemap():
enable_cache(vary_user=False)
collections = []
for collection in Collection.all_authz(Authz.from_role(None)):
updated_at = collection.updated_at.date().isoformat()
updated_at = max(settings.SITEMAP_FLOOR, updated_at)
collections.append({
'url': collection_url(collection.id),
'updated_at': updated_at
})
return render_xml('sitemap.xml', collections=collections)
def _get_credential_authz(credential):
if credential is None or not len(credential):
return
if " " in credential:
method, credential = credential.split(" ", 1)
if method == "Token":
return Authz.from_token(credential)
role = Role.by_api_key(credential)
if role is not None:
return Authz.from_role(role=role)
def test_admin(self):
authz = Authz.from_role(self.admin)
assert authz.logged_in is True, authz
assert authz.is_admin is True, authz.is_admin
assert authz.id == self.admin.id, authz.id
assert len(authz.roles) == 3, authz.roles
assert authz.can(self.public, authz.READ) is True, authz._collections
assert authz.can(self.public, authz.WRITE) is True, authz._collections
assert authz.can(self.private, authz.READ) is True, authz._collections
assert authz.can(self.private, authz.WRITE) is True, authz._collections
def sitemap_index():
enable_cache(vary_user=False)
collections = []
for collection in Collection.all_authz(Authz.from_role(None)):
collections.append({
'url':
url_for('collections_api.sitemap', id=collection.id),
'updated_at':
collection.updated_at.date().isoformat()
})
return render_xml('sitemap_index.xml', collections=collections)
def test_user(self):
authz = Authz.from_role(self.user)
assert authz.logged_in is True, authz
assert authz.is_admin is False, authz.is_admin
# assert authz.id == self.user.id, authz.id
# assert authz.role == self.user, authz.role
# assert len(authz.roles) == 4, (authz.roles, self.user.roles)
assert authz.can(self.public, authz.READ) is True, authz._collections
assert authz.can(self.public, authz.WRITE) is False, authz._collections
assert authz.can(self.private, authz.READ) is True, authz._collections
assert authz.can(self.private, authz.WRITE) is False, authz._collections
def test_scope(self):
authz = Authz.from_role(self.admin)
token = authz.to_token(scope="/bla")
with self.assertRaises(Unauthorized):
Authz.from_token(token)
with self.assertRaises(Unauthorized):
Authz.from_token(token, scope="/blubb")
sauthz = Authz.from_token(token, scope="/bla")
assert sauthz.id == authz.id
assert abs(sauthz.expire - authz.expire) < timedelta(seconds=2)
assert sauthz.expire > datetime.utcnow()
def crawldir(path, language=None, foreign_id=None):
"""Crawl the given directory."""
path = Path(path)
if foreign_id is None:
foreign_id = 'directory:%s' % slugify(path)
authz = Authz.from_role(Role.load_cli_user())
config = {'foreign_id': foreign_id, 'label': path.name, 'casefile': False}
create_collection(config, authz)
collection = Collection.by_foreign_id(foreign_id)
log.info('Crawling %s to %s (%s)...', path, foreign_id, collection.id)
crawl_directory(collection, path)
log.info('Complete. Make sure a worker is running :)')
def decode_authz():
authz = None
if 'Authorization' in request.headers:
credential = request.headers.get('Authorization')
authz = _get_credential_authz(credential)
if authz is None and 'api_key' in request.args:
authz = _get_credential_authz(request.args.get('api_key'))
authz = authz or Authz.from_role(role=None)
request.authz = authz
def _get_credential_authz(credential):
if credential is None or not len(credential):
return
if ' ' in credential:
mechanism, credential = credential.split(' ', 1)
authz = Authz.from_token(credential, scope=request.path)
if authz is not None:
return authz
role = Role.by_api_key(credential)
if role is not None:
return Authz.from_role(role=role)
def enable_authz(request):
authz = None
if "Authorization" in request.headers:
credential = request.headers.get("Authorization")
authz = _get_credential_authz(credential)
if authz is None and "api_key" in request.args:
authz = _get_credential_authz(request.args.get("api_key"))
authz = authz or Authz.from_role(role=None)
request.authz = authz
def metadata():
"""Get operational metadata for the frontend.
---
get:
summary: Retrieve system metadata from the application.
responses:
'200':
description: OK
content:
application/json:
schema:
type: object
tags:
- System
"""
locale = get_locale()
auth = {}
if settings.PASSWORD_LOGIN:
auth['password_login_uri'] = url_for('sessions_api.password_login')
auth['registration_uri'] = url_for('roles_api.create_code')
if settings.OAUTH:
auth['oauth_uri'] = url_for('sessions_api.oauth_init')
locales = settings.UI_LANGUAGES
locales = {l: Locale(l).get_language_name(l) for l in locales}
data = {
'status': 'ok',
'maintenance': request.authz.in_maintenance,
'app': {
'title': settings.APP_TITLE,
'description': settings.APP_DESCRIPTION,
'version': __version__,
'banner': settings.APP_BANNER,
'ui_uri': settings.APP_UI_URL,
'samples': settings.SAMPLE_SEARCHES,
'logo': settings.APP_LOGO,
'favicon': settings.APP_FAVICON,
'locale': str(locale),
'locales': locales
},
'categories': Collection.CATEGORIES,
'model': model,
'token': None,
'auth': auth
}
if settings.SINGLE_USER:
role = Role.load_cli_user()
authz = Authz.from_role(role)
data['token'] = authz.to_token(role=role)
return jsonify(data)
def password_login():
"""Provides email and password authentication."""
data = parse_request(LoginSchema)
role = Role.by_email(data.get('email'))
if role is None or not role.has_password:
return Unauthorized("Authentication has failed.")
if not role.check_password(data.get('password')):
return Unauthorized("Authentication has failed.")
db.session.commit()
update_role(role)
authz = Authz.from_role(role)
request.authz = authz
record_audit(Audit.ACT_LOGIN)
return jsonify({
'status': 'ok',
'token': authz.to_token(role=role)
})