def create():
require(not request.authz.in_maintenance, settings.PASSWORD_LOGIN)
data = parse_request(RoleCreateSchema)
try:
email = Role.SIGNATURE.loads(data.get('code'),
max_age=Role.SIGNATURE_MAX_AGE)
except BadSignature:
return jsonify({
'status': 'error',
'message': gettext('Invalid code')
}, status=400)
role = Role.by_email(email)
if role is not None:
return jsonify({
'status': 'error',
'message': gettext('Email is already registered')
}, status=409)
role = Role.load_or_create(
foreign_id='password:{}'.format(email),
type=Role.USER,
name=data.get('name') or email,
email=email
)
role.set_password(data.get('password'))
db.session.add(role)
db.session.commit()
update_role(role)
# Let the serializer return more info about this user
request.authz.id = role.id
tag_request(role_id=role.id)
return RoleSerializer.jsonify(role, status=201)
def reconcile(collection_id=None):
"""Reconciliation API, emulates Google Refine API."""
collection = None
if collection_id is not None:
collection = get_index_collection(collection_id)
query = request.values.get('query')
if query is not None:
# single
try:
query = json.loads(query)
except ValueError:
query = {'query': query}
return jsonify(reconcile_op(query, collection))
queries = request.values.get('queries')
if queries is not None:
# multiple requests in one query
try:
qs = json.loads(queries)
results = {}
for k, q in qs.items():
results[k] = reconcile_op(q, collection)
return jsonify(results)
except ValueError:
raise BadRequest()
return reconcile_index(collection)
def index(id):
collection = get_db_collection(id, request.authz.WRITE)
record_audit(Audit.ACT_COLLECTION, id=id)
roles = [r for r in Role.all_groups() if check_visible(r, request.authz)]
q = Permission.all()
q = q.filter(Permission.collection_id == collection.id)
permissions = []
for permission in q.all():
if not check_visible(permission.role, request.authz):
continue
permissions.append(permission)
if permission.role in roles:
roles.remove(permission.role)
# this workaround ensures that all groups are visible for the user to
# select in the UI even if they are not currently associated with the
# collection.
for role in roles:
if collection.casefile and role.is_public:
continue
permissions.append({
'collection_id': collection.id,
'write': False,
'read': False,
'role_id': str(role.id)
})
permissions = PermissionSerializer().serialize_many(permissions)
return jsonify({
'total': len(permissions),
'results': permissions
})
def handle_es_error(err):
log.error("ES [%s]: %r", err.error, err.info)
return jsonify({
'status': 'error',
'message': gettext('There was an error during search'),
'context': err.error
}, status=500)
def suggest_property():
prefix = request.args.get('prefix', '').lower().strip()
tag_request(prefix=prefix)
schema = request.args.get('schema', Entity.THING)
matches = []
for prop in model.get(schema).properties.values():
match = not len(prefix)
match = prefix in prop.name.lower()
match = match or prefix in prop.label.lower()
if match:
matches.append({
'id': prop.name,
'quid': prop.name,
'name': prop.label,
'r:score': 100,
'n:type': {
'id': '/properties/property',
'name': 'Property'
}
})
return jsonify({
"code": "/api/status/ok",
"status": "200 OK",
"prefix": request.args.get('prefix', ''),
"result": matches
})
def create_code():
data = parse_request(RoleCodeCreateSchema)
challenge_role(data)
return jsonify({
'status': 'ok',
'message': gettext('To proceed, please check your email.')
})
def ingest_upload(collection_id):
require(request.authz.can(collection_id, request.authz.WRITE))
sync = get_flag('sync')
meta, foreign_id = _load_metadata()
parent_id = _load_parent(collection_id, meta)
upload_dir = mkdtemp(prefix='aleph.upload.')
try:
path = None
content_hash = None
for storage in request.files.values():
path = safe_filename(storage.filename, default='upload')
path = os.path.join(upload_dir, path)
storage.save(path)
content_hash = checksum(path)
document = Document.by_keys(collection_id=collection_id,
parent_id=parent_id,
foreign_id=foreign_id,
content_hash=content_hash)
document.update(meta)
document.schema = Document.SCHEMA
if content_hash is None:
document.schema = Document.SCHEMA_FOLDER
ingest_document(document, path,
role_id=request.authz.id,
content_hash=content_hash)
finally:
shutil.rmtree(upload_dir)
if document.collection.casefile:
# Make sure collection counts are always accurate.
update_document(document, sync=sync)
return jsonify({
'status': 'ok',
'id': stringify(document.id)
}, status=201)
def generate(collection_id):
data = parse_request(XrefSchema)
collection = get_db_collection(collection_id, request.authz.WRITE)
args = {
"against_collection_ids": data.get("against_collection_ids")
}
xref_collection.apply_async([collection.id], kwargs=args, priority=5)
return jsonify({'status': 'accepted'}, status=202)
def _load_parent(collection_id, meta):
"""Determine the parent document for the document that is to be
ingested."""
parent_id = meta.get('parent_id')
if parent_id is None:
return
parent = Document.by_id(parent_id, collection_id=collection_id)
if parent is None:
raise BadRequest(response=jsonify({
'status': 'error',
'message': 'Cannot load parent document'
}, status=400))
return parent_id
def metadata():
locale = get_locale()
enable_cache(vary_user=False, vary=str(locale))
key = cache.key('metadata', locale)
data = cache.get_complex(key)
if data is not None:
return jsonify(data)
auth = {}
if settings.PASSWORD_LOGIN:
auth['password_login_uri'] = url_for('sessions_api.password_login')
auth['registration_uri'] = url_for('roles_api.create_code')
if settings.OAUTH:
auth['oauth_uri'] = url_for('sessions_api.oauth_init')
data = {
'status': 'ok',
'maintenance': request.authz.in_maintenance,
'app': {
'title': settings.APP_TITLE,
'description': settings.APP_DESCRIPTION,
'version': __version__,
'banner': settings.APP_BANNER,
'ui_uri': settings.APP_UI_URL,
'samples': settings.SAMPLE_SEARCHES,
'logo': settings.APP_LOGO,
'favicon': settings.APP_FAVICON,
'locale': str(locale),
'locales': settings.UI_LANGUAGES
},
'categories': Collection.CATEGORIES,
'countries': registry.country.names,
'languages': registry.language.names,
'model': model,
'auth': auth
}
cache.set_complex(key, data, expire=120)
return jsonify(data)
def openapi():
"""Generate an OpenAPI 3.0 documentation JSON file for the API."""
enable_cache(vary_user=False)
spec = get_openapi_spec(current_app)
for name, view in current_app.view_functions.items():
if name in (
"static",
"base_api.openapi",
"base_api.api_v1_message",
"sessions_api.oauth_callback",
):
continue
log.info("%s - %s", name, view.__qualname__)
spec.path(view=view)
return jsonify(spec.to_dict())
def _load_metadata():
"""Unpack the common, pre-defined metadata for all the uploaded files."""
try:
meta = json.loads(request.form.get('meta', '{}'))
except Exception as ex:
raise BadRequest(str(ex))
validate_data(meta, DocumentCreateSchema)
foreign_id = stringify(meta.get('foreign_id'))
if not len(request.files) and foreign_id is None:
raise BadRequest(response=jsonify({
'status': 'error',
'message': 'Directories need to have a foreign_id'
}, status=400))
return meta, foreign_id
def metadata():
locale = get_locale()
enable_cache(vary_user=False, vary=str(locale))
key = cache.key('metadata', locale)
data = cache.get_complex(key)
if data is not None:
return jsonify(data)
auth = {}
if settings.PASSWORD_LOGIN:
auth['password_login_uri'] = url_for('sessions_api.password_login')
auth['registration_uri'] = url_for('roles_api.create_code')
if settings.OAUTH:
auth['oauth_uri'] = url_for('sessions_api.oauth_init')
data = {
'status': 'ok',
'maintenance': request.authz.in_maintenance,
'app': {
'title': settings.APP_TITLE,
'version': __version__,
'ui_uri': settings.APP_UI_URL,
'samples': settings.SAMPLE_SEARCHES,
'logo': settings.APP_LOGO,
'favicon': settings.APP_FAVICON,
'locale': str(locale),
'locales': settings.UI_LANGUAGES
},
'categories': Collection.CATEGORIES,
'countries': registry.country.names,
'languages': registry.language.names,
'schemata': model,
'auth': auth
}
cache.set_complex(key, data, expire=120)
return jsonify(data)
def suggest():
require(request.authz.logged_in)
parser = QueryParser(request.args, request.authz, limit=10)
if parser.prefix is None or len(parser.prefix) < 3:
# Do not return 400 because it's a routine event.
return jsonify({
'status': 'error',
'message': gettext('prefix filter is too short'),
'results': [],
'total': 0
})
# this only returns users, not groups
q = Role.by_prefix(parser.prefix, exclude=parser.exclude)
result = DatabaseQueryResult(request, q, parser=parser)
return RoleSerializer.jsonify_result(result)
def suggest_type():
prefix = request.args.get('prefix', '').lower().strip()
matches = []
for schema in model:
match = not len(prefix)
match = match or prefix in schema.name.lower()
match = match or prefix in schema.label.lower()
if match and schema.matchable:
matches.append(get_freebase_type(schema))
return jsonify({
"code": "/api/status/ok",
"status": "200 OK",
"prefix": request.args.get('prefix', ''),
"result": matches
})
def handle_es_error(err):
message = err.error
try:
status = int(err.status_code)
except Exception:
status = 500
try:
for cause in err.info.get('error', {}).get('root_cause', []):
message = cause.get('reason', message)
except Exception as ex:
log.debug(ex)
return jsonify({
'status': 'error',
'message': message
}, status=status)
def references(id):
enable_cache()
entity = get_index_entity(id, request.authz.READ)
record_audit(Audit.ACT_ENTITY, id=id)
results = []
for prop, total in entity_references(entity, request.authz):
key = ('filter:properties.%s' % prop.name, id)
link = url_for('entities_api.index', _query=(key, ))
results.append({
'count': total,
'property': prop,
'schema': prop.schema.name,
'results': link
})
return jsonify({'status': 'ok', 'total': len(results), 'results': results})
def view(document_id):
enable_cache()
data = get_index_document(document_id)
document = get_db_document(document_id)
data['headers'] = document.headers
# TODO: should this be it's own API? Probably so, but for that it would
# be unclear if we should JSON wrap it, or serve plain with the correct
# MIME type?
if Document.SCHEMA_HTML in document.model.names:
data['html'] = sanitize_html(document.body_raw, document.source_url)
if Document.SCHEMA_TEXT in document.model.names:
data['text'] = document.body_text
if Document.SCHEMA_IMAGE in document.model.names:
data['text'] = document.body_text
return jsonify(data, schema=CombinedSchema)
def _load_parent(collection, meta):
"""Determine the parent document for the document that is to be
ingested."""
parent_id = meta.get('parent_id')
if parent_id is None:
return
parent = Document.by_id(parent_id, collection_id=collection.id)
if parent is None:
raise BadRequest(response=jsonify(
{
'status': 'error',
'message': 'Cannot load parent document'
},
status=400))
return parent
def ingest_upload(id):
collection = obj_or_404(Collection.by_id(id))
require(request.authz.can_write(collection.id))
meta, foreign_id = _load_metadata()
parent_id = _load_parent(collection, meta)
upload_dir = mkdtemp()
try:
documents = []
for storage in request.files.values():
path = safe_filename(storage.filename)
path = os.path.join(upload_dir, path)
storage.save(path)
content_hash = checksum(path)
document = Document.by_keys(collection=collection,
parent_id=parent_id,
foreign_id=foreign_id,
content_hash=content_hash)
document.mime_type = storage.mimetype
if storage.filename:
document.file_name = os.path.basename(storage.filename)
document.update(meta)
ingest_document(document, path,
role_id=request.authz.id)
documents.append(document)
if not len(request.files):
# If there is no files uploaded, try to create an empty
# directory instead. Maybe this should be more explicit,
# but it seemed like the most simple way of fitting it
# into the API.
document = Document.by_keys(collection=collection,
parent_id=parent_id,
foreign_id=foreign_id)
document.update(meta)
ingest_document(document, upload_dir,
role_id=request.authz.id)
documents.append(document)
finally:
shutil.rmtree(upload_dir)
# Update child counts in index.
if parent_id is not None:
index_document_id.delay(parent_id)
return jsonify({
'status': 'ok',
'documents': [DocumentSchema().dump(d).data for d in documents]
})
def _load_parent(collection, meta):
"""Determine the parent document for the document that is to be
ingested."""
parent = ensure_dict(meta.get("parent"))
parent_id = meta.get("parent_id", parent.get("id"))
if parent_id is None:
return
parent = Document.by_id(parent_id, collection=collection)
if parent is None:
raise BadRequest(
response=jsonify(
{"status": "error", "message": "Cannot load parent document"},
status=400,
)
)
return parent
def permissions_update(id):
# TODO: consider using a list to bundle permission writes
collection = get_collection(id, request.authz.WRITE)
data = parse_request(schema=PermissionSchema)
role = Role.all().filter(Role.id == data['role']['id']).first()
if role is None or not check_visible(role):
raise BadRequest()
perm = update_permission(role,
collection,
data['read'],
data['write'])
return jsonify({
'status': 'ok',
'updated': PermissionSchema().dump(perm)
})
def suggest_type():
prefix = request.args.get('prefix', '').lower().strip()
tag_request(prefix=prefix)
matches = []
for schema in model:
match = not len(prefix)
match = match or prefix in schema.name.lower()
match = match or prefix in schema.label.lower()
if match and schema.matchable:
matches.append(get_freebase_type(schema))
return jsonify({
"code": "/api/status/ok",
"status": "200 OK",
"prefix": request.args.get('prefix', ''),
"result": matches
})
def password_login():
"""Provides email and password authentication."""
data = parse_request(LoginSchema)
role = Role.by_email(data.get('email'))
if role is None or not role.has_password:
return Unauthorized("Authentication has failed.")
if not role.check_password(data.get('password')):
return Unauthorized("Authentication has failed.")
update_role(role)
db.session.commit()
return jsonify({
'status': 'ok',
'token': create_token(role)
})
def references(entity_id):
enable_cache()
entity = get_index_entity(entity_id, request.authz.READ)
tag_request(collection_id=entity.get('collection_id'))
results = []
for prop, total in entity_references(entity, request.authz):
results.append({
'count': total,
'property': prop,
'schema': prop.schema.name,
})
return jsonify({
'status': 'ok',
'total': len(results),
'results': results
})
def password_login():
"""Provides email and password authentication."""
require(settings.PASSWORD_LOGIN)
data = parse_request(LoginSchema)
role = Role.by_email(data.get('email'))
if role is None or not role.has_password:
raise BadRequest(gettext("Invalid user or password."))
if not role.check_password(data.get('password')):
raise BadRequest(gettext("Invalid user or password."))
db.session.commit()
update_role(role)
authz = Authz.from_role(role)
request.authz = authz
return jsonify({'status': 'ok', 'token': authz.to_token(role=role)})
def references(entity_id):
enable_cache()
entity = get_index_entity(entity_id, request.authz.READ)
record_audit(Audit.ACT_ENTITY, id=entity_id)
results = []
for prop, total in entity_references(entity, request.authz):
results.append({
'count': total,
'property': prop,
'schema': prop.schema.name,
})
return jsonify({
'status': 'ok',
'total': len(results),
'results': results
})
def tags(entity_id):
"""
---
get:
summary: Get entity tags
description: >-
Get tags for the entity with id `entity_id`. Tags include the query
string to make a search by that particular tag.
parameters:
- in: path
name: entity_id
required: true
schema:
type: string
responses:
'200':
description: OK
content:
application/json:
schema:
type: object
allOf:
- $ref: '#/components/schemas/QueryResponse'
properties:
results:
type: array
items:
$ref: '#/components/schemas/EntityTag'
tags:
- Entity
"""
enable_cache()
entity = get_index_entity(entity_id, request.authz.READ)
tag_request(collection_id=entity.get("collection_id"))
results = []
for (field, value, total) in entity_tags(entity, request.authz):
qvalue = quote(value.encode("utf-8"))
key = ("filter:%s" % field, qvalue)
qid = query_string([key])
results.append({
"id": qid,
"value": value,
"field": field,
"count": total
})
results.sort(key=lambda p: p["count"], reverse=True)
return jsonify({"status": "ok", "total": len(results), "results": results})
def content(entity_id):
"""
---
get:
summary: Get the content of an entity
description: >
Return the text and/or html content of the entity with id `entity_id`
parameters:
- in: path
name: entity_id
required: true
schema:
type: string
responses:
'200':
content:
application/json:
schema:
properties:
headers:
type: object
html:
type: string
text:
type: string
type: object
description: OK
'404':
description: Not Found
tags:
- Entity
"""
enable_cache()
entity = get_index_entity(entity_id, request.authz.READ, excludes=['text'])
tag_request(collection_id=entity.get('collection_id'))
proxy = model.get_proxy(entity)
html = proxy.first('bodyHtml', quiet=True)
source_url = proxy.first('sourceUrl', quiet=True)
encoding = proxy.first('encoding', quiet=True)
html = sanitize_html(html, source_url, encoding=encoding)
headers = proxy.first('headers', quiet=True)
headers = registry.json.unpack(headers)
return jsonify({
'headers': headers,
'text': proxy.first('bodyText', quiet=True),
'html': html
})
def statistics():
"""Get a summary of the data acessible to the current user.
---
get:
summary: System-wide user statistics.
description: >
Get a summary of the data acessible to the current user.
responses:
'200':
description: OK
content:
application/json:
schema:
type: object
tags:
- System
"""
enable_cache()
collections = request.authz.collections(request.authz.READ)
for collection_id in collections:
resolver.queue(request, Collection, collection_id)
resolver.resolve(request)
# Summarise stats. This is meant for display, so the counting is a bit
# inconsistent between counting all collections, and source collections
# only.
schemata = defaultdict(int)
countries = defaultdict(int)
categories = defaultdict(int)
for collection_id in collections:
data = resolver.get(request, Collection, collection_id)
if data is None or data.get('casefile'):
continue
categories[data.get('category')] += 1
things = get_collection_things(collection_id)
for schema, count in things.items():
schemata[schema] += count
for country in data.get('countries', []):
countries[country] += 1
return jsonify({
'collections': len(collections),
'schemata': dict(schemata),
'countries': dict(countries),
'categories': dict(categories),
'things': sum(schemata.values()),
})
def suggest():
"""
---
get:
summary: Suggest users matching a search prefix
description: >-
For a given `prefix`, suggest matching user accounts. For
security reasons, the prefix must be more than three
characters long.
parameters:
- in: query
name: prefix
required: true
schema:
type: string
responses:
'200':
description: OK
content:
application/json:
schema:
type: object
allOf:
- $ref: '#/components/schemas/QueryResponse'
properties:
results:
type: array
items:
$ref: '#/components/schemas/Role'
tags:
- Role
"""
require(request.authz.logged_in)
parser = QueryParser(request.args, request.authz, limit=10)
if parser.prefix is None or len(parser.prefix) < 3:
# Do not return 400 because it's a routine event.
return jsonify({
"status": "error",
"message": gettext("prefix filter is too short"),
"results": [],
"total": 0,
})
# this only returns users, not groups
exclude = ensure_list(parser.excludes.get("id"))
q = Role.by_prefix(parser.prefix, exclude=exclude)
result = DatabaseQueryResult(request, q, parser=parser)
return RoleSerializer.jsonify_result(result)
def ingest_upload(id):
collection = get_db_collection(id, request.authz.WRITE)
meta, foreign_id = _load_metadata(collection)
parent_id = _load_parent(collection, meta)
upload_dir = mkdtemp(prefix='aleph.upload.')
try:
documents = []
for storage in request.files.values():
path = safe_filename(storage.filename, default='upload')
path = os.path.join(upload_dir, path)
storage.save(path)
content_hash = checksum(path)
document = Document.by_keys(collection=collection,
parent_id=parent_id,
foreign_id=foreign_id,
content_hash=content_hash)
document.update(meta)
ingest_document(document, path, role_id=request.authz.id)
documents.append(document)
if not len(request.files):
# If there is no files uploaded, try to create an empty
# directory instead. Maybe this should be more explicit,
# but it seemed like the most simple way of fitting it
# into the API.
document = Document.by_keys(collection=collection,
parent_id=parent_id,
foreign_id=foreign_id)
document.schema = Document.SCHEMA_FOLDER
document.update(meta)
ingest_document(document,
None,
role_id=request.authz.id,
shallow=True)
documents.append(document)
finally:
shutil.rmtree(upload_dir)
# Update child counts in index.
if parent_id is not None:
index_document_id.apply_async([parent_id], priority=1)
return jsonify({
'status':
'ok',
'documents': [CombinedSchema().dump(d).data for d in documents]
})
def tags(entity_id):
"""
---
get:
summary: Get entity tags
description: >-
Get tags for the entity with id `entity_id`. Tags include the query
string to make a search by that particular tag.
parameters:
- in: path
name: entity_id
required: true
schema:
type: string
responses:
'200':
description: OK
content:
application/json:
schema:
type: object
allOf:
- $ref: '#/components/schemas/QueryResponse'
properties:
results:
type: array
items:
$ref: '#/components/schemas/EntityTag'
tags:
- Entity
"""
enable_cache()
entity = get_index_entity(entity_id, request.authz.READ)
tag_request(collection_id=entity.get('collection_id'))
results = []
for (field, value, total) in entity_tags(entity, request.authz):
qvalue = quote(value.encode('utf-8'))
key = ('filter:%s' % field, qvalue)
results.append({
'id': query_string([key]),
'value': value,
'field': field,
'count': total,
})
results.sort(key=lambda p: p['count'], reverse=True)
return jsonify({'status': 'ok', 'total': len(results), 'results': results})
def statistics(collection_id):
"""
---
get:
summary: Get a summary of collection contents
description: >
Get a listing of the most common entity types and attributes in the
given collection. The result is cached and can be somewhat out of
sync with the real numbers.
parameters:
- description: The collection ID.
in: path
name: collection_id
required: true
schema:
minimum: 1
type: integer
responses:
'200':
description: OK
content:
application/json:
schema:
type: object
properties:
schema:
type: object
names:
type: object
addresses:
type: object
countries:
type: object
languages:
type: object
phones:
type: object
emails:
type: object
ibans:
type: object
tags:
- Collection
"""
collection = get_db_collection(collection_id, request.authz.READ)
return jsonify(get_collection_stats(collection.id))
def password_login():
"""Provides email and password authentication."""
data = parse_request(schema=LoginSchema)
q = Role.by_email(data.get('email'))
q = q.filter(Role.password_digest != None) # noqa
role = q.first()
if role is None:
return Unauthorized("Authentication has failed.")
if not role.check_password(data.get('password')):
return Unauthorized("Authentication has failed.")
return jsonify({
'status': 'ok',
'token': create_token(role)
})
def references(entity_id):
enable_cache()
entity = get_index_entity(entity_id, request.authz.READ)
tag_request(collection_id=entity.get('collection_id'))
record_audit(Audit.ACT_ENTITY, id=entity_id)
results = []
for prop, total in entity_references(entity, request.authz):
results.append({
'count': total,
'property': prop,
'schema': prop.schema.name,
})
return jsonify({
'status': 'ok',
'total': len(results),
'results': results
})
def tags(entity_id):
enable_cache()
entity = get_index_entity(entity_id, request.authz.READ)
tag_request(collection_id=entity.get('collection_id'))
results = []
for (field, value, total) in entity_tags(entity, request.authz):
qvalue = quote(value.encode('utf-8'))
key = ('filter:%s' % field, qvalue)
results.append({
'id': query_string([key]),
'value': value,
'field': field,
'count': total,
})
results.sort(key=lambda p: p['count'], reverse=True)
return jsonify({'status': 'ok', 'total': len(results), 'results': results})
def _load_metadata():
"""Unpack the common, pre-defined metadata for all the uploaded files."""
try:
meta = json.loads(request.form.get("meta", "{}"))
except Exception as ex:
raise BadRequest(str(ex))
validate(meta, "DocumentIngest")
foreign_id = stringify(meta.get("foreign_id"))
if not len(request.files) and foreign_id is None:
raise BadRequest(
response=jsonify(
{"status": "error", "message": "Directories need to have a foreign_id"},
status=400,
)
)
return meta, foreign_id
def references(entity_id):
"""
---
get:
summary: Get entity references
description: >-
Get the schema-wise aggregation of references to the entity with id
`entity_id`. This can be used to find and display adjacent entities.
parameters:
- in: path
name: entity_id
required: true
schema:
type: string
responses:
'200':
description: OK
content:
application/json:
schema:
type: object
allOf:
- $ref: '#/components/schemas/QueryResponse'
properties:
results:
type: array
items:
$ref: '#/components/schemas/EntityReference'
tags:
- Entity
"""
enable_cache()
entity = get_index_entity(entity_id, request.authz.READ)
tag_request(collection_id=entity.get('collection_id'))
results = []
for prop, total in entity_references(entity, request.authz):
results.append({
'count': total,
'property': prop,
'schema': prop.schema.name,
})
return jsonify({
'status': 'ok',
'total': len(results),
'results': results
})
def status():
require(request.authz.logged_in)
status = get_active_collection_status()
active_collections = status.pop('datasets')
active_foreign_ids = set(active_collections.keys())
collections = request.authz.collections(request.authz.READ)
results = []
for fid in active_foreign_ids:
collection = Collection.by_foreign_id(fid)
if collection is None:
continue
if collection.id in collections:
result = active_collections[fid]
result['collection'] = collection.to_dict()
result['id'] = fid
results.append(result)
status['results'] = results
return jsonify(status)
def content(entity_id):
enable_cache()
entity = get_index_entity(entity_id, request.authz.READ)
tag_request(collection_id=entity.get('collection_id'))
for entity in entities_by_ids([entity_id],
schemata=entity.get('schema'),
excludes=['text']):
proxy = model.get_proxy(entity)
html = sanitize_html(proxy.first('bodyHtml', quiet=True),
proxy.first('sourceUrl', quiet=True))
headers = proxy.first('headers', quiet=True)
headers = registry.json.unpack(headers)
return jsonify({
'headers': headers,
'text': proxy.first('bodyText', quiet=True),
'html': html
})
return ('', 404)
def suggest_entity():
"""Suggest API, emulates Google Refine API."""
prefix = request.args.get('prefix', '')
args = {
'prefix': prefix,
'filter:schemata': request.args.getlist('type'),
'filter:collection_id': request.args.getlist('filter:collection_id')
}
parser = SearchQueryParser(args, request.authz)
query = EntitiesQuery(parser)
result = query.search()
matches = list(entity_matches(result))
return jsonify({
"code": "/api/status/ok",
"status": "200 OK",
"prefix": prefix,
"result": matches
})
def content(entity_id):
enable_cache()
entity = get_index_entity(entity_id, request.authz.READ)
tag_request(collection_id=entity.get('collection_id'))
for entity in entities_by_ids([entity_id],
schemata=entity.get('schema'),
excludes=['text']):
proxy = model.get_proxy(entity)
record_audit(Audit.ACT_ENTITY, id=entity_id)
html = sanitize_html(proxy.first('bodyHtml', quiet=True),
proxy.first('sourceUrl', quiet=True))
headers = proxy.first('headers', quiet=True)
headers = registry.json.unpack(headers)
return jsonify({
'headers': headers,
'text': proxy.first('bodyText', quiet=True),
'html': html
})
return ('', 404)
def password_login():
"""Provides email and password authentication."""
data = parse_request(LoginSchema)
role = Role.by_email(data.get('email'))
if role is None or not role.has_password:
return Unauthorized("Authentication has failed.")
if not role.check_password(data.get('password')):
return Unauthorized("Authentication has failed.")
db.session.commit()
update_role(role)
authz = Authz.from_role(role)
request.authz = authz
record_audit(Audit.ACT_LOGIN)
return jsonify({
'status': 'ok',
'token': authz.to_token(role=role)
})
def statistics():
"""Get a summary of the data acessible to the current user."""
enable_cache()
collections = request.authz.collections(request.authz.READ)
for collection_id in collections:
resolver.queue(request, Collection, collection_id)
for role_id in request.authz.roles:
resolver.queue(request, Role, role_id)
resolver.resolve(request)
# Summarise stats. This is meant for display, so the counting is a bit
# inconsistent between counting all collections, and source collections
# only.
schemata = defaultdict(int)
countries = defaultdict(int)
categories = defaultdict(int)
for collection_id in collections:
data = resolver.get(request, Collection, collection_id)
if data is None or data.get('casefile'):
continue
categories[data.get('category')] += 1
for schema, count in data.get('schemata', {}).items():
schemata[schema] += count
for country in data.get('countries', []):
countries[country] += 1
# Add a users roles to the home page:
groups = []
for role_id in request.authz.roles:
data = resolver.get(request, Role, role_id)
if data is None or data.get('type') != Role.GROUP:
continue
groups.append(RoleSerializer().serialize(data))
return jsonify({
'collections': len(collections),
'schemata': dict(schemata),
'countries': dict(countries),
'categories': dict(categories),
'groups': groups,
'things': sum(schemata.values()),
})
def suggest_entity():
"""Suggest API, emulates Google Refine API."""
prefix = request.args.get('prefix', '')
tag_request(prefix=prefix)
types = request.args.getlist('type') or Entity.THING
args = {
'prefix': prefix,
'filter:schemata': types,
'filter:collection_id': request.args.getlist('filter:collection_id')
}
parser = SearchQueryParser(args, request.authz)
query = EntitiesQuery(parser)
result = query.search()
matches = list(entity_matches(result))
return jsonify({
"code": "/api/status/ok",
"status": "200 OK",
"prefix": prefix,
"result": matches
})
def reconcile_index(collection=None):
domain = settings.APP_UI_URL.strip('/')
label = settings.APP_TITLE
suggest_query = []
schemata = list(model)
if collection is not None:
label = '%s (%s)' % (collection.get('label'), label)
suggest_query.append(('filter:collection_id', collection.get('id')))
schemata = [model.get(s) for s in collection.get('schemata').keys()]
return jsonify({
'name': label,
'identifierSpace': 'http://rdf.freebase.com/ns/type.object.id',
'schemaSpace': 'http://rdf.freebase.com/ns/type.object.id',
'view': {'url': entity_url('{{id}}')},
'preview': {
'url': entity_url('{{id}}'),
'width': 800,
'height': 400
},
'suggest': {
'entity': {
'service_url': domain,
'service_path': url_for('reconcile_api.suggest_entity',
_query=suggest_query,
_authorize=True,
_relative=True)
},
'type': {
'service_url': domain,
'service_path': url_for('reconcile_api.suggest_type',
_relative=True)
},
'property': {
'service_url': domain,
'service_path': url_for('reconcile_api.suggest_property',
_relative=True)
}
},
'defaultTypes': [get_freebase_type(s) for s in schemata if s.matchable]
})
def tags(entity_id):
enable_cache()
entity = get_index_entity(entity_id, request.authz.READ)
tag_request(collection_id=entity.get('collection_id'))
record_audit(Audit.ACT_ENTITY, id=entity_id)
results = []
for (field, value, total) in entity_tags(entity, request.authz):
qvalue = quote(value.encode('utf-8'))
key = ('filter:%s' % field, qvalue)
results.append({
'id': query_string([key]),
'value': value,
'field': field,
'count': total,
})
results.sort(key=lambda p: p['count'], reverse=True)
return jsonify({
'status': 'ok',
'total': len(results),
'results': results
})
def healthz():
return jsonify({'status': 'ok'})
def handle_bad_request(err):
return jsonify({
'status': 'error',
'message': err.description
}, status=400)
def handle_authz_error(err):
return jsonify({
'status': 'error',
'message': gettext('You are not authorized to do this.'),
'roles': request.authz.roles
}, status=403)
def handle_not_found_error(err):
return jsonify({
'status': 'error',
'message': gettext('This path does not exist.')
}, status=404)
def handle_server_error(err):
log.exception("%s: %s", type(err).__name__, err)
return jsonify({
'status': 'error',
'message': gettext('Internal server error.')
}, status=500)
def handle_invalid_data(err):
return jsonify({
'status': 'error',
'message': str(err),
'errors': err.errors
}, status=400)
def api_v1_message(path):
return jsonify({
'status': 'error',
'message': gettext('/api/1/ is deprecated, please use /api/2/.')
}, status=410)
def mark_read():
require(request.authz.logged_in)
role = Role.by_id(request.authz.id)
role.notified_at = datetime.utcnow()
db.session.commit()
return jsonify({'status': 'ok'}, status=202)
def handle_jwt_expired(err):
return jsonify({
'status': 'error',
'errors': gettext('Access token is invalid.')
}, status=401)