def collection_entity_save(collection):
collection = get_collection(collection, authz.WRITE)
data = request_data()
update_operation = 'id' in data
entities = get_loom_config().entities
schema = data.get('$schema')
if update_operation and schema is None:
schema = entities.get_schema(data['id'], right=authz.entity_right())
if schema not in get_loom_config().schemas.values():
raise BadRequest()
# this will raise if it fails:
validate(data, schema)
subject = entities.save(schema,
data,
collection_id=collection.id,
author=request.auth_user,
right=authz.entity_right())
collection_add_entity(collection, subject)
get_loom_indexer().index_one(subject, schema=schema)
entity = entities.get(subject,
schema=schema,
depth=2,
right=authz.entity_right())
return jsonify({
'status': 'ok',
'data': entity
},
status=200 if update_operation else 201)
def collection_entity_save(collection):
collection = get_collection(collection, authz.WRITE)
data = request_data()
update_operation = 'id' in data
entities = get_loom_config().entities
schema = data.get('$schema')
if update_operation and schema is None:
schema = entities.get_schema(data['id'], right=authz.entity_right())
if schema not in get_loom_config().schemas.values():
raise BadRequest()
# this will raise if it fails:
validate(data, schema)
subject = entities.save(schema, data, collection_id=collection.id,
author=request.auth_user,
right=authz.entity_right())
collection_add_entity(collection, subject)
get_loom_indexer().index_one(subject, schema=schema)
entity = entities.get(subject, schema=schema, depth=2,
right=authz.entity_right())
return jsonify({
'status': 'ok',
'data': entity
}, status=200 if update_operation else 201)
def collection_graph(collection):
collection = get_collection(collection, authz.READ)
network = Network(get_loom_config().resolver)
schema = request.args.get('$schema')
for entity in collection_entities(collection, depth=get_depth(3),
filter_schema=schema):
network.add(entity)
return jsonify(network)
def collection_remove_entity(collection, subject):
q = session.query(CollectionSubject).filter_by(subject=subject)
q = q.filter_by(collection_id=collection.id)
q.delete()
session.commit()
entities = get_loom_config().entities
entities.remove(subject, collection_id=collection.id)
def collection_graph(collection):
collection = get_collection(collection, authz.READ)
network = Network(get_loom_config().resolver)
schema = request.args.get('$schema')
for entity in collection_entities(collection,
depth=get_depth(3),
filter_schema=schema):
network.add(entity)
return jsonify(network)
def collection_entities(collection, depth=2, filter_schema=None):
config = get_loom_config()
# FIXME: this is a performance nightmare. Think about how to fix it.
results = []
q = config.entities.subjects(schema=filter_schema,
collection_id=collection.id,
right=authz.entity_right())
for (subject, schema) in q:
data = config.entities.get(subject, schema=schema, depth=depth,
right=authz.entity_right())
results.append(result_entity(data))
return results
def collection_entities(collection, depth=2, filter_schema=None):
config = get_loom_config()
# FIXME: this is a performance nightmare. Think about how to fix it.
results = []
q = config.entities.subjects(schema=filter_schema,
collection_id=collection.id,
right=authz.entity_right())
for (subject, schema) in q:
data = config.entities.get(subject,
schema=schema,
depth=depth,
right=authz.entity_right())
results.append(result_entity(data))
return results
def update_subjects(collection, data):
""" There must be a nicer way to do this in SQLA. """
# TODO: authz
subjects = data.get('subjects', [])
for cs in collection.subjects:
if cs.subject in subjects:
subjects.remove(cs.subject)
else:
session.delete(cs)
for subject in subjects:
if get_loom_config().entities.get_schema(subject):
cs = CollectionSubject(collection, subject)
session.add(cs)
else:
raise BadRequest()
def suggest_entity(args):
""" Auto-complete API. """
config = get_loom_config()
text = args.get('text')
options = []
if text is not None and len(text.strip()):
q = authz_filter({'match': {'$suggest': text}})
schema = args.get('$schema')
if schema is not None:
# Find matching sub-schemas as well.
schemas = config.implied_schemas(schema)
q = add_filter(q, {"terms": {"$schema": schemas}})
boost_collection = args.get('boost_collection')
if boost_collection is not None:
q = {
'function_score': {
'query': q,
'functions': [{
'boost_factor': 2,
'filter': {
'term': {
'$collections': boost_collection
}
}
}]
}
}
exclude_collection = args.get('exclude_collection')
if exclude_collection is not None:
q = add_filter(q, {'not': {
'term': {'$collections': exclude_collection}
}})
q = {
'size': 5,
'query': q,
'_source': ['name', 'id', '$schema']
}
result = get_es().search(index=get_es_index(), body=q)
for res in result.get('hits', {}).get('hits', []):
options.append(res.get('_source'))
return {
'text': text,
'options': options
}
def suggest_entity(args):
""" Auto-complete API. """
config = get_loom_config()
text = args.get('text')
options = []
if text is not None and len(text.strip()):
q = authz_filter({'match': {'$suggest': text}})
schema = args.get('$schema')
if schema is not None:
# Find matching sub-schemas as well.
schemas = config.implied_schemas(schema)
q = add_filter(q, {"terms": {"$schema": schemas}})
boost_collection = args.get('boost_collection')
if boost_collection is not None:
q = {
'function_score': {
'query':
q,
'functions': [{
'boost_factor': 2,
'filter': {
'term': {
'$collections': boost_collection
}
}
}]
}
}
exclude_collection = args.get('exclude_collection')
if exclude_collection is not None:
q = add_filter(
q, {'not': {
'term': {
'$collections': exclude_collection
}
}})
q = {'size': 5, 'query': q, '_source': ['name', 'id', '$schema']}
result = get_es().search(index=get_es_index(), body=q)
for res in result.get('hits', {}).get('hits', []):
options.append(res.get('_source'))
return {'text': text, 'options': options}
def test_basic_search(self):
self.setUpFixtures()
config = get_loom_config()
assert len(config.types), len(config.types)
res = self.client.get('/api/search')
assert res.json['total'] == 0, res.json
self.login()
res = self.client.get('/api/search')
assert res.json['total'] == 50, res.json
res = self.client.get('/api/search?limit=4')
assert len(res.json['results']) == 4, res.json
res = self.client.get('/api/search?offset=46')
assert len(res.json['results']) == 4, res.json
res = self.client.get('/api/search?q=Hazim')
assert res.json['total'] == 1, res.json['total']
assert not res.json['facets'], res.json['total']
res = self.client.get('/api/search?facet=jurisdiction_code')
facets = res.json['facets']
assert 'jurisdiction_code' in facets, facets
jur = facets['jurisdiction_code']
assert jur['values'][0]['key'] == 'BA', facets
ORG = "https://schema.occrp.org/generic/organization.json#"
res = self.client.get('/api/search?filter:$schema=' + ORG)
assert res.json['total'] == 8, res.json['total']
qs = '?filter:jurisdiction_code=BA&filter:$schema=' + ORG
res = self.client.get('/api/search' + qs)
assert res.json['total'] == 8, res.json['total']
qs = '?filter:jurisdiction_code=GB&filter:$schema=' + ORG
res = self.client.get('/api/search' + qs)
assert res.json['total'] == 0, res.json['total']
def entities_to_table(schema, entities):
""" Generate a flattened table from a set of entities, inlining
inline sub-entities. """
config = get_loom_config()
visitor = SchemaVisitor({'$ref': schema}, config.resolver)
rows = []
for entity in entities:
row = OrderedDict()
for prop in visitor.properties:
if prop.is_value and prop.inline:
row[prop.title] = entity.get(prop.name)
if prop.is_object:
child_entity = entity.get(prop.name, {})
if prop.inline:
for child_prop in prop.properties:
if not prop.inline:
continue
title = '%s: %s' % (prop.title, child_prop.title)
row[title] = child_entity.get(child_prop.name)
else:
row[prop.title] = child_entity.get('name',
child_entity.get('id'))
rows.append(row)
return visitor, rows
def entities_to_table(schema, entities):
""" Generate a flattened table from a set of entities, inlining
inline sub-entities. """
config = get_loom_config()
visitor = SchemaVisitor({'$ref': schema}, config.resolver)
rows = []
for entity in entities:
row = OrderedDict()
for prop in visitor.properties:
if prop.is_value and prop.inline:
row[prop.title] = entity.get(prop.name)
if prop.is_object:
child_entity = entity.get(prop.name, {})
if prop.inline:
for child_prop in prop.properties:
if not prop.inline:
continue
title = '%s: %s' % (prop.title, child_prop.title)
row[title] = child_entity.get(child_prop.name)
else:
row[prop.title] = child_entity.get('name',
child_entity.get('id'))
rows.append(row)
return visitor, rows
def validate(data, schema):
resolver = get_loom_config().resolver
_, schema = resolver.resolve(schema)
validator = Draft4Validator(schema, resolver=resolver,
format_checker=format_checker)
return validator.validate(data, schema)
def view(id):
entities = get_loom_config().entities
data = obj_or_404(entities.get(id, depth=get_depth(3),
right=authz.entity_right()))
return jsonify({'status': 'ok', 'data': result_entity(data)})
def view(id):
entities = get_loom_config().entities
data = obj_or_404(
entities.get(id, depth=get_depth(3), right=authz.entity_right()))
return jsonify({'status': 'ok', 'data': result_entity(data)})
