def get_instance_stats(authz):
query = {
'size': 0,
'query': {
'terms': {
'roles': list(authz.roles)
}
},
'aggs': {
'schema': {
'terms': {
'field': 'schema',
'size': 1000
}
}
}
}
result = es.search(index=entities_index(),
doc_type=entity_type(),
body=query)
aggregations = result.get('aggregations')
data = {'count': result.get('hits').get('total'), 'schemata': {}}
for schema in aggregations.get('schema').get('buckets'):
key = schema.get('key')
data['schemata'][key] = schema.get('doc_count')
return data
def generate_sitemap(collection_id):
"""Generate entries for a collection-based sitemap.xml file."""
# cf. https://www.sitemaps.org/protocol.html
query = {
'query': {
'bool': {
'filter': [
{'term': {'collection_id': collection_id}},
{'term': {'schemata': Entity.THING}},
authz_query(Authz.from_role(None))
]
}
},
'_source': {'includes': ['schemata', 'updated_at']}
}
scanner = scan(es, index=entities_index(), query=query)
# strictly, the limit for sitemap.xml is 50,000
for res in islice(scanner, 49500):
source = res.get('_source', {})
updated_at = source.get('updated_at', '').split('T', 1)[0]
if Document.SCHEMA in source.get('schemata', []):
url = document_url(res.get('_id'))
else:
url = entity_url(res.get('_id'))
yield (url, updated_at)
def get_instance_stats(authz):
query = {
'size': 0,
'query': {
'bool': {
'filter': [
authz_query(authz),
{'term': {'schemata': Entity.THING}}
]
}
},
'aggs': {
'schema': {'terms': {'field': 'schema', 'size': 1000}}
}
}
result = es.search(index=entities_index(),
body=query)
aggregations = result.get('aggregations')
data = {
'count': result.get('hits').get('total'),
'schemata': {}
}
for schema in aggregations.get('schema').get('buckets'):
key = schema.get('key')
data['schemata'][key] = schema.get('doc_count')
return data
def iter_entities(authz=None,
collection_id=None,
schemata=None,
includes=None,
excludes=None):
"""Scan all entities matching the given criteria."""
filters = []
if authz is not None:
filters.append(authz_query(authz))
if collection_id is not None:
filters.append({'term': {'collection_id': collection_id}})
if ensure_list(schemata):
filters.append({'terms': {'schemata': ensure_list(schemata)}})
source = {}
if ensure_list(includes):
source['includes'] = ensure_list(includes)
if ensure_list(excludes):
source['excludes'] = ensure_list(excludes)
query = {
'query': {
'bool': {
'filter': filters
}
},
'sort': ['_doc'],
'_source': source
}
for res in scan(es, index=entities_index(), query=query, scroll='1410m'):
yield unpack_result(res)
def __init__(self, request, query, parser=None, schema=None):
super(MatchQueryResult, self).__init__(request,
query,
parser=parser,
schema=schema)
ids = set()
for match in self.results:
ids.add(match.match_id)
ids.add(match.entity_id)
ids = {'ids': list(ids)}
result = es.mget(index=entities_index(),
doc_type=entity_type(),
body=ids)
for doc in result.get('docs', []):
entity = unpack_result(doc)
if entity is None:
continue
for match in self.results:
if match.match_id == entity['id']:
match.match = entity
if match.entity_id == entity['id']:
match.entity = entity
# Do not return results if the entity has been removed in the mean
# time. Not sure this is the ideal way of doing this, as it'll mess
# with pagination counts etc.
for match in list(self.results):
if not hasattr(match, 'match') or not hasattr(match, 'entity'):
self.results.remove(match)
def delete(id):
entity = get_db_entity(id, request.authz.WRITE)
delete_entity(entity)
db.session.commit()
update_collection(entity.collection)
refresh_index(entities_index())
return ('', 204)
def expand_group(node):
if node.type.group is None or node.value is None:
return
value = str(node.value)
query = {
'query': {
'term': {
node.type.group: value
}
},
'_source': {
'includes': ['schema', 'properties']
}
}
for res in scan(es, index=entities_index(), query=query):
entity_id = res.get('_id')
source = res.get('_source')
properties = source.get('properties')
schema = model.get(source.get('schema'))
for prop in schema.properties.values():
if prop.type != node.type:
continue
values = properties.get(prop.name)
values = node.type.normalize_set(values)
if value not in values:
continue
if prop.reverse:
yield Link(node, prop.reverse, entity_id)
else:
yield Link(node, prop, entity_id, inverted=True)
def get_entity(entity_id):
"""Fetch an entity from the index."""
result = es.get(index=entities_index(),
doc_type='doc',
id=entity_id,
ignore=[404],
_source_exclude=['text'])
return unpack_result(result)
def xref_collection(collection_id, other_id=None):
"""Cross-reference all the entities and documents in a collection."""
query = {'term': {'collection_id': collection_id}}
query = {'query': query, '_source': {'excludes': ['text', 'properties.*']}}
scanner = scan(es, index=entities_index(), query=query, scroll='30m')
for idx, res in enumerate(scanner):
res = unpack_result(res)
_xref_item(res, collection_id=other_id)
def delete_collection(collection_id, wait=True):
"""Delete all documents from a particular collection."""
query = {'term': {'collection_id': collection_id}}
query_delete(records_index(), query, wait=wait)
query_delete(entities_index(), query, wait=wait)
es.delete(index=collections_index(),
doc_type=collection_type(),
id=collection_id,
ignore=[404])
def create():
data = parse_request(EntityCreateSchema)
collection = get_db_collection(data['collection_id'], request.authz.WRITE)
entity = Entity.create(data, collection)
db.session.commit()
data = update_entity(entity)
update_collection(collection)
refresh_index(entities_index())
return serialize_data(data, CombinedSchema)
def get_document(document_id):
"""Fetch a document from the index."""
result = es.get(index=entities_index(),
doc_type=entity_type(),
id=document_id,
ignore=[404])
document = unpack_result(result)
if document is not None:
document.pop('text', None)
return document
def get_entity(entity_id):
"""Fetch an entity from the index."""
result = es.get(index=entities_index(),
doc_type=entity_type(),
id=entity_id,
ignore=[404])
entity = unpack_result(result)
if entity is not None:
entity.pop('text', None)
return entity
def entity_tags(entity, authz):
"""Do a search on tags of an entity."""
# NOTE: This must also work for documents.
FIELDS = [
'names',
'emails',
'phones',
'addresses',
'identifiers'
]
pivots = []
queries = []
# Go through all the tags which apply to this entity, and find how
# often they've been mentioned in other entities.
for field in FIELDS:
for value in entity.get(field, []):
if value is None:
continue
queries.append({})
queries.append({
'size': 0,
'query': {
'bool': {
'filter': [
authz_query(authz),
field_filter_query(field, value)
],
'must_not': [
{'ids': {'values': [entity.get('id')]}},
]
}
}
})
pivots.append((field, value))
if not len(queries):
return []
res = es.msearch(index=entities_index(), body=queries)
results = []
for (field, value), resp in zip(pivots, res.get('responses', [])):
total = resp.get('hits', {}).get('total')
if total > 0:
qvalue = quote(value.encode('utf-8'))
key = ('filter:%s' % field, qvalue)
results.append({
'id': query_string([key]),
'value': value,
'field': field,
'count': total
})
results.sort(key=lambda p: p['count'], reverse=True)
return results
def ingest_upload(id):
collection = get_db_collection(id, request.authz.WRITE)
meta, foreign_id = _load_metadata(collection)
parent_id = _load_parent(collection, meta)
upload_dir = mkdtemp(prefix='aleph.upload.')
try:
documents = []
for storage in request.files.values():
path = safe_filename(storage.filename, default='upload')
path = os.path.join(upload_dir, path)
storage.save(path)
content_hash = checksum(path)
document = Document.by_keys(collection=collection,
parent_id=parent_id,
foreign_id=foreign_id,
content_hash=content_hash)
document.update(meta)
document.uploader_id = request.authz.id
ingest_document(document, path)
documents.append(document)
if not len(request.files):
# If there is no files uploaded, try to create an empty
# directory instead. Maybe this should be more explicit,
# but it seemed like the most simple way of fitting it
# into the API.
document = Document.by_keys(collection=collection,
parent_id=parent_id,
foreign_id=foreign_id)
document.schema = Document.SCHEMA_FOLDER
document.update(meta)
document.uploader_id = request.authz.id
ingest_document(document, None)
documents.append(document)
finally:
shutil.rmtree(upload_dir)
if collection.casefile:
for document in documents:
params = {'document': document, 'collection': collection}
publish(Events.INGEST_DOCUMENT,
actor_id=document.uploader_id,
params=params)
# Update child counts in index.
if parent_id is not None:
index_document_id.apply_async([parent_id], priority=1)
refresh_index(index=entities_index())
return jsonify({
'status':
'ok',
'documents': [CombinedSchema().dump(d).data for d in documents]
})
def query_collection_contents(collection_id):
q = {
'query': {
'term': {
'collection_id': collection_id
}
},
'_source': {
'exclude': ['text']
}
}
res = scan(es, index=entities_index(), query=q)
return res
def get_collection_stats(collection_id):
"""Compute some statistics on the content of a collection."""
key = cache.key('cstats', collection_id)
data = cache.get_complex(key)
if data is not None:
return data
log.info("Generating collection stats: %s", collection_id)
query = {
'size': 0,
'query': {
'bool': {
'filter': [{
'term': {
'collection_id': collection_id
}
}]
}
},
'aggs': {
'schemata': {
'terms': {
'field': 'schema',
'size': 1000
}
},
'countries': {
'terms': {
'field': 'countries',
'size': 500
}
},
'languages': {
'terms': {
'field': 'languages',
'size': 10
}
},
}
}
result = search_safe(index=entities_index(), body=query)
aggregations = result.get('aggregations', {})
data = {'count': result['hits']['total']}
for facet in ['schemata', 'countries', 'languages']:
data[facet] = {}
for bucket in aggregations[facet]['buckets']:
data[facet][bucket['key']] = bucket['doc_count']
expire = randint(3600 * 3, 3600 * 12)
cache.set_complex(key, data, expire=expire)
return data
def update_collection_roles(collection):
"""Update the role visibility of objects which are part of collections."""
roles = ', '.join([str(r) for r in collection.roles])
body = {
'query': {
'term': {
'collection_id': collection.id
}
},
'script': {
'inline': 'ctx._source.roles = [%s]' % roles
}
}
query_update(entities_index(), body)
def entity_references(entity, authz):
"""Given a particular entity, find all the references to it from other
entities, grouped by the property where they are used."""
schema = model[entity.get('schema')]
# Generate all the possible mention locations.
properties = []
queries = []
for prop in model.properties:
if not prop.is_entity:
continue
if not schema.is_a(prop.range):
continue
field = 'properties.%s' % prop.name
queries.append({})
queries.append({
'size': 0,
'query': {
'bool': {
'filter': [
authz_query(authz),
{
'term': {
'schemata': prop.schema.name
}
},
{
'term': {
field: entity.get('id')
}
},
]
}
}
})
properties.append(prop)
# Run a count search (with schema facet?)
res = es.msearch(index=entities_index(), body=queries)
results = []
for prop, resp in zip(properties, res.get('responses', [])):
total = resp.get('hits', {}).get('total')
if total > 0:
results.append({
'count': total,
'property': prop,
'schema': prop.schema.name
})
return results
def check_alert(alert):
authz = Authz(role=alert.role)
query = alert_query(alert, authz)
found = 0
for result in scan(es, query=query, index=entities_index()):
entity = unpack_result(result)
found += 1
params = {'alert': alert, 'role': authz.role, 'entity': entity}
publish(Events.MATCH_ALERT,
actor_id=entity.get('uploader_id'),
params=params)
alert.update()
log.info('Found %d new results for: %s', found, alert.label)
db.session.commit()
def xref_collection(collection, other=None):
"""Cross-reference all the entities and documents in a collection."""
log.info("Cross-reference collection: %r", collection)
other_id = other.id if other is not None else None
query = {
'query': {
'term': {
'collection_id': collection.id
}
},
'_source': FIELDS_XREF
}
scanner = scan(es, index=entities_index(), query=query, scroll='15m')
for i, res in enumerate(scanner):
xref_item.apply_async([unpack_result(res), other_id], priority=4)
def _index_updates(collection, entities):
"""Look up existing index documents and generate an updated form.
This is necessary to make the index accumulative, i.e. if an entity or link
gets indexed twice with different field values, it'll add up the different
field values into a single record. This is to avoid overwriting the
document and losing field values. An alternative solution would be to
implement this in Groovy on the ES.
"""
common = {
'collection_id': collection.id,
'bulk': True,
'roles': collection.roles,
'updated_at': datetime.utcnow()
}
if not len(entities):
return
result = es.mget(index=entities_index(),
doc_type=entity_type(),
body={'ids': entities.keys()},
_source=['schema', 'properties', 'created_at'])
for doc in result.get('docs', []):
if not doc.get('found', False):
continue
entity_id = doc['_id']
entity = entities.get(entity_id)
existing = doc.get('_source')
combined = model.merge(existing, entity)
combined['created_at'] = existing.get('created_at')
entities[entity_id] = combined
for doc_id, entity in entities.items():
entity.pop('id', None)
entity.pop('data', None)
entity.update(common)
if 'created_at' not in entity:
entity['created_at'] = entity.get('updated_at')
schema = model.get(entity.get('schema'))
entity = finalize_index(entity, schema)
# pprint(entity)
yield {
'_id': doc_id,
'_index': entity_index(),
'_type': entity_type(),
'_source': entity
}
def delete_entities(collection_id):
"""Delete entities from a collection."""
query = {
'bool': {
'must_not': {
'term': {
'schemata': 'Document'
}
},
'must': {
'term': {
'collection_id': collection_id
}
}
}
}
query_delete(entities_index(), query)
def xref_collection(collection_id, other_id=None):
"""Cross-reference all the entities and documents in a collection."""
query = {'term': {'collection_id': collection_id}}
query = {
'query': query,
'_source': FIELDS_XREF
}
scanner = scan(es,
index=entities_index(),
query=query,
scroll='15m')
for i, res in enumerate(scanner):
res = unpack_result(res)
xref_item.apply_async(args=[res],
kwargs={'collection_id': other_id},
priority=4)
def _resolve_index(self, cache):
queries = OrderedDict()
for (type_, id_) in cache.keys():
if type_ in [Collection]:
index = collections_index()
queries[(type_, id_)] = {'_index': index, '_id': id_}
elif type_ in [Document, Entity]:
index = entities_index()
queries[(type_, id_)] = {'_index': index, '_id': id_}
if not len(queries):
return
results = es.mget(body={'docs': queries.values()},
_source_exclude=['text'])
for key, doc in zip(queries.keys(), results['docs']):
cache[key] = unpack_result(doc)
def export_collection(collection):
uri = collection_uri(collection.id)
g = Graph()
g.add((uri, RDF.type, DCMI.Collection))
g.add((uri, RDFS.label, Literal(collection.label)))
g.add((uri, DCMI.identifier, Literal(collection.foreign_id)))
g.add((uri, ALEPH.category, ALEPH[collection.category]))
for line in itergraph(g):
yield line
q = {'term': {'collection_id': collection.id}}
q = {'query': q, '_source': {'exclude': ['text']}}
for row in scan(es, index=entities_index(), query=q):
g = export_entity(unpack_result(row), uri)
for line in itergraph(g):
yield line
def delete_documents(collection_id):
"""Delete documents from a collection."""
query = {
'bool': {
'must': [{
'term': {
'schemata': 'Document'
}
}, {
'term': {
'collection_id': collection_id
}
}]
}
}
query_delete(entities_index(), query)
records_query = {'term': {'collection_id': collection_id}}
query_delete(records_index(), records_query)
def xref_item(proxy):
"""Cross-reference an entity or document, given as an indexed document."""
query = match_query(proxy)
if query == none_query():
return
query = {
'query': query,
'size': 100,
'_source': {'includes': ['schema', 'properties', 'collection_id']}
}
result = search_safe(index=entities_index(), body=query)
results = result.get('hits').get('hits')
for result in results:
result = unpack_result(result)
if result is not None:
other = model.get_proxy(result)
score = compare(model, proxy, other)
yield score, result.get('collection_id'), other
def entity_tags(entity, authz):
"""Do a search on tags of an entity."""
# NOTE: This must also work for documents.
FIELDS = [
'names',
'emails',
'phones',
'addresses',
'identifiers'
]
pivots = []
queries = []
# Go through all the tags which apply to this entity, and find how
# often they've been mentioned in other entities.
for field in FIELDS:
for value in entity.get(field, []):
if value is None or not len(value):
continue
queries.append({})
queries.append({
'size': 0,
'query': {
'bool': {
'filter': [
authz_query(authz),
field_filter_query(field, value)
],
'must_not': [
{'ids': {'values': [entity.get('id')]}},
]
}
}
})
pivots.append((field, value))
if not len(queries):
return
res = es.msearch(index=entities_index(), body=queries)
for (field, value), resp in zip(pivots, res.get('responses', [])):
total = resp.get('hits', {}).get('total')
if total is not None and total > 0:
yield (field, value, total)
def _xref_item(item, collection_id=None):
"""Cross-reference an entity or document, given as an indexed document."""
name = item.get('name') or item.get('title')
query = entity_query(item, collection_id=collection_id)
if 'match_none' in query:
return
query = {
'query': query,
'size': 10,
'_source': ['collection_id', 'name'],
}
result = search_safe(index=entities_index(), body=query)
results = result.get('hits').get('hits')
entity_id, document_id = None, None
if Document.SCHEMA in item.get('schemata'):
document_id = item.get('id')
else:
entity_id = item.get('id')
dq = db.session.query(Match)
dq = dq.filter(Match.entity_id == entity_id)
dq = dq.filter(Match.document_id == document_id)
if collection_id is not None:
dq = dq.filter(Match.match_collection_id == collection_id)
dq.delete()
for result in results:
source = result.get('_source', {})
log.info("Xref [%.1f]: %s <=> %s", result.get('_score'), name,
source.get('name'))
obj = Match()
obj.entity_id = entity_id
obj.document_id = document_id
obj.collection_id = item.get('collection_id')
obj.match_id = result.get('_id')
obj.match_collection_id = source.get('collection_id')
obj.score = result.get('_score')
db.session.add(obj)
db.session.commit()
