def delete_entity_references(entity_id):
"""Delete all entities associated with any collection.
This is used by the ``indexentities`` management command in order to clear
out any leftover entities in the index.
"""
q = {'query': {'term': {'entities.id': entity_id}}}
bulk_op(document_updates(q, entity_id))
flush_index()
def update_entity_references(entity, max_query=1000):
"""Same as above but runs in bulk for a particular entity."""
q = db.session.query(Reference.document_id)
q = q.filter(Reference.entity_id == entity.id)
q = q.filter(Entity.id == Reference.entity_id)
q = q.filter(Entity.state == Entity.STATE_ACTIVE)
documents = [str(r.document_id) for r in q]
for i in range(0, len(documents), max_query):
q = {'query': {'ids': {'values': documents[i:i + max_query]}}}
bulk_op(document_updates(q, entity.id, entity.collection_id))
flush_index()
def update_collection(collection):
"""Create or update a collection."""
if collection.deleted_at is not None:
index_delete(collection.id)
return
collection.updated_at = datetime.utcnow()
db.session.add(collection)
db.session.commit()
log.info("Updating: %r", collection)
index_collection(collection)
flush_index()
def delete_collection(collection_id):
"""Delete all documents from a particular collection."""
q = {'query': {'term': {'collection_id': collection_id}}, '_source': False}
def deletes():
for res in scan(es, query=q, index=es_index):
yield {
'_op_type': 'delete',
'_index': six.text_type(es_index),
'_type': res.get('_type'),
'_id': res.get('_id')
}
flush_index()
bulk_op(deletes())
flush_index()
def delete_pending():
"""Deletes any pending entities."""
deleted = 0
entities = db.session.query(Entity.id).filter(
Entity.state == Entity.STATE_PENDING)
to_delete = [
entities,
db.session.query(Reference).filter(Reference.entity_id.in_(entities))
]
for query in to_delete:
deleted += query.delete(synchronize_session='fetch')
log.debug('{} records deleted.'.format(deleted))
db.session.commit()
flush_index()
def delete_collection(collection_id):
"""Delete all documents from a particular collection."""
q = {'query': {'term': {'collection_id': collection_id}}, '_source': False}
def deletes():
types = [TYPE_RECORD, TYPE_DOCUMENT, TYPE_ENTITY]
for res in scan(es, query=q, index=es_index, doc_type=types):
yield {
'_op_type': 'delete',
'_index': six.text_type(es_index),
'_parent': res.get('_parent'),
'_type': res.get('_type'),
'_id': res.get('_id')
}
flush_index()
bulk_op(deletes())
def update_collection(collection, roles=False):
"""Create or update a collection."""
if collection.deleted_at is not None:
index_delete(collection.id)
return
collection.updated_at = datetime.utcnow()
db.session.add(collection)
db.session.commit()
log.info("Updating: %r", collection)
index_collection(collection)
if roles:
update_roles(collection)
if not collection.managed:
xref_collection.apply_async([collection.id], priority=2)
eq = db.session.query(Entity.id)
eq = eq.filter(Entity.collection_id == collection.id)
for entity in eq.all():
update_entity_full.apply_async([entity.id], priority=2)
flush_index()