def delete_collection(collection_id, wait=True):
"""Delete all documents from a particular collection."""
query = {'term': {'collection_id': collection_id}}
query_delete(records_index(), query, wait=wait)
query_delete(entities_index(), query, wait=wait)
es.delete(index=collections_index(),
doc_type=collection_type(),
id=collection_id,
ignore=[404])
def delete_documents(collection_id):
"""Delete documents from a collection."""
records_query = {'term': {'collection_id': collection_id}}
query_delete(records_index(), records_query)
refresh_index(index=records_index())
query = {
'bool': {
'must': [{
'term': {
'schemata': 'Document'
}
}, {
'term': {
'collection_id': collection_id
}
}]
}
}
query_delete(entities_index(), query)
def iter_records(document_id=None, collection_id=None):
"""Scan all records matching the given criteria."""
filters = []
if document_id is not None:
filters.append({'term': {'document_id': document_id}})
if collection_id is not None:
filters.append({'term': {'collection_id': collection_id}})
query = {'query': {'bool': {'filter': filters}}, 'sort': ['_doc']}
for res in scan(es, index=records_index(), query=query, scroll='1410m'):
yield unpack_result(res)
def index_records(document):
if not document.supports_records:
return
clear_records(document.id, refresh=False)
while True:
try:
bulk_op(generate_records(document))
refresh_index(index=records_index())
return
except BulkIndexError as exc:
log.exception(exc)
time.sleep(RETRY_DELAY)
def index_records(document):
if not document.supports_records:
return
clear_records(document.id)
for attempt in count():
try:
bulk_op(generate_records(document))
refresh_index(records_index())
return
except Exception as exc:
log.warning('Failed to index records: %s', exc)
backoff_cluster(failures=attempt)
def delete_documents(collection_id, wait=True):
"""Delete documents from a collection."""
query = {
'bool': {
'must': [{
'term': {
'schemata': 'Document'
}
}, {
'term': {
'collection_id': collection_id
}
}]
}
}
query_delete(entities_index(), query, wait=wait)
records_query = {'term': {'collection_id': collection_id}}
query_delete(records_index(), records_query, wait=wait)
def get_index(self):
return records_index()
def clear_records(document_id):
"""Delete all records associated with the given document."""
q = {'term': {'document_id': document_id}}
query_delete(records_index(), q)
def delete_document(document_id):
clear_records(document_id)
delete_entity(document_id)
refresh_index(index=records_index())
def clear_records(document_id, refresh=True):
"""Delete all records associated with the given document."""
q = {'term': {'document_id': document_id}}
query_delete(records_index(), q)
if refresh:
refresh_index(index=records_index())
def delete_records(document_id, sync=False):
"""Delete all records associated with the given document."""
q = {'term': {'document_id': document_id}}
query_delete(records_index(), q, wait_for_completion=sync, refresh=sync)
