def sync_es_transfer_status_with_storage_service(uuid, pending_deletion):
"""Update transfer's status in ES indices to match Storage Service.
This is a bit of a kludge that is made necessary by the fact that
the Storage Service does not update ElasticSearch directly when
a package's status has changed.
Updates to ES are visible in Backlog after running a new
search or refreshing the page.
:param uuid: Transfer UUID.
:param pending_deletion: Current pending_deletion value in ES.
:returns: Boolean indicating whether transfer should be kept in
search results (i.e. has not been deleted from Storage Service).
"""
keep_in_results = True
amclient = setup_amclient()
amclient.package_uuid = uuid
api_results = amclient.get_package_details()
if api_results in AMCLIENT_ERROR_CODES:
logger.warning(
"Package {} not found in Storage Service. AMClient error code: {}".format(
uuid, api_results
)
)
return keep_in_results
transfer_status = api_results.get("status")
if not transfer_status:
logger.warning(
"Status for package {} could not be retrived from Storage Service."
)
return keep_in_results
if transfer_status == es.STATUS_DELETE_REQUESTED and pending_deletion is False:
es_client = es.get_client()
es.mark_backlog_deletion_requested(es_client, uuid)
elif transfer_status == es.STATUS_UPLOADED and pending_deletion is True:
es_client = es.get_client()
es.revert_backlog_deletion_request(es_client, uuid)
elif transfer_status == es.STATUS_DELETED:
keep_in_results = False
es_client = es.get_client()
es.remove_backlog_transfer_files(es_client, uuid)
es.remove_backlog_transfer(es_client, uuid)
return keep_in_results
def check_and_remove_deleted_transfers(es_client):
"""
Check the storage service to see if transfers marked in ES as 'pending deletion' have been deleted yet. If so,
remove the transfer and its files from ES. This is a bit of a kludge (that we do elsewhere e.g. in the storage tab),
but it appears necessary as the storage service doesn't talk directly to ES.
:return: None
"""
query = {
'query': {
'bool': {
'must': {
'match': {
'pending_deletion': True
}
}
}
}
}
deletion_pending_results = es_client.search(body=query,
index='transfers',
doc_type='transfer',
fields='uuid,status')
for hit in deletion_pending_results['hits']['hits']:
transfer_uuid = hit['fields']['uuid'][0]
api_results = storage_service.get_file_info(uuid=transfer_uuid)
try:
status = api_results[0]['status']
except IndexError:
logger.info('Transfer not found in storage service: {}'.format(
transfer_uuid))
continue
if status == 'DELETED':
elasticSearchFunctions.remove_backlog_transfer_files(
es_client, transfer_uuid)
elasticSearchFunctions.remove_backlog_transfer(
es_client, transfer_uuid)
def check_and_remove_deleted_transfers(es_client):
"""
Check the storage service to see if transfers marked in ES as 'pending deletion' have been deleted yet. If so,
remove the transfer and its files from ES. This is a bit of a kludge (that we do elsewhere e.g. in the storage tab),
but it appears necessary as the storage service doesn't talk directly to ES.
:return: None
"""
query = {
"query": {
"bool": {
"must": {
"match": {
"pending_deletion": True
}
}
}
}
}
deletion_pending_results = es_client.search(body=query,
index="transfers",
_source="uuid,status")
for hit in deletion_pending_results["hits"]["hits"]:
transfer_uuid = hit["_source"]["uuid"]
api_results = storage_service.get_file_info(uuid=transfer_uuid)
try:
status = api_results[0]["status"]
except IndexError:
logger.info("Transfer not found in storage service: {}".format(
transfer_uuid))
continue
if status == "DELETED":
elasticSearchFunctions.remove_backlog_transfer_files(
es_client, transfer_uuid)
elasticSearchFunctions.remove_backlog_transfer(
es_client, transfer_uuid)