def sync_es_transfer_status_with_storage_service(uuid, pending_deletion): """Update transfer's status in ES indices to match Storage Service. This is a bit of a kludge that is made necessary by the fact that the Storage Service does not update ElasticSearch directly when a package's status has changed. Updates to ES are visible in Backlog after running a new search or refreshing the page. :param uuid: Transfer UUID. :param pending_deletion: Current pending_deletion value in ES. :returns: Boolean indicating whether transfer should be kept in search results (i.e. has not been deleted from Storage Service). """ keep_in_results = True amclient = setup_amclient() amclient.package_uuid = uuid api_results = amclient.get_package_details() if api_results in AMCLIENT_ERROR_CODES: logger.warning( "Package {} not found in Storage Service. AMClient error code: {}".format( uuid, api_results ) ) return keep_in_results transfer_status = api_results.get("status") if not transfer_status: logger.warning( "Status for package {} could not be retrived from Storage Service." ) return keep_in_results if transfer_status == es.STATUS_DELETE_REQUESTED and pending_deletion is False: es_client = es.get_client() es.mark_backlog_deletion_requested(es_client, uuid) elif transfer_status == es.STATUS_UPLOADED and pending_deletion is True: es_client = es.get_client() es.revert_backlog_deletion_request(es_client, uuid) elif transfer_status == es.STATUS_DELETED: keep_in_results = False es_client = es.get_client() es.remove_backlog_transfer_files(es_client, uuid) es.remove_backlog_transfer(es_client, uuid) return keep_in_results
def check_and_remove_deleted_transfers(es_client): """ Check the storage service to see if transfers marked in ES as 'pending deletion' have been deleted yet. If so, remove the transfer and its files from ES. This is a bit of a kludge (that we do elsewhere e.g. in the storage tab), but it appears necessary as the storage service doesn't talk directly to ES. :return: None """ query = { 'query': { 'bool': { 'must': { 'match': { 'pending_deletion': True } } } } } deletion_pending_results = es_client.search(body=query, index='transfers', doc_type='transfer', fields='uuid,status') for hit in deletion_pending_results['hits']['hits']: transfer_uuid = hit['fields']['uuid'][0] api_results = storage_service.get_file_info(uuid=transfer_uuid) try: status = api_results[0]['status'] except IndexError: logger.info('Transfer not found in storage service: {}'.format( transfer_uuid)) continue if status == 'DELETED': elasticSearchFunctions.remove_backlog_transfer_files( es_client, transfer_uuid) elasticSearchFunctions.remove_backlog_transfer( es_client, transfer_uuid)
def check_and_remove_deleted_transfers(es_client): """ Check the storage service to see if transfers marked in ES as 'pending deletion' have been deleted yet. If so, remove the transfer and its files from ES. This is a bit of a kludge (that we do elsewhere e.g. in the storage tab), but it appears necessary as the storage service doesn't talk directly to ES. :return: None """ query = { "query": { "bool": { "must": { "match": { "pending_deletion": True } } } } } deletion_pending_results = es_client.search(body=query, index="transfers", _source="uuid,status") for hit in deletion_pending_results["hits"]["hits"]: transfer_uuid = hit["_source"]["uuid"] api_results = storage_service.get_file_info(uuid=transfer_uuid) try: status = api_results[0]["status"] except IndexError: logger.info("Transfer not found in storage service: {}".format( transfer_uuid)) continue if status == "DELETED": elasticSearchFunctions.remove_backlog_transfer_files( es_client, transfer_uuid) elasticSearchFunctions.remove_backlog_transfer( es_client, transfer_uuid)