def approve_transfer(dirname, url, am_api_key, am_user):
"""
Approve transfer with dirname.
:returns: UUID of the approved transfer or None.
"""
LOGGER.info("Approving %s", dirname)
time.sleep(6)
am = AMClient(am_url=url, am_user_name=am_user, am_api_key=am_api_key)
try:
# Find the waiting transfers available to be approved via the am client
# interface.
waiting_transfers = am.unapproved_transfers()["results"]
except (KeyError, TypeError):
LOGGER.error(
"Request to unapproved transfers did not return the "
"expected response, see the request log"
)
return None
if not waiting_transfers:
LOGGER.warning("There are no waiting transfers.")
return None
res = list(
filter(
lambda waiting: fsencode(waiting["directory"]) == fsencode(dirname),
waiting_transfers,
)
)
if not res:
LOGGER.warning(
"Requested directory %s not found in the waiting " "transfers list", dirname
)
return None
LOGGER.info("Found waiting transfer: %s", res[0]["directory"])
# We can reuse the existing AM Client but we didn't know all the kwargs
# at the outset so we need to set its attributes here.
am.transfer_type = res[0]["type"]
am.transfer_directory = dirname
# Approve the transfer and return the UUID of the transfer approved.
approved = am.approve_transfer()
if isinstance(approved, int):
if errors.error_lookup(approved) is not None:
LOGGER.error("Error approving transfer: %s", errors.error_lookup(approved))
return None
# Get will return None, or the UUID.
return approved.get("uuid")
def approve_transfer(dirname, url, am_api_key, am_user):
"""
Approve transfer with dirname.
:returns: UUID of the approved transfer or None.
"""
LOGGER.info("Approving %s", dirname)
time.sleep(6)
am = AMClient(am_url=url, am_user_name=am_user, am_api_key=am_api_key)
try:
# Find the waiting transfers available to be approved via the am client
# interface.
waiting_transfers = am.unapproved_transfers()["results"]
except (KeyError, TypeError):
LOGGER.error(
"Request to unapproved transfers did not return the "
"expected response, see the request log"
)
return None
if not waiting_transfers:
LOGGER.warning("There are no waiting transfers.")
return None
res = list(
filter(
lambda waiting: fsencode(waiting["directory"]) == fsencode(dirname),
waiting_transfers,
)
)
if not res:
LOGGER.warning(
"Requested directory %s not found in the waiting " "transfers list", dirname
)
return None
LOGGER.info("Found waiting transfer: %s", res[0]["directory"])
# We can reuse the existing AM Client but we didn't know all the kwargs
# at the outset so we need to set its attributes here.
am.transfer_type = res[0]["type"]
am.transfer_directory = dirname
# Approve the transfer and return the UUID of the transfer approved.
approved = am.approve_transfer()
if isinstance(approved, int):
if errors.error_lookup(approved) is not None:
LOGGER.error("Error approving transfer: %s", errors.error_lookup(approved))
return None
# Get will return None, or the UUID.
return approved.get("uuid")
def test_get_package_details_invalid_uuid(self):
package_uuid = "23129471-baad-f00d-88b6-eb4714afb5ac"
response = amclient.AMClient(
ss_api_key=SS_API_KEY,
ss_user_name=SS_USER_NAME,
ss_url=SS_URL,
package_uuid=package_uuid).get_package_details()
assert errors.error_lookup(response) == \
errors.error_codes[errors.ERR_INVALID_RESPONSE]
def processing_exists(amclient, processing_name):
"""Check to see if a processing configuration exists inside Archivematica
so that it cab be called and used.
"""
amclient.processing_config = processing_name
resp = amclient.get_processing_config()
if errors.error_lookup(resp) is resp:
return True
return False
def processing_exists(amclient, processing_name):
"""Check to see if a processing configuration exists inside Archivematica
so that it cab be called and used.
"""
amclient.processing_config = processing_name
resp = amclient.get_processing_config()
if errors.error_lookup(resp) is resp:
return True
return False
def test_get_next_transfer_bad_source(self):
# Set bad TS Location UUID
ts_location_uuid = 'badd8d39-9cee-495e-b7ee-5e6292549bad'
# Test
path = transfer.get_next_transfer(SS_URL, SS_USER, SS_KEY,
ts_location_uuid, PATH_PREFIX, DEPTH,
COMPLETED, FILES)
# Verify
self.assertEqual(path,
errors.error_lookup(errors.ERR_INVALID_RESPONSE))
def __getattr__(self, name):
if name.startswith('print_'):
try:
method = name.replace('print_', '', 1)
res = getattr(self, method)()
# Shortening variable for PEP8 conformance.
err_lookup = errors.error_lookup
if isinstance(res, int):
self.stdout(
err_lookup.get(res,
err_lookup(errors.ERR_CLIENT_UNKNOWN)))
else:
self.stdout(res)
except requests.exceptions.InvalidURL:
self.stdout(errors.error_lookup(errors.ERR_INVALID_URL))
except BaseException:
self.stdout(errors.error_lookup(errors.ERR_CLIENT_UNKNOWN))
else:
raise AttributeError('AMClient has no method {0}'.format(name))
def test_get_ingest_status_invalid_uuid(self):
"""Test the response from the server for a request to find the status
of an ingest uuid that doesn't exist.
"""
response = amclient.AMClient(am_api_key=AM_API_KEY,
am_user_name=AM_USER_NAME,
am_url=AM_URL,
sip_uuid="63fcc1b0-f83d-47e6-"
"ac9d-a8f8d1fc2ab9").get_ingest_status()
assert errors.error_lookup(response) == \
errors.error_codes[errors.ERR_INVALID_RESPONSE]
def test_get_next_transfer_failed_auth(self):
# All default values
ss_user = '******'
ss_key = 'dne'
# Test
path = transfer.get_next_transfer(SS_URL, ss_user, ss_key,
TS_LOCATION_UUID, PATH_PREFIX, DEPTH,
COMPLETED, FILES)
# Verify
self.assertEqual(path,
errors.error_lookup(errors.ERR_INVALID_RESPONSE))
def test_approve_non_existing_transfer(self):
"""If a transfer isn't available for us to approve, test the response
from AMClient.py. The respons is a 404 and this is handled
specifically by utils.py and the return is an error code.
"""
response = amclient.AMClient(
am_api_key=AM_API_KEY,
am_user_name=AM_USER_NAME,
am_url=AM_URL,
transfer_directory="approve_2",
transfer_type="standard").approve_transfer()
assert errors.error_lookup(response) == \
errors.error_codes[errors.ERR_INVALID_RESPONSE]
def test_reingest_non_aip(self):
pipeline_uuid = 'bb033eff-131e-48d5-980f-c4edab0cb038'
aip_uuid = 'bb033eff-131e-48d5-980f-c4edab0cb038'
response = amclient.AMClient(
ss_api_key=SS_API_KEY,
ss_user_name=SS_USER_NAME,
ss_url=SS_URL,
pipeline_uuid=pipeline_uuid,
aip_uuid=aip_uuid,
reingest_type="standard",
processing_config="default").reingest_aip()
assert errors.error_lookup(response) == \
errors.error_codes[errors.ERR_INVALID_RESPONSE]
def test_get_non_existing_processing_config(self):
"""Test retrieval of a Processing MCP Config file that does not exist
in the Archivematica instance. Archivematica returns a 404 error and
a HTML result. This test is volatile to both changes in AM's handling
of this request failure in future, and changes to the error handling
in AMClient.py.
"""
response = amclient.AMClient(
am_api_key=AM_API_KEY,
am_user_name=AM_USER_NAME,
am_url=AM_URL,
processing_config="badf00d").get_processing_config()
assert errors.error_lookup(response) == \
errors.error_codes[errors.ERR_INVALID_RESPONSE]
def test_get_status_no_unit(self):
transfer_uuid = 'deadc0de-c0de-c0de-c0de-deadc0dec0de'
info = transfer.get_status(AM_URL, USER, API_KEY, SS_URL, SS_USER,
SS_KEY, transfer_uuid, 'transfer', session)
self.assertEqual(info,
errors.error_lookup(errors.ERR_INVALID_RESPONSE))
def get_status(
am_url,
am_user,
am_api_key,
ss_url,
ss_user,
ss_api_key,
unit_uuid,
unit_type,
hide_on_complete=False,
delete_on_complete=False,
):
"""
Get status of the SIP or Transfer with unit_uuid.
:param str unit_uuid: UUID of the unit to query for.
:param str unit_type: 'ingest' or 'transfer'
:param bool hide_on_complete: Hide the unit in the dashboard if COMPLETE
:returns: Dict with status of the unit from Archivematica or None.
"""
# Get status
url = "{}/api/{}/status/{}/".format(am_url, unit_type, unit_uuid)
params = {"username": am_user, "api_key": am_api_key}
unit_info = utils._call_url_json(url, params)
if isinstance(unit_info, int):
if errors.error_lookup(unit_info) is not None:
return errors.error_lookup(unit_info)
# If complete, hide in dashboard
if hide_on_complete and unit_info and unit_info.get("status") == "COMPLETE":
LOGGER.info("Hiding %s %s in dashboard", unit_type, unit_uuid)
url = "{}/api/{}/{}/delete/".format(am_url, unit_type, unit_uuid)
LOGGER.debug("Method: DELETE; URL: %s; params: %s;", url, params)
response = requests.delete(url, params=params)
LOGGER.debug("Response: %s", response)
# If Transfer is complete, get the SIP's status
if (
unit_info
and unit_type == "transfer"
and unit_info.get("status") == "COMPLETE"
and unit_info.get("sip_uuid") != "BACKLOG"
):
LOGGER.info(
"%s is a complete transfer, fetching SIP %s status.",
unit_uuid,
unit_info.get("sip_uuid"),
)
# Update DB to refer to this one
unit = models.retrieve_unit_by_type_and_uuid(
uuid=unit_uuid, unit_type=unit_type
)
models.update_unit_type_and_uuid(
unit=unit, unit_type="ingest", uuid=unit_info.get("sip_uuid")
)
# Get SIP status
url = "{}/api/ingest/status/{}/".format(am_url, unit_info.get("sip_uuid"))
unit_info = utils._call_url_json(url, params)
if isinstance(unit_info, int):
if errors.error_lookup(unit_info) is not None:
return errors.error_lookup(unit_info)
# If complete, hide in dashboard
if hide_on_complete and unit_info and unit_info.get("status") == "COMPLETE":
LOGGER.info("Hiding SIP %s in dashboard", unit.uuid)
url = "{}/api/ingest/{}/delete/".format(am_url, unit.uuid)
LOGGER.debug("Method: DELETE; URL: %s; params: %s;", url, params)
response = requests.delete(url, params=params)
LOGGER.debug("Response: %s", response)
# If complete and SIP status is 'UPLOADED', delete transfer source
# files
if delete_on_complete and unit_info and unit_info.get("status") == "COMPLETE":
am = AMClient(
ss_url=ss_url,
ss_user_name=ss_user,
ss_api_key=ss_api_key,
package_uuid=unit.uuid,
)
response = am.get_package_details()
if response.get("status") == "UPLOADED":
LOGGER.info(
"Deleting source files for SIP %s from watched " "directory",
unit.uuid,
)
try:
shutil.rmtree(unit.path)
LOGGER.info("Source files deleted for SIP %s " "deleted", unit.uuid)
except OSError as e:
LOGGER.warning(
"Error deleting source files: %s. If "
"running this module remotely the "
"script might not have access to the "
"transfer source",
e,
)
return unit_info
def test_get_status_no_unit(self):
transfer_uuid = "deadc0de-c0de-c0de-c0de-deadc0dec0de"
info = transfer.get_status(
AM_URL, USER, API_KEY, SS_URL, SS_USER, SS_KEY, transfer_uuid, "transfer"
)
self.assertEqual(info, errors.error_lookup(errors.ERR_INVALID_RESPONSE))
def test_get_status_not_json(self):
transfer_uuid = 'dfc8cf5f-b5b1-408c-88b1-34215964e9d6'
info = transfer.get_status(AM_URL, USER, API_KEY, SS_URL, SS_USER,
SS_KEY, transfer_uuid, 'transfer', session)
self.assertEqual(info,
errors.error_lookup(errors.ERR_INVALID_RESPONSE))
def get_status(
am_url,
am_user,
am_api_key,
ss_url,
ss_user,
ss_api_key,
unit_uuid,
unit_type,
hide_on_complete=False,
delete_on_complete=False,
):
"""
Get status of the SIP or Transfer with unit_uuid.
:param str unit_uuid: UUID of the unit to query for.
:param str unit_type: 'ingest' or 'transfer'
:param bool hide_on_complete: Hide the unit in the dashboard if COMPLETE
:returns: Dict with status of the unit from Archivematica or None.
"""
# Get status
url = "{}/api/{}/status/{}/".format(am_url, unit_type, unit_uuid)
params = {"username": am_user, "api_key": am_api_key}
unit_info = utils._call_url_json(url, params)
if isinstance(unit_info, int):
if errors.error_lookup(unit_info) is not None:
return errors.error_lookup(unit_info)
# If complete, hide in dashboard
if hide_on_complete and unit_info and unit_info.get(
"status") == "COMPLETE":
LOGGER.info("Hiding %s %s in dashboard", unit_type, unit_uuid)
url = "{}/api/{}/{}/delete/".format(am_url, unit_type, unit_uuid)
LOGGER.debug("Method: DELETE; URL: %s; params: %s;", url, params)
response = requests.delete(url, params=params)
LOGGER.debug("Response: %s", response)
# If Transfer is complete, get the SIP's status
if (unit_info and unit_type == "transfer"
and unit_info.get("status") == "COMPLETE"
and unit_info.get("sip_uuid") != "BACKLOG"):
LOGGER.info(
"%s is a complete transfer, fetching SIP %s status.",
unit_uuid,
unit_info.get("sip_uuid"),
)
# Update DB to refer to this one
unit = models.retrieve_unit_by_type_and_uuid(uuid=unit_uuid,
unit_type=unit_type)
models.update_unit_type_and_uuid(unit=unit,
unit_type="ingest",
uuid=unit_info.get("sip_uuid"))
# Get SIP status
url = "{}/api/ingest/status/{}/".format(am_url,
unit_info.get("sip_uuid"))
unit_info = utils._call_url_json(url, params)
if isinstance(unit_info, int):
if errors.error_lookup(unit_info) is not None:
return errors.error_lookup(unit_info)
# If complete, hide in dashboard
if hide_on_complete and unit_info and unit_info.get(
"status") == "COMPLETE":
LOGGER.info("Hiding SIP %s in dashboard", unit.uuid)
url = "{}/api/ingest/{}/delete/".format(am_url, unit.uuid)
LOGGER.debug("Method: DELETE; URL: %s; params: %s;", url, params)
response = requests.delete(url, params=params)
LOGGER.debug("Response: %s", response)
# If complete and SIP status is 'UPLOADED', delete transfer source
# files
if delete_on_complete and unit_info and unit_info.get(
"status") == "COMPLETE":
am = AMClient(
ss_url=ss_url,
ss_user_name=ss_user,
ss_api_key=ss_api_key,
package_uuid=unit.uuid,
)
response = am.get_package_details()
if response.get("status") == "UPLOADED":
LOGGER.info(
"Deleting source files for SIP %s from watched "
"directory",
unit.uuid,
)
try:
shutil.rmtree(unit.path)
LOGGER.info("Source files deleted for SIP %s "
"deleted", unit.uuid)
except OSError as e:
LOGGER.warning(
"Error deleting source files: %s. If "
"running this module remotely the "
"script might not have access to the "
"transfer source",
e,
)
return unit_info
def get_next_transfer(
ss_url,
ss_user,
ss_api_key,
ts_location_uuid,
path_prefix,
depth,
processed,
see_files,
):
"""
Helper to find the first directory that doesn't have an associated
transfer.
:param ss_url: URL of the Storage Service to query
:param ss_user: User on the Storage Service for authentication
:param ss_api_key: API key for user on the Storage Service for
authentication
:param ts_location_uuid: UUID of the transfer source Location
:param path_prefix: Relative path inside the Location to work with.
:param depth: Depth relative to path_prefix to create a transfer
from. Should be 1 or greater.
:param set processed: Set of the paths of processed by the automation
tools in the database. Ideally, relative to the
same transfer source location, including the same
path_prefix, and at the same depth. Paths include
those currently processing and completed.
:param bool see_files: Return files as well as folders to become
transfers.
:returns: Path relative to TS Location of the new transfer.
"""
# Get sorted list from source directory.
url = ss_url + "/api/v2/location/" + ts_location_uuid + "/browse/"
params = {"username": ss_user, "api_key": ss_api_key}
if path_prefix:
params["path"] = base64.b64encode(path_prefix)
browse_info = utils._call_url_json(url, params)
if isinstance(browse_info, int):
if errors.error_lookup(browse_info) is not None:
LOGGER.error("Error when browsing location: %s",
errors.error_lookup(browse_info))
return None
if browse_info is None:
return None
if see_files:
entries = browse_info["entries"]
else:
entries = browse_info["directories"]
entries = [base64.b64decode(e.encode("utf8")) for e in entries]
LOGGER.debug("Entries: %s", entries)
LOGGER.info("Total files or folders in transfer source location: %s",
len(entries))
entries = [os.path.join(path_prefix, e) for e in entries]
# If at the correct depth, check if any of these have not been made into
# transfers yet
if depth <= 1:
# Find the directories that are not already in the DB using sets
entries = set(entries) - processed
LOGGER.debug("New transfer candidates: %s", entries)
LOGGER.info("Unprocessed entries to choose from: %s", len(entries))
# Sort, take the first
entries = sorted(list(entries))
if not entries:
LOGGER.info("All potential transfers in %s have been created.",
path_prefix)
return None
target = entries[0]
return target
else: # if depth > 1
# Recurse on each directory
for entry in entries:
LOGGER.debug("New path: %s", entry)
target = get_next_transfer(
ss_url=ss_url,
ss_user=ss_user,
ss_api_key=ss_api_key,
ts_location_uuid=ts_location_uuid,
path_prefix=entry,
depth=depth - 1,
processed=processed,
see_files=see_files,
)
if target:
return target
return None
def reingest_full_and_approve(amclient, pipeline_uuid, aip_uuid,
processing_config="default", latency=None,
approval_retries=2):
"""Reingest an archivematica AIP.
This function will make three calls to the AM Client code:
1) To initiate the reingest.
2) To monitor the transfer status.
3) To approve it in the transfer workflow.
A latency argument makes up for the endpoint being neither particularly
synchronous or asynchronous. We don't want to mindlessly poll
the server to check that the AIP we want to reingest is in the pipeline.
"""
# Initialize an AIPs reingest.
amclient.pipeline_uuid = pipeline_uuid
amclient.aip_uuid = aip_uuid
amclient.processing_config = processing_config
reingest_aip = amclient.reingest_aip()
# If we successfully initialize the reingest we get a dict back that
# we can work with to monitor state. If we do not get a dict back we need
# to look at what went wrong.
if not isinstance(reingest_aip, dict):
LOGGER.error("Reingest failed with server error %s, returning.",
errors.error_lookup(reingest_aip))
return False, "Error calling reingest_aip."
reingest_uuid = reingest_aip["reingest_uuid"]
LOGGER.info("Reingest UUID to work with %s", reingest_uuid)
LOGGER.info("Checking status of %s once in transfer queue", reingest_uuid)
# We need to make sure that the approval is synchronized with the request
# to reingest. Occasionally a reingest will get stopped at approval when
# it doesn't need to if we just wait a little longer or try again.
for _ in range(approval_retries):
transfer = None
while not isinstance(transfer, dict):
if latency:
time.sleep(latency) # ~latency between call and AM actioning.
amclient.transfer_uuid = reingest_uuid
transfer = amclient.get_transfer_status()
LOGGER.info("Attempting to approve transfer following the "
"initialization of reingest.")
if transfer.get("status") == "USER_INPUT":
transfer_directory = transfer["directory"]
LOGGER.info("Approving reingest automatically. Directory to "
"approve %s", transfer_directory)
amclient.transfer_directory = transfer_directory
message = amclient.approve_transfer()
if message.get('error') is None:
LOGGER.info("Approval successful, returning True")
return True, message['uuid']
return False, "Error approving transfer."
return False, "Error retrieving transfer status."
def test_get_status_not_json(self):
transfer_uuid = "dfc8cf5f-b5b1-408c-88b1-34215964e9d6"
info = transfer.get_status(
AM_URL, USER, API_KEY, SS_URL, SS_USER, SS_KEY, transfer_uuid, "transfer"
)
self.assertEqual(info, errors.error_lookup(errors.ERR_INVALID_RESPONSE))
def get_next_transfer(
ss_url,
ss_user,
ss_api_key,
ts_location_uuid,
path_prefix,
depth,
processed,
see_files,
):
"""
Helper to find the first directory that doesn't have an associated
transfer.
:param ss_url: URL of the Storage Service to query
:param ss_user: User on the Storage Service for authentication
:param ss_api_key: API key for user on the Storage Service for
authentication
:param ts_location_uuid: UUID of the transfer source Location
:param path_prefix: Relative path inside the Location to work with.
:param depth: Depth relative to path_prefix to create a transfer
from. Should be 1 or greater.
:param set processed: Set of the paths of processed by the automation
tools in the database. Ideally, relative to the
same transfer source location, including the same
path_prefix, and at the same depth. Paths include
those currently processing and completed.
:param bool see_files: Return files as well as folders to become
transfers.
:returns: Path relative to TS Location of the new transfer.
"""
# Get sorted list from source directory.
url = ss_url + "/api/v2/location/" + ts_location_uuid + "/browse/"
params = {"username": ss_user, "api_key": ss_api_key}
if path_prefix:
params["path"] = base64.b64encode(path_prefix)
browse_info = utils._call_url_json(url, params)
if isinstance(browse_info, int):
if errors.error_lookup(browse_info) is not None:
LOGGER.error(
"Error when browsing location: %s", errors.error_lookup(browse_info)
)
return None
if browse_info is None:
return None
if see_files:
entries = browse_info["entries"]
else:
entries = browse_info["directories"]
entries = [base64.b64decode(e.encode("utf8")) for e in entries]
LOGGER.debug("Entries: %s", entries)
LOGGER.info("Total files or folders in transfer source location: %s", len(entries))
entries = [os.path.join(path_prefix, e) for e in entries]
# If at the correct depth, check if any of these have not been made into
# transfers yet
if depth <= 1:
# Find the directories that are not already in the DB using sets
entries = set(entries) - processed
LOGGER.debug("New transfer candidates: %s", entries)
LOGGER.info("Unprocessed entries to choose from: %s", len(entries))
# Sort, take the first
entries = sorted(list(entries))
if not entries:
LOGGER.info("All potential transfers in %s have been created.", path_prefix)
return None
target = entries[0]
return target
else: # if depth > 1
# Recurse on each directory
for entry in entries:
LOGGER.debug("New path: %s", entry)
target = get_next_transfer(
ss_url=ss_url,
ss_user=ss_user,
ss_api_key=ss_api_key,
ts_location_uuid=ts_location_uuid,
path_prefix=entry,
depth=depth - 1,
processed=processed,
see_files=see_files,
)
if target:
return target
return None
def reingest_full_and_approve(
amclient,
pipeline_uuid,
aip_uuid,
processing_config="default",
latency=None,
approval_retries=2,
):
"""Reingest an archivematica AIP.
This function will make three calls to the AM Client code:
1) To initiate the reingest.
2) To monitor the transfer status.
3) To approve it in the transfer workflow.
A latency argument makes up for the endpoint being neither particularly
synchronous or asynchronous. We don't want to mindlessly poll
the server to check that the AIP we want to reingest is in the pipeline.
"""
# Initialize an AIPs reingest.
amclient.pipeline_uuid = pipeline_uuid
amclient.aip_uuid = aip_uuid
amclient.processing_config = processing_config
reingest_aip = amclient.reingest_aip()
# If we successfully initialize the reingest we get a dict back that
# we can work with to monitor state. If we do not get a dict back we need
# to look at what went wrong.
if not isinstance(reingest_aip, dict):
LOGGER.error(
"Reingest failed with server error %s, returning.",
errors.error_lookup(reingest_aip),
)
return False, "Error calling reingest_aip."
reingest_uuid = reingest_aip["reingest_uuid"]
LOGGER.info("Reingest UUID to work with %s", reingest_uuid)
LOGGER.info("Checking status of %s once in transfer queue", reingest_uuid)
# We need to make sure that the approval is synchronized with the request
# to reingest. Occasionally a reingest will get stopped at approval when
# it doesn't need to if we just wait a little longer or try again.
for _ in range(approval_retries):
transfer = None
while not isinstance(transfer, dict):
if latency:
time.sleep(latency) # ~latency between call and AM actioning.
amclient.transfer_uuid = reingest_uuid
transfer = amclient.get_transfer_status()
LOGGER.info(
"Attempting to approve transfer following the "
"initialization of reingest."
)
if transfer.get("status") == "USER_INPUT":
transfer_directory = transfer["directory"]
LOGGER.info(
"Approving reingest automatically. Directory to " "approve %s",
transfer_directory,
)
amclient.transfer_directory = transfer_directory
message = amclient.approve_transfer()
if message.get("error") is None:
LOGGER.info("Approval successful, returning True")
return True, message["uuid"]
return False, "Error approving transfer."
return False, "Error retrieving transfer status."
def get_status(am_url,
am_user,
am_api_key,
ss_url,
ss_user,
ss_api_key,
unit_uuid,
unit_type,
hide_on_complete=False,
delete_on_complete=False):
"""
Get status of the SIP or Transfer with unit_uuid.
:param str unit_uuid: UUID of the unit to query for.
:param str unit_type: 'ingest' or 'transfer'
:param bool hide_on_complete: Hide the unit in the dashboard if COMPLETE
:returns: Dict with status of the unit from Archivematica or None.
"""
# Get status
url = "{}/api/{}/status/{}/".format(am_url, unit_type, unit_uuid)
params = {'username': am_user, 'api_key': am_api_key}
unit_info = utils._call_url_json(url, params)
if isinstance(unit_info, int):
if errors.error_lookup(unit_info) is not None:
return errors.error_lookup(unit_info)
# If complete, hide in dashboard
if hide_on_complete and unit_info and \
unit_info.get('status') == 'COMPLETE':
LOGGER.info('Hiding %s %s in dashboard', unit_type, unit_uuid)
url = "{}/api/{}/{}/delete/".format(am_url, unit_type, unit_uuid)
LOGGER.debug('Method: DELETE; URL: %s; params: %s;', url, params)
response = requests.delete(url, params=params)
LOGGER.debug('Response: %s', response)
# If Transfer is complete, get the SIP's status
if unit_info and unit_type == 'transfer' and \
unit_info.get('status') == 'COMPLETE' and \
unit_info.get('sip_uuid') != 'BACKLOG':
LOGGER.info('%s is a complete transfer, fetching SIP %s status.',
unit_uuid, unit_info.get('sip_uuid'))
# Update DB to refer to this one
unit = models.retrieve_unit_by_type_and_uuid(uuid=unit_uuid,
unit_type=unit_type)
models.update_unit_type_and_uuid(unit=unit,
unit_type="ingest",
uuid=unit_info.get('sip_uuid'))
# Get SIP status
url = "{}/api/ingest/status/{}/".\
format(am_url, unit_info.get('sip_uuid'))
unit_info = utils._call_url_json(url, params)
if isinstance(unit_info, int):
if errors.error_lookup(unit_info) is not None:
return errors.error_lookup(unit_info)
# If complete, hide in dashboard
if hide_on_complete and unit_info and \
unit_info.get('status') == 'COMPLETE':
LOGGER.info('Hiding SIP %s in dashboard', unit.uuid)
url = "{}/api/ingest/{}/delete/".format(am_url, unit.uuid)
LOGGER.debug('Method: DELETE; URL: %s; params: %s;', url, params)
response = requests.delete(url, params=params)
LOGGER.debug('Response: %s', response)
# If complete and SIP status is 'UPLOADED', delete transfer source
# files
if delete_on_complete and unit_info and \
unit_info.get('status') == 'COMPLETE':
am = AMClient(ss_url=ss_url,
ss_user_name=ss_user,
ss_api_key=ss_api_key,
package_uuid=unit.uuid)
response = am.get_package_details()
if response.get('status') == 'UPLOADED':
LOGGER.info(
'Deleting source files for SIP %s from watched '
'directory', unit.uuid)
try:
shutil.rmtree(unit.path)
LOGGER.info('Source files deleted for SIP %s '
'deleted', unit.uuid)
except OSError as e:
LOGGER.warning(
'Error deleting source files: %s. If '
'running this module remotely the '
'script might not have access to the '
'transfer source', e)
return unit_info
