def test_start_Transfer_unit_state(setup_session):
"""Test functions associated with setting unit state at the beginning of a
new automated transfer.
"""
transfer_uuid_one = str(uuid4())
models.add_new_transfer(uuid=transfer_uuid_one, path=b"/foo")
unit = models.transfer_session.query(models.Unit).filter_by(path=b"/foo").one()
models.transfer_failed_to_start(path=b"/bar")
unit = models.transfer_session.query(models.Unit).filter_by(path=b"/bar").one()
models.failed_to_approve(b"/foobar")
assert unit.current is False
assert unit.uuid == ""
assert unit.status == "FAILED"
unit = models.transfer_session.query(models.Unit).filter_by(path=b"/foobar").one()
assert unit.current is False
assert unit.uuid is None
def test_start_Transfer_unit_state(setup_session):
"""Test functions associated with setting unit state at the beginning of a
new automated transfer.
"""
transfer_uuid_one = str(uuid4())
models.add_new_transfer(uuid=transfer_uuid_one, path=b"/foo")
unit = models.transfer_session.query(models.Unit).\
filter_by(path=b"/foo").one()
models.transfer_failed_to_start(path=b"/bar")
unit = models.transfer_session.query(models.Unit).\
filter_by(path=b"/bar").one()
models.failed_to_approve(b"/foobar")
assert unit.current is False
assert unit.uuid == ""
assert unit.status == "FAILED"
unit = models.transfer_session.query(models.Unit).\
filter_by(path=b"/foobar").one()
assert unit.current is False
assert unit.uuid is None
def start_transfer(
ss_url,
ss_user,
ss_api_key,
ts_location_uuid,
ts_path,
depth,
am_url,
am_user,
am_api_key,
transfer_type,
see_files,
config_file,
):
"""
Starts a new transfer.
:param ss_url: URL of the Storage Service to query
:param ss_user: User on the Storage Service for authentication
:param ss_api_key: API key for user on the Storage Service for
authentication
:param ts_location_uuid: UUID of the transfer source Location
:param ts_path: Relative path inside the Location to work with.
:param depth: Depth relative to ts_path to create a transfer from. Should
be 1 or greater.
:param am_url: URL of Archivematica pipeline to start transfer on
:param am_user: User on Archivematica for authentication
:param am_api_key: API key for user on Archivematica for authentication
:param bool see_files: If true, start transfers from files as well as
directories
:param session: SQLAlchemy session with the DB
:returns: Tuple of Transfer information about the new transfer or None on
error.
"""
# Retrieve the next transfer to process.
processed = models.get_processed_transfer_paths()
target = get_next_transfer(
ss_url=ss_url,
ss_user=ss_user,
ss_api_key=ss_api_key,
ts_location_uuid=ts_location_uuid,
path_prefix=ts_path,
depth=depth,
processed=processed,
see_files=see_files,
)
if not target:
# Report the location UUID.
LOGGER.info(
"All potential transfers in Location ID: %s have been created. "
"Exiting",
ts_location_uuid,
)
return None
LOGGER.info("Starting with %s", target)
# Get accession ID
accession = get_accession_id(target)
LOGGER.info("Accession ID: %s", accession)
# Call the start transfer endpoint.
# Retrieve the directory name for Archivematica.
transfer_name, transfer_abs_path = call_start_transfer_endpoint(
am_url=am_url,
am_user=am_user,
am_api_key=am_api_key,
target=target,
transfer_type=transfer_type,
accession=accession,
ts_location_uuid=ts_location_uuid,
)
if not transfer_name:
LOGGER.info("Cannot begin transfer with target name: %s", target)
models.transfer_failed_to_start(target)
return None
# Run all pre-transfer scripts on the unapproved transfer directory.
LOGGER.info("Attempting to run pre-transfer scripts on: %s", transfer_name)
try:
run_pre_transfer_scripts(
config_file=config_file,
transfer_path=transfer_abs_path,
transfer_type=transfer_type,
)
except OSError as err:
LOGGER.error("Failed to run pre-transfer scripts: %s", err)
return None
# Approve transfer.
LOGGER.info("Ready to approve transfer")
retry_count = 3
for i in range(retry_count):
result = approve_transfer(transfer_name, am_url, am_api_key, am_user)
# Mark as started
if result:
LOGGER.info("Approved %s", result)
# Store the absolute path to help users to determine what type
# the transfer is, and where something it is.
new_transfer = models.add_new_transfer(uuid=result, path=target)
LOGGER.info("New transfer: %s", new_transfer)
break
LOGGER.info("Failed transfer approval, try %s of %s", i + 1,
retry_count)
else:
new_transfer = models.failed_to_approve(path=target)
LOGGER.warning("Transfer not approved: %s", transfer_name)
return None
# Start transfer completed successfully.
LOGGER.info("Finished %s", target)
return new_transfer
def start_transfer(
ss_url,
ss_user,
ss_api_key,
ts_location_uuid,
ts_path,
depth,
am_url,
am_user,
am_api_key,
transfer_type,
see_files,
config_file,
):
"""
Starts a new transfer.
:param ss_url: URL of the Storage Service to query
:param ss_user: User on the Storage Service for authentication
:param ss_api_key: API key for user on the Storage Service for
authentication
:param ts_location_uuid: UUID of the transfer source Location
:param ts_path: Relative path inside the Location to work with.
:param depth: Depth relative to ts_path to create a transfer from. Should
be 1 or greater.
:param am_url: URL of Archivematica pipeline to start transfer on
:param am_user: User on Archivematica for authentication
:param am_api_key: API key for user on Archivematica for authentication
:param bool see_files: If true, start transfers from files as well as
directories
:param session: SQLAlchemy session with the DB
:returns: Tuple of Transfer information about the new transfer or None on
error.
"""
# Retrieve the next transfer to process.
processed = models.get_processed_transfer_paths()
target = get_next_transfer(
ss_url=ss_url,
ss_user=ss_user,
ss_api_key=ss_api_key,
ts_location_uuid=ts_location_uuid,
path_prefix=ts_path,
depth=depth,
processed=processed,
see_files=see_files,
)
if not target:
# Report the location UUID.
LOGGER.info(
"All potential transfers in Location ID: %s have been created. " "Exiting",
ts_location_uuid,
)
return None
LOGGER.info("Starting with %s", target)
# Get accession ID
accession = get_accession_id(target)
LOGGER.info("Accession ID: %s", accession)
# Call the start transfer endpoint.
# Retrieve the directory name for Archivematica.
transfer_name, transfer_abs_path = call_start_transfer_endpoint(
am_url=am_url,
am_user=am_user,
am_api_key=am_api_key,
target=target,
transfer_type=transfer_type,
accession=accession,
ts_location_uuid=ts_location_uuid,
)
if not transfer_name:
LOGGER.info("Cannot begin transfer with target name: %s", target)
models.transfer_failed_to_start(target)
return None
# Run all pre-transfer scripts on the unapproved transfer directory.
LOGGER.info("Attempting to run pre-transfer scripts on: %s", transfer_name)
try:
run_pre_transfer_scripts(
config_file=config_file,
transfer_path=transfer_abs_path,
transfer_type=transfer_type,
)
except OSError as err:
LOGGER.error("Failed to run pre-transfer scripts: %s", err)
return None
# Approve transfer.
LOGGER.info("Ready to approve transfer")
retry_count = 3
for i in range(retry_count):
result = approve_transfer(transfer_name, am_url, am_api_key, am_user)
# Mark as started
if result:
LOGGER.info("Approved %s", result)
# Store the absolute path to help users to determine what type
# the transfer is, and where something it is.
new_transfer = models.add_new_transfer(uuid=result, path=target)
LOGGER.info("New transfer: %s", new_transfer)
break
LOGGER.info("Failed transfer approval, try %s of %s", i + 1, retry_count)
else:
new_transfer = models.failed_to_approve(path=target)
LOGGER.warning("Transfer not approved: %s", transfer_name)
return None
# Start transfer completed successfully.
LOGGER.info("Finished %s", target)
return new_transfer