def test_get_next_transfer_depth(self):
# Set depth
depth = 2
# Test
path = transfer.get_next_transfer(SS_URL, TS_LOCATION_UUID, PATH_PREFIX, depth, COMPLETED, FILES)
# Verify
assert path == b'SampleTransfers/BagTransfer/data'
def test_get_next_transfer_all_complete(self):
# Set completed set to be all elements
completed = {b'SampleTransfers/BagTransfer', b'SampleTransfers/CSVmetadata', b'SampleTransfers/DigitizationOutput', b'SampleTransfers/DSpaceExport', b'SampleTransfers/Images', b'SampleTransfers/ISODiskImage', b'SampleTransfers/Multimedia', b'SampleTransfers/OCRImage', b'SampleTransfers/OfficeDocs', b'SampleTransfers/RawCameraImages', b'SampleTransfers/structMapSample'}
# Test
path = transfer.get_next_transfer(SS_URL, TS_LOCATION_UUID, PATH_PREFIX, DEPTH, completed, FILES)
# Verify
assert path is None
def test_get_next_transfer_no_prefix(self):
# Set no prefix
path_prefix = b''
# Test
path = transfer.get_next_transfer(SS_URL, TS_LOCATION_UUID, path_prefix, DEPTH, COMPLETED, FILES)
# Verify
assert path == b'OPF format-corpus'
def test_get_next_transfer_bad_source(self):
# Set bad TS Location UUID
ts_location_uuid = 'badd8d39-9cee-495e-b7ee-5e6292549bad'
# Test
path = transfer.get_next_transfer(SS_URL, ts_location_uuid, PATH_PREFIX, DEPTH, COMPLETED, FILES)
# Verify
assert path is None
def test_get_next_transfer_all_complete(self):
# Set completed set to be all elements
completed = {
b"SampleTransfers/BagTransfer",
b"SampleTransfers/CSVmetadata",
b"SampleTransfers/DigitizationOutput",
b"SampleTransfers/DSpaceExport",
b"SampleTransfers/Images",
b"SampleTransfers/ISODiskImage",
b"SampleTransfers/Multimedia",
b"SampleTransfers/OCRImage",
b"SampleTransfers/OfficeDocs",
b"SampleTransfers/RawCameraImages",
b"SampleTransfers/structMapSample",
}
# Test
path = transfer.get_next_transfer(
SS_URL,
SS_USER,
SS_KEY,
TS_LOCATION_UUID,
PATH_PREFIX,
DEPTH,
completed,
FILES,
)
# Verify
self.assertEqual(path, None)
def test_get_next_transfer_existing_set(self):
# Set completed set
completed = {b'SampleTransfers/BagTransfer'}
# Test
path = transfer.get_next_transfer(SS_URL, TS_LOCATION_UUID, PATH_PREFIX, DEPTH, completed, FILES)
# Verify
assert path == b'SampleTransfers/CSVmetadata'
def test_get_next_transfer_failed_auth(self):
# All default values
ss_user = '******'
ss_key = 'dne'
# Test
path = transfer.get_next_transfer(SS_URL, ss_user, ss_key, TS_LOCATION_UUID, PATH_PREFIX, DEPTH, COMPLETED, FILES)
# Verify
assert path is None
def test_get_next_transfer_files(self):
# See files
files = True
completed = {b'SampleTransfers/BagTransfer'}
# Test
path = transfer.get_next_transfer(SS_URL, TS_LOCATION_UUID, PATH_PREFIX, DEPTH, completed, files)
# Verify
assert path == b'SampleTransfers/BagTransfer.zip'
def test_get_next_transfer_first_run(self):
# All default values
# Test
path = transfer.get_next_transfer(SS_URL, SS_USER, SS_KEY,
TS_LOCATION_UUID, PATH_PREFIX, DEPTH,
COMPLETED, FILES)
# Verify
self.assertEqual(path, b'SampleTransfers/BagTransfer')
def test_get_next_transfer_bad_source(self):
# Set bad TS Location UUID
ts_location_uuid = 'badd8d39-9cee-495e-b7ee-5e6292549bad'
# Test
path = transfer.get_next_transfer(SS_URL, SS_USER, SS_KEY,
ts_location_uuid, PATH_PREFIX, DEPTH,
COMPLETED, FILES)
# Verify
self.assertEqual(path,
errors.error_lookup(errors.ERR_INVALID_RESPONSE))
def test_get_next_transfer_failed_auth(self):
# All default values
ss_user = '******'
ss_key = 'dne'
# Test
path = transfer.get_next_transfer(SS_URL, ss_user, ss_key,
TS_LOCATION_UUID, PATH_PREFIX, DEPTH,
COMPLETED, FILES)
# Verify
assert path is None
def test_get_next_transfer_failed_auth(self):
# All default values
ss_user = '******'
ss_key = 'dne'
# Test
path = transfer.get_next_transfer(SS_URL, ss_user, ss_key,
TS_LOCATION_UUID, PATH_PREFIX, DEPTH,
COMPLETED, FILES)
# Verify
self.assertEqual(path,
errors.error_lookup(errors.ERR_INVALID_RESPONSE))
def test_get_next_transfer_first_run(self):
# All default values
# Test
path = transfer.get_next_transfer(
SS_URL,
SS_USER,
SS_KEY,
TS_LOCATION_UUID,
PATH_PREFIX,
DEPTH,
COMPLETED,
FILES,
)
# Verify
self.assertEqual(path, b"SampleTransfers/BagTransfer")
def test_get_next_transfer_depth(self):
# Set depth
depth = 2
# Test
path = transfer.get_next_transfer(
SS_URL,
SS_USER,
SS_KEY,
TS_LOCATION_UUID,
PATH_PREFIX,
depth,
COMPLETED,
FILES,
)
# Verify
self.assertEqual(path, b"SampleTransfers/BagTransfer/data")
def test_get_next_transfer_existing_set(self):
# Set completed set
completed = {b"SampleTransfers/BagTransfer"}
# Test
path = transfer.get_next_transfer(
SS_URL,
SS_USER,
SS_KEY,
TS_LOCATION_UUID,
PATH_PREFIX,
DEPTH,
completed,
FILES,
)
# Verify
self.assertEqual(path, b"SampleTransfers/CSVmetadata")
def test_get_next_transfer_no_prefix(self):
# Set no prefix
path_prefix = b""
# Test
path = transfer.get_next_transfer(
SS_URL,
SS_USER,
SS_KEY,
TS_LOCATION_UUID,
path_prefix,
DEPTH,
COMPLETED,
FILES,
)
# Verify
self.assertEqual(path, b"OPF format-corpus")
def test_get_next_transfer_bad_source(self):
# Set bad TS Location UUID
ts_location_uuid = "badd8d39-9cee-495e-b7ee-5e6292549bad"
# Test
path = transfer.get_next_transfer(
SS_URL,
SS_USER,
SS_KEY,
ts_location_uuid,
PATH_PREFIX,
DEPTH,
COMPLETED,
FILES,
)
# Verify
self.assertEqual(path, None)
def test_get_next_transfer_files(self):
# See files
files = True
completed = {b"SampleTransfers/BagTransfer"}
# Test
path = transfer.get_next_transfer(
SS_URL,
SS_USER,
SS_KEY,
TS_LOCATION_UUID,
PATH_PREFIX,
DEPTH,
completed,
files,
)
# Verify
self.assertEqual(path, b"SampleTransfers/BagTransfer.zip")
def test_get_next_transfer_failed_auth(self):
# All default values
ss_user = "******"
ss_key = "dne"
# Test
path = transfer.get_next_transfer(
SS_URL,
ss_user,
ss_key,
TS_LOCATION_UUID,
PATH_PREFIX,
DEPTH,
COMPLETED,
FILES,
)
# Verify.
self.assertEqual(path, None)
def _start_transfer(ss_url, ss_user, ss_api_key, ts_location_uuid, ts_path,
depth, am_url, am_user, am_api_key, transfer_type,
see_files, session):
"""
Start a new transfer.
This is an early implementation and an alternative to
``transfer.start_transfer`` that uses the new ``/api/v2beta/package`` API.
Because transfers start immediately we don't run the scripts inside the
pre-transfer directory like ``transfer.start_transfer`` does.
:param ss_url: URL of the Storage Sevice to query
:param ss_user: User on the Storage Service for authentication
:param ss_api_key: API key for user on the Storage Service for
authentication
:param ts_location_uuid: UUID of the transfer source Location
:param ts_path: Relative path inside the Location to work with.
:param depth: Depth relative to ts_path to create a transfer from. Should
be 1 or greater.
:param am_url: URL of Archivematica pipeline to start transfer on
:param am_user: User on Archivematica for authentication
:param am_api_key: API key for user on Archivematica for authentication
:param bool see_files: If true, start transfers from files as well as
directories
:param session: SQLAlchemy session with the DB
:returns: Tuple of Transfer information about the new transfer or None on
error.
"""
# Start new transfer
completed = {x[0] for x in session.query(Unit.path).all()}
target = get_next_transfer(ss_url, ss_user, ss_api_key, ts_location_uuid,
ts_path, depth, completed, see_files)
if not target:
LOGGER.warning(
"All potential transfers in %s have been created. Exiting",
ts_path)
return None
LOGGER.info("Starting with %s", target)
# Get accession ID
accession = get_accession_id(target)
LOGGER.info("Accession ID: %s", accession)
try:
result = _api_create_package(
am_url,
am_user,
am_api_key,
os.path.basename(target),
transfer_type,
accession,
ts_location_uuid,
target,
)
except (requests.exceptions.HTTPError, ValueError,
DashboardAPIError) as err:
LOGGER.error('Unable to start transfer: %s', err)
new_transfer = Unit(path=target,
unit_type='transfer',
status='FAILED',
current=False)
session.add(new_transfer)
return None
LOGGER.info('Package created: %s', result['id'])
new_transfer = Unit(uuid=result['id'],
path=target,
unit_type='transfer',
current=True)
LOGGER.info('New transfer: %s', new_transfer)
session.add(new_transfer)
return new_transfer
def test_get_next_transfer_first_run(self):
# All default values
# Test
path = transfer.get_next_transfer(SS_URL, TS_LOCATION_UUID, PATH_PREFIX, DEPTH, COMPLETED, FILES)
# Verify
assert path == b'SampleTransfers/BagTransfer'
