def test_move_rsync_command_decodes_paths(mocker):
popen = mocker.patch(
"subprocess.Popen",
return_value=mocker.Mock(
**{
"communicate.return_value": ("command output", None),
"returncode": 0
}),
)
space = Space()
space.move_rsync("source_dir", "destination_dir")
popen.assert_called_once_with(
[
"rsync",
"-t",
"-O",
"--protect-args",
"-vv",
"--chmod=Fug+rw,o-rwx,Dug+rwx,o-rwx",
"-r",
"source_dir",
"destination_dir",
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
def test_delete_uncompressed_path_local(
aipstore_uncompressed,
path,
aip_name,
remaining_directory,
deleted_directory,
mocker,
):
"""Test that local uncompressed paths are deleted as expected."""
# Initialize our space and create a path to delete.
sp = Space()
path_to_delete = os.path.join(str(aipstore_uncompressed), path, aip_name)
# rmtree is used to delete directories, we want to make sure we
# do call it from the delete function.
mocker.patch("shutil.rmtree", side_effect=shutil.rmtree)
# Make sure there is something to delete and delete it.
assert os.path.exists(path_to_delete)
sp._delete_path_local(path_to_delete)
# Verify that we called shutil to remove a directory, not a file.
shutil.rmtree.assert_called_with(path_to_delete)
# Make sure the path is gone.
assert not os.path.exists(path_to_delete)
# Make sure that the AIPSTORE part of the path is preserved.
aipstore = str(aipstore_uncompressed).split(path, 1)[0]
assert os.path.exists(aipstore)
# Assert none of the other AIPs have been deleted as a result of
# the function.
assert len(UNCOMPRESSED_AIPS) > 0
for aip_path, filename in UNCOMPRESSED_AIPS:
remaining_aip = os.path.join(aip_path, filename)
test_dir = os.path.join(path, aip_name)
if remaining_aip == test_dir:
continue
remaining_aip = os.path.join(str(aipstore_uncompressed), remaining_aip)
assert os.path.exists(remaining_aip)
# Ensure the package contains our simple manifest sample data.
assert len(os.listdir(remaining_aip)) == 1
assert os.listdir(remaining_aip)[0] == "some-manifest"
# Ensure that the correct parts of the quad-directory structure
# remain.
assert os.path.exists(os.path.join(aipstore, remaining_directory))
assert not os.path.exists(os.path.join(aipstore, deleted_directory))
def test_create_rsync_directory_commands_decode_paths(tmp_path, mocker):
temp_dir = tmp_path / "tmp"
temp_dir.mkdir()
dest_dir = "/a/mock/path/"
check_call = mocker.patch("subprocess.check_call")
mocker.patch("tempfile.mkdtemp", return_value=str(temp_dir))
space = Space()
space.create_rsync_directory(dest_dir, "user", "host")
check_call.assert_has_calls([
mocker.call([
"rsync",
"-vv",
"--protect-args",
"--chmod=ug=rwx,o=rx",
"--recursive",
os.path.join(str(temp_dir), ""),
"user@host:/",
]),
mocker.call([
"rsync",
"-vv",
"--protect-args",
"--chmod=ug=rwx,o=rx",
"--recursive",
os.path.join(str(temp_dir), ""),
"user@host:/a/",
]),
mocker.call([
"rsync",
"-vv",
"--protect-args",
"--chmod=ug=rwx,o=rx",
"--recursive",
os.path.join(str(temp_dir), ""),
"user@host:/a/mock/",
]),
mocker.call([
"rsync",
"-vv",
"--protect-args",
"--chmod=ug=rwx,o=rx",
"--recursive",
os.path.join(str(temp_dir), ""),
"user@host:/a/mock/path/",
]),
])
def test_delete_non_existant_path_local(tmpdir, aipstore_uncompressed):
"""Test the behavior when deleting non-existent paths and ensure
that when this is attempted there are no negative effects on the
AIP store.
"""
# Initialize our space and create a path to delete.
sp = Space()
path_to_delete = os.path.join(str(tmpdir), "does-not-exist")
# Make sure the path is essentially a nonsense path.
assert not os.path.exists(path_to_delete)
# There is no specific behavior for a path that doesn't exist, the
# function will fall through to a return of None.
assert sp._delete_path_local(path_to_delete) is None
# However unlikely, and in lieu of any other useful tests, make sure
# that there are no other side-effects i.e. the AIPstore is not affected.
assert len(UNCOMPRESSED_AIPS) > 0
for aip_path, filename in UNCOMPRESSED_AIPS:
remaining_aip = os.path.join(aip_path, filename)
remaining_aip = os.path.join(str(aipstore_uncompressed), remaining_aip)
assert os.path.exists(remaining_aip)
def test_move_from_storage_service(
mocker,
package,
ds_aip_collection,
metadata,
package_source_path,
package_mets_path,
ds_request_validity,
as_credentials_set,
as_credentials_valid,
upload_to_tsm,
):
mocker.patch("os.path.isfile",
return_value=getattr(package, "isfile", True))
dspace_rest_space = DSpaceREST(
space=Space(),
ds_rest_url=DS_REST_URL,
ds_user=DS_EMAIL,
ds_password=DS_PASSWORD,
ds_dip_collection=DFLT_DSPACE_DIP_COLLECTION,
ds_aip_collection=DFLT_DSPACE_AIP_COLLECTION,
as_archival_object=AS_ARCHIVAL_OBJECT,
verify_ssl=VERIFY_SSL,
upload_to_tsm=upload_to_tsm,
)
if as_credentials_set:
dspace_rest_space.as_url = AS_URL
dspace_rest_space.as_user = AS_USER
dspace_rest_space.as_password = AS_PASSWORD
dspace_rest_space.as_repository = AS_REPOSITORY
if not package:
with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo:
dspace_rest_space.move_from_storage_service(package_source_path,
AIP_DEST_PATH,
package=package)
assert str(excinfo.value) == ("DSpace requires package param.")
return
if package.package_type == Package.AIP and not package.isfile:
with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo:
dspace_rest_space.move_from_storage_service(package_source_path,
AIP_DEST_PATH,
package=package)
assert str(excinfo.value) == (
"Storing in DSpace does not support uncompressed AIPs.")
return
# Simple patches
mocker.patch("subprocess.Popen",
return_value=MockProcess(["fake-command"]))
mocker.patch("lxml.etree.parse",
return_value=etree.parse(package.fake_mets_file))
mocker.patch("os.remove")
if six.PY3:
mocker.patch("builtins.open")
else:
mocker.patch("__builtin__.open")
mocker.patch("os.listdir", return_value=[AIP_METS_FILENAME])
mocker.patch("scandir.walk", return_value=[("", [], [AIP_METS_FILENAME])])
# Patch ``requests.post``
def mock_requests_post(*args, **kwargs):
if (not ds_request_validity.is_valid) and (
ds_request_validity.url_substr in args[0]):
if ds_request_validity.raise_for_status:
return FakeDSpaceRESTPOSTResponse(
_raise=ds_request_validity.exc)
raise ds_request_validity.exc
return FakeDSpaceRESTPOSTResponse()
mocker.patch("requests.post", side_effect=mock_requests_post)
# Patch ``agentarchives.archivesspace.ArchivesSpaceClient``
if (not as_credentials_valid.is_valid
) and as_credentials_valid.method_that_raises == "add_digital_object":
fake_as_client = FakeArchivesSpaceClient(exc=as_credentials_valid.exc)
else:
fake_as_client = FakeArchivesSpaceClient()
mocker.patch("agentarchives.archivesspace.ArchivesSpaceClient",
return_value=fake_as_client)
if not as_credentials_valid.is_valid:
agentarchives.archivesspace.ArchivesSpaceClient.side_effect = (
as_credentials_valid.exc)
# Simulate AS request-related failure
if not as_credentials_valid.is_valid:
with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo:
dspace_rest_space.move_from_storage_service(package_source_path,
AIP_DEST_PATH,
package=package)
return
# Simulate AS "add digital object" failure
if (not as_credentials_valid.is_valid
) and as_credentials_valid.method_that_raises == "constructor":
with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo:
dspace_rest_space.move_from_storage_service(package_source_path,
AIP_DEST_PATH,
package=package)
assert excinfo.value.message == (
"Error depositing to DSpace or ArchiveSpace: Could not login to"
" ArchivesSpace server: {}, port: {}, user: {}, repository:"
" {}".format(AS_URL_NO_PORT, AS_PORT, AS_USER, AS_REPOSITORY))
return
if not ds_request_validity.is_valid:
with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo:
dspace_rest_space.move_from_storage_service(package_source_path,
AIP_DEST_PATH,
package=package)
return
# Call the test-targeting method in the happy path
dspace_rest_space.move_from_storage_service(package_source_path,
AIP_DEST_PATH,
package=package)
# Assertions about the 4 requests.post calls:
# 1. login to DSpace,
# 2. create a DSpace item
# 3. deposit a file to DSpace (.7z file for AIP; METS.xml file for DIP,
# which is contrived, but makes the testing easier.)
# 4. logout from DSpace.
(
actual_login_call,
(_, actual_create_item_args, actual_create_item_kwargs),
(_, actual_bitstream_args, actual_bitstream_kwargs),
actual_logout_call,
) = requests.post.mock_calls
assert actual_login_call == mocker.call(
DS_REST_LOGIN_URL,
cookies=None,
data={
"password": DS_PASSWORD,
"email": DS_EMAIL
},
headers=None,
verify=VERIFY_SSL,
)
assert actual_logout_call == mocker.call(
DS_REST_LOGOUT_URL,
cookies=COOKIES,
data=None,
headers=JSON_HEADERS,
verify=VERIFY_SSL,
)
assert actual_create_item_args == (
DS_REST_ITEM_CREATE_URL_TMPLT.format(ds_aip_collection), )
assert actual_create_item_kwargs["verify"] == VERIFY_SSL
assert actual_create_item_kwargs["cookies"] == COOKIES
assert json.loads(actual_create_item_kwargs["data"]) == {
"type": "item",
"metadata": metadata,
}
assert actual_create_item_kwargs["headers"] == JSON_HEADERS
assert actual_bitstream_kwargs["verify"] == VERIFY_SSL
assert actual_bitstream_kwargs["cookies"] == COOKIES
assert actual_bitstream_kwargs["headers"] == JSON_HEADERS
etree.parse.assert_called_once_with(package_mets_path)
if package.package_type == Package.DIP:
# No METS extraction happens with DIP, therefore no removal needed
os.remove.assert_not_called()
os.listdir.assert_called_with(package_source_path)
assert actual_bitstream_args == (DS_REST_DIP_DEPO_URL, )
if as_credentials_set:
archivesspace.ArchivesSpaceClient.assert_called_once_with(
AS_URL_NO_PORT, AS_USER, AS_PASSWORD, AS_PORT, AS_REPOSITORY)
assert fake_as_client.args == (
"/repositories/{}/archival_objects/{}".format(
AS_REPOSITORY, AS_ARCHIVAL_OBJECT),
PACKAGE_UUID,
)
assert fake_as_client.kwargs == {
"uri": DS_ITEM_HANDLE_URL,
"title": METS_1_DC_TITLE,
}
else:
archivesspace.ArchivesSpaceClient.assert_not_called()
return
os.remove.assert_called_once_with(package_mets_path)
os.listdir.assert_not_called()
archivesspace.ArchivesSpaceClient.assert_not_called()
assert actual_bitstream_args == (DS_REST_AIP_DEPO_URL, )
# Assertions about the subprocess.Popen call(s)
expected_unar_args = ([
"unar",
"-force-overwrite",
"-o",
AIP_SOURCE_PATH_DIR,
package_source_path,
AIP_EXTRACTED_METS_RELATIVE_PATH,
], )
if upload_to_tsm:
# 1. to unar to extract the AIP METS
# 2. to dsmc (Tivoli Storage Manager)
(
(_, actual_unar_args, _),
(_, actual_dsmc_args, _),
) = subprocess.Popen.mock_calls
assert actual_unar_args == expected_unar_args
assert actual_dsmc_args == (["dsmc", "archive", package_source_path], )
else:
# Assertions about the 1 subprocess.Popen call:
# 1. to unar to extract the AIP METS
((_, actual_unar_args, _), ) = subprocess.Popen.mock_calls
assert actual_unar_args == expected_unar_args
def test_move_from_storage_service(mocker, package, ds_aip_collection,
metadata, package_source_path,
package_mets_path, ds_request_validity,
as_credentials_set, as_credentials_valid,
upload_to_tsm):
mocker.patch('os.path.isfile',
return_value=getattr(package, 'isfile', True))
dspace_rest_space = DSpaceREST(
space=Space(),
ds_rest_url=DS_REST_URL,
ds_user=DS_EMAIL,
ds_password=DS_PASSWORD,
ds_dip_collection=DFLT_DSPACE_DIP_COLLECTION,
ds_aip_collection=DFLT_DSPACE_AIP_COLLECTION,
as_archival_object=AS_ARCHIVAL_OBJECT,
verify_ssl=VERIFY_SSL,
upload_to_tsm=upload_to_tsm)
if as_credentials_set:
dspace_rest_space.as_url = AS_URL
dspace_rest_space.as_user = AS_USER
dspace_rest_space.as_password = AS_PASSWORD
dspace_rest_space.as_repository = AS_REPOSITORY
if not package:
with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo:
dspace_rest_space.move_from_storage_service(package_source_path,
AIP_DEST_PATH,
package=package)
assert str(excinfo.value) == ('DSpace requires package param.')
return
if package.package_type == Package.AIP and not package.isfile:
with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo:
dspace_rest_space.move_from_storage_service(package_source_path,
AIP_DEST_PATH,
package=package)
assert str(excinfo.value) == (
'Storing in DSpace does not support uncompressed AIPs.')
return
# Simple patches
mocker.patch('subprocess.Popen',
return_value=MockProcess(['fake-command']))
mocker.patch('lxml.etree.parse',
return_value=etree.parse(package.fake_mets_file))
mocker.patch('os.remove')
mocker.patch('__builtin__.open')
mocker.patch('os.listdir', return_value=[AIP_METS_FILENAME])
# Patch ``requests.post``
def mock_requests_post(*args, **kwargs):
if ((not ds_request_validity.is_valid)
and (ds_request_validity.url_substr in args[0])):
if ds_request_validity.raise_for_status:
return FakeDSpaceRESTPOSTResponse(
_raise=ds_request_validity.exc)
raise ds_request_validity.exc
return FakeDSpaceRESTPOSTResponse()
mocker.patch('requests.post', side_effect=mock_requests_post)
# Patch ``agentarchives.archivesspace.ArchivesSpaceClient``
if ((not as_credentials_valid.is_valid) and
as_credentials_valid.method_that_raises == 'add_digital_object'):
fake_as_client = FakeArchivesSpaceClient(exc=as_credentials_valid.exc)
else:
fake_as_client = FakeArchivesSpaceClient()
mocker.patch('agentarchives.archivesspace.ArchivesSpaceClient',
return_value=fake_as_client)
if not as_credentials_valid.is_valid:
agentarchives.archivesspace.ArchivesSpaceClient.side_effect = (
as_credentials_valid.exc)
# Simulate AS request-related failure
if not as_credentials_valid.is_valid:
with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo:
dspace_rest_space.move_from_storage_service(package_source_path,
AIP_DEST_PATH,
package=package)
return
# Simulate AS "add digital object" failure
if ((not as_credentials_valid.is_valid)
and as_credentials_valid.method_that_raises == 'constructor'):
with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo:
dspace_rest_space.move_from_storage_service(package_source_path,
AIP_DEST_PATH,
package=package)
assert excinfo.value.message == (
'Error depositing to DSpace or ArchiveSpace: Could not login to'
' ArchivesSpace server: {}, port: {}, user: {}, repository:'
' {}'.format(AS_URL_NO_PORT, AS_PORT, AS_USER, AS_REPOSITORY))
return
if not ds_request_validity.is_valid:
with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo:
dspace_rest_space.move_from_storage_service(package_source_path,
AIP_DEST_PATH,
package=package)
return
# Call the test-targeting method in the happy path
dspace_rest_space.move_from_storage_service(package_source_path,
AIP_DEST_PATH,
package=package)
# Assertions about the 4 requests.post calls:
# 1. login to DSpace,
# 2. create a DSpace item
# 3. deposit a file to DSpace (.7z file for AIP; METS.xml file for DIP,
# which is contrived, but makes the testing easier.)
# 4. logout from DSpace.
(actual_login_call, (_, actual_create_item_args,
actual_create_item_kwargs), (_, actual_bitstream_args,
actual_bitstream_kwargs),
actual_logout_call) = requests.post.mock_calls
assert actual_login_call == mocker.call(DS_REST_LOGIN_URL,
cookies=None,
data={
'password': DS_PASSWORD,
'email': DS_EMAIL
},
headers=None,
verify=VERIFY_SSL)
assert actual_logout_call == mocker.call(DS_REST_LOGOUT_URL,
cookies=COOKIES,
data=None,
headers=JSON_HEADERS,
verify=VERIFY_SSL)
assert actual_create_item_args == (
DS_REST_ITEM_CREATE_URL_TMPLT.format(ds_aip_collection), )
assert actual_create_item_kwargs['verify'] == VERIFY_SSL
assert actual_create_item_kwargs['cookies'] == COOKIES
assert json.loads(actual_create_item_kwargs['data']) == {
'type': 'item',
'metadata': metadata
}
assert actual_create_item_kwargs['headers'] == JSON_HEADERS
assert actual_bitstream_kwargs['verify'] == VERIFY_SSL
assert actual_bitstream_kwargs['cookies'] == COOKIES
assert actual_bitstream_kwargs['headers'] == JSON_HEADERS
etree.parse.assert_called_once_with(package_mets_path)
if package.package_type == Package.DIP:
# No METS extraction happens with DIP, therefore no removal needed
os.remove.assert_not_called()
# Note: os.walk underlyingly calls os.listdir, so for the DIP case,
# os.listdir may be called several times
os.listdir.assert_called_with(package_source_path)
assert actual_bitstream_args == (DS_REST_DIP_DEPO_URL, )
if as_credentials_set:
archivesspace.ArchivesSpaceClient.assert_called_once_with(
AS_URL_NO_PORT, AS_USER, AS_PASSWORD, AS_PORT, AS_REPOSITORY)
assert fake_as_client.args == (
'/repositories/{}/archival_objects/{}'.format(
AS_REPOSITORY, AS_ARCHIVAL_OBJECT), PACKAGE_UUID)
assert fake_as_client.kwargs == {
'uri': DS_ITEM_HANDLE_URL,
'title': METS_1_DC_TITLE
}
else:
archivesspace.ArchivesSpaceClient.assert_not_called()
return
os.remove.assert_called_once_with(package_mets_path)
os.listdir.assert_not_called()
archivesspace.ArchivesSpaceClient.assert_not_called()
assert actual_bitstream_args == (DS_REST_AIP_DEPO_URL, )
# Assertions about the subprocess.Popen call(s)
expected_unar_args = ([
'unar', '-force-overwrite', '-o', AIP_SOURCE_PATH_DIR,
package_source_path, AIP_EXTRACTED_METS_RELATIVE_PATH
], )
if upload_to_tsm:
# 1. to unar to extract the AIP METS
# 2. to dsmc (Tivoli Storage Manager)
((_, actual_unar_args, _), (_, actual_dsmc_args,
_)) = subprocess.Popen.mock_calls
assert actual_unar_args == expected_unar_args
assert actual_dsmc_args == (['dsmc', 'archive', package_source_path], )
else:
# Assertions about the 1 subprocess.Popen call:
# 1. to unar to extract the AIP METS
((_, actual_unar_args, _), ) = subprocess.Popen.mock_calls
assert actual_unar_args == expected_unar_args