def test_move_rsync_command_decodes_paths(mocker): popen = mocker.patch( "subprocess.Popen", return_value=mocker.Mock( **{ "communicate.return_value": ("command output", None), "returncode": 0 }), ) space = Space() space.move_rsync("source_dir", "destination_dir") popen.assert_called_once_with( [ "rsync", "-t", "-O", "--protect-args", "-vv", "--chmod=Fug+rw,o-rwx,Dug+rwx,o-rwx", "-r", "source_dir", "destination_dir", ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, )
def test_delete_uncompressed_path_local( aipstore_uncompressed, path, aip_name, remaining_directory, deleted_directory, mocker, ): """Test that local uncompressed paths are deleted as expected.""" # Initialize our space and create a path to delete. sp = Space() path_to_delete = os.path.join(str(aipstore_uncompressed), path, aip_name) # rmtree is used to delete directories, we want to make sure we # do call it from the delete function. mocker.patch("shutil.rmtree", side_effect=shutil.rmtree) # Make sure there is something to delete and delete it. assert os.path.exists(path_to_delete) sp._delete_path_local(path_to_delete) # Verify that we called shutil to remove a directory, not a file. shutil.rmtree.assert_called_with(path_to_delete) # Make sure the path is gone. assert not os.path.exists(path_to_delete) # Make sure that the AIPSTORE part of the path is preserved. aipstore = str(aipstore_uncompressed).split(path, 1)[0] assert os.path.exists(aipstore) # Assert none of the other AIPs have been deleted as a result of # the function. assert len(UNCOMPRESSED_AIPS) > 0 for aip_path, filename in UNCOMPRESSED_AIPS: remaining_aip = os.path.join(aip_path, filename) test_dir = os.path.join(path, aip_name) if remaining_aip == test_dir: continue remaining_aip = os.path.join(str(aipstore_uncompressed), remaining_aip) assert os.path.exists(remaining_aip) # Ensure the package contains our simple manifest sample data. assert len(os.listdir(remaining_aip)) == 1 assert os.listdir(remaining_aip)[0] == "some-manifest" # Ensure that the correct parts of the quad-directory structure # remain. assert os.path.exists(os.path.join(aipstore, remaining_directory)) assert not os.path.exists(os.path.join(aipstore, deleted_directory))
def test_create_rsync_directory_commands_decode_paths(tmp_path, mocker): temp_dir = tmp_path / "tmp" temp_dir.mkdir() dest_dir = "/a/mock/path/" check_call = mocker.patch("subprocess.check_call") mocker.patch("tempfile.mkdtemp", return_value=str(temp_dir)) space = Space() space.create_rsync_directory(dest_dir, "user", "host") check_call.assert_has_calls([ mocker.call([ "rsync", "-vv", "--protect-args", "--chmod=ug=rwx,o=rx", "--recursive", os.path.join(str(temp_dir), ""), "user@host:/", ]), mocker.call([ "rsync", "-vv", "--protect-args", "--chmod=ug=rwx,o=rx", "--recursive", os.path.join(str(temp_dir), ""), "user@host:/a/", ]), mocker.call([ "rsync", "-vv", "--protect-args", "--chmod=ug=rwx,o=rx", "--recursive", os.path.join(str(temp_dir), ""), "user@host:/a/mock/", ]), mocker.call([ "rsync", "-vv", "--protect-args", "--chmod=ug=rwx,o=rx", "--recursive", os.path.join(str(temp_dir), ""), "user@host:/a/mock/path/", ]), ])
def test_delete_non_existant_path_local(tmpdir, aipstore_uncompressed): """Test the behavior when deleting non-existent paths and ensure that when this is attempted there are no negative effects on the AIP store. """ # Initialize our space and create a path to delete. sp = Space() path_to_delete = os.path.join(str(tmpdir), "does-not-exist") # Make sure the path is essentially a nonsense path. assert not os.path.exists(path_to_delete) # There is no specific behavior for a path that doesn't exist, the # function will fall through to a return of None. assert sp._delete_path_local(path_to_delete) is None # However unlikely, and in lieu of any other useful tests, make sure # that there are no other side-effects i.e. the AIPstore is not affected. assert len(UNCOMPRESSED_AIPS) > 0 for aip_path, filename in UNCOMPRESSED_AIPS: remaining_aip = os.path.join(aip_path, filename) remaining_aip = os.path.join(str(aipstore_uncompressed), remaining_aip) assert os.path.exists(remaining_aip)
def test_move_from_storage_service( mocker, package, ds_aip_collection, metadata, package_source_path, package_mets_path, ds_request_validity, as_credentials_set, as_credentials_valid, upload_to_tsm, ): mocker.patch("os.path.isfile", return_value=getattr(package, "isfile", True)) dspace_rest_space = DSpaceREST( space=Space(), ds_rest_url=DS_REST_URL, ds_user=DS_EMAIL, ds_password=DS_PASSWORD, ds_dip_collection=DFLT_DSPACE_DIP_COLLECTION, ds_aip_collection=DFLT_DSPACE_AIP_COLLECTION, as_archival_object=AS_ARCHIVAL_OBJECT, verify_ssl=VERIFY_SSL, upload_to_tsm=upload_to_tsm, ) if as_credentials_set: dspace_rest_space.as_url = AS_URL dspace_rest_space.as_user = AS_USER dspace_rest_space.as_password = AS_PASSWORD dspace_rest_space.as_repository = AS_REPOSITORY if not package: with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo: dspace_rest_space.move_from_storage_service(package_source_path, AIP_DEST_PATH, package=package) assert str(excinfo.value) == ("DSpace requires package param.") return if package.package_type == Package.AIP and not package.isfile: with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo: dspace_rest_space.move_from_storage_service(package_source_path, AIP_DEST_PATH, package=package) assert str(excinfo.value) == ( "Storing in DSpace does not support uncompressed AIPs.") return # Simple patches mocker.patch("subprocess.Popen", return_value=MockProcess(["fake-command"])) mocker.patch("lxml.etree.parse", return_value=etree.parse(package.fake_mets_file)) mocker.patch("os.remove") if six.PY3: mocker.patch("builtins.open") else: mocker.patch("__builtin__.open") mocker.patch("os.listdir", return_value=[AIP_METS_FILENAME]) mocker.patch("scandir.walk", return_value=[("", [], [AIP_METS_FILENAME])]) # Patch ``requests.post`` def mock_requests_post(*args, **kwargs): if (not ds_request_validity.is_valid) and ( ds_request_validity.url_substr in args[0]): if ds_request_validity.raise_for_status: return FakeDSpaceRESTPOSTResponse( _raise=ds_request_validity.exc) raise ds_request_validity.exc return FakeDSpaceRESTPOSTResponse() mocker.patch("requests.post", side_effect=mock_requests_post) # Patch ``agentarchives.archivesspace.ArchivesSpaceClient`` if (not as_credentials_valid.is_valid ) and as_credentials_valid.method_that_raises == "add_digital_object": fake_as_client = FakeArchivesSpaceClient(exc=as_credentials_valid.exc) else: fake_as_client = FakeArchivesSpaceClient() mocker.patch("agentarchives.archivesspace.ArchivesSpaceClient", return_value=fake_as_client) if not as_credentials_valid.is_valid: agentarchives.archivesspace.ArchivesSpaceClient.side_effect = ( as_credentials_valid.exc) # Simulate AS request-related failure if not as_credentials_valid.is_valid: with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo: dspace_rest_space.move_from_storage_service(package_source_path, AIP_DEST_PATH, package=package) return # Simulate AS "add digital object" failure if (not as_credentials_valid.is_valid ) and as_credentials_valid.method_that_raises == "constructor": with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo: dspace_rest_space.move_from_storage_service(package_source_path, AIP_DEST_PATH, package=package) assert excinfo.value.message == ( "Error depositing to DSpace or ArchiveSpace: Could not login to" " ArchivesSpace server: {}, port: {}, user: {}, repository:" " {}".format(AS_URL_NO_PORT, AS_PORT, AS_USER, AS_REPOSITORY)) return if not ds_request_validity.is_valid: with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo: dspace_rest_space.move_from_storage_service(package_source_path, AIP_DEST_PATH, package=package) return # Call the test-targeting method in the happy path dspace_rest_space.move_from_storage_service(package_source_path, AIP_DEST_PATH, package=package) # Assertions about the 4 requests.post calls: # 1. login to DSpace, # 2. create a DSpace item # 3. deposit a file to DSpace (.7z file for AIP; METS.xml file for DIP, # which is contrived, but makes the testing easier.) # 4. logout from DSpace. ( actual_login_call, (_, actual_create_item_args, actual_create_item_kwargs), (_, actual_bitstream_args, actual_bitstream_kwargs), actual_logout_call, ) = requests.post.mock_calls assert actual_login_call == mocker.call( DS_REST_LOGIN_URL, cookies=None, data={ "password": DS_PASSWORD, "email": DS_EMAIL }, headers=None, verify=VERIFY_SSL, ) assert actual_logout_call == mocker.call( DS_REST_LOGOUT_URL, cookies=COOKIES, data=None, headers=JSON_HEADERS, verify=VERIFY_SSL, ) assert actual_create_item_args == ( DS_REST_ITEM_CREATE_URL_TMPLT.format(ds_aip_collection), ) assert actual_create_item_kwargs["verify"] == VERIFY_SSL assert actual_create_item_kwargs["cookies"] == COOKIES assert json.loads(actual_create_item_kwargs["data"]) == { "type": "item", "metadata": metadata, } assert actual_create_item_kwargs["headers"] == JSON_HEADERS assert actual_bitstream_kwargs["verify"] == VERIFY_SSL assert actual_bitstream_kwargs["cookies"] == COOKIES assert actual_bitstream_kwargs["headers"] == JSON_HEADERS etree.parse.assert_called_once_with(package_mets_path) if package.package_type == Package.DIP: # No METS extraction happens with DIP, therefore no removal needed os.remove.assert_not_called() os.listdir.assert_called_with(package_source_path) assert actual_bitstream_args == (DS_REST_DIP_DEPO_URL, ) if as_credentials_set: archivesspace.ArchivesSpaceClient.assert_called_once_with( AS_URL_NO_PORT, AS_USER, AS_PASSWORD, AS_PORT, AS_REPOSITORY) assert fake_as_client.args == ( "/repositories/{}/archival_objects/{}".format( AS_REPOSITORY, AS_ARCHIVAL_OBJECT), PACKAGE_UUID, ) assert fake_as_client.kwargs == { "uri": DS_ITEM_HANDLE_URL, "title": METS_1_DC_TITLE, } else: archivesspace.ArchivesSpaceClient.assert_not_called() return os.remove.assert_called_once_with(package_mets_path) os.listdir.assert_not_called() archivesspace.ArchivesSpaceClient.assert_not_called() assert actual_bitstream_args == (DS_REST_AIP_DEPO_URL, ) # Assertions about the subprocess.Popen call(s) expected_unar_args = ([ "unar", "-force-overwrite", "-o", AIP_SOURCE_PATH_DIR, package_source_path, AIP_EXTRACTED_METS_RELATIVE_PATH, ], ) if upload_to_tsm: # 1. to unar to extract the AIP METS # 2. to dsmc (Tivoli Storage Manager) ( (_, actual_unar_args, _), (_, actual_dsmc_args, _), ) = subprocess.Popen.mock_calls assert actual_unar_args == expected_unar_args assert actual_dsmc_args == (["dsmc", "archive", package_source_path], ) else: # Assertions about the 1 subprocess.Popen call: # 1. to unar to extract the AIP METS ((_, actual_unar_args, _), ) = subprocess.Popen.mock_calls assert actual_unar_args == expected_unar_args
def test_move_from_storage_service(mocker, package, ds_aip_collection, metadata, package_source_path, package_mets_path, ds_request_validity, as_credentials_set, as_credentials_valid, upload_to_tsm): mocker.patch('os.path.isfile', return_value=getattr(package, 'isfile', True)) dspace_rest_space = DSpaceREST( space=Space(), ds_rest_url=DS_REST_URL, ds_user=DS_EMAIL, ds_password=DS_PASSWORD, ds_dip_collection=DFLT_DSPACE_DIP_COLLECTION, ds_aip_collection=DFLT_DSPACE_AIP_COLLECTION, as_archival_object=AS_ARCHIVAL_OBJECT, verify_ssl=VERIFY_SSL, upload_to_tsm=upload_to_tsm) if as_credentials_set: dspace_rest_space.as_url = AS_URL dspace_rest_space.as_user = AS_USER dspace_rest_space.as_password = AS_PASSWORD dspace_rest_space.as_repository = AS_REPOSITORY if not package: with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo: dspace_rest_space.move_from_storage_service(package_source_path, AIP_DEST_PATH, package=package) assert str(excinfo.value) == ('DSpace requires package param.') return if package.package_type == Package.AIP and not package.isfile: with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo: dspace_rest_space.move_from_storage_service(package_source_path, AIP_DEST_PATH, package=package) assert str(excinfo.value) == ( 'Storing in DSpace does not support uncompressed AIPs.') return # Simple patches mocker.patch('subprocess.Popen', return_value=MockProcess(['fake-command'])) mocker.patch('lxml.etree.parse', return_value=etree.parse(package.fake_mets_file)) mocker.patch('os.remove') mocker.patch('__builtin__.open') mocker.patch('os.listdir', return_value=[AIP_METS_FILENAME]) # Patch ``requests.post`` def mock_requests_post(*args, **kwargs): if ((not ds_request_validity.is_valid) and (ds_request_validity.url_substr in args[0])): if ds_request_validity.raise_for_status: return FakeDSpaceRESTPOSTResponse( _raise=ds_request_validity.exc) raise ds_request_validity.exc return FakeDSpaceRESTPOSTResponse() mocker.patch('requests.post', side_effect=mock_requests_post) # Patch ``agentarchives.archivesspace.ArchivesSpaceClient`` if ((not as_credentials_valid.is_valid) and as_credentials_valid.method_that_raises == 'add_digital_object'): fake_as_client = FakeArchivesSpaceClient(exc=as_credentials_valid.exc) else: fake_as_client = FakeArchivesSpaceClient() mocker.patch('agentarchives.archivesspace.ArchivesSpaceClient', return_value=fake_as_client) if not as_credentials_valid.is_valid: agentarchives.archivesspace.ArchivesSpaceClient.side_effect = ( as_credentials_valid.exc) # Simulate AS request-related failure if not as_credentials_valid.is_valid: with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo: dspace_rest_space.move_from_storage_service(package_source_path, AIP_DEST_PATH, package=package) return # Simulate AS "add digital object" failure if ((not as_credentials_valid.is_valid) and as_credentials_valid.method_that_raises == 'constructor'): with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo: dspace_rest_space.move_from_storage_service(package_source_path, AIP_DEST_PATH, package=package) assert excinfo.value.message == ( 'Error depositing to DSpace or ArchiveSpace: Could not login to' ' ArchivesSpace server: {}, port: {}, user: {}, repository:' ' {}'.format(AS_URL_NO_PORT, AS_PORT, AS_USER, AS_REPOSITORY)) return if not ds_request_validity.is_valid: with pytest.raises(dspace_rest.DSpaceRESTException) as excinfo: dspace_rest_space.move_from_storage_service(package_source_path, AIP_DEST_PATH, package=package) return # Call the test-targeting method in the happy path dspace_rest_space.move_from_storage_service(package_source_path, AIP_DEST_PATH, package=package) # Assertions about the 4 requests.post calls: # 1. login to DSpace, # 2. create a DSpace item # 3. deposit a file to DSpace (.7z file for AIP; METS.xml file for DIP, # which is contrived, but makes the testing easier.) # 4. logout from DSpace. (actual_login_call, (_, actual_create_item_args, actual_create_item_kwargs), (_, actual_bitstream_args, actual_bitstream_kwargs), actual_logout_call) = requests.post.mock_calls assert actual_login_call == mocker.call(DS_REST_LOGIN_URL, cookies=None, data={ 'password': DS_PASSWORD, 'email': DS_EMAIL }, headers=None, verify=VERIFY_SSL) assert actual_logout_call == mocker.call(DS_REST_LOGOUT_URL, cookies=COOKIES, data=None, headers=JSON_HEADERS, verify=VERIFY_SSL) assert actual_create_item_args == ( DS_REST_ITEM_CREATE_URL_TMPLT.format(ds_aip_collection), ) assert actual_create_item_kwargs['verify'] == VERIFY_SSL assert actual_create_item_kwargs['cookies'] == COOKIES assert json.loads(actual_create_item_kwargs['data']) == { 'type': 'item', 'metadata': metadata } assert actual_create_item_kwargs['headers'] == JSON_HEADERS assert actual_bitstream_kwargs['verify'] == VERIFY_SSL assert actual_bitstream_kwargs['cookies'] == COOKIES assert actual_bitstream_kwargs['headers'] == JSON_HEADERS etree.parse.assert_called_once_with(package_mets_path) if package.package_type == Package.DIP: # No METS extraction happens with DIP, therefore no removal needed os.remove.assert_not_called() # Note: os.walk underlyingly calls os.listdir, so for the DIP case, # os.listdir may be called several times os.listdir.assert_called_with(package_source_path) assert actual_bitstream_args == (DS_REST_DIP_DEPO_URL, ) if as_credentials_set: archivesspace.ArchivesSpaceClient.assert_called_once_with( AS_URL_NO_PORT, AS_USER, AS_PASSWORD, AS_PORT, AS_REPOSITORY) assert fake_as_client.args == ( '/repositories/{}/archival_objects/{}'.format( AS_REPOSITORY, AS_ARCHIVAL_OBJECT), PACKAGE_UUID) assert fake_as_client.kwargs == { 'uri': DS_ITEM_HANDLE_URL, 'title': METS_1_DC_TITLE } else: archivesspace.ArchivesSpaceClient.assert_not_called() return os.remove.assert_called_once_with(package_mets_path) os.listdir.assert_not_called() archivesspace.ArchivesSpaceClient.assert_not_called() assert actual_bitstream_args == (DS_REST_AIP_DEPO_URL, ) # Assertions about the subprocess.Popen call(s) expected_unar_args = ([ 'unar', '-force-overwrite', '-o', AIP_SOURCE_PATH_DIR, package_source_path, AIP_EXTRACTED_METS_RELATIVE_PATH ], ) if upload_to_tsm: # 1. to unar to extract the AIP METS # 2. to dsmc (Tivoli Storage Manager) ((_, actual_unar_args, _), (_, actual_dsmc_args, _)) = subprocess.Popen.mock_calls assert actual_unar_args == expected_unar_args assert actual_dsmc_args == (['dsmc', 'archive', package_source_path], ) else: # Assertions about the 1 subprocess.Popen call: # 1. to unar to extract the AIP METS ((_, actual_unar_args, _), ) = subprocess.Popen.mock_calls assert actual_unar_args == expected_unar_args