def __init__(self,
configuration,
majic_webservice_client=None,
file_system_comparer=None,
directory_synchroniser=None):
"""
Constructor
:param configuration: the configuration to use for the synchronisation
:param majic_webservice_client: the majic web service to use
:return: nothing
"""
super(Synchroniser, self).__init__()
self._config = configuration
if majic_webservice_client is None:
self._majic_webservice_client = MajicWebserviceClient(self._config)
else:
self._majic_webservice_client = majic_webservice_client
if file_system_comparer is None:
self._file_system_comparer = FileSystemComparer(self._config)
else:
self._file_system_comparer = file_system_comparer
if directory_synchroniser is None:
self._directory_synchroniser = DirectorySynchroniser(self._config)
else:
self._directory_synchroniser = directory_synchroniser
class Synchroniser(object):
"""
Provides services to synchronisation file
"""
def __init__(self,
configuration,
majic_webservice_client=None,
file_system_comparer=None,
directory_synchroniser=None):
"""
Constructor
:param configuration: the configuration to use for the synchronisation
:param majic_webservice_client: the majic web service to use
:return: nothing
"""
super(Synchroniser, self).__init__()
self._config = configuration
if majic_webservice_client is None:
self._majic_webservice_client = MajicWebserviceClient(self._config)
else:
self._majic_webservice_client = majic_webservice_client
if file_system_comparer is None:
self._file_system_comparer = FileSystemComparer(self._config)
else:
self._file_system_comparer = file_system_comparer
if directory_synchroniser is None:
self._directory_synchroniser = DirectorySynchroniser(self._config)
else:
self._directory_synchroniser = directory_synchroniser
def synchronise(self):
"""
Synchronise the files returned by the web service with those on the disc
:return: error code to exit with
"""
try:
log.debug("Starting to sync")
model_propeties = self._majic_webservice_client.get_properties_list_with_filtered_users()
self._file_system_comparer.perform_analysis(model_propeties)
self._file_system_comparer.add_extra_directories_to_sync()
new_count, updated_count, deleted_count = \
self._directory_synchroniser.synchronise_all(self._file_system_comparer)
log.info("Finished Synchronisation:")
log.info(" {}: Count of copied files and directories: ".format(new_count))
log.info(" {}: Count of directory permissions updated".format(updated_count))
log.info(" {}: Count of deleted directories".format(deleted_count))
return 0
except UserPrintableError as ex:
log.error(str(ex))
return 1
except Exception:
log.exception("Unknown error in synchronisation")
log.error("An unknown error occurred so files are not synced")
return 2
def setup_mocks(self, directory_contents_on_apache, reg_ex_to_ignore=""):
self.contents = directory_contents_on_apache
self.mock_apache_client = Mock(ApacheClient)
self.mock_apache_client.get_contents = self._find_contents
self.mock_file_handle = Mock()
self.mock_file_system_client = Mock(FileSystemClient)
self.mock_file_system_client.create_file = Mock(return_value=self.mock_file_handle)
self.mock_file_system_client.create_dir = Mock(return_value=True)
config = ConfigMother.test_configuration_with_values(reg_ex_to_ignore=reg_ex_to_ignore)
self.files_synchronisation = DirectorySynchroniser(config, self.mock_apache_client, self.mock_file_system_client)
class TestDirectorySynchronisation(unittest.TestCase):
def setup_mocks(self, directory_contents_on_apache, reg_ex_to_ignore=""):
self.contents = directory_contents_on_apache
self.mock_apache_client = Mock(ApacheClient)
self.mock_apache_client.get_contents = self._find_contents
self.mock_file_handle = Mock()
self.mock_file_system_client = Mock(FileSystemClient)
self.mock_file_system_client.create_file = Mock(return_value=self.mock_file_handle)
self.mock_file_system_client.create_dir = Mock(return_value=True)
config = ConfigMother.test_configuration_with_values(reg_ex_to_ignore=reg_ex_to_ignore)
self.files_synchronisation = DirectorySynchroniser(config, self.mock_apache_client, self.mock_file_system_client)
def _find_contents(self, directory):
return self.contents[directory]
def test_GIVEN_no_files_in_directory_WHEN_copy_new_THEN_directory_created_and_permissions_set(self):
model_run_path = "data/run1"
self.setup_mocks({model_run_path: []})
new_directories = [FileProperties(model_run_path, "owner", False, False)]
copied_count = self.files_synchronisation.copy_new(new_directories)
assert_that(self.mock_file_system_client.create_dir.called, is_(True), "Directory was created")
self.mock_file_system_client.create_dir.assert_called_with(model_run_path)
assert_that(self.mock_file_system_client.set_permissions.called, is_(True), "Permissions called")
self.mock_file_system_client.set_permissions.assert_called_with(new_directories[0])
assert_that(copied_count, is_(1), "copied_count")
def assert_file_downloaded(self, expected_file_name):
assert_that(self.mock_file_system_client.create_file.called, is_(True), "File was opened")
self.mock_file_system_client.create_file.assert_called_with(expected_file_name)
assert_that(self.mock_apache_client.download_file.called, is_(True), "Download called")
self.mock_apache_client.download_file.assert_called_with(expected_file_name, self.mock_file_handle)
def test_GIVEN_file_in_directory_WHEN_copy_new_THEN_directory_created_and_permissions_set(self):
model_run_dir = "data/run1"
self.new_directories = [FileProperties(model_run_dir, "owner", False, False)]
expected_file = "data.nc"
self.setup_mocks({model_run_dir: [expected_file]})
expected_file_name = "{}/{}".format(model_run_dir, expected_file)
copied_count = self.files_synchronisation.copy_new(self.new_directories)
self.assert_file_downloaded(expected_file_name)
assert_that(copied_count, is_(2), "copied_count")
def test_GIVEN_files_in_directory_some_not_nc_or_ncml_WHEN_copy_new_THEN_only_nc_and_ncml_files_copied(self):
model_run_dir = "data/run1"
self.new_directories = [FileProperties(model_run_dir, "owner", False, False)]
expected_files = ["data.nc", "copy.ncml"]
self.setup_mocks({model_run_dir: ["data.nc", "not to copy", "copy.ncml", "nottocopy.hi"]})
expected_file_names = ["{}/{}".format(self.new_directories[0].file_path, expected_file) for expected_file in expected_files]
copied_count = self.files_synchronisation.copy_new(self.new_directories)
assert_that(self.mock_file_system_client.create_file.called, is_(True), "File was opened")
call_list = self.mock_file_system_client.create_file.call_args_list
assert_that(call_list[0][0][0], is_(expected_file_names[0]))
assert_that(call_list[1][0][0], is_(expected_file_names[1]))
assert_that(self.mock_apache_client.download_file.called, is_(True), "Permissions called")
call_args = [x[0] for x in self.mock_apache_client.download_file.call_args_list]
assert_that(call_args[0], is_((expected_file_names[0], self.mock_file_handle)))
assert_that(call_args[1], is_((expected_file_names[1], self.mock_file_handle)))
assert_that(copied_count, is_(3), "copied_count")
def test_GIVEN_directory_in_directory_WHEN_copy_new_THEN_directory_created(self):
model_run_dir = "data/run1"
self.new_directories = [FileProperties(model_run_dir, "owner", False, False)]
expected_dir = "subdir"
expected_dir_name = "{}/{}".format(model_run_dir, expected_dir)
self.setup_mocks({model_run_dir: [expected_dir + '/'],
expected_dir_name: []})
copied_count = self.files_synchronisation.copy_new(self.new_directories)
assert_that(self.mock_file_system_client.create_dir.call_count, is_(2), "Parent and sub directory created count")
self.mock_file_system_client.create_dir.assert_called_with(expected_dir_name)
assert_that(copied_count, is_(2), "copied_count")
def test_GIVEN_file_in_sub_directory_WHEN_copy_new_THEN_directory_created_and_file_copied(self):
model_run_dir = "data/run1"
self.new_directories = [FileProperties(model_run_dir, "owner", False, False)]
expected_dir = "subdir"
expected_dir_name = "{}/{}".format(model_run_dir, expected_dir)
expected_filename = "my_file.nc"
expected_file_path = "{}/{}".format(expected_dir_name, expected_filename)
self.setup_mocks({model_run_dir: [expected_dir + '/'],
expected_dir_name: [expected_filename]})
copied_count = self.files_synchronisation.copy_new(self.new_directories)
assert_that(self.mock_file_system_client.create_dir.call_count, is_(2), "Parent and sub directory created count")
self.mock_file_system_client.create_dir.assert_called_with(expected_dir_name)
self.assert_file_downloaded(expected_file_path)
assert_that(copied_count, is_(3), "copied_count")
def test_GIVEN_apache_client_throws_error_WHEN_copy_new_THEN_next_directory_look_at(self):
model_run_path = "data/run1"
self.setup_mocks({model_run_path: []})
new_directories = [FileProperties(model_run_path, "owner", False, False),
FileProperties(model_run_path, "owner", False, False)]
self.mock_apache_client.get_contents = Mock(side_effect=ApacheClientError("Error"))
copied_count = self.files_synchronisation.copy_new(new_directories)
assert_that(self.mock_file_system_client.create_dir.called, is_(True), "Directory was created")
self.mock_file_system_client.create_dir.assert_called_with(model_run_path)
assert_that(self.mock_file_system_client.set_permissions.called, is_(True), "Permissions called")
self.mock_file_system_client.set_permissions.assert_called_with(new_directories[0])
assert_that(copied_count, is_(2), "copied_count")
def test_GIVEN_create_directory_throws_error_WHEN_copy_new_THEN_next_directory_look_at(self):
model_run_path = "data/run1"
self.setup_mocks({model_run_path: []})
new_directories = [FileProperties(model_run_path, "owner", False, False),
FileProperties(model_run_path, "owner", False, False)]
self.mock_file_system_client.create_dir = Mock(side_effect=FileSystemClientError("Error"))
copied_count = self.files_synchronisation.copy_new(new_directories)
assert_that(self.mock_file_system_client.create_dir.call_count, is_(2), "Directory was created count")
assert_that(self.mock_file_system_client.set_permissions.call_count, is_(0), "Set permissions is not called")
assert_that(copied_count, is_(0), "copied_count")
def test_GIVEN_create_file_throws_error_WHEN_copy_new_THEN_next_file_look_at(self):
model_run_path = "data/run1"
self.setup_mocks({model_run_path: ["file1.ncml", "file2.nc"]})
new_directories = [FileProperties(model_run_path, "owner", False, False)]
self.mock_file_system_client.create_file = Mock(side_effect=FileSystemClientError("Error"))
copied_count = self.files_synchronisation.copy_new(new_directories)
assert_that(self.mock_file_system_client.create_file.call_count, is_(2), "File opened")
assert_that(copied_count, is_(1), "copied_count")
def test_GIVEN_file_exists_already_WHEN_copy_new_THEN_file_is_not_copied(self):
model_run_path = "data/run1"
self.setup_mocks({model_run_path: ["file1.ncml", "file2.nc"]})
new_directories = [FileProperties(model_run_path, "owner", False, False)]
self.mock_file_system_client.create_file = Mock(return_value=None)
copied_count = self.files_synchronisation.copy_new(new_directories)
assert_that(self.mock_file_system_client.create_file.call_count, is_(2), "File opened")
assert_that(self.mock_apache_client.download_file.call_count, is_(0), "File downloaded")
assert_that(copied_count, is_(1), "copied_count")
def test_GIVEN_download_file_throws_error_WHEN_copy_new_THEN_next_file_look_at(self):
model_run_path = "data/run1"
self.setup_mocks({model_run_path: ["file1.ncml", "file2.nc"]})
new_directories = [FileProperties(model_run_path, "owner", False, False)]
self.mock_apache_client.download_file = Mock(side_effect=ApacheClientError("Error"))
copied_count = self.files_synchronisation.copy_new(new_directories)
assert_that(self.mock_apache_client.download_file.call_count, is_(2), "File download start")
assert_that(self.mock_file_system_client.close_and_delete_file.call_count, is_(2), "File is deleted when download fails")
assert_that(copied_count, is_(1), "copied_count")
def test_GIVEN_folder_WHEN_update_permissions_THEN_permissions_updated(self):
model_run_path = "data/run1"
self.setup_mocks({})
changed_directories = [FileProperties(model_run_path, "owner", False, False),
FileProperties(model_run_path, "owner", False, False)]
updated_count = self.files_synchronisation.update_permissions(changed_directories)
assert_that(self.mock_file_system_client.set_permissions.call_count, is_(2), "Permissions set on directories count")
assert_that(updated_count, is_(2), "updated count")
def test_GIVEN_folder_WHEN_update_permissions_THEN_permissions_updated(self):
model_run_path = "data/run1"
self.setup_mocks({})
changed_directories = [FileProperties(model_run_path, "owner", False, False),
FileProperties(model_run_path, "owner", False, False)]
updated_count = self.files_synchronisation.update_permissions(changed_directories)
assert_that(self.mock_file_system_client.set_permissions.call_count, is_(2), "Permissions set on directories count")
assert_that(updated_count, is_(2), "updated count")
def test_GIVEN_update_permissions_throws_WHEN_update_permissions_THEN_next_directory_looked_at(self):
model_run_path = "data/run1"
self.setup_mocks({})
self.mock_file_system_client.set_permissions = Mock(side_effect=FileSystemClientError("Error"))
changed_directories = [FileProperties(model_run_path, "owner", False, False),
FileProperties(model_run_path, "owner", False, False)]
updated_count = self.files_synchronisation.update_permissions(changed_directories)
assert_that(self.mock_file_system_client.set_permissions.call_count, is_(2), "Permissions set on directories count")
assert_that(updated_count, is_(0), "updated count")
def test_GIVEN_folders_WHEN_delete_THEN_folders_deleted(self):
model_run_path = "data/run1"
self.setup_mocks({})
deleted_directories = [FileProperties(model_run_path, "owner", False, False),
FileProperties(model_run_path, "owner", False, False)]
deleted_count = self.files_synchronisation.delete(deleted_directories)
assert_that(self.mock_file_system_client.delete_directory.call_count, is_(2), "Delete call count")
assert_that(deleted_count, is_(2), "deleted count")
def test_GIVEN_exception_thrown_on_delete_WHEN_delete_THEN_next_folder_deleted(self):
model_run_path = "data/run1"
self.setup_mocks({})
self.mock_file_system_client.delete_directory = Mock(side_effect=FileSystemClientError("Error"))
deleted_directories = [FileProperties(model_run_path, "owner", False, False),
FileProperties(model_run_path, "owner", False, False)]
deleted_count = self.files_synchronisation.delete(deleted_directories)
assert_that(self.mock_file_system_client.delete_directory.call_count, is_(2), "Delete call count")
assert_that(deleted_count, is_(0), "deleted count")
def test_GIVEN_one_change_new_and_delete_WHEN_sync_all_THEN_all_files_synched(self):
model_run_path_to_copy = "data/run1"
model_run_path_to_delete = "data/run2"
model_run_path_to_update = "data/run3"
model_run_path_to_resync = "data/run4"
filename = "blah.nc"
self.setup_mocks({model_run_path_to_copy: [],
model_run_path_to_resync: [filename]})
file_sync = Mock(FileSystemComparer)
file_sync.deleted_directories = [model_run_path_to_delete]
file_sync.new_directories = [FileProperties(model_run_path_to_copy, "owner", False, False)]
file_sync.changed_directories = [FileProperties(model_run_path_to_update, "owner", False, False)]
file_sync.existing_non_deleted_directories = [FileProperties(model_run_path_to_resync, "owner", False, False)]
self.mock_file_system_client.create_file = Mock(return_value=self.mock_file_handle)
new_count, updated_count, deleted_count = self.files_synchronisation.synchronise_all(file_sync)
assert_that(self.mock_file_system_client.create_dir.call_args_list[0][0][0], is_(model_run_path_to_copy))
assert_that(self.mock_file_system_client.create_dir.call_args_list[1][0][0], is_(model_run_path_to_resync))
self.mock_file_system_client.create_file.assert_called_with(model_run_path_to_resync + '/' + filename)
self.mock_file_system_client.set_permissions.assert_called_with(file_sync.changed_directories[0])
self.mock_file_system_client.delete_directory.assert_called_with(model_run_path_to_delete)
assert_that(new_count, is_(3), "updated count (plus one for erant existing update)")
assert_that(updated_count, is_(1), "updated count")
assert_that(deleted_count, is_(1), "deleted count")
def test_GIVEN_directory_matches_reg_ex_to_ignore_WHEN_copy_new_THEN_directory_not_created(self):
reg_ex_to_ignore = "run(\d)+/data/"
model_run_dir = "data/run13"
self.new_directories = [FileProperties(model_run_dir, "owner", False, False)]
expected_dir = "data"
expected_dir_name = "{}/{}".format(model_run_dir, expected_dir)
self.setup_mocks({model_run_dir: [expected_dir + '/'],
expected_dir_name: []},
reg_ex_to_ignore)
copied_count = self.files_synchronisation.copy_new(self.new_directories)
assert_that(self.mock_file_system_client.create_dir.call_count, is_(1), "Just parent created count")
self.mock_file_system_client.create_dir.assert_called_with(model_run_dir)
assert_that(copied_count, is_(1), "copied_count")
def test_GIVEN_file_matches_reg_ex_to_ignore_WHEN_copy_new_THEN_directory_not_created(self):
reg_ex_to_ignore = "run(\d)+/data/ output/.*\.dump\..*.nc"
model_run_dir = "data/run13"
self.new_directories = [FileProperties(model_run_dir, "owner", False, False)]
expected_dir = "output"
expected_dir_name = "{}/{}".format(model_run_dir, expected_dir)
filename = "majic.dump.spin1.196701.nc"
self.setup_mocks({model_run_dir: [expected_dir + '/'],
expected_dir_name: [filename]},
reg_ex_to_ignore)
copied_count = self.files_synchronisation.copy_new(self.new_directories)
assert_that(self.mock_file_system_client.create_dir.call_count, is_(2), "Copy parent and sub")
assert_that(self.mock_file_system_client.create_file.call_count, is_(0), "create file (not called because ignored)")
assert_that(copied_count, is_(2), "copied_count")
def test_GIVEN_new_file_in_directory_WHEN_copy_new_THEN_directory_not_created_permissions_set_as_writable_and_then_set_back(self):
# the directory must become writable to the process so that the process can update the directory if needed
model_run_path = "data/run1"
self.setup_mocks({model_run_path: []})
new_directories = [FileProperties(model_run_path, "owner", False, False)]
copied_count = self.files_synchronisation.copy_new(new_directories)
assert_that(self.mock_file_system_client.create_dir.called, is_(True), "Directory was created")
self.mock_file_system_client.create_dir.assert_called_with(model_run_path)
assert_that(self.mock_file_system_client.set_permissions.called, is_(True), "Permissions called")
self.mock_file_system_client.set_permissions.assert_called_with(new_directories[0])
assert_that(copied_count, is_(1), "copied_count")
