class Synchroniser(object):
"""
Provides services to synchronisation file
"""
def __init__(self,
configuration,
majic_webservice_client=None,
file_system_comparer=None,
directory_synchroniser=None):
"""
Constructor
:param configuration: the configuration to use for the synchronisation
:param majic_webservice_client: the majic web service to use
:return: nothing
"""
super(Synchroniser, self).__init__()
self._config = configuration
if majic_webservice_client is None:
self._majic_webservice_client = MajicWebserviceClient(self._config)
else:
self._majic_webservice_client = majic_webservice_client
if file_system_comparer is None:
self._file_system_comparer = FileSystemComparer(self._config)
else:
self._file_system_comparer = file_system_comparer
if directory_synchroniser is None:
self._directory_synchroniser = DirectorySynchroniser(self._config)
else:
self._directory_synchroniser = directory_synchroniser
def synchronise(self):
"""
Synchronise the files returned by the web service with those on the disc
:return: error code to exit with
"""
try:
log.debug("Starting to sync")
model_propeties = self._majic_webservice_client.get_properties_list_with_filtered_users()
self._file_system_comparer.perform_analysis(model_propeties)
self._file_system_comparer.add_extra_directories_to_sync()
new_count, updated_count, deleted_count = \
self._directory_synchroniser.synchronise_all(self._file_system_comparer)
log.info("Finished Synchronisation:")
log.info(" {}: Count of copied files and directories: ".format(new_count))
log.info(" {}: Count of directory permissions updated".format(updated_count))
log.info(" {}: Count of deleted directories".format(deleted_count))
return 0
except UserPrintableError as ex:
log.error(str(ex))
return 1
except Exception:
log.exception("Unknown error in synchronisation")
log.error("An unknown error occurred so files are not synced")
return 2
def test_GIVEN_non_run_directory_WHEN_compare_THEN_no_directories_deleted(self):
filelist = ["blah"]
self.mock_file_system_client.list_dirs = Mock(return_value=filelist)
model_properties = []
file_system_comparer = FileSystemComparer(self.config, self.mock_file_system_client)
file_system_comparer.perform_analysis(model_properties)
assert_that_file_system_comparer(file_system_comparer)
def test_GIVEN_one_extra_entry_on_file_list_WHEN_compare_THEN_extra_entry_on_del_list(self):
runids = [1, 10]
filelist = self.create_filelist_and_mock_file_stats(runids)
expected_deleted_dirs = ["{}/run{}".format(self.expected_path, runid) for runid in runids]
model_properties = []
file_system_comparer = FileSystemComparer(self.config, self.mock_file_system_client)
file_system_comparer.perform_analysis(model_properties)
assert_that_file_system_comparer(file_system_comparer, expected_deleted_dirs=expected_deleted_dirs)
def test_GIVEN_one_extra_entry_on_model_list_WHEN_compare_THEN_extra_entry_is_in_new_dirs_list(self):
runid = 12
expected_new_dir = [FileProperties("{}/run{}".format(self.expected_path, runid), self.model_owner, False, False)]
filelist = self.create_filelist_and_mock_file_stats([])
model_properties = RunModelPropertiesMother.create_model_run_properties([runid])
file_system_comparer = FileSystemComparer(self.config, self.mock_file_system_client)
file_system_comparer.perform_analysis(model_properties)
assert_that_file_system_comparer(file_system_comparer, expected_new_dirs=expected_new_dir)
def test_GIVEN_existing_file_owned_by_someone_else_WHEN_compare_THEN_directory_is_set_for_change(self):
runids = [23]
expected_owner = "different_username"
model_properties = RunModelPropertiesMother.create_model_run_properties(runids, owner=expected_owner)
filelist = self.create_filelist_and_mock_file_stats(runids)
expected_changed_dirs = [FileProperties("{}/{}".format(self.expected_path, file), expected_owner, False, False) for file in filelist]
file_system_comparer = FileSystemComparer(self.config, self.mock_file_system_client)
file_system_comparer.perform_analysis(model_properties)
assert_that_file_system_comparer(file_system_comparer, expected_changed_dirs=expected_changed_dirs,
existing_non_deleted_directories=expected_changed_dirs)
def test_GIVEN_existing_file_is_not_public_and_model_is_WHEN_compare_THEN_directory_is_set_for_change(self):
runids = [23]
model_properties = RunModelPropertiesMother.create_model_run_properties(runids, public=True)
filelist = self.create_filelist_and_mock_file_stats(runids)
for mock_file_stat in self.mock_file_stats.values():
mock_file_stat.is_published = False
expected_changed_dirs = [FileProperties("{}/{}".format(self.expected_path, file), self.model_owner, False, True) for file in filelist]
file_system_comparer = FileSystemComparer(self.config, self.mock_file_system_client)
file_system_comparer.perform_analysis(model_properties)
assert_that_file_system_comparer(file_system_comparer, expected_changed_dirs=expected_changed_dirs,
existing_non_deleted_directories=expected_changed_dirs)
def test_GIVEN_two_extra_models_WHEN_compare_THEN_models_appear_on_new_dirs_list(self):
runids = [100, 12]
expected_new_dirs = [FileProperties("{}/run{}".format(
self.expected_path, runid),
self.model_owner,
False,
False) for runid in runids]
filelist = self.create_filelist_and_mock_file_stats([])
model_properties = RunModelPropertiesMother.create_model_run_properties(runids)
file_system_comparer = FileSystemComparer(self.config, self.mock_file_system_client)
file_system_comparer.perform_analysis(model_properties)
assert_that_file_system_comparer(file_system_comparer, expected_new_dirs=expected_new_dirs)
def __init__(self,
configuration,
majic_webservice_client=None,
file_system_comparer=None,
directory_synchroniser=None):
"""
Constructor
:param configuration: the configuration to use for the synchronisation
:param majic_webservice_client: the majic web service to use
:return: nothing
"""
super(Synchroniser, self).__init__()
self._config = configuration
if majic_webservice_client is None:
self._majic_webservice_client = MajicWebserviceClient(self._config)
else:
self._majic_webservice_client = majic_webservice_client
if file_system_comparer is None:
self._file_system_comparer = FileSystemComparer(self._config)
else:
self._file_system_comparer = file_system_comparer
if directory_synchroniser is None:
self._directory_synchroniser = DirectorySynchroniser(self._config)
else:
self._directory_synchroniser = directory_synchroniser
def test_GIVEN_entries_in_file_list_same_as_model_list_WHEN_compare_THEN_no_extra_entires(self):
runids = [100, 12]
expected_new_dirs = []
expected_existing_non_deleted_directories = [FileProperties("{}/run{}".format(
self.expected_path, runid),
self.model_owner,
False,
False) for runid in runids]
filelist = self.create_filelist_and_mock_file_stats(runids)
model_properties = RunModelPropertiesMother.create_model_run_properties(runids)
file_system_comparer = FileSystemComparer(self.config, self.mock_file_system_client)
file_system_comparer.perform_analysis(model_properties)
assert_that_file_system_comparer(file_system_comparer, expected_new_dirs=expected_new_dirs,
existing_non_deleted_directories=expected_existing_non_deleted_directories)
def test_GIVEN_extra_directory_WHEN_add_extra_directories_to_sync_THEN_extra_directory_added(self):
added_dir = "data"
self.config = ConfigMother.test_configuration_with_values(extra_directories_to_sync=added_dir)
expected_added_dir = [FileProperties("data", getuser(), True, True)]
file_system_comparer = FileSystemComparer(self.config, self.mock_file_system_client)
file_system_comparer.new_directories = []
file_system_comparer.deleted_directories = []
file_system_comparer.existing_non_deleted_directories = []
file_system_comparer.changed_directories = []
file_system_comparer.add_extra_directories_to_sync()
assert_that_file_system_comparer(file_system_comparer, existing_non_deleted_directories=expected_added_dir)
