class TestFileSystemClient(unittest.TestCase): def setUp(self): self.root_dir = "/tmp/majic_test" # same as in delete_dir_test.sh try: os.mkdir(self.root_dir) except OSError as ex: if ex.errno != 17: raise ex else: self.tearDown() os.mkdir(self.root_dir) self.filename = os.path.join(self.root_dir, "test_file") f = open(self.filename, "w") f.write("testing") f.close() config = ConfigMother.test_configuration_with_values(file_root_path=self.root_dir) self.file_system_client = FileSystemClient(config) def tearDown(self): try: shutil.rmtree(self.root_dir) except OSError as e: if e.errno != 2: # file doesn't exist config = ConfigMother.test_configuration_with_values(file_root_path=self.root_dir) file_system_client = FileSystemClient(config) file_system_client.delete_directory("run1") shutil.rmtree(self.root_dir) def test_GIVEN_file_WHEN_from_path_THEN_properties_set(self): props = self.file_system_client.get_file_properties(self.filename) assert_that(props.file_path, is_(self.filename), "file path") assert_that(props.owner, is_(getpass.getuser()), "file path") def test_GIVEN_file_is_group_read_WHEN_from_path_THEN_properties_is_published_butnot_public(self): os.chmod(self.filename, S_IRUSR | S_IWUSR | S_IRGRP) props = self.file_system_client.get_file_properties(self.filename) assert_that(props.is_published, is_(True), "published") assert_that(props.is_public, is_(False), "public") def test_GIVEN_file_is_other_read_WHEN_from_path_THEN_properties_is_not_published_but_is_public(self): # this is a funny situation to be in os.chmod(self.filename, S_IRUSR | S_IWUSR | S_IROTH) props = self.file_system_client.get_file_properties(self.filename) assert_that(props.is_published, is_(False), "published") assert_that(props.is_public, is_(True), "public") def test_GIVEN_file_is_group_read_and_other_read_WHEN_from_path_THEN_properties_is_published_and_is_public(self): os.chmod(self.filename, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) props = self.file_system_client.get_file_properties(self.filename) assert_that(props.is_published, is_(True), "published") assert_that(props.is_public, is_(True), "public") def test_GIVEN_file_is_not_read_WHEN_from_path_THEN_properties_is_not_published_but_is_not_public(self): os.chmod(self.filename, S_IRUSR | S_IWUSR) props = self.file_system_client.get_file_properties(self.filename) assert_that(props.is_published, is_(False), "published") assert_that(props.is_public, is_(False), "public") def test_GIVEN_dir_WHEN_create_directory_created(self): file_path = "tmp" self.file_system_client.create_dir(file_path) assert_that(os.path.isdir(os.path.join(self.root_dir, file_path)), "Path is not directory or doesn't exist") def test_GIVEN_dir_in_non_existant_dir_WHEN_create_THEN_directory_tree_created(self): file_path = "tmp/sub/sub/end" self.file_system_client.create_dir(file_path) assert_that(os.path.isdir(os.path.join(self.root_dir, file_path)), "Path is not directory or doesn't exist") def test_GIVEN_dir_already_exists_WHEN_create_directory_created(self): file_path = "tmp" self.file_system_client.create_dir(file_path) self.file_system_client.create_dir(file_path) assert_that(os.path.isdir(os.path.join(self.root_dir, file_path)), "Path still exists") def test_GIVEN_any_other_error_WHEN_create_THEN_exception_thrown_as_apache_file_error(self): self.file_system_client._filesystem_root_for_app = "/data" assert_that(calling(self.file_system_client.create_dir).with_args("1"), raises(FileSystemClientError)) def test_GIVEN_dir_referenced_dir_outside_of_root_WHEN_create_directory_THEN_exception(self): file_path = "../john" assert_that( calling(self.file_system_client.create_dir).with_args(file_path), raises(Exception, "Code is trying to access a directory outside root"), ) def test_GIVEN_file_WHEN_create_THEN_file_created(self): file_path = "tmp" fileobj = self.file_system_client.create_file(file_path) fileobj.write("hi") assert_that(os.path.isfile(os.path.join(self.root_dir, file_path)), "Path is not a file or doesn't exist") def test_GIVEN_file_exists_WHEN_create_THEN_non_returned(self): file_path = self.filename fileobj = self.file_system_client.create_file(file_path) assert_that(fileobj, none(), "File object is none") def test_GIVEN_file_in_non_writable_directory_WHEN_create_THEN_file_created_after_permissions_changed(self): self.create_file_and_sub_dir() file_path = os.path.join(self.model_run_path, "tmp") self.file_system_client.set_permissions(FileProperties(self.model_run_path, "nobody", False, False)) fileobj = self.file_system_client.create_file(file_path) fileobj.write("hi") assert_that(os.path.isfile(os.path.join(self.root_dir, file_path)), "Path is not a file or doesn't exist") def test_GIVEN_dir_in_non_writable_directory_WHEN_create_THEN_file_created_after_permissions_changed(self): self.create_file_and_sub_dir() file_path = os.path.join(self.model_run_path, "tmp") self.file_system_client.set_permissions(FileProperties(self.model_run_path, "nobody", False, False)) self.file_system_client.create_dir(file_path) assert_that(os.path.isdir(os.path.join(self.root_dir, file_path)), "Path is not a file or doesn't exist") def test_GIVEN_any_error_thrown_WHEN_create_THEN_FileSystemClientError(self): file_path = "doesntexists/file.txt" assert_that(calling(self.file_system_client.create_file).with_args(file_path), raises(FileSystemClientError)) def test_GIVEN_file_WHEN_close_THEN_file_closed(self): file_path = "tmp" fileobj = self.file_system_client.create_file(file_path) self.file_system_client.close_file(fileobj) assert_that(fileobj.closed, "File is not closed") def test_GIVEN_exception_WHEN_close_THEN_FileSystemClientError(self): assert_that(calling(self.file_system_client.close_file).with_args(None), raises(FileSystemClientError)) def test_GIVEN_file_WHEN_close_and_delete_THEN_file_deleted(self): file_path = "tmp" fileobj = self.file_system_client.create_file(file_path) fileobj.write("hi") self.file_system_client.close_and_delete_file(fileobj) assert_that(os.path.isfile(os.path.join(self.root_dir, file_path)), is_(False), "Path exists") def test_GIVEN_dir_with_file_WHEN_delete_directory_THEN_directory_deleted(self): file_path = "run1" subdir = "blah" self.file_system_client.create_dir(file_path) sub_dir_path = os.path.join(file_path, subdir) self.file_system_client.create_dir(sub_dir_path) f = self.file_system_client.create_file(os.path.join(sub_dir_path, "file")) f.write("hi there") self.file_system_client.close_file(f) self.file_system_client.delete_directory(file_path) assert_that(os.path.exists(os.path.join(self.root_dir, file_path)), is_(False), "Path still exists") def test_GIVEN_error_WHEN_delete_directory_THEN_exception(self): file_path = "" assert_that( calling(self.file_system_client.delete_directory).with_args(file_path), raises(FileSystemClientError) ) def create_file_and_sub_dir(self): self.model_run_path = "run1" subdir = "blah" self.file_system_client.create_dir(self.model_run_path) sub_dir_path = os.path.join(self.model_run_path, subdir) self.file_system_client.create_dir(sub_dir_path) file_path = os.path.join(sub_dir_path, "file") f = self.file_system_client.create_file(file_path) f.write("hi there") self.file_system_client.close_file(f) return file_path def test_GIVEN_dir_WHEN_set_permissions_FF_THEN_premissions_set(self): file_path = self.create_file_and_sub_dir() file_property = FileProperties(file_path, getpass.getuser(), False, False) self.file_system_client.set_permissions(file_property) assert_that(self.file_system_client.get_file_properties(file_path), is_(file_property), "File properties") def test_GIVEN_dir_WHEN_set_permissions_FT_THEN_premissions_set(self): file_path = self.create_file_and_sub_dir() file_property = FileProperties(file_path, getpass.getuser(), False, True) self.file_system_client.set_permissions(file_property) assert_that(self.file_system_client.get_file_properties(file_path), is_(file_property), "File properties") def test_GIVEN_dir_WHEN_set_permissions_TF_THEN_premissions_set(self): file_path = self.create_file_and_sub_dir() file_property = FileProperties(file_path, getpass.getuser(), True, False) self.file_system_client.set_permissions(file_property) assert_that(self.file_system_client.get_file_properties(file_path), is_(file_property), "File properties") def test_GIVEN_dir_WHEN_set_permissions_TT_THEN_premissions_set(self): file_path = self.create_file_and_sub_dir() file_property = FileProperties(file_path, "root", True, True) self.file_system_client.set_permissions(file_property) assert_that(self.file_system_client.get_file_properties(file_path), is_(file_property), "File properties") def test_GIVEN_dir_WHEN_list_THEN_directories_returned(self): file_path = self.create_file_and_sub_dir() dirs = self.file_system_client.list_dirs("") assert_that(dirs, contains(self.model_run_path), "Directories")
class FileSystemComparer(object): """ Compare the list of directories from the data path on the file system with a list of model and detects changes On perform_analysis the following are populated: new_directories - with FileProperties for models that needs to be copied deleted_directories - with directory names which need deleting changed_directories - with FileProperties of directories which need their permissions updating existing_non_deleted_directories - directories which are neither new of deleted """ def __init__( self, config, file_system_client=None): """ Constructor :param config: configuration for file system comparer :param file_system_client: the file system client :return: """ self._config = config self.data_path = self._config.get(CONFIG_DATA_PATH, CONFIG_DATA_SECTION) if file_system_client is not None: self._file_system_client = file_system_client else: self._file_system_client = FileSystemClient(self._config) self.new_directories = None self.deleted_directories = None self.changed_directories = None self.existing_non_deleted_directories = None def perform_analysis(self, model_properties): """ Perform the analysis of the file system compared to the model properties :param model_properties: model properties expected to appear :return: nothing but set internal properties for new, changed and deleted files """ self.new_directories = [] self.deleted_directories = [] self.existing_non_deleted_directories = [] self.changed_directories = [] existing_run_ids = set() existing_directories = self._file_system_client.list_dirs(self.data_path) for existing_directory in existing_directories: match = re.match('run(\d+)', existing_directory) if match: existing_run_ids.add(int(match.group(1))) model_run_ids = set() file_properties_for_model_runs = {} for model_property in model_properties: model_run_id = model_property[JSON_MODEL_RUN_ID] model_run_ids.add(model_run_id) file_properties_for_model_runs[model_run_id] = FileProperties( self._create_model_dir(model_run_id), model_property[JSON_USER_NAME], model_property[JSON_IS_PUBLISHED], model_property.get(JSON_IS_PUBLIC, False)) for model_run_id in model_run_ids.difference(existing_run_ids): self.new_directories.append(file_properties_for_model_runs[model_run_id]) for model_run_id in existing_run_ids.difference(model_run_ids): self.deleted_directories.append(self._create_model_dir(model_run_id)) existing_run_ids.intersection_update(model_run_ids) for model_run_id in existing_run_ids: file_properties_to_set = file_properties_for_model_runs[model_run_id] self.existing_non_deleted_directories.append(file_properties_to_set) current_file_properties = self._file_system_client.get_file_properties(file_properties_to_set.file_path) if current_file_properties != file_properties_for_model_runs[model_run_id]: self.changed_directories.append(file_properties_to_set) def _create_model_dir(self, model_run_id): """ Create the model run dir path from the model run id :param model_run_id: model run id :return: the full path to the model run directory """ # ensure that the model id is a number so it can not possible contain path elements model_run_id_dir = str(model_run_id) assert(model_run_id_dir.isdigit()) dir_name = "{}{}".format(MODEL_RUN_DIR_PREFIX, model_run_id) return os.path.join(self.data_path, dir_name) def add_extra_directories_to_sync(self): """ Add extra directories to synchronise as public data :return: nothing """ owner = getuser() for dir_name in self._config.get(CONFIG_EXTRA_DIRS_TO_SYNC, CONFIG_DATA_SECTION).split(): dir_properties = FileProperties(dir_name, owner, True, True) self.existing_non_deleted_directories.append(dir_properties)