class TestFileSystemClient(unittest.TestCase):
def setUp(self):
self.root_dir = "/tmp/majic_test" # same as in delete_dir_test.sh
try:
os.mkdir(self.root_dir)
except OSError as ex:
if ex.errno != 17:
raise ex
else:
self.tearDown()
os.mkdir(self.root_dir)
self.filename = os.path.join(self.root_dir, "test_file")
f = open(self.filename, "w")
f.write("testing")
f.close()
config = ConfigMother.test_configuration_with_values(file_root_path=self.root_dir)
self.file_system_client = FileSystemClient(config)
def tearDown(self):
try:
shutil.rmtree(self.root_dir)
except OSError as e:
if e.errno != 2: # file doesn't exist
config = ConfigMother.test_configuration_with_values(file_root_path=self.root_dir)
file_system_client = FileSystemClient(config)
file_system_client.delete_directory("run1")
shutil.rmtree(self.root_dir)
def test_GIVEN_file_WHEN_from_path_THEN_properties_set(self):
props = self.file_system_client.get_file_properties(self.filename)
assert_that(props.file_path, is_(self.filename), "file path")
assert_that(props.owner, is_(getpass.getuser()), "file path")
def test_GIVEN_file_is_group_read_WHEN_from_path_THEN_properties_is_published_butnot_public(self):
os.chmod(self.filename, S_IRUSR | S_IWUSR | S_IRGRP)
props = self.file_system_client.get_file_properties(self.filename)
assert_that(props.is_published, is_(True), "published")
assert_that(props.is_public, is_(False), "public")
def test_GIVEN_file_is_other_read_WHEN_from_path_THEN_properties_is_not_published_but_is_public(self):
# this is a funny situation to be in
os.chmod(self.filename, S_IRUSR | S_IWUSR | S_IROTH)
props = self.file_system_client.get_file_properties(self.filename)
assert_that(props.is_published, is_(False), "published")
assert_that(props.is_public, is_(True), "public")
def test_GIVEN_file_is_group_read_and_other_read_WHEN_from_path_THEN_properties_is_published_and_is_public(self):
os.chmod(self.filename, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)
props = self.file_system_client.get_file_properties(self.filename)
assert_that(props.is_published, is_(True), "published")
assert_that(props.is_public, is_(True), "public")
def test_GIVEN_file_is_not_read_WHEN_from_path_THEN_properties_is_not_published_but_is_not_public(self):
os.chmod(self.filename, S_IRUSR | S_IWUSR)
props = self.file_system_client.get_file_properties(self.filename)
assert_that(props.is_published, is_(False), "published")
assert_that(props.is_public, is_(False), "public")
def test_GIVEN_dir_WHEN_create_directory_created(self):
file_path = "tmp"
self.file_system_client.create_dir(file_path)
assert_that(os.path.isdir(os.path.join(self.root_dir, file_path)), "Path is not directory or doesn't exist")
def test_GIVEN_dir_in_non_existant_dir_WHEN_create_THEN_directory_tree_created(self):
file_path = "tmp/sub/sub/end"
self.file_system_client.create_dir(file_path)
assert_that(os.path.isdir(os.path.join(self.root_dir, file_path)), "Path is not directory or doesn't exist")
def test_GIVEN_dir_already_exists_WHEN_create_directory_created(self):
file_path = "tmp"
self.file_system_client.create_dir(file_path)
self.file_system_client.create_dir(file_path)
assert_that(os.path.isdir(os.path.join(self.root_dir, file_path)), "Path still exists")
def test_GIVEN_any_other_error_WHEN_create_THEN_exception_thrown_as_apache_file_error(self):
self.file_system_client._filesystem_root_for_app = "/data"
assert_that(calling(self.file_system_client.create_dir).with_args("1"), raises(FileSystemClientError))
def test_GIVEN_dir_referenced_dir_outside_of_root_WHEN_create_directory_THEN_exception(self):
file_path = "../john"
assert_that(
calling(self.file_system_client.create_dir).with_args(file_path),
raises(Exception, "Code is trying to access a directory outside root"),
)
def test_GIVEN_file_WHEN_create_THEN_file_created(self):
file_path = "tmp"
fileobj = self.file_system_client.create_file(file_path)
fileobj.write("hi")
assert_that(os.path.isfile(os.path.join(self.root_dir, file_path)), "Path is not a file or doesn't exist")
def test_GIVEN_file_exists_WHEN_create_THEN_non_returned(self):
file_path = self.filename
fileobj = self.file_system_client.create_file(file_path)
assert_that(fileobj, none(), "File object is none")
def test_GIVEN_file_in_non_writable_directory_WHEN_create_THEN_file_created_after_permissions_changed(self):
self.create_file_and_sub_dir()
file_path = os.path.join(self.model_run_path, "tmp")
self.file_system_client.set_permissions(FileProperties(self.model_run_path, "nobody", False, False))
fileobj = self.file_system_client.create_file(file_path)
fileobj.write("hi")
assert_that(os.path.isfile(os.path.join(self.root_dir, file_path)), "Path is not a file or doesn't exist")
def test_GIVEN_dir_in_non_writable_directory_WHEN_create_THEN_file_created_after_permissions_changed(self):
self.create_file_and_sub_dir()
file_path = os.path.join(self.model_run_path, "tmp")
self.file_system_client.set_permissions(FileProperties(self.model_run_path, "nobody", False, False))
self.file_system_client.create_dir(file_path)
assert_that(os.path.isdir(os.path.join(self.root_dir, file_path)), "Path is not a file or doesn't exist")
def test_GIVEN_any_error_thrown_WHEN_create_THEN_FileSystemClientError(self):
file_path = "doesntexists/file.txt"
assert_that(calling(self.file_system_client.create_file).with_args(file_path), raises(FileSystemClientError))
def test_GIVEN_file_WHEN_close_THEN_file_closed(self):
file_path = "tmp"
fileobj = self.file_system_client.create_file(file_path)
self.file_system_client.close_file(fileobj)
assert_that(fileobj.closed, "File is not closed")
def test_GIVEN_exception_WHEN_close_THEN_FileSystemClientError(self):
assert_that(calling(self.file_system_client.close_file).with_args(None), raises(FileSystemClientError))
def test_GIVEN_file_WHEN_close_and_delete_THEN_file_deleted(self):
file_path = "tmp"
fileobj = self.file_system_client.create_file(file_path)
fileobj.write("hi")
self.file_system_client.close_and_delete_file(fileobj)
assert_that(os.path.isfile(os.path.join(self.root_dir, file_path)), is_(False), "Path exists")
def test_GIVEN_dir_with_file_WHEN_delete_directory_THEN_directory_deleted(self):
file_path = "run1"
subdir = "blah"
self.file_system_client.create_dir(file_path)
sub_dir_path = os.path.join(file_path, subdir)
self.file_system_client.create_dir(sub_dir_path)
f = self.file_system_client.create_file(os.path.join(sub_dir_path, "file"))
f.write("hi there")
self.file_system_client.close_file(f)
self.file_system_client.delete_directory(file_path)
assert_that(os.path.exists(os.path.join(self.root_dir, file_path)), is_(False), "Path still exists")
def test_GIVEN_error_WHEN_delete_directory_THEN_exception(self):
file_path = ""
assert_that(
calling(self.file_system_client.delete_directory).with_args(file_path), raises(FileSystemClientError)
)
def create_file_and_sub_dir(self):
self.model_run_path = "run1"
subdir = "blah"
self.file_system_client.create_dir(self.model_run_path)
sub_dir_path = os.path.join(self.model_run_path, subdir)
self.file_system_client.create_dir(sub_dir_path)
file_path = os.path.join(sub_dir_path, "file")
f = self.file_system_client.create_file(file_path)
f.write("hi there")
self.file_system_client.close_file(f)
return file_path
def test_GIVEN_dir_WHEN_set_permissions_FF_THEN_premissions_set(self):
file_path = self.create_file_and_sub_dir()
file_property = FileProperties(file_path, getpass.getuser(), False, False)
self.file_system_client.set_permissions(file_property)
assert_that(self.file_system_client.get_file_properties(file_path), is_(file_property), "File properties")
def test_GIVEN_dir_WHEN_set_permissions_FT_THEN_premissions_set(self):
file_path = self.create_file_and_sub_dir()
file_property = FileProperties(file_path, getpass.getuser(), False, True)
self.file_system_client.set_permissions(file_property)
assert_that(self.file_system_client.get_file_properties(file_path), is_(file_property), "File properties")
def test_GIVEN_dir_WHEN_set_permissions_TF_THEN_premissions_set(self):
file_path = self.create_file_and_sub_dir()
file_property = FileProperties(file_path, getpass.getuser(), True, False)
self.file_system_client.set_permissions(file_property)
assert_that(self.file_system_client.get_file_properties(file_path), is_(file_property), "File properties")
def test_GIVEN_dir_WHEN_set_permissions_TT_THEN_premissions_set(self):
file_path = self.create_file_and_sub_dir()
file_property = FileProperties(file_path, "root", True, True)
self.file_system_client.set_permissions(file_property)
assert_that(self.file_system_client.get_file_properties(file_path), is_(file_property), "File properties")
def test_GIVEN_dir_WHEN_list_THEN_directories_returned(self):
file_path = self.create_file_and_sub_dir()
dirs = self.file_system_client.list_dirs("")
assert_that(dirs, contains(self.model_run_path), "Directories")
class FileSystemComparer(object):
"""
Compare the list of directories from the data path on the file system with a list of model and detects changes
On perform_analysis the following are populated:
new_directories - with FileProperties for models that needs to be copied
deleted_directories - with directory names which need deleting
changed_directories - with FileProperties of directories which need their permissions updating
existing_non_deleted_directories - directories which are neither new of deleted
"""
def __init__(
self,
config,
file_system_client=None):
"""
Constructor
:param config: configuration for file system comparer
:param file_system_client: the file system client
:return:
"""
self._config = config
self.data_path = self._config.get(CONFIG_DATA_PATH, CONFIG_DATA_SECTION)
if file_system_client is not None:
self._file_system_client = file_system_client
else:
self._file_system_client = FileSystemClient(self._config)
self.new_directories = None
self.deleted_directories = None
self.changed_directories = None
self.existing_non_deleted_directories = None
def perform_analysis(self, model_properties):
"""
Perform the analysis of the file system compared to the model properties
:param model_properties: model properties expected to appear
:return: nothing but set internal properties for new, changed and deleted files
"""
self.new_directories = []
self.deleted_directories = []
self.existing_non_deleted_directories = []
self.changed_directories = []
existing_run_ids = set()
existing_directories = self._file_system_client.list_dirs(self.data_path)
for existing_directory in existing_directories:
match = re.match('run(\d+)', existing_directory)
if match:
existing_run_ids.add(int(match.group(1)))
model_run_ids = set()
file_properties_for_model_runs = {}
for model_property in model_properties:
model_run_id = model_property[JSON_MODEL_RUN_ID]
model_run_ids.add(model_run_id)
file_properties_for_model_runs[model_run_id] = FileProperties(
self._create_model_dir(model_run_id),
model_property[JSON_USER_NAME],
model_property[JSON_IS_PUBLISHED],
model_property.get(JSON_IS_PUBLIC, False))
for model_run_id in model_run_ids.difference(existing_run_ids):
self.new_directories.append(file_properties_for_model_runs[model_run_id])
for model_run_id in existing_run_ids.difference(model_run_ids):
self.deleted_directories.append(self._create_model_dir(model_run_id))
existing_run_ids.intersection_update(model_run_ids)
for model_run_id in existing_run_ids:
file_properties_to_set = file_properties_for_model_runs[model_run_id]
self.existing_non_deleted_directories.append(file_properties_to_set)
current_file_properties = self._file_system_client.get_file_properties(file_properties_to_set.file_path)
if current_file_properties != file_properties_for_model_runs[model_run_id]:
self.changed_directories.append(file_properties_to_set)
def _create_model_dir(self, model_run_id):
"""
Create the model run dir path from the model run id
:param model_run_id: model run id
:return: the full path to the model run directory
"""
# ensure that the model id is a number so it can not possible contain path elements
model_run_id_dir = str(model_run_id)
assert(model_run_id_dir.isdigit())
dir_name = "{}{}".format(MODEL_RUN_DIR_PREFIX, model_run_id)
return os.path.join(self.data_path, dir_name)
def add_extra_directories_to_sync(self):
"""
Add extra directories to synchronise as public data
:return: nothing
"""
owner = getuser()
for dir_name in self._config.get(CONFIG_EXTRA_DIRS_TO_SYNC, CONFIG_DATA_SECTION).split():
dir_properties = FileProperties(dir_name, owner, True, True)
self.existing_non_deleted_directories.append(dir_properties)
