def _export_linked_datatypes(self, project, zip_file):
files_helper = FilesHelper()
linked_paths = self._get_linked_datatypes_storage_path(project)
if not linked_paths:
# do not export an empty operation
return
# Make a import operation which will contain links to other projects
algo = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE, TVB_IMPORTER_CLASS)
op = model.Operation(None, project.id, algo.id, '')
op.project = project
op.algorithm = algo
op.id = 'links-to-external-projects'
op.start_now()
op.mark_complete(model.STATUS_FINISHED)
# write operation.xml to disk
files_helper.write_operation_metadata(op)
op_folder = files_helper.get_operation_folder(op.project.name, op.id)
operation_xml = files_helper.get_operation_meta_file_path(op.project.name, op.id)
op_folder_name = os.path.basename(op_folder)
# add operation.xml
zip_file.write(operation_xml, op_folder_name + '/' + os.path.basename(operation_xml))
# add linked datatypes to archive in the import operation
for pth in linked_paths:
zip_pth = op_folder_name + '/' + os.path.basename(pth)
zip_file.write(pth, zip_pth)
# remove these files, since we only want them in export archive
files_helper.remove_folder(op_folder)
def _export_linked_datatypes(self, project, zip_file):
files_helper = FilesHelper()
linked_paths = self._get_linked_datatypes_storage_path(project)
if not linked_paths:
# do not export an empty operation
return
# Make a import operation which will contain links to other projects
alg_group = dao.find_group(TVB_IMPORTER_MODULE, TVB_IMPORTER_CLASS)
algo = dao.get_algorithm_by_group(alg_group.id)
op = model.Operation(None, project.id, algo.id, '')
op.project = project
op.algorithm = algo
op.id = 'links-to-external-projects'
op.start_now()
op.mark_complete(model.STATUS_FINISHED)
# write operation.xml to disk
files_helper.write_operation_metadata(op)
op_folder = files_helper.get_operation_folder(op.project.name, op.id)
operation_xml = files_helper.get_operation_meta_file_path(
op.project.name, op.id)
op_folder_name = os.path.basename(op_folder)
# add operation.xml
zip_file.write(operation_xml,
op_folder_name + '/' + os.path.basename(operation_xml))
# add linked datatypes to archive in the import operation
for pth in linked_paths:
zip_pth = op_folder_name + '/' + os.path.basename(pth)
zip_file.write(pth, zip_pth)
# remove these files, since we only want them in export archive
files_helper.remove_folder(op_folder)
class FilesHelperTest(TransactionalTestCase):
"""
This class contains tests for the tvb.core.entities.file.files_helper module.
"""
PROJECT_NAME = "test_proj"
def setUp(self):
"""
Set up the context needed by the tests.
"""
self.files_helper = FilesHelper()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME)
def tearDown(self):
""" Remove generated project during tests. """
self.delete_project_folders()
def test_check_created(self):
""" Test standard flows for check created. """
self.files_helper.check_created()
self.assertTrue(os.path.exists(root_storage), "Storage not created!")
self.files_helper.check_created(os.path.join(root_storage, "test"))
self.assertTrue(os.path.exists(root_storage), "Storage not created!")
self.assertTrue(os.path.exists(os.path.join(root_storage, "test")), "Test directory not created!")
def test_get_project_folder(self):
"""
Test the get_project_folder method which should create a folder in case
it doesn't already exist.
"""
project_path = self.files_helper.get_project_folder(self.test_project)
self.assertTrue(os.path.exists(project_path), "Folder doesn't exist")
folder_path = self.files_helper.get_project_folder(self.test_project, "43")
self.assertTrue(os.path.exists(project_path), "Folder doesn't exist")
self.assertTrue(os.path.exists(folder_path), "Folder doesn't exist")
def test_rename_project_structure(self):
""" Try to rename the folder structure of a project. Standard flow. """
self.files_helper.get_project_folder(self.test_project)
path, name = self.files_helper.rename_project_structure(self.test_project.name, "new_name")
self.assertNotEqual(path, name, "Rename didn't take effect.")
def test_rename_structure_same_name(self):
""" Try to rename the folder structure of a project. Same name. """
self.files_helper.get_project_folder(self.test_project)
self.assertRaises(FileStructureException, self.files_helper.rename_project_structure,
self.test_project.name, self.PROJECT_NAME)
def test_remove_project_structure(self):
""" Check that remove project structure deletes the corresponding folder. Standard flow. """
full_path = self.files_helper.get_project_folder(self.test_project)
self.assertTrue(os.path.exists(full_path), "Folder was not created.")
self.files_helper.remove_project_structure(self.test_project.name)
self.assertFalse(os.path.exists(full_path), "Project folder not deleted.")
def test_write_project_metadata(self):
""" Write XML for test-project. """
self.files_helper.write_project_metadata(self.test_project)
expected_file = self.files_helper.get_project_meta_file_path(self.PROJECT_NAME)
self.assertTrue(os.path.exists(expected_file))
project_meta = XMLReader(expected_file).read_metadata()
loaded_project = model.Project(None, None)
loaded_project.from_dict(project_meta, self.test_user.id)
self.assertEqual(self.test_project.name, loaded_project.name)
self.assertEqual(self.test_project.description, loaded_project.description)
self.assertEqual(self.test_project.gid, loaded_project.gid)
expected_dict = self.test_project.to_dict()[1]
del expected_dict['last_updated']
found_dict = loaded_project.to_dict()[1]
del found_dict['last_updated']
self.assertDictContainsSubset(expected_dict, found_dict)
self.assertDictContainsSubset(found_dict, expected_dict)
def test_write_operation_metadata(self):
"""
Test that a correct XML is created for an operation.
"""
operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)
expected_file = self.files_helper.get_operation_meta_file_path(self.PROJECT_NAME, operation.id)
self.assertFalse(os.path.exists(expected_file))
self.files_helper.write_operation_metadata(operation)
self.assertTrue(os.path.exists(expected_file))
operation_meta = XMLReader(expected_file).read_metadata()
loaded_operation = model.Operation(None, None, None, None)
loaded_operation.from_dict(operation_meta, dao)
expected_dict = operation.to_dict()[1]
found_dict = loaded_operation.to_dict()[1]
for key, value in expected_dict.iteritems():
self.assertEqual(str(value), str(found_dict[key]))
# Now validate that operation metaData can be also updated
self.assertNotEqual("new_group_name", found_dict['user_group'])
self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id)
found_dict = XMLReader(expected_file).read_metadata()
self.assertEqual("new_group_name", found_dict['user_group'])
def test_remove_dt_happy_flow(self):
"""
Happy flow for removing a file related to a DataType.
"""
folder_path = self.files_helper.get_project_folder(self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
open(datatype.get_storage_file_path(), 'w')
self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!")
self.files_helper.remove_datatype(datatype)
self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not deleted!")
def test_remove_dt_non_existent(self):
"""
Try to call remove on a dataType with no H5 file.
Should work.
"""
folder_path = self.files_helper.get_project_folder(self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
self.assertFalse(os.path.exists(datatype.get_storage_file_path()))
self.files_helper.remove_datatype(datatype)
def test_move_datatype(self):
"""
Make sure associated H5 file is moved to a correct new location.
"""
folder_path = self.files_helper.get_project_folder(self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
open(datatype.get_storage_file_path(), 'w')
self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!")
self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '11', "43")
self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not moved!")
datatype.storage_path = self.files_helper.get_project_folder(self.PROJECT_NAME + '11', "43")
self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!")
def test_find_relative_path(self):
"""
Tests that relative path is computed properly.
"""
rel_path = self.files_helper.find_relative_path("/root/up/to/here/test/it/now", "/root/up/to/here")
self.assertEqual(rel_path, os.sep.join(["test", "it", "now"]), "Did not extract relative path as expected.")
def test_remove_files_valid(self):
"""
Pass a valid list of files and check they are all removed.
"""
file_list = ["test1", "test2", "test3"]
for file_n in file_list:
fp = open(file_n, 'w')
fp.write('test')
fp.close()
for file_n in file_list:
self.assertTrue(os.path.isfile(file_n))
self.files_helper.remove_files(file_list)
for file_n in file_list:
self.assertFalse(os.path.isfile(file_n))
def test_remove_folder(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
os.mkdir(folder_name)
self.assertTrue(os.path.isdir(folder_name), "Folder should be created.")
self.files_helper.remove_folder(folder_name)
self.assertFalse(os.path.isdir(folder_name), "Folder should be deleted.")
def test_remove_folder_non_existing_ignore_exc(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.")
self.files_helper.remove_folder(folder_name, ignore_errors=True)
def test_remove_folder_non_existing(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.")
self.assertRaises(FileStructureException, self.files_helper.remove_folder, folder_name, False)
class TestFilesHelper(TransactionalTestCase):
"""
This class contains tests for the tvb.core.entities.file.files_helper module.
"""
PROJECT_NAME = "test_proj"
def transactional_setup_method(self):
"""
Set up the context needed by the tests.
"""
self.files_helper = FilesHelper()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME)
def transactional_teardown_method(self):
""" Remove generated project during tests. """
self.delete_project_folders()
def test_check_created(self):
""" Test standard flows for check created. """
self.files_helper.check_created()
assert os.path.exists(root_storage), "Storage not created!"
self.files_helper.check_created(os.path.join(root_storage, "test"))
assert os.path.exists(root_storage), "Storage not created!"
assert os.path.exists(os.path.join(root_storage, "test")), "Test directory not created!"
def test_get_project_folder(self):
"""
Test the get_project_folder method which should create a folder in case
it doesn't already exist.
"""
project_path = self.files_helper.get_project_folder(self.test_project)
assert os.path.exists(project_path), "Folder doesn't exist"
folder_path = self.files_helper.get_project_folder(self.test_project, "43")
assert os.path.exists(project_path), "Folder doesn't exist"
assert os.path.exists(folder_path), "Folder doesn't exist"
def test_rename_project_structure(self):
""" Try to rename the folder structure of a project. Standard flow. """
self.files_helper.get_project_folder(self.test_project)
path, name = self.files_helper.rename_project_structure(self.test_project.name, "new_name")
assert path != name, "Rename didn't take effect."
def test_rename_structure_same_name(self):
""" Try to rename the folder structure of a project. Same name. """
self.files_helper.get_project_folder(self.test_project)
with pytest.raises(FileStructureException):
self.files_helper.rename_project_structure(self.test_project.name, self.PROJECT_NAME)
def test_remove_project_structure(self):
""" Check that remove project structure deletes the corresponding folder. Standard flow. """
full_path = self.files_helper.get_project_folder(self.test_project)
assert os.path.exists(full_path), "Folder was not created."
self.files_helper.remove_project_structure(self.test_project.name)
assert not os.path.exists(full_path), "Project folder not deleted."
def test_write_project_metadata(self):
""" Write XML for test-project. """
self.files_helper.write_project_metadata(self.test_project)
expected_file = self.files_helper.get_project_meta_file_path(self.PROJECT_NAME)
assert os.path.exists(expected_file)
project_meta = XMLReader(expected_file).read_metadata()
loaded_project = model_project.Project(None, None)
loaded_project.from_dict(project_meta, self.test_user.id)
assert self.test_project.name == loaded_project.name
assert self.test_project.description == loaded_project.description
assert self.test_project.gid == loaded_project.gid
expected_dict = self.test_project.to_dict()[1]
del expected_dict['last_updated']
found_dict = loaded_project.to_dict()[1]
del found_dict['last_updated']
self._dictContainsSubset(expected_dict, found_dict)
self._dictContainsSubset(found_dict, expected_dict)
def test_write_operation_metadata(self):
"""
Test that a correct XML is created for an operation.
"""
operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)
expected_file = self.files_helper.get_operation_meta_file_path(self.PROJECT_NAME, operation.id)
assert not os.path.exists(expected_file)
self.files_helper.write_operation_metadata(operation)
assert os.path.exists(expected_file)
operation_meta = XMLReader(expected_file).read_metadata()
loaded_operation = model_operation.Operation(None, None, None, None)
loaded_operation.from_dict(operation_meta, dao, user_id=self.test_user.id)
expected_dict = operation.to_dict()[1]
found_dict = loaded_operation.to_dict()[1]
for key, value in expected_dict.items():
assert str(value) == str(found_dict[key])
# Now validate that operation metaData can be also updated
assert "new_group_name" != found_dict['user_group']
self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id)
found_dict = XMLReader(expected_file).read_metadata()
assert "new_group_name" == found_dict['user_group']
def test_remove_dt_happy_flow(self, dummy_datatype_index_factory):
"""
Happy flow for removing a file related to a DataType.
"""
datatype = dummy_datatype_index_factory()
h5_path = h5.path_for_stored_index(datatype)
assert os.path.exists(h5_path), "Test file was not created!"
self.files_helper.remove_datatype_file(h5_path)
assert not os.path.exists(h5_path), "Test file was not deleted!"
def test_remove_dt_non_existent(self, dummy_datatype_index_factory):
"""
Try to call remove on a dataType with no H5 file.
Should work.
"""
datatype = dummy_datatype_index_factory()
h5_path = h5.path_for_stored_index(datatype)
wrong_path = os.path.join(h5_path, "WRONG_PATH")
assert not os.path.exists(wrong_path)
self.files_helper.remove_datatype_file(wrong_path)
def test_move_datatype(self, dummy_datatype_index_factory):
"""
Make sure associated H5 file is moved to a correct new location.
"""
datatype = dummy_datatype_index_factory(project=self.test_project)
old_file_path = h5.path_for_stored_index(datatype)
assert os.path.exists(old_file_path), "Test file was not created!"
full_path = h5.path_for_stored_index(datatype)
self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '2', "1", full_path)
assert not os.path.exists(old_file_path), "Test file was not moved!"
datatype.fk_from_operation = 43
new_file_path = os.path.join(self.files_helper.get_project_folder(self.PROJECT_NAME + '2', "1"), old_file_path.split("\\")[-1])
assert os.path.exists(new_file_path), "Test file was not created!"
def test_find_relative_path(self):
"""
Tests that relative path is computed properly.
"""
rel_path = self.files_helper.find_relative_path("/root/up/to/here/test/it/now", "/root/up/to/here")
assert rel_path == os.sep.join(["test", "it", "now"]), "Did not extract relative path as expected."
def test_remove_files_valid(self):
"""
Pass a valid list of files and check they are all removed.
"""
file_list = ["test1", "test2", "test3"]
for file_n in file_list:
fp = open(file_n, 'w')
fp.write('test')
fp.close()
for file_n in file_list:
assert os.path.isfile(file_n)
self.files_helper.remove_files(file_list)
for file_n in file_list:
assert not os.path.isfile(file_n)
def test_remove_folder(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
os.mkdir(folder_name)
assert os.path.isdir(folder_name), "Folder should be created."
self.files_helper.remove_folder(folder_name)
assert not os.path.isdir(folder_name), "Folder should be deleted."
def test_remove_folder_non_existing_ignore_exc(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
assert not os.path.isdir(folder_name), "Folder should not exist before call."
self.files_helper.remove_folder(folder_name, ignore_errors=True)
def test_remove_folder_non_existing(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
assert not os.path.isdir(folder_name), "Folder should not exist before call."
with pytest.raises(FileStructureException):
self.files_helper.remove_folder(folder_name, False)
def _dictContainsSubset(self, expected, actual, msg=None):
"""Checks whether actual is a superset of expected."""
missing = []
mismatched = []
for key, value in expected.items():
if key not in actual:
return False
elif value != actual[key]:
return False
return True
class DiagnoseDiskUsage(object):
FORMAT_DT = ' {:14} {:20} {:>12} {:>12} {:>12} {:>12}'
HEADER_DT = FORMAT_DT.format('', '', 'disk_size(kib)', 'db_size(kib)',
'delta(kib)', 'ratio(%)')
def __init__(self, prj_id):
self.file_helper = FilesHelper()
self.expected_files = set()
self.prj_disk_size, self.prj_db_size = 0, 0
try:
dao.session.open_session()
# We do not fetch the project using dao because dao will detach it from the session.
# We want to query on the fly on attribute access and this requires attached objects.
# This code is doing a tree traversal of the db.
# The query on attribute access style fits better than aggregating queries.
self.project = dao.session.query(Project).filter(
Project.id == prj_id).one()
self.expected_files.add(
self.file_helper.get_project_meta_file_path(self.project.name))
root_path = self.file_helper.get_project_folder(self.project)
print()
print('Reporting disk for project {} in {}'.format(
self.project.name, root_path))
print()
print(self.HEADER_DT)
for op in self.project.OPERATIONS:
self.analyse_operation(op)
print(self.HEADER_DT)
self.print_usage_line('Project', 'total', self.prj_disk_size,
self.prj_db_size)
print()
self.list_unexpected_project_files(root_path)
print()
finally:
dao.session.close_session()
@staticmethod
def get_h5_by_gid(root, gid):
for f in os.listdir(root):
fp = os.path.join(root, f)
if gid in f and os.path.isfile(fp):
return fp
@staticmethod
def get_file_kib_size(fp):
return int(round((os.path.getsize(fp) / 1024.)))
@staticmethod
def print_usage_line(col1, col2, actual, expected):
if expected != 0:
ratio = int(100.0 * actual / expected)
if ratio > 200:
ratio = "! %s" % ratio
else:
ratio = str(ratio)
else:
ratio = 'inf'
delta = actual - expected
if delta > 100:
delta = "! %s" % delta
else:
delta = str(delta)
print(
DiagnoseDiskUsage.FORMAT_DT.format(col1, col2,
'{:,}'.format(actual),
'{:,}'.format(expected), delta,
ratio))
def analyse_operation(self, op):
op_disk_size, op_db_size = 0, 0
print('Operation {} : {}'.format(op.id, op.algorithm.name))
for dt in op.DATA_TYPES:
if dt.type == 'DataTypeGroup':
# these have no h5
continue
op_pth = self.file_helper.get_operation_folder(
self.project.name, op.id)
dt_pth = self.get_h5_by_gid(op_pth, dt.gid)
dt_actual_disk_size = self.get_file_kib_size(dt_pth)
db_disk_size = dt.disk_size or 0
op_disk_size += dt_actual_disk_size
op_db_size += db_disk_size
self.print_usage_line(dt.gid[:12], dt.type, dt_actual_disk_size,
db_disk_size)
self.expected_files.add(dt_pth)
op_xml = self.file_helper.get_operation_meta_file_path(
self.project.name, op.id)
self.expected_files.add(op_xml)
self.prj_disk_size += op_disk_size
self.prj_db_size += op_db_size
self.print_usage_line('', 'total :', op_disk_size, op_db_size)
print()
def list_unexpected_project_files(self, root_path):
unexpected = []
for r, d, files in os.walk(root_path):
for f in files:
pth = os.path.join(r, f)
if pth not in self.expected_files:
unexpected.append(pth)
print('Unexpected project files :')
for f in unexpected:
print(f)
if not unexpected:
print('yey! none found')
class FilesHelperTest(TransactionalTestCase):
"""
This class contains tests for the tvb.core.entities.file.files_helper module.
"""
PROJECT_NAME = "test_proj"
def setUp(self):
"""
Set up the context needed by the tests.
"""
self.files_helper = FilesHelper()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user,
self.PROJECT_NAME)
def tearDown(self):
""" Remove generated project during tests. """
self.delete_project_folders()
def test_check_created(self):
""" Test standard flows for check created. """
self.files_helper.check_created()
self.assertTrue(os.path.exists(root_storage), "Storage not created!")
self.files_helper.check_created(os.path.join(root_storage, "test"))
self.assertTrue(os.path.exists(root_storage), "Storage not created!")
self.assertTrue(os.path.exists(os.path.join(root_storage, "test")),
"Test directory not created!")
def test_get_project_folder(self):
"""
Test the get_project_folder method which should create a folder in case
it doesn't already exist.
"""
project_path = self.files_helper.get_project_folder(self.test_project)
self.assertTrue(os.path.exists(project_path), "Folder doesn't exist")
folder_path = self.files_helper.get_project_folder(
self.test_project, "43")
self.assertTrue(os.path.exists(project_path), "Folder doesn't exist")
self.assertTrue(os.path.exists(folder_path), "Folder doesn't exist")
def test_rename_project_structure(self):
""" Try to rename the folder structure of a project. Standard flow. """
self.files_helper.get_project_folder(self.test_project)
path, name = self.files_helper.rename_project_structure(
self.test_project.name, "new_name")
self.assertNotEqual(path, name, "Rename didn't take effect.")
def test_rename_structure_same_name(self):
""" Try to rename the folder structure of a project. Same name. """
self.files_helper.get_project_folder(self.test_project)
self.assertRaises(FileStructureException,
self.files_helper.rename_project_structure,
self.test_project.name, self.PROJECT_NAME)
def test_remove_project_structure(self):
""" Check that remove project structure deletes the corresponding folder. Standard flow. """
full_path = self.files_helper.get_project_folder(self.test_project)
self.assertTrue(os.path.exists(full_path), "Folder was not created.")
self.files_helper.remove_project_structure(self.test_project.name)
self.assertFalse(os.path.exists(full_path),
"Project folder not deleted.")
def test_write_project_metadata(self):
""" Write XML for test-project. """
self.files_helper.write_project_metadata(self.test_project)
expected_file = self.files_helper.get_project_meta_file_path(
self.PROJECT_NAME)
self.assertTrue(os.path.exists(expected_file))
project_meta = XMLReader(expected_file).read_metadata()
loaded_project = model.Project(None, None)
loaded_project.from_dict(project_meta, self.test_user.id)
self.assertEqual(self.test_project.name, loaded_project.name)
self.assertEqual(self.test_project.description,
loaded_project.description)
self.assertEqual(self.test_project.gid, loaded_project.gid)
expected_dict = self.test_project.to_dict()[1]
del expected_dict['last_updated']
found_dict = loaded_project.to_dict()[1]
del found_dict['last_updated']
self.assertDictContainsSubset(expected_dict, found_dict)
self.assertDictContainsSubset(found_dict, expected_dict)
def test_write_operation_metadata(self):
"""
Test that a correct XML is created for an operation.
"""
operation = TestFactory.create_operation(
test_user=self.test_user, test_project=self.test_project)
expected_file = self.files_helper.get_operation_meta_file_path(
self.PROJECT_NAME, operation.id)
self.assertFalse(os.path.exists(expected_file))
self.files_helper.write_operation_metadata(operation)
self.assertTrue(os.path.exists(expected_file))
operation_meta = XMLReader(expected_file).read_metadata()
loaded_operation = model.Operation(None, None, None, None)
loaded_operation.from_dict(operation_meta, dao)
expected_dict = operation.to_dict()[1]
found_dict = loaded_operation.to_dict()[1]
for key, value in expected_dict.iteritems():
self.assertEqual(str(value), str(found_dict[key]))
# Now validate that operation metaData can be also updated
self.assertNotEqual("new_group_name", found_dict['user_group'])
self.files_helper.update_operation_metadata(self.PROJECT_NAME,
"new_group_name",
operation.id)
found_dict = XMLReader(expected_file).read_metadata()
self.assertEqual("new_group_name", found_dict['user_group'])
def test_remove_dt_happy_flow(self):
"""
Happy flow for removing a file related to a DataType.
"""
folder_path = self.files_helper.get_project_folder(
self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
open(datatype.get_storage_file_path(), 'w')
self.assertTrue(os.path.exists(datatype.get_storage_file_path()),
"Test file was not created!")
self.files_helper.remove_datatype(datatype)
self.assertFalse(os.path.exists(datatype.get_storage_file_path()),
"Test file was not deleted!")
def test_remove_dt_non_existent(self):
"""
Try to call remove on a dataType with no H5 file.
Should work.
"""
folder_path = self.files_helper.get_project_folder(
self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
self.assertFalse(os.path.exists(datatype.get_storage_file_path()))
self.files_helper.remove_datatype(datatype)
def test_move_datatype(self):
"""
Make sure associated H5 file is moved to a correct new location.
"""
folder_path = self.files_helper.get_project_folder(
self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
open(datatype.get_storage_file_path(), 'w')
self.assertTrue(os.path.exists(datatype.get_storage_file_path()),
"Test file was not created!")
self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '11',
"43")
self.assertFalse(os.path.exists(datatype.get_storage_file_path()),
"Test file was not moved!")
datatype.storage_path = self.files_helper.get_project_folder(
self.PROJECT_NAME + '11', "43")
self.assertTrue(os.path.exists(datatype.get_storage_file_path()),
"Test file was not created!")
def test_find_relative_path(self):
"""
Tests that relative path is computed properly.
"""
rel_path = self.files_helper.find_relative_path(
"/root/up/to/here/test/it/now", "/root/up/to/here")
self.assertEqual(rel_path, os.sep.join(["test", "it", "now"]),
"Did not extract relative path as expected.")
def test_remove_files_valid(self):
"""
Pass a valid list of files and check they are all removed.
"""
file_list = ["test1", "test2", "test3"]
for file_n in file_list:
fp = open(file_n, 'w')
fp.write('test')
fp.close()
for file_n in file_list:
self.assertTrue(os.path.isfile(file_n))
self.files_helper.remove_files(file_list)
for file_n in file_list:
self.assertFalse(os.path.isfile(file_n))
def test_remove_folder(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
os.mkdir(folder_name)
self.assertTrue(os.path.isdir(folder_name),
"Folder should be created.")
self.files_helper.remove_folder(folder_name)
self.assertFalse(os.path.isdir(folder_name),
"Folder should be deleted.")
def test_remove_folder_non_existing_ignore_exc(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
self.assertFalse(os.path.isdir(folder_name),
"Folder should not exist before call.")
self.files_helper.remove_folder(folder_name, ignore_errors=True)
def test_remove_folder_non_existing(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
self.assertFalse(os.path.isdir(folder_name),
"Folder should not exist before call.")
self.assertRaises(FileStructureException,
self.files_helper.remove_folder, folder_name, False)
