def export_project(self, project, optimize_size=False):
"""
Given a project root and the TVB storage_path, create a ZIP
ready for export.
:param project: project object which identifies project to be exported
"""
if project is None:
raise ExportException("Please provide project to be exported")
files_helper = FilesHelper()
project_folder = files_helper.get_project_folder(project)
project_datatypes = self._gather_project_datatypes(project, optimize_size)
to_be_exported_folders = []
considered_op_ids = []
min_dt_date = datetime.now()
if optimize_size:
## take only the DataType with visibility flag set ON
for dt in project_datatypes:
if dt[KEY_OPERATION_ID] not in considered_op_ids:
to_be_exported_folders.append({'folder': files_helper.get_project_folder(project,
str(dt[KEY_OPERATION_ID])),
'archive_path_prefix': str(dt[KEY_OPERATION_ID]) + os.sep})
considered_op_ids.append(dt[KEY_OPERATION_ID])
if min_dt_date > dt[KEY_DT_DATE]:
min_dt_date = dt[KEY_DT_DATE]
else:
to_be_exported_folders.append({'folder': project_folder,
'archive_path_prefix': '', 'exclude': ["TEMP"]})
if project_datatypes:
min_dt_date = min([dt[KEY_DT_DATE] for dt in project_datatypes])
# Compute path and name of the zip file
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, project.name, self.ZIP_FILE_EXTENSION)
export_folder = self._build_data_export_folder(project)
result_path = os.path.join(export_folder, zip_file_name)
with TvbZip(result_path, "w") as zip_file:
# Pack project [filtered] content into a ZIP file:
LOG.debug("Done preparing, now we will write folders " + str(len(to_be_exported_folders)))
LOG.debug(str(to_be_exported_folders))
for pack in to_be_exported_folders:
zip_file.write_folder(**pack)
LOG.debug("Done exporting files, now we will write the burst configurations...")
self._export_bursts(project, project_datatypes, zip_file)
LOG.debug("Done exporting burst configurations, now we will export linked DTs")
self._export_linked_datatypes(project, zip_file, min_dt_date)
## Make sure the Project.xml file gets copied:
if optimize_size:
LOG.debug("Done linked, now we write the project xml")
zip_file.write(files_helper.get_project_meta_file_path(project.name), files_helper.TVB_PROJECT_FILE)
LOG.debug("Done, closing")
return result_path
def export_project(self, project, optimize_size=False):
"""
Given a project root and the TVB storage_path, create a ZIP
ready for export.
:param project: project object which identifies project to be exported
"""
if project is None:
raise ExportException("Please provide project to be exported")
files_helper = FilesHelper()
project_folder = files_helper.get_project_folder(project)
project_datatypes = self._gather_project_datatypes(project, optimize_size)
to_be_exported_folders = []
considered_op_ids = []
if optimize_size:
## take only the DataType with visibility flag set ON
for dt in project_datatypes:
if dt[KEY_OPERATION_ID] not in considered_op_ids:
to_be_exported_folders.append({'folder': files_helper.get_project_folder(project,
str(dt[KEY_OPERATION_ID])),
'archive_path_prefix': str(dt[KEY_OPERATION_ID]) + os.sep})
considered_op_ids.append(dt[KEY_OPERATION_ID])
else:
to_be_exported_folders.append({'folder': project_folder,
'archive_path_prefix': '', 'exclude': ["TEMP"]})
# Compute path and name of the zip file
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, project.name, self.ZIP_FILE_EXTENSION)
export_folder = self._build_data_export_folder(project)
result_path = os.path.join(export_folder, zip_file_name)
with TvbZip(result_path, "w") as zip_file:
# Pack project [filtered] content into a ZIP file:
LOG.debug("Done preparing, now we will write folders " + str(len(to_be_exported_folders)))
LOG.debug(str(to_be_exported_folders))
for pack in to_be_exported_folders:
zip_file.write_folder(**pack)
LOG.debug("Done exporting files, now we will write the burst configurations...")
self._export_bursts(project, project_datatypes, zip_file)
LOG.debug("Done exporting burst configurations, now we will export linked DTs")
self._export_linked_datatypes(project, zip_file)
## Make sure the Project.xml file gets copied:
if optimize_size:
LOG.debug("Done linked, now we write the project xml")
zip_file.write(files_helper.get_project_meta_file_path(project.name), files_helper.TVB_PROJECT_FILE)
LOG.debug("Done, closing")
return result_path
class FilesHelperTest(TransactionalTestCase):
"""
This class contains tests for the tvb.core.entities.file.files_helper module.
"""
PROJECT_NAME = "test_proj"
def setUp(self):
"""
Set up the context needed by the tests.
"""
self.files_helper = FilesHelper()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME)
def tearDown(self):
""" Remove generated project during tests. """
self.delete_project_folders()
def test_check_created(self):
""" Test standard flows for check created. """
self.files_helper.check_created()
self.assertTrue(os.path.exists(root_storage), "Storage not created!")
self.files_helper.check_created(os.path.join(root_storage, "test"))
self.assertTrue(os.path.exists(root_storage), "Storage not created!")
self.assertTrue(os.path.exists(os.path.join(root_storage, "test")), "Test directory not created!")
def test_get_project_folder(self):
"""
Test the get_project_folder method which should create a folder in case
it doesn't already exist.
"""
project_path = self.files_helper.get_project_folder(self.test_project)
self.assertTrue(os.path.exists(project_path), "Folder doesn't exist")
folder_path = self.files_helper.get_project_folder(self.test_project, "43")
self.assertTrue(os.path.exists(project_path), "Folder doesn't exist")
self.assertTrue(os.path.exists(folder_path), "Folder doesn't exist")
def test_rename_project_structure(self):
""" Try to rename the folder structure of a project. Standard flow. """
self.files_helper.get_project_folder(self.test_project)
path, name = self.files_helper.rename_project_structure(self.test_project.name, "new_name")
self.assertNotEqual(path, name, "Rename didn't take effect.")
def test_rename_structure_same_name(self):
""" Try to rename the folder structure of a project. Same name. """
self.files_helper.get_project_folder(self.test_project)
self.assertRaises(FileStructureException, self.files_helper.rename_project_structure,
self.test_project.name, self.PROJECT_NAME)
def test_remove_project_structure(self):
""" Check that remove project structure deletes the corresponding folder. Standard flow. """
full_path = self.files_helper.get_project_folder(self.test_project)
self.assertTrue(os.path.exists(full_path), "Folder was not created.")
self.files_helper.remove_project_structure(self.test_project.name)
self.assertFalse(os.path.exists(full_path), "Project folder not deleted.")
def test_write_project_metadata(self):
""" Write XML for test-project. """
self.files_helper.write_project_metadata(self.test_project)
expected_file = self.files_helper.get_project_meta_file_path(self.PROJECT_NAME)
self.assertTrue(os.path.exists(expected_file))
project_meta = XMLReader(expected_file).read_metadata()
loaded_project = model.Project(None, None)
loaded_project.from_dict(project_meta, self.test_user.id)
self.assertEqual(self.test_project.name, loaded_project.name)
self.assertEqual(self.test_project.description, loaded_project.description)
self.assertEqual(self.test_project.gid, loaded_project.gid)
expected_dict = self.test_project.to_dict()[1]
del expected_dict['last_updated']
found_dict = loaded_project.to_dict()[1]
del found_dict['last_updated']
self.assertDictContainsSubset(expected_dict, found_dict)
self.assertDictContainsSubset(found_dict, expected_dict)
def test_write_operation_metadata(self):
"""
Test that a correct XML is created for an operation.
"""
operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)
expected_file = self.files_helper.get_operation_meta_file_path(self.PROJECT_NAME, operation.id)
self.assertFalse(os.path.exists(expected_file))
self.files_helper.write_operation_metadata(operation)
self.assertTrue(os.path.exists(expected_file))
operation_meta = XMLReader(expected_file).read_metadata()
loaded_operation = model.Operation(None, None, None, None)
loaded_operation.from_dict(operation_meta, dao)
expected_dict = operation.to_dict()[1]
found_dict = loaded_operation.to_dict()[1]
for key, value in expected_dict.iteritems():
self.assertEqual(str(value), str(found_dict[key]))
# Now validate that operation metaData can be also updated
self.assertNotEqual("new_group_name", found_dict['user_group'])
self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id)
found_dict = XMLReader(expected_file).read_metadata()
self.assertEqual("new_group_name", found_dict['user_group'])
def test_remove_dt_happy_flow(self):
"""
Happy flow for removing a file related to a DataType.
"""
folder_path = self.files_helper.get_project_folder(self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
open(datatype.get_storage_file_path(), 'w')
self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!")
self.files_helper.remove_datatype(datatype)
self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not deleted!")
def test_remove_dt_non_existent(self):
"""
Try to call remove on a dataType with no H5 file.
Should work.
"""
folder_path = self.files_helper.get_project_folder(self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
self.assertFalse(os.path.exists(datatype.get_storage_file_path()))
self.files_helper.remove_datatype(datatype)
def test_move_datatype(self):
"""
Make sure associated H5 file is moved to a correct new location.
"""
folder_path = self.files_helper.get_project_folder(self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
open(datatype.get_storage_file_path(), 'w')
self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!")
self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '11', "43")
self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not moved!")
datatype.storage_path = self.files_helper.get_project_folder(self.PROJECT_NAME + '11', "43")
self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!")
def test_find_relative_path(self):
"""
Tests that relative path is computed properly.
"""
rel_path = self.files_helper.find_relative_path("/root/up/to/here/test/it/now", "/root/up/to/here")
self.assertEqual(rel_path, os.sep.join(["test", "it", "now"]), "Did not extract relative path as expected.")
def test_remove_files_valid(self):
"""
Pass a valid list of files and check they are all removed.
"""
file_list = ["test1", "test2", "test3"]
for file_n in file_list:
fp = open(file_n, 'w')
fp.write('test')
fp.close()
for file_n in file_list:
self.assertTrue(os.path.isfile(file_n))
self.files_helper.remove_files(file_list)
for file_n in file_list:
self.assertFalse(os.path.isfile(file_n))
def test_remove_folder(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
os.mkdir(folder_name)
self.assertTrue(os.path.isdir(folder_name), "Folder should be created.")
self.files_helper.remove_folder(folder_name)
self.assertFalse(os.path.isdir(folder_name), "Folder should be deleted.")
def test_remove_folder_non_existing_ignore_exc(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.")
self.files_helper.remove_folder(folder_name, ignore_errors=True)
def test_remove_folder_non_existing(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.")
self.assertRaises(FileStructureException, self.files_helper.remove_folder, folder_name, False)
class TestFilesHelper(TransactionalTestCase):
"""
This class contains tests for the tvb.core.entities.file.files_helper module.
"""
PROJECT_NAME = "test_proj"
def transactional_setup_method(self):
"""
Set up the context needed by the tests.
"""
self.files_helper = FilesHelper()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME)
def transactional_teardown_method(self):
""" Remove generated project during tests. """
self.delete_project_folders()
def test_check_created(self):
""" Test standard flows for check created. """
self.files_helper.check_created()
assert os.path.exists(root_storage), "Storage not created!"
self.files_helper.check_created(os.path.join(root_storage, "test"))
assert os.path.exists(root_storage), "Storage not created!"
assert os.path.exists(os.path.join(root_storage, "test")), "Test directory not created!"
def test_get_project_folder(self):
"""
Test the get_project_folder method which should create a folder in case
it doesn't already exist.
"""
project_path = self.files_helper.get_project_folder(self.test_project)
assert os.path.exists(project_path), "Folder doesn't exist"
folder_path = self.files_helper.get_project_folder(self.test_project, "43")
assert os.path.exists(project_path), "Folder doesn't exist"
assert os.path.exists(folder_path), "Folder doesn't exist"
def test_rename_project_structure(self):
""" Try to rename the folder structure of a project. Standard flow. """
self.files_helper.get_project_folder(self.test_project)
path, name = self.files_helper.rename_project_structure(self.test_project.name, "new_name")
assert path != name, "Rename didn't take effect."
def test_rename_structure_same_name(self):
""" Try to rename the folder structure of a project. Same name. """
self.files_helper.get_project_folder(self.test_project)
with pytest.raises(FileStructureException):
self.files_helper.rename_project_structure(self.test_project.name, self.PROJECT_NAME)
def test_remove_project_structure(self):
""" Check that remove project structure deletes the corresponding folder. Standard flow. """
full_path = self.files_helper.get_project_folder(self.test_project)
assert os.path.exists(full_path), "Folder was not created."
self.files_helper.remove_project_structure(self.test_project.name)
assert not os.path.exists(full_path), "Project folder not deleted."
def test_write_project_metadata(self):
""" Write XML for test-project. """
self.files_helper.write_project_metadata(self.test_project)
expected_file = self.files_helper.get_project_meta_file_path(self.PROJECT_NAME)
assert os.path.exists(expected_file)
project_meta = XMLReader(expected_file).read_metadata()
loaded_project = model_project.Project(None, None)
loaded_project.from_dict(project_meta, self.test_user.id)
assert self.test_project.name == loaded_project.name
assert self.test_project.description == loaded_project.description
assert self.test_project.gid == loaded_project.gid
expected_dict = self.test_project.to_dict()[1]
del expected_dict['last_updated']
found_dict = loaded_project.to_dict()[1]
del found_dict['last_updated']
self._dictContainsSubset(expected_dict, found_dict)
self._dictContainsSubset(found_dict, expected_dict)
def test_write_operation_metadata(self):
"""
Test that a correct XML is created for an operation.
"""
operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)
expected_file = self.files_helper.get_operation_meta_file_path(self.PROJECT_NAME, operation.id)
assert not os.path.exists(expected_file)
self.files_helper.write_operation_metadata(operation)
assert os.path.exists(expected_file)
operation_meta = XMLReader(expected_file).read_metadata()
loaded_operation = model_operation.Operation(None, None, None, None)
loaded_operation.from_dict(operation_meta, dao, user_id=self.test_user.id)
expected_dict = operation.to_dict()[1]
found_dict = loaded_operation.to_dict()[1]
for key, value in expected_dict.items():
assert str(value) == str(found_dict[key])
# Now validate that operation metaData can be also updated
assert "new_group_name" != found_dict['user_group']
self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id)
found_dict = XMLReader(expected_file).read_metadata()
assert "new_group_name" == found_dict['user_group']
def test_remove_dt_happy_flow(self, dummy_datatype_index_factory):
"""
Happy flow for removing a file related to a DataType.
"""
datatype = dummy_datatype_index_factory()
h5_path = h5.path_for_stored_index(datatype)
assert os.path.exists(h5_path), "Test file was not created!"
self.files_helper.remove_datatype_file(h5_path)
assert not os.path.exists(h5_path), "Test file was not deleted!"
def test_remove_dt_non_existent(self, dummy_datatype_index_factory):
"""
Try to call remove on a dataType with no H5 file.
Should work.
"""
datatype = dummy_datatype_index_factory()
h5_path = h5.path_for_stored_index(datatype)
wrong_path = os.path.join(h5_path, "WRONG_PATH")
assert not os.path.exists(wrong_path)
self.files_helper.remove_datatype_file(wrong_path)
def test_move_datatype(self, dummy_datatype_index_factory):
"""
Make sure associated H5 file is moved to a correct new location.
"""
datatype = dummy_datatype_index_factory(project=self.test_project)
old_file_path = h5.path_for_stored_index(datatype)
assert os.path.exists(old_file_path), "Test file was not created!"
full_path = h5.path_for_stored_index(datatype)
self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '2', "1", full_path)
assert not os.path.exists(old_file_path), "Test file was not moved!"
datatype.fk_from_operation = 43
new_file_path = os.path.join(self.files_helper.get_project_folder(self.PROJECT_NAME + '2', "1"), old_file_path.split("\\")[-1])
assert os.path.exists(new_file_path), "Test file was not created!"
def test_find_relative_path(self):
"""
Tests that relative path is computed properly.
"""
rel_path = self.files_helper.find_relative_path("/root/up/to/here/test/it/now", "/root/up/to/here")
assert rel_path == os.sep.join(["test", "it", "now"]), "Did not extract relative path as expected."
def test_remove_files_valid(self):
"""
Pass a valid list of files and check they are all removed.
"""
file_list = ["test1", "test2", "test3"]
for file_n in file_list:
fp = open(file_n, 'w')
fp.write('test')
fp.close()
for file_n in file_list:
assert os.path.isfile(file_n)
self.files_helper.remove_files(file_list)
for file_n in file_list:
assert not os.path.isfile(file_n)
def test_remove_folder(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
os.mkdir(folder_name)
assert os.path.isdir(folder_name), "Folder should be created."
self.files_helper.remove_folder(folder_name)
assert not os.path.isdir(folder_name), "Folder should be deleted."
def test_remove_folder_non_existing_ignore_exc(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
assert not os.path.isdir(folder_name), "Folder should not exist before call."
self.files_helper.remove_folder(folder_name, ignore_errors=True)
def test_remove_folder_non_existing(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
assert not os.path.isdir(folder_name), "Folder should not exist before call."
with pytest.raises(FileStructureException):
self.files_helper.remove_folder(folder_name, False)
def _dictContainsSubset(self, expected, actual, msg=None):
"""Checks whether actual is a superset of expected."""
missing = []
mismatched = []
for key, value in expected.items():
if key not in actual:
return False
elif value != actual[key]:
return False
return True
class ExportManager(object):
"""
This class provides basic methods for exporting data types of projects in different formats.
"""
all_exporters = {} # Dictionary containing all available exporters
export_folder = None
EXPORT_FOLDER_NAME = "EXPORT_TMP"
EXPORTED_SIMULATION_NAME = "exported_simulation"
EXPORTED_SIMULATION_DTS_DIR = "datatypes"
ZIP_FILE_EXTENSION = "zip"
logger = get_logger(__name__)
def __init__(self):
# Here we register all available data type exporters
# If new exporters supported, they should be added here
self._register_exporter(TVBExporter())
self._register_exporter(TVBLinkedExporter())
self.export_folder = os.path.join(TvbProfile.current.TVB_STORAGE,
self.EXPORT_FOLDER_NAME)
self.files_helper = FilesHelper()
def _register_exporter(self, exporter):
"""
This method register into an internal format available exporters.
:param exporter: Instance of a data type exporter (extends ABCExporter)
"""
if exporter is not None:
self.all_exporters[exporter.__class__.__name__] = exporter
def get_exporters_for_data(self, data):
"""
Get available exporters for current data type.
:returns: a dictionary with the {exporter_id : label}
"""
if data is None:
raise InvalidExportDataException(
"Could not detect exporters for null data")
self.logger.debug("Trying to determine exporters valid for %s" %
data.type)
results = {}
# No exporter for None data
if data is None:
return results
for exporterId in self.all_exporters.keys():
exporter = self.all_exporters[exporterId]
if exporter.accepts(data):
results[exporterId] = exporter.get_label()
return results
def export_data(self, data, exporter_id, project):
"""
Export provided data using given exporter
:param data: data type to be exported
:param exporter_id: identifier of the exporter to be used
:param project: project that contains data to be exported
:returns: a tuple with the following elements
1. name of the file to be shown to user
2. full path of the export file (available for download)
3. boolean which specify if file can be deleted after download
"""
if data is None:
raise InvalidExportDataException(
"Could not export null data. Please select data to be exported"
)
if exporter_id is None:
raise ExportException(
"Please select the exporter to be used for this operation")
if exporter_id not in self.all_exporters:
raise ExportException(
"Provided exporter identifier is not a valid one")
exporter = self.all_exporters[exporter_id]
if project is None:
raise ExportException(
"Please provide the project where data files are stored")
# Now we start the real export
if not exporter.accepts(data):
raise InvalidExportDataException(
"Current data can not be exported by specified exporter")
# Now compute and create folder where to store exported data
# This will imply to generate a folder which is unique for each export
data_export_folder = None
try:
data_export_folder = self._build_data_export_folder(data)
self.logger.debug("Start export of data: %s" % data.type)
export_data = exporter.export(data, data_export_folder, project)
finally:
# In case export did not generated any file delete folder
if data_export_folder is not None and len(
os.listdir(data_export_folder)) == 0:
os.rmdir(data_export_folder)
return export_data
def _export_linked_datatypes(self, project, zip_file):
linked_paths = ProjectService().get_linked_datatypes_storage_path(
project)
if not linked_paths:
# do not export an empty operation
return
# Make a import operation which will contain links to other projects
algo = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE,
TVB_IMPORTER_CLASS)
op = model_operation.Operation(None, None, project.id, algo.id)
op.project = project
op.algorithm = algo
op.id = 'links-to-external-projects'
op.start_now()
op.mark_complete(model_operation.STATUS_FINISHED)
op_folder = self.files_helper.get_operation_folder(
op.project.name, op.id)
op_folder_name = os.path.basename(op_folder)
# add linked datatypes to archive in the import operation
for pth in linked_paths:
zip_pth = op_folder_name + '/' + os.path.basename(pth)
zip_file.write(pth, zip_pth)
# remove these files, since we only want them in export archive
self.files_helper.remove_folder(op_folder)
def export_project(self, project, optimize_size=False):
"""
Given a project root and the TVB storage_path, create a ZIP
ready for export.
:param project: project object which identifies project to be exported
"""
if project is None:
raise ExportException("Please provide project to be exported")
project_folder = self.files_helper.get_project_folder(project)
project_datatypes = dao.get_datatypes_in_project(
project.id, only_visible=optimize_size)
to_be_exported_folders = []
considered_op_ids = []
folders_to_exclude = self._get_op_with_errors(project.id)
if optimize_size:
# take only the DataType with visibility flag set ON
for dt in project_datatypes:
op_id = dt.fk_from_operation
if op_id not in considered_op_ids:
to_be_exported_folders.append({
'folder':
self.files_helper.get_project_folder(
project, str(op_id)),
'archive_path_prefix':
str(op_id) + os.sep,
'exclude':
folders_to_exclude
})
considered_op_ids.append(op_id)
else:
folders_to_exclude.append("TEMP")
to_be_exported_folders.append({
'folder': project_folder,
'archive_path_prefix': '',
'exclude': folders_to_exclude
})
# Compute path and name of the zip file
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, project.name,
self.ZIP_FILE_EXTENSION)
export_folder = self._build_data_export_folder(project)
result_path = os.path.join(export_folder, zip_file_name)
with TvbZip(result_path, "w") as zip_file:
# Pack project [filtered] content into a ZIP file:
self.logger.debug("Done preparing, now we will write folders " +
str(len(to_be_exported_folders)))
self.logger.debug(str(to_be_exported_folders))
for pack in to_be_exported_folders:
zip_file.write_folder(**pack)
self.logger.debug(
"Done exporting files, now we will export linked DTs")
self._export_linked_datatypes(project, zip_file)
# Make sure the Project.xml file gets copied:
if optimize_size:
self.logger.debug("Done linked, now we write the project xml")
zip_file.write(
self.files_helper.get_project_meta_file_path(project.name),
self.files_helper.TVB_PROJECT_FILE)
self.logger.debug("Done, closing")
return result_path
@staticmethod
def _get_op_with_errors(project_id):
"""
Get the operation folders with error base name as list.
"""
operations = dao.get_operations_with_error_in_project(project_id)
op_with_errors = []
for op in operations:
op_with_errors.append(op.id)
return op_with_errors
def _build_data_export_folder(self, data):
"""
This method computes the folder where results of an export operation will be
stored for a while (e.g until download is done; or for 1 day)
"""
now = datetime.now()
date_str = "%d-%d-%d_%d-%d-%d_%d" % (now.year, now.month, now.day,
now.hour, now.minute, now.second,
now.microsecond)
tmp_str = date_str + "@" + data.gid
data_export_folder = os.path.join(self.export_folder, tmp_str)
self.files_helper.check_created(data_export_folder)
return data_export_folder
def export_simulator_configuration(self, burst_id):
burst = dao.get_burst_by_id(burst_id)
if burst is None:
raise InvalidExportDataException("Could not find burst with ID " +
str(burst_id))
op_folder = self.files_helper.get_project_folder(
burst.project, str(burst.fk_simulation))
tmp_export_folder = self._build_data_export_folder(burst)
tmp_sim_folder = os.path.join(tmp_export_folder,
self.EXPORTED_SIMULATION_NAME)
if not os.path.exists(tmp_sim_folder):
os.makedirs(tmp_sim_folder)
all_view_model_paths, all_datatype_paths = h5.gather_references_of_view_model(
burst.simulator_gid, op_folder)
burst_path = h5.determine_filepath(burst.gid, op_folder)
all_view_model_paths.append(burst_path)
for vm_path in all_view_model_paths:
dest = os.path.join(tmp_sim_folder, os.path.basename(vm_path))
self.files_helper.copy_file(vm_path, dest)
for dt_path in all_datatype_paths:
dest = os.path.join(tmp_sim_folder,
self.EXPORTED_SIMULATION_DTS_DIR,
os.path.basename(dt_path))
self.files_helper.copy_file(dt_path, dest)
main_vm_path = h5.determine_filepath(burst.simulator_gid,
tmp_sim_folder)
H5File.remove_metadata_param(main_vm_path, 'history_gid')
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, str(burst_id),
self.ZIP_FILE_EXTENSION)
result_path = os.path.join(tmp_export_folder, zip_file_name)
with TvbZip(result_path, "w") as zip_file:
zip_file.write_folder(tmp_sim_folder)
self.files_helper.remove_folder(tmp_sim_folder)
return result_path
class DiagnoseDiskUsage(object):
FORMAT_DT = ' {:14} {:20} {:>12} {:>12} {:>12} {:>12}'
HEADER_DT = FORMAT_DT.format('', '', 'disk_size(kib)', 'db_size(kib)', 'delta(kib)', 'ratio(%)')
def __init__(self, prj_id):
self.file_helper = FilesHelper()
self.expected_files = set()
self.prj_disk_size, self.prj_db_size = 0, 0
try:
dao.session.open_session()
# We do not fetch the project using dao because dao will detach it from the session.
# We want to query on the fly on attribute access and this requires attached objects.
# This code is doing a tree traversal of the db.
# The query on attribute access style fits better than aggregating queries.
self.project = dao.session.query(Project).filter(Project.id == prj_id).one()
self.expected_files.add(self.file_helper.get_project_meta_file_path(self.project.name))
root_path = self.file_helper.get_project_folder(self.project)
print()
print('Reporting disk for project {} in {}'.format(self.project.name, root_path))
print()
print(self.HEADER_DT)
for op in self.project.OPERATIONS:
self.analyse_operation(op)
print(self.HEADER_DT)
self.print_usage_line('Project', 'total', self.prj_disk_size, self.prj_db_size)
print()
self.list_unexpected_project_files(root_path)
print()
finally:
dao.session.close_session()
@staticmethod
def get_h5_by_gid(root, gid):
for f in os.listdir(root):
fp = os.path.join(root, f)
if gid in f and os.path.isfile(fp):
return fp
@staticmethod
def get_file_kib_size(fp):
return int(round((os.path.getsize(fp) / 1024.)))
@staticmethod
def print_usage_line(col1, col2, actual, expected):
if expected != 0:
ratio = int(100.0 * actual / expected)
if ratio > 200:
ratio = "! %s" % ratio
else:
ratio = str(ratio)
else:
ratio = 'inf'
delta = actual - expected
if delta > 100:
delta = "! %s" % delta
else:
delta = str(delta)
print(DiagnoseDiskUsage.FORMAT_DT.format(col1, col2, '{:,}'.format(actual),
'{:,}'.format(expected), delta, ratio))
def analyse_operation(self, op):
op_disk_size, op_db_size = 0, 0
print('Operation {} : {}'.format(op.id, op.algorithm.name))
for dt in op.DATA_TYPES:
if dt.type == 'DataTypeGroup':
# these have no h5
continue
op_pth = self.file_helper.get_operation_folder(self.project.name, op.id)
dt_pth = self.get_h5_by_gid(op_pth, dt.gid)
dt_actual_disk_size = self.get_file_kib_size(dt_pth)
db_disk_size = dt.disk_size or 0
op_disk_size += dt_actual_disk_size
op_db_size += db_disk_size
self.print_usage_line(dt.gid[:12], dt.type, dt_actual_disk_size, db_disk_size)
self.expected_files.add(dt_pth)
self.prj_disk_size += op_disk_size
self.prj_db_size += op_db_size
self.print_usage_line('', 'total :', op_disk_size, op_db_size)
print()
def list_unexpected_project_files(self, root_path):
unexpected = []
for r, d, files in os.walk(root_path):
for f in files:
pth = os.path.join(r, f)
if pth not in self.expected_files:
unexpected.append(pth)
print('Unexpected project files :')
for f in unexpected:
print(f)
if not unexpected:
print('yey! none found')
class FilesHelperTest(TransactionalTestCase):
"""
This class contains tests for the tvb.core.entities.file.files_helper module.
"""
PROJECT_NAME = "test_proj"
def setUp(self):
"""
Set up the context needed by the tests.
"""
self.files_helper = FilesHelper()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user,
self.PROJECT_NAME)
def tearDown(self):
""" Remove generated project during tests. """
self.delete_project_folders()
def test_check_created(self):
""" Test standard flows for check created. """
self.files_helper.check_created()
self.assertTrue(os.path.exists(root_storage), "Storage not created!")
self.files_helper.check_created(os.path.join(root_storage, "test"))
self.assertTrue(os.path.exists(root_storage), "Storage not created!")
self.assertTrue(os.path.exists(os.path.join(root_storage, "test")),
"Test directory not created!")
def test_get_project_folder(self):
"""
Test the get_project_folder method which should create a folder in case
it doesn't already exist.
"""
project_path = self.files_helper.get_project_folder(self.test_project)
self.assertTrue(os.path.exists(project_path), "Folder doesn't exist")
folder_path = self.files_helper.get_project_folder(
self.test_project, "43")
self.assertTrue(os.path.exists(project_path), "Folder doesn't exist")
self.assertTrue(os.path.exists(folder_path), "Folder doesn't exist")
def test_rename_project_structure(self):
""" Try to rename the folder structure of a project. Standard flow. """
self.files_helper.get_project_folder(self.test_project)
path, name = self.files_helper.rename_project_structure(
self.test_project.name, "new_name")
self.assertNotEqual(path, name, "Rename didn't take effect.")
def test_rename_structure_same_name(self):
""" Try to rename the folder structure of a project. Same name. """
self.files_helper.get_project_folder(self.test_project)
self.assertRaises(FileStructureException,
self.files_helper.rename_project_structure,
self.test_project.name, self.PROJECT_NAME)
def test_remove_project_structure(self):
""" Check that remove project structure deletes the corresponding folder. Standard flow. """
full_path = self.files_helper.get_project_folder(self.test_project)
self.assertTrue(os.path.exists(full_path), "Folder was not created.")
self.files_helper.remove_project_structure(self.test_project.name)
self.assertFalse(os.path.exists(full_path),
"Project folder not deleted.")
def test_write_project_metadata(self):
""" Write XML for test-project. """
self.files_helper.write_project_metadata(self.test_project)
expected_file = self.files_helper.get_project_meta_file_path(
self.PROJECT_NAME)
self.assertTrue(os.path.exists(expected_file))
project_meta = XMLReader(expected_file).read_metadata()
loaded_project = model.Project(None, None)
loaded_project.from_dict(project_meta, self.test_user.id)
self.assertEqual(self.test_project.name, loaded_project.name)
self.assertEqual(self.test_project.description,
loaded_project.description)
self.assertEqual(self.test_project.gid, loaded_project.gid)
expected_dict = self.test_project.to_dict()[1]
del expected_dict['last_updated']
found_dict = loaded_project.to_dict()[1]
del found_dict['last_updated']
self.assertDictContainsSubset(expected_dict, found_dict)
self.assertDictContainsSubset(found_dict, expected_dict)
def test_write_operation_metadata(self):
"""
Test that a correct XML is created for an operation.
"""
operation = TestFactory.create_operation(
test_user=self.test_user, test_project=self.test_project)
expected_file = self.files_helper.get_operation_meta_file_path(
self.PROJECT_NAME, operation.id)
self.assertFalse(os.path.exists(expected_file))
self.files_helper.write_operation_metadata(operation)
self.assertTrue(os.path.exists(expected_file))
operation_meta = XMLReader(expected_file).read_metadata()
loaded_operation = model.Operation(None, None, None, None)
loaded_operation.from_dict(operation_meta, dao)
expected_dict = operation.to_dict()[1]
found_dict = loaded_operation.to_dict()[1]
for key, value in expected_dict.iteritems():
self.assertEqual(str(value), str(found_dict[key]))
# Now validate that operation metaData can be also updated
self.assertNotEqual("new_group_name", found_dict['user_group'])
self.files_helper.update_operation_metadata(self.PROJECT_NAME,
"new_group_name",
operation.id)
found_dict = XMLReader(expected_file).read_metadata()
self.assertEqual("new_group_name", found_dict['user_group'])
def test_remove_dt_happy_flow(self):
"""
Happy flow for removing a file related to a DataType.
"""
folder_path = self.files_helper.get_project_folder(
self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
open(datatype.get_storage_file_path(), 'w')
self.assertTrue(os.path.exists(datatype.get_storage_file_path()),
"Test file was not created!")
self.files_helper.remove_datatype(datatype)
self.assertFalse(os.path.exists(datatype.get_storage_file_path()),
"Test file was not deleted!")
def test_remove_dt_non_existent(self):
"""
Try to call remove on a dataType with no H5 file.
Should work.
"""
folder_path = self.files_helper.get_project_folder(
self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
self.assertFalse(os.path.exists(datatype.get_storage_file_path()))
self.files_helper.remove_datatype(datatype)
def test_move_datatype(self):
"""
Make sure associated H5 file is moved to a correct new location.
"""
folder_path = self.files_helper.get_project_folder(
self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
open(datatype.get_storage_file_path(), 'w')
self.assertTrue(os.path.exists(datatype.get_storage_file_path()),
"Test file was not created!")
self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '11',
"43")
self.assertFalse(os.path.exists(datatype.get_storage_file_path()),
"Test file was not moved!")
datatype.storage_path = self.files_helper.get_project_folder(
self.PROJECT_NAME + '11', "43")
self.assertTrue(os.path.exists(datatype.get_storage_file_path()),
"Test file was not created!")
def test_find_relative_path(self):
"""
Tests that relative path is computed properly.
"""
rel_path = self.files_helper.find_relative_path(
"/root/up/to/here/test/it/now", "/root/up/to/here")
self.assertEqual(rel_path, os.sep.join(["test", "it", "now"]),
"Did not extract relative path as expected.")
def test_remove_files_valid(self):
"""
Pass a valid list of files and check they are all removed.
"""
file_list = ["test1", "test2", "test3"]
for file_n in file_list:
fp = open(file_n, 'w')
fp.write('test')
fp.close()
for file_n in file_list:
self.assertTrue(os.path.isfile(file_n))
self.files_helper.remove_files(file_list)
for file_n in file_list:
self.assertFalse(os.path.isfile(file_n))
def test_remove_folder(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
os.mkdir(folder_name)
self.assertTrue(os.path.isdir(folder_name),
"Folder should be created.")
self.files_helper.remove_folder(folder_name)
self.assertFalse(os.path.isdir(folder_name),
"Folder should be deleted.")
def test_remove_folder_non_existing_ignore_exc(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
self.assertFalse(os.path.isdir(folder_name),
"Folder should not exist before call.")
self.files_helper.remove_folder(folder_name, ignore_errors=True)
def test_remove_folder_non_existing(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
self.assertFalse(os.path.isdir(folder_name),
"Folder should not exist before call.")
self.assertRaises(FileStructureException,
self.files_helper.remove_folder, folder_name, False)
