def run_export(project_id, loose_irrelevant=False):
s = ProjectService()
mng = ExportManager()
project = s.find_project(project_id)
export_file = mng.export_project(project, loose_irrelevant)
print("Check the exported file: %s" % export_file)
def downloadproject(self, project_id):
"""
Export the data from a whole project.
"""
current_project = self.project_service.find_project(project_id)
export_mng = ExportManager()
export_file = export_mng.export_project(current_project)
# Register export file for delete when download complete
# We force parent folder deletion because export process generated it.
self.mark_file_for_delete(export_file, True)
return serve_file(export_file, "application/x-download", "attachment")
def downloadproject(self, project_id):
"""
Export the data from a whole project.
"""
current_project = self.project_service.find_project(project_id)
export_mng = ExportManager()
export_file = export_mng.export_project(current_project)
# Register export file for delete when download complete
# We force parent folder deletion because export process generated it.
self.mark_file_for_delete(export_file, True)
return serve_file(export_file, "application/x-download", "attachment")
class ExportersTest(TransactionalTestCase):
"""
Test export functionality.
"""
TVB_EXPORTER = "TVBExporter"
CIFTI_EXPORTER = "CIFTIExporter"
def setUp(self):
self.export_manager = ExportManager()
self.datatypeFactory = DatatypesFactory()
self.project = self.datatypeFactory.get_project()
def tearDown(self):
"""
Clean-up tests data
"""
project = self.datatypeFactory.get_project()
FilesHelper().remove_project_structure(project.name)
# Remove EXPORT folder
export_folder = os.path.join(TvbProfile.current.TVB_STORAGE,
ExportManager.EXPORT_FOLDER_NAME)
if os.path.exists(export_folder):
shutil.rmtree(export_folder)
def test_get_exporters_for_data(self):
"""
Test retrieval of exporters that can be used for a given data.
"""
datatype = self.datatypeFactory.create_simple_datatype()
exporters = self.export_manager.get_exporters_for_data(datatype)
# Only TVB export can export any type of data type
self.assertEqual(1, len(exporters), "Incorrect number of exporters.")
def test_get_exporters_for_data_with_no_data(self):
"""
Test retrieval of exporters when data == None.
"""
self.assertRaises(InvalidExportDataException,
self.export_manager.get_exporters_for_data, None)
def test_tvb_export_of_simple_datatype(self):
"""
Test export of a data type which has no data stored on file system
"""
datatype = self.datatypeFactory.create_simple_datatype()
file_name, file_path, _ = self.export_manager.export_data(
datatype, self.TVB_EXPORTER, self.project)
self.assertTrue(file_name is not None,
"Export process should return a file name")
self.assertTrue(file_path is not None,
"Export process should return path to export file")
self.assertTrue(os.path.exists(file_path),
"Could not find export file: %s on disk." % file_path)
def test_tvb_export_of_datatype_with_storage(self):
"""
Test export of a data type which has no data stored on file system
"""
datatype = self.datatypeFactory.create_datatype_with_storage()
file_name, file_path, _ = self.export_manager.export_data(
datatype, self.TVB_EXPORTER, self.project)
self.assertTrue(file_name is not None,
"Export process should return a file name")
self.assertTrue(file_path is not None,
"Export process should return path to export file")
self.assertTrue(os.path.exists(file_path),
"Could not find export file: %s on disk." % file_path)
def test_tvb_export_for_datatype_group(self):
"""
This method checks export of a data type group
"""
datatype_group = self.datatypeFactory.create_datatype_group()
file_name, file_path, _ = self.export_manager.export_data(
datatype_group, self.TVB_EXPORTER, self.project)
self.assertTrue(file_name is not None,
"Export process should return a file name")
self.assertTrue(file_path is not None,
"Export process should return path to export file")
self.assertTrue(os.path.exists(file_path),
"Could not find export file: %s on disk." % file_path)
# Now check if the generated file is a correct ZIP file
self.assertTrue(zipfile.is_zipfile(file_path),
"Generated file is not a valid ZIP file")
with closing(zipfile.ZipFile(file_path)) as zip_file:
list_of_files = zip_file.namelist()
count_datatypes = dao.count_datatypes_in_group(datatype_group.id)
# Check if ZIP files contains files for data types + operation
self.assertEqual(
count_datatypes * 2, len(list_of_files),
"Should have 2 x nr datatypes files, one for operations one for datatypes"
)
def test_export_with_invalid_data(self):
"""
Test scenarios when data provided to export method is invalid
"""
# Test with no datatype
self.assertRaises(InvalidExportDataException,
self.export_manager.export_data, None,
self.TVB_EXPORTER, self.project)
# Test with no exporter
datatype = self.datatypeFactory.create_datatype_with_storage()
self.assertRaises(ExportException, self.export_manager.export_data,
datatype, None, self.project)
# test with wrong exporter
self.assertRaises(ExportException, self.export_manager.export_data,
datatype, "wrong_exporter", self.project)
# test with no project folder
self.assertRaises(ExportException, self.export_manager.export_data,
datatype, self.TVB_EXPORTER, None)
def test_export_project_failure(self):
"""
This method tests export of project with None data
"""
self.assertRaises(ExportException, self.export_manager.export_project,
None)
def tet_export_project(self):
"""
Test export of a project
"""
project = self.datatypeFactory.get_project()
export_file = self.export_manager.export_project(project)
self.assertTrue(export_file is not None,
"Export process should return path to export file")
self.assertTrue(
os.path.exists(export_file),
"Could not find export file: %s on disk." % export_file)
# Now check if the generated file is a correct ZIP file
self.assertTrue(zipfile.is_zipfile(export_file),
"Generated file is not a valid ZIP file")
class TestExporters(TransactionalTestCase):
"""
Test export functionality.
"""
TVB_EXPORTER = "TVBExporter"
CIFTI_EXPORTER = "CIFTIExporter"
def transactional_setup_method(self):
self.export_manager = ExportManager()
self.test_user = TestFactory.create_user('Exporter_Tests_User1')
self.test_project = TestFactory.create_project(
self.test_user, 'Exporter_Tests_Project1')
def transactional_teardown_method(self):
"""
Clean-up tests data
"""
user = TestFactory.create_user('Exporter_Tests_User2')
project = TestFactory.create_project(user, 'Exporter_Tests_Project2')
StorageInterface().remove_project_structure(project.name)
# Remove EXPORT folder
export_folder = os.path.join(TvbProfile.current.TVB_STORAGE,
ExportManager.EXPORT_FOLDER_NAME)
if os.path.exists(export_folder):
shutil.rmtree(export_folder)
def test_get_exporters_for_data(self, dummy_datatype_index_factory):
"""
Test retrieval of exporters that can be used for a given data.
"""
datatype = dummy_datatype_index_factory()
exporters = self.export_manager.get_exporters_for_data(datatype)
# Only TVB export can export any type of data type
assert 1, len(exporters) == "Incorrect number of exporters."
def test_get_exporters_for_data_with_no_data(self):
"""
Test retrieval of exporters when data == None.
"""
with pytest.raises(InvalidExportDataException):
self.export_manager.get_exporters_for_data(None)
def test_tvb_export_of_simple_datatype(self, dummy_datatype_index_factory):
"""
Test export of a data type which has no data stored on file system
"""
datatype = dummy_datatype_index_factory()
_, file_path, _ = self.export_manager.export_data(
datatype, self.TVB_EXPORTER, self.test_project)
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file: %s on disk." % file_path
def test_tvb_export_of_datatype_with_storage(self,
dummy_datatype_index_factory):
"""
Test export of a data type which has no data stored on file system
"""
datatype = dummy_datatype_index_factory()
_, file_path, _ = self.export_manager.export_data(
datatype, self.TVB_EXPORTER, self.test_project)
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file: %s on disk." % file_path
def test_tvb_export_for_datatype_group(self, datatype_group_factory):
"""
This method checks export of a data type group
"""
datatype_group = datatype_group_factory(project=self.test_project,
store_vm=True)
file_name, file_path, _ = self.export_manager.export_data(
datatype_group, self.TVB_EXPORTER, self.test_project)
assert file_name is not None, "Export process should return a file name"
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file: %s on disk." % file_path
# Now check if the generated file is a correct ZIP file
assert zipfile.is_zipfile(
file_path), "Generated file is not a valid ZIP file"
with closing(zipfile.ZipFile(file_path)) as zip_file:
list_of_files = zip_file.namelist()
list_of_folders = []
for file in list_of_files:
dir_name = os.path.dirname(file)
if dir_name not in list_of_folders:
list_of_folders.append(dir_name)
count_datatypes = dao.count_datatypes_in_group(datatype_group.id)
# Check if ZIP files contains files for data types and view models (multiple H5 files in case of a Sim)
assert count_datatypes == len(list_of_folders)
assert count_datatypes * 6 == len(list_of_files)
def test_export_with_invalid_data(self, dummy_datatype_index_factory):
"""
Test scenarios when data provided to export method is invalid
"""
# Test with no datatype
with pytest.raises(InvalidExportDataException):
self.export_manager.export_data(None, self.TVB_EXPORTER,
self.test_project)
# Test with no exporter
datatype = dummy_datatype_index_factory()
with pytest.raises(ExportException):
self.export_manager.export_data(datatype, None, self.test_project)
# test with wrong exporter
with pytest.raises(ExportException):
self.export_manager.export_data(datatype, "wrong_exporter",
self.test_project)
# test with no project folder
with pytest.raises(ExportException):
self.export_manager.export_data(datatype, self.TVB_EXPORTER, None)
def test_export_project_failure(self):
"""
This method tests export of project with None data
"""
with pytest.raises(ExportException):
self.export_manager.export_project(None)
def test_export_project(self, project_factory, user_factory):
"""
Test export of a project
"""
user = user_factory(username='******')
project = project_factory(user)
export_file = self.export_manager.export_project(project)
assert export_file is not None, "Export process should return path to export file"
assert os.path.exists(
export_file
), "Could not find export file: %s on disk." % export_file
# Now check if the generated file is a correct ZIP file
assert zipfile.is_zipfile(
export_file), "Generated file is not a valid ZIP file"
def test_export_simulator_configuration(self, operation_factory,
connectivity_index_factory):
"""
Test export of a simulator configuration
"""
conn_gid = uuid.UUID(connectivity_index_factory().gid)
operation = operation_factory(is_simulation=True,
store_vm=True,
test_project=self.test_project,
conn_gid=conn_gid)
burst_configuration = BurstConfiguration(self.test_project.id)
burst_configuration.fk_simulation = operation.id
burst_configuration.simulator_gid = operation.view_model_gid
burst_configuration.name = "Test_burst"
burst_configuration = dao.store_entity(burst_configuration)
op_folder = StorageInterface().get_project_folder(
self.test_project.name, str(operation.id))
BurstService().store_burst_configuration(burst_configuration,
op_folder)
export_file = self.export_manager.export_simulator_configuration(
burst_configuration.id)
assert export_file is not None, "Export process should return path to export file"
assert os.path.exists(
export_file
), "Could not find export file: %s on disk." % export_file
assert zipfile.is_zipfile(
export_file), "Generated file is not a valid ZIP file"
class TestImportExportProjectWithLinksTest(_BaseLinksTest):
@pytest.fixture()
def initialize_linked_projects(self, initialize_two_projects):
"""
Adds to the _BaseLinksTest setup the following
2 links from src to dest project
Import/export services
"""
dest_id = self.dest_project.id
self.flow_service.create_link([self.red_datatype.id], dest_id)
self.flow_service.create_link([self.blue_datatype.id], dest_id)
self.export_mng = ExportManager()
def test_export(self, initialize_linked_projects):
export_file = self.export_mng.export_project(self.dest_project)
with TvbZip(export_file) as z:
assert 'links-to-external-projects/Operation.xml' in z.namelist()
def _export_and_remove(self, project):
"""export the project and remove it"""
export_file = self.export_mng.export_project(project)
self.project_service.remove_project(project.id)
return export_file
def _import(self, export_file, user_id):
""" import a project zip for a user """
# instantiated for every use because it is stateful
import_service = ImportService()
import_service.import_project_structure(export_file, user_id)
return import_service.created_projects[0].id
def test_links_recreated_on_import(self, initialize_linked_projects):
export_file = self._export_and_remove(self.dest_project)
imported_proj_id = self._import(export_file, self.dst_usr_id)
assert 1 == self.red_datatypes_in(imported_proj_id)
assert 1 == self.blue_datatypes_in(imported_proj_id)
links = dao.get_linked_datatypes_in_project(imported_proj_id)
assert 2 == len(links)
def test_datatypes_recreated_on_import(self, initialize_linked_projects):
export_file = self._export_and_remove(self.dest_project)
self.project_service.remove_project(self.src_project.id)
# both projects have been deleted
# import should recreate links as datatypes
imported_proj_id = self._import(export_file, self.dst_usr_id)
assert 1 == self.red_datatypes_in(imported_proj_id)
assert 1 == self.blue_datatypes_in(imported_proj_id)
links = dao.get_linked_datatypes_in_project(imported_proj_id)
assert 0 == len(links)
def test_datatypes_and_links_recreated_on_import(
self, initialize_linked_projects):
export_file = self._export_and_remove(self.dest_project)
# remove datatype 2 from source project
self.project_service.remove_datatype(self.src_project.id,
self.blue_datatype.gid)
imported_proj_id = self._import(export_file, self.dst_usr_id)
# both datatypes should be recreated
assert 1 == self.red_datatypes_in(imported_proj_id)
assert 1 == self.blue_datatypes_in(imported_proj_id)
# only datatype 1 should be a link
links = dao.get_linked_datatypes_in_project(imported_proj_id)
assert 1 == len(links)
assert self.red_datatype.gid == links[0].gid
@pytest.fixture()
def create_interlinked_projects(self):
def build():
"""
Project src will have 3 datatypes, and a link to the VW from the dest project.
Project dest will have the derived VW and links
"""
# add a connectivity to src project and link it to dest project
zip_path = os.path.join(os.path.dirname(tvb_data.__file__),
'connectivity', 'connectivity_96.zip')
conn = TestFactory.import_zip_connectivity(self.dst_user,
self.src_project,
zip_path, "John")
self.flow_service.create_link([conn.id], self.dest_project.id)
# in dest derive a ValueWrapper from the linked conn
vw_gid = TestFactory.create_value_wrapper(self.dst_user,
self.dest_project)[1]
vw = dao.get_datatype_by_gid(vw_gid)
# then link the time series in the src project
self.flow_service.create_link([vw.id], self.src_project.id)
assert 3 == len(dao.get_datatypes_in_project(self.src_project.id))
assert 1 == len(
dao.get_linked_datatypes_in_project(self.src_project.id))
assert 1 == len(dao.get_datatypes_in_project(self.dest_project.id))
assert 3 == len(
dao.get_linked_datatypes_in_project(self.dest_project.id))
return build
def test_create_interlinked_projects(self, initialize_linked_projects,
create_interlinked_projects):
create_interlinked_projects()
def test_linked_datatype_dependencies_restored_on_import(
self, initialize_linked_projects, create_interlinked_projects):
create_interlinked_projects()
# export both then remove them
export_file_src = self._export_and_remove(self.src_project)
assert 4 == len(dao.get_datatypes_in_project(self.dest_project.id))
assert 0 == len(
dao.get_linked_datatypes_in_project(self.dest_project.id))
export_file_dest = self._export_and_remove(self.dest_project)
# importing both projects should work
imported_id_1 = self._import(export_file_src, self.src_usr_id)
assert 4 == len(dao.get_datatypes_in_project(imported_id_1))
assert 0 == len(dao.get_linked_datatypes_in_project(imported_id_1))
imported_id_2 = self._import(export_file_dest, self.dst_usr_id)
assert 0 == len(dao.get_datatypes_in_project(imported_id_2))
assert 4 == len(dao.get_linked_datatypes_in_project(imported_id_2))
def test_linked_datatype_dependencies_restored_on_import_inverse_order(
self, initialize_linked_projects, create_interlinked_projects):
create_interlinked_projects()
# export both then remove them
export_file_src = self._export_and_remove(self.src_project)
assert 4 == len(dao.get_datatypes_in_project(self.dest_project.id))
assert 0 == len(
dao.get_linked_datatypes_in_project(self.dest_project.id))
export_file_dest = self._export_and_remove(self.dest_project)
# importing dest before src should work
imported_id_2 = self._import(export_file_dest, self.dst_usr_id)
assert 4 == len(dao.get_datatypes_in_project(imported_id_2))
assert 0 == len(dao.get_linked_datatypes_in_project(imported_id_2))
imported_id_1 = self._import(export_file_src, self.src_usr_id)
assert 0 == len(dao.get_datatypes_in_project(imported_id_1))
assert 4 == len(dao.get_linked_datatypes_in_project(imported_id_1))
class ImportExportProjectWithLinksTest(_BaseLinksTest):
def setUp(self):
"""
Adds to the _BaseLinksTest setup the following
2 links from src to dest project
Import/export services
"""
self.clean_database(delete_folders=True)
self._initialize_two_projects()
dest_id = self.dest_project.id
self.flow_service.create_link([self.red_datatype.id], dest_id)
self.flow_service.create_link([self.blue_datatype.id], dest_id)
self.export_mng = ExportManager()
def test_export(self):
export_file = self.export_mng.export_project(self.dest_project)
with TvbZip(export_file) as z:
self.assertTrue('links-to-external-projects/Operation.xml' in z.namelist())
def _export_and_remove(self, project):
"""export the project and remove it"""
export_file = self.export_mng.export_project(project)
self.project_service.remove_project(project.id)
return export_file
def _import(self, export_file, user_id):
""" import a project zip for a user """
# instantiated for every use because it is stateful
import_service = ImportService()
import_service.import_project_structure(export_file, user_id)
return import_service.created_projects[0].id
def test_links_recreated_on_import(self):
export_file = self._export_and_remove(self.dest_project)
imported_proj_id = self._import(export_file, self.dest_usr_id)
self.assertEqual(1, self.red_datatypes_in(imported_proj_id))
self.assertEqual(1, self.blue_datatypes_in(imported_proj_id))
links = dao.get_linked_datatypes_in_project(imported_proj_id)
self.assertEqual(2, len(links))
def test_datatypes_recreated_on_import(self):
export_file = self._export_and_remove(self.dest_project)
self.project_service.remove_project(self.src_project.id)
# both projects have been deleted
# import should recreate links as datatypes
imported_proj_id = self._import(export_file, self.dest_usr_id)
self.assertEqual(1, self.red_datatypes_in(imported_proj_id))
self.assertEqual(1, self.blue_datatypes_in(imported_proj_id))
links = dao.get_linked_datatypes_in_project(imported_proj_id)
self.assertEqual(0, len(links))
def test_datatypes_and_links_recreated_on_import(self):
export_file = self._export_and_remove(self.dest_project)
# remove datatype 2 from source project
self.project_service.remove_datatype(self.src_project.id, self.blue_datatype.gid)
imported_proj_id = self._import(export_file, self.dest_usr_id)
# both datatypes should be recreated
self.assertEqual(1, self.red_datatypes_in(imported_proj_id))
self.assertEqual(1, self.blue_datatypes_in(imported_proj_id))
# only datatype 1 should be a link
links = dao.get_linked_datatypes_in_project(imported_proj_id)
self.assertEqual(1, len(links))
self.assertEquals(self.red_datatype.gid, links[0].gid)
def _create_interlinked_projects(self):
"""
Extend the two projects created in setup.
Project src will have 3 datatypes, one a connectivity, and a link to the time series from the dest project.
Project dest will have 3 links to the datatypes in src and a time series derived from the linked connectivity
"""
# add a connectivity to src project and link it to dest project
_, conn = self.datatype_factory_src.create_connectivity()
self.flow_service.create_link([conn.id], self.dest_project.id)
# in dest derive a time series from the linked conn
ts = self.datatype_factory_dest.create_timeseries(conn)
# then link the time series in the src project
self.flow_service.create_link([ts.id], self.src_project.id)
self.assertEqual(3, len(dao.get_datatypes_in_project(self.src_project.id)))
self.assertEqual(1, len(dao.get_linked_datatypes_in_project(self.src_project.id)))
self.assertEqual(1, len(dao.get_datatypes_in_project(self.dest_project.id)))
self.assertEqual(3, len(dao.get_linked_datatypes_in_project(self.dest_project.id)))
def test_create_interlinked_projects(self):
self._create_interlinked_projects()
def test_linked_datatype_dependencies_restored_on_import(self):
self._create_interlinked_projects()
# export both then remove them
export_file_src = self._export_and_remove(self.src_project)
self.assertEqual(4, len(dao.get_datatypes_in_project(self.dest_project.id)))
self.assertEqual(0, len(dao.get_linked_datatypes_in_project(self.dest_project.id)))
export_file_dest = self._export_and_remove(self.dest_project)
# importing both projects should work
imported_id_1 = self._import(export_file_src, self.src_usr_id)
self.assertEqual(4, len(dao.get_datatypes_in_project(imported_id_1)))
self.assertEqual(0, len(dao.get_linked_datatypes_in_project(imported_id_1)))
imported_id_2 = self._import(export_file_dest, self.dest_usr_id)
self.assertEqual(0, len(dao.get_datatypes_in_project(imported_id_2)))
self.assertEqual(4, len(dao.get_linked_datatypes_in_project(imported_id_2)))
def test_linked_datatype_dependencies_restored_on_import_inverse_order(self):
self._create_interlinked_projects()
# export both then remove them
export_file_src = self._export_and_remove(self.src_project)
self.assertEqual(4, len(dao.get_datatypes_in_project(self.dest_project.id)))
self.assertEqual(0, len(dao.get_linked_datatypes_in_project(self.dest_project.id)))
export_file_dest = self._export_and_remove(self.dest_project)
# importing dest before src should work
imported_id_2 = self._import(export_file_dest, self.dest_usr_id)
self.assertEqual(4, len(dao.get_datatypes_in_project(imported_id_2)))
self.assertEqual(0, len(dao.get_linked_datatypes_in_project(imported_id_2)))
imported_id_1 = self._import(export_file_src, self.src_usr_id)
self.assertEqual(0, len(dao.get_datatypes_in_project(imported_id_1)))
self.assertEqual(4, len(dao.get_linked_datatypes_in_project(imported_id_1)))
class ExportersTest(TransactionalTestCase):
"""
Test export functionality.
"""
TVB_EXPORTER = "TVBExporter"
CIFTI_EXPORTER = "CIFTIExporter"
def setUp(self):
self.export_manager = ExportManager()
self.datatypeFactory = DatatypesFactory()
self.project = self.datatypeFactory.get_project()
def tearDown(self):
"""
Clean-up tests data
"""
project = self.datatypeFactory.get_project()
FilesHelper().remove_project_structure(project.name)
# Remove EXPORT folder
export_folder = os.path.join(TvbProfile.current.TVB_STORAGE, ExportManager.EXPORT_FOLDER_NAME)
if os.path.exists(export_folder):
shutil.rmtree(export_folder)
def test_get_exporters_for_data(self):
"""
Test retrieval of exporters that can be used for a given data.
"""
datatype = self.datatypeFactory.create_simple_datatype()
exporters = self.export_manager.get_exporters_for_data(datatype)
# Only TVB export can export any type of data type
self.assertEqual(1, len(exporters), "Incorrect number of exporters.")
def test_get_exporters_for_data_with_no_data(self):
"""
Test retrieval of exporters when data == None.
"""
self.assertRaises(InvalidExportDataException, self.export_manager.get_exporters_for_data, None)
def test_tvb_export_of_simple_datatype(self):
"""
Test export of a data type which has no data stored on file system
"""
datatype = self.datatypeFactory.create_simple_datatype()
file_name, file_path, _ = self.export_manager.export_data(datatype, self.TVB_EXPORTER, self.project)
self.assertTrue(file_name is not None, "Export process should return a file name")
self.assertTrue(file_path is not None, "Export process should return path to export file")
self.assertTrue(os.path.exists(file_path), "Could not find export file: %s on disk." % file_path)
def test_tvb_export_of_datatype_with_storage(self):
"""
Test export of a data type which has no data stored on file system
"""
datatype = self.datatypeFactory.create_datatype_with_storage()
file_name, file_path, _ = self.export_manager.export_data(datatype, self.TVB_EXPORTER, self.project)
self.assertTrue(file_name is not None, "Export process should return a file name")
self.assertTrue(file_path is not None, "Export process should return path to export file")
self.assertTrue(os.path.exists(file_path), "Could not find export file: %s on disk." % file_path)
def test_tvb_export_for_datatype_group(self):
"""
This method checks export of a data type group
"""
datatype_group = self.datatypeFactory.create_datatype_group()
file_name, file_path, _ = self.export_manager.export_data(datatype_group, self.TVB_EXPORTER, self.project)
self.assertTrue(file_name is not None, "Export process should return a file name")
self.assertTrue(file_path is not None, "Export process should return path to export file")
self.assertTrue(os.path.exists(file_path), "Could not find export file: %s on disk." % file_path)
# Now check if the generated file is a correct ZIP file
self.assertTrue(zipfile.is_zipfile(file_path), "Generated file is not a valid ZIP file")
with closing(zipfile.ZipFile(file_path)) as zip_file:
list_of_files = zip_file.namelist()
count_datatypes = dao.count_datatypes_in_group(datatype_group.id)
# Check if ZIP files contains files for data types + operation
self.assertEqual(count_datatypes * 2, len(list_of_files),
"Should have 2 x nr datatypes files, one for operations one for datatypes")
def test_export_with_invalid_data(self):
"""
Test scenarios when data provided to export method is invalid
"""
# Test with no datatype
self.assertRaises(InvalidExportDataException, self.export_manager.export_data,
None, self.TVB_EXPORTER, self.project)
# Test with no exporter
datatype = self.datatypeFactory.create_datatype_with_storage()
self.assertRaises(ExportException, self.export_manager.export_data, datatype, None, self.project)
# test with wrong exporter
self.assertRaises(ExportException, self.export_manager.export_data, datatype, "wrong_exporter", self.project)
# test with no project folder
self.assertRaises(ExportException, self.export_manager.export_data, datatype, self.TVB_EXPORTER, None)
def test_export_project_failure(self):
"""
This method tests export of project with None data
"""
self.assertRaises(ExportException, self.export_manager.export_project, None)
def tet_export_project(self):
"""
Test export of a project
"""
project = self.datatypeFactory.get_project()
export_file = self.export_manager.export_project(project)
self.assertTrue(export_file is not None, "Export process should return path to export file")
self.assertTrue(os.path.exists(export_file), "Could not find export file: %s on disk." % export_file)
# Now check if the generated file is a correct ZIP file
self.assertTrue(zipfile.is_zipfile(export_file), "Generated file is not a valid ZIP file")
class TestImportExportProjectWithLinksTest(_BaseLinksTest):
@pytest.fixture()
def transactional_setup_fixture(self, initialize_two_projects):
"""
Adds to the _BaseLinksTest setup the following
2 links from src to dest project
Import/export services
"""
dest_id = self.dest_project.id
self.flow_service.create_link([self.red_datatype.id], dest_id)
self.flow_service.create_link([self.blue_datatype.id], dest_id)
self.export_mng = ExportManager()
def test_export(self, transactional_setup_fixture):
export_file = self.export_mng.export_project(self.dest_project)
with TvbZip(export_file) as z:
assert 'links-to-external-projects/Operation.xml' in z.namelist()
def _export_and_remove(self, project):
"""export the project and remove it"""
export_file = self.export_mng.export_project(project)
self.project_service.remove_project(project.id)
return export_file
def _import(self, export_file, user_id):
""" import a project zip for a user """
# instantiated for every use because it is stateful
import_service = ImportService()
import_service.import_project_structure(export_file, user_id)
return import_service.created_projects[0].id
def test_links_recreated_on_import(self, transactional_setup_fixture):
export_file = self._export_and_remove(self.dest_project)
imported_proj_id = self._import(export_file, self.dst_usr_id)
assert 1 == self.red_datatypes_in(imported_proj_id)
assert 0 == self.blue_datatypes_in(imported_proj_id)
links = dao.get_linked_datatypes_in_project(imported_proj_id)
assert 2 == len(links)
def test_datatypes_recreated_on_import(self, transactional_setup_fixture):
export_file = self._export_and_remove(self.dest_project)
self.project_service.remove_project(self.src_project.id)
# both projects have been deleted
# import should recreate links as datatypes
imported_proj_id = self._import(export_file, self.dst_usr_id)
assert 1 == self.red_datatypes_in(imported_proj_id)
assert 1 == self.blue_datatypes_in(imported_proj_id)
links = dao.get_linked_datatypes_in_project(imported_proj_id)
assert 0 == len(links)
def test_datatypes_and_links_recreated_on_import(self, transactional_setup_fixture):
export_file = self._export_and_remove(self.dest_project)
# remove datatype 2 from source project
self.project_service.remove_datatype(self.src_project.id, self.blue_datatype.gid)
imported_proj_id = self._import(export_file, self.dst_usr_id)
# both datatypes should be recreated
assert 1 == self.red_datatypes_in(imported_proj_id)
assert 1 == self.blue_datatypes_in(imported_proj_id)
# only datatype 1 should be a link
links = dao.get_linked_datatypes_in_project(imported_proj_id)
assert 1 == len(links)
assert self.red_datatype.gid == links[0].gid
@pytest.fixture()
def create_interlinked_projects(self, region_mapping_factory, time_series_region_index_factory):
"""
Extend the two projects created in setup.
Project src will have 3 datatypes, one a connectivity, and a link to the time series from the dest project.
Project dest will have 3 links to the datatypes in src and a time series derived from the linked connectivity
"""
# add a connectivity to src project and link it to dest project
zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_96.zip')
TestFactory.import_zip_connectivity(self.dst_user, self.dest_project, zip_path, "John")
conn = TestFactory.get_entity(self.dest_project, ConnectivityIndex)
self.flow_service.create_link([conn.id], self.dest_project.id)
# in dest derive a time series from the linked conn
path = os.path.join(os.path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip')
surface = TestFactory.import_surface_zip(self.dst_user, self.dest_project, path, CORTICAL)
TXT_FILE = os.path.join(os.path.dirname(demo_data.__file__), 'regionMapping_16k_76.txt')
region_mapping = TestFactory.import_region_mapping(self.dst_user, self.dest_project, TXT_FILE,
surface.gid, conn.gid)
ts = time_series_region_index_factory(connectivity=h5.load_from_index(conn), region_mapping=h5.load_from_index(region_mapping))
# then link the time series in the src project
self.flow_service.create_link([ts.id], self.src_project.id)
assert 3 == len(dao.get_datatypes_in_project(self.src_project.id))
assert 1 == len(dao.get_linked_datatypes_in_project(self.src_project.id))
assert 1 == len(dao.get_datatypes_in_project(self.dest_project.id))
assert 3 == len(dao.get_linked_datatypes_in_project(self.dest_project.id))
def test_create_interlinked_projects(self, transactional_setup_fixture, create_interlinked_projects):
create_interlinked_projects()
def test_linked_datatype_dependencies_restored_on_import(self, transactional_setup_fixture, create_interlinked_projects):
create_interlinked_projects()
# export both then remove them
export_file_src = self._export_and_remove(self.src_project)
assert 4 == len(dao.get_datatypes_in_project(self.dest_project.id))
assert 0 == len(dao.get_linked_datatypes_in_project(self.dest_project.id))
export_file_dest = self._export_and_remove(self.dest_project)
# importing both projects should work
imported_id_1 = self._import(export_file_src, self.src_usr_id)
assert 4 == len(dao.get_datatypes_in_project(imported_id_1))
assert 0 == len(dao.get_linked_datatypes_in_project(imported_id_1))
imported_id_2 = self._import(export_file_dest, self.dest_usr_id)
assert 0 == len(dao.get_datatypes_in_project(imported_id_2))
assert 4 == len(dao.get_linked_datatypes_in_project(imported_id_2))
def test_linked_datatype_dependencies_restored_on_import_inverse_order(self, transactional_setup_fixture, create_interlinked_projects):
create_interlinked_projects()
# export both then remove them
export_file_src = self._export_and_remove(self.src_project)
assert 4 == len(dao.get_datatypes_in_project(self.dest_project.id))
assert 0 == len(dao.get_linked_datatypes_in_project(self.dest_project.id))
export_file_dest = self._export_and_remove(self.dest_project)
# importing dest before src should work
imported_id_2 = self._import(export_file_dest, self.dest_usr_id)
assert 4 == len(dao.get_datatypes_in_project(imported_id_2))
assert 0 == len(dao.get_linked_datatypes_in_project(imported_id_2))
imported_id_1 = self._import(export_file_src, self.src_usr_id)
assert 0 == len(dao.get_datatypes_in_project(imported_id_1))
assert 4 == len(dao.get_linked_datatypes_in_project(imported_id_1))
class TestExporters(TransactionalTestCase):
"""
Test export functionality.
"""
TVB_EXPORTER = "TVBExporter"
CIFTI_EXPORTER = "CIFTIExporter"
def transactional_setup_method(self):
self.export_manager = ExportManager()
self.test_user = TestFactory.create_user('Exporter_Tests_User1')
self.test_project = TestFactory.create_project(
self.test_user, 'Exporter_Tests_Project1')
def transactional_teardown_method(self):
"""
Clean-up tests data
"""
user = TestFactory.create_user('Exporter_Tests_User2')
project = TestFactory.create_project(user, 'Exporter_Tests_Project2')
FilesHelper().remove_project_structure(project.name)
# Remove EXPORT folder
export_folder = os.path.join(TvbProfile.current.TVB_STORAGE,
ExportManager.EXPORT_FOLDER_NAME)
if os.path.exists(export_folder):
shutil.rmtree(export_folder)
def test_get_exporters_for_data(self, dummy_datatype_index_factory):
"""
Test retrieval of exporters that can be used for a given data.
"""
datatype = dummy_datatype_index_factory()
exporters = self.export_manager.get_exporters_for_data(datatype)
# Only TVB export can export any type of data type
assert 1, len(exporters) == "Incorrect number of exporters."
def test_get_exporters_for_data_with_no_data(self):
"""
Test retrieval of exporters when data == None.
"""
with pytest.raises(InvalidExportDataException):
self.export_manager.get_exporters_for_data(None)
def test_tvb_export_of_simple_datatype(self, dummy_datatype_index_factory):
"""
Test export of a data type which has no data stored on file system
"""
datatype = dummy_datatype_index_factory()
file_name, file_path, _ = self.export_manager.export_data(
datatype, self.TVB_EXPORTER, self.test_project)
assert file_name is not None, "Export process should return a file name"
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file: %s on disk." % file_path
def test_tvb_export_of_datatype_with_storage(self,
dummy_datatype_index_factory):
"""
Test export of a data type which has no data stored on file system
"""
datatype = dummy_datatype_index_factory()
file_name, file_path, _ = self.export_manager.export_data(
datatype, self.TVB_EXPORTER, self.test_project)
assert file_name is not None, "Export process should return a file name"
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file: %s on disk." % file_path
def test_tvb_export_for_datatype_group(self, datatype_group_factory):
"""
This method checks export of a data type group
"""
datatype_group = datatype_group_factory(project=self.test_project)
file_name, file_path, _ = self.export_manager.export_data(
datatype_group, self.TVB_EXPORTER, self.test_project)
assert file_name is not None, "Export process should return a file name"
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file: %s on disk." % file_path
# Now check if the generated file is a correct ZIP file
assert zipfile.is_zipfile(
file_path), "Generated file is not a valid ZIP file"
with closing(zipfile.ZipFile(file_path)) as zip_file:
list_of_files = zip_file.namelist()
count_datatypes = dao.count_datatypes_in_group(datatype_group.id)
# Check if ZIP files contains files for data types
assert count_datatypes == len(list_of_files)
def test_export_with_invalid_data(self, dummy_datatype_index_factory):
"""
Test scenarios when data provided to export method is invalid
"""
# Test with no datatype
with pytest.raises(InvalidExportDataException):
self.export_manager.export_data(None, self.TVB_EXPORTER,
self.test_project)
# Test with no exporter
datatype = dummy_datatype_index_factory()
with pytest.raises(ExportException):
self.export_manager.export_data(datatype, None, self.test_project)
# test with wrong exporter
with pytest.raises(ExportException):
self.export_manager.export_data(datatype, "wrong_exporter",
self.test_project)
# test with no project folder
with pytest.raises(ExportException):
self.export_manager.export_data(datatype, self.TVB_EXPORTER, None)
def test_export_project_failure(self):
"""
This method tests export of project with None data
"""
with pytest.raises(ExportException):
self.export_manager.export_project(None)
def test_export_project(self, project_factory, user_factory):
"""
Test export of a project
"""
user = user_factory(username='******')
project = project_factory(user)
export_file = self.export_manager.export_project(project)
assert export_file is not None, "Export process should return path to export file"
assert os.path.exists(
export_file
), "Could not find export file: %s on disk." % export_file
# Now check if the generated file is a correct ZIP file
assert zipfile.is_zipfile(
export_file), "Generated file is not a valid ZIP file"
def test_export_simulator_configuration(self, operation_factory):
"""
Test export of a simulator configuration
"""
operation = operation_factory()
simulator = Simulator()
simulator_index = SimulatorIndex()
simulator_index.fill_from_has_traits(simulator)
simulator_index.fk_from_operation = operation.id
simulator_index = dao.store_entity(simulator_index)
burst_configuration = BurstConfiguration(self.test_project.id,
simulator_index.id)
burst_configuration = dao.store_entity(burst_configuration)
simulator_index.fk_parent_burst = burst_configuration.id
simulator_index = dao.store_entity(simulator_index)
simulator_h5 = h5.path_for_stored_index(simulator_index)
with SimulatorH5(simulator_h5) as h5_file:
h5_file.store(simulator)
export_file = self.export_manager.export_simulator_configuration(
burst_configuration.id)
assert export_file is not None, "Export process should return path to export file"
assert os.path.exists(
export_file
), "Could not find export file: %s on disk." % export_file
assert zipfile.is_zipfile(
export_file), "Generated file is not a valid ZIP file"
class TestExporters(TransactionalTestCase):
"""
Test export functionality.
"""
TVB_EXPORTER = "TVBExporter"
TVB_LINKED_EXPORTER = "TVBLinkedExporter"
def transactional_setup_method(self):
self.export_manager = ExportManager()
self.test_user = TestFactory.create_user('Exporter_Tests_User1')
self.test_project = TestFactory.create_project(
self.test_user, 'Exporter_Tests_Project1')
def transactional_teardown_method(self):
"""
Clean-up tests data
"""
# Remove EXPORT folder
export_folder = os.path.join(TvbProfile.current.TVB_STORAGE,
StorageInterface.EXPORT_FOLDER_NAME)
StorageInterface.remove_folder(export_folder, True)
def test_get_exporters_for_data(self, dummy_datatype_index_factory):
"""
Test retrieval of exporters that can be used for a given data.
"""
datatype = dummy_datatype_index_factory()
exporters = self.export_manager.get_exporters_for_data(datatype)
# Only TVB export can export any type of data type
assert len(exporters) == 2, "Incorrect number of exporters."
def test_get_exporters_for_data_with_no_data(self):
"""
Test retrieval of exporters when data == None.
"""
with pytest.raises(InvalidExportDataException):
self.export_manager.get_exporters_for_data(None)
def test_tvb_export_of_simple_datatype(self, dummy_datatype_index_factory):
"""
Test export of a data type which has no data stored on file system
"""
datatype = dummy_datatype_index_factory()
_, file_path, _ = self.export_manager.export_data(
datatype, self.TVB_EXPORTER, self.test_project)
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file: %s on disk." % file_path
@staticmethod
def compare_files(original_path, decrypted_file_path):
buffer_size = TvbProfile.current.hpc.CRYPT_BUFFER_SIZE
with open(original_path, 'rb') as f_original:
with open(decrypted_file_path, 'rb') as f_decrypted:
while True:
original_content_chunk = f_original.read(buffer_size)
decrypted_content_chunk = f_decrypted.read(buffer_size)
assert original_content_chunk == decrypted_content_chunk, \
"Original and Decrypted chunks are not equal, so decryption hasn't been done correctly!"
# check if EOF was reached
if len(original_content_chunk) < buffer_size:
break
def test_tvb_export_of_simple_datatype_with_encryption(
self, dummy_datatype_index_factory):
"""
Test export of an encrypted data type which has no data stored on file system
"""
datatype = dummy_datatype_index_factory()
storage_interface = StorageInterface()
import_export_encryption_handler = StorageInterface.get_import_export_encryption_handler(
)
import_export_encryption_handler.generate_public_private_key_pair(
TvbProfile.current.TVB_TEMP_FOLDER)
_, file_path, _ = self.export_manager.export_data(
datatype, self.TVB_EXPORTER, self.test_project,
os.path.join(TvbProfile.current.TVB_TEMP_FOLDER,
import_export_encryption_handler.PUBLIC_KEY_NAME))
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file: %s on disk." % file_path
result = storage_interface.unpack_zip(
file_path, TvbProfile.current.TVB_TEMP_FOLDER)
encrypted_password_path = import_export_encryption_handler.extract_encrypted_password_from_list(
result)
decrypted_file_path = import_export_encryption_handler.decrypt_content(
encrypted_password_path, result,
os.path.join(TvbProfile.current.TVB_TEMP_FOLDER,
import_export_encryption_handler.PRIVATE_KEY_NAME))[0]
original_path = h5.path_for_stored_index(datatype)
self.compare_files(original_path, decrypted_file_path)
def test_tvb_linked_export_of_simple_datatype(
self, connectivity_index_factory, surface_index_factory,
region_mapping_index_factory):
"""
Test export of a data type and its linked data types which have no data stored on file system
"""
conn = connectivity_index_factory()
_, surface = surface_index_factory(cortical=True)
region_mapping_index = region_mapping_index_factory(
conn_gid=conn.gid, surface_gid=surface.gid.hex)
_, file_path, _ = self.export_manager.export_data(
region_mapping_index, self.TVB_LINKED_EXPORTER, self.test_project)
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file;: %s on disk." % file_path
def test_tvb_linked_export_of_simple_datatype_with_encryption(
self, connectivity_index_factory, surface_index_factory,
region_mapping_index_factory):
"""
Test export of an encrypted data type and its linked data types which have no data stored on file system
"""
conn = connectivity_index_factory()
surface_idx, surface = surface_index_factory(cortical=True)
region_mapping_index = region_mapping_index_factory(
conn_gid=conn.gid, surface_gid=surface.gid.hex)
storage_interface = StorageInterface()
import_export_encryption_handler = StorageInterface.get_import_export_encryption_handler(
)
import_export_encryption_handler.generate_public_private_key_pair(
TvbProfile.current.TVB_TEMP_FOLDER)
_, file_path, _ = self.export_manager.export_data(
region_mapping_index, self.TVB_LINKED_EXPORTER, self.test_project,
os.path.join(TvbProfile.current.TVB_TEMP_FOLDER,
import_export_encryption_handler.PUBLIC_KEY_NAME))
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file;: %s on disk." % file_path
result = storage_interface.unpack_zip(
file_path, TvbProfile.current.TVB_TEMP_FOLDER)
encrypted_password = import_export_encryption_handler.extract_encrypted_password_from_list(
result)
decrypted_file_paths = import_export_encryption_handler.decrypt_content(
encrypted_password, result,
os.path.join(TvbProfile.current.TVB_TEMP_FOLDER,
import_export_encryption_handler.PRIVATE_KEY_NAME))
original_conn_path = h5.path_for_stored_index(conn)
decrypted_conn_path, idx = (decrypted_file_paths[0], 0) if 'Connectivity' in decrypted_file_paths[0] else \
(decrypted_file_paths[1], 1) if 'Connectivity' in decrypted_file_paths[1] else (decrypted_file_paths[2], 2)
self.compare_files(original_conn_path, decrypted_conn_path)
original_surface_path = h5.path_for_stored_index(surface_idx)
del decrypted_file_paths[idx]
decrypted_surface_path, idx = (decrypted_file_paths[0], 0) if 'Surface' in decrypted_file_paths[0] else \
(decrypted_file_paths[1], 1)
self.compare_files(original_surface_path, decrypted_surface_path)
original_rm_path = h5.path_for_stored_index(region_mapping_index)
del decrypted_file_paths[idx]
self.compare_files(original_rm_path, decrypted_file_paths[0])
def test_tvb_export_of_datatype_with_storage(self,
dummy_datatype_index_factory):
"""
Test export of a data type which has no data stored on file system
"""
datatype = dummy_datatype_index_factory()
_, file_path, _ = self.export_manager.export_data(
datatype, self.TVB_EXPORTER, self.test_project)
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file;: %s on disk." % file_path
def test_export_datatype_with_links(self, region_mapping_index_factory,
user_factory, project_factory):
"""
This is a test for exporting region mapping with links, that results in importing:
connectivity, surface and region mapping all from one zip.
"""
self.test_user = user_factory()
self.test_project = project_factory(self.test_user)
region_mapping_index = region_mapping_index_factory()
export_manager = ExportManager()
_, exported_h5_file, _ = export_manager.export_data(
region_mapping_index, self.TVB_LINKED_EXPORTER, self.test_project)
assert zipfile.is_zipfile(
exported_h5_file), "Generated file is not a valid ZIP file"
with zipfile.ZipFile(exported_h5_file, 'r') as zipObj:
# Get list of files names in zip
dts_in_zip = len(zipObj.namelist())
assert 3 == dts_in_zip
has_conn = False
has_surface = False
for dt_file in zipObj.namelist():
if dt_file.find(region_mapping_index.fk_connectivity_gid) > -1:
has_conn = True
if dt_file.find(region_mapping_index.fk_surface_gid):
has_surface = True
assert has_conn is True, "Connectivity was exported in zip"
assert has_surface is True, "Surface was exported in zip"
def test_tvb_export_for_datatype_group(self, datatype_group_factory):
"""
This method checks export of a data type group
"""
ts_datatype_group, dm_datatype_group = datatype_group_factory(
project=self.test_project, store_vm=True)
file_name, file_path, _ = self.export_manager.export_data(
dm_datatype_group, self.TVB_EXPORTER, self.test_project)
assert file_name is not None, "Export process should return a file name"
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file: %s on disk." % file_path
# Now check if the generated file is a correct ZIP file
assert zipfile.is_zipfile(
file_path), "Generated file is not a valid ZIP file"
with closing(zipfile.ZipFile(file_path)) as zip_file:
list_of_files = zip_file.namelist()
list_of_folders = []
for file in list_of_files:
dir_name = os.path.dirname(file)
if dir_name not in list_of_folders:
list_of_folders.append(dir_name)
count_datatypes = dao.count_datatypes_in_group(
ts_datatype_group.id)
count_datatypes += dao.count_datatypes_in_group(
dm_datatype_group.id)
# Check if ZIP files contains files for data types and view models (multiple H5 files in case of a Sim)
assert count_datatypes == len(list_of_folders)
assert (count_datatypes / 2) * 6 + (count_datatypes /
2) * 2 == len(list_of_files)
def test_tvb_export_for_datatype_group_with_links(self,
datatype_group_factory):
"""
This method checks export of a data type group with Links
"""
ts_datatype_group, dm_datatype_group = datatype_group_factory(
project=self.test_project,
store_vm=True,
use_time_series_region=True)
file_name, file_path, _ = self.export_manager.export_data(
ts_datatype_group, self.TVB_LINKED_EXPORTER, self.test_project)
assert file_name is not None, "Export process should return a file name"
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file: %s on disk." % file_path
# Now check if the generated file is a correct ZIP file
assert zipfile.is_zipfile(
file_path), "Generated file is not a valid ZIP file"
with closing(zipfile.ZipFile(file_path)) as zip_file:
list_of_files = zip_file.namelist()
list_of_folders = []
links_folder_found = False
for file in list_of_files:
dir_name = os.path.dirname(file)
if not links_folder_found:
if "Links" in dir_name:
links_folder_found = True
assert file_path is not None, "Export process should return path to export file"
if dir_name not in list_of_folders:
list_of_folders.append(dir_name)
assert links_folder_found is not None, "Links folder was not exported"
count_datatypes = dao.count_datatypes_in_group(
ts_datatype_group.id)
count_datatypes += dao.count_datatypes_in_group(
dm_datatype_group.id)
# Check if ZIP files contains files for data types and view models (multiple H5 files in case of a Sim)
# +1 For Links folder
assert count_datatypes + 1 == len(list_of_folders)
# +3 for the 3 files in Links folder: Connectivity, Surface, Region Mapping
# time series have 6 files, datatype measures have 2 files
assert (count_datatypes / 2) * 6 + (
count_datatypes / 2) * 2 + 3 == len(list_of_files)
def test_tvb_export_for_encrypted_datatype_group_with_links(
self, datatype_group_factory):
"""
This method checks export of an encrypted data type group
"""
ts_datatype_group, dm_datatype_group = datatype_group_factory(
project=self.test_project, store_vm=True)
storage_interface = StorageInterface()
import_export_encryption_handler = StorageInterface.get_import_export_encryption_handler(
)
import_export_encryption_handler.generate_public_private_key_pair(
TvbProfile.current.TVB_TEMP_FOLDER)
file_name, file_path, _ = self.export_manager.export_data(
dm_datatype_group, self.TVB_EXPORTER, self.test_project,
os.path.join(TvbProfile.current.TVB_TEMP_FOLDER,
import_export_encryption_handler.PUBLIC_KEY_NAME))
assert file_name is not None, "Export process should return a file name"
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file: %s on disk." % file_path
# Now check if the generated file is a correct ZIP file
assert zipfile.is_zipfile(
file_path), "Generated file is not a valid ZIP file"
result = storage_interface.unpack_zip(
file_path, TvbProfile.current.TVB_TEMP_FOLDER)
encrypted_password = import_export_encryption_handler.extract_encrypted_password_from_list(
result)
decrypted_file_paths = import_export_encryption_handler.decrypt_content(
encrypted_password, result,
os.path.join(TvbProfile.current.TVB_TEMP_FOLDER,
import_export_encryption_handler.PRIVATE_KEY_NAME))
# Here we only test if the length of decrypted_file_paths is the one expected
assert len(decrypted_file_paths) == len(
result
), "Number of decrypted data type group files is not correct!"
def test_export_with_invalid_data(self, dummy_datatype_index_factory):
"""
Test scenarios when data provided to export method is invalid
"""
# Test with no datatype
with pytest.raises(InvalidExportDataException):
self.export_manager.export_data(None, self.TVB_EXPORTER,
self.test_project)
# Test with no exporter
datatype = dummy_datatype_index_factory()
with pytest.raises(ExportException):
self.export_manager.export_data(datatype, None, self.test_project)
# test with wrong exporter
with pytest.raises(ExportException):
self.export_manager.export_data(datatype, "wrong_exporter",
self.test_project)
# test with no project folder
with pytest.raises(ExportException):
self.export_manager.export_data(datatype, self.TVB_EXPORTER, None)
def test_export_project_failure(self):
"""
This method tests export of project with None data
"""
with pytest.raises(ExportException):
self.export_manager.export_project(None)
def test_export_project(self, project_factory, user_factory):
"""
Test export of a project
"""
user = user_factory(username='******')
project = project_factory(user)
export_file = self.export_manager.export_project(project)
assert export_file is not None, "Export process should return path to export file"
assert os.path.exists(
export_file
), "Could not find export file: %s on disk." % export_file
# Now check if the generated file is a correct ZIP file
assert zipfile.is_zipfile(
export_file), "Generated file is not a valid ZIP file"
def test_export_simulator_configuration(self, operation_factory,
connectivity_index_factory):
"""
Test export of a simulator configuration
"""
conn_gid = uuid.UUID(connectivity_index_factory().gid)
operation = operation_factory(is_simulation=True,
store_vm=True,
test_project=self.test_project,
conn_gid=conn_gid)
burst_configuration = BurstConfiguration(self.test_project.id)
burst_configuration.fk_simulation = operation.id
burst_configuration.simulator_gid = operation.view_model_gid
burst_configuration.name = "Test_burst"
burst_configuration = dao.store_entity(burst_configuration)
BurstService().store_burst_configuration(burst_configuration)
export_file = self.export_manager.export_simulator_configuration(
burst_configuration.id)
assert export_file is not None, "Export process should return path to export file"
assert os.path.exists(
export_file
), "Could not find export file: %s on disk." % export_file
assert zipfile.is_zipfile(
export_file), "Generated file is not a valid ZIP file"
class ImportExportProjectWithLinksTest(_BaseLinksTest):
def setUpTVB(self):
"""
Adds to the _BaseLinksTest setup the following
2 links from src to dest project
Import/export services
"""
_BaseLinksTest.setUpTVB(self)
dest_id = self.dest_project.id
self.flow_service.create_link([self.red_datatype.id], dest_id)
self.flow_service.create_link([self.blue_datatype.id], dest_id)
self.export_mng = ExportManager()
self.import_service = ImportService()
def test_export(self):
export_file = self.export_mng.export_project(self.dest_project)
with TvbZip(export_file) as z:
self.assertTrue('links-to-external-projects/Operation.xml' in z.namelist())
def _export_and_remove_dest(self):
"""export the destination project and remove it"""
dest_id = self.dest_project.id
export_file = self.export_mng.export_project(self.dest_project)
self.project_service.remove_project(dest_id)
return export_file
def _import_dest(self, export_file):
self.import_service.import_project_structure(export_file, self.user.id)
return self.import_service.created_projects[0].id
def test_links_recreated_on_import(self):
export_file = self._export_and_remove_dest()
imported_proj_id = self._import_dest(export_file)
self.assertEqual(1, self.red_datatypes_in(imported_proj_id))
self.assertEqual(1, self.blue_datatypes_in(imported_proj_id))
links = dao.get_linked_datatypes_for_project(imported_proj_id)
self.assertEqual(2, len(links))
def test_datatypes_recreated_on_import(self):
export_file = self._export_and_remove_dest()
self.project_service.remove_project(self.src_project.id)
# both projects have been deleted
# import should recreate links as datatypes
imported_proj_id = self._import_dest(export_file)
self.assertEqual(1, self.red_datatypes_in(imported_proj_id))
self.assertEqual(1, self.blue_datatypes_in(imported_proj_id))
links = dao.get_linked_datatypes_for_project(imported_proj_id)
self.assertEqual(0, len(links))
def test_datatypes_and_links_recreated_on_import(self):
export_file = self._export_and_remove_dest()
# remove datatype 2 from source project
self.project_service.remove_datatype(self.src_project.id, self.blue_datatype.gid)
imported_proj_id = self._import_dest(export_file)
# both datatypes should be recreated
self.assertEqual(1, self.red_datatypes_in(imported_proj_id))
self.assertEqual(1, self.blue_datatypes_in(imported_proj_id))
# only datatype 1 should be a link
links = dao.get_linked_datatypes_for_project(imported_proj_id)
self.assertEqual(1, len(links))
self.assertEquals(self.red_datatype.gid, links[0].gid)
