def setup_method(self): """ Sets up necessary files for the tests. """ self.meta_writer = XMLWriter(self.WRITABLE_METADATA) self.result_path = os.path.join(os.path.dirname(__file__), "Operation.xml")
class TestMetaDataWriteXML: """ Tests for XMLWriter. """ WRITABLE_METADATA = TestMetaDataReadXML.EXPECTED_DICTIONARY def setup_method(self): """ Sets up necessary files for the tests. """ self.meta_writer = XMLWriter(self.WRITABLE_METADATA) self.result_path = os.path.join(os.path.dirname(__file__), "Operation.xml") def teardown_method(self): """ Remove created XML file. """ os.remove(self.result_path) def test_write_metadata(self): """ Test that an XML file is created and correct data is written in it. """ assert not os.path.exists(self.result_path) self.meta_writer.write_metadata_in_xml(self.result_path) assert os.path.exists(self.result_path) reader = XMLReader(self.result_path) meta_data = reader.read_metadata_from_xml() for key, value in TestMetaDataReadXML.EXPECTED_DICTIONARY.items(): found_value = meta_data[key] assert value == found_value
def write_image_metadata(self, figure, meta_entity, images_folder): """ Writes figure meta-data into XML file """ _, dict_data = figure.to_dict() XMLWriter(meta_entity).write_metadata_in_xml( self._compute_image_metadata_file(figure, images_folder))
def write_project_metadata_from_dict(project_path, meta_entity, project_file): project_cfg_file = os.path.join(project_path, project_file) XMLWriter(meta_entity).write_metadata_in_xml(project_cfg_file) os.chmod(project_path, TvbProfile.current.ACCESS_MODE_TVB_FILES)
class StorageInterface: TEMP_FOLDER = "TEMP" IMAGES_FOLDER = "IMAGES" PROJECTS_FOLDER = "PROJECTS" ROOT_NODE_PATH = "/" TVB_FILE_EXTENSION = XMLWriter.FILE_EXTENSION TVB_STORAGE_FILE_EXTENSION = ".h5" TVB_ZIP_FILE_EXTENSION = ".zip" TVB_PROJECT_FILE = "Project" + TVB_FILE_EXTENSION ZIP_FILE_EXTENSION = "zip" OPERATION_FOLDER_PREFIX = "Operation_" logger = get_logger(__name__) def __init__(self): self.files_helper = FilesHelper() self.data_encryption_handler = encryption_handler self.folders_queue_consumer = FoldersQueueConsumer() # object attributes which have parameters in their constructor will be lazily instantiated self.tvb_zip = None self.storage_manager = None self.xml_reader = None self.xml_writer = None self.encryption_handler = None # FilesHelper methods start here # def check_created(self, path=TvbProfile.current.TVB_STORAGE): self.files_helper.check_created(path) @staticmethod def get_projects_folder(): return FilesHelper.get_projects_folder() def get_project_folder(self, project_name, *sub_folders): return self.files_helper.get_project_folder(project_name, *sub_folders) def get_temp_folder(self, project_name): return self.files_helper.get_project_folder(project_name, self.TEMP_FOLDER) def remove_project_structure(self, project_name): self.files_helper.remove_project_structure(project_name) def get_project_meta_file_path(self, project_name): return self.files_helper.get_project_meta_file_path(project_name) def read_project_metadata(self, project_path): return FilesHelper.read_project_metadata(project_path, self.TVB_PROJECT_FILE) def write_project_metadata_from_dict(self, project_path, meta_entity): FilesHelper.write_project_metadata_from_dict(project_path, meta_entity, self.TVB_PROJECT_FILE) def write_project_metadata(self, meta_dictionary): self.files_helper.write_project_metadata(meta_dictionary, self.TVB_PROJECT_FILE) def remove_operation_data(self, project_name, operation_id): self.files_helper.remove_operation_data(project_name, operation_id) def remove_datatype_file(self, h5_file): self.files_helper.remove_datatype_file(h5_file) self.push_folder_to_sync(FilesHelper.get_project_folder_from_h5(h5_file)) def get_images_folder(self, project_name): return self.files_helper.get_images_folder(project_name, self.IMAGES_FOLDER) def write_image_metadata(self, figure, meta_entity): self.files_helper.write_image_metadata(figure, meta_entity, self.IMAGES_FOLDER) def remove_image_metadata(self, figure): self.files_helper.remove_image_metadata(figure, self.IMAGES_FOLDER) def get_allen_mouse_cache_folder(self, project_name): return self.files_helper.get_allen_mouse_cache_folder(project_name) def get_tumor_dataset_folder(self): return self.files_helper.get_tumor_dataset_folder() def zip_folders(self, all_datatypes, project_name, zip_full_path): operation_folders = [] for data_type in all_datatypes: operation_folder = self.get_project_folder(project_name, str(data_type.fk_from_operation)) operation_folders.append(operation_folder) FilesHelper.zip_folders(zip_full_path, operation_folders, self.OPERATION_FOLDER_PREFIX) @staticmethod def zip_folder(result_name, folder_root): FilesHelper.zip_folder(result_name, folder_root) def unpack_zip(self, uploaded_zip, folder_path): return self.files_helper.unpack_zip(uploaded_zip, folder_path) @staticmethod def copy_file(source, dest, dest_postfix=None, buffer_size=1024 * 1024): FilesHelper.copy_file(source, dest, dest_postfix, buffer_size) @staticmethod def remove_files(file_list, ignore_exception=False): FilesHelper.remove_files(file_list, ignore_exception) @staticmethod def remove_folder(folder_path, ignore_errors=False): FilesHelper.remove_folder(folder_path, ignore_errors) @staticmethod def compute_size_on_disk(file_path): return FilesHelper.compute_size_on_disk(file_path) @staticmethod def compute_recursive_h5_disk_usage(start_path='.'): return FilesHelper.compute_recursive_h5_disk_usage(start_path) # TvbZip methods start here # def write_zip_folder(self, dest_path, folder, exclude=None): self.tvb_zip = TvbZip(dest_path, "w") self.tvb_zip.write_zip_folder(folder, exclude) self.tvb_zip.close() def write_zip_folder_with_links(self, dest_path, folder, linked_paths, op, exclude=None): self.tvb_zip = TvbZip(dest_path, "w") self.tvb_zip.write_zip_folder(folder, exclude) self.logger.debug("Done exporting files, now we will export linked DTs") if linked_paths is not None: self.export_datatypes(linked_paths, op) self.tvb_zip.close() def get_filenames_in_zip(self, dest_path, mode="r"): self.tvb_zip = TvbZip(dest_path, mode) name_list = self.tvb_zip.namelist() self.tvb_zip.close() return name_list def open_tvb_zip(self, dest_path, name, mode="r"): self.tvb_zip = TvbZip(dest_path, mode) file = self.tvb_zip.open(name) self.tvb_zip.close() return file # Return a HDF5 storage methods to call the methods from there # @staticmethod def get_storage_manager(file_full_path): return HDF5StorageManager(file_full_path) # XML methods start here # def read_metadata_from_xml(self, xml_path): self.xml_reader = XMLReader(xml_path) return self.xml_reader.read_metadata_from_xml() def write_metadata_in_xml(self, entity, final_path): self.xml_writer = XMLWriter(entity) return self.xml_writer.write_metadata_in_xml(final_path) # Encryption Handler methods start here # def cleanup_encryption_handler(self, dir_gid): self.encryption_handler = EncryptionHandler(dir_gid) self.encryption_handler.cleanup_encryption_handler() @staticmethod def generate_random_password(pass_size): return EncryptionHandler.generate_random_password(pass_size) def get_encrypted_dir(self, dir_gid): self.encryption_handler = EncryptionHandler(dir_gid) return self.encryption_handler.get_encrypted_dir() def get_password_file(self, dir_gid): self.encryption_handler = EncryptionHandler(dir_gid) return self.encryption_handler.get_password_file() def encrypt_inputs(self, dir_gid, files_to_encrypt, subdir=None): self.encryption_handler = EncryptionHandler(dir_gid) return self.encryption_handler.encrypt_inputs(files_to_encrypt, subdir) def decrypt_results_to_dir(self, dir_gid, dir, from_subdir=None): self.encryption_handler = EncryptionHandler(dir_gid) return self.encryption_handler.decrypt_results_to_dir(dir, from_subdir) def decrypt_files_to_dir(self, dir_gid, files, dir): self.encryption_handler = EncryptionHandler(dir_gid) return self.encryption_handler.decrypt_files_to_dir(files, dir) def get_current_enc_dirname(self, dir_gid): self.encryption_handler = EncryptionHandler(dir_gid) return self.encryption_handler.current_enc_dirname # Data Encryption Handler methods start here # def inc_project_usage_count(self, folder): return self.data_encryption_handler.inc_project_usage_count(folder) def inc_running_op_count(self, folder): return self.data_encryption_handler.inc_running_op_count(folder) def dec_running_op_count(self, folder): return self.data_encryption_handler.dec_running_op_count(folder) def check_and_delete(self, folder): self.data_encryption_handler.dec_running_op_count(folder) return self.data_encryption_handler.check_and_delete(folder) @staticmethod def sync_folders(folder): DataEncryptionHandler.sync_folders(folder) def set_project_active(self, project, linked_dt=None): self.data_encryption_handler.set_project_active(project, linked_dt) def set_project_inactive(self, project): self.data_encryption_handler.set_project_inactive(project.name) def push_folder_to_sync(self, project_name): project_folder = self.get_project_folder(project_name) self.data_encryption_handler.push_folder_to_sync(project_folder) @staticmethod def encryption_enabled(): return DataEncryptionHandler.encryption_enabled() @staticmethod def startup_cleanup(): return DataEncryptionHandler.startup_cleanup() # Folders Queue Consumer methods start here # def run(self): return self.folders_queue_consumer.run() def start(self): self.folders_queue_consumer.start() def mark_stop(self): self.folders_queue_consumer.mark_stop() def join(self): self.folders_queue_consumer.join() # Generic methods start here def ends_with_tvb_file_extension(self, file): return file.endswith(self.TVB_FILE_EXTENSION) def ends_with_tvb_storage_file_extension(self, file): return file.endswith(self.TVB_STORAGE_FILE_EXTENSION) def is_in_usage(self, project_folder): return self.data_encryption_handler.is_in_usage(project_folder) def rename_project(self, current_proj_name, new_name): project_folder = self.get_project_folder(current_proj_name) if self.encryption_enabled() and not self.data_encryption_handler.is_in_usage(project_folder): raise RenameWhileSyncEncryptingException( "A project can not be renamed while sync encryption operations are running") self.files_helper.rename_project_structure(current_proj_name, new_name) encrypted_path = DataEncryptionHandler.compute_encrypted_folder_path(project_folder) if os.path.exists(encrypted_path): new_encrypted_path = DataEncryptionHandler.compute_encrypted_folder_path( self.get_project_folder(new_name)) os.rename(encrypted_path, new_encrypted_path) def remove_project(self, project): project_folder = self.get_project_folder(project.name) self.remove_project_structure(project.name) encrypted_path = DataEncryptionHandler.compute_encrypted_folder_path(project_folder) if os.path.exists(encrypted_path): self.remove_folder(encrypted_path) if os.path.exists(DataEncryptionHandler.project_key_path(project.id)): os.remove(DataEncryptionHandler.project_key_path(project.id)) def move_datatype_with_sync(self, to_project, to_project_path, new_op_id, full_path, vm_full_path): self.set_project_active(to_project) self.sync_folders(to_project_path) self.files_helper.move_datatype(to_project.name, str(new_op_id), full_path) self.files_helper.move_datatype(to_project.name, str(new_op_id), vm_full_path) self.sync_folders(to_project_path) self.set_project_inactive(to_project) def export_datatypes(self, paths, operation): op_folder = self.get_project_folder(operation.project.name, operation.id) op_folder_name = os.path.basename(op_folder) # add linked datatypes to archive in the import operation for pth in paths: zip_pth = op_folder_name + '/' + os.path.basename(pth) self.tvb_zip.write(pth, zip_pth) # remove these files, since we only want them in export archive self.remove_folder(op_folder) def build_data_export_folder(self, data, export_folder): now = datetime.now() date_str = "%d-%d-%d_%d-%d-%d_%d" % (now.year, now.month, now.day, now.hour, now.minute, now.second, now.microsecond) tmp_str = date_str + "@" + data.gid data_export_folder = os.path.join(export_folder, tmp_str) self.check_created(data_export_folder) return data_export_folder def export_project(self, project, folders_to_exclude, export_folder, linked_paths, op): project_folder = self.get_project_folder(project.name) folders_to_exclude.append("TEMP") # Compute path and name of the zip file now = datetime.now() date_str = now.strftime("%Y-%m-%d_%H-%M") zip_file_name = "%s_%s.%s" % (date_str, project.name, self.ZIP_FILE_EXTENSION) export_folder = self.build_data_export_folder(project, export_folder) result_path = os.path.join(export_folder, zip_file_name) # Pack project [filtered] content into a ZIP file: self.logger.debug("Done preparing, now we will write the folder.") self.logger.debug(project_folder) self.write_zip_folder_with_links(result_path, project_folder, linked_paths, op, folders_to_exclude) # Make sure the Project.xml file gets copied: self.logger.debug("Done, closing") return result_path
def write_metadata_in_xml(self, entity, final_path): self.xml_writer = XMLWriter(entity) return self.xml_writer.write_metadata_in_xml(final_path)
class StorageInterface: TEMP_FOLDER = "TEMP" IMAGES_FOLDER = "IMAGES" PROJECTS_FOLDER = "PROJECTS" ROOT_NODE_PATH = "/" TVB_STORAGE_FILE_EXTENSION = ".h5" TVB_ZIP_FILE_EXTENSION = ".zip" TVB_XML_FILE_EXTENSION = ".xml" TVB_PROJECT_FILE = "Project" + TVB_XML_FILE_EXTENSION FILE_NAME_STRUCTURE = '{}_{}.h5' OPERATION_FOLDER_PREFIX = "Operation_" EXPORTED_SIMULATION_NAME = "exported_simulation" EXPORTED_SIMULATION_DTS_DIR = "datatypes" export_folder = None EXPORT_FOLDER_NAME = "EXPORT_TMP" EXPORT_FOLDER = os.path.join(TvbProfile.current.TVB_STORAGE, EXPORT_FOLDER_NAME) logger = get_logger(__name__) def __init__(self): self.files_helper = FilesHelper() self.data_encryption_handler = encryption_handler self.folders_queue_consumer = FoldersQueueConsumer() # object attributes which have parameters in their constructor will be lazily instantiated self.tvb_zip = None self.xml_reader = None self.xml_writer = None self.encryption_handler = None # FilesHelper methods start here # def check_created(self, path=TvbProfile.current.TVB_STORAGE): self.files_helper.check_created(path) @staticmethod def get_projects_folder(): return FilesHelper.get_projects_folder() def get_project_folder(self, project_name, *sub_folders): return self.files_helper.get_project_folder(project_name, *sub_folders) def get_temp_folder(self, project_name): return self.files_helper.get_project_folder(project_name, self.TEMP_FOLDER) def get_project_meta_file_path(self, project_name): return self.files_helper.get_project_meta_file_path(project_name) def read_project_metadata(self, project_path): return FilesHelper.read_project_metadata(project_path, self.TVB_PROJECT_FILE) def write_project_metadata_from_dict(self, project_path, meta_entity): FilesHelper.write_project_metadata_from_dict(project_path, meta_entity, self.TVB_PROJECT_FILE) def write_project_metadata(self, meta_dictionary): self.files_helper.write_project_metadata(meta_dictionary, self.TVB_PROJECT_FILE) def remove_operation_data(self, project_name, operation_id): self.files_helper.remove_operation_data(project_name, operation_id) def get_images_folder(self, project_name): return self.files_helper.get_images_folder(project_name, self.IMAGES_FOLDER) def write_image_metadata(self, figure, meta_entity): self.files_helper.write_image_metadata(figure, meta_entity, self.IMAGES_FOLDER) def remove_figure(self, figure): self.files_helper.remove_figure(figure, self.IMAGES_FOLDER) self.push_folder_to_sync(figure.project.name) def get_allen_mouse_cache_folder(self, project_name): return self.files_helper.get_allen_mouse_cache_folder(project_name) def get_tumor_dataset_folder(self): return self.files_helper.get_tumor_dataset_folder() @staticmethod def copy_file(source, dest, dest_postfix=None, buffer_size=1024 * 1024): FilesHelper.copy_file(source, dest, dest_postfix, buffer_size) @staticmethod def remove_files(file_list, ignore_exception=False): FilesHelper.remove_files(file_list, ignore_exception) @staticmethod def remove_folder(folder_path, ignore_errors=False): FilesHelper.remove_folder(folder_path, ignore_errors) @staticmethod def compute_size_on_disk(file_path): return FilesHelper.compute_size_on_disk(file_path) @staticmethod def compute_recursive_h5_disk_usage(start_path='.'): return FilesHelper.compute_recursive_h5_disk_usage(start_path) # TvbZip methods start here # def write_zip_folder(self, dest_path, folder, linked_paths=None, op=None, exclude=[]): self.tvb_zip = TvbZip(dest_path, "w") self.tvb_zip.write_zip_folder(folder, exclude) self.logger.debug( "Done exporting files, now we will export linked DTs") if linked_paths is not None and op is not None: self.__export_datatypes(linked_paths, op) self.tvb_zip.close() def unpack_zip(self, uploaded_zip, folder_path): self.tvb_zip = TvbZip(uploaded_zip, "r") unpacked_folder = self.tvb_zip.unpack_zip(folder_path) self.tvb_zip.close() return unpacked_folder def get_filenames_in_zip(self, dest_path, mode="r"): self.tvb_zip = TvbZip(dest_path, mode) name_list = self.tvb_zip.namelist() self.tvb_zip.close() return name_list def open_tvb_zip(self, dest_path, name, mode="r"): self.tvb_zip = TvbZip(dest_path, mode) file = self.tvb_zip.open(name) self.tvb_zip.close() return file # Return a HDF5 storage methods to call the methods from there # @staticmethod def get_storage_manager(file_full_path): return HDF5StorageManager(file_full_path) # XML methods start here # def read_metadata_from_xml(self, xml_path): self.xml_reader = XMLReader(xml_path) return self.xml_reader.read_metadata_from_xml() def write_metadata_in_xml(self, entity, final_path): self.xml_writer = XMLWriter(entity) return self.xml_writer.write_metadata_in_xml(final_path) # Encryption Handler methods start here # def cleanup_encryption_handler(self, dir_gid): self.encryption_handler = EncryptionHandler(dir_gid) self.encryption_handler.cleanup_encryption_handler() @staticmethod def generate_random_password(pass_size): return EncryptionHandler.generate_random_password(pass_size) def get_encrypted_dir(self, dir_gid): self.encryption_handler = EncryptionHandler(dir_gid) return self.encryption_handler.get_encrypted_dir() def get_password_file(self, dir_gid): self.encryption_handler = EncryptionHandler(dir_gid) return self.encryption_handler.get_password_file() def encrypt_inputs(self, dir_gid, files_to_encrypt, subdir=None): self.encryption_handler = EncryptionHandler(dir_gid) return self.encryption_handler.encrypt_inputs(files_to_encrypt, subdir) def decrypt_results_to_dir(self, dir_gid, dir, from_subdir=None): self.encryption_handler = EncryptionHandler(dir_gid) return self.encryption_handler.decrypt_results_to_dir(dir, from_subdir) def decrypt_files_to_dir(self, dir_gid, files, dir): self.encryption_handler = EncryptionHandler(dir_gid) return self.encryption_handler.decrypt_files_to_dir(files, dir) def get_current_enc_dirname(self, dir_gid): self.encryption_handler = EncryptionHandler(dir_gid) return self.encryption_handler.current_enc_dirname # Data Encryption Handler methods start here # def inc_project_usage_count(self, folder): return self.data_encryption_handler.inc_project_usage_count(folder) def inc_running_op_count(self, folder): return self.data_encryption_handler.inc_running_op_count(folder) def dec_running_op_count(self, folder): return self.data_encryption_handler.dec_running_op_count(folder) def check_and_delete(self, folder): self.data_encryption_handler.dec_running_op_count(folder) return self.data_encryption_handler.check_and_delete(folder) @staticmethod def sync_folders(folder): DataEncryptionHandler.sync_folders(folder) def set_project_active(self, project, linked_dt=None): self.data_encryption_handler.set_project_active(project, linked_dt) def set_project_inactive(self, project): self.data_encryption_handler.set_project_inactive(project.name) def push_folder_to_sync(self, project_name): project_folder = self.get_project_folder(project_name) self.data_encryption_handler.push_folder_to_sync(project_folder) @staticmethod def encryption_enabled(): return DataEncryptionHandler.encryption_enabled() @staticmethod def startup_cleanup(): return DataEncryptionHandler.startup_cleanup() # Folders Queue Consumer methods start here # def run(self): return self.folders_queue_consumer.run() def start(self): self.folders_queue_consumer.start() def mark_stop(self): self.folders_queue_consumer.mark_stop() def join(self): self.folders_queue_consumer.join() # Generic methods start here def get_file_by_gid(self, project_name, op_id, dt_gid): op_path = self.files_helper.get_project_folder(project_name, str(op_id)) for f in os.listdir(op_path): fp = os.path.join(op_path, f) if dt_gid in f and os.path.isfile(fp): return fp def get_filename(self, class_name, gid): return self.FILE_NAME_STRUCTURE.format(class_name, gid.hex) def path_for(self, op_id, h5_file_class, gid, project_name, dt_class): operation_dir = self.files_helper.get_project_folder( project_name, str(op_id)) return self.path_by_dir(operation_dir, h5_file_class, gid, dt_class) def path_by_dir(self, base_dir, h5_file_class, gid, dt_class): if isinstance(gid, str): gid = uuid.UUID(gid) fname = self.get_filename(dt_class or h5_file_class.file_name_base(), gid) return os.path.join(base_dir, fname) def ends_with_tvb_file_extension(self, file): return file.endswith(self.TVB_XML_FILE_EXTENSION) def ends_with_tvb_storage_file_extension(self, file): return file.endswith(self.TVB_STORAGE_FILE_EXTENSION) def is_in_usage(self, project_folder): return self.data_encryption_handler.is_in_usage(project_folder) def rename_project(self, current_proj_name, new_name): project_folder = self.get_project_folder(current_proj_name) if self.encryption_enabled( ) and not self.data_encryption_handler.is_in_usage(project_folder): raise RenameWhileSyncEncryptingException( "A project can not be renamed while sync encryption operations are running" ) self.files_helper.rename_project_structure(current_proj_name, new_name) encrypted_path = DataEncryptionHandler.compute_encrypted_folder_path( project_folder) if os.path.exists(encrypted_path): new_encrypted_path = DataEncryptionHandler.compute_encrypted_folder_path( self.get_project_folder(new_name)) os.rename(encrypted_path, new_encrypted_path) def remove_project(self, project, sync_for_encryption=False): project_folder = self.get_project_folder(project.name) if sync_for_encryption: self.sync_folders(project_folder) try: self.remove_folder(project_folder) self.logger.debug("Project folders were removed for " + project.name) except OSError: self.logger.exception("A problem occurred while removing folder.") raise FileStructureException( "Permission denied. Make sure you have write access on TVB folder!" ) encrypted_path = DataEncryptionHandler.compute_encrypted_folder_path( project_folder) FilesHelper.remove_files([ encrypted_path, DataEncryptionHandler.project_key_path(project.id) ], True) def move_datatype_with_sync(self, to_project, to_project_path, new_op_id, full_path, vm_full_path): self.set_project_active(to_project) self.sync_folders(to_project_path) self.files_helper.move_datatype(to_project.name, str(new_op_id), full_path) self.files_helper.move_datatype(to_project.name, str(new_op_id), vm_full_path) self.sync_folders(to_project_path) self.set_project_inactive(to_project) # Exporting related methods start here def __export_datatypes(self, paths, operation): op_folder = self.get_project_folder(operation.project.name, operation.id) op_folder_name = os.path.basename(op_folder) # add linked datatypes to archive in the import operation for pth in paths: zip_pth = op_folder_name + '/' + os.path.basename(pth) self.tvb_zip.write(pth, zip_pth) # remove these files, since we only want them in export archive self.remove_folder(op_folder) def __build_data_export_folder(self, data, export_folder): now = datetime.now() date_str = "%d-%d-%d_%d-%d-%d_%d" % (now.year, now.month, now.day, now.hour, now.minute, now.second, now.microsecond) tmp_str = date_str + "@" + data.gid data_export_folder = os.path.join(export_folder, tmp_str) self.check_created(data_export_folder) return data_export_folder def export_project(self, project, folders_to_exclude, linked_paths, op): """ This method is used to export a project as a ZIP file. :param project: project to be exported. :param folders_to_exclude: a list of paths to folders inside of a project folder which should not be exported. :param linked_paths: a list of links to datatypes for the project to be exported :param op: operation for links to exported datatypes (if any) """ project_folder = self.get_project_folder(project.name) folders_to_exclude.append("TEMP") # Compute path and name of the zip file now = datetime.now() date_str = now.strftime("%Y-%m-%d_%H-%M") zip_file_name = "%s_%s.%s" % (date_str, project.name, self.TVB_ZIP_FILE_EXTENSION) export_folder = self.__build_data_export_folder( project, self.EXPORT_FOLDER) result_path = os.path.join(export_folder, zip_file_name) # Pack project [filtered] content into a ZIP file: self.logger.debug("Done preparing, now we will write the folder.") self.logger.debug(project_folder) self.write_zip_folder(result_path, project_folder, linked_paths, op, folders_to_exclude) # Make sure the Project.xml file gets copied: self.logger.debug("Done, closing") return result_path def export_simulator_configuration(self, burst, all_view_model_paths, all_datatype_paths, zip_filename): """ This method is used to export a simulator configuration as a ZIP file :param burst: BurstConfiguration of the simulation to be exported :param all_view_model_paths: a list of paths to all view model files of the simulation :param all_datatype_paths: a list of paths to all datatype files of the simulation :param zip_filename: name of the file to be exported """ tmp_export_folder = self.__build_data_export_folder( burst, self.EXPORT_FOLDER) tmp_sim_folder = os.path.join(tmp_export_folder, self.EXPORTED_SIMULATION_NAME) if not os.path.exists(tmp_sim_folder): os.makedirs(tmp_sim_folder) for vm_path in all_view_model_paths: dest = os.path.join(tmp_sim_folder, os.path.basename(vm_path)) self.copy_file(vm_path, dest) for dt_path in all_datatype_paths: dest = os.path.join(tmp_sim_folder, self.EXPORTED_SIMULATION_DTS_DIR, os.path.basename(dt_path)) self.copy_file(dt_path, dest) result_path = os.path.join(tmp_export_folder, zip_filename) self.write_zip_folder(result_path, tmp_sim_folder) self.remove_folder(tmp_sim_folder) return result_path def export_datatypes(self, dt_path_list, data, download_file_name): """ This method is used to export a list of datatypes as a ZIP file. :param dt_path_list: a list of paths to be exported (there are more than one when exporting with links) :param data: data to be exported :param download_file_name: name of the zip file to be downloaded """ export_folder = self.__build_data_export_folder( data, self.EXPORT_FOLDER) file_destination = None for dt_path in dt_path_list: file_destination = os.path.join(export_folder, os.path.basename(dt_path)) if not os.path.exists(file_destination): self.copy_file(dt_path, file_destination) self.get_storage_manager(file_destination).remove_metadata( 'parent_burst', check_existence=True) if len(dt_path_list) == 1: return file_destination export_data_zip_path = os.path.join(os.path.dirname(export_folder), download_file_name) self.write_zip_folder(export_data_zip_path, export_folder) return export_data_zip_path def export_datatypes_structure(self, all_datatypes, data, download_file_name, project_name): """ This method is used to export a list of datatypes as a ZIP file, while preserving the folder structure (eg: operation folders). It is only used during normal tvb exporting for datatype groups. :param all_datatypes: list of datatype paths to be exported (more than 1 if we export a datatype group) :param data: data to be exported :param download_file_name: name of the ZIP file to be exported :param project_name: name of the project in which the data to be exported exists """ export_folder = self.__build_data_export_folder( data, self.EXPORT_FOLDER) zip_full_path = os.path.join(export_folder, download_file_name) operation_folders = [] for data_type in all_datatypes: operation_folder = self.get_project_folder( project_name, str(data_type.fk_from_operation)) operation_folders.append(operation_folder) self.tvb_zip = TvbZip(zip_full_path, "w") self.tvb_zip.write_zip_folders(operation_folders, []) self.tvb_zip.close() return zip_full_path