class ProjectUpdateManager(UpdateManager): """ This goes through all the scripts that are newer than the version number written in the current project metadata xml, and executes them on the project folder. """ def __init__(self, project_path): self.project_path = project_path self.storage_interface = StorageInterface() # This assumes that old project metadata file can be parsed by current version. self.project_meta = self.storage_interface.read_project_metadata( project_path) from_version = int(self.project_meta.get('version', 0)) super(ProjectUpdateManager, self).__init__(project_update_scripts, from_version, TvbProfile.current.version.PROJECT_VERSION) def run_all_updates(self): """ Upgrade the project to the latest structure Go through all update scripts, from project version up to the current_version in the code """ super(ProjectUpdateManager, self).run_all_updates(project_path=self.project_path) # update project version in metadata self.project_meta['version'] = self.current_version self.storage_interface.write_project_metadata_from_dict( self.project_path, self.project_meta)
def __init__(self): # Here we register all available data type exporters # If new exporters supported, they should be added here self._register_exporter(TVBExporter()) self._register_exporter(TVBLinkedExporter()) self.export_folder = os.path.join(TvbProfile.current.TVB_STORAGE, self.EXPORT_FOLDER_NAME) self.storage_interface = StorageInterface()
def _import_projects_from_folder(self, temp_folder): """ Process each project from the uploaded pack, to extract names. """ project_roots = [] for root, _, files in os.walk(temp_folder): if StorageInterface.TVB_PROJECT_FILE in files: project_roots.append(root) for temp_project_path in project_roots: update_manager = ProjectUpdateManager(temp_project_path) update_manager.run_all_updates() project = self.__populate_project(temp_project_path) # Populate the internal list of create projects so far, for cleaning up folders, in case of failure self.created_projects.append(project) # Ensure project final folder exists on disk project_path = self.storage_interface.get_project_folder(project.name) shutil.move(os.path.join(temp_project_path, StorageInterface.TVB_PROJECT_FILE), project_path) # Now import project operations with their results self.import_project_operations(project, temp_project_path) # Import images and move them from temp into target self._store_imported_images(project, temp_project_path, project.name) if StorageInterface.encryption_enabled(): StorageInterface.sync_folders(project_path) shutil.rmtree(project_path)
def test_tvb_export_of_simple_datatype_with_encryption( self, dummy_datatype_index_factory): """ Test export of an encrypted data type which has no data stored on file system """ datatype = dummy_datatype_index_factory() storage_interface = StorageInterface() import_export_encryption_handler = StorageInterface.get_import_export_encryption_handler( ) import_export_encryption_handler.generate_public_private_key_pair( TvbProfile.current.TVB_TEMP_FOLDER) _, file_path, _ = self.export_manager.export_data( datatype, self.TVB_EXPORTER, self.test_project, os.path.join(TvbProfile.current.TVB_TEMP_FOLDER, import_export_encryption_handler.PUBLIC_KEY_NAME)) assert file_path is not None, "Export process should return path to export file" assert os.path.exists( file_path), "Could not find export file: %s on disk." % file_path result = storage_interface.unpack_zip( file_path, TvbProfile.current.TVB_TEMP_FOLDER) encrypted_password_path = import_export_encryption_handler.extract_encrypted_password_from_list( result) decrypted_file_path = import_export_encryption_handler.decrypt_content( encrypted_password_path, result, os.path.join(TvbProfile.current.TVB_TEMP_FOLDER, import_export_encryption_handler.PRIVATE_KEY_NAME))[0] original_path = h5.path_for_stored_index(datatype) self.compare_files(original_path, decrypted_file_path)
def _do_operation_launch(self, op, sim_gid, mocker, is_pse=False): # Prepare encrypted dir self.storage_interface = StorageInterface() self.dir_gid = sim_gid job_encrypted_inputs = HPCSchedulerClient()._prepare_input(op, sim_gid) self.storage_interface.encrypt_inputs(sim_gid, job_encrypted_inputs) encrypted_dir = self.storage_interface.get_encrypted_dir(self.dir_gid) mocker.patch('tvb.core.operation_hpc_launcher._request_passfile', _request_passfile_dummy) mocker.patch( 'tvb.core.operation_hpc_launcher._update_operation_status', _update_operation_status) # Call do_operation_launch similarly to CSCS env plain_dir = self.storage_interface.get_project_folder( self.test_project.name, 'plain') do_operation_launch(sim_gid.hex, 1000, is_pse, '', op.id, plain_dir) assert len(os.listdir(encrypted_dir)) == 7 output_path = os.path.join(encrypted_dir, HPCSchedulerClient.OUTPUT_FOLDER) assert os.path.exists(output_path) expected_files = 2 if is_pse: expected_files = 3 assert len(os.listdir(output_path)) == expected_files return output_path
def transactional_setup_method(self): """ Reset the database before each test. """ self.project_service = ProjectService() self.storage_interface = StorageInterface() self.test_user = TestFactory.create_user()
def stage_out_to_operation_folder(working_dir, operation, simulator_gid): # type: (Storage, Operation, typing.Union[uuid.UUID, str]) -> (list, Operation, str) encrypted_files = HPCSchedulerClient._stage_out_results( working_dir, simulator_gid) simulation_results = list() metric_encrypted_file = None metric_vm_encrypted_file = None for encrypted_file in encrypted_files: if os.path.basename(encrypted_file).startswith( DatatypeMeasureH5.file_name_base()): metric_encrypted_file = encrypted_file elif os.path.basename(encrypted_file).startswith( MEASURE_METRICS_MODEL_CLASS): metric_vm_encrypted_file = encrypted_file else: simulation_results.append(encrypted_file) storage_interface = StorageInterface() metric_op, metric_file = HPCSchedulerClient._handle_metric_results( metric_encrypted_file, metric_vm_encrypted_file, operation, storage_interface, simulator_gid) project = dao.get_project_by_id(operation.fk_launched_in) operation_dir = HPCSchedulerClient.storage_interface.get_project_folder( project.name, str(operation.id)) h5_filenames = storage_interface.decrypt_files_to_dir( simulator_gid, simulation_results, operation_dir) storage_interface.cleanup_encryption_handler(simulator_gid) LOGGER.info("Decrypted h5: {}".format(h5_filenames)) LOGGER.info("Metric op: {}".format(metric_op)) LOGGER.info("Metric file: {}".format(metric_file)) return h5_filenames, metric_op, metric_file
def do_operation_launch(simulator_gid, available_disk_space, is_group_launch, base_url, operation_id, plain_dir='/root/plain'): try: log.info("Preparing HPC launch for simulation with id={}".format( simulator_gid)) populate_datatypes_registry() log.info("Current TVB profile has HPC run=: {}".format( TvbProfile.current.hpc.IS_HPC_RUN)) storage_interface = StorageInterface() _request_passfile(simulator_gid, operation_id, base_url, storage_interface.get_password_file(simulator_gid)) storage_interface.decrypt_results_to_dir(simulator_gid, plain_dir) log.info("Current wdir is: {}".format(plain_dir)) view_model = h5.load_view_model(simulator_gid, plain_dir) adapter_instance = HPCSimulatorAdapter(plain_dir, is_group_launch) _update_operation_status(STATUS_STARTED, simulator_gid, operation_id, base_url) adapter_instance._prelaunch(None, view_model, available_disk_space) _encrypt_results(adapter_instance, storage_interface, simulator_gid) _update_operation_status(STATUS_FINISHED, simulator_gid, operation_id, base_url) except Exception as excep: log.error("Could not execute operation {}".format(str(sys.argv[1]))) log.exception(excep) _update_operation_status(STATUS_ERROR, simulator_gid, operation_id, base_url) raise excep
def store_view_model(operation, project, view_model): storage_path = StorageInterface().get_project_folder( project.name, str(operation.id)) h5.store_view_model(view_model, storage_path) view_model_size_on_disk = StorageInterface.compute_recursive_h5_disk_usage( storage_path) operation.view_model_disk_size = view_model_size_on_disk dao.store_entity(operation)
def transactional_setup_method(self): self.test_user = TestFactory.create_user('Rest_User') self.test_project = TestFactory.create_project(self.test_user, 'Rest_Project', users=[self.test_user.id]) self.operations_resource = GetOperationsInProjectResource() self.status_resource = GetOperationStatusResource() self.results_resource = GetOperationResultsResource() self.launch_resource = LaunchOperationResource() self.storage_interface = StorageInterface()
def __init__(self, path): self.bi_hemispheric = False self.vertices, self.normals, self.triangles = [], [], [] self.hemisphere_mask = [] self._read_vertices = 0 self.storage_interface = StorageInterface() self._read(path)
def transactional_teardown_method(self): """ Clean-up tests data """ # Remove EXPORT folder export_folder = os.path.join(TvbProfile.current.TVB_STORAGE, StorageInterface.EXPORT_FOLDER_NAME) StorageInterface.remove_folder(export_folder, True)
def setup_method(self): self.storage_interface = StorageInterface() self.dir_gid = '123' self.encryption_handler = self.storage_interface.get_encryption_handler( self.dir_gid) self.clean_database() self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user)
def run(self): """ Get the required data from the operation queue and launch the operation. """ operation_id = self.operation_id run_params = [TvbProfile.current.PYTHON_INTERPRETER_PATH, '-m', 'tvb.core.operation_async_launcher', str(operation_id), TvbProfile.CURRENT_PROFILE_NAME] current_operation = dao.get_operation_by_id(operation_id) storage_interface = StorageInterface() project_folder = storage_interface.get_project_folder(current_operation.project.name) in_usage = storage_interface.is_in_usage(project_folder) storage_interface.inc_running_op_count(project_folder) if not in_usage: storage_interface.sync_folders(project_folder) # In the exceptional case where the user pressed stop while the Thread startup is done, # We should no longer launch the operation. if self.stopped() is False: env = os.environ.copy() env['PYTHONPATH'] = os.pathsep.join(sys.path) # anything that was already in $PYTHONPATH should have been reproduced in sys.path launched_process = Popen(run_params, stdout=PIPE, stderr=PIPE, env=env) LOGGER.debug("Storing pid=%s for operation id=%s launched on local machine." % (operation_id, launched_process.pid)) op_ident = OperationProcessIdentifier(operation_id, pid=launched_process.pid) dao.store_entity(op_ident) if self.stopped(): # In the exceptional case where the user pressed stop while the Thread startup is done. # and stop_operation is concurrently asking about OperationProcessIdentity. self.stop_pid(launched_process.pid) subprocess_result = launched_process.communicate() LOGGER.info("Finished with launch of operation %s" % operation_id) returned = launched_process.wait() LOGGER.info("Return code: {}. Stopped: {}".format(returned, self.stopped())) LOGGER.info("Thread: {}".format(self)) if returned != 0 and not self.stopped(): # Process did not end as expected. (e.g. Segmentation fault) burst_service = BurstService() operation = dao.get_operation_by_id(self.operation_id) LOGGER.error("Operation suffered fatal failure! Exit code: %s Exit message: %s" % (returned, subprocess_result)) burst_service.persist_operation_state(operation, STATUS_ERROR, "Operation failed unexpectedly! Please check the log files.") del launched_process storage_interface.check_and_delete(project_folder) # Give back empty spot now that you finished your operation CURRENT_ACTIVE_THREADS.remove(self) LOCKS_QUEUE.put(1)
def _adapt_epileptor_simulations(): """ Previous Simulations on EpileptorWithPermitivity model, should be converted to use the Epileptor model. As the parameters from the two models are having different ranges and defaults, we do not translate parameters, we only set the Epileptor as model instead of EpileptorPermittivityCoupling, and leave the model params to defaults. """ session = SA_SESSIONMAKER() epileptor_old = "EpileptorPermittivityCoupling" epileptor_new = "Epileptor" param_model = "model" try: all_ep_ops = session.query(model.Operation).filter( model.Operation.parameters.ilike('%"' + epileptor_old + '"%')).all() storage_interface = StorageInterface() all_bursts = dict() for ep_op in all_ep_ops: try: op_params = parse_json_parameters(ep_op.parameters) if op_params[param_model] != epileptor_old: LOGGER.debug("Skipping op " + str(op_params[param_model]) + " -- " + str(ep_op)) continue LOGGER.debug("Updating " + str(op_params)) op_params[param_model] = epileptor_new ep_op.parameters = json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder) LOGGER.debug("New params:" + ep_op.parameters) storage_interface.write_operation_metadata(ep_op) burst = dao.get_burst_for_operation_id(ep_op.id) if burst is not None: LOGGER.debug("Updating burst:" + str(burst)) burst.prepare_after_load() burst.simulator_configuration[param_model] = { 'value': epileptor_new } burst._simulator_configuration = json.dumps( burst.simulator_configuration, cls=MapAsJson.MapAsJsonEncoder) if burst.id not in all_bursts: all_bursts[burst.id] = burst except Exception: LOGGER.exception("Could not process " + str(ep_op)) session.add_all(all_ep_ops) session.add_all(list(all_bursts.values())) session.commit() except Exception: LOGGER.exception("Could not update Simulation Epileptor Params") finally: session.close()
class TVBLinkedExporter(ABCExporter): """ """ def __init__(self): self.storage_interface = StorageInterface() def get_supported_types(self): return [DataType] def get_label(self): return "TVB Format with links" def export(self, data, data_export_folder, project): """ Exports data type: 1. If data is a normal data type, simply exports storage file (HDF format) 2. If data is a DataTypeGroup creates a zip with all files for all data types """ self.copy_dt_to_export_folder(data, data_export_folder) export_data_zip_path = self.get_export_data_zip_path( data, data_export_folder) return self.export_data_with_references(export_data_zip_path, data_export_folder) def get_export_data_zip_path(self, data, data_export_folder): zip_file_name = self.get_export_file_name(data) return os.path.join(os.path.dirname(data_export_folder), zip_file_name) def export_data_with_references(self, export_data_zip_path, data_export_folder): self.storage_interface.write_zip_folder(export_data_zip_path, data_export_folder) return None, export_data_zip_path, True def copy_dt_to_export_folder(self, data, data_export_folder): data_path = h5.path_for_stored_index(data) with H5File.from_file(data_path) as f: file_destination = os.path.join(data_export_folder, os.path.basename(data_path)) if not os.path.exists(file_destination): self.storage_interface.copy_file(data_path, file_destination) sub_dt_refs = f.gather_references() for _, ref_gid in sub_dt_refs: if ref_gid: dt = load.load_entity_by_gid(ref_gid) self.copy_dt_to_export_folder(dt, data_export_folder) H5File.remove_metadata_param(file_destination, 'parent_burst') def get_export_file_extension(self, data): return "zip" def skip_group_datatypes(self): return True
def __init__(self): self.generic_attributes = GenericAttributes() self.generic_attributes.subject = DataTypeMetaData.DEFAULT_SUBJECT self.storage_interface = StorageInterface() # Will be populate with current running operation's identifier self.operation_id = None self.user_id = None self.submitted_form = None self.log = get_logger(self.__class__.__module__)
def setup_method(self): """ Prepare the usage of a different config file for this class only. """ StorageInterface.remove_files([TEST_CONFIG_FILE, TestSQLiteProfile.DEFAULT_STORAGE]) self.old_config_file = TvbProfile.current.TVB_CONFIG_FILE TvbProfile.current.__class__.TVB_CONFIG_FILE = TEST_CONFIG_FILE TvbProfile._build_profile_class(TvbProfile.CURRENT_PROFILE_NAME) self.settings_service = SettingsService()
class TVBExporter(ABCExporter): """ This exporter simply provides for download data in TVB format """ def __init__(self): self.storage_interface = StorageInterface() def get_supported_types(self): return [DataType] def get_label(self): return "TVB Format" def export(self, data, export_folder, project): """ Exports data type: 1. If data is a normal data type, simply exports storage file (HDF format) 2. If data is a DataTypeGroup creates a zip with all files for all data types """ download_file_name = self.get_export_file_name(data) if self.is_data_a_group(data): all_datatypes = self._get_all_data_types_arr(data) if all_datatypes is None or len(all_datatypes) == 0: raise ExportException( "Could not export a data type group with no data") zip_file = os.path.join(export_folder, download_file_name) # Create ZIP archive self.storage_interface.zip_folders(all_datatypes, project.name, zip_file) return download_file_name, zip_file, True else: data_file = self.copy_dt_to_export_folder(data, export_folder) return None, data_file, True def copy_dt_to_export_folder(self, data, data_export_folder): data_path = h5.path_for_stored_index(data) file_destination = os.path.join(data_export_folder, os.path.basename(data_path)) if not os.path.exists(file_destination): self.storage_interface.copy_file(data_path, file_destination) H5File.remove_metadata_param(file_destination, 'parent_burst') return file_destination def get_export_file_extension(self, data): if self.is_data_a_group(data): return "zip" else: return "h5"
def editone(self, project_id=None, cancel=False, save=False, delete=False, **data): """ Create or change Project. When project_id is empty we create a new entity, otherwise we are to edit and existent one. """ if cherrypy.request.method == 'POST' and cancel: raise cherrypy.HTTPRedirect('/project') if cherrypy.request.method == 'POST' and delete: self._remove_project(project_id) raise cherrypy.HTTPRedirect('/project/viewall') current_user = common.get_logged_user() is_create = False if project_id is None or not int(project_id): is_create = True data["administrator"] = current_user.display_name admin_username = current_user.username else: current_project = self.project_service.find_project(project_id) if not save: # Only when we do not have submitted data, # populate fields with initial values for edit. data = dict(name=current_project.name, description=current_project.description) data["administrator"] = current_project.administrator.display_name admin_username = current_project.administrator.username self._mark_selected(current_project) data["project_id"] = project_id template_specification = dict(mainContent="project/editone", data=data, isCreate=is_create, title="Create new project" if is_create else "Edit " + data["name"], editUsersEnabled=(current_user.username == admin_username)) try: if cherrypy.request.method == 'POST' and save: data = EditForm().to_python(data) saved_project = self.project_service.store_project(current_user, is_create, project_id, **data) if StorageInterface.encryption_enabled() and is_create: project_folder = StorageInterface().get_project_folder(saved_project.name) StorageInterface.sync_folders(project_folder) shutil.rmtree(project_folder) self._mark_selected(saved_project) raise cherrypy.HTTPRedirect('/project/viewall') except formencode.Invalid as excep: self.logger.debug(str(excep)) template_specification[common.KEY_ERRORS] = excep.unpack_errors() except ProjectServiceException as excep: self.logger.debug(str(excep)) common.set_error_message(excep.message) raise cherrypy.HTTPRedirect('/project/viewall') all_users, members, pages = self.user_service.get_users_for_project(current_user.username, project_id) template_specification['usersList'] = all_users template_specification['usersMembers'] = [m.id for m in members] template_specification['usersPages'] = pages template_specification['usersCurrentPage'] = 1 return self.fill_default_attributes(template_specification, 'properties')
def __init__(self, project_path): self.project_path = project_path self.storage_interface = StorageInterface() # This assumes that old project metadata file can be parsed by current version. self.project_meta = self.storage_interface.read_project_metadata( project_path) from_version = int(self.project_meta.get('version', 0)) super(ProjectUpdateManager, self).__init__(project_update_scripts, from_version, TvbProfile.current.version.PROJECT_VERSION)
def teardown_method(self): """ Reset the database when test is done. """ # Delete TEMP folder StorageInterface.remove_folder(TvbProfile.current.TVB_TEMP_FOLDER) # Delete folder where data was exported if self.zip_path: StorageInterface.remove_folder(os.path.split(self.zip_path)[0]) self.delete_project_folders()
def test_tvb_linked_export_of_simple_datatype_with_encryption( self, connectivity_index_factory, surface_index_factory, region_mapping_index_factory): """ Test export of an encrypted data type and its linked data types which have no data stored on file system """ conn = connectivity_index_factory() surface_idx, surface = surface_index_factory(cortical=True) region_mapping_index = region_mapping_index_factory( conn_gid=conn.gid, surface_gid=surface.gid.hex) storage_interface = StorageInterface() import_export_encryption_handler = StorageInterface.get_import_export_encryption_handler( ) import_export_encryption_handler.generate_public_private_key_pair( TvbProfile.current.TVB_TEMP_FOLDER) _, file_path, _ = self.export_manager.export_data( region_mapping_index, self.TVB_LINKED_EXPORTER, self.test_project, os.path.join(TvbProfile.current.TVB_TEMP_FOLDER, import_export_encryption_handler.PUBLIC_KEY_NAME)) assert file_path is not None, "Export process should return path to export file" assert os.path.exists( file_path), "Could not find export file;: %s on disk." % file_path result = storage_interface.unpack_zip( file_path, TvbProfile.current.TVB_TEMP_FOLDER) encrypted_password = import_export_encryption_handler.extract_encrypted_password_from_list( result) decrypted_file_paths = import_export_encryption_handler.decrypt_content( encrypted_password, result, os.path.join(TvbProfile.current.TVB_TEMP_FOLDER, import_export_encryption_handler.PRIVATE_KEY_NAME)) original_conn_path = h5.path_for_stored_index(conn) decrypted_conn_path, idx = (decrypted_file_paths[0], 0) if 'Connectivity' in decrypted_file_paths[0] else \ (decrypted_file_paths[1], 1) if 'Connectivity' in decrypted_file_paths[1] else (decrypted_file_paths[2], 2) self.compare_files(original_conn_path, decrypted_conn_path) original_surface_path = h5.path_for_stored_index(surface_idx) del decrypted_file_paths[idx] decrypted_surface_path, idx = (decrypted_file_paths[0], 0) if 'Surface' in decrypted_file_paths[0] else \ (decrypted_file_paths[1], 1) self.compare_files(original_surface_path, decrypted_surface_path) original_rm_path = h5.path_for_stored_index(region_mapping_index) del decrypted_file_paths[idx] self.compare_files(original_rm_path, decrypted_file_paths[0])
def import_conn_h5(project_id, h5_path): project = dao.get_project_by_id(project_id) TvbProfile.set_profile(TvbProfile.COMMAND_PROFILE) now = datetime.now() date_str = "%d-%d-%d_%d-%d-%d_%d" % (now.year, now.month, now.day, now.hour, now.minute, now.second, now.microsecond) uq_name = "%s-Connectivity" % date_str new_path = os.path.join(TvbProfile.current.TVB_TEMP_FOLDER, uq_name) StorageInterface.copy_file(h5_path, new_path) importer = ABCAdapter.build_adapter_from_class(TVBImporter) view_model = importer.get_view_model_class()() view_model.data_file = new_path return OperationService().fire_operation(importer, project.administrator, project_id, view_model=view_model)
def delete_project_folders(): """ This method deletes folders for all projects from TVB folder. This is done without any check on database. You might get projects in DB but no folder for them on disk. """ BaseStorageTestCase.delete_projects_folders() for folder in [ os.path.join(TvbProfile.current.TVB_STORAGE, StorageInterface.EXPORT_FOLDER_NAME), os.path.join(TvbProfile.current.TVB_STORAGE, StorageInterface.TEMP_FOLDER) ]: StorageInterface.remove_folder(folder, True) os.makedirs(folder)
def load_burst_read_only(self, burst_config_id): try: burst_config = self.burst_service.load_burst_configuration( burst_config_id) storage_path = StorageInterface().get_project_folder( self.context.project.name, str(burst_config.fk_simulation)) simulator = h5.load_view_model(burst_config.simulator_gid, storage_path) last_loaded_form_url = self.get_url_for_final_fragment( burst_config) self.context.init_session_at_burst_loading(burst_config, simulator, last_loaded_form_url) form = self.prepare_first_fragment() self.monitors_handler.build_list_of_monitors_from_view_models( self.context.simulator) rendering_rules = SimulatorFragmentRenderingRules( form, SimulatorWizzardURLs.SET_CONNECTIVITY_URL, is_simulation_readonly_load=True, is_first_fragment=True) return rendering_rules.to_dict() except Exception: # Most probably Burst was removed. Delete it from session, so that client # has a good chance to get a good response on refresh self.logger.exception("Error loading burst") self.context.remove_burst_config_from_session() raise
def test_adapter_huge_memory_requirement(self, test_adapter_factory): """ Test that an MemoryException is raised in case adapter cant launch due to lack of memory. """ # Prepare adapter test_adapter_factory(adapter_class=DummyAdapterHugeMemoryRequired) adapter = TestFactory.create_adapter("tvb.tests.framework.adapters.dummy_adapter3", "DummyAdapterHugeMemoryRequired") # Simulate receiving POST data form = DummyAdapterHugeMemoryRequiredForm() view_model = form.get_view_model()() view_model.test = 5 # Prepare operation for launch operation = Operation(view_model.gid.hex, self.test_user.id, self.test_project.id, adapter.stored_adapter.id, status=STATUS_STARTED) operation = dao.store_entity(operation) # Store ViewModel in H5 parent_folder = StorageInterface().get_project_folder(self.test_project.name, str(operation.id)) h5.store_view_model(view_model, parent_folder) # Launch operation with pytest.raises(NoMemoryAvailableException): OperationService().initiate_prelaunch(operation, adapter)
def test_export_simulator_configuration(self, operation_factory, connectivity_index_factory): """ Test export of a simulator configuration """ conn_gid = uuid.UUID(connectivity_index_factory().gid) operation = operation_factory(is_simulation=True, store_vm=True, test_project=self.test_project, conn_gid=conn_gid) burst_configuration = BurstConfiguration(self.test_project.id) burst_configuration.fk_simulation = operation.id burst_configuration.simulator_gid = operation.view_model_gid burst_configuration.name = "Test_burst" burst_configuration = dao.store_entity(burst_configuration) op_folder = StorageInterface().get_project_folder( self.test_project.name, str(operation.id)) BurstService().store_burst_configuration(burst_configuration, op_folder) export_file = self.export_manager.export_simulator_configuration( burst_configuration.id) assert export_file is not None, "Export process should return path to export file" assert os.path.exists( export_file ), "Could not find export file: %s on disk." % export_file assert zipfile.is_zipfile( export_file), "Generated file is not a valid ZIP file"
def _configure_job(simulator_gid, available_space, is_group_launch, operation_id): # type: (str, int, bool, int) -> (dict, list) bash_entrypoint = os.path.join( os.environ[HPCSchedulerClient.TVB_BIN_ENV_KEY], HPCSettings.HPC_LAUNCHER_SH_SCRIPT) base_url = TvbProfile.current.web.BASE_URL inputs_in_container = os.path.join( HPCSchedulerClient.CONTAINER_INPUT_FOLDER, StorageInterface.get_encryption_handler( simulator_gid).current_enc_dirname) # Build job configuration JSON my_job = { HPCSettings.UNICORE_EXE_KEY: os.path.basename(bash_entrypoint), HPCSettings.UNICORE_ARGS_KEY: [ simulator_gid, available_space, is_group_launch, base_url, inputs_in_container, HPCSchedulerClient.HOME_FOLDER_MOUNT, operation_id ], HPCSettings.UNICORE_RESOURCER_KEY: { "CPUs": "1" } } if HPCSchedulerClient.CSCS_PROJECT in os.environ: my_job[HPCSettings.UNICORE_PROJECT_KEY] = os.environ[ HPCSchedulerClient.CSCS_PROJECT] return my_job, bash_entrypoint
def update(input_file, burst_match_dict=None): """ In order to avoid segmentation faults when updating a batch of files just start every conversion on a different Python process. :param input_file: the file that needs to be converted to a newer file storage version. This should be a file that still uses TVB 2.0 storage """ if not os.path.isfile(input_file): raise FileVersioningException("The input path %s received for upgrading from 2 -> 3 is not a " "valid file on the disk." % input_file) folder, file_name = os.path.split(input_file) storage_manager = StorageInterface.get_storage_manager(input_file) root_metadata = storage_manager.get_metadata() class_name = root_metadata[DataTypeMetaData.KEY_CLASS_NAME] if class_name == "LocalConnectivity": root_metadata[DataTypeMetaData.KEY_MODULE] = "tvb.datatypes.local_connectivity" storage_manager.set_metadata(root_metadata) update_localconnectivity_metadata(folder, file_name) elif class_name == "RegionMapping": root_metadata[DataTypeMetaData.KEY_MODULE] = "tvb.datatypes.region_mapping" root_metadata[TvbProfile.current.version.DATA_VERSION_ATTRIBUTE] = TvbProfile.current.version.DATA_VERSION storage_manager.set_metadata(root_metadata)