def launch(self, view_model): # type: (CrossCorrelateAdapterModel) -> [CrossCorrelationIndex] """ Launch algorithm and build results. Compute the node-pairwise cross-correlation of the source 4D TimeSeries represented by the index given as input. Return a CrossCorrelationIndex. Create a CrossCorrelationH5 that contains the cross-correlation sequences for all possible combinations of the nodes. See: http://www.scipy.org/doc/api_docs/SciPy.signal.signaltools.html#correlate :param view_model: the ViewModel keeping the algorithm inputs :return: the cross correlation index for the given time series :rtype: `CrossCorrelationIndex` """ # --------- Prepare CrossCorrelationIndex and CrossCorrelationH5 objects for result ------------## cross_corr_index = CrossCorrelationIndex() cross_corr_h5_path = h5.path_for(self.storage_path, CrossCorrelationH5, cross_corr_index.gid) cross_corr_h5 = CrossCorrelationH5(cross_corr_h5_path) node_slice = [slice(self.input_shape[0]), None, slice(self.input_shape[2]), slice(self.input_shape[3])] # ---------- Iterate over slices and compose final result ------------## small_ts = TimeSeries() with h5.h5_file_for_index(self.input_time_series_index) as ts_h5: small_ts.sample_period = ts_h5.sample_period.load() small_ts.sample_period_unit = ts_h5.sample_period_unit.load() partial_cross_corr = None for var in range(self.input_shape[1]): node_slice[1] = slice(var, var + 1) small_ts.data = ts_h5.read_data_slice(tuple(node_slice)) partial_cross_corr = self._compute_cross_correlation(small_ts, ts_h5) cross_corr_h5.write_data_slice(partial_cross_corr) partial_cross_corr.source.gid = view_model.time_series partial_cross_corr.gid = uuid.UUID(cross_corr_index.gid) cross_corr_index.fill_from_has_traits(partial_cross_corr) self.fill_index_from_h5(cross_corr_index, cross_corr_h5) cross_corr_h5.store(partial_cross_corr, scalars_only=True) cross_corr_h5.close() return cross_corr_index
def launch(self, view_model): # type: (NodeCoherenceModel) -> [CoherenceSpectrumIndex] """ Launch algorithm and build results. """ # --------- Prepare a CoherenceSpectrum object for result ------------## coherence_spectrum_index = CoherenceSpectrumIndex() time_series_h5 = h5.h5_file_for_index(self.input_time_series_index) dest_path = h5.path_for(self.storage_path, CoherenceSpectrumH5, coherence_spectrum_index.gid) coherence_h5 = CoherenceSpectrumH5(dest_path) coherence_h5.gid.store(uuid.UUID(coherence_spectrum_index.gid)) coherence_h5.source.store(time_series_h5.gid.load()) coherence_h5.nfft.store(self.algorithm.nfft) # ------------- NOTE: Assumes 4D, Simulator timeSeries. --------------## input_shape = time_series_h5.data.shape node_slice = [ slice(input_shape[0]), None, slice(input_shape[2]), slice(input_shape[3]) ] # ---------- Iterate over slices and compose final result ------------## small_ts = TimeSeries() small_ts.sample_period = time_series_h5.sample_period.load() partial_coh = None for var in range(input_shape[1]): node_slice[1] = slice(var, var + 1) small_ts.data = time_series_h5.read_data_slice(tuple(node_slice)) self.algorithm.time_series = small_ts partial_coh = self.algorithm.evaluate() coherence_h5.write_data_slice(partial_coh) coherence_h5.frequency.store(partial_coh.frequency) coherence_h5.close() coherence_spectrum_index.ndim = len(coherence_h5.array_data.shape) time_series_h5.close() coherence_spectrum_index.fk_source_gid = self.input_time_series_index.gid coherence_spectrum_index.nfft = partial_coh.nfft coherence_spectrum_index.frequencies = partial_coh.frequency return coherence_spectrum_index
def test_node_coherence_adapter(self, tmpdir, time_series_index_factory): # algorithm returns complex values instead of float storage_folder = str(tmpdir) ts_index = time_series_index_factory() node_coherence_adapter = NodeCoherenceAdapter() node_coherence_adapter.storage_path = storage_folder view_model = node_coherence_adapter.get_view_model_class()() view_model.time_series = ts_index.gid node_coherence_adapter.configure(view_model) disk = node_coherence_adapter.get_required_disk_size(view_model) mem = node_coherence_adapter.get_required_memory_size(view_model) coherence_spectrum_idx = node_coherence_adapter.launch(view_model) result_h5 = h5.path_for(storage_folder, CoherenceSpectrumH5, coherence_spectrum_idx.gid) assert os.path.exists(result_h5)
def test_cross_correlation_adapter(self, tmpdir, time_series_index_factory): storage_folder = str(tmpdir) ts_index = time_series_index_factory() cross_correlation_adapter = CrossCorrelateAdapter() cross_correlation_adapter.storage_path = storage_folder view_model = cross_correlation_adapter.get_view_model_class()() view_model.time_series = ts_index.gid cross_correlation_adapter.configure(view_model) disk = cross_correlation_adapter.get_required_disk_size(view_model) mem = cross_correlation_adapter.get_required_memory_size(view_model) cross_correlation_idx = cross_correlation_adapter.launch(view_model) result_h5 = h5.path_for(storage_folder, CrossCorrelationH5, cross_correlation_idx.gid) assert os.path.exists(result_h5)
def launch(self, view_model): # type: (ICAAdapterModel) -> [IndependentComponentsIndex] """ Launch algorithm and build results. :param view_model: the ViewModel keeping the algorithm inputs :return: the ica index for the specified time series """ # --------------------- Prepare result entities ---------------------## ica_index = IndependentComponentsIndex() result_path = h5.path_for(self.storage_path, IndependentComponentsH5, ica_index.gid) ica_h5 = IndependentComponentsH5(path=result_path) # ------------- NOTE: Assumes 4D, Simulator timeSeries. --------------## time_series_h5 = h5.h5_file_for_index(self.input_time_series_index) input_shape = time_series_h5.data.shape node_slice = [ slice(input_shape[0]), None, slice(input_shape[2]), slice(input_shape[3]) ] # ---------- Iterate over slices and compose final result ------------## small_ts = TimeSeries() for var in range(input_shape[1]): node_slice[1] = slice(var, var + 1) small_ts.data = time_series_h5.read_data_slice(tuple(node_slice)) partial_ica = compute_ica_decomposition(small_ts, view_model.n_components) ica_h5.write_data_slice(partial_ica) time_series_h5.close() partial_ica.source.gid = view_model.time_series partial_ica.gid = uuid.UUID(ica_index.gid) ica_h5.store(partial_ica, scalars_only=True) ica_h5.close() ica_index.fill_from_has_traits(partial_ica) return ica_index
def launch(self, view_model): # type: (ICAAdapterModel) -> [IndependentComponentsIndex] """ Launch algorithm and build results. """ # --------- Prepare a IndependentComponents object for result ----------## ica_index = IndependentComponentsIndex() ica_index.fk_source_gid = view_model.time_series.hex time_series_h5 = h5.h5_file_for_index(self.input_time_series_index) result_path = h5.path_for(self.storage_path, IndependentComponentsH5, ica_index.gid) ica_h5 = IndependentComponentsH5(path=result_path) ica_h5.gid.store(uuid.UUID(ica_index.gid)) ica_h5.source.store(view_model.time_series) ica_h5.n_components.store(self.algorithm.n_components) # ------------- NOTE: Assumes 4D, Simulator timeSeries. --------------## input_shape = time_series_h5.data.shape node_slice = [ slice(input_shape[0]), None, slice(input_shape[2]), slice(input_shape[3]) ] # ---------- Iterate over slices and compose final result ------------## small_ts = TimeSeries() for var in range(input_shape[1]): node_slice[1] = slice(var, var + 1) small_ts.data = time_series_h5.read_data_slice(tuple(node_slice)) self.algorithm.time_series = small_ts partial_ica = self.algorithm.evaluate() ica_h5.write_data_slice(partial_ica) array_metadata = ica_h5.unmixing_matrix.get_cached_metadata() ica_index.array_has_complex = array_metadata.has_complex ica_index.shape = json.dumps(ica_h5.unmixing_matrix.shape) ica_index.ndim = len(ica_h5.unmixing_matrix.shape) ica_h5.close() time_series_h5.close() return ica_index
def __copy_linked_datatype_before_delete(self, op, datatype, project, fk_to_project): new_op = Operation(op.view_model_gid, dao.get_system_user().id, fk_to_project, datatype.parent_operation.fk_from_algo, datatype.parent_operation.status, datatype.parent_operation.start_date, datatype.parent_operation.completion_date, datatype.parent_operation.fk_operation_group, datatype.parent_operation.additional_info, datatype.parent_operation.user_group, datatype.parent_operation.range_values) new_op.visible = datatype.parent_operation.visible new_op = dao.store_entity(new_op) to_project = self.find_project(fk_to_project) to_project_path = self.storage_interface.get_project_folder(to_project.name) full_path = h5.path_for_stored_index(datatype) old_folder = self.storage_interface.get_project_folder(project.name, str(op.id)) file_paths = h5.gather_references_of_view_model(op.view_model_gid, old_folder, only_view_models=True)[0] file_paths.append(full_path) # The BurstConfiguration h5 file has to be moved only when we handle the time series which has the operation # folder containing the file if datatype.is_ts and datatype.fk_parent_burst is not None: bc_path = h5.path_for(datatype.parent_operation.id, BurstConfigurationH5, datatype.fk_parent_burst, project.name) if os.path.exists(bc_path): file_paths.append(bc_path) bc = dao.get_burst_for_operation_id(op.id) bc.fk_simulation = new_op.id dao.store_entity(bc) # Move all files to the new operation folder self.storage_interface.move_datatype_with_sync(to_project, to_project_path, new_op.id, file_paths) datatype.fk_from_operation = new_op.id datatype.parent_operation = new_op dao.store_entity(datatype) return new_op
def launch(self, view_model): # type: (NodeCovarianceAdapterModel) -> [CovarianceIndex] """ Launch algorithm and build results. :returns: the `CovarianceIndex` built with the given time_series index as source """ # Create an index for the computed covariance. covariance_index = CovarianceIndex() covariance_h5_path = h5.path_for(self.storage_path, CovarianceH5, covariance_index.gid) covariance_h5 = CovarianceH5(covariance_h5_path) # NOTE: Assumes 4D, Simulator timeSeries. node_slice = [ slice(self.input_shape[0]), None, slice(self.input_shape[2]), None ] with h5.h5_file_for_index(self.input_time_series_index) as ts_h5: for mode in range(self.input_shape[3]): for var in range(self.input_shape[1]): small_ts = TimeSeries() node_slice[1] = slice(var, var + 1) node_slice[3] = slice(mode, mode + 1) small_ts.data = ts_h5.read_data_slice(tuple(node_slice)) partial_cov = self._compute_node_covariance( small_ts, ts_h5) covariance_h5.write_data_slice(partial_cov.array_data) ts_array_metadata = covariance_h5.array_data.get_cached_metadata() covariance_index.source_gid = self.input_time_series_index.gid covariance_index.subtype = type(covariance_index).__name__ covariance_index.array_data_min = ts_array_metadata.min covariance_index.array_data_max = ts_array_metadata.max covariance_index.array_data_mean = ts_array_metadata.mean covariance_index.ndim = len(covariance_h5.array_data.shape) covariance_h5.gid.store(uuid.UUID(covariance_index.gid)) covariance_h5.source.store(view_model.time_series) covariance_h5.close() return covariance_index
def launch(self, view_model): # type: (PCAAdapterModel) -> [PrincipalComponentsIndex] """ Launch algorithm and build results. :param view_model: the ViewModel keeping the algorithm inputs :return: the `PrincipalComponentsIndex` object built with the given timeseries as source """ # --------------------- Prepare result entities ----------------------## principal_components_index = PrincipalComponentsIndex() dest_path = h5.path_for(self.storage_path, PrincipalComponentsH5, principal_components_index.gid) pca_h5 = PrincipalComponentsH5(path=dest_path) # ------------- NOTE: Assumes 4D, Simulator timeSeries. --------------## time_series_h5 = h5.h5_file_for_index(self.input_time_series_index) input_shape = time_series_h5.data.shape node_slice = [ slice(input_shape[0]), None, slice(input_shape[2]), slice(input_shape[3]) ] # ---------- Iterate over slices and compose final result ------------## small_ts = TimeSeries() for var in range(input_shape[1]): node_slice[1] = slice(var, var + 1) small_ts.data = time_series_h5.read_data_slice(tuple(node_slice)) self.time_series = small_ts.gid partial_pca = compute_pca(small_ts) pca_h5.write_data_slice(partial_pca) time_series_h5.close() partial_pca.source.gid = view_model.time_series partial_pca.gid = uuid.UUID(principal_components_index.gid) principal_components_index.fill_from_has_traits(partial_pca) pca_h5.store(partial_pca, scalars_only=True) pca_h5.close() return principal_components_index
def launch(self, data_file, dataset_name, connectivity): """ Execute import operations: """ try: data = self.read_matlab_data(data_file, dataset_name) measurement_count, node_count = data.shape if node_count != connectivity.number_of_regions: raise LaunchException( 'The measurements are for %s nodes but the selected connectivity' ' contains %s nodes' % (node_count, connectivity.number_of_regions)) measures = [] for i in range(measurement_count): cm_idx = ConnectivityMeasureIndex() cm_idx.type = ConnectivityMeasureIndex.__name__ cm_idx.connectivity_gid = connectivity.gid cm_data = data[i, :] cm_idx.array_data_ndim = cm_data.ndim cm_idx.ndim = cm_data.ndim cm_idx.array_data_min, cm_idx.array_data_max, cm_idx.array_data_mean = from_ndarray( cm_data) cm_h5_path = h5.path_for(self.storage_path, ConnectivityMeasureH5, cm_idx.gid) with ConnectivityMeasureH5(cm_h5_path) as cm_h5: cm_h5.array_data.store(data[i, :]) cm_h5.connectivity.store(uuid.UUID(connectivity.gid)) cm_h5.gid.store(uuid.UUID(cm_idx.gid)) cm_idx.user_tag_2 = "nr.-%d" % (i + 1) cm_idx.user_tag_3 = "conn_%d" % node_count measures.append(cm_idx) return measures except ParseException as excep: logger = get_logger(__name__) logger.exception(excep) raise LaunchException(excep)
def launch(self, view_model): # type: (BalloonModelAdapterModel) -> [TimeSeriesRegionIndex] """ Launch algorithm and build results. :param time_series: the input time-series used as neural activation in the Balloon Model :returns: the simulated BOLD signal :rtype: `TimeSeries` """ input_time_series_h5 = h5.h5_file_for_index(self.input_time_series_index) time_line = input_time_series_h5.read_time_page(0, self.input_shape[0]) bold_signal_index = TimeSeriesRegionIndex() bold_signal_h5_path = h5.path_for(self.storage_path, TimeSeriesRegionH5, bold_signal_index.gid) bold_signal_h5 = TimeSeriesRegionH5(bold_signal_h5_path) bold_signal_h5.gid.store(uuid.UUID(bold_signal_index.gid)) self._fill_result_h5(bold_signal_h5, input_time_series_h5) # ---------- Iterate over slices and compose final result ------------## node_slice = [slice(self.input_shape[0]), slice(self.input_shape[1]), None, slice(self.input_shape[3])] small_ts = TimeSeries() small_ts.sample_period = self.input_time_series_index.sample_period small_ts.sample_period_unit = self.input_time_series_index.sample_period_unit small_ts.time = time_line for node in range(self.input_shape[2]): node_slice[2] = slice(node, node + 1) small_ts.data = input_time_series_h5.read_data_slice(tuple(node_slice)) self.algorithm.time_series = small_ts partial_bold = self.algorithm.evaluate() bold_signal_h5.write_data_slice_on_grow_dimension(partial_bold.data, grow_dimension=2) bold_signal_h5.write_time_slice(time_line) bold_signal_shape = bold_signal_h5.data.shape bold_signal_h5.nr_dimensions.store(len(bold_signal_shape)) bold_signal_h5.close() input_time_series_h5.close() self._fill_result_index(bold_signal_index, bold_signal_shape) return bold_signal_index
def test_export(self): op = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project) burst_config = BurstConfiguration(self.test_project.id) burst_config.fk_simulation = op.id burst_config.simulator_gid = self.session_stored_simulator.gid.hex burst_config = dao.store_entity(burst_config) storage_path = FilesHelper().get_project_folder(self.test_project, str(op.id)) h5_path = h5.path_for(storage_path, SimulatorH5, self.session_stored_simulator.gid) with SimulatorH5(h5_path) as h5_file: h5_file.store(self.session_stored_simulator) burst = dao.get_bursts_for_project(self.test_project.id) self.sess_mock['burst_id'] = str(burst[0].id) with patch('cherrypy.session', self.sess_mock, create=True): common.add2session(common.KEY_BURST_CONFIG, self.session_stored_simulator) common.add2session(common.KEY_BURST_CONFIG, burst_config) result = self.simulator_controller.export(str(burst[0].id)) assert path.exists(result.input.name), "Simulation was not exported!"
def launch_operation(self, project_gid, algorithm_class, view_model, temp_folder): h5_file_path = h5.path_for(temp_folder, ViewModelH5, view_model.gid) h5_file = ViewModelH5(h5_file_path, view_model) h5_file.store(view_model) h5_file.close() model_file_obj = open(h5_file_path, 'rb') files = {RequestFileKey.LAUNCH_ANALYZERS_MODEL_FILE.value: (os.path.basename(h5_file_path), model_file_obj)} if issubclass(algorithm_class, ABCUploader): for key in algorithm_class().get_form_class().get_upload_information().keys(): path = getattr(view_model, key) data_file_obj = open(path, 'rb') files[key] = (os.path.basename(path), data_file_obj) return self.secured_request().post(self.build_request_url(RestLink.LAUNCH_OPERATION.compute_url(True, { LinkPlaceholder.PROJECT_GID.value: project_gid, LinkPlaceholder.ALG_MODULE.value: algorithm_class.__module__, LinkPlaceholder.ALG_CLASSNAME.value: algorithm_class.__name__ })), files=files)
def launch(self, view_model): # type: (NodeCovarianceAdapterModel) -> [CovarianceIndex] """ Launch algorithm and build results. :param view_model: the ViewModel keeping the algorithm inputs :return: the `CovarianceIndex` built with the given time_series index as source """ # -------------------- Prepare result entities ---------------------## covariance_index = CovarianceIndex() covariance_h5_path = h5.path_for(self.storage_path, CovarianceH5, covariance_index.gid) covariance_h5 = CovarianceH5(covariance_h5_path) # ------------ NOTE: Assumes 4D, Simulator timeSeries -------------## node_slice = [ slice(self.input_shape[0]), None, slice(self.input_shape[2]), None ] ts_h5 = h5.h5_file_for_index(self.input_time_series_index) for mode in range(self.input_shape[3]): for var in range(self.input_shape[1]): small_ts = TimeSeries() node_slice[1] = slice(var, var + 1) node_slice[3] = slice(mode, mode + 1) small_ts.data = ts_h5.read_data_slice(tuple(node_slice)) partial_cov = self._compute_node_covariance(small_ts, ts_h5) covariance_h5.write_data_slice(partial_cov.array_data) ts_h5.close() partial_cov.source.gid = view_model.time_series partial_cov.gid = uuid.UUID(covariance_index.gid) covariance_index.fill_from_has_traits(partial_cov) self.fill_index_from_h5(covariance_index, covariance_h5) covariance_h5.store(partial_cov, scalars_only=True) covariance_h5.close() return covariance_index
def launch(self, view_model): datatype = self._base_before_launch(view_model.data_file, view_model.region_volume) # note the streaming parsing, we do not load the dataset in memory at once tract_gen, hdr = trackvis.read(view_model.data_file, as_generator=True) vox2ras = _SpaceTransform(hdr) tract_start_indices = [0] tract_region = [] with TractsH5(path_for(self.storage_path, TractsH5, datatype.gid)) as tracts_h5: # we process tracts in bigger chunks to optimize disk write costs for tract_bundle in chunk_iter(tract_gen, self.READ_CHUNK): tract_bundle = [tr[0] for tr in tract_bundle] for tr in tract_bundle: tract_start_indices.append(tract_start_indices[-1] + len(tr)) if view_model.region_volume is not None: tract_region.append(self._get_tract_region(tr[0])) vertices = numpy.concatenate(tract_bundle) # in voxel space datatype.vertices = vox2ras.transform(vertices) tracts_h5.write_vertices_slice(datatype.vertices) datatype.tract_start_idx = numpy.array(tract_start_indices) datatype.tract_region = numpy.array(tract_region, dtype=numpy.int16) tracts_h5.tract_region.store(datatype.tract_region) tracts_h5.tract_start_idx.store(datatype.tract_start_idx) tracts_h5.region_volume_map.store(view_model.region_volume) self.region_volume_h5.close() tracts_index = TractsIndex() tracts_index.fill_from_has_traits(datatype) return tracts_index
def launch(self, view_model): # type: (PCAAdapterModel) -> [PrincipalComponentsIndex] """ Launch algorithm and build results. :returns: the `PrincipalComponents` object built with the given timeseries as source """ # --------- Prepare a PrincipalComponents object for result ----------## principal_components_index = PrincipalComponentsIndex() principal_components_index.fk_source_gid = view_model.time_series.hex time_series_h5 = h5.h5_file_for_index(self.input_time_series_index) dest_path = h5.path_for(self.storage_path, PrincipalComponentsH5, principal_components_index.gid) pca_h5 = PrincipalComponentsH5(path=dest_path) pca_h5.source.store(time_series_h5.gid.load()) pca_h5.gid.store(uuid.UUID(principal_components_index.gid)) # ------------- NOTE: Assumes 4D, Simulator timeSeries. --------------## input_shape = time_series_h5.data.shape node_slice = [ slice(input_shape[0]), None, slice(input_shape[2]), slice(input_shape[3]) ] # ---------- Iterate over slices and compose final result ------------## small_ts = TimeSeries() for var in range(input_shape[1]): node_slice[1] = slice(var, var + 1) small_ts.data = time_series_h5.read_data_slice(tuple(node_slice)) self.algorithm.time_series = small_ts partial_pca = self.algorithm.evaluate() pca_h5.write_data_slice(partial_pca) pca_h5.close() time_series_h5.close() return principal_components_index
def create_region_ts(self, data_shape, connectivity): if connectivity.number_of_regions != data_shape[1]: raise LaunchException( "Data has %d channels but the connectivity has %d nodes" % (data_shape[1], connectivity.number_of_regions)) ts_idx = TimeSeriesRegionIndex() ts_idx.fk_connectivity_gid = connectivity.gid region_map_indexes = dao.get_generic_entity(RegionMappingIndex, connectivity.gid, 'fk_connectivity_gid') ts_idx.has_surface_mapping = False if len(region_map_indexes) > 0: ts_idx.fk_region_mapping_gid = region_map_indexes[0].gid ts_idx.has_surface_mapping = True ts_h5_path = h5.path_for(self.storage_path, TimeSeriesRegionH5, ts_idx.gid) ts_h5 = TimeSeriesRegionH5(ts_h5_path) ts_h5.connectivity.store(uuid.UUID(connectivity.gid)) return TimeSeriesRegion(), ts_idx, ts_h5
def test_server_launch_operation(self, mocker, time_series_index_factory): self._mock_user(mocker) algorithm_module = "tvb.adapters.analyzers.fourier_adapter" algorithm_class = "FourierAdapter" input_ts_index = time_series_index_factory() fft_model = FFTAdapterModel() fft_model.time_series = UUID(input_ts_index.gid) fft_model.window_function = list(SUPPORTED_WINDOWING_FUNCTIONS)[0] input_folder = self.files_helper.get_project_folder(self.test_project) view_model_h5_path = h5.path_for(input_folder, ViewModelH5, fft_model.gid) view_model_h5 = ViewModelH5(view_model_h5_path, fft_model) view_model_h5.store(fft_model) view_model_h5.close() # Mock flask.request.files to return a dictionary request_mock = mocker.patch.object(flask, 'request') fp = open(view_model_h5_path, 'rb') request_mock.files = { RequestFileKey.LAUNCH_ANALYZERS_MODEL_FILE.value: FileStorage(fp, os.path.basename(view_model_h5_path)) } # Mock launch_operation() call and current_user mocker.patch.object(OperationService, 'launch_operation') operation_gid, status = self.launch_resource.post( project_gid=self.test_project.gid, algorithm_module=algorithm_module, algorithm_classname=algorithm_class) fp.close() assert type(operation_gid) is str assert len(operation_gid) > 0
def test_fmri_balloon_adapter(self, tmpdir, time_series_region_index_factory, connectivity_factory, region_mapping_factory, surface_factory): # To be fixed once we have the migrated importers storage_folder = str(tmpdir) connectivity = connectivity_factory() surface = surface_factory() region_mapping = region_mapping_factory(surface=surface, connectivity=connectivity) ts_index = time_series_region_index_factory(connectivity=connectivity, region_mapping=region_mapping) fmri_balloon_adapter = BalloonModelAdapter() fmri_balloon_adapter.storage_path = storage_folder view_model = fmri_balloon_adapter.get_view_model_class()() view_model.time_series = ts_index.gid fmri_balloon_adapter.configure(view_model) disk = fmri_balloon_adapter.get_required_disk_size(view_model) mem = fmri_balloon_adapter.get_required_memory_size(view_model) ts_index = fmri_balloon_adapter.launch(view_model) result_h5 = h5.path_for(storage_folder, TimeSeriesRegionH5, ts_index.gid) assert os.path.exists(result_h5)
def test_adapter_huge_memory_requirement(self, test_adapter_factory): """ Test that an MemoryException is raised in case adapter cant launch due to lack of memory. """ # Prepare adapter test_adapter_factory(adapter_class=TestAdapterHugeMemoryRequired) adapter = TestFactory.create_adapter( "tvb.tests.framework.adapters.testadapter3", "TestAdapterHugeMemoryRequired") # Simulate receiving POST data form = TestAdapterHugeMemoryRequiredForm() adapter.submit_form(form) view_model = form.get_view_model()() view_model.test = 5 # Prepare operation for launch operation = model_operation.Operation( self.test_user.id, self.test_project.id, adapter.stored_adapter.id, json.dumps({'gid': view_model.gid.hex}), json.dumps({}), status=model_operation.STATUS_STARTED) operation = dao.store_entity(operation) # Store ViewModel in H5 parent_folder = FilesHelper().get_project_folder( self.test_project, str(operation.id)) view_model_path = os.path.join( parent_folder, h5.path_for(parent_folder, ViewModelH5, view_model.gid)) with ViewModelH5(view_model_path, view_model) as view_model_h5: view_model_h5.store(view_model) # Launch operation with pytest.raises(NoMemoryAvailableException): OperationService().initiate_prelaunch(operation, adapter)
def load_datatype_from_file(self, storage_folder, file_name, op_id, datatype_group=None, move=True, final_storage=None): """ Creates an instance of datatype from storage / H5 file :returns: DatatypeIndex """ self.logger.debug("Loading DataType from file: %s" % file_name) datatype, generic_attributes = h5.load_with_references( os.path.join(storage_folder, file_name)) index_class = h5.REGISTRY.get_index_for_datatype(datatype.__class__) datatype_index = index_class() datatype_index.fill_from_has_traits(datatype) datatype_index.fill_from_generic_attributes(generic_attributes) # Add all the required attributes if datatype_group is not None: datatype_index.fk_datatype_group = datatype_group.id datatype_index.fk_from_operation = op_id associated_file = h5.path_for_stored_index(datatype_index) if os.path.exists(associated_file): datatype_index.disk_size = FilesHelper.compute_size_on_disk( associated_file) # Now move storage file into correct folder if necessary if move and final_storage is not None: current_file = os.path.join(storage_folder, file_name) h5_type = h5.REGISTRY.get_h5file_for_datatype(datatype.__class__) final_path = h5.path_for(final_storage, h5_type, datatype.gid) if final_path != current_file and move: shutil.move(current_file, final_path) return datatype_index
def test_fcd_adapter(self, tmpdir, time_series_region_index_factory, connectivity_factory, region_mapping_factory, surface_factory): storage_folder = str(tmpdir) connectivity = connectivity_factory() surface = surface_factory() region_mapping = region_mapping_factory(surface=surface, connectivity=connectivity) ts_index = time_series_region_index_factory(connectivity=connectivity, region_mapping=region_mapping) fcd_adapter = FunctionalConnectivityDynamicsAdapter() fcd_adapter.storage_path = storage_folder view_model = fcd_adapter.get_view_model_class()() view_model.sw = 0.5 view_model.sp = 0.2 view_model.time_series = ts_index.gid fcd_adapter.configure(view_model) disk = fcd_adapter.get_required_disk_size(view_model) mem = fcd_adapter.get_required_memory_size(view_model) fcd_idx = fcd_adapter.launch(view_model) result_h5 = h5.path_for(storage_folder, FcdH5, fcd_idx[0].gid) assert os.path.exists(result_h5)
def launch(self, view_model): # type: (TrackImporterModel) -> [TractsIndex] datatype = self._base_before_launch(view_model.data_file, view_model.region_volume) tracts_h5 = TractsH5( path_for(self.storage_path, TractsH5, datatype.gid)) tract_start_indices = [0] tract_region = [] for tractf in sorted( self.storage_interface.get_filenames_in_zip( view_model.data_file)): # one track per file if not tractf.endswith( '.txt'): # omit directories and other non track files continue vertices_file = self.storage_interface.open_tvb_zip( view_model.data_file, tractf) datatype.tract_vertices = numpy.loadtxt(vertices_file, dtype=numpy.float32) tract_start_indices.append(tract_start_indices[-1] + len(datatype.tract_vertices)) tracts_h5.write_vertices_slice(datatype.tract_vertices) if view_model.region_volume is not None: tract_region.append( self._get_tract_region(datatype.tract_vertices[0])) vertices_file.close() tracts_h5.close() self.region_volume_h5.close() datatype.tract_start_idx = tract_start_indices datatype.tract_region = numpy.array(tract_region, dtype=numpy.int16) return datatype
def launch(self, view_model): # type: (FFTAdapterModel) -> [FourierSpectrumIndex] """ Launch algorithm and build results. :param time_series: the input time series to which the fft is to be applied :param segment_length: the block size which determines the frequency resolution \ of the resulting power spectra :param window_function: windowing functions can be applied before the FFT is performed :type window_function: None; ‘hamming’; ‘bartlett’; ‘blackman’; ‘hanning’ :returns: the fourier spectrum for the specified time series :rtype: `FourierSpectrumIndex` """ fft_index = FourierSpectrumIndex() fft_index.fk_source_gid = view_model.time_series.hex block_size = int(math.floor(self.input_shape[2] / self.memory_factor)) blocks = int(math.ceil(self.input_shape[2] / block_size)) input_time_series_h5 = h5.h5_file_for_index( self.input_time_series_index) dest_path = h5.path_for(self.storage_path, FourierSpectrumH5, fft_index.gid) spectra_file = FourierSpectrumH5(dest_path) spectra_file.gid.store(uuid.UUID(fft_index.gid)) spectra_file.source.store(uuid.UUID(self.input_time_series_index.gid)) # ------------- NOTE: Assumes 4D, Simulator timeSeries. -------------- node_slice = [ slice(self.input_shape[0]), slice(self.input_shape[1]), None, slice(self.input_shape[3]) ] # ---------- Iterate over slices and compose final result ------------ small_ts = TimeSeries() small_ts.sample_period = input_time_series_h5.sample_period.load() for block in range(blocks): node_slice[2] = slice( block * block_size, min([(block + 1) * block_size, self.input_shape[2]]), 1) small_ts.data = input_time_series_h5.read_data_slice( tuple(node_slice)) self.algorithm.time_series = small_ts partial_result = self.algorithm.evaluate() if blocks <= 1 and len(partial_result.array_data) == 0: self.add_operation_additional_info( "Fourier produced empty result (most probably due to a very short input TimeSeries)." ) return None spectra_file.write_data_slice(partial_result) fft_index.ndim = len(spectra_file.array_data.shape) input_time_series_h5.close() fft_index.windowing_function = self.algorithm.window_function fft_index.segment_length = self.algorithm.segment_length fft_index.detrend = self.algorithm.detrend fft_index.frequency_step = partial_result.freq_step fft_index.max_frequency = partial_result.max_freq spectra_file.segment_length.store(self.algorithm.segment_length) spectra_file.windowing_function.store( str(self.algorithm.window_function)) spectra_file.close() self.log.debug("partial segment_length is %s" % (str(partial_result.segment_length))) return fft_index
def launch(self, view_model): # type: (SimulatorAdapterModel) -> [TimeSeriesIndex, SimulationHistoryIndex] """ Called from the GUI to launch a simulation. *: string class name of chosen model, etc... *_parameters: dictionary of parameters for chosen model, etc... connectivity: tvb.datatypes.connectivity.Connectivity object. surface: tvb.datatypes.surfaces.CorticalSurface: or None. stimulus: tvb.datatypes.patters.* object """ result_h5 = dict() result_indexes = dict() start_time = self.algorithm.current_step * self.algorithm.integrator.dt self.algorithm.configure(full_configure=False) if self.branch_simulation_state_gid is not None: history_index = dao.get_datatype_by_gid( self.branch_simulation_state_gid.hex) history = h5.load_from_index(history_index) assert isinstance(history, SimulationHistory) history.fill_into(self.algorithm) region_map, region_volume_map = self._try_load_region_mapping() for monitor in self.algorithm.monitors: m_name = type(monitor).__name__ ts = monitor.create_time_series(self.algorithm.connectivity, self.algorithm.surface, region_map, region_volume_map) self.log.debug("Monitor created the TS") ts.start_time = start_time ts_index_class = h5.REGISTRY.get_index_for_datatype(type(ts)) ts_index = ts_index_class() ts_index.fill_from_has_traits(ts) ts_index.data_ndim = 4 ts_index.state = 'INTERMEDIATE' state_variable_dimension_name = ts.labels_ordering[1] if m_name in self.HAVE_STATE_VARIABLES: selected_vois = [ self.algorithm.model.variables_of_interest[idx] for idx in monitor.voi ] ts.labels_dimensions[ state_variable_dimension_name] = selected_vois ts_index.labels_dimensions = json.dumps(ts.labels_dimensions) ts_h5_class = h5.REGISTRY.get_h5file_for_datatype(type(ts)) ts_h5_path = h5.path_for(self.storage_path, ts_h5_class, ts.gid) ts_h5 = ts_h5_class(ts_h5_path) ts_h5.store(ts, scalars_only=True, store_references=False) ts_h5.sample_rate.store(ts.sample_rate) ts_h5.nr_dimensions.store(ts_index.data_ndim) if self.algorithm.surface: ts_index.fk_surface_gid = self.algorithm.surface.region_mapping_data.surface.gid.hex ts_h5.surface.store(self.algorithm.surface.gid) else: ts_h5.store_references(ts) result_indexes[m_name] = ts_index result_h5[m_name] = ts_h5 # Run simulation self.log.debug("Starting simulation...") for result in self.algorithm( simulation_length=self.algorithm.simulation_length): for j, monitor in enumerate(self.algorithm.monitors): if result[j] is not None: m_name = type(monitor).__name__ ts_h5 = result_h5[m_name] ts_h5.write_time_slice([result[j][0]]) ts_h5.write_data_slice([result[j][1]]) self.log.debug( "Completed simulation, starting to store simulation state ") # Now store simulator history, at the simulation end results = [] if not self._is_group_launch(): simulation_history = SimulationHistory() simulation_history.populate_from(self.algorithm) history_index = h5.store_complete(simulation_history, self.storage_path) results.append(history_index) self.log.debug("Simulation state persisted, returning results ") for monitor in self.algorithm.monitors: m_name = type(monitor).__name__ ts_shape = result_h5[m_name].read_data_shape() result_indexes[m_name].fill_shape(ts_shape) result_h5[m_name].close() self.log.debug("%s: Adapter simulation finished!!" % str(self)) results.extend(result_indexes.values()) return results
Demo script on how to use tvb-framework default read/write capabilities .. moduleauthor:: Lia Domide <*****@*****.**> """ from tvb.core.neocom import h5 from tvb.basic.profile import TvbProfile from tvb.datatypes.connectivity import Connectivity from tvb.adapters.datatypes.h5.connectivity_h5 import ConnectivityH5 if __name__ == '__main__': TvbProfile.set_profile(TvbProfile.COMMAND_PROFILE) # Read from a ZIP conn_ht = Connectivity.from_file() conn_ht.configure() # Store in a given folder the HasTraits entity PATH = "." h5.store_complete(conn_ht, PATH) # Reproduce the just written file name containing GUID file_name = h5.path_for(PATH, ConnectivityH5, conn_ht.gid) # Load back from a file name a HasTraits instance conn_back = h5.load(file_name) # Check that the loaded and written entities are correct assert conn_ht.number_of_regions == 76 assert conn_ht.number_of_regions == conn_back.number_of_regions
def launch(self, simulator_gid): """ Called from the GUI to launch a simulation. *: string class name of chosen model, etc... *_parameters: dictionary of parameters for chosen model, etc... connectivity: tvb.datatypes.connectivity.Connectivity object. surface: tvb.datatypes.surfaces.CorticalSurface: or None. stimulus: tvb.datatypes.patters.* object """ result_h5 = dict() result_indexes = dict() start_time = self.algorithm.current_step * self.algorithm.integrator.dt self.algorithm.configure(full_configure=False) if self.branch_simulation_state_gid is not None: simulation_state_index = dao.get_datatype_by_gid( self.branch_simulation_state_gid.hex) self.branch_simulation_state_path = h5.path_for_stored_index( simulation_state_index) with SimulationStateH5(self.branch_simulation_state_path ) as branch_simulation_state_h5: branch_simulation_state_h5.load_into(self.algorithm) region_map, region_volume_map = self._try_load_region_mapping() for monitor in self.algorithm.monitors: m_name = monitor.__class__.__name__ ts = monitor.create_time_series(self.algorithm.connectivity, self.algorithm.surface, region_map, region_volume_map) self.log.debug("Monitor created the TS") ts.start_time = start_time ts_index_class = h5.REGISTRY.get_index_for_datatype(type(ts)) ts_index = ts_index_class() ts_index.fill_from_has_traits(ts) ts_index.data_ndim = 4 ts_index.state = 'INTERMEDIATE' # state_variable_dimension_name = ts.labels_ordering[1] # if ts_index.user_tag_1: # ts_index.labels_dimensions[state_variable_dimension_name] = ts.user_tag_1.split(';') # elif m_name in self.HAVE_STATE_VARIABLES: # selected_vois = [self.algorithm.model.variables_of_interest[idx] for idx in monitor.voi] # ts.labels_dimensions[state_variable_dimension_name] = selected_vois ts_h5_class = h5.REGISTRY.get_h5file_for_datatype(type(ts)) ts_h5_path = h5.path_for(self.storage_path, ts_h5_class, ts.gid) ts_h5 = ts_h5_class(ts_h5_path) ts_h5.store(ts, scalars_only=True, store_references=False) ts_h5.sample_rate.store(ts.sample_rate) ts_h5.nr_dimensions.store(ts_index.data_ndim) if self.algorithm.surface: ts_index.surface_gid = self.algorithm.surface.region_mapping_data.surface.gid.hex ts_h5.surface.store(self.algorithm.surface.gid) else: ts_index.connectivity_gid = self.algorithm.connectivity.gid.hex ts_h5.connectivity.store(self.algorithm.connectivity.gid) if region_map: ts_index.region_mapping_gid = region_map.gid.hex ts_h5.region_mapping.store(region_map.gid) if region_volume_map: ts_index.region_mapping_volume_gid = region_volume_map.gid.hex ts_h5.region_mapping_volume.store(region_volume_map.gid) result_indexes[m_name] = ts_index result_h5[m_name] = ts_h5 # Run simulation self.log.debug("Starting simulation...") for result in self.algorithm(simulation_length=self.simulation_length): for j, monitor in enumerate(self.algorithm.monitors): if result[j] is not None: m_name = monitor.__class__.__name__ ts_h5 = result_h5[m_name] ts_h5.write_time_slice([result[j][0]]) ts_h5.write_data_slice([result[j][1]]) self.log.debug( "Completed simulation, starting to store simulation state ") # Populate H5 file for simulator state. This step could also be done while running sim, in background. if not self._is_group_launch(): simulation_state_index = SimulationStateIndex() simulation_state_path = h5.path_for(self.storage_path, SimulationStateH5, self.algorithm.gid) with SimulationStateH5( simulation_state_path) as simulation_state_h5: simulation_state_h5.store(self.algorithm) self._capture_operation_results([simulation_state_index]) self.log.debug("Simulation state persisted, returning results ") for monitor in self.algorithm.monitors: m_name = monitor.__class__.__name__ ts_shape = result_h5[m_name].read_data_shape() result_indexes[m_name].fill_shape(ts_shape) result_h5[m_name].close() # self.log.info("%s: Adapter simulation finished!!" % str(self)) return list(result_indexes.values())
def _update_vm_generic_operation_tag(view_model, operation): project = dao.get_project_by_id(operation.fk_launched_in) h5_path = h5.path_for(operation.id, ViewModelH5, view_model.gid, project.name, type(view_model).__name__) with ViewModelH5(h5_path, view_model) as vm_h5: vm_h5.operation_tag.store(operation.user_group)
def store_burst_configuration(self, burst_config, storage_path): bc_path = h5.path_for(storage_path, BurstConfigurationH5, burst_config.gid) with BurstConfigurationH5(bc_path) as bc_h5: bc_h5.store(burst_config)
def path_for(self, h5_file_class, gid, dt_class=None): project = dao.get_project_by_id(self.current_project_id) return h5.path_for(self.operation_id, h5_file_class, gid, project.name, dt_class)