def configure(self, view_model): # type: (PearsonCorrelationCoefficientAdapterModel) -> None """ Store the input shape to be later used to estimate memory usage. :param time_series: the input time-series index for which correlation coefficient should be computed :param t_start: the physical time interval start for the analysis :param t_end: physical time, interval end """ if view_model.t_start >= view_model.t_end or view_model.t_start < 0: raise LaunchException( "Can not launch operation without monitors selected !!!") self.input_time_series_index = self.load_entity_by_gid( view_model.time_series.hex) self.input_shape = (int((view_model.t_end - view_model.t_start) / self.input_time_series_index.sample_period), self.input_time_series_index.data_length_2d, self.input_time_series_index.data_length_3d, self.input_time_series_index.data_length_4d)
def launch(self, view_model): # type: (ProjectionMatrixImporterModel) -> [ProjectionMatrixIndex] """ Creates ProjectionMatrix entity from uploaded data. :raises LaunchException: when * no projection_file or sensors are specified * the dataset is invalid * number of sensors is different from the one in dataset """ if view_model.projection_file is None: raise LaunchException("Please select MATLAB file which contains data to import") if view_model.sensors is None: raise LaunchException("No sensors selected. Please initiate upload again and select one.") if view_model.surface is None: raise LaunchException("No source selected. Please initiate upload again and select a source.") surface_index = self.load_entity_by_gid(view_model.surface.hex) expected_surface_shape = surface_index.number_of_vertices sensors_index = self.load_entity_by_gid(view_model.sensors.hex) expected_sensors_shape = sensors_index.number_of_sensors self.logger.debug("Reading projection matrix from uploaded file...") if view_model.projection_file.endswith(".mat"): projection_data = self.read_matlab_data(view_model.projection_file, view_model.dataset_name) else: projection_data = self.read_list_data(view_model.projection_file) if projection_data is None or len(projection_data) == 0: raise LaunchException("Invalid (empty) dataset...") if projection_data.shape[0] != expected_sensors_shape: raise LaunchException("Invalid Projection Matrix shape[0]: %d Expected: %d" % (projection_data.shape[0], expected_sensors_shape)) if projection_data.shape[1] != expected_surface_shape: raise LaunchException("Invalid Projection Matrix shape[1]: %d Expected: %d" % (projection_data.shape[1], expected_surface_shape)) projection_matrix_type = determine_projection_type(sensors_index) surface_ht = h5.load_from_index(surface_index) sensors_ht = h5.load_from_index(sensors_index) projection_matrix = ProjectionMatrix(sources=surface_ht, sensors=sensors_ht, projection_type=projection_matrix_type, projection_data=projection_data) return h5.store_complete(projection_matrix, self.storage_path)
def _parse_header(self): """ Reads the ordinals from the header and updates self.permutation """ self.line += 1 try: ordinals = [int(v) for v in self.rows[0]] except ValueError: raise LaunchException("Invalid ordinal in header %s" % self.rows[0]) header_i = list(enumerate(ordinals)) header_i.sort( key=lambda i__ordinal: i__ordinal[1]) # sort by the column ordinal inverse_permutation = [i for i, ordinal_ in header_i] for i in range(len(self.permutation)): self.permutation[inverse_permutation[i]] = i self.rows = self.rows[1:] # consume header
def initiate_operation(self, current_user, project_id, adapter_instance, temporary_storage, method_name=ABCAdapter.LAUNCH_METHOD, visible=True, **kwargs): """ Gets the parameters of the computation from the previous inputs form, and launches a computation (on the cluster or locally). Invoke custom method on an Adapter Instance. Make sure when the operation has finished that the correct results are stored into DB. """ if not isinstance(adapter_instance, ABCAdapter): self.logger.warning("Inconsistent Adapter Class:" + str(adapter_instance.__class__)) raise LaunchException("Developer Exception!!") # Prepare Files parameters files = {} kw2 = copy(kwargs) for i, j in kwargs.iteritems(): if isinstance(j, FieldStorage) or isinstance(j, Part): files[i] = j del kw2[i] temp_files = {} try: for i, j in files.iteritems(): if j.file is None: kw2[i] = None continue uq_name = utils.date2string(datetime.now(), True) + '_' + str(i) # We have to add original file name to end, in case file processing # involves file extension reading file_name = TEMPORARY_PREFIX + uq_name + '_' + j.filename file_name = os.path.join(temporary_storage, file_name) kw2[i] = file_name temp_files[i] = file_name file_obj = open(file_name, 'wb') file_obj.write(j.file.read()) file_obj.close() self.logger.debug("Will store file:" + file_name) kwargs = kw2 except Exception, excep: self._handle_exception(excep, temp_files, "Could not launch operation: invalid input files!")
def launch(self, surface_type, data_file, should_center=False): """ Execute import operations: """ try: surface = make_surface(surface_type) if surface is None: raise ParseException("Could not determine surface type! %s" % surface_type) surface.storage_path = self.storage_path surface.zero_based_triangles = True with open(data_file) as f: obj = ObjSurface(f) if should_center: vertices = center_vertices(obj.vertices) else: vertices = obj.vertices surface.vertices = vertices surface.triangles = obj.triangles if obj.have_normals: self.log.debug("OBJ came with normals included") surface.vertex_normals = obj.normals else: self.log.warning( "OBJ came without normals. We will try to compute them...") validation_result = surface.validate() if validation_result.warnings: self.add_operation_additional_info(validation_result.summary()) return [surface] except ParseException as excep: self.log.exception(excep) raise LaunchException(excep)
def __get_range_values(self, kwargs, ranger_name): """ For the ranger given by ranger_name look in kwargs and return the array with all the possible values. """ if ranger_name not in kwargs: return None if str(kwargs[ranger_name]) not in kwargs: return None range_values = [] try: range_data = json.loads(str(kwargs[str(kwargs[ranger_name])])) except Exception: try: range_data = [x.strip() for x in str(kwargs[str(kwargs[ranger_name])]).split(',') if len(x.strip()) > 0] return range_data except Exception, excep: self.logger.warning("Could not launch operation !") self.logger.exception(excep) raise LaunchException("Could not launch with no data from:" + str(ranger_name))
def create_region_ts(self, data_shape, connectivity): if connectivity.number_of_regions != data_shape[1]: raise LaunchException( "Data has %d channels but the connectivity has %d nodes" % (data_shape[1], connectivity.number_of_regions)) ts_idx = TimeSeriesRegionIndex() ts_idx.fk_connectivity_gid = connectivity.gid region_map_indexes = dao.get_generic_entity(RegionMappingIndex, connectivity.gid, 'fk_connectivity_gid') ts_idx.has_surface_mapping = False if len(region_map_indexes) > 0: ts_idx.fk_region_mapping_gid = region_map_indexes[0].gid ts_idx.has_surface_mapping = True ts_h5_path = self.path_for(TimeSeriesRegionH5, ts_idx.gid) ts_h5 = TimeSeriesRegionH5(ts_h5_path) ts_h5.connectivity.store(uuid.UUID(connectivity.gid)) return TimeSeriesRegion(), ts_idx, ts_h5
def configure(self, simulator_gid): """ Make preparations for the adapter launch. """ self.log.debug("%s: Configuring simulator adapter..." % str(self)) self.algorithm, history_gid = SimulatorService().deserialize_simulator( simulator_gid, self.storage_path) self.branch_simulation_state_gid = history_gid # for monitor in self.algorithm.monitors: # if issubclass(monitor, Projection): # # TODO: add a service that loads a RM with Surface and Connectivity # pass try: self.algorithm.preconfigure() except ValueError as err: raise LaunchException( "Failed to configure simulator due to invalid Input Values. It could be because " "of an incompatibility between different version of TVB code.", err)
def configure(self, view_model): # type: (SimulatorAdapterModel) -> None """ Make preparations for the adapter launch. """ self.log.debug("%s: Configuring simulator adapter..." % str(self)) self.algorithm = self._prepare_simulator_from_view_model(view_model) self.branch_simulation_state_gid = view_model.history_gid # for monitor in self.algorithm.monitors: # if issubclass(monitor, Projection): # # TODO: add a service that loads a RM with Surface and Connectivity # pass try: self.algorithm.preconfigure() except ValueError as err: raise LaunchException( "Failed to configure simulator due to invalid Input Values. It could be because " "of an incompatibility between different version of TVB code.", err)
def launch(self, view_model): # type: (GIFTISurfaceImporterModel) -> [SurfaceIndex] """ Execute import operations: """ parser = GIFTIParser(self.storage_path, self.operation_id) try: surface = parser.parse(view_model.data_file, view_model.data_file_part2, view_model.file_type, should_center=view_model.should_center) surface.compute_triangle_normals() surface.compute_vertex_normals() validation_result = surface.validate() if validation_result.warnings: self.add_operation_additional_info(validation_result.summary()) surface_idx = h5.store_complete(surface, self.storage_path) return [surface_idx] except ParseException as excep: logger = get_logger(__name__) logger.exception(excep) raise LaunchException(excep)
def _ensure_region_mapping_index(self, region_mapping_volume, measure=None): # type: (RegionVolumeMappingIndex, DataTypeMatrix) -> RegionVolumeMappingIndex if region_mapping_volume is None: if measure is not None and hasattr(measure, "fk_connectivity_gid"): region_mapping_volume = dao.get_generic_entity( RegionVolumeMappingIndex, measure.fk_connectivity_gid, 'fk_connectivity_gid') if region_mapping_volume is not None and len( region_mapping_volume): region_mapping_volume = region_mapping_volume[0] else: region_mapping_volume = None else: region_mapping_volume = dao.try_load_last_entity_of_type( self.current_project_id, RegionVolumeMappingIndex) if region_mapping_volume is None: raise LaunchException( 'You should have a compatible volume mapping to launch this viewer' ) return region_mapping_volume
def launch(self, projection_file, surface, sensors, dataset_name=DEFAULT_DATASET_NAME): """ Creates ProjectionMatrix entity from uploaded data. """ if surface is None: raise LaunchException( "No source selected. Please initiate upload again and select a source." ) if isinstance(sensors, SensorsEEG): projection_matrix = ProjectionSurfaceEEG( storage_path=self.storage_path) else: projection_matrix = ProjectionSurfaceMEG( storage_path=self.storage_path) expected_shape = surface.number_of_vertices return self.generic_launch(projection_file, dataset_name, surface, sensors, expected_shape, projection_matrix)
def _decrypt_content(view_model, trait_upload_field_name): if TvbProfile.current.UPLOAD_KEY_PATH is None or not os.path.exists( TvbProfile.current.UPLOAD_KEY_PATH): raise LaunchException( "We can not process Encrypted files at this moment, " "due to missing PK for decryption! Please contact the administrator!" ) upload_path = getattr(view_model, trait_upload_field_name) # Get the encrypted password with open(view_model.encrypted_aes_key, 'rb') as f: encrypted_password = f.read() # Read the private key with open(TvbProfile.current.UPLOAD_KEY_PATH, "rb") as key_file: private_key = serialization.load_pem_private_key( key_file.read(), password=None, backend=default_backend()) # Decrypt the password using the private key decrypted_password = private_key.decrypt( encrypted_password, padding.OAEP(mgf=padding.MGF1(algorithm=hashes.SHA256()), algorithm=hashes.SHA256(), label=None)) decrypted_password = decrypted_password.decode() # Get path to decrypted file decrypted_download_path = upload_path.replace(ENCRYPTED_DATA_SUFFIX, DECRYPTED_DATA_SUFFIX) # Use the decrypted password to decrypt the message pyAesCrypt.decryptFile(upload_path, decrypted_download_path, decrypted_password, TvbProfile.current.hpc.CRYPT_BUFFER_SIZE) view_model.__setattr__(trait_upload_field_name, decrypted_download_path)
def _fit_topology(self, figure, topography, topography_data, color_bar_min, color_bar_max): """ Trim data, to make sure everything is inside the head contour. """ x_arr = topography_data["x_arr"] y_arr = topography_data["y_arr"] circle_x = topography_data["circle_x"] circle_y = topography_data["circle_y"] rad = topography_data["rad"] try: topo = griddata(topography_data["sproj"][:, 0], topography_data["sproj"][:, 1], numpy.ravel(numpy.array(topography)), x_arr, y_arr, interp='linear') except KeyError, err: self.log.exception("Could not execute matplotlib.mlab.griddata...") raise LaunchException( "The measure points location is not compatible with this viewer " "(maybe all on one line)!", err)
def compute_parameters(self, input_data, colors=None, rays=None, step=None): """ Build the required HTML response to be displayed. :raises LaunchException: when number of regions in input_data is less than 3 """ if input_data.number_of_regions <= 3: raise LaunchException('The connectivity matrix you selected has fewer nodes than acceptable for display!') normalized_weights = self._normalize_weights(input_data.ordered_weights) weights = Connectivity2DViewer._get_weights(normalized_weights, input_data.hemispheres) # Compute shapes and colors ad adjacent data norm_rays, min_ray, max_ray = self._normalize_rays(rays, input_data.number_of_regions) colors, step = self._prepare_colors(colors, input_data.number_of_regions, step) if numpy.all((input_data.hemispheres == False)): right_json = "" else: right_json = self._get_json(input_data.ordered_labels[input_data.hemispheres], input_data.ordered_centres[input_data.hemispheres], weights[1], math.pi, 1, 2, numpy.asarray(norm_rays)[input_data.hemispheres], numpy.asarray(colors)[input_data.hemispheres], X_CANVAS_SMALL, Y_CANVAS_SMALL) if numpy.all((input_data.hemispheres == True)): left_json = "" else: left_json = self._get_json(input_data.ordered_labels[~input_data.hemispheres], input_data.ordered_centres[~input_data.hemispheres], weights[0], math.pi, 1, 2, numpy.asarray(norm_rays)[~input_data.hemispheres], numpy.asarray(colors)[~input_data.hemispheres], X_CANVAS_SMALL, Y_CANVAS_SMALL) full_json = self._get_json(input_data.ordered_labels, input_data.ordered_centres, normalized_weights, math.pi, 0, 1, norm_rays, colors, X_CANVAS_FULL, Y_CANVAS_FULL) params = dict(bothHemisphereJson=full_json, rightHemisphereJson=right_json, leftHemisphereJson=left_json, stepValue=step or max_ray, firstColor=self.DEFAULT_COLOR, secondColor=self.OTHER_COLOR, minRay=min_ray, maxRay=max_ray) return params, {}
def _fill_apriori_data(self, operations): """ Gather apriori data from the operations. Also gather the datatype gid's""" for metric in self.metrics: self.apriori_data[metric] = numpy.zeros((self.apriori_x.size, self.apriori_y.size)) # An 2D array of GIDs which is used later to launch overlay for a DataType self.datatypes_gids = [[None for _ in self.range2] for _ in self.range1] for operation in operations: self.log.debug("Gathering data from operation : %s" % operation.id) range_values = eval(operation.range_values) key_1 = range_values[self.range1_name] index_x = self.range1.index(key_1) key_2 = range_values[self.range2_name] index_y = self.range2.index(key_2) if operation.status == STATUS_STARTED: raise LaunchException("Not all operations from this range are complete. Cannot view until then.") operation_results = dao.get_results_for_operation(operation.id) if operation_results: datatype = operation_results[0] self.datatypes_gids[index_x][index_y] = datatype.gid if datatype.type == "DatatypeMeasure": measures = dao.get_generic_entity(DatatypeMeasure, datatype.id) else: measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype') else: self.datatypes_gids[index_x][index_y] = None measures = None for metric in self.metrics: if measures: self.apriori_data[metric][index_x][index_y] = measures[0].metrics[metric] else: self.apriori_data[metric][index_x][index_y] = numpy.NaN
def launch(self, file_type, data_file, data_file_part2, should_center=False): """ Execute import operations: """ parser = GIFTIParser(self.storage_path, self.operation_id) try: surface = parser.parse(data_file, data_file_part2, file_type, should_center=should_center) validation_result = surface.validate() if validation_result.warnings: self.add_operation_additional_info(validation_result.summary()) return [surface] except ParseException, excep: logger = get_logger(__name__) logger.exception(excep) raise LaunchException(excep)
def launch(self, annotations, region_map=None, **kwarg): if region_map is None: region_map = dao.get_generic_entity(RegionMapping, annotations.connectivity.gid, '_connectivity') if len(region_map) < 1: raise LaunchException( "Can not launch this viewer unless we have at least a RegionMapping for the current Connectivity!") region_map = region_map[0] boundary_url = region_map.surface.get_url_for_region_boundaries(region_map) url_vertices_pick, url_normals_pick, url_triangles_pick = region_map.surface.get_urls_for_pick_rendering() url_vertices, url_normals, _, url_triangles, url_region_map = \ region_map.surface.get_urls_for_rendering(True, region_map) params = dict(title="Connectivity Annotations Visualizer", baseUrl=TvbProfile.current.web.BASE_URL, annotationsTreeUrl=self.paths2url(annotations, 'tree_json'), urlTriangleToRegion=self.paths2url(region_map, "get_triangles_mapping"), urlActivationPatterns=self.paths2url(annotations, "get_activation_patterns"), minValue=0, maxValue=annotations.connectivity.number_of_regions - 1, urlColors=json.dumps(url_region_map), urlVerticesPick=json.dumps(url_vertices_pick), urlTrianglesPick=json.dumps(url_triangles_pick), urlNormalsPick=json.dumps(url_normals_pick), brainCenter=json.dumps(region_map.surface.center()), urlVertices=json.dumps(url_vertices), urlTriangles=json.dumps(url_triangles), urlNormals=json.dumps(url_normals), urlRegionBoundaries=boundary_url) return self.build_display_result("annotations/annotations_view", params, pages={"controlPage": "annotations/annotations_controls"})
def launch(self, projection_file, surface, sensors, dataset_name=DEFAULT_DATASET_NAME): """ Creates ProjectionMatrix entity from uploaded data. :raises LaunchException: when * no projection_file or sensors are specified * the dataset is invalid * number of sensors is different from the one in dataset """ if projection_file is None: raise LaunchException("Please select MATLAB file which contains data to import") if sensors is None: raise LaunchException("No sensors selected. Please initiate upload again and select one.") if surface is None: raise LaunchException("No source selected. Please initiate upload again and select a source.") expected_shape = surface.number_of_vertices self.logger.debug("Reading projection matrix from uploaded file...") if projection_file.endswith(".mat"): eeg_projection_data = self.read_matlab_data(projection_file, dataset_name) else: eeg_projection_data = self.read_list_data(projection_file) if eeg_projection_data is None or len(eeg_projection_data) == 0: raise LaunchException("Invalid (empty) dataset...") if eeg_projection_data.shape[0] != sensors.number_of_sensors: raise LaunchException("Invalid Projection Matrix shape[0]: %d Expected: %d" % (eeg_projection_data.shape[0], sensors.number_of_sensors)) if eeg_projection_data.shape[1] != expected_shape: raise LaunchException("Invalid Projection Matrix shape[1]: %d Expected: %d" % (eeg_projection_data.shape[1], expected_shape)) self.logger.debug("Creating Projection Matrix instance") projection_matrix = build_projection_instance(sensors, self.storage_path) projection_matrix.sources = surface projection_matrix.sensors = sensors if eeg_projection_data is not None: projection_matrix.projection_data = eeg_projection_data return [projection_matrix]
def prepare_operations(self, user_id, project_id, algorithm, category, metadata, visible=True, **kwargs): """ Do all the necessary preparations for storing an operation. If it's the case of a range of values create an operation group and multiple operations for each possible instance from the range. :param metadata: Initial MetaData with potential Burst identification inside. """ operations = [] available_args, group = self._prepare_group(project_id, kwargs) if len(available_args) > TvbProfile.current.MAX_RANGE_NUMBER: raise LaunchException( "Too big range specified. You should limit the" " resulting operations to %d" % TvbProfile.current.MAX_RANGE_NUMBER) else: self.logger.debug("Launching a range with %d operations..." % len(available_args)) group_id = None if group is not None: group_id = group.id metadata, user_group = self._prepare_metadata(metadata, category, group, kwargs) self.logger.debug("Saving Operation(userId=" + str(user_id) + ",projectId=" + str(project_id) + "," + str(metadata) + ",algorithmId=" + str(algorithm.id) + ", ops_group= " + str(group_id) + ")") visible_operation = visible and category.display is False meta_str = json.dumps(metadata) for (one_set_of_args, range_vals) in available_args: range_values = json.dumps(range_vals) if range_vals else None operation = model.Operation(user_id, project_id, algorithm.id, json.dumps( one_set_of_args, cls=MapAsJson.MapAsJsonEncoder), meta_str, op_group_id=group_id, user_group=user_group, range_values=range_values) operation.visible = visible_operation operations.append(operation) operations = dao.store_entities(operations) if group is not None: burst_id = None if DataTypeMetaData.KEY_BURST in metadata: burst_id = metadata[DataTypeMetaData.KEY_BURST] datatype_group = model.DataTypeGroup( group, operation_id=operations[0].id, fk_parent_burst=burst_id, state=metadata[DataTypeMetaData.KEY_STATE]) dao.store_entity(datatype_group) return operations, group
def create_eeg_ts(self, data, sensors): if sensors.number_of_sensors != data.shape[1]: raise LaunchException( "Data has %d channels but the sensors have %d" % (data.shape[1], sensors.number_of_sensors)) return TimeSeriesEEG(storage_path=self.storage_path, sensors=sensors)
def launch(self, view_model): # type: (TopographicViewerModel) -> dict connectivities_idx = [] measures_ht = [] for measure in [ view_model.data_0, view_model.data_1, view_model.data_2 ]: if measure is not None: measure_index = self.load_entity_by_gid(measure) measures_ht.append(h5.load_from_index(measure_index)) conn_index = self.load_entity_by_gid( measure_index.fk_connectivity_gid) connectivities_idx.append(conn_index) with h5.h5_file_for_index(connectivities_idx[0]) as conn_h5: centres = conn_h5.centres.load() sensor_locations = TopographyCalculations.normalize_sensors(centres) sensor_number = len(sensor_locations) arrays = [] titles = [] min_vals = [] max_vals = [] data_array = [] data_arrays = [] for i, measure in enumerate(measures_ht): if connectivities_idx[i].number_of_regions != sensor_number: raise Exception("Use the same connectivity!!!") arrays.append(measure.array_data.tolist()) titles.append(measure.title) min_vals.append(measure.array_data.min()) max_vals.append(measure.array_data.max()) color_bar_min = min(min_vals) color_bar_max = max(max_vals) for i, array_data in enumerate(arrays): try: data_array = TopographyCalculations.compute_topography_data( array_data, sensor_locations) # We always access the first element because only one connectivity can be used at one time first_label = h5.load_from_index( connectivities_idx[0]).hemispheres[0] if first_label: data_array = numpy.rot90(data_array, k=1, axes=(0, 1)) else: data_array = numpy.rot90(data_array, k=-1, axes=(0, 1)) if numpy.any(numpy.isnan(array_data)): titles[i] = titles[i] + " - Topography contains nan" if not numpy.any(array_data): titles[i] = titles[i] + " - Topography data is all zeros" data_arrays.append( ABCDisplayer.dump_with_precision(data_array.flat)) except KeyError as err: self.log.exception(err) raise LaunchException( "The measure points location is not compatible with this viewer ", err) params = dict(matrix_datas=data_arrays, matrix_shape=json.dumps(data_array.squeeze().shape), titles=titles, vmin=color_bar_min, vmax=color_bar_max) return self.build_display_result( "topographic/view", params, pages={"controlPage": "topographic/controls"})
def launch(self, uploaded, surface_type, zero_based_triangles=False, should_center=False): """ Execute import operations: unpack ZIP and build Surface object as result. :param uploaded: an archive containing the Surface data to be imported :param surface_type: a string from the following\: \ "Skin Air", "Skull Skin", "Brain Skull", "Cortical Surface", "EEG Cap", "Face" :returns: a subclass of `Surface` DataType :raises LaunchException: when * `uploaded` is missing * `surface_type` is invalid :raises RuntimeError: when triangles contain an invalid vertex index """ if uploaded is None: raise LaunchException( "Please select ZIP file which contains data to import") self.logger.debug("Start to import surface: '%s' from file: %s" % (surface_type, uploaded)) try: zip_surface = ZipSurfaceParser(uploaded) except IOError: exception_str = "Did not find the specified ZIP at %s" % uploaded raise LaunchException(exception_str) # Detect and instantiate correct surface type self.logger.debug("Create surface instance") surface = self._make_surface(surface_type) surface.storage_path = self.storage_path surface.zero_based_triangles = zero_based_triangles if should_center: vertices = center_vertices(zip_surface.vertices) else: vertices = zip_surface.vertices surface.vertices = vertices if len(zip_surface.normals) != 0: surface.vertex_normals = zip_surface.normals if zero_based_triangles: surface.triangles = zip_surface.triangles else: surface.triangles = zip_surface.triangles - 1 if zip_surface.bi_hemispheric: self.logger.info("Hemispheres detected") surface.hemisphere_mask = zip_surface.hemisphere_mask surface.triangle_normals = None # Now check if the triangles of the surface are valid triangles_min_vertex = numpy.amin(surface.triangles) if triangles_min_vertex < 0: if triangles_min_vertex == -1 and not zero_based_triangles: raise LaunchException( "Triangles contain a negative vertex index. Maybe you have a ZERO based surface." ) else: raise LaunchException( "Your triangles contain a negative vertex index: %d" % triangles_min_vertex) no_of_vertices = len(surface.vertices) triangles_max_vertex = numpy.amax(surface.triangles) if triangles_max_vertex >= no_of_vertices: if triangles_max_vertex == no_of_vertices and zero_based_triangles: raise LaunchException( "Your triangles contain an invalid vertex index: %d. " "Maybe your surface is NOT ZERO Based." % triangles_max_vertex) else: raise LaunchException( "Your triangles contain an invalid vertex index: %d." % triangles_max_vertex) validation_result = surface.validate() if validation_result.warnings: self.add_operation_additional_info(validation_result.summary()) self.logger.debug("Surface ready to be stored") return surface
def launch(self, view_model): # type: (RegionMappingImporterModel) -> [RegionMappingIndex] """ Creates region mapping from uploaded data. :raises LaunchException: when * a parameter is None or missing * archive has more than one file * uploaded files are empty * number of vertices in imported file is different to the number of surface vertices * imported file has negative values * imported file has regions which are not in connectivity """ if view_model.mapping_file is None: raise LaunchException( "Please select mappings file which contains data to import") if view_model.surface is None: raise LaunchException( "No surface selected. Please initiate upload again and select a brain surface." ) if view_model.connectivity is None: raise LaunchException( "No connectivity selected. Please initiate upload again and select one." ) self.logger.debug("Reading mappings from uploaded file") if zipfile.is_zipfile(view_model.mapping_file): tmp_folder = tempfile.mkdtemp( prefix='region_mapping_zip_', dir=TvbProfile.current.TVB_TEMP_FOLDER) try: files = FilesHelper().unpack_zip(view_model.mapping_file, tmp_folder) if len(files) > 1: raise LaunchException( "Please upload a ZIP file containing only one file.") array_data = self.read_list_data(files[0], dtype=numpy.int32) finally: if os.path.exists(tmp_folder): shutil.rmtree(tmp_folder) else: array_data = self.read_list_data(view_model.mapping_file, dtype=numpy.int32) # Now we do some checks before building final RegionMapping if array_data is None or len(array_data) == 0: raise LaunchException( "Uploaded file does not contains any data. Please initiate upload with another file." ) # Check if we have a mapping for each surface vertex. surface_index = self.load_entity_by_gid(view_model.surface) if len(array_data) != surface_index.number_of_vertices: msg = "Imported file contains a different number of values than the number of surface vertices. " \ "Imported: %d values while surface has: %d vertices." % ( len(array_data), surface_index.number_of_vertices) raise LaunchException(msg) # Now check if the values from imported file correspond to connectivity regions if array_data.min() < 0: raise LaunchException( "Imported file contains negative values. Please fix problem and re-import file" ) connectivity_index = self.load_entity_by_gid(view_model.connectivity) if array_data.max() >= connectivity_index.number_of_regions: msg = "Imported file contains invalid regions. Found region: %d while selected connectivity has: %d " \ "regions defined (0 based)." % (array_data.max(), connectivity_index.number_of_regions) raise LaunchException(msg) self.logger.debug("Creating RegionMapping instance") connectivity_ht = h5.load_from_index(connectivity_index) surface_ht = h5.load_from_index(surface_index) region_mapping = RegionMapping(surface=surface_ht, connectivity=connectivity_ht, array_data=array_data) return h5.store_complete(region_mapping, self.storage_path)
def plot(self, figure, operation_group, metric, range1_name, range2_name): """ Do the plot for the given figure. Also need operation group, metric and ranges in order to compute the data to be plotted. """ operations = dao.get_operations_in_group(operation_group.id) # Data from which to interpolate larger 2-D space apriori_x = numpy.array(self.range1) apriori_y = numpy.array(self.range2) apriori_data = numpy.zeros((apriori_x.size, apriori_y.size)) # An 2D array of GIDs which is used later to launch overlay for a DataType datatypes_gids = [[None for _ in self.range2] for _ in self.range1] for operation_ in operations: range_values = eval(operation_.range_values) key_1 = range_values[range1_name] index_x = self.range1.index(key_1) key_2 = range_values[range2_name] index_y = self.range2.index(key_2) if operation_.status != model.STATUS_FINISHED: raise LaunchException("Not all operations from this range are complete. Cannot view until then.") datatype = dao.get_results_for_operation(operation_.id)[0] datatypes_gids[index_x][index_y] = datatype.gid if datatype.type == "DatatypeMeasure": measures = dao.get_generic_entity(DatatypeMeasure, datatype.id) else: measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype') if measures: apriori_data[index_x][index_y] = measures[0].metrics[metric] else: apriori_data[index_x][index_y] = 0 # Attempt order-3 interpolation. kx = ky = 3 if len(self.range1) <= 3 or len(self.range2) <= 3: # Number of points is too small, just do linear interpolation kx = ky = 1 s = interpolate.RectBivariateSpline(apriori_x, apriori_y, apriori_data, kx=kx, ky=ky) # Get data of higher resolution that we'll plot later on posteriori_x = numpy.arange(self.range1[0], self.range1[-1], (self.range1[-1] - self.range1[0]) / RESOLUTION[0]) posteriori_y = numpy.arange(self.range2[0], self.range2[-1], (self.range2[-1] - self.range2[0]) / RESOLUTION[1]) posteriori_data = numpy.rot90(s(posteriori_x, posteriori_y)) self.interp_models[figure.number] = s # Do actual plot. axes = figure.gca() img = axes.imshow(posteriori_data, extent=(min(self.range1), max(self.range1), min(self.range2), max(self.range2)), aspect='auto', interpolation='nearest') axes.set_title("Interpolated values for metric %s" % (metric,)) figure.colorbar(img) axes.set_xlabel(range1_name) axes.set_ylabel(range2_name) def format_coord(x, y): return 'x=%1.4f, y=%1.4f' % (x, y) axes.format_coord = format_coord return datatypes_gids
def launch(self, view_model): # type: (ZIPSurfaceImporterModel) -> [SurfaceIndex] """ Execute import operations: unpack ZIP and build Surface object as result :raises LaunchException: when * `uploaded` is missing * `surface_type` is invalid :raises RuntimeError: when triangles contain an invalid vertex index """ if view_model.uploaded is None: raise LaunchException( "Please select ZIP file which contains data to import") self.logger.debug("Start to import surface: '%s' from file: %s" % (view_model.surface_type, view_model.uploaded)) try: zip_surface = ZipSurfaceParser(view_model.uploaded) except IOError: exception_str = "Did not find the specified ZIP at %s" % view_model.uploaded raise LaunchException(exception_str) # Detect and instantiate correct surface type self.logger.debug("Create surface instance") surface = self._make_surface(view_model.surface_type) surface.zero_based_triangles = view_model.zero_based_triangles if view_model.should_center: vertices = center_vertices(zip_surface.vertices) else: vertices = zip_surface.vertices surface.vertices = vertices if len(zip_surface.normals) != 0: surface.vertex_normals = zip_surface.normals if view_model.zero_based_triangles: surface.triangles = zip_surface.triangles else: surface.triangles = zip_surface.triangles - 1 if zip_surface.bi_hemispheric: self.logger.info("Hemispheres detected") surface.hemisphere_mask = zip_surface.hemisphere_mask surface.compute_triangle_normals() # Now check if the triangles of the surface are valid triangles_min_vertex = numpy.amin(surface.triangles) if triangles_min_vertex < 0: if triangles_min_vertex == -1 and not view_model.zero_based_triangles: raise LaunchException( "Triangles contain a negative vertex index. Maybe you have a ZERO based surface." ) else: raise LaunchException( "Your triangles contain a negative vertex index: %d" % triangles_min_vertex) no_of_vertices = len(surface.vertices) triangles_max_vertex = numpy.amax(surface.triangles) if triangles_max_vertex >= no_of_vertices: if triangles_max_vertex == no_of_vertices and view_model.zero_based_triangles: raise LaunchException( "Your triangles contain an invalid vertex index: %d. " "Maybe your surface is NOT ZERO Based." % triangles_max_vertex) else: raise LaunchException( "Your triangles contain an invalid vertex index: %d." % triangles_max_vertex) validation_result = surface.validate() if validation_result.warnings: self.add_operation_additional_info(validation_result.summary()) surface.configure() self.logger.debug("Surface ready to be stored") surf_idx = h5.store_complete(surface, self.storage_path) self.generic_attributes.user_tag_1 = surface.surface_type return surf_idx
def generate_preview(self, **kwargs): """ Should be implemented by all visualizers that can be used by portlets. """ raise LaunchException("%s used as Portlet but doesn't implement 'generate_preview'" % self.__class__)
def configure(self, model, model_parameters, integrator, integrator_parameters, connectivity, monitors, monitors_parameters=None, surface=None, surface_parameters=None, stimulus=None, coupling=None, coupling_parameters=None, initial_conditions=None, conduction_speed=None, simulation_length=0, simulation_state=None): """ Make preparations for the adapter launch. """ self.log.debug("available_couplings: %s..." % str(self.available_couplings)) self.log.debug("coupling: %s..." % str(coupling)) self.log.debug("coupling_parameters: %s..." % str(coupling_parameters)) self.log.debug("%s: Initializing Model..." % str(self)) noise_framework.build_noise(model_parameters) model_instance = self.available_models[str(model)](**model_parameters) self._validate_model_parameters(model_instance, connectivity, surface) self.log.debug("%s: Initializing Integration scheme..." % str(self)) noise_framework.build_noise(integrator_parameters) integr = self.available_integrators[integrator]( **integrator_parameters) self.log.debug("%s: Instantiating Monitors..." % str(self)) monitors_list = [] for monitor_name in monitors: if (monitors_parameters is not None) and (str(monitor_name) in monitors_parameters): current_monitor_parameters = monitors_parameters[str( monitor_name)] HRFKernelEquation.build_equation_from_dict( 'hrf_kernel', current_monitor_parameters, True) monitors_list.append( self.available_monitors[str(monitor_name)]( **current_monitor_parameters)) else: ### We have monitors without any UI settable parameter. monitors_list.append( self.available_monitors[str(monitor_name)]()) if len(monitors) < 1: raise LaunchException( "Can not launch operation without monitors selected !!!") self.log.debug("%s: Initializing Coupling..." % str(self)) coupling_inst = self.available_couplings[str(coupling)]( **coupling_parameters) self.log.debug("Initializing Cortex...") if self._is_surface_simulation(surface, surface_parameters): cortex_entity = Cortex(use_storage=False).populate_cortex( surface, surface_parameters) if cortex_entity.region_mapping_data.connectivity.number_of_regions != connectivity.number_of_regions: raise LaunchException( "Incompatible RegionMapping -- Connectivity !!") if cortex_entity.region_mapping_data.surface.number_of_vertices != surface.number_of_vertices: raise LaunchException( "Incompatible RegionMapping -- Surface !!") select_loc_conn = cortex_entity.local_connectivity if select_loc_conn is not None and select_loc_conn.surface.number_of_vertices != surface.number_of_vertices: raise LaunchException( "Incompatible LocalConnectivity -- Surface !!") else: cortex_entity = None self.log.debug("%s: Instantiating requested simulator..." % str(self)) if conduction_speed not in (0.0, None): connectivity.speed = numpy.array([conduction_speed]) else: raise LaunchException("conduction speed cannot be 0 or missing") self.algorithm = Simulator(connectivity=connectivity, coupling=coupling_inst, surface=cortex_entity, stimulus=stimulus, model=model_instance, integrator=integr, monitors=monitors_list, initial_conditions=initial_conditions, conduction_speed=conduction_speed) self.simulation_length = simulation_length self.log.debug("%s: Initializing storage..." % str(self)) try: self.algorithm.preconfigure() except ValueError as err: raise LaunchException( "Failed to configure simulator due to invalid Input Values. It could be because " "of an incompatibility between different version of TVB code.", err)
class OperationService: """ Class responsible for preparing an operation launch. It will prepare parameters, and decide if the operation is to be executed immediately, or to be sent on the cluster. """ ATT_UID = "uid" def __init__(self): self.logger = get_logger(self.__class__.__module__) self.workflow_service = WorkflowService() self.file_helper = FilesHelper() ########################################################################################## ######## Methods related to launching operations start here ############################## ########################################################################################## def initiate_operation(self, current_user, project_id, adapter_instance, temporary_storage, visible=True, **kwargs): """ Gets the parameters of the computation from the previous inputs form, and launches a computation (on the cluster or locally). Invoke custom method on an Adapter Instance. Make sure when the operation has finished that the correct results are stored into DB. """ if not isinstance(adapter_instance, ABCAdapter): self.logger.warning("Inconsistent Adapter Class:" + str(adapter_instance.__class__)) raise LaunchException("Developer Exception!!") # Prepare Files parameters files = {} kw2 = copy(kwargs) for i, j in kwargs.iteritems(): if isinstance(j, FieldStorage) or isinstance(j, Part): files[i] = j del kw2[i] temp_files = {} try: for i, j in files.iteritems(): if j.file is None: kw2[i] = None continue uq_name = utils.date2string(datetime.now(), True) + '_' + str(i) # We have to add original file name to end, in case file processing # involves file extension reading file_name = TEMPORARY_PREFIX + uq_name + '_' + j.filename file_name = os.path.join(temporary_storage, file_name) kw2[i] = file_name temp_files[i] = file_name with open(file_name, 'wb') as file_obj: file_obj.write(j.file.read()) self.logger.debug("Will store file:" + file_name) kwargs = kw2 except Exception, excep: self._handle_exception( excep, temp_files, "Could not launch operation: invalid input files!") ### Store Operation entity. algo = adapter_instance.stored_adapter algo_category = dao.get_category_by_id(algo.fk_category) operations = self.prepare_operations(current_user.id, project_id, algo, algo_category, {}, visible, **kwargs)[0] if isinstance(adapter_instance, ABCSynchronous): if len(operations) > 1: raise LaunchException( "Synchronous operations are not supporting ranges!") if len(operations) < 1: self.logger.warning("No operation was defined") raise LaunchException("Invalid empty Operation!!!") return self.initiate_prelaunch(operations[0], adapter_instance, temp_files, **kwargs) else: return self._send_to_cluster(operations, adapter_instance, current_user.username)
def launch(self, view_model): # type: (ConnectivityAnnotationsViewModel) -> dict annotations_index = self.load_entity_by_gid( view_model.annotations_index) if view_model.connectivity_index is None: connectivity_index = self.load_entity_by_gid( annotations_index.fk_connectivity_gid) else: connectivity_index = self.load_entity_by_gid( view_model.connectivity_index) if view_model.region_mapping_index is None: region_map = dao.get_generic_entity(RegionMappingIndex, connectivity_index.gid, 'fk_connectivity_gid') if len(region_map) < 1: raise LaunchException( "Can not launch this viewer unless we have at least a RegionMapping for the current Connectivity!" ) region_mapping_index = region_map[0] else: region_mapping_index = self.load_entity_by_gid( view_model.region_mapping_index) boundary_url = SurfaceURLGenerator.get_url_for_region_boundaries( region_mapping_index.fk_surface_gid, region_mapping_index.gid, self.stored_adapter.id) surface_index = self.load_entity_by_gid( region_mapping_index.fk_surface_gid) surface_h5 = h5.h5_file_for_index(surface_index) assert isinstance(surface_h5, SurfaceH5) url_vertices_pick, url_normals_pick, url_triangles_pick = SurfaceURLGenerator.get_urls_for_pick_rendering( surface_h5) url_vertices, url_normals, _, url_triangles, url_region_map = SurfaceURLGenerator.get_urls_for_rendering( surface_h5, region_mapping_index.gid) params = dict(title="Connectivity Annotations Visualizer", annotationsTreeUrl=URLGenerator.build_url( self.stored_adapter.id, 'tree_json', view_model.annotations_index), urlTriangleToRegion=URLGenerator.build_url( self.stored_adapter.id, "get_triangles_mapping", region_mapping_index.gid), urlActivationPatterns=URLGenerator.paths2url( view_model.annotations_index, "get_activation_patterns"), minValue=0, maxValue=connectivity_index.number_of_regions - 1, urlColors=json.dumps(url_region_map), urlVerticesPick=json.dumps(url_vertices_pick), urlTrianglesPick=json.dumps(url_triangles_pick), urlNormalsPick=json.dumps(url_normals_pick), brainCenter=json.dumps(surface_h5.center()), urlVertices=json.dumps(url_vertices), urlTriangles=json.dumps(url_triangles), urlNormals=json.dumps(url_normals), urlRegionBoundaries=boundary_url) return self.build_display_result( "annotations/annotations_view", params, pages={"controlPage": "annotations/annotations_controls"})