class SimulatorH5(SimulatorConfigurationH5): def __init__(self, path): super(SimulatorH5, self).__init__(path) self.connectivity = Reference(Simulator.connectivity, self) self.conduction_speed = Scalar(Simulator.conduction_speed, self) self.coupling = Reference(Simulator.coupling, self) self.surface = Reference(Simulator.surface, self) self.stimulus = Reference(Simulator.stimulus, self) self.model = Reference(Simulator.model, self) self.integrator = Reference(Simulator.integrator, self) self.initial_conditions = DataSet(Simulator.initial_conditions, self) self.monitors = Json(Simulator.monitors, self) self.simulation_length = Scalar(Simulator.simulation_length, self) self.simulation_state = Reference(Attr(field_type=uuid.UUID), self, name='simulation_state') def store(self, datatype, scalars_only=False, store_references=False): # type: (Simulator, bool, bool) -> None self.gid.store(datatype.gid) self.connectivity.store(datatype.connectivity) self.conduction_speed.store(datatype.conduction_speed) self.initial_conditions.store(datatype.initial_conditions) self.simulation_length.store(datatype.simulation_length) integrator_gid = self.store_config_as_reference(datatype.integrator) self.integrator.store(integrator_gid) coupling_gid = self.store_config_as_reference(datatype.coupling) self.coupling.store(coupling_gid) model_gid = self.store_config_as_reference(datatype.model) self.model.store(model_gid) monitor_gids = [] for monitor in datatype.monitors: monitor_gid = self.store_config_as_reference(monitor).hex monitor_gids.append(monitor_gid) self.monitors.store(monitor_gids) if datatype.surface: cortex_gid = self.store_config_as_reference(datatype.surface) self.surface.store(cortex_gid) if datatype.stimulus: self.stimulus.store(datatype.stimulus) self.type.store(self.get_full_class_name(type(datatype))) def load_into(self, datatype): # type: (Simulator) -> None datatype.conduction_speed = self.conduction_speed.load() datatype.initial_conditions = self.initial_conditions.load() datatype.simulation_length = self.simulation_length.load() datatype.integrator = self.load_from_reference(self.integrator.load()) datatype.coupling = self.load_from_reference(self.coupling.load()) datatype.model = self.load_from_reference(self.model.load()) monitors = [] for monitor in self.monitors.load(): monitors.append(self.load_from_reference(monitor)) datatype.monitors = monitors if self.surface.load(): datatype.surface = self.load_from_reference(self.surface.load())
def __init__(self, path): super(CosineH5, self).__init__(path) self.equation = Scalar(Cosine.equation, self) self.parameters = Json(Cosine.parameters, self)
def __init__(self, path): super(ValueWrapperH5, self).__init__(path) self.data_value = Scalar(Attr(str), self, name='data_value') self.data_type = Scalar(Attr(str), self, name='data_type') self.data_name = Scalar(Attr(str), self, name='data_name')
def __init__(self, path): super(LinearH5, self).__init__(path) self.equation = Scalar(Linear.equation, self) self.parameters = Json(Linear.parameters, self)
def __init__(self, path): super(GeneralizedSigmoidH5, self).__init__(path) self.equation = Scalar(GeneralizedSigmoid.equation, self) self.parameters = Json(GeneralizedSigmoid.parameters, self)
def __init__(self, path): super(DoubleExponentialH5, self).__init__(path) self.equation = Scalar(DoubleExponential.equation, self) self.parameters = Json(DoubleExponential.parameters, self)
def __init__(self, path): super(MixtureOfGammasH5, self).__init__(path) self.equation = Scalar(MixtureOfGammas.equation, self) self.parameters = Json(MixtureOfGammas.parameters, self)
class BurstConfigurationH5(H5File): def __init__(self, path): super(BurstConfigurationH5, self).__init__(path) self.name = Scalar(Attr(str), self, name='name') self.status = Scalar(Attr(str), self, name='status') self.error_message = Scalar(Attr(str, required=False), self, name='error_message') self.start_time = Scalar(Attr(str), self, name='start_time') self.finish_time = Scalar(Attr(str, required=False), self, name='finish_time') self.simulator = Reference(Attr(uuid.UUID), self, name='simulator') self.range1 = Scalar(Attr(str, required=False), self, name='range1') self.range2 = Scalar(Attr(str, required=False), self, name='range2') def store(self, burst_config, scalars_only=False, store_references=True): # type (BurstConfiguration, bool, bool) -> None self.gid.store(uuid.UUID(burst_config.gid)) self.name.store(burst_config.name) self.status.store(burst_config.status) self.error_message.store(burst_config.error_message or 'None') self.start_time.store(date2string(burst_config.start_time)) self.finish_time.store(date2string(burst_config.finish_time)) self.simulator.store(uuid.UUID(burst_config.simulator_gid)) self.range1.store(burst_config.range1) self.range2.store(burst_config.range2) def load_into(self, burst_config): # type (BurstConfiguration) -> None burst_config.gid = self.gid.load().hex burst_config.name = self.name.load() burst_config.status = self.status.load() burst_config.error_message = self.error_message.load() burst_config.start_time = string2date(self.start_time.load()) finish_time = self.finish_time.load() if finish_time and finish_time != 'None': burst_config.finish_time = string2date(finish_time) burst_config.simulator_gid = self.simulator.load().hex try: burst_config.range1 = self.range1.load() except MissingDataSetException: burst_config.range1 = None try: burst_config.range2 = self.range2.load() except MissingDataSetException: burst_config.range2 = None
def __init__(self, path): super(BurstConfigurationH5, self).__init__(path) self.name = Scalar(Attr(str), self, name='name') self.status = Scalar(Attr(str), self, name='status') self.error_message = Scalar(Attr(str, required=False), self, name='error_message') self.start_time = Scalar(Attr(str), self, name='start_time') self.finish_time = Scalar(Attr(str, required=False), self, name='finish_time') self.simulator = Reference(Attr(uuid.UUID), self, name='simulator') self.range1 = Scalar(Attr(str, required=False), self, name='range1') self.range2 = Scalar(Attr(str, required=False), self, name='range2')
def __init__(self, path): super(StructuralMRIH5, self).__init__(path) self.array_data = DataSet(StructuralMRI.array_data, self) self.weighting = Scalar(StructuralMRI.weighting, self) self.volume = Reference(StructuralMRI.volume, self)
class TimeSeriesH5(H5File): def __init__(self, path): super(TimeSeriesH5, self).__init__(path) self.title = Scalar(TimeSeries.title) self.data = DataSet(TimeSeries.data, expand_dimension=0) self.nr_dimensions = Scalar(TimeSeries.nr_dimensions) # omitted length_nd , these are indexing props, to be removed from datatype too self.labels_ordering = Json(TimeSeries.labels_ordering) self.labels_dimensions = Json(TimeSeries.labels_dimensions) self.time = DataSet(TimeSeries.time, expand_dimension=0) self.start_time = Scalar(TimeSeries.start_time) self.sample_period = Scalar(TimeSeries.sample_period) self.sample_period_unit = Scalar(TimeSeries.sample_period_unit) self.sample_rate = Scalar(TimeSeries.sample_rate) self._end_accessor_declarations() # omitted has_surface_mapping, has_volume_mapping, indexing props, to be removed fro datatype too # experiment: load header data eagerly, see surface for a lazy approach # as we do not explicitly make a difference between opening for read or write # the file might not yet exist, so loading headers makes no sense if self.storage_manager.is_valid_hdf5_file(): self._sample_period = self.sample_period.load() self._start_time = self.start_time.load() # experimental port of some of the data access apis from the datatype # NOTE: some methods can not be here as they load data from dependent data types # or they assume that dependent data has been loaded # Those belong to a higher level where dependent h5 files are handles and # partially loaded datatypes are filled def read_data_shape(self): return self.data.shape def read_data_slice(self, data_slice): """ Expose chunked-data access. """ return self.data[data_slice] def read_time_page(self, current_page, page_size, max_size=None): """ Compute time for current page. :param current_page: Starting from 0 """ # todo: why are we even storing the time array if we return a synthetized version? current_page = int(current_page) page_size = int(page_size) if max_size is None: max_size = page_size else: max_size = int(max_size) page_real_size = page_size * self._sample_period start_time = self._start_time + current_page * page_real_size end_time = start_time + min(page_real_size, max_size * self._sample_period) return numpy.arange(start_time, end_time, self._sample_period) def read_channels_page(self, from_idx, to_idx, step=None, specific_slices=None, channels_list=None): """ Read and return only the data page for the specified channels list. :param from_idx: the starting time idx from which to read data :param to_idx: the end time idx up until to which you read data :param step: increments in which to read the data. Optional, default to 1. :param specific_slices: optional parameter. If speficied slices the data accordingly. :param channels_list: the list of channels for which we want data """ if channels_list: channels_list = json.loads(channels_list) for i in range(len(channels_list)): channels_list[i] = int(channels_list[i]) if channels_list: channel_slice = tuple(channels_list) else: channel_slice = slice(None) data_page = self.read_data_page(from_idx, to_idx, step, specific_slices) # This is just a 1D array like in the case of Global Average monitor. # No need for the channels list if len(data_page.shape) == 1: return data_page.reshape(data_page.shape[0], 1) else: return data_page[:, channel_slice] def read_data_page(self, from_idx, to_idx, step=None, specific_slices=None): """ Retrieve one page of data (paging done based on time). """ from_idx, to_idx = int(from_idx), int(to_idx) if isinstance(specific_slices, basestring): specific_slices = json.loads(specific_slices) if step is None: step = 1 else: step = int(step) slices = [] overall_shape = self.data.shape for i in range(len(overall_shape)): if i == 0: # Time slice slices.append( slice(from_idx, min(to_idx, overall_shape[0]), step)) continue if i == 2: # Read full of the main_dimension (space for the simulator) slices.append(slice(overall_shape[i])) continue if specific_slices is None: slices.append(slice(0, 1)) else: slices.append( slice(specific_slices[i], min(specific_slices[i] + 1, overall_shape[i]), 1)) data = self.data[tuple(slices)] if len(data) == 1: # Do not allow time dimension to get squeezed, a 2D result need to # come out of this method. data = data.squeeze() data = data.reshape((1, len(data))) else: data = data.squeeze() return data def write_time_slice(self, partial_result): """ Append a new value to the ``time`` attribute. """ self.time.append(partial_result) def write_data_slice(self, partial_result, grow_dimension=0): """ Append a chunk of time-series data to the ``data`` attribute. """ self.data.append(partial_result) def get_min_max_values(self): """ Retrieve the minimum and maximum values from the metadata. :returns: (minimum_value, maximum_value) """ metadata = self.data.get_cached_metadata() return metadata.min, metadata.max
class TimeSeriesH5(H5File): def __init__(self, path): super(TimeSeriesH5, self).__init__(path) self.title = Scalar(TimeSeries.title, self) self.data = DataSet(TimeSeries.data, self, expand_dimension=0) self.nr_dimensions = Scalar(Int(), self, name="nr_dimensions") # omitted length_nd , these are indexing props, to be removed from datatype too self.labels_ordering = Json(TimeSeries.labels_ordering, self) self.labels_dimensions = Json(TimeSeries.labels_dimensions, self) self.time = DataSet(TimeSeries.time, self, expand_dimension=0) self.start_time = Scalar(TimeSeries.start_time, self) self.sample_period = Scalar(TimeSeries.sample_period, self) self.sample_period_unit = Scalar(TimeSeries.sample_period_unit, self) self.sample_rate = Scalar(Float(), self, name="sample_rate") # omitted has_surface_mapping, has_volume_mapping, as they are indexing props to be filled only in DB # experiment: load header data eagerly, see surface for a lazy approach # as we do not explicitly make a difference between opening for read or write # the file might not yet exist, so loading headers makes no sense if not self.is_new_file: self._sample_period = self.sample_period.load() self._start_time = self.start_time.load() # experimental port of some of the data access apis from the datatype # NOTE: some methods can not be here as they load data from dependent data types # or they assume that dependent data has been loaded # Those belong to a higher level where dependent h5 files are handles and # partially loaded datatypes are filled def read_data_shape(self): return self.data.shape def read_data_slice(self, data_slice): """ Expose chunked-data access. """ return self.data[data_slice] def read_time_page(self, current_page, page_size, max_size=None): """ Compute time for current page. :param current_page: Starting from 0 """ # todo: why are we even storing the time array if we return a synthetized version? current_page = int(current_page) page_size = int(page_size) if max_size is None: max_size = page_size else: max_size = int(max_size) page_real_size = page_size * self._sample_period start_time = self._start_time + current_page * page_real_size end_time = start_time + min(page_real_size, max_size * self._sample_period) return numpy.arange(start_time, end_time, self._sample_period) def read_channels_page(self, from_idx, to_idx, step=None, specific_slices=None, channels_list=None): """ Read and return only the data page for the specified channels list. :param from_idx: the starting time idx from which to read data :param to_idx: the end time idx up until to which you read data :param step: increments in which to read the data. Optional, default to 1. :param specific_slices: optional parameter. If speficied slices the data accordingly. :param channels_list: the list of channels for which we want data """ if channels_list: channels_list = json.loads(channels_list) for i in range(len(channels_list)): channels_list[i] = int(channels_list[i]) if channels_list: channel_slice = tuple(channels_list) else: channel_slice = slice(None) data_page = self.read_data_page(from_idx, to_idx, step, specific_slices) # This is just a 1D array like in the case of Global Average monitor. # No need for the channels list if len(data_page.shape) == 1: return data_page.reshape(data_page.shape[0], 1) else: return data_page[:, channel_slice] def read_data_page(self, from_idx, to_idx, step=None, specific_slices=None): """ Retrieve one page of data (paging done based on time). """ from_idx, to_idx = int(from_idx), int(to_idx) if isinstance(specific_slices, str): specific_slices = json.loads(specific_slices) if step is None: step = 1 else: step = int(step) slices = [] overall_shape = self.data.shape for i in range(len(overall_shape)): if i == 0: # Time slice slices.append( slice(from_idx, min(to_idx, overall_shape[0]), step)) continue if i == 2: # Read full of the main_dimension (space for the simulator) slices.append(slice(overall_shape[i])) continue if specific_slices is None: slices.append(slice(0, 1)) else: slices.append(slice(specific_slices[i], min(specific_slices[i] + 1, overall_shape[i]), 1)) data = self.data[tuple(slices)] data = data.squeeze() if len(data.shape) == 1: # Do not allow time dimension to get squeezed, a 2D result need to # come out of this method. data = data.reshape((1, len(data))) return data def write_time_slice(self, partial_result): """ Append a new value to the ``time`` attribute. """ self.time.append(partial_result) def write_data_slice(self, partial_result): """ Append a chunk of time-series data to the ``data`` attribute. """ self.data.append(partial_result) def write_data_slice_on_grow_dimension(self, partial_result, grow_dimension=0): self.data.append(partial_result, grow_dimension=grow_dimension) def get_min_max_values(self): """ Retrieve the minimum and maximum values from the metadata. :returns: (minimum_value, maximum_value) """ metadata = self.data.get_cached_metadata() return metadata.min, metadata.max def get_space_labels(self): """ It assumes that we want to select in the 3'rd dimension, and generates labels for each point in that dimension. Subclasses are more specific. :return: An array of strings. """ if self.nr_dimensions.load() > 2: return ['signal-%d' % i for i in range(self.data.shape[2])] else: return [] def get_grouped_space_labels(self): """ :return: A list of label groups. A label group is a tuple (name, [(label_idx, label)...]). Default all labels in a group named '' """ return [('', list(enumerate(self.get_space_labels())))] def get_default_selection(self): """ :return: The measure point indices that have to be shown by default. By default show all. """ return list(range(min(NO_OF_DEFAULT_SELECTED_CHANNELS, self.data.shape[2]))) def get_measure_points_selection_gid(self): """ :return: a datatype gid with which to obtain al valid measure point selection for this time series We have to decide if the default should be all selections or none """ return '' def store_references(self, ts): pass
def __init__(self, path): super(IntegratorH5, self).__init__(path) self.dt = Scalar(Integrator.dt, self)
def __init__(self, path): super(VolumeH5, self).__init__(path) self.origin = DataSet(Volume.origin, self) self.voxel_size = DataSet(Volume.voxel_size, self) self.voxel_unit = Scalar(Volume.voxel_unit, self)
def __init__(self, path): super(CortexH5, self).__init__(path) self.local_connectivity = Reference(Cortex.local_connectivity, self) self.region_mapping_data = Reference(Cortex.region_mapping_data, self) self.coupling_strength = Scalar(Cortex.coupling_strength, self)
def __init__(self, path): super(NoiseH5, self).__init__(path) self.ntau = Scalar(Noise.ntau, self) # TODO: serialize random_stream? self.noise_seed = Scalar(Noise.noise_seed, self)
def __init__(self, path): super(GammaH5, self).__init__(path) self.equation = Scalar(Gamma.equation, self) self.parameters = Json(Gamma.parameters, self)
def __init__(self, path): super(ConnectivityMeasureH5, self).__init__(path) self.array_data = DataSet(ConnectivityMeasure.array_data, self) self.connectivity = Reference(ConnectivityMeasure.connectivity, self) self.title = Scalar(Attr(str), self, name='title')
def __init__(self, path): super(FirstOrderVolterraH5, self).__init__(path) self.equation = Scalar(FirstOrderVolterra.equation, self) self.parameters = Json(FirstOrderVolterra.parameters, self)
class SurfaceH5(H5File): def __init__(self, path): super(SurfaceH5, self).__init__(path) self.vertices = DataSet(Surface.vertices, self) self.triangles = DataSet(Surface.triangles, self) self.vertex_normals = DataSet(Surface.vertex_normals, self) self.triangle_normals = DataSet(Surface.triangle_normals, self) self.number_of_vertices = Scalar(Surface.number_of_vertices, self) self.number_of_triangles = Scalar(Surface.number_of_triangles, self) self.edge_mean_length = Scalar(Surface.edge_mean_length, self) self.edge_min_length = Scalar(Surface.edge_min_length, self) self.edge_max_length = Scalar(Surface.edge_max_length, self) self.zero_based_triangles = Scalar(Surface.zero_based_triangles, self) self.split_triangles = DataSet(NArray(dtype=int), self, name="split_triangles") self.number_of_split_slices = Scalar(Int(), self, name="number_of_split_slices") self.split_slices = Json(Attr(field_type=dict), self, name="split_slices") self.bi_hemispheric = Scalar(Surface.bi_hemispheric, self) self.surface_type = Scalar(Surface.surface_type, self) self.valid_for_simulations = Scalar(Surface.valid_for_simulations, self) # cached header like information, needed to interpret the rest of the file # Load the data that is required in order to interpret the file format # number_of_vertices and split_slices are needed for the get_vertices_slice read call if not self.is_new_file: self._split_slices = self.split_slices.load() self._split_triangles = self.split_triangles.load() self._number_of_vertices = self.number_of_vertices.load() self._number_of_triangles = self.number_of_triangles.load() self._number_of_split_slices = self.number_of_split_slices.load() self._bi_hemispheric = self.bi_hemispheric.load() # else: this is a new file def store(self, datatype, scalars_only=False, store_references=True): # type: (Surface, bool, bool) -> None super(SurfaceH5, self).store(datatype, scalars_only=scalars_only, store_references=store_references) # When any of the header fields change we have to update our cache of them # As they are an invariant of SurfaceH5 we don't do that in the accessors but here. # This implies that direct public writes to them via the accessors will break the invariant. # todo: should we make the accessors private? In complex formats like this one they are private # for this type direct writes to accessors should not be done self._number_of_vertices = datatype.number_of_vertices self._number_of_triangles = datatype.number_of_triangles self._bi_hemispheric = datatype.bi_hemispheric self.prepare_slices(datatype) self.number_of_split_slices.store(self._number_of_split_slices) self.split_slices.store(self._split_slices) self.split_triangles.store(self._split_triangles) def read_subtype_attr(self): return self.surface_type.load() def center(self): """ Compute the center of the surface as the mean spot on all the three axes. """ # is this different from return numpy.mean(self.vertices, axis=0) ? return [float(numpy.mean(self.vertices[:, 0])), float(numpy.mean(self.vertices[:, 1])), float(numpy.mean(self.vertices[:, 2]))] def get_number_of_split_slices(self): return self._number_of_split_slices def prepare_slices(self, datatype): """ Before storing Surface in H5, make sure vertices/triangles are split in slices that are readable by WebGL. WebGL only supports triangle indices in interval [0.... 2^16] """ # Do not split when size is conveniently small: if self._number_of_vertices <= SPLIT_MAX_SIZE + SPLIT_BUFFER_SIZE and not self._bi_hemispheric: self._number_of_split_slices = 1 self._split_slices = {0: {KEY_TRIANGLES: {KEY_START: 0, KEY_END: self._number_of_triangles}, KEY_VERTICES: {KEY_START: 0, KEY_END: self._number_of_vertices}, KEY_HEMISPHERE: HEMISPHERE_UNKNOWN}} self._split_triangles = numpy.array([], dtype=numpy.int32) return # Compute the number of split slices: left_hemisphere_slices = 0 left_hemisphere_vertices_no = 0 if self._bi_hemispheric: # when more than one hemisphere right_hemisphere_vertices_no = numpy.count_nonzero(datatype.hemisphere_mask) left_hemisphere_vertices_no = self._number_of_vertices - right_hemisphere_vertices_no LOG.debug("Right %d Left %d" % (right_hemisphere_vertices_no, left_hemisphere_vertices_no)) left_hemisphere_slices = self._get_slices_number(left_hemisphere_vertices_no) self._number_of_split_slices = left_hemisphere_slices self._number_of_split_slices += self._get_slices_number(right_hemisphere_vertices_no) LOG.debug("Hemispheres Total %d Left %d" % (self._number_of_split_slices, left_hemisphere_slices)) else: # when a single hemisphere self._number_of_split_slices = self._get_slices_number(self._number_of_vertices) LOG.debug("Start to compute surface split triangles and vertices") split_triangles = [] ignored_triangles_counter = 0 self._split_slices = {} for i in range(self._number_of_split_slices): split_triangles.append([]) if not self._bi_hemispheric: self._split_slices[i] = {KEY_VERTICES: {KEY_START: i * SPLIT_MAX_SIZE, KEY_END: min(self._number_of_vertices, (i + 1) * SPLIT_MAX_SIZE + SPLIT_BUFFER_SIZE)}, KEY_HEMISPHERE: HEMISPHERE_UNKNOWN} else: if i < left_hemisphere_slices: self._split_slices[i] = {KEY_VERTICES: {KEY_START: i * SPLIT_MAX_SIZE, KEY_END: min(left_hemisphere_vertices_no, (i + 1) * SPLIT_MAX_SIZE + SPLIT_BUFFER_SIZE)}, KEY_HEMISPHERE: HEMISPHERE_LEFT} else: self._split_slices[i] = {KEY_VERTICES: {KEY_START: left_hemisphere_vertices_no + (i - left_hemisphere_slices) * SPLIT_MAX_SIZE, KEY_END: min(self._number_of_vertices, left_hemisphere_vertices_no + SPLIT_MAX_SIZE * (i + 1 - left_hemisphere_slices) + SPLIT_BUFFER_SIZE)}, KEY_HEMISPHERE: HEMISPHERE_RIGHT} # Iterate Triangles and find the slice where it fits best, based on its vertices indexes: for i in range(self._number_of_triangles): current_triangle = [datatype.triangles[i][j] for j in range(3)] fit_slice, transformed_triangle = self._find_slice(current_triangle) if fit_slice is not None: split_triangles[fit_slice].append(transformed_triangle) else: # triangle ignored, as it has vertices over multiple slices. ignored_triangles_counter += 1 continue final_split_triangles = [] last_triangles_idx = 0 # Concatenate triangles, to be stored in a single HDF5 array. for slice_idx, split_ in enumerate(split_triangles): self._split_slices[slice_idx][KEY_TRIANGLES] = {KEY_START: last_triangles_idx, KEY_END: last_triangles_idx + len(split_)} final_split_triangles.extend(split_) last_triangles_idx += len(split_) self._split_triangles = numpy.array(final_split_triangles, dtype=numpy.int32) if ignored_triangles_counter > 0: LOG.warning("Ignored triangles from multiple hemispheres: " + str(ignored_triangles_counter)) LOG.debug("End compute surface split triangles and vertices " + str(self._split_slices)) @staticmethod def _get_slices_number(vertices_number): """ Slices are for vertices [SPLIT_MAX_SIZE * i ... SPLIT_MAX_SIZE * (i + 1) + SPLIT_BUFFER_SIZE] Slices will overlap : |........SPLIT_MAX_SIZE|...SPLIT_BUFFER_SIZE| <-- split 1 |......... SPLIT_MAX_SIZE|...SPLIT_BUFFER_SIZE| <-- split 2 If we have trailing data smaller than the SPLIT_BUFFER_SIZE, then we no longer split but we need to have at least 1 slice. """ slices_number, trailing = divmod(vertices_number, SPLIT_MAX_SIZE) if trailing > SPLIT_BUFFER_SIZE or (slices_number == 0 and trailing > 0): slices_number += 1 return slices_number def _find_slice(self, triangle): mn = min(triangle) mx = max(triangle) for i in range(self._number_of_split_slices): v = self._split_slices[i][KEY_VERTICES] # extracted for performance slice_start = v[KEY_START] if slice_start <= mn and mx < v[KEY_END]: return i, [triangle[j] - slice_start for j in range(3)] return None, triangle def get_slice_vertex_boundaries(self, slice_idx): if str(slice_idx) in self._split_slices: start_idx = max(0, self._split_slices[str(slice_idx)][KEY_VERTICES][KEY_START]) end_idx = min(self._split_slices[str(slice_idx)][KEY_VERTICES][KEY_END], self._number_of_vertices) return start_idx, end_idx else: LOG.warning("Could not access slice indices, possibly due to an incompatibility with code update!") return 0, min(SPLIT_BUFFER_SIZE, self._number_of_vertices) def _get_slice_triangle_boundaries(self, slice_idx): if str(slice_idx) in self._split_slices: start_idx = max(0, self._split_slices[str(slice_idx)][KEY_TRIANGLES][KEY_START]) end_idx = min(self._split_slices[str(slice_idx)][KEY_TRIANGLES][KEY_END], self._number_of_triangles) return start_idx, end_idx else: LOG.warn("Could not access slice indices, possibly due to an incompatibility with code update!") return 0, self._number_of_triangles def get_vertices_slice(self, slice_number=0): """ Read vertices slice, to be used by WebGL visualizer. """ slice_number = int(slice_number) start_idx, end_idx = self.get_slice_vertex_boundaries(slice_number) return self.vertices[start_idx: end_idx: 1] def get_vertex_normals_slice(self, slice_number=0): """ Read vertex-normal slice, to be used by WebGL visualizer. """ slice_number = int(slice_number) start_idx, end_idx = self.get_slice_vertex_boundaries(slice_number) return self.vertex_normals[start_idx: end_idx: 1] def get_triangles_slice(self, slice_number=0): """ Read split-triangles slice, to be used by WebGL visualizer. """ if self._number_of_split_slices == 1: return self.triangles.load() slice_number = int(slice_number) start_idx, end_idx = self._get_slice_triangle_boundaries(slice_number) return self._split_triangles[start_idx: end_idx: 1] def get_lines_slice(self, slice_number=0): """ Read the gl lines values for the current slice number. """ return Surface._triangles_to_lines(self.get_triangles_slice(slice_number)) def get_slices_to_hemisphere_mask(self): """ :return: a vector af length number_of_slices, with 1 when current chunk belongs to the Right hemisphere """ if not self._bi_hemispheric or self._split_slices is None: return None result = [1] * self._number_of_split_slices for key, value in self._split_slices.items(): if value[KEY_HEMISPHERE] == HEMISPHERE_LEFT: result[int(key)] = 0 return result # todo: many of these do not belong in the data access layer but higher, adapter or gui layer ####################################### Split for Picking ####################################### def get_pick_vertices_slice(self, slice_number=0): """ Read vertices slice, to be used by WebGL visualizer with pick. """ slice_number = int(slice_number) slice_triangles = self.triangles[ slice_number * SPLIT_PICK_MAX_TRIANGLE: min(self._number_of_triangles, (slice_number + 1) * SPLIT_PICK_MAX_TRIANGLE) ] result_vertices = [] cache_vertices = self.vertices.load() for triang in slice_triangles: result_vertices.append(cache_vertices[triang[0]]) result_vertices.append(cache_vertices[triang[1]]) result_vertices.append(cache_vertices[triang[2]]) return numpy.array(result_vertices) def get_pick_vertex_normals_slice(self, slice_number=0): """ Read vertex-normals slice, to be used by WebGL visualizer with pick. """ slice_number = int(slice_number) slice_triangles = self.triangles[ slice_number * SPLIT_PICK_MAX_TRIANGLE: min(self.number_of_triangles.load(), (slice_number + 1) * SPLIT_PICK_MAX_TRIANGLE) ] result_normals = [] cache_vertex_normals = self.vertex_normals.load() for triang in slice_triangles: result_normals.append(cache_vertex_normals[triang[0]]) result_normals.append(cache_vertex_normals[triang[1]]) result_normals.append(cache_vertex_normals[triang[2]]) return numpy.array(result_normals) def get_pick_triangles_slice(self, slice_number=0): """ Read triangles slice, to be used by WebGL visualizer with pick. """ slice_number = int(slice_number) no_of_triangles = (min(self._number_of_triangles, (slice_number + 1) * SPLIT_PICK_MAX_TRIANGLE) - slice_number * SPLIT_PICK_MAX_TRIANGLE) triangles_array = numpy.arange(no_of_triangles * 3).reshape((no_of_triangles, 3)) return triangles_array
def __init__(self, path): super(DiscreteEquationH5, self).__init__(path) self.equation = Scalar(DiscreteEquation.equation, self)
def __init__(self, path): super(SurfaceH5, self).__init__(path) self.vertices = DataSet(Surface.vertices, self) self.triangles = DataSet(Surface.triangles, self) self.vertex_normals = DataSet(Surface.vertex_normals, self) self.triangle_normals = DataSet(Surface.triangle_normals, self) self.number_of_vertices = Scalar(Surface.number_of_vertices, self) self.number_of_triangles = Scalar(Surface.number_of_triangles, self) self.edge_mean_length = Scalar(Surface.edge_mean_length, self) self.edge_min_length = Scalar(Surface.edge_min_length, self) self.edge_max_length = Scalar(Surface.edge_max_length, self) self.zero_based_triangles = Scalar(Surface.zero_based_triangles, self) self.split_triangles = DataSet(NArray(dtype=int), self, name="split_triangles") self.number_of_split_slices = Scalar(Int(), self, name="number_of_split_slices") self.split_slices = Json(Attr(field_type=dict), self, name="split_slices") self.bi_hemispheric = Scalar(Surface.bi_hemispheric, self) self.surface_type = Scalar(Surface.surface_type, self) self.valid_for_simulations = Scalar(Surface.valid_for_simulations, self) # cached header like information, needed to interpret the rest of the file # Load the data that is required in order to interpret the file format # number_of_vertices and split_slices are needed for the get_vertices_slice read call if not self.is_new_file: self._split_slices = self.split_slices.load() self._split_triangles = self.split_triangles.load() self._number_of_vertices = self.number_of_vertices.load() self._number_of_triangles = self.number_of_triangles.load() self._number_of_split_slices = self.number_of_split_slices.load() self._bi_hemispheric = self.bi_hemispheric.load()
def __init__(self, path): super(DoubleGaussianH5, self).__init__(path) self.equation = Scalar(DoubleGaussian.equation, self) self.parameters = Json(DoubleGaussian.parameters, self)
class H5File(object): """ A H5 based file format. This class implements reading and writing to a *specific* h5 based file format. A subclass of this defines a new file format. """ KEY_WRITTEN_BY = 'written_by' is_new_file = False def __init__(self, path): # type: (str) -> None self.path = path self.storage_manager = StorageInterface.get_storage_manager(self.path) # would be nice to have an opened state for the chunked api instead of the close_file=False # common scalar headers self.gid = Uuid(HasTraits.gid, self) self.written_by = Scalar(Attr(str), self, name=self.KEY_WRITTEN_BY) self.create_date = Scalar(Attr(str), self, name='create_date') self.type = Scalar(Attr(str), self, name='type') # Generic attributes descriptors self.generic_attributes = GenericAttributes() self.invalid = Scalar(Attr(bool), self, name='invalid') self.is_nan = Scalar(Attr(bool), self, name='is_nan') self.subject = Scalar(Attr(str), self, name='subject') self.state = Scalar(Attr(str), self, name='state') self.user_tag_1 = Scalar(Attr(str), self, name='user_tag_1') self.user_tag_2 = Scalar(Attr(str), self, name='user_tag_2') self.user_tag_3 = Scalar(Attr(str), self, name='user_tag_3') self.user_tag_4 = Scalar(Attr(str), self, name='user_tag_4') self.user_tag_5 = Scalar(Attr(str), self, name='user_tag_5') self.operation_tag = Scalar(Attr(str, required=False), self, name='operation_tag') self.parent_burst = Uuid(Attr(uuid.UUID, required=False), self, name='parent_burst') self.visible = Scalar(Attr(bool), self, name='visible') self.metadata_cache = None # Keep a list with datasets for which we should write metadata before closing the file self.expandable_datasets = [] if not self.storage_manager.is_valid_tvb_file(): self.written_by.store(self.get_class_path()) self.is_new_file = True @classmethod def file_name_base(cls): return cls.__name__.replace("H5", "") def read_subtype_attr(self): return None def get_class_path(self): return self.__class__.__module__ + '.' + self.__class__.__name__ def iter_accessors(self): # type: () -> typing.Generator[Accessor] for accessor in self.__dict__.values(): if isinstance(accessor, Accessor): yield accessor def iter_datasets(self): for dataset in self.__dict__.values(): if isinstance(dataset, DataSet): yield dataset def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() def close(self): for dataset in self.expandable_datasets: self.storage_manager.set_metadata(dataset.meta.to_dict(), dataset.field_name) self.storage_manager.close_file() def store(self, datatype, scalars_only=False, store_references=True): # type: (HasTraits, bool, bool) -> None for accessor in self.iter_accessors(): f_name = accessor.trait_attribute.field_name if f_name is None: # skipp attribute that does not seem to belong to a traited type # accessor is an independent Accessor continue if scalars_only and not isinstance(accessor, Scalar): continue if not store_references and isinstance(accessor, Reference): continue accessor.store(getattr(datatype, f_name)) def load_into(self, datatype): # type: (HasTraits) -> None for accessor in self.iter_accessors(): if isinstance(accessor, (Reference, ReferenceList)): # we do not load references recursively continue f_name = accessor.trait_attribute.field_name if f_name is None: # skipp attribute that does not seem to belong to a traited type continue # handle optional data, that will be missing from the h5 files try: value = accessor.load() except MissingDataSetException: if accessor.trait_attribute.required: raise else: value = None if isinstance(accessor, JsonFinal): current_attr = getattr(datatype, f_name) for k, v in current_attr.items(): current_attr[k] = value[k] else: try: setattr(datatype, f_name, value) except TraitFinalAttributeError: if getattr(datatype, f_name) != value: raise else: LOGGER.info( 'Cannot overwrite Final attribute: {} on {}, but it already has the expected value' .format(f_name, type(datatype).__name__)) def store_generic_attributes(self, generic_attributes, create=True): # type: (GenericAttributes, bool) -> None # write_metadata creation time, serializer class name, etc if create: self.create_date.store(date2string(datetime.now())) self.generic_attributes.fill_from(generic_attributes) self.invalid.store(self.generic_attributes.invalid) self.is_nan.store(self.generic_attributes.is_nan) self.subject.store(self.generic_attributes.subject) self.state.store(self.generic_attributes.state) self.user_tag_1.store(self.generic_attributes.user_tag_1) self.user_tag_2.store(self.generic_attributes.user_tag_2) self.user_tag_3.store(self.generic_attributes.user_tag_3) self.user_tag_4.store(self.generic_attributes.user_tag_4) self.user_tag_5.store(self.generic_attributes.user_tag_5) self.operation_tag.store(self.generic_attributes.operation_tag) self.visible.store(self.generic_attributes.visible) if self.generic_attributes.parent_burst is not None: self.parent_burst.store( uuid.UUID(self.generic_attributes.parent_burst)) def load_generic_attributes(self): # type: () -> GenericAttributes self.generic_attributes.invalid = self.invalid.load() self.generic_attributes.is_nan = self.is_nan.load() self.generic_attributes.subject = self.subject.load() self.generic_attributes.state = self.state.load() self.generic_attributes.user_tag_1 = self.user_tag_1.load() self.generic_attributes.user_tag_2 = self.user_tag_2.load() self.generic_attributes.user_tag_3 = self.user_tag_3.load() self.generic_attributes.user_tag_4 = self.user_tag_4.load() self.generic_attributes.user_tag_5 = self.user_tag_5.load() self.generic_attributes.visible = self.visible.load() self.generic_attributes.create_date = string2date( str(self.create_date.load())) or None try: self.generic_attributes.operation_tag = self.operation_tag.load() except MissingDataSetException: self.generic_attributes.operation_tag = None try: burst = self.parent_burst.load() self.generic_attributes.parent_burst = burst.hex if burst is not None else None except MissingDataSetException: self.generic_attributes.parent_burst = None return self.generic_attributes def gather_references(self, datatype_cls=None): ret = [] for accessor in self.iter_accessors(): trait_attribute = None if datatype_cls: if hasattr(datatype_cls, accessor.field_name): trait_attribute = getattr(datatype_cls, accessor.field_name) if not trait_attribute: trait_attribute = accessor.trait_attribute if isinstance(accessor, Reference): ret.append((trait_attribute, accessor.load())) if isinstance(accessor, ReferenceList): hex_gids = accessor.load() gids = [uuid.UUID(hex_gid) for hex_gid in hex_gids] ret.append((trait_attribute, gids)) return ret def determine_datatype_from_file(self): config_type = self.type.load() package, cls_name = config_type.rsplit('.', 1) module = importlib.import_module(package) datatype_cls = getattr(module, cls_name) return datatype_cls @staticmethod def determine_type(path): # type: (str) -> typing.Type[HasTraits] type_class_fqn = H5File.get_metadata_param(path, 'type') if type_class_fqn is None: return HasTraits package, cls_name = type_class_fqn.rsplit('.', 1) module = importlib.import_module(package) cls = getattr(module, cls_name) return cls @staticmethod def get_metadata_param(path, param): meta = StorageInterface.get_storage_manager(path).get_metadata() return meta.get(param) def store_metadata_param(self, key, value): self.storage_manager.set_metadata({key: value}) @staticmethod def remove_metadata_param(file_path, param): storage_manager = StorageInterface.get_storage_manager(file_path) if param in storage_manager.get_metadata(): storage_manager.remove_metadata(param) @staticmethod def h5_class_from_file(path): # type: (str) -> typing.Type[H5File] h5file_class_fqn = H5File.get_metadata_param(path, H5File.KEY_WRITTEN_BY) if h5file_class_fqn is None: return H5File(path) package, cls_name = h5file_class_fqn.rsplit('.', 1) module = importlib.import_module(package) cls = getattr(module, cls_name) return cls @staticmethod def from_file(path): # type: (str) -> H5File cls = H5File.h5_class_from_file(path) return cls(path) def __repr__(self): return '<{}("{}")>'.format(type(self).__name__, self.path)
def __init__(self, path): super(SinusoidH5, self).__init__(path) self.equation = Scalar(Sinusoid.equation, self) self.parameters = Json(Sinusoid.parameters, self)
def __init__(self, path): # type: (str) -> None self.path = path self.storage_manager = StorageInterface.get_storage_manager(self.path) # would be nice to have an opened state for the chunked api instead of the close_file=False # common scalar headers self.gid = Uuid(HasTraits.gid, self) self.written_by = Scalar(Attr(str), self, name=self.KEY_WRITTEN_BY) self.create_date = Scalar(Attr(str), self, name='create_date') self.type = Scalar(Attr(str), self, name='type') # Generic attributes descriptors self.generic_attributes = GenericAttributes() self.invalid = Scalar(Attr(bool), self, name='invalid') self.is_nan = Scalar(Attr(bool), self, name='is_nan') self.subject = Scalar(Attr(str), self, name='subject') self.state = Scalar(Attr(str), self, name='state') self.user_tag_1 = Scalar(Attr(str), self, name='user_tag_1') self.user_tag_2 = Scalar(Attr(str), self, name='user_tag_2') self.user_tag_3 = Scalar(Attr(str), self, name='user_tag_3') self.user_tag_4 = Scalar(Attr(str), self, name='user_tag_4') self.user_tag_5 = Scalar(Attr(str), self, name='user_tag_5') self.operation_tag = Scalar(Attr(str, required=False), self, name='operation_tag') self.parent_burst = Uuid(Attr(uuid.UUID, required=False), self, name='parent_burst') self.visible = Scalar(Attr(bool), self, name='visible') self.metadata_cache = None # Keep a list with datasets for which we should write metadata before closing the file self.expandable_datasets = [] if not self.storage_manager.is_valid_tvb_file(): self.written_by.store(self.get_class_path()) self.is_new_file = True
def __init__(self, path): super(EquationH5, self).__init__(path) self.equation = Scalar(Equation.equation, self) self.parameters = Json(Equation.parameters, self)
def __init__(self, path): super(StimuliRegionH5, self).__init__(path) self.spatial = Scalar(Attr(str), self, name='spatial') self.temporal = Scalar(Attr(str), self, name='temporal') self.connectivity = Reference(StimuliRegion.connectivity, self) self.weight = DataSet(StimuliRegion.weight, self)