def _resolve_manifest(manifest, configdir): result = manifest.copy() for k, v in result.items(): if not isinstance(v, str): raise BluepySnapError('{} should be a string value.'.format(v)) if not Path(v).is_absolute() and not v.startswith("$"): if configdir is None: raise BluepySnapError( "Dictionary config with relative paths is not allowed." ) result[k] = str(Path(configdir, v).resolve()) while True: update = False for k, v in result.items(): if v.count('$') > 1: raise BluepySnapError( '{} is not a valid anchor : contains more than one sub anchor.' .format(k)) if v.startswith('$'): tokens = v.split('/', 1) resolved = result[tokens[0]] if '$' not in resolved: result[k] = str(Path(resolved, *tokens[1:])) update = True if not update: break assert '${configdir}' not in result result['${configdir}'] = configdir return result
def get_filepath(self, node_id): """Return path to model file corresponding to `node_id`. Args: node_id (int|CircuitNodeId): node id Returns: Path: path to the model file of neuron """ if not isinstance(node_id, (int, CircuitNodeId)): raise BluepySnapError("node_id must be a int or a CircuitNodeId") node = self._population.get(node_id, [Node.MODEL_TYPE, Node.MODEL_TEMPLATE]) if node[Node.MODEL_TYPE] == "biophysical": models_dir = self._config_components.get( "biophysical_neuron_models_dir") if models_dir is None: raise BluepySnapError( "Missing 'biophysical_neuron_models_dir' in Sonata config") else: models_dir = self._config_components.get("point_neuron_models_dir") if models_dir is None: raise BluepySnapError( "Missing 'point_neuron_models_dir' in Sonata config") template = node[Node.MODEL_TEMPLATE] assert ':' in template, "Format of 'model_template' must be <schema>:<resource>." schema, resource = template.split(':', 1) resource = Path(resource).with_suffix(f'.{schema}') if resource.is_absolute(): return resource return Path(models_dir, resource)
def _resolve_manifest(manifest, configdir): result = manifest.copy() assert '${configdir}' not in result result['${configdir}'] = configdir for k, v in six.iteritems(result): if v.startswith('.'): result[k] = str(Path(configdir, v).resolve()) while True: update = False for k, v in six.iteritems(result): if v.count('$') > 1: raise BluepySnapError( '{} is not a valid anchor : contains more than one sub anchor.' .format(k)) if v.startswith('$'): tokens = v.split('/', 1) resolved = result[tokens[0]] if '$' not in resolved: result[k] = str(Path(resolved, *tokens[1:])) update = True if not update: break for k, v in result.items(): if not v.startswith('/'): raise BluepySnapError( "{} cannot be resolved as an abs path.".format(k)) return result
def spikes_firing_rate_histogram(filtered_report, time_binsize=None, ax=None): # pragma: no cover """Spike firing rate histogram. This plot shows the number of nodes firing during a range of time. Args: time_binsize(None/int/float): bin size (milliseconds). If None, a binning heuristic is used to create an histogram with ~100 spikes per bin in average. ax(matplotlib.Axis): matplotlib Axis to draw on (if not specified, pyplot.gca() is used). Returns: matplotlib.Axis: Axis containing firing rate histogram. Notes: If no axis is provided through the ax=ax keyword argument, then a default layout is set using pyplot.gca(). """ # pylint: disable=too-many-locals plt = _get_pyplot() if time_binsize is not None and time_binsize <= 0: raise BluepySnapError( "Invalid time_binsize = {}. Should be > 0.".format(time_binsize)) spike_report = filtered_report.spike_report times = filtered_report.report.index node_count = filtered_report.report[['ids', 'population' ]].drop_duplicates().shape[0] if len(times) == 0: raise BluepySnapError("No data to display. You should check your " "'group' query: {}.".format(spike_report.group)) time_start = np.min(times) time_stop = np.max(times) if time_binsize is None: # heuristic for a nice bin size (~100 spikes per bin on average) time_binsize = min(50.0, (time_stop - time_start) / ((len(times) / 100.) + 1.)) bins = np.append(np.arange(time_start, time_stop, time_binsize), time_stop) hist, bin_edges = np.histogram(times, bins=bins) freq = 1.0 * hist / node_count / (0.001 * time_binsize) if ax is None: ax = plt.gca() ax.set_xlabel('Time [ms]') ax.set_ylabel('PSTH [Hz]') # use the middle of the bins instead of the start of the bin ax.plot(0.5 * (bin_edges[1:] + bin_edges[:-1]), freq, label="PSTH", drawstyle='steps-mid') return ax
def spikes_isi(filtered_report, use_frequency=False, binsize=None, ax=None): # pragma: no cover # pylint: disable=too-many-locals """Interspike interval histogram. This plots show the binned time/frequency interval between to spikes for neurons. Args: use_frequency(bool): use inverse interspike interval times (Hz) binsize(None/int/float): bin size in milliseconds or Hz. If None is used the binning is delegated to matplolib and is done automatically. ax(matplotlib.Axis): matplotlib Axis to draw on (if not specified, pyplot.gca() is used). Returns: matplotlib.Axis: axis containing the interspike interval histogram. Notes: If no axis is provided through the ax=ax keyword argument, then a default layout is set using pyplot.gca(). """ plt = _get_pyplot() if binsize is not None and binsize <= 0: raise BluepySnapError( "Invalid binsize = {}. Should be > 0.".format(binsize)) gb = filtered_report.report.groupby(["ids", "population"]) values = np.concatenate( [np.diff(node_spikes.index.to_numpy()) for _, node_spikes in gb]) if len(values) == 0: raise BluepySnapError("No data to display. You should check your " "'group' query: {}.".format( filtered_report.spike_report.group)) if use_frequency: values = values[values > 0] # filter out zero intervals values = 1000.0 / values if binsize is None: bins = 'auto' else: bins = np.arange(0, np.max(values), binsize) if ax is None: ax = plt.gca() if use_frequency: ax.set_xlabel('Frequency [Hz]') else: ax.set_xlabel('Interspike interval [ms]') ax.set_ylabel('Bin weight') ax.hist(values, bins=bins, edgecolor='black', density=True) return ax
def _node_ids_by_filter(self, queries, raise_missing_prop): """Return node IDs if their properties match the `queries` dict. `props` values could be: pairs (range match for floating dtype fields) scalar or iterables (exact or "one of" match for other fields) You can use the special operators '$or' and '$and' also to combine different queries together. Examples: >>> _node_ids_by_filter({ Node.X: (0, 1), Node.MTYPE: 'L1_SLAC' }) >>> _node_ids_by_filter({ Node.LAYER: [2, 3] }) >>> _node_ids_by_filter({'$or': [{ Node.LAYER: [2, 3]}, >>> { Node.X: (0, 1), Node.MTYPE: 'L1_SLAC' }]}) """ queries = self._resolve_nodesets(queries) if raise_missing_prop: properties = query.get_properties(queries) if not properties.issubset(self._data.columns): unknown_props = properties - set(self._data.columns) raise BluepySnapError( f"Unknown node properties: {unknown_props}") idx = query.resolve_ids(self._data, self.name, queries) return self._data.index[idx].values
def __getitem__(self, population_name): """Access the NetworkObjectPopulation corresponding to the population 'population_name'.""" try: return self._populations[population_name] except KeyError: raise BluepySnapError("{} not a {} population.".format( population_name, self.__class__))
def pathway_edges(self, source=None, target=None, properties=None): """Get edges corresponding to ``source`` -> ``target`` connections. Args: source: source node group target: target node group properties: None / edge property name / list of edge property names Returns: List of edge IDs, if ``properties`` is None; Pandas Series indexed by edge IDs if ``properties`` is string; Pandas DataFrame indexed by edge IDs if ``properties`` is list. """ if source is None and target is None: raise BluepySnapError( "Either `source` or `target` should be specified") source_node_ids = _resolve_node_ids(self.source, source) target_edge_ids = _resolve_node_ids(self.target, target) if source_node_ids is None: selection = self._population.afferent_edges(target_edge_ids) elif target_edge_ids is None: selection = self._population.efferent_edges(source_node_ids) else: selection = self._population.connecting_edges( source_node_ids, target_edge_ids) return self._get(selection, properties)
def __getitem__(self, population_name): """Access the NetworkObjectPopulation corresponding to the population 'population_name'.""" try: return self._populations[population_name] except KeyError as e: raise BluepySnapError( f"{population_name} not a {self.__class__} population.") from e
def __getitem__(self, population_name): """Access the NodePopulation corresponding to the population 'population_name'.""" try: return self._populations[population_name] except KeyError: raise BluepySnapError( "{} not a node population.".format(population_name))
def circuit(self): """Access to the circuit used for the simulation.""" from bluepysnap.circuit import Circuit if "network" not in self._config: raise BluepySnapError( "No 'network' set in the simulation/global config file.") return Circuit(self._config["network"])
def pathway_edges(self, source=None, target=None, properties=None): """Get edges corresponding to ``source`` -> ``target`` connections. Args: source: source node group target: target node group properties: None / edge property name / list of edge property names Returns: CircuitEdgeIDs, if ``properties`` is None; Pandas Series indexed by CircuitEdgeIDs if ``properties`` is string; Pandas DataFrame indexed by CircuitEdgeIDs if ``properties`` is list. """ if source is None and target is None: raise BluepySnapError( "Either `source` or `target` should be specified") source_ids = self._circuit.nodes.ids(source) target_ids = self._circuit.nodes.ids(target) result = self._get_ids_from_pop( lambda x: (x.pathway_edges(source_ids, target_ids), x.name), CircuitEdgeIds) if properties: return self.get(result, properties) return result
def _update(d, index, value): if index not in d: d[index] = value elif d[index] != value: raise BluepySnapError("Same property with different " "dtype. {}: {}!= {}".format( index, value, d[index]))
def _nodes(self, population_name): """Returns the NodePopulation corresponding to population.""" result = self._edge_storage.circuit.nodes.get(population_name) if result is None: raise BluepySnapError("Undefined node population: '%s'" % population_name) return result
def get(self, edge_ids=None, properties=None): # pylint: disable=arguments-renamed """Edge properties as pandas DataFrame. Args: edge_ids (int/CircuitEdgeId/CircuitEdgeIds/sequence): same as Edges.ids(). properties (None/str/list): an edge property name or a list of edge property names. If set to None ids are returned. Returns: pandas.Series/pandas.DataFrame: A pandas Series indexed by edge IDs if ``properties`` is scalar. A pandas DataFrame indexed by edge IDs if ``properties`` is list. Notes: The Edges.property_names function will give you all the usable properties for the `properties` argument. """ if edge_ids is None: raise BluepySnapError("You need to set edge_ids in get.") if properties is None: Deprecate.warn( "Returning ids with get/properties is deprecated and will be removed in 1.0.0. " "Please use Edges.ids instead.") return edge_ids return super().get(edge_ids, properties)
def get(self, group=None, t_start=None, t_stop=None): """Fetch data from the report. Args: group (None/int/list/np.array/dict): Get spikes filtered by group. See NodePopulation. t_start (float): Include only frames occurring at or after this time. t_stop (float): Include only frames occurring at or before this time. Returns: pandas.DataFrame: frame as columns indexed by timestamps. """ ids = self._resolve(group).tolist() try: view = self._frame_population.get(node_ids=ids, tstart=t_start, tstop=t_stop) except SonataError as e: raise BluepySnapError(e) if len(view.ids) == 0: return pd.DataFrame() res = pd.DataFrame(data=view.data, columns=pd.MultiIndex.from_arrays( np.asarray(view.ids).T), index=view.times).sort_index(axis=1) # rename from multi index to index cannot be achieved easily through df.rename res.columns = self._wrap_columns(res.columns) return res
def get(self, group=None, t_start=None, t_stop=None): """Fetch spikes from the report. Args: group (None/int/list/np.array/dict): Get spikes filtered by group. See NodePopulation. t_start (float): Include only spikes occurring at or after this time. t_stop (float): Include only spikes occurring at or before this time. Returns: pandas.Series: return spiking node_ids indexed by sorted spike time. """ node_ids = self._resolve_nodes(group).tolist() series_name = "ids" try: res = self._spike_population.get(node_ids=node_ids, tstart=t_start, tstop=t_stop) except SonataError as e: raise BluepySnapError(e) from e if not res: return pd.Series( data=[], index=pd.Index([], name="times"), name=series_name, dtype=IDS_DTYPE ) res = pd.DataFrame(data=res, columns=[series_name, "times"]).set_index("times")[series_name] if self._sorted_by != "by_time": res.sort_index(inplace=True) return res.astype(IDS_DTYPE)
def _edge_ids_by_filter(self, queries, raise_missing_prop): """Return edge IDs if their properties match the `queries` dict. `props` values could be: pairs (range match for floating dtype fields) scalar or iterables (exact or "one of" match for other fields) You can use the special operators '$or' and '$and' also to combine different queries together. Examples: >>> self._edge_ids_by_filter({ Edge.POST_SECTION_ID: (0, 1), >>> Edge.AXONAL_DELAY: (.5, 2.) }) >>> self._edge_ids_by_filter({'$or': [{ Edge.PRE_X_CENTER: [2, 3]}, >>> { Edge.POST_SECTION_POS: (0, 1), >>> Edge.SYN_WEIGHT: (0.,1.4) }]}) """ properties = query.get_properties(queries) unknown_props = properties - self.property_names if raise_missing_prop and unknown_props: raise BluepySnapError(f"Unknown edge properties: {unknown_props}") res = [] ids = self.ids(None) chunk_size = int(1e8) for chunk in np.array_split(ids, 1 + len(ids) // chunk_size): data = self.get(chunk, properties - unknown_props) res.extend(chunk[query.resolve_ids(data, self.name, queries)]) return np.array(res, dtype=IDS_DTYPE)
def _properties_mask(self, queries, raise_missing_prop): """Return mask of node IDs with rows matching `props` dict.""" # pylint: disable=assignment-from-no-return circuit_keys = {POPULATION_KEY, NODE_ID_KEY, NODE_SET_KEY} unknown_props = set(queries) - set(self._data.columns) - circuit_keys if unknown_props: if raise_missing_prop: raise BluepySnapError("Unknown node properties: [{0}]".format( ", ".join(unknown_props))) return np.full(len(self._data), fill_value=False) queries, mask = self._circuit_mask(queries) if not mask.any(): # Avoid fail and/or processing time if wrong population or no nodes return mask for prop, values in six.iteritems(queries): prop = self._data[prop] if np.issubdtype(prop.dtype.type, np.floating): v1, v2 = values prop_mask = np.logical_and(prop >= v1, prop <= v2) elif isinstance(values, six.string_types) and values.startswith('regex:'): prop_mask = _complex_query(prop, {'$regex': values[6:]}) elif isinstance(values, collections.Mapping): prop_mask = _complex_query(prop, values) else: prop_mask = np.in1d(prop, values) mask = np.logical_and(mask, prop_mask) return mask
def log(self): """Context manager for the spike log file.""" path = Path(self.config["output_dir"]) / self.config["log_file"] if not path.exists(): raise BluepySnapError( "Cannot find the log file for the spike report.") yield open(str(path), "r")
def euler2mat(az, ay, ax): """Build 3x3 rotation matrices from az, ay, ax rotation angles (in that order). Args: az: rotation angles around Z (Nx1 NumPy array; radians) ay: rotation angles around Y (Nx1 NumPy array; radians) ax: rotation angles around X (Nx1 NumPy array; radians) Returns: List with Nx3x3 rotation matrices corresponding to each of N angle triplets. See Also: https://en.wikipedia.org/wiki/Euler_angles#Rotation_matrix (R = X1 * Y2 * Z3) """ if len(az) != len(ay) or len(az) != len(ax): raise BluepySnapError("All angles must have the same length.") c1, s1 = np.cos(ax), np.sin(ax) c2, s2 = np.cos(ay), np.sin(ay) c3, s3 = np.cos(az), np.sin(az) mm = np.array([ [c2 * c3, -c2 * s3, s2], [c1 * s3 + c3 * s1 * s2, c1 * c3 - s1 * s2 * s3, -c2 * s1], [s1 * s3 - c1 * c3 * s2, c3 * s1 + c1 * s2 * s3, c1 * c2], ]) return [mm[..., i] for i in range(len(az))]
def get(self, group=None, t_start=None, t_stop=None): """Fetch data from the report. Args: group (None/int/list/np.array/dict): Get frames filtered by group. See NodePopulation. t_start (float): Include only frames occurring at or after this time. t_stop (float): Include only frames occurring at or before this time. Returns: pandas.DataFrame: frame as columns indexed by timestamps. """ ids = self._resolve(group).tolist() try: view = self._frame_population.get(node_ids=ids, tstart=t_start, tstop=t_stop) except SonataError as e: raise BluepySnapError(e) from e if len(view.ids) == 0: return pd.DataFrame() # cell ids and section ids in the columns are enforced to be int64 # to avoid issues with numpy automatic conversions and to ensure that # the results are the same regardless of the libsonata version [NSETM-1766] res = pd.DataFrame( data=view.data, columns=pd.MultiIndex.from_arrays(ensure_ids(view.ids).T), index=view.times, ).sort_index(axis=1) # rename from multi index to index cannot be achieved easily through df.rename res.columns = self._wrap_columns(res.columns) return res
def nodes(self): """Returns the NodePopulation corresponding to this report.""" result = self.frame_report.simulation.circuit.nodes.get( self._population_name) if result is None: raise BluepySnapError("Undefined node population: '%s'" % self._population_name) return result
def _complex_query(prop, query): result = np.full(len(prop), True) for key, value in query.items(): if key == REGEX_KEY: result = np.logical_and(result, prop.str.match(value + "\\Z")) else: raise BluepySnapError("Unknown query modifier: '%s'" % key) return result
def get(cls, const_name): """Get a constant from a string name.""" try: res = getattr(cls, const_name) except AttributeError: raise BluepySnapError("{} does not have a '{}' member".format( cls, const_name)) return res
def get(cls, const_name): """Get a constant from a string name.""" try: res = getattr(cls, const_name) except AttributeError as e: raise BluepySnapError( "{cls} does not have a '{const_name}' member") from e return res
def ids(self, group=None, limit=None, sample=None, raise_missing_property=True): """Edge IDs corresponding to edges ``edge_ids``. Args: group (None/int/CircuitEdgeId/CircuitEdgeIds/sequence): Which IDs will be returned depends on the type of the ``group`` argument: - ``None``: return all IDs. - ``int``, ``CircuitEdgeId``: return a single edge ID. - ``CircuitEdgeIds`` return IDs of edges the edge population in an array. - ``sequence``: return IDs of edges in an array. sample (int): If specified, randomly choose ``sample`` number of IDs from the match result. If the size of the sample is greater than the size of the EdgePopulation then all ids are taken and shuffled. limit (int): If specified, return the first ``limit`` number of IDs from the match result. If limit is greater than the size of the population all node IDs are returned. raise_missing_property (bool): if True, raises if a property is not listed in this population. Otherwise the ids are just not selected if a property is missing. Returns: numpy.array: A numpy array of IDs. """ if group is None: result = self._population.select_all().flatten() elif isinstance(group, CircuitEdgeIds): result = group.filter_population(self.name).get_ids() elif isinstance(group, np.ndarray): result = group elif isinstance(group, Mapping): result = self._edge_ids_by_filter( queries=group, raise_missing_prop=raise_missing_property) else: result = utils.ensure_list(group) # test if first value is a CircuitEdgeId if yes then all values must be CircuitEdgeId if isinstance(first(result, None), CircuitEdgeId): try: result = [ cid.id for cid in result if cid.population == self.name ] except AttributeError as e: raise BluepySnapError( "All values from a list must be of type int or CircuitEdgeId." ) from e if sample is not None: if len(result) > 0: result = np.random.choice(result, min(sample, len(result)), replace=False) if limit is not None: result = result[:limit] return utils.ensure_ids(result)
def _complex_query(prop, query): # pylint: disable=assignment-from-no-return result = np.full(len(prop), True) for key, value in six.iteritems(query): if key == '$regex': result = np.logical_and(result, prop.str.match(value + "\\Z")) else: raise BluepySnapError("Unknown query modifier: '%s'" % key) return result
def frame_trace(filtered_report, plot_type='mean', ax=None): # pragma: no cover """Returns a plot displaying the voltage of a node or a compartment as a function of time. Args: plot_type (str): string either `all` or `mean`. `all` will plot the first 15 traces from the group. `mean` will plot the mean value of the node ax: A plot axis object that will be updated Returns: matplotlib.Axis: axis containing the soma's traces. """ # pylint: disable=too-many-locals plt = _get_pyplot() if ax is None: ax = plt.gca() data_units = filtered_report.frame_report.data_units if plot_type == "mean": ax.set_ylabel('Avg volt. [{}]'.format(data_units)) elif plot_type == "all": ax.set_ylabel('Voltage [{}]'.format(data_units)) ax.set_xlabel("Time [{}]".format( filtered_report.frame_report.time_units)) ax.set_xlim([ filtered_report.report.index.min(), filtered_report.report.index.max() ]) if plot_type == "mean": ax.plot(filtered_report.report.T.mean()) elif plot_type == "all": max_per_pop = 15 levels = filtered_report.report.columns.levels slicer = [] # create a slicer that will slice only on the last level of the columns # that is, node_id for the soma report, element_id for the compartment report for i, _ in enumerate(levels): max_ = levels[i][:max_per_pop][-1] slicer.append( slice(None) if i != len(levels) - 1 else slice(None, max_)) data = filtered_report.report.loc[:, tuple(slicer)].T # create [[(pop1, id1), (pop1, id2),...], [(pop2, id1), (pop2, id2),...]] indexes = [[(pop, idx) for idx in data.loc[pop].index] for pop in levels[0]] # try to keep the maximum of ids from each population kept_ids = list(roundrobin(*indexes))[:max_per_pop] for _, row in data.loc[kept_ids].iterrows(): ax.plot(row) else: raise BluepySnapError( "Unknown plot_type {}. Should be 'mean or 'all'.".format( plot_type)) return ax
def data_units(self): """Returns the data unit for this report.""" units = { self._frame_reader[pop].data_units for pop in self.population_names } if len(units) > 1: raise BluepySnapError( "Multiple data units found in the different populations.") return units.pop()