def _resolve_manifest(manifest, configdir):
result = manifest.copy()
for k, v in result.items():
if not isinstance(v, str):
raise BluepySnapError('{} should be a string value.'.format(v))
if not Path(v).is_absolute() and not v.startswith("$"):
if configdir is None:
raise BluepySnapError(
"Dictionary config with relative paths is not allowed."
)
result[k] = str(Path(configdir, v).resolve())
while True:
update = False
for k, v in result.items():
if v.count('$') > 1:
raise BluepySnapError(
'{} is not a valid anchor : contains more than one sub anchor.'
.format(k))
if v.startswith('$'):
tokens = v.split('/', 1)
resolved = result[tokens[0]]
if '$' not in resolved:
result[k] = str(Path(resolved, *tokens[1:]))
update = True
if not update:
break
assert '${configdir}' not in result
result['${configdir}'] = configdir
return result
def get_filepath(self, node_id):
"""Return path to model file corresponding to `node_id`.
Args:
node_id (int|CircuitNodeId): node id
Returns:
Path: path to the model file of neuron
"""
if not isinstance(node_id, (int, CircuitNodeId)):
raise BluepySnapError("node_id must be a int or a CircuitNodeId")
node = self._population.get(node_id,
[Node.MODEL_TYPE, Node.MODEL_TEMPLATE])
if node[Node.MODEL_TYPE] == "biophysical":
models_dir = self._config_components.get(
"biophysical_neuron_models_dir")
if models_dir is None:
raise BluepySnapError(
"Missing 'biophysical_neuron_models_dir' in Sonata config")
else:
models_dir = self._config_components.get("point_neuron_models_dir")
if models_dir is None:
raise BluepySnapError(
"Missing 'point_neuron_models_dir' in Sonata config")
template = node[Node.MODEL_TEMPLATE]
assert ':' in template, "Format of 'model_template' must be <schema>:<resource>."
schema, resource = template.split(':', 1)
resource = Path(resource).with_suffix(f'.{schema}')
if resource.is_absolute():
return resource
return Path(models_dir, resource)
def _resolve_manifest(manifest, configdir):
result = manifest.copy()
assert '${configdir}' not in result
result['${configdir}'] = configdir
for k, v in six.iteritems(result):
if v.startswith('.'):
result[k] = str(Path(configdir, v).resolve())
while True:
update = False
for k, v in six.iteritems(result):
if v.count('$') > 1:
raise BluepySnapError(
'{} is not a valid anchor : contains more than one sub anchor.'
.format(k))
if v.startswith('$'):
tokens = v.split('/', 1)
resolved = result[tokens[0]]
if '$' not in resolved:
result[k] = str(Path(resolved, *tokens[1:]))
update = True
if not update:
break
for k, v in result.items():
if not v.startswith('/'):
raise BluepySnapError(
"{} cannot be resolved as an abs path.".format(k))
return result
def spikes_firing_rate_histogram(filtered_report,
time_binsize=None,
ax=None): # pragma: no cover
"""Spike firing rate histogram.
This plot shows the number of nodes firing during a range of time.
Args:
time_binsize(None/int/float): bin size (milliseconds). If None, a binning heuristic is used
to create an histogram with ~100 spikes per bin in average.
ax(matplotlib.Axis): matplotlib Axis to draw on (if not specified, pyplot.gca() is used).
Returns:
matplotlib.Axis: Axis containing firing rate histogram.
Notes:
If no axis is provided through the ax=ax keyword argument,
then a default layout is set using pyplot.gca().
"""
# pylint: disable=too-many-locals
plt = _get_pyplot()
if time_binsize is not None and time_binsize <= 0:
raise BluepySnapError(
"Invalid time_binsize = {}. Should be > 0.".format(time_binsize))
spike_report = filtered_report.spike_report
times = filtered_report.report.index
node_count = filtered_report.report[['ids', 'population'
]].drop_duplicates().shape[0]
if len(times) == 0:
raise BluepySnapError("No data to display. You should check your "
"'group' query: {}.".format(spike_report.group))
time_start = np.min(times)
time_stop = np.max(times)
if time_binsize is None:
# heuristic for a nice bin size (~100 spikes per bin on average)
time_binsize = min(50.0, (time_stop - time_start) /
((len(times) / 100.) + 1.))
bins = np.append(np.arange(time_start, time_stop, time_binsize), time_stop)
hist, bin_edges = np.histogram(times, bins=bins)
freq = 1.0 * hist / node_count / (0.001 * time_binsize)
if ax is None:
ax = plt.gca()
ax.set_xlabel('Time [ms]')
ax.set_ylabel('PSTH [Hz]')
# use the middle of the bins instead of the start of the bin
ax.plot(0.5 * (bin_edges[1:] + bin_edges[:-1]),
freq,
label="PSTH",
drawstyle='steps-mid')
return ax
def spikes_isi(filtered_report,
use_frequency=False,
binsize=None,
ax=None): # pragma: no cover
# pylint: disable=too-many-locals
"""Interspike interval histogram.
This plots show the binned time/frequency interval between to spikes for neurons.
Args:
use_frequency(bool): use inverse interspike interval times (Hz)
binsize(None/int/float): bin size in milliseconds or Hz. If None is used the binning is
delegated to matplolib and is done automatically.
ax(matplotlib.Axis): matplotlib Axis to draw on (if not specified, pyplot.gca() is used).
Returns:
matplotlib.Axis: axis containing the interspike interval histogram.
Notes:
If no axis is provided through the ax=ax keyword argument,
then a default layout is set using pyplot.gca().
"""
plt = _get_pyplot()
if binsize is not None and binsize <= 0:
raise BluepySnapError(
"Invalid binsize = {}. Should be > 0.".format(binsize))
gb = filtered_report.report.groupby(["ids", "population"])
values = np.concatenate(
[np.diff(node_spikes.index.to_numpy()) for _, node_spikes in gb])
if len(values) == 0:
raise BluepySnapError("No data to display. You should check your "
"'group' query: {}.".format(
filtered_report.spike_report.group))
if use_frequency:
values = values[values > 0] # filter out zero intervals
values = 1000.0 / values
if binsize is None:
bins = 'auto'
else:
bins = np.arange(0, np.max(values), binsize)
if ax is None:
ax = plt.gca()
if use_frequency:
ax.set_xlabel('Frequency [Hz]')
else:
ax.set_xlabel('Interspike interval [ms]')
ax.set_ylabel('Bin weight')
ax.hist(values, bins=bins, edgecolor='black', density=True)
return ax
def _node_ids_by_filter(self, queries, raise_missing_prop):
"""Return node IDs if their properties match the `queries` dict.
`props` values could be:
pairs (range match for floating dtype fields)
scalar or iterables (exact or "one of" match for other fields)
You can use the special operators '$or' and '$and' also to combine different queries
together.
Examples:
>>> _node_ids_by_filter({ Node.X: (0, 1), Node.MTYPE: 'L1_SLAC' })
>>> _node_ids_by_filter({ Node.LAYER: [2, 3] })
>>> _node_ids_by_filter({'$or': [{ Node.LAYER: [2, 3]},
>>> { Node.X: (0, 1), Node.MTYPE: 'L1_SLAC' }]})
"""
queries = self._resolve_nodesets(queries)
if raise_missing_prop:
properties = query.get_properties(queries)
if not properties.issubset(self._data.columns):
unknown_props = properties - set(self._data.columns)
raise BluepySnapError(
f"Unknown node properties: {unknown_props}")
idx = query.resolve_ids(self._data, self.name, queries)
return self._data.index[idx].values
def __getitem__(self, population_name):
"""Access the NetworkObjectPopulation corresponding to the population 'population_name'."""
try:
return self._populations[population_name]
except KeyError:
raise BluepySnapError("{} not a {} population.".format(
population_name, self.__class__))
def pathway_edges(self, source=None, target=None, properties=None):
"""Get edges corresponding to ``source`` -> ``target`` connections.
Args:
source: source node group
target: target node group
properties: None / edge property name / list of edge property names
Returns:
List of edge IDs, if ``properties`` is None;
Pandas Series indexed by edge IDs if ``properties`` is string;
Pandas DataFrame indexed by edge IDs if ``properties`` is list.
"""
if source is None and target is None:
raise BluepySnapError(
"Either `source` or `target` should be specified")
source_node_ids = _resolve_node_ids(self.source, source)
target_edge_ids = _resolve_node_ids(self.target, target)
if source_node_ids is None:
selection = self._population.afferent_edges(target_edge_ids)
elif target_edge_ids is None:
selection = self._population.efferent_edges(source_node_ids)
else:
selection = self._population.connecting_edges(
source_node_ids, target_edge_ids)
return self._get(selection, properties)
def __getitem__(self, population_name):
"""Access the NetworkObjectPopulation corresponding to the population 'population_name'."""
try:
return self._populations[population_name]
except KeyError as e:
raise BluepySnapError(
f"{population_name} not a {self.__class__} population.") from e
def __getitem__(self, population_name):
"""Access the NodePopulation corresponding to the population 'population_name'."""
try:
return self._populations[population_name]
except KeyError:
raise BluepySnapError(
"{} not a node population.".format(population_name))
def circuit(self):
"""Access to the circuit used for the simulation."""
from bluepysnap.circuit import Circuit
if "network" not in self._config:
raise BluepySnapError(
"No 'network' set in the simulation/global config file.")
return Circuit(self._config["network"])
def pathway_edges(self, source=None, target=None, properties=None):
"""Get edges corresponding to ``source`` -> ``target`` connections.
Args:
source: source node group
target: target node group
properties: None / edge property name / list of edge property names
Returns:
CircuitEdgeIDs, if ``properties`` is None;
Pandas Series indexed by CircuitEdgeIDs if ``properties`` is string;
Pandas DataFrame indexed by CircuitEdgeIDs if ``properties`` is list.
"""
if source is None and target is None:
raise BluepySnapError(
"Either `source` or `target` should be specified")
source_ids = self._circuit.nodes.ids(source)
target_ids = self._circuit.nodes.ids(target)
result = self._get_ids_from_pop(
lambda x: (x.pathway_edges(source_ids, target_ids), x.name),
CircuitEdgeIds)
if properties:
return self.get(result, properties)
return result
def _update(d, index, value):
if index not in d:
d[index] = value
elif d[index] != value:
raise BluepySnapError("Same property with different "
"dtype. {}: {}!= {}".format(
index, value, d[index]))
def _nodes(self, population_name):
"""Returns the NodePopulation corresponding to population."""
result = self._edge_storage.circuit.nodes.get(population_name)
if result is None:
raise BluepySnapError("Undefined node population: '%s'" %
population_name)
return result
def get(self, edge_ids=None, properties=None): # pylint: disable=arguments-renamed
"""Edge properties as pandas DataFrame.
Args:
edge_ids (int/CircuitEdgeId/CircuitEdgeIds/sequence): same as Edges.ids().
properties (None/str/list): an edge property name or a list of edge property names.
If set to None ids are returned.
Returns:
pandas.Series/pandas.DataFrame:
A pandas Series indexed by edge IDs if ``properties`` is scalar.
A pandas DataFrame indexed by edge IDs if ``properties`` is list.
Notes:
The Edges.property_names function will give you all the usable properties
for the `properties` argument.
"""
if edge_ids is None:
raise BluepySnapError("You need to set edge_ids in get.")
if properties is None:
Deprecate.warn(
"Returning ids with get/properties is deprecated and will be removed in 1.0.0. "
"Please use Edges.ids instead.")
return edge_ids
return super().get(edge_ids, properties)
def get(self, group=None, t_start=None, t_stop=None):
"""Fetch data from the report.
Args:
group (None/int/list/np.array/dict): Get spikes filtered by group. See NodePopulation.
t_start (float): Include only frames occurring at or after this time.
t_stop (float): Include only frames occurring at or before this time.
Returns:
pandas.DataFrame: frame as columns indexed by timestamps.
"""
ids = self._resolve(group).tolist()
try:
view = self._frame_population.get(node_ids=ids,
tstart=t_start,
tstop=t_stop)
except SonataError as e:
raise BluepySnapError(e)
if len(view.ids) == 0:
return pd.DataFrame()
res = pd.DataFrame(data=view.data,
columns=pd.MultiIndex.from_arrays(
np.asarray(view.ids).T),
index=view.times).sort_index(axis=1)
# rename from multi index to index cannot be achieved easily through df.rename
res.columns = self._wrap_columns(res.columns)
return res
def get(self, group=None, t_start=None, t_stop=None):
"""Fetch spikes from the report.
Args:
group (None/int/list/np.array/dict): Get spikes filtered by group. See NodePopulation.
t_start (float): Include only spikes occurring at or after this time.
t_stop (float): Include only spikes occurring at or before this time.
Returns:
pandas.Series: return spiking node_ids indexed by sorted spike time.
"""
node_ids = self._resolve_nodes(group).tolist()
series_name = "ids"
try:
res = self._spike_population.get(node_ids=node_ids, tstart=t_start, tstop=t_stop)
except SonataError as e:
raise BluepySnapError(e) from e
if not res:
return pd.Series(
data=[], index=pd.Index([], name="times"), name=series_name, dtype=IDS_DTYPE
)
res = pd.DataFrame(data=res, columns=[series_name, "times"]).set_index("times")[series_name]
if self._sorted_by != "by_time":
res.sort_index(inplace=True)
return res.astype(IDS_DTYPE)
def _edge_ids_by_filter(self, queries, raise_missing_prop):
"""Return edge IDs if their properties match the `queries` dict.
`props` values could be:
pairs (range match for floating dtype fields)
scalar or iterables (exact or "one of" match for other fields)
You can use the special operators '$or' and '$and' also to combine different queries
together.
Examples:
>>> self._edge_ids_by_filter({ Edge.POST_SECTION_ID: (0, 1),
>>> Edge.AXONAL_DELAY: (.5, 2.) })
>>> self._edge_ids_by_filter({'$or': [{ Edge.PRE_X_CENTER: [2, 3]},
>>> { Edge.POST_SECTION_POS: (0, 1),
>>> Edge.SYN_WEIGHT: (0.,1.4) }]})
"""
properties = query.get_properties(queries)
unknown_props = properties - self.property_names
if raise_missing_prop and unknown_props:
raise BluepySnapError(f"Unknown edge properties: {unknown_props}")
res = []
ids = self.ids(None)
chunk_size = int(1e8)
for chunk in np.array_split(ids, 1 + len(ids) // chunk_size):
data = self.get(chunk, properties - unknown_props)
res.extend(chunk[query.resolve_ids(data, self.name, queries)])
return np.array(res, dtype=IDS_DTYPE)
def _properties_mask(self, queries, raise_missing_prop):
"""Return mask of node IDs with rows matching `props` dict."""
# pylint: disable=assignment-from-no-return
circuit_keys = {POPULATION_KEY, NODE_ID_KEY, NODE_SET_KEY}
unknown_props = set(queries) - set(self._data.columns) - circuit_keys
if unknown_props:
if raise_missing_prop:
raise BluepySnapError("Unknown node properties: [{0}]".format(
", ".join(unknown_props)))
return np.full(len(self._data), fill_value=False)
queries, mask = self._circuit_mask(queries)
if not mask.any():
# Avoid fail and/or processing time if wrong population or no nodes
return mask
for prop, values in six.iteritems(queries):
prop = self._data[prop]
if np.issubdtype(prop.dtype.type, np.floating):
v1, v2 = values
prop_mask = np.logical_and(prop >= v1, prop <= v2)
elif isinstance(values,
six.string_types) and values.startswith('regex:'):
prop_mask = _complex_query(prop, {'$regex': values[6:]})
elif isinstance(values, collections.Mapping):
prop_mask = _complex_query(prop, values)
else:
prop_mask = np.in1d(prop, values)
mask = np.logical_and(mask, prop_mask)
return mask
def log(self):
"""Context manager for the spike log file."""
path = Path(self.config["output_dir"]) / self.config["log_file"]
if not path.exists():
raise BluepySnapError(
"Cannot find the log file for the spike report.")
yield open(str(path), "r")
def euler2mat(az, ay, ax):
"""Build 3x3 rotation matrices from az, ay, ax rotation angles (in that order).
Args:
az: rotation angles around Z (Nx1 NumPy array; radians)
ay: rotation angles around Y (Nx1 NumPy array; radians)
ax: rotation angles around X (Nx1 NumPy array; radians)
Returns:
List with Nx3x3 rotation matrices corresponding to each of N angle triplets.
See Also:
https://en.wikipedia.org/wiki/Euler_angles#Rotation_matrix (R = X1 * Y2 * Z3)
"""
if len(az) != len(ay) or len(az) != len(ax):
raise BluepySnapError("All angles must have the same length.")
c1, s1 = np.cos(ax), np.sin(ax)
c2, s2 = np.cos(ay), np.sin(ay)
c3, s3 = np.cos(az), np.sin(az)
mm = np.array([
[c2 * c3, -c2 * s3, s2],
[c1 * s3 + c3 * s1 * s2, c1 * c3 - s1 * s2 * s3, -c2 * s1],
[s1 * s3 - c1 * c3 * s2, c3 * s1 + c1 * s2 * s3, c1 * c2],
])
return [mm[..., i] for i in range(len(az))]
def get(self, group=None, t_start=None, t_stop=None):
"""Fetch data from the report.
Args:
group (None/int/list/np.array/dict): Get frames filtered by group. See NodePopulation.
t_start (float): Include only frames occurring at or after this time.
t_stop (float): Include only frames occurring at or before this time.
Returns:
pandas.DataFrame: frame as columns indexed by timestamps.
"""
ids = self._resolve(group).tolist()
try:
view = self._frame_population.get(node_ids=ids,
tstart=t_start,
tstop=t_stop)
except SonataError as e:
raise BluepySnapError(e) from e
if len(view.ids) == 0:
return pd.DataFrame()
# cell ids and section ids in the columns are enforced to be int64
# to avoid issues with numpy automatic conversions and to ensure that
# the results are the same regardless of the libsonata version [NSETM-1766]
res = pd.DataFrame(
data=view.data,
columns=pd.MultiIndex.from_arrays(ensure_ids(view.ids).T),
index=view.times,
).sort_index(axis=1)
# rename from multi index to index cannot be achieved easily through df.rename
res.columns = self._wrap_columns(res.columns)
return res
def nodes(self):
"""Returns the NodePopulation corresponding to this report."""
result = self.frame_report.simulation.circuit.nodes.get(
self._population_name)
if result is None:
raise BluepySnapError("Undefined node population: '%s'" %
self._population_name)
return result
def _complex_query(prop, query):
result = np.full(len(prop), True)
for key, value in query.items():
if key == REGEX_KEY:
result = np.logical_and(result, prop.str.match(value + "\\Z"))
else:
raise BluepySnapError("Unknown query modifier: '%s'" % key)
return result
def get(cls, const_name):
"""Get a constant from a string name."""
try:
res = getattr(cls, const_name)
except AttributeError:
raise BluepySnapError("{} does not have a '{}' member".format(
cls, const_name))
return res
def get(cls, const_name):
"""Get a constant from a string name."""
try:
res = getattr(cls, const_name)
except AttributeError as e:
raise BluepySnapError(
"{cls} does not have a '{const_name}' member") from e
return res
def ids(self,
group=None,
limit=None,
sample=None,
raise_missing_property=True):
"""Edge IDs corresponding to edges ``edge_ids``.
Args:
group (None/int/CircuitEdgeId/CircuitEdgeIds/sequence): Which IDs will be
returned depends on the type of the ``group`` argument:
- ``None``: return all IDs.
- ``int``, ``CircuitEdgeId``: return a single edge ID.
- ``CircuitEdgeIds`` return IDs of edges the edge population in an array.
- ``sequence``: return IDs of edges in an array.
sample (int): If specified, randomly choose ``sample`` number of
IDs from the match result. If the size of the sample is greater than
the size of the EdgePopulation then all ids are taken and shuffled.
limit (int): If specified, return the first ``limit`` number of
IDs from the match result. If limit is greater than the size of the population
all node IDs are returned.
raise_missing_property (bool): if True, raises if a property is not listed in this
population. Otherwise the ids are just not selected if a property is missing.
Returns:
numpy.array: A numpy array of IDs.
"""
if group is None:
result = self._population.select_all().flatten()
elif isinstance(group, CircuitEdgeIds):
result = group.filter_population(self.name).get_ids()
elif isinstance(group, np.ndarray):
result = group
elif isinstance(group, Mapping):
result = self._edge_ids_by_filter(
queries=group, raise_missing_prop=raise_missing_property)
else:
result = utils.ensure_list(group)
# test if first value is a CircuitEdgeId if yes then all values must be CircuitEdgeId
if isinstance(first(result, None), CircuitEdgeId):
try:
result = [
cid.id for cid in result if cid.population == self.name
]
except AttributeError as e:
raise BluepySnapError(
"All values from a list must be of type int or CircuitEdgeId."
) from e
if sample is not None:
if len(result) > 0:
result = np.random.choice(result,
min(sample, len(result)),
replace=False)
if limit is not None:
result = result[:limit]
return utils.ensure_ids(result)
def _complex_query(prop, query):
# pylint: disable=assignment-from-no-return
result = np.full(len(prop), True)
for key, value in six.iteritems(query):
if key == '$regex':
result = np.logical_and(result, prop.str.match(value + "\\Z"))
else:
raise BluepySnapError("Unknown query modifier: '%s'" % key)
return result
def frame_trace(filtered_report,
plot_type='mean',
ax=None): # pragma: no cover
"""Returns a plot displaying the voltage of a node or a compartment as a function of time.
Args:
plot_type (str): string either `all` or `mean`. `all` will plot the first 15 traces from the
group. `mean` will plot the mean value of the node
ax: A plot axis object that will be updated
Returns:
matplotlib.Axis: axis containing the soma's traces.
"""
# pylint: disable=too-many-locals
plt = _get_pyplot()
if ax is None:
ax = plt.gca()
data_units = filtered_report.frame_report.data_units
if plot_type == "mean":
ax.set_ylabel('Avg volt. [{}]'.format(data_units))
elif plot_type == "all":
ax.set_ylabel('Voltage [{}]'.format(data_units))
ax.set_xlabel("Time [{}]".format(
filtered_report.frame_report.time_units))
ax.set_xlim([
filtered_report.report.index.min(),
filtered_report.report.index.max()
])
if plot_type == "mean":
ax.plot(filtered_report.report.T.mean())
elif plot_type == "all":
max_per_pop = 15
levels = filtered_report.report.columns.levels
slicer = []
# create a slicer that will slice only on the last level of the columns
# that is, node_id for the soma report, element_id for the compartment report
for i, _ in enumerate(levels):
max_ = levels[i][:max_per_pop][-1]
slicer.append(
slice(None) if i != len(levels) - 1 else slice(None, max_))
data = filtered_report.report.loc[:, tuple(slicer)].T
# create [[(pop1, id1), (pop1, id2),...], [(pop2, id1), (pop2, id2),...]]
indexes = [[(pop, idx) for idx in data.loc[pop].index]
for pop in levels[0]]
# try to keep the maximum of ids from each population
kept_ids = list(roundrobin(*indexes))[:max_per_pop]
for _, row in data.loc[kept_ids].iterrows():
ax.plot(row)
else:
raise BluepySnapError(
"Unknown plot_type {}. Should be 'mean or 'all'.".format(
plot_type))
return ax
def data_units(self):
"""Returns the data unit for this report."""
units = {
self._frame_reader[pop].data_units
for pop in self.population_names
}
if len(units) > 1:
raise BluepySnapError(
"Multiple data units found in the different populations.")
return units.pop()
