def test_one_compartment_report():
population = 'p1'
output_file = tempfile.mkstemp(suffix='h5')[1]
cr = CompartmentReport(output_file,
mode='w',
default_population=population,
tstart=0.0,
tstop=100.0,
dt=0.1)
cr.add_cell(node_id=0, element_ids=[0], element_pos=[0.0])
for i in range(1000):
cr.record_cell(0, [i / 100.0], tstep=i)
cr.close()
report_h5 = h5py.File(output_file, 'r')
report_grp = report_h5['/report/{}'.format(population)]
assert ('data' in report_grp)
data_ds = report_grp['data'][()]
assert (report_grp['data'].size == 1000)
assert (np.isreal(data_ds.dtype))
assert (data_ds[0] == 0.00)
assert (data_ds[-1] == 9.99)
assert ('mapping' in report_grp)
mapping_grp = report_grp['mapping']
assert (all(mapping_grp['element_ids'][()] == [0]))
assert (mapping_grp['element_pos'][()] == [0.0])
assert (mapping_grp['index_pointer'][()].size == 2)
assert (mapping_grp['node_ids'][()] == [0])
assert (np.allclose(mapping_grp['time'][()], [0.0, 100.0, 0.1]))
os.remove(output_file)
def test_multi_population_report():
cells = [(0, 10, 'v1'), (1, 50, 'v1'), (2, 100, 'v1'), (3, 1, 'v1'),
(4, 200, 'v1'), (0, 100, 'v2'), (1, 50, 'v2')]
rank_cells = [c for c in cells[rank::nhosts]]
output_file = os.path.join(cpath, 'output/multi_population_report.h5')
cr = CompartmentReport(output_file,
mode='w',
tstart=0.0,
tstop=100.0,
dt=0.1,
variable='Vm',
units='mV')
for node_id, n_elements, pop in rank_cells:
cr.add_cell(node_id=node_id,
population=pop,
element_ids=np.arange(n_elements),
element_pos=np.zeros(n_elements))
for i in range(1000):
for node_id, n_elements, pop in rank_cells:
cr.record_cell(node_id,
population=pop,
vals=[node_id + i / 1000.0] * n_elements,
tstep=i)
cr.close()
if rank == 0:
report_h5 = h5py.File(output_file, 'r')
report_grp = report_h5['/report/{}'.format('v1')]
assert ('data' in report_grp)
data_ds = report_grp['data'][()]
assert (report_grp['data'].shape == (1000, 361))
assert (np.isreal(data_ds.dtype))
assert ('mapping' in report_grp)
mapping_grp = report_grp['mapping']
assert (mapping_grp['element_ids'].size == 361)
assert (mapping_grp['element_pos'].size == 361)
assert (mapping_grp['index_pointer'].size == 6)
assert (np.all(np.sort(mapping_grp['node_ids'][()]) == np.arange(5)))
assert (np.allclose(mapping_grp['time'][()], [0.0, 100.0, 0.1]))
report_grp = report_h5['/report/{}'.format('v2')]
assert ('data' in report_grp)
data_ds = report_grp['data'][()]
assert (report_grp['data'].shape == (1000, 150))
assert (np.isreal(data_ds.dtype))
assert ('mapping' in report_grp)
mapping_grp = report_grp['mapping']
assert (mapping_grp['element_ids'].size == 150)
assert (mapping_grp['element_pos'].size == 150)
assert (mapping_grp['index_pointer'].size == 3)
assert (np.all(np.sort(mapping_grp['node_ids'][()]) == [0, 1]))
assert (np.allclose(mapping_grp['time'][()], [0.0, 100.0, 0.1]))
os.remove(output_file)
barrier()
def test_compartment_reader():
report = CompartmentReport(output_file, 'r')
assert (len(report.populations) == 2)
# Check v1 population
assert ('v1' in report.populations)
v1_grp = report['v1']
assert (np.all(np.sort(v1_grp.node_ids()) == np.arange(5)))
assert (v1_grp.tstart() == 0.0)
assert (v1_grp.tstop() == 100.0)
assert (v1_grp.dt() == 0.1)
assert (v1_grp.units() == 'mV')
assert (v1_grp.n_elements() == 361)
assert (v1_grp.element_pos().size == 361)
assert (v1_grp.element_ids().size == 361)
assert (v1_grp.data().shape == (1000, 361))
assert (v1_grp.data(0).shape == (1000, 10))
assert (v1_grp.data(0, time_window=(0.0, 50.0)).shape == (500, 10))
assert (np.all(np.unique(v1_grp.data(0)) == [0.0]))
assert (v1_grp.data(1).shape == (1000, 50))
assert (np.all(np.unique(v1_grp.data(1)) == [1.0]))
assert (v1_grp.data(2).shape == (1000, 100))
assert (np.all(np.unique(v1_grp.data(2)) == [2.0]))
assert (v1_grp.data(3).shape == (1000, 1))
assert (np.all(np.unique(v1_grp.data(3)) == [3.0]))
assert (v1_grp.data(4).shape == (1000, 200))
assert (np.all(np.unique(v1_grp.data(4)) == [4.0]))
# Check v2 population
assert ('v2' in report.populations)
v1_grp = report['v2']
assert (np.all(np.sort(v1_grp.node_ids()) == np.arange(2)))
assert (v1_grp.tstart() == 0.0)
assert (v1_grp.tstop() == 100.0)
assert (v1_grp.dt() == 0.1)
assert (v1_grp.units() == 'mV')
assert (v1_grp.n_elements() == 150)
assert (v1_grp.element_pos().size == 150)
assert (v1_grp.element_ids().size == 150)
assert (v1_grp.data().shape == (1000, 150))
assert (v1_grp.data(0).shape == (1000, 100))
assert (v1_grp.data(0, time_window=(0.0, 50.0)).shape == (500, 100))
assert (np.all(np.unique(v1_grp.data(0)) == [0.0]))
assert (v1_grp.data(1).shape == (1000, 50))
assert (np.all(np.unique(v1_grp.data(1)) == [1.0]))
def load_reports(config_file):
cfg = ConfigDict.from_json(config_file)
reports = []
for report_name, report in cfg.reports.items():
if report['module'] not in ['membrane_report', 'multimeter_report']:
continue
report_file = report[
'file_name'] if 'file_name' in report else '{}.h5'.format(
report_name)
report_file = report_file if os.path.isabs(
report_file) else os.path.join(cfg.output_dir, report_file)
reports.append(CompartmentReport(report_file, 'r'))
return reports
def get_var_recorder(node_recording_df):
if self._var_recorder is None:
self._var_recorder = CompartmentReport(
self._file_name,
mode='w',
variable=self._variable_name[0],
default_population=self._population,
tstart=node_recording_df['time'].min(),
tstop=node_recording_df['time'].max(),
dt=self._interval,
n_steps=len(node_recording_df),
mpi_size=1)
if self._to_h5 and MPI_RANK == 0:
for gid in self._gids:
self._var_recorder.add_cell(
gid,
element_ids=[0],
element_pos=[0.0],
population=self._population)
self._var_recorder.initialize()
return self._var_recorder
def test_multi_compartment_report():
population = 'cortical'
output_file = tempfile.mkstemp(suffix='h5')[1]
n_elements = 50
cr = CompartmentReport(output_file,
mode='w',
default_population=population,
tstart=0.0,
tstop=100.0,
dt=0.1)
cr.add_cell(node_id=0,
element_ids=np.arange(n_elements),
element_pos=[0.5] * n_elements)
cr.initialize()
for i in range(1000):
cr.record_cell(0, [i + j for j in range(n_elements)], tstep=i)
cr.close()
report_h5 = h5py.File(output_file, 'r')
report_grp = report_h5['/report/{}'.format(population)]
assert ('data' in report_grp)
data_ds = report_grp['data'][()]
assert (report_grp['data'].shape == (1000, n_elements))
assert (np.isreal(data_ds.dtype))
assert (data_ds[0, 0] == 0.0)
assert (data_ds[999, n_elements - 1] == 999.0 + n_elements - 1)
assert ('mapping' in report_grp)
mapping_grp = report_grp['mapping']
assert (np.allclose(mapping_grp['element_ids'][()], np.arange(n_elements)))
assert (np.allclose(mapping_grp['element_pos'][()], [0.5] * n_elements))
assert (mapping_grp['index_pointer'][()].size == 2)
assert (mapping_grp['node_ids'][()] == [0])
assert (np.allclose(mapping_grp['time'][()], [0.0, 100.0, 0.1]))
os.remove(output_file)
def test_block_record():
cells = [(0, 10), (1, 50), (2, 100), (3, 1), (4, 200)]
total_elements = sum(n_elements for _, n_elements in cells)
rank_cells = [c for c in cells[rank::nhosts]]
output_file = os.path.join(cpath, 'output/multi_compartment_report.h5')
population = 'cortical'
cr = CompartmentReport(output_file,
mode='w',
default_population=population,
tstart=0.0,
tstop=100.0,
dt=0.1,
variable='mebrane_potential',
units='mV')
for node_id, n_elements in rank_cells:
cr.add_cell(node_id=node_id,
element_ids=np.arange(n_elements),
element_pos=np.zeros(n_elements))
for node_id, n_elements in rank_cells:
cr.record_cell_block(node_id,
np.full((1000, n_elements),
fill_value=node_id + 1),
beg_step=0,
end_step=1000)
cr.close()
if rank == 0:
report_h5 = h5py.File(output_file, 'r')
report_grp = report_h5['/report/{}'.format(population)]
assert ('data' in report_grp)
data_ds = report_grp['data'][()]
assert (report_grp['data'].shape == (1000, total_elements))
assert (np.isreal(data_ds.dtype))
assert ('mapping' in report_grp)
mapping_grp = report_grp['mapping']
assert (mapping_grp['element_ids'].size == total_elements)
assert (mapping_grp['element_pos'].size == total_elements)
assert (mapping_grp['index_pointer'].size == 6)
assert (np.all(np.sort(mapping_grp['node_ids'][()]) == np.arange(5)))
assert (np.allclose(mapping_grp['time'][()], [0.0, 100.0, 0.1]))
os.remove(output_file)
barrier()
def test_custom_columns():
cells = [(0, 10), (1, 50), (2, 100), (3, 1), (4, 200)]
total_elements = sum(n_elements for _, n_elements in cells)
rank_cells = [c for c in cells[rank::nhosts]]
output_file = os.path.join(cpath, 'output/multi_compartment_report.h5')
population = 'cortical'
cr = CompartmentReport(output_file,
mode='w',
default_population=population,
tstart=0.0,
tstop=100.0,
dt=0.1,
variable='mebrane_potential',
units='mV')
for node_id, n_elements in rank_cells:
cr.add_cell(node_id=node_id,
element_ids=np.arange(n_elements),
element_pos=np.zeros(n_elements),
synapses=[node_id * 2] * n_elements)
for i in range(1000):
for node_id, n_elements in rank_cells:
cr.record_cell(node_id, [node_id + i / 1000.0] * n_elements,
tstep=i)
cr.close()
if rank == 0:
report_h5 = h5py.File(output_file, 'r')
report_grp = report_h5['/report/{}'.format(population)]
assert ('mapping' in report_grp)
mapping_grp = report_grp['mapping']
assert (mapping_grp['element_ids'].size == total_elements)
assert (mapping_grp['element_pos'].size == total_elements)
assert (mapping_grp['index_pointer'].size == 6)
assert (np.all(np.sort(mapping_grp['node_ids'][()]) == np.arange(5)))
assert (np.allclose(mapping_grp['time'][()], [0.0, 100.0, 0.1]))
assert ('synapses' in mapping_grp.keys())
assert (mapping_grp['synapses'][()].size == total_elements)
os.remove(output_file)
barrier()
def build_file():
rank_cells = [(0, 10, 'v1'), (1, 50, 'v1'), (2, 100, 'v1'), (3, 1, 'v1'),
(4, 200, 'v1'), (0, 100, 'v2'), (1, 50, 'v2')]
cr = CompartmentReport(output_file,
mode='w',
tstart=0.0,
tstop=100.0,
dt=0.1,
variable='Vm',
units='mV')
for node_id, n_elements, pop in rank_cells:
cr.add_cell(node_id=node_id,
population=pop,
element_ids=np.arange(n_elements),
element_pos=np.zeros(n_elements))
for i in range(1000):
for node_id, n_elements, pop in rank_cells:
cr.record_cell(node_id,
population=pop,
vals=[node_id] * n_elements,
tstep=i)
cr.close()
class MultimeterMod(object):
def __init__(self,
tmp_dir,
file_name,
variable_name,
cells,
tstart=None,
tstop=None,
interval=None,
to_h5=True,
delete_dat=True,
**opt_params):
"""For recording neuron properties using a NEST multimeter object
:param tmp_dir: ouput directory
:param file_name: Name of (SONATA hdf5) file that will be saved to
:param variable_name: A list of the variable(s) being recorded. Must be valid according to the cells
:param cells: A node-set or list of gids to record from
:param tstart: Start time of the recording (if None will default to sim.tstart)
:param tstop: Stop time of recording (if None will default to sim.tstop)
:param interval: Recording time step (if None will default to sim.dt)
:param to_h5: True to save to sonata .h5 format (default: True)
:param delete_dat: True to delete the .dat files created by NEST (default True)
:param opt_params:
"""
self._output_dir = tmp_dir
self._file_name = file_name if os.path.isabs(
file_name) else os.path.join(self._output_dir, file_name)
self._variable_name = variable_name
self._node_set = cells
self._tstart = tstart
self._tstop = tstop
self._interval = interval
self._to_h5 = to_h5
self._delete_dat = delete_dat
self._gids = None # global ids will be the NEST ids assigned to each cell
self._multimeter = None
self._population = None
self._min_delay = 1.0 # Required for calculating steps recorded
self.__output_label = os.path.join(
self._output_dir,
'__bmtk_nest_{}'.format(os.path.basename(self._file_name)))
self._var_recorder = None # CellVarRecorder(self._file_name, self._output_dir, self._variable_name, buffer_data=False)
def initialize(self, sim):
node_set = sim.net.get_node_set(self._node_set)
self._gids = list(set(node_set.gids()))
self._gids.sort()
self._population = node_set.population_names()[0]
self._tstart = self._tstart or sim.tstart
self._tstop = self._tstop or sim.tstop
self._interval = self._interval or sim.dt
self._multimeter = create_multimeter(self._tstart, self._tstop,
self._variable_name,
self.__output_label)
nest.SetStatus(self._multimeter, 'interval', self._interval)
nest.Connect(self._multimeter, self._gids)
def finalize(self, sim):
io.barrier(
) # Makes sure all nodes finish, but not sure if actually required by nest
# min_delay needs to be fetched after simulation otherwise the value will be off. There also seems to be some
# MPI barrier inside GetKernelStatus
self._min_delay = nest.GetKernelStatus('min_delay')
if self._to_h5 and MPI_RANK == 0:
# Initialize hdf5 file including preallocated data block of recorded variables
# Unfortantely with NEST the final time-step recorded can't be calculated in advanced, and even with the
# same min/max_delay can be different. We need to read the output-file to get n_steps
def get_var_recorder(node_recording_df):
if self._var_recorder is None:
self._var_recorder = CompartmentReport(
self._file_name,
mode='w',
variable=self._variable_name[0],
default_population=self._population,
tstart=node_recording_df['time'].min(),
tstop=node_recording_df['time'].max(),
dt=self._interval,
n_steps=len(node_recording_df),
mpi_size=1)
if self._to_h5 and MPI_RANK == 0:
for gid in self._gids:
pop_id = gid_map.get_pool_id(gid)
self._var_recorder.add_cell(
pop_id.node_id,
element_ids=[0],
element_pos=[0.0],
population=pop_id.population)
self._var_recorder.initialize()
return self._var_recorder
gid_map = sim.net.gid_map
for nest_file in glob.glob('{}*'.format(self.__output_label)):
# report_df = pd.read_csv(nest_file, index_col=False, names=['nest_id', 'time']+self._variable_name,
# sep='\t', comment='#')
report_df = read_dat(nest_file, self._variable_name)
# print(report_df)
# exit()
for grp_id, grp_df in report_df.groupby(by='nest_id'):
pop_id = gid_map.get_pool_id(grp_id)
vr = get_var_recorder(grp_df)
for var_name in self._variable_name:
vr.record_cell_block(
node_id=pop_id.node_id,
vals=grp_df[var_name],
beg_step=0,
end_step=vr[pop_id.population].n_steps(),
population=pop_id.population)
if self._delete_dat:
# remove csv file created by nest
os.remove(nest_file)
self._var_recorder.close()
io.barrier()