def test_multi_sub_sorting_extractor(self):
N = self.RX.get_num_frames()
SX_multi = se.MultiSortingExtractor(
sortings=[self.SX, self.SX, self.SX], )
SX_multi.set_unit_property(unit_id=1, property_name='dummy', value=5)
SX_sub = se.SubSortingExtractor(parent_sorting=SX_multi, start_frame=0)
self._check_sortings_equal(SX_multi, SX_sub)
self.assertEqual(SX_multi.get_unit_property(1, 'dummy'),
SX_sub.get_unit_property(1, 'dummy'))
N = self.RX.get_num_frames()
SX_multi = se.MultiSortingExtractor(sortings=[self.SX, self.SX2], )
SX_sub1 = se.SubSortingExtractor(parent_sorting=SX_multi,
start_frame=0,
end_frame=N)
self._check_sortings_equal(SX_multi, SX_sub1)
def get_result(self):
sorting_list = self.get_result_list()
if len(sorting_list) == 1:
sorting = sorting_list[0]
else:
for i, sorting in enumerate(sorting_list):
property_name = self.recording_list[i].get_channel_property(
self.recording_list[i].get_channel_ids()[0],
self.grouping_property)
if sorting is not None:
for unit in sorting.get_unit_ids():
sorting.set_unit_property(unit, self.grouping_property,
property_name)
# reassemble the sorting outputs
sorting_list = [sort for sort in sorting_list if sort is not None]
multi_sorting = se.MultiSortingExtractor(sortings=sorting_list)
sorting = multi_sorting
if self.delete_folders:
for out in self.output_folders:
if self.verbose:
print("Removing ", str(out))
shutil.rmtree(str(out), ignore_errors=True)
sorting.set_sampling_frequency(
self.recording_list[0].get_sampling_frequency())
return sorting
def _spikeSortByProperty(recording, spikesorter, property, parallel, **kwargs):
'''
Parameters
----------
recording
sorter
kwargs
Returns
-------
'''
recording_list = se.getSubExtractorsByProperty(recording, property)
sorting_list = _parallelSpikeSorting(recording_list, spikesorter, parallel,
**kwargs)
# add group property
for i, sorting in enumerate(sorting_list):
group = recording_list[i].getChannelProperty(
recording_list[i].getChannelIds()[0], 'group')
for unit in sorting.getUnitIds():
sorting.setUnitProperty(unit, 'group', group)
# reassemble the sorting outputs
multi_sorting = se.MultiSortingExtractor(sortings=sorting_list)
return multi_sorting
def test_multi_sub_sorting_extractor(self):
N = self.RX.get_num_frames()
SX_multi = se.MultiSortingExtractor(
sortings=[self.SX, self.SX, self.SX], start_frames=[0, N, 2 * N])
SX_sub = se.SubSortingExtractor(parent_sorting=SX_multi,
start_frame=N,
end_frame=2 * N)
self._check_sortings_equal(self.SX, SX_sub)
self.assertEqual(SX_multi.get_sampling_frequency(),
self.SX.get_sampling_frequency())
self.assertEqual(SX_sub.get_sampling_frequency(),
self.SX.get_sampling_frequency())
N = self.RX.get_num_frames()
SX_multi = se.MultiSortingExtractor(
sortings=[self.SX, self.SX, self.SX], start_frames=[0, N, 2 * N])
SX_sub = se.SubSortingExtractor(parent_sorting=SX_multi, start_frame=0)
self._check_sortings_equal(SX_multi, SX_sub)
N = self.RX.get_num_frames()
SX_multi = se.MultiSortingExtractor(
sortings=[self.SX, self.SX, self.SX], start_frames=[2 * N, 0, N])
SX_sub = se.SubSortingExtractor(parent_sorting=SX_multi,
start_frame=N,
end_frame=2 * N)
self._check_sortings_equal(self.SX, SX_sub)
N = self.RX.get_num_frames()
SX_multi = se.MultiSortingExtractor(
sortings=[self.SX, self.SX, self.SX], start_frames=[0, 0, 0])
SX_sub = se.SubSortingExtractor(parent_sorting=SX_multi, start_frame=0)
self._check_sortings_equal(SX_multi, SX_sub)
N = self.RX.get_num_frames()
SX_multi = se.MultiSortingExtractor(sortings=[self.SX, self.SX2],
start_frames=[0, 0])
SX_sub1 = se.SubSortingExtractor(parent_sorting=SX_multi,
start_frame=0,
end_frame=N)
self._check_sortings_equal(SX_multi, SX_sub1)
def test_multi_sub_sorting_extractor(self):
N = self.RX.getNumFrames()
SX_multi = se.MultiSortingExtractor(
sortings=[self.SX, self.SX, self.SX], start_frames=[0, N, 2 * N])
SX_sub = se.SubSortingExtractor(parent_sorting=SX_multi,
start_frame=N,
end_frame=2 * N)
self._check_sortings_equal(self.SX, SX_sub)
N = self.RX.getNumFrames()
SX_multi = se.MultiSortingExtractor(
sortings=[self.SX, self.SX, self.SX], start_frames=[0, N, 2 * N])
SX_sub = se.SubSortingExtractor(parent_sorting=SX_multi, start_frame=0)
self._check_sortings_equal(SX_multi, SX_sub)
N = self.RX.getNumFrames()
SX_multi = se.MultiSortingExtractor(
sortings=[self.SX, self.SX, self.SX], start_frames=[2 * N, 0, N])
SX_sub = se.SubSortingExtractor(parent_sorting=SX_multi,
start_frame=N,
end_frame=2 * N)
self._check_sortings_equal(self.SX, SX_sub)
N = self.RX.getNumFrames()
SX_multi = se.MultiSortingExtractor(
sortings=[self.SX, self.SX, self.SX], start_frames=[0, 0, 0])
SX_sub = se.SubSortingExtractor(parent_sorting=SX_multi, start_frame=0)
self._check_sortings_equal(SX_multi, SX_sub)
N = self.RX.getNumFrames()
SX_multi = se.MultiSortingExtractor(sortings=[self.SX, self.SX2],
start_frames=[0, 0])
SX_sub1 = se.SubSortingExtractor(parent_sorting=SX_multi,
start_frame=0,
end_frame=N)
self._check_sortings_equal(SX_multi, SX_sub1)
def test_dump_load_multi_sub_extractor(self):
# generate dumpable formats
path1 = self.test_dir + '/mda'
path2 = path1 + '/firings_true.mda'
se.MdaRecordingExtractor.write_recording(self.RX, path1)
se.MdaSortingExtractor.write_sorting(self.SX, path2)
RX_mda = se.MdaRecordingExtractor(path1)
SX_mda = se.MdaSortingExtractor(path2)
RX_multi_chan = se.MultiRecordingChannelExtractor(recordings=[RX_mda, RX_mda, RX_mda])
check_dumping(RX_multi_chan)
RX_multi_time = se.MultiRecordingTimeExtractor(recordings=[RX_mda, RX_mda, RX_mda], )
check_dumping(RX_multi_time)
RX_multi_chan = se.SubRecordingExtractor(RX_mda, channel_ids=[0, 1])
check_dumping(RX_multi_chan)
SX_sub = se.SubSortingExtractor(SX_mda, unit_ids=[1, 2])
check_dumping(SX_sub)
SX_multi = se.MultiSortingExtractor(sortings=[SX_mda, SX_mda, SX_mda])
check_dumping(SX_multi)
def get_result_from_folder(output_folder):
output_folder = Path(output_folder)
sortings = []
recording_folders = [
f for f in os.listdir(output_folder)
if (os.path.isdir(os.path.join(output_folder, f))
and f.startswith('recording'))
]
for rf in recording_folders:
result_fname = str(output_folder / rf)
sorting = se.CombinatoSortingExtractor(datapath=result_fname)
for u in sorting.get_unit_ids():
sorting.set_unit_property(unit_id=u,
property_name='channel_id',
value=int(rf[9:]))
sortings.append(sorting)
return se.MultiSortingExtractor(sortings)
def test_neuroscope_extractors(self):
# NeuroscopeRecordingExtractor tests
nscope_dir = Path(self.test_dir) / 'neuroscope_rec0'
dat_file = nscope_dir / 'neuroscope_rec0.dat'
se.NeuroscopeRecordingExtractor.write_recording(self.RX, nscope_dir)
RX_ns = se.NeuroscopeRecordingExtractor(dat_file)
check_recording_return_types(RX_ns)
check_recordings_equal(self.RX, RX_ns, force_dtype='int32')
check_dumping(RX_ns)
check_recording_return_types(RX_ns)
check_recordings_equal(self.RX, RX_ns, force_dtype='int32')
check_dumping(RX_ns)
del RX_ns
# overwrite
nscope_dir = Path(self.test_dir) / 'neuroscope_rec1'
dat_file = nscope_dir / 'neuroscope_rec1.dat'
se.NeuroscopeRecordingExtractor.write_recording(recording=self.RX,
save_path=nscope_dir)
RX_ns = se.NeuroscopeRecordingExtractor(dat_file)
check_recording_return_types(RX_ns)
check_recordings_equal(self.RX, RX_ns)
check_dumping(RX_ns)
# NeuroscopeSortingExtractor tests
nscope_dir = Path(self.test_dir) / 'neuroscope_sort0'
sort_name = 'neuroscope_sort0'
initial_sorting_resfile = Path(
self.test_dir) / sort_name / f'{sort_name}.res'
initial_sorting_clufile = Path(
self.test_dir) / sort_name / f'{sort_name}.clu'
se.NeuroscopeSortingExtractor.write_sorting(self.SX, nscope_dir)
SX_neuroscope = se.NeuroscopeSortingExtractor(
resfile_path=initial_sorting_resfile,
clufile_path=initial_sorting_clufile)
check_sorting_return_types(SX_neuroscope)
check_sortings_equal(self.SX, SX_neuroscope)
check_dumping(SX_neuroscope)
SX_neuroscope_no_mua = se.NeuroscopeSortingExtractor(
resfile_path=initial_sorting_resfile,
clufile_path=initial_sorting_clufile,
keep_mua_units=False)
check_sorting_return_types(SX_neuroscope_no_mua)
check_dumping(SX_neuroscope_no_mua)
# Test for extra argument 'keep_mua_units' resulted in the right output
SX_neuroscope_no_mua = se.NeuroscopeSortingExtractor(
resfile_path=initial_sorting_resfile,
clufile_path=initial_sorting_clufile,
keep_mua_units=False)
check_sorting_return_types(SX_neuroscope_no_mua)
check_dumping(SX_neuroscope_no_mua)
num_original_units = len(SX_neuroscope.get_unit_ids())
self.assertEqual(list(SX_neuroscope.get_unit_ids()),
list(range(1, num_original_units + 1)))
self.assertEqual(list(SX_neuroscope_no_mua.get_unit_ids()),
list(range(1, num_original_units)))
# Tests for the auto-detection of format for NeuroscopeSortingExtractor
SX_neuroscope_from_fp = se.NeuroscopeSortingExtractor(
folder_path=nscope_dir)
check_sorting_return_types(SX_neuroscope_from_fp)
check_sortings_equal(self.SX, SX_neuroscope_from_fp)
check_dumping(SX_neuroscope_from_fp)
# Tests for the NeuroscopeMultiSortingExtractor
nscope_dir = Path(self.test_dir) / 'neuroscope_sort1'
SX_multisorting = se.MultiSortingExtractor(
sortings=[self.SX, self.SX]) # re-using same SX for simplicity
se.NeuroscopeMultiSortingExtractor.write_sorting(
SX_multisorting, nscope_dir)
SX_neuroscope_mse = se.NeuroscopeMultiSortingExtractor(nscope_dir)
check_sorting_return_types(SX_neuroscope_mse)
check_sortings_equal(SX_multisorting, SX_neuroscope_mse)
check_dumping(SX_neuroscope_mse)
def yuta2nwb(
session_path='D:/BuzsakiData/SenzaiY/YutaMouse41/YutaMouse41-150903',
# '/Users/bendichter/Desktop/Buzsaki/SenzaiBuzsaki2017/YutaMouse41/YutaMouse41-150903',
subject_xls=None,
include_spike_waveforms=True,
stub=True,
cache_spec=True):
subject_path, session_id = os.path.split(session_path)
fpath_base = os.path.split(subject_path)[0]
identifier = session_id
mouse_number = session_id[9:11]
if '-' in session_id:
subject_id, date_text = session_id.split('-')
b = False
else:
subject_id, date_text = session_id.split('b')
b = True
if subject_xls is None:
subject_xls = os.path.join(subject_path,
'YM' + mouse_number + ' exp_sheet.xlsx')
else:
if not subject_xls[-4:] == 'xlsx':
subject_xls = os.path.join(subject_xls,
'YM' + mouse_number + ' exp_sheet.xlsx')
session_start_time = dateparse(date_text, yearfirst=True)
df = pd.read_excel(subject_xls)
subject_data = {}
for key in [
'genotype', 'DOB', 'implantation', 'Probe', 'Surgery',
'virus injection', 'mouseID'
]:
names = df.iloc[:, 0]
if key in names.values:
subject_data[key] = df.iloc[np.argmax(names == key), 1]
if isinstance(subject_data['DOB'], datetime):
age = session_start_time - subject_data['DOB']
else:
age = None
subject = Subject(subject_id=subject_id,
age=str(age),
genotype=subject_data['genotype'],
species='mouse')
nwbfile = NWBFile(
session_description='mouse in open exploration and theta maze',
identifier=identifier,
session_start_time=session_start_time.astimezone(),
file_create_date=datetime.now().astimezone(),
experimenter='Yuta Senzai',
session_id=session_id,
institution='NYU',
lab='Buzsaki',
subject=subject,
related_publications='DOI:10.1016/j.neuron.2016.12.011')
print('reading and writing raw position data...', end='', flush=True)
ns.add_position_data(nwbfile, session_path)
shank_channels = ns.get_shank_channels(session_path)[:8]
nshanks = len(shank_channels)
all_shank_channels = np.concatenate(shank_channels)
print('setting up electrodes...', end='', flush=True)
hilus_csv_path = os.path.join(fpath_base, 'early_session_hilus_chans.csv')
lfp_channel = get_reference_elec(subject_xls,
hilus_csv_path,
session_start_time,
session_id,
b=b)
custom_column = [{
'name': 'theta_reference',
'description':
'this electrode was used to calculate LFP canonical bands',
'data': all_shank_channels == lfp_channel
}]
ns.write_electrode_table(nwbfile,
session_path,
custom_columns=custom_column,
max_shanks=max_shanks)
print('reading raw electrode data...', end='', flush=True)
if stub:
# example recording extractor for fast testing
xml_filepath = os.path.join(session_path, session_id + '.xml')
xml_root = et.parse(xml_filepath).getroot()
acq_sampling_frequency = float(
xml_root.find('acquisitionSystem').find('samplingRate').text)
num_channels = 4
num_frames = 10000
X = np.random.normal(0, 1, (num_channels, num_frames))
geom = np.random.normal(0, 1, (num_channels, 2))
X = (X * 100).astype(int)
sre = se.NumpyRecordingExtractor(
timeseries=X, sampling_frequency=acq_sampling_frequency, geom=geom)
else:
nre = se.NeuroscopeRecordingExtractor('{}/{}.dat'.format(
session_path, session_id))
sre = se.SubRecordingExtractor(nre, channel_ids=all_shank_channels)
print('writing raw electrode data...', end='', flush=True)
se.NwbRecordingExtractor.add_electrical_series(sre, nwbfile)
print('done.')
print('reading spiking units...', end='', flush=True)
if stub:
spike_times = [200, 300, 400]
num_frames = 10000
allshanks = []
for k in range(nshanks):
SX = se.NumpySortingExtractor()
for j in range(len(spike_times)):
SX.add_unit(unit_id=j + 1,
times=np.sort(
np.random.uniform(0, num_frames,
spike_times[j])))
allshanks.append(SX)
se_allshanks = se.MultiSortingExtractor(allshanks)
se_allshanks.set_sampling_frequency(acq_sampling_frequency)
else:
se_allshanks = se.NeuroscopeMultiSortingExtractor(session_path,
keep_mua_units=False)
electrode_group = []
for shankn in np.arange(1, nshanks + 1, dtype=int):
for id in se_allshanks.sortings[shankn - 1].get_unit_ids():
electrode_group.append(nwbfile.electrode_groups['shank' +
str(shankn)])
df_unit_features = get_UnitFeatureCell_features(fpath_base, session_id,
session_path)
celltype_names = []
for celltype_id, region_id in zip(df_unit_features['fineCellType'].values,
df_unit_features['region'].values):
if celltype_id == 1:
if region_id == 3:
celltype_names.append('pyramidal cell')
elif region_id == 4:
celltype_names.append('granule cell')
else:
raise Exception('unknown type')
elif not np.isfinite(celltype_id):
celltype_names.append('missing')
else:
celltype_names.append(celltype_dict[celltype_id])
# Add custom column data into the SortingExtractor so it can be written by the converter
# Note there is currently a hidden assumption that the way in which the NeuroscopeSortingExtractor
# merges the cluster IDs matches one-to-one with the get_UnitFeatureCell_features extraction
property_descriptions = {
'cell_type': 'name of cell type',
'global_id': 'global id for cell for entire experiment',
'shank_id': '0-indexed id of cluster of shank',
'electrode_group': 'the electrode group that each spike unit came from'
}
property_values = {
'cell_type': celltype_names,
'global_id': df_unit_features['unitID'].values,
'shank_id': [x - 2 for x in df_unit_features['unitIDshank'].values],
# - 2 b/c the get_UnitFeatureCell_features removes 0 and 1 IDs from each shank
'electrode_group': electrode_group
}
for unit_id in se_allshanks.get_unit_ids():
for property_name in property_descriptions.keys():
se_allshanks.set_unit_property(
unit_id, property_name,
property_values[property_name][unit_id])
se.NwbSortingExtractor.write_sorting(
se_allshanks,
nwbfile=nwbfile,
property_descriptions=property_descriptions)
print('done.')
# Read and write LFP's
print('reading LFPs...', end='', flush=True)
lfp_fs, all_channels_lfp_data = ns.read_lfp(session_path, stub=stub)
lfp_data = all_channels_lfp_data[:, all_shank_channels]
print('writing LFPs...', flush=True)
# lfp_data[:int(len(lfp_data)/4)]
lfp_ts = ns.write_lfp(nwbfile,
lfp_data,
lfp_fs,
name='lfp',
description='lfp signal for all shank electrodes')
# Read and add special environmental electrodes
for name, channel in special_electrode_dict.items():
ts = TimeSeries(
name=name,
description=
'environmental electrode recorded inline with neural data',
data=all_channels_lfp_data[:, channel],
rate=lfp_fs,
unit='V',
#conversion=np.nan,
resolution=np.nan)
nwbfile.add_acquisition(ts)
# compute filtered LFP
print('filtering LFP...', end='', flush=True)
all_lfp_phases = []
for passband in ('theta', 'gamma'):
lfp_fft = filter_lfp(
lfp_data[:, all_shank_channels == lfp_channel].ravel(),
lfp_fs,
passband=passband)
lfp_phase, _ = hilbert_lfp(lfp_fft)
all_lfp_phases.append(lfp_phase[:, np.newaxis])
data = np.dstack(all_lfp_phases)
print('done.', flush=True)
if include_spike_waveforms:
print('writing waveforms...', end='', flush=True)
nshanks = min((max_shanks, len(ns.get_shank_channels(session_path))))
for shankn in np.arange(nshanks, dtype=int) + 1:
# Get spike activty from .spk file on a per-shank and per-sample basis
ns.write_spike_waveforms(nwbfile, session_path, shankn, stub=stub)
print('done.', flush=True)
# Get the LFP Decomposition Series
decomp_series = DecompositionSeries(
name='LFPDecompositionSeries',
description='Theta and Gamma phase for reference LFP',
data=data,
rate=lfp_fs,
source_timeseries=lfp_ts,
metric='phase',
unit='radians')
decomp_series.add_band(band_name='theta', band_limits=(4, 10))
decomp_series.add_band(band_name='gamma', band_limits=(30, 80))
check_module(nwbfile, 'ecephys',
'contains processed extracellular electrophysiology data'
).add_data_interface(decomp_series)
[nwbfile.add_stimulus(x) for x in ns.get_events(session_path)]
# create epochs corresponding to experiments/environments for the mouse
sleep_state_fpath = os.path.join(session_path,
'{}--StatePeriod.mat'.format(session_id))
exist_pos_data = any(
os.path.isfile(
os.path.join(session_path, '{}__{}.mat'.format(
session_id, task_type['name']))) for task_type in task_types)
if exist_pos_data:
nwbfile.add_epoch_column('label', 'name of epoch')
for task_type in task_types:
label = task_type['name']
file = os.path.join(session_path, session_id + '__' + label + '.mat')
if os.path.isfile(file):
print('loading position for ' + label + '...', end='', flush=True)
pos_obj = Position(name=label + '_position')
matin = loadmat(file)
tt = matin['twhl_norm'][:, 0]
exp_times = find_discontinuities(tt)
if 'conversion' in task_type:
conversion = task_type['conversion']
else:
conversion = np.nan
for pos_type in ('twhl_norm', 'twhl_linearized'):
if pos_type in matin:
pos_data_norm = matin[pos_type][:, 1:]
spatial_series_object = SpatialSeries(
name=label + '_{}_spatial_series'.format(pos_type),
data=H5DataIO(pos_data_norm, compression='gzip'),
reference_frame='unknown',
conversion=conversion,
resolution=np.nan,
timestamps=H5DataIO(tt, compression='gzip'))
pos_obj.add_spatial_series(spatial_series_object)
check_module(
nwbfile, 'behavior',
'contains processed behavioral data').add_data_interface(
pos_obj)
for i, window in enumerate(exp_times):
nwbfile.add_epoch(start_time=window[0],
stop_time=window[1],
label=label + '_' + str(i))
print('done.')
# there are occasional mismatches between the matlab struct and the neuroscope files
# regions: 3: 'CA3', 4: 'DG'
trialdata_path = os.path.join(session_path,
session_id + '__EightMazeRun.mat')
if os.path.isfile(trialdata_path):
trials_data = loadmat(trialdata_path)['EightMazeRun']
trialdatainfo_path = os.path.join(fpath_base, 'EightMazeRunInfo.mat')
trialdatainfo = [
x[0] for x in loadmat(trialdatainfo_path)['EightMazeRunInfo'][0]
]
features = trialdatainfo[:7]
features[:2] = 'start_time', 'stop_time',
[
nwbfile.add_trial_column(x, 'description')
for x in features[4:] + ['condition']
]
for trial_data in trials_data:
if trial_data[3]:
cond = 'run_left'
else:
cond = 'run_right'
nwbfile.add_trial(start_time=trial_data[0],
stop_time=trial_data[1],
condition=cond,
error_run=trial_data[4],
stim_run=trial_data[5],
both_visit=trial_data[6])
"""
mono_syn_fpath = os.path.join(session_path, session_id+'-MonoSynConvClick.mat')
matin = loadmat(mono_syn_fpath)
exc = matin['FinalExcMonoSynID']
inh = matin['FinalInhMonoSynID']
#exc_obj = CatCellInfo(name='excitatory_connections',
# indices_values=[], cell_index=exc[:, 0] - 1, indices=exc[:, 1] - 1)
#module_cellular.add_container(exc_obj)
#inh_obj = CatCellInfo(name='inhibitory_connections',
# indices_values=[], cell_index=inh[:, 0] - 1, indices=inh[:, 1] - 1)
#module_cellular.add_container(inh_obj)
"""
if os.path.isfile(sleep_state_fpath):
matin = loadmat(sleep_state_fpath)['StatePeriod']
table = TimeIntervals(name='states',
description='sleep states of animal')
table.add_column(name='label', description='sleep state')
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append({
'start_time': row[0],
'stop_time': row[1],
'label': name
})
[
table.add_row(**row)
for row in sorted(data, key=lambda x: x['start_time'])
]
check_module(nwbfile, 'behavior',
'contains behavioral data').add_data_interface(table)
print('writing NWB file...', end='', flush=True)
if stub:
out_fname = session_path + '_stub.nwb'
else:
out_fname = session_path + '.nwb'
with NWBHDF5IO(out_fname, mode='w') as io:
io.write(nwbfile, cache_spec=cache_spec)
print('done.')
print('testing read...', end='', flush=True)
# test read
with NWBHDF5IO(out_fname, mode='r') as io:
io.read()
print('done.')