def process_consensus(project, action_id, sorters, min_agreement=None):
action = project.actions[action_id]
# if exdir_path is None:
exdir_path = _get_data_path(action)
exdir_file = exdir.File(exdir_path, plugins=[exdir.plugins.quantities])
sorting_list = []
sorter_names = []
for sorter in sorters:
phy_dir = str(exdir_file['processing']['electrophysiology']['spikesorting'][sorter]['phy'].directory)
phy_params = str(
exdir_file['processing']['electrophysiology']['spikesorting'][sorter]['phy'].directory / 'params.py')
sorting_phy = se.PhySortingExtractor(phy_dir)
sorting_list.append(sorting_phy)
sorter_names.append(sorter)
recording = se.PhyRecordingExtractor(phy_dir)
mcmp = sc.compare_multiple_sorters(sorting_list=sorting_list, name_list=sorter_names, verbose=True)
if min_agreement is None:
min_agreement = len(sorter_names) - 1
agr = mcmp.get_agreement_sorting(minimum_agreement_count=min_agreement)
print(agr.get_unit_ids())
for u in agr.get_unit_ids():
print(agr.get_unit_property(u, 'sorter_unit_ids'))
agr.get_unit_property_names(u)
consensus_dir = exdir_file['processing']['electrophysiology']['spikesorting'].require_group('consensus').require_raw('phy').directory
st.postprocessing.export_to_phy(recording, agr, output_folder=consensus_dir,
ms_before=0.5, ms_after=2, verbose=True,
grouping_property='group')
def test_export_to_phy():
folder = 'test'
if os.path.isdir(folder):
shutil.rmtree(folder)
rec, sort = se.example_datasets.create_dumpable_extractors(folder=folder,
duration=10,
num_channels=8)
export_to_phy(rec, sort, output_folder='phy')
rec.set_channel_groups([0, 0, 0, 0, 1, 1, 1, 1])
export_to_phy(rec,
sort,
output_folder='phy_group',
grouping_property='group',
recompute_info=True)
export_to_phy(rec,
sort,
output_folder='max_channels',
max_channels_per_template=4,
recompute_info=True)
export_to_phy(rec,
sort,
output_folder='phy_no_feat',
grouping_property='group',
compute_pc_features=False,
recompute_info=True)
rec_phy = se.PhyRecordingExtractor('phy')
rec_phyg = se.PhyRecordingExtractor('phy_group')
assert np.allclose(rec.get_traces(), rec_phy.get_traces())
assert np.allclose(rec.get_traces(), rec_phyg.get_traces())
assert not (Path('phy_no_feat') / 'pc_features.npy').is_file()
assert not (Path('phy_no_feat') / 'pc_feature_ind.npy').is_file()
sort_phy = se.PhySortingExtractor('phy', load_waveforms=True)
sort_phyg = se.PhySortingExtractor('phy_group', load_waveforms=True)
assert np.allclose(sort_phy.get_unit_spike_train(0),
sort.get_unit_spike_train(sort.get_unit_ids()[0]))
assert np.allclose(sort_phyg.get_unit_spike_train(2),
sort.get_unit_spike_train(sort.get_unit_ids()[2]))
assert sort_phy.get_unit_spike_features(1, 'waveforms').shape[1] == 8
assert sort_phyg.get_unit_spike_features(3, 'waveforms').shape[1] == 4
shutil.rmtree('test')
def test_export_to_phy():
rec, sort = se.example_datasets.toy_example(duration=10, num_channels=8)
export_to_phy(rec, sort, output_folder='phy')
rec.set_channel_groups(rec.get_channel_ids(), [0, 0, 0, 0, 1, 1, 1, 1])
export_to_phy(rec,
sort,
output_folder='phy_group',
grouping_property='group')
rec_phy = se.PhyRecordingExtractor('phy')
rec_phyg = se.PhyRecordingExtractor('phy_group')
assert np.allclose(rec.get_traces(), rec_phy.get_traces())
assert np.allclose(rec.get_traces(), rec_phyg.get_traces())
sort_phy = se.PhySortingExtractor('phy', load_waveforms=True)
sort_phyg = se.PhySortingExtractor('phy_group', load_waveforms=True)
assert np.allclose(sort_phy.get_unit_spike_train(0),
sort.get_unit_spike_train(sort.get_unit_ids()[0]))
assert np.allclose(sort_phyg.get_unit_spike_train(2),
sort.get_unit_spike_train(sort.get_unit_ids()[2]))
assert sort_phy.get_unit_spike_features(1, 'waveforms').shape[1] == 8
assert sort_phyg.get_unit_spike_features(3, 'waveforms').shape[1] == 4
def process_save_phy(project, action_id, sorter, save_waveforms=True, check_exists=False):
action = project.actions[action_id]
exdir_path = _get_data_path(action)
if exdir_path is None:
print('No "main" in data for action {}'.format(action.id))
return
exdir_file = exdir.File(exdir_path, plugins=exdir.plugins.quantities)
elphys = exdir_file['processing']['electrophysiology']
phy_folder = elphys['spikesorting'][sorter]['phy'].directory
if check_exists:
unit_ids = set([int(b) for a in elphys.values() if 'UnitTimes' in a for b in a['UnitTimes']])
sorting_ = se.PhySortingExtractor(phy_folder, exclude_groups=['noise'])
if set(sorting_.get_unit_ids()) == unit_ids:
print('Unit ids are the same in phy and exdir.')
return
sorting = se.PhySortingExtractor(phy_folder, exclude_cluster_groups=['noise'])
if save_waveforms:
recording = se.PhyRecordingExtractor(phy_folder)
# workaround to avoid grouping in windows
waveforms = st.postprocessing.get_unit_waveforms(recording, sorting, max_spikes_per_unit=None, memmap=False,
save_property_or_features=False, n_jobs=8, verbose=True)
if "group" in sorting.get_shared_unit_property_names():
channel_groups = recording.get_channel_groups()
for (wf, unit) in zip(waveforms, sorting.get_unit_ids()):
unit_group = sorting.get_unit_property(unit, "group")
channel_unit_group = np.where(channel_groups == int(unit_group))[0]
waveform_group = wf[:, channel_unit_group]
sorting.set_unit_spike_features(unit, "waveforms", waveform_group)
else:
for (wf, unit) in zip(waveforms, sorting.get_unit_ids()):
sorting.set_unit_spike_features(unit, "waveforms", wf)
se.ExdirSortingExtractor.write_sorting(sorting, exdir_path, sampling_frequency=sorting.params['sample_rate'],
save_waveforms=True, verbose=True)
def test_export_to_phy():
folder = 'test'
if os.path.isdir(folder):
shutil.rmtree(folder)
rec, sort = se.example_datasets.toy_example(dump_folder=folder,
dumpable=True,
duration=10,
num_channels=8)
export_to_phy(rec, sort, output_folder='phy')
rec.set_channel_groups([0, 0, 0, 0, 1, 1, 1, 1])
export_to_phy(rec,
sort,
output_folder='phy_group',
grouping_property='group',
recompute_info=True)
export_to_phy(rec,
sort,
output_folder='phy_max_channels',
max_channels_per_template=4,
recompute_info=True)
export_to_phy(rec,
sort,
output_folder='phy_no_feat',
grouping_property='group',
compute_pc_features=False,
recompute_info=True)
export_to_phy(rec,
sort,
output_folder='phy_no_amp',
compute_amplitudes=False)
export_to_phy(rec,
sort,
output_folder='phy_no_amp_feat',
compute_amplitudes=False,
compute_pc_features=False)
export_to_phy(rec, sort, output_folder='phy_par', n_jobs=2)
rec_phy = se.PhyRecordingExtractor('phy')
rec_phyg = se.PhyRecordingExtractor('phy_group')
assert np.allclose(rec.get_traces(), rec_phy.get_traces())
assert np.allclose(rec.get_traces(), rec_phyg.get_traces())
assert not (Path('phy_no_feat') / 'pc_features.npy').is_file()
assert not (Path('phy_no_feat') / 'pc_feature_ind.npy').is_file()
assert not (Path('phy_no_amp') / 'amplitudes.npy').is_file()
assert not (Path('phy_no_amp_feat') / 'amplitudes.npy').is_file()
assert not (Path('phy_no_amp_feat') / 'pc_features.npy').is_file()
assert not (Path('phy_no_amp_feat') / 'pc_feature_ind.npy').is_file()
sort_phy = se.PhySortingExtractor('phy', load_waveforms=True)
sort_phyg = se.PhySortingExtractor('phy_group', load_waveforms=True)
assert np.allclose(sort_phy.get_unit_spike_train(0),
sort.get_unit_spike_train(sort.get_unit_ids()[0]))
assert np.allclose(sort_phyg.get_unit_spike_train(2),
sort.get_unit_spike_train(sort.get_unit_ids()[2]))
assert sort_phy.get_unit_spike_features(1, 'waveforms').shape[1] == 8
assert sort_phyg.get_unit_spike_features(3, 'waveforms').shape[1] == 4
rec.set_channel_groups([0, 0, 0, 0, 1, 1, 1, 1])
recrm = remove_bad_channels(rec, [1, 2, 5])
export_to_phy(recrm,
sort,
output_folder='phy_rm',
grouping_property='group',
recompute_info=True,
verbose=True)
templates_ind = np.load('phy_rm/template_ind.npy')
assert len(np.where(templates_ind == -1)[0]) > 0 # removed channels are -1
try:
shutil.rmtree('test')
shutil.rmtree('phy')
shutil.rmtree('phy_group')
shutil.rmtree('phy_max_channels')
shutil.rmtree('phy_no_feat')
shutil.rmtree('phy_no_amp')
shutil.rmtree('phy_no_amp_feat')
shutil.rmtree('phy_rm')
shutil.rmtree('phy_par')
except:
print("Could not delete some test folders")