def process_sorting_record(sorting_record, comparison_result_list):
try:
params = {
'recording_uri': sorting_record[RECORDING_URI_KEY],
'gt_uri': sorting_record[GROUND_TRUTH_URI_KEY],
'firings_uri': sorting_record[SORTING_FIRINGS_URI_KEY]
}
quality_metric_job = hi.Job(compute_quality_metrics_hi, params)
ground_truth_comparison_job = hi.Job(
compute_ground_truth_comparison_hi, params)
comparison = make_comparison_entry(sorting_record, quality_metric_job,
ground_truth_comparison_job)
comparison_result_list.append(comparison)
except KeyError:
print(
f"One of sorting/recording/gt-sorting keys missing from {json.dumps(sorting_record)}. Skipping..."
)
def queue_sort(sorter: SorterRecord, recording: RecordingRecord) -> hi.Job:
if sorter.sorter_name not in KNOWN_SORTERS.keys():
raise Exception(
f'Sorter {sorter.sorter_name} was requested but is not recognized.'
)
sort_fn = KNOWN_SORTERS[sorter.sorter_name]
base_recording = sv.LabboxEphysRecordingExtractor(recording.recording_uri,
download=True)
params = {'recording_object': base_recording.object()}
return hi.Job(sort_fn, params)
def test_sorting(sorter_func):
import sortingview as sv
recording_name = 'paired_kampff/2014_11_25_Pair_3_0'
recording_uri = 'sha1://a205f87cef8b7f86df7a09cddbc79a1fbe5df60f/2014_11_25_Pair_3_0.json'
sorting_uri = 'sha1://c656add63d85a17840980084a1ff1cdc662a2cd5/2014_11_25_Pair_3_0.firings_true.json'
recording = sv.LabboxEphysRecordingExtractor(recording_uri, download=True)
sorting_true = sv.LabboxEphysSortingExtractor(sorting_uri)
channel_ids = recording.get_channel_ids()
samplerate = recording.get_sampling_frequency()
num_timepoints = recording.get_num_frames()
print(f'{recording_name}')
print(
f'Recording has {len(channel_ids)} channels and {num_timepoints} timepoints (samplerate: {samplerate})'
)
unit_ids = sorting_true.get_unit_ids()
spike_train = sorting_true.get_unit_spike_train(unit_id=unit_ids[0])
print(f'Unit {unit_ids[0]} has {len(spike_train)} events')
jh = hi.ParallelJobHandler(num_workers=4)
# jh = hi.SlurmJobHandler(num_jobs_per_allocation=4, max_simultaneous_allocations=4, srun_command='')
log = hi.Log()
with hi.Config(use_container=True,
job_handler=jh,
log=log,
show_console=True):
sorting_object = hi.Job(sorter_func, {
'recording_object': recording.object()
}).wait().return_value
sorting = sv.LabboxEphysSortingExtractor(sorting_object)
unit_ids = sorting.get_unit_ids()
spike_train = sorting.get_unit_spike_train(unit_id=unit_ids[0])
print(f'Unit {unit_ids[0]} has {len(spike_train)} events')
def main():
(params, std_args) = init_configuration()
study_sets = load_study_records(params.study_source_file)
study_matrix = parse_sorters(params.sorter_spec_file,
list(study_sets.keys()))
sorting_matrix = populate_sorting_matrix(study_matrix, study_sets)
sorting_matrix = remove_preexisting_records(sorting_matrix,
params.workspace_uri)
hither_config = extract_hither_config(std_args)
jobs: hi.Job = []
try:
with hi.Config(**hither_config):
sortings = list(sorting_loop(sorting_matrix))
with hi.Config(job_handler=None, job_cache=None):
for sorting in sortings:
p = {
'sorting_entry': sorting,
'workspace_uri': params.workspace_uri
}
jobs.append(hi.Job(hi_post_result_to_workspace, p))
hi.wait(None)
finally:
call_cleanup(hither_config)
def test_sorting(sorter_func,
*,
show_console=True,
job_handler: Union[None, hi.JobHandler] = None):
import sortingview as sv
recording_name = 'paired_kampff/2014_11_25_Pair_3_0'
recording_uri = 'sha1://a205f87cef8b7f86df7a09cddbc79a1fbe5df60f/2014_11_25_Pair_3_0.json'
sorting_uri = 'sha1://c656add63d85a17840980084a1ff1cdc662a2cd5/2014_11_25_Pair_3_0.firings_true.json'
recording = sv.LabboxEphysRecordingExtractor(recording_uri, download=True)
sorting_true = sv.LabboxEphysSortingExtractor(sorting_uri)
channel_ids = recording.get_channel_ids()
samplerate = recording.get_sampling_frequency()
num_timepoints = recording.get_num_frames()
print(f'{recording_name}')
print(
f'Recording has {len(channel_ids)} channels and {num_timepoints} timepoints (samplerate: {samplerate})'
)
unit_ids = sorting_true.get_unit_ids()
spike_train = sorting_true.get_unit_spike_train(unit_id=unit_ids[0])
print(f'Unit {unit_ids[0]} has {len(spike_train)} events')
with hi.Config(use_container=True,
show_console=show_console,
job_handler=job_handler):
sorting_object = hi.Job(sorter_func, {
'recording_object': recording.object()
}).wait().return_value
sorting = sv.LabboxEphysSortingExtractor(sorting_object)
unit_ids = sorting.get_unit_ids()
spike_train = sorting.get_unit_spike_train(unit_id=unit_ids[0])
print(f'Unit {unit_ids[0]} has {len(spike_train)} events')
def run_sorter_docker(sorter_name,
recording,
output_folder,
delete_output_folder=False,
grouping_property=None,
parallel=False,
verbose=False,
raise_error=True,
n_jobs=-1,
joblib_backend='loky',
use_docker=True,
container=None,
**params):
if use_docker:
# if container is None:
# assert sorter_name in default_docker_images, f"Default docker image for {sorter_name} not found"
# docker_image = default_docker_images[sorter_name]
#
# print(f"Running in docker image {docker_image.get_name()}")
output_folder = Path(output_folder).absolute()
output_folder.mkdir(exist_ok=True, parents=True)
# dump recording with relative file paths to docker container /input folder
dump_dict_container, input_directory = modify_input_folder(
recording.dump_to_dict(), '/input')
with hither.Config(use_container=False, show_console=True):
kwargs = dict(recording_dict=dump_dict_container,
sorter_name=sorter_name,
output_folder=str(output_folder),
delete_output_folder=False,
grouping_property=grouping_property,
parallel=parallel,
verbose=verbose,
raise_error=raise_error,
n_jobs=n_jobs,
joblib_backend=joblib_backend)
kwargs.update(params)
kwargs.update({
'input_directory': str(input_directory),
'output_directory': str(output_folder)
})
sorting_job = hither.Job(run_sorter_docker_with_container, kwargs)
sorting_job.wait()
sorting = se.NpzSortingExtractor(output_folder / "sorting_docker.npz")
else:
# standard call
sorting = ss.run_sorter(sorter_name,
recording,
output_folder=output_folder,
delete_output_folder=delete_output_folder,
grouping_property=grouping_property,
parallel=parallel,
verbose=verbose,
raise_error=raise_error,
n_jobs=n_jobs,
joblib_backend=joblib_backend,
**params)
return sorting