def populate_extractors(
entry: RecordingEntry
) -> Tuple[sv.LabboxEphysRecordingExtractor, sv.LabboxEphysSortingExtractor,
sv.LabboxEphysSortingExtractor]:
recording = sv.LabboxEphysRecordingExtractor(entry.recording_uri,
download=True)
sample_rate = recording.get_sampling_frequency()
sorting_true = sv.LabboxEphysSortingExtractor(entry.sorting_true_uri,
samplerate=sample_rate)
sorting = sv.LabboxEphysSortingExtractor(entry.sorting_object,
samplerate=sample_rate)
return (recording, sorting_true, sorting)
def populate_extractors(workspace_uri: str, rec_uri: str, gt_uri: str,
sorting_result: Any) -> HydratedObjects:
workspace = sv.load_workspace(workspace_uri)
recording = sv.LabboxEphysRecordingExtractor(rec_uri, download=True)
sample_rate = recording.get_sampling_frequency()
gt_sort = sv.LabboxEphysSortingExtractor(gt_uri, samplerate=sample_rate)
sorting = sv.LabboxEphysSortingExtractor(sorting_result,
samplerate=sample_rate)
return HydratedObjects(workspace=workspace,
recording=recording,
gt_sort=gt_sort,
sorting=sorting)
def compute_quality_metrics_hi(recording_uri, gt_uri, firings_uri):
# gt_uri is not needed, but including it lets this method and the ground truth comparison use the same consistent kwargs parameters.
print_per_verbose(
1,
f"Computing quality metrics for recording {recording_uri} and sorting {firings_uri}. Fetching Extractors..."
)
print_per_verbose(
2, f"Execting sv.LabboxEphysRecordingExtractor({recording_uri})")
recording = sv.LabboxEphysRecordingExtractor(recording_uri)
sample_rate = recording.get_sampling_frequency()
print_per_verbose(2, f"Found sample rate {sample_rate}.")
sorting_object = {
'sorting_format': 'mda',
'data': {
'firings': firings_uri,
'samplerate': sample_rate
}
}
print_per_verbose(
2,
f"(Comparison evaluation) Executing sv.LabboxEphysSortingExtractor({json.dumps(sorting_object)})"
)
sorting = sv.LabboxEphysSortingExtractor(sorting_object)
print_per_verbose(2, f"Executing quality metrics")
try:
qm = compute_quality_metrics(recording, sorting)
except Exception as e:
print(f"WARNING: Problem in compute_quality_metrics:\n{e}")
qm = f"Quality metric computation for recording {recording_uri} sorting {firings_uri} returned error:\n{e}"
return qm
def make_output_record(job: SortingJob) -> OutputRecord:
errored = job.sorting_job.status == "error"
if (errored):
stored_sorting = None
else:
sorting = sv.LabboxEphysSortingExtractor(
job.sorting_job.result.return_value)
stored_sorting = sv.LabboxEphysSortingExtractor.store_sorting(sorting)
console = kc.store_json(job.sorting_job._console_lines)
try:
elapsed = job.sorting_job.timestamp_completed - job.sorting_job.timestamp_started
except:
elapsed = 0.0
# Can't do this--sorting_job is not json-serializable.
# TODO: Implement a __str__ method for hi2.Job
# print_per_verbose(3, f"sorting_job: {json.dumps(job.sorting_job, indent=4)}")
record: OutputRecord = {
'recordingName': job.recording_name,
'studyName': job.study_name,
'sorterName': job.sorter_name,
'sortingParameters': job.params,
'consoleOutUri': console,
'cpuTimeSec': elapsed,
'errored': errored,
'startTime': _fmt_time(job.sorting_job.timestamp_started),
'endTime': _fmt_time(job.sorting_job.timestamp_completed),
'sortingOutput': stored_sorting,
'recordingUri': job.recording_uri,
'groundTruthUri': job.ground_truth_uri
}
return record
def get_sorting_output(self, study_name, recording_name, sorter_name):
# get study
dataset = self._df.query(f"studyName == '{study_name}'")
assert len(dataset) > 0, f"Study '{study_name}' not found"
# get recording
dataset = dataset.query(f"recordingName == '{recording_name}'")
assert len(dataset) > 0, f"Recording '{recording_name}' not found"
# get sorting uri
dataset = dataset.query(f"sorterName == '{sorter_name}'")
assert len(dataset) == 1, f"Sorting output '{sorter_name}' not found"
firings_uri = dataset["firings"].values[0]
# get samplint rate
recording = self.get_gt_recording(study_name,
recording_name,
download=False)
sorting_object = {
'sorting_format': 'mda',
'data': {
'firings': firings_uri,
'samplerate': recording.get_sampling_frequency()
}
}
sorting = sv.LabboxEphysSortingExtractor(sorting_object)
return sorting
def get_gt_sorting_output(self, study_name, recording_name):
# get study
dataset = self._df.query(f"studyName == '{study_name}'")
assert len(dataset) > 0, f"Study '{study_name}' not found"
# get recording
dataset = dataset.query(f"recordingName == '{recording_name}'")
assert len(dataset) > 0, f"Recording '{recording_name}' not found"
firings_uri = dataset.iloc[0]["sortingTrueUri"]
sorting = sv.LabboxEphysSortingExtractor(firings_uri)
return sorting
def compute_ground_truth_comparison_hi(recording_uri, gt_uri, firings_uri):
print_per_verbose(
1,
f"Computing ground truth comparison for ground truth {gt_uri} and sorting {firings_uri} (recording {recording_uri})"
)
print_per_verbose(3, f'Fetching sample rate from {recording_uri}')
recording = sv.LabboxEphysRecordingExtractor(recording_uri)
sample_rate = recording.get_sampling_frequency()
print_per_verbose(3, f'Got sample rate {sample_rate}')
print_per_verbose(2, f"Building sorting object for ground truth {gt_uri}")
gt_firings = kc.load_json(gt_uri)['firings']
print_per_verbose(2, f"Got ground truth firings {gt_firings}")
gt_sorting_obj = {
'sorting_format': 'mda',
'data': {
'firings': gt_firings,
'samplerate': sample_rate
}
}
gt_sorting = sv.LabboxEphysSortingExtractor(gt_sorting_obj)
print_per_verbose(
2, f"Building sorting object for sorting with firings {firings_uri}")
sorting_obj = {
'sorting_format': 'mda',
'data': {
'firings': firings_uri,
'samplerate': sample_rate
}
}
sorting = sv.LabboxEphysSortingExtractor(sorting_obj)
print_per_verbose(2, f"Executing ground-truth comparison")
try:
gt = compare_with_ground_truth(sorting, gt_sorting)
except Exception as e:
print(f"WARNING: Problem in compute_ground_truth_comparison:\n{e}")
gt = f"Ground truth comparison for gt {gt_uri} and sorting {firings_uri} returned error:\n{e}"
return gt
def test_sorting(sorter_func):
import sortingview as sv
recording_name = 'paired_kampff/2014_11_25_Pair_3_0'
recording_uri = 'sha1://a205f87cef8b7f86df7a09cddbc79a1fbe5df60f/2014_11_25_Pair_3_0.json'
sorting_uri = 'sha1://c656add63d85a17840980084a1ff1cdc662a2cd5/2014_11_25_Pair_3_0.firings_true.json'
recording = sv.LabboxEphysRecordingExtractor(recording_uri, download=True)
sorting_true = sv.LabboxEphysSortingExtractor(sorting_uri)
channel_ids = recording.get_channel_ids()
samplerate = recording.get_sampling_frequency()
num_timepoints = recording.get_num_frames()
print(f'{recording_name}')
print(
f'Recording has {len(channel_ids)} channels and {num_timepoints} timepoints (samplerate: {samplerate})'
)
unit_ids = sorting_true.get_unit_ids()
spike_train = sorting_true.get_unit_spike_train(unit_id=unit_ids[0])
print(f'Unit {unit_ids[0]} has {len(spike_train)} events')
jh = hi.ParallelJobHandler(num_workers=4)
# jh = hi.SlurmJobHandler(num_jobs_per_allocation=4, max_simultaneous_allocations=4, srun_command='')
log = hi.Log()
with hi.Config(use_container=True,
job_handler=jh,
log=log,
show_console=True):
sorting_object = hi.Job(sorter_func, {
'recording_object': recording.object()
}).wait().return_value
sorting = sv.LabboxEphysSortingExtractor(sorting_object)
unit_ids = sorting.get_unit_ids()
spike_train = sorting.get_unit_spike_train(unit_id=unit_ids[0])
print(f'Unit {unit_ids[0]} has {len(spike_train)} events')
def test_sorting(sorter_func,
*,
show_console=True,
job_handler: Union[None, hi.JobHandler] = None):
import sortingview as sv
recording_name = 'paired_kampff/2014_11_25_Pair_3_0'
recording_uri = 'sha1://a205f87cef8b7f86df7a09cddbc79a1fbe5df60f/2014_11_25_Pair_3_0.json'
sorting_uri = 'sha1://c656add63d85a17840980084a1ff1cdc662a2cd5/2014_11_25_Pair_3_0.firings_true.json'
recording = sv.LabboxEphysRecordingExtractor(recording_uri, download=True)
sorting_true = sv.LabboxEphysSortingExtractor(sorting_uri)
channel_ids = recording.get_channel_ids()
samplerate = recording.get_sampling_frequency()
num_timepoints = recording.get_num_frames()
print(f'{recording_name}')
print(
f'Recording has {len(channel_ids)} channels and {num_timepoints} timepoints (samplerate: {samplerate})'
)
unit_ids = sorting_true.get_unit_ids()
spike_train = sorting_true.get_unit_spike_train(unit_id=unit_ids[0])
print(f'Unit {unit_ids[0]} has {len(spike_train)} events')
with hi.Config(use_container=True,
show_console=show_console,
job_handler=job_handler):
sorting_object = hi.Job(sorter_func, {
'recording_object': recording.object()
}).wait().return_value
sorting = sv.LabboxEphysSortingExtractor(sorting_object)
unit_ids = sorting.get_unit_ids()
spike_train = sorting.get_unit_spike_train(unit_id=unit_ids[0])
print(f'Unit {unit_ids[0]} has {len(spike_train)} events')