def get_timeseries_segment(recording_object, ds_factor, segment_num,
segment_size):
import time
recording0 = le.LabboxEphysRecordingExtractor(recording_object,
download=False)
t1 = segment_num * segment_size * ds_factor
t2 = ((segment_num + 1) * segment_size * ds_factor)
if t2 > recording0.get_num_frames():
t2 = int(recording0.get_num_frames() / ds_factor) * ds_factor
traces = recording0.get_traces(start_frame=t1, end_frame=t2)
M = traces.shape[0]
N = traces.shape[1]
if ds_factor > 1:
N2 = int(N / ds_factor)
traces_reshaped = traces.reshape((M, N2, ds_factor))
traces_min = np.min(traces_reshaped, axis=2)
traces_max = np.max(traces_reshaped, axis=2)
traces = np.zeros((M, N2 * 2))
traces[:, 0::2] = traces_min
traces[:, 1::2] = traces_max
data_b64 = _mda32_to_base64(traces)
# elapsed = time.time() - timer
return dict(data_b64=data_b64)
def old_fetch_average_waveform_plot_data(recording_object, sorting_object,
unit_id):
import labbox_ephys as le
R = le.LabboxEphysRecordingExtractor(recording_object)
S = le.LabboxEphysSortingExtractor(sorting_object)
start_frame = 0
end_frame = R.get_sampling_frequency() * 30
R0 = se.SubRecordingExtractor(parent_recording=R,
start_frame=start_frame,
end_frame=end_frame)
S0 = se.SubSortingExtractor(parent_sorting=S,
start_frame=start_frame,
end_frame=end_frame)
times0 = S0.get_unit_spike_train(unit_id=unit_id)
if len(times0) == 0:
# no waveforms found
return dict(channel_id=None, average_waveform=None)
try:
average_waveform = st.postprocessing.get_unit_templates(
recording=R0, sorting=S0, unit_ids=[unit_id])[0]
except:
raise Exception(f'Error getting unit templates for unit {unit_id}')
channel_maximums = np.max(np.abs(average_waveform), axis=1)
maxchan_index = np.argmax(channel_maximums)
maxchan_id = R0.get_channel_ids()[maxchan_index]
return dict(channel_id=maxchan_id,
average_waveform=average_waveform[maxchan_index, :].tolist())
def main():
recording_object = {
"data": {
"geom": [[1, 0], [2, 0], [3, 0], [4, 0]],
"params": {
"samplerate": 30000,
"spike_sign": -1
},
"raw":
"sha1dir://fb52d510d2543634e247e0d2d1d4390be9ed9e20.synth_magland/datasets_noise10_K10_C4/001_synth/raw.mda"
},
"recording_format": "mda"
}
sorting_object = {
"data": {
"firings":
"sha1dir://fb52d510d2543634e247e0d2d1d4390be9ed9e20.synth_magland/datasets_noise10_K10_C4/001_synth/firings_true.mda",
"samplerate": 30000
},
"sorting_format": "mda"
}
recording = le.LabboxEphysRecordingExtractor(recording_object)
sorting = le.LabboxEphysSortingExtractor(sorting_object)
h5_path = le.prepare_snippets_h5.run(
sorting_object=sorting_object,
recording_object=recording_object).wait()
print(h5_path)
def get_sorting_info(sorting_object, recording_object):
sorting = le.LabboxEphysSortingExtractor(sorting_object)
recording = le.LabboxEphysRecordingExtractor(recording_object, download=False)
return dict(
unit_ids=_to_int_list(sorting.get_unit_ids()),
samplerate=recording.get_sampling_frequency()
)
def calculate_timeseries_info(recording_object):
recording0 = le.LabboxEphysRecordingExtractor(recording_object, download=False)
traces0 = recording0.get_traces(
channel_ids=recording0.get_channel_ids(),
start_frame=0, end_frame=min(recording0.get_num_frames(), 25000
))
y_offsets = -np.mean(traces0, axis=1)
for m in range(traces0.shape[0]):
traces0[m, :] = traces0[m, :] + y_offsets[m]
vv = np.percentile(np.abs(traces0), 90)
y_scale_factor = 1 / (2 * vv) if vv > 0 else 1
# segment_size_times_num_channels = 1000000
segment_size_times_num_channels = 100000
segment_size = int(np.ceil(segment_size_times_num_channels / recording0.get_num_channels()))
return dict(
samplerate=recording0.get_sampling_frequency(),
num_channels=len(recording0.get_channel_ids()),
channel_ids=recording0.get_channel_ids(),
channel_locations=geom_from_recording(recording0).tolist(),
num_timepoints=recording0.get_num_frames(),
y_offsets=y_offsets.astype(float).tolist(),
y_scale_factor=float(y_scale_factor),
initial_y_scale_factor=1,
segment_size=segment_size
)
def get_franklab_datajoint_importable_recordings(config):
import labbox_ephys as le
os.environ['DJ_SUPPORT_FILEPATH_MANAGEMENT'] = 'TRUE'
import datajoint as dj
dj.config['enable_python_native_blobs'] = True
dj.config['database.port'] = config['port']
dj.config['database.user'] = config['user']
dj.config['database.password'] = config['password']
# must config dj prior to importing
import nwb_datajoint as nwbdj
ret = []
for session in nwbdj.Session():
nwb_file_sha1 = session['nwb_file_sha1']
nwb_file_name = session['nwb_file_name']
nwb_path = f'sha1://{nwb_file_sha1}/file.nwb'
try:
recording = le.LabboxEphysRecordingExtractor(nwb_path)
recording_object = recording.object()
except:
traceback.print_exc()
print(f'Warning: problem loading recording: {nwb_path}')
recording_object = None
if recording_object is not None:
info = le.get_recording_info(recording_object)
ret.append(dict(
label=nwb_file_name,
path=nwb_path,
recording_object=recording_object,
recording_info=info
))
return ret
def preload_download_recording_2(recording_object):
import kachery_p2p as kp
try:
X = le.LabboxEphysRecordingExtractor(recording_object, download=True)
except:
return {'success': False}
return {'success': True}
def get_recording_info(recording_object):
recording = le.LabboxEphysRecordingExtractor(recording_object,
download=False)
return dict(sampling_frequency=recording.get_sampling_frequency(),
channel_ids=recording.get_channel_ids(),
channel_groups=recording.get_channel_groups().tolist(),
geom=geom_from_recording(recording).tolist(),
num_frames=recording.get_num_frames(),
is_local=recording.is_local())
def javascript_state_changed(self, prev_state, state):
self._set_status('running', 'Running TimeseriesView')
self._create_efficient_access = state.get('create_efficient_access',
False)
if not self._recording:
self._set_status('running', 'Loading recording')
recording0 = state.get('recording', None)
if not recording0:
self._set_error('Missing: recording')
return
try:
self._recording = le.LabboxEphysRecordingExtractor(recording0)
except Exception as err:
traceback.print_exc()
self._set_error('Problem initiating recording: {}'.format(err))
return
self._set_status('running', 'Loading recording data')
traces0 = self._recording.get_traces(
channel_ids=self._recording.get_channel_ids(),
start_frame=0,
end_frame=min(self._recording.get_num_frames(), 25000))
y_offsets = -np.mean(traces0, axis=1)
for m in range(traces0.shape[0]):
traces0[m, :] = traces0[m, :] + y_offsets[m]
vv = np.percentile(np.abs(traces0), 90)
y_scale_factor = 1 / (2 * vv) if vv > 0 else 1
self._segment_size = int(
np.ceil(self._segment_size_times_num_channels /
self._recording.get_num_channels()))
try:
channel_locations = self._recording.get_channel_locations()
except:
channel_locations = None
self.set_state(
dict(num_channels=self._recording.get_num_channels(),
channel_ids=self._recording.get_channel_ids(),
channel_locations=channel_locations,
num_timepoints=self._recording.get_num_frames(),
y_offsets=y_offsets,
y_scale_factor=y_scale_factor,
samplerate=self._recording.get_sampling_frequency(),
segment_size=self._segment_size,
status_message='Loaded recording.'))
# SR = state.get('segmentsRequested', {})
# for key in SR.keys():
# aa = SR[key]
# if not self.get_python_state(key, None):
# self.set_state(dict(status_message='Loading segment {}'.format(key)))
# data0 = self._load_data(aa['ds'], aa['ss'])
# data0_base64 = _mda32_to_base64(data0)
# state0 = {}
# state0[key] = dict(data=data0_base64, ds=aa['ds'], ss=aa['ss'])
# self.set_state(state0)
# self.set_state(dict(status_message='Loaded segment {}'.format(key)))
self._set_status('finished', '')
def mountainsort4b(recording_object: dict,
detect_sign=-1,
adjacency_radius=50,
clip_size=50,
detect_threshold=3,
detect_interval=10,
freq_min=300,
freq_max=6000,
whiten=True,
curation=False,
filter=True):
# Unfortunately we need to duplicate wrapper code from spikeforest2 due to trickiness in running code in containers. Will need to think about this
# import spiketoolkit as st
import spikesorters as ss
import labbox_ephys as le
recording = le.LabboxEphysRecordingExtractor(recording_object)
# for quick testing
# import spikeextractors as se
# recording = se.SubRecordingExtractor(parent_recording=recording_object, start_frame=0, end_frame=30000 * 1)
# Preprocessing
# print('Preprocessing...')
# recording = st.preprocessing.bandpass_filter(recording_object, freq_min=300, freq_max=6000)
# recording = st.preprocessing.whiten(recording_object)
# Sorting
print('Sorting...')
with hi.TemporaryDirectory() as tmpdir:
sorter = ss.Mountainsort4Sorter(recording=recording,
output_folder=tmpdir,
delete_output_folder=False)
num_workers = os.environ.get('NUM_WORKERS', None)
if num_workers:
num_workers = int(num_workers)
else:
num_workers = 0
sorter.set_params(detect_sign=detect_sign,
adjacency_radius=adjacency_radius,
clip_size=clip_size,
detect_threshold=detect_threshold,
detect_interval=detect_interval,
num_workers=num_workers,
curation=curation,
whiten=whiten,
filter=filter,
freq_min=freq_min,
freq_max=freq_max)
timer = sorter.run()
print('#SF-SORTER-RUNTIME#{:.3f}#'.format(timer))
sorting = sorter.get_result()
sorting_object = _create_sorting_object(sorting)
return dict(sorting_object=sorting_object)
def preload_check_recording_downloaded_2(recording_object):
import kachery_p2p as kp
try:
kp._experimental_config(nop2p=True)
X = le.LabboxEphysRecordingExtractor(recording_object, download=False)
except:
return {'isLocal': False}
finally:
kp._experimental_config(nop2p=False)
return {'isLocal': True}
def get_gt_recording(self, study_name, recording_name, download=False):
# get study
dataset = self._df.query(f"studyName == '{study_name}'")
assert len(dataset) > 0, f"Study '{study_name}' not found"
# get recording
dataset = dataset.query(f"recordingName == '{recording_name}'")
assert len(dataset) > 0, f"Recording '{recording_name}' not found"
recording_uri = dataset.iloc[0]["recordingUri"]
recording = le.LabboxEphysRecordingExtractor(recording_uri,
download=download)
return recording
def main():
snippets_h5_uri = 'sha1://55c0cb6a63231236b6948b0dd422e6fedc75c5b5/real_snippets.h5?manifest=b124474caccccdba135d9550ec544a88caf531aa'
recording_obj = {
'recording_format': 'snippets1',
'data': {
'snippets_h5_uri': snippets_h5_uri
}
}
sorting_obj = {
'sorting_format': 'snippets1',
'data': {
'snippets_h5_uri': snippets_h5_uri
}
}
recording = le.LabboxEphysRecordingExtractor(recording_obj)
sorting = le.LabboxEphysSortingExtractor(sorting_obj)
print(recording.get_sampling_frequency())
print(recording.get_channel_ids())
le_recordings = []
le_sortings = []
le_recordings.append(
dict(recordingId='loren_example1',
recordingLabel='loren_example1',
recordingPath=ka.store_object(recording_obj,
basename='loren_example1.json'),
recordingObject=recording_obj,
description='''
Example from Loren Frank
'''.strip()))
le_sortings.append(
dict(sortingId='loren_example1:mountainsort4',
sortingLabel='loren_example1:mountainsort4',
sortingPath=ka.store_object(
sorting_obj, basename='loren_example-mountainsort4.json'),
sortingObject=sorting_obj,
recordingId='loren_example1',
recordingPath=ka.store_object(recording_obj,
basename='loren_example1.json'),
recordingObject=recording_obj,
description='''
Example from Loren Frank (MountainSort4)
'''.strip()))
feed_uri = create_labbox_ephys_feed(le_recordings, le_sortings)
print(feed_uri)
def main():
snippets_h5_uri = 'sha1://5fc6996dfed9e7fd577bc85194d982a1ba52085e/real_snippets_1.h5?manifest=741f23273c3121aada6d9bdb67009c8c2ae1ed77'
recording_obj = {
'recording_format': 'snippets1',
'data': {
'snippets_h5_uri': snippets_h5_uri
}
}
sorting_obj = {
'sorting_format': 'snippets1',
'data': {
'snippets_h5_uri': snippets_h5_uri
}
}
recording = le.LabboxEphysRecordingExtractor(recording_obj)
sorting = le.LabboxEphysSortingExtractor(sorting_obj)
print(recording.get_sampling_frequency())
print(recording.get_channel_ids())
le_recordings = []
le_sortings = []
le_recordings.append(
dict(recordingId='loren_example1',
recordingLabel='loren_example1',
recordingPath=ka.store_object(recording_obj,
basename='loren_example1.json'),
recordingObject=recording_obj,
description='''
Example from Loren Frank
'''.strip()))
le_sortings.append(
dict(sortingId='loren_example1:mountainsort4',
sortingLabel='loren_example1:mountainsort4',
sortingPath=ka.store_object(
sorting_obj, basename='loren_example-mountainsort4.json'),
sortingObject=sorting_obj,
recordingId='loren_example1',
recordingPath=ka.store_object(recording_obj,
basename='loren_example1.json'),
recordingObject=recording_obj,
description='''
Example from Loren Frank (MountainSort4)
'''.strip()))
feed_uri = create_labbox_ephys_feed(le_recordings, le_sortings)
print(feed_uri)
def main():
# Sorting: cortexlab-single-phase-3 Curated (good units) for cortexlab-single-phase-3 (full)
recording_object = kp.load_object(
'sha1://8b222e25bc4d9c792e4490ca322b5338e0795596/cortexlab-single-phase-3.json'
)
sorting_object = {
"sorting_format": "h5_v1",
"data": {
"h5_path":
"sha1://68029d0eded8ca7d8f95c16dea81318966ae9b55/sorting.h5?manifest=12b0d8e37c7050a6fe636d4c16ed143bbd5dab0c"
}
}
recording = le.LabboxEphysRecordingExtractor(recording_object)
sorting = le.LabboxEphysSortingExtractor(sorting_object)
h5_path = le.prepare_snippets_h5.run(sorting_object=sorting_object,
recording_object=recording_object,
start_frame=0,
end_frame=30000 * 240).wait()
print(h5_path)
def get_isi_violation_rates(sorting_object, recording_object, configuration={}):
import labbox_ephys as le
import spikemetrics as sm
S = le.LabboxEphysSortingExtractor(sorting_object)
R = le.LabboxEphysRecordingExtractor(recording_object)
samplerate = R.get_sampling_frequency()
# duration_sec = R.get_num_frames() / samplerate
isi_threshold_msec = configuration.get('isi_threshold_msec', 2.5)
unit_ids = configuration.get('unit_ids', S.get_unit_ids())
ret = {}
for id in unit_ids:
spike_train = S.get_unit_spike_train(unit_id=id)
ret[str(id)], _ = sm.metrics.isi_violations( #_ is total violations
spike_train=spike_train,
duration=R.get_num_frames(),
isi_threshold=isi_threshold_msec / 1000 * samplerate
)
return ret
def get_timeseries_segment(recording_object, ds_factor, segment_num,
segment_size):
import time
recording0 = le.LabboxEphysRecordingExtractor(recording_object,
download=False)
t1 = segment_num * segment_size * ds_factor
t2 = ((segment_num + 1) * segment_size * ds_factor)
if t2 > recording0.get_num_frames():
t2 = int(recording0.get_num_frames() / ds_factor) * ds_factor
traces = recording0.get_traces(start_frame=t1, end_frame=t2)
M = traces.shape[0]
N = traces.shape[1]
if ds_factor > 1:
N2 = int(N / ds_factor)
traces_reshaped = traces.reshape((M, N2, ds_factor))
traces_min = np.min(traces_reshaped, axis=2)
traces_max = np.max(traces_reshaped, axis=2)
traces = np.zeros((M, N2 * 2), dtype=np.float32)
traces[:, 0::2] = traces_min
traces[:, 1::2] = traces_max
return {'traces': traces.astype(np.float32)}
def prepare_snippets_h5(recording_object,
sorting_object,
start_frame=None,
end_frame=None,
max_events_per_unit=None,
max_neighborhood_size=15):
if recording_object['recording_format'] == 'snippets1':
return recording_object['data']['snippets_h5_uri']
import labbox_ephys as le
recording = le.LabboxEphysRecordingExtractor(recording_object)
sorting = le.LabboxEphysSortingExtractor(sorting_object)
with hi.TemporaryDirectory() as tmpdir:
save_path = tmpdir + '/snippets.h5'
prepare_snippets_h5_from_extractors(
recording=recording,
sorting=sorting,
output_h5_path=save_path,
start_frame=start_frame,
end_frame=end_frame,
max_events_per_unit=max_events_per_unit,
max_neighborhood_size=max_neighborhood_size)
return ka.store_file(save_path)
"label":
"SF/PAIRED_KAMPFF/paired_kampff/2014_11_25_Pair_3_0",
"recording_uri":
"sha1://a205f87cef8b7f86df7a09cddbc79a1fbe5df60f/SF/PAIRED_KAMPFF/paired_kampff/2014_11_25_Pair_3_0.json",
"sorting_true_uri":
"sha1://1cd517687aeca7ecdfaa9695680038d142a75031/firings_true.mda"
}
# To find more examples, see: https://github.com/flatironinstitute/spikeforest_recordings
# However: note that some processing needs to be done to the files in this repo (to add the manifests to the raw data). This is WIP
# Adjust these values ###########################
X = X1 # Select example from above
feed_name = 'labbox-ephys-default'
workspace_name = 'default'
#################################################
recording_label = X['label']
recording_uri = X['recording_uri']
sorting_true_uri = X['sorting_true_uri']
recording = le.LabboxEphysRecordingExtractor(recording_uri, download=True)
sorting_true = le.LabboxEphysSortingExtractor(sorting_true_uri,
samplerate=30000)
sorting_label = 'true'
feed = kp.load_feed(feed_name, create=True)
workspace = le.load_workspace(workspace_name=workspace_name, feed=feed)
print(f'Feed URI: {feed.get_uri()}')
R_id = workspace.add_recording(recording=recording, label=recording_label)
S_id = workspace.add_sorting(sorting=sorting_true,
recording_id=R_id,
label=sorting_label)
end_frame=None,
max_events_per_unit=1000,
max_neighborhood_size=2
)
# Example display some contents of the file
with h5py.File(output_h5_path, 'r') as f:
unit_ids = np.array(f.get('unit_ids'))
sampling_frequency = np.array(f.get('sampling_frequency'))[0].item()
if np.isnan(sampling_frequency):
print('WARNING: sampling frequency is nan. Using 30000 for now. Please correct the snippets file.')
sampling_frequency = 30000
print(f'Unit IDs: {unit_ids}')
print(f'Sampling freq: {sampling_frequency}')
for unit_id in unit_ids:
unit_spike_train = np.array(f.get(f'unit_spike_trains/{unit_id}'))
unit_waveforms = np.array(f.get(f'unit_waveforms/{unit_id}/waveforms'))
unit_waveforms_channel_ids = np.array(f.get(f'unit_waveforms/{unit_id}/channel_ids'))
print(f'Unit {unit_id} | Tot num events: {len(unit_spike_train)} | shape of subsampled snippets: {unit_waveforms.shape}')
recording = le.LabboxEphysRecordingExtractor({
'recording_format': 'snippets1',
'data': {
'snippets_h5_uri': ka.store_file(output_h5_path)
}
})
print(f'Channel IDs: {recording.get_channel_ids()}')
print(f'Num. frames: {recording.get_num_frames()}')
for channel_id in recording.get_channel_ids():
print(f'Channel {channel_id}: {recording.get_channel_property(channel_id, "location")}')
def get_structure(sorting_object, recording_object):
import labbox_ephys as le
S = le.LabboxEphysSortingExtractor(sorting_object)
R = le.LabboxEphysRecordingExtractor(recording_object)
return S, R
def download_recording(recording_object):
recording = le.LabboxEphysRecordingExtractor(recording_object,
download=False)
recording.download()
def get_sorting_object(sorting_path, recording_object):
recording = le.LabboxEphysRecordingExtractor(recording_object,
download=False)
sorting = le.LabboxEphysSortingExtractor(
sorting_path, samplerate=recording.get_sampling_frequency())
return sorting.object()
def get_recording_object(recording_path):
recording = le.LabboxEphysRecordingExtractor(recording_path,
download=False)
return recording.object()
def recording_is_downloaded(recording_object):
recording = le.LabboxEphysRecordingExtractor(recording_object,
download=False)
return recording.is_local()
