def ingest():
# ========== Insert new "Session" ===========
data_dir = get_ephys_root_data_dir()
# Folder structure: root / subject / session / probe / .ap.meta
sessions, sess_folder_names = [], []
for subj_key in subject.Subject.fetch('KEY'):
subj_dir = data_dir / subj_key['subject']
if subj_dir.exists():
for meta_filepath in subj_dir.rglob('*.ap.meta'):
sess_folder = meta_filepath.parent.parent.name
if sess_folder not in sess_folder_names:
npx_meta = neuropixels.NeuropixelsMeta(meta_filepath)
sessions.append({
**subj_key, 'session_datetime':
npx_meta.recording_time
})
sess_folder_names.append(sess_folder)
print(f'Inserting {len(sessions)} session(s)')
Session.insert(sessions, skip_duplicates=True)
# ========== Insert new "ProbeInsertion" ===========
probe_insertions = []
for sess_key in Session.fetch('KEY'):
subj_dir = data_dir / sess_key['subject']
if subj_dir.exists():
for meta_filepath in subj_dir.rglob('*.ap.meta'):
npx_meta = neuropixels.NeuropixelsMeta(meta_filepath)
probe = {
'probe_type': npx_meta.probe_model,
'probe': npx_meta.probe_SN
}
ephys.Probe.insert1(probe, skip_duplicates=True)
probe_dir = meta_filepath.parent
probe_number = re.search('(imec)?\d{1}$',
probe_dir.name).group()
probe_number = int(probe_number.replace(
'imec',
'')) if 'imec' in probe_number else int(probe_number)
probe_insertions.append({
**sess_key,
**probe, 'insertion_number':
int(probe_number)
})
print(f'Inserting {len(probe_insertions)} probe_insertion(s)')
ephys.ProbeInsertion.insert(probe_insertions,
ignore_extra_fields=True,
skip_duplicates=True)
def make(self, key):
npx_dir = EphysRecording._get_npx_data_dir(key)
meta_filepath = next(pathlib.Path(npx_dir).glob('*.ap.meta'))
npx_meta = neuropixels.NeuropixelsMeta(meta_filepath)
ks_dir = Unit._get_ks_data_dir(key)
ks = kilosort.Kilosort(ks_dir)
# -- Remove 0-spike units
withspike_idx = [
i for i, u in enumerate(ks.data['cluster_ids'])
if (ks.data['spike_clusters'] == u).any()
]
valid_units = ks.data['cluster_ids'][withspike_idx]
valid_unit_labels = ks.data['cluster_groups'][withspike_idx]
# -- Get channel and electrode-site mapping
e_config_key = (EphysRecording * ElectrodeConfig & key).fetch1('KEY')
chn2electrodes = get_npx_chn2electrode_map(npx_meta, e_config_key)
# -- Insert unit, label, peak-chn
units = []
for unit, unit_lbl in zip(valid_units, valid_unit_labels):
if (ks.data['spike_clusters'] == unit).any():
unit_channel, _ = ks.get_best_channel(unit)
units.append({
'unit': unit,
'cluster_quality_label': unit_lbl,
**chn2electrodes[unit_channel]
})
self.insert([{**key, **u} for u in units])
def make(self, key):
npx_dir = EphysRecording._get_npx_data_dir(key)
meta_filepath = next(pathlib.Path(npx_dir).glob('*.ap.meta'))
npx_meta = neuropixels.NeuropixelsMeta(meta_filepath)
if re.search('(1.0|2.0)', npx_meta.probe_model):
eg_members = []
probe_type = {'probe_type': npx_meta.probe_model}
q_electrodes = ProbeType.Electrode & probe_type
for shank, shank_col, shank_row, is_used in npx_meta.shankmap[
'data']:
electrode = (q_electrodes & {
'shank': shank,
'shank_col': shank_col,
'shank_row': shank_row
}).fetch1('KEY')
eg_members.append({**electrode, 'is_used': is_used})
else:
raise NotImplementedError(
'Processing for neuropixels probe model {} not yet implemented'
.format(npx_meta.probe_model))
# ---- compute hash for the electrode config (hash of dict of all ElectrodeConfig.Electrode) ----
ec_hash = uuid.UUID(
utils.dict_to_hash({k['electrode']: k
for k in eg_members}))
el_list = sorted([k['electrode'] for k in eg_members])
el_jumps = [-1] + np.where(
np.diff(el_list) > 1)[0].tolist() + [len(el_list) - 1]
ec_name = '; '.join([
f'{el_list[s + 1]}-{el_list[e]}'
for s, e in zip(el_jumps[:-1], el_jumps[1:])
])
e_config = {**probe_type, 'electrode_config_name': ec_name}
# ---- make new ElectrodeConfig if needed ----
if not (ElectrodeConfig & {'electrode_config_uuid': ec_hash}):
ElectrodeConfig.insert1({
**e_config, 'electrode_config_uuid':
ec_hash
})
ElectrodeConfig.Electrode.insert({
**e_config,
**m
} for m in eg_members)
self.insert1({**key, **e_config})
def get_ephys_probe_data_dir(probe_key):
root_dir = get_ephys_root_data_dir()
subj = probe_key['subject']
probe_no = probe_key['insertion_number']
sess_date_string = probe_key['session_datetime'].strftime('%m%d%y')
dir_pattern = f'*{subj}_{sess_date_string}*_imec{probe_no}'
npx_meta_pattern = f'{subj}_{sess_date_string}*imec{probe_no}.ap.meta'
try:
npx_meta_fp = next(
root_dir.rglob('/'.join([dir_pattern, npx_meta_pattern])))
except StopIteration:
return None
npx_meta = neuropixels.NeuropixelsMeta(npx_meta_fp)
# ensuring time difference between behavior-start and ephys-start is no more than 2 minutes - this is to handle multiple sessions in a day
start_time_difference = abs(
(npx_meta.recording_time -
probe_key['session_datetime']).total_seconds())
if start_time_difference <= 120:
return npx_meta_fp.parent
def make(self, key):
units = {
u['unit']: u
for u in (Unit & key).fetch(as_dict=True, order_by='unit')
}
npx_dir = EphysRecording._get_npx_data_dir(key)
meta_filepath = next(pathlib.Path(npx_dir).glob('*.ap.meta'))
npx_meta = neuropixels.NeuropixelsMeta(meta_filepath)
ks_dir = Unit._get_ks_data_dir(key)
ks = kilosort.Kilosort(ks_dir)
# -- Get channel and electrode-site mapping
e_config_key = (EphysRecording * ElectrodeConfig & key).fetch1('KEY')
chn2electrodes = get_npx_chn2electrode_map(npx_meta, e_config_key)
is_qc = (Clustering & key).fetch1('quality_control')
unit_waveforms, unit_peak_waveforms = [], []
if is_qc:
unit_wfs = np.load(ks_dir /
'mean_waveforms.npy') # unit x channel x sample
for unit_no, unit_wf in zip(ks.data['cluster_ids'], unit_wfs):
if unit_no in units:
for chn, chn_wf in zip(ks.data['channel_map'], unit_wf):
unit_waveforms.append({
**units[unit_no],
**chn2electrodes[chn], 'waveform_mean':
chn_wf
})
if chn2electrodes[chn]['electrode'] == units[unit_no][
'electrode']:
unit_peak_waveforms.append({
**units[unit_no], 'peak_chn_waveform_mean':
chn_wf
})
else:
npx_recording = neuropixels.Neuropixels(npx_dir)
for unit_no, unit_dict in units.items():
spks = (UnitSpikeTimes & unit_dict).fetch1('unit_spike_times')
wfs = npx_recording.extract_spike_waveforms(
spks, ks.data['channel_map']) # (sample x channel x spike)
wfs = wfs.transpose((1, 2, 0)) # (channel x spike x sample)
for chn, chn_wf in zip(ks.data['channel_map'], wfs):
unit_waveforms.append({
**unit_dict,
**chn2electrodes[chn], 'waveform_mean':
chn_wf.mean(axis=0),
'waveforms':
chn_wf
})
if chn2electrodes[chn]['electrode'] == unit_dict[
'electrode']:
unit_peak_waveforms.append({
**unit_dict, 'peak_chn_waveform_mean':
chn_wf.mean(axis=0)
})
self.insert(unit_peak_waveforms, ignore_extra_fields=True)
self.Electrode.insert(unit_waveforms, ignore_extra_fields=True)
subject.Subject.insert(subjects, skip_duplicates=True)
# ========== Insert new "Session" ===========
data_dir = get_ephys_root_data_dir()
sessions = []
for subj_key in subject.Subject.fetch('KEY'):
subj_dir = data_dir / subj_key['subject']
if subj_dir.exists():
try:
meta_filepath = next(subj_dir.rglob('*.ap.meta'))
except StopIteration:
continue
npx_meta = neuropixels.NeuropixelsMeta(meta_filepath)
sessions.append({**subj_key, 'session_datetime': npx_meta.recording_time})
print(f'Inserting {len(sessions)} session(s)')
Session.insert(sessions, skip_duplicates=True)
# ========== Insert new "ProbeInsertion" ===========
probe_insertions = []
for sess_key in Session.fetch('KEY'):
subj_dir = data_dir / sess_key['subject']
if subj_dir.exists():
for meta_filepath in subj_dir.rglob('*.ap.meta'):
npx_meta = neuropixels.NeuropixelsMeta(meta_filepath)
probe = {'probe_type': npx_meta.probe_model, 'probe': npx_meta.probe_SN}