def _make_tuples(self, key):
# Get DFF threshold to compute dff
s, m = key['sess_name'], key['mouse']
tc = key['trial_class']
session = (ManualTrials & key).fetch1("manual_sess_name")
f, keys, subkeys, allkeys = open_hdf(summary_file)
trial = dict(f['all'][tc][session][key['trial_name']])
rois_trials = [k for k in trial.keys() if 'C_ROI' in k]
rois_dff = (RoiDFF & f"sess_name='{s}'"
& f"mouse='{m}'").fetch('roi_id')
for roi, roi_dff in zip(rois_trials, rois_dff):
roi_trace = trial[roi][()]
try:
dffth = (RoiDFF & f"sess_name='{s}'" & f"mouse='{m}'"
& f"roi_id='{roi_dff}'").fetch1('dff_th')
except Exception as e:
raise ValueError(f'AAAA Skipping: {e}')
rkey = key.copy()
rkey['roi_id'] = roi
rkey['signal'] = roi_trace
rkey['dff'] = (roi_trace - dffth) / dffth
manual_insert_skip_duplicate(self, rkey)
def _make_tuples(self, key):
session_fld = get_session_folder(**key)
fps = (Recording & key).fetch1('fps_behav')
# Load recordings AI file
aifile = (Recording & key).fetch1("aifile")
f, keys, subkeys, allkeys = open_hdf(os.path.join(session_fld, aifile))
# Get the start and end time in behaviour frames
camera_triggers = f['AI']['0'][()]
microscope_triggers = f['AI']['1'][()]
starts, ends = get_times_signal_high_and_low(microscope_triggers)
cam_starts, cam_ends = get_times_signal_high_and_low(camera_triggers)
# Get the start end time of recording in sample numbers
print('\nExactring start end of individual recordings')
rec_starts = [starts[0]] + [
starts[s] for s in list(
np.where(derivative(starts) > self.sampling_frequency)[0])
]
rec_ends = []
for rs in rec_starts:
_ends = np.array([e for e in ends if e > rs])
try:
nxt = np.where(
derivative(_ends) > self.sampling_frequency)[0][0]
rec_ends.append(_ends[nxt - 1])
except:
rec_ends.append(ends[-1])
if len(rec_ends) != len(rec_starts):
raise ValueError('Something went wrong')
startends = [(s, e) for s, e in zip(rec_starts, rec_ends)]
# Go from sample number to frame numbers
def smpl2frm(val):
return np.int32(round((val / self.sampling_frequency) * fps))
startends = [(smpl2frm(s), smpl2frm(e)) for s, e in startends]
# Get is recording
roi_data = f['Fiji_ROI_1'][()]
is_recording = np.zeros_like(roi_data)
for start, end in startends:
is_recording[start:end - 20] = 1
key['is_ca_recording'] = is_recording
key['starts'] = np.array([s for s, e in startends])
key['ends'] = np.array([e for s, e in startends])
key['camera_frames'] = camera_triggers
key['microscope_frames'] = camera_triggers
key['n_frames'] = len(is_recording)
manual_insert_skip_duplicate(self, key)
def _make_tuples(self, key):
sub = [
f for f in fld.glob('*') if f.is_dir() and key['mouse'] in str(f)
][0]
hdfs = sorted([f for f in sub.glob('*.hdf5') if 'Fiji-tag' in f.name])
exp_names = [f.name.split('_Fiji')[0] for f in hdfs]
exp_data = {}
for exp in exp_names:
h = [h for h in hdfs if exp in str(h)]
v = [f for f in sub.glob('*.mp4') if exp in str(f)]
if len(h) != 1 or len(v) != 1:
continue
# raise ValueError(f'h:{h}\nv:{v}')
exp_data[exp] = dict(hdf=h[0], video=v[0])
for name, data in exp_data.items():
if '_t' in name:
splitter = '_t'
else:
splitter = '_v'
_, _, _, fps, _ = get_video_params(
get_cap_from_file(str(data['video'])))
try:
f, keys, subkeys, allkeys = open_hdf(str(data['hdf']))
except Exception as e:
print(f'Failed to open AI file: {data["hdf"].name}:\n{e}')
return
roi = [k for k in keys if 'Fiji_ROI' in k][0]
sig = f[roi][()]
is_rec = np.zeros_like(sig)
is_rec[sig > 0] = 1
rec_starts = np.where(derivative(is_rec) > 0)[0]
rec_ends = np.where(derivative(is_rec) < 0)[0]
ekey = key.copy()
ekey['date'] = name.split('_')[0]
ekey['rec'] = int(name.split(splitter)[1][0])
ekey['name'] = name
ekey['hdf_path'] = str(data['hdf'])
ekey['video_path'] = str(data['video'])
ekey['video_fps'] = fps
ekey['is_ca_rec'] = is_rec
ekey['ca_rec_starts'] = rec_starts
ekey['ca_rec_ends'] = rec_ends
manual_insert_skip_duplicate(self, ekey)
def _make_tuples(self, key):
session_fld = get_session_folder(**key)
recs = load_yaml(metadatafile)['sessions'][key['mouse']][
key['sess_name']]
for n, rec in enumerate(sorted(recs)):
# Get files
rec_files = [f for f in os.listdir(session_fld) if rec in f]
videos = [
f for f in rec_files
if f.endswith(".mp4") or f.endswith(".avi")
]
if len(videos) != 1:
if len(set([get_file_name(f) for f in videos])) == 1:
video = get_file_name(videos[0]) + ".mp4"
else:
raise ValueError
else:
video = videos[0]
temp_fld = os.path.join(self.temp_files_fld, key['mouse'])
temp_rec_files = [f for f in os.listdir(temp_fld) if rec in f]
ais = [fl for fl in temp_rec_files if fl == f"{rec}_Fiji-tag.hdf5"]
if not ais:
continue
if len(ais) != 1: raise ValueError(f'Found ais: {ais}')
else: ai = ais[0]
# Open video and get number of frames
nframes, width, height, fps, _ = get_video_params(
get_cap_from_file(os.path.join(session_fld, video)))
# Open AI file and get number of samples
f, keys, subkeys, allkeys = open_hdf(os.path.join(session_fld, ai))
n_samples = len(f['AI']['0'][()])
rkey = key.copy()
rkey['rec_name'] = rec
rkey['rec_n'] = n
rkey['videofile'] = video
rkey['aifile'] = ai
rkey['n_frames'] = nframes
rkey['fps_behav'] = fps
rkey['n_samples'] = n_samples
manual_insert_skip_duplicate(self, rkey)
def populate(self):
f, keys, subkeys, allkeys = open_hdf(summary_file)
for trial_class in subkeys['all']:
sessions = list(dict(f['all'][trial_class]).keys())
for session in sessions:
trials = list(dict(f['all'][trial_class][session]).keys())
for trial in trials:
tkey = dict(
trial_class=trial_class,
mouse=session.split("_")[1],
sess_name=session.split("_")[2],
frame=int(trial.split("_")[1]),
trial_name=trial,
manual_sess_name=session,
)
manual_insert_skip_duplicate(self, tkey)
def _make_tuples(self, key):
session_fld = get_session_folder(**key)
recs = load_yaml(metadatafile)['sessions'][key['mouse']][
key['sess_name']]
# Get each session's recordin AI file
for n, rec in enumerate(sorted(recs)):
rec_files = [f for f in os.listdir(session_fld) if rec in f]
ais = [fl for fl in rec_files if fl == f"{rec}.hdf5"]
if len(ais) != 1: raise ValueError
else: ai = ais[0]
f, keys, subkeys, allkeys = open_hdf(os.path.join(session_fld, ai))
# Get stim start and protocol name
try:
protocol_names = dict(
f['Audio Stimulation'])['Audio Stimuli Names'][(
)].decode("utf-8")
except KeyError as err:
return
protocol_names = protocol_names.replace("[", "").replace(
"]", "").replace("'", "")
protocol_names = protocol_names.split(", u")
protocol_names = [pc.split("\\\\")[-1] for pc in protocol_names]
start_frames = np.where(
dict(f['Audio Stimulation'])['Audio Stimuli Start Indices'][
()] > 0)[0]
if len(start_frames) != len(protocol_names): raise ValueError
for sf, pc in zip(start_frames, protocol_names):
skey = key.copy()
skey['frame'] = sf
skey['protocol_name'] = pc
skey['rec_name'] = rec
manual_insert_skip_duplicate(self, skey)
def _make_tuples(self, key):
# Insert into main table
manual_insert_skip_duplicate(self, key)
# Get number of frmes per recording
frames_shift = (Sessions & key).fetch1('frames_shift')
# Iterate over recordings
hdf = (Experiment & key).fetch('hdf_path')
for rec_n, h in enumerate(hdf):
h = h.replace('.hdf5', '_forFede.hdf5')
try:
f, keys, subkeys, allkeys = open_hdf(h)
except FileExistsError:
print(f'File: {Path(h).name} does not exist!')
# Get tag entries for each type of event
for k in keys:
if k in ('audio', 'visual', 'auditory', 'visual_audio'):
self.process_stim_evoked(k, f, frames_shift, key, rec_n)
else:
self.process_other(k, f, frames_shift, key, rec_n)
def _make_tuples(self, key):
session_fld = get_session_folder(**key)
aifile = (Recording & key).fetch1("aifile")
f, keys, subkeys, allkeys = open_hdf(os.path.join(session_fld, aifile))
rois = [k for k in keys if 'Fiji_ROI' in k]
print(f"{key['rec_name']} -> {len(rois)} rois")
rec_name = key['rec_name']
is_recording = (TiffTimes
& f"rec_name='{rec_name}'").fetch1("is_ca_recording")
for roi in rois:
roi_trace = f[roi][()]
if len(roi_trace) != len(is_recording):
raise ValueError('oops')
rkey = key.copy()
rkey['roi_id'] = roi
rkey['signal'] = roi_trace
manual_insert_skip_duplicate(self, rkey)
def get_trial_content(trial_class, session, trial_name):
f, keys, subkeys, allkeys = open_hdf(summary_file)
return dict(f['all'][trial_class][session][trial_name])
def _make_tuples(self, key):
# Get all experiments for this mouse, by date
exps = pd.DataFrame((Experiment & key).fetch(as_dict=True))
if len(exps) == 0:
return
# Loop over dates
for date in exps.date.unique():
es = exps.loc[exps.date == date].sort_values('name', axis=0)
# get stuff
n_frames = [len(r.is_ca_rec) for i, r in es.iterrows()]
frames_shift = [0] + list(np.cumsum(n_frames[:-1]))
if len(es.video_fps.unique()) > 1:
raise ValueError('More than one video fps')
is_ca_rec = np.concatenate(es.is_ca_rec.values)
ca_rec_starts = np.concatenate([
np.array(ca) + frames_shift[n]
for n, ca in enumerate(es.ca_rec_starts.values)
])
ca_rec_ends = np.concatenate([
np.array(ca) + frames_shift[n]
for n, ca in enumerate(es.ca_rec_ends.values)
])
# f, ax = plt.subplots()
# ax.plot(is_ca_rec)
# for s,e in zip(ca_rec_starts, ca_rec_ends):
# ax.axvline(s, color='g')
# ax.axvline(e, color='m')
# plt.show()
# Check that ROIs match
roi_data = [open_hdf(f) for f in es.hdf_path]
rois = sorted([k for k in roi_data[0][1] if 'Fiji_ROI' in k])
if len(roi_data) > 1:
for (_, keys, _, _) in roi_data[1:]:
for k in keys:
if 'Fiji_ROI' in k and k not in rois:
raise ValueError('Unrecognized ROI')
# Stack ROIs
rois_stacks = {}
for roi in rois:
rois_stacks[roi] = np.concatenate(
[f[roi][()] for f, _, _, _ in roi_data])
# Stack tracking
trackings = []
for i, s in es.iterrows():
name = s['name']
trackings.append((Trackings * Trackings.BodyPartTracking & key
& 'bp="body"' & f'date="{s.date}"'
& f'name="{name}"').fetch('x',
'y',
'speed',
'dir_of_mvmt',
as_dict=True)[0])
keys = list(trackings[0].keys())
tracking = {
k: np.concatenate([t[k] for t in trackings])
for k in keys
}
# time to insert in the table
mkey = key.copy()
mkey['date'] = date
mkey['video_fps'] = es.video_fps.values[0]
mkey['is_ca_rec'] = is_ca_rec
mkey['ca_rec_starts'] = ca_rec_starts
mkey['ca_rec_ends'] = ca_rec_ends
mkey['exps_nframes'] = n_frames
mkey['frames_shift'] = frames_shift
manual_insert_skip_duplicate(self, mkey)
tkey = key.copy()
tkey['date'] = date
tkey['x'] = tracking['x']
tkey['y'] = tracking['y']
tkey['s'] = tracking['speed']
tkey['dir_of_mvmt'] = tracking['dir_of_mvmt']
manual_insert_skip_duplicate(self.Tracking, tkey)
for roi, data in rois_stacks.items():
rkey = key.copy()
rkey['date'] = date
rkey['roi_name'] = roi
rkey['sig'] = data
manual_insert_skip_duplicate(self.RoiTrace, rkey)