def sort_files():
""" sorts raw files into the correct folders """
logger.info("Sorting raw behavior files")
fls = files(raw_data_folder / "tosort")
if isinstance(fls, list):
logger.debug(f"Sorting {len(fls)} files")
for f in track(fls, description="sorting", transient=True):
src = raw_data_folder / "tosort" / f.name
if f.suffix == ".avi":
dst = raw_data_folder / "video" / f.name
elif f.suffix == ".bin" or f.suffix == ".csv":
dst = raw_data_folder / "analog_inputs" / f.name
else:
logger.warning(f"File not recognized: {f}")
continue
if dst.exists():
logger.debug(f"Destinatoin file already exists, skipping")
else:
logger.info(f"Moving file '{src}' to '{dst}'")
shutil.move(src, dst)
else:
logger.warning(f"Expected files list got: {fls}")
def save_bouts_JSON():
tracking = {}
logger.info("Fetching bouts")
bouts = pd.DataFrame(
(LocomotionBouts & 'complete="true"' & 'direction="outbound"').fetch())
logger.info(f"Got {len(bouts)} bouts")
for i, bout in track(bouts.iterrows(), total=len(bouts)):
save_name = f"{i}_complete_bout.json"
if (save_folder / save_name).exists():
continue
else:
if bout["name"] not in tracking.keys():
tracking[bout["name"]] = Tracking.get_session_tracking(
bout["name"], movement=False, body_only=False)
time.sleep(1)
trk = LocomotionBouts.get_bout_tracking(bout,
tracking[bout["name"]])
trk = {
bp: np.array(list(v.values()))
for bp, v in trk.to_dict().items()
}
bout_dict = {**bout.to_dict(), **trk}
for k, v in bout_dict.items():
if isinstance(v, np.ndarray):
bout_dict[k] = list(v)
recorder.add_data(bout_dict, f"{i}_complete_bout", fmt="json")
def save_rois():
tracking = {}
for ROI in arena.ROIs_dict.keys():
# fetch from database
crossings = pd.DataFrame((ROICrossing * ROICrossing.InitialCondition
& f'roi="{ROI}"'
& "mouse_exits=1").fetch())
logger.info(f"Loaded {len(crossings)} crossings for: '{ROI}'")
for i, crossing in track(crossings.iterrows(), total=len(crossings)):
save_name = f"crossing_{ROI}_{i}.json"
if (save_folder / save_name).exists():
continue
if crossing["name"] not in tracking.keys():
tracking[crossing["name"]] = Tracking.get_session_tracking(
crossing["name"], movement=False, body_only=False)
time.sleep(1)
crossing_trk = ROICrossing.get_crossing_tracking(
crossing, tracking[crossing["name"]])
roi_crossing_dict = {**crossing.to_dict(), **crossing_trk}
for k, v in roi_crossing_dict.items():
if isinstance(v, np.ndarray):
roi_crossing_dict[k] = list(v)
recorder.add_data(roi_crossing_dict,
f"crossing_{ROI}_{i}",
fmt="json")
recorder.add_data(crossings, f"{ROI}_crossings", fmt="h5")
def save_bouts_h5():
bouts_files = files(
save_folder,
"*_complete_bout.json",
)
logger.info(f"Found {len(bouts_files)} bouts JSON files")
# load from json and save as .h5
bouts_json = []
for bf in track(
bouts_files,
description="loading bouts JSON",
transient=True,
):
bouts_json.append(from_json(bf))
recorder.add_data(pd.DataFrame(bouts_json), f"complete_bouts", fmt="h5")
def load_complete_bouts(
duration_max: float = 10,
keep: int = -1,
linearize_to: Path = None,
window: int = 4,
trim: bool = True,
) -> List[Locomotion]:
"""
Loads complete bouts saved as a .h5 file
"""
# load and clean complete bouts
_bouts = pd.read_hdf(
paths.analysis_folder / "behavior" / "saved_data" /
"complete_bouts.h5",
key="hdf",
)
_bouts = _bouts.loc[(_bouts.start_roi == 0)
& (_bouts.duration < duration_max)]
_bouts = _bouts.sort_values("duration").iloc[:keep]
logger.debug(f"Kept {len(_bouts)} bouts")
# turn into Locomotion bouts
bouts = []
for i, bout in track(_bouts.iterrows(),
total=len(_bouts),
description="loading bouts..."):
locomotion_bout = Locomotion(mouse,
bout,
fps=60,
smoothing_window=window)
# add extra stuff from the locomotion bout (e.g. gcoord)
for key in bout.index:
if "_x" not in key and "_y" not in key:
if isinstance(bout[key], list):
bout_data = np.array(bout[key])
else:
bout_data = bout[key]
setattr(locomotion_bout, key, bout_data)
bouts.append(locomotion_bout)
return bouts
module load miniconda
module load cuda/9.0
conda activate dlc
export DLClight=True
echo "running tracking"
python dlc_on_hpc.py '/nfs/winstor/branco/Federico/Locomotion/dlc/loco-FC-2021-10-14/config.yaml' '/VIDEO' '/DEST'
"""
# export CUDA_VISIBLE_DEVICES=0
logger.info(
f"Found {len(to_track)} videos left to track. Generating bash files.")
for video in track(to_track):
winstor_video_path = winstor_folder / "raw" / "video" / video.name
# winstor_video_path = winstor_folder / "dlc" / "videos" / video.name
winstor_save_path = winstor_folder / "raw" / "tracking"
bash_content = BASH_TEMPLATE.replace("VIDEO",
str(winstor_video_path)).replace(
"DEST", str(winstor_save_path))
bash_path = bash_folder / (video.stem + ".sh")
with open(bash_path, "w") as fout:
fout.write(bash_content)
# dos2unix
content, outsize = "", 0
with open(bash_path, "rb") as infile:
axes = f.subplot_mosaic(
"""
AABBB
AACCC
"""
)
draw.ROI(ROI, ax=axes["A"])
SKIP = 0
N = 10 # len(_bouts)
S, T = np.full((80, N), np.nan), np.full((80, N), np.nan)
_S, _T = np.full((80, N), np.nan), np.full((80, N), np.nan)
simulated = []
for i in track(range(N)):
bout = paths[i]
trajectory, _time = MSD(
bout, skip=SKIP, start_frame=2, end_frame=-2
).simulate()
# plot
draw.Tracking(bout.x, bout.y, zorder=-1, ax=axes["A"], alpha=0.8)
draw.Tracking(
trajectory.x,
trajectory.y,
zorder=10,
ax=axes["A"],
color="red",
alpha=0.8,
)
def animate_one(ROI: str, crossing_id: int, FPS: int):
scale = 1 / 30 # to scale velocity vectors
save_folder = (paths.analysis_folder / "behavior" /
"roi_crossings_animations")
# ----------------------------------- prep ----------------------------------- #
# load tracking
bouts = pd.read_hdf(paths.analysis_folder / "behavior" / "roi_crossings" /
f"{ROI}_crossings.h5")
bout = bouts.sort_values("duration").iloc[crossing_id]
logger.info(f"Animating bout {crossing_id} - {round(bout.duration, 2)}s")
# tracking: dict = ROICrossing.get_crossing_tracking(bout.crossing_id)
# generate a path from tracking
path = Path(bout.x, bout.y)
# create fig and start camera
if ROI == "T4":
f = plt.figure(figsize=(12, 12))
elif "S" in ROI:
f = plt.figure(figsize=(8, 14))
else:
f = plt.figure(figsize=(5, 14))
axes = f.subplot_mosaic("""
AA
AA
BB
""")
camera = Camera(f)
# ----------------------------------- plot ----------------------------------- #
n_interpolated_frames = int(60 / FPS)
n_frames = len(bout.x) - 1
trajectory = GrowingPath()
for frame in track(range(n_frames), total=n_frames):
# repeat each frame N times interpolating between frames
for interpol in np.linspace(0, 1, n_interpolated_frames):
# get interpolated quantities
x = interpolate_at_frame(path.x, frame, interpol)
y = interpolate_at_frame(path.y, frame, interpol)
speed = interpolate_at_frame(path.speed, frame, interpol)
trajectory.update(x, y, speed=speed)
# time elapsed
_time = (np.arange(len(trajectory.speed)) / n_interpolated_frames /
60)
# plot the arena
draw.ROI(ROI, set_ax=True, shade=False, ax=axes["A"])
# plot tracking so far
draw.Tracking(trajectory.x, trajectory.y, lw=3, ax=axes["A"])
draw.Dot(x, y, s=50, ax=axes["A"])
# plot speed and velocity vectors
draw.Arrow(
x,
y,
interpolate_at_frame(path.velocity.angle,
frame,
interpol,
method="step"),
L=scale *
interpolate_at_frame(path.velocity.magnitude, frame, interpol),
color=colors.velocity,
outline=True,
width=2,
ax=axes["A"],
)
draw.Arrow(
x,
y,
path.acceleration.angle[frame],
L=4 * scale * path.acceleration.magnitude[frame],
color=colors.acceleration,
outline=True,
width=2,
ax=axes["A"],
zorder=200,
)
# plot speed trace
axes["B"].fill_between(_time,
0,
trajectory.speed,
alpha=0.5,
color=colors.speed)
axes["B"].plot(_time, trajectory.speed, lw=4, color=colors.speed)
axes["A"].set(title=f"{ROI} - crossing: {crossing_id}")
axes["B"].set(xlabel="time (s)", ylabel="speed (cm/s)")
camera.snap()
# ------------------------------- video creation ------------------------------- #
save_path = save_folder / f"{ROI}_{bout.crossing_id}.mp4"
logger.info(f"Saving animation @: '{save_path}'")
animation = camera.animate(interval=1000 / FPS)
animation.save(save_path, fps=FPS)
logger.info("Done!")
plt.cla()
plt.close(f)
del camera
del animation
# ----------------------------------- plot ----------------------------------- #
f = plot_roi_crossing(bout, step=4)
recorder.add_figures(svg=False)
plt.close(f)
from tpd import recorder
from myterial import salmon, teal, indigo
import draw
from data.dbase.db_tables import Tracking, ValidatedSession
from fcutils.progress import track
folder = Path(r"D:\Dropbox (UCL)\Rotation_vte\Locomotion\analysis")
recorder.start(base_folder=folder, folder_name="all_tracking", timestamp=False)
'''
Plot the tracking for each session to check that everything is ok
'''
sessions = ValidatedSession().fetch('name')
for session in track(sessions):
save_name = folder / 'all_tracking' / (session + '.png')
if save_name.exists() or 'open' in session:
continue
tracking = Tracking.get_session_tracking(session,
body_only=False,
movement=False)
if tracking.empty:
logger.info(f'"{session}" - no tracking')
continue
body = tracking.loc[tracking.bpname == 'body'].iloc[0]
snout = tracking.loc[tracking.bpname == 'snout'].iloc[0]
paw = tracking.loc[tracking.bpname == 'right_fl'].iloc[0]
def get_rois_crossings(
tracking: pd.DataFrame,
ROI: arena.ROI,
max_duration:float,
min_speed:float,
) -> dict:
# get every time the mouse enters the ROI
enters = sorted([x for x in set(get_onset_offset(tracking.global_coord > ROI.g_0, 0.5)[0]) if x > 0])
# loop over the results
results = []
for start in track(enters, transient=True):
if tracking.speed[start] < min_speed or start < 60:
continue
# check if anywhere the mouse left the ROI
gcoord = tracking.global_coord[start:start+max_duration+1]
if gcoord[0] > ROI.g_0 + 0.1:
continue
if np.any(gcoord < ROI.g_0):
continue
elif np.any(gcoord > ROI.g_1):
end = np.where(tracking.global_coord[start:] >= ROI.g_1)[0][0] + start
exit = True
else:
end = start + max_duration
exit = False
if end > len(tracking.x):
continue
if tracking.x.min()<0 or tracking.x.max()>40:
continue
duration = (end - start) / 60
if duration < .2 or duration > max_duration/60:
continue
if tracking.global_coord[end] > ROI.g_1 + 0.05:
continue
if end <= start:
continue
# get data
results.append(dict(
roi = ROI.name,
start_frame = start,
end_frame = end,
duration = (end - start) / 60,
mouse_exits = 1 if exit else -1,
gcoord = tracking.global_coord[start:end],
x = tracking.x[start:end]+0.5,
y = tracking.y[start:end]+0.5,
speed = tracking.speed[start:end],
theta = tracking.theta[start:end],
thetadot = tracking.thetadot[start:end],
thetadotdot = tracking.thetadotdot[start:end],
acceleration = tracking.acceleration[start:end],
))
return results
def fill_session_table(table):
logger.info("Filling in session table")
in_table = table.fetch("name")
mice = from_yaml("data\dbase\mice.yaml")
# Load recordings sessoins metadata
recorded_sessions = pd.read_excel(
table.recordings_metadata_path, engine="odf"
)
# Get the videos of all sessions
vids = [f for f in files(raw_data_folder / "video") if ".avi" in f.name]
for video in track(vids, description="Adding sessions", transient=True):
# Get session data
name = video.name.split("_video")[0]
if name in in_table:
continue
if "opto" in name:
logger.info(
f"Skipping session {name} because its OPTOGENETICS session"
)
continue
if "test" in name.lower() in name.lower():
logger.info(f"Skipping session {name} as it is a test")
continue
# get date and mouse
try:
date = name.split("_")[1]
mouse = [
m["mouse"]["mouse_id"]
for m in mice
if m["mouse"]["mouse_id"] in name
][0]
except IndexError:
logger.warning(
f"Skipping session {name} because couldnt figure out the mouse or date it was done on"
)
continue
key = dict(mouse_id=mouse, name=name, date=date)
# get file paths
key["video_file_path"] = (
raw_data_folder / "video" / (name + "_video.avi")
)
key["ai_file_path"] = (
raw_data_folder / "analog_inputs" / (name + "_analog.bin")
)
key["csv_file_path"] = (
raw_data_folder / "analog_inputs" / (name + "_data.csv")
)
if (
not key["video_file_path"].exists()
or not key["ai_file_path"].exists()
):
raise FileNotFoundError(
f"Either video or AI files not found for session: {name} with data:\n{key}"
)
# get ephys files & arena type
if name in recorded_sessions["bonsai filename"].values:
rec = recorded_sessions.loc[
recorded_sessions["bonsai filename"] == name
].iloc[0]
base_path = (
table.recordings_raw_data_path
/ rec["recording folder"]
/ (rec["recording folder"] + "_imec0")
/ (rec["recording folder"] + "_t0.imec0")
)
key["ephys_ap_data_path"] = str(base_path) + ".ap.bin"
key["ephys_ap_meta_path"] = str(base_path) + ".ap.meta"
key["arena"] = rec.arena
key["is_recording"] = 1
key["date"] = rec.date
else:
key["ephys_ap_data_path"] = ""
key["ephys_ap_meta_path"] = ""
key["arena"] = "hairpin"
key["is_recording"] = 0
# add to table
insert_entry_in_table(key["name"], "name", key, table)
def test_track():
N = np.linspace(0, 1000, 1)
for n in track(np.arange(N), transient=True):
pass