def sort_files():
""" sorts raw files into the correct folders """
logger.info("Sorting raw behavior files")
fls = files(raw_data_folder / "tosort")
if isinstance(fls, list):
logger.debug(f"Sorting {len(fls)} files")
for f in track(fls, description="sorting", transient=True):
src = raw_data_folder / "tosort" / f.name
if f.suffix == ".avi":
dst = raw_data_folder / "video" / f.name
elif f.suffix == ".bin" or f.suffix == ".csv":
dst = raw_data_folder / "analog_inputs" / f.name
else:
logger.warning(f"File not recognized: {f}")
continue
if dst.exists():
logger.debug(f"Destinatoin file already exists, skipping")
else:
logger.info(f"Moving file '{src}' to '{dst}'")
shutil.move(src, dst)
else:
logger.warning(f"Expected files list got: {fls}")
def save_bouts_h5():
bouts_files = files(
save_folder,
"*_complete_bout.json",
)
logger.info(f"Found {len(bouts_files)} bouts JSON files")
# load from json and save as .h5
bouts_json = []
for bf in track(
bouts_files,
description="loading bouts JSON",
transient=True,
):
bouts_json.append(from_json(bf))
recorder.add_data(pd.DataFrame(bouts_json), f"complete_bouts", fmt="h5")
from loguru import logger
from fcutils.progress import track
from fcutils.path import files
"""
Creates a set of bash scripts to analyze files with DLC on HPC
"""
# videos_folder = Path(r'W:\swc\branco\Federico\Locomotion\dlc\videos')
videos_folder = Path(r"W:\swc\branco\Federico\Locomotion\raw\video")
tracking_folder = Path(r"W:\swc\branco\Federico\Locomotion\raw\tracking")
bash_folder = Path(r"W:\swc\branco\Federico\Locomotion\dlc\dlc_individuals")
winstor_folder = PurePosixPath("nfs/winstor/branco/Federico/Locomotion")
# get videos that have not been tracked yet
videos = files(videos_folder, "FC_*.avi")
videos = [
v for v in videos if "test" not in v.name.lower() and "t_" not in v.name
]
trackings = [f.name for f in files(tracking_folder, "*.h5")]
logger.info(f"Found {len(videos)} videos and {len(trackings)} tracking files")
to_track = []
for video in videos:
tracked = [f for f in trackings if video.stem in f]
if not tracked:
to_track.append(video)
# prepare bash files
BASH_TEMPLATE = """#! /bin/bash
from pathlib import Path
import numpy as np
from myterial.utils import make_palette
import brainrender as br
from myterial import blue_grey_dark, blue_grey_light
from fcutils.path import files
# ----------------------------------- prep ----------------------------------- #
br.settings.SHOW_AXES = False
br.settings.ROOT_ALPHA = 0.1
probes_folder = Path(
r"D:\Dropbox (UCL)\Rotation_vte\Locomotion\reconstructed_probe_location")
probes_files = files(probes_folder, pattern="*_0.npy")
save_folder = Path(
r"D:\Dropbox (UCL)\Rotation_vte\Presentations\Presentations\Fiete lab")
colors = make_palette(blue_grey_dark, blue_grey_light, len(probes_files))
# ----------------------------- create renderings ---------------------------- #
for repeat in range(3):
# scene and brain regions
scene = br.Scene(inset=False, screenshots_folder=save_folder)
scene.add_brain_region("CUN", "GRN", "PPN", alpha=0.9)
# add probes
for n, probe in enumerate(probes_files):
sys.path.append("./")
import matplotlib.pyplot as plt
import pandas as pd
from myterial import pink
import draw
f, ax = plt.subplots(figsize=(7, 10))
# load and draw tracking data
from fcutils.path import files
for fp in files(
"/Users/federicoclaudi/Dropbox (UCL)/Rotation_vte/Locomotion/analysis/control/",
"*.h5",
):
tracking = pd.read_hdf(fp, key="hdf")
tracking.x = 20 - tracking.x + 20
# draw.Tracking.scatter(tracking.x, tracking.y, c=tracking.theta, vmin=0, vmax=360, cmap='bwr', lw=1, ec='k')
draw.Tracking(tracking.x, tracking.y, alpha=0.7)
# draw hairpin arena
draw.Hairpin(ax)
# draw waypoints
wps = Waypoints()
for wp in wps:
draw.Arrow(wp.x, wp.y, wp.theta, 2, width=4, color=pink, outline=True)
# fit dubin path
["k", "b", "r", "m", green]
# scene and brain regions
scene = br.Scene()
scene.add_brain_region("CUN", "GRN", alpha=.88)
scene.add_brain_region("IC",
"PRNr",
"PRNc",
"SCm",
alpha=0.25,
silhouette=False)
scene.add_brain_region("PPN", alpha=0.8, silhouette=False)
# add probes
for n, probe in enumerate(files(probes_folder, pattern="*_0.npy")):
coords = np.load(probe)
probe_actor = scene.add(
br.actors.Points(coords, colors=colors[probe.stem], radius=40))
logger.info(
f"Adding probe from {probe.name} with color: {colors[probe.stem]}")
# slice
plane = scene.atlas.get_plane(norm=(0, 0, 1),
pos=scene.root._mesh.centerOfMass())
scene.slice(plane)
# render
cam = {
"pos": (7196, 247, -38602),
"viewup": (0, -1, 0),
def test_files(fld):
from_str = path.files(fld)
from_pth = path.files(Path(fld))
assert len(from_str) == len(from_pth)
def fill_session_table(table):
logger.info("Filling in session table")
in_table = table.fetch("name")
mice = from_yaml("data\dbase\mice.yaml")
# Load recordings sessoins metadata
recorded_sessions = pd.read_excel(
table.recordings_metadata_path, engine="odf"
)
# Get the videos of all sessions
vids = [f for f in files(raw_data_folder / "video") if ".avi" in f.name]
for video in track(vids, description="Adding sessions", transient=True):
# Get session data
name = video.name.split("_video")[0]
if name in in_table:
continue
if "opto" in name:
logger.info(
f"Skipping session {name} because its OPTOGENETICS session"
)
continue
if "test" in name.lower() in name.lower():
logger.info(f"Skipping session {name} as it is a test")
continue
# get date and mouse
try:
date = name.split("_")[1]
mouse = [
m["mouse"]["mouse_id"]
for m in mice
if m["mouse"]["mouse_id"] in name
][0]
except IndexError:
logger.warning(
f"Skipping session {name} because couldnt figure out the mouse or date it was done on"
)
continue
key = dict(mouse_id=mouse, name=name, date=date)
# get file paths
key["video_file_path"] = (
raw_data_folder / "video" / (name + "_video.avi")
)
key["ai_file_path"] = (
raw_data_folder / "analog_inputs" / (name + "_analog.bin")
)
key["csv_file_path"] = (
raw_data_folder / "analog_inputs" / (name + "_data.csv")
)
if (
not key["video_file_path"].exists()
or not key["ai_file_path"].exists()
):
raise FileNotFoundError(
f"Either video or AI files not found for session: {name} with data:\n{key}"
)
# get ephys files & arena type
if name in recorded_sessions["bonsai filename"].values:
rec = recorded_sessions.loc[
recorded_sessions["bonsai filename"] == name
].iloc[0]
base_path = (
table.recordings_raw_data_path
/ rec["recording folder"]
/ (rec["recording folder"] + "_imec0")
/ (rec["recording folder"] + "_t0.imec0")
)
key["ephys_ap_data_path"] = str(base_path) + ".ap.bin"
key["ephys_ap_meta_path"] = str(base_path) + ".ap.meta"
key["arena"] = rec.arena
key["is_recording"] = 1
key["date"] = rec.date
else:
key["ephys_ap_data_path"] = ""
key["ephys_ap_meta_path"] = ""
key["arena"] = "hairpin"
key["is_recording"] = 0
# add to table
insert_entry_in_table(key["name"], "name", key, table)