def get_not_tracked_files():
videos = [f for f in listdir(raw_video_fld) if f.endswith(".mp4")]
tracked = [
get_file_name(f) for f in listdir(raw_tracking_fld)
if f.endswith(".h5")
]
not_tracked = []
for video in videos:
is_tracked = [f for f in tracked if get_file_name(video) in f]
if not is_tracked:
not_tracked.append(video)
print(f"Found {len(not_tracked)} videos to track.")
for vid in not_tracked:
print(f" {vid}")
return not_tracked
def _make_tuples(self, key):
# Check if we have a tracking data file
tracking_fname = Session._get_formatted_date(
key['date']) + f'_{key["mouse_id"]}_video'
tracking_file = [
f for f in listdir(raw_tracking_fld) if tracking_fname in f
]
if not tracking_file:
print(
f"No tracking file for session {Session._get_formatted_date(key['date'])} - {key['mouse_id']}"
)
return
else:
tracking_file = tracking_file[0]
# Insert entry into main class
self.insert1(key)
# Load and clean tracking data
bp_tracking = prepare_tracking_data(tracking_file,
median_filter=True,
fisheye=False,
common_coord=False,
compute=True)
bones_tracking = compute_body_segments(bp_tracking, self.bsegments)
# Insert into the bodyparts tracking
for bp, tracking in bp_tracking.items():
bp_key = key.copy()
bp_key['bp'] = bp
bp_key['x'] = tracking.x.values
bp_key['y'] = tracking.y.values
bp_key['speed'] = tracking.speed.values
bp_key['dir_of_mvmt'] = tracking.direction_of_movement.values
bp_key['angular_velocity'] = tracking.angular_velocity.values
self.BodyPartTracking.insert1(bp_key)
# Insert into the body segment data
for bone, (bp1, bp2) in self.bsegments.items():
segment_key = key.copy()
segment_key['bp1'] = bp1
segment_key['bp2'] = bp2
segment_key['orientation'] = bones_tracking[bone][
'orientation'].values
segment_key['angular_velocity'] = bones_tracking[bone][
'angular_velocity'].values
self.BodySegmentTracking.insert1(segment_key)
def convert_intputs_ais_to_pandas():
files = [f for f in listdir(raw_analog_inputs_fld)]
tdms = [f for f in files if f.endswith(".tdms")]
h5 = [get_file_name(f) for f in files if f.endswith(".h5")]
to_convert = [f for f in tdms if get_file_name(f) not in h5]
print(
f'Found {len(to_convert)} tdms files to convert. e.g.: {to_convert[0]}\n'
)
pool = mp.Pool(mp.cpu_count() - 2)
pool.map(get_analog_inputs_clean_dataframe, to_convert)
pool.close()
def __init__(self,
calibration_images_folder,
camera_name,
checkerboard_shape,
dark_threshold,
images_extention='.png',
save_folder=None,
save_inverted_map=True):
"""
Initialise and run fish eye calibration
:param calibration_images_folder: path to folder with calibration images
:param images_extention: file type of images
:param camera_name: str, used to save the maps
:param checkerboard_shape: tuple with #size of the checkerboard (# of vertices in each dimension, not including those on the edge)
:param dark_threshold: int, The algorithm is finnicky and likes saturation, so set pixels with values below the dark_threshold to black
:param save_folder: str, path to a directory where to save the output
:save_inverted_map: bool, if true an inverted map used for tracking data correction is saved
"""
# ----------------------------------- setup ---------------------------------- #
check_create_folder(calibration_images_folder, raise_error=True)
self.calibration_images = [
f for f in listdir(calibration_images_folder)
if images_extention in f
]
print('Found ' + str(len(self.calibration_images)) +
' images for fisheye calibration.')
if save_folder is None:
self.save_folder = calibration_images_folder
else:
check_create_folder(save_folder)
self.save_folder = save_folder
self.camera_name = camera_name
self.checkerboard_shape = checkerboard_shape
self.dark_threshold = dark_threshold
self.save_inverted_map = save_inverted_map
mouse=mouse,
injected=injected)
bp_tracking = fetch_tracking(for_bp=True, **filt)
bones_tracking = fetch_tracking(for_bp=False, **filt)
tracking = fetch_tracking_processed(**filt)
tracking = get_frames_state(tracking)
tracking = get_when_in_center(tracking, center, radius)
n_frames = len(tracking.x)
if use_real_video:
data = Session._get_formatted_date(tracking.date.iloc[0])
video = [
f for f in listdir(raw_video_fld)
if mouse in f and data in f and f.endswith(".mp4")
][0]
videocap = get_cap_from_file(video)
# %%
# -------------------------------- Write clip -------------------------------- #
savepath = os.path.join(output_fld,
f'{mouse}_{injected}_real_{use_cluster}.mp4')
writer = open_cvwriter(savepath,
w=frame_shape[0],
h=frame_shape[1],
framerate=fps,
format=".mp4",
iscolor=True)
def get_not_converted_videos(convert_videos, fps=None):
raw_vids = [f for f in listdir(raw_video_fld) if f.endswith(".tdms")]
converted = [
f.split(".")[0] for f in listdir(raw_video_fld) if f.endswith(".mp4")
]
to_convert = []
for raw in raw_vids:
if raw.split(".")[0] not in converted:
to_convert.append(raw)
print("\n\nFound {} videos to convert.".format(len(to_convert)))
for vid in to_convert:
print(" ", vid)
if convert_videos:
for video in raw_vids:
# Get metadata file
vid = get_file_name(video)
metadata = [f for f in listdir(raw_metadata_fld) if vid in f][0]
tdmsconvert(video, metadata, fps=fps)
else:
for video in to_convert:
name = get_file_name(video)
metadata = [f for f in listdir(raw_metadata_fld) if name in f][0]
# Create sbatch scripts to run the conversion on HPC
template = open(
os.path.join(bash_scripts, "run_on_hpc_template.txt"),
"r").read()
newbash = template.replace("out.out", "output/{}.out".format(name))
newbash = newbash.replace("err.err", "output/{}.err".format(name))
newbash = newbash.replace(
"VIDEO",
os.path.join(hpc_raw_video_fld,
os.path.split(video)[-1]).replace("\\", "/"))
newbash = newbash.replace(
"METADATA",
os.path.join(hpc_raw_metadata_fld,
os.path.split(metadata)[-1]).replace("\\", "/"))
newbash = newbash.replace("FPS", str(fps))
# Write output
script_name = os.path.join(bash_scripts, "individuals",
name[:-5] + ".sh")
f = open(script_name, "w")
f.write(newbash)
f.close()
print("Created bash script at: " + script_name)
print("""
With VPN active, use these commands to connect to HPC and submit converting jobs.
Make sure first that batchrun_tdmsconv_on_hpc doesn't include old bash scripts
ssh [email protected]
cd ../../../ # get to where winstor is
cd winstor/branco/Federico/Locomotion/bash_scripts/
bash batchrun_tdmsconv_on_hpc.sh # submit jobs
squeue -u federicoc -i 30 # to watch progress
""")
def sort_mantis_files():
exp_dirs = get_subdirs(raw_tosort_fld)
if not exp_dirs:
return
# Start logging
logging.disable(logging.NOTSET)
fancylog.start_logging(raw_tosort_fld,
package,
verbose=True,
filename='mantis_sorter')
logging.info("Starting to process mantis files")
# Loop over subdirs
for subdir in exp_dirs:
# --------------------------------- GET FILES -------------------------------- #
logging.info(" processing: {}".format(subdir))
files = [f for f in listdir(subdir) if f.endswith('.tdms')]
if not files: continue
if len(files) > 3:
raise NotImplementedError("Can't deal with this many files!")
elif len(files) < 3:
raise ValueError("Found too few files")
for f in files:
for i in range(10):
if "({})".format(i + 1) in f:
raise NotImplementedError(
"Cannot deal with how files are organised in the folder, sorry. "
)
# Infer what the experiment name is
metadata_file = [f for f in files if 'meta.tdms' in f]
if not metadata_file:
logging.warning("Failed to find metadata file")
raise FileNotFoundError("Could not find metadata file")
else:
metadata_file = metadata_file[0]
# Get AI file
exp_name = os.path.split(metadata_file)[-1].split("(0)")[0]
inputs_file = [f for f in files if f.endswith(exp_name + '(0).tdms')]
if not inputs_file:
logging.warning("Failed to find analog inputs file")
raise FileNotFoundError("Could not find analog inputs file")
else:
inputs_file = inputs_file[0]
# Get video file
video_file = [
f for f in files if f != inputs_file and f != metadata_file
][0]
# ---------------------- CHECK IF MANTIS DROPPED FRAMES ---------------------- #
camera_name = os.path.split(video_file)[-1].split("(0)-")[-1].split(
".")[0]
check = check_mantis_dropped_frames(subdir,
camera_name,
exp_name,
skip_analog_inputs=True)
if check:
logging.info(" Mantis didn't drop any frames for video file")
else:
logging.info(" Mantis dropped some frames, darn it.")
# -------------------------------- MOVE FILES -------------------------------- #
# Get destination files
subshort = os.path.split(subdir)[-1]
vdest = os.path.join(raw_video_fld, subshort + '_video.tdms')
mdest = os.path.join(raw_metadata_fld,
subshort + '_video_metadata.tdms')
adest = os.path.join(raw_analog_inputs_fld, subshort + '_AI.tdms')
logging.info(" Video file: {} -> {}".format(video_file, vdest))
logging.info(" Metadata file: {} -> {}".format(
metadata_file, mdest))
logging.info(" Analog inputs file: {} -> {}".format(
inputs_file, adest))
# Move files
for src, dest in tqdm(
zip([video_file, metadata_file, inputs_file],
[vdest, mdest, adest])):
if os.path.isfile(dest):
logging.warning(
" The destination file {} already exists, stopping to avoid overwriting"
.format(dest))
raise FileExistsError(
" The destination file {} already exists, stopping to avoid overwriting"
.format(dest))
os.rename(src, dest)
# disable logging
logging.disable(sys.maxsize)
def get_files_for_session(self, session_id=None, mouse_id=None, date=None):
# Get session given args and check all went okay
if session_id is not None:
session = (self & f"session_id={session_id}").fetch(as_dict=True)
elif mouse_id is not None and date is not None:
session = (self & f"mouse_id='{mouse_id}'"
& f"date='{date}'").fetch(as_dict=True)
else:
raise ValueError(
"Need to pass either session id or mouse id + date")
if not session:
print("Couldn't find any session with args: {} - {} - {}".format(
session_id, mouse_id, date))
return None
elif len(session) > 1:
raise ValueError("Found too many sessions")
else:
session = session[0]
# Create name
sessname = self._get_formatted_date(
session['date']) + f"_{session['mouse_id']}"
# Get files
video_tdms = [
f for f in listdir(raw_video_fld)
if sessname in f and f.endswith(".tdms")
][0]
metadata_tmds = [
f for f in listdir(raw_metadata_fld)
if sessname in f and f.endswith(".tdms")
][0]
inputs_tmds = [
f for f in listdir(raw_analog_inputs_fld)
if sessname in f and f.endswith(".tdms")
][0]
try:
converted_vid = [
f for f in listdir(raw_video_fld)
if sessname in f and f.endswith(".mp4")
][0]
except:
converted_vid = None
try:
tracking = [
f for f in listdir(raw_tracking_fld)
if sessname in f and f.endswith(".h5")
][0]
except:
tracking = None
files = dict(
raw_video=video_tdms,
raw_metadata=metadata_tmds,
raw_inputs=inputs_tmds,
converted_video=converted_vid,
trackingdata_file=tracking,
)
return files