def get_not_tracked_files(): videos = [f for f in listdir(raw_video_fld) if f.endswith(".mp4")] tracked = [ get_file_name(f) for f in listdir(raw_tracking_fld) if f.endswith(".h5") ] not_tracked = [] for video in videos: is_tracked = [f for f in tracked if get_file_name(video) in f] if not is_tracked: not_tracked.append(video) print(f"Found {len(not_tracked)} videos to track.") for vid in not_tracked: print(f" {vid}") return not_tracked
def _make_tuples(self, key): # Check if we have a tracking data file tracking_fname = Session._get_formatted_date( key['date']) + f'_{key["mouse_id"]}_video' tracking_file = [ f for f in listdir(raw_tracking_fld) if tracking_fname in f ] if not tracking_file: print( f"No tracking file for session {Session._get_formatted_date(key['date'])} - {key['mouse_id']}" ) return else: tracking_file = tracking_file[0] # Insert entry into main class self.insert1(key) # Load and clean tracking data bp_tracking = prepare_tracking_data(tracking_file, median_filter=True, fisheye=False, common_coord=False, compute=True) bones_tracking = compute_body_segments(bp_tracking, self.bsegments) # Insert into the bodyparts tracking for bp, tracking in bp_tracking.items(): bp_key = key.copy() bp_key['bp'] = bp bp_key['x'] = tracking.x.values bp_key['y'] = tracking.y.values bp_key['speed'] = tracking.speed.values bp_key['dir_of_mvmt'] = tracking.direction_of_movement.values bp_key['angular_velocity'] = tracking.angular_velocity.values self.BodyPartTracking.insert1(bp_key) # Insert into the body segment data for bone, (bp1, bp2) in self.bsegments.items(): segment_key = key.copy() segment_key['bp1'] = bp1 segment_key['bp2'] = bp2 segment_key['orientation'] = bones_tracking[bone][ 'orientation'].values segment_key['angular_velocity'] = bones_tracking[bone][ 'angular_velocity'].values self.BodySegmentTracking.insert1(segment_key)
def convert_intputs_ais_to_pandas(): files = [f for f in listdir(raw_analog_inputs_fld)] tdms = [f for f in files if f.endswith(".tdms")] h5 = [get_file_name(f) for f in files if f.endswith(".h5")] to_convert = [f for f in tdms if get_file_name(f) not in h5] print( f'Found {len(to_convert)} tdms files to convert. e.g.: {to_convert[0]}\n' ) pool = mp.Pool(mp.cpu_count() - 2) pool.map(get_analog_inputs_clean_dataframe, to_convert) pool.close()
def __init__(self, calibration_images_folder, camera_name, checkerboard_shape, dark_threshold, images_extention='.png', save_folder=None, save_inverted_map=True): """ Initialise and run fish eye calibration :param calibration_images_folder: path to folder with calibration images :param images_extention: file type of images :param camera_name: str, used to save the maps :param checkerboard_shape: tuple with #size of the checkerboard (# of vertices in each dimension, not including those on the edge) :param dark_threshold: int, The algorithm is finnicky and likes saturation, so set pixels with values below the dark_threshold to black :param save_folder: str, path to a directory where to save the output :save_inverted_map: bool, if true an inverted map used for tracking data correction is saved """ # ----------------------------------- setup ---------------------------------- # check_create_folder(calibration_images_folder, raise_error=True) self.calibration_images = [ f for f in listdir(calibration_images_folder) if images_extention in f ] print('Found ' + str(len(self.calibration_images)) + ' images for fisheye calibration.') if save_folder is None: self.save_folder = calibration_images_folder else: check_create_folder(save_folder) self.save_folder = save_folder self.camera_name = camera_name self.checkerboard_shape = checkerboard_shape self.dark_threshold = dark_threshold self.save_inverted_map = save_inverted_map
mouse=mouse, injected=injected) bp_tracking = fetch_tracking(for_bp=True, **filt) bones_tracking = fetch_tracking(for_bp=False, **filt) tracking = fetch_tracking_processed(**filt) tracking = get_frames_state(tracking) tracking = get_when_in_center(tracking, center, radius) n_frames = len(tracking.x) if use_real_video: data = Session._get_formatted_date(tracking.date.iloc[0]) video = [ f for f in listdir(raw_video_fld) if mouse in f and data in f and f.endswith(".mp4") ][0] videocap = get_cap_from_file(video) # %% # -------------------------------- Write clip -------------------------------- # savepath = os.path.join(output_fld, f'{mouse}_{injected}_real_{use_cluster}.mp4') writer = open_cvwriter(savepath, w=frame_shape[0], h=frame_shape[1], framerate=fps, format=".mp4", iscolor=True)
def get_not_converted_videos(convert_videos, fps=None): raw_vids = [f for f in listdir(raw_video_fld) if f.endswith(".tdms")] converted = [ f.split(".")[0] for f in listdir(raw_video_fld) if f.endswith(".mp4") ] to_convert = [] for raw in raw_vids: if raw.split(".")[0] not in converted: to_convert.append(raw) print("\n\nFound {} videos to convert.".format(len(to_convert))) for vid in to_convert: print(" ", vid) if convert_videos: for video in raw_vids: # Get metadata file vid = get_file_name(video) metadata = [f for f in listdir(raw_metadata_fld) if vid in f][0] tdmsconvert(video, metadata, fps=fps) else: for video in to_convert: name = get_file_name(video) metadata = [f for f in listdir(raw_metadata_fld) if name in f][0] # Create sbatch scripts to run the conversion on HPC template = open( os.path.join(bash_scripts, "run_on_hpc_template.txt"), "r").read() newbash = template.replace("out.out", "output/{}.out".format(name)) newbash = newbash.replace("err.err", "output/{}.err".format(name)) newbash = newbash.replace( "VIDEO", os.path.join(hpc_raw_video_fld, os.path.split(video)[-1]).replace("\\", "/")) newbash = newbash.replace( "METADATA", os.path.join(hpc_raw_metadata_fld, os.path.split(metadata)[-1]).replace("\\", "/")) newbash = newbash.replace("FPS", str(fps)) # Write output script_name = os.path.join(bash_scripts, "individuals", name[:-5] + ".sh") f = open(script_name, "w") f.write(newbash) f.close() print("Created bash script at: " + script_name) print(""" With VPN active, use these commands to connect to HPC and submit converting jobs. Make sure first that batchrun_tdmsconv_on_hpc doesn't include old bash scripts ssh [email protected] cd ../../../ # get to where winstor is cd winstor/branco/Federico/Locomotion/bash_scripts/ bash batchrun_tdmsconv_on_hpc.sh # submit jobs squeue -u federicoc -i 30 # to watch progress """)
def sort_mantis_files(): exp_dirs = get_subdirs(raw_tosort_fld) if not exp_dirs: return # Start logging logging.disable(logging.NOTSET) fancylog.start_logging(raw_tosort_fld, package, verbose=True, filename='mantis_sorter') logging.info("Starting to process mantis files") # Loop over subdirs for subdir in exp_dirs: # --------------------------------- GET FILES -------------------------------- # logging.info(" processing: {}".format(subdir)) files = [f for f in listdir(subdir) if f.endswith('.tdms')] if not files: continue if len(files) > 3: raise NotImplementedError("Can't deal with this many files!") elif len(files) < 3: raise ValueError("Found too few files") for f in files: for i in range(10): if "({})".format(i + 1) in f: raise NotImplementedError( "Cannot deal with how files are organised in the folder, sorry. " ) # Infer what the experiment name is metadata_file = [f for f in files if 'meta.tdms' in f] if not metadata_file: logging.warning("Failed to find metadata file") raise FileNotFoundError("Could not find metadata file") else: metadata_file = metadata_file[0] # Get AI file exp_name = os.path.split(metadata_file)[-1].split("(0)")[0] inputs_file = [f for f in files if f.endswith(exp_name + '(0).tdms')] if not inputs_file: logging.warning("Failed to find analog inputs file") raise FileNotFoundError("Could not find analog inputs file") else: inputs_file = inputs_file[0] # Get video file video_file = [ f for f in files if f != inputs_file and f != metadata_file ][0] # ---------------------- CHECK IF MANTIS DROPPED FRAMES ---------------------- # camera_name = os.path.split(video_file)[-1].split("(0)-")[-1].split( ".")[0] check = check_mantis_dropped_frames(subdir, camera_name, exp_name, skip_analog_inputs=True) if check: logging.info(" Mantis didn't drop any frames for video file") else: logging.info(" Mantis dropped some frames, darn it.") # -------------------------------- MOVE FILES -------------------------------- # # Get destination files subshort = os.path.split(subdir)[-1] vdest = os.path.join(raw_video_fld, subshort + '_video.tdms') mdest = os.path.join(raw_metadata_fld, subshort + '_video_metadata.tdms') adest = os.path.join(raw_analog_inputs_fld, subshort + '_AI.tdms') logging.info(" Video file: {} -> {}".format(video_file, vdest)) logging.info(" Metadata file: {} -> {}".format( metadata_file, mdest)) logging.info(" Analog inputs file: {} -> {}".format( inputs_file, adest)) # Move files for src, dest in tqdm( zip([video_file, metadata_file, inputs_file], [vdest, mdest, adest])): if os.path.isfile(dest): logging.warning( " The destination file {} already exists, stopping to avoid overwriting" .format(dest)) raise FileExistsError( " The destination file {} already exists, stopping to avoid overwriting" .format(dest)) os.rename(src, dest) # disable logging logging.disable(sys.maxsize)
def get_files_for_session(self, session_id=None, mouse_id=None, date=None): # Get session given args and check all went okay if session_id is not None: session = (self & f"session_id={session_id}").fetch(as_dict=True) elif mouse_id is not None and date is not None: session = (self & f"mouse_id='{mouse_id}'" & f"date='{date}'").fetch(as_dict=True) else: raise ValueError( "Need to pass either session id or mouse id + date") if not session: print("Couldn't find any session with args: {} - {} - {}".format( session_id, mouse_id, date)) return None elif len(session) > 1: raise ValueError("Found too many sessions") else: session = session[0] # Create name sessname = self._get_formatted_date( session['date']) + f"_{session['mouse_id']}" # Get files video_tdms = [ f for f in listdir(raw_video_fld) if sessname in f and f.endswith(".tdms") ][0] metadata_tmds = [ f for f in listdir(raw_metadata_fld) if sessname in f and f.endswith(".tdms") ][0] inputs_tmds = [ f for f in listdir(raw_analog_inputs_fld) if sessname in f and f.endswith(".tdms") ][0] try: converted_vid = [ f for f in listdir(raw_video_fld) if sessname in f and f.endswith(".mp4") ][0] except: converted_vid = None try: tracking = [ f for f in listdir(raw_tracking_fld) if sessname in f and f.endswith(".h5") ][0] except: tracking = None files = dict( raw_video=video_tdms, raw_metadata=metadata_tmds, raw_inputs=inputs_tmds, converted_video=converted_vid, trackingdata_file=tracking, ) return files