def plot_trajectories( config, videos, videotype=".avi", shuffle=1, trainingsetindex=0, filtered=False, displayedbodyparts="all", displayedindividuals="all", showfigures=False, destfolder=None, modelprefix="", imagetype=".png", resolution=100, linewidth=1.0, track_method="", ): """ Plots the trajectories of various bodyparts across the video. Parameters ---------- config : string Full path of the config.yaml file as a string. videos : list A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored. videotype: string, optional Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi`` shuffle: list, optional List of integers specifying the shuffle indices of the training dataset. The default is [1] trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). filtered: bool, default false Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions displayedbodyparts: list of strings, optional This select the body parts that are plotted in the video. Either ``all``, then all body parts from config.yaml are used, or a list of strings that are a subset of the full list. E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts. showfigures: bool, default false If true then plots are also displayed. destfolder: string, optional Specifies the destination folder that was used for storing analysis data (default is the path of the video). imagetype: string, default ".png" Specifies the output image format, tested '.tif', '.jpg', '.svg' and ".png". resolution: int, default 100 Specifies the resolution (in dpi) of saved figures. Note higher resolution figures take longer to generate. linewidth: float, default 1.0 Specifies width of line for line and histogram plots. track_method: string, optional Specifies the tracker used to generate the data. Empty by default (corresponding to a single animal project). For multiple animals, must be either 'box', 'skeleton', or 'ellipse' and will be taken from the config.yaml file if none is given. Example -------- for labeling the frames >>> deeplabcut.plot_trajectories('home/alex/analysis/project/reaching-task/config.yaml',['/home/alex/analysis/project/videos/reachingvideo1.avi']) -------- """ cfg = auxiliaryfunctions.read_config(config) track_method = auxfun_multianimal.get_track_method( cfg, track_method=track_method) trainFraction = cfg["TrainingFraction"][trainingsetindex] DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction, modelprefix=modelprefix ) # automatically loads corresponding model (even training iteration based on snapshot index) bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, displayedbodyparts) individuals = auxfun_multianimal.IntersectionofIndividualsandOnesGivenbyUser( cfg, displayedindividuals) Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype) if not len(Videos): print( "No videos found. Make sure you passed a list of videos and that *videotype* is right." ) return failed = [] for video in Videos: if destfolder is None: videofolder = str(Path(video).parents[0]) else: videofolder = destfolder vname = str(Path(video).stem) print("Loading ", video, "and data.") try: df, _, _, suffix = auxiliaryfunctions.load_analyzed_data( videofolder, vname, DLCscorer, filtered, track_method) failed.append(False) tmpfolder = os.path.join(videofolder, "plot-poses", vname) auxiliaryfunctions.attempttomakefolder(tmpfolder, recursive=True) # Keep only the individuals and bodyparts that were labeled labeled_bpts = [ bp for bp in df.columns.get_level_values("bodyparts").unique() if bp in bodyparts ] # Either display the animals defined in the config if they are found # in the dataframe, or all the trajectories regardless of their names try: animals = set(df.columns.get_level_values("individuals")) except KeyError: animals = {""} for animal in animals.intersection(individuals) or animals: PlottingResults( tmpfolder, df, cfg, labeled_bpts, animal, showfigures, suffix + animal + imagetype, resolution=resolution, linewidth=linewidth, ) except FileNotFoundError as e: failed.append(True) print(e) try: _ = auxiliaryfunctions.load_detection_data( video, DLCscorer, track_method) print('Call "deeplabcut.stitch_tracklets()"' " prior to plotting the trajectories.") except FileNotFoundError as e: print(e) print( f"Make sure {video} was previously analyzed, and that " f'detections were successively converted to tracklets using "deeplabcut.convert_detections2tracklets()" ' f'and "deeplabcut.stitch_tracklets()".') if not all(failed): print( 'Plots created! Please check the directory "plot-poses" within the video directory' ) else: print( f"Plots could not be created! " f"Videos were not evaluated with the current scorer {DLCscorer}.")
def extract_outlier_frames( config, videos, videotype=".avi", shuffle=1, trainingsetindex=0, outlieralgorithm="jump", comparisonbodyparts="all", epsilon=20, p_bound=0.01, ARdegree=3, MAdegree=1, alpha=0.01, extractionalgorithm="kmeans", automatic=False, cluster_resizewidth=30, cluster_color=False, opencv=True, savelabeled=False, destfolder=None, modelprefix="", track_method="", ): """ Extracts the outlier frames in case, the predictions are not correct for a certain video from the cropped video running from start to stop as defined in config.yaml. Another crucial parameter in config.yaml is how many frames to extract 'numframes2extract'. Parameter ---------- config : string Full path of the config.yaml file as a string. videos : list A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored. videotype: string, optional Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi`` shuffle : int, optional The shuffle index of training dataset. The extracted frames will be stored in the labeled-dataset for the corresponding shuffle of training dataset. Default is set to 1 trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). outlieralgorithm: 'fitting', 'jump', 'uncertain', or 'manual' String specifying the algorithm used to detect the outliers. Currently, deeplabcut supports three methods + a manual GUI option. 'Fitting' fits a Auto Regressive Integrated Moving Average model to the data and computes the distance to the estimated data. Larger distances than epsilon are then potentially identified as outliers. The methods 'jump' identifies larger jumps than 'epsilon' in any body part; and 'uncertain' looks for frames with confidence below p_bound. The default is set to ``jump``. comparisonbodyparts: list of strings, optional This selects the body parts for which the comparisons with the outliers are carried out. Either ``all``, then all body parts from config.yaml are used orr a list of strings that are a subset of the full list. E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts. p_bound: float between 0 and 1, optional For outlieralgorithm 'uncertain' this parameter defines the likelihood below, below which a body part will be flagged as a putative outlier. epsilon; float,optional Meaning depends on outlieralgoritm. The default is set to 20 pixels. For outlieralgorithm 'fitting': Float bound according to which frames are picked when the (average) body part estimate deviates from model fit For outlieralgorithm 'jump': Float bound specifying the distance by which body points jump from one frame to next (Euclidean distance) ARdegree: int, optional For outlieralgorithm 'fitting': Autoregressive degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part) see https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html MAdegree: int For outlieralgorithm 'fitting': MovingAvarage degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part) See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html alpha: float Significance level for detecting outliers based on confidence interval of fitted ARIMA model. Only the distance is used however. extractionalgorithm : string, optional String specifying the algorithm to use for selecting the frames from the identified putatative outlier frames. Currently, deeplabcut supports either ``kmeans`` or ``uniform`` based selection (same logic as for extract_frames). The default is set to``uniform``, if provided it must be either ``uniform`` or ``kmeans``. automatic : bool, optional Set it to True, if you want to extract outliers without being asked for user feedback. cluster_resizewidth: number, default: 30 For k-means one can change the width to which the images are downsampled (aspect ratio is fixed). cluster_color: bool, default: False If false then each downsampled image is treated as a grayscale vector (discarding color information). If true, then the color channels are considered. This increases the computational complexity. opencv: bool, default: True Uses openCV for loading & extractiong (otherwise moviepy (legacy)) savelabeled: bool, default: False If true also saves frame with predicted labels in each folder. destfolder: string, optional Specifies the destination folder that was used for storing analysis data (default is the path of the video). track_method: string, optional Specifies the tracker used to generate the data. Empty by default (corresponding to a single animal project). For multiple animals, must be either 'box', 'skeleton', or 'ellipse' and will be taken from the config.yaml file if none is given. Examples Windows example for extracting the frames with default settings >>> deeplabcut.extract_outlier_frames('C:\\myproject\\reaching-task\\config.yaml',['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi']) -------- for extracting the frames with default settings >>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi']) -------- for extracting the frames with kmeans >>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'],extractionalgorithm='kmeans') -------- for extracting the frames with kmeans and epsilon = 5 pixels. >>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'],epsilon = 5,extractionalgorithm='kmeans') -------- """ cfg = auxiliaryfunctions.read_config(config) bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, comparisonbodyparts) if not len(bodyparts): raise ValueError("No valid bodyparts were selected.") track_method = auxfun_multianimal.get_track_method( cfg, track_method=track_method) DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction=cfg["TrainingFraction"][trainingsetindex], modelprefix=modelprefix, ) Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype) if len(Videos) == 0: print("No suitable videos found in", videos) for video in Videos: if destfolder is None: videofolder = str(Path(video).parents[0]) else: videofolder = destfolder vname = os.path.splitext(os.path.basename(video))[0] try: df, dataname, _, _ = auxiliaryfunctions.load_analyzed_data( videofolder, vname, DLCscorer, track_method=track_method) nframes = len(df) startindex = max([int(np.floor(nframes * cfg["start"])), 0]) stopindex = min([int(np.ceil(nframes * cfg["stop"])), nframes]) Index = np.arange(stopindex - startindex) + startindex df = df.iloc[Index] mask = df.columns.get_level_values("bodyparts").isin(bodyparts) df_temp = df.loc[:, mask] Indices = [] if outlieralgorithm == "uncertain": p = df_temp.xs("likelihood", level="coords", axis=1) ind = df_temp.index[(p < p_bound).any(axis=1)].tolist() Indices.extend(ind) elif outlieralgorithm == "jump": temp_dt = df_temp.diff(axis=0)**2 temp_dt.drop("likelihood", axis=1, level="coords", inplace=True) sum_ = temp_dt.sum(axis=1, level=1) ind = df_temp.index[(sum_ > epsilon**2).any(axis=1)].tolist() Indices.extend(ind) elif outlieralgorithm == "fitting": d, o = compute_deviations(df_temp, dataname, p_bound, alpha, ARdegree, MAdegree) # Some heuristics for extracting frames based on distance: ind = np.flatnonzero( d > epsilon ) # time points with at least average difference of epsilon if ( len(ind) < cfg["numframes2pick"] * 2 and len(d) > cfg["numframes2pick"] * 2 ): # if too few points qualify, extract the most distant ones. ind = np.argsort(d)[::-1][:cfg["numframes2pick"] * 2] Indices.extend(ind) elif outlieralgorithm == "manual": wd = Path(config).resolve().parents[0] os.chdir(str(wd)) from deeplabcut.gui import outlier_frame_extraction_toolbox outlier_frame_extraction_toolbox.show( config, video, shuffle, df, savelabeled, cfg.get("multianimalproject", False), ) # Run always except when the outlieralgorithm == manual. if not outlieralgorithm == "manual": Indices = np.sort(list(set(Indices))) # remove repetitions. print( "Method ", outlieralgorithm, " found ", len(Indices), " putative outlier frames.", ) print( "Do you want to proceed with extracting ", cfg["numframes2pick"], " of those?", ) if outlieralgorithm == "uncertain" or outlieralgorithm == "jump": print( "If this list is very large, perhaps consider changing the parameters " "(start, stop, p_bound, comparisonbodyparts) or use a different method." ) elif outlieralgorithm == "fitting": print( "If this list is very large, perhaps consider changing the parameters " "(start, stop, epsilon, ARdegree, MAdegree, alpha, comparisonbodyparts) " "or use a different method.") if not automatic: askuser = input("yes/no") else: askuser = "******" if (askuser == "y" or askuser == "yes" or askuser == "Ja" or askuser == "ha"): # multilanguage support :) # Now extract from those Indices! ExtractFramesbasedonPreselection( Indices, extractionalgorithm, df, video, cfg, config, opencv, cluster_resizewidth, cluster_color, savelabeled, ) else: print( "Nothing extracted, please change the parameters and start again..." ) except FileNotFoundError as e: print(e) print( "It seems the video has not been analyzed yet, or the video is not found! " "You can only refine the labels after the a video is analyzed. Please run 'analyze_video' first. " "Or, please double check your video file path")
def proc_video( videos, destfolder, filtered, DLCscorer, DLCscorerlegacy, track_method, cfg, individuals, color_by, bodyparts, codec, bodyparts2connect, trailpoints, save_frames, outputframerate, Frames2plot, draw_skeleton, skeleton_color, displaycropped, fastmode, keypoints_only, video, ): """Helper function for create_videos Parameters ---------- """ videofolder = Path(video).parents[0] if destfolder is None: destfolder = videofolder # where your folder with videos is. auxiliaryfunctions.attempttomakefolder(destfolder) os.chdir(destfolder) # THE VIDEO IS STILL IN THE VIDEO FOLDER print("Starting to process video: {}".format(video)) vname = str(Path(video).stem) if filtered: videooutname1 = os.path.join(vname + DLCscorer + "filtered_labeled.mp4") videooutname2 = os.path.join(vname + DLCscorerlegacy + "filtered_labeled.mp4") else: videooutname1 = os.path.join(vname + DLCscorer + "_labeled.mp4") videooutname2 = os.path.join(vname + DLCscorerlegacy + "_labeled.mp4") if os.path.isfile(videooutname1) or os.path.isfile(videooutname2): print("Labeled video {} already created.".format(vname)) else: print("Loading {} and data.".format(video)) try: df, filepath, _, _ = auxiliaryfunctions.load_analyzed_data( destfolder, vname, DLCscorer, filtered, track_method) metadata = auxiliaryfunctions.load_video_metadata( destfolder, vname, DLCscorer) if cfg.get("multianimalproject", False): s = "_id" if color_by == "individual" else "_bp" else: s = "" videooutname = filepath.replace(".h5", f"{s}_labeled.mp4") if os.path.isfile(videooutname): print("Labeled video already created. Skipping...") return if all(individuals): df = df.loc(axis=1)[:, individuals] cropping = metadata["data"]["cropping"] [x1, x2, y1, y2] = metadata["data"]["cropping_parameters"] labeled_bpts = [ bp for bp in df.columns.get_level_values("bodyparts").unique() if bp in bodyparts ] if keypoints_only: # Mask rather than drop unwanted bodyparts to ensure consistent coloring mask = df.columns.get_level_values("bodyparts").isin(bodyparts) df.loc[:, ~mask] = np.nan inds = None if bodyparts2connect: all_bpts = df.columns.get_level_values("bodyparts")[::3] inds = get_segment_indices(bodyparts2connect, all_bpts) create_video_with_keypoints_only( df, videooutname, inds, cfg["pcutoff"], cfg["dotsize"], cfg["alphavalue"], skeleton_color=skeleton_color, color_by=color_by, colormap=cfg["colormap"], ) elif not fastmode: tmpfolder = os.path.join(str(videofolder), "temp-" + vname) if save_frames: auxiliaryfunctions.attempttomakefolder(tmpfolder) clip = vp(video) CreateVideoSlow( videooutname, clip, df, tmpfolder, cfg["dotsize"], cfg["colormap"], cfg["alphavalue"], cfg["pcutoff"], trailpoints, cropping, x1, x2, y1, y2, save_frames, labeled_bpts, outputframerate, Frames2plot, bodyparts2connect, skeleton_color, draw_skeleton, displaycropped, color_by, ) else: if displaycropped: # then the cropped video + the labels is depicted clip = vp( fname=video, sname=videooutname, codec=codec, sw=x2 - x1, sh=y2 - y1, fps=outputframerate, ) else: # then the full video + the (perhaps in cropped mode analyzed labels) are depicted clip = vp(fname=video, sname=videooutname, codec=codec, fps=outputframerate) CreateVideo( clip, df, cfg["pcutoff"], cfg["dotsize"], cfg["colormap"], labeled_bpts, trailpoints, cropping, x1, x2, y1, y2, bodyparts2connect, skeleton_color, draw_skeleton, displaycropped, color_by, ) except FileNotFoundError as e: print(e)
def analyzeskeleton( config, videos, videotype="", shuffle=1, trainingsetindex=0, filtered=False, save_as_csv=False, destfolder=None, modelprefix="", track_method="", ): """Extracts length and orientation of each "bone" of the skeleton. The bone and skeleton information is defined in the config file. Parameters ---------- config: str Full path of the config.yaml file. videos: list[str] The full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored. videotype: str, optional, default="" Checks for the extension of the video in case the input to the video is a directory. Only videos with this extension are analyzed. If left unspecified, videos with common extensions ('avi', 'mp4', 'mov', 'mpeg', 'mkv') are kept. shuffle : int, optional, default=1 The shuffle index of training dataset. The extracted frames will be stored in the labeled-dataset for the corresponding shuffle of training dataset. trainingsetindex: int, optional, default=0 Integer specifying which TrainingsetFraction to use. Note that TrainingFraction is a list in config.yaml. filtered: bool, optional, default=False Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with ``deeplabcut.filterpredictions``. save_as_csv: bool, optional, default=False Saves the predictions in a .csv file. destfolder: string or None, optional, default=None Specifies the destination folder for analysis data. If ``None``, the path of the video is used. Note that for subsequent analysis this folder also needs to be passed. modelprefix: str, optional, default="" Directory containing the deeplabcut models to use when evaluating the network. By default, the models are assumed to exist in the project folder. track_method: string, optional, default="" Specifies the tracker used to generate the data. Empty by default (corresponding to a single animal project). For multiple animals, must be either 'box', 'skeleton', or 'ellipse' and will be taken from the config.yaml file if none is given. Returns ------- None """ # Load config file, scorer and videos cfg = auxiliaryfunctions.read_config(config) if not cfg["skeleton"]: raise ValueError("No skeleton defined in the config.yaml.") track_method = auxfun_multianimal.get_track_method( cfg, track_method=track_method) DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction=cfg["TrainingFraction"][trainingsetindex], modelprefix=modelprefix, ) Videos = auxiliaryfunctions.get_list_of_videos(videos, videotype) for video in Videos: print("Processing %s" % (video)) if destfolder is None: destfolder = str(Path(video).parents[0]) vname = Path(video).stem try: df, filepath, scorer, _ = auxiliaryfunctions.load_analyzed_data( destfolder, vname, DLCscorer, filtered, track_method) output_name = filepath.replace(".h5", f"_skeleton.h5") if os.path.isfile(output_name): print( f"Skeleton in video {vname} already processed. Skipping..." ) continue bones = {} if "individuals" in df.columns.names: for animal_name, df_ in df.groupby(level="individuals", axis=1): temp = df_.droplevel(["scorer", "individuals"], axis=1) if animal_name != "single": for bp1, bp2 in cfg["skeleton"]: name = "{}_{}_{}".format(animal_name, bp1, bp2) bones[name] = analyzebone(temp[bp1], temp[bp2]) else: for bp1, bp2 in cfg["skeleton"]: name = "{}_{}".format(bp1, bp2) bones[name] = analyzebone(df[scorer][bp1], df[scorer][bp2]) skeleton = pd.concat(bones, axis=1) skeleton.to_hdf(output_name, "df_with_missing", format="table", mode="w") if save_as_csv: skeleton.to_csv(output_name.replace(".h5", ".csv")) except FileNotFoundError as e: print(e) continue
def analyzeskeleton( config, videos, videotype="avi", shuffle=1, trainingsetindex=0, filtered=False, save_as_csv=False, destfolder=None, modelprefix="", track_method="", ): """ Extracts length and orientation of each "bone" of the skeleton as defined in the config file. Parameter ---------- config : string Full path of the config.yaml file as a string. videos : list A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored. shuffle : int, optional The shufle index of training dataset. The extracted frames will be stored in the labeled-dataset for the corresponding shuffle of training dataset. Default is set to 1 trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). filtered: bool, default false Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions save_as_csv: bool, optional Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False`` destfolder: string, optional Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this folder also needs to be passed. track_method: string, optional Specifies the tracker used to generate the data. Empty by default (corresponding to a single animal project). For multiple animals, must be either 'box', 'skeleton', or 'ellipse'. """ # Load config file, scorer and videos cfg = auxiliaryfunctions.read_config(config) if not cfg["skeleton"]: raise ValueError("No skeleton defined in the config.yaml.") DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction=cfg["TrainingFraction"][trainingsetindex], modelprefix=modelprefix, ) Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype) for video in Videos: print("Processing %s" % (video)) if destfolder is None: destfolder = str(Path(video).parents[0]) vname = Path(video).stem try: df, filepath, scorer, _ = auxiliaryfunctions.load_analyzed_data( destfolder, vname, DLCscorer, filtered, track_method) output_name = filepath.replace(".h5", f"_skeleton.h5") if os.path.isfile(output_name): print( f"Skeleton in video {vname} already processed. Skipping..." ) continue bones = {} if "individuals" in df.columns.names: for animal_name, df_ in df.groupby(level="individuals", axis=1): temp = df_.droplevel(["scorer", "individuals"], axis=1) if animal_name != "single": for bp1, bp2 in cfg["skeleton"]: name = "{}_{}_{}".format(animal_name, bp1, bp2) bones[name] = analyzebone(temp[bp1], temp[bp2]) else: for bp1, bp2 in cfg["skeleton"]: name = "{}_{}".format(bp1, bp2) bones[name] = analyzebone(df[scorer][bp1], df[scorer][bp2]) skeleton = pd.concat(bones, axis=1) skeleton.to_hdf(output_name, "df_with_missing", format="table", mode="w") if save_as_csv: skeleton.to_csv(output_name.replace(".h5", ".csv")) except FileNotFoundError as e: print(e) continue
def filterpredictions( config, video, videotype="avi", shuffle=1, trainingsetindex=0, filtertype="median", windowlength=5, p_bound=0.001, ARdegree=3, MAdegree=1, alpha=0.01, save_as_csv=True, destfolder=None, modelprefix="", track_method="", ): """ Fits frame-by-frame pose predictions with ARIMA model (filtertype='arima') or median filter (default). Parameter ---------- config : string Full path of the config.yaml file as a string. video : string Full path of the video to extract the frame from. Make sure that this video is already analyzed. shuffle : int, optional The shufle index of training dataset. The extracted frames will be stored in the labeled-dataset for the corresponding shuffle of training dataset. Default is set to 1 trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). filtertype: string Select which filter, 'arima', 'median' or 'spline'. windowlength: int For filtertype='median' filters the input array using a local window-size given by windowlength. The array will automatically be zero-padded. https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.medfilt.html The windowlenght should be an odd number. If filtertype='spline', windowlength is the maximal gap size to fill. p_bound: float between 0 and 1, optional For filtertype 'arima' this parameter defines the likelihood below, below which a body part will be consided as missing data for filtering purposes. ARdegree: int, optional For filtertype 'arima' Autoregressive degree of Sarimax model degree. see https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html MAdegree: int For filtertype 'arima' Moving Avarage degree of Sarimax model degree. See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html alpha: float Significance level for detecting outliers based on confidence interval of fitted SARIMAX model. save_as_csv: bool, optional Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False`` destfolder: string, optional Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this folder also needs to be passed. Example -------- Arima model: deeplabcut.filterpredictions('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,filterype='arima',ARdegree=5,MAdegree=2) Use median filter over 10bins: deeplabcut.filterpredictions('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,windowlength=10) One can then use the filtered rather than the frame-by-frame predictions by calling: deeplabcut.plot_trajectories('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,filtered=True) deeplabcut.create_labeled_video('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,filtered=True) -------- Returns filtered pandas array with the same structure as normal output of network. """ cfg = auxiliaryfunctions.read_config(config) DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction=cfg["TrainingFraction"][trainingsetindex], modelprefix=modelprefix, ) Videos = auxiliaryfunctions.Getlistofvideos(video, videotype) if not len(Videos): print( "No video(s) were found. Please check your paths and/or 'video_type'." ) return for video in Videos: if destfolder is None: destfolder = str(Path(video).parents[0]) print("Filtering with %s model %s" % (filtertype, video)) vname = Path(video).stem try: _ = auxiliaryfunctions.load_analyzed_data(destfolder, vname, DLCscorer, True, track_method) print(f"Data from {vname} were already filtered. Skipping...") except FileNotFoundError: # Data haven't been filtered yet try: df, filepath, _, _ = auxiliaryfunctions.load_analyzed_data( destfolder, vname, DLCscorer, track_method=track_method) nrows = df.shape[0] if filtertype == "arima": temp = df.values.reshape((nrows, -1, 3)) placeholder = np.empty_like(temp) for i in range(temp.shape[1]): x, y, p = temp[:, i].T meanx, _ = FitSARIMAXModel(x, p, p_bound, alpha, ARdegree, MAdegree, False) meany, _ = FitSARIMAXModel(y, p, p_bound, alpha, ARdegree, MAdegree, False) meanx[0] = x[0] meany[0] = y[0] placeholder[:, i] = np.c_[meanx, meany, p] data = pd.DataFrame( placeholder.reshape((nrows, -1)), columns=df.columns, index=df.index, ) elif filtertype == "median": data = df.copy() mask = data.columns.get_level_values( "coords") != "likelihood" data.loc[:, mask] = df.loc[:, mask].apply(signal.medfilt, args=(windowlength, ), axis=0) elif filtertype == "spline": data = df.copy() mask_data = data.columns.get_level_values("coords").isin( ("x", "y")) xy = data.loc[:, mask_data].values prob = data.loc[:, ~mask_data].values missing = np.isnan(xy) xy_filled = columnwise_spline_interp(xy, windowlength) filled = ~np.isnan(xy_filled) xy[filled] = xy_filled[filled] inds = np.argwhere(missing & filled) if inds.size: # Retrieve original individual label indices inds[:, 1] //= 2 inds = np.unique(inds, axis=0) prob[inds[:, 0], inds[:, 1]] = 0.01 data.loc[:, ~mask_data] = prob data.loc[:, mask_data] = xy else: raise ValueError(f"Unknown filter type {filtertype}") outdataname = filepath.replace(".h5", "_filtered.h5") data.to_hdf(outdataname, "df_with_missing", format="table", mode="w") if save_as_csv: print("Saving filtered csv poses!") data.to_csv(outdataname.split(".h5")[0] + ".csv") except FileNotFoundError as e: print(e) continue
def create_labeled_video( config, videos, videotype="avi", shuffle=1, trainingsetindex=0, filtered=False, fastmode=True, save_frames=False, Frames2plot=None, displayedbodyparts="all", displayedindividuals="all", codec="mp4v", outputframerate=None, destfolder=None, draw_skeleton=False, trailpoints=0, displaycropped=False, color_by="bodypart", modelprefix="", track_method="", ): """ Labels the bodyparts in a video. Make sure the video is already analyzed by the function 'analyze_video' Parameters ---------- config : string Full path of the config.yaml file as a string. videos : list A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored. videotype: string, optional Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi`` shuffle : int, optional Number of shuffles of training dataset. Default is set to 1. trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). filtered: bool, default false Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions videotype: string, optional Checks for the extension of the video in case the input is a directory.\nOnly videos with this extension are analyzed. The default is ``.avi`` fastmode: bool If true uses openCV (much faster but less customization of video) vs matplotlib (if false). You can also "save_frames" individually or not in the matplotlib mode (if you set the "save_frames" variable accordingly). However, using matplotlib to create the frames it therefore allows much more flexible (one can set transparency of markers, crop, and easily customize). save_frames: bool If true creates each frame individual and then combines into a video. This variant is relatively slow as it stores all individual frames. Frames2plot: List of indices If not None & save_frames=True then the frames corresponding to the index will be plotted. For example, Frames2plot=[0,11] will plot the first and the 12th frame. displayedbodyparts: list of strings, optional This selects the body parts that are plotted in the video. Either ``all``, then all body parts from config.yaml are used orr a list of strings that are a subset of the full list. E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts. displayedindividuals: list of strings, optional Individuals plotted in the video. By default, all individuals present in the config will be showed. codec: codec for labeled video. Options see http://www.fourcc.org/codecs.php [depends on your ffmpeg installation.] outputframerate: positive number, output frame rate for labeled video (only available for the mode with saving frames.) By default: None, which results in the original video rate. destfolder: string, optional Specifies the destination folder that was used for storing analysis data (default is the path of the video). draw_skeleton: bool If ``True`` adds a line connecting the body parts making a skeleton on on each frame. The body parts to be connected and the color of these connecting lines are specified in the config file. By default: ``False`` trailpoints: int Number of revious frames whose body parts are plotted in a frame (for displaying history). Default is set to 0. displaycropped: bool, optional Specifies whether only cropped frame is displayed (with labels analyzed therein), or the original frame with the labels analyzed in the cropped subset. color_by : string, optional (default='bodypart') Coloring rule. By default, each bodypart is colored differently. If set to 'individual', points belonging to a single individual are colored the same. Examples -------- If you want to create the labeled video for only 1 video >>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi']) -------- If you want to create the labeled video for only 1 video and store the individual frames >>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'],fastmode=True, save_frames=True) -------- If you want to create the labeled video for multiple videos >>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi']) -------- If you want to create the labeled video for all the videos (as .avi extension) in a directory. >>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/']) -------- If you want to create the labeled video for all the videos (as .mp4 extension) in a directory. >>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'],videotype='mp4') -------- """ cfg = auxiliaryfunctions.read_config(config) trainFraction = cfg["TrainingFraction"][trainingsetindex] DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction, modelprefix=modelprefix ) # automatically loads corresponding model (even training iteration based on snapshot index) if save_frames: fastmode = False # otherwise one cannot save frames bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, displayedbodyparts) individuals = auxfun_multianimal.IntersectionofIndividualsandOnesGivenbyUser( cfg, displayedindividuals) if draw_skeleton: bodyparts2connect = cfg["skeleton"] skeleton_color = cfg["skeleton_color"] else: bodyparts2connect = None skeleton_color = None start_path = os.getcwd() Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype) if not len(Videos): print( "No video(s) were found. Please check your paths and/or 'video_type'." ) return for video in Videos: videofolder = Path(video).parents[0] if destfolder is None: destfolder = videofolder # where your folder with videos is. auxiliaryfunctions.attempttomakefolder(destfolder) os.chdir(destfolder) # THE VIDEO IS STILL IN THE VIDEO FOLDER videotype = Path(video).suffix print("Starting % ", destfolder, videos) vname = str(Path(video).stem) # if notanalyzed: # notanalyzed,outdataname,sourcedataname,DLCscorer=auxiliaryfunctions.CheckifPostProcessing(folder,vname,DLCscorer,DLCscorerlegacy,suffix='checking') if filtered == True: videooutname1 = os.path.join(vname + DLCscorer + "filtered_labeled.mp4") videooutname2 = os.path.join(vname + DLCscorerlegacy + "filtered_labeled.mp4") else: videooutname1 = os.path.join(vname + DLCscorer + "_labeled.mp4") videooutname2 = os.path.join(vname + DLCscorerlegacy + "_labeled.mp4") if os.path.isfile(videooutname1) or os.path.isfile(videooutname2): print("Labeled video already created.") else: print("Loading ", video, "and data.") try: df, filepath, _, _ = auxiliaryfunctions.load_analyzed_data( destfolder, vname, DLCscorer, filtered, track_method) metadata = auxiliaryfunctions.load_video_metadata( destfolder, vname, DLCscorer) if cfg.get("multianimalproject", False): s = "_id" if color_by == "individual" else "_bp" else: s = "" videooutname = filepath.replace(".h5", f"{s}_labeled.mp4") if os.path.isfile(videooutname): print("Labeled video already created. Skipping...") continue if all(individuals): df = df.loc(axis=1)[:, individuals] cropping = metadata["data"]["cropping"] [x1, x2, y1, y2] = metadata["data"]["cropping_parameters"] labeled_bpts = [ bp for bp in df.columns.get_level_values( "bodyparts").unique() if bp in bodyparts ] if not fastmode: tmpfolder = os.path.join(str(videofolder), "temp-" + vname) if save_frames: auxiliaryfunctions.attempttomakefolder(tmpfolder) clip = vp(video) CreateVideoSlow( videooutname, clip, df, tmpfolder, cfg["dotsize"], cfg["colormap"], cfg["alphavalue"], cfg["pcutoff"], trailpoints, cropping, x1, x2, y1, y2, save_frames, labeled_bpts, outputframerate, Frames2plot, bodyparts2connect, skeleton_color, draw_skeleton, displaycropped, color_by, ) else: if (displaycropped ): # then the cropped video + the labels is depicted clip = vp( fname=video, sname=videooutname, codec=codec, sw=x2 - x1, sh=y2 - y1, ) else: # then the full video + the (perhaps in cropped mode analyzed labels) are depicted clip = vp(fname=video, sname=videooutname, codec=codec) CreateVideo( clip, df, cfg["pcutoff"], cfg["dotsize"], cfg["colormap"], labeled_bpts, trailpoints, cropping, x1, x2, y1, y2, bodyparts2connect, skeleton_color, draw_skeleton, displaycropped, color_by, ) except FileNotFoundError as e: print(e) continue os.chdir(start_path)
def proc_video( videos, destfolder, filtered, DLCscorer, DLCscorerlegacy, track_method, cfg, individuals, color_by, bodyparts, codec, bodyparts2connect, trailpoints, save_frames, outputframerate, Frames2plot, draw_skeleton, skeleton_color, displaycropped, fastmode, video, ): """Helper function for create_videos Parameters ---------- """ videofolder = Path(video).parents[0] if destfolder is None: destfolder = videofolder # where your folder with videos is. auxiliaryfunctions.attempttomakefolder(destfolder) os.chdir(destfolder) # THE VIDEO IS STILL IN THE VIDEO FOLDER print("Starting to process video: {}".format(video)) vname = str(Path(video).stem) if filtered: videooutname1 = os.path.join(vname + DLCscorer + "filtered_labeled.mp4") videooutname2 = os.path.join(vname + DLCscorerlegacy + "filtered_labeled.mp4") else: videooutname1 = os.path.join(vname + DLCscorer + "_labeled.mp4") videooutname2 = os.path.join(vname + DLCscorerlegacy + "_labeled.mp4") if os.path.isfile(videooutname1) or os.path.isfile(videooutname2): print("Labeled video {} already created.".format(vname)) else: print("Loading {} and data.".format(video)) try: df, filepath, _, _ = auxiliaryfunctions.load_analyzed_data( destfolder, vname, DLCscorer, filtered, track_method ) metadata = auxiliaryfunctions.load_video_metadata( destfolder, vname, DLCscorer ) if cfg.get("multianimalproject", False): s = "_id" if color_by == "individual" else "_bp" else: s = "" # Adds support for multi_output mode. Adds extra bodyparts found in the data but not in the config.yaml. # Only works if "multi_output_format" is set to "separate-bodyparts". cmp_set = set( idx[1] for idx in df if (idx[1] in bodyparts or is_extension_part(idx[1], bodyparts)) ) bodyparts = [ bp for bp in df.columns.get_level_values("bodyparts").unique() if (bp in cmp_set) ] # print(bodyparts) videooutname = filepath.replace(".h5", f"{s}_labeled.mp4") if os.path.isfile(videooutname): print("Labeled video already created. Skipping...") return if all(individuals): df = df.loc(axis=1)[:, individuals] cropping = metadata["data"]["cropping"] [x1, x2, y1, y2] = metadata["data"]["cropping_parameters"] labeled_bpts = [ bp for bp in df.columns.get_level_values("bodyparts").unique() if bp in bodyparts ] if not fastmode: tmpfolder = os.path.join(str(videofolder), "temp-" + vname) if save_frames: auxiliaryfunctions.attempttomakefolder(tmpfolder) clip = vp(video) CreateVideoSlow( videooutname, clip, df, tmpfolder, cfg["dotsize"], cfg["colormap"], cfg["alphavalue"], cfg["pcutoff"], trailpoints, cropping, x1, x2, y1, y2, save_frames, labeled_bpts, outputframerate, Frames2plot, bodyparts2connect, skeleton_color, draw_skeleton, displaycropped, color_by, ) else: if displaycropped: # then the cropped video + the labels is depicted clip = vp( fname=video, sname=videooutname, codec=codec, sw=x2 - x1, sh=y2 - y1, ) else: # then the full video + the (perhaps in cropped mode analyzed labels) are depicted clip = vp(fname=video, sname=videooutname, codec=codec) CreateVideo( clip, df, cfg["pcutoff"], cfg["dotsize"], cfg["colormap"], labeled_bpts, trailpoints, cropping, x1, x2, y1, y2, bodyparts2connect, skeleton_color, draw_skeleton, displaycropped, color_by, ) except FileNotFoundError as e: print(e)