def MakeLabeledPlots(folder, DataCombined, cfg, Labels, Colorscheme, cc, scale): tmpfolder = str(folder) + "_labeled" auxiliaryfunctions.attempttomakefolder(tmpfolder) for index, imagename in enumerate(DataCombined.index.values): image = io.imread(os.path.join(cfg["project_path"], imagename)) plt.axis("off") if np.ndim(image) == 2: h, w = np.shape(image) else: h, w, nc = np.shape(image) plt.figure( frameon=False, figsize=(w * 1.0 / 100 * scale, h * 1.0 / 100 * scale) ) plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0) plt.imshow(image, "gray") if index == 0: print("They are stored in the following folder: %s." % tmpfolder) # folder) for c, bp in enumerate(cfg["bodyparts"]): plt.plot( DataCombined[cfg["scorer"]][bp]["x"].values[index], DataCombined[cfg["scorer"]][bp]["y"].values[index], Labels[cc], color=Colorscheme(c), alpha=cfg["alphavalue"], ms=cfg["dotsize"], ) plt.xlim(0, w) plt.ylim(0, h) plt.axis("off") plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0) plt.gca().invert_yaxis() # plt.savefig(str(Path(tmpfolder)/imagename.split(os.sep)[-1])) plt.savefig( os.path.join(tmpfolder, str(Path(imagename).name)) ) # create file name also on Windows for Unix projects (and vice versa) plt.close("all")
def make_labeled_images_from_dataframe( df, cfg, destfolder="", scale=1.0, dpi=100, keypoint="+", draw_skeleton=True, color_by="bodypart", ): """ Write labeled frames to disk from a DataFrame. Parameters ---------- df : pd.DataFrame DataFrame containing the labeled data. Typically, the DataFrame is obtained through pandas.read_csv() or pandas.read_hdf(). cfg : dict Project configuration. destfolder : string, optional Destination folder into which images will be stored. By default, same location as the labeled data. Note that the folder will be created if it does not exist. scale : float, optional Up/downscale the output dimensions. By default, outputs are of the same dimensions as the original images. dpi : int, optional Output resolution. 100 dpi by default. keypoint : str, optional Keypoint appearance. By default, keypoints are marked by a + sign. Refer to https://matplotlib.org/3.2.1/api/markers_api.html for a list of all possible options. draw_skeleton : bool, optional Whether to draw the animal skeleton as defined in *cfg*. True by default. color_by : str, optional Color scheme of the keypoints. Must be either 'bodypart' or 'individual'. By default, keypoints are colored relative to the bodypart they represent. """ bodyparts = df.columns.get_level_values("bodyparts") bodypart_names = bodyparts.unique() nbodyparts = len(bodypart_names) bodyparts = bodyparts[::2] if color_by == "bodypart": map_ = bodyparts.map(dict(zip(bodypart_names, range(nbodyparts)))) cmap = get_cmap(nbodyparts, cfg["colormap"]) colors = cmap(map_) elif color_by == "individual": try: individuals = df.columns.get_level_values("individuals") individual_names = individuals.unique().to_list() nindividuals = len(individual_names) individuals = individuals[::2] map_ = individuals.map( dict(zip(individual_names, range(nindividuals)))) cmap = get_cmap(nindividuals, cfg["colormap"]) colors = cmap(map_) except KeyError as e: raise Exception( "Coloring by individuals is only valid for multi-animal data" ) from e else: raise ValueError( "`color_by` must be either `bodypart` or `individual`.") bones = [] if draw_skeleton: for bp1, bp2 in cfg["skeleton"]: match1, match2 = [], [] for j, bp in enumerate(bodyparts): if bp == bp1: match1.append(j) elif bp == bp2: match2.append(j) bones.extend(zip(match1, match2)) ind_bones = tuple(zip(*bones)) sep = "/" if "/" in df.index[0] else "\\" images = cfg["project_path"] + sep + df.index if sep != os.path.sep: images = images.str.replace(sep, os.path.sep) if not destfolder: destfolder = os.path.dirname(images[0]) tmpfolder = destfolder + "_labeled" attempttomakefolder(tmpfolder) ic = io.imread_collection(images.to_list()) h, w = ic[0].shape[:2] fig, ax = prepare_figure_axes(w, h, scale, dpi) im = ax.imshow(np.zeros((h, w)), "gray") scat = ax.scatter([], [], s=cfg["dotsize"], alpha=cfg["alphavalue"], marker=keypoint) scat.set_color(colors) xy = df.values.reshape((df.shape[0], -1, 2)) segs = xy[:, ind_bones].swapaxes(1, 2) coll = LineCollection([], colors=cfg["skeleton_color"]) ax.add_collection(coll) for i in trange(len(ic)): coords = xy[i] im.set_array(ic[i]) scat.set_offsets(coords) if ind_bones: coll.set_segments(segs[i]) imagename = os.path.basename(ic.files[i]) fig.savefig( os.path.join(tmpfolder, imagename.replace(".png", f"_{color_by}.png"))) plt.close(fig)
def plot_trajectories(config, videos, videotype='.avi', shuffle=1, trainingsetindex=0, filtered=False, displayedbodyparts='all', showfigures=False, destfolder=None): """ Plots the trajectories of various bodyparts across the video. Parameters ---------- config : string Full path of the config.yaml file as a string. videos : list A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored. videotype: string, optional Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi`` shuffle: list, optional List of integers specifying the shuffle indices of the training dataset. The default is [1] trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). filtered: bool, default false Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcutcore.filterpredictions displayedbodyparts: list of strings, optional This select the body parts that are plotted in the video. Either ``all``, then all body parts from config.yaml are used, or a list of strings that are a subset of the full list. E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts. showfigures: bool, default false If true then plots are also displayed. destfolder: string, optional Specifies the destination folder that was used for storing analysis data (default is the path of the video). Example -------- for labeling the frames >>> deeplabcutcore.plot_trajectories('home/alex/analysis/project/reaching-task/config.yaml',['/home/alex/analysis/project/videos/reachingvideo1.avi']) -------- """ cfg = auxiliaryfunctions.read_config(config) trainFraction = cfg['TrainingFraction'][trainingsetindex] DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction ) #automatically loads corresponding model (even training iteration based on snapshot index) bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, displayedbodyparts) Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype) for video in Videos: print(video) if destfolder is None: videofolder = str(Path(video).parents[0]) else: videofolder = destfolder vname = str(Path(video).stem) print("Starting % ", videofolder, video) notanalyzed, dataname, DLCscorer = auxiliaryfunctions.CheckifNotAnalyzed( videofolder, vname, DLCscorer, DLCscorerlegacy, flag='checking') if notanalyzed: print("The video was not analyzed with this scorer:", DLCscorer) else: #LoadData print("Loading ", video, "and data.") datafound, metadata, Dataframe, DLCscorer, suffix = auxiliaryfunctions.LoadAnalyzedData( str(videofolder), vname, DLCscorer, filtered ) #returns boolean variable if data was found and metadata + pandas array if datafound: basefolder = videofolder auxiliaryfunctions.attempttomakefolder(basefolder) auxiliaryfunctions.attempttomakefolder( os.path.join(basefolder, 'plot-poses')) tmpfolder = os.path.join(basefolder, 'plot-poses', vname) auxiliaryfunctions.attempttomakefolder(tmpfolder) PlottingResults(tmpfolder, Dataframe, DLCscorer, cfg, bodyparts, showfigures, suffix + '.png') print( 'Plots created! Please check the directory "plot-poses" within the video directory' )
def evaluate_network(config, Shuffles=[1], trainingsetindex=0, plotting=None, show_errors=True, comparisonbodyparts="all", gputouse=None, rescale=False): """ Evaluates the network based on the saved models at different stages of the training network.\n The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'. Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models. Parameters ---------- config : string Full path of the config.yaml file as a string. Shuffles: list, optional List of integers specifying the shuffle indices of the training dataset. The default is [1] trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This variable can also be set to "all". plotting: bool, optional Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False`` show_errors: bool, optional Display train and test errors. The default is `True`` comparisonbodyparts: list of bodyparts, Default is "all". The average error will be computed for those body parts only (Has to be a subset of the body parts). gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None. See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries rescale: bool, default False Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the original size! Examples -------- If you do not want to plot >>> deeplabcutcore.evaluate_network('/analysis/project/reaching-task/config.yaml', Shuffles=[1]) -------- If you want to plot >>> deeplabcutcore.evaluate_network('/analysis/project/reaching-task/config.yaml',Shuffles=[1],True) """ import os #import skimage.color #from skimage.io import imread from deeplabcutcore.utils.auxfun_videos import imread, imresize from deeplabcutcore.pose_estimation_tensorflow.nnet import predict from deeplabcutcore.pose_estimation_tensorflow.config import load_config from deeplabcutcore.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input from deeplabcutcore.utils import auxiliaryfunctions import tensorflow as tf if 'TF_CUDNN_USE_AUTOTUNE' in os.environ: del os.environ[ 'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training tf.compat.v1.reset_default_graph() os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # # tf.logging.set_verbosity(tf.logging.WARN) start_path = os.getcwd() # Read file path for pose_config file. >> pass it on cfg = auxiliaryfunctions.read_config(config) if gputouse is not None: #gpu selectinon os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse) if trainingsetindex == 'all': TrainingFractions = cfg["TrainingFraction"] else: if trainingsetindex < len( cfg["TrainingFraction"]) and trainingsetindex >= 0: TrainingFractions = [ cfg["TrainingFraction"][int(trainingsetindex)] ] else: raise Exception('Please check the trainingsetindex! ', trainingsetindex, ' should be an integer from 0 .. ', int(len(cfg["TrainingFraction"]) - 1)) # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) Data = pd.read_hdf( os.path.join(cfg["project_path"], str(trainingsetfolder), 'CollectedData_' + cfg["scorer"] + '.h5'), 'df_with_missing') # Get list of body parts to evaluate network for comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, comparisonbodyparts) # Make folder for evaluation auxiliaryfunctions.attempttomakefolder( str(cfg["project_path"] + "/evaluation-results/")) for shuffle in Shuffles: for trainFraction in TrainingFractions: ################################################## # Load and setup CNN part detector ################################################## datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) modelfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg))) path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml' # Load meta data data, trainIndices, testIndices, trainFraction = auxiliaryfunctions.LoadMetadata( os.path.join(cfg["project_path"], metadatafn)) try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, trainFraction)) #change batch size, if it was edited during analysis! dlc_cfg['batch_size'] = 1 #in case this was edited for analysis. #Create folder structure to store results. evaluationfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetEvaluationFolder( trainFraction, shuffle, cfg))) auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True) #path_train_config = modelfolder / 'train' / 'pose_cfg.yaml' # Check which snapshots are available and sort them by # iterations Snapshots = np.array([ fn.split('.')[0] for fn in os.listdir(os.path.join(str(modelfolder), 'train')) if "index" in fn ]) try: #check if any where found? Snapshots[0] except IndexError: raise FileNotFoundError( "Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so." % (shuffle, trainFraction)) increasing_indices = np.argsort( [int(m.split('-')[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] if cfg["snapshotindex"] == -1: snapindices = [-1] elif cfg["snapshotindex"] == "all": snapindices = range(len(Snapshots)) elif cfg["snapshotindex"] < len(Snapshots): snapindices = [cfg["snapshotindex"]] else: raise ValueError( "Invalid choice, only -1 (last), any integer up to last, or all (as string)!" ) final_result = [] ########################### RESCALING (to global scale) if rescale == True: scale = dlc_cfg['global_scale'] Data = pd.read_hdf( os.path.join(cfg["project_path"], str(trainingsetfolder), 'CollectedData_' + cfg["scorer"] + '.h5'), 'df_with_missing') * scale else: scale = 1 ################################################## # Compute predictions over images ################################################## for snapindex in snapindices: dlc_cfg['init_weights'] = os.path.join( str(modelfolder), 'train', Snapshots[snapindex] ) #setting weights to corresponding snapshot. trainingsiterations = ( dlc_cfg['init_weights'].split(os.sep)[-1] ).split( '-' )[-1] #read how many training siterations that corresponds to. # Name for deeplabcut net (based on its parameters) DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction, trainingsiterations) print("Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations) notanalyzed, resultsfilename, DLCscorer = auxiliaryfunctions.CheckifNotEvaluated( str(evaluationfolder), DLCscorer, DLCscorerlegacy, Snapshots[snapindex]) if notanalyzed: # Specifying state of model (snapshot / training state) sess, inputs, outputs = predict.setup_pose_prediction( dlc_cfg) Numimages = len(Data.index) PredicteData = np.zeros( (Numimages, 3 * len(dlc_cfg['all_joints_names']))) print("Analyzing data...") for imageindex, imagename in tqdm(enumerate(Data.index)): image = imread(os.path.join(cfg['project_path'], imagename), mode='RGB') if scale != 1: image = imresize(image, scale) #image = skimage.color.gray2rgb(image) image_batch = data_to_input(image) # Compute prediction with the CNN outputs_np = sess.run(outputs, feed_dict={inputs: image_batch}) scmap, locref = predict.extract_cnn_output( outputs_np, dlc_cfg) # Extract maximum scoring location from the heatmap, assume 1 person pose = predict.argmax_pose_predict( scmap, locref, dlc_cfg.stride) PredicteData[imageindex, :] = pose.flatten( ) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts! sess.close() #closes the current tf session index = pd.MultiIndex.from_product( [[DLCscorer], dlc_cfg['all_joints_names'], ['x', 'y', 'likelihood']], names=['scorer', 'bodyparts', 'coords']) # Saving results DataMachine = pd.DataFrame(PredicteData, columns=index, index=Data.index.values) DataMachine.to_hdf(resultsfilename, 'df_with_missing', format='table', mode='w') print("Done and results stored for snapshot: ", Snapshots[snapindex]) DataCombined = pd.concat([Data.T, DataMachine.T], axis=0, sort=False).T RMSE, RMSEpcutoff = pairwisedistances( DataCombined, cfg["scorer"], DLCscorer, cfg["pcutoff"], comparisonbodyparts) testerror = np.nanmean( RMSE.iloc[testIndices].values.flatten()) trainerror = np.nanmean( RMSE.iloc[trainIndices].values.flatten()) testerrorpcutoff = np.nanmean( RMSEpcutoff.iloc[testIndices].values.flatten()) trainerrorpcutoff = np.nanmean( RMSEpcutoff.iloc[trainIndices].values.flatten()) results = [ trainingsiterations, int(100 * trainFraction), shuffle, np.round(trainerror, 2), np.round(testerror, 2), cfg["pcutoff"], np.round(trainerrorpcutoff, 2), np.round(testerrorpcutoff, 2) ] final_result.append(results) if show_errors == True: print("Results for", trainingsiterations, " training iterations:", int(100 * trainFraction), shuffle, "train error:", np.round(trainerror, 2), "pixels. Test error:", np.round(testerror, 2), " pixels.") print("With pcutoff of", cfg["pcutoff"], " train error:", np.round(trainerrorpcutoff, 2), "pixels. Test error:", np.round(testerrorpcutoff, 2), "pixels") if scale != 1: print( "The predictions have been calculated for rescaled images (and rescaled ground truth). Scale:", scale) print( "Thereby, the errors are given by the average distances between the labels by DLC and the scorer." ) if plotting == True: print("Plotting...") foldername = os.path.join( str(evaluationfolder), 'LabeledImages_' + DLCscorer + '_' + Snapshots[snapindex]) auxiliaryfunctions.attempttomakefolder(foldername) Plotting( cfg, comparisonbodyparts, DLCscorer, trainIndices, DataCombined * 1. / scale, foldername ) #Rescaling coordinates to have figure in original size! tf.compat.v1.reset_default_graph() #print(final_result) else: DataMachine = pd.read_hdf(resultsfilename, 'df_with_missing') if plotting == True: DataCombined = pd.concat([Data.T, DataMachine.T], axis=0, sort=False).T print( "Plotting...(attention scale might be inconsistent in comparison to when data was analyzed; i.e. if you used rescale)" ) foldername = os.path.join( str(evaluationfolder), 'LabeledImages_' + DLCscorer + '_' + Snapshots[snapindex]) auxiliaryfunctions.attempttomakefolder(foldername) Plotting(cfg, comparisonbodyparts, DLCscorer, trainIndices, DataCombined * 1. / scale, foldername) if len(final_result) > 0: #Only append if results were calculated make_results_file(final_result, evaluationfolder, DLCscorer) print( "The network is evaluated and the results are stored in the subdirectory 'evaluation_results'." ) print( "If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos." ) print( "Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)" ) #returning to intial folder os.chdir(str(start_path))
def ExtractFramesbasedonPreselection( Index, extractionalgorithm, Dataframe, dataname, scorer, video, cfg, config, opencv=True, cluster_resizewidth=30, cluster_color=False, savelabeled=True, ): from deeplabcutcore.create_project import add start = cfg["start"] stop = cfg["stop"] numframes2extract = cfg["numframes2pick"] bodyparts = cfg["bodyparts"] videofolder = str(Path(video).parents[0]) vname = str(Path(video).stem) tmpfolder = os.path.join(cfg["project_path"], "labeled-data", vname) if os.path.isdir(tmpfolder): print("Frames from video", vname, " already extracted (more will be added)!") else: auxiliaryfunctions.attempttomakefolder(tmpfolder) nframes = np.size(Dataframe.index) print("Loading video...") if opencv: import cv2 cap = cv2.VideoCapture(video) fps = cap.get(5) duration = nframes * 1.0 / fps size = (int(cap.get(4)), int(cap.get(3))) else: from moviepy.editor import VideoFileClip clip = VideoFileClip(video) fps = clip.fps duration = clip.duration size = clip.size if cfg["cropping"]: # one might want to adjust coords = (cfg["x1"], cfg["x2"], cfg["y1"], cfg["y2"]) else: coords = None print("Duration of video [s]: ", duration, ", recorded @ ", fps, "fps!") print("Overall # of frames: ", nframes, "with (cropped) frame dimensions: ") if extractionalgorithm == "uniform": if opencv: frames2pick = frameselectiontools.UniformFramescv2( cap, numframes2extract, start, stop, Index ) else: frames2pick = frameselectiontools.UniformFrames( clip, numframes2extract, start, stop, Index ) elif extractionalgorithm == "kmeans": if opencv: frames2pick = frameselectiontools.KmeansbasedFrameselectioncv2( cap, numframes2extract, start, stop, cfg["cropping"], coords, Index, resizewidth=cluster_resizewidth, color=cluster_color, ) else: if cfg["cropping"]: clip = clip.crop(y1=cfg["y1"], y2=cfg["x2"], x1=cfg["x1"], x2=cfg["x2"]) frames2pick = frameselectiontools.KmeansbasedFrameselection( clip, numframes2extract, start, stop, Index, resizewidth=cluster_resizewidth, color=cluster_color, ) else: print( "Please implement this method yourself! Currently the options are 'kmeans', 'jump', 'uniform'." ) frames2pick = [] # Extract frames + frames with plotted labels and store them in folder (with name derived from video name) nder labeled-data print("Let's select frames indices:", frames2pick) colors = visualization.get_cmap(len(bodyparts), cfg["colormap"]) strwidth = int(np.ceil(np.log10(nframes))) # width for strings for index in frames2pick: ##tqdm(range(0,nframes,10)): if opencv: PlottingSingleFramecv2( cap, cv2, cfg["cropping"], coords, Dataframe, bodyparts, tmpfolder, index, scorer, cfg["dotsize"], cfg["pcutoff"], cfg["alphavalue"], colors, strwidth, savelabeled, ) else: PlottingSingleFrame( clip, Dataframe, bodyparts, tmpfolder, index, scorer, cfg["dotsize"], cfg["pcutoff"], cfg["alphavalue"], colors, strwidth, savelabeled, ) plt.close("all") # close videos if opencv: cap.release() else: clip.close() del clip # Extract annotations based on DeepLabCut and store in the folder (with name derived from video name) under labeled-data if len(frames2pick) > 0: # Dataframe = pd.read_hdf(os.path.join(videofolder,dataname+'.h5')) DF = Dataframe.iloc[frames2pick] DF.index = [ os.path.join( "labeled-data", vname, "img" + str(index).zfill(strwidth) + ".png" ) for index in DF.index ] # exchange index number by file names. machinefile = os.path.join( tmpfolder, "machinelabels-iter" + str(cfg["iteration"]) + ".h5" ) if Path(machinefile).is_file(): Data = pd.read_hdf(machinefile, "df_with_missing") DataCombined = pd.concat([Data, DF]) # drop duplicate labels: DataCombined = DataCombined[~DataCombined.index.duplicated(keep="first")] DataCombined.to_hdf(machinefile, key="df_with_missing", mode="w") DataCombined.to_csv( os.path.join(tmpfolder, "machinelabels.csv") ) # this is always the most current one (as reading is from h5) else: DF.to_hdf(machinefile, key="df_with_missing", mode="w") DF.to_csv(os.path.join(tmpfolder, "machinelabels.csv")) try: if cfg["cropping"]: add.add_new_videos( config, [video], coords=[coords] ) # make sure you pass coords as a list else: add.add_new_videos(config, [video], coords=None) except: # can we make a catch here? - in fact we should drop indices from DataCombined if they are in CollectedData.. [ideal behavior; currently this is pretty unlikely] print( "AUTOMATIC ADDING OF VIDEO TO CONFIG FILE FAILED! You need to do this manually for including it in the config.yaml file!" ) print("Videopath:", video, "Coordinates for cropping:", coords) pass print( "The outlier frames are extracted. They are stored in the subdirectory labeled-data\%s." % vname ) print( "Once you extracted frames for all videos, use 'refine_labels' to manually correct the labels." ) else: print("No frames were extracted.")
def create_labeled_video( config, videos, videotype="avi", shuffle=1, trainingsetindex=0, filtered=False, save_frames=False, Frames2plot=None, delete=False, displayedbodyparts="all", codec="mp4v", outputframerate=None, destfolder=None, draw_skeleton=False, trailpoints=0, displaycropped=False, ): """ Labels the bodyparts in a video. Make sure the video is already analyzed by the function 'analyze_video' Parameters ---------- config : string Full path of the config.yaml file as a string. videos : list A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored. videotype: string, optional Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi`` shuffle : int, optional Number of shuffles of training dataset. Default is set to 1. trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). filtered: bool, default false Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcutcore.filterpredictions videotype: string, optional Checks for the extension of the video in case the input is a directory.\nOnly videos with this extension are analyzed. The default is ``.avi`` save_frames: bool If true creates each frame individual and then combines into a video. This variant is relatively slow as it stores all individual frames. However, it uses matplotlib to create the frames and is therefore much more flexible (one can set transparency of markers, crop, and easily customize). Frames2plot: List of indices If not None & save_frames=True then the frames corresponding to the index will be plotted. For example, Frames2plot=[0,11] will plot the first and the 12th frame. delete: bool If true then the individual frames created during the video generation will be deleted. displayedbodyparts: list of strings, optional This select the body parts that are plotted in the video. Either ``all``, then all body parts from config.yaml are used orr a list of strings that are a subset of the full list. E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts. codec: codec for labeled video. Options see http://www.fourcc.org/codecs.php [depends on your ffmpeg installation.] outputframerate: positive number, output frame rate for labeled video (only available for the mode with saving frames.) By default: None, which results in the original video rate. destfolder: string, optional Specifies the destination folder that was used for storing analysis data (default is the path of the video). draw_skeleton: bool If ``True`` adds a line connecting the body parts making a skeleton on on each frame. The body parts to be connected and the color of these connecting lines are specified in the config file. By default: ``False`` trailpoints: int Number of revious frames whose body parts are plotted in a frame (for displaying history). Default is set to 0. displaycropped: bool, optional Specifies whether only cropped frame is displayed (with labels analyzed therein), or the original frame with the labels analyzed in the cropped subset. Examples -------- If you want to create the labeled video for only 1 video >>> deeplabcutcore.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi']) -------- If you want to create the labeled video for only 1 video and store the individual frames >>> deeplabcutcore.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'],save_frames=True) -------- If you want to create the labeled video for multiple videos >>> deeplabcutcore.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi']) -------- If you want to create the labeled video for all the videos (as .avi extension) in a directory. >>> deeplabcutcore.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/']) -------- If you want to create the labeled video for all the videos (as .mp4 extension) in a directory. >>> deeplabcutcore.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'],videotype='mp4') -------- """ cfg = auxiliaryfunctions.read_config(config) start_path = os.getcwd() # record cwd to return to this directory in the end trainFraction = cfg["TrainingFraction"][trainingsetindex] DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction ) # automatically loads corresponding model (even training iteration based on snapshot index) bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, displayedbodyparts ) if draw_skeleton: bodyparts2connect = cfg["skeleton"] skeleton_color = cfg["skeleton_color"] else: bodyparts2connect = None skeleton_color = None Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype) for video in Videos: if destfolder is None: videofolder = Path(video).parents[0] # where your folder with videos is. else: videofolder = destfolder os.chdir(str(videofolder)) videotype = Path(video).suffix print("Starting % ", videofolder, videos) vname = str(Path(video).stem) # if notanalyzed: # notanalyzed,outdataname,sourcedataname,DLCscorer=auxiliaryfunctions.CheckifPostProcessing(folder,vname,DLCscorer,DLCscorerlegacy,suffix='checking') if filtered == True: videooutname1 = os.path.join(vname + DLCscorer + "filtered_labeled.mp4") videooutname2 = os.path.join( vname + DLCscorerlegacy + "filtered_labeled.mp4" ) else: videooutname1 = os.path.join(vname + DLCscorer + "_labeled.mp4") videooutname2 = os.path.join(vname + DLCscorerlegacy + "_labeled.mp4") if os.path.isfile(videooutname1) or os.path.isfile(videooutname2): print("Labeled video already created.") else: print("Loading ", video, "and data.") datafound, metadata, Dataframe, DLCscorer, suffix = auxiliaryfunctions.LoadAnalyzedData( str(videofolder), vname, DLCscorer, filtered ) # returns boolean variable if data was found and metadata + pandas array videooutname = os.path.join(vname + DLCscorer + suffix + "_labeled.mp4") if datafound and not os.path.isfile( videooutname ): # checking again, for this loader video could exist # Loading cropping data used during analysis cropping = metadata["data"]["cropping"] [x1, x2, y1, y2] = metadata["data"]["cropping_parameters"] if save_frames == True: tmpfolder = os.path.join(str(videofolder), "temp-" + vname) auxiliaryfunctions.attempttomakefolder(tmpfolder) clip = vp(video) CreateVideoSlow( videooutname, clip, Dataframe, tmpfolder, cfg["dotsize"], cfg["colormap"], cfg["alphavalue"], cfg["pcutoff"], trailpoints, cropping, x1, x2, y1, y2, delete, DLCscorer, bodyparts, outputframerate, Frames2plot, bodyparts2connect, skeleton_color, draw_skeleton, displaycropped, ) else: if ( displaycropped ): # then the cropped video + the labels is depicted clip = vp( fname=video, sname=videooutname, codec=codec, sw=x2 - x1, sh=y2 - y1, ) CreateVideo( clip, Dataframe, cfg["pcutoff"], cfg["dotsize"], cfg["colormap"], DLCscorer, bodyparts, trailpoints, cropping, x1, x2, y1, y2, bodyparts2connect, skeleton_color, draw_skeleton, displaycropped, ) else: # then the full video + the (perhaps in cropped mode analyzed labels) are depicted clip = vp(fname=video, sname=videooutname, codec=codec) CreateVideo( clip, Dataframe, cfg["pcutoff"], cfg["dotsize"], cfg["colormap"], DLCscorer, bodyparts, trailpoints, cropping, x1, x2, y1, y2, bodyparts2connect, skeleton_color, draw_skeleton, displaycropped, ) os.chdir(str(start_path))
def create_training_dataset(config,num_shuffles=1,Shuffles=None,windows2linux=False,userfeedback=False, trainIndexes=None,testIndexes=None, net_type=None,augmenter_type=None): """ Creates a training dataset. Labels from all the extracted frames are merged into a single .h5 file.\n Only the videos included in the config file are used to create this dataset.\n [OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project. Parameter ---------- config : string Full path of the config.yaml file as a string. num_shuffles : int, optional Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1. Shuffles: list of shuffles. Alternatively the user can also give a list of shuffles (integers!). windows2linux: bool. The annotation files contain path formated according to your operating system. If you label on windows but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths. userfeedback: bool, optional If this is set to false, then all requested train/test splits are created (no matter if they already exist). If you want to assure that previous splits etc. are not overwritten, then set this to True and you will be asked for each split. trainIndexes: list of lists, optional (default=None) List of one or multiple lists containing train indexes. A list containing two lists of training indexes will produce two splits. testIndexes: list of lists, optional (default=None) List of test indexes. net_type: string Type of networks. Currently resnet_50, resnet_101, resnet_152, mobilenet_v2_1.0,mobilenet_v2_0.75, mobilenet_v2_0.5, and mobilenet_v2_0.35 are supported. augmenter_type: string Type of augmenter. Currently default, imgaug, tensorpack, and deterministic are supported. Example -------- >>> deeplabcutcore.create_training_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1) Windows: >>> deeplabcutcore.create_training_dataset('C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5]) -------- """ import scipy.io as sio # Loading metadata from config file: cfg = auxiliaryfunctions.read_config(config) scorer = cfg['scorer'] project_path = cfg['project_path'] # Create path for training sets & store data there trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) #Path concatenation OS platform independent auxiliaryfunctions.attempttomakefolder(Path(os.path.join(project_path,str(trainingsetfolder))),recursive=True) Data = merge_annotateddatasets(cfg,project_path,Path(os.path.join(project_path,trainingsetfolder)),windows2linux) Data = Data[scorer] #extract labeled data #loading & linking pretrained models if net_type is None: #loading & linking pretrained models net_type =cfg.get('default_net_type', 'resnet_50') else: if 'resnet' in net_type or 'mobilenet' in net_type: pass else: raise ValueError('Invalid network type:', net_type) if augmenter_type is None: augmenter_type=cfg.get('default_augmenter', 'default') else: if augmenter_type in ['default','imgaug','tensorpack','deterministic']: pass else: raise ValueError('Invalid augmenter type:', augmenter_type) import deeplabcutcore parent_path = Path(os.path.dirname(deeplabcutcore.__file__)) defaultconfigfile = str(parent_path / 'pose_cfg.yaml') model_path,num_shuffles=auxfun_models.Check4weights(net_type,parent_path,num_shuffles) #if the model does not exist >> throws error! if Shuffles is None: Shuffles = range(1, num_shuffles + 1) else: Shuffles = [i for i in Shuffles if isinstance(i, int)] #print(trainIndexes,testIndexes, Shuffles, augmenter_type,net_type) if trainIndexes is None and testIndexes is None: splits = [(trainFraction, shuffle, SplitTrials(range(len(Data.index)), trainFraction)) for trainFraction in cfg['TrainingFraction'] for shuffle in Shuffles] else: if len(trainIndexes) != len(testIndexes) != len(Shuffles): raise ValueError('Number of Shuffles and train and test indexes should be equal.') splits = [] for shuffle, (train_inds, test_inds) in enumerate(zip(trainIndexes, testIndexes)): trainFraction = round(len(train_inds) * 1./ (len(train_inds) + len(test_inds)), 2) print(f"You passed a split with the following fraction: {int(100 * trainFraction)}%") splits.append((trainFraction, Shuffles[shuffle], (train_inds, test_inds))) bodyparts = cfg['bodyparts'] nbodyparts = len(bodyparts) for trainFraction, shuffle, (trainIndexes, testIndexes) in splits: if len(trainIndexes)>0: if userfeedback: trainposeconfigfile, _, _ = training.return_train_network_path(config, shuffle=shuffle, trainFraction=trainFraction) if trainposeconfigfile.is_file(): askuser=input ("The model folder is already present. If you continue, it will overwrite the existing model (split). Do you want to continue?(yes/no): ") if askuser=='no'or askuser=='No' or askuser=='N' or askuser=='No': raise Exception("Use the Shuffles argument as a list to specify a different shuffle index. Check out the help for more details.") #################################################### # Generating data structure with labeled information & frame metadata (for deep cut) #################################################### # Make training file! datafilename, metadatafilename = auxiliaryfunctions.GetDataandMetaDataFilenames(trainingsetfolder, trainFraction, shuffle, cfg) ################################################################################ # Saving data file (convert to training file for deeper cut (*.mat)) ################################################################################ data, MatlabData = format_training_data(Data, trainIndexes, nbodyparts, project_path) sio.savemat(os.path.join(project_path,datafilename), {'dataset': MatlabData}) ################################################################################ # Saving metadata (Pickle file) ################################################################################ auxiliaryfunctions.SaveMetadata(os.path.join(project_path,metadatafilename),data, trainIndexes, testIndexes, trainFraction) ################################################################################ # Creating file structure for training & # Test files as well as pose_yaml files (containing training and testing information) ################################################################################# modelfoldername=auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg) auxiliaryfunctions.attempttomakefolder(Path(config).parents[0] / modelfoldername,recursive=True) auxiliaryfunctions.attempttomakefolder(str(Path(config).parents[0] / modelfoldername)+ '/train') auxiliaryfunctions.attempttomakefolder(str(Path(config).parents[0] / modelfoldername)+ '/test') path_train_config = str(os.path.join(cfg['project_path'],Path(modelfoldername),'train','pose_cfg.yaml')) path_test_config = str(os.path.join(cfg['project_path'],Path(modelfoldername),'test','pose_cfg.yaml')) #str(cfg['proj_path']+'/'+Path(modelfoldername) / 'test' / 'pose_cfg.yaml') items2change = { "dataset": datafilename, "metadataset": metadatafilename, "num_joints": len(bodyparts), "all_joints": [[i] for i in range(len(bodyparts))], "all_joints_names": [str(bpt) for bpt in bodyparts], "init_weights": model_path, "project_path": str(cfg['project_path']), "net_type": net_type, "dataset_type": augmenter_type, } trainingdata = MakeTrain_pose_yaml(items2change,path_train_config,defaultconfigfile) keys2save = [ "dataset", "num_joints", "all_joints", "all_joints_names", "net_type", 'init_weights', 'global_scale', 'location_refinement', 'locref_stdev' ] MakeTest_pose_yaml(trainingdata, keys2save,path_test_config) print("The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!") return splits
def mergeandsplit(config,trainindex=0,uniform=True,windows2linux=False): """ This function allows additional control over "create_training_dataset". Merge annotated data sets (from different folders) and split data in a specific way, returns the split variables (train/test indices). Importantly, this allows one to freeze a split. One can also either create a uniform split (uniform = True; thereby indexing TrainingFraction in config file) or leave-one-folder out split by passing the index of the corrensponding video from the config.yaml file as variable trainindex. Parameter ---------- config : string Full path of the config.yaml file as a string. trainindex: int, optional Either (in case uniform = True) indexes which element of TrainingFraction in the config file should be used (note it is a list!). Alternatively (uniform = False) indexes which folder is dropped, i.e. the first if trainindex=0, the second if trainindex =1, etc. uniform: bool, optional Perform uniform split (disregarding folder structure in labeled data), or (if False) leave one folder out. windows2linux: bool. The annotation files contain path formated according to your operating system. If you label on windows but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths. Examples -------- To create a leave-one-folder-out model: >>> trainIndexes, testIndexes=deeplabcutcore.mergeandsplit(config,trainindex=0,uniform=False) returns the indices for the first video folder (as defined in config file) as testIndexes and all others as trainIndexes. You can then create the training set by calling (e.g. defining it as Shuffle 3): >>> deeplabcutcore.create_training_dataset(config,Shuffles=[3],trainIndexes=trainIndexes,testIndexes=testIndexes) To freeze a (uniform) split: >>> trainIndexes, testIndexes=deeplabcutcore.mergeandsplit(config,trainindex=0,uniform=True) You can then create two model instances that have the identical trainingset. Thereby you can assess the role of various parameters on the performance of DLC. >>> deeplabcutcore.create_training_dataset(config,Shuffles=[0],trainIndexes=trainIndexes,testIndexes=testIndexes) >>> deeplabcutcore.create_training_dataset(config,Shuffles=[1],trainIndexes=trainIndexes,testIndexes=testIndexes) -------- """ # Loading metadata from config file: cfg = auxiliaryfunctions.read_config(config) scorer = cfg['scorer'] project_path = cfg['project_path'] # Create path for training sets & store data there trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) #Path concatenation OS platform independent auxiliaryfunctions.attempttomakefolder(Path(os.path.join(project_path,str(trainingsetfolder))),recursive=True) fn=os.path.join(project_path,trainingsetfolder,'CollectedData_'+cfg['scorer']) try: Data= pd.read_hdf(fn+'.h5', 'df_with_missing') except FileNotFoundError: Data = merge_annotateddatasets(cfg,project_path,Path(os.path.join(project_path,trainingsetfolder)),windows2linux=windows2linux) Data = Data[scorer] #extract labeled data if uniform==True: TrainingFraction = cfg['TrainingFraction'] trainFraction=TrainingFraction[trainindex] trainIndexes, testIndexes = SplitTrials(range(len(Data.index)), trainFraction) else: #leave one folder out split videos = cfg['video_sets'].keys() test_video_name = [Path(i).stem for i in videos][trainindex] print("Excluding the following folder (from training):", test_video_name) trainIndexes, testIndexes=[],[] for index,name in enumerate(Data.index): #print(index,name.split(os.sep)[1]) if test_video_name==name.split(os.sep)[1]: #this is the video name #print(name,test_video_name) testIndexes.append(index) else: trainIndexes.append(index) return trainIndexes, testIndexes