def __init__(self, dlc_cfg_path, model_snapshot): self.dlc_cfg = load_config(dlc_cfg_path) self.dlc_cfg["init_weights"] = model_snapshot self.dlc_cfg['batch_size'] = 1 self.sess, self.inputs, self.outputs = setup_pose_prediction( self.dlc_cfg) self.tracked_parts = self.dlc_cfg['all_joints_names']
def test_net(visualise, cache_scoremaps): logging.basicConfig(level=logging.INFO) cfg = load_config() dataset = create_dataset(cfg) dataset.set_shuffle(False) dataset.set_test_mode(True) sess, inputs, outputs = setup_pose_prediction(cfg) if cache_scoremaps: out_dir = cfg.scoremap_dir if not os.path.exists(out_dir): os.makedirs(out_dir) num_images = dataset.num_images predictions = np.zeros((num_images, ), dtype=np.object) for k in range(num_images): print("processing image {}/{}".format(k, num_images - 1)) batch = dataset.next_batch() outputs_np = sess.run(outputs, feed_dict={inputs: batch[Batch.inputs]}) scmap, locref = extract_cnn_output(outputs_np, cfg) pose = argmax_pose_predict(scmap, locref, cfg.stride) pose_refscale = np.copy(pose) pose_refscale[:, 0:2] /= cfg.global_scale predictions[k] = pose_refscale if visualise: img = np.squeeze(batch[Batch.inputs]).astype("uint8") visualize.show_heatmaps(cfg, img, scmap, pose) visualize.waitforbuttonpress() if cache_scoremaps: base = os.path.basename(batch[Batch.data_item].im_path) raw_name = os.path.splitext(base)[0] out_fn = os.path.join(out_dir, raw_name + ".mat") scipy.io.savemat(out_fn, mdict={"scoremaps": scmap.astype("float32")}) out_fn = os.path.join(out_dir, raw_name + "_locreg" + ".mat") if cfg.location_refinement: scipy.io.savemat( out_fn, mdict={"locreg_pred": locref.astype("float32")}) scipy.io.savemat("predictions.mat", mdict={"joints": predictions}) sess.close()
def load_deeplabcut(): """ Loads TensorFlow with predefined in config DeepLabCut model :return: tuple of DeepLabCut config, TensorFlow session, inputs and outputs """ model = os.path.join(DLC_PATH, models_folder, MODEL) cfg = load_config(os.path.join(model, 'test/pose_cfg.yaml')) snapshots = sorted([sn.split('.')[0] for sn in os.listdir(model + '/train/') if "index" in sn]) cfg['init_weights'] = model + '/train/' + snapshots[-1] sess, inputs, outputs = predict.setup_pose_prediction(cfg) return cfg, sess, inputs, outputs
def analyze_videos(config,videos,shuffle=1,trainingsetindex=0,videotype='avi',gputouse=None,save_as_csv=False, destfolder=None): """ Makes prediction based on a trained network. The index of the trained network is specified by parameters in the config file (in particular the variable 'snapshotindex') You can crop the video (before analysis), by changing 'cropping'=True and setting 'x1','x2','y1','y2' in the config file. The same cropping parameters will then be used for creating the video. Output: The labels are stored as MultiIndex Pandas Array, which contains the name of the network, body part name, (x, y) label position \n in pixels, and the likelihood for each frame per body part. These arrays are stored in an efficient Hierarchical Data Format (HDF) \n in the same directory, where the video is stored. However, if the flag save_as_csv is set to True, the data can also be exported in \n comma-separated values format (.csv), which in turn can be imported in many programs, such as MATLAB, R, Prism, etc. Parameters ---------- config : string Full path of the config.yaml file as a string. videos : list A list of strings containing the full paths to videos for analysis or a path to the directory where all the videos with same extension are stored. shuffle: int, optional An integer specifying the shuffle index of the training dataset used for training the network. The default is 1. trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). videotype: string, optional Checks for the extension of the video in case the input to the video is a directory.\nOnly videos with this extension are analyzed. The default is ``.avi`` gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None. See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries save_as_csv: bool, optional Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False`` destfolder: string, optional Specifies the destination folder for analysis data (default is the path of the video) Examples -------- If you want to analyze only 1 video >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi']) -------- If you want to analyze all videos of type avi in a folder: >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos'],videotype='.avi') -------- If you want to analyze multiple videos >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi']) -------- If you want to analyze multiple videos with shuffle = 2 >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'], shuffle=2) -------- If you want to analyze multiple videos with shuffle = 2 and save results as an additional csv file too >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'], shuffle=2,save_as_csv=True) -------- """ if 'TF_CUDNN_USE_AUTOTUNE' in os.environ: del os.environ['TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training tf.reset_default_graph() start_path=os.getcwd() #record cwd to return to this directory in the end cfg = auxiliaryfunctions.read_config(config) trainFraction = cfg['TrainingFraction'][trainingsetindex] modelfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg))) path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml' try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError("It seems the model for shuffle %s and trainFraction %s does not exist."%(shuffle,trainFraction)) # Check which snapshots are available and sort them by # iterations try: Snapshots = np.array([fn.split('.')[0]for fn in os.listdir(os.path.join(modelfolder , 'train'))if "index" in fn]) except FileNotFoundError: raise FileNotFoundError("Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s."%(shuffle,shuffle)) if cfg['snapshotindex'] == 'all': print("Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!") snapshotindex = -1 else: snapshotindex=cfg['snapshotindex'] increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder) ################################################## # Load and setup CNN part detector ################################################## # Check if data already was generated: dlc_cfg['init_weights'] = os.path.join(modelfolder , 'train', Snapshots[snapshotindex]) trainingsiterations = (dlc_cfg['init_weights'].split(os.sep)[-1]).split('-')[-1] #update batchsize (based on parameters in config.yaml) dlc_cfg['batch_size']=cfg['batch_size'] # Name for scorer: DLCscorer = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations=trainingsiterations) sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg) pdindex = pd.MultiIndex.from_product([[DLCscorer], dlc_cfg['all_joints_names'], ['x', 'y', 'likelihood']],names=['scorer', 'bodyparts', 'coords']) if gputouse is not None: #gpu selectinon os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse) ################################################## # Datafolder ################################################## #checks if input is a directory if [os.path.isdir(i) for i in videos] == [True]:#os.path.isdir(video)==True: """ Analyzes all the videos in the directory. """ print("Analyzing all the videos in the directory") videofolder= videos[0] os.chdir(videofolder) videolist=[fn for fn in os.listdir(os.curdir) if (videotype in fn) and ('_labeled.mp4' not in fn)] #exclude labeled-videos! Videos = sample(videolist,len(videolist)) # this is useful so multiple nets can be used to analzye simultanously else: if isinstance(videos,str): if os.path.isfile(videos): # #or just one direct path! Videos=[videos] else: Videos=[] else: Videos=[v for v in videos if os.path.isfile(v)] if len(Videos)>0: #looping over videos for video in Videos: AnalyzeVideo(video,DLCscorer,trainFraction,cfg,dlc_cfg,sess,inputs, outputs,pdindex,save_as_csv, destfolder) os.chdir(str(start_path)) print("The videos are analyzed. Now your research can truly start! \n You can create labeled videos with 'create_labeled_video'.") print("If the tracking is not satisfactory for some videos, consider expanding the training set. You can use the function 'extract_outlier_frames' to extract any outlier frames!")
def load_model(cfg, shuffle=1, trainingsetindex=0, TFGPUinference=True, modelprefix=""): """ Loads a tensorflow session with a DLC model from the associated configuration Return a tensorflow session with DLC model given cfg and shuffle Parameters: ----------- cfg : dict Configuration read from the project's main config.yaml file shuffle : int, optional which shuffle to use trainingsetindex : int. optional which training fraction to use, identified by its index TFGPUinference : bool, optional use tensorflow inference model? default = True Returns: -------- sess : tensorflow session tensorflow session with DLC model from the provided configuration, shuffle, and trainingsetindex checkpoint file path : string the path to the checkpoint file associated with the loaded model """ ######################## ### find snapshot to use ######################## train_fraction = cfg["TrainingFraction"][trainingsetindex] model_folder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetModelFolder( train_fraction, shuffle, cfg, modelprefix=modelprefix ) ), ) path_test_config = os.path.normpath(model_folder + "/test/pose_cfg.yaml") path_train_config = os.path.normpath(model_folder + "/train/pose_cfg.yaml") try: dlc_cfg = load_config(str(path_train_config)) # dlc_cfg_train = load_config(str(path_train_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, train_fraction) ) # Check which snapshots are available and sort them by # iterations try: Snapshots = np.array( [ fn.split(".")[0] for fn in os.listdir(os.path.join(model_folder, "train")) if "index" in fn ] ) except FileNotFoundError: raise FileNotFoundError( "Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before trying to export.\n Use the function 'train_network' to train the network for shuffle %s." % (shuffle, shuffle) ) if len(Snapshots) == 0: raise FileNotFoundError( "The train folder for iteration %s and shuffle %s exists, but no snapshots were found.\n Please train this model before trying to export.\n Use the function 'train_network' to train the network for iteration %s shuffle %s." % (cfg["iteration"], shuffle, cfg["iteration"], shuffle) ) if cfg["snapshotindex"] == "all": print( "Snapshotindex is set to 'all' in the config.yaml file. Changing snapshot index to -1!" ) snapshotindex = -1 else: snapshotindex = cfg["snapshotindex"] increasing_indices = np.argsort([int(m.split("-")[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] #################################### ### Load and setup CNN part detector #################################### # Check if data already was generated: dlc_cfg["init_weights"] = os.path.join( model_folder, "train", Snapshots[snapshotindex] ) trainingsiterations = (dlc_cfg["init_weights"].split(os.sep)[-1]).split("-")[-1] dlc_cfg["num_outputs"] = cfg.get("num_outputs", dlc_cfg.get("num_outputs", 1)) dlc_cfg["batch_size"] = None # load network if TFGPUinference: sess, _, _ = predict.setup_GPUpose_prediction(dlc_cfg) output = ["concat_1"] else: sess, _, _ = predict.setup_pose_prediction(dlc_cfg) if dlc_cfg["location_refinement"]: output = ["Sigmoid", "pose/locref_pred/block4/BiasAdd"] else: output = ["Sigmoid", "pose/part_pred/block4/BiasAdd"] input = tf.get_default_graph().get_operations()[0].name return sess, input, output, dlc_cfg
def evaluate_multianimal_full( config, Shuffles=[1], trainingsetindex=0, plotting=None, show_errors=True, comparisonbodyparts="all", gputouse=None, modelprefix="", c_engine=False, ): from deeplabcut.pose_estimation_tensorflow.nnet import predict from deeplabcut.pose_estimation_tensorflow.nnet import ( predict_multianimal as predictma, ) from deeplabcut.utils import auxiliaryfunctions, auxfun_multianimal import tensorflow as tf if "TF_CUDNN_USE_AUTOTUNE" in os.environ: del os.environ[ "TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training tf.reset_default_graph() os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" # if gputouse is not None: # gpu selectinon os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse) start_path = os.getcwd() ################################################## # Load data... ################################################## cfg = auxiliaryfunctions.read_config(config) if trainingsetindex == "all": TrainingFractions = cfg["TrainingFraction"] else: TrainingFractions = [cfg["TrainingFraction"][trainingsetindex]] # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) Data = pd.read_hdf( os.path.join( cfg["project_path"], str(trainingsetfolder), "CollectedData_" + cfg["scorer"] + ".h5", ), "df_with_missing", ) # Handle data previously annotated on a different platform sep = "/" if "/" in Data.index[0] else "\\" if sep != os.path.sep: Data.index = Data.index.str.replace(sep, os.path.sep) # Get list of body parts to evaluate network for comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, comparisonbodyparts) all_bpts = np.asarray( len(cfg["individuals"]) * cfg["multianimalbodyparts"] + cfg["uniquebodyparts"]) colors = visualization.get_cmap(len(comparisonbodyparts), name=cfg["colormap"]) # Make folder for evaluation auxiliaryfunctions.attempttomakefolder( str(cfg["project_path"] + "/evaluation-results/")) for shuffle in Shuffles: for trainFraction in TrainingFractions: ################################################## # Load and setup CNN part detector ################################################## datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) modelfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetModelFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" # Load meta data ( data, trainIndices, testIndices, trainFraction, ) = auxiliaryfunctions.LoadMetadata( os.path.join(cfg["project_path"], metadatafn)) try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, trainFraction)) # TODO: IMPLEMENT for different batch sizes? dlc_cfg["batch_size"] = 1 # due to differently sized images!!! joints = dlc_cfg["all_joints_names"] # Create folder structure to store results. evaluationfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetEvaluationFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True) # path_train_config = modelfolder / 'train' / 'pose_cfg.yaml' # Check which snapshots are available and sort them by # iterations Snapshots = np.array([ fn.split(".")[0] for fn in os.listdir(os.path.join(str(modelfolder), "train")) if "index" in fn ]) if len(Snapshots) == 0: print( "Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so." % (shuffle, trainFraction)) else: increasing_indices = np.argsort( [int(m.split("-")[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] if cfg["snapshotindex"] == -1: snapindices = [-1] elif cfg["snapshotindex"] == "all": snapindices = range(len(Snapshots)) elif cfg["snapshotindex"] < len(Snapshots): snapindices = [cfg["snapshotindex"]] else: print( "Invalid choice, only -1 (last), any integer up to last, or all (as string)!" ) final_result = [] ################################################## # Compute predictions over images ################################################## for snapindex in snapindices: dlc_cfg["init_weights"] = os.path.join( str(modelfolder), "train", Snapshots[snapindex] ) # setting weights to corresponding snapshot. trainingsiterations = ( dlc_cfg["init_weights"].split(os.sep)[-1] ).split( "-" )[-1] # read how many training siterations that corresponds to. # name for deeplabcut net (based on its parameters) DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction, trainingsiterations, modelprefix=modelprefix, ) print( "Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations, ) ( notanalyzed, resultsfilename, DLCscorer, ) = auxiliaryfunctions.CheckifNotEvaluated( str(evaluationfolder), DLCscorer, DLCscorerlegacy, Snapshots[snapindex], ) if os.path.isfile( resultsfilename.split(".h5")[0] + "_full.pickle"): print("Model already evaluated.", resultsfilename) else: if plotting: foldername = os.path.join( str(evaluationfolder), "LabeledImages_" + DLCscorer + "_" + Snapshots[snapindex], ) auxiliaryfunctions.attempttomakefolder(foldername) # print(dlc_cfg) # Specifying state of model (snapshot / training state) sess, inputs, outputs = predict.setup_pose_prediction( dlc_cfg) PredicteData = {} dist = np.full((len(Data), len(all_bpts)), np.nan) conf = np.full_like(dist, np.nan) distnorm = np.full(len(Data), np.nan) print("Analyzing data...") for imageindex, imagename in tqdm(enumerate( Data.index)): image_path = os.path.join(cfg["project_path"], imagename) image = io.imread(image_path) frame = img_as_ubyte(skimage.color.gray2rgb(image)) GT = Data.iloc[imageindex] df = GT.unstack("coords").reindex( joints, level='bodyparts') # Evaluate PAF edge lengths to calibrate `distnorm` temp = GT.unstack("bodyparts")[joints] xy = temp.values.reshape( (-1, 2, temp.shape[1])).swapaxes(1, 2) edges = xy[:, dlc_cfg["partaffinityfield_graph"]] lengths = np.sum( (edges[:, :, 0] - edges[:, :, 1])**2, axis=2) distnorm[imageindex] = np.nanmax(lengths) # FIXME Is having an empty array vs nan really that necessary?! groundtruthidentity = list( df.index.get_level_values( "individuals").to_numpy().reshape((-1, 1))) groundtruthcoordinates = list( df.values[:, np.newaxis]) for i, coords in enumerate(groundtruthcoordinates): if np.isnan(coords).any(): groundtruthcoordinates[i] = np.empty( (0, 2), dtype=float) groundtruthidentity[i] = np.array( [], dtype=str) PredicteData[imagename] = {} PredicteData[imagename]["index"] = imageindex pred = predictma.get_detectionswithcostsandGT( frame, groundtruthcoordinates, dlc_cfg, sess, inputs, outputs, outall=False, nms_radius=dlc_cfg.nmsradius, det_min_score=dlc_cfg.minconfidence, c_engine=c_engine, ) PredicteData[imagename]["prediction"] = pred PredicteData[imagename]["groundtruth"] = [ groundtruthidentity, groundtruthcoordinates, GT, ] coords_pred = pred["coordinates"][0] probs_pred = pred["confidence"] for bpt, xy_gt in df.groupby(level="bodyparts"): inds_gt = np.flatnonzero( np.all(~np.isnan(xy_gt), axis=1)) n_joint = joints.index(bpt) xy = coords_pred[n_joint] if inds_gt.size and xy.size: # Pick the predictions closest to ground truth, # rather than the ones the model has most confident in d = cdist(xy_gt.iloc[inds_gt], xy) rows, cols = linear_sum_assignment(d) min_dists = d[rows, cols] inds = np.flatnonzero(all_bpts == bpt) sl = imageindex, inds[inds_gt[rows]] dist[sl] = min_dists conf[sl] = probs_pred[n_joint][ cols].squeeze() if plotting: fig = visualization.make_multianimal_labeled_image( frame, groundtruthcoordinates, coords_pred, probs_pred, colors, cfg["dotsize"], cfg["alphavalue"], cfg["pcutoff"], ) visualization.save_labeled_frame( fig, image_path, foldername, imageindex in trainIndices, ) sess.close() # closes the current tf session # Compute all distance statistics df_dist = pd.DataFrame(dist, columns=df.index) df_conf = pd.DataFrame(conf, columns=df.index) df_joint = pd.concat([df_dist, df_conf], keys=["rmse", "conf"], names=["metrics"], axis=1) df_joint = df_joint.reorder_levels(list( np.roll(df_joint.columns.names, -1)), axis=1) df_joint.sort_index(axis=1, level=["individuals", "bodyparts"], ascending=[True, True], inplace=True) write_path = os.path.join( evaluationfolder, f"dist_{trainingsiterations}.csv") df_joint.to_csv(write_path) # Calculate overall prediction error error = df_joint.xs("rmse", level="metrics", axis=1) mask = df_joint.xs("conf", level="metrics", axis=1) >= cfg["pcutoff"] error_masked = error[mask] error_train = np.nanmean(error.iloc[trainIndices]) error_train_cut = np.nanmean( error_masked.iloc[trainIndices]) error_test = np.nanmean(error.iloc[testIndices]) error_test_cut = np.nanmean( error_masked.iloc[testIndices]) results = [ trainingsiterations, int(100 * trainFraction), shuffle, np.round(error_train, 2), np.round(error_test, 2), cfg["pcutoff"], np.round(error_train_cut, 2), np.round(error_test_cut, 2), ] final_result.append(results) # For OKS/PCK, compute the standard deviation error across all frames sd = df_dist.groupby("bodyparts", axis=1).mean().std(axis=0) sd["distnorm"] = np.sqrt(np.nanmax(distnorm)) sd.to_csv(write_path.replace("dist.csv", "sd.csv")) if show_errors: string = "Results for {} training iterations: {}, shuffle {}:\n" \ "Train error: {} pixels. Test error: {} pixels.\n" \ "With pcutoff of {}:\n" \ "Train error: {} pixels. Test error: {} pixels." print(string.format(*results)) print("##########################################") print( "Average Euclidean distance to GT per individual (in pixels)" ) print( error_masked.groupby( 'individuals', axis=1).mean().mean().to_string()) print( "Average Euclidean distance to GT per bodypart (in pixels)" ) print( error_masked.groupby( 'bodyparts', axis=1).mean().mean().to_string()) PredicteData["metadata"] = { "nms radius": dlc_cfg.nmsradius, "minimal confidence": dlc_cfg.minconfidence, "PAFgraph": dlc_cfg.partaffinityfield_graph, "all_joints": [[i] for i in range(len(dlc_cfg.all_joints))], "all_joints_names": [ dlc_cfg.all_joints_names[i] for i in range(len(dlc_cfg.all_joints)) ], "stride": dlc_cfg.get("stride", 8), } print( "Done and results stored for snapshot: ", Snapshots[snapindex], ) dictionary = { "Scorer": DLCscorer, "DLC-model-config file": dlc_cfg, "trainIndices": trainIndices, "testIndices": testIndices, "trainFraction": trainFraction, } metadata = {"data": dictionary} auxfun_multianimal.SaveFullMultiAnimalData( PredicteData, metadata, resultsfilename) tf.reset_default_graph() if len(final_result ) > 0: # Only append if results were calculated make_results_file(final_result, evaluationfolder, DLCscorer) # returning to intial folder os.chdir(str(start_path))
def analyze_videos(config, videos, videotype='avi', shuffle=1, trainingsetindex=0, gputouse=None, save_as_csv=False, destfolder=None, cropping=None): """ Makes prediction based on a trained network. The index of the trained network is specified by parameters in the config file (in particular the variable 'snapshotindex') You can crop the video (before analysis), by changing 'cropping'=True and setting 'x1','x2','y1','y2' in the config file. The same cropping parameters will then be used for creating the video. Note: you can also pass cropping = [x1,x2,y1,y2] coordinates directly, that then will be used for all videos. You can of course loop over videos & pass specific coordinates for each case. Output: The labels are stored as MultiIndex Pandas Array, which contains the name of the network, body part name, (x, y) label position \n in pixels, and the likelihood for each frame per body part. These arrays are stored in an efficient Hierarchical Data Format (HDF) \n in the same directory, where the video is stored. However, if the flag save_as_csv is set to True, the data can also be exported in \n comma-separated values format (.csv), which in turn can be imported in many programs, such as MATLAB, R, Prism, etc. Parameters ---------- config : string Full path of the config.yaml file as a string. videos : list A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored. videotype: string, optional Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi`` shuffle: int, optional An integer specifying the shuffle index of the training dataset used for training the network. The default is 1. trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None. See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries save_as_csv: bool, optional Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False`` destfolder: string, optional Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this folder also needs to be passed. Examples -------- Windows example for analyzing 1 video >>> deeplabcut.analyze_videos('C:\\myproject\\reaching-task\\config.yaml',['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi']) -------- If you want to analyze only 1 video >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi']) -------- If you want to analyze all videos of type avi in a folder: >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos'],videotype='.avi') -------- If you want to analyze multiple videos >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi']) -------- If you want to analyze multiple videos with shuffle = 2 >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'], shuffle=2) -------- If you want to analyze multiple videos with shuffle = 2 and save results as an additional csv file too >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'], shuffle=2,save_as_csv=True) -------- """ if 'TF_CUDNN_USE_AUTOTUNE' in os.environ: del os.environ[ 'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training if gputouse is not None: #gpu selection os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse) vers = (tf.__version__).split('.') if int(vers[0]) == 1 and int(vers[1]) > 12: TF = tf.compat.v1 else: TF = tf TF.reset_default_graph() start_path = os.getcwd( ) #record cwd to return to this directory in the end cfg = auxiliaryfunctions.read_config(config) if cropping is not None: cfg['cropping'] = True cfg['x1'], cfg['x2'], cfg['y1'], cfg['y2'] = cropping print("Overwriting cropping parameters:", cropping) print( "These are used for all videos, but won't be save to the cfg file." ) trainFraction = cfg['TrainingFraction'][trainingsetindex] modelfolder = os.path.join( cfg["project_path"], str(auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg))) path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml' try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, trainFraction)) # Check which snapshots are available and sort them by # iterations try: Snapshots = np.array([ fn.split('.')[0] for fn in os.listdir(os.path.join(modelfolder, 'train')) if "index" in fn ]) except FileNotFoundError: raise FileNotFoundError( "Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s." % (shuffle, shuffle)) if cfg['snapshotindex'] == 'all': print( "Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!" ) snapshotindex = -1 else: snapshotindex = cfg['snapshotindex'] increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder) ################################################## # Load and setup CNN part detector ################################################## # Check if data already was generated: dlc_cfg['init_weights'] = os.path.join(modelfolder, 'train', Snapshots[snapshotindex]) trainingsiterations = (dlc_cfg['init_weights'].split( os.sep)[-1]).split('-')[-1] #update batchsize (based on parameters in config.yaml) dlc_cfg['batch_size'] = cfg['batch_size'] # update number of outputs dlc_cfg['num_outputs'] = cfg.get('num_outputs', 1) print('num_outputs = ', dlc_cfg['num_outputs']) # Name for scorer: DLCscorer = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction, trainingsiterations=trainingsiterations) sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg) xyz_labs_orig = ['x', 'y', 'likelihood'] suffix = [str(s + 1) for s in range(dlc_cfg['num_outputs'])] suffix[0] = '' # first one has empty suffix for backwards compatibility xyz_labs = [x + s for s in suffix for x in xyz_labs_orig] pdindex = pd.MultiIndex.from_product( [[DLCscorer], dlc_cfg['all_joints_names'], xyz_labs], names=['scorer', 'bodyparts', 'coords']) ################################################## # Datafolder ################################################## Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype) if len(Videos) > 0: #looping over videos for video in Videos: AnalyzeVideo(video, DLCscorer, trainFraction, cfg, dlc_cfg, sess, inputs, outputs, pdindex, save_as_csv, destfolder) os.chdir(str(start_path)) print( "The videos are analyzed. Now your research can truly start! \n You can create labeled videos with 'create_labeled_video'." ) print( "If the tracking is not satisfactory for some videos, consider expanding the training set. You can use the function 'extract_outlier_frames' to extract any outlier frames!" ) else: print("No video was found in the path/ or single video with path:", videos) print( "Perhaps the videotype is distinct from the videos in the path, I was looking for:", videotype) return DLCscorer
def evaluate_network( config, Shuffles=[1], trainingsetindex=0, plotting=None, show_errors=True, comparisonbodyparts="all", gputouse=None, rescale=False, modelprefix="", c_engine=False, ): """ Evaluates the network based on the saved models at different stages of the training network.\n The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'. Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models. Parameters ---------- config : string Full path of the config.yaml file as a string. Shuffles: list, optional List of integers specifying the shuffle indices of the training dataset. The default is [1] trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This variable can also be set to "all". plotting: bool, optional Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False`` show_errors: bool, optional Display train and test errors. The default is `True`` comparisonbodyparts: list of bodyparts, Default is "all". The average error will be computed for those body parts only (Has to be a subset of the body parts). gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None. See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries rescale: bool, default False Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the original size! Examples -------- If you do not want to plot >>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml', Shuffles=[1]) -------- If you want to plot >>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',Shuffles=[1],True) """ import os start_path = os.getcwd() from deeplabcut.utils import auxiliaryfunctions cfg = auxiliaryfunctions.read_config(config) if cfg.get("multianimalproject", False): from deeplabcut.pose_estimation_tensorflow.evaluate_multianimal import ( evaluate_multianimal_full, ) # TODO: Make this code not so redundant! evaluate_multianimal_full( config, Shuffles, trainingsetindex, plotting, show_errors, comparisonbodyparts, gputouse, modelprefix, c_engine=c_engine, ) else: from deeplabcut.utils.auxfun_videos import imread, imresize from deeplabcut.pose_estimation_tensorflow.nnet import predict from deeplabcut.pose_estimation_tensorflow.config import load_config from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import ( data_to_input, ) from deeplabcut.utils import auxiliaryfunctions import tensorflow as tf if "TF_CUDNN_USE_AUTOTUNE" in os.environ: del os.environ[ "TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training tf.reset_default_graph() os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" # # tf.logging.set_verbosity(tf.logging.WARN) start_path = os.getcwd() # Read file path for pose_config file. >> pass it on cfg = auxiliaryfunctions.read_config(config) if gputouse is not None: # gpu selectinon os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse) if trainingsetindex == "all": TrainingFractions = cfg["TrainingFraction"] else: if (trainingsetindex < len(cfg["TrainingFraction"]) and trainingsetindex >= 0): TrainingFractions = [ cfg["TrainingFraction"][int(trainingsetindex)] ] else: raise Exception( "Please check the trainingsetindex! ", trainingsetindex, " should be an integer from 0 .. ", int(len(cfg["TrainingFraction"]) - 1), ) # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) Data = pd.read_hdf( os.path.join( cfg["project_path"], str(trainingsetfolder), "CollectedData_" + cfg["scorer"] + ".h5", ), "df_with_missing", ) # Get list of body parts to evaluate network for comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, comparisonbodyparts) # Make folder for evaluation auxiliaryfunctions.attempttomakefolder( str(cfg["project_path"] + "/evaluation-results/")) for shuffle in Shuffles: for trainFraction in TrainingFractions: ################################################## # Load and setup CNN part detector ################################################## datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) modelfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetModelFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" # Load meta data ( data, trainIndices, testIndices, trainFraction, ) = auxiliaryfunctions.LoadMetadata( os.path.join(cfg["project_path"], metadatafn)) try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, trainFraction)) # change batch size, if it was edited during analysis! dlc_cfg[ "batch_size"] = 1 # in case this was edited for analysis. # Create folder structure to store results. evaluationfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetEvaluationFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True) # path_train_config = modelfolder / 'train' / 'pose_cfg.yaml' # Check which snapshots are available and sort them by # iterations Snapshots = np.array([ fn.split(".")[0] for fn in os.listdir( os.path.join(str(modelfolder), "train")) if "index" in fn ]) try: # check if any where found? Snapshots[0] except IndexError: raise FileNotFoundError( "Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so." % (shuffle, trainFraction)) increasing_indices = np.argsort( [int(m.split("-")[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] if cfg["snapshotindex"] == -1: snapindices = [-1] elif cfg["snapshotindex"] == "all": snapindices = range(len(Snapshots)) elif cfg["snapshotindex"] < len(Snapshots): snapindices = [cfg["snapshotindex"]] else: raise ValueError( "Invalid choice, only -1 (last), any integer up to last, or all (as string)!" ) final_result = [] ########################### RESCALING (to global scale) if rescale == True: scale = dlc_cfg["global_scale"] Data = (pd.read_hdf( os.path.join( cfg["project_path"], str(trainingsetfolder), "CollectedData_" + cfg["scorer"] + ".h5", ), "df_with_missing", ) * scale) else: scale = 1 ################################################## # Compute predictions over images ################################################## for snapindex in snapindices: dlc_cfg["init_weights"] = os.path.join( str(modelfolder), "train", Snapshots[snapindex] ) # setting weights to corresponding snapshot. trainingsiterations = ( dlc_cfg["init_weights"].split(os.sep)[-1] ).split( "-" )[-1] # read how many training siterations that corresponds to. # Name for deeplabcut net (based on its parameters) DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction, trainingsiterations, modelprefix=modelprefix, ) print( "Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations, ) ( notanalyzed, resultsfilename, DLCscorer, ) = auxiliaryfunctions.CheckifNotEvaluated( str(evaluationfolder), DLCscorer, DLCscorerlegacy, Snapshots[snapindex], ) if notanalyzed: # Specifying state of model (snapshot / training state) sess, inputs, outputs = predict.setup_pose_prediction( dlc_cfg) Numimages = len(Data.index) PredicteData = np.zeros( (Numimages, 3 * len(dlc_cfg["all_joints_names"]))) print("Analyzing data...") for imageindex, imagename in tqdm(enumerate( Data.index)): image = imread(os.path.join( cfg["project_path"], imagename), mode="RGB") if scale != 1: image = imresize(image, scale) image_batch = data_to_input(image) # Compute prediction with the CNN outputs_np = sess.run( outputs, feed_dict={inputs: image_batch}) scmap, locref = predict.extract_cnn_output( outputs_np, dlc_cfg) # Extract maximum scoring location from the heatmap, assume 1 person pose = predict.argmax_pose_predict( scmap, locref, dlc_cfg.stride) PredicteData[imageindex, :] = ( pose.flatten() ) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts! sess.close() # closes the current tf session index = pd.MultiIndex.from_product( [ [DLCscorer], dlc_cfg["all_joints_names"], ["x", "y", "likelihood"], ], names=["scorer", "bodyparts", "coords"], ) # Saving results DataMachine = pd.DataFrame(PredicteData, columns=index, index=Data.index.values) DataMachine.to_hdf(resultsfilename, "df_with_missing", format="table", mode="w") print( "Done and results stored for snapshot: ", Snapshots[snapindex], ) DataCombined = pd.concat([Data.T, DataMachine.T], axis=0, sort=False).T RMSE, RMSEpcutoff = pairwisedistances( DataCombined, cfg["scorer"], DLCscorer, cfg["pcutoff"], comparisonbodyparts, ) testerror = np.nanmean( RMSE.iloc[testIndices].values.flatten()) trainerror = np.nanmean( RMSE.iloc[trainIndices].values.flatten()) testerrorpcutoff = np.nanmean( RMSEpcutoff.iloc[testIndices].values.flatten()) trainerrorpcutoff = np.nanmean( RMSEpcutoff.iloc[trainIndices].values.flatten()) results = [ trainingsiterations, int(100 * trainFraction), shuffle, np.round(trainerror, 2), np.round(testerror, 2), cfg["pcutoff"], np.round(trainerrorpcutoff, 2), np.round(testerrorpcutoff, 2), ] final_result.append(results) if show_errors == True: print( "Results for", trainingsiterations, " training iterations:", int(100 * trainFraction), shuffle, "train error:", np.round(trainerror, 2), "pixels. Test error:", np.round(testerror, 2), " pixels.", ) print( "With pcutoff of", cfg["pcutoff"], " train error:", np.round(trainerrorpcutoff, 2), "pixels. Test error:", np.round(testerrorpcutoff, 2), "pixels", ) if scale != 1: print( "The predictions have been calculated for rescaled images (and rescaled ground truth). Scale:", scale, ) print( "Thereby, the errors are given by the average distances between the labels by DLC and the scorer." ) if plotting == True: print("Plotting...") foldername = os.path.join( str(evaluationfolder), "LabeledImages_" + DLCscorer + "_" + Snapshots[snapindex], ) auxiliaryfunctions.attempttomakefolder(foldername) Plotting( cfg, comparisonbodyparts, DLCscorer, trainIndices, DataCombined * 1.0 / scale, foldername, ) # Rescaling coordinates to have figure in original size! tf.reset_default_graph() # print(final_result) else: DataMachine = pd.read_hdf(resultsfilename, "df_with_missing") if plotting == True: DataCombined = pd.concat([Data.T, DataMachine.T], axis=0, sort=False).T print( "Plotting...(attention scale might be inconsistent in comparison to when data was analyzed; i.e. if you used rescale)" ) foldername = os.path.join( str(evaluationfolder), "LabeledImages_" + DLCscorer + "_" + Snapshots[snapindex], ) auxiliaryfunctions.attempttomakefolder(foldername) Plotting( cfg, comparisonbodyparts, DLCscorer, trainIndices, DataCombined * 1.0 / scale, foldername, ) if len(final_result ) > 0: # Only append if results were calculated make_results_file(final_result, evaluationfolder, DLCscorer) print( "The network is evaluated and the results are stored in the subdirectory 'evaluation_results'." ) print( "If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos." ) print( "Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)" ) # returning to intial folder os.chdir(str(start_path))
def extract_maps( config, shuffle=0, trainingsetindex=0, gputouse=None, rescale=False, Indices=None, modelprefix="", ): """ Extracts the scoremap, locref, partaffinityfields (if available). Returns a dictionary indexed by: trainingsetfraction, snapshotindex, and imageindex for those keys, each item contains: (image,scmap,locref,paf,bpt names,partaffinity graph, imagename, True/False if this image was in trainingset) ---------- config : string Full path of the config.yaml file as a string. shuffle: integer integers specifying shuffle index of the training dataset. The default is 0. trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This variable can also be set to "all". rescale: bool, default False Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the original size! Examples -------- If you want to extract the data for image 0 and 103 (of the training set) for model trained with shuffle 0. >>> deeplabcut.extract_maps(configfile,0,Indices=[0,103]) """ from deeplabcut.utils.auxfun_videos import imread, imresize from deeplabcut.pose_estimation_tensorflow.nnet import predict from deeplabcut.pose_estimation_tensorflow.nnet import ( predict_multianimal as predictma, ) from deeplabcut.pose_estimation_tensorflow.config import load_config from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input from deeplabcut.utils import auxiliaryfunctions from tqdm import tqdm import tensorflow as tf vers = (tf.__version__).split(".") if int(vers[0]) == 1 and int(vers[1]) > 12: TF = tf.compat.v1 else: TF = tf import pandas as pd from pathlib import Path import numpy as np TF.reset_default_graph() os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" # # tf.logging.set_verbosity(tf.logging.WARN) start_path = os.getcwd() # Read file path for pose_config file. >> pass it on cfg = auxiliaryfunctions.read_config(config) if gputouse is not None: # gpu selectinon os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse) if trainingsetindex == "all": TrainingFractions = cfg["TrainingFraction"] else: if trainingsetindex < len( cfg["TrainingFraction"]) and trainingsetindex >= 0: TrainingFractions = [ cfg["TrainingFraction"][int(trainingsetindex)] ] else: raise Exception( "Please check the trainingsetindex! ", trainingsetindex, " should be an integer from 0 .. ", int(len(cfg["TrainingFraction"]) - 1), ) # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) Data = pd.read_hdf( os.path.join( cfg["project_path"], str(trainingsetfolder), "CollectedData_" + cfg["scorer"] + ".h5", ), "df_with_missing", ) # Make folder for evaluation auxiliaryfunctions.attempttomakefolder( str(cfg["project_path"] + "/evaluation-results/")) Maps = {} for trainFraction in TrainingFractions: Maps[trainFraction] = {} ################################################## # Load and setup CNN part detector ################################################## datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) modelfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" # Load meta data ( data, trainIndices, testIndices, trainFraction, ) = auxiliaryfunctions.LoadMetadata( os.path.join(cfg["project_path"], metadatafn)) try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, trainFraction)) # change batch size, if it was edited during analysis! dlc_cfg["batch_size"] = 1 # in case this was edited for analysis. # Create folder structure to store results. evaluationfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetEvaluationFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True) # path_train_config = modelfolder / 'train' / 'pose_cfg.yaml' # Check which snapshots are available and sort them by # iterations Snapshots = np.array([ fn.split(".")[0] for fn in os.listdir(os.path.join(str(modelfolder), "train")) if "index" in fn ]) try: # check if any where found? Snapshots[0] except IndexError: raise FileNotFoundError( "Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so." % (shuffle, trainFraction)) increasing_indices = np.argsort( [int(m.split("-")[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] if cfg["snapshotindex"] == -1: snapindices = [-1] elif cfg["snapshotindex"] == "all": snapindices = range(len(Snapshots)) elif cfg["snapshotindex"] < len(Snapshots): snapindices = [cfg["snapshotindex"]] else: print( "Invalid choice, only -1 (last), any integer up to last, or all (as string)!" ) ########################### RESCALING (to global scale) scale = dlc_cfg["global_scale"] if rescale else 1 Data *= scale bptnames = [ dlc_cfg["all_joints_names"][i] for i in range(len(dlc_cfg["all_joints"])) ] for snapindex in snapindices: dlc_cfg["init_weights"] = os.path.join( str(modelfolder), "train", Snapshots[snapindex] ) # setting weights to corresponding snapshot. trainingsiterations = ( dlc_cfg["init_weights"].split(os.sep)[-1] ).split("-")[ -1] # read how many training siterations that corresponds to. # Name for deeplabcut net (based on its parameters) # DLCscorer,DLCscorerlegacy = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations) # notanalyzed, resultsfilename, DLCscorer=auxiliaryfunctions.CheckifNotEvaluated(str(evaluationfolder),DLCscorer,DLCscorerlegacy,Snapshots[snapindex]) # print("Extracting maps for ", DLCscorer, " with # of trainingiterations:", trainingsiterations) # if notanalyzed: #this only applies to ask if h5 exists... # Specifying state of model (snapshot / training state) sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg) Numimages = len(Data.index) PredicteData = np.zeros( (Numimages, 3 * len(dlc_cfg["all_joints_names"]))) print("Analyzing data...") if Indices is None: Indices = enumerate(Data.index) else: Ind = [Data.index[j] for j in Indices] Indices = enumerate(Ind) DATA = {} for imageindex, imagename in tqdm(Indices): image = imread(os.path.join(cfg["project_path"], imagename), mode="RGB") if scale != 1: image = imresize(image, scale) image_batch = data_to_input(image) # Compute prediction with the CNN outputs_np = sess.run(outputs, feed_dict={inputs: image_batch}) if cfg.get("multianimalproject", False): scmap, locref, paf = predictma.extract_cnn_output( outputs_np, dlc_cfg) pagraph = dlc_cfg["partaffinityfield_graph"] else: scmap, locref = predict.extract_cnn_output( outputs_np, dlc_cfg) paf = None pagraph = [] if imageindex in testIndices: trainingfram = False else: trainingfram = True DATA[imageindex] = [ image, scmap, locref, paf, bptnames, pagraph, imagename, trainingfram, ] Maps[trainFraction][Snapshots[snapindex]] = DATA os.chdir(str(start_path)) return Maps
def evaluate_multiview_network(config,videos,projection_matrices,multiview_step,snapshot_index=None,Shuffles=[1],plotting = None,show_errors = True,comparisonbodyparts="all",gputouse=None): """ Evaluates the network based on the saved models at different stages of the training network.\n The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'. Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models. Parameters ---------- config : string Full path of the config.yaml file as a string. videos: list of strings Name of each video, one per viewpoint. Must be in the same order that it was in for training projection_matrices: list of arrays Projection matrix for each viewpoint. Each is a 3x4 array multiview_step: 1 or 2. Indicates whether network was trained with train_multiview_network_step_1 or train_multiview_network_step_2 Shuffles: list, optional List of integers specifying the shuffle indices of the training dataset. The default is [1] plotting: bool, optional Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False`` show_errors: bool, optional Display train and test errors. The default is `True`` comparisonbodyparts: list of bodyparts, Default is "all". The average error will be computed for those body parts only (Has to be a subset of the body parts). gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None. See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries Examples -------- If you do not want to plot >>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml', shuffle=[1]) -------- If you want to plot >>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',shuffle=[1],True) """ import os from skimage import io import skimage.color from deeplabcut.pose_estimation_tensorflow.nnet import predict as ptf_predict from deeplabcut.pose_estimation_tensorflow.config import load_config from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input from deeplabcut.utils import auxiliaryfunctions, visualization import tensorflow as tf if 'TF_CUDNN_USE_AUTOTUNE' in os.environ: del os.environ['TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training tf.reset_default_graph() os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # # tf.logging.set_verbosity(tf.logging.WARN) start_path=os.getcwd() # Read file path for pose_config file. >> pass it on cfg = auxiliaryfunctions.read_config(config) if gputouse is not None: #gpu selectinon os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse) # Loading human annotatated data trainingsetfolder=auxiliaryfunctions.GetTrainingSetFolder(cfg) Datas = [pd.read_hdf(os.path.join(cfg['project_path'], 'labeled-data', video, 'CollectedData_'+cfg['scorer']+'.h5'), 'df_with_missing') for video in videos] # Get list of body parts to evaluate network for comparisonbodyparts=auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(cfg,comparisonbodyparts) # Make folder for evaluation auxiliaryfunctions.attempttomakefolder(str(cfg["project_path"]+"/evaluation-results/")) for shuffle in Shuffles: for trainFraction in cfg["TrainingFraction"]: ################################################## # Load and setup CNN part detector ################################################## datafn,metadatafn=auxiliaryfunctions.GetDataandMetaDataFilenames(trainingsetfolder,trainFraction,shuffle,cfg) modelfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg))) path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml' # Load meta data metadatas = [] for video in videos: m = ('-'+video).join(os.path.splitext(metadatafn)) data, trainIndices, testIndices, trainFraction=auxiliaryfunctions.LoadMetadata(os.path.join(cfg["project_path"],m)) metadatas.append(data) try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError("It seems the model for shuffle %s and trainFraction %s does not exist."%(shuffle,trainFraction)) #change batch size, if it was edited during analysis! dlc_cfg['batch_size']=1 #in case this was edited for analysis. #Create folder structure to store results. evaluationfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetEvaluationFolder(trainFraction,shuffle,cfg))) auxiliaryfunctions.attempttomakefolder(evaluationfolder,recursive=True) #path_train_config = modelfolder / 'train' / 'pose_cfg.yaml' dlc_cfg.multiview_step = multiview_step dlc_cfg.projection_matrices = projection_matrices # Check which snapshots are available and sort them by # iterations Snapshots = np.array([fn.split('.')[0]for fn in os.listdir(os.path.join(str(modelfolder), 'train'))if "index" in fn]) try: #check if any where found? Snapshots[0] except IndexError: raise FileNotFoundError("Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."%(shuffle,trainFraction)) increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] if snapshot_index is not None: snapindices = [i for i in range(len(Snapshots)) if int(Snapshots[i].split('-')[1].split('.')[0])==snapshot_index] elif cfg["snapshotindex"] == -1: snapindices = [-1] elif cfg["snapshotindex"] == "all": snapindices = range(len(Snapshots)) elif cfg["snapshotindex"]<len(Snapshots): snapindices=[cfg["snapshotindex"]] else: print("Invalid choice, only -1 (last), any integer up to last, or all (as string)!") final_result=[] ################################################## # Compute predictions over images ################################################## for snapindex in snapindices: dlc_cfg['init_weights'] = os.path.join(str(modelfolder),'train',Snapshots[snapindex]) #setting weights to corresponding snapshot. trainingsiterations = (dlc_cfg['init_weights'].split(os.sep)[-1]).split('-')[-1] #read how many training siterations that corresponds to. #name for deeplabcut net (based on its parameters) DLCscorer = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations) print("Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations) resultsfilename=os.path.join(str(evaluationfolder),DLCscorer + '-' + Snapshots[snapindex]+ '.h5') try: DataMachine = pd.read_hdf(resultsfilename,'df_with_missing') print("This net has already been evaluated!") except FileNotFoundError: # Specifying state of model (snapshot / training state) sess, inputs, outputs = ptf_predict.setup_pose_prediction(dlc_cfg) Numimages = len(Datas[0].index) PredicteDatas = np.zeros((Numimages,len(Datas), 3 * len(dlc_cfg['all_joints_names']))) imagesizes = [] print("Analyzing data...") if multiview_step == 1: for imageindex in tqdm(range(len(Datas[0].index))): imagenames = [Data.index[imageindex] for Data in Datas] images = [io.imread(os.path.join(cfg['project_path'],imagename),mode='RGB') for imagename in imagenames] images = [skimage.color.gray2rgb(image) for image in images] image_batch = images imagesizes.append([image.shape for image in images]) # Compute prediction with the CNN outputs_np = sess.run(outputs, feed_dict={inputs: image_batch}) scmap, locref = ptf_predict.extract_cnn_output(outputs_np, dlc_cfg) # Extract maximum scoring location from the heatmap, assume 1 person pose = ptf_predict.argmax_pose_predict(scmap, locref, dlc_cfg.stride) PredicteDatas[imageindex] = pose.reshape([pose.shape[0], -1]) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts! sess.close() #closes the current tf session index = pd.MultiIndex.from_product( [[DLCscorer], dlc_cfg['all_joints_names'], ['x', 'y', 'likelihood']], names=['scorer', 'bodyparts', 'coords']) # Saving results for i, video in enumerate(videos): print('Evaluating 2D predictions on video %s'%video) Data = Datas[i] DataMachine = pd.DataFrame(PredicteDatas[:,i], columns=index, index=Data.index.values) r = ('-'+video).join(os.path.splitext(resultsfilename)) DataMachine.to_hdf(r,'df_with_missing',format='table',mode='w') print("Done and results stored for snapshot: ", Snapshots[snapindex]) DataCombined = pd.concat([Data.T, DataMachine.T], axis=0).T RMSE,RMSEpcutoff = pairwisedistances(DataCombined, cfg["scorer"], DLCscorer,cfg["pcutoff"],comparisonbodyparts) testerror = np.nanmean(RMSE.iloc[testIndices].values.flatten()) trainerror = np.nanmean(RMSE.iloc[trainIndices].values.flatten()) testerrorpcutoff = np.nanmean(RMSEpcutoff.iloc[testIndices].values.flatten()) trainerrorpcutoff = np.nanmean(RMSEpcutoff.iloc[trainIndices].values.flatten()) results = [trainingsiterations,int(100 * trainFraction),shuffle,np.round(trainerror,2),np.round(testerror,2),cfg["pcutoff"],np.round(trainerrorpcutoff,2), np.round(testerrorpcutoff,2)] final_result.append(results) if show_errors == True: print("Results for",trainingsiterations," training iterations:", int(100 * trainFraction), shuffle, "train error:",np.round(trainerror,2), "pixels. Test error:", np.round(testerror,2)," pixels.") print("With pcutoff of", cfg["pcutoff"]," train error:",np.round(trainerrorpcutoff,2), "pixels. Test error:", np.round(testerrorpcutoff,2), "pixels") print("Thereby, the errors are given by the average distances between the labels by DLC and the scorer.") if plotting == True: print("Plotting...") colors = visualization.get_cmap(len(comparisonbodyparts),name=cfg['colormap']) foldername=os.path.join(str(evaluationfolder),'LabeledImages_' + DLCscorer + '_' + Snapshots[snapindex]+'_'+video) auxiliaryfunctions.attempttomakefolder(foldername) NumFrames=np.size(DataCombined.index) for ind in np.arange(NumFrames): visualization.PlottingandSaveLabeledFrame(DataCombined,ind,trainIndices,cfg,colors,comparisonbodyparts,DLCscorer,foldername) # get predictions in homogeneous pixel coordinates # pixel coordinates have (0,0) in the top-left, and the bottom-right coordinate is (h,w) predictions = PredicteDatas.reshape(Numimages, len(Datas), len(dlc_cfg['all_joints_names']), 3) scores = np.copy(predictions[:,:,:,2]) predictions[:,:,:,2] = 1.0 # homogeneous coordinates; (x,y,1). Top-left corner is (-width/2, -height/2, 1); Bottom-right corner is opposite. Shape is num_images x num_views x num_joints x 3 num_ims, num_views, num_joints, _ = predictions.shape # get labels in homogeneous pixel coordinates labels = np.array([Data.values.reshape(num_ims, num_joints, 2) for Data in Datas]) # num_views x num_ims x num_joints x (x,y) labels = np.transpose(labels, [1, 2, 0, 3]) # num_ims x num_joints x num_views x (x,y) labels = np.concatenate([labels, np.ones([num_ims, num_joints, num_views, 1])], axis=3) # solve linear system to get labels in 3D # helpful explanation of equation found on pg 5 here: https://hal.inria.fr/inria-00524401/PDF/Sturm-cvpr05.pdf labs = labels.reshape([num_ims * num_joints, num_views, 3]).astype(np.float) confidences = ~np.isnan(np.sum(labs, axis=2)) valid = np.sum(~np.isnan(np.sum(labs, axis=2)), axis=1) >= 2 labs[~confidences] = 0 labels3d = project_3d(projection_matrices, labs, confidences=confidences) labels3d[~valid] = np.nan labels3d = labels3d.reshape([num_ims, num_joints, 3]) # solve linear system to get 3D predictions preds = np.transpose(predictions, [0, 2, 1, 3]) # num_ims x num_joints x num_views x 3 preds = preds.reshape([num_ims*num_joints, num_views, 3]) preds3d = project_3d(projection_matrices, preds) preds3d = preds3d.reshape([num_ims, num_joints, 3]) # try it with confidence weighting scores = np.transpose(scores, [0, 2, 1]) # num_images x num_joints x num_views scores = np.reshape(scores, [num_ims*num_joints, num_views]) preds3d_weighted = project_3d(projection_matrices, preds, confidences=scores) preds3d_weighted = preds3d_weighted.reshape([num_ims, num_joints, 3]) # try it with the pcutoff scores2 = np.copy(scores) scores2[scores2 < cfg["pcutoff"]] = 0 preds3d_weighted_cutoff = project_3d(projection_matrices, preds, confidences=scores2) preds3d_weighted_cutoff = preds3d_weighted_cutoff.reshape([num_ims, num_joints, 3]) print("\n\n3D errors:") RMSE_train = np.nanmean(np.nansum((preds3d[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5) RMSE_test = np.nanmean(np.nansum((preds3d[testIndices] - labels3d[testIndices])**2, axis=2)**0.5) RMSE_train_weighted = np.nanmean(np.nansum((preds3d_weighted[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5) RMSE_test_weighted = np.nanmean(np.nansum((preds3d_weighted[testIndices] - labels3d[testIndices])**2, axis=2)**0.5) RMSE_train_weighted_cutoff = np.nanmean(np.nansum((preds3d_weighted_cutoff[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5) RMSE_test_weighted_cutoff = np.nanmean(np.nansum((preds3d_weighted_cutoff[testIndices] - labels3d[testIndices])**2, axis=2)**0.5) print("RMSE train: ", RMSE_train) print("RMSE test: ", RMSE_test) print("RMSE train weighted: ", RMSE_train_weighted) print("RMSE test weighted: ", RMSE_test_weighted) print("RMSE train weighted cutoff: ", RMSE_train_weighted_cutoff) print("RMSE test weighted cutoff: ", RMSE_test_weighted_cutoff) tail = np.nansum((preds3d_weighted - labels3d)**2, axis=2)**0.5 tail = np.sort(tail[~np.isnan(tail)]) tail = tail[-10:] print('10 worst predictions: ', tail) tf.reset_default_graph() elif multiview_step==2: preds3d = [] for imageindex in tqdm(range(len(Datas[0].index))): imagenames = [Data.index[imageindex] for Data in Datas] images = [io.imread(os.path.join(cfg['project_path'],imagename),mode='RGB') for imagename in imagenames] images = [skimage.color.gray2rgb(image) for image in images] image_batch = images # Compute prediction with the CNN outputs_np = sess.run(outputs, feed_dict={inputs: image_batch}) pred_3d = outputs_np[2] preds3d.append(pred_3d) sess.close() #closes the current tf session preds3d = np.array(preds3d) # num_ims x num_joints x (x,y,z) num_ims, num_joints = preds3d.shape[:2] num_views = dlc_cfg.num_views # get labels in homogeneous pixel coordinates labels = np.array([Data.values.reshape(num_ims, num_joints, 2) for Data in Datas]) # num_views x num_ims x num_joints x (x,y) labels = np.transpose(labels, [1, 2, 0, 3]) # num_ims x num_joints x num_views x (x,y) labels = np.concatenate([labels, np.ones([num_ims, num_joints, num_views, 1])], axis=3) # solve linear system to get labels in 3D # helpful explanation of equation found on pg 5 here: https://hal.inria.fr/inria-00524401/PDF/Sturm-cvpr05.pdf labs = labels.reshape([num_ims * num_joints, num_views, 3]).astype(np.float) confidences = ~np.isnan(np.sum(labs, axis=2)) valid = np.sum(~np.isnan(np.sum(labs, axis=2)), axis=1) >= 2 labs[~confidences] = 0 labels3d = project_3d(projection_matrices, labs, confidences=confidences) labels3d[~valid] = np.nan labels3d = labels3d.reshape([num_ims, num_joints, 3]) print("\n\n3D errors (units are determined by projection matrices):") RMSE_train = np.nanmean(np.nansum((preds3d[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5) RMSE_test = np.nanmean(np.nansum((preds3d[testIndices] - labels3d[testIndices])**2, axis=2)**0.5) print("RMSE train: ", RMSE_train) print("RMSE test: ", RMSE_test) tail = np.nansum((preds3d- labels3d)**2, axis=2)**0.5 tail = np.sort(tail[~np.isnan(tail)]) tail = tail[-10:] print('10 worst predictions: ', tail) tf.reset_default_graph() else: print('invalid multiview_step given') return make_results_file(final_result,evaluationfolder,DLCscorer) print("The network is evaluated and the results are stored in the subdirectory 'evaluation_results'.") print("If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos.") print("Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)") #returning to intial folder os.chdir(str(start_path))
def analyze_videos(config,videos, videotype='avi', shuffle=1, trainingsetindex=0, gputouse=None, save_as_csv=False, destfolder=None, batchsize=None, cropping=None,get_nframesfrommetadata=True, TFGPUinference=True,dynamic=(False,.5,10)): """ Makes prediction based on a trained network. The index of the trained network is specified by parameters in the config file (in particular the variable 'snapshotindex') You can crop the video (before analysis), by changing 'cropping'=True and setting 'x1','x2','y1','y2' in the config file. The same cropping parameters will then be used for creating the video. Output: The labels are stored as MultiIndex Pandas Array, which contains the name of the network, body part name, (x, y) label position \n in pixels, and the likelihood for each frame per body part. These arrays are stored in an efficient Hierarchical Data Format (HDF) \n in the same directory, where the video is stored. However, if the flag save_as_csv is set to True, the data can also be exported in \n comma-separated values format (.csv), which in turn can be imported in many programs, such as MATLAB, R, Prism, etc. Parameters ---------- config : string Full path of the config.yaml file as a string. videos : list A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored. videotype: string, optional Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi`` shuffle: int, optional An integer specifying the shuffle index of the training dataset used for training the network. The default is 1. trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None. See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries save_as_csv: bool, optional Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False`` destfolder: string, optional Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this folder also needs to be passed. batchsize: int, default from pose_cfg.yaml Change batch size for inference; if given overwrites value in pose_cfg.yaml TFGPUinference: bool, default: True Perform inference on GPU with Tensorflow code. Introduced in "Pretraining boosts out-of-domain robustness for pose estimation" by Alexander Mathis, Mert Yüksekgönül, Byron Rogers, Matthias Bethge, Mackenzie W. Mathis Source: https://arxiv.org/abs/1909.11229 dynamic: triple containing (state,detectiontreshold,margin) If the state is true, then dynamic cropping will be performed. That means that if an object is detected (i.e. any body part > detectiontreshold), then object boundaries are computed according to the smallest/largest x position and smallest/largest y position of all body parts. This window is expanded by the margin and from then on only the posture within this crop is analyzed (until the object is lost, i.e. <detectiontreshold). The current position is utilized for updating the crop window for the next frame (this is why the margin is important and should be set large enough given the movement of the animal). Examples -------- Windows example for analyzing 1 video >>> deeplabcut.analyze_videos('C:\\myproject\\reaching-task\\config.yaml',['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi']) -------- If you want to analyze only 1 video >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi']) -------- If you want to analyze all videos of type avi in a folder: >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos'],videotype='.avi') -------- If you want to analyze multiple videos >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi']) -------- If you want to analyze multiple videos with shuffle = 2 >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'], shuffle=2) -------- If you want to analyze multiple videos with shuffle = 2 and save results as an additional csv file too >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'], shuffle=2,save_as_csv=True) -------- """ if 'TF_CUDNN_USE_AUTOTUNE' in os.environ: del os.environ['TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training if gputouse is not None: #gpu selection os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse) tf.reset_default_graph() start_path=os.getcwd() #record cwd to return to this directory in the end cfg = auxiliaryfunctions.read_config(config) trainFraction = cfg['TrainingFraction'][trainingsetindex] if cropping is not None: cfg['cropping']=True cfg['x1'],cfg['x2'],cfg['y1'],cfg['y2']=cropping print("Overwriting cropping parameters:", cropping) print("These are used for all videos, but won't be save to the cfg file.") modelfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg))) path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml' try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError("It seems the model for shuffle %s and trainFraction %s does not exist."%(shuffle,trainFraction)) # Check which snapshots are available and sort them by # iterations try: Snapshots = np.array([fn.split('.')[0]for fn in os.listdir(os.path.join(modelfolder , 'train'))if "index" in fn]) except FileNotFoundError: raise FileNotFoundError("Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s."%(shuffle,shuffle)) if cfg['snapshotindex'] == 'all': print("Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!") snapshotindex = -1 else: snapshotindex=cfg['snapshotindex'] increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder) ################################################## # Load and setup CNN part detector ################################################## # Check if data already was generated: dlc_cfg['init_weights'] = os.path.join(modelfolder , 'train', Snapshots[snapshotindex]) trainingsiterations = (dlc_cfg['init_weights'].split(os.sep)[-1]).split('-')[-1] # Update number of output and batchsize dlc_cfg['num_outputs'] = cfg.get('num_outputs', dlc_cfg.get('num_outputs', 1)) if batchsize==None: #update batchsize (based on parameters in config.yaml) dlc_cfg['batch_size']=cfg['batch_size'] else: dlc_cfg['batch_size']=batchsize cfg['batch_size']=batchsize if dynamic[0]: #state=true #(state,detectiontreshold,margin)=dynamic print("Starting analysis in dynamic cropping mode with parameters:", dynamic) dlc_cfg['num_outputs']=1 TFGPUinference=False dlc_cfg['batch_size']=1 print("Switching batchsize to 1, num_outputs (per animal) to 1 and TFGPUinference to False (all these features are not supported in this mode).") # Name for scorer: DLCscorer,DLCscorerlegacy = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations=trainingsiterations) if dlc_cfg['num_outputs']>1: if TFGPUinference: print("Switching to numpy-based keypoint extraction code, as multiple point extraction is not supported by TF code currently.") TFGPUinference=False print("Extracting ", dlc_cfg['num_outputs'], "instances per bodypart") xyz_labs_orig = ['x', 'y', 'likelihood'] suffix = [str(s+1) for s in range(dlc_cfg['num_outputs'])] suffix[0] = '' # first one has empty suffix for backwards compatibility xyz_labs = [x+s for s in suffix for x in xyz_labs_orig] else: xyz_labs = ['x', 'y', 'likelihood'] #sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg) if TFGPUinference: sess, inputs, outputs = predict.setup_GPUpose_prediction(dlc_cfg) else: sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg) pdindex = pd.MultiIndex.from_product([[DLCscorer], dlc_cfg['all_joints_names'], xyz_labs], names=['scorer', 'bodyparts', 'coords']) ################################################## # Datafolder ################################################## Videos=auxiliaryfunctions.Getlistofvideos(videos,videotype) if len(Videos)>0: #looping over videos for video in Videos: DLCscorer=AnalyzeVideo(video,DLCscorer,DLCscorerlegacy,trainFraction,cfg,dlc_cfg,sess,inputs, outputs,pdindex,save_as_csv, destfolder,TFGPUinference,dynamic) os.chdir(str(start_path)) print("The videos are analyzed. Now your research can truly start! \n You can create labeled videos with 'create_labeled_video'.") print("If the tracking is not satisfactory for some videos, consider expanding the training set. You can use the function 'extract_outlier_frames' to extract any outlier frames!") return DLCscorer #note: this is either DLCscorer or DLCscorerlegacy depending on what was used! else: print("No video/s found. Please check your path!") return DLCscorer
def generate_prediction(MAX_PREDICTION_STEPS=1000): """ Generator for predicting image MAX_PREDICTION_STEPS : Number of predictions that should be done before re-initializing """ ################################################## # Clone arguments from deeplabcut.evaluate_network ################################################## config = "/root/DLCROS_ws/Surgical_Tool_Tracking/ForwardPassDeepLabCut/DaVinci-Ambar-2019-10-31/config.yaml" Shuffles = [1] plotting = None show_errors = True comparisonbodyparts = "all" gputouse = None # Suppress scientific notation while printing np.set_printoptions(suppress=True) ################################################## # SETUP everything until image prediction ################################################## if 'TF_CUDNN_USE_AUTOTUNE' in os.environ: del os.environ[ 'TF_CUDNN_USE_AUTOTUNE'] # was potentially set during training vers = tf.__version__.split('.') if int(vers[0]) == 1 and int(vers[1]) > 12: TF = tf.compat.v1 else: TF = tf TF.reset_default_graph() os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' #tf.logging.set_verbosity(tf.logging.WARN) start_path = os.getcwd() # Read file path for pose_config file. >> pass it on cfg = auxiliaryfunctions.read_config(config) if gputouse is not None: # gpu selectinon os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse) ############## # Cloning for-loop variables shuffle = Shuffles[0] trainFraction = cfg["TrainingFraction"][0] ############## trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) # Get list of body parts to evaluate network for comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, comparisonbodyparts) ################################################## # Load and setup CNN part detector ################################################## modelfolder = os.path.join( cfg["project_path"], str(auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg))) path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml' # Load meta data # data, trainIndices, testIndices, trainFraction = auxiliaryfunctions.LoadMetadata( # os.path.join(cfg["project_path"], metadatafn)) try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle s and trainFraction %s does not exist." ) dlc_cfg['batch_size'] = 1 # in case this was edited for analysis. # Check which snapshots are available and sort them by # iterations Snapshots = np.array([ fn.split('.')[0] for fn in os.listdir(os.path.join(str(modelfolder), 'train')) if "index" in fn ]) try: # check if any where found? Snapshots[0] except IndexError: raise FileNotFoundError( "Snapshots not found! It seems the dataset for shuffle and " "trainFraction is not trained.\nPlease train it before evaluating." "\nUse the function 'train_network' to do so.") increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] if cfg["snapshotindex"] == -1: snapindices = [-1] elif cfg["snapshotindex"] == "all": snapindices = range(len(Snapshots)) elif cfg["snapshotindex"] < len(Snapshots): snapindices = [cfg["snapshotindex"]] else: print( "Invalid choice, only -1 (last), any integer up to last, or all (as string)!" ) ################################################## # Compute predictions over image ################################################## for snapindex in snapindices: dlc_cfg['init_weights'] = os.path.join( str(modelfolder), 'train', Snapshots[snapindex]) # setting weights to corresponding snapshot. trainingsiterations = (dlc_cfg['init_weights'].split( os.sep)[-1]).split('-')[ -1] # read how many training siterations that corresponds to. # name for deeplabcut net (based on its parameters) DLCscorer = auxiliaryfunctions.GetScorerName(cfg, shuffle, trainFraction, trainingsiterations) print("Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations) # Specifying state of model (snapshot / training state) sess, inputs, outputs = ptf_predict.setup_pose_prediction(dlc_cfg) # Using GPU for prediction # Specifying state of model (snapshot / training state) # sess, inputs, outputs = ptf_predict.setup_GPUpose_prediction(dlc_cfg) print("Analyzing test image ...") imagename = "img034.png" image = io.imread(imagename, plugin='matplotlib') count = 0 start_time = time.time() while count < MAX_PREDICTION_STEPS: ################################################## # Predict for test image once, and wait for future images to arrive ################################################## print("Calling predict_single_image") pose = predict_single_image(image, sess, inputs, outputs, dlc_cfg) ################################################## # Yield prediction to caller ################################################## image = ( yield pose ) # Receive image here ( Refer https://stackabuse.com/python-generators/ for sending/receiving in generators) step_time = time.time() print(f"time: {step_time-start_time}") start_time = step_time count += 1 if count == MAX_PREDICTION_STEPS: print( f"Restart prediction system, Steps have exceeded {MAX_PREDICTION_STEPS}" ) sess.close() # closes the current tf session TF.reset_default_graph()
################################################## # Compute predictions over image ################################################## for snapindex in snapindices: dlc_cfg['init_weights'] = os.path.join(str(modelfolder), 'train', Snapshots[snapindex]) # setting weights to corresponding snapshot. trainingsiterations = (dlc_cfg['init_weights'].split(os.sep)[-1]).split('-')[ -1] # read how many training siterations that corresponds to. # name for deeplabcut net (based on its parameters) DLCscorer = auxiliaryfunctions.GetScorerName(cfg, shuffle, trainFraction, trainingsiterations) print("Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations) # Specifying state of model (snapshot / training state) sess, inputs, outputs = ptf_predict.setup_pose_prediction(dlc_cfg) # Using GPU for prediction # Specifying state of model (snapshot / training state) # sess, inputs, outputs = ptf_predict.setup_GPUpose_prediction(dlc_cfg) print("Analyzing image...") ################################################## # Wait for image to arrive ################################################## image_arrived = True # dikeo imagename = "img034.png"
def analyze_videos_multiview(config, videos, projection_matrices, multiview_step, output_folder, snapshot_index=None, make_labeled_video=True, shuffle=1, trainingsetindex=0): """ videos: list of strings each string is the path to a video. Each video should pertain to a different view projection_matrices: list of matrices each projection matrix is a 3x4 numpy array multiview_step: int either 1 or 2, denoting whether network was trained via output_folder: string a path to a folder in which to write output make_labeled_video: bool, optional if True, make a video out of the labeled frames and write it to output_folder """ from threading import Thread, Lock from queue import Queue if 'TF_CUDNN_USE_AUTOTUNE' in os.environ: del os.environ[ 'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training if multiview_step != 1 and multiview_step != 2: print('multiview_step should be either 1 or 2') return tf.reset_default_graph() start_path = os.getcwd( ) #record cwd to return to this directory in the end cfg = auxiliaryfunctions.read_config(config) trainFraction = cfg['TrainingFraction'][trainingsetindex] modelfolder = os.path.join( cfg["project_path"], str(auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg))) path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml' try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, trainFraction)) dlc_cfg.multiview_step = multiview_step if multiview_step == 2: dlc_cfg.projection_matrices = projection_matrices # Check which snapshots are available and sort them by # iterations try: Snapshots = np.array([ fn.split('.')[0] for fn in os.listdir(os.path.join(modelfolder, 'train')) if "index" in fn ]) except FileNotFoundError: raise FileNotFoundError( "Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s." % (shuffle, shuffle)) increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] if snapshot_index is not None: snapshotindex = -1 for i in range(len(Snapshots)): if int(Snapshots[i].split('-')[1].split('.')[0]) == snapshot_index: snapshotindex = i elif cfg['snapshotindex'] == 'all': print( "Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!" ) snapshotindex = -1 else: snapshotindex = cfg['snapshotindex'] print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder) ################################################## # Load and setup CNN part detector ################################################## # Check if data already was generated: dlc_cfg['init_weights'] = os.path.join(modelfolder, 'train', Snapshots[snapshotindex]) trainingsiterations = (dlc_cfg['init_weights'].split( os.sep)[-1]).split('-')[-1] dlc_cfg['batch_size'] = 1 # Name for scorer: DLCscorer = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction, trainingsiterations=trainingsiterations) sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg) caps = [cv2.VideoCapture(video) for video in videos] for cap in caps: fps = cap.get( 5 ) #https://docs.opencv.org/2.4/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get nframes = int(cap.get(7)) duration = nframes * 1. / fps size = (int(cap.get(4)), int(cap.get(3))) ny, nx = size print("Duration of video [s]: ", round(duration, 2), ", recorded with ", round(fps, 2), "fps!") print("Overall # of frames: ", nframes, " found with (before cropping) frame dimensions: ", nx, ny) start = time.time() print('Extracting pose') qs = [Queue(maxsize=128) for _ in caps] q = Queue(maxsize=128) def extract_one(cap, q): while cap.isOpened(): ret, frame = cap.read() if ret: # frame = cv2.resize(frame, (482, 256)) q.put(frame) else: break q.put(None) def extract_all(q): while True: res = [qq.get() for qq in qs] if any([frame is None for frame in res]): q.put(None) break q.put(res) ts = [Thread(target=extract_one, args=z) for z in zip(caps, qs)] t = Thread(target=extract_all, args=(q, )) for tt in ts: tt.start() t.start() poses = [] counter = 0 for _ in tqdm(range(nframes)): frames = q.get() if frames is not None: frames = [ cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) for frame in frames ] frames = [img_as_ubyte(frame) for frame in frames] out = sess.run(outputs, feed_dict={inputs: frames}) if multiview_step == 2: poses.append(out[2]) continue scmap, locref = predict.extract_cnn_output(out, dlc_cfg) pose = predict.argmax_pose_predict(scmap, locref, dlc_cfg.stride) poses.append(pose) else: nframes = counter break counter += 1 print('Extracted pose for %d frames' % nframes) if multiview_step == 1: poses = np.array(poses) # nframes x num_views x num_joints x 3 num_views = poses.shape[1] results = poses.reshape([nframes, -1]) pdindex = pd.MultiIndex.from_product( [[DLCscorer], ['view_%d' % i for i in range(poses.shape[1])], dlc_cfg['all_joints_names'], ['x', 'y', 'likelihood']], names=['scorer', 'views', 'bodyparts', 'coords']) results = pd.DataFrame(data=results, columns=pdindex) results.to_hdf(os.path.join(output_folder, '2dposes.h5'), key='results') results.to_csv(os.path.join(output_folder, '2dposes.csv')) poses = np.transpose(poses, [0, 2, 1, 3]).reshape([-1, num_views, 3]) # / [[[482, 256, 1]]] scores = np.copy(poses[:, :, 2]) poses[:, :, 2] = 1 preds3d = project_3d(projection_matrices, poses, confidences=scores) preds3d[~np.isfinite(preds3d)] = 0 preds3d = preds3d.reshape([nframes, -1]) elif multiview_step == 2: preds3d = np.array(poses) # nframes x num_joints x 3 preds3d = preds3d.reshape([nframes, -1]) pdindex = pd.MultiIndex.from_product( [[DLCscorer], dlc_cfg['all_joints_names'], ['x', 'y', 'z']], names=['scorer', 'bodyparts', 'coords']) results = pd.DataFrame(preds3d, columns=pdindex) results.to_hdf(os.path.join(output_folder, '3dposes.h5'), key='results') results.to_csv(os.path.join(output_folder, '3dposes.csv')) if make_labeled_video: print('making 3d video') make_3d_labeled_video(preds3d.reshape([nframes, -1, 3]), output_folder)
def analyze_time_lapse_frames(config,directory,frametype='.png',shuffle=1,trainingsetindex=0,gputouse=None,save_as_csv=False): """ Analyzed all images (of type = frametype) in a folder and stores the output in one file. You can crop the frames (before analysis), by changing 'cropping'=True and setting 'x1','x2','y1','y2' in the config file. Output: The labels are stored as MultiIndex Pandas Array, which contains the name of the network, body part name, (x, y) label position \n in pixels, and the likelihood for each frame per body part. These arrays are stored in an efficient Hierarchical Data Format (HDF) \n in the same directory, where the video is stored. However, if the flag save_as_csv is set to True, the data can also be exported in \n comma-separated values format (.csv), which in turn can be imported in many programs, such as MATLAB, R, Prism, etc. Parameters ---------- config : string Full path of the config.yaml file as a string. directory: string Full path to directory containing the frames that shall be analyzed frametype: string, optional Checks for the file extension of the frames. Only images with this extension are analyzed. The default is ``.png`` shuffle: int, optional An integer specifying the shuffle index of the training dataset used for training the network. The default is 1. trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None. See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries save_as_csv: bool, optional Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False`` Examples -------- If you want to analyze all frames in /analysis/project/timelapseexperiment1 >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml','/analysis/project/timelapseexperiment1') -------- If you want to analyze all frames in /analysis/project/timelapseexperiment1 >>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml','/analysis/project/timelapseexperiment1', frametype='.bmp') -------- Note: for test purposes one can extract all frames from a video with ffmeg, e.g. ffmpeg -i testvideo.avi thumb%04d.png """ if 'TF_CUDNN_USE_AUTOTUNE' in os.environ: del os.environ['TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training tf.reset_default_graph() start_path=os.getcwd() #record cwd to return to this directory in the end cfg = auxiliaryfunctions.read_config(config) trainFraction = cfg['TrainingFraction'][trainingsetindex] modelfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg))) path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml' try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError("It seems the model for shuffle %s and trainFraction %s does not exist."%(shuffle,trainFraction)) # Check which snapshots are available and sort them by # iterations try: Snapshots = np.array([fn.split('.')[0]for fn in os.listdir(os.path.join(modelfolder , 'train'))if "index" in fn]) except FileNotFoundError: raise FileNotFoundError("Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s."%(shuffle,shuffle)) if cfg['snapshotindex'] == 'all': print("Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!") snapshotindex = -1 else: snapshotindex=cfg['snapshotindex'] increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder) ################################################## # Load and setup CNN part detector ################################################## # Check if data already was generated: dlc_cfg['init_weights'] = os.path.join(modelfolder , 'train', Snapshots[snapshotindex]) trainingsiterations = (dlc_cfg['init_weights'].split(os.sep)[-1]).split('-')[-1] #update batchsize (based on parameters in config.yaml) dlc_cfg['batch_size']=cfg['batch_size'] # Name for scorer: DLCscorer = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations=trainingsiterations) sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg) pdindex = pd.MultiIndex.from_product([[DLCscorer], dlc_cfg['all_joints_names'], ['x', 'y', 'likelihood']],names=['scorer', 'bodyparts', 'coords']) if gputouse is not None: #gpu selectinon os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse) ################################################## # Loading the images ################################################## #checks if input is a directory if os.path.isdir(directory)==True: """ Analyzes all the frames in the directory. """ print("Analyzing all frames in the directory: ", directory) os.chdir(directory) framelist=np.sort([fn for fn in os.listdir(os.curdir) if (frametype in fn)]) vname = Path(directory).stem dataname = os.path.join(directory,vname + DLCscorer + '.h5') try: # Attempt to load data... pd.read_hdf(dataname) print("Frames already analyzed!", dataname) except FileNotFoundError: nframes = len(framelist) if nframes>1: start = time.time() PredicteData,nframes,nx,ny=GetPosesofFrames(cfg,dlc_cfg, sess, inputs, outputs,directory,framelist,nframes,dlc_cfg['batch_size']) stop = time.time() if cfg['cropping']==True: coords=[cfg['x1'],cfg['x2'],cfg['y1'],cfg['y2']] else: coords=[0, nx, 0, ny] dictionary = { "start": start, "stop": stop, "run_duration": stop - start, "Scorer": DLCscorer, "config file": dlc_cfg, "batch_size": dlc_cfg["batch_size"], "frame_dimensions": (ny, nx), "nframes": nframes, "cropping": cfg['cropping'], "cropping_parameters": coords } metadata = {'data': dictionary} print("Saving results in %s..." %(directory)) auxiliaryfunctions.SaveData(PredicteData[:nframes,:], metadata, dataname, pdindex, framelist,save_as_csv) print("The folder was analyzed. Now your research can truly start!") print("If the tracking is not satisfactory for some frome, consider expanding the training set.") else: print("No frames were found. Consider changing the path or the frametype.") os.chdir(str(start_path))
def evaluate_network(config, Shuffles=[1], plotting=None, show_errors=True, comparisonbodyparts="all", gputouse=None): """ Evaluates the network based on the saved models at different stages of the training network.\n The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'. Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models. Parameters ---------- config : string Full path of the config.yaml file as a string. Shuffles: list, optional List of integers specifying the shuffle indices of the training dataset. The default is [1] plotting: bool, optional Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False`` show_errors: bool, optional Display train and test errors. The default is `True`` comparisonbodyparts: list of bodyparts, Default is "all". The average error will be computed for those body parts only (Has to be a subset of the body parts). gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None. See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries Examples -------- If you do not want to plot >>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml', shuffle=[1]) -------- If you want to plot >>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',shuffle=[1],True) """ import os from skimage import io import skimage.color from deeplabcut.pose_estimation_tensorflow.nnet import predict as ptf_predict from deeplabcut.pose_estimation_tensorflow.config import load_config from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input from deeplabcut.utils import auxiliaryfunctions, visualization import tensorflow as tf if 'TF_CUDNN_USE_AUTOTUNE' in os.environ: del os.environ[ 'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training vers = (tf.__version__).split('.') if int(vers[0]) == 1 and int(vers[1]) > 12: TF = tf.compat.v1 else: TF = tf TF.reset_default_graph() os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # # tf.logging.set_verbosity(tf.logging.WARN) start_path = os.getcwd() # Read file path for pose_config file. >> pass it on cfg = auxiliaryfunctions.read_config(config) if gputouse is not None: #gpu selectinon os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse) # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) Data = pd.read_hdf( os.path.join(cfg["project_path"], str(trainingsetfolder), 'CollectedData_' + cfg["scorer"] + '.h5'), 'df_with_missing') # Get list of body parts to evaluate network for comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, comparisonbodyparts) # Make folder for evaluation auxiliaryfunctions.attempttomakefolder( str(cfg["project_path"] + "/evaluation-results/")) for shuffle in Shuffles: for trainFraction in cfg["TrainingFraction"]: ################################################## # Load and setup CNN part detector ################################################## datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) modelfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg))) path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml' # Load meta data data, trainIndices, testIndices, trainFraction = auxiliaryfunctions.LoadMetadata( os.path.join(cfg["project_path"], metadatafn)) try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, trainFraction)) #change batch size, if it was edited during analysis! dlc_cfg['batch_size'] = 1 #in case this was edited for analysis. #Create folder structure to store results. evaluationfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetEvaluationFolder( trainFraction, shuffle, cfg))) auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True) #path_train_config = modelfolder / 'train' / 'pose_cfg.yaml' # Check which snapshots are available and sort them by # iterations Snapshots = np.array([ fn.split('.')[0] for fn in os.listdir(os.path.join(str(modelfolder), 'train')) if "index" in fn ]) try: #check if any where found? Snapshots[0] except IndexError: raise FileNotFoundError( "Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so." % (shuffle, trainFraction)) increasing_indices = np.argsort( [int(m.split('-')[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] if cfg["snapshotindex"] == -1: snapindices = [-1] elif cfg["snapshotindex"] == "all": snapindices = range(len(Snapshots)) elif cfg["snapshotindex"] < len(Snapshots): snapindices = [cfg["snapshotindex"]] else: print( "Invalid choice, only -1 (last), any integer up to last, or all (as string)!" ) final_result = [] ################################################## # Compute predictions over images ################################################## for snapindex in snapindices: dlc_cfg['init_weights'] = os.path.join( str(modelfolder), 'train', Snapshots[snapindex] ) #setting weights to corresponding snapshot. trainingsiterations = ( dlc_cfg['init_weights'].split(os.sep)[-1] ).split( '-' )[-1] #read how many training siterations that corresponds to. #name for deeplabcut net (based on its parameters) DLCscorer = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction, trainingsiterations) print("Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations) resultsfilename = os.path.join( str(evaluationfolder), DLCscorer + '-' + Snapshots[snapindex] + '.h5') try: DataMachine = pd.read_hdf(resultsfilename, 'df_with_missing') print("This net has already been evaluated!") except FileNotFoundError: # Specifying state of model (snapshot / training state) sess, inputs, outputs = ptf_predict.setup_pose_prediction( dlc_cfg) Numimages = len(Data.index) PredicteData = np.zeros( (Numimages, 3 * len(dlc_cfg['all_joints_names']))) print("Analyzing data...") for imageindex, imagename in tqdm(enumerate(Data.index)): image = io.imread(os.path.join(cfg['project_path'], imagename), mode='RGB') image = skimage.color.gray2rgb(image) image_batch = data_to_input(image) # Compute prediction with the CNN outputs_np = sess.run(outputs, feed_dict={inputs: image_batch}) scmap, locref = ptf_predict.extract_cnn_output( outputs_np, dlc_cfg) # Extract maximum scoring location from the heatmap, assume 1 person pose = ptf_predict.argmax_pose_predict( scmap, locref, dlc_cfg.stride) PredicteData[imageindex, :] = pose.flatten( ) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts! sess.close() #closes the current tf session index = pd.MultiIndex.from_product( [[DLCscorer], dlc_cfg['all_joints_names'], ['x', 'y', 'likelihood']], names=['scorer', 'bodyparts', 'coords']) # Saving results DataMachine = pd.DataFrame(PredicteData, columns=index, index=Data.index.values) DataMachine.to_hdf(resultsfilename, 'df_with_missing', format='table', mode='w') print("Done and results stored for snapshot: ", Snapshots[snapindex]) DataCombined = pd.concat([Data.T, DataMachine.T], axis=0).T RMSE, RMSEpcutoff = pairwisedistances( DataCombined, cfg["scorer"], DLCscorer, cfg["pcutoff"], comparisonbodyparts) testerror = np.nanmean( RMSE.iloc[testIndices].values.flatten()) trainerror = np.nanmean( RMSE.iloc[trainIndices].values.flatten()) testerrorpcutoff = np.nanmean( RMSEpcutoff.iloc[testIndices].values.flatten()) trainerrorpcutoff = np.nanmean( RMSEpcutoff.iloc[trainIndices].values.flatten()) results = [ trainingsiterations, int(100 * trainFraction), shuffle, np.round(trainerror, 2), np.round(testerror, 2), cfg["pcutoff"], np.round(trainerrorpcutoff, 2), np.round(testerrorpcutoff, 2) ] final_result.append(results) if show_errors == True: print("Results for", trainingsiterations, " training iterations:", int(100 * trainFraction), shuffle, "train error:", np.round(trainerror, 2), "pixels. Test error:", np.round(testerror, 2), " pixels.") print("With pcutoff of", cfg["pcutoff"], " train error:", np.round(trainerrorpcutoff, 2), "pixels. Test error:", np.round(testerrorpcutoff, 2), "pixels") print( "Thereby, the errors are given by the average distances between the labels by DLC and the scorer." ) if plotting == True: print("Plotting...") colors = visualization.get_cmap( len(comparisonbodyparts), name=cfg['colormap']) foldername = os.path.join( str(evaluationfolder), 'LabeledImages_' + DLCscorer + '_' + Snapshots[snapindex]) auxiliaryfunctions.attempttomakefolder(foldername) NumFrames = np.size(DataCombined.index) for ind in np.arange(NumFrames): visualization.PlottingandSaveLabeledFrame( DataCombined, ind, trainIndices, cfg, colors, comparisonbodyparts, DLCscorer, foldername) TF.reset_default_graph() #print(final_result) make_results_file(final_result, evaluationfolder, DLCscorer) print( "The network is evaluated and the results are stored in the subdirectory 'evaluation_results'." ) print( "If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos." ) print( "Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)" ) #returning to intial folder os.chdir(str(start_path))
def evaluate_multianimal_full( config, Shuffles=[1], trainingsetindex=0, plotting=None, show_errors=True, comparisonbodyparts="all", gputouse=None, modelprefix="", c_engine=False, ): """ WIP multi animal project. """ import os from deeplabcut.pose_estimation_tensorflow.nnet import predict from deeplabcut.pose_estimation_tensorflow.nnet import ( predict_multianimal as predictma, ) from deeplabcut.utils import auxiliaryfunctions, auxfun_multianimal import tensorflow as tf if "TF_CUDNN_USE_AUTOTUNE" in os.environ: del os.environ[ "TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training tf.reset_default_graph() os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" # if gputouse is not None: # gpu selectinon os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse) start_path = os.getcwd() ################################################## # Load data... ################################################## cfg = auxiliaryfunctions.read_config(config) if trainingsetindex == "all": TrainingFractions = cfg["TrainingFraction"] else: TrainingFractions = [cfg["TrainingFraction"][trainingsetindex]] # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) Data = pd.read_hdf( os.path.join( cfg["project_path"], str(trainingsetfolder), "CollectedData_" + cfg["scorer"] + ".h5", ), "df_with_missing", ) # Get list of body parts to evaluate network for comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, comparisonbodyparts) colors = visualization.get_cmap(len(comparisonbodyparts), name=cfg["colormap"]) # Make folder for evaluation auxiliaryfunctions.attempttomakefolder( str(cfg["project_path"] + "/evaluation-results/")) for shuffle in Shuffles: for trainFraction in TrainingFractions: ################################################## # Load and setup CNN part detector ################################################## datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) modelfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetModelFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" # Load meta data ( data, trainIndices, testIndices, trainFraction, ) = auxiliaryfunctions.LoadMetadata( os.path.join(cfg["project_path"], metadatafn)) try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, trainFraction)) # TODO: IMPLEMENT for different batch sizes? dlc_cfg["batch_size"] = 1 # due to differently sized images!!! # Create folder structure to store results. evaluationfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetEvaluationFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True) # path_train_config = modelfolder / 'train' / 'pose_cfg.yaml' # Check which snapshots are available and sort them by # iterations Snapshots = np.array([ fn.split(".")[0] for fn in os.listdir(os.path.join(str(modelfolder), "train")) if "index" in fn ]) if len(Snapshots) == 0: print( "Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so." % (shuffle, trainFraction)) else: increasing_indices = np.argsort( [int(m.split("-")[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] if cfg["snapshotindex"] == -1: snapindices = [-1] elif cfg["snapshotindex"] == "all": snapindices = range(len(Snapshots)) elif cfg["snapshotindex"] < len(Snapshots): snapindices = [cfg["snapshotindex"]] else: print( "Invalid choice, only -1 (last), any integer up to last, or all (as string)!" ) ( individuals, uniquebodyparts, multianimalbodyparts, ) = auxfun_multianimal.extractindividualsandbodyparts(cfg) final_result = [] ################################################## # Compute predictions over images ################################################## for snapindex in snapindices: dlc_cfg["init_weights"] = os.path.join( str(modelfolder), "train", Snapshots[snapindex] ) # setting weights to corresponding snapshot. trainingsiterations = ( dlc_cfg["init_weights"].split(os.sep)[-1] ).split( "-" )[-1] # read how many training siterations that corresponds to. # name for deeplabcut net (based on its parameters) DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction, trainingsiterations, modelprefix=modelprefix, ) print( "Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations, ) ( notanalyzed, resultsfilename, DLCscorer, ) = auxiliaryfunctions.CheckifNotEvaluated( str(evaluationfolder), DLCscorer, DLCscorerlegacy, Snapshots[snapindex], ) if os.path.isfile( resultsfilename.split(".h5")[0] + "_full.pickle"): print("Model already evaluated.", resultsfilename) else: if plotting: foldername = os.path.join( str(evaluationfolder), "LabeledImages_" + DLCscorer + "_" + Snapshots[snapindex], ) auxiliaryfunctions.attempttomakefolder(foldername) # print(dlc_cfg) # Specifying state of model (snapshot / training state) sess, inputs, outputs = predict.setup_pose_prediction( dlc_cfg) PredicteData = {} print("Analyzing data...") for imageindex, imagename in tqdm(enumerate( Data.index)): image_path = os.path.join(cfg["project_path"], imagename) image = io.imread(image_path) frame = img_as_ubyte(skimage.color.gray2rgb(image)) GT = Data.iloc[imageindex] # Storing GT data as dictionary, so it can be used for calculating connection costs groundtruthcoordinates = [] groundtruthidentity = [] for bptindex, bpt in enumerate( dlc_cfg["all_joints_names"]): coords = np.zeros([len(individuals), 2 ]) * np.nan identity = [] for prfxindex, prefix in enumerate( individuals): if bpt in uniquebodyparts and prefix == "single": coords[prfxindex, :] = np.array([ GT[cfg["scorer"]][prefix][bpt] ["x"], GT[cfg["scorer"]][prefix][bpt] ["y"], ]) identity.append(prefix) elif (bpt in multianimalbodyparts and prefix != "single"): coords[prfxindex, :] = np.array([ GT[cfg["scorer"]][prefix][bpt] ["x"], GT[cfg["scorer"]][prefix][bpt] ["y"], ]) identity.append(prefix) else: identity.append("nix") groundtruthcoordinates.append( coords[np.isfinite(coords[:, 0]), :]) groundtruthidentity.append( np.array(identity)[np.isfinite(coords[:, 0])]) PredicteData[imagename] = {} PredicteData[imagename]["index"] = imageindex pred = predictma.get_detectionswithcostsandGT( frame, groundtruthcoordinates, dlc_cfg, sess, inputs, outputs, outall=False, nms_radius=dlc_cfg.nmsradius, det_min_score=dlc_cfg.minconfidence, c_engine=c_engine, ) PredicteData[imagename]["prediction"] = pred PredicteData[imagename]["groundtruth"] = [ groundtruthidentity, groundtruthcoordinates, GT, ] if plotting: coords_pred = pred["coordinates"][0] probs_pred = pred["confidence"] fig = visualization.make_multianimal_labeled_image( frame, groundtruthcoordinates, coords_pred, probs_pred, colors, cfg["dotsize"], cfg["alphavalue"], cfg["pcutoff"], ) visualization.save_labeled_frame( fig, image_path, foldername, imageindex in trainIndices, ) sess.close() # closes the current tf session PredicteData["metadata"] = { "nms radius": dlc_cfg.nmsradius, "minimal confidence": dlc_cfg.minconfidence, "PAFgraph": dlc_cfg.partaffinityfield_graph, "all_joints": [[i] for i in range(len(dlc_cfg.all_joints))], "all_joints_names": [ dlc_cfg.all_joints_names[i] for i in range(len(dlc_cfg.all_joints)) ], "stride": dlc_cfg.get("stride", 8), } print( "Done and results stored for snapshot: ", Snapshots[snapindex], ) dictionary = { "Scorer": DLCscorer, "DLC-model-config file": dlc_cfg, "trainIndices": trainIndices, "testIndices": testIndices, "trainFraction": trainFraction, } metadata = {"data": dictionary} auxfun_multianimal.SaveFullMultiAnimalData( PredicteData, metadata, resultsfilename) tf.reset_default_graph() # returning to intial folder os.chdir(str(start_path))
def analyze_stream(config, destfolder, shuffle=1, trainingsetindex=0, gputouse=0, save_as_csv=False, save_frames=True, cropping=None, baseline=True, name="default_animal"): """ Makes prediction based on a trained network. The index of the trained network is specified by parameters in the config file (in particular the variable 'snapshotindex') You can crop the video (before analysis), by changing 'cropping'=True and setting 'x1','x2','y1','y2' in the config file. The same cropping parameters will then be used for creating the video. Output: The labels are stored as MultiIndex Pandas Array, which contains the name of the network, body part name, (x, y) label position \n in pixels, and the likelihood for each frame per body part. These arrays are stored in an efficient Hierarchical Data Format (HDF) \n in the same directory, where the video is stored. However, if the flag save_as_csv is set to True, the data can also be exported in \n comma-separated values format (.csv), which in turn can be imported in many programs, such as MATLAB, R, Prism, etc. Parameters ---------- config : string Full path of the config.yaml file as a string. destfolder : string Full path of the directory to which you want to output data and (optionally) saved frames. shuffle: int, optional An integer specifying the shuffle index of the training dataset used for training the network. The default is 1. trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None. See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries save_as_csv: bool, optional Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False`` save_frames: bool, optional Labels and saves each frame of the stream to the destfolder defined above. The default is ``False``; if provided it must be either ``True`` or ``False`` cropping: bool, optional Selects whether to apply cropping to each frame or not. Not recommended as it increases computational overhead. baseline: bool, optional Selects whether the current trial is a baseline trial (movement tracking but no reinforcement) or a training trial (movement tracking with reinforcement via water reward). If True, current trial is baseline trial; else (e.g. False), current trial is training trial. name: string, optional Pass in the name/subject ID of the animal to be observed in the current trial. This will ensure that the animal is named consistently in data output and trial metadata. Examples -------- If you want to analyze a stream without saving anything >>> deeplabcut.analyze_stream('/analysis/project/reaching-task/config.yaml','/analysis/project/reaching-task/output') -------- If you want to analyze a stream and save just the labelled frames >>> deeplabcut.analyze_stream('/analysis/project/reaching-task/config.yaml','/analysis/project/reaching-task/output', save_frames=True) -------- If you want to analyze a stream and save both the frames and a .csv file with the coordinates of the labels >>> deeplabcut.analyze_stream('/analysis/project/reaching-task/config.yaml','/analysis/project/reaching-task/output', save_as_csv=True, save_frames=True) -------- """ if 'TF_CUDNN_USE_AUTOTUNE' in os.environ: del os.environ[ 'TF_CUDNN_USE_AUTOTUNE'] # was potentially set during training if gputouse is not None: # gpu selection os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse) tf.reset_default_graph() start_path = os.getcwd( ) # record cwd to return to this directory in the end cfg = auxiliaryfunctions.read_config(config) if cropping is not None: cfg['cropping'] = True cfg['x1'], cfg['x2'], cfg['y1'], cfg['y2'] = cropping print("Overwriting cropping parameters:", cropping) print( "These are used for all videos, but won't be save to the cfg file." ) trainFraction = cfg['TrainingFraction'][trainingsetindex] modelfolder = os.path.join( cfg["project_path"], str(auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg))) path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml' try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, trainFraction)) # Check which snapshots are available and sort them by # iterations try: Snapshots = np.array([ fn.split('.')[0] for fn in os.listdir(os.path.join(modelfolder, 'train')) if "index" in fn ]) except FileNotFoundError: raise FileNotFoundError( "Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s." % (shuffle, shuffle)) if cfg['snapshotindex'] == 'all': print( "Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!" ) snapshotindex = -1 else: snapshotindex = cfg['snapshotindex'] increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder) ################################################## # Load and setup CNN part detector ################################################## # Check if data already was generated: dlc_cfg['init_weights'] = os.path.join(modelfolder, 'train', Snapshots[snapshotindex]) trainingsiterations = (dlc_cfg['init_weights'].split( os.sep)[-1]).split('-')[-1] # update batchsize (based on parameters in config.yaml) dlc_cfg['batch_size'] = cfg['batch_size'] # Name for scorer: DLCscorer = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction, trainingsiterations=trainingsiterations) sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg) pdindex = pd.MultiIndex.from_product( [[DLCscorer], dlc_cfg['all_joints_names'], ['x', 'y', 'likelihood']], names=['scorer', 'bodyparts', 'coords']) if gputouse is not None: # gpu selectinon os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse) ################################################## # Set up data buffer and global variables to be used in threads ################################################## global PredicteData PredicteData = np.zeros((50000, 3 * len(dlc_cfg['all_joints_names']))) global led_arr led_arr = np.zeros((50000, 7)) global x_range global y_range global acc_range x_range = list(range(0, (3 * len(dlc_cfg['all_joints_names'])), 3)) y_range = list(range(1, (3 * len(dlc_cfg['all_joints_names'])), 3)) acc_range = list(range(2, (3 * len(dlc_cfg['all_joints_names'])), 3)) global colors colors = [(0, 0, 255), (0, 165, 255), (0, 255, 255), (0, 255, 0), (255, 0, 0), (240, 32, 160), (0, 0, 255), (0, 165, 255)] global empty_count empty_count = 0 global threshold_count AnalyzeStream(DLCscorer, trainFraction, cfg, dlc_cfg, sess, inputs, outputs, pdindex, save_as_csv, save_frames, destfolder, name, baseline)