def make_batch(self, components): data_item = DataItem.from_dict(components[0]) mirror = components[2] part_score_targets = components[3] part_score_weights = components[4] locref_targets = components[5] locref_mask = components[6] im_file = data_item.im_path # logging.debug('image %s', im_file) # print('image: {}'.format(im_file)) # logging.debug('mirror %r', mirror) img = data_item.image # augmented image batch = {Batch.inputs: img} if self.has_gt: batch.update({ Batch.part_score_targets: part_score_targets, Batch.part_score_weights: part_score_weights, Batch.locref_targets: locref_targets, Batch.locref_mask: locref_mask }) batch = {key: data_to_input(data) for (key, data) in batch.items()} batch[Batch.data_item] = data_item return batch
def predict_single_image(image, sess, inputs, outputs, dlc_cfg): """ Returns pose for one single image :param image: :return: """ # assert image = skimage.color.gray2rgb(image) image_batch = data_to_input(image) # Compute prediction with the CNN outputs_np = sess.run(outputs, feed_dict={inputs: image_batch}) scmap, locref = ptf_predict.extract_cnn_output(outputs_np, dlc_cfg) # Extract maximum scoring location from the heatmap, assume 1 person pose = ptf_predict.argmax_pose_predict(scmap, locref, dlc_cfg.stride) return pose
def validate(self, sess, trainingsiterations): final_result = [] # TODO:: Adapt to training/validation loss value for imageindex, imagename in tqdm(enumerate(self.Data.index)): image = io.imread(os.path.join(self.cfg['project_path'], imagename), mode='RGB') image = skimage.color.gray2rgb(image) image_batch = data_to_input(image) # Compute prediction with the CNN #[loss_val, summary] = sess.run([total_loss, merged_summaries], # feed_dict={inputs: image_batch}) outputs_np = sess.run(self.outputs, feed_dict={self.inputs: image_batch}) scmap, locref = ptf_predict.extract_cnn_output( outputs_np, self.pose_cfg) # Extract maximum scoring location from the heatmap, assume 1 person pose = ptf_predict.argmax_pose_predict(scmap, locref, self.pose_cfg.stride) self.PredictedData[imageindex, :] = pose.flatten( ) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts! DLCscorer = 'Predictor' index = pd.MultiIndex.from_product( [[DLCscorer], self.pose_cfg['all_joints_names'], ['x', 'y', 'likelihood']], names=['scorer', 'bodyparts', 'coords']) # Saving results DataMachine = pd.DataFrame(self.PredictedData, columns=index, index=self.Data.index.values) #DataMachine.to_hdf(resultsfilename,'df_with_missing',format='table',mode='w') print("Validated validation set") DataCombined = pd.concat([self.Data.T, DataMachine.T], axis=0).T RMSE, RMSEpcutoff = evaluate.pairwisedistances( DataCombined, self.cfg["scorer"], DLCscorer, self.cfg["pcutoff"], self.comparisonbodyparts) validerror = np.nanmean(RMSE.iloc[self.validIndices].values.flatten()) trainerror = np.nanmean(RMSE.iloc[self.trainIndices].values.flatten()) validerrorpcutoff = np.nanmean( RMSEpcutoff.iloc[self.validIndices].values.flatten()) trainerrorpcutoff = np.nanmean( RMSEpcutoff.iloc[self.trainIndices].values.flatten()) results = [ trainingsiterations, np.round(trainerror, 2), np.round(validerror, 2), self.cfg["pcutoff"], np.round(trainerrorpcutoff, 2), np.round(validerrorpcutoff, 2) ] final_result.append(results) print("Results for", trainingsiterations, " training iterations:", "train error:", np.round(trainerror, 2), "pixels. Validation error:", np.round(validerror, 2), " pixels.") print("With pcutoff of", self.cfg["pcutoff"], " train error:", np.round(trainerrorpcutoff, 2), "pixels. Validation error:", np.round(validerrorpcutoff, 2), "pixels") print( "Thereby, the errors are given by the average distances between the labels by DLC and the scorer." ) # Write to file self.lrf.write("{}, {:.5f}, {:.5f}, {:.5f}, {:.5f}\n".format( trainingsiterations, trainerror, trainerrorpcutoff, validerror, validerrorpcutoff)) self.lrf.flush() return validerror
def evaluate_network( config, Shuffles=[1], trainingsetindex=0, plotting=None, show_errors=True, comparisonbodyparts="all", gputouse=None, rescale=False, modelprefix="", c_engine=False, ): """ Evaluates the network based on the saved models at different stages of the training network.\n The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'. Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models. Parameters ---------- config : string Full path of the config.yaml file as a string. Shuffles: list, optional List of integers specifying the shuffle indices of the training dataset. The default is [1] trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This variable can also be set to "all". plotting: bool, optional Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False`` show_errors: bool, optional Display train and test errors. The default is `True`` comparisonbodyparts: list of bodyparts, Default is "all". The average error will be computed for those body parts only (Has to be a subset of the body parts). gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None. See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries rescale: bool, default False Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the original size! Examples -------- If you do not want to plot >>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml', Shuffles=[1]) -------- If you want to plot >>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',Shuffles=[1],True) """ import os start_path = os.getcwd() from deeplabcut.utils import auxiliaryfunctions cfg = auxiliaryfunctions.read_config(config) if cfg.get("multianimalproject", False): from deeplabcut.pose_estimation_tensorflow.evaluate_multianimal import ( evaluate_multianimal_full, ) # TODO: Make this code not so redundant! evaluate_multianimal_full( config, Shuffles, trainingsetindex, plotting, show_errors, comparisonbodyparts, gputouse, modelprefix, c_engine=c_engine, ) else: from deeplabcut.utils.auxfun_videos import imread, imresize from deeplabcut.pose_estimation_tensorflow.nnet import predict from deeplabcut.pose_estimation_tensorflow.config import load_config from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import ( data_to_input, ) from deeplabcut.utils import auxiliaryfunctions import tensorflow as tf if "TF_CUDNN_USE_AUTOTUNE" in os.environ: del os.environ[ "TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training tf.reset_default_graph() os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" # # tf.logging.set_verbosity(tf.logging.WARN) start_path = os.getcwd() # Read file path for pose_config file. >> pass it on cfg = auxiliaryfunctions.read_config(config) if gputouse is not None: # gpu selectinon os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse) if trainingsetindex == "all": TrainingFractions = cfg["TrainingFraction"] else: if (trainingsetindex < len(cfg["TrainingFraction"]) and trainingsetindex >= 0): TrainingFractions = [ cfg["TrainingFraction"][int(trainingsetindex)] ] else: raise Exception( "Please check the trainingsetindex! ", trainingsetindex, " should be an integer from 0 .. ", int(len(cfg["TrainingFraction"]) - 1), ) # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) Data = pd.read_hdf( os.path.join( cfg["project_path"], str(trainingsetfolder), "CollectedData_" + cfg["scorer"] + ".h5", ), "df_with_missing", ) # Get list of body parts to evaluate network for comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, comparisonbodyparts) # Make folder for evaluation auxiliaryfunctions.attempttomakefolder( str(cfg["project_path"] + "/evaluation-results/")) for shuffle in Shuffles: for trainFraction in TrainingFractions: ################################################## # Load and setup CNN part detector ################################################## datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) modelfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetModelFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" # Load meta data ( data, trainIndices, testIndices, trainFraction, ) = auxiliaryfunctions.LoadMetadata( os.path.join(cfg["project_path"], metadatafn)) try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, trainFraction)) # change batch size, if it was edited during analysis! dlc_cfg[ "batch_size"] = 1 # in case this was edited for analysis. # Create folder structure to store results. evaluationfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetEvaluationFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True) # path_train_config = modelfolder / 'train' / 'pose_cfg.yaml' # Check which snapshots are available and sort them by # iterations Snapshots = np.array([ fn.split(".")[0] for fn in os.listdir( os.path.join(str(modelfolder), "train")) if "index" in fn ]) try: # check if any where found? Snapshots[0] except IndexError: raise FileNotFoundError( "Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so." % (shuffle, trainFraction)) increasing_indices = np.argsort( [int(m.split("-")[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] if cfg["snapshotindex"] == -1: snapindices = [-1] elif cfg["snapshotindex"] == "all": snapindices = range(len(Snapshots)) elif cfg["snapshotindex"] < len(Snapshots): snapindices = [cfg["snapshotindex"]] else: raise ValueError( "Invalid choice, only -1 (last), any integer up to last, or all (as string)!" ) final_result = [] ########################### RESCALING (to global scale) if rescale == True: scale = dlc_cfg["global_scale"] Data = (pd.read_hdf( os.path.join( cfg["project_path"], str(trainingsetfolder), "CollectedData_" + cfg["scorer"] + ".h5", ), "df_with_missing", ) * scale) else: scale = 1 ################################################## # Compute predictions over images ################################################## for snapindex in snapindices: dlc_cfg["init_weights"] = os.path.join( str(modelfolder), "train", Snapshots[snapindex] ) # setting weights to corresponding snapshot. trainingsiterations = ( dlc_cfg["init_weights"].split(os.sep)[-1] ).split( "-" )[-1] # read how many training siterations that corresponds to. # Name for deeplabcut net (based on its parameters) DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction, trainingsiterations, modelprefix=modelprefix, ) print( "Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations, ) ( notanalyzed, resultsfilename, DLCscorer, ) = auxiliaryfunctions.CheckifNotEvaluated( str(evaluationfolder), DLCscorer, DLCscorerlegacy, Snapshots[snapindex], ) if notanalyzed: # Specifying state of model (snapshot / training state) sess, inputs, outputs = predict.setup_pose_prediction( dlc_cfg) Numimages = len(Data.index) PredicteData = np.zeros( (Numimages, 3 * len(dlc_cfg["all_joints_names"]))) print("Analyzing data...") for imageindex, imagename in tqdm(enumerate( Data.index)): image = imread(os.path.join( cfg["project_path"], imagename), mode="RGB") if scale != 1: image = imresize(image, scale) image_batch = data_to_input(image) # Compute prediction with the CNN outputs_np = sess.run( outputs, feed_dict={inputs: image_batch}) scmap, locref = predict.extract_cnn_output( outputs_np, dlc_cfg) # Extract maximum scoring location from the heatmap, assume 1 person pose = predict.argmax_pose_predict( scmap, locref, dlc_cfg.stride) PredicteData[imageindex, :] = ( pose.flatten() ) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts! sess.close() # closes the current tf session index = pd.MultiIndex.from_product( [ [DLCscorer], dlc_cfg["all_joints_names"], ["x", "y", "likelihood"], ], names=["scorer", "bodyparts", "coords"], ) # Saving results DataMachine = pd.DataFrame(PredicteData, columns=index, index=Data.index.values) DataMachine.to_hdf(resultsfilename, "df_with_missing", format="table", mode="w") print( "Done and results stored for snapshot: ", Snapshots[snapindex], ) DataCombined = pd.concat([Data.T, DataMachine.T], axis=0, sort=False).T RMSE, RMSEpcutoff = pairwisedistances( DataCombined, cfg["scorer"], DLCscorer, cfg["pcutoff"], comparisonbodyparts, ) testerror = np.nanmean( RMSE.iloc[testIndices].values.flatten()) trainerror = np.nanmean( RMSE.iloc[trainIndices].values.flatten()) testerrorpcutoff = np.nanmean( RMSEpcutoff.iloc[testIndices].values.flatten()) trainerrorpcutoff = np.nanmean( RMSEpcutoff.iloc[trainIndices].values.flatten()) results = [ trainingsiterations, int(100 * trainFraction), shuffle, np.round(trainerror, 2), np.round(testerror, 2), cfg["pcutoff"], np.round(trainerrorpcutoff, 2), np.round(testerrorpcutoff, 2), ] final_result.append(results) if show_errors == True: print( "Results for", trainingsiterations, " training iterations:", int(100 * trainFraction), shuffle, "train error:", np.round(trainerror, 2), "pixels. Test error:", np.round(testerror, 2), " pixels.", ) print( "With pcutoff of", cfg["pcutoff"], " train error:", np.round(trainerrorpcutoff, 2), "pixels. Test error:", np.round(testerrorpcutoff, 2), "pixels", ) if scale != 1: print( "The predictions have been calculated for rescaled images (and rescaled ground truth). Scale:", scale, ) print( "Thereby, the errors are given by the average distances between the labels by DLC and the scorer." ) if plotting == True: print("Plotting...") foldername = os.path.join( str(evaluationfolder), "LabeledImages_" + DLCscorer + "_" + Snapshots[snapindex], ) auxiliaryfunctions.attempttomakefolder(foldername) Plotting( cfg, comparisonbodyparts, DLCscorer, trainIndices, DataCombined * 1.0 / scale, foldername, ) # Rescaling coordinates to have figure in original size! tf.reset_default_graph() # print(final_result) else: DataMachine = pd.read_hdf(resultsfilename, "df_with_missing") if plotting == True: DataCombined = pd.concat([Data.T, DataMachine.T], axis=0, sort=False).T print( "Plotting...(attention scale might be inconsistent in comparison to when data was analyzed; i.e. if you used rescale)" ) foldername = os.path.join( str(evaluationfolder), "LabeledImages_" + DLCscorer + "_" + Snapshots[snapindex], ) auxiliaryfunctions.attempttomakefolder(foldername) Plotting( cfg, comparisonbodyparts, DLCscorer, trainIndices, DataCombined * 1.0 / scale, foldername, ) if len(final_result ) > 0: # Only append if results were calculated make_results_file(final_result, evaluationfolder, DLCscorer) print( "The network is evaluated and the results are stored in the subdirectory 'evaluation_results'." ) print( "If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos." ) print( "Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)" ) # returning to intial folder os.chdir(str(start_path))
def extract_maps( config, shuffle=0, trainingsetindex=0, gputouse=None, rescale=False, Indices=None, modelprefix="", ): """ Extracts the scoremap, locref, partaffinityfields (if available). Returns a dictionary indexed by: trainingsetfraction, snapshotindex, and imageindex for those keys, each item contains: (image,scmap,locref,paf,bpt names,partaffinity graph, imagename, True/False if this image was in trainingset) ---------- config : string Full path of the config.yaml file as a string. shuffle: integer integers specifying shuffle index of the training dataset. The default is 0. trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This variable can also be set to "all". rescale: bool, default False Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the original size! Examples -------- If you want to extract the data for image 0 and 103 (of the training set) for model trained with shuffle 0. >>> deeplabcut.extract_maps(configfile,0,Indices=[0,103]) """ from deeplabcut.utils.auxfun_videos import imread, imresize from deeplabcut.pose_estimation_tensorflow.nnet import predict from deeplabcut.pose_estimation_tensorflow.nnet import ( predict_multianimal as predictma, ) from deeplabcut.pose_estimation_tensorflow.config import load_config from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input from deeplabcut.utils import auxiliaryfunctions from tqdm import tqdm import tensorflow as tf vers = (tf.__version__).split(".") if int(vers[0]) == 1 and int(vers[1]) > 12: TF = tf.compat.v1 else: TF = tf import pandas as pd from pathlib import Path import numpy as np TF.reset_default_graph() os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" # # tf.logging.set_verbosity(tf.logging.WARN) start_path = os.getcwd() # Read file path for pose_config file. >> pass it on cfg = auxiliaryfunctions.read_config(config) if gputouse is not None: # gpu selectinon os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse) if trainingsetindex == "all": TrainingFractions = cfg["TrainingFraction"] else: if trainingsetindex < len( cfg["TrainingFraction"]) and trainingsetindex >= 0: TrainingFractions = [ cfg["TrainingFraction"][int(trainingsetindex)] ] else: raise Exception( "Please check the trainingsetindex! ", trainingsetindex, " should be an integer from 0 .. ", int(len(cfg["TrainingFraction"]) - 1), ) # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) Data = pd.read_hdf( os.path.join( cfg["project_path"], str(trainingsetfolder), "CollectedData_" + cfg["scorer"] + ".h5", ), "df_with_missing", ) # Make folder for evaluation auxiliaryfunctions.attempttomakefolder( str(cfg["project_path"] + "/evaluation-results/")) Maps = {} for trainFraction in TrainingFractions: Maps[trainFraction] = {} ################################################## # Load and setup CNN part detector ################################################## datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) modelfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" # Load meta data ( data, trainIndices, testIndices, trainFraction, ) = auxiliaryfunctions.LoadMetadata( os.path.join(cfg["project_path"], metadatafn)) try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, trainFraction)) # change batch size, if it was edited during analysis! dlc_cfg["batch_size"] = 1 # in case this was edited for analysis. # Create folder structure to store results. evaluationfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetEvaluationFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True) # path_train_config = modelfolder / 'train' / 'pose_cfg.yaml' # Check which snapshots are available and sort them by # iterations Snapshots = np.array([ fn.split(".")[0] for fn in os.listdir(os.path.join(str(modelfolder), "train")) if "index" in fn ]) try: # check if any where found? Snapshots[0] except IndexError: raise FileNotFoundError( "Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so." % (shuffle, trainFraction)) increasing_indices = np.argsort( [int(m.split("-")[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] if cfg["snapshotindex"] == -1: snapindices = [-1] elif cfg["snapshotindex"] == "all": snapindices = range(len(Snapshots)) elif cfg["snapshotindex"] < len(Snapshots): snapindices = [cfg["snapshotindex"]] else: print( "Invalid choice, only -1 (last), any integer up to last, or all (as string)!" ) ########################### RESCALING (to global scale) scale = dlc_cfg["global_scale"] if rescale else 1 Data *= scale bptnames = [ dlc_cfg["all_joints_names"][i] for i in range(len(dlc_cfg["all_joints"])) ] for snapindex in snapindices: dlc_cfg["init_weights"] = os.path.join( str(modelfolder), "train", Snapshots[snapindex] ) # setting weights to corresponding snapshot. trainingsiterations = ( dlc_cfg["init_weights"].split(os.sep)[-1] ).split("-")[ -1] # read how many training siterations that corresponds to. # Name for deeplabcut net (based on its parameters) # DLCscorer,DLCscorerlegacy = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations) # notanalyzed, resultsfilename, DLCscorer=auxiliaryfunctions.CheckifNotEvaluated(str(evaluationfolder),DLCscorer,DLCscorerlegacy,Snapshots[snapindex]) # print("Extracting maps for ", DLCscorer, " with # of trainingiterations:", trainingsiterations) # if notanalyzed: #this only applies to ask if h5 exists... # Specifying state of model (snapshot / training state) sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg) Numimages = len(Data.index) PredicteData = np.zeros( (Numimages, 3 * len(dlc_cfg["all_joints_names"]))) print("Analyzing data...") if Indices is None: Indices = enumerate(Data.index) else: Ind = [Data.index[j] for j in Indices] Indices = enumerate(Ind) DATA = {} for imageindex, imagename in tqdm(Indices): image = imread(os.path.join(cfg["project_path"], imagename), mode="RGB") if scale != 1: image = imresize(image, scale) image_batch = data_to_input(image) # Compute prediction with the CNN outputs_np = sess.run(outputs, feed_dict={inputs: image_batch}) if cfg.get("multianimalproject", False): scmap, locref, paf = predictma.extract_cnn_output( outputs_np, dlc_cfg) pagraph = dlc_cfg["partaffinityfield_graph"] else: scmap, locref = predict.extract_cnn_output( outputs_np, dlc_cfg) paf = None pagraph = [] if imageindex in testIndices: trainingfram = False else: trainingfram = True DATA[imageindex] = [ image, scmap, locref, paf, bptnames, pagraph, imagename, trainingfram, ] Maps[trainFraction][Snapshots[snapindex]] = DATA os.chdir(str(start_path)) return Maps
def make_batch(self, data_item, scale, mirror): im_file = data_item.im_path logging.debug('image %s', im_file) logging.debug('mirror %r', mirror) #print(im_file, os.getcwd()) #print(self.cfg.project_path) image = imread(os.path.join(self.cfg.project_path,im_file), mode='RGB') if self.has_gt: joints = np.copy(data_item.joints) if self.cfg.crop: #adapted cropping for DLC if np.random.rand()<self.cfg.cropratio: #1. get center of joints j=np.random.randint(np.shape(joints)[1]) #pick a random joint # draw random crop dimensions & subtract joint points #print(joints,j,'ahah') joints,image=CropImage(joints,image,joints[0,j,1],joints[0,j,2],self.cfg) #if self.has_gt: # joints[0,:, 1] -= x0 # joints[0,:, 2] -= y0 ''' print(joints) import matplotlib.pyplot as plt plt.clf() plt.imshow(image) plt.plot(joints[0,:,1],joints[0,:,2],'.') plt.savefig("abc"+str(np.random.randint(int(1e6)))+".png") ''' else: pass #no cropping! img = imresize(image, scale) if scale != 1 else image scaled_img_size = arr(img.shape[0:2]) if mirror: img = np.fliplr(img) batch = {Batch.inputs: img} if self.has_gt: stride = self.cfg.stride if mirror: joints = [self.mirror_joints(person_joints, self.symmetric_joints, image.shape[1]) for person_joints in joints] sm_size = np.ceil(scaled_img_size / (stride * 2)).astype(int) * 2 scaled_joints = [person_joints[:, 1:3] * scale for person_joints in joints] joint_id = [person_joints[:, 0].astype(int) for person_joints in joints] part_score_targets, part_score_weights, locref_targets, locref_mask = self.compute_target_part_scoremap( joint_id, scaled_joints, data_item, sm_size, scale) batch.update({ Batch.part_score_targets: part_score_targets, Batch.part_score_weights: part_score_weights, Batch.locref_targets: locref_targets, Batch.locref_mask: locref_mask }) batch = {key: data_to_input(data) for (key, data) in batch.items()} batch[Batch.data_item] = data_item return batch
def evaluate_network(config, Shuffles=[1], plotting=None, show_errors=True, comparisonbodyparts="all", gputouse=None): """ Evaluates the network based on the saved models at different stages of the training network.\n The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'. Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models. Parameters ---------- config : string Full path of the config.yaml file as a string. Shuffles: list, optional List of integers specifying the shuffle indices of the training dataset. The default is [1] plotting: bool, optional Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False`` show_errors: bool, optional Display train and test errors. The default is `True`` comparisonbodyparts: list of bodyparts, Default is "all". The average error will be computed for those body parts only (Has to be a subset of the body parts). gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None. See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries Examples -------- If you do not want to plot >>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml', shuffle=[1]) -------- If you want to plot >>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',shuffle=[1],True) """ import os from skimage import io import skimage.color from deeplabcut.pose_estimation_tensorflow.nnet import predict as ptf_predict from deeplabcut.pose_estimation_tensorflow.config import load_config from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input from deeplabcut.utils import auxiliaryfunctions, visualization import tensorflow as tf if 'TF_CUDNN_USE_AUTOTUNE' in os.environ: del os.environ[ 'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training vers = (tf.__version__).split('.') if int(vers[0]) == 1 and int(vers[1]) > 12: TF = tf.compat.v1 else: TF = tf TF.reset_default_graph() os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # # tf.logging.set_verbosity(tf.logging.WARN) start_path = os.getcwd() # Read file path for pose_config file. >> pass it on cfg = auxiliaryfunctions.read_config(config) if gputouse is not None: #gpu selectinon os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse) # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) Data = pd.read_hdf( os.path.join(cfg["project_path"], str(trainingsetfolder), 'CollectedData_' + cfg["scorer"] + '.h5'), 'df_with_missing') # Get list of body parts to evaluate network for comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, comparisonbodyparts) # Make folder for evaluation auxiliaryfunctions.attempttomakefolder( str(cfg["project_path"] + "/evaluation-results/")) for shuffle in Shuffles: for trainFraction in cfg["TrainingFraction"]: ################################################## # Load and setup CNN part detector ################################################## datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) modelfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg))) path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml' # Load meta data data, trainIndices, testIndices, trainFraction = auxiliaryfunctions.LoadMetadata( os.path.join(cfg["project_path"], metadatafn)) try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, trainFraction)) #change batch size, if it was edited during analysis! dlc_cfg['batch_size'] = 1 #in case this was edited for analysis. #Create folder structure to store results. evaluationfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetEvaluationFolder( trainFraction, shuffle, cfg))) auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True) #path_train_config = modelfolder / 'train' / 'pose_cfg.yaml' # Check which snapshots are available and sort them by # iterations Snapshots = np.array([ fn.split('.')[0] for fn in os.listdir(os.path.join(str(modelfolder), 'train')) if "index" in fn ]) try: #check if any where found? Snapshots[0] except IndexError: raise FileNotFoundError( "Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so." % (shuffle, trainFraction)) increasing_indices = np.argsort( [int(m.split('-')[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] if cfg["snapshotindex"] == -1: snapindices = [-1] elif cfg["snapshotindex"] == "all": snapindices = range(len(Snapshots)) elif cfg["snapshotindex"] < len(Snapshots): snapindices = [cfg["snapshotindex"]] else: print( "Invalid choice, only -1 (last), any integer up to last, or all (as string)!" ) final_result = [] ################################################## # Compute predictions over images ################################################## for snapindex in snapindices: dlc_cfg['init_weights'] = os.path.join( str(modelfolder), 'train', Snapshots[snapindex] ) #setting weights to corresponding snapshot. trainingsiterations = ( dlc_cfg['init_weights'].split(os.sep)[-1] ).split( '-' )[-1] #read how many training siterations that corresponds to. #name for deeplabcut net (based on its parameters) DLCscorer = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction, trainingsiterations) print("Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations) resultsfilename = os.path.join( str(evaluationfolder), DLCscorer + '-' + Snapshots[snapindex] + '.h5') try: DataMachine = pd.read_hdf(resultsfilename, 'df_with_missing') print("This net has already been evaluated!") except FileNotFoundError: # Specifying state of model (snapshot / training state) sess, inputs, outputs = ptf_predict.setup_pose_prediction( dlc_cfg) Numimages = len(Data.index) PredicteData = np.zeros( (Numimages, 3 * len(dlc_cfg['all_joints_names']))) print("Analyzing data...") for imageindex, imagename in tqdm(enumerate(Data.index)): image = io.imread(os.path.join(cfg['project_path'], imagename), mode='RGB') image = skimage.color.gray2rgb(image) image_batch = data_to_input(image) # Compute prediction with the CNN outputs_np = sess.run(outputs, feed_dict={inputs: image_batch}) scmap, locref = ptf_predict.extract_cnn_output( outputs_np, dlc_cfg) # Extract maximum scoring location from the heatmap, assume 1 person pose = ptf_predict.argmax_pose_predict( scmap, locref, dlc_cfg.stride) PredicteData[imageindex, :] = pose.flatten( ) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts! sess.close() #closes the current tf session index = pd.MultiIndex.from_product( [[DLCscorer], dlc_cfg['all_joints_names'], ['x', 'y', 'likelihood']], names=['scorer', 'bodyparts', 'coords']) # Saving results DataMachine = pd.DataFrame(PredicteData, columns=index, index=Data.index.values) DataMachine.to_hdf(resultsfilename, 'df_with_missing', format='table', mode='w') print("Done and results stored for snapshot: ", Snapshots[snapindex]) DataCombined = pd.concat([Data.T, DataMachine.T], axis=0).T RMSE, RMSEpcutoff = pairwisedistances( DataCombined, cfg["scorer"], DLCscorer, cfg["pcutoff"], comparisonbodyparts) testerror = np.nanmean( RMSE.iloc[testIndices].values.flatten()) trainerror = np.nanmean( RMSE.iloc[trainIndices].values.flatten()) testerrorpcutoff = np.nanmean( RMSEpcutoff.iloc[testIndices].values.flatten()) trainerrorpcutoff = np.nanmean( RMSEpcutoff.iloc[trainIndices].values.flatten()) results = [ trainingsiterations, int(100 * trainFraction), shuffle, np.round(trainerror, 2), np.round(testerror, 2), cfg["pcutoff"], np.round(trainerrorpcutoff, 2), np.round(testerrorpcutoff, 2) ] final_result.append(results) if show_errors == True: print("Results for", trainingsiterations, " training iterations:", int(100 * trainFraction), shuffle, "train error:", np.round(trainerror, 2), "pixels. Test error:", np.round(testerror, 2), " pixels.") print("With pcutoff of", cfg["pcutoff"], " train error:", np.round(trainerrorpcutoff, 2), "pixels. Test error:", np.round(testerrorpcutoff, 2), "pixels") print( "Thereby, the errors are given by the average distances between the labels by DLC and the scorer." ) if plotting == True: print("Plotting...") colors = visualization.get_cmap( len(comparisonbodyparts), name=cfg['colormap']) foldername = os.path.join( str(evaluationfolder), 'LabeledImages_' + DLCscorer + '_' + Snapshots[snapindex]) auxiliaryfunctions.attempttomakefolder(foldername) NumFrames = np.size(DataCombined.index) for ind in np.arange(NumFrames): visualization.PlottingandSaveLabeledFrame( DataCombined, ind, trainIndices, cfg, colors, comparisonbodyparts, DLCscorer, foldername) TF.reset_default_graph() #print(final_result) make_results_file(final_result, evaluationfolder, DLCscorer) print( "The network is evaluated and the results are stored in the subdirectory 'evaluation_results'." ) print( "If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos." ) print( "Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)" ) #returning to intial folder os.chdir(str(start_path))
def make_batch(self, data_item, scale, mirror): im_file = data_item.im_path logging.debug("image %s", im_file) logging.debug("mirror %r", mirror) image = imread(os.path.join(self.cfg['project_path'], im_file), mode="RGB") if self.has_gt: joints = np.copy(data_item.joints) if self.cfg['crop']: # adapted cropping for DLC if np.random.rand() < self.cfg['cropratio']: j = np.random.randint(np.shape(joints)[1]) # pick a random joint joints, image = CropImage( joints, image, joints[0, j, 1], joints[0, j, 2], self.cfg ) """ print(joints) import matplotlib.pyplot as plt plt.clf() plt.imshow(image) plt.plot(joints[0,:,1],joints[0,:,2],'.') plt.savefig("abc"+str(np.random.randint(int(1e6)))+".png") """ else: pass # no cropping! img = imresize(image, scale) if scale != 1 else image scaled_img_size = arr(img.shape[0:2]) if mirror: img = np.fliplr(img) batch = {Batch.inputs: img} if self.has_gt: stride = self.cfg['stride'] if mirror: joints = [ self.mirror_joints( person_joints, self.symmetric_joints, image.shape[1] ) for person_joints in joints ] sm_size = np.ceil(scaled_img_size / (stride * 2)).astype(int) * 2 scaled_joints = [person_joints[:, 1:3] * scale for person_joints in joints] joint_id = [person_joints[:, 0].astype(int) for person_joints in joints] ( part_score_targets, part_score_weights, locref_targets, locref_mask, ) = self.compute_target_part_scoremap( joint_id, scaled_joints, data_item, sm_size, scale ) batch.update( { Batch.part_score_targets: part_score_targets, Batch.part_score_weights: part_score_weights, Batch.locref_targets: locref_targets, Batch.locref_mask: locref_mask, } ) batch = {key: data_to_input(data) for (key, data) in batch.items()} batch[Batch.data_item] = data_item return batch
def make_batch(self, data_item, scale, mirror): im_file = data_item.im_path logging.debug("image %s", im_file) logging.debug("mirror %r", mirror) # print(im_file, os.getcwd()) # print(self.cfg.project_path) vid_fname = os.path.join(self.cfg.project_path, im_file) image = imread(vid_fname, mode="RGB") # print("Full image filename: ", vid_fname) # print("Shape of read image: ", image.shape) if self.has_gt: joints = np.copy(data_item.joints) if self.cfg.crop: # adapted cropping for DLC if np.random.rand() < self.cfg.cropratio: j = np.random.randint(np.shape(joints)[1]) # pick a random joint joints, image = CropImage( joints, image, joints[0, j, 1], joints[0, j, 2], self.cfg ) """ print(joints) import matplotlib.pyplot as plt plt.clf() plt.imshow(image) plt.plot(joints[0,:,1],joints[0,:,2],'.') plt.savefig("abc"+str(np.random.randint(int(1e6)))+".png") """ else: pass # no cropping! # Charlie addition if not self.cfg['using_z_slices']: img = imresize(image, scale) if scale != 1 else image scaled_img_size = arr(img.shape[0:2]) else: # img = imresize(image, scale) if scale < 1 else image # if scale != 1: # zspan = range(image.shape[0]) # img = np.array([imresize(image[z,...], scale) for z in zspan]) # print(f"{img.shape}") # else: # img = image img = image # Just ignore scale scaled_img_size = arr(img.shape[:3]) # Ignore color if mirror: img = np.fliplr(img) batch = {Batch.inputs: img} if self.has_gt: stride = self.cfg.stride if mirror: joints = [ self.mirror_joints( person_joints, self.symmetric_joints, image.shape[1] ) for person_joints in joints ] # print("Input size: ", scaled_img_size) # print("Stride: ", stride) sm_size = np.ceil(scaled_img_size / (stride * 2)).astype(int) * 2 if self.cfg.using_z_slices: sm_size[0] = scaled_img_size[0] # z should not be "strided" print(f"Resized to {sm_size} from {image.shape} using scale {scale}") if not self.cfg.using_z_slices: scaled_joints = [person_joints[:, 1:3] * scale for person_joints in joints] else: # print("Scale ", scale) scaled_joints = [person_joints[:, 1:4] * scale for person_joints in joints] # [print("Person joints ", person_joints) for person_joints in joints] joint_id = [person_joints[:, 0].astype(int) for person_joints in joints] if not self.cfg.using_z_slices: compute = self.compute_target_part_scoremap else: compute = self.compute_target_part_scoremap_slices ( part_score_targets, part_score_weights, locref_targets, locref_mask, ) = compute(joint_id, scaled_joints, data_item, sm_size, scale) # print("part_score_targets: ", part_score_targets.shape) # print("locref_targets: ", locref_targets.shape) batch.update( { Batch.part_score_targets: part_score_targets, Batch.part_score_weights: part_score_weights, Batch.locref_targets: locref_targets, Batch.locref_mask: locref_mask, } ) batch = {key: data_to_input(data) for (key, data) in batch.items()} batch[Batch.data_item] = data_item return batch