def evaluate_multianimal_full( config, Shuffles=[1], trainingsetindex=0, plotting=None, show_errors=True, comparisonbodyparts="all", gputouse=None, modelprefix="", c_engine=False, ): """ WIP multi animal project. """ import os from deeplabcut.pose_estimation_tensorflow.nnet import predict from deeplabcut.pose_estimation_tensorflow.nnet import ( predict_multianimal as predictma, ) from deeplabcut.utils import auxiliaryfunctions, auxfun_multianimal import tensorflow as tf if "TF_CUDNN_USE_AUTOTUNE" in os.environ: del os.environ[ "TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training tf.reset_default_graph() os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" # if gputouse is not None: # gpu selectinon os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse) start_path = os.getcwd() ################################################## # Load data... ################################################## cfg = auxiliaryfunctions.read_config(config) if trainingsetindex == "all": TrainingFractions = cfg["TrainingFraction"] else: TrainingFractions = [cfg["TrainingFraction"][trainingsetindex]] # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) Data = pd.read_hdf( os.path.join( cfg["project_path"], str(trainingsetfolder), "CollectedData_" + cfg["scorer"] + ".h5", ), "df_with_missing", ) # Get list of body parts to evaluate network for comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, comparisonbodyparts) colors = visualization.get_cmap(len(comparisonbodyparts), name=cfg["colormap"]) # Make folder for evaluation auxiliaryfunctions.attempttomakefolder( str(cfg["project_path"] + "/evaluation-results/")) for shuffle in Shuffles: for trainFraction in TrainingFractions: ################################################## # Load and setup CNN part detector ################################################## datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) modelfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetModelFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" # Load meta data ( data, trainIndices, testIndices, trainFraction, ) = auxiliaryfunctions.LoadMetadata( os.path.join(cfg["project_path"], metadatafn)) try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, trainFraction)) # TODO: IMPLEMENT for different batch sizes? dlc_cfg["batch_size"] = 1 # due to differently sized images!!! # Create folder structure to store results. evaluationfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetEvaluationFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True) # path_train_config = modelfolder / 'train' / 'pose_cfg.yaml' # Check which snapshots are available and sort them by # iterations Snapshots = np.array([ fn.split(".")[0] for fn in os.listdir(os.path.join(str(modelfolder), "train")) if "index" in fn ]) if len(Snapshots) == 0: print( "Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so." % (shuffle, trainFraction)) else: increasing_indices = np.argsort( [int(m.split("-")[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] if cfg["snapshotindex"] == -1: snapindices = [-1] elif cfg["snapshotindex"] == "all": snapindices = range(len(Snapshots)) elif cfg["snapshotindex"] < len(Snapshots): snapindices = [cfg["snapshotindex"]] else: print( "Invalid choice, only -1 (last), any integer up to last, or all (as string)!" ) ( individuals, uniquebodyparts, multianimalbodyparts, ) = auxfun_multianimal.extractindividualsandbodyparts(cfg) final_result = [] ################################################## # Compute predictions over images ################################################## for snapindex in snapindices: dlc_cfg["init_weights"] = os.path.join( str(modelfolder), "train", Snapshots[snapindex] ) # setting weights to corresponding snapshot. trainingsiterations = ( dlc_cfg["init_weights"].split(os.sep)[-1] ).split( "-" )[-1] # read how many training siterations that corresponds to. # name for deeplabcut net (based on its parameters) DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction, trainingsiterations, modelprefix=modelprefix, ) print( "Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations, ) ( notanalyzed, resultsfilename, DLCscorer, ) = auxiliaryfunctions.CheckifNotEvaluated( str(evaluationfolder), DLCscorer, DLCscorerlegacy, Snapshots[snapindex], ) if os.path.isfile( resultsfilename.split(".h5")[0] + "_full.pickle"): print("Model already evaluated.", resultsfilename) else: if plotting: foldername = os.path.join( str(evaluationfolder), "LabeledImages_" + DLCscorer + "_" + Snapshots[snapindex], ) auxiliaryfunctions.attempttomakefolder(foldername) # print(dlc_cfg) # Specifying state of model (snapshot / training state) sess, inputs, outputs = predict.setup_pose_prediction( dlc_cfg) PredicteData = {} print("Analyzing data...") for imageindex, imagename in tqdm(enumerate( Data.index)): image_path = os.path.join(cfg["project_path"], imagename) image = io.imread(image_path) frame = img_as_ubyte(skimage.color.gray2rgb(image)) GT = Data.iloc[imageindex] # Storing GT data as dictionary, so it can be used for calculating connection costs groundtruthcoordinates = [] groundtruthidentity = [] for bptindex, bpt in enumerate( dlc_cfg["all_joints_names"]): coords = np.zeros([len(individuals), 2 ]) * np.nan identity = [] for prfxindex, prefix in enumerate( individuals): if bpt in uniquebodyparts and prefix == "single": coords[prfxindex, :] = np.array([ GT[cfg["scorer"]][prefix][bpt] ["x"], GT[cfg["scorer"]][prefix][bpt] ["y"], ]) identity.append(prefix) elif (bpt in multianimalbodyparts and prefix != "single"): coords[prfxindex, :] = np.array([ GT[cfg["scorer"]][prefix][bpt] ["x"], GT[cfg["scorer"]][prefix][bpt] ["y"], ]) identity.append(prefix) else: identity.append("nix") groundtruthcoordinates.append( coords[np.isfinite(coords[:, 0]), :]) groundtruthidentity.append( np.array(identity)[np.isfinite(coords[:, 0])]) PredicteData[imagename] = {} PredicteData[imagename]["index"] = imageindex pred = predictma.get_detectionswithcostsandGT( frame, groundtruthcoordinates, dlc_cfg, sess, inputs, outputs, outall=False, nms_radius=dlc_cfg.nmsradius, det_min_score=dlc_cfg.minconfidence, c_engine=c_engine, ) PredicteData[imagename]["prediction"] = pred PredicteData[imagename]["groundtruth"] = [ groundtruthidentity, groundtruthcoordinates, GT, ] if plotting: coords_pred = pred["coordinates"][0] probs_pred = pred["confidence"] fig = visualization.make_multianimal_labeled_image( frame, groundtruthcoordinates, coords_pred, probs_pred, colors, cfg["dotsize"], cfg["alphavalue"], cfg["pcutoff"], ) visualization.save_labeled_frame( fig, image_path, foldername, imageindex in trainIndices, ) sess.close() # closes the current tf session PredicteData["metadata"] = { "nms radius": dlc_cfg.nmsradius, "minimal confidence": dlc_cfg.minconfidence, "PAFgraph": dlc_cfg.partaffinityfield_graph, "all_joints": [[i] for i in range(len(dlc_cfg.all_joints))], "all_joints_names": [ dlc_cfg.all_joints_names[i] for i in range(len(dlc_cfg.all_joints)) ], "stride": dlc_cfg.get("stride", 8), } print( "Done and results stored for snapshot: ", Snapshots[snapindex], ) dictionary = { "Scorer": DLCscorer, "DLC-model-config file": dlc_cfg, "trainIndices": trainIndices, "testIndices": testIndices, "trainFraction": trainFraction, } metadata = {"data": dictionary} auxfun_multianimal.SaveFullMultiAnimalData( PredicteData, metadata, resultsfilename) tf.reset_default_graph() # returning to intial folder os.chdir(str(start_path))
def evaluate_multianimal_full( config, Shuffles=[1], trainingsetindex=0, plotting=None, show_errors=True, comparisonbodyparts="all", gputouse=None, modelprefix="", c_engine=False, ): from deeplabcut.pose_estimation_tensorflow.nnet import predict from deeplabcut.pose_estimation_tensorflow.nnet import ( predict_multianimal as predictma, ) from deeplabcut.utils import auxiliaryfunctions, auxfun_multianimal import tensorflow as tf if "TF_CUDNN_USE_AUTOTUNE" in os.environ: del os.environ[ "TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training tf.reset_default_graph() os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" # if gputouse is not None: # gpu selectinon os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse) start_path = os.getcwd() ################################################## # Load data... ################################################## cfg = auxiliaryfunctions.read_config(config) if trainingsetindex == "all": TrainingFractions = cfg["TrainingFraction"] else: TrainingFractions = [cfg["TrainingFraction"][trainingsetindex]] # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) Data = pd.read_hdf( os.path.join( cfg["project_path"], str(trainingsetfolder), "CollectedData_" + cfg["scorer"] + ".h5", ), "df_with_missing", ) # Handle data previously annotated on a different platform sep = "/" if "/" in Data.index[0] else "\\" if sep != os.path.sep: Data.index = Data.index.str.replace(sep, os.path.sep) # Get list of body parts to evaluate network for comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, comparisonbodyparts) all_bpts = np.asarray( len(cfg["individuals"]) * cfg["multianimalbodyparts"] + cfg["uniquebodyparts"]) colors = visualization.get_cmap(len(comparisonbodyparts), name=cfg["colormap"]) # Make folder for evaluation auxiliaryfunctions.attempttomakefolder( str(cfg["project_path"] + "/evaluation-results/")) for shuffle in Shuffles: for trainFraction in TrainingFractions: ################################################## # Load and setup CNN part detector ################################################## datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) modelfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetModelFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" # Load meta data ( data, trainIndices, testIndices, trainFraction, ) = auxiliaryfunctions.LoadMetadata( os.path.join(cfg["project_path"], metadatafn)) try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, trainFraction)) # TODO: IMPLEMENT for different batch sizes? dlc_cfg["batch_size"] = 1 # due to differently sized images!!! joints = dlc_cfg["all_joints_names"] # Create folder structure to store results. evaluationfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetEvaluationFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True) # path_train_config = modelfolder / 'train' / 'pose_cfg.yaml' # Check which snapshots are available and sort them by # iterations Snapshots = np.array([ fn.split(".")[0] for fn in os.listdir(os.path.join(str(modelfolder), "train")) if "index" in fn ]) if len(Snapshots) == 0: print( "Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so." % (shuffle, trainFraction)) else: increasing_indices = np.argsort( [int(m.split("-")[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] if cfg["snapshotindex"] == -1: snapindices = [-1] elif cfg["snapshotindex"] == "all": snapindices = range(len(Snapshots)) elif cfg["snapshotindex"] < len(Snapshots): snapindices = [cfg["snapshotindex"]] else: print( "Invalid choice, only -1 (last), any integer up to last, or all (as string)!" ) final_result = [] ################################################## # Compute predictions over images ################################################## for snapindex in snapindices: dlc_cfg["init_weights"] = os.path.join( str(modelfolder), "train", Snapshots[snapindex] ) # setting weights to corresponding snapshot. trainingsiterations = ( dlc_cfg["init_weights"].split(os.sep)[-1] ).split( "-" )[-1] # read how many training siterations that corresponds to. # name for deeplabcut net (based on its parameters) DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction, trainingsiterations, modelprefix=modelprefix, ) print( "Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations, ) ( notanalyzed, resultsfilename, DLCscorer, ) = auxiliaryfunctions.CheckifNotEvaluated( str(evaluationfolder), DLCscorer, DLCscorerlegacy, Snapshots[snapindex], ) if os.path.isfile( resultsfilename.split(".h5")[0] + "_full.pickle"): print("Model already evaluated.", resultsfilename) else: if plotting: foldername = os.path.join( str(evaluationfolder), "LabeledImages_" + DLCscorer + "_" + Snapshots[snapindex], ) auxiliaryfunctions.attempttomakefolder(foldername) # print(dlc_cfg) # Specifying state of model (snapshot / training state) sess, inputs, outputs = predict.setup_pose_prediction( dlc_cfg) PredicteData = {} dist = np.full((len(Data), len(all_bpts)), np.nan) conf = np.full_like(dist, np.nan) distnorm = np.full(len(Data), np.nan) print("Analyzing data...") for imageindex, imagename in tqdm(enumerate( Data.index)): image_path = os.path.join(cfg["project_path"], imagename) image = io.imread(image_path) frame = img_as_ubyte(skimage.color.gray2rgb(image)) GT = Data.iloc[imageindex] df = GT.unstack("coords").reindex( joints, level='bodyparts') # Evaluate PAF edge lengths to calibrate `distnorm` temp = GT.unstack("bodyparts")[joints] xy = temp.values.reshape( (-1, 2, temp.shape[1])).swapaxes(1, 2) edges = xy[:, dlc_cfg["partaffinityfield_graph"]] lengths = np.sum( (edges[:, :, 0] - edges[:, :, 1])**2, axis=2) distnorm[imageindex] = np.nanmax(lengths) # FIXME Is having an empty array vs nan really that necessary?! groundtruthidentity = list( df.index.get_level_values( "individuals").to_numpy().reshape((-1, 1))) groundtruthcoordinates = list( df.values[:, np.newaxis]) for i, coords in enumerate(groundtruthcoordinates): if np.isnan(coords).any(): groundtruthcoordinates[i] = np.empty( (0, 2), dtype=float) groundtruthidentity[i] = np.array( [], dtype=str) PredicteData[imagename] = {} PredicteData[imagename]["index"] = imageindex pred = predictma.get_detectionswithcostsandGT( frame, groundtruthcoordinates, dlc_cfg, sess, inputs, outputs, outall=False, nms_radius=dlc_cfg.nmsradius, det_min_score=dlc_cfg.minconfidence, c_engine=c_engine, ) PredicteData[imagename]["prediction"] = pred PredicteData[imagename]["groundtruth"] = [ groundtruthidentity, groundtruthcoordinates, GT, ] coords_pred = pred["coordinates"][0] probs_pred = pred["confidence"] for bpt, xy_gt in df.groupby(level="bodyparts"): inds_gt = np.flatnonzero( np.all(~np.isnan(xy_gt), axis=1)) n_joint = joints.index(bpt) xy = coords_pred[n_joint] if inds_gt.size and xy.size: # Pick the predictions closest to ground truth, # rather than the ones the model has most confident in d = cdist(xy_gt.iloc[inds_gt], xy) rows, cols = linear_sum_assignment(d) min_dists = d[rows, cols] inds = np.flatnonzero(all_bpts == bpt) sl = imageindex, inds[inds_gt[rows]] dist[sl] = min_dists conf[sl] = probs_pred[n_joint][ cols].squeeze() if plotting: fig = visualization.make_multianimal_labeled_image( frame, groundtruthcoordinates, coords_pred, probs_pred, colors, cfg["dotsize"], cfg["alphavalue"], cfg["pcutoff"], ) visualization.save_labeled_frame( fig, image_path, foldername, imageindex in trainIndices, ) sess.close() # closes the current tf session # Compute all distance statistics df_dist = pd.DataFrame(dist, columns=df.index) df_conf = pd.DataFrame(conf, columns=df.index) df_joint = pd.concat([df_dist, df_conf], keys=["rmse", "conf"], names=["metrics"], axis=1) df_joint = df_joint.reorder_levels(list( np.roll(df_joint.columns.names, -1)), axis=1) df_joint.sort_index(axis=1, level=["individuals", "bodyparts"], ascending=[True, True], inplace=True) write_path = os.path.join( evaluationfolder, f"dist_{trainingsiterations}.csv") df_joint.to_csv(write_path) # Calculate overall prediction error error = df_joint.xs("rmse", level="metrics", axis=1) mask = df_joint.xs("conf", level="metrics", axis=1) >= cfg["pcutoff"] error_masked = error[mask] error_train = np.nanmean(error.iloc[trainIndices]) error_train_cut = np.nanmean( error_masked.iloc[trainIndices]) error_test = np.nanmean(error.iloc[testIndices]) error_test_cut = np.nanmean( error_masked.iloc[testIndices]) results = [ trainingsiterations, int(100 * trainFraction), shuffle, np.round(error_train, 2), np.round(error_test, 2), cfg["pcutoff"], np.round(error_train_cut, 2), np.round(error_test_cut, 2), ] final_result.append(results) # For OKS/PCK, compute the standard deviation error across all frames sd = df_dist.groupby("bodyparts", axis=1).mean().std(axis=0) sd["distnorm"] = np.sqrt(np.nanmax(distnorm)) sd.to_csv(write_path.replace("dist.csv", "sd.csv")) if show_errors: string = "Results for {} training iterations: {}, shuffle {}:\n" \ "Train error: {} pixels. Test error: {} pixels.\n" \ "With pcutoff of {}:\n" \ "Train error: {} pixels. Test error: {} pixels." print(string.format(*results)) print("##########################################") print( "Average Euclidean distance to GT per individual (in pixels)" ) print( error_masked.groupby( 'individuals', axis=1).mean().mean().to_string()) print( "Average Euclidean distance to GT per bodypart (in pixels)" ) print( error_masked.groupby( 'bodyparts', axis=1).mean().mean().to_string()) PredicteData["metadata"] = { "nms radius": dlc_cfg.nmsradius, "minimal confidence": dlc_cfg.minconfidence, "PAFgraph": dlc_cfg.partaffinityfield_graph, "all_joints": [[i] for i in range(len(dlc_cfg.all_joints))], "all_joints_names": [ dlc_cfg.all_joints_names[i] for i in range(len(dlc_cfg.all_joints)) ], "stride": dlc_cfg.get("stride", 8), } print( "Done and results stored for snapshot: ", Snapshots[snapindex], ) dictionary = { "Scorer": DLCscorer, "DLC-model-config file": dlc_cfg, "trainIndices": trainIndices, "testIndices": testIndices, "trainFraction": trainFraction, } metadata = {"data": dictionary} auxfun_multianimal.SaveFullMultiAnimalData( PredicteData, metadata, resultsfilename) tf.reset_default_graph() if len(final_result ) > 0: # Only append if results were calculated make_results_file(final_result, evaluationfolder, DLCscorer) # returning to intial folder os.chdir(str(start_path))