def test_net(visualise, cache_scoremaps):
logging.basicConfig(level=logging.INFO)
cfg = load_config()
dataset = create_dataset(cfg)
dataset.set_shuffle(False)
dataset.set_test_mode(True)
sess, inputs, outputs = setup_pose_prediction(cfg)
if cache_scoremaps:
out_dir = cfg.scoremap_dir
if not os.path.exists(out_dir):
os.makedirs(out_dir)
num_images = dataset.num_images
predictions = np.zeros((num_images, ), dtype=np.object)
for k in range(num_images):
print("processing image {}/{}".format(k, num_images - 1))
batch = dataset.next_batch()
outputs_np = sess.run(outputs, feed_dict={inputs: batch[Batch.inputs]})
scmap, locref = extract_cnn_output(outputs_np, cfg)
pose = argmax_pose_predict(scmap, locref, cfg.stride)
pose_refscale = np.copy(pose)
pose_refscale[:, 0:2] /= cfg.global_scale
predictions[k] = pose_refscale
if visualise:
img = np.squeeze(batch[Batch.inputs]).astype("uint8")
visualize.show_heatmaps(cfg, img, scmap, pose)
visualize.waitforbuttonpress()
if cache_scoremaps:
base = os.path.basename(batch[Batch.data_item].im_path)
raw_name = os.path.splitext(base)[0]
out_fn = os.path.join(out_dir, raw_name + ".mat")
scipy.io.savemat(out_fn,
mdict={"scoremaps": scmap.astype("float32")})
out_fn = os.path.join(out_dir, raw_name + "_locreg" + ".mat")
if cfg.location_refinement:
scipy.io.savemat(
out_fn, mdict={"locreg_pred": locref.astype("float32")})
scipy.io.savemat("predictions.mat", mdict={"joints": predictions})
sess.close()
def predict_single_image(image, sess, inputs, outputs, dlc_cfg):
"""
Returns pose for one single image
:param image:
:return:
"""
# assert
image = skimage.color.gray2rgb(image)
image_batch = data_to_input(image)
# Compute prediction with the CNN
outputs_np = sess.run(outputs, feed_dict={inputs: image_batch})
scmap, locref = ptf_predict.extract_cnn_output(outputs_np, dlc_cfg)
# Extract maximum scoring location from the heatmap, assume 1 person
pose = ptf_predict.argmax_pose_predict(scmap, locref, dlc_cfg.stride)
return pose
def validate(self, sess, trainingsiterations):
final_result = []
# TODO:: Adapt to training/validation loss value
for imageindex, imagename in tqdm(enumerate(self.Data.index)):
image = io.imread(os.path.join(self.cfg['project_path'],
imagename),
mode='RGB')
image = skimage.color.gray2rgb(image)
image_batch = data_to_input(image)
# Compute prediction with the CNN
#[loss_val, summary] = sess.run([total_loss, merged_summaries],
# feed_dict={inputs: image_batch})
outputs_np = sess.run(self.outputs,
feed_dict={self.inputs: image_batch})
scmap, locref = ptf_predict.extract_cnn_output(
outputs_np, self.pose_cfg)
# Extract maximum scoring location from the heatmap, assume 1 person
pose = ptf_predict.argmax_pose_predict(scmap, locref,
self.pose_cfg.stride)
self.PredictedData[imageindex, :] = pose.flatten(
) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts!
DLCscorer = 'Predictor'
index = pd.MultiIndex.from_product(
[[DLCscorer], self.pose_cfg['all_joints_names'],
['x', 'y', 'likelihood']],
names=['scorer', 'bodyparts', 'coords'])
# Saving results
DataMachine = pd.DataFrame(self.PredictedData,
columns=index,
index=self.Data.index.values)
#DataMachine.to_hdf(resultsfilename,'df_with_missing',format='table',mode='w')
print("Validated validation set")
DataCombined = pd.concat([self.Data.T, DataMachine.T], axis=0).T
RMSE, RMSEpcutoff = evaluate.pairwisedistances(
DataCombined, self.cfg["scorer"], DLCscorer, self.cfg["pcutoff"],
self.comparisonbodyparts)
validerror = np.nanmean(RMSE.iloc[self.validIndices].values.flatten())
trainerror = np.nanmean(RMSE.iloc[self.trainIndices].values.flatten())
validerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[self.validIndices].values.flatten())
trainerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[self.trainIndices].values.flatten())
results = [
trainingsiterations,
np.round(trainerror, 2),
np.round(validerror, 2), self.cfg["pcutoff"],
np.round(trainerrorpcutoff, 2),
np.round(validerrorpcutoff, 2)
]
final_result.append(results)
print("Results for",
trainingsiterations, " training iterations:", "train error:",
np.round(trainerror, 2), "pixels. Validation error:",
np.round(validerror, 2), " pixels.")
print("With pcutoff of", self.cfg["pcutoff"], " train error:",
np.round(trainerrorpcutoff, 2), "pixels. Validation error:",
np.round(validerrorpcutoff, 2), "pixels")
print(
"Thereby, the errors are given by the average distances between the labels by DLC and the scorer."
)
# Write to file
self.lrf.write("{}, {:.5f}, {:.5f}, {:.5f}, {:.5f}\n".format(
trainingsiterations, trainerror, trainerrorpcutoff, validerror,
validerrorpcutoff))
self.lrf.flush()
return validerror
def evaluate_network(
config,
Shuffles=[1],
trainingsetindex=0,
plotting=None,
show_errors=True,
comparisonbodyparts="all",
gputouse=None,
rescale=False,
modelprefix="",
c_engine=False,
):
"""
Evaluates the network based on the saved models at different stages of the training network.\n
The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'.
Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
Shuffles: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This
variable can also be set to "all".
plotting: bool, optional
Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False``
show_errors: bool, optional
Display train and test errors. The default is `True``
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
rescale: bool, default False
Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every
image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported
in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated
on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the
original size!
Examples
--------
If you do not want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml', Shuffles=[1])
--------
If you want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',Shuffles=[1],True)
"""
import os
start_path = os.getcwd()
from deeplabcut.utils import auxiliaryfunctions
cfg = auxiliaryfunctions.read_config(config)
if cfg.get("multianimalproject", False):
from deeplabcut.pose_estimation_tensorflow.evaluate_multianimal import (
evaluate_multianimal_full, )
# TODO: Make this code not so redundant!
evaluate_multianimal_full(
config,
Shuffles,
trainingsetindex,
plotting,
show_errors,
comparisonbodyparts,
gputouse,
modelprefix,
c_engine=c_engine,
)
else:
from deeplabcut.utils.auxfun_videos import imread, imresize
from deeplabcut.pose_estimation_tensorflow.nnet import predict
from deeplabcut.pose_estimation_tensorflow.config import load_config
from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import (
data_to_input, )
from deeplabcut.utils import auxiliaryfunctions
import tensorflow as tf
if "TF_CUDNN_USE_AUTOTUNE" in os.environ:
del os.environ[
"TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training
tf.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
# tf.logging.set_verbosity(tf.logging.WARN)
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
if (trainingsetindex < len(cfg["TrainingFraction"])
and trainingsetindex >= 0):
TrainingFractions = [
cfg["TrainingFraction"][int(trainingsetindex)]
]
else:
raise Exception(
"Please check the trainingsetindex! ",
trainingsetindex,
" should be an integer from 0 .. ",
int(len(cfg["TrainingFraction"]) - 1),
)
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
)
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# change batch size, if it was edited during analysis!
dlc_cfg[
"batch_size"] = 1 # in case this was edited for analysis.
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
# path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0] for fn in os.listdir(
os.path.join(str(modelfolder), "train"))
if "index" in fn
])
try: # check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
raise ValueError(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
final_result = []
########################### RESCALING (to global scale)
if rescale == True:
scale = dlc_cfg["global_scale"]
Data = (pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
) * scale)
else:
scale = 1
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg["init_weights"].split(os.sep)[-1]
).split(
"-"
)[-1] # read how many training siterations that corresponds to.
# Name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix,
)
print(
"Running ",
DLCscorer,
" with # of trainingiterations:",
trainingsiterations,
)
(
notanalyzed,
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.CheckifNotEvaluated(
str(evaluationfolder),
DLCscorer,
DLCscorerlegacy,
Snapshots[snapindex],
)
if notanalyzed:
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = predict.setup_pose_prediction(
dlc_cfg)
Numimages = len(Data.index)
PredicteData = np.zeros(
(Numimages, 3 * len(dlc_cfg["all_joints_names"])))
print("Analyzing data...")
for imageindex, imagename in tqdm(enumerate(
Data.index)):
image = imread(os.path.join(
cfg["project_path"], imagename),
mode="RGB")
if scale != 1:
image = imresize(image, scale)
image_batch = data_to_input(image)
# Compute prediction with the CNN
outputs_np = sess.run(
outputs, feed_dict={inputs: image_batch})
scmap, locref = predict.extract_cnn_output(
outputs_np, dlc_cfg)
# Extract maximum scoring location from the heatmap, assume 1 person
pose = predict.argmax_pose_predict(
scmap, locref, dlc_cfg.stride)
PredicteData[imageindex, :] = (
pose.flatten()
) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts!
sess.close() # closes the current tf session
index = pd.MultiIndex.from_product(
[
[DLCscorer],
dlc_cfg["all_joints_names"],
["x", "y", "likelihood"],
],
names=["scorer", "bodyparts", "coords"],
)
# Saving results
DataMachine = pd.DataFrame(PredicteData,
columns=index,
index=Data.index.values)
DataMachine.to_hdf(resultsfilename,
"df_with_missing",
format="table",
mode="w")
print(
"Done and results stored for snapshot: ",
Snapshots[snapindex],
)
DataCombined = pd.concat([Data.T, DataMachine.T],
axis=0,
sort=False).T
RMSE, RMSEpcutoff = pairwisedistances(
DataCombined,
cfg["scorer"],
DLCscorer,
cfg["pcutoff"],
comparisonbodyparts,
)
testerror = np.nanmean(
RMSE.iloc[testIndices].values.flatten())
trainerror = np.nanmean(
RMSE.iloc[trainIndices].values.flatten())
testerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[testIndices].values.flatten())
trainerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[trainIndices].values.flatten())
results = [
trainingsiterations,
int(100 * trainFraction),
shuffle,
np.round(trainerror, 2),
np.round(testerror, 2),
cfg["pcutoff"],
np.round(trainerrorpcutoff, 2),
np.round(testerrorpcutoff, 2),
]
final_result.append(results)
if show_errors == True:
print(
"Results for",
trainingsiterations,
" training iterations:",
int(100 * trainFraction),
shuffle,
"train error:",
np.round(trainerror, 2),
"pixels. Test error:",
np.round(testerror, 2),
" pixels.",
)
print(
"With pcutoff of",
cfg["pcutoff"],
" train error:",
np.round(trainerrorpcutoff, 2),
"pixels. Test error:",
np.round(testerrorpcutoff, 2),
"pixels",
)
if scale != 1:
print(
"The predictions have been calculated for rescaled images (and rescaled ground truth). Scale:",
scale,
)
print(
"Thereby, the errors are given by the average distances between the labels by DLC and the scorer."
)
if plotting == True:
print("Plotting...")
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" +
Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
Plotting(
cfg,
comparisonbodyparts,
DLCscorer,
trainIndices,
DataCombined * 1.0 / scale,
foldername,
) # Rescaling coordinates to have figure in original size!
tf.reset_default_graph()
# print(final_result)
else:
DataMachine = pd.read_hdf(resultsfilename,
"df_with_missing")
if plotting == True:
DataCombined = pd.concat([Data.T, DataMachine.T],
axis=0,
sort=False).T
print(
"Plotting...(attention scale might be inconsistent in comparison to when data was analyzed; i.e. if you used rescale)"
)
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" +
Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
Plotting(
cfg,
comparisonbodyparts,
DLCscorer,
trainIndices,
DataCombined * 1.0 / scale,
foldername,
)
if len(final_result
) > 0: # Only append if results were calculated
make_results_file(final_result, evaluationfolder,
DLCscorer)
print(
"The network is evaluated and the results are stored in the subdirectory 'evaluation_results'."
)
print(
"If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos."
)
print(
"Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)"
)
# returning to intial folder
os.chdir(str(start_path))
def extract_maps(
config,
shuffle=0,
trainingsetindex=0,
gputouse=None,
rescale=False,
Indices=None,
modelprefix="",
):
"""
Extracts the scoremap, locref, partaffinityfields (if available).
Returns a dictionary indexed by: trainingsetfraction, snapshotindex, and imageindex
for those keys, each item contains: (image,scmap,locref,paf,bpt names,partaffinity graph, imagename, True/False if this image was in trainingset)
----------
config : string
Full path of the config.yaml file as a string.
shuffle: integer
integers specifying shuffle index of the training dataset. The default is 0.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This
variable can also be set to "all".
rescale: bool, default False
Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every
image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported
in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated
on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the
original size!
Examples
--------
If you want to extract the data for image 0 and 103 (of the training set) for model trained with shuffle 0.
>>> deeplabcut.extract_maps(configfile,0,Indices=[0,103])
"""
from deeplabcut.utils.auxfun_videos import imread, imresize
from deeplabcut.pose_estimation_tensorflow.nnet import predict
from deeplabcut.pose_estimation_tensorflow.nnet import (
predict_multianimal as predictma, )
from deeplabcut.pose_estimation_tensorflow.config import load_config
from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input
from deeplabcut.utils import auxiliaryfunctions
from tqdm import tqdm
import tensorflow as tf
vers = (tf.__version__).split(".")
if int(vers[0]) == 1 and int(vers[1]) > 12:
TF = tf.compat.v1
else:
TF = tf
import pandas as pd
from pathlib import Path
import numpy as np
TF.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
# tf.logging.set_verbosity(tf.logging.WARN)
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
if trainingsetindex < len(
cfg["TrainingFraction"]) and trainingsetindex >= 0:
TrainingFractions = [
cfg["TrainingFraction"][int(trainingsetindex)]
]
else:
raise Exception(
"Please check the trainingsetindex! ",
trainingsetindex,
" should be an integer from 0 .. ",
int(len(cfg["TrainingFraction"]) - 1),
)
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
)
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
Maps = {}
for trainFraction in TrainingFractions:
Maps[trainFraction] = {}
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# change batch size, if it was edited during analysis!
dlc_cfg["batch_size"] = 1 # in case this was edited for analysis.
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
# path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
])
try: # check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
########################### RESCALING (to global scale)
scale = dlc_cfg["global_scale"] if rescale else 1
Data *= scale
bptnames = [
dlc_cfg["all_joints_names"][i]
for i in range(len(dlc_cfg["all_joints"]))
]
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg["init_weights"].split(os.sep)[-1]
).split("-")[
-1] # read how many training siterations that corresponds to.
# Name for deeplabcut net (based on its parameters)
# DLCscorer,DLCscorerlegacy = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations)
# notanalyzed, resultsfilename, DLCscorer=auxiliaryfunctions.CheckifNotEvaluated(str(evaluationfolder),DLCscorer,DLCscorerlegacy,Snapshots[snapindex])
# print("Extracting maps for ", DLCscorer, " with # of trainingiterations:", trainingsiterations)
# if notanalyzed: #this only applies to ask if h5 exists...
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
Numimages = len(Data.index)
PredicteData = np.zeros(
(Numimages, 3 * len(dlc_cfg["all_joints_names"])))
print("Analyzing data...")
if Indices is None:
Indices = enumerate(Data.index)
else:
Ind = [Data.index[j] for j in Indices]
Indices = enumerate(Ind)
DATA = {}
for imageindex, imagename in tqdm(Indices):
image = imread(os.path.join(cfg["project_path"], imagename),
mode="RGB")
if scale != 1:
image = imresize(image, scale)
image_batch = data_to_input(image)
# Compute prediction with the CNN
outputs_np = sess.run(outputs, feed_dict={inputs: image_batch})
if cfg.get("multianimalproject", False):
scmap, locref, paf = predictma.extract_cnn_output(
outputs_np, dlc_cfg)
pagraph = dlc_cfg["partaffinityfield_graph"]
else:
scmap, locref = predict.extract_cnn_output(
outputs_np, dlc_cfg)
paf = None
pagraph = []
if imageindex in testIndices:
trainingfram = False
else:
trainingfram = True
DATA[imageindex] = [
image,
scmap,
locref,
paf,
bptnames,
pagraph,
imagename,
trainingfram,
]
Maps[trainFraction][Snapshots[snapindex]] = DATA
os.chdir(str(start_path))
return Maps
def evaluate_network(config,
Shuffles=[1],
plotting=None,
show_errors=True,
comparisonbodyparts="all",
gputouse=None):
"""
Evaluates the network based on the saved models at different stages of the training network.\n
The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'.
Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
Shuffles: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
plotting: bool, optional
Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False``
show_errors: bool, optional
Display train and test errors. The default is `True``
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
Examples
--------
If you do not want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml', shuffle=[1])
--------
If you want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',shuffle=[1],True)
"""
import os
from skimage import io
import skimage.color
from deeplabcut.pose_estimation_tensorflow.nnet import predict as ptf_predict
from deeplabcut.pose_estimation_tensorflow.config import load_config
from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input
from deeplabcut.utils import auxiliaryfunctions, visualization
import tensorflow as tf
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ[
'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
vers = (tf.__version__).split('.')
if int(vers[0]) == 1 and int(vers[1]) > 12:
TF = tf.compat.v1
else:
TF = tf
TF.reset_default_graph()
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' #
# tf.logging.set_verbosity(tf.logging.WARN)
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: #gpu selectinon
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(cfg["project_path"], str(trainingsetfolder),
'CollectedData_' + cfg["scorer"] + '.h5'),
'df_with_missing')
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in cfg["TrainingFraction"]:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(trainFraction, shuffle,
cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
# Load meta data
data, trainIndices, testIndices, trainFraction = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
#change batch size, if it was edited during analysis!
dlc_cfg['batch_size'] = 1 #in case this was edited for analysis.
#Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg)))
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
#path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split('.')[0]
for fn in os.listdir(os.path.join(str(modelfolder), 'train'))
if "index" in fn
])
try: #check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
increasing_indices = np.argsort(
[int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
final_result = []
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg['init_weights'] = os.path.join(
str(modelfolder), 'train', Snapshots[snapindex]
) #setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg['init_weights'].split(os.sep)[-1]
).split(
'-'
)[-1] #read how many training siterations that corresponds to.
#name for deeplabcut net (based on its parameters)
DLCscorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, trainingsiterations)
print("Running ", DLCscorer, " with # of trainingiterations:",
trainingsiterations)
resultsfilename = os.path.join(
str(evaluationfolder),
DLCscorer + '-' + Snapshots[snapindex] + '.h5')
try:
DataMachine = pd.read_hdf(resultsfilename,
'df_with_missing')
print("This net has already been evaluated!")
except FileNotFoundError:
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = ptf_predict.setup_pose_prediction(
dlc_cfg)
Numimages = len(Data.index)
PredicteData = np.zeros(
(Numimages, 3 * len(dlc_cfg['all_joints_names'])))
print("Analyzing data...")
for imageindex, imagename in tqdm(enumerate(Data.index)):
image = io.imread(os.path.join(cfg['project_path'],
imagename),
mode='RGB')
image = skimage.color.gray2rgb(image)
image_batch = data_to_input(image)
# Compute prediction with the CNN
outputs_np = sess.run(outputs,
feed_dict={inputs: image_batch})
scmap, locref = ptf_predict.extract_cnn_output(
outputs_np, dlc_cfg)
# Extract maximum scoring location from the heatmap, assume 1 person
pose = ptf_predict.argmax_pose_predict(
scmap, locref, dlc_cfg.stride)
PredicteData[imageindex, :] = pose.flatten(
) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts!
sess.close() #closes the current tf session
index = pd.MultiIndex.from_product(
[[DLCscorer], dlc_cfg['all_joints_names'],
['x', 'y', 'likelihood']],
names=['scorer', 'bodyparts', 'coords'])
# Saving results
DataMachine = pd.DataFrame(PredicteData,
columns=index,
index=Data.index.values)
DataMachine.to_hdf(resultsfilename,
'df_with_missing',
format='table',
mode='w')
print("Done and results stored for snapshot: ",
Snapshots[snapindex])
DataCombined = pd.concat([Data.T, DataMachine.T], axis=0).T
RMSE, RMSEpcutoff = pairwisedistances(
DataCombined, cfg["scorer"], DLCscorer, cfg["pcutoff"],
comparisonbodyparts)
testerror = np.nanmean(
RMSE.iloc[testIndices].values.flatten())
trainerror = np.nanmean(
RMSE.iloc[trainIndices].values.flatten())
testerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[testIndices].values.flatten())
trainerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[trainIndices].values.flatten())
results = [
trainingsiterations,
int(100 * trainFraction), shuffle,
np.round(trainerror, 2),
np.round(testerror, 2), cfg["pcutoff"],
np.round(trainerrorpcutoff, 2),
np.round(testerrorpcutoff, 2)
]
final_result.append(results)
if show_errors == True:
print("Results for",
trainingsiterations, " training iterations:",
int(100 * trainFraction), shuffle,
"train error:",
np.round(trainerror, 2), "pixels. Test error:",
np.round(testerror, 2), " pixels.")
print("With pcutoff of",
cfg["pcutoff"], " train error:",
np.round(trainerrorpcutoff,
2), "pixels. Test error:",
np.round(testerrorpcutoff, 2), "pixels")
print(
"Thereby, the errors are given by the average distances between the labels by DLC and the scorer."
)
if plotting == True:
print("Plotting...")
colors = visualization.get_cmap(
len(comparisonbodyparts), name=cfg['colormap'])
foldername = os.path.join(
str(evaluationfolder), 'LabeledImages_' +
DLCscorer + '_' + Snapshots[snapindex])
auxiliaryfunctions.attempttomakefolder(foldername)
NumFrames = np.size(DataCombined.index)
for ind in np.arange(NumFrames):
visualization.PlottingandSaveLabeledFrame(
DataCombined, ind, trainIndices, cfg, colors,
comparisonbodyparts, DLCscorer, foldername)
TF.reset_default_graph()
#print(final_result)
make_results_file(final_result, evaluationfolder, DLCscorer)
print(
"The network is evaluated and the results are stored in the subdirectory 'evaluation_results'."
)
print(
"If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos."
)
print(
"Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)"
)
#returning to intial folder
os.chdir(str(start_path))
def analyze_videos_multiview(config,
videos,
projection_matrices,
multiview_step,
output_folder,
snapshot_index=None,
make_labeled_video=True,
shuffle=1,
trainingsetindex=0):
"""
videos: list of strings
each string is the path to a video. Each video should pertain to a different view
projection_matrices: list of matrices
each projection matrix is a 3x4 numpy array
multiview_step: int
either 1 or 2, denoting whether network was trained via
output_folder: string
a path to a folder in which to write output
make_labeled_video: bool, optional
if True, make a video out of the labeled frames and write it to output_folder
"""
from threading import Thread, Lock
from queue import Queue
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ[
'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
if multiview_step != 1 and multiview_step != 2:
print('multiview_step should be either 1 or 2')
return
tf.reset_default_graph()
start_path = os.getcwd(
) #record cwd to return to this directory in the end
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
modelfolder = os.path.join(
cfg["project_path"],
str(auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
dlc_cfg.multiview_step = multiview_step
if multiview_step == 2:
dlc_cfg.projection_matrices = projection_matrices
# Check which snapshots are available and sort them by # iterations
try:
Snapshots = np.array([
fn.split('.')[0]
for fn in os.listdir(os.path.join(modelfolder, 'train'))
if "index" in fn
])
except FileNotFoundError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s."
% (shuffle, shuffle))
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if snapshot_index is not None:
snapshotindex = -1
for i in range(len(Snapshots)):
if int(Snapshots[i].split('-')[1].split('.')[0]) == snapshot_index:
snapshotindex = i
elif cfg['snapshotindex'] == 'all':
print(
"Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!"
)
snapshotindex = -1
else:
snapshotindex = cfg['snapshotindex']
print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder)
##################################################
# Load and setup CNN part detector
##################################################
# Check if data already was generated:
dlc_cfg['init_weights'] = os.path.join(modelfolder, 'train',
Snapshots[snapshotindex])
trainingsiterations = (dlc_cfg['init_weights'].split(
os.sep)[-1]).split('-')[-1]
dlc_cfg['batch_size'] = 1
# Name for scorer:
DLCscorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, trainingsiterations=trainingsiterations)
sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
caps = [cv2.VideoCapture(video) for video in videos]
for cap in caps:
fps = cap.get(
5
) #https://docs.opencv.org/2.4/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get
nframes = int(cap.get(7))
duration = nframes * 1. / fps
size = (int(cap.get(4)), int(cap.get(3)))
ny, nx = size
print("Duration of video [s]: ", round(duration, 2), ", recorded with ",
round(fps, 2), "fps!")
print("Overall # of frames: ", nframes,
" found with (before cropping) frame dimensions: ", nx, ny)
start = time.time()
print('Extracting pose')
qs = [Queue(maxsize=128) for _ in caps]
q = Queue(maxsize=128)
def extract_one(cap, q):
while cap.isOpened():
ret, frame = cap.read()
if ret:
# frame = cv2.resize(frame, (482, 256))
q.put(frame)
else:
break
q.put(None)
def extract_all(q):
while True:
res = [qq.get() for qq in qs]
if any([frame is None for frame in res]):
q.put(None)
break
q.put(res)
ts = [Thread(target=extract_one, args=z) for z in zip(caps, qs)]
t = Thread(target=extract_all, args=(q, ))
for tt in ts:
tt.start()
t.start()
poses = []
counter = 0
for _ in tqdm(range(nframes)):
frames = q.get()
if frames is not None:
frames = [
cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) for frame in frames
]
frames = [img_as_ubyte(frame) for frame in frames]
out = sess.run(outputs, feed_dict={inputs: frames})
if multiview_step == 2:
poses.append(out[2])
continue
scmap, locref = predict.extract_cnn_output(out, dlc_cfg)
pose = predict.argmax_pose_predict(scmap, locref, dlc_cfg.stride)
poses.append(pose)
else:
nframes = counter
break
counter += 1
print('Extracted pose for %d frames' % nframes)
if multiview_step == 1:
poses = np.array(poses) # nframes x num_views x num_joints x 3
num_views = poses.shape[1]
results = poses.reshape([nframes, -1])
pdindex = pd.MultiIndex.from_product(
[[DLCscorer], ['view_%d' % i for i in range(poses.shape[1])],
dlc_cfg['all_joints_names'], ['x', 'y', 'likelihood']],
names=['scorer', 'views', 'bodyparts', 'coords'])
results = pd.DataFrame(data=results, columns=pdindex)
results.to_hdf(os.path.join(output_folder, '2dposes.h5'),
key='results')
results.to_csv(os.path.join(output_folder, '2dposes.csv'))
poses = np.transpose(poses,
[0, 2, 1, 3]).reshape([-1, num_views,
3]) # / [[[482, 256, 1]]]
scores = np.copy(poses[:, :, 2])
poses[:, :, 2] = 1
preds3d = project_3d(projection_matrices, poses, confidences=scores)
preds3d[~np.isfinite(preds3d)] = 0
preds3d = preds3d.reshape([nframes, -1])
elif multiview_step == 2:
preds3d = np.array(poses) # nframes x num_joints x 3
preds3d = preds3d.reshape([nframes, -1])
pdindex = pd.MultiIndex.from_product(
[[DLCscorer], dlc_cfg['all_joints_names'], ['x', 'y', 'z']],
names=['scorer', 'bodyparts', 'coords'])
results = pd.DataFrame(preds3d, columns=pdindex)
results.to_hdf(os.path.join(output_folder, '3dposes.h5'), key='results')
results.to_csv(os.path.join(output_folder, '3dposes.csv'))
if make_labeled_video:
print('making 3d video')
make_3d_labeled_video(preds3d.reshape([nframes, -1, 3]), output_folder)
def evaluate_multiview_network(config,videos,projection_matrices,multiview_step,snapshot_index=None,Shuffles=[1],plotting = None,show_errors = True,comparisonbodyparts="all",gputouse=None):
"""
Evaluates the network based on the saved models at different stages of the training network.\n
The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'.
Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos: list of strings
Name of each video, one per viewpoint. Must be in the same order that it was in for training
projection_matrices: list of arrays
Projection matrix for each viewpoint. Each is a 3x4 array
multiview_step:
1 or 2. Indicates whether network was trained with train_multiview_network_step_1 or train_multiview_network_step_2
Shuffles: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
plotting: bool, optional
Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False``
show_errors: bool, optional
Display train and test errors. The default is `True``
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
Examples
--------
If you do not want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml', shuffle=[1])
--------
If you want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',shuffle=[1],True)
"""
import os
from skimage import io
import skimage.color
from deeplabcut.pose_estimation_tensorflow.nnet import predict as ptf_predict
from deeplabcut.pose_estimation_tensorflow.config import load_config
from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input
from deeplabcut.utils import auxiliaryfunctions, visualization
import tensorflow as tf
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ['TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
tf.reset_default_graph()
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' #
# tf.logging.set_verbosity(tf.logging.WARN)
start_path=os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: #gpu selectinon
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
# Loading human annotatated data
trainingsetfolder=auxiliaryfunctions.GetTrainingSetFolder(cfg)
Datas = [pd.read_hdf(os.path.join(cfg['project_path'], 'labeled-data', video, 'CollectedData_'+cfg['scorer']+'.h5'), 'df_with_missing') for video in videos]
# Get list of body parts to evaluate network for
comparisonbodyparts=auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(cfg,comparisonbodyparts)
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(str(cfg["project_path"]+"/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in cfg["TrainingFraction"]:
##################################################
# Load and setup CNN part detector
##################################################
datafn,metadatafn=auxiliaryfunctions.GetDataandMetaDataFilenames(trainingsetfolder,trainFraction,shuffle,cfg)
modelfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
# Load meta data
metadatas = []
for video in videos:
m = ('-'+video).join(os.path.splitext(metadatafn))
data, trainIndices, testIndices, trainFraction=auxiliaryfunctions.LoadMetadata(os.path.join(cfg["project_path"],m))
metadatas.append(data)
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError("It seems the model for shuffle %s and trainFraction %s does not exist."%(shuffle,trainFraction))
#change batch size, if it was edited during analysis!
dlc_cfg['batch_size']=1 #in case this was edited for analysis.
#Create folder structure to store results.
evaluationfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetEvaluationFolder(trainFraction,shuffle,cfg)))
auxiliaryfunctions.attempttomakefolder(evaluationfolder,recursive=True)
#path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
dlc_cfg.multiview_step = multiview_step
dlc_cfg.projection_matrices = projection_matrices
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([fn.split('.')[0]for fn in os.listdir(os.path.join(str(modelfolder), 'train'))if "index" in fn])
try: #check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError("Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."%(shuffle,trainFraction))
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if snapshot_index is not None:
snapindices = [i for i in range(len(Snapshots)) if int(Snapshots[i].split('-')[1].split('.')[0])==snapshot_index]
elif cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"]<len(Snapshots):
snapindices=[cfg["snapshotindex"]]
else:
print("Invalid choice, only -1 (last), any integer up to last, or all (as string)!")
final_result=[]
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg['init_weights'] = os.path.join(str(modelfolder),'train',Snapshots[snapindex]) #setting weights to corresponding snapshot.
trainingsiterations = (dlc_cfg['init_weights'].split(os.sep)[-1]).split('-')[-1] #read how many training siterations that corresponds to.
#name for deeplabcut net (based on its parameters)
DLCscorer = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations)
print("Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations)
resultsfilename=os.path.join(str(evaluationfolder),DLCscorer + '-' + Snapshots[snapindex]+ '.h5')
try:
DataMachine = pd.read_hdf(resultsfilename,'df_with_missing')
print("This net has already been evaluated!")
except FileNotFoundError:
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = ptf_predict.setup_pose_prediction(dlc_cfg)
Numimages = len(Datas[0].index)
PredicteDatas = np.zeros((Numimages,len(Datas), 3 * len(dlc_cfg['all_joints_names'])))
imagesizes = []
print("Analyzing data...")
if multiview_step == 1:
for imageindex in tqdm(range(len(Datas[0].index))):
imagenames = [Data.index[imageindex] for Data in Datas]
images = [io.imread(os.path.join(cfg['project_path'],imagename),mode='RGB') for imagename in imagenames]
images = [skimage.color.gray2rgb(image) for image in images]
image_batch = images
imagesizes.append([image.shape for image in images])
# Compute prediction with the CNN
outputs_np = sess.run(outputs, feed_dict={inputs: image_batch})
scmap, locref = ptf_predict.extract_cnn_output(outputs_np, dlc_cfg)
# Extract maximum scoring location from the heatmap, assume 1 person
pose = ptf_predict.argmax_pose_predict(scmap, locref, dlc_cfg.stride)
PredicteDatas[imageindex] = pose.reshape([pose.shape[0], -1]) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts!
sess.close() #closes the current tf session
index = pd.MultiIndex.from_product(
[[DLCscorer], dlc_cfg['all_joints_names'], ['x', 'y', 'likelihood']],
names=['scorer', 'bodyparts', 'coords'])
# Saving results
for i, video in enumerate(videos):
print('Evaluating 2D predictions on video %s'%video)
Data = Datas[i]
DataMachine = pd.DataFrame(PredicteDatas[:,i], columns=index, index=Data.index.values)
r = ('-'+video).join(os.path.splitext(resultsfilename))
DataMachine.to_hdf(r,'df_with_missing',format='table',mode='w')
print("Done and results stored for snapshot: ", Snapshots[snapindex])
DataCombined = pd.concat([Data.T, DataMachine.T], axis=0).T
RMSE,RMSEpcutoff = pairwisedistances(DataCombined, cfg["scorer"], DLCscorer,cfg["pcutoff"],comparisonbodyparts)
testerror = np.nanmean(RMSE.iloc[testIndices].values.flatten())
trainerror = np.nanmean(RMSE.iloc[trainIndices].values.flatten())
testerrorpcutoff = np.nanmean(RMSEpcutoff.iloc[testIndices].values.flatten())
trainerrorpcutoff = np.nanmean(RMSEpcutoff.iloc[trainIndices].values.flatten())
results = [trainingsiterations,int(100 * trainFraction),shuffle,np.round(trainerror,2),np.round(testerror,2),cfg["pcutoff"],np.round(trainerrorpcutoff,2), np.round(testerrorpcutoff,2)]
final_result.append(results)
if show_errors == True:
print("Results for",trainingsiterations," training iterations:", int(100 * trainFraction), shuffle, "train error:",np.round(trainerror,2), "pixels. Test error:", np.round(testerror,2)," pixels.")
print("With pcutoff of", cfg["pcutoff"]," train error:",np.round(trainerrorpcutoff,2), "pixels. Test error:", np.round(testerrorpcutoff,2), "pixels")
print("Thereby, the errors are given by the average distances between the labels by DLC and the scorer.")
if plotting == True:
print("Plotting...")
colors = visualization.get_cmap(len(comparisonbodyparts),name=cfg['colormap'])
foldername=os.path.join(str(evaluationfolder),'LabeledImages_' + DLCscorer + '_' + Snapshots[snapindex]+'_'+video)
auxiliaryfunctions.attempttomakefolder(foldername)
NumFrames=np.size(DataCombined.index)
for ind in np.arange(NumFrames):
visualization.PlottingandSaveLabeledFrame(DataCombined,ind,trainIndices,cfg,colors,comparisonbodyparts,DLCscorer,foldername)
# get predictions in homogeneous pixel coordinates
# pixel coordinates have (0,0) in the top-left, and the bottom-right coordinate is (h,w)
predictions = PredicteDatas.reshape(Numimages, len(Datas), len(dlc_cfg['all_joints_names']), 3)
scores = np.copy(predictions[:,:,:,2])
predictions[:,:,:,2] = 1.0 # homogeneous coordinates; (x,y,1). Top-left corner is (-width/2, -height/2, 1); Bottom-right corner is opposite. Shape is num_images x num_views x num_joints x 3
num_ims, num_views, num_joints, _ = predictions.shape
# get labels in homogeneous pixel coordinates
labels = np.array([Data.values.reshape(num_ims, num_joints, 2) for Data in Datas]) # num_views x num_ims x num_joints x (x,y)
labels = np.transpose(labels, [1, 2, 0, 3]) # num_ims x num_joints x num_views x (x,y)
labels = np.concatenate([labels, np.ones([num_ims, num_joints, num_views, 1])], axis=3)
# solve linear system to get labels in 3D
# helpful explanation of equation found on pg 5 here: https://hal.inria.fr/inria-00524401/PDF/Sturm-cvpr05.pdf
labs = labels.reshape([num_ims * num_joints, num_views, 3]).astype(np.float)
confidences = ~np.isnan(np.sum(labs, axis=2))
valid = np.sum(~np.isnan(np.sum(labs, axis=2)), axis=1) >= 2
labs[~confidences] = 0
labels3d = project_3d(projection_matrices, labs, confidences=confidences)
labels3d[~valid] = np.nan
labels3d = labels3d.reshape([num_ims, num_joints, 3])
# solve linear system to get 3D predictions
preds = np.transpose(predictions, [0, 2, 1, 3]) # num_ims x num_joints x num_views x 3
preds = preds.reshape([num_ims*num_joints, num_views, 3])
preds3d = project_3d(projection_matrices, preds)
preds3d = preds3d.reshape([num_ims, num_joints, 3])
# try it with confidence weighting
scores = np.transpose(scores, [0, 2, 1]) # num_images x num_joints x num_views
scores = np.reshape(scores, [num_ims*num_joints, num_views])
preds3d_weighted = project_3d(projection_matrices, preds, confidences=scores)
preds3d_weighted = preds3d_weighted.reshape([num_ims, num_joints, 3])
# try it with the pcutoff
scores2 = np.copy(scores)
scores2[scores2 < cfg["pcutoff"]] = 0
preds3d_weighted_cutoff = project_3d(projection_matrices, preds, confidences=scores2)
preds3d_weighted_cutoff = preds3d_weighted_cutoff.reshape([num_ims, num_joints, 3])
print("\n\n3D errors:")
RMSE_train = np.nanmean(np.nansum((preds3d[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5)
RMSE_test = np.nanmean(np.nansum((preds3d[testIndices] - labels3d[testIndices])**2, axis=2)**0.5)
RMSE_train_weighted = np.nanmean(np.nansum((preds3d_weighted[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5)
RMSE_test_weighted = np.nanmean(np.nansum((preds3d_weighted[testIndices] - labels3d[testIndices])**2, axis=2)**0.5)
RMSE_train_weighted_cutoff = np.nanmean(np.nansum((preds3d_weighted_cutoff[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5)
RMSE_test_weighted_cutoff = np.nanmean(np.nansum((preds3d_weighted_cutoff[testIndices] - labels3d[testIndices])**2, axis=2)**0.5)
print("RMSE train: ", RMSE_train)
print("RMSE test: ", RMSE_test)
print("RMSE train weighted: ", RMSE_train_weighted)
print("RMSE test weighted: ", RMSE_test_weighted)
print("RMSE train weighted cutoff: ", RMSE_train_weighted_cutoff)
print("RMSE test weighted cutoff: ", RMSE_test_weighted_cutoff)
tail = np.nansum((preds3d_weighted - labels3d)**2, axis=2)**0.5
tail = np.sort(tail[~np.isnan(tail)])
tail = tail[-10:]
print('10 worst predictions: ', tail)
tf.reset_default_graph()
elif multiview_step==2:
preds3d = []
for imageindex in tqdm(range(len(Datas[0].index))):
imagenames = [Data.index[imageindex] for Data in Datas]
images = [io.imread(os.path.join(cfg['project_path'],imagename),mode='RGB') for imagename in imagenames]
images = [skimage.color.gray2rgb(image) for image in images]
image_batch = images
# Compute prediction with the CNN
outputs_np = sess.run(outputs, feed_dict={inputs: image_batch})
pred_3d = outputs_np[2]
preds3d.append(pred_3d)
sess.close() #closes the current tf session
preds3d = np.array(preds3d) # num_ims x num_joints x (x,y,z)
num_ims, num_joints = preds3d.shape[:2]
num_views = dlc_cfg.num_views
# get labels in homogeneous pixel coordinates
labels = np.array([Data.values.reshape(num_ims, num_joints, 2) for Data in Datas]) # num_views x num_ims x num_joints x (x,y)
labels = np.transpose(labels, [1, 2, 0, 3]) # num_ims x num_joints x num_views x (x,y)
labels = np.concatenate([labels, np.ones([num_ims, num_joints, num_views, 1])], axis=3)
# solve linear system to get labels in 3D
# helpful explanation of equation found on pg 5 here: https://hal.inria.fr/inria-00524401/PDF/Sturm-cvpr05.pdf
labs = labels.reshape([num_ims * num_joints, num_views, 3]).astype(np.float)
confidences = ~np.isnan(np.sum(labs, axis=2))
valid = np.sum(~np.isnan(np.sum(labs, axis=2)), axis=1) >= 2
labs[~confidences] = 0
labels3d = project_3d(projection_matrices, labs, confidences=confidences)
labels3d[~valid] = np.nan
labels3d = labels3d.reshape([num_ims, num_joints, 3])
print("\n\n3D errors (units are determined by projection matrices):")
RMSE_train = np.nanmean(np.nansum((preds3d[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5)
RMSE_test = np.nanmean(np.nansum((preds3d[testIndices] - labels3d[testIndices])**2, axis=2)**0.5)
print("RMSE train: ", RMSE_train)
print("RMSE test: ", RMSE_test)
tail = np.nansum((preds3d- labels3d)**2, axis=2)**0.5
tail = np.sort(tail[~np.isnan(tail)])
tail = tail[-10:]
print('10 worst predictions: ', tail)
tf.reset_default_graph()
else:
print('invalid multiview_step given')
return
make_results_file(final_result,evaluationfolder,DLCscorer)
print("The network is evaluated and the results are stored in the subdirectory 'evaluation_results'.")
print("If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos.")
print("Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)")
#returning to intial folder
os.chdir(str(start_path))
