def create_labeled_video(config,
videos,
videotype='avi',
shuffle=1,
trainingsetindex=0,
save_frames=False,
Frames2plot=None,
delete=False,
displayedbodyparts='all',
codec='mp4v',
outputframerate=None,
destfolder=None):
"""
Labels the bodyparts in a video. Make sure the video is already analyzed by the function 'analyze_video'
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
Number of shuffles of training dataset. Default is set to 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
videotype: string, optional
Checks for the extension of the video in case the input is a directory.\nOnly videos with this extension are analyzed. The default is ``.avi``
save_frames: bool
If true creates each frame individual and then combines into a video. This variant is relatively slow as
it stores all individual frames. However, it uses matplotlib to create the frames and is therefore much more flexible (one can set transparency of markers, crop, and easily customize).
Frames2plot: List of indices
If not None & save_frames=True then the frames corresponding to the index will be plotted. For example, Frames2plot=[0,11] will plot the first and the 12th frame.
delete: bool
If true then the individual frames created during the video generation will be deleted.
displayedbodyparts: list of strings, optional
This select the body parts that are plotted in the video. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
codec: codec for labeled video. Options see http://www.fourcc.org/codecs.php [depends on your ffmpeg installation.]
outputframerate: positive number, output frame rate for labeled video (only available for the mode with saving frames.) By default: None, which results in the original video rate.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
Examples
--------
If you want to create the labeled video for only 1 video
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to create the labeled video for only 1 video and store the individual frames
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'],save_frames=True)
--------
If you want to create the labeled video for multiple videos
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to create the labeled video for all the videos (as .avi extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'])
--------
If you want to create the labeled video for all the videos (as .mp4 extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'],videotype='mp4')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
DLCscorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction
) #automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
if destfolder is None:
#videofolder = str(Path(video).parents[0])
videofolder = Path(video).parents[
0] #where your folder with videos is.
else:
videofolder = destfolder
os.chdir(str(videofolder))
videotype = Path(video).suffix
print("Starting % ", videofolder, videos)
vname = str(Path(video).stem)
if os.path.isfile(
os.path.join(str(videofolder),
vname + DLCscorer + '_labeled.mp4')):
print("Labeled video already created.")
else:
print("Loading ", video, "and data.")
dataname = os.path.join(str(videofolder),
vname + DLCscorer + '.h5')
try:
Dataframe = pd.read_hdf(dataname)
metadata = auxiliaryfunctions.LoadVideoMetadata(dataname)
#print(metadata)
datanames = [dataname]
except FileNotFoundError:
datanames = [
fn for fn in os.listdir(os.curdir)
if (vname in fn) and (".h5" in fn) and "resnet" in fn
]
if len(datanames) == 0:
print("The video was not analyzed with this scorer:",
DLCscorer)
print(
"No other scorers were found, please use the function 'analyze_videos' first."
)
elif len(datanames) > 0:
print("The video was not analyzed with this scorer:",
DLCscorer)
print("Other scorers were found, however:", datanames)
DLCscorer = 'DeepCut' + (
datanames[0].split('DeepCut')[1]).split('.h5')[0]
print("Creating labeled video for:", DLCscorer,
" instead.")
Dataframe = pd.read_hdf(datanames[0])
metadata = auxiliaryfunctions.LoadVideoMetadata(
datanames[0])
if len(datanames) > 0:
#Loading cropping data used during analysis
cropping = metadata['data']["cropping"]
[x1, x2, y1, y2] = metadata['data']["cropping_parameters"]
print(cropping, x1, x2, y1, y2)
if save_frames == True:
tmpfolder = os.path.join(str(videofolder), 'temp-' + vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder)
clip = vp(video)
#CreateVideoSlow(clip,Dataframe,tmpfolder,cfg["dotsize"],cfg["colormap"],cfg["alphavalue"],cfg["pcutoff"],cfg["cropping"],cfg["x1"],cfg["x2"],cfg["y1"],cfg["y2"],delete,DLCscorer,bodyparts)
CreateVideoSlow(clip, Dataframe, tmpfolder, cfg["dotsize"],
cfg["colormap"], cfg["alphavalue"],
cfg["pcutoff"], cropping, x1, x2, y1, y2,
delete, DLCscorer, bodyparts,
outputframerate, Frames2plot)
else:
clip = vp(fname=video,
sname=os.path.join(vname + DLCscorer +
'_labeled.mp4'),
codec=codec)
if cropping:
print(
"Fast video creation has currently not been implemented for cropped videos. Please use 'save_frames=True' to get the video."
)
else:
CreateVideo(clip, Dataframe, cfg["pcutoff"],
cfg["dotsize"], cfg["colormap"], DLCscorer,
bodyparts, cropping, x1, x2, y1,
y2) #NEED TO ADD CROPPING!
def evaluate_multiview_network(config,videos,projection_matrices,multiview_step,snapshot_index=None,Shuffles=[1],plotting = None,show_errors = True,comparisonbodyparts="all",gputouse=None):
"""
Evaluates the network based on the saved models at different stages of the training network.\n
The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'.
Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos: list of strings
Name of each video, one per viewpoint. Must be in the same order that it was in for training
projection_matrices: list of arrays
Projection matrix for each viewpoint. Each is a 3x4 array
multiview_step:
1 or 2. Indicates whether network was trained with train_multiview_network_step_1 or train_multiview_network_step_2
Shuffles: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
plotting: bool, optional
Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False``
show_errors: bool, optional
Display train and test errors. The default is `True``
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
Examples
--------
If you do not want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml', shuffle=[1])
--------
If you want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',shuffle=[1],True)
"""
import os
from skimage import io
import skimage.color
from deeplabcut.pose_estimation_tensorflow.nnet import predict as ptf_predict
from deeplabcut.pose_estimation_tensorflow.config import load_config
from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input
from deeplabcut.utils import auxiliaryfunctions, visualization
import tensorflow as tf
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ['TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
tf.reset_default_graph()
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' #
# tf.logging.set_verbosity(tf.logging.WARN)
start_path=os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: #gpu selectinon
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
# Loading human annotatated data
trainingsetfolder=auxiliaryfunctions.GetTrainingSetFolder(cfg)
Datas = [pd.read_hdf(os.path.join(cfg['project_path'], 'labeled-data', video, 'CollectedData_'+cfg['scorer']+'.h5'), 'df_with_missing') for video in videos]
# Get list of body parts to evaluate network for
comparisonbodyparts=auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(cfg,comparisonbodyparts)
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(str(cfg["project_path"]+"/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in cfg["TrainingFraction"]:
##################################################
# Load and setup CNN part detector
##################################################
datafn,metadatafn=auxiliaryfunctions.GetDataandMetaDataFilenames(trainingsetfolder,trainFraction,shuffle,cfg)
modelfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
# Load meta data
metadatas = []
for video in videos:
m = ('-'+video).join(os.path.splitext(metadatafn))
data, trainIndices, testIndices, trainFraction=auxiliaryfunctions.LoadMetadata(os.path.join(cfg["project_path"],m))
metadatas.append(data)
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError("It seems the model for shuffle %s and trainFraction %s does not exist."%(shuffle,trainFraction))
#change batch size, if it was edited during analysis!
dlc_cfg['batch_size']=1 #in case this was edited for analysis.
#Create folder structure to store results.
evaluationfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetEvaluationFolder(trainFraction,shuffle,cfg)))
auxiliaryfunctions.attempttomakefolder(evaluationfolder,recursive=True)
#path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
dlc_cfg.multiview_step = multiview_step
dlc_cfg.projection_matrices = projection_matrices
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([fn.split('.')[0]for fn in os.listdir(os.path.join(str(modelfolder), 'train'))if "index" in fn])
try: #check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError("Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."%(shuffle,trainFraction))
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if snapshot_index is not None:
snapindices = [i for i in range(len(Snapshots)) if int(Snapshots[i].split('-')[1].split('.')[0])==snapshot_index]
elif cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"]<len(Snapshots):
snapindices=[cfg["snapshotindex"]]
else:
print("Invalid choice, only -1 (last), any integer up to last, or all (as string)!")
final_result=[]
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg['init_weights'] = os.path.join(str(modelfolder),'train',Snapshots[snapindex]) #setting weights to corresponding snapshot.
trainingsiterations = (dlc_cfg['init_weights'].split(os.sep)[-1]).split('-')[-1] #read how many training siterations that corresponds to.
#name for deeplabcut net (based on its parameters)
DLCscorer = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations)
print("Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations)
resultsfilename=os.path.join(str(evaluationfolder),DLCscorer + '-' + Snapshots[snapindex]+ '.h5')
try:
DataMachine = pd.read_hdf(resultsfilename,'df_with_missing')
print("This net has already been evaluated!")
except FileNotFoundError:
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = ptf_predict.setup_pose_prediction(dlc_cfg)
Numimages = len(Datas[0].index)
PredicteDatas = np.zeros((Numimages,len(Datas), 3 * len(dlc_cfg['all_joints_names'])))
imagesizes = []
print("Analyzing data...")
if multiview_step == 1:
for imageindex in tqdm(range(len(Datas[0].index))):
imagenames = [Data.index[imageindex] for Data in Datas]
images = [io.imread(os.path.join(cfg['project_path'],imagename),mode='RGB') for imagename in imagenames]
images = [skimage.color.gray2rgb(image) for image in images]
image_batch = images
imagesizes.append([image.shape for image in images])
# Compute prediction with the CNN
outputs_np = sess.run(outputs, feed_dict={inputs: image_batch})
scmap, locref = ptf_predict.extract_cnn_output(outputs_np, dlc_cfg)
# Extract maximum scoring location from the heatmap, assume 1 person
pose = ptf_predict.argmax_pose_predict(scmap, locref, dlc_cfg.stride)
PredicteDatas[imageindex] = pose.reshape([pose.shape[0], -1]) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts!
sess.close() #closes the current tf session
index = pd.MultiIndex.from_product(
[[DLCscorer], dlc_cfg['all_joints_names'], ['x', 'y', 'likelihood']],
names=['scorer', 'bodyparts', 'coords'])
# Saving results
for i, video in enumerate(videos):
print('Evaluating 2D predictions on video %s'%video)
Data = Datas[i]
DataMachine = pd.DataFrame(PredicteDatas[:,i], columns=index, index=Data.index.values)
r = ('-'+video).join(os.path.splitext(resultsfilename))
DataMachine.to_hdf(r,'df_with_missing',format='table',mode='w')
print("Done and results stored for snapshot: ", Snapshots[snapindex])
DataCombined = pd.concat([Data.T, DataMachine.T], axis=0).T
RMSE,RMSEpcutoff = pairwisedistances(DataCombined, cfg["scorer"], DLCscorer,cfg["pcutoff"],comparisonbodyparts)
testerror = np.nanmean(RMSE.iloc[testIndices].values.flatten())
trainerror = np.nanmean(RMSE.iloc[trainIndices].values.flatten())
testerrorpcutoff = np.nanmean(RMSEpcutoff.iloc[testIndices].values.flatten())
trainerrorpcutoff = np.nanmean(RMSEpcutoff.iloc[trainIndices].values.flatten())
results = [trainingsiterations,int(100 * trainFraction),shuffle,np.round(trainerror,2),np.round(testerror,2),cfg["pcutoff"],np.round(trainerrorpcutoff,2), np.round(testerrorpcutoff,2)]
final_result.append(results)
if show_errors == True:
print("Results for",trainingsiterations," training iterations:", int(100 * trainFraction), shuffle, "train error:",np.round(trainerror,2), "pixels. Test error:", np.round(testerror,2)," pixels.")
print("With pcutoff of", cfg["pcutoff"]," train error:",np.round(trainerrorpcutoff,2), "pixels. Test error:", np.round(testerrorpcutoff,2), "pixels")
print("Thereby, the errors are given by the average distances between the labels by DLC and the scorer.")
if plotting == True:
print("Plotting...")
colors = visualization.get_cmap(len(comparisonbodyparts),name=cfg['colormap'])
foldername=os.path.join(str(evaluationfolder),'LabeledImages_' + DLCscorer + '_' + Snapshots[snapindex]+'_'+video)
auxiliaryfunctions.attempttomakefolder(foldername)
NumFrames=np.size(DataCombined.index)
for ind in np.arange(NumFrames):
visualization.PlottingandSaveLabeledFrame(DataCombined,ind,trainIndices,cfg,colors,comparisonbodyparts,DLCscorer,foldername)
# get predictions in homogeneous pixel coordinates
# pixel coordinates have (0,0) in the top-left, and the bottom-right coordinate is (h,w)
predictions = PredicteDatas.reshape(Numimages, len(Datas), len(dlc_cfg['all_joints_names']), 3)
scores = np.copy(predictions[:,:,:,2])
predictions[:,:,:,2] = 1.0 # homogeneous coordinates; (x,y,1). Top-left corner is (-width/2, -height/2, 1); Bottom-right corner is opposite. Shape is num_images x num_views x num_joints x 3
num_ims, num_views, num_joints, _ = predictions.shape
# get labels in homogeneous pixel coordinates
labels = np.array([Data.values.reshape(num_ims, num_joints, 2) for Data in Datas]) # num_views x num_ims x num_joints x (x,y)
labels = np.transpose(labels, [1, 2, 0, 3]) # num_ims x num_joints x num_views x (x,y)
labels = np.concatenate([labels, np.ones([num_ims, num_joints, num_views, 1])], axis=3)
# solve linear system to get labels in 3D
# helpful explanation of equation found on pg 5 here: https://hal.inria.fr/inria-00524401/PDF/Sturm-cvpr05.pdf
labs = labels.reshape([num_ims * num_joints, num_views, 3]).astype(np.float)
confidences = ~np.isnan(np.sum(labs, axis=2))
valid = np.sum(~np.isnan(np.sum(labs, axis=2)), axis=1) >= 2
labs[~confidences] = 0
labels3d = project_3d(projection_matrices, labs, confidences=confidences)
labels3d[~valid] = np.nan
labels3d = labels3d.reshape([num_ims, num_joints, 3])
# solve linear system to get 3D predictions
preds = np.transpose(predictions, [0, 2, 1, 3]) # num_ims x num_joints x num_views x 3
preds = preds.reshape([num_ims*num_joints, num_views, 3])
preds3d = project_3d(projection_matrices, preds)
preds3d = preds3d.reshape([num_ims, num_joints, 3])
# try it with confidence weighting
scores = np.transpose(scores, [0, 2, 1]) # num_images x num_joints x num_views
scores = np.reshape(scores, [num_ims*num_joints, num_views])
preds3d_weighted = project_3d(projection_matrices, preds, confidences=scores)
preds3d_weighted = preds3d_weighted.reshape([num_ims, num_joints, 3])
# try it with the pcutoff
scores2 = np.copy(scores)
scores2[scores2 < cfg["pcutoff"]] = 0
preds3d_weighted_cutoff = project_3d(projection_matrices, preds, confidences=scores2)
preds3d_weighted_cutoff = preds3d_weighted_cutoff.reshape([num_ims, num_joints, 3])
print("\n\n3D errors:")
RMSE_train = np.nanmean(np.nansum((preds3d[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5)
RMSE_test = np.nanmean(np.nansum((preds3d[testIndices] - labels3d[testIndices])**2, axis=2)**0.5)
RMSE_train_weighted = np.nanmean(np.nansum((preds3d_weighted[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5)
RMSE_test_weighted = np.nanmean(np.nansum((preds3d_weighted[testIndices] - labels3d[testIndices])**2, axis=2)**0.5)
RMSE_train_weighted_cutoff = np.nanmean(np.nansum((preds3d_weighted_cutoff[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5)
RMSE_test_weighted_cutoff = np.nanmean(np.nansum((preds3d_weighted_cutoff[testIndices] - labels3d[testIndices])**2, axis=2)**0.5)
print("RMSE train: ", RMSE_train)
print("RMSE test: ", RMSE_test)
print("RMSE train weighted: ", RMSE_train_weighted)
print("RMSE test weighted: ", RMSE_test_weighted)
print("RMSE train weighted cutoff: ", RMSE_train_weighted_cutoff)
print("RMSE test weighted cutoff: ", RMSE_test_weighted_cutoff)
tail = np.nansum((preds3d_weighted - labels3d)**2, axis=2)**0.5
tail = np.sort(tail[~np.isnan(tail)])
tail = tail[-10:]
print('10 worst predictions: ', tail)
tf.reset_default_graph()
elif multiview_step==2:
preds3d = []
for imageindex in tqdm(range(len(Datas[0].index))):
imagenames = [Data.index[imageindex] for Data in Datas]
images = [io.imread(os.path.join(cfg['project_path'],imagename),mode='RGB') for imagename in imagenames]
images = [skimage.color.gray2rgb(image) for image in images]
image_batch = images
# Compute prediction with the CNN
outputs_np = sess.run(outputs, feed_dict={inputs: image_batch})
pred_3d = outputs_np[2]
preds3d.append(pred_3d)
sess.close() #closes the current tf session
preds3d = np.array(preds3d) # num_ims x num_joints x (x,y,z)
num_ims, num_joints = preds3d.shape[:2]
num_views = dlc_cfg.num_views
# get labels in homogeneous pixel coordinates
labels = np.array([Data.values.reshape(num_ims, num_joints, 2) for Data in Datas]) # num_views x num_ims x num_joints x (x,y)
labels = np.transpose(labels, [1, 2, 0, 3]) # num_ims x num_joints x num_views x (x,y)
labels = np.concatenate([labels, np.ones([num_ims, num_joints, num_views, 1])], axis=3)
# solve linear system to get labels in 3D
# helpful explanation of equation found on pg 5 here: https://hal.inria.fr/inria-00524401/PDF/Sturm-cvpr05.pdf
labs = labels.reshape([num_ims * num_joints, num_views, 3]).astype(np.float)
confidences = ~np.isnan(np.sum(labs, axis=2))
valid = np.sum(~np.isnan(np.sum(labs, axis=2)), axis=1) >= 2
labs[~confidences] = 0
labels3d = project_3d(projection_matrices, labs, confidences=confidences)
labels3d[~valid] = np.nan
labels3d = labels3d.reshape([num_ims, num_joints, 3])
print("\n\n3D errors (units are determined by projection matrices):")
RMSE_train = np.nanmean(np.nansum((preds3d[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5)
RMSE_test = np.nanmean(np.nansum((preds3d[testIndices] - labels3d[testIndices])**2, axis=2)**0.5)
print("RMSE train: ", RMSE_train)
print("RMSE test: ", RMSE_test)
tail = np.nansum((preds3d- labels3d)**2, axis=2)**0.5
tail = np.sort(tail[~np.isnan(tail)])
tail = tail[-10:]
print('10 worst predictions: ', tail)
tf.reset_default_graph()
else:
print('invalid multiview_step given')
return
make_results_file(final_result,evaluationfolder,DLCscorer)
print("The network is evaluated and the results are stored in the subdirectory 'evaluation_results'.")
print("If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos.")
print("Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)")
#returning to intial folder
os.chdir(str(start_path))
def generate_prediction(MAX_PREDICTION_STEPS=1000):
"""
Generator for predicting image
MAX_PREDICTION_STEPS : Number of predictions that should be done before re-initializing
"""
##################################################
# Clone arguments from deeplabcut.evaluate_network
##################################################
config = "/root/DLCROS_ws/Surgical_Tool_Tracking/ForwardPassDeepLabCut/DaVinci-Ambar-2019-10-31/config.yaml"
Shuffles = [1]
plotting = None
show_errors = True
comparisonbodyparts = "all"
gputouse = None
# Suppress scientific notation while printing
np.set_printoptions(suppress=True)
##################################################
# SETUP everything until image prediction
##################################################
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ[
'TF_CUDNN_USE_AUTOTUNE'] # was potentially set during training
vers = tf.__version__.split('.')
if int(vers[0]) == 1 and int(vers[1]) > 12:
TF = tf.compat.v1
else:
TF = tf
TF.reset_default_graph()
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
#tf.logging.set_verbosity(tf.logging.WARN)
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: # gpu selectinon
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
##############
# Cloning for-loop variables
shuffle = Shuffles[0]
trainFraction = cfg["TrainingFraction"][0]
##############
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
##################################################
# Load and setup CNN part detector
##################################################
modelfolder = os.path.join(
cfg["project_path"],
str(auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
# Load meta data
# data, trainIndices, testIndices, trainFraction = auxiliaryfunctions.LoadMetadata(
# os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle s and trainFraction %s does not exist."
)
dlc_cfg['batch_size'] = 1 # in case this was edited for analysis.
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split('.')[0]
for fn in os.listdir(os.path.join(str(modelfolder), 'train'))
if "index" in fn
])
try: # check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle and "
"trainFraction is not trained.\nPlease train it before evaluating."
"\nUse the function 'train_network' to do so.")
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
##################################################
# Compute predictions over image
##################################################
for snapindex in snapindices:
dlc_cfg['init_weights'] = os.path.join(
str(modelfolder), 'train',
Snapshots[snapindex]) # setting weights to corresponding snapshot.
trainingsiterations = (dlc_cfg['init_weights'].split(
os.sep)[-1]).split('-')[
-1] # read how many training siterations that corresponds to.
# name for deeplabcut net (based on its parameters)
DLCscorer = auxiliaryfunctions.GetScorerName(cfg, shuffle,
trainFraction,
trainingsiterations)
print("Running ", DLCscorer, " with # of trainingiterations:",
trainingsiterations)
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = ptf_predict.setup_pose_prediction(dlc_cfg)
# Using GPU for prediction
# Specifying state of model (snapshot / training state)
# sess, inputs, outputs = ptf_predict.setup_GPUpose_prediction(dlc_cfg)
print("Analyzing test image ...")
imagename = "img034.png"
image = io.imread(imagename, plugin='matplotlib')
count = 0
start_time = time.time()
while count < MAX_PREDICTION_STEPS:
##################################################
# Predict for test image once, and wait for future images to arrive
##################################################
print("Calling predict_single_image")
pose = predict_single_image(image, sess, inputs, outputs, dlc_cfg)
##################################################
# Yield prediction to caller
##################################################
image = (
yield pose
) # Receive image here ( Refer https://stackabuse.com/python-generators/ for sending/receiving in generators)
step_time = time.time()
print(f"time: {step_time-start_time}")
start_time = step_time
count += 1
if count == MAX_PREDICTION_STEPS:
print(
f"Restart prediction system, Steps have exceeded {MAX_PREDICTION_STEPS}"
)
sess.close() # closes the current tf session
TF.reset_default_graph()
def evaluate_network(
config,
Shuffles=[1],
trainingsetindex=0,
plotting=None,
show_errors=True,
comparisonbodyparts="all",
gputouse=None,
rescale=False,
modelprefix="",
c_engine=False,
):
"""
Evaluates the network based on the saved models at different stages of the training network.\n
The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'.
Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
Shuffles: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This
variable can also be set to "all".
plotting: bool, optional
Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False``
show_errors: bool, optional
Display train and test errors. The default is `True``
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
rescale: bool, default False
Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every
image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported
in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated
on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the
original size!
Examples
--------
If you do not want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml', Shuffles=[1])
--------
If you want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',Shuffles=[1],True)
"""
import os
start_path = os.getcwd()
from deeplabcut.utils import auxiliaryfunctions
cfg = auxiliaryfunctions.read_config(config)
if cfg.get("multianimalproject", False):
from deeplabcut.pose_estimation_tensorflow.evaluate_multianimal import (
evaluate_multianimal_full, )
# TODO: Make this code not so redundant!
evaluate_multianimal_full(
config=config,
Shuffles=Shuffles,
trainingsetindex=trainingsetindex,
plotting=plotting,
comparisonbodyparts=comparisonbodyparts,
gputouse=gputouse,
modelprefix=modelprefix,
c_engine=c_engine,
)
else:
from deeplabcut.utils.auxfun_videos import imread, imresize
from deeplabcut.pose_estimation_tensorflow.nnet import predict
from deeplabcut.pose_estimation_tensorflow.config import load_config
from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import (
data_to_input, )
from deeplabcut.utils import auxiliaryfunctions
import tensorflow as tf
if "TF_CUDNN_USE_AUTOTUNE" in os.environ:
del os.environ[
"TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training
tf.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
# tf.logging.set_verbosity(tf.logging.WARN)
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
if (trainingsetindex < len(cfg["TrainingFraction"])
and trainingsetindex >= 0):
TrainingFractions = [
cfg["TrainingFraction"][int(trainingsetindex)]
]
else:
raise Exception(
"Please check the trainingsetindex! ",
trainingsetindex,
" should be an integer from 0 .. ",
int(len(cfg["TrainingFraction"]) - 1),
)
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
)
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# change batch size, if it was edited during analysis!
dlc_cfg[
"batch_size"] = 1 # in case this was edited for analysis.
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
# path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0] for fn in os.listdir(
os.path.join(str(modelfolder), "train"))
if "index" in fn
])
try: # check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
raise ValueError(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
final_result = []
########################### RESCALING (to global scale)
if rescale == True:
scale = dlc_cfg["global_scale"]
Data = (pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
) * scale)
else:
scale = 1
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg["init_weights"].split(os.sep)[-1]
).split(
"-"
)[-1] # read how many training siterations that corresponds to.
# Name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix,
)
print(
"Running ",
DLCscorer,
" with # of trainingiterations:",
trainingsiterations,
)
(
notanalyzed,
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.CheckifNotEvaluated(
str(evaluationfolder),
DLCscorer,
DLCscorerlegacy,
Snapshots[snapindex],
)
if notanalyzed:
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = predict.setup_pose_prediction(
dlc_cfg)
Numimages = len(Data.index)
PredicteData = np.zeros(
(Numimages, 3 * len(dlc_cfg["all_joints_names"])))
print("Analyzing data...")
for imageindex, imagename in tqdm(enumerate(
Data.index)):
image = imread(os.path.join(
cfg["project_path"], imagename),
mode="RGB")
if scale != 1:
image = imresize(image, scale)
image_batch = data_to_input(image)
# Compute prediction with the CNN
outputs_np = sess.run(
outputs, feed_dict={inputs: image_batch})
scmap, locref = predict.extract_cnn_output(
outputs_np, dlc_cfg)
# Extract maximum scoring location from the heatmap, assume 1 person
pose = predict.argmax_pose_predict(
scmap, locref, dlc_cfg.stride)
PredicteData[imageindex, :] = (
pose.flatten()
) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts!
sess.close() # closes the current tf session
index = pd.MultiIndex.from_product(
[
[DLCscorer],
dlc_cfg["all_joints_names"],
["x", "y", "likelihood"],
],
names=["scorer", "bodyparts", "coords"],
)
# Saving results
DataMachine = pd.DataFrame(PredicteData,
columns=index,
index=Data.index.values)
DataMachine.to_hdf(resultsfilename,
"df_with_missing",
format="table",
mode="w")
print(
"Done and results stored for snapshot: ",
Snapshots[snapindex],
)
DataCombined = pd.concat([Data.T, DataMachine.T],
axis=0,
sort=False).T
RMSE, RMSEpcutoff = pairwisedistances(
DataCombined,
cfg["scorer"],
DLCscorer,
cfg["pcutoff"],
comparisonbodyparts,
)
testerror = np.nanmean(
RMSE.iloc[testIndices].values.flatten())
trainerror = np.nanmean(
RMSE.iloc[trainIndices].values.flatten())
testerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[testIndices].values.flatten())
trainerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[trainIndices].values.flatten())
results = [
trainingsiterations,
int(100 * trainFraction),
shuffle,
np.round(trainerror, 2),
np.round(testerror, 2),
cfg["pcutoff"],
np.round(trainerrorpcutoff, 2),
np.round(testerrorpcutoff, 2),
]
final_result.append(results)
if show_errors:
print(
"Results for",
trainingsiterations,
" training iterations:",
int(100 * trainFraction),
shuffle,
"train error:",
np.round(trainerror, 2),
"pixels. Test error:",
np.round(testerror, 2),
" pixels.",
)
print(
"With pcutoff of",
cfg["pcutoff"],
" train error:",
np.round(trainerrorpcutoff, 2),
"pixels. Test error:",
np.round(testerrorpcutoff, 2),
"pixels",
)
if scale != 1:
print(
"The predictions have been calculated for rescaled images (and rescaled ground truth). Scale:",
scale,
)
print(
"Thereby, the errors are given by the average distances between the labels by DLC and the scorer."
)
if plotting == True:
print("Plotting...")
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" +
Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
Plotting(
cfg,
comparisonbodyparts,
DLCscorer,
trainIndices,
DataCombined * 1.0 / scale,
foldername,
) # Rescaling coordinates to have figure in original size!
tf.reset_default_graph()
# print(final_result)
else:
DataMachine = pd.read_hdf(resultsfilename,
"df_with_missing")
if plotting == True:
DataCombined = pd.concat([Data.T, DataMachine.T],
axis=0,
sort=False).T
print(
"Plotting...(attention scale might be inconsistent in comparison to when data was analyzed; i.e. if you used rescale)"
)
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" +
Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
Plotting(
cfg,
comparisonbodyparts,
DLCscorer,
trainIndices,
DataCombined * 1.0 / scale,
foldername,
)
if len(final_result
) > 0: # Only append if results were calculated
make_results_file(final_result, evaluationfolder,
DLCscorer)
print(
"The network is evaluated and the results are stored in the subdirectory 'evaluation_results'."
)
print(
"If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos."
)
print(
"Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)"
)
# returning to intial folder
os.chdir(str(start_path))
def extract_outlier_frames(config,
videos,
videotype='avi',
shuffle=1,
trainingsetindex=0,
outlieralgorithm='jump',
comparisonbodyparts='all',
epsilon=20,
p_bound=.01,
ARdegree=3,
MAdegree=1,
alpha=.01,
extractionalgorithm='kmeans',
automatic=False,
cluster_resizewidth=30,
cluster_color=False,
opencv=True,
savelabeled=True,
destfolder=None):
"""
Extracts the outlier frames in case, the predictions are not correct for a certain video from the cropped video running from
start to stop as defined in config.yaml.
Another crucial parameter in config.yaml is how many frames to extract 'numframes2extract'.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
The shufle index of training dataset. The extracted frames will be stored in the labeled-dataset for
the corresponding shuffle of training dataset. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
outlieralgorithm: 'fitting', 'jump', or 'uncertain', optional
String specifying the algorithm used to detect the outliers. Currently, deeplabcut supports three methods. 'Fitting'
fits a Auto Regressive Integrated Moving Average model to the data and computes the distance to the estimated data. Larger distances than
epsilon are then potentially identified as outliers. The methods 'jump' identifies larger jumps than 'epsilon' in any body part; and 'uncertain'
looks for frames with confidence below p_bound. The default is set to ``jump``.
comparisonbodyparts: list of strings, optional
This select the body parts for which the comparisons with the outliers are carried out. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
p_bound: float between 0 and 1, optional
For outlieralgorithm 'uncertain' this parameter defines the likelihood below, below which a body part will be flagged as a putative outlier.
epsilon; float,optional
Meaning depends on outlieralgoritm. The default is set to 20 pixels.
For outlieralgorithm 'fitting': Float bound according to which frames are picked when the (average) body part estimate deviates from model fit
For outlieralgorithm 'jump': Float bound specifying the distance by which body points jump from one frame to next (Euclidean distance)
ARdegree: int, optional
For outlieralgorithm 'fitting': Autoregressive degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part)
see https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
MAdegree: int
For outlieralgorithm 'fitting': MovingAvarage degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part)
See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
alpha: float
Significance level for detecting outliers based on confidence interval of fitted ARIMA model. Only the distance is used however.
extractionalgorithm : string, optional
String specifying the algorithm to use for selecting the frames from the identified putatative outlier frames. Currently, deeplabcut
supports either ``kmeans`` or ``uniform`` based selection (same logic as for extract_frames).
The default is set to``uniform``, if provided it must be either ``uniform`` or ``kmeans``.
automatic : bool, optional
Set it to True, if you want to extract outliers without being asked for user feedback.
cluster_resizewidth: number, default: 30
For k-means one can change the width to which the images are downsampled (aspect ratio is fixed).
cluster_color: bool, default: False
If false then each downsampled image is treated as a grayscale vector (discarding color information). If true, then the color channels are considered. This increases
the computational complexity.
opencv: bool, default: True
Uses openCV for loading & extractiong (otherwise moviepy (legacy))
savelabeled: bool, default: True
If true also saves frame with predicted labels in each folder.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
Examples
Windows example for extracting the frames with default settings
>>> deeplabcut.extract_outlier_frames('C:\\myproject\\reaching-task\\config.yaml',['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi'])
--------
for extracting the frames with default settings
>>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'])
--------
for extracting the frames with kmeans
>>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'],extractionalgorithm='kmeans')
--------
for extracting the frames with kmeans and epsilon = 5 pixels.
>>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'],epsilon = 5,extractionalgorithm='kmeans')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
scorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction=cfg['TrainingFraction'][trainingsetindex])
print("network parameters:", scorer)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
dataname = str(Path(video).stem) + scorer
try:
Dataframe = pd.read_hdf(os.path.join(videofolder,
dataname + '.h5'))
nframes = np.size(Dataframe.index)
#extract min and max index based on start stop interval.
startindex = max([int(np.floor(nframes * cfg['start'])), 0])
stopindex = min([int(np.ceil(nframes * cfg['stop'])), nframes])
Index = np.arange(stopindex - startindex) + startindex
#figure out body part list:
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
Indices = []
if outlieralgorithm == 'uncertain': #necessary parameters: considered body parts and
for bpindex, bp in enumerate(bodyparts):
if bp in cfg[
'bodyparts']: #filter [who knows what users put in...]
p = Dataframe[scorer][bp]['likelihood'].values[Index]
Indices.extend(
np.where(p < p_bound)[0] + startindex
) # all indices between start and stop that are below p_bound.
elif outlieralgorithm == 'jump':
for bpindex, bp in enumerate(bodyparts):
if bp in cfg[
'bodyparts']: #filter [who knows what users put in...]
dx = np.diff(Dataframe[scorer][bp]['x'].values[Index])
dy = np.diff(Dataframe[scorer][bp]['y'].values[Index])
# all indices between start and stop with jump larger than epsilon (leading up to this point!)
Indices.extend(
np.where((dx**2 + dy**2) > epsilon**2)[0] +
startindex + 1)
elif outlieralgorithm == 'fitting':
#deviation_dataname = str(Path(videofolder)/Path(dataname))
# Calculate deviatons for video
[d, o] = ComputeDeviations(Dataframe, cfg, bodyparts, scorer,
dataname, p_bound, alpha, ARdegree,
MAdegree)
#Some heuristics for extracting frames based on distance:
Indices = np.where(
d > epsilon
)[0] # time points with at least average difference of epsilon
if len(Index) < cfg['numframes2pick'] * 2 and len(d) > cfg[
'numframes2pick'] * 2: # if too few points qualify, extract the most distant ones.
Indices = np.argsort(d)[::-1][:cfg['numframes2pick'] * 2]
elif outlieralgorithm == 'manual':
wd = Path(config).resolve().parents[0]
os.chdir(str(wd))
from deeplabcut.refine_training_dataset import outlier_frame_extraction_toolbox
outlier_frame_extraction_toolbox.show(config, video, shuffle,
Dataframe, scorer,
savelabeled)
# Run always except when the outlieralgorithm == manual.
if not outlieralgorithm == 'manual':
Indices = np.sort(list(set(Indices))) #remove repetitions.
print("Method ", outlieralgorithm, " found ", len(Indices),
" putative outlier frames.")
print("Do you want to proceed with extracting ",
cfg['numframes2pick'], " of those?")
if outlieralgorithm == 'uncertain':
print(
"If this list is very large, perhaps consider changing the paramters (start, stop, p_bound, comparisonbodyparts) or use a different method."
)
elif outlieralgorithm == 'jump':
print(
"If this list is very large, perhaps consider changing the paramters (start, stop, epsilon, comparisonbodyparts) or use a different method."
)
elif outlieralgorithm == 'fitting':
print(
"If this list is very large, perhaps consider changing the paramters (start, stop, epsilon, ARdegree, MAdegree, alpha, comparisonbodyparts) or use a different method."
)
if automatic == False:
askuser = input("yes/no")
else:
askuser = '******'
if askuser == 'y' or askuser == 'yes' or askuser == 'Ja' or askuser == 'ha': # multilanguage support :)
#Now extract from those Indices!
ExtractFramesbasedonPreselection(
Indices, extractionalgorithm, Dataframe, dataname,
scorer, video, cfg, config, opencv,
cluster_resizewidth, cluster_color, savelabeled)
else:
print(
"Nothing extracted, change parameters and start again..."
)
except FileNotFoundError:
print(
"The video has not been analyzed yet!. You can only refine the labels, after the pose has been estimate. Please run 'analyze_video' first."
)
def plot_trajectories(config,
videos,
videotype='.avi',
shuffle=1,
trainingsetindex=0,
filtered=False,
displayedbodyparts='all',
showfigures=False,
destfolder=None,
outformat='.pdf'):
"""
Plots the trajectories of various bodyparts across the video.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
displayedbodyparts: list of strings, optional
This select the body parts that are plotted in the video.
Either ``all``, then all body parts from config.yaml are used,
or a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
showfigures: bool, default false
If true then plots are also displayed.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
Example
--------
for labeling the frames
>>> deeplabcut.plot_trajectories('home/alex/analysis/project/reaching-task/config.yaml',['/home/alex/analysis/project/videos/reachingvideo1.avi'])
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction
) #automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
print(video)
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
vname = str(Path(video).stem)
print("Starting % ", videofolder, video)
notanalyzed, dataname, DLCscorer = auxiliaryfunctions.CheckifNotAnalyzed(
videofolder, vname, DLCscorer, DLCscorerlegacy, flag='checking')
if notanalyzed:
print("The video was not analyzed with this scorer:", DLCscorer)
else:
#LoadData
print("Loading ", video, "and data.")
datafound, metadata, Dataframe, DLCscorer, suffix = auxiliaryfunctions.LoadAnalyzedData(
str(videofolder), vname, DLCscorer, filtered
) #returns boolean variable if data was found and metadata + pandas array
if datafound:
basefolder = videofolder
auxiliaryfunctions.attempttomakefolder(basefolder)
auxiliaryfunctions.attempttomakefolder(
os.path.join(basefolder, 'plot-poses'))
tmpfolder = os.path.join(basefolder, 'plot-poses', vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder)
PlottingResults(tmpfolder, Dataframe, DLCscorer, cfg,
bodyparts, showfigures, suffix + outformat)
print(
'Plots created! Please check the directory "plot-poses" within the video directory'
)
def evaluate_multianimal_crossvalidate(
config,
Shuffles=[1],
trainingsetindex=0,
pbounds=None,
edgewisecondition=True,
target="rpck_train",
inferencecfg=None,
init_points=20,
n_iter=50,
log_file=None,
dcorr=10.0,
leastbpts=1,
printingintermediatevalues=True,
modelprefix="",
plotting=False,
):
"""
Cross-validate inference parameters on evaluation data; optimal parameters will be stored in "inference_cfg.yaml".
They will then be then used for inference (for analysis of videos). Performs Bayesian Optimization with https://github.com/fmfn/BayesianOptimization
This is a crucial step. The most important variable (in inferencecfg) to cross-validate is minimalnumberofconnections.
Pass a reasonable range to optimize (e.g. if you have 5 edges from 1 to 5. If you have 4 bpts and 11 connections from 3 to 9).
config: string
Full path of the config.yaml file as a string.
shuffle: int, optional
An integer specifying the shuffle index of the training dataset used for training the network. The default is 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
pbounds: dictionary of variables with ranges to crossvalidate.
By default: pbounds = {
'pafthreshold': (0.05, 0.7),
'detectionthresholdsquare': (0, 0.9),
'minimalnumberofconnections': (1, # connections in your skeleton),
}
inferencecfg: dict, OPTIONAL
For the variables that are *not* crossvalidated the parameters from inference_cfg.yaml are used, or
you can overwrite them by passing a dictinary with your preferred parameters.
edgewisecondition: bool, default True
Estimates Euclidean distances for each skeleton edge and uses those distance for excluding possible connections.
If false, uses only one distance for all bodyparts (which is obviously suboptimal).
target: string, default='rpck_train'
What metric to optimize. Options are pck/rpck/rmse on train/test set.
init_points: int, optional (default=10)
Number of random initial explorations. Probing random regions helps diversify the exploration space.
Parameter from BayesianOptimization.
n_iter: int, optional (default=20)
Number of iterations of Bayesian optimization to perform.
The larger it is, the higher the likelihood of finding a good extremum.
Parameter from BayesianOptimization.
log_file: str, optional (default=None)
Path to a JSON file containing the progress of a previous Bayesian optimization run.
Note that previously probed points will not be evaluated again.
dcorr: float,
Distance thereshold for percent correct keypoints / relative percent correct keypoints (see paper).
leastbpts: integer (should be a small number)
If an animals has less or equal as many body parts in an image it will not be used
for cross validation. Imagine e.g. if only a single bodypart is present, then
if animals need a certain minimal number of bodyparts for assembly (minimalnumberofconnections),
this might not be predictable.
printingintermediatevalues: bool, default True
If intermediate metrics RMSE/hits/.. per sample should be printed.
Examples
--------
first run evalute:
deeplabcut.evaluate_network(path_config_file,Shuffles=[shuffle],plotting=True)
Then e.g. for finding inference parameters to minimize rmse on test set:
deeplabcut.evaluate_multianimal_crossvalidate(path_config_file,Shuffles=[shuffle],target='rmse_test')
"""
from deeplabcut.pose_estimation_tensorflow.lib import crossvalutils
from deeplabcut.utils import auxfun_multianimal, auxiliaryfunctions
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
)
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, "all"
)
colors = visualization.get_cmap(len(comparisonbodyparts), name=cfg["colormap"])
# wild guesses for a wide range:
maxconnections = len(cfg["skeleton"])
minconnections = 1 # len(cfg['multianimalbodyparts'])-1
_pbounds = {
"pafthreshold": (0.05, 0.7),
"detectionthresholdsquare": (
0.0,
0.9,
), # TODO: set to minimum (from pose_cfg.yaml)
"minimalnumberofconnections": (minconnections, maxconnections),
}
if pbounds is not None:
_pbounds.update(pbounds)
if "rpck" in target or "pck" in target:
maximize = True
if "rmse" in target:
maximize = False # i.e. minimize
for shuffle in Shuffles:
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix
)
),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True)
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg
)
_, trainIndices, testIndices, _ = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn)
)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix
)
),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction)
)
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array(
[
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
]
)
snapindex = -1
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (dlc_cfg["init_weights"].split(os.sep)[-1]).split("-")[
-1
] # read how many training siterations that corresponds to.
DLCscorer, _ = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, trainingsiterations, modelprefix=modelprefix
)
path_inference_config = Path(modelfolder) / "test" / "inference_cfg.yaml"
if inferencecfg is None: # then load or initialize
inferencecfg = auxfun_multianimal.read_inferencecfg(
path_inference_config, cfg
)
else:
auxfun_multianimal.check_inferencecfg_sanity(cfg, inferencecfg)
# Pick distance threshold for (r)PCK from the statistics computed during evaluation
stats_file = os.path.join(evaluationfolder, "sd.csv")
if os.path.isfile(stats_file):
stats = pd.read_csv(stats_file, header=None, index_col=0)
inferencecfg['distnormalization'] = np.round(
stats.loc["distnorm", 1], 2
).item()
stats = stats.drop("distnorm")
dcorr = (
2 * stats.mean().squeeze()
) # Taken as 2*SD error between predictions and ground truth
else:
dcorr = 10
inferencecfg['topktoretain'] = np.inf
inferencecfg, opt = crossvalutils.bayesian_search(
config,
inferencecfg,
_pbounds,
edgewisecondition=edgewisecondition,
shuffle=shuffle,
trainingsetindex=trainingsetindex,
target=target,
maximize=maximize,
init_points=init_points,
n_iter=n_iter,
acq="ei",
log_file=log_file,
dcorr=dcorr,
leastbpts=leastbpts,
modelprefix=modelprefix,
printingintermediatevalues=printingintermediatevalues,
)
# update number of individuals to retain.
inferencecfg['topktoretain'] = len(cfg["individuals"]) + 1 * (
len(cfg["uniquebodyparts"]) > 0
)
# calculating result at best best solution
DataOptParams, poses_gt, poses = crossvalutils.compute_crossval_metrics(
config, inferencecfg, shuffle, trainingsetindex, modelprefix
)
path_inference_config = str(path_inference_config)
# print("Quantification:", DataOptParams.head())
# DataOptParams.to_hdf(
# path_inference_config.split(".yaml")[0] + ".h5",
# "df_with_missing",
# format="table",
# mode="w",
# )
DataOptParams.to_csv(os.path.join(evaluationfolder, "results.csv"))
print("Saving optimal inference parameters...")
print(DataOptParams.to_string())
auxiliaryfunctions.write_plainconfig(path_inference_config, dict(inferencecfg))
# Store best predictions
max_indivs = max(pose.shape[0] for pose in poses)
bpts = dlc_cfg["all_joints_names"]
container = np.full((len(poses), max_indivs * len(bpts) * 3), np.nan)
for n, pose in enumerate(poses):
temp = pose.flatten()
container[n, : len(temp)] = temp
header = pd.MultiIndex.from_product(
[
[DLCscorer],
[f"individual{i}" for i in range(1, max_indivs + 1)],
bpts,
["x", "y", "likelihood"],
],
names=["scorer", "individuals", "bodyparts", "coords"],
)
df = pd.DataFrame(container, columns=header)
df.to_hdf(
os.path.join(evaluationfolder, f"{DLCscorer}.h5"), key="df_with_missing"
)
if plotting:
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" + Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
for imageindex, imagename in tqdm(enumerate(Data.index)):
image_path = os.path.join(cfg["project_path"], imagename)
image = io.imread(image_path)
frame = img_as_ubyte(skimage.color.gray2rgb(image))
groundtruthcoordinates = poses_gt[imageindex]
coords_pred = poses[imageindex][:, :, :2]
probs_pred = poses[imageindex][:, :, -1:]
fig = visualization.make_multianimal_labeled_image(
frame,
groundtruthcoordinates,
coords_pred,
probs_pred,
colors,
cfg["dotsize"],
cfg["alphavalue"],
cfg["pcutoff"],
)
visualization.save_labeled_frame(
fig, image_path, foldername, imageindex in trainIndices
)
def ExtractFramesbasedonPreselection(
Index,
extractionalgorithm,
data,
video,
cfg,
config,
opencv=True,
cluster_resizewidth=30,
cluster_color=False,
savelabeled=True,
with_annotations=True,
):
from deeplabcut.create_project import add
start = cfg["start"]
stop = cfg["stop"]
numframes2extract = cfg["numframes2pick"]
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(cfg, "all")
videofolder = str(Path(video).parents[0])
vname = str(Path(video).stem)
tmpfolder = os.path.join(cfg["project_path"], "labeled-data", vname)
if os.path.isdir(tmpfolder):
print("Frames from video", vname, " already extracted (more will be added)!")
else:
auxiliaryfunctions.attempttomakefolder(tmpfolder, recursive=True)
nframes = len(data)
print("Loading video...")
if opencv:
vid = VideoWriter(video)
fps = vid.fps
duration = vid.calc_duration()
else:
from moviepy.editor import VideoFileClip
clip = VideoFileClip(video)
fps = clip.fps
duration = clip.duration
if cfg["cropping"]: # one might want to adjust
coords = (cfg["x1"], cfg["x2"], cfg["y1"], cfg["y2"])
else:
coords = None
print("Duration of video [s]: ", duration, ", recorded @ ", fps, "fps!")
print("Overall # of frames: ", nframes, "with (cropped) frame dimensions: ")
if extractionalgorithm == "uniform":
if opencv:
frames2pick = frameselectiontools.UniformFramescv2(
vid, numframes2extract, start, stop, Index
)
else:
frames2pick = frameselectiontools.UniformFrames(
clip, numframes2extract, start, stop, Index
)
elif extractionalgorithm == "kmeans":
if opencv:
frames2pick = frameselectiontools.KmeansbasedFrameselectioncv2(
vid,
numframes2extract,
start,
stop,
cfg["cropping"],
coords,
Index,
resizewidth=cluster_resizewidth,
color=cluster_color,
)
else:
if cfg["cropping"]:
clip = clip.crop(y1=cfg["y1"], y2=cfg["x2"], x1=cfg["x1"], x2=cfg["x2"])
frames2pick = frameselectiontools.KmeansbasedFrameselection(
clip,
numframes2extract,
start,
stop,
Index,
resizewidth=cluster_resizewidth,
color=cluster_color,
)
else:
print(
"Please implement this method yourself! Currently the options are 'kmeans', 'jump', 'uniform'."
)
frames2pick = []
# Extract frames + frames with plotted labels and store them in folder (with name derived from video name) nder labeled-data
print("Let's select frames indices:", frames2pick)
colors = visualization.get_cmap(len(bodyparts), cfg["colormap"])
strwidth = int(np.ceil(np.log10(nframes))) # width for strings
for index in frames2pick: ##tqdm(range(0,nframes,10)):
if opencv:
PlottingSingleFramecv2(
vid,
cfg["cropping"],
coords,
data,
bodyparts,
tmpfolder,
index,
cfg["dotsize"],
cfg["pcutoff"],
cfg["alphavalue"],
colors,
strwidth,
savelabeled,
)
else:
PlottingSingleFrame(
clip,
data,
bodyparts,
tmpfolder,
index,
cfg["dotsize"],
cfg["pcutoff"],
cfg["alphavalue"],
colors,
strwidth,
savelabeled,
)
plt.close("all")
# close videos
if opencv:
vid.close()
else:
clip.close()
del clip
# Extract annotations based on DeepLabCut and store in the folder (with name derived from video name) under labeled-data
if len(frames2pick) > 0:
try:
if cfg["cropping"]:
add.add_new_videos(
config, [video], coords=[coords]
) # make sure you pass coords as a list
else:
add.add_new_videos(config, [video], coords=None)
except: # can we make a catch here? - in fact we should drop indices from DataCombined if they are in CollectedData.. [ideal behavior; currently this is pretty unlikely]
print(
"AUTOMATIC ADDING OF VIDEO TO CONFIG FILE FAILED! You need to do this manually for including it in the config.yaml file!"
)
print("Videopath:", video, "Coordinates for cropping:", coords)
pass
if with_annotations:
machinefile = os.path.join(
tmpfolder, "machinelabels-iter" + str(cfg["iteration"]) + ".h5"
)
if isinstance(data, pd.DataFrame):
df = data.loc[frames2pick]
df.index = pd.MultiIndex.from_tuples([
("labeled-data", vname, "img" + str(index).zfill(strwidth) + ".png")
for index in df.index
]) # exchange index number by file names.
elif isinstance(data, dict):
idx = pd.MultiIndex.from_tuples([
("labeled-data", vname, "img" + str(index).zfill(strwidth) + ".png")
for index in frames2pick
])
filename = os.path.join(
str(tmpfolder), f"CollectedData_{cfg['scorer']}.h5"
)
try:
df_temp = pd.read_hdf(filename, "df_with_missing")
columns = df_temp.columns
except FileNotFoundError:
columns = pd.MultiIndex.from_product(
[
[cfg["scorer"]],
cfg["individuals"],
cfg["multianimalbodyparts"],
["x", "y"],
],
names=["scorer", "individuals", "bodyparts", "coords"],
)
if cfg["uniquebodyparts"]:
columns2 = pd.MultiIndex.from_product(
[
[cfg["scorer"]],
["single"],
cfg["uniquebodyparts"],
["x", "y"],
],
names=["scorer", "individuals", "bodyparts", "coords"],
)
df_temp = pd.concat(
(
pd.DataFrame(columns=columns),
pd.DataFrame(columns=columns2),
)
)
columns = df_temp.columns
array = np.full((len(frames2pick), len(columns)), np.nan)
for i, index in enumerate(frames2pick):
data_temp = data.get(index)
if data_temp is not None:
vals = np.concatenate(data_temp)[:, :2].flatten()
array[i, : len(vals)] = vals
df = pd.DataFrame(array, index=idx, columns=columns)
else:
return
if Path(machinefile).is_file():
Data = pd.read_hdf(machinefile, "df_with_missing")
conversioncode.guarantee_multiindex_rows(Data)
DataCombined = pd.concat([Data, df])
# drop duplicate labels:
DataCombined = DataCombined[
~DataCombined.index.duplicated(keep="first")
]
DataCombined.to_hdf(machinefile, key="df_with_missing", mode="w")
DataCombined.to_csv(
os.path.join(tmpfolder, "machinelabels.csv")
) # this is always the most current one (as reading is from h5)
else:
df.to_hdf(machinefile, key="df_with_missing", mode="w")
df.to_csv(os.path.join(tmpfolder, "machinelabels.csv"))
print(
"The outlier frames are extracted. They are stored in the subdirectory labeled-data\%s."
% vname
)
print(
"Once you extracted frames for all videos, use 'refine_labels' to manually correct the labels."
)
else:
print("No frames were extracted.")
def filterpredictions(config,
video,
shuffle=1,
trainingsetindex=0,
comparisonbodyparts='all',
p_bound=.01,
ARdegree=3,
MAdegree=1,
alpha=.01):
"""
Fits frame-by-frame pose predictions with SARIMAX model.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
video : string
Full path of the video to extract the frame from. Make sure that this video is already analyzed.
shuffle : int, optional
The shufle index of training dataset. The extracted frames will be stored in the labeled-dataset for
the corresponding shuffle of training dataset. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
comparisonbodyparts: list of strings, optional
This select the body parts for which SARIMAX models are fit. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
p_bound: float between 0 and 1, optional
For outlieralgorithm 'uncertain' this parameter defines the likelihood below, below which a body part will be flagged as a putative outlier.
ARdegree: int, optional
For outlieralgorithm 'fitting': Autoregressive degree of Sarimax model degree.
see https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
MAdegree: int
For outlieralgorithm 'fitting': Moving Avarage degree of Sarimax model degree.
See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
alpha: float
Significance level for detecting outliers based on confidence interval of fitted SARIMAX model.
Example
--------
tba
--------
Returns filtered pandas array (incl. confidence interval), original data, distance and average outlier vector.
"""
cfg = auxiliaryfunctions.read_config(config)
scorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction=cfg['TrainingFraction'][trainingsetindex])
print("network parameters:", scorer)
videofolder = str(Path(video).parents[0])
dataname = str(Path(video).stem) + scorer
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
try:
Dataframe = pd.read_hdf(os.path.join(videofolder, dataname + '.h5'))
except FileExistsError:
print("Could not find data.")
data, d, o = ComputeDeviations(Dataframe,
cfg,
bodyparts,
scorer,
dataname,
p_bound,
alpha,
ARdegree,
MAdegree,
storeoutput='full')
return data, Dataframe, d, o
def create_labeled_video(config,
videos,
videotype='avi',
shuffle=1,
trainingsetindex=0,
filtered=False,
save_frames=False,
Frames2plot=None,
delete=False,
displayedbodyparts='all',
codec='mp4v',
outputframerate=None,
destfolder=None,
draw_skeleton=False,
trailpoints=0,
displaycropped=False):
"""
Labels the bodyparts in a video. Make sure the video is already analyzed by the function 'analyze_video'
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
Number of shuffles of training dataset. Default is set to 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
videotype: string, optional
Checks for the extension of the video in case the input is a directory.\nOnly videos with this extension are analyzed. The default is ``.avi``
save_frames: bool
If true creates each frame individual and then combines into a video. This variant is relatively slow as
it stores all individual frames. However, it uses matplotlib to create the frames and is therefore much more flexible (one can set transparency of markers, crop, and easily customize).
Frames2plot: List of indices
If not None & save_frames=True then the frames corresponding to the index will be plotted. For example, Frames2plot=[0,11] will plot the first and the 12th frame.
delete: bool
If true then the individual frames created during the video generation will be deleted.
displayedbodyparts: list of strings, optional
This select the body parts that are plotted in the video. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
codec: codec for labeled video. Options see http://www.fourcc.org/codecs.php [depends on your ffmpeg installation.]
outputframerate: positive number, output frame rate for labeled video (only available for the mode with saving frames.) By default: None, which results in the original video rate.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
draw_skeleton: bool
If ``True`` adds a line connecting the body parts making a skeleton on on each frame. The body parts to be connected and the color of these connecting lines are specified in the config file. By default: ``False``
trailpoints: int
Number of revious frames whose body parts are plotted in a frame (for displaying history). Default is set to 0.
displaycropped: bool, optional
Specifies whether only cropped frame is displayed (with labels analyzed therein), or the original frame with the labels analyzed in the cropped subset.
Examples
--------
If you want to create the labeled video for only 1 video
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to create the labeled video for only 1 video and store the individual frames
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'],save_frames=True)
--------
If you want to create the labeled video for multiple videos
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to create the labeled video for all the videos (as .avi extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'])
--------
If you want to create the labeled video for all the videos (as .mp4 extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'],videotype='mp4')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction
) #automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
if draw_skeleton:
bodyparts2connect = cfg['skeleton']
skeleton_color = cfg['skeleton_color']
else:
bodyparts2connect = None
skeleton_color = None
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
if destfolder is None:
videofolder = Path(video).parents[
0] #where your folder with videos is.
else:
videofolder = destfolder
os.chdir(str(videofolder))
videotype = Path(video).suffix
print("Starting % ", videofolder, videos)
vname = str(Path(video).stem)
#if notanalyzed:
#notanalyzed,outdataname,sourcedataname,DLCscorer=auxiliaryfunctions.CheckifPostProcessing(folder,vname,DLCscorer,DLCscorerlegacy,suffix='checking')
if filtered == True:
videooutname1 = os.path.join(vname + DLCscorer +
'filtered_labeled.mp4')
videooutname2 = os.path.join(vname + DLCscorerlegacy +
'filtered_labeled.mp4')
else:
videooutname1 = os.path.join(vname + DLCscorer + '_labeled.mp4')
videooutname2 = os.path.join(vname + DLCscorerlegacy +
'_labeled.mp4')
if os.path.isfile(videooutname1) or os.path.isfile(videooutname2):
print("Labeled video already created.")
else:
print("Loading ", video, "and data.")
datafound, metadata, Dataframe, DLCscorer, suffix = auxiliaryfunctions.LoadAnalyzedData(
str(videofolder), vname, DLCscorer, filtered
) #returns boolean variable if data was found and metadata + pandas array
videooutname = os.path.join(vname + DLCscorer + suffix +
'_labeled.mp4')
if datafound and not os.path.isfile(
videooutname
): #checking again, for this loader video could exist
#Loading cropping data used during analysis
cropping = metadata['data']["cropping"]
[x1, x2, y1, y2] = metadata['data']["cropping_parameters"]
if save_frames == True:
tmpfolder = os.path.join(str(videofolder), 'temp-' + vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder)
clip = vp(video)
CreateVideoSlow(videooutname, clip, Dataframe, tmpfolder,
cfg["dotsize"], cfg["colormap"],
cfg["alphavalue"], cfg["pcutoff"],
trailpoints, cropping, x1, x2, y1, y2,
delete, DLCscorer, bodyparts,
outputframerate, Frames2plot,
bodyparts2connect, skeleton_color,
draw_skeleton, displaycropped)
else:
if displaycropped: #then the cropped video + the labels is depicted
clip = vp(fname=video,
sname=videooutname,
codec=codec,
sw=x2 - x1,
sh=y2 - y1)
CreateVideo(clip, Dataframe, cfg["pcutoff"],
cfg["dotsize"], cfg["colormap"], DLCscorer,
bodyparts, trailpoints, cropping, x1, x2,
y1, y2, bodyparts2connect, skeleton_color,
draw_skeleton, displaycropped)
else: #then the full video + the (perhaps in cropped mode analyzed labels) are depicted
clip = vp(fname=video, sname=videooutname, codec=codec)
CreateVideo(clip, Dataframe, cfg["pcutoff"],
cfg["dotsize"], cfg["colormap"], DLCscorer,
bodyparts, trailpoints, cropping, x1, x2,
y1, y2, bodyparts2connect, skeleton_color,
draw_skeleton, displaycropped)
def extract_outlier_frames(
config,
videos,
videotype=".avi",
shuffle=1,
trainingsetindex=0,
outlieralgorithm="jump",
comparisonbodyparts="all",
epsilon=20,
p_bound=0.01,
ARdegree=3,
MAdegree=1,
alpha=0.01,
extractionalgorithm="kmeans",
automatic=False,
cluster_resizewidth=30,
cluster_color=False,
opencv=True,
savelabeled=False,
destfolder=None,
modelprefix="",
track_method="",
):
"""
Extracts the outlier frames in case, the predictions are not correct for a certain video from the cropped video running from
start to stop as defined in config.yaml.
Another crucial parameter in config.yaml is how many frames to extract 'numframes2extract'.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
The shuffle index of training dataset. The extracted frames will be stored in the labeled-dataset for
the corresponding shuffle of training dataset. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
outlieralgorithm: 'fitting', 'jump', 'uncertain', or 'manual'
String specifying the algorithm used to detect the outliers. Currently, deeplabcut supports three methods + a manual GUI option. 'Fitting'
fits a Auto Regressive Integrated Moving Average model to the data and computes the distance to the estimated data. Larger distances than
epsilon are then potentially identified as outliers. The methods 'jump' identifies larger jumps than 'epsilon' in any body part; and 'uncertain'
looks for frames with confidence below p_bound. The default is set to ``jump``.
comparisonbodyparts: list of strings, optional
This selects the body parts for which the comparisons with the outliers are carried out. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
p_bound: float between 0 and 1, optional
For outlieralgorithm 'uncertain' this parameter defines the likelihood below, below which a body part will be flagged as a putative outlier.
epsilon; float,optional
Meaning depends on outlieralgoritm. The default is set to 20 pixels.
For outlieralgorithm 'fitting': Float bound according to which frames are picked when the (average) body part estimate deviates from model fit
For outlieralgorithm 'jump': Float bound specifying the distance by which body points jump from one frame to next (Euclidean distance)
ARdegree: int, optional
For outlieralgorithm 'fitting': Autoregressive degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part)
see https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
MAdegree: int
For outlieralgorithm 'fitting': MovingAvarage degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part)
See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
alpha: float
Significance level for detecting outliers based on confidence interval of fitted ARIMA model. Only the distance is used however.
extractionalgorithm : string, optional
String specifying the algorithm to use for selecting the frames from the identified putatative outlier frames. Currently, deeplabcut
supports either ``kmeans`` or ``uniform`` based selection (same logic as for extract_frames).
The default is set to``uniform``, if provided it must be either ``uniform`` or ``kmeans``.
automatic : bool, optional
Set it to True, if you want to extract outliers without being asked for user feedback.
cluster_resizewidth: number, default: 30
For k-means one can change the width to which the images are downsampled (aspect ratio is fixed).
cluster_color: bool, default: False
If false then each downsampled image is treated as a grayscale vector (discarding color information). If true, then the color channels are considered. This increases
the computational complexity.
opencv: bool, default: True
Uses openCV for loading & extractiong (otherwise moviepy (legacy))
savelabeled: bool, default: False
If true also saves frame with predicted labels in each folder.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
track_method: string, optional
Specifies the tracker used to generate the data. Empty by default (corresponding to a single animal project).
For multiple animals, must be either 'box', 'skeleton', or 'ellipse' and will be taken from the config.yaml file if none is given.
Examples
Windows example for extracting the frames with default settings
>>> deeplabcut.extract_outlier_frames('C:\\myproject\\reaching-task\\config.yaml',['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi'])
--------
for extracting the frames with default settings
>>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'])
--------
for extracting the frames with kmeans
>>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'],extractionalgorithm='kmeans')
--------
for extracting the frames with kmeans and epsilon = 5 pixels.
>>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'],epsilon = 5,extractionalgorithm='kmeans')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts
)
if not len(bodyparts):
raise ValueError("No valid bodyparts were selected.")
track_method = auxfun_multianimal.get_track_method(cfg, track_method=track_method)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction=cfg["TrainingFraction"][trainingsetindex],
modelprefix=modelprefix,
)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
if len(Videos) == 0:
print("No suitable videos found in", videos)
for video in Videos:
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
vname = os.path.splitext(os.path.basename(video))[0]
try:
df, dataname, _, _ = auxiliaryfunctions.load_analyzed_data(
videofolder, vname, DLCscorer, track_method=track_method
)
nframes = len(df)
startindex = max([int(np.floor(nframes * cfg["start"])), 0])
stopindex = min([int(np.ceil(nframes * cfg["stop"])), nframes])
Index = np.arange(stopindex - startindex) + startindex
df = df.iloc[Index]
mask = df.columns.get_level_values("bodyparts").isin(bodyparts)
df_temp = df.loc[:, mask]
Indices = []
if outlieralgorithm == "uncertain":
p = df_temp.xs("likelihood", level="coords", axis=1)
ind = df_temp.index[(p < p_bound).any(axis=1)].tolist()
Indices.extend(ind)
elif outlieralgorithm == "jump":
temp_dt = df_temp.diff(axis=0) ** 2
temp_dt.drop("likelihood", axis=1, level="coords", inplace=True)
sum_ = temp_dt.sum(axis=1, level=1)
ind = df_temp.index[(sum_ > epsilon ** 2).any(axis=1)].tolist()
Indices.extend(ind)
elif outlieralgorithm == "fitting":
d, o = compute_deviations(
df_temp, dataname, p_bound, alpha, ARdegree, MAdegree
)
# Some heuristics for extracting frames based on distance:
ind = np.flatnonzero(
d > epsilon
) # time points with at least average difference of epsilon
if (
len(ind) < cfg["numframes2pick"] * 2
and len(d) > cfg["numframes2pick"] * 2
): # if too few points qualify, extract the most distant ones.
ind = np.argsort(d)[::-1][: cfg["numframes2pick"] * 2]
Indices.extend(ind)
elif outlieralgorithm == "manual":
wd = Path(config).resolve().parents[0]
os.chdir(str(wd))
from deeplabcut.gui import outlier_frame_extraction_toolbox
outlier_frame_extraction_toolbox.show(
config,
video,
shuffle,
df,
savelabeled,
cfg.get("multianimalproject", False),
)
# Run always except when the outlieralgorithm == manual.
if not outlieralgorithm == "manual":
Indices = np.sort(list(set(Indices))) # remove repetitions.
print(
"Method ",
outlieralgorithm,
" found ",
len(Indices),
" putative outlier frames.",
)
print(
"Do you want to proceed with extracting ",
cfg["numframes2pick"],
" of those?",
)
if outlieralgorithm == "uncertain" or outlieralgorithm == "jump":
print(
"If this list is very large, perhaps consider changing the parameters "
"(start, stop, p_bound, comparisonbodyparts) or use a different method."
)
elif outlieralgorithm == "fitting":
print(
"If this list is very large, perhaps consider changing the parameters "
"(start, stop, epsilon, ARdegree, MAdegree, alpha, comparisonbodyparts) "
"or use a different method."
)
if not automatic:
askuser = input("yes/no")
else:
askuser = "******"
if (
askuser == "y"
or askuser == "yes"
or askuser == "Ja"
or askuser == "ha"
): # multilanguage support :)
# Now extract from those Indices!
ExtractFramesbasedonPreselection(
Indices,
extractionalgorithm,
df,
video,
cfg,
config,
opencv,
cluster_resizewidth,
cluster_color,
savelabeled,
)
else:
print(
"Nothing extracted, please change the parameters and start again..."
)
except FileNotFoundError as e:
print(e)
print(
"It seems the video has not been analyzed yet, or the video is not found! "
"You can only refine the labels after the a video is analyzed. Please run 'analyze_video' first. "
"Or, please double check your video file path"
)
def create_labeled_video(
config,
videos,
videotype="avi",
shuffle=1,
trainingsetindex=0,
filtered=False,
fastmode=True,
save_frames=False,
Frames2plot=None,
displayedbodyparts="all",
displayedindividuals="all",
codec="mp4v",
outputframerate=None,
destfolder=None,
draw_skeleton=False,
trailpoints=0,
displaycropped=False,
color_by="bodypart",
modelprefix="",
track_method="",
):
"""
Labels the bodyparts in a video. Make sure the video is already analyzed by the function 'analyze_video'
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
Number of shuffles of training dataset. Default is set to 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
videotype: string, optional
Checks for the extension of the video in case the input is a directory.\nOnly videos with this extension are analyzed. The default is ``.avi``
fastmode: bool
If true uses openCV (much faster but less customization of video) vs matplotlib (if false). You can also
"save_frames" individually or not in the matplotlib mode (if you set the "save_frames" variable accordingly).
However, using matplotlib to create the frames it therefore allows much more flexible (one can set transparency of markers, crop, and easily customize).
save_frames: bool
If true creates each frame individual and then combines into a video. This variant is relatively slow as
it stores all individual frames.
Frames2plot: List of indices
If not None & save_frames=True then the frames corresponding to the index will be plotted. For example, Frames2plot=[0,11] will plot the first and the 12th frame.
displayedbodyparts: list of strings, optional
This selects the body parts that are plotted in the video. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
displayedindividuals: list of strings, optional
Individuals plotted in the video. By default, all individuals present in the config will be showed.
codec: codec for labeled video. Options see http://www.fourcc.org/codecs.php [depends on your ffmpeg installation.]
outputframerate: positive number, output frame rate for labeled video (only available for the mode with saving frames.) By default: None, which results in the original video rate.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
draw_skeleton: bool
If ``True`` adds a line connecting the body parts making a skeleton on on each frame. The body parts to be connected and the color of these connecting lines are specified in the config file. By default: ``False``
trailpoints: int
Number of revious frames whose body parts are plotted in a frame (for displaying history). Default is set to 0.
displaycropped: bool, optional
Specifies whether only cropped frame is displayed (with labels analyzed therein), or the original frame with the labels analyzed in the cropped subset.
color_by : string, optional (default='bodypart')
Coloring rule. By default, each bodypart is colored differently.
If set to 'individual', points belonging to a single individual are colored the same.
Examples
--------
If you want to create the labeled video for only 1 video
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to create the labeled video for only 1 video and store the individual frames
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'],fastmode=True, save_frames=True)
--------
If you want to create the labeled video for multiple videos
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to create the labeled video for all the videos (as .avi extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'])
--------
If you want to create the labeled video for all the videos (as .mp4 extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'],videotype='mp4')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, modelprefix=modelprefix
) # automatically loads corresponding model (even training iteration based on snapshot index)
if save_frames:
fastmode = False # otherwise one cannot save frames
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
individuals = auxfun_multianimal.IntersectionofIndividualsandOnesGivenbyUser(
cfg, displayedindividuals)
if draw_skeleton:
bodyparts2connect = cfg["skeleton"]
skeleton_color = cfg["skeleton_color"]
else:
bodyparts2connect = None
skeleton_color = None
start_path = os.getcwd()
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
if not len(Videos):
print(
"No video(s) were found. Please check your paths and/or 'video_type'."
)
return
for video in Videos:
videofolder = Path(video).parents[0]
if destfolder is None:
destfolder = videofolder # where your folder with videos is.
auxiliaryfunctions.attempttomakefolder(destfolder)
os.chdir(destfolder) # THE VIDEO IS STILL IN THE VIDEO FOLDER
videotype = Path(video).suffix
print("Starting % ", destfolder, videos)
vname = str(Path(video).stem)
# if notanalyzed:
# notanalyzed,outdataname,sourcedataname,DLCscorer=auxiliaryfunctions.CheckifPostProcessing(folder,vname,DLCscorer,DLCscorerlegacy,suffix='checking')
if filtered == True:
videooutname1 = os.path.join(vname + DLCscorer +
"filtered_labeled.mp4")
videooutname2 = os.path.join(vname + DLCscorerlegacy +
"filtered_labeled.mp4")
else:
videooutname1 = os.path.join(vname + DLCscorer + "_labeled.mp4")
videooutname2 = os.path.join(vname + DLCscorerlegacy +
"_labeled.mp4")
if os.path.isfile(videooutname1) or os.path.isfile(videooutname2):
print("Labeled video already created.")
else:
print("Loading ", video, "and data.")
try:
df, filepath, _, _ = auxiliaryfunctions.load_analyzed_data(
destfolder, vname, DLCscorer, filtered, track_method)
metadata = auxiliaryfunctions.load_video_metadata(
destfolder, vname, DLCscorer)
if cfg.get("multianimalproject", False):
s = "_id" if color_by == "individual" else "_bp"
else:
s = ""
videooutname = filepath.replace(".h5", f"{s}_labeled.mp4")
if os.path.isfile(videooutname):
print("Labeled video already created. Skipping...")
continue
if all(individuals):
df = df.loc(axis=1)[:, individuals]
cropping = metadata["data"]["cropping"]
[x1, x2, y1, y2] = metadata["data"]["cropping_parameters"]
labeled_bpts = [
bp for bp in df.columns.get_level_values(
"bodyparts").unique() if bp in bodyparts
]
if not fastmode:
tmpfolder = os.path.join(str(videofolder), "temp-" + vname)
if save_frames:
auxiliaryfunctions.attempttomakefolder(tmpfolder)
clip = vp(video)
CreateVideoSlow(
videooutname,
clip,
df,
tmpfolder,
cfg["dotsize"],
cfg["colormap"],
cfg["alphavalue"],
cfg["pcutoff"],
trailpoints,
cropping,
x1,
x2,
y1,
y2,
save_frames,
labeled_bpts,
outputframerate,
Frames2plot,
bodyparts2connect,
skeleton_color,
draw_skeleton,
displaycropped,
color_by,
)
else:
if (displaycropped
): # then the cropped video + the labels is depicted
clip = vp(
fname=video,
sname=videooutname,
codec=codec,
sw=x2 - x1,
sh=y2 - y1,
)
else: # then the full video + the (perhaps in cropped mode analyzed labels) are depicted
clip = vp(fname=video, sname=videooutname, codec=codec)
CreateVideo(
clip,
df,
cfg["pcutoff"],
cfg["dotsize"],
cfg["colormap"],
labeled_bpts,
trailpoints,
cropping,
x1,
x2,
y1,
y2,
bodyparts2connect,
skeleton_color,
draw_skeleton,
displaycropped,
color_by,
)
except FileNotFoundError as e:
print(e)
continue
os.chdir(start_path)
def ExtractFramesbasedonPreselection(
Index,
extractionalgorithm,
Dataframe,
dataname,
video,
cfg,
config,
opencv=True,
cluster_resizewidth=30,
cluster_color=False,
savelabeled=True,
):
from deeplabcut.create_project import add
start = cfg["start"]
stop = cfg["stop"]
numframes2extract = cfg["numframes2pick"]
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(cfg, "all")
videofolder = str(Path(video).parents[0])
vname = str(Path(video).stem)
tmpfolder = os.path.join(cfg["project_path"], "labeled-data", vname)
if os.path.isdir(tmpfolder):
print("Frames from video", vname, " already extracted (more will be added)!")
else:
auxiliaryfunctions.attempttomakefolder(tmpfolder)
nframes = len(Dataframe)
print("Loading video...")
if opencv:
cap = cv2.VideoCapture(video)
fps = cap.get(5)
duration = nframes * 1.0 / fps
size = (int(cap.get(4)), int(cap.get(3)))
else:
from moviepy.editor import VideoFileClip
clip = VideoFileClip(video)
fps = clip.fps
duration = clip.duration
size = clip.size
if cfg["cropping"]: # one might want to adjust
coords = (cfg["x1"], cfg["x2"], cfg["y1"], cfg["y2"])
else:
coords = None
print("Duration of video [s]: ", duration, ", recorded @ ", fps, "fps!")
print(
"Overall # of frames: ", nframes, "with (cropped) frame dimensions: ",
)
if extractionalgorithm == "uniform":
if opencv:
frames2pick = frameselectiontools.UniformFramescv2(
cap, numframes2extract, start, stop, Index
)
else:
frames2pick = frameselectiontools.UniformFrames(
clip, numframes2extract, start, stop, Index
)
elif extractionalgorithm == "kmeans":
if opencv:
frames2pick = frameselectiontools.KmeansbasedFrameselectioncv2(
cap,
numframes2extract,
start,
stop,
cfg["cropping"],
coords,
Index,
resizewidth=cluster_resizewidth,
color=cluster_color,
)
else:
if cfg["cropping"]:
clip = clip.crop(y1=cfg["y1"], y2=cfg["x2"], x1=cfg["x1"], x2=cfg["x2"])
frames2pick = frameselectiontools.KmeansbasedFrameselection(
clip,
numframes2extract,
start,
stop,
Index,
resizewidth=cluster_resizewidth,
color=cluster_color,
)
else:
print(
"Please implement this method yourself! Currently the options are 'kmeans', 'jump', 'uniform'."
)
frames2pick = []
# Extract frames + frames with plotted labels and store them in folder (with name derived from video name) nder labeled-data
print("Let's select frames indices:", frames2pick)
colors = visualization.get_cmap(len(bodyparts), cfg["colormap"])
strwidth = int(np.ceil(np.log10(nframes))) # width for strings
for index in frames2pick: ##tqdm(range(0,nframes,10)):
if opencv:
PlottingSingleFramecv2(
cap,
cfg["cropping"],
coords,
Dataframe,
bodyparts,
tmpfolder,
index,
cfg["dotsize"],
cfg["pcutoff"],
cfg["alphavalue"],
colors,
strwidth,
savelabeled,
)
else:
PlottingSingleFrame(
clip,
Dataframe,
bodyparts,
tmpfolder,
index,
cfg["dotsize"],
cfg["pcutoff"],
cfg["alphavalue"],
colors,
strwidth,
savelabeled,
)
plt.close("all")
# close videos
if opencv:
cap.release()
else:
clip.close()
del clip
# Extract annotations based on DeepLabCut and store in the folder (with name derived from video name) under labeled-data
if len(frames2pick) > 0:
DF = Dataframe.loc[frames2pick]
DF.index = [
os.path.join(
"labeled-data", vname, "img" + str(index).zfill(strwidth) + ".png"
)
for index in DF.index
] # exchange index number by file names.
machinefile = os.path.join(
tmpfolder, "machinelabels-iter" + str(cfg["iteration"]) + ".h5"
)
if Path(machinefile).is_file():
Data = pd.read_hdf(machinefile, "df_with_missing")
DataCombined = pd.concat([Data, DF])
# drop duplicate labels:
DataCombined = DataCombined[~DataCombined.index.duplicated(keep="first")]
DataCombined.to_hdf(machinefile, key="df_with_missing", mode="w")
DataCombined.to_csv(
os.path.join(tmpfolder, "machinelabels.csv")
) # this is always the most current one (as reading is from h5)
else:
DF.to_hdf(machinefile, key="df_with_missing", mode="w")
DF.to_csv(os.path.join(tmpfolder, "machinelabels.csv"))
try:
if cfg["cropping"]:
add.add_new_videos(
config, [video], coords=[coords]
) # make sure you pass coords as a list
else:
add.add_new_videos(config, [video], coords=None)
except: # can we make a catch here? - in fact we should drop indices from DataCombined if they are in CollectedData.. [ideal behavior; currently this is pretty unlikely]
print(
"AUTOMATIC ADDING OF VIDEO TO CONFIG FILE FAILED! You need to do this manually for including it in the config.yaml file!"
)
print("Videopath:", video, "Coordinates for cropping:", coords)
pass
print(
"The outlier frames are extracted. They are stored in the subdirectory labeled-data\%s."
% vname
)
print(
"Once you extracted frames for all videos, use 'refine_labels' to manually correct the labels."
)
else:
print("No frames were extracted.")
def create_labeled_video(
config,
videos,
videotype="",
shuffle=1,
trainingsetindex=0,
filtered=False,
fastmode=True,
save_frames=False,
keypoints_only=False,
Frames2plot=None,
displayedbodyparts="all",
displayedindividuals="all",
codec="mp4v",
outputframerate=None,
destfolder=None,
draw_skeleton=False,
trailpoints=0,
displaycropped=False,
color_by="bodypart",
modelprefix="",
track_method="",
):
"""Labels the bodyparts in a video.
Make sure the video is already analyzed by the function
``deeplabcut.analyze_videos``.
Parameters
----------
config : string
Full path of the config.yaml file.
videos : list[str]
A list of strings containing the full paths to videos for analysis or a path
to the directory, where all the videos with same extension are stored.
videotype: str, optional, default=""
Checks for the extension of the video in case the input to the video is a
directory. Only videos with this extension are analyzed.
If left unspecified, videos with common extensions
('avi', 'mp4', 'mov', 'mpeg', 'mkv') are kept.
shuffle : int, optional, default=1
Number of shuffles of training dataset.
trainingsetindex: int, optional, default=0
Integer specifying which TrainingsetFraction to use.
Note that TrainingFraction is a list in config.yaml.
filtered: bool, optional, default=False
Boolean variable indicating if filtered output should be plotted rather than
frame-by-frame predictions. Filtered version can be calculated with
``deeplabcut.filterpredictions``.
fastmode: bool, optional, default=True
If ``True``, uses openCV (much faster but less customization of video) instead
of matplotlib if ``False``. You can also "save_frames" individually or not in
the matplotlib mode (if you set the "save_frames" variable accordingly).
However, using matplotlib to create the frames it therefore allows much more
flexible (one can set transparency of markers, crop, and easily customize).
save_frames: bool, optional, default=False
If ``True``, creates each frame individual and then combines into a video.
Setting this to ``True`` is relatively slow as it stores all individual frames.
keypoints_only: bool, optional, default=False
By default, both video frames and keypoints are visible. If ``True``, only the
keypoints are shown. These clips are an hommage to Johansson movies,
see https://www.youtube.com/watch?v=1F5ICP9SYLU and of course his seminal
paper: "Visual perception of biological motion and a model for its analysis"
by Gunnar Johansson in Perception & Psychophysics 1973.
Frames2plot: List[int] or None, optional, default=None
If not ``None`` and ``save_frames=True`` then the frames corresponding to the
index will be plotted. For example, ``Frames2plot=[0,11]`` will plot the first
and the 12th frame.
displayedbodyparts: list[str] or str, optional, default="all"
This selects the body parts that are plotted in the video. If ``all``, then all
body parts from config.yaml are used. If a list of strings that are a subset of
the full list. E.g. ['hand','Joystick'] for the demo
Reaching-Mackenzie-2018-08-30/config.yaml to select only these body parts.
displayedindividuals: list[str] or str, optional, default="all"
Individuals plotted in the video.
By default, all individuals present in the config will be showed.
codec: str, optional, default="mp4v"
Codec for labeled video. For available options, see
http://www.fourcc.org/codecs.php. Note that this depends on your ffmpeg
installation.
outputframerate: int or None, optional, default=None
Positive number, output frame rate for labeled video (only available for the
mode with saving frames.) If ``None``, which results in the original video
rate.
destfolder: string or None, optional, default=None
Specifies the destination folder that was used for storing analysis data. If
``None``, the path of the video file is used.
draw_skeleton: bool, optional, default=False
If ``True`` adds a line connecting the body parts making a skeleton on each
frame. The body parts to be connected and the color of these connecting lines
are specified in the config file.
trailpoints: int, optional, default=0
Number of previous frames whose body parts are plotted in a frame
(for displaying history).
displaycropped: bool, optional, default=False
Specifies whether only cropped frame is displayed (with labels analyzed
therein), or the original frame with the labels analyzed in the cropped subset.
color_by : string, optional, default='bodypart'
Coloring rule. By default, each bodypart is colored differently.
If set to 'individual', points belonging to a single individual are colored the
same.
modelprefix: str, optional, default=""
Directory containing the deeplabcut models to use when evaluating the network.
By default, the models are assumed to exist in the project folder.
track_method: string, optional, default=""
Specifies the tracker used to generate the data.
Empty by default (corresponding to a single animal project).
For multiple animals, must be either 'box', 'skeleton', or 'ellipse' and will
be taken from the config.yaml file if none is given.
Returns
-------
None
Examples
--------
Create the labeled video for a single video
>>> deeplabcut.create_labeled_video(
'/analysis/project/reaching-task/config.yaml',
['/analysis/project/videos/reachingvideo1.avi'],
)
Create the labeled video for a single video and store the individual frames
>>> deeplabcut.create_labeled_video(
'/analysis/project/reaching-task/config.yaml',
['/analysis/project/videos/reachingvideo1.avi'],
fastmode=True,
save_frames=True,
)
Create the labeled video for multiple videos
>>> deeplabcut.create_labeled_video(
'/analysis/project/reaching-task/config.yaml',
[
'/analysis/project/videos/reachingvideo1.avi',
'/analysis/project/videos/reachingvideo2.avi',
],
)
Create the labeled video for all the videos with an .avi extension in a directory.
>>> deeplabcut.create_labeled_video(
'/analysis/project/reaching-task/config.yaml',
['/analysis/project/videos/'],
)
Create the labeled video for all the videos with an .mp4 extension in a directory.
>>> deeplabcut.create_labeled_video(
'/analysis/project/reaching-task/config.yaml',
['/analysis/project/videos/'],
videotype='mp4',
)
"""
cfg = auxiliaryfunctions.read_config(config)
track_method = auxfun_multianimal.get_track_method(
cfg, track_method=track_method)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, modelprefix=modelprefix
) # automatically loads corresponding model (even training iteration based on snapshot index)
if save_frames:
fastmode = False # otherwise one cannot save frames
keypoints_only = False
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
individuals = auxfun_multianimal.IntersectionofIndividualsandOnesGivenbyUser(
cfg, displayedindividuals)
if draw_skeleton:
bodyparts2connect = cfg["skeleton"]
skeleton_color = cfg["skeleton_color"]
else:
bodyparts2connect = None
skeleton_color = None
start_path = os.getcwd()
Videos = auxiliaryfunctions.get_list_of_videos(videos, videotype)
if not Videos:
return
func = partial(
proc_video,
videos,
destfolder,
filtered,
DLCscorer,
DLCscorerlegacy,
track_method,
cfg,
individuals,
color_by,
bodyparts,
codec,
bodyparts2connect,
trailpoints,
save_frames,
outputframerate,
Frames2plot,
draw_skeleton,
skeleton_color,
displaycropped,
fastmode,
keypoints_only,
)
with Pool(min(os.cpu_count(), len(Videos))) as pool:
pool.map(func, Videos)
os.chdir(start_path)
def extract_maps(
config,
shuffle=0,
trainingsetindex=0,
comparisonbodyparts="all",
gputouse=None,
rescale=False,
Indices=None,
modelprefix="",
):
"""
Extracts the scoremap, locref, partaffinityfields (if available).
Returns a dictionary indexed by: trainingsetfraction, snapshotindex, and imageindex
for those keys, each item contains: (image,scmap,locref,paf,bpt names,partaffinity graph, imagename, True/False if this image was in trainingset)
----------
config : string
Full path of the config.yaml file as a string.
shuffle: integer
integers specifying shuffle index of the training dataset. The default is 0.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This
variable can also be set to "all".
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
rescale: bool, default False
Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every
image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported
in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated
on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the
original size!
Examples
--------
If you want to extract the data for image 0 and 103 (of the training set) for model trained with shuffle 0.
>>> deeplabcut.extract_maps(configfile,0,Indices=[0,103])
"""
from deeplabcut.utils.auxfun_videos import imread, imresize
from deeplabcut.pose_estimation_tensorflow.nnet import predict
from deeplabcut.pose_estimation_tensorflow.nnet import (
predict_multianimal as predictma, )
from deeplabcut.pose_estimation_tensorflow.config import load_config
from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input
from deeplabcut.utils import auxiliaryfunctions
from tqdm import tqdm
import tensorflow as tf
vers = (tf.__version__).split(".")
if int(vers[0]) == 1 and int(vers[1]) > 12:
TF = tf.compat.v1
else:
TF = tf
import pandas as pd
from pathlib import Path
import numpy as np
TF.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
# tf.logging.set_verbosity(tf.logging.WARN)
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
if trainingsetindex < len(
cfg["TrainingFraction"]) and trainingsetindex >= 0:
TrainingFractions = [
cfg["TrainingFraction"][int(trainingsetindex)]
]
else:
raise Exception(
"Please check the trainingsetindex! ",
trainingsetindex,
" should be an integer from 0 .. ",
int(len(cfg["TrainingFraction"]) - 1),
)
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
)
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
Maps = {}
for trainFraction in TrainingFractions:
Maps[trainFraction] = {}
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# change batch size, if it was edited during analysis!
dlc_cfg["batch_size"] = 1 # in case this was edited for analysis.
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
# path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
])
try: # check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
########################### RESCALING (to global scale)
if rescale == True:
scale = dlc_cfg["global_scale"]
Data = (pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
) * scale)
else:
scale = 1
bptnames = [
dlc_cfg["all_joints_names"][i]
for i in range(len(dlc_cfg["all_joints"]))
]
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg["init_weights"].split(os.sep)[-1]
).split("-")[
-1] # read how many training siterations that corresponds to.
# Name for deeplabcut net (based on its parameters)
# DLCscorer,DLCscorerlegacy = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations)
# notanalyzed, resultsfilename, DLCscorer=auxiliaryfunctions.CheckifNotEvaluated(str(evaluationfolder),DLCscorer,DLCscorerlegacy,Snapshots[snapindex])
# print("Extracting maps for ", DLCscorer, " with # of trainingiterations:", trainingsiterations)
# if notanalyzed: #this only applies to ask if h5 exists...
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
Numimages = len(Data.index)
PredicteData = np.zeros(
(Numimages, 3 * len(dlc_cfg["all_joints_names"])))
print("Analyzing data...")
if Indices is None:
Indices = enumerate(Data.index)
else:
Ind = [Data.index[j] for j in Indices]
Indices = enumerate(Ind)
DATA = {}
for imageindex, imagename in tqdm(Indices):
image = imread(os.path.join(cfg["project_path"], imagename),
mode="RGB")
if scale != 1:
image = imresize(image, scale)
image_batch = data_to_input(image)
# Compute prediction with the CNN
outputs_np = sess.run(outputs, feed_dict={inputs: image_batch})
if cfg.get("multianimalproject", False):
scmap, locref, paf = predictma.extract_cnn_output(
outputs_np, dlc_cfg)
pagraph = dlc_cfg["partaffinityfield_graph"]
else:
scmap, locref = predict.extract_cnn_output(
outputs_np, dlc_cfg)
paf = None
pagraph = []
if imageindex in testIndices:
trainingfram = False
else:
trainingfram = True
DATA[imageindex] = [
image,
scmap,
locref,
paf,
bptnames,
pagraph,
imagename,
trainingfram,
]
# return DATA
Maps[trainFraction][Snapshots[snapindex]] = DATA
os.chdir(str(start_path))
return Maps
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: # gpu selectinon
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
##############
# Cloning for-loop variables
shuffle = Shuffles[0]
trainFraction = cfg["TrainingFraction"][0]
##############
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(cfg, comparisonbodyparts)
##################################################
# Load and setup CNN part detector
##################################################
modelfolder = os.path.join(cfg["project_path"], str(auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
# Load meta data
# data, trainIndices, testIndices, trainFraction = auxiliaryfunctions.LoadMetadata(
# os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle s and trainFraction %s does not exist.")
def extract_save_all_maps(
config,
shuffle=1,
trainingsetindex=0,
comparisonbodyparts="all",
gputouse=None,
rescale=False,
Indices=None,
modelprefix="",
dest_folder=None,
nplots_per_row=None,
):
"""
Extracts the scoremap, location refinement field and part affinity field prediction of the model. The maps
will be rescaled to the size of the input image and stored in the corresponding model folder in /evaluation-results.
----------
config : string
Full path of the config.yaml file as a string.
shuffle: integer
integers specifying shuffle index of the training dataset. The default is 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This
variable can also be set to "all".
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
Indices: default None
For which images shall the scmap/locref and paf be computed? Give a list of images
nplots_per_row: int, optional (default=None)
Number of plots per row in grid plots. By default, calculated to approximate a squared grid of plots
Examples
--------
Calculated maps for images 0, 1 and 33.
>>> deeplabcut.extract_save_all_maps('/analysis/project/reaching-task/config.yaml', shuffle=1,Indices=[0,1,33])
"""
from deeplabcut.utils.auxiliaryfunctions import (
read_config,
attempttomakefolder,
GetEvaluationFolder,
)
from tqdm import tqdm
cfg = read_config(config)
data = extract_maps(
config,
shuffle,
trainingsetindex,
comparisonbodyparts,
gputouse,
rescale,
Indices,
modelprefix,
)
if not nplots_per_row:
from deeplabcut.utils import auxiliaryfunctions
bpts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
nplots_per_row = int(np.floor(np.sqrt(len(bpts))))
print("Saving plots...")
for frac, values in data.items():
if not dest_folder:
# dest_folder = os.path.join(cfg['project_path'], 'maps')
dest_folder = os.path.join(
cfg["project_path"],
str(
GetEvaluationFolder(frac,
shuffle,
cfg,
modelprefix=modelprefix)),
"maps",
)
attempttomakefolder(dest_folder)
dest_path = os.path.join(dest_folder, "{}_{}_{}_{}_{}_{}.png")
for snap, maps in values.items():
for imagenr in tqdm(maps):
(
image,
scmap,
locref,
paf,
bptnames,
pafgraph,
impath,
trainingframe,
) = maps[imagenr]
label = "train" if trainingframe else "test"
imname = os.path.split(os.path.splitext(impath)[0])[1]
if not os.path.isfile(
dest_path.format(imagenr, "scmap", label, shuffle,
frac, snap)):
scmap, (locref_x, locref_y), paf = resize_all_maps(
image, scmap, locref, paf)
fig1, _ = visualize_scoremaps(
image,
scmap,
labels=bptnames,
nplots_per_row=nplots_per_row)
fig2, _ = visualize_locrefs(
image,
scmap,
locref_x,
locref_y,
labels=bptnames,
nplots_per_row=nplots_per_row,
)
fig3, _ = visualize_locrefs(
image,
scmap,
locref_x,
locref_y,
zoom_width=100,
labels=bptnames,
nplots_per_row=nplots_per_row,
)
if paf is not None:
fig4, _ = visualize_paf(
image,
paf,
pafgraph,
labels=bptnames,
nplots_per_row=nplots_per_row,
)
fig1.savefig(
dest_path.format(imname, "scmap", label, shuffle, frac,
snap))
fig2.savefig(
dest_path.format(imname, "locref", label, shuffle,
frac, snap))
fig3.savefig(
dest_path.format(imname, "locrefzoom", label, shuffle,
frac, snap))
if paf is not None:
fig4.savefig(
dest_path.format(imname, "paf", label, shuffle,
frac, snap))
plt.close("all")
def evaluate_multianimal_full(
config,
Shuffles=[1],
trainingsetindex=0,
plotting=None,
show_errors=True,
comparisonbodyparts="all",
gputouse=None,
modelprefix="",
c_engine=False,
):
"""
WIP multi animal project.
"""
import os
from deeplabcut.pose_estimation_tensorflow.nnet import predict
from deeplabcut.pose_estimation_tensorflow.nnet import (
predict_multianimal as predictma, )
from deeplabcut.utils import auxiliaryfunctions, auxfun_multianimal
from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input
import tensorflow as tf
if "TF_CUDNN_USE_AUTOTUNE" in os.environ:
del os.environ[
"TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training
tf.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
start_path = os.getcwd()
##################################################
# Load data...
##################################################
cfg = auxiliaryfunctions.read_config(config)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
TrainingFractions = [cfg["TrainingFraction"][trainingsetindex]]
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
)
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
colors = visualization.get_cmap(len(comparisonbodyparts),
name=cfg["colormap"])
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# TODO: IMPLEMENT for different batch sizes?
dlc_cfg["batch_size"] = 1 # due to differently sized images!!!
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
# path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
])
if len(Snapshots) == 0:
print(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
else:
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
final_result = []
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg["init_weights"].split(os.sep)[-1]
).split(
"-"
)[-1] # read how many training siterations that corresponds to.
# name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix,
)
print(
"Running ",
DLCscorer,
" with # of trainingiterations:",
trainingsiterations,
)
(
notanalyzed,
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.CheckifNotEvaluated(
str(evaluationfolder),
DLCscorer,
DLCscorerlegacy,
Snapshots[snapindex],
)
if os.path.isfile(
resultsfilename.split(".h5")[0] + "_full.pickle"):
print("Model already evaluated.", resultsfilename)
else:
if plotting:
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" +
Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
# print(dlc_cfg)
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = predict.setup_pose_prediction(
dlc_cfg)
PredicteData = {}
print("Analyzing data...")
for imageindex, imagename in tqdm(enumerate(
Data.index)):
image_path = os.path.join(cfg["project_path"],
imagename)
image = io.imread(image_path)
frame = img_as_ubyte(skimage.color.gray2rgb(image))
GT = Data.iloc[imageindex]
# Storing GT data as dictionary, so it can be used for calculating connection costs
groundtruthcoordinates = []
groundtruthidentity = []
for bptindex, bpt in enumerate(
dlc_cfg["all_joints_names"]):
coords = np.zeros([len(individuals), 2
]) * np.nan
identity = []
for prfxindex, prefix in enumerate(
individuals):
if bpt in uniquebodyparts and prefix == "single":
coords[prfxindex, :] = np.array([
GT[cfg["scorer"]][prefix][bpt]
["x"],
GT[cfg["scorer"]][prefix][bpt]
["y"],
])
identity.append(prefix)
elif (bpt in multianimalbodyparts
and prefix != "single"):
coords[prfxindex, :] = np.array([
GT[cfg["scorer"]][prefix][bpt]
["x"],
GT[cfg["scorer"]][prefix][bpt]
["y"],
])
identity.append(prefix)
else:
identity.append("nix")
groundtruthcoordinates.append(
coords[np.isfinite(coords[:, 0]), :])
groundtruthidentity.append(
np.array(identity)[np.isfinite(coords[:,
0])])
PredicteData[imagename] = {}
PredicteData[imagename]["index"] = imageindex
pred = predictma.get_detectionswithcostsandGT(
frame,
groundtruthcoordinates,
dlc_cfg,
sess,
inputs,
outputs,
outall=False,
nms_radius=dlc_cfg.nmsradius,
det_min_score=dlc_cfg.minconfidence,
c_engine=c_engine,
)
PredicteData[imagename]["prediction"] = pred
PredicteData[imagename]["groundtruth"] = [
groundtruthidentity,
groundtruthcoordinates,
GT,
]
if plotting:
coords_pred = pred["coordinates"][0]
probs_pred = pred["confidence"]
fig = visualization.make_multianimal_labeled_image(
frame,
groundtruthcoordinates,
coords_pred,
probs_pred,
colors,
cfg["dotsize"],
cfg["alphavalue"],
cfg["pcutoff"],
)
visualization.save_labeled_frame(
fig,
image_path,
foldername,
imageindex in trainIndices,
)
sess.close() # closes the current tf session
PredicteData["metadata"] = {
"nms radius":
dlc_cfg.nmsradius,
"minimal confidence":
dlc_cfg.minconfidence,
"PAFgraph":
dlc_cfg.partaffinityfield_graph,
"all_joints":
[[i] for i in range(len(dlc_cfg.all_joints))],
"all_joints_names": [
dlc_cfg.all_joints_names[i]
for i in range(len(dlc_cfg.all_joints))
],
"stride":
dlc_cfg.get("stride", 8),
}
print(
"Done and results stored for snapshot: ",
Snapshots[snapindex],
)
dictionary = {
"Scorer": DLCscorer,
"DLC-model-config file": dlc_cfg,
"trainIndices": trainIndices,
"testIndices": testIndices,
"trainFraction": trainFraction,
}
metadata = {"data": dictionary}
auxfun_multianimal.SaveFullMultiAnimalData(
PredicteData, metadata, resultsfilename)
tf.reset_default_graph()
# returning to intial folder
os.chdir(str(start_path))
def evaluate_multianimal_full(
config,
Shuffles=[1],
trainingsetindex=0,
plotting=None,
show_errors=True,
comparisonbodyparts="all",
gputouse=None,
modelprefix="",
c_engine=False,
):
from deeplabcut.pose_estimation_tensorflow.nnet import predict
from deeplabcut.pose_estimation_tensorflow.nnet import (
predict_multianimal as predictma,
)
from deeplabcut.utils import auxiliaryfunctions, auxfun_multianimal
import tensorflow as tf
if "TF_CUDNN_USE_AUTOTUNE" in os.environ:
del os.environ["TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training
tf.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
start_path = os.getcwd()
##################################################
# Load data...
##################################################
cfg = auxiliaryfunctions.read_config(config)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
TrainingFractions = [cfg["TrainingFraction"][trainingsetindex]]
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
)
# Handle data previously annotated on a different platform
sep = "/" if "/" in Data.index[0] else "\\"
if sep != os.path.sep:
Data.index = Data.index.str.replace(sep, os.path.sep)
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts
)
all_bpts = np.asarray(
len(cfg["individuals"]) * cfg["multianimalbodyparts"] + cfg["uniquebodyparts"]
)
colors = visualization.get_cmap(len(comparisonbodyparts), name=cfg["colormap"])
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/")
)
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg
)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix
)
),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn)
)
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction)
)
# TODO: IMPLEMENT for different batch sizes?
dlc_cfg["batch_size"] = 1 # due to differently sized images!!!
joints = dlc_cfg["all_joints_names"]
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix
)
),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True)
# path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array(
[
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
]
)
if len(Snapshots) == 0:
print(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction)
)
else:
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots]
)
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
final_result = []
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg["init_weights"].split(os.sep)[-1]
).split("-")[
-1
] # read how many training siterations that corresponds to.
# name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix,
)
print(
"Running ",
DLCscorer,
" with # of trainingiterations:",
trainingsiterations,
)
(
notanalyzed,
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.CheckifNotEvaluated(
str(evaluationfolder),
DLCscorer,
DLCscorerlegacy,
Snapshots[snapindex],
)
if os.path.isfile(resultsfilename.split(".h5")[0] + "_full.pickle"):
print("Model already evaluated.", resultsfilename)
else:
if plotting:
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_"
+ DLCscorer
+ "_"
+ Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
# print(dlc_cfg)
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
PredicteData = {}
dist = np.full((len(Data), len(all_bpts)), np.nan)
conf = np.full_like(dist, np.nan)
distnorm = np.full(len(Data), np.nan)
print("Analyzing data...")
for imageindex, imagename in tqdm(enumerate(Data.index)):
image_path = os.path.join(cfg["project_path"], imagename)
image = io.imread(image_path)
if image.ndim == 2 or image.shape[-1] == 1:
image = skimage.color.gray2rgb(image)
frame = img_as_ubyte(image)
GT = Data.iloc[imageindex]
df = GT.unstack("coords").reindex(joints, level='bodyparts')
# Evaluate PAF edge lengths to calibrate `distnorm`
temp_xy = GT.unstack("bodyparts")[joints]
xy = temp_xy.values.reshape((-1, 2, temp_xy.shape[1])).swapaxes(
1, 2
)
if dlc_cfg['partaffinityfield_predict']:
edges = xy[:, dlc_cfg["partaffinityfield_graph"]]
lengths = np.sum(
(edges[:, :, 0] - edges[:, :, 1]) ** 2, axis=2
)
distnorm[imageindex] = np.nanmax(lengths)
# FIXME Is having an empty array vs nan really that necessary?!
groundtruthidentity = list(
df.index.get_level_values("individuals")
.to_numpy()
.reshape((-1, 1))
)
groundtruthcoordinates = list(df.values[:, np.newaxis])
for i, coords in enumerate(groundtruthcoordinates):
if np.isnan(coords).any():
groundtruthcoordinates[i] = np.empty(
(0, 2), dtype=float
)
groundtruthidentity[i] = np.array([], dtype=str)
PredicteData[imagename] = {}
PredicteData[imagename]["index"] = imageindex
pred = predictma.get_detectionswithcostsandGT(
frame,
groundtruthcoordinates,
dlc_cfg,
sess,
inputs,
outputs,
outall=False,
nms_radius=dlc_cfg['nmsradius'],
det_min_score=dlc_cfg['minconfidence'],
c_engine=c_engine,
)
PredicteData[imagename]["prediction"] = pred
PredicteData[imagename]["groundtruth"] = [
groundtruthidentity,
groundtruthcoordinates,
GT,
]
coords_pred = pred["coordinates"][0]
probs_pred = pred["confidence"]
for bpt, xy_gt in df.groupby(level="bodyparts"):
inds_gt = np.flatnonzero(
np.all(~np.isnan(xy_gt), axis=1)
)
n_joint = joints.index(bpt)
xy = coords_pred[n_joint]
if inds_gt.size and xy.size:
# Pick the predictions closest to ground truth,
# rather than the ones the model has most confident in
d = cdist(xy_gt.iloc[inds_gt], xy)
rows, cols = linear_sum_assignment(d)
min_dists = d[rows, cols]
inds = np.flatnonzero(all_bpts == bpt)
sl = imageindex, inds[inds_gt[rows]]
dist[sl] = min_dists
conf[sl] = probs_pred[n_joint][cols].squeeze()
if plotting:
gt = (temp_xy.values
.reshape((-1, 2, temp_xy.shape[1]))
.T.swapaxes(1, 2))
fig = visualization.make_multianimal_labeled_image(
frame,
gt,
coords_pred,
probs_pred,
colors,
cfg["dotsize"],
cfg["alphavalue"],
cfg["pcutoff"],
)
visualization.save_labeled_frame(
fig,
image_path,
foldername,
imageindex in trainIndices,
)
sess.close() # closes the current tf session
# Compute all distance statistics
df_dist = pd.DataFrame(dist, columns=df.index)
df_conf = pd.DataFrame(conf, columns=df.index)
df_joint = pd.concat(
[df_dist, df_conf],
keys=["rmse", "conf"],
names=["metrics"],
axis=1
)
df_joint = df_joint.reorder_levels(
list(np.roll(df_joint.columns.names, -1)), axis=1
)
df_joint.sort_index(
axis=1,
level=["individuals", "bodyparts"],
ascending=[True, True],
inplace=True
)
write_path = os.path.join(evaluationfolder, f"dist_{trainingsiterations}.csv")
df_joint.to_csv(write_path)
# Calculate overall prediction error
error = df_joint.xs("rmse", level="metrics", axis=1)
mask = df_joint.xs("conf", level="metrics", axis=1) >= cfg["pcutoff"]
error_masked = error[mask]
error_train = np.nanmean(error.iloc[trainIndices])
error_train_cut = np.nanmean(error_masked.iloc[trainIndices])
error_test = np.nanmean(error.iloc[testIndices])
error_test_cut = np.nanmean(error_masked.iloc[testIndices])
results = [
trainingsiterations,
int(100 * trainFraction),
shuffle,
np.round(error_train, 2),
np.round(error_test, 2),
cfg["pcutoff"],
np.round(error_train_cut, 2),
np.round(error_test_cut, 2),
]
final_result.append(results)
# For OKS/PCK, compute the standard deviation error across all frames
sd = df_dist.groupby("bodyparts", axis=1).mean().std(axis=0)
sd["distnorm"] = np.sqrt(np.nanmax(distnorm))
sd.to_csv(write_path.replace("dist.csv", "sd.csv"))
if show_errors:
string = "Results for {} training iterations: {}, shuffle {}:\n" \
"Train error: {} pixels. Test error: {} pixels.\n" \
"With pcutoff of {}:\n" \
"Train error: {} pixels. Test error: {} pixels."
print(string.format(*results))
print("##########################################")
print("Average Euclidean distance to GT per individual (in pixels)")
print(error_masked.groupby('individuals', axis=1).mean().mean().to_string())
print("Average Euclidean distance to GT per bodypart (in pixels)")
print(error_masked.groupby('bodyparts', axis=1).mean().mean().to_string())
PredicteData["metadata"] = {
"nms radius": dlc_cfg['nmsradius'],
"minimal confidence": dlc_cfg['minconfidence'],
"PAFgraph": dlc_cfg['partaffinityfield_graph'],
"all_joints": [[i] for i in range(len(dlc_cfg['all_joints']))],
"all_joints_names": [
dlc_cfg['all_joints_names'][i]
for i in range(len(dlc_cfg['all_joints']))
],
"stride": dlc_cfg.get("stride", 8),
}
print(
"Done and results stored for snapshot: ",
Snapshots[snapindex],
)
dictionary = {
"Scorer": DLCscorer,
"DLC-model-config file": dlc_cfg,
"trainIndices": trainIndices,
"testIndices": testIndices,
"trainFraction": trainFraction,
}
metadata = {"data": dictionary}
auxfun_multianimal.SaveFullMultiAnimalData(
PredicteData, metadata, resultsfilename
)
tf.reset_default_graph()
if len(final_result) > 0: # Only append if results were calculated
make_results_file(final_result, evaluationfolder, DLCscorer)
# returning to intial folder
os.chdir(str(start_path))
def evaluate_multianimal_full(
config,
Shuffles=[1],
trainingsetindex=0,
plotting=False,
show_errors=True,
comparisonbodyparts="all",
gputouse=None,
modelprefix="",
):
from deeplabcut.pose_estimation_tensorflow.core import (
predict,
predict_multianimal as predictma,
)
from deeplabcut.utils import (
auxiliaryfunctions,
auxfun_multianimal,
auxfun_videos,
conversioncode,
)
import tensorflow as tf
if "TF_CUDNN_USE_AUTOTUNE" in os.environ:
del os.environ[
"TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training
tf.compat.v1.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
start_path = os.getcwd()
if plotting is True:
plotting = "bodypart"
##################################################
# Load data...
##################################################
cfg = auxiliaryfunctions.read_config(config)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
TrainingFractions = [cfg["TrainingFraction"][trainingsetindex]]
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
))
conversioncode.guarantee_multiindex_rows(Data)
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
all_bpts = np.asarray(
len(cfg["individuals"]) * cfg["multianimalbodyparts"] +
cfg["uniquebodyparts"])
colors = visualization.get_cmap(len(comparisonbodyparts),
name=cfg["colormap"])
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
pipeline = iaa.Sequential(random_order=False)
pre_resize = dlc_cfg.get("pre_resize")
if pre_resize:
width, height = pre_resize
pipeline.add(iaa.Resize({"height": height, "width": width}))
# TODO: IMPLEMENT for different batch sizes?
dlc_cfg["batch_size"] = 1 # due to differently sized images!!!
stride = dlc_cfg["stride"]
# Ignore best edges possibly defined during a prior evaluation
_ = dlc_cfg.pop("paf_best", None)
joints = dlc_cfg["all_joints_names"]
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
# path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
])
if len(Snapshots) == 0:
print(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
else:
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
final_result = []
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg["init_weights"].split(os.sep)[-1]
).split(
"-"
)[-1] # read how many training siterations that corresponds to.
# name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix,
)
print(
"Running ",
DLCscorer,
" with # of trainingiterations:",
trainingsiterations,
)
(
notanalyzed,
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.CheckifNotEvaluated(
str(evaluationfolder),
DLCscorer,
DLCscorerlegacy,
Snapshots[snapindex],
)
data_path = resultsfilename.split(
".h5")[0] + "_full.pickle"
if plotting:
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" +
Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
if plotting == "bodypart":
fig, ax = visualization.create_minimal_figure()
if os.path.isfile(data_path):
print("Model already evaluated.", resultsfilename)
else:
sess, inputs, outputs = predict.setup_pose_prediction(
dlc_cfg)
PredicteData = {}
dist = np.full((len(Data), len(all_bpts)), np.nan)
conf = np.full_like(dist, np.nan)
print("Network Evaluation underway...")
for imageindex, imagename in tqdm(enumerate(
Data.index)):
image_path = os.path.join(cfg["project_path"],
*imagename)
frame = auxfun_videos.imread(image_path,
mode="skimage")
GT = Data.iloc[imageindex]
if not GT.any():
continue
# Pass the image and the keypoints through the resizer;
# this has no effect if no augmenters were added to it.
keypoints = [
GT.to_numpy().reshape((-1, 2)).astype(float)
]
frame_, keypoints = pipeline(images=[frame],
keypoints=keypoints)
frame = frame_[0]
GT[:] = keypoints[0].flatten()
df = GT.unstack("coords").reindex(
joints, level="bodyparts")
# FIXME Is having an empty array vs nan really that necessary?!
groundtruthidentity = list(
df.index.get_level_values(
"individuals").to_numpy().reshape((-1, 1)))
groundtruthcoordinates = list(
df.values[:, np.newaxis])
for i, coords in enumerate(groundtruthcoordinates):
if np.isnan(coords).any():
groundtruthcoordinates[i] = np.empty(
(0, 2), dtype=float)
groundtruthidentity[i] = np.array(
[], dtype=str)
# Form 2D array of shape (n_rows, 4) where the last dimension
# is (sample_index, peak_y, peak_x, bpt_index) to slice the PAFs.
temp = df.reset_index(level="bodyparts").dropna()
temp["bodyparts"].replace(
dict(zip(joints, range(len(joints)))),
inplace=True,
)
temp["sample"] = 0
peaks_gt = temp.loc[:, [
"sample", "y", "x", "bodyparts"
]].to_numpy()
peaks_gt[:, 1:3] = (peaks_gt[:, 1:3] -
stride // 2) / stride
pred = predictma.predict_batched_peaks_and_costs(
dlc_cfg,
np.expand_dims(frame, axis=0),
sess,
inputs,
outputs,
peaks_gt.astype(int),
)
if not pred:
continue
else:
pred = pred[0]
PredicteData[imagename] = {}
PredicteData[imagename]["index"] = imageindex
PredicteData[imagename]["prediction"] = pred
PredicteData[imagename]["groundtruth"] = [
groundtruthidentity,
groundtruthcoordinates,
GT,
]
coords_pred = pred["coordinates"][0]
probs_pred = pred["confidence"]
for bpt, xy_gt in df.groupby(level="bodyparts"):
inds_gt = np.flatnonzero(
np.all(~np.isnan(xy_gt), axis=1))
n_joint = joints.index(bpt)
xy = coords_pred[n_joint]
if inds_gt.size and xy.size:
# Pick the predictions closest to ground truth,
# rather than the ones the model has most confident in
xy_gt_values = xy_gt.iloc[inds_gt].values
neighbors = _find_closest_neighbors(
xy_gt_values, xy, k=3)
found = neighbors != -1
min_dists = np.linalg.norm(
xy_gt_values[found] -
xy[neighbors[found]],
axis=1,
)
inds = np.flatnonzero(all_bpts == bpt)
sl = imageindex, inds[inds_gt[found]]
dist[sl] = min_dists
conf[sl] = probs_pred[n_joint][
neighbors[found]].squeeze()
if plotting == "bodypart":
temp_xy = GT.unstack(
"bodyparts")[joints].values
gt = temp_xy.reshape(
(-1, 2,
temp_xy.shape[1])).T.swapaxes(1, 2)
h, w, _ = np.shape(frame)
fig.set_size_inches(w / 100, h / 100)
ax.set_xlim(0, w)
ax.set_ylim(0, h)
ax.invert_yaxis()
ax = visualization.make_multianimal_labeled_image(
frame,
gt,
coords_pred,
probs_pred,
colors,
cfg["dotsize"],
cfg["alphavalue"],
cfg["pcutoff"],
ax=ax,
)
visualization.save_labeled_frame(
fig,
image_path,
foldername,
imageindex in trainIndices,
)
visualization.erase_artists(ax)
sess.close() # closes the current tf session
# Compute all distance statistics
df_dist = pd.DataFrame(dist, columns=df.index)
df_conf = pd.DataFrame(conf, columns=df.index)
df_joint = pd.concat(
[df_dist, df_conf],
keys=["rmse", "conf"],
names=["metrics"],
axis=1,
)
df_joint = df_joint.reorder_levels(list(
np.roll(df_joint.columns.names, -1)),
axis=1)
df_joint.sort_index(
axis=1,
level=["individuals", "bodyparts"],
ascending=[True, True],
inplace=True,
)
write_path = os.path.join(
evaluationfolder,
f"dist_{trainingsiterations}.csv")
df_joint.to_csv(write_path)
# Calculate overall prediction error
error = df_joint.xs("rmse", level="metrics", axis=1)
mask = (df_joint.xs("conf", level="metrics", axis=1) >=
cfg["pcutoff"])
error_masked = error[mask]
error_train = np.nanmean(error.iloc[trainIndices])
error_train_cut = np.nanmean(
error_masked.iloc[trainIndices])
error_test = np.nanmean(error.iloc[testIndices])
error_test_cut = np.nanmean(
error_masked.iloc[testIndices])
results = [
trainingsiterations,
int(100 * trainFraction),
shuffle,
np.round(error_train, 2),
np.round(error_test, 2),
cfg["pcutoff"],
np.round(error_train_cut, 2),
np.round(error_test_cut, 2),
]
final_result.append(results)
if show_errors:
string = (
"Results for {} training iterations, training fraction of {}, and shuffle {}:\n"
"Train error: {} pixels. Test error: {} pixels.\n"
"With pcutoff of {}:\n"
"Train error: {} pixels. Test error: {} pixels."
)
print(string.format(*results))
print("##########################################")
print(
"Average Euclidean distance to GT per individual (in pixels; test-only)"
)
print(error_masked.iloc[testIndices].groupby(
"individuals",
axis=1).mean().mean().to_string())
print(
"Average Euclidean distance to GT per bodypart (in pixels; test-only)"
)
print(error_masked.iloc[testIndices].groupby(
"bodyparts", axis=1).mean().mean().to_string())
PredicteData["metadata"] = {
"nms radius":
dlc_cfg["nmsradius"],
"minimal confidence":
dlc_cfg["minconfidence"],
"sigma":
dlc_cfg.get("sigma", 1),
"PAFgraph":
dlc_cfg["partaffinityfield_graph"],
"PAFinds":
np.arange(len(dlc_cfg["partaffinityfield_graph"])),
"all_joints":
[[i] for i in range(len(dlc_cfg["all_joints"]))],
"all_joints_names": [
dlc_cfg["all_joints_names"][i]
for i in range(len(dlc_cfg["all_joints"]))
],
"stride":
dlc_cfg.get("stride", 8),
}
print(
"Done and results stored for snapshot: ",
Snapshots[snapindex],
)
dictionary = {
"Scorer": DLCscorer,
"DLC-model-config file": dlc_cfg,
"trainIndices": trainIndices,
"testIndices": testIndices,
"trainFraction": trainFraction,
}
metadata = {"data": dictionary}
_ = auxfun_multianimal.SaveFullMultiAnimalData(
PredicteData, metadata, resultsfilename)
tf.compat.v1.reset_default_graph()
n_multibpts = len(cfg["multianimalbodyparts"])
if n_multibpts == 1:
continue
# Skip data-driven skeleton selection unless
# the model was trained on the full graph.
max_n_edges = n_multibpts * (n_multibpts - 1) // 2
n_edges = len(dlc_cfg["partaffinityfield_graph"])
if n_edges == max_n_edges:
print("Selecting best skeleton...")
n_graphs = 10
paf_inds = None
else:
n_graphs = 1
paf_inds = [list(range(n_edges))]
(
results,
paf_scores,
best_assemblies,
) = crossvalutils.cross_validate_paf_graphs(
config,
str(path_test_config).replace("pose_", "inference_"),
data_path,
data_path.replace("_full.", "_meta."),
n_graphs=n_graphs,
paf_inds=paf_inds,
oks_sigma=dlc_cfg.get("oks_sigma", 0.1),
margin=dlc_cfg.get("bbox_margin", 0),
symmetric_kpts=dlc_cfg.get("symmetric_kpts"),
)
if plotting == "individual":
assemblies, assemblies_unique, image_paths = best_assemblies
fig, ax = visualization.create_minimal_figure()
n_animals = len(cfg["individuals"])
if cfg["uniquebodyparts"]:
n_animals += 1
colors = visualization.get_cmap(n_animals,
name=cfg["colormap"])
for k, v in tqdm(assemblies.items()):
imname = image_paths[k]
image_path = os.path.join(cfg["project_path"],
*imname)
frame = auxfun_videos.imread(image_path,
mode="skimage")
h, w, _ = np.shape(frame)
fig.set_size_inches(w / 100, h / 100)
ax.set_xlim(0, w)
ax.set_ylim(0, h)
ax.invert_yaxis()
gt = [
s.to_numpy().reshape((-1, 2)) for _, s in
Data.loc[imname].groupby("individuals")
]
coords_pred = []
coords_pred += [ass.xy for ass in v]
probs_pred = []
probs_pred += [ass.data[:, 2:3] for ass in v]
if assemblies_unique is not None:
unique = assemblies_unique.get(k, None)
if unique is not None:
coords_pred.append(unique[:, :2])
probs_pred.append(unique[:, 2:3])
while len(coords_pred) < len(gt):
coords_pred.append(np.full((1, 2), np.nan))
probs_pred.append(np.full((1, 2), np.nan))
ax = visualization.make_multianimal_labeled_image(
frame,
gt,
coords_pred,
probs_pred,
colors,
cfg["dotsize"],
cfg["alphavalue"],
cfg["pcutoff"],
ax=ax,
)
visualization.save_labeled_frame(
fig,
image_path,
foldername,
k in trainIndices,
)
visualization.erase_artists(ax)
df = results[1].copy()
df.loc(axis=0)[("mAP_train", "mean")] = [
d[0]["mAP"] for d in results[2]
]
df.loc(axis=0)[("mAR_train", "mean")] = [
d[0]["mAR"] for d in results[2]
]
df.loc(axis=0)[("mAP_test", "mean")] = [
d[1]["mAP"] for d in results[2]
]
df.loc(axis=0)[("mAR_test", "mean")] = [
d[1]["mAR"] for d in results[2]
]
with open(data_path.replace("_full.", "_map."),
"wb") as file:
pickle.dump((df, paf_scores), file)
if len(final_result
) > 0: # Only append if results were calculated
make_results_file(final_result, evaluationfolder,
DLCscorer)
os.chdir(str(start_path))
def return_evaluate_network_data(
config,
shuffle=0,
trainingsetindex=0,
comparisonbodyparts="all",
Snapindex=None,
rescale=False,
fulldata=False,
show_errors=True,
modelprefix="",
returnjustfns=True,
):
"""
Returns the results for (previously evaluated) network. deeplabcut.evaluate_network(..)
Returns list of (per model): [trainingsiterations,trainfraction,shuffle,trainerror,testerror,pcutoff,trainerrorpcutoff,testerrorpcutoff,Snapshots[snapindex],scale,net_type]
If fulldata=True, also returns (the complete annotation and prediction array)
Returns list of: (DataMachine, Data, data, trainIndices, testIndices, trainFraction, DLCscorer,comparisonbodyparts, cfg, Snapshots[snapindex])
----------
config : string
Full path of the config.yaml file as a string.
shuffle: integer
integers specifying shuffle index of the training dataset. The default is 0.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This
variable can also be set to "all".
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
rescale: bool, default False
Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every
image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported
in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated
on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the
original size!
Examples
--------
If you do not want to plot
>>> deeplabcut._evaluate_network_data('/analysis/project/reaching-task/config.yaml', shuffle=[1])
--------
If you want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',shuffle=[1],True)
"""
import os
from deeplabcut.pose_estimation_tensorflow.config import load_config
from deeplabcut.utils import auxiliaryfunctions
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
# Data=pd.read_hdf(os.path.join(cfg["project_path"],str(trainingsetfolder),'CollectedData_' + cfg["scorer"] + '.h5'),'df_with_missing')
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
##################################################
# Load data...
##################################################
trainFraction = cfg["TrainingFraction"][trainingsetindex]
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
data, trainIndices, testIndices, trainFraction = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
########################### RESCALING (to global scale)
if rescale == True:
scale = dlc_cfg["global_scale"]
print("Rescaling Data to ", scale)
Data = (pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
) * scale)
else:
scale = 1
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
)
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
])
if len(Snapshots) == 0:
print(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
snapindices = []
else:
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if Snapindex == None:
Snapindex = cfg["snapshotindex"]
if Snapindex == -1:
snapindices = [-1]
elif Snapindex == "all":
snapindices = range(len(Snapshots))
elif Snapindex < len(Snapshots):
snapindices = [Snapindex]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
DATA = []
results = []
resultsfns = []
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train",
Snapshots[snapindex]) # setting weights to corresponding snapshot.
trainingsiterations = (dlc_cfg["init_weights"].split(
os.sep)[-1]).split("-")[
-1] # read how many training siterations that corresponds to.
# name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix)
if not returnjustfns:
print(
"Retrieving ",
DLCscorer,
" with # of trainingiterations:",
trainingsiterations,
)
(
notanalyzed,
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.CheckifNotEvaluated(str(evaluationfolder),
DLCscorer, DLCscorerlegacy,
Snapshots[snapindex])
# resultsfilename=os.path.join(str(evaluationfolder),DLCscorer + '-' + str(Snapshots[snapindex])+ '.h5') # + '-' + str(snapshot)+ ' #'-' + Snapshots[snapindex]+ '.h5')
print(resultsfilename)
resultsfns.append(resultsfilename)
if not returnjustfns:
if not notanalyzed and os.path.isfile(
resultsfilename): # data exists..
DataMachine = pd.read_hdf(resultsfilename, "df_with_missing")
DataCombined = pd.concat([Data.T, DataMachine.T], axis=0).T
RMSE, RMSEpcutoff = pairwisedistances(
DataCombined,
cfg["scorer"],
DLCscorer,
cfg["pcutoff"],
comparisonbodyparts,
)
testerror = np.nanmean(RMSE.iloc[testIndices].values.flatten())
trainerror = np.nanmean(
RMSE.iloc[trainIndices].values.flatten())
testerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[testIndices].values.flatten())
trainerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[trainIndices].values.flatten())
if show_errors == True:
print(
"Results for",
trainingsiterations,
" training iterations:",
int(100 * trainFraction),
shuffle,
"train error:",
np.round(trainerror, 2),
"pixels. Test error:",
np.round(testerror, 2),
" pixels.",
)
print(
"With pcutoff of",
cfg["pcutoff"],
" train error:",
np.round(trainerrorpcutoff, 2),
"pixels. Test error:",
np.round(testerrorpcutoff, 2),
"pixels",
)
print("Snapshot", Snapshots[snapindex])
r = [
trainingsiterations,
int(100 * trainFraction),
shuffle,
np.round(trainerror, 2),
np.round(testerror, 2),
cfg["pcutoff"],
np.round(trainerrorpcutoff, 2),
np.round(testerrorpcutoff, 2),
Snapshots[snapindex],
scale,
dlc_cfg["net_type"],
]
results.append(r)
else:
print("Model not trained/evaluated!")
if fulldata == True:
DATA.append([
DataMachine,
Data,
data,
trainIndices,
testIndices,
trainFraction,
DLCscorer,
comparisonbodyparts,
cfg,
evaluationfolder,
Snapshots[snapindex],
])
os.chdir(start_path)
if returnjustfns:
return resultsfns
else:
if fulldata == True:
return DATA, results
else:
return results
def plot_trajectories(
config,
videos,
videotype=".avi",
shuffle=1,
trainingsetindex=0,
filtered=False,
displayedbodyparts="all",
displayedindividuals="all",
showfigures=False,
destfolder=None,
modelprefix="",
track_method="",
imagetype=".png",
resolution=100,
linewidth=1.0,
):
"""
Plots the trajectories of various bodyparts across the video.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
displayedbodyparts: list of strings, optional
This select the body parts that are plotted in the video.
Either ``all``, then all body parts from config.yaml are used,
or a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
showfigures: bool, default false
If true then plots are also displayed.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
imagetype: string, default ".png"
Specifies the output image format, tested '.tif', '.jpg', '.svg' and ".png".
resolution: int, default 100
Specifies the resolution (in dpi) of saved figures. Note higher resolution figures take longer to generate.
linewidth: float, default 1.0
Specifies width of line for line and histogram plots.
Example
--------
for labeling the frames
>>> deeplabcut.plot_trajectories('home/alex/analysis/project/reaching-task/config.yaml',['/home/alex/analysis/project/videos/reachingvideo1.avi'])
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, modelprefix=modelprefix
) # automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
individuals = auxfun_multianimal.IntersectionofIndividualsandOnesGivenbyUser(
cfg, displayedindividuals)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
if not len(Videos):
print(
"No videos found. Make sure you passed a list of videos and that *videotype* is right."
)
return
failed = []
for video in Videos:
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
vname = str(Path(video).stem)
print("Loading ", video, "and data.")
try:
df, _, _, suffix = auxiliaryfunctions.load_analyzed_data(
videofolder, vname, DLCscorer, filtered, track_method)
failed.append(False)
tmpfolder = os.path.join(videofolder, "plot-poses", vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder, recursive=True)
# Keep only the individuals and bodyparts that were labeled
labeled_bpts = [
bp for bp in df.columns.get_level_values("bodyparts").unique()
if bp in bodyparts
]
for animal in individuals:
PlottingResults(
tmpfolder,
df,
cfg,
labeled_bpts,
animal,
showfigures,
suffix + animal + imagetype,
resolution=resolution,
linewidth=linewidth,
)
except FileNotFoundError as e:
failed.append(True)
print(e)
try:
_ = auxiliaryfunctions.load_detection_data(
video, DLCscorer, track_method)
print(
'Call "deeplabcut.refine_training_dataset.convert_raw_tracks_to_h5()"'
" prior to plotting the trajectories.")
except FileNotFoundError as e:
print(e)
print(
f"Make sure {video} was previously analyzed, and that "
f'detections were successively converted to tracklets using "deeplabcut.convert_detections2tracklets()" '
f'and "deeplabcut.convert_raw_tracks_to_h5()".')
if not all(failed):
print(
'Plots created! Please check the directory "plot-poses" within the video directory'
)
else:
print(
f"Plots could not be created! "
f"Videos were not evaluated with the current scorer {DLCscorer}.")
def evaluate_network(config,
Shuffles=[1],
plotting=None,
show_errors=True,
comparisonbodyparts="all",
gputouse=None):
"""
Evaluates the network based on the saved models at different stages of the training network.\n
The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'.
Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
Shuffles: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
plotting: bool, optional
Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False``
show_errors: bool, optional
Display train and test errors. The default is `True``
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
Examples
--------
If you do not want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml', shuffle=[1])
--------
If you want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',shuffle=[1],True)
"""
import os
from skimage import io
import skimage.color
from deeplabcut.pose_estimation_tensorflow.nnet import predict as ptf_predict
from deeplabcut.pose_estimation_tensorflow.config import load_config
from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input
from deeplabcut.utils import auxiliaryfunctions, visualization
import tensorflow as tf
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ[
'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
vers = (tf.__version__).split('.')
if int(vers[0]) == 1 and int(vers[1]) > 12:
TF = tf.compat.v1
else:
TF = tf
TF.reset_default_graph()
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' #
# tf.logging.set_verbosity(tf.logging.WARN)
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: #gpu selectinon
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(cfg["project_path"], str(trainingsetfolder),
'CollectedData_' + cfg["scorer"] + '.h5'),
'df_with_missing')
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in cfg["TrainingFraction"]:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(trainFraction, shuffle,
cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
# Load meta data
data, trainIndices, testIndices, trainFraction = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
#change batch size, if it was edited during analysis!
dlc_cfg['batch_size'] = 1 #in case this was edited for analysis.
#Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg)))
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
#path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split('.')[0]
for fn in os.listdir(os.path.join(str(modelfolder), 'train'))
if "index" in fn
])
try: #check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
increasing_indices = np.argsort(
[int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
final_result = []
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg['init_weights'] = os.path.join(
str(modelfolder), 'train', Snapshots[snapindex]
) #setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg['init_weights'].split(os.sep)[-1]
).split(
'-'
)[-1] #read how many training siterations that corresponds to.
#name for deeplabcut net (based on its parameters)
DLCscorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, trainingsiterations)
print("Running ", DLCscorer, " with # of trainingiterations:",
trainingsiterations)
resultsfilename = os.path.join(
str(evaluationfolder),
DLCscorer + '-' + Snapshots[snapindex] + '.h5')
try:
DataMachine = pd.read_hdf(resultsfilename,
'df_with_missing')
print("This net has already been evaluated!")
except FileNotFoundError:
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = ptf_predict.setup_pose_prediction(
dlc_cfg)
Numimages = len(Data.index)
PredicteData = np.zeros(
(Numimages, 3 * len(dlc_cfg['all_joints_names'])))
print("Analyzing data...")
for imageindex, imagename in tqdm(enumerate(Data.index)):
image = io.imread(os.path.join(cfg['project_path'],
imagename),
mode='RGB')
image = skimage.color.gray2rgb(image)
image_batch = data_to_input(image)
# Compute prediction with the CNN
outputs_np = sess.run(outputs,
feed_dict={inputs: image_batch})
scmap, locref = ptf_predict.extract_cnn_output(
outputs_np, dlc_cfg)
# Extract maximum scoring location from the heatmap, assume 1 person
pose = ptf_predict.argmax_pose_predict(
scmap, locref, dlc_cfg.stride)
PredicteData[imageindex, :] = pose.flatten(
) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts!
sess.close() #closes the current tf session
index = pd.MultiIndex.from_product(
[[DLCscorer], dlc_cfg['all_joints_names'],
['x', 'y', 'likelihood']],
names=['scorer', 'bodyparts', 'coords'])
# Saving results
DataMachine = pd.DataFrame(PredicteData,
columns=index,
index=Data.index.values)
DataMachine.to_hdf(resultsfilename,
'df_with_missing',
format='table',
mode='w')
print("Done and results stored for snapshot: ",
Snapshots[snapindex])
DataCombined = pd.concat([Data.T, DataMachine.T], axis=0).T
RMSE, RMSEpcutoff = pairwisedistances(
DataCombined, cfg["scorer"], DLCscorer, cfg["pcutoff"],
comparisonbodyparts)
testerror = np.nanmean(
RMSE.iloc[testIndices].values.flatten())
trainerror = np.nanmean(
RMSE.iloc[trainIndices].values.flatten())
testerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[testIndices].values.flatten())
trainerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[trainIndices].values.flatten())
results = [
trainingsiterations,
int(100 * trainFraction), shuffle,
np.round(trainerror, 2),
np.round(testerror, 2), cfg["pcutoff"],
np.round(trainerrorpcutoff, 2),
np.round(testerrorpcutoff, 2)
]
final_result.append(results)
if show_errors == True:
print("Results for",
trainingsiterations, " training iterations:",
int(100 * trainFraction), shuffle,
"train error:",
np.round(trainerror, 2), "pixels. Test error:",
np.round(testerror, 2), " pixels.")
print("With pcutoff of",
cfg["pcutoff"], " train error:",
np.round(trainerrorpcutoff,
2), "pixels. Test error:",
np.round(testerrorpcutoff, 2), "pixels")
print(
"Thereby, the errors are given by the average distances between the labels by DLC and the scorer."
)
if plotting == True:
print("Plotting...")
colors = visualization.get_cmap(
len(comparisonbodyparts), name=cfg['colormap'])
foldername = os.path.join(
str(evaluationfolder), 'LabeledImages_' +
DLCscorer + '_' + Snapshots[snapindex])
auxiliaryfunctions.attempttomakefolder(foldername)
NumFrames = np.size(DataCombined.index)
for ind in np.arange(NumFrames):
visualization.PlottingandSaveLabeledFrame(
DataCombined, ind, trainIndices, cfg, colors,
comparisonbodyparts, DLCscorer, foldername)
TF.reset_default_graph()
#print(final_result)
make_results_file(final_result, evaluationfolder, DLCscorer)
print(
"The network is evaluated and the results are stored in the subdirectory 'evaluation_results'."
)
print(
"If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos."
)
print(
"Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)"
)
#returning to intial folder
os.chdir(str(start_path))
def plot_trajectories(config,
videos,
videotype='.avi',
shuffle=1,
trainingsetindex=0,
filtered=False,
showfigures=False,
displayedbodyparts='all',
destfolder=None):
"""
Plots the trajectories of various bodyparts across the video.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
showfigures: bool, default false
If true then plots are also displayed.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
Example
--------
for labeling the frames
>>> deeplabcut.plot_trajectories('home/alex/analysis/project/reaching-task/config.yaml',['/home/alex/analysis/project/videos/reachingvideo1.avi'])
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
DLCscorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction
) #automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
print(video)
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
videotype = str(Path(video).suffix)
print("Starting % ", videofolder, videos)
basefolder = videofolder
auxiliaryfunctions.attempttomakefolder(basefolder)
RunTrajectoryAnalysis(video, basefolder, DLCscorer, videofolder, cfg,
showfigures, bodyparts, filtered)
print(
'Plots created! Please check the directory "plot-poses" within the video directory'
)
def create_labeled_video(
config,
videos,
videotype="avi",
shuffle=1,
trainingsetindex=0,
filtered=False,
fastmode=True,
save_frames=False,
Frames2plot=None,
displayedbodyparts="all",
displayedindividuals="all",
codec="mp4v",
outputframerate=None,
destfolder=None,
draw_skeleton=False,
trailpoints=0,
displaycropped=False,
color_by="bodypart",
modelprefix="",
track_method="",
):
"""
Labels the bodyparts in a video. Make sure the video is already analyzed by the function 'analyze_video'
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
Number of shuffles of training dataset. Default is set to 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
fastmode: bool
If true uses openCV (much faster but less customization of video) vs matplotlib (if false). You can also
"save_frames" individually or not in the matplotlib mode (if you set the "save_frames" variable accordingly).
However, using matplotlib to create the frames it therefore allows much more flexible (one can set transparency of markers, crop, and easily customize).
save_frames: bool
If true creates each frame individual and then combines into a video. This variant is relatively slow as
it stores all individual frames.
Frames2plot: List of indices
If not None & save_frames=True then the frames corresponding to the index will be plotted. For example, Frames2plot=[0,11] will plot the first and the 12th frame.
displayedbodyparts: list of strings, optional
This selects the body parts that are plotted in the video. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
displayedindividuals: list of strings, optional
Individuals plotted in the video. By default, all individuals present in the config will be showed.
codec: codec for labeled video. Options see http://www.fourcc.org/codecs.php [depends on your ffmpeg installation.]
outputframerate: positive number, output frame rate for labeled video (only available for the mode with saving frames.) By default: None, which results in the original video rate.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
draw_skeleton: bool
If ``True`` adds a line connecting the body parts making a skeleton on on each frame. The body parts to be connected and the color of these connecting lines are specified in the config file. By default: ``False``
trailpoints: int
Number of revious frames whose body parts are plotted in a frame (for displaying history). Default is set to 0.
displaycropped: bool, optional
Specifies whether only cropped frame is displayed (with labels analyzed therein), or the original frame with the labels analyzed in the cropped subset.
color_by : string, optional (default='bodypart')
Coloring rule. By default, each bodypart is colored differently.
If set to 'individual', points belonging to a single individual are colored the same.
Examples
--------
If you want to create the labeled video for only 1 video
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to create the labeled video for only 1 video and store the individual frames
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'],fastmode=True, save_frames=True)
--------
If you want to create the labeled video for multiple videos
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to create the labeled video for all the videos (as .avi extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'])
--------
If you want to create the labeled video for all the videos (as .mp4 extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'],videotype='mp4')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, modelprefix=modelprefix
) # automatically loads corresponding model (even training iteration based on snapshot index)
if save_frames:
fastmode = False # otherwise one cannot save frames
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts
)
individuals = auxfun_multianimal.IntersectionofIndividualsandOnesGivenbyUser(
cfg, displayedindividuals
)
if draw_skeleton:
bodyparts2connect = cfg["skeleton"]
skeleton_color = cfg["skeleton_color"]
else:
bodyparts2connect = None
skeleton_color = None
start_path = os.getcwd()
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
if not Videos:
print("No video(s) were found. Please check your paths and/or 'video_type'.")
return
func = partial(
proc_video,
videos,
destfolder,
filtered,
DLCscorer,
DLCscorerlegacy,
track_method,
cfg,
individuals,
color_by,
bodyparts,
codec,
bodyparts2connect,
trailpoints,
save_frames,
outputframerate,
Frames2plot,
draw_skeleton,
skeleton_color,
displaycropped,
fastmode,
)
with Pool(min(os.cpu_count(), len(Videos))) as pool:
pool.map(func, Videos)
os.chdir(start_path)
def plot_trajectories(
config,
videos,
videotype="",
shuffle=1,
trainingsetindex=0,
filtered=False,
displayedbodyparts="all",
displayedindividuals="all",
showfigures=False,
destfolder=None,
modelprefix="",
imagetype=".png",
resolution=100,
linewidth=1.0,
track_method="",
):
"""Plots the trajectories of various bodyparts across the video.
Parameters
----------
config: str
Full path of the config.yaml file.
videos: list[str]
Full paths to videos for analysis or a path to the directory, where all the
videos with same extension are stored.
videotype: str, optional, default=""
Checks for the extension of the video in case the input to the video is a
directory. Only videos with this extension are analyzed.
If left unspecified, videos with common extensions
('avi', 'mp4', 'mov', 'mpeg', 'mkv') are kept.
shuffle: int, optional, default=1
Integer specifying the shuffle index of the training dataset.
trainingsetindex: int, optional, default=0
Integer specifying which TrainingsetFraction to use.
Note that TrainingFraction is a list in config.yaml.
filtered: bool, optional, default=False
Boolean variable indicating if filtered output should be plotted rather than
frame-by-frame predictions. Filtered version can be calculated with
``deeplabcut.filterpredictions``.
displayedbodyparts: list[str] or str, optional, default="all"
This select the body parts that are plotted in the video.
Either ``all``, then all body parts from config.yaml are used,
or a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml
to select only these two body parts.
showfigures: bool, optional, default=False
If ``True`` then plots are also displayed.
destfolder: string or None, optional, default=None
Specifies the destination folder that was used for storing analysis data. If
``None``, the path of the video is used.
modelprefix: str, optional, default=""
Directory containing the deeplabcut models to use when evaluating the network.
By default, the models are assumed to exist in the project folder.
imagetype: string, optional, default=".png"
Specifies the output image format - '.tif', '.jpg', '.svg' and ".png".
resolution: int, optional, default=100
Specifies the resolution (in dpi) of saved figures.
Note higher resolution figures take longer to generate.
linewidth: float, optional, default=1.0
Specifies width of line for line and histogram plots.
track_method: string, optional, default=""
Specifies the tracker used to generate the data.
Empty by default (corresponding to a single animal project).
For multiple animals, must be either 'box', 'skeleton', or 'ellipse' and will
be taken from the config.yaml file if none is given.
Returns
-------
None
Examples
--------
To label the frames
>>> deeplabcut.plot_trajectories(
'home/alex/analysis/project/reaching-task/config.yaml',
['/home/alex/analysis/project/videos/reachingvideo1.avi'],
)
"""
cfg = auxiliaryfunctions.read_config(config)
track_method = auxfun_multianimal.get_track_method(cfg, track_method=track_method)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, modelprefix=modelprefix
) # automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts
)
individuals = auxfun_multianimal.IntersectionofIndividualsandOnesGivenbyUser(
cfg, displayedindividuals
)
Videos = auxiliaryfunctions.get_list_of_videos(videos, videotype)
if not len(Videos):
print(
"No videos found. Make sure you passed a list of videos and that *videotype* is right."
)
return
failed = []
for video in Videos:
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
vname = str(Path(video).stem)
print("Loading ", video, "and data.")
try:
df, _, _, suffix = auxiliaryfunctions.load_analyzed_data(
videofolder, vname, DLCscorer, filtered, track_method
)
failed.append(False)
tmpfolder = os.path.join(videofolder, "plot-poses", vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder, recursive=True)
# Keep only the individuals and bodyparts that were labeled
labeled_bpts = [
bp
for bp in df.columns.get_level_values("bodyparts").unique()
if bp in bodyparts
]
# Either display the animals defined in the config if they are found
# in the dataframe, or all the trajectories regardless of their names
try:
animals = set(df.columns.get_level_values("individuals"))
except KeyError:
animals = {""}
for animal in animals.intersection(individuals) or animals:
PlottingResults(
tmpfolder,
df,
cfg,
labeled_bpts,
animal,
showfigures,
suffix + animal + imagetype,
resolution=resolution,
linewidth=linewidth,
)
except FileNotFoundError as e:
failed.append(True)
print(e)
try:
_ = auxiliaryfunctions.load_detection_data(
video, DLCscorer, track_method
)
print(
'Call "deeplabcut.stitch_tracklets()"'
" prior to plotting the trajectories."
)
except FileNotFoundError as e:
print(e)
print(
f"Make sure {video} was previously analyzed, and that "
f'detections were successively converted to tracklets using "deeplabcut.convert_detections2tracklets()" '
f'and "deeplabcut.stitch_tracklets()".'
)
if not all(failed):
print(
'Plots created! Please check the directory "plot-poses" within the video directory'
)
else:
print(
f"Plots could not be created! "
f"Videos were not evaluated with the current scorer {DLCscorer}."
)
