def MakeLabeledPlots(folder, DataCombined, cfg, Labels, Colorscheme, cc, scale):
tmpfolder = str(folder) + "_labeled"
auxiliaryfunctions.attempttomakefolder(tmpfolder)
for index, imagename in enumerate(DataCombined.index.values):
image = io.imread(os.path.join(cfg["project_path"], imagename))
plt.axis("off")
if np.ndim(image) == 2:
h, w = np.shape(image)
else:
h, w, nc = np.shape(image)
plt.figure(
frameon=False, figsize=(w * 1.0 / 100 * scale, h * 1.0 / 100 * scale)
)
plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0)
plt.imshow(image, "gray")
if index == 0:
print("They are stored in the following folder: %s." % tmpfolder) # folder)
for c, bp in enumerate(cfg["bodyparts"]):
plt.plot(
DataCombined[cfg["scorer"]][bp]["x"].values[index],
DataCombined[cfg["scorer"]][bp]["y"].values[index],
Labels[cc],
color=Colorscheme(c),
alpha=cfg["alphavalue"],
ms=cfg["dotsize"],
)
plt.xlim(0, w)
plt.ylim(0, h)
plt.axis("off")
plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0)
plt.gca().invert_yaxis()
# plt.savefig(str(Path(tmpfolder)/imagename.split(os.sep)[-1]))
plt.savefig(
os.path.join(tmpfolder, str(Path(imagename).name))
) # create file name also on Windows for Unix projects (and vice versa)
plt.close("all")
def make_labeled_images_from_dataframe(
df,
cfg,
destfolder="",
scale=1.0,
dpi=100,
keypoint="+",
draw_skeleton=True,
color_by="bodypart",
):
"""
Write labeled frames to disk from a DataFrame.
Parameters
----------
df : pd.DataFrame
DataFrame containing the labeled data. Typically, the DataFrame is obtained
through pandas.read_csv() or pandas.read_hdf().
cfg : dict
Project configuration.
destfolder : string, optional
Destination folder into which images will be stored. By default, same location as the labeled data.
Note that the folder will be created if it does not exist.
scale : float, optional
Up/downscale the output dimensions.
By default, outputs are of the same dimensions as the original images.
dpi : int, optional
Output resolution. 100 dpi by default.
keypoint : str, optional
Keypoint appearance. By default, keypoints are marked by a + sign.
Refer to https://matplotlib.org/3.2.1/api/markers_api.html for a list of all possible options.
draw_skeleton : bool, optional
Whether to draw the animal skeleton as defined in *cfg*. True by default.
color_by : str, optional
Color scheme of the keypoints. Must be either 'bodypart' or 'individual'.
By default, keypoints are colored relative to the bodypart they represent.
"""
bodyparts = df.columns.get_level_values("bodyparts")
bodypart_names = bodyparts.unique()
nbodyparts = len(bodypart_names)
bodyparts = bodyparts[::2]
if color_by == "bodypart":
map_ = bodyparts.map(dict(zip(bodypart_names, range(nbodyparts))))
cmap = get_cmap(nbodyparts, cfg["colormap"])
colors = cmap(map_)
elif color_by == "individual":
try:
individuals = df.columns.get_level_values("individuals")
individual_names = individuals.unique().to_list()
nindividuals = len(individual_names)
individuals = individuals[::2]
map_ = individuals.map(
dict(zip(individual_names, range(nindividuals))))
cmap = get_cmap(nindividuals, cfg["colormap"])
colors = cmap(map_)
except KeyError as e:
raise Exception(
"Coloring by individuals is only valid for multi-animal data"
) from e
else:
raise ValueError(
"`color_by` must be either `bodypart` or `individual`.")
bones = []
if draw_skeleton:
for bp1, bp2 in cfg["skeleton"]:
match1, match2 = [], []
for j, bp in enumerate(bodyparts):
if bp == bp1:
match1.append(j)
elif bp == bp2:
match2.append(j)
bones.extend(zip(match1, match2))
ind_bones = tuple(zip(*bones))
sep = "/" if "/" in df.index[0] else "\\"
images = cfg["project_path"] + sep + df.index
if sep != os.path.sep:
images = images.str.replace(sep, os.path.sep)
if not destfolder:
destfolder = os.path.dirname(images[0])
tmpfolder = destfolder + "_labeled"
attempttomakefolder(tmpfolder)
ic = io.imread_collection(images.to_list())
h, w = ic[0].shape[:2]
fig, ax = prepare_figure_axes(w, h, scale, dpi)
im = ax.imshow(np.zeros((h, w)), "gray")
scat = ax.scatter([], [],
s=cfg["dotsize"],
alpha=cfg["alphavalue"],
marker=keypoint)
scat.set_color(colors)
xy = df.values.reshape((df.shape[0], -1, 2))
segs = xy[:, ind_bones].swapaxes(1, 2)
coll = LineCollection([], colors=cfg["skeleton_color"])
ax.add_collection(coll)
for i in trange(len(ic)):
coords = xy[i]
im.set_array(ic[i])
scat.set_offsets(coords)
if ind_bones:
coll.set_segments(segs[i])
imagename = os.path.basename(ic.files[i])
fig.savefig(
os.path.join(tmpfolder,
imagename.replace(".png", f"_{color_by}.png")))
plt.close(fig)
def plot_trajectories(config,
videos,
videotype='.avi',
shuffle=1,
trainingsetindex=0,
filtered=False,
displayedbodyparts='all',
showfigures=False,
destfolder=None):
"""
Plots the trajectories of various bodyparts across the video.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcutcore.filterpredictions
displayedbodyparts: list of strings, optional
This select the body parts that are plotted in the video.
Either ``all``, then all body parts from config.yaml are used,
or a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
showfigures: bool, default false
If true then plots are also displayed.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
Example
--------
for labeling the frames
>>> deeplabcutcore.plot_trajectories('home/alex/analysis/project/reaching-task/config.yaml',['/home/alex/analysis/project/videos/reachingvideo1.avi'])
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction
) #automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
print(video)
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
vname = str(Path(video).stem)
print("Starting % ", videofolder, video)
notanalyzed, dataname, DLCscorer = auxiliaryfunctions.CheckifNotAnalyzed(
videofolder, vname, DLCscorer, DLCscorerlegacy, flag='checking')
if notanalyzed:
print("The video was not analyzed with this scorer:", DLCscorer)
else:
#LoadData
print("Loading ", video, "and data.")
datafound, metadata, Dataframe, DLCscorer, suffix = auxiliaryfunctions.LoadAnalyzedData(
str(videofolder), vname, DLCscorer, filtered
) #returns boolean variable if data was found and metadata + pandas array
if datafound:
basefolder = videofolder
auxiliaryfunctions.attempttomakefolder(basefolder)
auxiliaryfunctions.attempttomakefolder(
os.path.join(basefolder, 'plot-poses'))
tmpfolder = os.path.join(basefolder, 'plot-poses', vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder)
PlottingResults(tmpfolder, Dataframe, DLCscorer, cfg,
bodyparts, showfigures, suffix + '.png')
print(
'Plots created! Please check the directory "plot-poses" within the video directory'
)
def evaluate_network(config,
Shuffles=[1],
trainingsetindex=0,
plotting=None,
show_errors=True,
comparisonbodyparts="all",
gputouse=None,
rescale=False):
"""
Evaluates the network based on the saved models at different stages of the training network.\n
The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'.
Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
Shuffles: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This
variable can also be set to "all".
plotting: bool, optional
Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False``
show_errors: bool, optional
Display train and test errors. The default is `True``
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
rescale: bool, default False
Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every
image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported
in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated
on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the
original size!
Examples
--------
If you do not want to plot
>>> deeplabcutcore.evaluate_network('/analysis/project/reaching-task/config.yaml', Shuffles=[1])
--------
If you want to plot
>>> deeplabcutcore.evaluate_network('/analysis/project/reaching-task/config.yaml',Shuffles=[1],True)
"""
import os
#import skimage.color
#from skimage.io import imread
from deeplabcutcore.utils.auxfun_videos import imread, imresize
from deeplabcutcore.pose_estimation_tensorflow.nnet import predict
from deeplabcutcore.pose_estimation_tensorflow.config import load_config
from deeplabcutcore.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input
from deeplabcutcore.utils import auxiliaryfunctions
import tensorflow as tf
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ[
'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
tf.compat.v1.reset_default_graph()
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' #
# tf.logging.set_verbosity(tf.logging.WARN)
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: #gpu selectinon
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
if trainingsetindex == 'all':
TrainingFractions = cfg["TrainingFraction"]
else:
if trainingsetindex < len(
cfg["TrainingFraction"]) and trainingsetindex >= 0:
TrainingFractions = [
cfg["TrainingFraction"][int(trainingsetindex)]
]
else:
raise Exception('Please check the trainingsetindex! ',
trainingsetindex,
' should be an integer from 0 .. ',
int(len(cfg["TrainingFraction"]) - 1))
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(cfg["project_path"], str(trainingsetfolder),
'CollectedData_' + cfg["scorer"] + '.h5'),
'df_with_missing')
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(trainFraction, shuffle,
cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
# Load meta data
data, trainIndices, testIndices, trainFraction = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
#change batch size, if it was edited during analysis!
dlc_cfg['batch_size'] = 1 #in case this was edited for analysis.
#Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg)))
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
#path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split('.')[0]
for fn in os.listdir(os.path.join(str(modelfolder), 'train'))
if "index" in fn
])
try: #check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
increasing_indices = np.argsort(
[int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
raise ValueError(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
final_result = []
########################### RESCALING (to global scale)
if rescale == True:
scale = dlc_cfg['global_scale']
Data = pd.read_hdf(
os.path.join(cfg["project_path"], str(trainingsetfolder),
'CollectedData_' + cfg["scorer"] + '.h5'),
'df_with_missing') * scale
else:
scale = 1
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg['init_weights'] = os.path.join(
str(modelfolder), 'train', Snapshots[snapindex]
) #setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg['init_weights'].split(os.sep)[-1]
).split(
'-'
)[-1] #read how many training siterations that corresponds to.
# Name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, trainingsiterations)
print("Running ", DLCscorer, " with # of trainingiterations:",
trainingsiterations)
notanalyzed, resultsfilename, DLCscorer = auxiliaryfunctions.CheckifNotEvaluated(
str(evaluationfolder), DLCscorer, DLCscorerlegacy,
Snapshots[snapindex])
if notanalyzed:
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = predict.setup_pose_prediction(
dlc_cfg)
Numimages = len(Data.index)
PredicteData = np.zeros(
(Numimages, 3 * len(dlc_cfg['all_joints_names'])))
print("Analyzing data...")
for imageindex, imagename in tqdm(enumerate(Data.index)):
image = imread(os.path.join(cfg['project_path'],
imagename),
mode='RGB')
if scale != 1:
image = imresize(image, scale)
#image = skimage.color.gray2rgb(image)
image_batch = data_to_input(image)
# Compute prediction with the CNN
outputs_np = sess.run(outputs,
feed_dict={inputs: image_batch})
scmap, locref = predict.extract_cnn_output(
outputs_np, dlc_cfg)
# Extract maximum scoring location from the heatmap, assume 1 person
pose = predict.argmax_pose_predict(
scmap, locref, dlc_cfg.stride)
PredicteData[imageindex, :] = pose.flatten(
) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts!
sess.close() #closes the current tf session
index = pd.MultiIndex.from_product(
[[DLCscorer], dlc_cfg['all_joints_names'],
['x', 'y', 'likelihood']],
names=['scorer', 'bodyparts', 'coords'])
# Saving results
DataMachine = pd.DataFrame(PredicteData,
columns=index,
index=Data.index.values)
DataMachine.to_hdf(resultsfilename,
'df_with_missing',
format='table',
mode='w')
print("Done and results stored for snapshot: ",
Snapshots[snapindex])
DataCombined = pd.concat([Data.T, DataMachine.T],
axis=0,
sort=False).T
RMSE, RMSEpcutoff = pairwisedistances(
DataCombined, cfg["scorer"], DLCscorer, cfg["pcutoff"],
comparisonbodyparts)
testerror = np.nanmean(
RMSE.iloc[testIndices].values.flatten())
trainerror = np.nanmean(
RMSE.iloc[trainIndices].values.flatten())
testerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[testIndices].values.flatten())
trainerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[trainIndices].values.flatten())
results = [
trainingsiterations,
int(100 * trainFraction), shuffle,
np.round(trainerror, 2),
np.round(testerror, 2), cfg["pcutoff"],
np.round(trainerrorpcutoff, 2),
np.round(testerrorpcutoff, 2)
]
final_result.append(results)
if show_errors == True:
print("Results for",
trainingsiterations, " training iterations:",
int(100 * trainFraction), shuffle,
"train error:",
np.round(trainerror, 2), "pixels. Test error:",
np.round(testerror, 2), " pixels.")
print("With pcutoff of",
cfg["pcutoff"], " train error:",
np.round(trainerrorpcutoff,
2), "pixels. Test error:",
np.round(testerrorpcutoff, 2), "pixels")
if scale != 1:
print(
"The predictions have been calculated for rescaled images (and rescaled ground truth). Scale:",
scale)
print(
"Thereby, the errors are given by the average distances between the labels by DLC and the scorer."
)
if plotting == True:
print("Plotting...")
foldername = os.path.join(
str(evaluationfolder), 'LabeledImages_' +
DLCscorer + '_' + Snapshots[snapindex])
auxiliaryfunctions.attempttomakefolder(foldername)
Plotting(
cfg, comparisonbodyparts, DLCscorer, trainIndices,
DataCombined * 1. / scale, foldername
) #Rescaling coordinates to have figure in original size!
tf.compat.v1.reset_default_graph()
#print(final_result)
else:
DataMachine = pd.read_hdf(resultsfilename,
'df_with_missing')
if plotting == True:
DataCombined = pd.concat([Data.T, DataMachine.T],
axis=0,
sort=False).T
print(
"Plotting...(attention scale might be inconsistent in comparison to when data was analyzed; i.e. if you used rescale)"
)
foldername = os.path.join(
str(evaluationfolder), 'LabeledImages_' +
DLCscorer + '_' + Snapshots[snapindex])
auxiliaryfunctions.attempttomakefolder(foldername)
Plotting(cfg, comparisonbodyparts, DLCscorer,
trainIndices, DataCombined * 1. / scale,
foldername)
if len(final_result) > 0: #Only append if results were calculated
make_results_file(final_result, evaluationfolder, DLCscorer)
print(
"The network is evaluated and the results are stored in the subdirectory 'evaluation_results'."
)
print(
"If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos."
)
print(
"Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)"
)
#returning to intial folder
os.chdir(str(start_path))
def ExtractFramesbasedonPreselection(
Index,
extractionalgorithm,
Dataframe,
dataname,
scorer,
video,
cfg,
config,
opencv=True,
cluster_resizewidth=30,
cluster_color=False,
savelabeled=True,
):
from deeplabcutcore.create_project import add
start = cfg["start"]
stop = cfg["stop"]
numframes2extract = cfg["numframes2pick"]
bodyparts = cfg["bodyparts"]
videofolder = str(Path(video).parents[0])
vname = str(Path(video).stem)
tmpfolder = os.path.join(cfg["project_path"], "labeled-data", vname)
if os.path.isdir(tmpfolder):
print("Frames from video", vname, " already extracted (more will be added)!")
else:
auxiliaryfunctions.attempttomakefolder(tmpfolder)
nframes = np.size(Dataframe.index)
print("Loading video...")
if opencv:
import cv2
cap = cv2.VideoCapture(video)
fps = cap.get(5)
duration = nframes * 1.0 / fps
size = (int(cap.get(4)), int(cap.get(3)))
else:
from moviepy.editor import VideoFileClip
clip = VideoFileClip(video)
fps = clip.fps
duration = clip.duration
size = clip.size
if cfg["cropping"]: # one might want to adjust
coords = (cfg["x1"], cfg["x2"], cfg["y1"], cfg["y2"])
else:
coords = None
print("Duration of video [s]: ", duration, ", recorded @ ", fps, "fps!")
print("Overall # of frames: ", nframes, "with (cropped) frame dimensions: ")
if extractionalgorithm == "uniform":
if opencv:
frames2pick = frameselectiontools.UniformFramescv2(
cap, numframes2extract, start, stop, Index
)
else:
frames2pick = frameselectiontools.UniformFrames(
clip, numframes2extract, start, stop, Index
)
elif extractionalgorithm == "kmeans":
if opencv:
frames2pick = frameselectiontools.KmeansbasedFrameselectioncv2(
cap,
numframes2extract,
start,
stop,
cfg["cropping"],
coords,
Index,
resizewidth=cluster_resizewidth,
color=cluster_color,
)
else:
if cfg["cropping"]:
clip = clip.crop(y1=cfg["y1"], y2=cfg["x2"], x1=cfg["x1"], x2=cfg["x2"])
frames2pick = frameselectiontools.KmeansbasedFrameselection(
clip,
numframes2extract,
start,
stop,
Index,
resizewidth=cluster_resizewidth,
color=cluster_color,
)
else:
print(
"Please implement this method yourself! Currently the options are 'kmeans', 'jump', 'uniform'."
)
frames2pick = []
# Extract frames + frames with plotted labels and store them in folder (with name derived from video name) nder labeled-data
print("Let's select frames indices:", frames2pick)
colors = visualization.get_cmap(len(bodyparts), cfg["colormap"])
strwidth = int(np.ceil(np.log10(nframes))) # width for strings
for index in frames2pick: ##tqdm(range(0,nframes,10)):
if opencv:
PlottingSingleFramecv2(
cap,
cv2,
cfg["cropping"],
coords,
Dataframe,
bodyparts,
tmpfolder,
index,
scorer,
cfg["dotsize"],
cfg["pcutoff"],
cfg["alphavalue"],
colors,
strwidth,
savelabeled,
)
else:
PlottingSingleFrame(
clip,
Dataframe,
bodyparts,
tmpfolder,
index,
scorer,
cfg["dotsize"],
cfg["pcutoff"],
cfg["alphavalue"],
colors,
strwidth,
savelabeled,
)
plt.close("all")
# close videos
if opencv:
cap.release()
else:
clip.close()
del clip
# Extract annotations based on DeepLabCut and store in the folder (with name derived from video name) under labeled-data
if len(frames2pick) > 0:
# Dataframe = pd.read_hdf(os.path.join(videofolder,dataname+'.h5'))
DF = Dataframe.iloc[frames2pick]
DF.index = [
os.path.join(
"labeled-data", vname, "img" + str(index).zfill(strwidth) + ".png"
)
for index in DF.index
] # exchange index number by file names.
machinefile = os.path.join(
tmpfolder, "machinelabels-iter" + str(cfg["iteration"]) + ".h5"
)
if Path(machinefile).is_file():
Data = pd.read_hdf(machinefile, "df_with_missing")
DataCombined = pd.concat([Data, DF])
# drop duplicate labels:
DataCombined = DataCombined[~DataCombined.index.duplicated(keep="first")]
DataCombined.to_hdf(machinefile, key="df_with_missing", mode="w")
DataCombined.to_csv(
os.path.join(tmpfolder, "machinelabels.csv")
) # this is always the most current one (as reading is from h5)
else:
DF.to_hdf(machinefile, key="df_with_missing", mode="w")
DF.to_csv(os.path.join(tmpfolder, "machinelabels.csv"))
try:
if cfg["cropping"]:
add.add_new_videos(
config, [video], coords=[coords]
) # make sure you pass coords as a list
else:
add.add_new_videos(config, [video], coords=None)
except: # can we make a catch here? - in fact we should drop indices from DataCombined if they are in CollectedData.. [ideal behavior; currently this is pretty unlikely]
print(
"AUTOMATIC ADDING OF VIDEO TO CONFIG FILE FAILED! You need to do this manually for including it in the config.yaml file!"
)
print("Videopath:", video, "Coordinates for cropping:", coords)
pass
print(
"The outlier frames are extracted. They are stored in the subdirectory labeled-data\%s."
% vname
)
print(
"Once you extracted frames for all videos, use 'refine_labels' to manually correct the labels."
)
else:
print("No frames were extracted.")
def create_labeled_video(
config,
videos,
videotype="avi",
shuffle=1,
trainingsetindex=0,
filtered=False,
save_frames=False,
Frames2plot=None,
delete=False,
displayedbodyparts="all",
codec="mp4v",
outputframerate=None,
destfolder=None,
draw_skeleton=False,
trailpoints=0,
displaycropped=False,
):
"""
Labels the bodyparts in a video. Make sure the video is already analyzed by the function 'analyze_video'
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
Number of shuffles of training dataset. Default is set to 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcutcore.filterpredictions
videotype: string, optional
Checks for the extension of the video in case the input is a directory.\nOnly videos with this extension are analyzed. The default is ``.avi``
save_frames: bool
If true creates each frame individual and then combines into a video. This variant is relatively slow as
it stores all individual frames. However, it uses matplotlib to create the frames and is therefore much more flexible (one can set transparency of markers, crop, and easily customize).
Frames2plot: List of indices
If not None & save_frames=True then the frames corresponding to the index will be plotted. For example, Frames2plot=[0,11] will plot the first and the 12th frame.
delete: bool
If true then the individual frames created during the video generation will be deleted.
displayedbodyparts: list of strings, optional
This select the body parts that are plotted in the video. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
codec: codec for labeled video. Options see http://www.fourcc.org/codecs.php [depends on your ffmpeg installation.]
outputframerate: positive number, output frame rate for labeled video (only available for the mode with saving frames.) By default: None, which results in the original video rate.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
draw_skeleton: bool
If ``True`` adds a line connecting the body parts making a skeleton on on each frame. The body parts to be connected and the color of these connecting lines are specified in the config file. By default: ``False``
trailpoints: int
Number of revious frames whose body parts are plotted in a frame (for displaying history). Default is set to 0.
displaycropped: bool, optional
Specifies whether only cropped frame is displayed (with labels analyzed therein), or the original frame with the labels analyzed in the cropped subset.
Examples
--------
If you want to create the labeled video for only 1 video
>>> deeplabcutcore.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to create the labeled video for only 1 video and store the individual frames
>>> deeplabcutcore.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'],save_frames=True)
--------
If you want to create the labeled video for multiple videos
>>> deeplabcutcore.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to create the labeled video for all the videos (as .avi extension) in a directory.
>>> deeplabcutcore.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'])
--------
If you want to create the labeled video for all the videos (as .mp4 extension) in a directory.
>>> deeplabcutcore.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'],videotype='mp4')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
start_path = os.getcwd() # record cwd to return to this directory in the end
trainFraction = cfg["TrainingFraction"][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction
) # automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts
)
if draw_skeleton:
bodyparts2connect = cfg["skeleton"]
skeleton_color = cfg["skeleton_color"]
else:
bodyparts2connect = None
skeleton_color = None
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
if destfolder is None:
videofolder = Path(video).parents[0] # where your folder with videos is.
else:
videofolder = destfolder
os.chdir(str(videofolder))
videotype = Path(video).suffix
print("Starting % ", videofolder, videos)
vname = str(Path(video).stem)
# if notanalyzed:
# notanalyzed,outdataname,sourcedataname,DLCscorer=auxiliaryfunctions.CheckifPostProcessing(folder,vname,DLCscorer,DLCscorerlegacy,suffix='checking')
if filtered == True:
videooutname1 = os.path.join(vname + DLCscorer + "filtered_labeled.mp4")
videooutname2 = os.path.join(
vname + DLCscorerlegacy + "filtered_labeled.mp4"
)
else:
videooutname1 = os.path.join(vname + DLCscorer + "_labeled.mp4")
videooutname2 = os.path.join(vname + DLCscorerlegacy + "_labeled.mp4")
if os.path.isfile(videooutname1) or os.path.isfile(videooutname2):
print("Labeled video already created.")
else:
print("Loading ", video, "and data.")
datafound, metadata, Dataframe, DLCscorer, suffix = auxiliaryfunctions.LoadAnalyzedData(
str(videofolder), vname, DLCscorer, filtered
) # returns boolean variable if data was found and metadata + pandas array
videooutname = os.path.join(vname + DLCscorer + suffix + "_labeled.mp4")
if datafound and not os.path.isfile(
videooutname
): # checking again, for this loader video could exist
# Loading cropping data used during analysis
cropping = metadata["data"]["cropping"]
[x1, x2, y1, y2] = metadata["data"]["cropping_parameters"]
if save_frames == True:
tmpfolder = os.path.join(str(videofolder), "temp-" + vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder)
clip = vp(video)
CreateVideoSlow(
videooutname,
clip,
Dataframe,
tmpfolder,
cfg["dotsize"],
cfg["colormap"],
cfg["alphavalue"],
cfg["pcutoff"],
trailpoints,
cropping,
x1,
x2,
y1,
y2,
delete,
DLCscorer,
bodyparts,
outputframerate,
Frames2plot,
bodyparts2connect,
skeleton_color,
draw_skeleton,
displaycropped,
)
else:
if (
displaycropped
): # then the cropped video + the labels is depicted
clip = vp(
fname=video,
sname=videooutname,
codec=codec,
sw=x2 - x1,
sh=y2 - y1,
)
CreateVideo(
clip,
Dataframe,
cfg["pcutoff"],
cfg["dotsize"],
cfg["colormap"],
DLCscorer,
bodyparts,
trailpoints,
cropping,
x1,
x2,
y1,
y2,
bodyparts2connect,
skeleton_color,
draw_skeleton,
displaycropped,
)
else: # then the full video + the (perhaps in cropped mode analyzed labels) are depicted
clip = vp(fname=video, sname=videooutname, codec=codec)
CreateVideo(
clip,
Dataframe,
cfg["pcutoff"],
cfg["dotsize"],
cfg["colormap"],
DLCscorer,
bodyparts,
trailpoints,
cropping,
x1,
x2,
y1,
y2,
bodyparts2connect,
skeleton_color,
draw_skeleton,
displaycropped,
)
os.chdir(str(start_path))
def create_training_dataset(config,num_shuffles=1,Shuffles=None,windows2linux=False,userfeedback=False,
trainIndexes=None,testIndexes=None,
net_type=None,augmenter_type=None):
"""
Creates a training dataset. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
userfeedback: bool, optional
If this is set to false, then all requested train/test splits are created (no matter if they already exist). If you
want to assure that previous splits etc. are not overwritten, then set this to True and you will be asked for each split.
trainIndexes: list of lists, optional (default=None)
List of one or multiple lists containing train indexes.
A list containing two lists of training indexes will produce two splits.
testIndexes: list of lists, optional (default=None)
List of test indexes.
net_type: string
Type of networks. Currently resnet_50, resnet_101, resnet_152, mobilenet_v2_1.0,mobilenet_v2_0.75, mobilenet_v2_0.5, and mobilenet_v2_0.35 are supported.
augmenter_type: string
Type of augmenter. Currently default, imgaug, tensorpack, and deterministic are supported.
Example
--------
>>> deeplabcutcore.create_training_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcutcore.create_training_dataset('C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
import scipy.io as sio
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg['scorer']
project_path = cfg['project_path']
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) #Path concatenation OS platform independent
auxiliaryfunctions.attempttomakefolder(Path(os.path.join(project_path,str(trainingsetfolder))),recursive=True)
Data = merge_annotateddatasets(cfg,project_path,Path(os.path.join(project_path,trainingsetfolder)),windows2linux)
Data = Data[scorer] #extract labeled data
#loading & linking pretrained models
if net_type is None: #loading & linking pretrained models
net_type =cfg.get('default_net_type', 'resnet_50')
else:
if 'resnet' in net_type or 'mobilenet' in net_type:
pass
else:
raise ValueError('Invalid network type:', net_type)
if augmenter_type is None:
augmenter_type=cfg.get('default_augmenter', 'default')
else:
if augmenter_type in ['default','imgaug','tensorpack','deterministic']:
pass
else:
raise ValueError('Invalid augmenter type:', augmenter_type)
import deeplabcutcore
parent_path = Path(os.path.dirname(deeplabcutcore.__file__))
defaultconfigfile = str(parent_path / 'pose_cfg.yaml')
model_path,num_shuffles=auxfun_models.Check4weights(net_type,parent_path,num_shuffles) #if the model does not exist >> throws error!
if Shuffles is None:
Shuffles = range(1, num_shuffles + 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
#print(trainIndexes,testIndexes, Shuffles, augmenter_type,net_type)
if trainIndexes is None and testIndexes is None:
splits = [(trainFraction, shuffle, SplitTrials(range(len(Data.index)), trainFraction))
for trainFraction in cfg['TrainingFraction'] for shuffle in Shuffles]
else:
if len(trainIndexes) != len(testIndexes) != len(Shuffles):
raise ValueError('Number of Shuffles and train and test indexes should be equal.')
splits = []
for shuffle, (train_inds, test_inds) in enumerate(zip(trainIndexes, testIndexes)):
trainFraction = round(len(train_inds) * 1./ (len(train_inds) + len(test_inds)), 2)
print(f"You passed a split with the following fraction: {int(100 * trainFraction)}%")
splits.append((trainFraction, Shuffles[shuffle], (train_inds, test_inds)))
bodyparts = cfg['bodyparts']
nbodyparts = len(bodyparts)
for trainFraction, shuffle, (trainIndexes, testIndexes) in splits:
if len(trainIndexes)>0:
if userfeedback:
trainposeconfigfile, _, _ = training.return_train_network_path(config, shuffle=shuffle, trainFraction=trainFraction)
if trainposeconfigfile.is_file():
askuser=input ("The model folder is already present. If you continue, it will overwrite the existing model (split). Do you want to continue?(yes/no): ")
if askuser=='no'or askuser=='No' or askuser=='N' or askuser=='No':
raise Exception("Use the Shuffles argument as a list to specify a different shuffle index. Check out the help for more details.")
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
# Make training file!
datafilename, metadatafilename = auxiliaryfunctions.GetDataandMetaDataFilenames(trainingsetfolder,
trainFraction, shuffle, cfg)
################################################################################
# Saving data file (convert to training file for deeper cut (*.mat))
################################################################################
data, MatlabData = format_training_data(Data, trainIndexes, nbodyparts, project_path)
sio.savemat(os.path.join(project_path,datafilename), {'dataset': MatlabData})
################################################################################
# Saving metadata (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(os.path.join(project_path,metadatafilename),data, trainIndexes, testIndexes, trainFraction)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername=auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg)
auxiliaryfunctions.attempttomakefolder(Path(config).parents[0] / modelfoldername,recursive=True)
auxiliaryfunctions.attempttomakefolder(str(Path(config).parents[0] / modelfoldername)+ '/train')
auxiliaryfunctions.attempttomakefolder(str(Path(config).parents[0] / modelfoldername)+ '/test')
path_train_config = str(os.path.join(cfg['project_path'],Path(modelfoldername),'train','pose_cfg.yaml'))
path_test_config = str(os.path.join(cfg['project_path'],Path(modelfoldername),'test','pose_cfg.yaml'))
#str(cfg['proj_path']+'/'+Path(modelfoldername) / 'test' / 'pose_cfg.yaml')
items2change = {
"dataset": datafilename,
"metadataset": metadatafilename,
"num_joints": len(bodyparts),
"all_joints": [[i] for i in range(len(bodyparts))],
"all_joints_names": [str(bpt) for bpt in bodyparts],
"init_weights": model_path,
"project_path": str(cfg['project_path']),
"net_type": net_type,
"dataset_type": augmenter_type,
}
trainingdata = MakeTrain_pose_yaml(items2change,path_train_config,defaultconfigfile)
keys2save = [
"dataset", "num_joints", "all_joints", "all_joints_names",
"net_type", 'init_weights', 'global_scale', 'location_refinement',
'locref_stdev'
]
MakeTest_pose_yaml(trainingdata, keys2save,path_test_config)
print("The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!")
return splits
def mergeandsplit(config,trainindex=0,uniform=True,windows2linux=False):
"""
This function allows additional control over "create_training_dataset".
Merge annotated data sets (from different folders) and split data in a specific way, returns the split variables (train/test indices).
Importantly, this allows one to freeze a split.
One can also either create a uniform split (uniform = True; thereby indexing TrainingFraction in config file) or leave-one-folder out split
by passing the index of the corrensponding video from the config.yaml file as variable trainindex.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
trainindex: int, optional
Either (in case uniform = True) indexes which element of TrainingFraction in the config file should be used (note it is a list!).
Alternatively (uniform = False) indexes which folder is dropped, i.e. the first if trainindex=0, the second if trainindex =1, etc.
uniform: bool, optional
Perform uniform split (disregarding folder structure in labeled data), or (if False) leave one folder out.
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
Examples
--------
To create a leave-one-folder-out model:
>>> trainIndexes, testIndexes=deeplabcutcore.mergeandsplit(config,trainindex=0,uniform=False)
returns the indices for the first video folder (as defined in config file) as testIndexes and all others as trainIndexes.
You can then create the training set by calling (e.g. defining it as Shuffle 3):
>>> deeplabcutcore.create_training_dataset(config,Shuffles=[3],trainIndexes=trainIndexes,testIndexes=testIndexes)
To freeze a (uniform) split:
>>> trainIndexes, testIndexes=deeplabcutcore.mergeandsplit(config,trainindex=0,uniform=True)
You can then create two model instances that have the identical trainingset. Thereby you can assess the role of various parameters on the performance of DLC.
>>> deeplabcutcore.create_training_dataset(config,Shuffles=[0],trainIndexes=trainIndexes,testIndexes=testIndexes)
>>> deeplabcutcore.create_training_dataset(config,Shuffles=[1],trainIndexes=trainIndexes,testIndexes=testIndexes)
--------
"""
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg['scorer']
project_path = cfg['project_path']
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) #Path concatenation OS platform independent
auxiliaryfunctions.attempttomakefolder(Path(os.path.join(project_path,str(trainingsetfolder))),recursive=True)
fn=os.path.join(project_path,trainingsetfolder,'CollectedData_'+cfg['scorer'])
try:
Data= pd.read_hdf(fn+'.h5', 'df_with_missing')
except FileNotFoundError:
Data = merge_annotateddatasets(cfg,project_path,Path(os.path.join(project_path,trainingsetfolder)),windows2linux=windows2linux)
Data = Data[scorer] #extract labeled data
if uniform==True:
TrainingFraction = cfg['TrainingFraction']
trainFraction=TrainingFraction[trainindex]
trainIndexes, testIndexes = SplitTrials(range(len(Data.index)), trainFraction)
else: #leave one folder out split
videos = cfg['video_sets'].keys()
test_video_name = [Path(i).stem for i in videos][trainindex]
print("Excluding the following folder (from training):", test_video_name)
trainIndexes, testIndexes=[],[]
for index,name in enumerate(Data.index):
#print(index,name.split(os.sep)[1])
if test_video_name==name.split(os.sep)[1]: #this is the video name
#print(name,test_video_name)
testIndexes.append(index)
else:
trainIndexes.append(index)
return trainIndexes, testIndexes
