def Plotting(cfg, comparisonbodyparts, DLCscorer, trainIndices, DataCombined,
foldername):
from deeplabcut.utils import visualization
colors = visualization.get_cmap(len(comparisonbodyparts),
name=cfg['colormap'])
NumFrames = np.size(DataCombined.index)
for ind in tqdm(np.arange(NumFrames)):
visualization.PlottingandSaveLabeledFrame(DataCombined, ind,
trainIndices, cfg, colors,
comparisonbodyparts,
DLCscorer, foldername)
def Plotting(cfg, comparisonbodyparts, DLCscorer, trainIndices, DataCombined,
foldername):
""" Function used for plotting GT and predictions """
from deeplabcut.utils import visualization
colors = visualization.get_cmap(len(comparisonbodyparts),
name=cfg["colormap"])
NumFrames = np.size(DataCombined.index)
for ind in tqdm(np.arange(NumFrames)):
visualization.plot_and_save_labeled_frame(
DataCombined,
ind,
trainIndices,
cfg,
colors,
comparisonbodyparts,
DLCscorer,
foldername,
)
def CreateVideoSlow(
videooutname,
clip,
Dataframe,
tmpfolder,
dotsize,
colormap,
alphavalue,
pcutoff,
trailpoints,
cropping,
x1,
x2,
y1,
y2,
save_frames,
bodyparts2plot,
outputframerate,
Frames2plot,
bodyparts2connect,
skeleton_color,
draw_skeleton,
displaycropped,
color_by,
):
"""Creating individual frames with labeled body parts and making a video"""
# scorer=np.unique(Dataframe.columns.get_level_values(0))[0]
# bodyparts2plot = list(np.unique(Dataframe.columns.get_level_values(1)))
if displaycropped:
ny, nx = y2 - y1, x2 - x1
else:
ny, nx = clip.height(), clip.width()
fps = clip.fps()
if outputframerate is None: # by def. same as input rate.
outputframerate = fps
nframes = clip.nframes
duration = nframes / fps
print("Duration of video [s]: {}, recorded with {} fps!".format(
round(duration, 2), round(fps, 2)))
print(
"Overall # of frames: {} with cropped frame dimensions: {} {}".format(
nframes, nx, ny))
print("Generating frames and creating video.")
df_x, df_y, df_likelihood = Dataframe.values.reshape(
(len(Dataframe), -1, 3)).T
if cropping and not displaycropped:
df_x += x1
df_y += y1
bpts = Dataframe.columns.get_level_values("bodyparts")
all_bpts = bpts.values[::3]
if draw_skeleton:
bpts2connect = get_segment_indices(bodyparts2connect, all_bpts)
bplist = bpts.unique().to_list()
nbodyparts = len(bplist)
if Dataframe.columns.nlevels == 3:
nindividuals = 1
map2bp = list(range(len(all_bpts)))
map2id = [0 for _ in map2bp]
else:
nindividuals = len(
Dataframe.columns.get_level_values("individuals").unique())
map2bp = [bplist.index(bp) for bp in all_bpts]
nbpts_per_ind = (
Dataframe.groupby(level="individuals", axis=1).size().values // 3)
map2id = []
for i, j in enumerate(nbpts_per_ind):
map2id.extend([i] * j)
keep = np.flatnonzero(np.isin(all_bpts, bodyparts2plot))
bpts2color = [(ind, map2bp[ind], map2id[ind]) for ind in keep]
if color_by == "individual":
colors = visualization.get_cmap(nindividuals, name=colormap)
else:
colors = visualization.get_cmap(nbodyparts, name=colormap)
nframes_digits = int(np.ceil(np.log10(nframes)))
if nframes_digits > 9:
raise Exception(
"Your video has more than 10**9 frames, we recommend chopping it up."
)
if Frames2plot is None:
Index = set(range(nframes))
else:
Index = {int(k) for k in Frames2plot if 0 <= k < nframes}
# Prepare figure
prev_backend = plt.get_backend()
plt.switch_backend("agg")
dpi = 100
fig = plt.figure(frameon=False, figsize=(nx / dpi, ny / dpi))
ax = fig.add_subplot(111)
writer = FFMpegWriter(fps=outputframerate, codec="h264")
with writer.saving(fig, videooutname,
dpi=dpi), np.errstate(invalid="ignore"):
for index in trange(min(nframes, len(Dataframe))):
imagename = tmpfolder + "/file" + str(index).zfill(
nframes_digits) + ".png"
image = img_as_ubyte(clip.load_frame())
if index in Index: # then extract the frame!
if cropping and displaycropped:
image = image[y1:y2, x1:x2]
ax.imshow(image)
if draw_skeleton:
for bpt1, bpt2 in bpts2connect:
if np.all(
df_likelihood[[bpt1, bpt2], index] > pcutoff):
ax.plot(
[df_x[bpt1, index], df_x[bpt2, index]],
[df_y[bpt1, index], df_y[bpt2, index]],
color=skeleton_color,
alpha=alphavalue,
)
for ind, num_bp, num_ind in bpts2color:
if df_likelihood[ind, index] > pcutoff:
if color_by == "bodypart":
color = colors(num_bp)
else:
color = colors(num_ind)
if trailpoints > 0:
ax.scatter(
df_x[ind][max(0, index - trailpoints):index],
df_y[ind][max(0, index - trailpoints):index],
s=dotsize**2,
color=color,
alpha=alphavalue * 0.75,
)
ax.scatter(
df_x[ind, index],
df_y[ind, index],
s=dotsize**2,
color=color,
alpha=alphavalue,
)
ax.set_xlim(0, nx)
ax.set_ylim(0, ny)
ax.axis("off")
ax.invert_yaxis()
fig.subplots_adjust(left=0,
bottom=0,
right=1,
top=1,
wspace=0,
hspace=0)
if save_frames:
fig.savefig(imagename)
writer.grab_frame()
ax.clear()
print("Labeled video {} successfully created.".format(videooutname))
plt.switch_backend(prev_backend)
def evaluate_multianimal_full(
config,
Shuffles=[1],
trainingsetindex=0,
plotting=False,
show_errors=True,
comparisonbodyparts="all",
gputouse=None,
modelprefix="",
):
from deeplabcut.pose_estimation_tensorflow.core import (
predict,
predict_multianimal as predictma,
)
from deeplabcut.utils import (
auxiliaryfunctions,
auxfun_multianimal,
auxfun_videos,
conversioncode,
)
import tensorflow as tf
if "TF_CUDNN_USE_AUTOTUNE" in os.environ:
del os.environ["TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training
tf.compat.v1.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
start_path = os.getcwd()
if plotting is True:
plotting = "bodypart"
##################################################
# Load data...
##################################################
cfg = auxiliaryfunctions.read_config(config)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
TrainingFractions = [cfg["TrainingFraction"][trainingsetindex]]
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
)
)
conversioncode.guarantee_multiindex_rows(Data)
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts
)
all_bpts = np.asarray(
len(cfg["individuals"]) * cfg["multianimalbodyparts"] + cfg["uniquebodyparts"]
)
colors = visualization.get_cmap(len(comparisonbodyparts), name=cfg["colormap"])
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/")
)
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg
)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix
)
),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn)
)
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction)
)
pipeline = iaa.Sequential(random_order=False)
pre_resize = dlc_cfg.get("pre_resize")
if pre_resize:
width, height = pre_resize
pipeline.add(iaa.Resize({"height": height, "width": width}))
# TODO: IMPLEMENT for different batch sizes?
dlc_cfg["batch_size"] = 1 # due to differently sized images!!!
stride = dlc_cfg["stride"]
# Ignore best edges possibly defined during a prior evaluation
_ = dlc_cfg.pop("paf_best", None)
joints = dlc_cfg["all_joints_names"]
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix
)
),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True)
# path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array(
[
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
]
)
if len(Snapshots) == 0:
print(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction)
)
else:
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots]
)
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
final_result = []
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg["init_weights"].split(os.sep)[-1]
).split("-")[
-1
] # read how many training siterations that corresponds to.
# name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix,
)
print(
"Running ",
DLCscorer,
" with # of trainingiterations:",
trainingsiterations,
)
(
notanalyzed,
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.CheckifNotEvaluated(
str(evaluationfolder),
DLCscorer,
DLCscorerlegacy,
Snapshots[snapindex],
)
data_path = resultsfilename.split(".h5")[0] + "_full.pickle"
if plotting:
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" + Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
if plotting == "bodypart":
fig, ax = visualization.create_minimal_figure()
if os.path.isfile(data_path):
print("Model already evaluated.", resultsfilename)
else:
(sess, inputs, outputs,) = predict.setup_pose_prediction(
dlc_cfg
)
PredicteData = {}
dist = np.full((len(Data), len(all_bpts)), np.nan)
conf = np.full_like(dist, np.nan)
print("Network Evaluation underway...")
for imageindex, imagename in tqdm(enumerate(Data.index)):
image_path = os.path.join(cfg["project_path"], *imagename)
frame = auxfun_videos.imread(image_path, mode="skimage")
GT = Data.iloc[imageindex]
if not GT.any():
continue
# Pass the image and the keypoints through the resizer;
# this has no effect if no augmenters were added to it.
keypoints = [GT.to_numpy().reshape((-1, 2)).astype(float)]
frame_, keypoints = pipeline(
images=[frame], keypoints=keypoints
)
frame = frame_[0]
GT[:] = keypoints[0].flatten()
df = GT.unstack("coords").reindex(joints, level="bodyparts")
# FIXME Is having an empty array vs nan really that necessary?!
groundtruthidentity = list(
df.index.get_level_values("individuals")
.to_numpy()
.reshape((-1, 1))
)
groundtruthcoordinates = list(df.values[:, np.newaxis])
for i, coords in enumerate(groundtruthcoordinates):
if np.isnan(coords).any():
groundtruthcoordinates[i] = np.empty(
(0, 2), dtype=float
)
groundtruthidentity[i] = np.array([], dtype=str)
# Form 2D array of shape (n_rows, 4) where the last dimension
# is (sample_index, peak_y, peak_x, bpt_index) to slice the PAFs.
temp = df.reset_index(level="bodyparts").dropna()
temp["bodyparts"].replace(
dict(zip(joints, range(len(joints)))), inplace=True,
)
temp["sample"] = 0
peaks_gt = temp.loc[
:, ["sample", "y", "x", "bodyparts"]
].to_numpy()
peaks_gt[:, 1:3] = (peaks_gt[:, 1:3] - stride // 2) / stride
pred = predictma.predict_batched_peaks_and_costs(
dlc_cfg,
np.expand_dims(frame, axis=0),
sess,
inputs,
outputs,
peaks_gt.astype(int),
)
if not pred:
continue
else:
pred = pred[0]
PredicteData[imagename] = {}
PredicteData[imagename]["index"] = imageindex
PredicteData[imagename]["prediction"] = pred
PredicteData[imagename]["groundtruth"] = [
groundtruthidentity,
groundtruthcoordinates,
GT,
]
coords_pred = pred["coordinates"][0]
probs_pred = pred["confidence"]
for bpt, xy_gt in df.groupby(level="bodyparts"):
inds_gt = np.flatnonzero(
np.all(~np.isnan(xy_gt), axis=1)
)
n_joint = joints.index(bpt)
xy = coords_pred[n_joint]
if inds_gt.size and xy.size:
# Pick the predictions closest to ground truth,
# rather than the ones the model has most confident in
xy_gt_values = xy_gt.iloc[inds_gt].values
neighbors = _find_closest_neighbors(
xy_gt_values, xy, k=3
)
found = neighbors != -1
min_dists = np.linalg.norm(
xy_gt_values[found] - xy[neighbors[found]],
axis=1,
)
inds = np.flatnonzero(all_bpts == bpt)
sl = imageindex, inds[inds_gt[found]]
dist[sl] = min_dists
conf[sl] = probs_pred[n_joint][
neighbors[found]
].squeeze()
if plotting == "bodypart":
temp_xy = GT.unstack("bodyparts")[joints].values
gt = temp_xy.reshape(
(-1, 2, temp_xy.shape[1])
).T.swapaxes(1, 2)
h, w, _ = np.shape(frame)
fig.set_size_inches(w / 100, h / 100)
ax.set_xlim(0, w)
ax.set_ylim(0, h)
ax.invert_yaxis()
ax = visualization.make_multianimal_labeled_image(
frame,
gt,
coords_pred,
probs_pred,
colors,
cfg["dotsize"],
cfg["alphavalue"],
cfg["pcutoff"],
ax=ax,
)
visualization.save_labeled_frame(
fig,
image_path,
foldername,
imageindex in trainIndices,
)
visualization.erase_artists(ax)
sess.close() # closes the current tf session
# Compute all distance statistics
df_dist = pd.DataFrame(dist, columns=df.index)
df_conf = pd.DataFrame(conf, columns=df.index)
df_joint = pd.concat(
[df_dist, df_conf],
keys=["rmse", "conf"],
names=["metrics"],
axis=1,
)
df_joint = df_joint.reorder_levels(
list(np.roll(df_joint.columns.names, -1)), axis=1
)
df_joint.sort_index(
axis=1,
level=["individuals", "bodyparts"],
ascending=[True, True],
inplace=True,
)
write_path = os.path.join(
evaluationfolder, f"dist_{trainingsiterations}.csv"
)
df_joint.to_csv(write_path)
# Calculate overall prediction error
error = df_joint.xs("rmse", level="metrics", axis=1)
mask = (
df_joint.xs("conf", level="metrics", axis=1)
>= cfg["pcutoff"]
)
error_masked = error[mask]
error_train = np.nanmean(error.iloc[trainIndices])
error_train_cut = np.nanmean(error_masked.iloc[trainIndices])
error_test = np.nanmean(error.iloc[testIndices])
error_test_cut = np.nanmean(error_masked.iloc[testIndices])
results = [
trainingsiterations,
int(100 * trainFraction),
shuffle,
np.round(error_train, 2),
np.round(error_test, 2),
cfg["pcutoff"],
np.round(error_train_cut, 2),
np.round(error_test_cut, 2),
]
final_result.append(results)
if show_errors:
string = (
"Results for {} training iterations, training fraction of {}, and shuffle {}:\n"
"Train error: {} pixels. Test error: {} pixels.\n"
"With pcutoff of {}:\n"
"Train error: {} pixels. Test error: {} pixels."
)
print(string.format(*results))
print("##########################################")
print(
"Average Euclidean distance to GT per individual (in pixels; test-only)"
)
print(
error_masked.iloc[testIndices]
.groupby("individuals", axis=1)
.mean()
.mean()
.to_string()
)
print(
"Average Euclidean distance to GT per bodypart (in pixels; test-only)"
)
print(
error_masked.iloc[testIndices]
.groupby("bodyparts", axis=1)
.mean()
.mean()
.to_string()
)
PredicteData["metadata"] = {
"nms radius": dlc_cfg["nmsradius"],
"minimal confidence": dlc_cfg["minconfidence"],
"sigma": dlc_cfg.get("sigma", 1),
"PAFgraph": dlc_cfg["partaffinityfield_graph"],
"PAFinds": np.arange(
len(dlc_cfg["partaffinityfield_graph"])
),
"all_joints": [
[i] for i in range(len(dlc_cfg["all_joints"]))
],
"all_joints_names": [
dlc_cfg["all_joints_names"][i]
for i in range(len(dlc_cfg["all_joints"]))
],
"stride": dlc_cfg.get("stride", 8),
}
print(
"Done and results stored for snapshot: ",
Snapshots[snapindex],
)
dictionary = {
"Scorer": DLCscorer,
"DLC-model-config file": dlc_cfg,
"trainIndices": trainIndices,
"testIndices": testIndices,
"trainFraction": trainFraction,
}
metadata = {"data": dictionary}
_ = auxfun_multianimal.SaveFullMultiAnimalData(
PredicteData, metadata, resultsfilename
)
tf.compat.v1.reset_default_graph()
n_multibpts = len(cfg["multianimalbodyparts"])
if n_multibpts == 1:
continue
# Skip data-driven skeleton selection unless
# the model was trained on the full graph.
max_n_edges = n_multibpts * (n_multibpts - 1) // 2
n_edges = len(dlc_cfg["partaffinityfield_graph"])
if n_edges == max_n_edges:
print("Selecting best skeleton...")
n_graphs = 10
paf_inds = None
else:
n_graphs = 1
paf_inds = [list(range(n_edges))]
(
results,
paf_scores,
best_assemblies,
) = crossvalutils.cross_validate_paf_graphs(
config,
str(path_test_config).replace("pose_", "inference_"),
data_path,
data_path.replace("_full.", "_meta."),
n_graphs=n_graphs,
paf_inds=paf_inds,
oks_sigma=dlc_cfg.get("oks_sigma", 0.1),
margin=dlc_cfg.get("bbox_margin", 0),
symmetric_kpts=dlc_cfg.get("symmetric_kpts"),
)
if plotting == "individual":
assemblies, assemblies_unique, image_paths = best_assemblies
fig, ax = visualization.create_minimal_figure()
n_animals = len(cfg["individuals"])
if cfg["uniquebodyparts"]:
n_animals += 1
colors = visualization.get_cmap(n_animals, name=cfg["colormap"])
for k, v in tqdm(assemblies.items()):
imname = image_paths[k]
image_path = os.path.join(cfg["project_path"], *imname)
frame = auxfun_videos.imread(image_path, mode="skimage")
h, w, _ = np.shape(frame)
fig.set_size_inches(w / 100, h / 100)
ax.set_xlim(0, w)
ax.set_ylim(0, h)
ax.invert_yaxis()
gt = [
s.to_numpy().reshape((-1, 2))
for _, s in Data.loc[imname].groupby("individuals")
]
coords_pred = []
coords_pred += [ass.xy for ass in v]
probs_pred = []
probs_pred += [ass.data[:, 2:3] for ass in v]
if assemblies_unique is not None:
unique = assemblies_unique.get(k, None)
if unique is not None:
coords_pred.append(unique[:, :2])
probs_pred.append(unique[:, 2:3])
while len(coords_pred) < len(gt):
coords_pred.append(np.full((1, 2), np.nan))
probs_pred.append(np.full((1, 2), np.nan))
ax = visualization.make_multianimal_labeled_image(
frame,
gt,
coords_pred,
probs_pred,
colors,
cfg["dotsize"],
cfg["alphavalue"],
cfg["pcutoff"],
ax=ax,
)
visualization.save_labeled_frame(
fig, image_path, foldername, k in trainIndices,
)
visualization.erase_artists(ax)
df = results[1].copy()
df.loc(axis=0)[("mAP_train", "mean")] = [
d[0]["mAP"] for d in results[2]
]
df.loc(axis=0)[("mAR_train", "mean")] = [
d[0]["mAR"] for d in results[2]
]
df.loc(axis=0)[("mAP_test", "mean")] = [
d[1]["mAP"] for d in results[2]
]
df.loc(axis=0)[("mAR_test", "mean")] = [
d[1]["mAR"] for d in results[2]
]
with open(data_path.replace("_full.", "_map."), "wb") as file:
pickle.dump((df, paf_scores), file)
if len(final_result) > 0: # Only append if results were calculated
make_results_file(final_result, evaluationfolder, DLCscorer)
os.chdir(str(start_path))
def ExtractFramesbasedonPreselection(
Index,
extractionalgorithm,
data,
video,
cfg,
config,
opencv=True,
cluster_resizewidth=30,
cluster_color=False,
savelabeled=True,
with_annotations=True,
):
from deeplabcut.create_project import add
start = cfg["start"]
stop = cfg["stop"]
numframes2extract = cfg["numframes2pick"]
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, "all")
videofolder = str(Path(video).parents[0])
vname = str(Path(video).stem)
tmpfolder = os.path.join(cfg["project_path"], "labeled-data", vname)
if os.path.isdir(tmpfolder):
print("Frames from video", vname,
" already extracted (more will be added)!")
else:
auxiliaryfunctions.attempttomakefolder(tmpfolder, recursive=True)
nframes = len(data)
print("Loading video...")
if opencv:
vid = VideoWriter(video)
fps = vid.fps
duration = vid.calc_duration()
else:
from moviepy.editor import VideoFileClip
clip = VideoFileClip(video)
fps = clip.fps
duration = clip.duration
if cfg["cropping"]: # one might want to adjust
coords = (cfg["x1"], cfg["x2"], cfg["y1"], cfg["y2"])
else:
coords = None
print("Duration of video [s]: ", duration, ", recorded @ ", fps, "fps!")
print("Overall # of frames: ", nframes,
"with (cropped) frame dimensions: ")
if extractionalgorithm == "uniform":
if opencv:
frames2pick = frameselectiontools.UniformFramescv2(
vid, numframes2extract, start, stop, Index)
else:
frames2pick = frameselectiontools.UniformFrames(
clip, numframes2extract, start, stop, Index)
elif extractionalgorithm == "kmeans":
if opencv:
frames2pick = frameselectiontools.KmeansbasedFrameselectioncv2(
vid,
numframes2extract,
start,
stop,
cfg["cropping"],
coords,
Index,
resizewidth=cluster_resizewidth,
color=cluster_color,
)
else:
if cfg["cropping"]:
clip = clip.crop(y1=cfg["y1"],
y2=cfg["x2"],
x1=cfg["x1"],
x2=cfg["x2"])
frames2pick = frameselectiontools.KmeansbasedFrameselection(
clip,
numframes2extract,
start,
stop,
Index,
resizewidth=cluster_resizewidth,
color=cluster_color,
)
else:
print(
"Please implement this method yourself! Currently the options are 'kmeans', 'jump', 'uniform'."
)
frames2pick = []
# Extract frames + frames with plotted labels and store them in folder (with name derived from video name) nder labeled-data
print("Let's select frames indices:", frames2pick)
colors = visualization.get_cmap(len(bodyparts), cfg["colormap"])
strwidth = int(np.ceil(np.log10(nframes))) # width for strings
for index in frames2pick: ##tqdm(range(0,nframes,10)):
if opencv:
PlottingSingleFramecv2(
vid,
cfg["cropping"],
coords,
data,
bodyparts,
tmpfolder,
index,
cfg["dotsize"],
cfg["pcutoff"],
cfg["alphavalue"],
colors,
strwidth,
savelabeled,
)
else:
PlottingSingleFrame(
clip,
data,
bodyparts,
tmpfolder,
index,
cfg["dotsize"],
cfg["pcutoff"],
cfg["alphavalue"],
colors,
strwidth,
savelabeled,
)
plt.close("all")
# close videos
if opencv:
vid.close()
else:
clip.close()
del clip
# Extract annotations based on DeepLabCut and store in the folder (with name derived from video name) under labeled-data
if len(frames2pick) > 0:
try:
if cfg["cropping"]:
add.add_new_videos(
config, [video],
coords=[coords]) # make sure you pass coords as a list
else:
add.add_new_videos(config, [video], coords=None)
except: # can we make a catch here? - in fact we should drop indices from DataCombined if they are in CollectedData.. [ideal behavior; currently this is pretty unlikely]
print(
"AUTOMATIC ADDING OF VIDEO TO CONFIG FILE FAILED! You need to do this manually for including it in the config.yaml file!"
)
print("Videopath:", video, "Coordinates for cropping:", coords)
pass
if with_annotations:
machinefile = os.path.join(
tmpfolder,
"machinelabels-iter" + str(cfg["iteration"]) + ".h5")
if isinstance(data, pd.DataFrame):
df = data.loc[frames2pick]
df.index = [
os.path.join(
"labeled-data",
vname,
"img" + str(index).zfill(strwidth) + ".png",
) for index in df.index
] # exchange index number by file names.
elif isinstance(data, dict):
idx = [
os.path.join(
"labeled-data",
vname,
"img" + str(index).zfill(strwidth) + ".png",
) for index in frames2pick
]
filename = os.path.join(str(tmpfolder),
f"CollectedData_{cfg['scorer']}.h5")
try:
df_temp = pd.read_hdf(filename, "df_with_missing")
columns = df_temp.columns
except FileNotFoundError:
columns = pd.MultiIndex.from_product(
[
[cfg["scorer"]],
cfg["individuals"],
cfg["multianimalbodyparts"],
["x", "y"],
],
names=["scorer", "individuals", "bodyparts", "coords"],
)
if cfg["uniquebodyparts"]:
columns2 = pd.MultiIndex.from_product(
[
[cfg["scorer"]],
["single"],
cfg["uniquebodyparts"],
["x", "y"],
],
names=[
"scorer", "individuals", "bodyparts", "coords"
],
)
df_temp = pd.concat((
pd.DataFrame(columns=columns),
pd.DataFrame(columns=columns2),
))
columns = df_temp.columns
array = np.full((len(frames2pick), len(columns)), np.nan)
for i, index in enumerate(frames2pick):
data_temp = data.get(index)
if data_temp is not None:
vals = np.concatenate(data_temp)[:, :2].flatten()
array[i, :len(vals)] = vals
df = pd.DataFrame(array, index=idx, columns=columns)
else:
return
if Path(machinefile).is_file():
Data = pd.read_hdf(machinefile, "df_with_missing")
DataCombined = pd.concat([Data, df])
# drop duplicate labels:
DataCombined = DataCombined[~DataCombined.index.duplicated(
keep="first")]
DataCombined.to_hdf(machinefile,
key="df_with_missing",
mode="w")
DataCombined.to_csv(
os.path.join(tmpfolder, "machinelabels.csv")
) # this is always the most current one (as reading is from h5)
else:
df.to_hdf(machinefile, key="df_with_missing", mode="w")
df.to_csv(os.path.join(tmpfolder, "machinelabels.csv"))
print(
"The outlier frames are extracted. They are stored in the subdirectory labeled-data\%s."
% vname)
print(
"Once you extracted frames for all videos, use 'refine_labels' to manually correct the labels."
)
else:
print("No frames were extracted.")
def evaluate_multianimal_full(
config,
Shuffles=[1],
trainingsetindex=0,
plotting=None,
show_errors=True,
comparisonbodyparts="all",
gputouse=None,
modelprefix="",
c_engine=False,
):
from deeplabcut.pose_estimation_tensorflow.nnet import predict
from deeplabcut.pose_estimation_tensorflow.nnet import (
predict_multianimal as predictma, )
from deeplabcut.utils import auxiliaryfunctions, auxfun_multianimal
import tensorflow as tf
if "TF_CUDNN_USE_AUTOTUNE" in os.environ:
del os.environ[
"TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training
tf.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
start_path = os.getcwd()
##################################################
# Load data...
##################################################
cfg = auxiliaryfunctions.read_config(config)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
TrainingFractions = [cfg["TrainingFraction"][trainingsetindex]]
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
)
# Handle data previously annotated on a different platform
sep = "/" if "/" in Data.index[0] else "\\"
if sep != os.path.sep:
Data.index = Data.index.str.replace(sep, os.path.sep)
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
all_bpts = np.asarray(
len(cfg["individuals"]) * cfg["multianimalbodyparts"] +
cfg["uniquebodyparts"])
colors = visualization.get_cmap(len(comparisonbodyparts),
name=cfg["colormap"])
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# TODO: IMPLEMENT for different batch sizes?
dlc_cfg["batch_size"] = 1 # due to differently sized images!!!
joints = dlc_cfg["all_joints_names"]
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
# path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
])
if len(Snapshots) == 0:
print(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
else:
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
final_result = []
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg["init_weights"].split(os.sep)[-1]
).split(
"-"
)[-1] # read how many training siterations that corresponds to.
# name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix,
)
print(
"Running ",
DLCscorer,
" with # of trainingiterations:",
trainingsiterations,
)
(
notanalyzed,
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.CheckifNotEvaluated(
str(evaluationfolder),
DLCscorer,
DLCscorerlegacy,
Snapshots[snapindex],
)
if os.path.isfile(
resultsfilename.split(".h5")[0] + "_full.pickle"):
print("Model already evaluated.", resultsfilename)
else:
if plotting:
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" +
Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
# print(dlc_cfg)
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = predict.setup_pose_prediction(
dlc_cfg)
PredicteData = {}
dist = np.full((len(Data), len(all_bpts)), np.nan)
conf = np.full_like(dist, np.nan)
distnorm = np.full(len(Data), np.nan)
print("Analyzing data...")
for imageindex, imagename in tqdm(enumerate(
Data.index)):
image_path = os.path.join(cfg["project_path"],
imagename)
image = io.imread(image_path)
frame = img_as_ubyte(skimage.color.gray2rgb(image))
GT = Data.iloc[imageindex]
df = GT.unstack("coords").reindex(
joints, level='bodyparts')
# Evaluate PAF edge lengths to calibrate `distnorm`
temp = GT.unstack("bodyparts")[joints]
xy = temp.values.reshape(
(-1, 2, temp.shape[1])).swapaxes(1, 2)
edges = xy[:, dlc_cfg["partaffinityfield_graph"]]
lengths = np.sum(
(edges[:, :, 0] - edges[:, :, 1])**2, axis=2)
distnorm[imageindex] = np.nanmax(lengths)
# FIXME Is having an empty array vs nan really that necessary?!
groundtruthidentity = list(
df.index.get_level_values(
"individuals").to_numpy().reshape((-1, 1)))
groundtruthcoordinates = list(
df.values[:, np.newaxis])
for i, coords in enumerate(groundtruthcoordinates):
if np.isnan(coords).any():
groundtruthcoordinates[i] = np.empty(
(0, 2), dtype=float)
groundtruthidentity[i] = np.array(
[], dtype=str)
PredicteData[imagename] = {}
PredicteData[imagename]["index"] = imageindex
pred = predictma.get_detectionswithcostsandGT(
frame,
groundtruthcoordinates,
dlc_cfg,
sess,
inputs,
outputs,
outall=False,
nms_radius=dlc_cfg.nmsradius,
det_min_score=dlc_cfg.minconfidence,
c_engine=c_engine,
)
PredicteData[imagename]["prediction"] = pred
PredicteData[imagename]["groundtruth"] = [
groundtruthidentity,
groundtruthcoordinates,
GT,
]
coords_pred = pred["coordinates"][0]
probs_pred = pred["confidence"]
for bpt, xy_gt in df.groupby(level="bodyparts"):
inds_gt = np.flatnonzero(
np.all(~np.isnan(xy_gt), axis=1))
n_joint = joints.index(bpt)
xy = coords_pred[n_joint]
if inds_gt.size and xy.size:
# Pick the predictions closest to ground truth,
# rather than the ones the model has most confident in
d = cdist(xy_gt.iloc[inds_gt], xy)
rows, cols = linear_sum_assignment(d)
min_dists = d[rows, cols]
inds = np.flatnonzero(all_bpts == bpt)
sl = imageindex, inds[inds_gt[rows]]
dist[sl] = min_dists
conf[sl] = probs_pred[n_joint][
cols].squeeze()
if plotting:
fig = visualization.make_multianimal_labeled_image(
frame,
groundtruthcoordinates,
coords_pred,
probs_pred,
colors,
cfg["dotsize"],
cfg["alphavalue"],
cfg["pcutoff"],
)
visualization.save_labeled_frame(
fig,
image_path,
foldername,
imageindex in trainIndices,
)
sess.close() # closes the current tf session
# Compute all distance statistics
df_dist = pd.DataFrame(dist, columns=df.index)
df_conf = pd.DataFrame(conf, columns=df.index)
df_joint = pd.concat([df_dist, df_conf],
keys=["rmse", "conf"],
names=["metrics"],
axis=1)
df_joint = df_joint.reorder_levels(list(
np.roll(df_joint.columns.names, -1)),
axis=1)
df_joint.sort_index(axis=1,
level=["individuals", "bodyparts"],
ascending=[True, True],
inplace=True)
write_path = os.path.join(
evaluationfolder,
f"dist_{trainingsiterations}.csv")
df_joint.to_csv(write_path)
# Calculate overall prediction error
error = df_joint.xs("rmse", level="metrics", axis=1)
mask = df_joint.xs("conf", level="metrics",
axis=1) >= cfg["pcutoff"]
error_masked = error[mask]
error_train = np.nanmean(error.iloc[trainIndices])
error_train_cut = np.nanmean(
error_masked.iloc[trainIndices])
error_test = np.nanmean(error.iloc[testIndices])
error_test_cut = np.nanmean(
error_masked.iloc[testIndices])
results = [
trainingsiterations,
int(100 * trainFraction),
shuffle,
np.round(error_train, 2),
np.round(error_test, 2),
cfg["pcutoff"],
np.round(error_train_cut, 2),
np.round(error_test_cut, 2),
]
final_result.append(results)
# For OKS/PCK, compute the standard deviation error across all frames
sd = df_dist.groupby("bodyparts",
axis=1).mean().std(axis=0)
sd["distnorm"] = np.sqrt(np.nanmax(distnorm))
sd.to_csv(write_path.replace("dist.csv", "sd.csv"))
if show_errors:
string = "Results for {} training iterations: {}, shuffle {}:\n" \
"Train error: {} pixels. Test error: {} pixels.\n" \
"With pcutoff of {}:\n" \
"Train error: {} pixels. Test error: {} pixels."
print(string.format(*results))
print("##########################################")
print(
"Average Euclidean distance to GT per individual (in pixels)"
)
print(
error_masked.groupby(
'individuals',
axis=1).mean().mean().to_string())
print(
"Average Euclidean distance to GT per bodypart (in pixels)"
)
print(
error_masked.groupby(
'bodyparts',
axis=1).mean().mean().to_string())
PredicteData["metadata"] = {
"nms radius":
dlc_cfg.nmsradius,
"minimal confidence":
dlc_cfg.minconfidence,
"PAFgraph":
dlc_cfg.partaffinityfield_graph,
"all_joints":
[[i] for i in range(len(dlc_cfg.all_joints))],
"all_joints_names": [
dlc_cfg.all_joints_names[i]
for i in range(len(dlc_cfg.all_joints))
],
"stride":
dlc_cfg.get("stride", 8),
}
print(
"Done and results stored for snapshot: ",
Snapshots[snapindex],
)
dictionary = {
"Scorer": DLCscorer,
"DLC-model-config file": dlc_cfg,
"trainIndices": trainIndices,
"testIndices": testIndices,
"trainFraction": trainFraction,
}
metadata = {"data": dictionary}
auxfun_multianimal.SaveFullMultiAnimalData(
PredicteData, metadata, resultsfilename)
tf.reset_default_graph()
if len(final_result
) > 0: # Only append if results were calculated
make_results_file(final_result, evaluationfolder,
DLCscorer)
# returning to intial folder
os.chdir(str(start_path))
def __init__(self, parent, config, video, shuffle, Dataframe, savelabeled,
multianimal):
super(MainFrame,
self).__init__("DeepLabCut2.0 - Manual Outlier Frame Extraction",
parent)
###################################################################################################################################################
# Spliting the frame into top and bottom panels. Bottom panels contains the widgets. The top panel is for showing images and plotting!
# topSplitter = wx.SplitterWindow(self)
#
# self.image_panel = ImagePanel(topSplitter, config,video,shuffle,Dataframe,self.gui_size)
# self.widget_panel = WidgetPanel(topSplitter)
#
# topSplitter.SplitHorizontally(self.image_panel, self.widget_panel,sashPosition=self.gui_size[1]*0.83)#0.9
# topSplitter.SetSashGravity(1)
# sizer = wx.BoxSizer(wx.VERTICAL)
# sizer.Add(topSplitter, 1, wx.EXPAND)
# self.SetSizer(sizer)
# Spliting the frame into top and bottom panels. Bottom panels contains the widgets. The top panel is for showing images and plotting!
topSplitter = wx.SplitterWindow(self)
vSplitter = wx.SplitterWindow(topSplitter)
self.image_panel = ImagePanel(vSplitter, self.gui_size)
self.choice_panel = ScrollPanel(vSplitter)
vSplitter.SplitVertically(self.image_panel,
self.choice_panel,
sashPosition=self.gui_size[0] * 0.8)
vSplitter.SetSashGravity(1)
self.widget_panel = WidgetPanel(topSplitter)
topSplitter.SplitHorizontally(vSplitter,
self.widget_panel,
sashPosition=self.gui_size[1] *
0.83) # 0.9
topSplitter.SetSashGravity(1)
sizer = wx.BoxSizer(wx.VERTICAL)
sizer.Add(topSplitter, 1, wx.EXPAND)
self.SetSizer(sizer)
###################################################################################################################################################
# Add Buttons to the WidgetPanel and bind them to their respective functions.
widgetsizer = wx.WrapSizer(orient=wx.HORIZONTAL)
self.load_button_sizer = wx.BoxSizer(wx.VERTICAL)
self.help_button_sizer = wx.BoxSizer(wx.VERTICAL)
self.help = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Help")
self.help_button_sizer.Add(self.help, 1, wx.ALL, 15)
# widgetsizer.Add(self.help , 1, wx.ALL, 15)
self.help.Bind(wx.EVT_BUTTON, self.helpButton)
widgetsizer.Add(self.help_button_sizer, 1, wx.ALL, 0)
self.grab = wx.Button(self.widget_panel,
id=wx.ID_ANY,
label="Grab Frames")
widgetsizer.Add(self.grab, 1, wx.ALL, 15)
self.grab.Bind(wx.EVT_BUTTON, self.grabFrame)
self.grab.Enable(True)
widgetsizer.AddStretchSpacer(5)
self.slider = wx.Slider(
self.widget_panel,
id=wx.ID_ANY,
value=0,
minValue=0,
maxValue=1,
size=(200, -1),
style=wx.SL_HORIZONTAL | wx.SL_AUTOTICKS | wx.SL_LABELS,
)
widgetsizer.Add(self.slider, 1, wx.ALL, 5)
self.slider.Bind(wx.EVT_SLIDER, self.OnSliderScroll)
widgetsizer.AddStretchSpacer(5)
self.start_frames_sizer = wx.BoxSizer(wx.VERTICAL)
self.end_frames_sizer = wx.BoxSizer(wx.VERTICAL)
self.start_frames_sizer.AddSpacer(15)
# self.startFrame = wx.SpinCtrl(self.widget_panel, value='0', size=(100, -1), min=0, max=120)
self.startFrame = wx.SpinCtrl(self.widget_panel,
value="0",
size=(100, -1)) # ,style=wx.SP_VERTICAL)
self.startFrame.Enable(False)
self.start_frames_sizer.Add(self.startFrame, 1,
wx.EXPAND | wx.ALIGN_LEFT, 15)
start_text = wx.StaticText(self.widget_panel,
label="Start Frame Index")
self.start_frames_sizer.Add(start_text, 1, wx.EXPAND | wx.ALIGN_LEFT,
15)
self.checkBox = wx.CheckBox(self.widget_panel,
id=wx.ID_ANY,
label="Range of frames")
self.checkBox.Bind(wx.EVT_CHECKBOX, self.activate_frame_range)
self.start_frames_sizer.Add(self.checkBox, 1,
wx.EXPAND | wx.ALIGN_LEFT, 15)
#
self.end_frames_sizer.AddSpacer(15)
self.endFrame = wx.SpinCtrl(self.widget_panel,
value="1",
size=(160, -1)) # , min=1, max=120)
self.endFrame.Enable(False)
self.end_frames_sizer.Add(self.endFrame, 1, wx.EXPAND | wx.ALIGN_LEFT,
15)
end_text = wx.StaticText(self.widget_panel, label="Number of Frames")
self.end_frames_sizer.Add(end_text, 1, wx.EXPAND | wx.ALIGN_LEFT, 15)
self.updateFrame = wx.Button(self.widget_panel,
id=wx.ID_ANY,
label="Update")
self.end_frames_sizer.Add(self.updateFrame, 1,
wx.EXPAND | wx.ALIGN_LEFT, 15)
self.updateFrame.Bind(wx.EVT_BUTTON, self.updateSlider)
self.updateFrame.Enable(False)
widgetsizer.Add(self.start_frames_sizer, 1, wx.ALL, 0)
widgetsizer.AddStretchSpacer(5)
widgetsizer.Add(self.end_frames_sizer, 1, wx.ALL, 0)
widgetsizer.AddStretchSpacer(15)
self.quit = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Quit")
widgetsizer.Add(self.quit, 1, wx.ALL, 15)
self.quit.Bind(wx.EVT_BUTTON, self.quitButton)
self.quit.Enable(True)
self.widget_panel.SetSizer(widgetsizer)
self.widget_panel.SetSizerAndFit(widgetsizer)
# Variables initialization
self.numberFrames = 0
self.currFrame = 0
self.figure = Figure()
self.axes = self.figure.add_subplot(111)
self.drs = []
self.extract_range_frame = False
self.firstFrame = 0
self.Colorscheme = []
# Read confing file
self.cfg = auxiliaryfunctions.read_config(config)
self.Task = self.cfg["Task"]
self.start = self.cfg["start"]
self.stop = self.cfg["stop"]
self.date = self.cfg["date"]
self.trainFraction = self.cfg["TrainingFraction"]
self.trainFraction = self.trainFraction[0]
self.videos = self.cfg["video_sets"].keys()
self.bodyparts = self.cfg["bodyparts"]
self.colormap = plt.get_cmap(self.cfg["colormap"])
self.colormap = self.colormap.reversed()
self.markerSize = self.cfg["dotsize"]
self.alpha = self.cfg["alphavalue"]
self.iterationindex = self.cfg["iteration"]
self.cropping = self.cfg["cropping"]
self.video_names = [Path(i).stem for i in self.videos]
self.config_path = Path(config)
self.video_source = Path(video).resolve()
self.shuffle = shuffle
self.Dataframe = Dataframe
self.savelabeled = savelabeled
self.multianimal = multianimal
if self.multianimal:
from deeplabcut.utils import auxfun_multianimal
(
self.individual_names,
self.uniquebodyparts,
self.multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(self.cfg)
self.choiceBox, self.visualization_rdb = self.choice_panel.addRadioButtons(
)
self.Colorscheme = visualization.get_cmap(
len(self.individual_names), self.cfg["colormap"])
self.visualization_rdb.Bind(wx.EVT_RADIOBOX, self.clear_plot)
# Read the video file
self.vid = VideoWriter(str(self.video_source))
if self.cropping:
self.vid.set_bbox(self.cfg["x1"], self.cfg["x2"], self.cfg["y1"],
self.cfg["y2"])
self.filename = Path(self.video_source).name
self.numberFrames = len(self.vid)
self.strwidth = int(np.ceil(np.log10(self.numberFrames)))
# Set the values of slider and range of frames
self.startFrame.SetMax(self.numberFrames - 1)
self.slider.SetMax(self.numberFrames - 1)
self.endFrame.SetMax(self.numberFrames - 1)
self.startFrame.Bind(wx.EVT_SPINCTRL, self.updateSlider) # wx.EVT_SPIN
# Set the status bar
self.statusbar.SetStatusText("Working on video: {}".format(
self.filename))
# Adding the video file to the config file.
if self.vid.name not in self.video_names:
add.add_new_videos(self.config_path, [self.video_source])
self.update()
self.plot_labels()
self.widget_panel.Layout()
def plot(self,im):
"""
Plots and call auxfun_drag class for moving and removing points.
"""
#small hack in case there are any 0 intensity images!
img = io.imread(im)
maxIntensity = np.max(img)
if maxIntensity == 0:
maxIntensity = np.max(img) + 255
divider = make_axes_locatable(self.axes)
cax = divider.append_axes("right", size="5%", pad=0.05)
self.drs= []
if self.visualization_rdb.GetSelection() == 0: #i.e. for color scheme for individuals
self.Colorscheme = visualization.get_cmap(len(self.individual_names),self.cfg['colormap'])
self.norm,self.colorIndex = self.image_panel.getColorIndices(im,self.individual_names)
cbar = self.figure.colorbar(self.ax, cax=cax,spacing='proportional', ticks=self.colorIndex)
cbar.set_ticklabels(self.individual_names)
else: #i.e. for color scheme for all bodyparts
self.Colorscheme = visualization.get_cmap(len(self.all_bodyparts),self.cfg['colormap'])
self.norm,self.colorIndex = self.image_panel.getColorIndices(im,self.all_bodyparts)
cbar = self.figure.colorbar(self.ax, cax=cax,spacing='proportional', ticks=self.colorIndex)
cbar.set_ticklabels(self.all_bodyparts)
for ci,ind in enumerate(self.individual_names):
col_idx = 0 #variable for iterating through the colorscheme for all bodyparts
image_points = []
if ind == 'single':
if self.visualization_rdb.GetSelection() == 0:
for c, bp in enumerate(self.uniquebodyparts):
self.points = [self.Dataframe[self.scorer][ind][bp]['x'].values[self.iter],self.Dataframe[self.scorer][ind][bp]['y'].values[self.iter],self.Dataframe[self.scorer][ind][bp]['likelihood'].values[self.iter]]
self.likelihood = self.points[2]
if self.likelihood < self.threshold:
self.circle = [patches.Circle((self.points[0], self.points[1]), radius=self.markerSize, facecolor = 'None', edgecolor = self.Colorscheme(ci) , alpha=self.alpha)]
else:
self.circle = [patches.Circle((self.points[0], self.points[1]), radius=self.markerSize, fc = self.Colorscheme(ci) , alpha=self.alpha)]
self.axes.add_patch(self.circle[0])
self.dr = auxfun_drag_multi_individuals.DraggablePoint(self.circle[0],ind,bp,self.likelihood)
self.dr.connect()
self.dr.coords = MainFrame.getLabels(self,self.iter,ind,self.uniquebodyparts)[c]
self.drs.append(self.dr)
self.updatedCoords.append(self.dr.coords)
else:
for c, bp in enumerate(self.uniquebodyparts):
self.points = [self.Dataframe[self.scorer][ind][bp]['x'].values[self.iter],self.Dataframe[self.scorer][ind][bp]['y'].values[self.iter],self.Dataframe[self.scorer][ind][bp]['likelihood'].values[self.iter]]
self.likelihood = self.points[2]
if self.likelihood < self.threshold:
self.circle = [patches.Circle((self.points[0], self.points[1]), radius=self.markerSize, fc = 'None', edgecolor = self.Colorscheme(col_idx) , alpha=self.alpha)]
else:
self.circle = [patches.Circle((self.points[0], self.points[1]), radius=self.markerSize, fc = self.Colorscheme(col_idx) , alpha=self.alpha)]
self.axes.add_patch(self.circle[0])
col_idx = col_idx + 1
self.dr = auxfun_drag_multi_individuals.DraggablePoint(self.circle[0],ind,bp,self.likelihood)
self.dr.connect()
self.dr.coords = MainFrame.getLabels(self,self.iter,ind,self.uniquebodyparts)[c]
self.drs.append(self.dr)
self.updatedCoords.append(self.dr.coords)
else:
if self.visualization_rdb.GetSelection() == 0:
for c, bp in enumerate(self.multianimalbodyparts):
self.points = [self.Dataframe[self.scorer][ind][bp]['x'].values[self.iter],self.Dataframe[self.scorer][ind][bp]['y'].values[self.iter],self.Dataframe[self.scorer][ind][bp]['likelihood'].values[self.iter]]
self.likelihood = self.points[2]
if self.likelihood < self.threshold:
self.circle = [patches.Circle((self.points[0], self.points[1]), radius=self.markerSize, fc = 'None', edgecolor= self.Colorscheme(ci) , alpha=self.alpha)]
else:
self.circle = [patches.Circle((self.points[0], self.points[1]), radius=self.markerSize, fc = self.Colorscheme(ci) , alpha=self.alpha)]
self.axes.add_patch(self.circle[0])
self.dr = auxfun_drag_multi_individuals.DraggablePoint(self.circle[0],ind,bp,self.likelihood)
self.dr.connect()
self.dr.coords = MainFrame.getLabels(self,self.iter,ind,self.multianimalbodyparts)[c]
self.drs.append(self.dr)
self.updatedCoords.append(self.dr.coords)
else:
for c, bp in enumerate(self.multianimalbodyparts):
self.points = [self.Dataframe[self.scorer][ind][bp]['x'].values[self.iter],self.Dataframe[self.scorer][ind][bp]['y'].values[self.iter],self.Dataframe[self.scorer][ind][bp]['likelihood'].values[self.iter]]
self.likelihood = self.points[2]
if self.likelihood < self.threshold:
self.circle = [patches.Circle((self.points[0], self.points[1]), radius=self.markerSize, fc = 'None', edgecolor = self.Colorscheme(col_idx) , alpha=self.alpha)]
else:
self.circle = [patches.Circle((self.points[0], self.points[1]), radius=self.markerSize, fc = self.Colorscheme(col_idx) , alpha=self.alpha)]
self.axes.add_patch(self.circle[0])
col_idx = col_idx + 1
self.dr = auxfun_drag_multi_individuals.DraggablePoint(self.circle[0],ind,bp,self.likelihood)
self.dr.connect()
self.dr.coords = MainFrame.getLabels(self,self.iter,ind,self.multianimalbodyparts)[c]
self.drs.append(self.dr)
self.updatedCoords.append(self.dr.coords)
self.figure.canvas.draw()
def evaluate_multianimal_crossvalidate(
config,
Shuffles=[1],
trainingsetindex=0,
pbounds=None,
edgewisecondition=True,
target="rpck_train",
inferencecfg=None,
init_points=20,
n_iter=50,
dcorr=10.0,
leastbpts=1,
printingintermediatevalues=True,
modelprefix="",
plotting=False,
):
"""
Crossvalidate inference parameters on evaluation data; optimal parametrs will be stored in " inference_cfg.yaml".
They will then be then used for inference (for analysis of videos). Performs Bayesian Optimization with https://github.com/fmfn/BayesianOptimization
This is a crucial step. The most important variable (in inferencecfg) to cross-validate is minimalnumberofconnections. Pass
a reasonable range to optimze (e.g. if you have 5 edges from 1 to 5. If you have 4 bpts and 11 connections from 3 to 9).
config: string
Full path of the config.yaml file as a string.
shuffle: int, optional
An integer specifying the shuffle index of the training dataset used for training the network. The default is 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
pbounds: dictionary of variables with ranges to crossvalidate.
By default: pbounds = {
'pafthreshold': (0.05, 0.7),
'detectionthresholdsquare': (0, 0.9),
'minimalnumberofconnections': (1, # connections in your skeleton),
}
inferencecfg: dict, OPTIONAL
For the variables that are *not* crossvalidated the parameters from inference_cfg.yaml are used, or
you can overwrite them by passing a dictinary with your preferred parameters.
edgewisecondition: bool, default True
Estimates Euclidean distances for each skeleton edge and uses those distance for excluding possible connections.
If false, uses only one distance for all bodyparts (which is obviously suboptimal).
target: string, default='rpck_train'
What metric to optimize. Options are pck/rpck/rmse on train/test set.
init_points: int, optional (default=10)
Number of random initial explorations. Probing random regions helps diversify the exploration space.
Parameter from BayesianOptimization.
n_iter: int, optional (default=20)
Number of iterations of Bayesian optimization to perform.
The larger it is, the higher the likelihood of finding a good extremum.
Parameter from BayesianOptimization.
dcorr: float,
Distance thereshold for percent correct keypoints / relative percent correct keypoints (see paper).
leastbpts: integer (should be a small number)
If an animals has less or equal as many body parts in an image it will not be used
for cross validation. Imagine e.g. if only a single bodypart is present, then
if animals need a certain minimal number of bodyparts for assembly (minimalnumberofconnections),
this might not be predictable.
printingintermediatevalues: bool, default True
If intermediate metrics RMSE/hits/.. per sample should be printed.
Examples
--------
first run evalute:
deeplabcut.evaluate_network(path_config_file,Shuffles=[shuffle],plotting=True)
Then e.g. for finding inference parameters to minimize rmse on test set:
deeplabcut.evaluate_multianimal_crossvalidate(path_config_file,Shuffles=[shuffle],target='rmse_test')
"""
from deeplabcut.pose_estimation_tensorflow.lib import crossvalutils
from deeplabcut.utils import auxfun_multianimal, auxiliaryfunctions
from easydict import EasyDict as edict
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
)
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, "all")
colors = visualization.get_cmap(len(comparisonbodyparts),
name=cfg["colormap"])
# wild guesses for a wide range:
maxconnections = len(cfg["skeleton"])
minconnections = 1 # len(cfg['multianimalbodyparts'])-1
_pbounds = {
"pafthreshold": (0.05, 0.7),
"detectionthresholdsquare": (
0,
0.9,
), # TODO: set to minimum (from pose_cfg.yaml)
"minimalnumberofconnections": (minconnections, maxconnections),
}
if pbounds is not None:
_pbounds.update(pbounds)
if "rpck" in target or "pck" in target:
maximize = True
if "rmse" in target:
maximize = False # i.e. minimize
for shuffle in Shuffles:
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
_, trainIndices, testIndices, _ = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
])
snapindex = -1
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train",
Snapshots[snapindex]) # setting weights to corresponding snapshot.
trainingsiterations = (dlc_cfg["init_weights"].split(
os.sep)[-1]).split("-")[
-1] # read how many training siterations that corresponds to.
DLCscorer, _ = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix)
path_inference_config = Path(
modelfolder) / "test" / "inference_cfg.yaml"
if inferencecfg is None: # then load or initialize
inferencecfg = auxfun_multianimal.read_inferencecfg(
path_inference_config, cfg)
else:
inferencecfg = edict(inferencecfg)
auxfun_multianimal.check_inferencecfg_sanity(cfg, inferencecfg)
inferencecfg.topktoretain = np.inf
inferencecfg, opt = crossvalutils.bayesian_search(
config,
inferencecfg,
_pbounds,
edgewisecondition=edgewisecondition,
shuffle=shuffle,
trainingsetindex=trainingsetindex,
target=target,
maximize=maximize,
init_points=init_points,
n_iter=n_iter,
acq="ei",
dcorr=dcorr,
leastbpts=leastbpts,
modelprefix=modelprefix,
)
# update number of individuals to retain.
inferencecfg.topktoretain = len(
cfg["individuals"]) + 1 * (len(cfg["uniquebodyparts"]) > 0)
# calculating result at best best solution
DataOptParams, poses_gt, poses = crossvalutils.compute_crossval_metrics(
config, inferencecfg, shuffle, trainingsetindex, modelprefix)
path_inference_config = str(path_inference_config)
# print("Quantification:", DataOptParams.head())
DataOptParams.to_hdf(
path_inference_config.split(".yaml")[0] + ".h5",
"df_with_missing",
format="table",
mode="w",
)
DataOptParams.to_csv(path_inference_config.split(".yaml")[0] + ".csv")
print("Saving optimal inference parameters...")
print(DataOptParams.to_string())
auxiliaryfunctions.write_plainconfig(path_inference_config,
dict(inferencecfg))
# Store best predictions
max_indivs = max(pose.shape[0] for pose in poses)
bpts = dlc_cfg["all_joints_names"]
container = np.full((len(poses), max_indivs * len(bpts) * 3), np.nan)
for n, pose in enumerate(poses):
temp = pose.flatten()
container[n, :len(temp)] = temp
header = pd.MultiIndex.from_product(
[
[DLCscorer],
[f"individual{i}" for i in range(1, max_indivs + 1)],
bpts,
["x", "y", "likelihood"],
],
names=["scorer", "individuals", "bodyparts", "coords"],
)
df = pd.DataFrame(container, columns=header)
df.to_hdf(os.path.join(evaluationfolder, f"{DLCscorer}.h5"),
key="df_with_missing")
if plotting:
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" + Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
for imageindex, imagename in tqdm(enumerate(Data.index)):
image_path = os.path.join(cfg["project_path"], imagename)
image = io.imread(image_path)
frame = img_as_ubyte(skimage.color.gray2rgb(image))
groundtruthcoordinates = poses_gt[imageindex]
coords_pred = poses[imageindex][:, :, :2]
probs_pred = poses[imageindex][:, :, -1:]
fig = visualization.make_multianimal_labeled_image(
frame,
groundtruthcoordinates,
coords_pred,
probs_pred,
colors,
cfg["dotsize"],
cfg["alphavalue"],
cfg["pcutoff"],
)
visualization.save_labeled_frame(fig, image_path, foldername,
imageindex in trainIndices)
def evaluate_network(config,
Shuffles=[1],
plotting=None,
show_errors=True,
comparisonbodyparts="all",
gputouse=None):
"""
Evaluates the network based on the saved models at different stages of the training network.\n
The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'.
Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
Shuffles: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
plotting: bool, optional
Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False``
show_errors: bool, optional
Display train and test errors. The default is `True``
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
Examples
--------
If you do not want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml', shuffle=[1])
--------
If you want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',shuffle=[1],True)
"""
import os
from skimage import io
import skimage.color
from deeplabcut.pose_estimation_tensorflow.nnet import predict as ptf_predict
from deeplabcut.pose_estimation_tensorflow.config import load_config
from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input
from deeplabcut.utils import auxiliaryfunctions, visualization
import tensorflow as tf
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ[
'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
vers = (tf.__version__).split('.')
if int(vers[0]) == 1 and int(vers[1]) > 12:
TF = tf.compat.v1
else:
TF = tf
TF.reset_default_graph()
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' #
# tf.logging.set_verbosity(tf.logging.WARN)
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: #gpu selectinon
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(cfg["project_path"], str(trainingsetfolder),
'CollectedData_' + cfg["scorer"] + '.h5'),
'df_with_missing')
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in cfg["TrainingFraction"]:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(trainFraction, shuffle,
cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
# Load meta data
data, trainIndices, testIndices, trainFraction = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
#change batch size, if it was edited during analysis!
dlc_cfg['batch_size'] = 1 #in case this was edited for analysis.
#Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg)))
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
#path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split('.')[0]
for fn in os.listdir(os.path.join(str(modelfolder), 'train'))
if "index" in fn
])
try: #check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
increasing_indices = np.argsort(
[int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
final_result = []
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg['init_weights'] = os.path.join(
str(modelfolder), 'train', Snapshots[snapindex]
) #setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg['init_weights'].split(os.sep)[-1]
).split(
'-'
)[-1] #read how many training siterations that corresponds to.
#name for deeplabcut net (based on its parameters)
DLCscorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, trainingsiterations)
print("Running ", DLCscorer, " with # of trainingiterations:",
trainingsiterations)
resultsfilename = os.path.join(
str(evaluationfolder),
DLCscorer + '-' + Snapshots[snapindex] + '.h5')
try:
DataMachine = pd.read_hdf(resultsfilename,
'df_with_missing')
print("This net has already been evaluated!")
except FileNotFoundError:
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = ptf_predict.setup_pose_prediction(
dlc_cfg)
Numimages = len(Data.index)
PredicteData = np.zeros(
(Numimages, 3 * len(dlc_cfg['all_joints_names'])))
print("Analyzing data...")
for imageindex, imagename in tqdm(enumerate(Data.index)):
image = io.imread(os.path.join(cfg['project_path'],
imagename),
mode='RGB')
image = skimage.color.gray2rgb(image)
image_batch = data_to_input(image)
# Compute prediction with the CNN
outputs_np = sess.run(outputs,
feed_dict={inputs: image_batch})
scmap, locref = ptf_predict.extract_cnn_output(
outputs_np, dlc_cfg)
# Extract maximum scoring location from the heatmap, assume 1 person
pose = ptf_predict.argmax_pose_predict(
scmap, locref, dlc_cfg.stride)
PredicteData[imageindex, :] = pose.flatten(
) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts!
sess.close() #closes the current tf session
index = pd.MultiIndex.from_product(
[[DLCscorer], dlc_cfg['all_joints_names'],
['x', 'y', 'likelihood']],
names=['scorer', 'bodyparts', 'coords'])
# Saving results
DataMachine = pd.DataFrame(PredicteData,
columns=index,
index=Data.index.values)
DataMachine.to_hdf(resultsfilename,
'df_with_missing',
format='table',
mode='w')
print("Done and results stored for snapshot: ",
Snapshots[snapindex])
DataCombined = pd.concat([Data.T, DataMachine.T], axis=0).T
RMSE, RMSEpcutoff = pairwisedistances(
DataCombined, cfg["scorer"], DLCscorer, cfg["pcutoff"],
comparisonbodyparts)
testerror = np.nanmean(
RMSE.iloc[testIndices].values.flatten())
trainerror = np.nanmean(
RMSE.iloc[trainIndices].values.flatten())
testerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[testIndices].values.flatten())
trainerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[trainIndices].values.flatten())
results = [
trainingsiterations,
int(100 * trainFraction), shuffle,
np.round(trainerror, 2),
np.round(testerror, 2), cfg["pcutoff"],
np.round(trainerrorpcutoff, 2),
np.round(testerrorpcutoff, 2)
]
final_result.append(results)
if show_errors == True:
print("Results for",
trainingsiterations, " training iterations:",
int(100 * trainFraction), shuffle,
"train error:",
np.round(trainerror, 2), "pixels. Test error:",
np.round(testerror, 2), " pixels.")
print("With pcutoff of",
cfg["pcutoff"], " train error:",
np.round(trainerrorpcutoff,
2), "pixels. Test error:",
np.round(testerrorpcutoff, 2), "pixels")
print(
"Thereby, the errors are given by the average distances between the labels by DLC and the scorer."
)
if plotting == True:
print("Plotting...")
colors = visualization.get_cmap(
len(comparisonbodyparts), name=cfg['colormap'])
foldername = os.path.join(
str(evaluationfolder), 'LabeledImages_' +
DLCscorer + '_' + Snapshots[snapindex])
auxiliaryfunctions.attempttomakefolder(foldername)
NumFrames = np.size(DataCombined.index)
for ind in np.arange(NumFrames):
visualization.PlottingandSaveLabeledFrame(
DataCombined, ind, trainIndices, cfg, colors,
comparisonbodyparts, DLCscorer, foldername)
TF.reset_default_graph()
#print(final_result)
make_results_file(final_result, evaluationfolder, DLCscorer)
print(
"The network is evaluated and the results are stored in the subdirectory 'evaluation_results'."
)
print(
"If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos."
)
print(
"Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)"
)
#returning to intial folder
os.chdir(str(start_path))
def __init__(self, parent, config, video, shuffle, Dataframe, savelabeled,
multianimal):
# Settting the GUI size and panels design
displays = (wx.Display(i) for i in range(wx.Display.GetCount())
) # Gets the number of displays
screenSizes = [
display.GetGeometry().GetSize() for display in displays
] # Gets the size of each display
index = 0 # For display 1.
screenWidth = screenSizes[index][0]
screenHeight = screenSizes[index][1]
self.gui_size = (screenWidth * 0.7, screenHeight * 0.85)
wx.Frame.__init__(
self,
parent,
id=wx.ID_ANY,
title="DeepLabCut2.0 - Manual Outlier Frame Extraction",
size=wx.Size(self.gui_size),
pos=wx.DefaultPosition,
style=wx.RESIZE_BORDER | wx.DEFAULT_FRAME_STYLE | wx.TAB_TRAVERSAL,
)
self.statusbar = self.CreateStatusBar()
self.statusbar.SetStatusText("")
self.SetSizeHints(
wx.Size(self.gui_size)
) # This sets the minimum size of the GUI. It can scale now!
###################################################################################################################################################
# Spliting the frame into top and bottom panels. Bottom panels contains the widgets. The top panel is for showing images and plotting!
# topSplitter = wx.SplitterWindow(self)
#
# self.image_panel = ImagePanel(topSplitter, config,video,shuffle,Dataframe,self.gui_size)
# self.widget_panel = WidgetPanel(topSplitter)
#
# topSplitter.SplitHorizontally(self.image_panel, self.widget_panel,sashPosition=self.gui_size[1]*0.83)#0.9
# topSplitter.SetSashGravity(1)
# sizer = wx.BoxSizer(wx.VERTICAL)
# sizer.Add(topSplitter, 1, wx.EXPAND)
# self.SetSizer(sizer)
# Spliting the frame into top and bottom panels. Bottom panels contains the widgets. The top panel is for showing images and plotting!
topSplitter = wx.SplitterWindow(self)
vSplitter = wx.SplitterWindow(topSplitter)
self.image_panel = ImagePanel(vSplitter, self.gui_size)
self.choice_panel = ScrollPanel(vSplitter)
vSplitter.SplitVertically(self.image_panel,
self.choice_panel,
sashPosition=self.gui_size[0] * 0.8)
vSplitter.SetSashGravity(1)
self.widget_panel = WidgetPanel(topSplitter)
topSplitter.SplitHorizontally(vSplitter,
self.widget_panel,
sashPosition=self.gui_size[1] *
0.83) # 0.9
topSplitter.SetSashGravity(1)
sizer = wx.BoxSizer(wx.VERTICAL)
sizer.Add(topSplitter, 1, wx.EXPAND)
self.SetSizer(sizer)
###################################################################################################################################################
# Add Buttons to the WidgetPanel and bind them to their respective functions.
widgetsizer = wx.WrapSizer(orient=wx.HORIZONTAL)
self.load_button_sizer = wx.BoxSizer(wx.VERTICAL)
self.help_button_sizer = wx.BoxSizer(wx.VERTICAL)
self.help = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Help")
self.help_button_sizer.Add(self.help, 1, wx.ALL, 15)
# widgetsizer.Add(self.help , 1, wx.ALL, 15)
self.help.Bind(wx.EVT_BUTTON, self.helpButton)
widgetsizer.Add(self.help_button_sizer, 1, wx.ALL, 0)
self.grab = wx.Button(self.widget_panel,
id=wx.ID_ANY,
label="Grab Frames")
widgetsizer.Add(self.grab, 1, wx.ALL, 15)
self.grab.Bind(wx.EVT_BUTTON, self.grabFrame)
self.grab.Enable(True)
widgetsizer.AddStretchSpacer(5)
self.slider = wx.Slider(
self.widget_panel,
id=wx.ID_ANY,
value=0,
minValue=0,
maxValue=1,
size=(200, -1),
style=wx.SL_HORIZONTAL | wx.SL_AUTOTICKS | wx.SL_LABELS,
)
widgetsizer.Add(self.slider, 1, wx.ALL, 5)
self.slider.Bind(wx.EVT_SLIDER, self.OnSliderScroll)
widgetsizer.AddStretchSpacer(5)
self.start_frames_sizer = wx.BoxSizer(wx.VERTICAL)
self.end_frames_sizer = wx.BoxSizer(wx.VERTICAL)
self.start_frames_sizer.AddSpacer(15)
# self.startFrame = wx.SpinCtrl(self.widget_panel, value='0', size=(100, -1), min=0, max=120)
self.startFrame = wx.SpinCtrl(self.widget_panel,
value="0",
size=(100, -1)) # ,style=wx.SP_VERTICAL)
self.startFrame.Enable(False)
self.start_frames_sizer.Add(self.startFrame, 1,
wx.EXPAND | wx.ALIGN_LEFT, 15)
start_text = wx.StaticText(self.widget_panel,
label="Start Frame Index")
self.start_frames_sizer.Add(start_text, 1, wx.EXPAND | wx.ALIGN_LEFT,
15)
self.checkBox = wx.CheckBox(self.widget_panel,
id=wx.ID_ANY,
label="Range of frames")
self.checkBox.Bind(wx.EVT_CHECKBOX, self.activate_frame_range)
self.start_frames_sizer.Add(self.checkBox, 1,
wx.EXPAND | wx.ALIGN_LEFT, 15)
#
self.end_frames_sizer.AddSpacer(15)
self.endFrame = wx.SpinCtrl(self.widget_panel,
value="1",
size=(160, -1)) # , min=1, max=120)
self.endFrame.Enable(False)
self.end_frames_sizer.Add(self.endFrame, 1, wx.EXPAND | wx.ALIGN_LEFT,
15)
end_text = wx.StaticText(self.widget_panel, label="Number of Frames")
self.end_frames_sizer.Add(end_text, 1, wx.EXPAND | wx.ALIGN_LEFT, 15)
self.updateFrame = wx.Button(self.widget_panel,
id=wx.ID_ANY,
label="Update")
self.end_frames_sizer.Add(self.updateFrame, 1,
wx.EXPAND | wx.ALIGN_LEFT, 15)
self.updateFrame.Bind(wx.EVT_BUTTON, self.updateSlider)
self.updateFrame.Enable(False)
widgetsizer.Add(self.start_frames_sizer, 1, wx.ALL, 0)
widgetsizer.AddStretchSpacer(5)
widgetsizer.Add(self.end_frames_sizer, 1, wx.ALL, 0)
widgetsizer.AddStretchSpacer(15)
self.quit = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Quit")
widgetsizer.Add(self.quit, 1, wx.ALL, 15)
self.quit.Bind(wx.EVT_BUTTON, self.quitButton)
self.quit.Enable(True)
self.widget_panel.SetSizer(widgetsizer)
self.widget_panel.SetSizerAndFit(widgetsizer)
# Variables initialization
self.numberFrames = 0
self.currFrame = 0
self.figure = Figure()
self.axes = self.figure.add_subplot(111)
self.drs = []
self.extract_range_frame = False
self.firstFrame = 0
self.Colorscheme = []
# self.cropping = False
# Read confing file
self.cfg = auxiliaryfunctions.read_config(config)
self.Task = self.cfg["Task"]
self.start = self.cfg["start"]
self.stop = self.cfg["stop"]
self.date = self.cfg["date"]
self.trainFraction = self.cfg["TrainingFraction"]
self.trainFraction = self.trainFraction[0]
self.videos = self.cfg["video_sets"].keys()
self.bodyparts = self.cfg["bodyparts"]
self.colormap = plt.get_cmap(self.cfg["colormap"])
self.colormap = self.colormap.reversed()
self.markerSize = self.cfg["dotsize"]
self.alpha = self.cfg["alphavalue"]
self.iterationindex = self.cfg["iteration"]
self.cropping = self.cfg["cropping"]
self.video_names = [Path(i).stem for i in self.videos]
self.config_path = Path(config)
self.video_source = Path(video).resolve()
self.shuffle = shuffle
self.Dataframe = Dataframe
self.savelabeled = savelabeled
self.multianimal = multianimal
if self.multianimal:
from deeplabcut.utils import auxfun_multianimal
(
self.individual_names,
self.uniquebodyparts,
self.multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(self.cfg)
self.choiceBox, self.visualization_rdb = self.choice_panel.addRadioButtons(
)
self.Colorscheme = visualization.get_cmap(
len(self.individual_names), self.cfg["colormap"])
self.visualization_rdb.Bind(wx.EVT_RADIOBOX, self.clear_plot)
# Read the video file
self.vid = cv2.VideoCapture(str(self.video_source))
self.videoPath = os.path.dirname(self.video_source)
self.filename = Path(self.video_source).name
self.numberFrames = int(self.vid.get(cv2.CAP_PROP_FRAME_COUNT))
self.strwidth = int(np.ceil(np.log10(self.numberFrames)))
# Set the values of slider and range of frames
self.startFrame.SetMax(self.numberFrames - 1)
self.slider.SetMax(self.numberFrames - 1)
self.endFrame.SetMax(self.numberFrames - 1)
self.startFrame.Bind(wx.EVT_SPINCTRL, self.updateSlider) # wx.EVT_SPIN
# Set the status bar
self.statusbar.SetStatusText("Working on video: {}".format(
os.path.split(str(self.video_source))[-1]))
# Adding the video file to the config file.
if not (str(self.video_source.stem) in self.video_names):
add.add_new_videos(self.config_path, [self.video_source])
self.filename = Path(self.video_source).name
self.update()
self.plot_labels()
self.widget_panel.Layout()
def plot(self, im):
"""
Plots and call auxfun_drag class for moving and removing points.
"""
# small hack in case there are any 0 intensity images!
img = io.imread(im)
maxIntensity = np.max(img)
if maxIntensity == 0:
maxIntensity = np.max(img) + 255
divider = make_axes_locatable(self.axes)
cax = divider.append_axes("right", size="5%", pad=0.05)
self.drs = []
if (self.visualization_rdb.GetSelection() == 0
): # i.e. for color scheme for individuals
self.Colorscheme = visualization.get_cmap(
len(self.individual_names), self.cfg["colormap"])
self.norm, self.colorIndex = self.image_panel.getColorIndices(
im, self.individual_names)
cbar = self.figure.colorbar(self.ax,
cax=cax,
spacing="proportional",
ticks=self.colorIndex)
cbar.set_ticklabels(self.individual_names)
else: # i.e. for color scheme for all bodyparts
self.Colorscheme = visualization.get_cmap(len(self.all_bodyparts),
self.cfg["colormap"])
self.norm, self.colorIndex = self.image_panel.getColorIndices(
im, self.all_bodyparts)
cbar = self.figure.colorbar(self.ax,
cax=cax,
spacing="proportional",
ticks=self.colorIndex)
cbar.set_ticklabels(self.all_bodyparts)
for ci, ind in enumerate(self.individual_names):
col_idx = (
0 # variable for iterating through the colorscheme for all bodyparts
)
image_points = []
if ind == "single":
if self.visualization_rdb.GetSelection() == 0:
for c, bp in enumerate(self.uniquebodyparts):
self.points = [
self.Dataframe[self.scorer][ind][bp]["x"].values[
self.iter],
self.Dataframe[self.scorer][ind][bp]["y"].values[
self.iter],
self.Dataframe[self.scorer][ind][bp]
["likelihood"].values[self.iter],
]
self.likelihood = self.points[2]
# fix move to corner
if self.move2corner:
ny, nx = np.shape(img)[0], np.shape(img)[1]
if self.points[0] > nx or self.points[0] < 0:
print("fixing x for ", bp)
self.points[0] = self.center[0]
if self.points[1] > ny or self.points[1] < 0:
print("fixing y for ", bp)
self.points[1] = self.center[1]
if self.likelihood < self.threshold:
self.circle = [
patches.Circle(
(self.points[0], self.points[1]),
radius=self.markerSize,
facecolor="None",
edgecolor=self.Colorscheme(ci),
alpha=self.alpha,
)
]
else:
self.circle = [
patches.Circle(
(self.points[0], self.points[1]),
radius=self.markerSize,
fc=self.Colorscheme(ci),
alpha=self.alpha,
)
]
self.axes.add_patch(self.circle[0])
self.dr = auxfun_drag.DraggablePoint(
self.circle[0],
bp,
individual_names=ind,
likelihood=self.likelihood,
)
self.dr.connect()
self.dr.coords = MainFrame.getLabels(
self, self.iter, ind, self.uniquebodyparts)[c]
self.drs.append(self.dr)
self.updatedCoords.append(self.dr.coords)
else:
for c, bp in enumerate(self.uniquebodyparts):
self.points = [
self.Dataframe[self.scorer][ind][bp]["x"].values[
self.iter],
self.Dataframe[self.scorer][ind][bp]["y"].values[
self.iter],
self.Dataframe[self.scorer][ind][bp]
["likelihood"].values[self.iter],
]
self.likelihood = self.points[2]
# fix move to corner
if self.move2corner:
ny, nx = np.shape(img)[0], np.shape(img)[1]
if self.points[0] > nx or self.points[0] < 0:
print("fixing x for ", bp)
self.points[0] = self.center[0]
if self.points[1] > ny or self.points[1] < 0:
print("fixing y for ", bp)
self.points[1] = self.center[1]
if self.likelihood < self.threshold:
self.circle = [
patches.Circle(
(self.points[0], self.points[1]),
radius=self.markerSize,
fc="None",
edgecolor=self.Colorscheme(col_idx),
alpha=self.alpha,
)
]
else:
self.circle = [
patches.Circle(
(self.points[0], self.points[1]),
radius=self.markerSize,
fc=self.Colorscheme(col_idx),
alpha=self.alpha,
)
]
self.axes.add_patch(self.circle[0])
col_idx = col_idx + 1
self.dr = auxfun_drag.DraggablePoint(
self.circle[0],
bp,
individual_names=ind,
likelihood=self.likelihood,
)
self.dr.connect()
self.dr.coords = MainFrame.getLabels(
self, self.iter, ind, self.uniquebodyparts)[c]
self.drs.append(self.dr)
self.updatedCoords.append(self.dr.coords)
else:
if self.visualization_rdb.GetSelection() == 0:
for c, bp in enumerate(self.multianimalbodyparts):
self.points = [
self.Dataframe[self.scorer][ind][bp]["x"].values[
self.iter],
self.Dataframe[self.scorer][ind][bp]["y"].values[
self.iter],
self.Dataframe[self.scorer][ind][bp]
["likelihood"].values[self.iter],
]
self.likelihood = self.points[2]
# fix move to corner
if self.move2corner:
ny, nx = np.shape(img)[0], np.shape(img)[1]
if self.points[0] > nx or self.points[0] < 0:
print("fixing x for ", bp)
self.points[0] = self.center[0]
if self.points[1] > ny or self.points[1] < 0:
print("fixing y for ", bp)
self.points[1] = self.center[1]
if self.likelihood < self.threshold:
self.circle = [
patches.Circle(
(self.points[0], self.points[1]),
radius=self.markerSize,
fc="None",
edgecolor=self.Colorscheme(ci),
alpha=self.alpha,
)
]
else:
self.circle = [
patches.Circle(
(self.points[0], self.points[1]),
radius=self.markerSize,
fc=self.Colorscheme(ci),
alpha=self.alpha,
)
]
self.axes.add_patch(self.circle[0])
self.dr = auxfun_drag.DraggablePoint(
self.circle[0],
bp,
individual_names=ind,
likelihood=self.likelihood,
)
self.dr.connect()
self.dr.coords = MainFrame.getLabels(
self, self.iter, ind, self.multianimalbodyparts)[c]
self.drs.append(self.dr)
self.updatedCoords.append(self.dr.coords)
else:
for c, bp in enumerate(self.multianimalbodyparts):
self.points = [
self.Dataframe[self.scorer][ind][bp]["x"].values[
self.iter],
self.Dataframe[self.scorer][ind][bp]["y"].values[
self.iter],
self.Dataframe[self.scorer][ind][bp]
["likelihood"].values[self.iter],
]
self.likelihood = self.points[2]
# fix move to corner
if self.move2corner:
ny, nx = np.shape(img)[0], np.shape(img)[1]
if self.points[0] > nx or self.points[0] < 0:
print("fixing x for ", bp)
self.points[0] = self.center[0]
if self.points[1] > ny or self.points[1] < 0:
print("fixing y for ", bp)
self.points[1] = self.center[1]
if self.likelihood < self.threshold:
self.circle = [
patches.Circle(
(self.points[0], self.points[1]),
radius=self.markerSize,
fc="None",
edgecolor=self.Colorscheme(col_idx),
alpha=self.alpha,
)
]
else:
self.circle = [
patches.Circle(
(self.points[0], self.points[1]),
radius=self.markerSize,
fc=self.Colorscheme(col_idx),
alpha=self.alpha,
)
]
self.axes.add_patch(self.circle[0])
col_idx = col_idx + 1
self.dr = auxfun_drag.DraggablePoint(
self.circle[0],
bp,
individual_names=ind,
likelihood=self.likelihood,
)
self.dr.connect()
self.dr.coords = MainFrame.getLabels(
self, self.iter, ind, self.multianimalbodyparts)[c]
self.drs.append(self.dr)
self.updatedCoords.append(self.dr.coords)
self.figure.canvas.draw()
def __init__(self, parent, config):
super(MainFrame, self).__init__(
"DeepLabCut - Refinement ToolBox",
parent,
)
self.Bind(wx.EVT_CHAR_HOOK, self.OnKeyPressed)
###################################################################################################################################################
# Spliting the frame into top and bottom panels. Bottom panels contains the widgets. The top panel is for showing images and plotting!
topSplitter = wx.SplitterWindow(self)
vSplitter = wx.SplitterWindow(topSplitter)
self.image_panel = ImagePanel(vSplitter, config, self.gui_size)
self.choice_panel = ScrollPanel(vSplitter)
# self.choice_panel.SetupScrolling(scroll_x=True, scroll_y=True, scrollToTop=False)
# self.choice_panel.SetupScrolling(scroll_x=True, scrollToTop=False)
vSplitter.SplitVertically(self.image_panel,
self.choice_panel,
sashPosition=self.gui_size[0] * 0.8)
vSplitter.SetSashGravity(1)
self.widget_panel = WidgetPanel(topSplitter)
topSplitter.SplitHorizontally(vSplitter,
self.widget_panel,
sashPosition=self.gui_size[1] *
0.83) # 0.9
topSplitter.SetSashGravity(1)
sizer = wx.BoxSizer(wx.VERTICAL)
sizer.Add(topSplitter, 1, wx.EXPAND)
self.SetSizer(sizer)
###################################################################################################################################################
# Add Buttons to the WidgetPanel and bind them to their respective functions.
widgetsizer = wx.WrapSizer(orient=wx.HORIZONTAL)
self.load = wx.Button(self.widget_panel,
id=wx.ID_ANY,
label="Load labels")
widgetsizer.Add(self.load, 1, wx.ALL, 15)
self.load.Bind(wx.EVT_BUTTON, self.browseDir)
self.prev = wx.Button(self.widget_panel,
id=wx.ID_ANY,
label="<<Previous")
widgetsizer.Add(self.prev, 1, wx.ALL, 15)
self.prev.Bind(wx.EVT_BUTTON, self.prevImage)
self.prev.Enable(False)
self.next = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Next>>")
widgetsizer.Add(self.next, 1, wx.ALL, 15)
self.next.Bind(wx.EVT_BUTTON, self.nextImage)
self.next.Enable(False)
self.help = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Help")
widgetsizer.Add(self.help, 1, wx.ALL, 15)
self.help.Bind(wx.EVT_BUTTON, self.helpButton)
self.help.Enable(True)
self.zoom = wx.ToggleButton(self.widget_panel, label="Zoom")
widgetsizer.Add(self.zoom, 1, wx.ALL, 15)
self.zoom.Bind(wx.EVT_TOGGLEBUTTON, self.zoomButton)
self.widget_panel.SetSizer(widgetsizer)
self.zoom.Enable(False)
self.home = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Home")
widgetsizer.Add(self.home, 1, wx.ALL, 15)
self.home.Bind(wx.EVT_BUTTON, self.homeButton)
self.widget_panel.SetSizer(widgetsizer)
self.home.Enable(False)
self.pan = wx.ToggleButton(self.widget_panel,
id=wx.ID_ANY,
label="Pan")
widgetsizer.Add(self.pan, 1, wx.ALL, 15)
self.pan.Bind(wx.EVT_TOGGLEBUTTON, self.panButton)
self.widget_panel.SetSizer(widgetsizer)
self.pan.Enable(False)
self.lock = wx.CheckBox(self.widget_panel,
id=wx.ID_ANY,
label="Lock View")
widgetsizer.Add(self.lock, 1, wx.ALL, 15)
self.lock.Bind(wx.EVT_CHECKBOX, self.lockChecked)
self.widget_panel.SetSizer(widgetsizer)
self.lock.Enable(False)
self.save = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Save")
widgetsizer.Add(self.save, 1, wx.ALL, 15)
self.save.Bind(wx.EVT_BUTTON, self.saveDataSet)
self.save.Enable(False)
widgetsizer.AddStretchSpacer(15)
self.quit = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Quit")
widgetsizer.Add(self.quit, 1, wx.ALL, 15)
self.quit.Bind(wx.EVT_BUTTON, self.quitButton)
self.widget_panel.SetSizer(widgetsizer)
self.widget_panel.SetSizerAndFit(widgetsizer)
self.widget_panel.Layout()
###############################################################################################################################
# Variable initialization
self.currentDirectory = os.getcwd()
self.index = []
self.iter = []
self.threshold = []
self.file = 0
self.updatedCoords = []
self.drs = []
self.cfg = auxiliaryfunctions.read_config(config)
self.humanscorer = self.cfg["scorer"]
self.move2corner = self.cfg["move2corner"]
self.center = self.cfg["corner2move2"]
self.colormap = plt.get_cmap(self.cfg["colormap"])
self.colormap = self.colormap.reversed()
self.markerSize = self.cfg["dotsize"]
self.alpha = self.cfg["alphavalue"]
self.iterationindex = self.cfg["iteration"]
self.project_path = self.cfg["project_path"]
self.bodyparts = self.cfg["bodyparts"]
self.threshold = 0.1
self.img_size = (10, 6) # (imgW, imgH) # width, height in inches.
self.preview = False
self.view_locked = False
# Workaround for MAC - xlim and ylim changed events seem to be triggered too often so need to make sure that the
# xlim and ylim have actually changed before turning zoom off
self.prezoom_xlim = []
self.prezoom_ylim = []
from deeplabcut.utils import auxfun_multianimal
(
self.individual_names,
self.uniquebodyparts,
self.multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(self.cfg)
# self.choiceBox,self.visualization_rdb = self.choice_panel.addRadioButtons()
self.Colorscheme = visualization.get_cmap(len(self.individual_names),
self.cfg["colormap"])
def evaluate_multiview_network(config,videos,projection_matrices,multiview_step,snapshot_index=None,Shuffles=[1],plotting = None,show_errors = True,comparisonbodyparts="all",gputouse=None):
"""
Evaluates the network based on the saved models at different stages of the training network.\n
The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'.
Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos: list of strings
Name of each video, one per viewpoint. Must be in the same order that it was in for training
projection_matrices: list of arrays
Projection matrix for each viewpoint. Each is a 3x4 array
multiview_step:
1 or 2. Indicates whether network was trained with train_multiview_network_step_1 or train_multiview_network_step_2
Shuffles: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
plotting: bool, optional
Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False``
show_errors: bool, optional
Display train and test errors. The default is `True``
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
Examples
--------
If you do not want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml', shuffle=[1])
--------
If you want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',shuffle=[1],True)
"""
import os
from skimage import io
import skimage.color
from deeplabcut.pose_estimation_tensorflow.nnet import predict as ptf_predict
from deeplabcut.pose_estimation_tensorflow.config import load_config
from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input
from deeplabcut.utils import auxiliaryfunctions, visualization
import tensorflow as tf
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ['TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
tf.reset_default_graph()
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' #
# tf.logging.set_verbosity(tf.logging.WARN)
start_path=os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: #gpu selectinon
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
# Loading human annotatated data
trainingsetfolder=auxiliaryfunctions.GetTrainingSetFolder(cfg)
Datas = [pd.read_hdf(os.path.join(cfg['project_path'], 'labeled-data', video, 'CollectedData_'+cfg['scorer']+'.h5'), 'df_with_missing') for video in videos]
# Get list of body parts to evaluate network for
comparisonbodyparts=auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(cfg,comparisonbodyparts)
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(str(cfg["project_path"]+"/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in cfg["TrainingFraction"]:
##################################################
# Load and setup CNN part detector
##################################################
datafn,metadatafn=auxiliaryfunctions.GetDataandMetaDataFilenames(trainingsetfolder,trainFraction,shuffle,cfg)
modelfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
# Load meta data
metadatas = []
for video in videos:
m = ('-'+video).join(os.path.splitext(metadatafn))
data, trainIndices, testIndices, trainFraction=auxiliaryfunctions.LoadMetadata(os.path.join(cfg["project_path"],m))
metadatas.append(data)
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError("It seems the model for shuffle %s and trainFraction %s does not exist."%(shuffle,trainFraction))
#change batch size, if it was edited during analysis!
dlc_cfg['batch_size']=1 #in case this was edited for analysis.
#Create folder structure to store results.
evaluationfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetEvaluationFolder(trainFraction,shuffle,cfg)))
auxiliaryfunctions.attempttomakefolder(evaluationfolder,recursive=True)
#path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
dlc_cfg.multiview_step = multiview_step
dlc_cfg.projection_matrices = projection_matrices
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([fn.split('.')[0]for fn in os.listdir(os.path.join(str(modelfolder), 'train'))if "index" in fn])
try: #check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError("Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."%(shuffle,trainFraction))
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if snapshot_index is not None:
snapindices = [i for i in range(len(Snapshots)) if int(Snapshots[i].split('-')[1].split('.')[0])==snapshot_index]
elif cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"]<len(Snapshots):
snapindices=[cfg["snapshotindex"]]
else:
print("Invalid choice, only -1 (last), any integer up to last, or all (as string)!")
final_result=[]
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg['init_weights'] = os.path.join(str(modelfolder),'train',Snapshots[snapindex]) #setting weights to corresponding snapshot.
trainingsiterations = (dlc_cfg['init_weights'].split(os.sep)[-1]).split('-')[-1] #read how many training siterations that corresponds to.
#name for deeplabcut net (based on its parameters)
DLCscorer = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations)
print("Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations)
resultsfilename=os.path.join(str(evaluationfolder),DLCscorer + '-' + Snapshots[snapindex]+ '.h5')
try:
DataMachine = pd.read_hdf(resultsfilename,'df_with_missing')
print("This net has already been evaluated!")
except FileNotFoundError:
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = ptf_predict.setup_pose_prediction(dlc_cfg)
Numimages = len(Datas[0].index)
PredicteDatas = np.zeros((Numimages,len(Datas), 3 * len(dlc_cfg['all_joints_names'])))
imagesizes = []
print("Analyzing data...")
if multiview_step == 1:
for imageindex in tqdm(range(len(Datas[0].index))):
imagenames = [Data.index[imageindex] for Data in Datas]
images = [io.imread(os.path.join(cfg['project_path'],imagename),mode='RGB') for imagename in imagenames]
images = [skimage.color.gray2rgb(image) for image in images]
image_batch = images
imagesizes.append([image.shape for image in images])
# Compute prediction with the CNN
outputs_np = sess.run(outputs, feed_dict={inputs: image_batch})
scmap, locref = ptf_predict.extract_cnn_output(outputs_np, dlc_cfg)
# Extract maximum scoring location from the heatmap, assume 1 person
pose = ptf_predict.argmax_pose_predict(scmap, locref, dlc_cfg.stride)
PredicteDatas[imageindex] = pose.reshape([pose.shape[0], -1]) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts!
sess.close() #closes the current tf session
index = pd.MultiIndex.from_product(
[[DLCscorer], dlc_cfg['all_joints_names'], ['x', 'y', 'likelihood']],
names=['scorer', 'bodyparts', 'coords'])
# Saving results
for i, video in enumerate(videos):
print('Evaluating 2D predictions on video %s'%video)
Data = Datas[i]
DataMachine = pd.DataFrame(PredicteDatas[:,i], columns=index, index=Data.index.values)
r = ('-'+video).join(os.path.splitext(resultsfilename))
DataMachine.to_hdf(r,'df_with_missing',format='table',mode='w')
print("Done and results stored for snapshot: ", Snapshots[snapindex])
DataCombined = pd.concat([Data.T, DataMachine.T], axis=0).T
RMSE,RMSEpcutoff = pairwisedistances(DataCombined, cfg["scorer"], DLCscorer,cfg["pcutoff"],comparisonbodyparts)
testerror = np.nanmean(RMSE.iloc[testIndices].values.flatten())
trainerror = np.nanmean(RMSE.iloc[trainIndices].values.flatten())
testerrorpcutoff = np.nanmean(RMSEpcutoff.iloc[testIndices].values.flatten())
trainerrorpcutoff = np.nanmean(RMSEpcutoff.iloc[trainIndices].values.flatten())
results = [trainingsiterations,int(100 * trainFraction),shuffle,np.round(trainerror,2),np.round(testerror,2),cfg["pcutoff"],np.round(trainerrorpcutoff,2), np.round(testerrorpcutoff,2)]
final_result.append(results)
if show_errors == True:
print("Results for",trainingsiterations," training iterations:", int(100 * trainFraction), shuffle, "train error:",np.round(trainerror,2), "pixels. Test error:", np.round(testerror,2)," pixels.")
print("With pcutoff of", cfg["pcutoff"]," train error:",np.round(trainerrorpcutoff,2), "pixels. Test error:", np.round(testerrorpcutoff,2), "pixels")
print("Thereby, the errors are given by the average distances between the labels by DLC and the scorer.")
if plotting == True:
print("Plotting...")
colors = visualization.get_cmap(len(comparisonbodyparts),name=cfg['colormap'])
foldername=os.path.join(str(evaluationfolder),'LabeledImages_' + DLCscorer + '_' + Snapshots[snapindex]+'_'+video)
auxiliaryfunctions.attempttomakefolder(foldername)
NumFrames=np.size(DataCombined.index)
for ind in np.arange(NumFrames):
visualization.PlottingandSaveLabeledFrame(DataCombined,ind,trainIndices,cfg,colors,comparisonbodyparts,DLCscorer,foldername)
# get predictions in homogeneous pixel coordinates
# pixel coordinates have (0,0) in the top-left, and the bottom-right coordinate is (h,w)
predictions = PredicteDatas.reshape(Numimages, len(Datas), len(dlc_cfg['all_joints_names']), 3)
scores = np.copy(predictions[:,:,:,2])
predictions[:,:,:,2] = 1.0 # homogeneous coordinates; (x,y,1). Top-left corner is (-width/2, -height/2, 1); Bottom-right corner is opposite. Shape is num_images x num_views x num_joints x 3
num_ims, num_views, num_joints, _ = predictions.shape
# get labels in homogeneous pixel coordinates
labels = np.array([Data.values.reshape(num_ims, num_joints, 2) for Data in Datas]) # num_views x num_ims x num_joints x (x,y)
labels = np.transpose(labels, [1, 2, 0, 3]) # num_ims x num_joints x num_views x (x,y)
labels = np.concatenate([labels, np.ones([num_ims, num_joints, num_views, 1])], axis=3)
# solve linear system to get labels in 3D
# helpful explanation of equation found on pg 5 here: https://hal.inria.fr/inria-00524401/PDF/Sturm-cvpr05.pdf
labs = labels.reshape([num_ims * num_joints, num_views, 3]).astype(np.float)
confidences = ~np.isnan(np.sum(labs, axis=2))
valid = np.sum(~np.isnan(np.sum(labs, axis=2)), axis=1) >= 2
labs[~confidences] = 0
labels3d = project_3d(projection_matrices, labs, confidences=confidences)
labels3d[~valid] = np.nan
labels3d = labels3d.reshape([num_ims, num_joints, 3])
# solve linear system to get 3D predictions
preds = np.transpose(predictions, [0, 2, 1, 3]) # num_ims x num_joints x num_views x 3
preds = preds.reshape([num_ims*num_joints, num_views, 3])
preds3d = project_3d(projection_matrices, preds)
preds3d = preds3d.reshape([num_ims, num_joints, 3])
# try it with confidence weighting
scores = np.transpose(scores, [0, 2, 1]) # num_images x num_joints x num_views
scores = np.reshape(scores, [num_ims*num_joints, num_views])
preds3d_weighted = project_3d(projection_matrices, preds, confidences=scores)
preds3d_weighted = preds3d_weighted.reshape([num_ims, num_joints, 3])
# try it with the pcutoff
scores2 = np.copy(scores)
scores2[scores2 < cfg["pcutoff"]] = 0
preds3d_weighted_cutoff = project_3d(projection_matrices, preds, confidences=scores2)
preds3d_weighted_cutoff = preds3d_weighted_cutoff.reshape([num_ims, num_joints, 3])
print("\n\n3D errors:")
RMSE_train = np.nanmean(np.nansum((preds3d[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5)
RMSE_test = np.nanmean(np.nansum((preds3d[testIndices] - labels3d[testIndices])**2, axis=2)**0.5)
RMSE_train_weighted = np.nanmean(np.nansum((preds3d_weighted[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5)
RMSE_test_weighted = np.nanmean(np.nansum((preds3d_weighted[testIndices] - labels3d[testIndices])**2, axis=2)**0.5)
RMSE_train_weighted_cutoff = np.nanmean(np.nansum((preds3d_weighted_cutoff[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5)
RMSE_test_weighted_cutoff = np.nanmean(np.nansum((preds3d_weighted_cutoff[testIndices] - labels3d[testIndices])**2, axis=2)**0.5)
print("RMSE train: ", RMSE_train)
print("RMSE test: ", RMSE_test)
print("RMSE train weighted: ", RMSE_train_weighted)
print("RMSE test weighted: ", RMSE_test_weighted)
print("RMSE train weighted cutoff: ", RMSE_train_weighted_cutoff)
print("RMSE test weighted cutoff: ", RMSE_test_weighted_cutoff)
tail = np.nansum((preds3d_weighted - labels3d)**2, axis=2)**0.5
tail = np.sort(tail[~np.isnan(tail)])
tail = tail[-10:]
print('10 worst predictions: ', tail)
tf.reset_default_graph()
elif multiview_step==2:
preds3d = []
for imageindex in tqdm(range(len(Datas[0].index))):
imagenames = [Data.index[imageindex] for Data in Datas]
images = [io.imread(os.path.join(cfg['project_path'],imagename),mode='RGB') for imagename in imagenames]
images = [skimage.color.gray2rgb(image) for image in images]
image_batch = images
# Compute prediction with the CNN
outputs_np = sess.run(outputs, feed_dict={inputs: image_batch})
pred_3d = outputs_np[2]
preds3d.append(pred_3d)
sess.close() #closes the current tf session
preds3d = np.array(preds3d) # num_ims x num_joints x (x,y,z)
num_ims, num_joints = preds3d.shape[:2]
num_views = dlc_cfg.num_views
# get labels in homogeneous pixel coordinates
labels = np.array([Data.values.reshape(num_ims, num_joints, 2) for Data in Datas]) # num_views x num_ims x num_joints x (x,y)
labels = np.transpose(labels, [1, 2, 0, 3]) # num_ims x num_joints x num_views x (x,y)
labels = np.concatenate([labels, np.ones([num_ims, num_joints, num_views, 1])], axis=3)
# solve linear system to get labels in 3D
# helpful explanation of equation found on pg 5 here: https://hal.inria.fr/inria-00524401/PDF/Sturm-cvpr05.pdf
labs = labels.reshape([num_ims * num_joints, num_views, 3]).astype(np.float)
confidences = ~np.isnan(np.sum(labs, axis=2))
valid = np.sum(~np.isnan(np.sum(labs, axis=2)), axis=1) >= 2
labs[~confidences] = 0
labels3d = project_3d(projection_matrices, labs, confidences=confidences)
labels3d[~valid] = np.nan
labels3d = labels3d.reshape([num_ims, num_joints, 3])
print("\n\n3D errors (units are determined by projection matrices):")
RMSE_train = np.nanmean(np.nansum((preds3d[trainIndices] - labels3d[trainIndices])**2, axis=2)**0.5)
RMSE_test = np.nanmean(np.nansum((preds3d[testIndices] - labels3d[testIndices])**2, axis=2)**0.5)
print("RMSE train: ", RMSE_train)
print("RMSE test: ", RMSE_test)
tail = np.nansum((preds3d- labels3d)**2, axis=2)**0.5
tail = np.sort(tail[~np.isnan(tail)])
tail = tail[-10:]
print('10 worst predictions: ', tail)
tf.reset_default_graph()
else:
print('invalid multiview_step given')
return
make_results_file(final_result,evaluationfolder,DLCscorer)
print("The network is evaluated and the results are stored in the subdirectory 'evaluation_results'.")
print("If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos.")
print("Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)")
#returning to intial folder
os.chdir(str(start_path))
def PlottingResults(
tmpfolder,
Dataframe,
cfg,
bodyparts2plot,
individuals2plot,
showfigures=False,
suffix=".png",
resolution=100,
linewidth=1.0,
):
""" Plots poses vs time; pose x vs pose y; histogram of differences and likelihoods."""
pcutoff = cfg["pcutoff"]
colors = visualization.get_cmap(len(bodyparts2plot), name=cfg["colormap"])
alphavalue = cfg["alphavalue"]
if individuals2plot:
Dataframe = Dataframe.loc(axis=1)[:, individuals2plot]
animal_bpts = Dataframe.columns.get_level_values("bodyparts")
# Pose X vs pose Y
fig1 = plt.figure(figsize=(8, 6))
ax1 = fig1.add_subplot(111)
ax1.set_xlabel("X position in pixels")
ax1.set_ylabel("Y position in pixels")
ax1.invert_yaxis()
# Poses vs time
fig2 = plt.figure(figsize=(10, 3))
ax2 = fig2.add_subplot(111)
ax2.set_xlabel("Frame Index")
ax2.set_ylabel("X-(dashed) and Y- (solid) position in pixels")
# Likelihoods
fig3 = plt.figure(figsize=(10, 3))
ax3 = fig3.add_subplot(111)
ax3.set_xlabel("Frame Index")
ax3.set_ylabel("Likelihood (use to set pcutoff)")
# Histograms
fig4 = plt.figure()
ax4 = fig4.add_subplot(111)
ax4.set_ylabel("Count")
ax4.set_xlabel("DeltaX and DeltaY")
bins = np.linspace(0, np.amax(Dataframe.max()), 100)
with np.errstate(invalid="ignore"):
for bpindex, bp in enumerate(bodyparts2plot):
if (
bp in animal_bpts
): # Avoid 'unique' bodyparts only present in the 'single' animal
prob = Dataframe.xs((bp, "likelihood"), level=(-2, -1),
axis=1).values.squeeze()
mask = prob < pcutoff
temp_x = np.ma.array(
Dataframe.xs((bp, "x"), level=(-2, -1),
axis=1).values.squeeze(),
mask=mask,
)
temp_y = np.ma.array(
Dataframe.xs((bp, "y"), level=(-2, -1),
axis=1).values.squeeze(),
mask=mask,
)
ax1.plot(temp_x,
temp_y,
".",
color=colors(bpindex),
alpha=alphavalue)
ax2.plot(
temp_x,
"--",
color=colors(bpindex),
linewidth=linewidth,
alpha=alphavalue,
)
ax2.plot(
temp_y,
"-",
color=colors(bpindex),
linewidth=linewidth,
alpha=alphavalue,
)
ax3.plot(
prob,
"-",
color=colors(bpindex),
linewidth=linewidth,
alpha=alphavalue,
)
Histogram(temp_x,
colors(bpindex),
bins,
ax4,
linewidth=linewidth)
Histogram(temp_y,
colors(bpindex),
bins,
ax4,
linewidth=linewidth)
sm = plt.cm.ScalarMappable(
cmap=plt.get_cmap(cfg["colormap"]),
norm=plt.Normalize(vmin=0, vmax=len(bodyparts2plot) - 1),
)
sm._A = []
for ax in ax1, ax2, ax3, ax4:
cbar = plt.colorbar(sm, ax=ax, ticks=range(len(bodyparts2plot)))
cbar.set_ticklabels(bodyparts2plot)
fig1.savefig(
os.path.join(tmpfolder, "trajectory" + suffix),
bbox_inches="tight",
dpi=resolution,
)
fig2.savefig(os.path.join(tmpfolder, "plot" + suffix),
bbox_inches="tight",
dpi=resolution)
fig3.savefig(
os.path.join(tmpfolder, "plot-likelihood" + suffix),
bbox_inches="tight",
dpi=resolution,
)
fig4.savefig(os.path.join(tmpfolder, "hist" + suffix),
bbox_inches="tight",
dpi=resolution)
if not showfigures:
plt.close("all")
else:
plt.show()
def evaluate_multianimal_full(
config,
Shuffles=[1],
trainingsetindex=0,
plotting=None,
show_errors=True,
comparisonbodyparts="all",
gputouse=None,
modelprefix="",
c_engine=False,
):
"""
WIP multi animal project.
"""
import os
from deeplabcut.pose_estimation_tensorflow.nnet import predict
from deeplabcut.pose_estimation_tensorflow.nnet import (
predict_multianimal as predictma, )
from deeplabcut.utils import auxiliaryfunctions, auxfun_multianimal
import tensorflow as tf
if "TF_CUDNN_USE_AUTOTUNE" in os.environ:
del os.environ[
"TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training
tf.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
start_path = os.getcwd()
##################################################
# Load data...
##################################################
cfg = auxiliaryfunctions.read_config(config)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
TrainingFractions = [cfg["TrainingFraction"][trainingsetindex]]
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
)
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
colors = visualization.get_cmap(len(comparisonbodyparts),
name=cfg["colormap"])
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# TODO: IMPLEMENT for different batch sizes?
dlc_cfg["batch_size"] = 1 # due to differently sized images!!!
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
# path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
])
if len(Snapshots) == 0:
print(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
else:
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
final_result = []
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg["init_weights"].split(os.sep)[-1]
).split(
"-"
)[-1] # read how many training siterations that corresponds to.
# name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix,
)
print(
"Running ",
DLCscorer,
" with # of trainingiterations:",
trainingsiterations,
)
(
notanalyzed,
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.CheckifNotEvaluated(
str(evaluationfolder),
DLCscorer,
DLCscorerlegacy,
Snapshots[snapindex],
)
if os.path.isfile(
resultsfilename.split(".h5")[0] + "_full.pickle"):
print("Model already evaluated.", resultsfilename)
else:
if plotting:
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" +
Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
# print(dlc_cfg)
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = predict.setup_pose_prediction(
dlc_cfg)
PredicteData = {}
print("Analyzing data...")
for imageindex, imagename in tqdm(enumerate(
Data.index)):
image_path = os.path.join(cfg["project_path"],
imagename)
image = io.imread(image_path)
frame = img_as_ubyte(skimage.color.gray2rgb(image))
GT = Data.iloc[imageindex]
# Storing GT data as dictionary, so it can be used for calculating connection costs
groundtruthcoordinates = []
groundtruthidentity = []
for bptindex, bpt in enumerate(
dlc_cfg["all_joints_names"]):
coords = np.zeros([len(individuals), 2
]) * np.nan
identity = []
for prfxindex, prefix in enumerate(
individuals):
if bpt in uniquebodyparts and prefix == "single":
coords[prfxindex, :] = np.array([
GT[cfg["scorer"]][prefix][bpt]
["x"],
GT[cfg["scorer"]][prefix][bpt]
["y"],
])
identity.append(prefix)
elif (bpt in multianimalbodyparts
and prefix != "single"):
coords[prfxindex, :] = np.array([
GT[cfg["scorer"]][prefix][bpt]
["x"],
GT[cfg["scorer"]][prefix][bpt]
["y"],
])
identity.append(prefix)
else:
identity.append("nix")
groundtruthcoordinates.append(
coords[np.isfinite(coords[:, 0]), :])
groundtruthidentity.append(
np.array(identity)[np.isfinite(coords[:,
0])])
PredicteData[imagename] = {}
PredicteData[imagename]["index"] = imageindex
pred = predictma.get_detectionswithcostsandGT(
frame,
groundtruthcoordinates,
dlc_cfg,
sess,
inputs,
outputs,
outall=False,
nms_radius=dlc_cfg.nmsradius,
det_min_score=dlc_cfg.minconfidence,
c_engine=c_engine,
)
PredicteData[imagename]["prediction"] = pred
PredicteData[imagename]["groundtruth"] = [
groundtruthidentity,
groundtruthcoordinates,
GT,
]
if plotting:
coords_pred = pred["coordinates"][0]
probs_pred = pred["confidence"]
fig = visualization.make_multianimal_labeled_image(
frame,
groundtruthcoordinates,
coords_pred,
probs_pred,
colors,
cfg["dotsize"],
cfg["alphavalue"],
cfg["pcutoff"],
)
visualization.save_labeled_frame(
fig,
image_path,
foldername,
imageindex in trainIndices,
)
sess.close() # closes the current tf session
PredicteData["metadata"] = {
"nms radius":
dlc_cfg.nmsradius,
"minimal confidence":
dlc_cfg.minconfidence,
"PAFgraph":
dlc_cfg.partaffinityfield_graph,
"all_joints":
[[i] for i in range(len(dlc_cfg.all_joints))],
"all_joints_names": [
dlc_cfg.all_joints_names[i]
for i in range(len(dlc_cfg.all_joints))
],
"stride":
dlc_cfg.get("stride", 8),
}
print(
"Done and results stored for snapshot: ",
Snapshots[snapindex],
)
dictionary = {
"Scorer": DLCscorer,
"DLC-model-config file": dlc_cfg,
"trainIndices": trainIndices,
"testIndices": testIndices,
"trainFraction": trainFraction,
}
metadata = {"data": dictionary}
auxfun_multianimal.SaveFullMultiAnimalData(
PredicteData, metadata, resultsfilename)
tf.reset_default_graph()
# returning to intial folder
os.chdir(str(start_path))
def plot_labels(self):
"""
Plots the labels of the analyzed video
"""
self.vid.set_to_frame(self.currFrame)
frame = self.vid.read_frame()
if frame is not None:
divider = make_axes_locatable(self.axes)
cax = divider.append_axes("right", size="5%", pad=0.05)
if self.multianimal:
# take into account of all the bodyparts for the colorscheme. Sort the bodyparts to have same order as in the config file
self.all_bodyparts = np.array(self.multianimalbodyparts +
self.uniquebodyparts)
_, return_idx = np.unique(self.all_bodyparts,
return_index=True)
self.all_bodyparts = list(
self.all_bodyparts[np.sort(return_idx)])
if (self.visualization_rdb.GetSelection() == 0
): # i.e. for color scheme for individuals
self.Colorscheme = visualization.get_cmap(
len(self.individual_names), self.cfg["colormap"])
self.norm, self.colorIndex = self.image_panel.getColorIndices(
frame, self.individual_names)
cbar = self.figure.colorbar(self.ax,
cax=cax,
spacing="proportional",
ticks=self.colorIndex)
cbar.set_ticklabels(self.individual_names)
else: # i.e. for color scheme for all bodyparts
self.Colorscheme = visualization.get_cmap(
len(self.all_bodyparts), self.cfg["colormap"])
self.norm, self.colorIndex = self.image_panel.getColorIndices(
frame, self.all_bodyparts)
cbar = self.figure.colorbar(self.ax,
cax=cax,
spacing="proportional",
ticks=self.colorIndex)
cbar.set_ticklabels(self.all_bodyparts)
for ci, ind in enumerate(self.individual_names):
col_idx = (
0
) # variable for iterating through the colorscheme for all bodyparts
image_points = []
if ind == "single":
if self.visualization_rdb.GetSelection() == 0:
for c, bp in enumerate(self.uniquebodyparts):
pts = self.Dataframe.xs(
(ind, bp),
level=("individuals", "bodyparts"),
axis=1,
).values
self.circle = patches.Circle(
pts[self.currFrame, :2],
radius=self.markerSize,
fc=self.Colorscheme(ci),
alpha=self.alpha,
)
self.axes.add_patch(self.circle)
else:
for c, bp in enumerate(self.uniquebodyparts):
pts = self.Dataframe.xs(
(ind, bp),
level=("individuals", "bodyparts"),
axis=1,
).values
self.circle = patches.Circle(
pts[self.currFrame, :2],
radius=self.markerSize,
fc=self.Colorscheme(col_idx),
alpha=self.alpha,
)
self.axes.add_patch(self.circle)
col_idx = col_idx + 1
else:
if self.visualization_rdb.GetSelection() == 0:
for c, bp in enumerate(self.multianimalbodyparts):
pts = self.Dataframe.xs(
(ind, bp),
level=("individuals", "bodyparts"),
axis=1,
).values
self.circle = patches.Circle(
pts[self.currFrame, :2],
radius=self.markerSize,
fc=self.Colorscheme(ci),
alpha=self.alpha,
)
self.axes.add_patch(self.circle)
else:
for c, bp in enumerate(self.multianimalbodyparts):
pts = self.Dataframe.xs(
(ind, bp),
level=("individuals", "bodyparts"),
axis=1,
).values
self.circle = patches.Circle(
pts[self.currFrame, :2],
radius=self.markerSize,
fc=self.Colorscheme(col_idx),
alpha=self.alpha,
)
self.axes.add_patch(self.circle)
col_idx = col_idx + 1
self.figure.canvas.draw()
else:
self.norm, self.colorIndex = self.image_panel.getColorIndices(
frame, self.bodyparts)
cbar = self.figure.colorbar(self.ax,
cax=cax,
spacing="proportional",
ticks=self.colorIndex)
cbar.set_ticklabels(self.bodyparts)
for bpindex, bp in enumerate(self.bodyparts):
color = self.colormap(self.norm(self.colorIndex[bpindex]))
self.points = [
self.Dataframe.xs((bp, "x"), level=(-2, -1),
axis=1).values[self.currFrame],
self.Dataframe.xs((bp, "y"), level=(-2, -1),
axis=1).values[self.currFrame],
1.0,
]
circle = [
patches.Circle(
(self.points[0], self.points[1]),
radius=self.markerSize,
fc=color,
alpha=self.alpha,
)
]
self.axes.add_patch(circle[0])
self.figure.canvas.draw()
else:
print("Invalid frame")
def ExtractFramesbasedonPreselection(Index,
extractionalgorithm,
Dataframe,
dataname,
scorer,
video,
cfg,
config,
opencv=True,
cluster_resizewidth=30,
cluster_color=False,
savelabeled=True):
from deeplabcut.create_project import add
start = cfg['start']
stop = cfg['stop']
numframes2extract = cfg['numframes2pick']
bodyparts = cfg['bodyparts']
videofolder = str(Path(video).parents[0])
vname = str(Path(video).stem)
tmpfolder = os.path.join(cfg['project_path'], 'labeled-data', vname)
if os.path.isdir(tmpfolder):
print("Frames from video", vname,
" already extracted (more will be added)!")
else:
auxiliaryfunctions.attempttomakefolder(tmpfolder)
nframes = np.size(Dataframe.index)
print("Loading video...")
if opencv:
import cv2
cap = cv2.VideoCapture(video)
fps = cap.get(5)
duration = nframes * 1. / fps
size = (int(cap.get(4)), int(cap.get(3)))
else:
from moviepy.editor import VideoFileClip
clip = VideoFileClip(video)
fps = clip.fps
duration = clip.duration
size = clip.size
if cfg['cropping']: # one might want to adjust
coords = (cfg['x1'], cfg['x2'], cfg['y1'], cfg['y2'])
else:
coords = None
print("Duration of video [s]: ", duration, ", recorded @ ", fps, "fps!")
print(
"Overall # of frames: ",
nframes,
"with (cropped) frame dimensions: ",
)
if extractionalgorithm == 'uniform':
if opencv:
frames2pick = frameselectiontools.UniformFramescv2(
cap, numframes2extract, start, stop, Index)
else:
frames2pick = frameselectiontools.UniformFrames(
clip, numframes2extract, start, stop, Index)
elif extractionalgorithm == 'kmeans':
if opencv:
frames2pick = frameselectiontools.KmeansbasedFrameselectioncv2(
cap,
numframes2extract,
start,
stop,
cfg['cropping'],
coords,
Index,
resizewidth=cluster_resizewidth,
color=cluster_color)
else:
if cfg['cropping']:
clip = clip.crop(y1=cfg['y1'],
y2=cfg['x2'],
x1=cfg['x1'],
x2=cfg['x2'])
frames2pick = frameselectiontools.KmeansbasedFrameselection(
clip,
numframes2extract,
start,
stop,
Index,
resizewidth=cluster_resizewidth,
color=cluster_color)
else:
print(
"Please implement this method yourself! Currently the options are 'kmeans', 'jump', 'uniform'."
)
frames2pick = []
# Extract frames + frames with plotted labels and store them in folder (with name derived from video name) nder labeled-data
print("Let's select frames indices:", frames2pick)
colors = visualization.get_cmap(len(bodyparts), cfg['colormap'])
strwidth = int(np.ceil(np.log10(nframes))) #width for strings
for index in frames2pick: ##tqdm(range(0,nframes,10)):
if opencv:
PlottingSingleFramecv2(cap, cv2, cfg['cropping'], coords,
Dataframe, bodyparts, tmpfolder, index,
scorer, cfg['dotsize'], cfg['pcutoff'],
cfg['alphavalue'], colors, strwidth,
savelabeled)
else:
PlottingSingleFrame(clip, Dataframe, bodyparts, tmpfolder, index,
scorer, cfg['dotsize'], cfg['pcutoff'],
cfg['alphavalue'], colors, strwidth,
savelabeled)
plt.close("all")
#close videos
if opencv:
cap.release()
else:
clip.close()
del clip
# Extract annotations based on DeepLabCut and store in the folder (with name derived from video name) under labeled-data
if len(frames2pick) > 0:
#Dataframe = pd.read_hdf(os.path.join(videofolder,dataname+'.h5'))
DF = Dataframe.ix[frames2pick]
DF.index = [
os.path.join('labeled-data', vname,
"img" + str(index).zfill(strwidth) + ".png")
for index in DF.index
] #exchange index number by file names.
machinefile = os.path.join(
tmpfolder, 'machinelabels-iter' + str(cfg['iteration']) + '.h5')
if Path(machinefile).is_file():
Data = pd.read_hdf(machinefile, 'df_with_missing')
DataCombined = pd.concat([Data, DF])
#drop duplicate labels:
DataCombined = DataCombined[~DataCombined.index.duplicated(
keep='first')]
DataCombined.to_hdf(machinefile, key='df_with_missing', mode='w')
DataCombined.to_csv(
os.path.join(tmpfolder, "machinelabels.csv")
) #this is always the most current one (as reading is from h5)
else:
DF.to_hdf(machinefile, key='df_with_missing', mode='w')
DF.to_csv(os.path.join(tmpfolder, "machinelabels.csv"))
try:
if cfg['cropping']:
add.add_new_videos(
config, [video],
coords=[coords]) # make sure you pass coords as a list
else:
add.add_new_videos(config, [video], coords=None)
except: #can we make a catch here? - in fact we should drop indices from DataCombined if they are in CollectedData.. [ideal behavior; currently this is pretty unlikely]
print(
"AUTOMATIC ADDING OF VIDEO TO CONFIG FILE FAILED! You need to do this manually for including it in the config.yaml file!"
)
print("Videopath:", video, "Coordinates for cropping:", coords)
pass
print(
"The outlier frames are extracted. They are stored in the subdirectory labeled-data\%s."
% vname)
print(
"Once you extracted frames for all videos, use 'refine_labels' to manually correct the labels."
)
else:
print("No frames were extracted.")
def __init__(self, parent,config):
# Settting the GUI size and panels design
displays = (wx.Display(i) for i in range(wx.Display.GetCount())) # Gets the number of displays
screenSizes = [display.GetGeometry().GetSize() for display in displays] # Gets the size of each display
index = 0 # For display 1.
screenWidth = screenSizes[index][0]
screenHeight = screenSizes[index][1]
self.gui_size = (screenWidth*0.7,screenHeight*0.85)
wx.Frame.__init__ ( self, parent, id = wx.ID_ANY, title = 'DeepLabCut2.0 - Refinement ToolBox',
size = wx.Size(self.gui_size), pos = wx.DefaultPosition, style = wx.RESIZE_BORDER|wx.DEFAULT_FRAME_STYLE|wx.TAB_TRAVERSAL )
self.statusbar = self.CreateStatusBar()
self.statusbar.SetStatusText("")
self.Bind(wx.EVT_CHAR_HOOK, self.OnKeyPressed)
self.SetSizeHints(wx.Size(self.gui_size)) # This sets the minimum size of the GUI. It can scale now!
###################################################################################################################################################
# Spliting the frame into top and bottom panels. Bottom panels contains the widgets. The top panel is for showing images and plotting!
topSplitter = wx.SplitterWindow(self)
vSplitter = wx.SplitterWindow(topSplitter)
self.image_panel = ImagePanel(vSplitter, config,self.gui_size)
self.choice_panel = ScrollPanel(vSplitter)
# self.choice_panel.SetupScrolling(scroll_x=True, scroll_y=True, scrollToTop=False)
# self.choice_panel.SetupScrolling(scroll_x=True, scrollToTop=False)
vSplitter.SplitVertically(self.image_panel,self.choice_panel, sashPosition=self.gui_size[0]*0.8)
vSplitter.SetSashGravity(1)
self.widget_panel = WidgetPanel(topSplitter)
topSplitter.SplitHorizontally(vSplitter, self.widget_panel,sashPosition=self.gui_size[1]*0.83)#0.9
topSplitter.SetSashGravity(1)
sizer = wx.BoxSizer(wx.VERTICAL)
sizer.Add(topSplitter, 1, wx.EXPAND)
self.SetSizer(sizer)
###################################################################################################################################################
# Add Buttons to the WidgetPanel and bind them to their respective functions.
widgetsizer = wx.WrapSizer(orient=wx.HORIZONTAL)
self.load = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Load labels")
widgetsizer.Add(self.load , 1, wx.ALL, 15)
self.load.Bind(wx.EVT_BUTTON, self.browseDir)
self.prev = wx.Button(self.widget_panel, id=wx.ID_ANY, label="<<Previous")
widgetsizer.Add(self.prev , 1, wx.ALL, 15)
self.prev.Bind(wx.EVT_BUTTON, self.prevImage)
self.prev.Enable(False)
self.next = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Next>>")
widgetsizer.Add(self.next , 1, wx.ALL, 15)
self.next.Bind(wx.EVT_BUTTON, self.nextImage)
self.next.Enable(False)
self.help = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Help")
widgetsizer.Add(self.help , 1, wx.ALL, 15)
self.help.Bind(wx.EVT_BUTTON, self.helpButton)
self.help.Enable(True)
self.zoom = wx.ToggleButton(self.widget_panel, label="Zoom")
widgetsizer.Add(self.zoom , 1, wx.ALL, 15)
self.zoom.Bind(wx.EVT_TOGGLEBUTTON, self.zoomButton)
self.widget_panel.SetSizer(widgetsizer)
self.zoom.Enable(False)
self.home = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Home")
widgetsizer.Add(self.home , 1, wx.ALL,15)
self.home.Bind(wx.EVT_BUTTON, self.homeButton)
self.widget_panel.SetSizer(widgetsizer)
self.home.Enable(False)
self.pan = wx.ToggleButton(self.widget_panel, id=wx.ID_ANY, label="Pan")
widgetsizer.Add(self.pan , 1, wx.ALL, 15)
self.pan.Bind(wx.EVT_TOGGLEBUTTON, self.panButton)
self.widget_panel.SetSizer(widgetsizer)
self.pan.Enable(False)
self.lock = wx.CheckBox(self.widget_panel, id=wx.ID_ANY, label="Lock View")
widgetsizer.Add(self.lock, 1, wx.ALL, 15)
self.lock.Bind(wx.EVT_CHECKBOX, self.lockChecked)
self.widget_panel.SetSizer(widgetsizer)
self.lock.Enable(False)
self.save = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Save")
widgetsizer.Add(self.save , 1, wx.ALL, 15)
self.save.Bind(wx.EVT_BUTTON, self.saveDataSet)
self.save.Enable(False)
widgetsizer.AddStretchSpacer(15)
self.quit = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Quit")
widgetsizer.Add(self.quit , 1, wx.ALL|wx.ALIGN_RIGHT, 15)
self.quit.Bind(wx.EVT_BUTTON, self.quitButton)
self.widget_panel.SetSizer(widgetsizer)
self.widget_panel.SetSizerAndFit(widgetsizer)
self.widget_panel.Layout()
###############################################################################################################################
# Variable initialization
self.currentDirectory = os.getcwd()
self.index = []
self.iter = []
self.threshold = []
self.file = 0
self.updatedCoords = []
self.drs = []
self.cfg = auxiliaryfunctions.read_config(config)
self.humanscorer = self.cfg['scorer']
self.move2corner = self.cfg['move2corner']
self.center = self.cfg['corner2move2']
self.colormap = plt.get_cmap(self.cfg['colormap'])
self.colormap = self.colormap.reversed()
self.markerSize = self.cfg['dotsize']
self.alpha = self.cfg['alphavalue']
self.iterationindex = self.cfg['iteration']
self.project_path=self.cfg['project_path']
self.bodyparts = self.cfg['bodyparts']
self.threshold = 0.1
self.img_size = (10,6)# (imgW, imgH) # width, height in inches.
self.preview = False
self.view_locked=False
# Workaround for MAC - xlim and ylim changed events seem to be triggered too often so need to make sure that the
# xlim and ylim have actually changed before turning zoom off
self.prezoom_xlim=[]
self.prezoom_ylim=[]
from deeplabcut.utils import auxfun_multianimal
self.individual_names,self.uniquebodyparts,self.multianimalbodyparts = auxfun_multianimal.extractindividualsandbodyparts(self.cfg)
# self.choiceBox,self.visualization_rdb = self.choice_panel.addRadioButtons()
self.Colorscheme = visualization.get_cmap(len(self.individual_names),self.cfg['colormap'])