def return_train_network_path(config, shuffle=1, trainingsetindex=0):
''' Returns the training and test pose config file names as well as the folder where the snapshot is
Parameter
----------
config : string
Full path of the config.yaml file as a string.
shuffle: int, optional
Integer value specifying the shuffle index to select for training. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
Returns the triple: trainposeconfigfile, testposeconfigfile, snapshotfolder
'''
from deeplabcut.utils import auxiliaryfunctions
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
modelfoldername = auxiliaryfunctions.GetModelFolder(
cfg["TrainingFraction"][trainingsetindex], shuffle, cfg)
trainposeconfigfile = Path(
os.path.join(cfg['project_path'], str(modelfoldername), "train",
"pose_cfg.yaml"))
testposeconfigfile = Path(
os.path.join(cfg['project_path'], str(modelfoldername), "test",
"pose_cfg.yaml"))
snapshotfolder = Path(
os.path.join(cfg['project_path'], str(modelfoldername), 'train'))
return trainposeconfigfile, testposeconfigfile, snapshotfolder
def return_train_network_path(config, shuffle, trainFraction):
''' Returns the training and test pose config file names as well as the folder where the snapshot is
Parameter
----------
config : string
Full path of the config.yaml file as a string.
shuffle: int
Integer value specifying the shuffle index to select for training.
trainFraction: float
Float specifying which training set fraction to use.
Returns the triple: trainposeconfigfile, testposeconfigfile, snapshotfolder
'''
from deeplabcut.utils import auxiliaryfunctions
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
trainposeconfigfile = Path(
os.path.join(cfg['project_path'], str(modelfoldername), "train",
"pose_cfg.yaml"))
testposeconfigfile = Path(
os.path.join(cfg['project_path'], str(modelfoldername), "test",
"pose_cfg.yaml"))
snapshotfolder = Path(
os.path.join(cfg['project_path'], str(modelfoldername), 'train'))
return trainposeconfigfile, testposeconfigfile, snapshotfolder
def edit_pose_config(self, event):
"""
"""
self.shuffles.Enable(True)
self.trainingindex.Enable(True)
self.display_iters.Enable(True)
self.save_iters.Enable(True)
self.max_iters.Enable(True)
self.snapshots.Enable(True)
# Read the pose config file
cfg = auxiliaryfunctions.read_config(self.config)
trainFraction = cfg["TrainingFraction"][self.trainingindex.GetValue()]
# print(os.path.join(cfg['project_path'],auxiliaryfunctions.GetModelFolder(trainFraction, self.shuffles.GetValue(),cfg),'train','pose_cfg.yaml'))
self.pose_cfg_path = os.path.join(
cfg["project_path"],
auxiliaryfunctions.GetModelFolder(trainFraction,
self.shuffles.GetValue(), cfg),
"train",
"pose_cfg.yaml",
)
# let the user open the file with default text editor. Also make it mac compatible
if sys.platform == "darwin":
self.file_open_bool = subprocess.call(["open", self.pose_cfg_path])
self.file_open_bool = True
else:
self.file_open_bool = webbrowser.open(self.pose_cfg_path)
if self.file_open_bool:
self.pose_cfg = auxiliaryfunctions.read_plainconfig(
self.pose_cfg_path)
else:
raise FileNotFoundError("File not found!")
def get_snapshot_path(snapshot, dlcpath, shuffle=1, trainingsetindex=0):
"""Get the full path for the snapshot.
Parameters
----------
snapshot : snapshot name, str
dlcpath : the path for the DLC project, str
shuffle : shuffle index, int, optional
default value is 1
Returns
-------
snapshot_path : full path of the snapshot
config_path : full path of the configuration file
"""
dlc_base_path = Path(dlcpath)
config_path = dlc_base_path / 'config.yaml'
cfg = auxiliaryfunctions.read_config(config_path)
modelfoldername = auxiliaryfunctions.GetModelFolder(
cfg["TrainingFraction"][trainingsetindex], shuffle, cfg)
train_path = dlc_base_path / modelfoldername / 'train'
snapshot_path = str(train_path / snapshot)
return snapshot_path, config_path
def cross_validate(self, event):
trainingsetindex = self.trainingset.GetValue()
shuffle = [self.shuffles.GetValue()]
cfg = auxiliaryfunctions.read_config(self.config)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
self.inf_cfg_path = os.path.join(
cfg["project_path"],
auxiliaryfunctions.GetModelFolder(trainFraction,
self.shuffles.GetValue(), cfg),
"test",
"inference_cfg.yaml",
)
# Read from edited inf. file first ...
print(self.inf_cfg_path)
print("optimizing parameters using " + self.targettypes.GetValue() +
" as a target...")
deeplabcut.evaluate_multianimal_crossvalidate(
self.config,
Shuffles=shuffle,
trainingsetindex=trainingsetindex,
edgewisecondition=self.edgeWise.GetStringSelection(),
leastbpts=self.infg.GetValue(),
init_points=self.inpts.GetValue(),
n_iter=self.n_iter.GetValue(),
target=self.targettypes.GetValue(),
)
def add_train_shuffle(get_max_shuffle_idx,
cfg,
trainIndexes,
testIndexes,
schedule_config={}):
project_path = Path(cfg["project_path"])
path_config_file = project_path / "config.yaml"
TrainingFraction = cfg["TrainingFraction"]
iteration = cfg['iteration']
modelfoldername = auxiliaryfunctions.GetModelFolder(
TrainingFraction[iteration], get_max_shuffle_idx, cfg)
path_train_folder = project_path / Path(modelfoldername)
if path_train_folder.exists():
print('Iteration {} shuffle {} already exists'.format(
iteration, get_max_shuffle_idx))
return
else:
print("\nAdding shuffle {}".format(get_max_shuffle_idx))
#get_max_shuffle_idx #schedule_config['project_path'] = str(str(project_path))
#get_max_shuffle_idx = largestshuffleindex + schedule_config_idx + 1
# Create dataset for that shuffle with deterministic data
print("\nCreating shuffle {}".format(get_max_shuffle_idx))
create_training_dataset(
str(path_config_file),
Shuffles=[get_max_shuffle_idx],
trainIndexes=trainIndexes,
testIndexes=testIndexes,
items2change_pose=schedule_config,
)
return
def update_checkpoint(self, new_weights="none"):
if new_weights == 'none':
print('No checkpoint provided.')
else:
print('Updating init_weights with %s..' % new_weights)
from deeplabcut.utils import auxiliaryfunctions
# load config.yaml
main_config = auxiliaryfunctions.read_config(
self.full_config_path())
# Update train and test config.yaml paths
trainingsetindex = 0
shuffle = 1
modelfoldername = auxiliaryfunctions.GetModelFolder(
main_config["TrainingFraction"][trainingsetindex], shuffle,
main_config)
path_train_config = os.path.join(main_config['project_path'],
Path(modelfoldername), 'train',
'pose_cfg.yaml')
path_test_config = os.path.join(main_config['project_path'],
Path(modelfoldername), 'test',
'pose_cfg.yaml')
# Update training pose_cfg.yaml
if os.path.exists(path_train_config):
with open(path_train_config, "r") as ymlfile:
cfg_train = yaml.load(ymlfile)
# Update init_weights path
cfg_train['init_weights'] = os.path.join(
main_config['project_path'], Path(modelfoldername),
'train', new_weights)
with open(path_train_config, 'w') as ymlfile:
yaml.dump(cfg_train, ymlfile)
# Update testing pose_cfg.yaml
# if os.path.exists(path_test_config):
# with open(path_test_config, "r") as ymlfile:
# cfg_test = yaml.load(ymlfile)
# # Update init_weights path
# cfg_test['init_weights'] = os.path.join(main_config['project_path'], #Path(modelfoldername), 'test',new_weights)
#with open(path_test_config, 'w') as ymlfile:
# yaml.dump(cfg_test, ymlfile)
print('done.')
def get_train_config(cfg, shuffle=1):
from deeplabcut.utils import auxiliaryfunctions
from deeplabcut.pose_estimation_tensorflow.config import load_config
project_path = cfg['project_path']
iteration = cfg['iteration']
TrainingFraction = cfg['TrainingFraction'][iteration]
modelfolder = os.path.join(
project_path,
str(auxiliaryfunctions.GetModelFolder(TrainingFraction, shuffle, cfg)))
path_test_config = Path(modelfolder) / 'train' / 'pose_cfg.yaml'
print(path_test_config)
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, TrainingFraction))
return dlc_cfg
def init_session(cfg, gputouse=None, shuffle=1, trainIndex=0, locate_on_gpu=False):
if isinstance(cfg, (str, _Path)):
cfg = _load_config(cfg)
TF.reset_default_graph()
projpath = cfg['project_path']
trainFraction = cfg['TrainingFraction'][trainIndex]
modelfolder = projpath / _aux.GetModelFolder(trainFraction,shuffle,cfg)
dlc_cfg = _get_pose_config(cfg, modelfolder, shuffle=shuffle, trainIndex=trainIndex)
snapshot, iteration = _get_snapshot(cfg, modelfolder, shuffle=shuffle)
dlc_cfg['init_weights'] = str(snapshot)
#update batchsize (based on parameters in config.yaml)
dlc_cfg['batch_size'] = cfg['batch_size']
# update number of outputs
dlc_cfg['num_outputs'] = cfg.get('num_outputs', 1)
print('num_outputs = ', dlc_cfg['num_outputs'])
DLCscorer = _aux.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations=iteration)
cls = DirectTFSession if cfg["batch_size"] == 1 else BatchTFSession
return cls(dlc_cfg, *(_dlc_setup_pose_prediction(dlc_cfg, locate_on_gpu=locate_on_gpu)))
def edit_inf_config(self, event):
# Read the infer config file
cfg = auxiliaryfunctions.read_config(self.config)
#trainFraction = cfg["TrainingFraction"][trainingsetindex]
self.inf_cfg_path = os.path.join(
cfg["project_path"],
auxiliaryfunctions.GetModelFolder(trainFraction,
self.shuffle.GetValue(), cfg),
"test",
"inference_cfg.yaml",
)
# let the user open the file with default text editor. Also make it mac compatible
if sys.platform == "darwin":
self.file_open_bool = subprocess.call(["open", self.inf_cfg_path])
self.file_open_bool = True
else:
self.file_open_bool = webbrowser.open(self.inf_cfg_path)
if self.file_open_bool:
self.inf_cfg = auxiliaryfunctions.read_config(self.inf_cfg_path)
else:
raise FileNotFoundError("File not found!")
def evaluate_multianimal_full(
config,
Shuffles=[1],
trainingsetindex=0,
plotting=False,
show_errors=True,
comparisonbodyparts="all",
gputouse=None,
modelprefix="",
):
from deeplabcut.pose_estimation_tensorflow.core import (
predict,
predict_multianimal as predictma,
)
from deeplabcut.utils import (
auxiliaryfunctions,
auxfun_multianimal,
auxfun_videos,
conversioncode,
)
import tensorflow as tf
if "TF_CUDNN_USE_AUTOTUNE" in os.environ:
del os.environ["TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training
tf.compat.v1.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
start_path = os.getcwd()
if plotting is True:
plotting = "bodypart"
##################################################
# Load data...
##################################################
cfg = auxiliaryfunctions.read_config(config)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
TrainingFractions = [cfg["TrainingFraction"][trainingsetindex]]
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
)
)
conversioncode.guarantee_multiindex_rows(Data)
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts
)
all_bpts = np.asarray(
len(cfg["individuals"]) * cfg["multianimalbodyparts"] + cfg["uniquebodyparts"]
)
colors = visualization.get_cmap(len(comparisonbodyparts), name=cfg["colormap"])
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/")
)
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg
)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix
)
),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn)
)
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction)
)
pipeline = iaa.Sequential(random_order=False)
pre_resize = dlc_cfg.get("pre_resize")
if pre_resize:
width, height = pre_resize
pipeline.add(iaa.Resize({"height": height, "width": width}))
# TODO: IMPLEMENT for different batch sizes?
dlc_cfg["batch_size"] = 1 # due to differently sized images!!!
stride = dlc_cfg["stride"]
# Ignore best edges possibly defined during a prior evaluation
_ = dlc_cfg.pop("paf_best", None)
joints = dlc_cfg["all_joints_names"]
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix
)
),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True)
# path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array(
[
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
]
)
if len(Snapshots) == 0:
print(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction)
)
else:
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots]
)
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
final_result = []
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg["init_weights"].split(os.sep)[-1]
).split("-")[
-1
] # read how many training siterations that corresponds to.
# name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix,
)
print(
"Running ",
DLCscorer,
" with # of trainingiterations:",
trainingsiterations,
)
(
notanalyzed,
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.CheckifNotEvaluated(
str(evaluationfolder),
DLCscorer,
DLCscorerlegacy,
Snapshots[snapindex],
)
data_path = resultsfilename.split(".h5")[0] + "_full.pickle"
if plotting:
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" + Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
if plotting == "bodypart":
fig, ax = visualization.create_minimal_figure()
if os.path.isfile(data_path):
print("Model already evaluated.", resultsfilename)
else:
(sess, inputs, outputs,) = predict.setup_pose_prediction(
dlc_cfg
)
PredicteData = {}
dist = np.full((len(Data), len(all_bpts)), np.nan)
conf = np.full_like(dist, np.nan)
print("Network Evaluation underway...")
for imageindex, imagename in tqdm(enumerate(Data.index)):
image_path = os.path.join(cfg["project_path"], *imagename)
frame = auxfun_videos.imread(image_path, mode="skimage")
GT = Data.iloc[imageindex]
if not GT.any():
continue
# Pass the image and the keypoints through the resizer;
# this has no effect if no augmenters were added to it.
keypoints = [GT.to_numpy().reshape((-1, 2)).astype(float)]
frame_, keypoints = pipeline(
images=[frame], keypoints=keypoints
)
frame = frame_[0]
GT[:] = keypoints[0].flatten()
df = GT.unstack("coords").reindex(joints, level="bodyparts")
# FIXME Is having an empty array vs nan really that necessary?!
groundtruthidentity = list(
df.index.get_level_values("individuals")
.to_numpy()
.reshape((-1, 1))
)
groundtruthcoordinates = list(df.values[:, np.newaxis])
for i, coords in enumerate(groundtruthcoordinates):
if np.isnan(coords).any():
groundtruthcoordinates[i] = np.empty(
(0, 2), dtype=float
)
groundtruthidentity[i] = np.array([], dtype=str)
# Form 2D array of shape (n_rows, 4) where the last dimension
# is (sample_index, peak_y, peak_x, bpt_index) to slice the PAFs.
temp = df.reset_index(level="bodyparts").dropna()
temp["bodyparts"].replace(
dict(zip(joints, range(len(joints)))), inplace=True,
)
temp["sample"] = 0
peaks_gt = temp.loc[
:, ["sample", "y", "x", "bodyparts"]
].to_numpy()
peaks_gt[:, 1:3] = (peaks_gt[:, 1:3] - stride // 2) / stride
pred = predictma.predict_batched_peaks_and_costs(
dlc_cfg,
np.expand_dims(frame, axis=0),
sess,
inputs,
outputs,
peaks_gt.astype(int),
)
if not pred:
continue
else:
pred = pred[0]
PredicteData[imagename] = {}
PredicteData[imagename]["index"] = imageindex
PredicteData[imagename]["prediction"] = pred
PredicteData[imagename]["groundtruth"] = [
groundtruthidentity,
groundtruthcoordinates,
GT,
]
coords_pred = pred["coordinates"][0]
probs_pred = pred["confidence"]
for bpt, xy_gt in df.groupby(level="bodyparts"):
inds_gt = np.flatnonzero(
np.all(~np.isnan(xy_gt), axis=1)
)
n_joint = joints.index(bpt)
xy = coords_pred[n_joint]
if inds_gt.size and xy.size:
# Pick the predictions closest to ground truth,
# rather than the ones the model has most confident in
xy_gt_values = xy_gt.iloc[inds_gt].values
neighbors = _find_closest_neighbors(
xy_gt_values, xy, k=3
)
found = neighbors != -1
min_dists = np.linalg.norm(
xy_gt_values[found] - xy[neighbors[found]],
axis=1,
)
inds = np.flatnonzero(all_bpts == bpt)
sl = imageindex, inds[inds_gt[found]]
dist[sl] = min_dists
conf[sl] = probs_pred[n_joint][
neighbors[found]
].squeeze()
if plotting == "bodypart":
temp_xy = GT.unstack("bodyparts")[joints].values
gt = temp_xy.reshape(
(-1, 2, temp_xy.shape[1])
).T.swapaxes(1, 2)
h, w, _ = np.shape(frame)
fig.set_size_inches(w / 100, h / 100)
ax.set_xlim(0, w)
ax.set_ylim(0, h)
ax.invert_yaxis()
ax = visualization.make_multianimal_labeled_image(
frame,
gt,
coords_pred,
probs_pred,
colors,
cfg["dotsize"],
cfg["alphavalue"],
cfg["pcutoff"],
ax=ax,
)
visualization.save_labeled_frame(
fig,
image_path,
foldername,
imageindex in trainIndices,
)
visualization.erase_artists(ax)
sess.close() # closes the current tf session
# Compute all distance statistics
df_dist = pd.DataFrame(dist, columns=df.index)
df_conf = pd.DataFrame(conf, columns=df.index)
df_joint = pd.concat(
[df_dist, df_conf],
keys=["rmse", "conf"],
names=["metrics"],
axis=1,
)
df_joint = df_joint.reorder_levels(
list(np.roll(df_joint.columns.names, -1)), axis=1
)
df_joint.sort_index(
axis=1,
level=["individuals", "bodyparts"],
ascending=[True, True],
inplace=True,
)
write_path = os.path.join(
evaluationfolder, f"dist_{trainingsiterations}.csv"
)
df_joint.to_csv(write_path)
# Calculate overall prediction error
error = df_joint.xs("rmse", level="metrics", axis=1)
mask = (
df_joint.xs("conf", level="metrics", axis=1)
>= cfg["pcutoff"]
)
error_masked = error[mask]
error_train = np.nanmean(error.iloc[trainIndices])
error_train_cut = np.nanmean(error_masked.iloc[trainIndices])
error_test = np.nanmean(error.iloc[testIndices])
error_test_cut = np.nanmean(error_masked.iloc[testIndices])
results = [
trainingsiterations,
int(100 * trainFraction),
shuffle,
np.round(error_train, 2),
np.round(error_test, 2),
cfg["pcutoff"],
np.round(error_train_cut, 2),
np.round(error_test_cut, 2),
]
final_result.append(results)
if show_errors:
string = (
"Results for {} training iterations, training fraction of {}, and shuffle {}:\n"
"Train error: {} pixels. Test error: {} pixels.\n"
"With pcutoff of {}:\n"
"Train error: {} pixels. Test error: {} pixels."
)
print(string.format(*results))
print("##########################################")
print(
"Average Euclidean distance to GT per individual (in pixels; test-only)"
)
print(
error_masked.iloc[testIndices]
.groupby("individuals", axis=1)
.mean()
.mean()
.to_string()
)
print(
"Average Euclidean distance to GT per bodypart (in pixels; test-only)"
)
print(
error_masked.iloc[testIndices]
.groupby("bodyparts", axis=1)
.mean()
.mean()
.to_string()
)
PredicteData["metadata"] = {
"nms radius": dlc_cfg["nmsradius"],
"minimal confidence": dlc_cfg["minconfidence"],
"sigma": dlc_cfg.get("sigma", 1),
"PAFgraph": dlc_cfg["partaffinityfield_graph"],
"PAFinds": np.arange(
len(dlc_cfg["partaffinityfield_graph"])
),
"all_joints": [
[i] for i in range(len(dlc_cfg["all_joints"]))
],
"all_joints_names": [
dlc_cfg["all_joints_names"][i]
for i in range(len(dlc_cfg["all_joints"]))
],
"stride": dlc_cfg.get("stride", 8),
}
print(
"Done and results stored for snapshot: ",
Snapshots[snapindex],
)
dictionary = {
"Scorer": DLCscorer,
"DLC-model-config file": dlc_cfg,
"trainIndices": trainIndices,
"testIndices": testIndices,
"trainFraction": trainFraction,
}
metadata = {"data": dictionary}
_ = auxfun_multianimal.SaveFullMultiAnimalData(
PredicteData, metadata, resultsfilename
)
tf.compat.v1.reset_default_graph()
n_multibpts = len(cfg["multianimalbodyparts"])
if n_multibpts == 1:
continue
# Skip data-driven skeleton selection unless
# the model was trained on the full graph.
max_n_edges = n_multibpts * (n_multibpts - 1) // 2
n_edges = len(dlc_cfg["partaffinityfield_graph"])
if n_edges == max_n_edges:
print("Selecting best skeleton...")
n_graphs = 10
paf_inds = None
else:
n_graphs = 1
paf_inds = [list(range(n_edges))]
(
results,
paf_scores,
best_assemblies,
) = crossvalutils.cross_validate_paf_graphs(
config,
str(path_test_config).replace("pose_", "inference_"),
data_path,
data_path.replace("_full.", "_meta."),
n_graphs=n_graphs,
paf_inds=paf_inds,
oks_sigma=dlc_cfg.get("oks_sigma", 0.1),
margin=dlc_cfg.get("bbox_margin", 0),
symmetric_kpts=dlc_cfg.get("symmetric_kpts"),
)
if plotting == "individual":
assemblies, assemblies_unique, image_paths = best_assemblies
fig, ax = visualization.create_minimal_figure()
n_animals = len(cfg["individuals"])
if cfg["uniquebodyparts"]:
n_animals += 1
colors = visualization.get_cmap(n_animals, name=cfg["colormap"])
for k, v in tqdm(assemblies.items()):
imname = image_paths[k]
image_path = os.path.join(cfg["project_path"], *imname)
frame = auxfun_videos.imread(image_path, mode="skimage")
h, w, _ = np.shape(frame)
fig.set_size_inches(w / 100, h / 100)
ax.set_xlim(0, w)
ax.set_ylim(0, h)
ax.invert_yaxis()
gt = [
s.to_numpy().reshape((-1, 2))
for _, s in Data.loc[imname].groupby("individuals")
]
coords_pred = []
coords_pred += [ass.xy for ass in v]
probs_pred = []
probs_pred += [ass.data[:, 2:3] for ass in v]
if assemblies_unique is not None:
unique = assemblies_unique.get(k, None)
if unique is not None:
coords_pred.append(unique[:, :2])
probs_pred.append(unique[:, 2:3])
while len(coords_pred) < len(gt):
coords_pred.append(np.full((1, 2), np.nan))
probs_pred.append(np.full((1, 2), np.nan))
ax = visualization.make_multianimal_labeled_image(
frame,
gt,
coords_pred,
probs_pred,
colors,
cfg["dotsize"],
cfg["alphavalue"],
cfg["pcutoff"],
ax=ax,
)
visualization.save_labeled_frame(
fig, image_path, foldername, k in trainIndices,
)
visualization.erase_artists(ax)
df = results[1].copy()
df.loc(axis=0)[("mAP_train", "mean")] = [
d[0]["mAP"] for d in results[2]
]
df.loc(axis=0)[("mAR_train", "mean")] = [
d[0]["mAR"] for d in results[2]
]
df.loc(axis=0)[("mAP_test", "mean")] = [
d[1]["mAP"] for d in results[2]
]
df.loc(axis=0)[("mAR_test", "mean")] = [
d[1]["mAR"] for d in results[2]
]
with open(data_path.replace("_full.", "_map."), "wb") as file:
pickle.dump((df, paf_scores), file)
if len(final_result) > 0: # Only append if results were calculated
make_results_file(final_result, evaluationfolder, DLCscorer)
os.chdir(str(start_path))
def evaluate_network(
config,
Shuffles=[1],
trainingsetindex=0,
plotting=False,
show_errors=True,
comparisonbodyparts="all",
gputouse=None,
rescale=False,
modelprefix="",
):
"""
Evaluates the network based on the saved models at different stages of the training network.\n
The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'.
Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
Shuffles: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This
variable can also be set to "all".
plotting: bool or str, optional
Plots the predictions on the train and test images.
The default is ``False``; if provided it must be either ``True``, ``False``, "bodypart", or "individual".
Setting to ``True`` defaults as "bodypart" for multi-animal projects.
show_errors: bool, optional
Display train and test errors. The default is `True``
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
rescale: bool, default False
Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every
image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported
in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated
on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the
original size!
Examples
--------
If you do not want to plot, just evaluate shuffle 1.
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml', Shuffles=[1])
--------
If you want to plot and evaluate shuffle 0 and 1.
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',Shuffles=[0, 1],plotting = True)
--------
If you want to plot assemblies for a maDLC project:
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',Shuffles=[1],plotting = "individual")
Note: this defaults to standard plotting for single-animal projects.
"""
if plotting not in (True, False, "bodypart", "individual"):
raise ValueError(f"Unknown value for `plotting`={plotting}")
import os
start_path = os.getcwd()
from deeplabcut.utils import auxiliaryfunctions
cfg = auxiliaryfunctions.read_config(config)
if cfg.get("multianimalproject", False):
from .evaluate_multianimal import evaluate_multianimal_full
# TODO: Make this code not so redundant!
evaluate_multianimal_full(
config=config,
Shuffles=Shuffles,
trainingsetindex=trainingsetindex,
plotting=plotting,
comparisonbodyparts=comparisonbodyparts,
gputouse=gputouse,
modelprefix=modelprefix,
)
else:
from deeplabcut.utils.auxfun_videos import imread, imresize
from deeplabcut.pose_estimation_tensorflow.core import predict
from deeplabcut.pose_estimation_tensorflow.config import load_config
from deeplabcut.pose_estimation_tensorflow.datasets.utils import data_to_input
from deeplabcut.utils import auxiliaryfunctions, conversioncode
import tensorflow as tf
# If a string was passed in, auto-convert to True for backward compatibility
plotting = bool(plotting)
if "TF_CUDNN_USE_AUTOTUNE" in os.environ:
del os.environ[
"TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training
tf.compat.v1.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
# tf.logging.set_verbosity(tf.logging.WARN)
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
if (trainingsetindex < len(cfg["TrainingFraction"])
and trainingsetindex >= 0):
TrainingFractions = [
cfg["TrainingFraction"][int(trainingsetindex)]
]
else:
raise Exception(
"Please check the trainingsetindex! ",
trainingsetindex,
" should be an integer from 0 .. ",
int(len(cfg["TrainingFraction"]) - 1),
)
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
))
# Get list of body parts to evaluate network for
comparisonbodyparts = (
auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts))
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# change batch size, if it was edited during analysis!
dlc_cfg[
"batch_size"] = 1 # in case this was edited for analysis.
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
# path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0] for fn in os.listdir(
os.path.join(str(modelfolder), "train"))
if "index" in fn
])
try: # check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
raise ValueError(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
final_result = []
########################### RESCALING (to global scale)
if rescale:
scale = dlc_cfg["global_scale"]
Data = (pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
)) * scale)
else:
scale = 1
conversioncode.guarantee_multiindex_rows(Data)
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg["init_weights"].split(os.sep)[-1]
).split(
"-"
)[-1] # read how many training siterations that corresponds to.
# Name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix,
)
print(
"Running ",
DLCscorer,
" with # of training iterations:",
trainingsiterations,
)
(
notanalyzed,
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.CheckifNotEvaluated(
str(evaluationfolder),
DLCscorer,
DLCscorerlegacy,
Snapshots[snapindex],
)
if notanalyzed:
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = predict.setup_pose_prediction(
dlc_cfg)
Numimages = len(Data.index)
PredicteData = np.zeros(
(Numimages, 3 * len(dlc_cfg["all_joints_names"])))
print("Running evaluation ...")
for imageindex, imagename in tqdm(enumerate(
Data.index)):
image = imread(
os.path.join(cfg["project_path"], *imagename),
mode="skimage",
)
if scale != 1:
image = imresize(image, scale)
image_batch = data_to_input(image)
# Compute prediction with the CNN
outputs_np = sess.run(
outputs, feed_dict={inputs: image_batch})
scmap, locref = predict.extract_cnn_output(
outputs_np, dlc_cfg)
# Extract maximum scoring location from the heatmap, assume 1 person
pose = predict.argmax_pose_predict(
scmap, locref, dlc_cfg["stride"])
PredicteData[imageindex, :] = (
pose.flatten()
) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts!
sess.close() # closes the current tf session
index = pd.MultiIndex.from_product(
[
[DLCscorer],
dlc_cfg["all_joints_names"],
["x", "y", "likelihood"],
],
names=["scorer", "bodyparts", "coords"],
)
# Saving results
DataMachine = pd.DataFrame(PredicteData,
columns=index,
index=Data.index)
DataMachine.to_hdf(resultsfilename, "df_with_missing")
print(
"Analysis is done and the results are stored (see evaluation-results) for snapshot: ",
Snapshots[snapindex],
)
DataCombined = pd.concat([Data.T, DataMachine.T],
axis=0,
sort=False).T
RMSE, RMSEpcutoff = pairwisedistances(
DataCombined,
cfg["scorer"],
DLCscorer,
cfg["pcutoff"],
comparisonbodyparts,
)
testerror = np.nanmean(
RMSE.iloc[testIndices].values.flatten())
trainerror = np.nanmean(
RMSE.iloc[trainIndices].values.flatten())
testerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[testIndices].values.flatten())
trainerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[trainIndices].values.flatten())
results = [
trainingsiterations,
int(100 * trainFraction),
shuffle,
np.round(trainerror, 2),
np.round(testerror, 2),
cfg["pcutoff"],
np.round(trainerrorpcutoff, 2),
np.round(testerrorpcutoff, 2),
]
final_result.append(results)
if show_errors:
print(
"Results for",
trainingsiterations,
" training iterations:",
int(100 * trainFraction),
shuffle,
"train error:",
np.round(trainerror, 2),
"pixels. Test error:",
np.round(testerror, 2),
" pixels.",
)
print(
"With pcutoff of",
cfg["pcutoff"],
" train error:",
np.round(trainerrorpcutoff, 2),
"pixels. Test error:",
np.round(testerrorpcutoff, 2),
"pixels",
)
if scale != 1:
print(
"The predictions have been calculated for rescaled images (and rescaled ground truth). Scale:",
scale,
)
print(
"Thereby, the errors are given by the average distances between the labels by DLC and the scorer."
)
if plotting:
print("Plotting...")
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" +
Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
Plotting(
cfg,
comparisonbodyparts,
DLCscorer,
trainIndices,
DataCombined * 1.0 / scale,
foldername,
) # Rescaling coordinates to have figure in original size!
tf.compat.v1.reset_default_graph()
# print(final_result)
else:
DataMachine = pd.read_hdf(resultsfilename)
conversioncode.guarantee_multiindex_rows(DataMachine)
if plotting:
DataCombined = pd.concat([Data.T, DataMachine.T],
axis=0,
sort=False).T
print(
"Plotting...(attention scale might be inconsistent in comparison to when data was analyzed; i.e. if you used rescale)"
)
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" +
Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
Plotting(
cfg,
comparisonbodyparts,
DLCscorer,
trainIndices,
DataCombined * 1.0 / scale,
foldername,
)
if len(final_result
) > 0: # Only append if results were calculated
make_results_file(final_result, evaluationfolder,
DLCscorer)
print(
"The network is evaluated and the results are stored in the subdirectory 'evaluation_results'."
)
print(
"Please check the results, then choose the best model (snapshot) for prediction. You can update the config.yaml file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos."
)
print(
"Otherwise, consider adding more labeled-data and retraining the network (see DeepLabCut workflow Fig 2, Nath 2019)"
)
# returning to initial folder
os.chdir(str(start_path))
def calculatepafdistancebounds(config,
shuffle=0,
trainingsetindex=0,
modelprefix="",
numdigits=0,
onlytrain=False):
"""
Returns distances along paf edges in train/test data
----------
config : string
Full path of the config.yaml file as a string.
shuffle: integer
integers specifying shuffle index of the training dataset. The default is 0.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This
variable can also be set to "all".
numdigits: number of digits to round for distances.
"""
import os
from deeplabcut.utils import auxiliaryfunctions, auxfun_multianimal
from deeplabcut.pose_estimation_tensorflow.config import load_config
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if cfg["multianimalproject"]:
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
# Load meta data & annotations
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
))[cfg["scorer"]]
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
dlc_cfg = load_config(str(path_test_config))
# get the graph!
partaffinityfield_graph = dlc_cfg["partaffinityfield_graph"]
jointnames = [
dlc_cfg["all_joints_names"][i]
for i in range(len(dlc_cfg["all_joints"]))
]
path_inferencebounds_config = (Path(modelfolder) / "test" /
"inferencebounds.yaml")
inferenceboundscfg = {}
for pi, edge in enumerate(partaffinityfield_graph):
j1, j2 = jointnames[edge[0]], jointnames[edge[1]]
ds_within = []
ds_across = []
for ind in individuals:
for ind2 in individuals:
if ind != "single" and ind2 != "single":
if (ind, j1, "x") in Data.keys() and (
ind2,
j2,
"y",
) in Data.keys():
distances = (np.sqrt(
(Data[ind, j1, "x"] - Data[ind2, j2, "x"])**2 +
(Data[ind, j1, "y"] - Data[ind2, j2, "y"])**2)
/ dlc_cfg["stride"])
else:
distances = None
if distances is not None:
if onlytrain:
distances = distances.iloc[trainIndices]
if ind == ind2:
ds_within.extend(distances.values.flatten())
else:
ds_across.extend(distances.values.flatten())
edgeencoding = str(edge[0]) + "_" + str(edge[1])
inferenceboundscfg[edgeencoding] = {}
if len(ds_within) > 0:
inferenceboundscfg[edgeencoding]["intra_max"] = str(
round(np.nanmax(ds_within), numdigits))
inferenceboundscfg[edgeencoding]["intra_min"] = str(
round(np.nanmin(ds_within), numdigits))
else:
inferenceboundscfg[edgeencoding]["intra_max"] = str(
1e5) # large number (larger than any image diameter)
inferenceboundscfg[edgeencoding]["intra_min"] = str(0)
# NOTE: the inter-animal distances are currently not used, but are interesting to compare to intra_*
if len(ds_across) > 0:
inferenceboundscfg[edgeencoding]["inter_max"] = str(
round(np.nanmax(ds_across), numdigits))
inferenceboundscfg[edgeencoding]["inter_min"] = str(
round(np.nanmin(ds_across), numdigits))
else:
inferenceboundscfg[edgeencoding]["inter_max"] = str(
1e5
) # large number (larger than image diameters in typical experiments)
inferenceboundscfg[edgeencoding]["inter_min"] = str(0)
auxiliaryfunctions.write_plainconfig(str(path_inferencebounds_config),
dict(inferenceboundscfg))
return inferenceboundscfg
else:
print("You might as well bring owls to Athens.")
return {}
def return_evaluate_network_data(
config,
shuffle=0,
trainingsetindex=0,
comparisonbodyparts="all",
Snapindex=None,
rescale=False,
fulldata=False,
show_errors=True,
modelprefix="",
returnjustfns=True,
):
"""
Returns the results for (previously evaluated) network. deeplabcut.evaluate_network(..)
Returns list of (per model): [trainingsiterations,trainfraction,shuffle,trainerror,testerror,pcutoff,trainerrorpcutoff,testerrorpcutoff,Snapshots[snapindex],scale,net_type]
If fulldata=True, also returns (the complete annotation and prediction array)
Returns list of: (DataMachine, Data, data, trainIndices, testIndices, trainFraction, DLCscorer,comparisonbodyparts, cfg, Snapshots[snapindex])
----------
config : string
Full path of the config.yaml file as a string.
shuffle: integer
integers specifying shuffle index of the training dataset. The default is 0.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This
variable can also be set to "all".
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
rescale: bool, default False
Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every
image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported
in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated
on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the
original size!
Examples
--------
If you do not want to plot
>>> deeplabcut._evaluate_network_data('/analysis/project/reaching-task/config.yaml', shuffle=[1])
--------
If you want to plot
>>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',shuffle=[1],True)
"""
import os
from deeplabcut.pose_estimation_tensorflow.config import load_config
from deeplabcut.utils import auxiliaryfunctions
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
# Data=pd.read_hdf(os.path.join(cfg["project_path"],str(trainingsetfolder),'CollectedData_' + cfg["scorer"] + '.h5'),'df_with_missing')
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
##################################################
# Load data...
##################################################
trainFraction = cfg["TrainingFraction"][trainingsetindex]
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
data, trainIndices, testIndices, trainFraction = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
########################### RESCALING (to global scale)
if rescale == True:
scale = dlc_cfg["global_scale"]
print("Rescaling Data to ", scale)
Data = (pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
)) * scale)
else:
scale = 1
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
))
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
])
if len(Snapshots) == 0:
print(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
snapindices = []
else:
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if Snapindex == None:
Snapindex = cfg["snapshotindex"]
if Snapindex == -1:
snapindices = [-1]
elif Snapindex == "all":
snapindices = range(len(Snapshots))
elif Snapindex < len(Snapshots):
snapindices = [Snapindex]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
DATA = []
results = []
resultsfns = []
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train",
Snapshots[snapindex]) # setting weights to corresponding snapshot.
trainingsiterations = (dlc_cfg["init_weights"].split(
os.sep)[-1]).split("-")[
-1] # read how many training siterations that corresponds to.
# name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix)
if not returnjustfns:
print(
"Retrieving ",
DLCscorer,
" with # of trainingiterations:",
trainingsiterations,
)
(
notanalyzed,
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.CheckifNotEvaluated(str(evaluationfolder),
DLCscorer, DLCscorerlegacy,
Snapshots[snapindex])
# resultsfilename=os.path.join(str(evaluationfolder),DLCscorer + '-' + str(Snapshots[snapindex])+ '.h5') # + '-' + str(snapshot)+ ' #'-' + Snapshots[snapindex]+ '.h5')
print(resultsfilename)
resultsfns.append(resultsfilename)
if not returnjustfns:
if not notanalyzed and os.path.isfile(
resultsfilename): # data exists..
DataMachine = pd.read_hdf(resultsfilename)
DataCombined = pd.concat([Data.T, DataMachine.T], axis=0).T
RMSE, RMSEpcutoff = pairwisedistances(
DataCombined,
cfg["scorer"],
DLCscorer,
cfg["pcutoff"],
comparisonbodyparts,
)
testerror = np.nanmean(RMSE.iloc[testIndices].values.flatten())
trainerror = np.nanmean(
RMSE.iloc[trainIndices].values.flatten())
testerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[testIndices].values.flatten())
trainerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[trainIndices].values.flatten())
if show_errors == True:
print(
"Results for",
trainingsiterations,
" training iterations:",
int(100 * trainFraction),
shuffle,
"train error:",
np.round(trainerror, 2),
"pixels. Test error:",
np.round(testerror, 2),
" pixels.",
)
print(
"With pcutoff of",
cfg["pcutoff"],
" train error:",
np.round(trainerrorpcutoff, 2),
"pixels. Test error:",
np.round(testerrorpcutoff, 2),
"pixels",
)
print("Snapshot", Snapshots[snapindex])
r = [
trainingsiterations,
int(100 * trainFraction),
shuffle,
np.round(trainerror, 2),
np.round(testerror, 2),
cfg["pcutoff"],
np.round(trainerrorpcutoff, 2),
np.round(testerrorpcutoff, 2),
Snapshots[snapindex],
scale,
dlc_cfg["net_type"],
]
results.append(r)
else:
print("Model not trained/evaluated!")
if fulldata == True:
DATA.append([
DataMachine,
Data,
data,
trainIndices,
testIndices,
trainFraction,
DLCscorer,
comparisonbodyparts,
cfg,
evaluationfolder,
Snapshots[snapindex],
])
os.chdir(start_path)
if returnjustfns:
return resultsfns
else:
if fulldata == True:
return DATA, results
else:
return results
def load_model(cfg, shuffle=1, trainingsetindex=0, TFGPUinference=True, modelprefix=""):
"""
Loads a tensorflow session with a DLC model from the associated configuration
Return a tensorflow session with DLC model given cfg and shuffle
Parameters:
-----------
cfg : dict
Configuration read from the project's main config.yaml file
shuffle : int, optional
which shuffle to use
trainingsetindex : int. optional
which training fraction to use, identified by its index
TFGPUinference : bool, optional
use tensorflow inference model? default = True
Returns:
--------
sess : tensorflow session
tensorflow session with DLC model from the provided configuration, shuffle, and trainingsetindex
checkpoint file path : string
the path to the checkpoint file associated with the loaded model
"""
########################
### find snapshot to use
########################
train_fraction = cfg["TrainingFraction"][trainingsetindex]
model_folder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
train_fraction, shuffle, cfg, modelprefix=modelprefix
)
),
)
path_test_config = os.path.normpath(model_folder + "/test/pose_cfg.yaml")
path_train_config = os.path.normpath(model_folder + "/train/pose_cfg.yaml")
try:
dlc_cfg = load_config(str(path_train_config))
# dlc_cfg_train = load_config(str(path_train_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, train_fraction)
)
# Check which snapshots are available and sort them by # iterations
try:
Snapshots = np.array(
[
fn.split(".")[0]
for fn in os.listdir(os.path.join(model_folder, "train"))
if "index" in fn
]
)
except FileNotFoundError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before trying to export.\n Use the function 'train_network' to train the network for shuffle %s."
% (shuffle, shuffle)
)
if len(Snapshots) == 0:
raise FileNotFoundError(
"The train folder for iteration %s and shuffle %s exists, but no snapshots were found.\n Please train this model before trying to export.\n Use the function 'train_network' to train the network for iteration %s shuffle %s."
% (cfg["iteration"], shuffle, cfg["iteration"], shuffle)
)
if cfg["snapshotindex"] == "all":
print(
"Snapshotindex is set to 'all' in the config.yaml file. Changing snapshot index to -1!"
)
snapshotindex = -1
else:
snapshotindex = cfg["snapshotindex"]
increasing_indices = np.argsort([int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
####################################
### Load and setup CNN part detector
####################################
# Check if data already was generated:
dlc_cfg["init_weights"] = os.path.join(
model_folder, "train", Snapshots[snapshotindex]
)
trainingsiterations = (dlc_cfg["init_weights"].split(os.sep)[-1]).split("-")[-1]
dlc_cfg["num_outputs"] = cfg.get("num_outputs", dlc_cfg.get("num_outputs", 1))
dlc_cfg["batch_size"] = None
# load network
if TFGPUinference:
sess, _, _ = predict.setup_GPUpose_prediction(dlc_cfg)
output = ["concat_1"]
else:
sess, _, _ = predict.setup_pose_prediction(dlc_cfg)
if dlc_cfg["location_refinement"]:
output = ["Sigmoid", "pose/locref_pred/block4/BiasAdd"]
else:
output = ["Sigmoid", "pose/part_pred/block4/BiasAdd"]
input = tf.get_default_graph().get_operations()[0].name
return sess, input, output, dlc_cfg
def update_project_paths(self):
print('Updating paths..')
from deeplabcut.utils import auxiliaryfunctions
# load config.yaml
main_config = auxiliaryfunctions.read_config(self.full_config_path())
# Update path in main config
new_project_dir = self.project_dir.resolve()
main_config['project_path'] = str(new_project_dir)
# Update video path. NOTE: the video path name
for old_vid in list(main_config["video_sets"]):
new_vid = str(new_project_dir / "videos" / Path(old_vid).name)
main_config["video_sets"][new_vid] = main_config["video_sets"].pop(
old_vid)
# Write dictionary to yaml config file
auxiliaryfunctions.write_config(self.full_config_path(), main_config)
# Update train and test config.yaml paths
trainingsetindex = 0
shuffle = 1
modelfoldername = auxiliaryfunctions.GetModelFolder(
main_config["TrainingFraction"][trainingsetindex], shuffle,
main_config)
path_train_config = os.path.join(main_config['project_path'],
Path(modelfoldername), 'train',
'pose_cfg.yaml')
path_test_config = os.path.join(main_config['project_path'],
Path(modelfoldername), 'test',
'pose_cfg.yaml')
# Update training pose_cfg.yaml
if os.path.exists(path_train_config):
#train(str(poseconfigfile),displayiters,saveiters,maxiters,max_to_keep=max_snapshots_to_keep) #pass on path and file name for pose_cfg.yaml!
with open(path_train_config, "r") as ymlfile:
cfg_train = yaml.load(ymlfile, Loader=yaml.FullLoader)
cfg_train['project_path'] = str(new_project_dir)
old_dataset_train = os.path.join(*cfg_train['dataset'].split(
'\\')) #str(Path(cfg_train['dataset']))
cfg_train['dataset'] = old_dataset_train
old_metadataset = os.path.join(*cfg_train['metadataset'].split(
'\\')) #str(Path(cfg_train['metadataset']))
cfg_train['metadataset'] = old_metadataset
# if Path(Path.cwd().parent /
# init_loc = input("Please specificy directory to resnet_v1_50.ckpt")
cfg_train['init_weights'] = str(Path.cwd().parent /
"resnet_v1_50.ckpt")
with open(path_train_config, 'w') as ymlfile:
yaml.dump(cfg_train, ymlfile)
# Update MATLAB file contining training files
if os.path.exists(self.project_dir / cfg_train['dataset']):
import scipy.io as sio
# Load Matlab file dataset annotation
mlab = sio.loadmat(self.project_dir / cfg_train['dataset'])
num_images = mlab['dataset'].shape[1]
for i in range(num_images):
oldFilePath = mlab['dataset'][0, i][0][0]
newFilePath = os.path.join(
*oldFilePath.split('\\')) #str(Path(oldFilePath))
mlab['dataset'][0, i][0][0] = newFilePath
# Saving mat file
sio.savemat(
os.path.join(self.project_dir / cfg_train['dataset']),
mlab)
# Update testing pose_cfg.yaml
if os.path.exists(path_test_config):
#train(str(poseconfigfile),displayiters,saveiters,maxiters,max_to_keep=max_snapshots_to_keep) #pass on path and file name for pose_cfg.yaml!
with open(path_test_config, "r") as ymlfile:
cfg_test = yaml.load(ymlfile, Loader=yaml.FullLoader)
cfg_test['init_weights'] = str(Path.cwd().parent /
"resnet_v1_50.ckpt")
old_dataset_test = os.path.join(*cfg_test['dataset'].split(
'\\')) #str(Path(cfg_test['dataset']))
cfg_test['dataset'] = old_dataset_test
with open(path_test_config, 'w') as ymlfile:
yaml.dump(cfg_test, ymlfile)
print('done.')
def create_multianimaltraining_dataset(
config,
num_shuffles=1,
Shuffles=None,
windows2linux=False,
net_type=None,
numdigits=2,
crop_size=(400, 400),
crop_sampling="hybrid",
paf_graph=None,
trainIndices=None,
testIndices=None,
):
"""
Creates a training dataset for multi-animal datasets. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Imporant differences to standard:
- stores coordinates with numdigits as many digits
- creates
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
net_type: string
Type of networks. Currently resnet_50, resnet_101, and resnet_152, efficientnet-b0, efficientnet-b1, efficientnet-b2, efficientnet-b3,
efficientnet-b4, efficientnet-b5, and efficientnet-b6 as well as dlcrnet_ms5 are supported (not the MobileNets!).
See Lauer et al. 2021 https://www.biorxiv.org/content/10.1101/2021.04.30.442096v1
numdigits: int, optional
crop_size: tuple of int, optional
Dimensions (width, height) of the crops for data augmentation.
Default is 400x400.
crop_sampling: str, optional
Crop centers sampling method. Must be either:
"uniform" (randomly over the image),
"keypoints" (randomly over the annotated keypoints),
"density" (weighing preferentially dense regions of keypoints),
or "hybrid" (alternating randomly between "uniform" and "density").
Default is "hybrid".
paf_graph: list of lists, optional (default=None)
If not None, overwrite the default complete graph. This is useful for advanced users who
already know a good graph, or simply want to use a specific one. Note that, in that case,
the data-driven selection procedure upon model evaluation will be skipped.
trainIndices: list of lists, optional (default=None)
List of one or multiple lists containing train indexes.
A list containing two lists of training indexes will produce two splits.
testIndices: list of lists, optional (default=None)
List of one or multiple lists containing test indexes.
Example
--------
>>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
>>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml', Shuffles=[0,1,2], trainIndices=[trainInd1, trainInd2, trainInd3], testIndices=[testInd1, testInd2, testInd3])
Windows:
>>> deeplabcut.create_multianimaltraining_dataset(r'C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
if windows2linux:
warnings.warn(
"`windows2linux` has no effect since 2.2.0.4 and will be removed in 2.2.1.",
FutureWarning,
)
if len(crop_size) != 2 or not all(isinstance(v, int) for v in crop_size):
raise ValueError(
"Crop size must be a tuple of two integers (width, height).")
if crop_sampling not in ("uniform", "keypoints", "density", "hybrid"):
raise ValueError(
f"Invalid sampling {crop_sampling}. Must be "
f"either 'uniform', 'keypoints', 'density', or 'hybrid.")
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg["scorer"]
project_path = cfg["project_path"]
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
full_training_path = Path(project_path, trainingsetfolder)
auxiliaryfunctions.attempttomakefolder(full_training_path, recursive=True)
Data = merge_annotateddatasets(cfg, full_training_path)
if Data is None:
return
Data = Data[scorer]
if net_type is None: # loading & linking pretrained models
net_type = cfg.get("default_net_type", "dlcrnet_ms5")
elif not any(net in net_type for net in ("resnet", "eff", "dlc", "mob")):
raise ValueError(f"Unsupported network {net_type}.")
multi_stage = False
### dlcnet_ms5: backbone resnet50 + multi-fusion & multi-stage module
### dlcr101_ms5/dlcr152_ms5: backbone resnet101/152 + multi-fusion & multi-stage module
if all(net in net_type for net in ("dlcr", "_ms5")):
num_layers = re.findall("dlcr([0-9]*)", net_type)[0]
if num_layers == "":
num_layers = 50
net_type = "resnet_{}".format(num_layers)
multi_stage = True
dataset_type = "multi-animal-imgaug"
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
if paf_graph is None: # Automatically form a complete PAF graph
partaffinityfield_graph = [
list(edge)
for edge in combinations(range(len(multianimalbodyparts)), 2)
]
else:
# Ignore possible connections between 'multi' and 'unique' body parts;
# one can never be too careful...
to_ignore = auxfun_multianimal.filter_unwanted_paf_connections(
cfg, paf_graph)
partaffinityfield_graph = [
edge for i, edge in enumerate(paf_graph) if i not in to_ignore
]
auxfun_multianimal.validate_paf_graph(cfg, partaffinityfield_graph)
print("Utilizing the following graph:", partaffinityfield_graph)
# Disable the prediction of PAFs if the graph is empty
partaffinityfield_predict = bool(partaffinityfield_graph)
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, Path(dlcparent_path), num_shuffles)
if Shuffles is None:
Shuffles = range(1, num_shuffles + 1, 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
# print(trainIndices,testIndices, Shuffles, augmenter_type,net_type)
if trainIndices is None and testIndices is None:
splits = []
for shuffle in Shuffles: # Creating shuffles starting from 1
for train_frac in cfg["TrainingFraction"]:
train_inds, test_inds = SplitTrials(range(len(Data)),
train_frac)
splits.append((train_frac, shuffle, (train_inds, test_inds)))
else:
if len(trainIndices) != len(testIndices) != len(Shuffles):
raise ValueError(
"Number of Shuffles and train and test indexes should be equal."
)
splits = []
for shuffle, (train_inds,
test_inds) in enumerate(zip(trainIndices, testIndices)):
trainFraction = round(
len(train_inds) * 1.0 / (len(train_inds) + len(test_inds)), 2)
print(
f"You passed a split with the following fraction: {int(100 * trainFraction)}%"
)
# Now that the training fraction is guaranteed to be correct,
# the values added to pad the indices are removed.
train_inds = np.asarray(train_inds)
train_inds = train_inds[train_inds != -1]
test_inds = np.asarray(test_inds)
test_inds = test_inds[test_inds != -1]
splits.append(
(trainFraction, Shuffles[shuffle], (train_inds, test_inds)))
for trainFraction, shuffle, (trainIndices, testIndices) in splits:
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
print(
"Creating training data for: Shuffle:",
shuffle,
"TrainFraction: ",
trainFraction,
)
# Make training file!
data = format_multianimal_training_data(
Data,
trainIndices,
cfg["project_path"],
numdigits,
)
if len(trainIndices) > 0:
(
datafilename,
metadatafilename,
) = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
################################################################################
# Saving metadata and data file (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename),
data,
trainIndices,
testIndices,
trainFraction,
)
datafilename = datafilename.split(".mat")[0] + ".pickle"
import pickle
with open(os.path.join(project_path, datafilename), "wb") as f:
# Pickle the 'labeled-data' dictionary using the highest protocol available.
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
auxiliaryfunctions.attempttomakefolder(Path(config).parents[0] /
modelfoldername,
recursive=True)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "train"))
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "test"))
path_train_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"train",
"pose_cfg.yaml",
))
path_test_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"pose_cfg.yaml",
))
path_inference_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"inference_cfg.yaml",
))
jointnames = [str(bpt) for bpt in multianimalbodyparts]
jointnames.extend([str(bpt) for bpt in uniquebodyparts])
items2change = {
"dataset":
datafilename,
"metadataset":
metadatafilename,
"num_joints":
len(multianimalbodyparts) +
len(uniquebodyparts), # cfg["uniquebodyparts"]),
"all_joints": [[i] for i in range(
len(multianimalbodyparts) + len(uniquebodyparts))
], # cfg["uniquebodyparts"]))],
"all_joints_names":
jointnames,
"init_weights":
model_path,
"project_path":
str(cfg["project_path"]),
"net_type":
net_type,
"multi_stage":
multi_stage,
"pairwise_loss_weight":
0.1,
"pafwidth":
20,
"partaffinityfield_graph":
partaffinityfield_graph,
"partaffinityfield_predict":
partaffinityfield_predict,
"weigh_only_present_joints":
False,
"num_limbs":
len(partaffinityfield_graph),
"dataset_type":
dataset_type,
"optimizer":
"adam",
"batch_size":
8,
"multi_step": [[1e-4, 7500], [5 * 1e-5, 12000], [1e-5,
200000]],
"save_iters":
10000,
"display_iters":
500,
"num_idchannel":
len(cfg["individuals"]) if cfg.get("identity", False) else 0,
"crop_size":
list(crop_size),
"crop_sampling":
crop_sampling,
}
trainingdata = MakeTrain_pose_yaml(items2change, path_train_config,
defaultconfigfile)
keys2save = [
"dataset",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"multi_stage",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
"dataset_type",
"partaffinityfield_predict",
"pairwise_predict",
"partaffinityfield_graph",
"num_limbs",
"dataset_type",
"num_idchannel",
]
MakeTest_pose_yaml(
trainingdata,
keys2save,
path_test_config,
nmsradius=5.0,
minconfidence=0.01,
sigma=1,
locref_smooth=False,
) # setting important def. values for inference
# Setting inference cfg file:
defaultinference_configfile = os.path.join(dlcparent_path,
"inference_cfg.yaml")
items2change = {
"minimalnumberofconnections":
int(len(cfg["multianimalbodyparts"]) / 2),
"topktoretain":
len(cfg["individuals"]) + 1 *
(len(cfg["uniquebodyparts"]) > 0),
"withid":
cfg.get("identity", False),
}
MakeInference_yaml(items2change, path_inference_config,
defaultinference_configfile)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
else:
pass
def create_training_dataset(config,
num_shuffles=1,
Shuffles=None,
windows2linux=False):
"""
Creates a training dataset. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL]Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
Example
--------
>>> deeplabcut.create_training_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcut.create_training_dataset('C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
from skimage import io
import scipy.io as sio
import deeplabcut
import subprocess
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg['scorer']
project_path = cfg['project_path']
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(
cfg) #Path concatenation OS platform independent
auxiliaryfunctions.attempttomakefolder(Path(
os.path.join(project_path, str(trainingsetfolder))),
recursive=True)
Data = merge_annotateddatasets(
cfg, project_path, Path(os.path.join(project_path, trainingsetfolder)),
windows2linux)
Data = Data[scorer] #extract labeled data
#set model type. we will allow more in the future.
if cfg['resnet'] == 50:
net_type = 'resnet_' + str(cfg['resnet'])
resnet_path = str(
Path(deeplabcut.__file__).parents[0] /
'pose_estimation_tensorflow/models/pretrained/resnet_v1_50.ckpt')
elif cfg['resnet'] == 101:
net_type = 'resnet_' + str(cfg['resnet'])
resnet_path = str(
Path(deeplabcut.__file__).parents[0] /
'pose_estimation_tensorflow/models/pretrained/resnet_v1_101.ckpt')
else:
print(
"Currently only ResNet 50 or 101 supported, please change 'resnet' entry in config.yaml!"
)
num_shuffles = -1 #thus the loop below is empty...
if not Path(resnet_path).is_file():
"""
Downloads the ImageNet pretrained weights for ResNet.
"""
start = os.getcwd()
os.chdir(str(Path(resnet_path).parents[0]))
print("Downloading the pretrained model (ResNets)....")
subprocess.call("download.sh", shell=True)
os.chdir(start)
if Shuffles == None:
Shuffles = range(1, num_shuffles + 1, 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
bodyparts = cfg['bodyparts']
TrainingFraction = cfg['TrainingFraction']
for shuffle in Shuffles: # Creating shuffles starting from 1
for trainFraction in TrainingFraction:
trainIndexes, testIndexes = SplitTrials(range(len(Data.index)),
trainFraction)
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
# Make training file!
data = []
for jj in trainIndexes:
H = {}
# load image to get dimensions:
filename = Data.index[jj]
im = io.imread(os.path.join(cfg['project_path'], filename))
H['image'] = filename
if np.ndim(im) == 3:
H['size'] = np.array(
[np.shape(im)[2],
np.shape(im)[0],
np.shape(im)[1]])
else:
# print "Grayscale!"
H['size'] = np.array([1, np.shape(im)[0], np.shape(im)[1]])
indexjoints = 0
joints = np.zeros((len(bodyparts), 3)) * np.nan
for bpindex, bodypart in enumerate(bodyparts):
if Data[bodypart]['x'][jj] < np.shape(
im)[1] and Data[bodypart]['y'][jj] < np.shape(
im)[0]: #are labels in image?
joints[indexjoints, 0] = int(bpindex)
joints[indexjoints, 1] = Data[bodypart]['x'][jj]
joints[indexjoints, 2] = Data[bodypart]['y'][jj]
indexjoints += 1
joints = joints[np.where(np.prod(
np.isfinite(joints),
1))[0], :] # drop NaN, i.e. lines for missing body parts
assert (np.prod(np.array(joints[:, 2]) < np.shape(im)[0])
) # y coordinate within image?
assert (np.prod(np.array(joints[:, 1]) < np.shape(im)[1])
) # x coordinate within image?
H['joints'] = np.array(joints, dtype=int)
if np.size(joints) > 0: #exclude images without labels
data.append(H)
if len(trainIndexes) > 0:
datafilename, metadatafilename = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
################################################################################
# Saving metadata (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename), data,
trainIndexes, testIndexes, trainFraction)
################################################################################
# Saving data file (convert to training file for deeper cut (*.mat))
################################################################################
DTYPE = [('image', 'O'), ('size', 'O'), ('joints', 'O')]
MatlabData = np.array(
[(np.array([data[item]['image']],
dtype='U'), np.array([data[item]['size']]),
boxitintoacell(data[item]['joints']))
for item in range(len(data))],
dtype=DTYPE)
sio.savemat(os.path.join(project_path, datafilename),
{'dataset': MatlabData})
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
auxiliaryfunctions.attempttomakefolder(
Path(config).parents[0] / modelfoldername, recursive=True)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername) + '/' +
'/train')
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername) + '/' +
'/test')
path_train_config = str(
os.path.join(cfg['project_path'], Path(modelfoldername),
'train', 'pose_cfg.yaml'))
path_test_config = str(
os.path.join(cfg['project_path'], Path(modelfoldername),
'test', 'pose_cfg.yaml'))
#str(cfg['proj_path']+'/'+Path(modelfoldername) / 'test' / 'pose_cfg.yaml')
items2change = {
"dataset": datafilename,
"metadataset": metadatafilename,
"num_joints": len(bodyparts),
"all_joints": [[i] for i in range(len(bodyparts))],
"all_joints_names": [str(bpt) for bpt in bodyparts],
"init_weights": resnet_path,
"project_path": str(cfg['project_path']),
"net_type": net_type
}
defaultconfigfile = str(
Path(deeplabcut.__file__).parents[0] / 'pose_cfg.yaml')
trainingdata = MakeTrain_pose_yaml(items2change,
path_train_config,
defaultconfigfile)
keys2save = [
"dataset", "num_joints", "all_joints", "all_joints_names",
"net_type", 'init_weights', 'global_scale',
'location_refinement', 'locref_stdev'
]
MakeTest_pose_yaml(trainingdata, keys2save, path_test_config)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
def evaluate_multianimal_full(
config,
Shuffles=[1],
trainingsetindex=0,
plotting=None,
show_errors=True,
comparisonbodyparts="all",
gputouse=None,
modelprefix="",
c_engine=False,
):
"""
WIP multi animal project.
"""
import os
from deeplabcut.pose_estimation_tensorflow.nnet import predict
from deeplabcut.pose_estimation_tensorflow.nnet import (
predict_multianimal as predictma, )
from deeplabcut.utils import auxiliaryfunctions, auxfun_multianimal
import tensorflow as tf
if "TF_CUDNN_USE_AUTOTUNE" in os.environ:
del os.environ[
"TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training
tf.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
start_path = os.getcwd()
##################################################
# Load data...
##################################################
cfg = auxiliaryfunctions.read_config(config)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
TrainingFractions = [cfg["TrainingFraction"][trainingsetindex]]
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
)
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
colors = visualization.get_cmap(len(comparisonbodyparts),
name=cfg["colormap"])
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# TODO: IMPLEMENT for different batch sizes?
dlc_cfg["batch_size"] = 1 # due to differently sized images!!!
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
# path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
])
if len(Snapshots) == 0:
print(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
else:
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
final_result = []
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg["init_weights"].split(os.sep)[-1]
).split(
"-"
)[-1] # read how many training siterations that corresponds to.
# name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix,
)
print(
"Running ",
DLCscorer,
" with # of trainingiterations:",
trainingsiterations,
)
(
notanalyzed,
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.CheckifNotEvaluated(
str(evaluationfolder),
DLCscorer,
DLCscorerlegacy,
Snapshots[snapindex],
)
if os.path.isfile(
resultsfilename.split(".h5")[0] + "_full.pickle"):
print("Model already evaluated.", resultsfilename)
else:
if plotting:
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" +
Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
# print(dlc_cfg)
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = predict.setup_pose_prediction(
dlc_cfg)
PredicteData = {}
print("Analyzing data...")
for imageindex, imagename in tqdm(enumerate(
Data.index)):
image_path = os.path.join(cfg["project_path"],
imagename)
image = io.imread(image_path)
frame = img_as_ubyte(skimage.color.gray2rgb(image))
GT = Data.iloc[imageindex]
# Storing GT data as dictionary, so it can be used for calculating connection costs
groundtruthcoordinates = []
groundtruthidentity = []
for bptindex, bpt in enumerate(
dlc_cfg["all_joints_names"]):
coords = np.zeros([len(individuals), 2
]) * np.nan
identity = []
for prfxindex, prefix in enumerate(
individuals):
if bpt in uniquebodyparts and prefix == "single":
coords[prfxindex, :] = np.array([
GT[cfg["scorer"]][prefix][bpt]
["x"],
GT[cfg["scorer"]][prefix][bpt]
["y"],
])
identity.append(prefix)
elif (bpt in multianimalbodyparts
and prefix != "single"):
coords[prfxindex, :] = np.array([
GT[cfg["scorer"]][prefix][bpt]
["x"],
GT[cfg["scorer"]][prefix][bpt]
["y"],
])
identity.append(prefix)
else:
identity.append("nix")
groundtruthcoordinates.append(
coords[np.isfinite(coords[:, 0]), :])
groundtruthidentity.append(
np.array(identity)[np.isfinite(coords[:,
0])])
PredicteData[imagename] = {}
PredicteData[imagename]["index"] = imageindex
pred = predictma.get_detectionswithcostsandGT(
frame,
groundtruthcoordinates,
dlc_cfg,
sess,
inputs,
outputs,
outall=False,
nms_radius=dlc_cfg.nmsradius,
det_min_score=dlc_cfg.minconfidence,
c_engine=c_engine,
)
PredicteData[imagename]["prediction"] = pred
PredicteData[imagename]["groundtruth"] = [
groundtruthidentity,
groundtruthcoordinates,
GT,
]
if plotting:
coords_pred = pred["coordinates"][0]
probs_pred = pred["confidence"]
fig = visualization.make_multianimal_labeled_image(
frame,
groundtruthcoordinates,
coords_pred,
probs_pred,
colors,
cfg["dotsize"],
cfg["alphavalue"],
cfg["pcutoff"],
)
visualization.save_labeled_frame(
fig,
image_path,
foldername,
imageindex in trainIndices,
)
sess.close() # closes the current tf session
PredicteData["metadata"] = {
"nms radius":
dlc_cfg.nmsradius,
"minimal confidence":
dlc_cfg.minconfidence,
"PAFgraph":
dlc_cfg.partaffinityfield_graph,
"all_joints":
[[i] for i in range(len(dlc_cfg.all_joints))],
"all_joints_names": [
dlc_cfg.all_joints_names[i]
for i in range(len(dlc_cfg.all_joints))
],
"stride":
dlc_cfg.get("stride", 8),
}
print(
"Done and results stored for snapshot: ",
Snapshots[snapindex],
)
dictionary = {
"Scorer": DLCscorer,
"DLC-model-config file": dlc_cfg,
"trainIndices": trainIndices,
"testIndices": testIndices,
"trainFraction": trainFraction,
}
metadata = {"data": dictionary}
auxfun_multianimal.SaveFullMultiAnimalData(
PredicteData, metadata, resultsfilename)
tf.reset_default_graph()
# returning to intial folder
os.chdir(str(start_path))
def analyze_videos(config,
videos,
videotype='avi',
shuffle=1,
trainingsetindex=0,
gputouse=None,
save_as_csv=False,
destfolder=None,
cropping=None):
"""
Makes prediction based on a trained network. The index of the trained network is specified by parameters in the config file (in particular the variable 'snapshotindex')
You can crop the video (before analysis), by changing 'cropping'=True and setting 'x1','x2','y1','y2' in the config file. The same cropping parameters will then be used for creating the video.
Note: you can also pass cropping = [x1,x2,y1,y2] coordinates directly, that then will be used for all videos. You can of course loop over videos & pass specific coordinates for each case.
Output: The labels are stored as MultiIndex Pandas Array, which contains the name of the network, body part name, (x, y) label position \n
in pixels, and the likelihood for each frame per body part. These arrays are stored in an efficient Hierarchical Data Format (HDF) \n
in the same directory, where the video is stored. However, if the flag save_as_csv is set to True, the data can also be exported in \n
comma-separated values format (.csv), which in turn can be imported in many programs, such as MATLAB, R, Prism, etc.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: int, optional
An integer specifying the shuffle index of the training dataset used for training the network. The default is 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this
folder also needs to be passed.
Examples
--------
Windows example for analyzing 1 video
>>> deeplabcut.analyze_videos('C:\\myproject\\reaching-task\\config.yaml',['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi'])
--------
If you want to analyze only 1 video
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to analyze all videos of type avi in a folder:
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos'],videotype='.avi')
--------
If you want to analyze multiple videos
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to analyze multiple videos with shuffle = 2
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'], shuffle=2)
--------
If you want to analyze multiple videos with shuffle = 2 and save results as an additional csv file too
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'], shuffle=2,save_as_csv=True)
--------
"""
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ[
'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
if gputouse is not None: #gpu selection
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
vers = (tf.__version__).split('.')
if int(vers[0]) == 1 and int(vers[1]) > 12:
TF = tf.compat.v1
else:
TF = tf
TF.reset_default_graph()
start_path = os.getcwd(
) #record cwd to return to this directory in the end
cfg = auxiliaryfunctions.read_config(config)
if cropping is not None:
cfg['cropping'] = True
cfg['x1'], cfg['x2'], cfg['y1'], cfg['y2'] = cropping
print("Overwriting cropping parameters:", cropping)
print(
"These are used for all videos, but won't be save to the cfg file."
)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
modelfolder = os.path.join(
cfg["project_path"],
str(auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# Check which snapshots are available and sort them by # iterations
try:
Snapshots = np.array([
fn.split('.')[0]
for fn in os.listdir(os.path.join(modelfolder, 'train'))
if "index" in fn
])
except FileNotFoundError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s."
% (shuffle, shuffle))
if cfg['snapshotindex'] == 'all':
print(
"Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!"
)
snapshotindex = -1
else:
snapshotindex = cfg['snapshotindex']
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder)
##################################################
# Load and setup CNN part detector
##################################################
# Check if data already was generated:
dlc_cfg['init_weights'] = os.path.join(modelfolder, 'train',
Snapshots[snapshotindex])
trainingsiterations = (dlc_cfg['init_weights'].split(
os.sep)[-1]).split('-')[-1]
#update batchsize (based on parameters in config.yaml)
dlc_cfg['batch_size'] = cfg['batch_size']
# update number of outputs
dlc_cfg['num_outputs'] = cfg.get('num_outputs', 1)
print('num_outputs = ', dlc_cfg['num_outputs'])
# Name for scorer:
DLCscorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, trainingsiterations=trainingsiterations)
sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
xyz_labs_orig = ['x', 'y', 'likelihood']
suffix = [str(s + 1) for s in range(dlc_cfg['num_outputs'])]
suffix[0] = '' # first one has empty suffix for backwards compatibility
xyz_labs = [x + s for s in suffix for x in xyz_labs_orig]
pdindex = pd.MultiIndex.from_product(
[[DLCscorer], dlc_cfg['all_joints_names'], xyz_labs],
names=['scorer', 'bodyparts', 'coords'])
##################################################
# Datafolder
##################################################
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
if len(Videos) > 0:
#looping over videos
for video in Videos:
AnalyzeVideo(video, DLCscorer, trainFraction, cfg, dlc_cfg, sess,
inputs, outputs, pdindex, save_as_csv, destfolder)
os.chdir(str(start_path))
print(
"The videos are analyzed. Now your research can truly start! \n You can create labeled videos with 'create_labeled_video'."
)
print(
"If the tracking is not satisfactory for some videos, consider expanding the training set. You can use the function 'extract_outlier_frames' to extract any outlier frames!"
)
else:
print("No video was found in the path/ or single video with path:",
videos)
print(
"Perhaps the videotype is distinct from the videos in the path, I was looking for:",
videotype)
return DLCscorer
def create_multianimaltraining_dataset(
config,
num_shuffles=1,
Shuffles=None,
windows2linux=False,
net_type=None,
numdigits=2,
paf_graph=None,
):
"""
Creates a training dataset for multi-animal datasets. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Imporant differences to standard:
- stores coordinates with numdigits as many digits
- creates
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
net_type: string
Type of networks. Currently resnet_50, resnet_101, and resnet_152, efficientnet-b0, efficientnet-b1, efficientnet-b2, efficientnet-b3,
efficientnet-b4, efficientnet-b5, and efficientnet-b6 as well as dlcrnet_ms5 are supported (not the MobileNets!).
See Lauer et al. 2021 https://www.biorxiv.org/content/10.1101/2021.04.30.442096v1
numdigits: int, optional
paf_graph: list of lists, optional (default=None)
If not None, overwrite the default complete graph. This is useful for advanced users who
already know a good graph, or simply want to use a specific one. Note that, in that case,
the data-driven selection procedure upon model evaluation will be skipped.
Example
--------
>>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcut.create_multianimaltraining_dataset(r'C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg["scorer"]
project_path = cfg["project_path"]
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
full_training_path = Path(project_path, trainingsetfolder)
auxiliaryfunctions.attempttomakefolder(full_training_path, recursive=True)
Data = merge_annotateddatasets(cfg, full_training_path, windows2linux)
if Data is None:
return
Data = Data[scorer]
def strip_cropped_image_name(path):
# utility function to split different crops from same image into either train or test!
head, filename = os.path.split(path)
if cfg["croppedtraining"]:
filename = filename.split("c")[0]
return os.path.join(head, filename)
img_names = Data.index.map(strip_cropped_image_name).unique()
if net_type is None: # loading & linking pretrained models
net_type = cfg.get("default_net_type", "dlcrnet_ms5")
elif not any(net in net_type for net in ("resnet", "eff", "dlc")):
raise ValueError(f"Unsupported network {net_type}.")
multi_stage = False
if net_type == "dlcrnet_ms5":
net_type = "resnet_50"
multi_stage = True
dataset_type = "multi-animal-imgaug"
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
if paf_graph is None: # Automatically form a complete PAF graph
partaffinityfield_graph = [
list(edge)
for edge in combinations(range(len(multianimalbodyparts)), 2)
]
else:
# Ignore possible connections between 'multi' and 'unique' body parts;
# one can never be too careful...
to_ignore = auxfun_multianimal.filter_unwanted_paf_connections(
cfg, paf_graph)
partaffinityfield_graph = [
edge for i, edge in enumerate(paf_graph) if i not in to_ignore
]
auxfun_multianimal.validate_paf_graph(cfg, partaffinityfield_graph)
print("Utilizing the following graph:", partaffinityfield_graph)
partaffinityfield_predict = True
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, Path(dlcparent_path), num_shuffles)
if Shuffles is None:
Shuffles = range(1, num_shuffles + 1, 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
TrainingFraction = cfg["TrainingFraction"]
for shuffle in Shuffles: # Creating shuffles starting from 1
for trainFraction in TrainingFraction:
train_inds_temp, test_inds_temp = SplitTrials(
range(len(img_names)), trainFraction)
# Map back to the original indices.
temp = [
re.escape(name) for i, name in enumerate(img_names)
if i in test_inds_temp
]
mask = Data.index.str.contains("|".join(temp))
testIndices = np.flatnonzero(mask)
trainIndices = np.flatnonzero(~mask)
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
print(
"Creating training data for: Shuffle:",
shuffle,
"TrainFraction: ",
trainFraction,
)
# Make training file!
data = format_multianimal_training_data(
Data,
trainIndices,
cfg["project_path"],
numdigits,
)
if len(trainIndices) > 0:
(
datafilename,
metadatafilename,
) = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
################################################################################
# Saving metadata and data file (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename),
data,
trainIndices,
testIndices,
trainFraction,
)
datafilename = datafilename.split(".mat")[0] + ".pickle"
import pickle
with open(os.path.join(project_path, datafilename), "wb") as f:
# Pickle the 'labeled-data' dictionary using the highest protocol available.
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
auxiliaryfunctions.attempttomakefolder(
Path(config).parents[0] / modelfoldername, recursive=True)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "train"))
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "test"))
path_train_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"train",
"pose_cfg.yaml",
))
path_test_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"pose_cfg.yaml",
))
path_inference_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"inference_cfg.yaml",
))
jointnames = [str(bpt) for bpt in multianimalbodyparts]
jointnames.extend([str(bpt) for bpt in uniquebodyparts])
items2change = {
"dataset":
datafilename,
"metadataset":
metadatafilename,
"num_joints":
len(multianimalbodyparts) +
len(uniquebodyparts), # cfg["uniquebodyparts"]),
"all_joints": [[i] for i in range(
len(multianimalbodyparts) + len(uniquebodyparts))
], # cfg["uniquebodyparts"]))],
"all_joints_names":
jointnames,
"init_weights":
model_path,
"project_path":
str(cfg["project_path"]),
"net_type":
net_type,
"multi_stage":
multi_stage,
"pairwise_loss_weight":
0.1,
"pafwidth":
20,
"partaffinityfield_graph":
partaffinityfield_graph,
"partaffinityfield_predict":
partaffinityfield_predict,
"weigh_only_present_joints":
False,
"num_limbs":
len(partaffinityfield_graph),
"dataset_type":
dataset_type,
"optimizer":
"adam",
"batch_size":
8,
"multi_step": [[1e-4, 7500], [5 * 1e-5, 12000],
[1e-5, 200000]],
"save_iters":
10000,
"display_iters":
500,
"num_idchannel":
len(cfg["individuals"])
if cfg.get("identity", False) else 0,
}
trainingdata = MakeTrain_pose_yaml(items2change,
path_train_config,
defaultconfigfile)
keys2save = [
"dataset",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"multi_stage",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
"dataset_type",
"partaffinityfield_predict",
"pairwise_predict",
"partaffinityfield_graph",
"num_limbs",
"dataset_type",
"num_idchannel",
]
MakeTest_pose_yaml(
trainingdata,
keys2save,
path_test_config,
nmsradius=5.0,
minconfidence=0.01,
) # setting important def. values for inference
# Setting inference cfg file:
defaultinference_configfile = os.path.join(
dlcparent_path, "inference_cfg.yaml")
items2change = {
"minimalnumberofconnections":
int(len(cfg["multianimalbodyparts"]) / 2),
"topktoretain":
len(cfg["individuals"]) + 1 *
(len(cfg["uniquebodyparts"]) > 0),
"withid":
cfg.get("identity", False),
}
MakeInference_yaml(items2change, path_inference_config,
defaultinference_configfile)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
else:
pass
def bayesian_search(
config_path,
inferencecfg,
pbounds,
edgewisecondition=True,
shuffle=1,
trainingsetindex=0,
modelprefix="",
snapshotindex=-1,
target="rpck_test",
maximize=True,
init_points=20,
n_iter=50,
acq="ei",
log_file=None,
dcorr=5,
leastbpts=3,
printingintermediatevalues=True,
): #
if "rpck" in target:
assert maximize == True
if "rmse" in target:
assert maximize == False
cfg = auxiliaryfunctions.read_config(config_path)
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
cfg["TrainingFraction"][int(trainingsetindex)],
shuffle,
cfg,
modelprefix=modelprefix,
)),
)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
cfg["TrainingFraction"][int(trainingsetindex)],
cfg["iteration"],
modelprefix=modelprefix,
)
# load params
fns = return_evaluate_network_data(
config_path,
shuffle=shuffle,
trainingsetindex=trainingsetindex,
modelprefix=modelprefix,
)
predictionsfn = fns[snapshotindex]
data, metadata = auxfun_multianimal.LoadFullMultiAnimalData(predictionsfn)
params = set_up_evaluation(data)
columns = ["train_iter", "train_frac", "shuffle"]
columns += [
"_".join((b, a)) for a in ("train", "test")
for b in ("rmse", "hits", "misses", "falsepos", "ndetects", "pck",
"rpck")
]
train_iter = trainingsetindex # int(predictionsfn.split('-')[-1].split('.')[0])
train_frac = cfg["TrainingFraction"][
train_iter] # int(predictionsfn.split('trainset')[1].split('shuffle')[0])
trainIndices = metadata["data"]["trainIndices"]
testIndices = metadata["data"]["testIndices"]
if edgewisecondition:
mf = str(
auxiliaryfunctions.GetModelFolder(
cfg["TrainingFraction"][int(trainingsetindex)],
shuffle,
cfg,
modelprefix=modelprefix,
))
modelfolder = os.path.join(cfg["project_path"], mf)
path_inferencebounds_config = (Path(modelfolder) / "test" /
"inferencebounds.yaml")
try:
inferenceboundscfg = auxiliaryfunctions.read_plainconfig(
path_inferencebounds_config)
except FileNotFoundError:
print("Computing distances...")
from deeplabcut.pose_estimation_tensorflow import calculatepafdistancebounds
inferenceboundscfg = calculatepafdistancebounds(
config_path, shuffle, trainingsetindex)
auxiliaryfunctions.write_plainconfig(path_inferencebounds_config,
inferenceboundscfg)
partaffinityfield_graph = params["paf_graph"]
upperbound = np.array([
float(inferenceboundscfg[str(edge[0]) + "_" +
str(edge[1])]["intra_max"])
for edge in partaffinityfield_graph
])
lowerbound = np.array([
float(inferenceboundscfg[str(edge[0]) + "_" +
str(edge[1])]["intra_min"])
for edge in partaffinityfield_graph
])
upperbound *= inferencecfg["upperbound_factor"]
lowerbound *= inferencecfg["lowerbound_factor"]
else:
lowerbound = None
upperbound = None
def dlc_hyperparams(**kwargs):
inferencecfg.update(kwargs)
# Ensure type consistency
for k, (bound, _) in pbounds.items():
inferencecfg[k] = type(bound)(inferencecfg[k])
stats = compute_crossval_metrics_preloadeddata(
params,
columns,
inferencecfg,
data,
trainIndices,
testIndices,
train_iter,
train_frac,
shuffle,
lowerbound,
upperbound,
dcorr=dcorr,
leastbpts=leastbpts,
)
# stats = compute_crossval_metrics(config_path, inferencecfg, shuffle,trainingsetindex,
# dcorr=dcorr,leastbpts=leastbpts,modelprefix=modelprefix)
if printingintermediatevalues:
print(
"rpck",
stats["rpck_test"].values[0],
"rpck train:",
stats["rpck_train"].values[0],
)
print(
"rmse",
stats["rmse_test"].values[0],
"miss",
stats["misses_test"].values[0],
"hit",
stats["hits_test"].values[0],
)
# val = stats['rmse_test'].values[0]*(1+stats['misses_test'].values[0]*1./stats['hits_test'].values[0])
val = stats[target].values[0]
if np.isnan(val):
if maximize: # pck case
val = -1e9 # random small number
else: # RMSE, return a large RMSE
val = 1e9
if not maximize:
val = -val
return val
opt = BayesianOptimization(f=dlc_hyperparams,
pbounds=pbounds,
random_state=42)
if log_file:
load_logs(opt, log_file)
logger = JSONLogger(path=os.path.join(evaluationfolder, "opti_log" +
DLCscorer + ".json"))
opt.subscribe(Events.OPTIMIZATION_STEP, logger)
opt.maximize(init_points=init_points, n_iter=n_iter, acq=acq)
inferencecfg.update(opt.max["params"])
for k, (bound, _) in pbounds.items():
tmp = type(bound)(inferencecfg[k])
if isinstance(tmp, np.floating):
tmp = np.round(tmp, 2).item()
inferencecfg[k] = tmp
return inferencecfg, opt
def analyze_videos(config,videos,shuffle=1,trainingsetindex=0,videotype='avi',gputouse=None,save_as_csv=False, destfolder=None):
"""
Makes prediction based on a trained network. The index of the trained network is specified by parameters in the config file (in particular the variable 'snapshotindex')
You can crop the video (before analysis), by changing 'cropping'=True and setting 'x1','x2','y1','y2' in the config file. The same cropping parameters will then be used for creating the video.
Output: The labels are stored as MultiIndex Pandas Array, which contains the name of the network, body part name, (x, y) label position \n
in pixels, and the likelihood for each frame per body part. These arrays are stored in an efficient Hierarchical Data Format (HDF) \n
in the same directory, where the video is stored. However, if the flag save_as_csv is set to True, the data can also be exported in \n
comma-separated values format (.csv), which in turn can be imported in many programs, such as MATLAB, R, Prism, etc.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory where all the videos with same extension are stored.
shuffle: int, optional
An integer specifying the shuffle index of the training dataset used for training the network. The default is 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\nOnly videos with this extension are analyzed. The default is ``.avi``
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video)
Examples
--------
If you want to analyze only 1 video
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to analyze all videos of type avi in a folder:
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos'],videotype='.avi')
--------
If you want to analyze multiple videos
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to analyze multiple videos with shuffle = 2
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'], shuffle=2)
--------
If you want to analyze multiple videos with shuffle = 2 and save results as an additional csv file too
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'], shuffle=2,save_as_csv=True)
--------
"""
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ['TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
tf.reset_default_graph()
start_path=os.getcwd() #record cwd to return to this directory in the end
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
modelfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError("It seems the model for shuffle %s and trainFraction %s does not exist."%(shuffle,trainFraction))
# Check which snapshots are available and sort them by # iterations
try:
Snapshots = np.array([fn.split('.')[0]for fn in os.listdir(os.path.join(modelfolder , 'train'))if "index" in fn])
except FileNotFoundError:
raise FileNotFoundError("Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s."%(shuffle,shuffle))
if cfg['snapshotindex'] == 'all':
print("Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!")
snapshotindex = -1
else:
snapshotindex=cfg['snapshotindex']
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder)
##################################################
# Load and setup CNN part detector
##################################################
# Check if data already was generated:
dlc_cfg['init_weights'] = os.path.join(modelfolder , 'train', Snapshots[snapshotindex])
trainingsiterations = (dlc_cfg['init_weights'].split(os.sep)[-1]).split('-')[-1]
#update batchsize (based on parameters in config.yaml)
dlc_cfg['batch_size']=cfg['batch_size']
# Name for scorer:
DLCscorer = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations=trainingsiterations)
sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
pdindex = pd.MultiIndex.from_product([[DLCscorer], dlc_cfg['all_joints_names'], ['x', 'y', 'likelihood']],names=['scorer', 'bodyparts', 'coords'])
if gputouse is not None: #gpu selectinon
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
##################################################
# Datafolder
##################################################
#checks if input is a directory
if [os.path.isdir(i) for i in videos] == [True]:#os.path.isdir(video)==True:
"""
Analyzes all the videos in the directory.
"""
print("Analyzing all the videos in the directory")
videofolder= videos[0]
os.chdir(videofolder)
videolist=[fn for fn in os.listdir(os.curdir) if (videotype in fn) and ('_labeled.mp4' not in fn)] #exclude labeled-videos!
Videos = sample(videolist,len(videolist)) # this is useful so multiple nets can be used to analzye simultanously
else:
if isinstance(videos,str):
if os.path.isfile(videos): # #or just one direct path!
Videos=[videos]
else:
Videos=[]
else:
Videos=[v for v in videos if os.path.isfile(v)]
if len(Videos)>0:
#looping over videos
for video in Videos:
AnalyzeVideo(video,DLCscorer,trainFraction,cfg,dlc_cfg,sess,inputs, outputs,pdindex,save_as_csv, destfolder)
os.chdir(str(start_path))
print("The videos are analyzed. Now your research can truly start! \n You can create labeled videos with 'create_labeled_video'.")
print("If the tracking is not satisfactory for some videos, consider expanding the training set. You can use the function 'extract_outlier_frames' to extract any outlier frames!")
def train_network(config, shuffle=1, trainingsetindex=0, gputouse=None,
max_snapshots_to_keep=5, autotune=False, displayiters=None,
saveiters=None, maxiters=None):
"""Trains the network with the labels in the training dataset.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
shuffle: int, optional
Integer value specifying the shuffle index to select for training. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
Additional parameters:
max_snapshots_to_keep: int, or None. Sets how many snapshots are kept, i.e. states of the trained network. Every savinginteration many times
a snapshot is stored, however only the last max_snapshots_to_keep many are kept! If you change this to None, then all are kept.
See: https://github.com/AlexEMG/DeepLabCut/issues/8#issuecomment-387404835
autotune: property of TensorFlow, somehow faster if 'false' (as Eldar found out, see https://github.com/tensorflow/tensorflow/issues/13317). Default: False
displayiters: this variable is actually set in pose_config.yaml. However, you can overwrite it with this hack. Don't use this regularly, just if you are too lazy to dig out
the pose_config.yaml file for the corresponding project. If None, the value from there is used, otherwise it is overwritten! Default: None
saveiters: this variable is actually set in pose_config.yaml. However, you can overwrite it with this hack. Don't use this regularly, just if you are too lazy to dig out
the pose_config.yaml file for the corresponding project. If None, the value from there is used, otherwise it is overwritten! Default: None
maxiters: this variable is actually set in pose_config.yaml. However, you can overwrite it with this hack. Don't use this regularly, just if you are too lazy to dig out
the pose_config.yaml file for the corresponding project. If None, the value from there is used, otherwise it is overwritten! Default: None
Example
--------
for training the network for first shuffle of the training dataset.
>>> deeplabcut.train_network('/analysis/project/reaching-task/config.yaml')
--------
for training the network for second shuffle of the training dataset.
>>> deeplabcut.train_network('/analysis/project/reaching-task/config.yaml',shuffle=2)
--------
"""
import tensorflow as tf
# reload logger.
import importlib
import logging
importlib.reload(logging)
logging.shutdown()
from deeplabcut.pose_estimation_tensorflow.train import train
from deeplabcut.utils import auxiliaryfunctions
tf.reset_default_graph()
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
modelfoldername = auxiliaryfunctions.GetModelFolder(
cfg["TrainingFraction"][trainingsetindex], shuffle, cfg)
poseconfigfile = Path(
os.path.join(cfg['project_path'], str(modelfoldername), "train",
"pose_cfg.yaml"))
if not poseconfigfile.is_file():
print("The training datafile ", poseconfigfile, " is not present.")
print(
"Probably, the training dataset for this secific shuffle index was not created.")
print(
"Try with a different shuffle/trainingsetfraction or use function 'create_training_dataset' to create a new trainingdataset with this shuffle index.")
else:
# Set environment variables
if autotune is not False: # see: https://github.com/tensorflow/tensorflow/issues/13317
os.environ['TF_CUDNN_USE_AUTOTUNE'] = '0'
if gputouse is not None:
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
print('starts here:')
print(cfg)
print(saveiters)
print(str(poseconfigfile))
print(max_snapshots_to_keep)
try:
train(str(poseconfigfile), displayiters, saveiters, maxiters,
max_to_keep=max_snapshots_to_keep) # pass on path and file name for pose_cfg.yaml!
except BaseException as e:
raise e
finally:
os.chdir(str(start_path))
print(
"The network is now trained and ready to evaluate. Use the function 'evaluate_network' to evaluate the network.")
def analyze_time_lapse_frames(config,directory,frametype='.png',shuffle=1,trainingsetindex=0,gputouse=None,save_as_csv=False):
"""
Analyzed all images (of type = frametype) in a folder and stores the output in one file.
You can crop the frames (before analysis), by changing 'cropping'=True and setting 'x1','x2','y1','y2' in the config file.
Output: The labels are stored as MultiIndex Pandas Array, which contains the name of the network, body part name, (x, y) label position \n
in pixels, and the likelihood for each frame per body part. These arrays are stored in an efficient Hierarchical Data Format (HDF) \n
in the same directory, where the video is stored. However, if the flag save_as_csv is set to True, the data can also be exported in \n
comma-separated values format (.csv), which in turn can be imported in many programs, such as MATLAB, R, Prism, etc.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
directory: string
Full path to directory containing the frames that shall be analyzed
frametype: string, optional
Checks for the file extension of the frames. Only images with this extension are analyzed. The default is ``.png``
shuffle: int, optional
An integer specifying the shuffle index of the training dataset used for training the network. The default is 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
Examples
--------
If you want to analyze all frames in /analysis/project/timelapseexperiment1
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml','/analysis/project/timelapseexperiment1')
--------
If you want to analyze all frames in /analysis/project/timelapseexperiment1
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml','/analysis/project/timelapseexperiment1', frametype='.bmp')
--------
Note: for test purposes one can extract all frames from a video with ffmeg, e.g. ffmpeg -i testvideo.avi thumb%04d.png
"""
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ['TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
tf.reset_default_graph()
start_path=os.getcwd() #record cwd to return to this directory in the end
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
modelfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError("It seems the model for shuffle %s and trainFraction %s does not exist."%(shuffle,trainFraction))
# Check which snapshots are available and sort them by # iterations
try:
Snapshots = np.array([fn.split('.')[0]for fn in os.listdir(os.path.join(modelfolder , 'train'))if "index" in fn])
except FileNotFoundError:
raise FileNotFoundError("Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s."%(shuffle,shuffle))
if cfg['snapshotindex'] == 'all':
print("Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!")
snapshotindex = -1
else:
snapshotindex=cfg['snapshotindex']
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder)
##################################################
# Load and setup CNN part detector
##################################################
# Check if data already was generated:
dlc_cfg['init_weights'] = os.path.join(modelfolder , 'train', Snapshots[snapshotindex])
trainingsiterations = (dlc_cfg['init_weights'].split(os.sep)[-1]).split('-')[-1]
#update batchsize (based on parameters in config.yaml)
dlc_cfg['batch_size']=cfg['batch_size']
# Name for scorer:
DLCscorer = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations=trainingsiterations)
sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
pdindex = pd.MultiIndex.from_product([[DLCscorer], dlc_cfg['all_joints_names'], ['x', 'y', 'likelihood']],names=['scorer', 'bodyparts', 'coords'])
if gputouse is not None: #gpu selectinon
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
##################################################
# Loading the images
##################################################
#checks if input is a directory
if os.path.isdir(directory)==True:
"""
Analyzes all the frames in the directory.
"""
print("Analyzing all frames in the directory: ", directory)
os.chdir(directory)
framelist=np.sort([fn for fn in os.listdir(os.curdir) if (frametype in fn)])
vname = Path(directory).stem
dataname = os.path.join(directory,vname + DLCscorer + '.h5')
try:
# Attempt to load data...
pd.read_hdf(dataname)
print("Frames already analyzed!", dataname)
except FileNotFoundError:
nframes = len(framelist)
if nframes>1:
start = time.time()
PredicteData,nframes,nx,ny=GetPosesofFrames(cfg,dlc_cfg, sess, inputs, outputs,directory,framelist,nframes,dlc_cfg['batch_size'])
stop = time.time()
if cfg['cropping']==True:
coords=[cfg['x1'],cfg['x2'],cfg['y1'],cfg['y2']]
else:
coords=[0, nx, 0, ny]
dictionary = {
"start": start,
"stop": stop,
"run_duration": stop - start,
"Scorer": DLCscorer,
"config file": dlc_cfg,
"batch_size": dlc_cfg["batch_size"],
"frame_dimensions": (ny, nx),
"nframes": nframes,
"cropping": cfg['cropping'],
"cropping_parameters": coords
}
metadata = {'data': dictionary}
print("Saving results in %s..." %(directory))
auxiliaryfunctions.SaveData(PredicteData[:nframes,:], metadata, dataname, pdindex, framelist,save_as_csv)
print("The folder was analyzed. Now your research can truly start!")
print("If the tracking is not satisfactory for some frome, consider expanding the training set.")
else:
print("No frames were found. Consider changing the path or the frametype.")
os.chdir(str(start_path))
def train_network(
config,
shuffle=1,
trainingsetindex=0,
max_snapshots_to_keep=5,
displayiters=None,
saveiters=None,
maxiters=None,
allow_growth=False,
gputouse=None,
autotune=False,
keepdeconvweights=True,
modelprefix="",
):
"""Trains the network with the labels in the training dataset.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
shuffle: int, optional
Integer value specifying the shuffle index to select for training. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
Additional parameters:
max_snapshots_to_keep: int, or None. Sets how many snapshots are kept, i.e. states of the trained network. Every savinginteration many times
a snapshot is stored, however only the last max_snapshots_to_keep many are kept! If you change this to None, then all are kept.
See: https://github.com/AlexEMG/DeepLabCut/issues/8#issuecomment-387404835
displayiters: this variable is actually set in pose_config.yaml. However, you can overwrite it with this hack. Don't use this regularly, just if you are too lazy to dig out
the pose_config.yaml file for the corresponding project. If None, the value from there is used, otherwise it is overwritten! Default: None
saveiters: this variable is actually set in pose_config.yaml. However, you can overwrite it with this hack. Don't use this regularly, just if you are too lazy to dig out
the pose_config.yaml file for the corresponding project. If None, the value from there is used, otherwise it is overwritten! Default: None
maxiters: this variable is actually set in pose_config.yaml. However, you can overwrite it with this hack. Don't use this regularly, just if you are too lazy to dig out
the pose_config.yaml file for the corresponding project. If None, the value from there is used, otherwise it is overwritten! Default: None
allow_growth: bool, default false.
For some smaller GPUs the memory issues happen. If true, the memory allocator does not pre-allocate the entire specified
GPU memory region, instead starting small and growing as needed. See issue: https://forum.image.sc/t/how-to-stop-running-out-of-vram/30551/2
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
autotune: property of TensorFlow, somehow faster if 'false' (as Eldar found out, see https://github.com/tensorflow/tensorflow/issues/13317). Default: False
keepdeconvweights: bool, default: true
Also restores the weights of the deconvolution layers (and the backbone) when training from a snapshot. Note that if you change the number of bodyparts, you need to
set this to false for re-training.
Example
--------
for training the network for first shuffle of the training dataset.
>>> deeplabcut.train_network('/analysis/project/reaching-task/config.yaml')
--------
for training the network for second shuffle of the training dataset.
>>> deeplabcut.train_network('/analysis/project/reaching-task/config.yaml',shuffle=2,keepdeconvweights=True)
--------
"""
import tensorflow as tf
# reload logger.
import importlib
import logging
importlib.reload(logging)
logging.shutdown()
from deeplabcut.utils import auxiliaryfunctions
tf.compat.v1.reset_default_graph()
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
modelfoldername = auxiliaryfunctions.GetModelFolder(
cfg["TrainingFraction"][trainingsetindex],
shuffle,
cfg,
modelprefix=modelprefix)
poseconfigfile = Path(
os.path.join(cfg["project_path"], str(modelfoldername), "train",
"pose_cfg.yaml"))
if not poseconfigfile.is_file():
print("The training datafile ", poseconfigfile, " is not present.")
print(
"Probably, the training dataset for this specific shuffle index was not created."
)
print(
"Try with a different shuffle/trainingsetfraction or use function 'create_training_dataset' to create a new trainingdataset with this shuffle index."
)
else:
# Set environment variables
if (autotune is not False
): # see: https://github.com/tensorflow/tensorflow/issues/13317
os.environ["TF_CUDNN_USE_AUTOTUNE"] = "0"
if gputouse is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
try:
cfg_dlc = auxiliaryfunctions.read_plainconfig(poseconfigfile)
if "multi-animal" in cfg_dlc["dataset_type"]:
from deeplabcut.pose_estimation_tensorflow.core.train_multianimal import train
print("Selecting multi-animal trainer")
train(
str(poseconfigfile),
displayiters,
saveiters,
maxiters,
max_to_keep=max_snapshots_to_keep,
keepdeconvweights=keepdeconvweights,
allow_growth=allow_growth,
) # pass on path and file name for pose_cfg.yaml!
else:
from deeplabcut.pose_estimation_tensorflow.core.train import train
print("Selecting single-animal trainer")
train(
str(poseconfigfile),
displayiters,
saveiters,
maxiters,
max_to_keep=max_snapshots_to_keep,
keepdeconvweights=keepdeconvweights,
allow_growth=allow_growth,
) # pass on path and file name for pose_cfg.yaml!
except BaseException as e:
raise e
finally:
os.chdir(str(start_path))
print(
"The network is now trained and ready to evaluate. Use the function 'evaluate_network' to evaluate the network."
)
def create_training_dataset(
config,
num_shuffles=1,
Shuffles=None,
windows2linux=False,
userfeedback=False,
trainIndices=None,
testIndices=None,
net_type=None,
augmenter_type=None,
):
"""
Creates a training dataset. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
userfeedback: bool, optional
If this is set to false, then all requested train/test splits are created (no matter if they already exist). If you
want to assure that previous splits etc. are not overwritten, then set this to True and you will be asked for each split.
trainIndices: list of lists, optional (default=None)
List of one or multiple lists containing train indexes.
A list containing two lists of training indexes will produce two splits.
testIndices: list of lists, optional (default=None)
List of one or multiple lists containing test indexes.
net_type: string
Type of networks. Currently resnet_50, resnet_101, resnet_152, mobilenet_v2_1.0,mobilenet_v2_0.75, mobilenet_v2_0.5, and mobilenet_v2_0.35 are supported.
augmenter_type: string
Type of augmenter. Currently default, imgaug, tensorpack, and deterministic are supported.
Example
--------
>>> deeplabcut.create_training_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcut.create_training_dataset('C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
import scipy.io as sio
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
if cfg.get("multianimalproject", False):
from deeplabcut.generate_training_dataset.multiple_individuals_trainingsetmanipulation import (
create_multianimaltraining_dataset, )
create_multianimaltraining_dataset(config, num_shuffles, Shuffles,
windows2linux, net_type)
else:
scorer = cfg["scorer"]
project_path = cfg["project_path"]
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(
cfg) # Path concatenation OS platform independent
auxiliaryfunctions.attempttomakefolder(Path(
os.path.join(project_path, str(trainingsetfolder))),
recursive=True)
Data = merge_annotateddatasets(
cfg, Path(os.path.join(project_path, trainingsetfolder)),
windows2linux)
if Data is None:
return
Data = Data[scorer] # extract labeled data
# loading & linking pretrained models
if net_type is None: # loading & linking pretrained models
net_type = cfg.get("default_net_type", "resnet_50")
else:
if "resnet" in net_type or "mobilenet" in net_type:
pass
else:
raise ValueError("Invalid network type:", net_type)
if augmenter_type is None:
augmenter_type = cfg.get("default_augmenter", "imgaug")
if augmenter_type is None: # this could be in config.yaml for old projects!
# updating variable if null/None! #backwardscompatability
auxiliaryfunctions.edit_config(config,
{"default_augmenter": "imgaug"})
augmenter_type = "imgaug"
else:
if augmenter_type in [
"default",
"scalecrop",
"imgaug",
"tensorpack",
"deterministic",
]:
pass
else:
raise ValueError("Invalid augmenter type:", augmenter_type)
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, Path(dlcparent_path), num_shuffles)
if Shuffles is None:
Shuffles = range(1, num_shuffles + 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
# print(trainIndices,testIndices, Shuffles, augmenter_type,net_type)
if trainIndices is None and testIndices is None:
splits = [(
trainFraction,
shuffle,
SplitTrials(range(len(Data.index)), trainFraction),
) for trainFraction in cfg["TrainingFraction"]
for shuffle in Shuffles]
else:
if len(trainIndices) != len(testIndices) != len(Shuffles):
raise ValueError(
"Number of Shuffles and train and test indexes should be equal."
)
splits = []
for shuffle, (train_inds, test_inds) in enumerate(
zip(trainIndices, testIndices)):
trainFraction = round(
len(train_inds) * 1.0 / (len(train_inds) + len(test_inds)),
2)
print(
f"You passed a split with the following fraction: {int(100 * trainFraction)}%"
)
splits.append((trainFraction, Shuffles[shuffle], (train_inds,
test_inds)))
bodyparts = cfg["bodyparts"]
nbodyparts = len(bodyparts)
for trainFraction, shuffle, (trainIndices, testIndices) in splits:
if len(trainIndices) > 0:
if userfeedback:
trainposeconfigfile, _, _ = training.return_train_network_path(
config,
shuffle=shuffle,
trainingsetindex=cfg["TrainingFraction"].index(
trainFraction),
)
if trainposeconfigfile.is_file():
askuser = input(
"The model folder is already present. If you continue, it will overwrite the existing model (split). Do you want to continue?(yes/no): "
)
if (askuser == "no" or askuser == "No"
or askuser == "N" or askuser == "No"):
raise Exception(
"Use the Shuffles argument as a list to specify a different shuffle index. Check out the help for more details."
)
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
# Make training file!
(
datafilename,
metadatafilename,
) = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
################################################################################
# Saving data file (convert to training file for deeper cut (*.mat))
################################################################################
data, MatlabData = format_training_data(
Data, trainIndices, nbodyparts, project_path)
sio.savemat(os.path.join(project_path, datafilename),
{"dataset": MatlabData})
################################################################################
# Saving metadata (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename),
data,
trainIndices,
testIndices,
trainFraction,
)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
auxiliaryfunctions.attempttomakefolder(
Path(config).parents[0] / modelfoldername, recursive=True)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername) + "/train")
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername) + "/test")
path_train_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"train",
"pose_cfg.yaml",
))
path_test_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"pose_cfg.yaml",
))
# str(cfg['proj_path']+'/'+Path(modelfoldername) / 'test' / 'pose_cfg.yaml')
items2change = {
"dataset": datafilename,
"metadataset": metadatafilename,
"num_joints": len(bodyparts),
"all_joints": [[i] for i in range(len(bodyparts))],
"all_joints_names": [str(bpt) for bpt in bodyparts],
"init_weights": model_path,
"project_path": str(cfg["project_path"]),
"net_type": net_type,
"dataset_type": augmenter_type,
}
items2drop = {}
if augmenter_type == "scalecrop":
# these values are dropped as scalecrop
# doesn't have rotation implemented
items2drop = {"rotation": 0, "rotratio": 0.0}
trainingdata = MakeTrain_pose_yaml(items2change,
path_train_config,
defaultconfigfile,
items2drop)
keys2save = [
"dataset",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
]
MakeTest_pose_yaml(trainingdata, keys2save, path_test_config)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
return splits
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
datafile, _ = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder,
0.8,
1,
cfg,
)
datafile = datafile.split(".mat")[0] + ".pickle"
with open(os.path.join(cfg["project_path"], datafile), "rb") as f:
pickledata = pickle.load(f)
num_images = len(pickledata)
assert all(len(pickledata[i]["image"]) == 3 for i in range(num_images))
print("Editing pose config...")
model_folder = auxiliaryfunctions.GetModelFolder(
TRAIN_SIZE, 1, cfg, cfg["project_path"]
)
pose_config_path = os.path.join(model_folder, "train", "pose_cfg.yaml")
edits = {
"global_scale": 0.5,
"batch_size": 1,
"save_iters": N_ITER,
"display_iters": N_ITER // 2,
"crop_size": [200, 200],
# "multi_step": [[0.001, N_ITER]],
}
deeplabcut.auxiliaryfunctions.edit_config(pose_config_path, edits)
print("Pose config edited.")
print("Training network...")
deeplabcut.train_network(config_path, maxiters=N_ITER)
def evaluate_multianimal_crossvalidate(
config,
Shuffles=[1],
trainingsetindex=0,
pbounds=None,
edgewisecondition=True,
target="rpck_train",
inferencecfg=None,
init_points=20,
n_iter=50,
dcorr=10.0,
leastbpts=1,
printingintermediatevalues=True,
modelprefix="",
plotting=False,
):
"""
Crossvalidate inference parameters on evaluation data; optimal parametrs will be stored in " inference_cfg.yaml".
They will then be then used for inference (for analysis of videos). Performs Bayesian Optimization with https://github.com/fmfn/BayesianOptimization
This is a crucial step. The most important variable (in inferencecfg) to cross-validate is minimalnumberofconnections. Pass
a reasonable range to optimze (e.g. if you have 5 edges from 1 to 5. If you have 4 bpts and 11 connections from 3 to 9).
config: string
Full path of the config.yaml file as a string.
shuffle: int, optional
An integer specifying the shuffle index of the training dataset used for training the network. The default is 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
pbounds: dictionary of variables with ranges to crossvalidate.
By default: pbounds = {
'pafthreshold': (0.05, 0.7),
'detectionthresholdsquare': (0, 0.9),
'minimalnumberofconnections': (1, # connections in your skeleton),
}
inferencecfg: dict, OPTIONAL
For the variables that are *not* crossvalidated the parameters from inference_cfg.yaml are used, or
you can overwrite them by passing a dictinary with your preferred parameters.
edgewisecondition: bool, default True
Estimates Euclidean distances for each skeleton edge and uses those distance for excluding possible connections.
If false, uses only one distance for all bodyparts (which is obviously suboptimal).
target: string, default='rpck_train'
What metric to optimize. Options are pck/rpck/rmse on train/test set.
init_points: int, optional (default=10)
Number of random initial explorations. Probing random regions helps diversify the exploration space.
Parameter from BayesianOptimization.
n_iter: int, optional (default=20)
Number of iterations of Bayesian optimization to perform.
The larger it is, the higher the likelihood of finding a good extremum.
Parameter from BayesianOptimization.
dcorr: float,
Distance thereshold for percent correct keypoints / relative percent correct keypoints (see paper).
leastbpts: integer (should be a small number)
If an animals has less or equal as many body parts in an image it will not be used
for cross validation. Imagine e.g. if only a single bodypart is present, then
if animals need a certain minimal number of bodyparts for assembly (minimalnumberofconnections),
this might not be predictable.
printingintermediatevalues: bool, default True
If intermediate metrics RMSE/hits/.. per sample should be printed.
Examples
--------
first run evalute:
deeplabcut.evaluate_network(path_config_file,Shuffles=[shuffle],plotting=True)
Then e.g. for finding inference parameters to minimize rmse on test set:
deeplabcut.evaluate_multianimal_crossvalidate(path_config_file,Shuffles=[shuffle],target='rmse_test')
"""
from deeplabcut.pose_estimation_tensorflow.lib import crossvalutils
from deeplabcut.utils import auxfun_multianimal, auxiliaryfunctions
from easydict import EasyDict as edict
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
)
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, "all")
colors = visualization.get_cmap(len(comparisonbodyparts),
name=cfg["colormap"])
# wild guesses for a wide range:
maxconnections = len(cfg["skeleton"])
minconnections = 1 # len(cfg['multianimalbodyparts'])-1
_pbounds = {
"pafthreshold": (0.05, 0.7),
"detectionthresholdsquare": (
0,
0.9,
), # TODO: set to minimum (from pose_cfg.yaml)
"minimalnumberofconnections": (minconnections, maxconnections),
}
if pbounds is not None:
_pbounds.update(pbounds)
if "rpck" in target or "pck" in target:
maximize = True
if "rmse" in target:
maximize = False # i.e. minimize
for shuffle in Shuffles:
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
_, trainIndices, testIndices, _ = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
])
snapindex = -1
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train",
Snapshots[snapindex]) # setting weights to corresponding snapshot.
trainingsiterations = (dlc_cfg["init_weights"].split(
os.sep)[-1]).split("-")[
-1] # read how many training siterations that corresponds to.
DLCscorer, _ = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix)
path_inference_config = Path(
modelfolder) / "test" / "inference_cfg.yaml"
if inferencecfg is None: # then load or initialize
inferencecfg = auxfun_multianimal.read_inferencecfg(
path_inference_config, cfg)
else:
inferencecfg = edict(inferencecfg)
auxfun_multianimal.check_inferencecfg_sanity(cfg, inferencecfg)
inferencecfg.topktoretain = np.inf
inferencecfg, opt = crossvalutils.bayesian_search(
config,
inferencecfg,
_pbounds,
edgewisecondition=edgewisecondition,
shuffle=shuffle,
trainingsetindex=trainingsetindex,
target=target,
maximize=maximize,
init_points=init_points,
n_iter=n_iter,
acq="ei",
dcorr=dcorr,
leastbpts=leastbpts,
modelprefix=modelprefix,
)
# update number of individuals to retain.
inferencecfg.topktoretain = len(
cfg["individuals"]) + 1 * (len(cfg["uniquebodyparts"]) > 0)
# calculating result at best best solution
DataOptParams, poses_gt, poses = crossvalutils.compute_crossval_metrics(
config, inferencecfg, shuffle, trainingsetindex, modelprefix)
path_inference_config = str(path_inference_config)
# print("Quantification:", DataOptParams.head())
DataOptParams.to_hdf(
path_inference_config.split(".yaml")[0] + ".h5",
"df_with_missing",
format="table",
mode="w",
)
DataOptParams.to_csv(path_inference_config.split(".yaml")[0] + ".csv")
print("Saving optimal inference parameters...")
print(DataOptParams.to_string())
auxiliaryfunctions.write_plainconfig(path_inference_config,
dict(inferencecfg))
# Store best predictions
max_indivs = max(pose.shape[0] for pose in poses)
bpts = dlc_cfg["all_joints_names"]
container = np.full((len(poses), max_indivs * len(bpts) * 3), np.nan)
for n, pose in enumerate(poses):
temp = pose.flatten()
container[n, :len(temp)] = temp
header = pd.MultiIndex.from_product(
[
[DLCscorer],
[f"individual{i}" for i in range(1, max_indivs + 1)],
bpts,
["x", "y", "likelihood"],
],
names=["scorer", "individuals", "bodyparts", "coords"],
)
df = pd.DataFrame(container, columns=header)
df.to_hdf(os.path.join(evaluationfolder, f"{DLCscorer}.h5"),
key="df_with_missing")
if plotting:
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" + Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
for imageindex, imagename in tqdm(enumerate(Data.index)):
image_path = os.path.join(cfg["project_path"], imagename)
image = io.imread(image_path)
frame = img_as_ubyte(skimage.color.gray2rgb(image))
groundtruthcoordinates = poses_gt[imageindex]
coords_pred = poses[imageindex][:, :, :2]
probs_pred = poses[imageindex][:, :, -1:]
fig = visualization.make_multianimal_labeled_image(
frame,
groundtruthcoordinates,
coords_pred,
probs_pred,
colors,
cfg["dotsize"],
cfg["alphavalue"],
cfg["pcutoff"],
)
visualization.save_labeled_frame(fig, image_path, foldername,
imageindex in trainIndices)
def evaluate_multianimal_full(
config,
Shuffles=[1],
trainingsetindex=0,
plotting=None,
show_errors=True,
comparisonbodyparts="all",
gputouse=None,
modelprefix="",
c_engine=False,
):
from deeplabcut.pose_estimation_tensorflow.nnet import predict
from deeplabcut.pose_estimation_tensorflow.nnet import (
predict_multianimal as predictma, )
from deeplabcut.utils import auxiliaryfunctions, auxfun_multianimal
import tensorflow as tf
if "TF_CUDNN_USE_AUTOTUNE" in os.environ:
del os.environ[
"TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training
tf.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
start_path = os.getcwd()
##################################################
# Load data...
##################################################
cfg = auxiliaryfunctions.read_config(config)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
TrainingFractions = [cfg["TrainingFraction"][trainingsetindex]]
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
)
# Handle data previously annotated on a different platform
sep = "/" if "/" in Data.index[0] else "\\"
if sep != os.path.sep:
Data.index = Data.index.str.replace(sep, os.path.sep)
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
all_bpts = np.asarray(
len(cfg["individuals"]) * cfg["multianimalbodyparts"] +
cfg["uniquebodyparts"])
colors = visualization.get_cmap(len(comparisonbodyparts),
name=cfg["colormap"])
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# TODO: IMPLEMENT for different batch sizes?
dlc_cfg["batch_size"] = 1 # due to differently sized images!!!
joints = dlc_cfg["all_joints_names"]
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
# path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
])
if len(Snapshots) == 0:
print(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
else:
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
final_result = []
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg["init_weights"].split(os.sep)[-1]
).split(
"-"
)[-1] # read how many training siterations that corresponds to.
# name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix,
)
print(
"Running ",
DLCscorer,
" with # of trainingiterations:",
trainingsiterations,
)
(
notanalyzed,
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.CheckifNotEvaluated(
str(evaluationfolder),
DLCscorer,
DLCscorerlegacy,
Snapshots[snapindex],
)
if os.path.isfile(
resultsfilename.split(".h5")[0] + "_full.pickle"):
print("Model already evaluated.", resultsfilename)
else:
if plotting:
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_" + DLCscorer + "_" +
Snapshots[snapindex],
)
auxiliaryfunctions.attempttomakefolder(foldername)
# print(dlc_cfg)
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = predict.setup_pose_prediction(
dlc_cfg)
PredicteData = {}
dist = np.full((len(Data), len(all_bpts)), np.nan)
conf = np.full_like(dist, np.nan)
distnorm = np.full(len(Data), np.nan)
print("Analyzing data...")
for imageindex, imagename in tqdm(enumerate(
Data.index)):
image_path = os.path.join(cfg["project_path"],
imagename)
image = io.imread(image_path)
frame = img_as_ubyte(skimage.color.gray2rgb(image))
GT = Data.iloc[imageindex]
df = GT.unstack("coords").reindex(
joints, level='bodyparts')
# Evaluate PAF edge lengths to calibrate `distnorm`
temp = GT.unstack("bodyparts")[joints]
xy = temp.values.reshape(
(-1, 2, temp.shape[1])).swapaxes(1, 2)
edges = xy[:, dlc_cfg["partaffinityfield_graph"]]
lengths = np.sum(
(edges[:, :, 0] - edges[:, :, 1])**2, axis=2)
distnorm[imageindex] = np.nanmax(lengths)
# FIXME Is having an empty array vs nan really that necessary?!
groundtruthidentity = list(
df.index.get_level_values(
"individuals").to_numpy().reshape((-1, 1)))
groundtruthcoordinates = list(
df.values[:, np.newaxis])
for i, coords in enumerate(groundtruthcoordinates):
if np.isnan(coords).any():
groundtruthcoordinates[i] = np.empty(
(0, 2), dtype=float)
groundtruthidentity[i] = np.array(
[], dtype=str)
PredicteData[imagename] = {}
PredicteData[imagename]["index"] = imageindex
pred = predictma.get_detectionswithcostsandGT(
frame,
groundtruthcoordinates,
dlc_cfg,
sess,
inputs,
outputs,
outall=False,
nms_radius=dlc_cfg.nmsradius,
det_min_score=dlc_cfg.minconfidence,
c_engine=c_engine,
)
PredicteData[imagename]["prediction"] = pred
PredicteData[imagename]["groundtruth"] = [
groundtruthidentity,
groundtruthcoordinates,
GT,
]
coords_pred = pred["coordinates"][0]
probs_pred = pred["confidence"]
for bpt, xy_gt in df.groupby(level="bodyparts"):
inds_gt = np.flatnonzero(
np.all(~np.isnan(xy_gt), axis=1))
n_joint = joints.index(bpt)
xy = coords_pred[n_joint]
if inds_gt.size and xy.size:
# Pick the predictions closest to ground truth,
# rather than the ones the model has most confident in
d = cdist(xy_gt.iloc[inds_gt], xy)
rows, cols = linear_sum_assignment(d)
min_dists = d[rows, cols]
inds = np.flatnonzero(all_bpts == bpt)
sl = imageindex, inds[inds_gt[rows]]
dist[sl] = min_dists
conf[sl] = probs_pred[n_joint][
cols].squeeze()
if plotting:
fig = visualization.make_multianimal_labeled_image(
frame,
groundtruthcoordinates,
coords_pred,
probs_pred,
colors,
cfg["dotsize"],
cfg["alphavalue"],
cfg["pcutoff"],
)
visualization.save_labeled_frame(
fig,
image_path,
foldername,
imageindex in trainIndices,
)
sess.close() # closes the current tf session
# Compute all distance statistics
df_dist = pd.DataFrame(dist, columns=df.index)
df_conf = pd.DataFrame(conf, columns=df.index)
df_joint = pd.concat([df_dist, df_conf],
keys=["rmse", "conf"],
names=["metrics"],
axis=1)
df_joint = df_joint.reorder_levels(list(
np.roll(df_joint.columns.names, -1)),
axis=1)
df_joint.sort_index(axis=1,
level=["individuals", "bodyparts"],
ascending=[True, True],
inplace=True)
write_path = os.path.join(
evaluationfolder,
f"dist_{trainingsiterations}.csv")
df_joint.to_csv(write_path)
# Calculate overall prediction error
error = df_joint.xs("rmse", level="metrics", axis=1)
mask = df_joint.xs("conf", level="metrics",
axis=1) >= cfg["pcutoff"]
error_masked = error[mask]
error_train = np.nanmean(error.iloc[trainIndices])
error_train_cut = np.nanmean(
error_masked.iloc[trainIndices])
error_test = np.nanmean(error.iloc[testIndices])
error_test_cut = np.nanmean(
error_masked.iloc[testIndices])
results = [
trainingsiterations,
int(100 * trainFraction),
shuffle,
np.round(error_train, 2),
np.round(error_test, 2),
cfg["pcutoff"],
np.round(error_train_cut, 2),
np.round(error_test_cut, 2),
]
final_result.append(results)
# For OKS/PCK, compute the standard deviation error across all frames
sd = df_dist.groupby("bodyparts",
axis=1).mean().std(axis=0)
sd["distnorm"] = np.sqrt(np.nanmax(distnorm))
sd.to_csv(write_path.replace("dist.csv", "sd.csv"))
if show_errors:
string = "Results for {} training iterations: {}, shuffle {}:\n" \
"Train error: {} pixels. Test error: {} pixels.\n" \
"With pcutoff of {}:\n" \
"Train error: {} pixels. Test error: {} pixels."
print(string.format(*results))
print("##########################################")
print(
"Average Euclidean distance to GT per individual (in pixels)"
)
print(
error_masked.groupby(
'individuals',
axis=1).mean().mean().to_string())
print(
"Average Euclidean distance to GT per bodypart (in pixels)"
)
print(
error_masked.groupby(
'bodyparts',
axis=1).mean().mean().to_string())
PredicteData["metadata"] = {
"nms radius":
dlc_cfg.nmsradius,
"minimal confidence":
dlc_cfg.minconfidence,
"PAFgraph":
dlc_cfg.partaffinityfield_graph,
"all_joints":
[[i] for i in range(len(dlc_cfg.all_joints))],
"all_joints_names": [
dlc_cfg.all_joints_names[i]
for i in range(len(dlc_cfg.all_joints))
],
"stride":
dlc_cfg.get("stride", 8),
}
print(
"Done and results stored for snapshot: ",
Snapshots[snapindex],
)
dictionary = {
"Scorer": DLCscorer,
"DLC-model-config file": dlc_cfg,
"trainIndices": trainIndices,
"testIndices": testIndices,
"trainFraction": trainFraction,
}
metadata = {"data": dictionary}
auxfun_multianimal.SaveFullMultiAnimalData(
PredicteData, metadata, resultsfilename)
tf.reset_default_graph()
if len(final_result
) > 0: # Only append if results were calculated
make_results_file(final_result, evaluationfolder,
DLCscorer)
# returning to intial folder
os.chdir(str(start_path))
