def cross_validate_paf_graphs(
config,
inference_config,
full_data_file,
metadata_file,
output_name="",
pcutoff=0.1,
greedy=False,
add_discarded=True,
calibrate=False,
overwrite_config=True,
):
cfg = auxiliaryfunctions.read_config(config)
inf_cfg = auxiliaryfunctions.read_plainconfig(inference_config)
inf_cfg_temp = inf_cfg.copy()
inf_cfg_temp["pcutoff"] = pcutoff
with open(full_data_file, "rb") as file:
data = pickle.load(file)
with open(metadata_file, "rb") as file:
metadata = pickle.load(file)
params = _set_up_evaluation(data)
to_ignore = auxfun_multianimal.filter_unwanted_paf_connections(
cfg, params["paf_graph"])
paf_inds, paf_scores = _get_n_best_paf_graphs(data,
metadata,
params["paf_graph"],
ignore_inds=to_ignore)
if calibrate:
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
calibration_file = os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
)
else:
calibration_file = ""
results = _benchmark_paf_graphs(
cfg,
inf_cfg_temp,
data,
paf_inds,
greedy,
add_discarded,
calibration_file=calibration_file,
)
# Select optimal PAF graph
df = results[1]
size_opt = np.argmax(
(1 - df.loc["miss", "mean"]) * df.loc["purity", "mean"])
pose_config = inference_config.replace("inference_cfg", "pose_cfg")
if not overwrite_config:
shutil.copy(pose_config, pose_config.replace(".yaml", "_old.yaml"))
inds = list(paf_inds[size_opt])
auxiliaryfunctions.edit_config(pose_config,
{"paf_best": [int(ind) for ind in inds]})
if output_name:
with open(output_name, "wb") as file:
pickle.dump([results], file)
return results
def create_multianimaltraining_dataset(
config,
num_shuffles=1,
Shuffles=None,
windows2linux=False,
net_type=None,
numdigits=2,
crop_size=(400, 400),
crop_sampling="hybrid",
paf_graph=None,
trainIndices=None,
testIndices=None,
):
"""
Creates a training dataset for multi-animal datasets. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Imporant differences to standard:
- stores coordinates with numdigits as many digits
- creates
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
net_type: string
Type of networks. Currently resnet_50, resnet_101, and resnet_152, efficientnet-b0, efficientnet-b1, efficientnet-b2, efficientnet-b3,
efficientnet-b4, efficientnet-b5, and efficientnet-b6 as well as dlcrnet_ms5 are supported (not the MobileNets!).
See Lauer et al. 2021 https://www.biorxiv.org/content/10.1101/2021.04.30.442096v1
numdigits: int, optional
crop_size: tuple of int, optional
Dimensions (width, height) of the crops for data augmentation.
Default is 400x400.
crop_sampling: str, optional
Crop centers sampling method. Must be either:
"uniform" (randomly over the image),
"keypoints" (randomly over the annotated keypoints),
"density" (weighing preferentially dense regions of keypoints),
or "hybrid" (alternating randomly between "uniform" and "density").
Default is "hybrid".
paf_graph: list of lists, optional (default=None)
If not None, overwrite the default complete graph. This is useful for advanced users who
already know a good graph, or simply want to use a specific one. Note that, in that case,
the data-driven selection procedure upon model evaluation will be skipped.
trainIndices: list of lists, optional (default=None)
List of one or multiple lists containing train indexes.
A list containing two lists of training indexes will produce two splits.
testIndices: list of lists, optional (default=None)
List of one or multiple lists containing test indexes.
Example
--------
>>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
>>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml', Shuffles=[0,1,2], trainIndices=[trainInd1, trainInd2, trainInd3], testIndices=[testInd1, testInd2, testInd3])
Windows:
>>> deeplabcut.create_multianimaltraining_dataset(r'C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
if windows2linux:
warnings.warn(
"`windows2linux` has no effect since 2.2.0.4 and will be removed in 2.2.1.",
FutureWarning,
)
if len(crop_size) != 2 or not all(isinstance(v, int) for v in crop_size):
raise ValueError(
"Crop size must be a tuple of two integers (width, height).")
if crop_sampling not in ("uniform", "keypoints", "density", "hybrid"):
raise ValueError(
f"Invalid sampling {crop_sampling}. Must be "
f"either 'uniform', 'keypoints', 'density', or 'hybrid.")
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg["scorer"]
project_path = cfg["project_path"]
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
full_training_path = Path(project_path, trainingsetfolder)
auxiliaryfunctions.attempttomakefolder(full_training_path, recursive=True)
Data = merge_annotateddatasets(cfg, full_training_path)
if Data is None:
return
Data = Data[scorer]
if net_type is None: # loading & linking pretrained models
net_type = cfg.get("default_net_type", "dlcrnet_ms5")
elif not any(net in net_type for net in ("resnet", "eff", "dlc", "mob")):
raise ValueError(f"Unsupported network {net_type}.")
multi_stage = False
### dlcnet_ms5: backbone resnet50 + multi-fusion & multi-stage module
### dlcr101_ms5/dlcr152_ms5: backbone resnet101/152 + multi-fusion & multi-stage module
if all(net in net_type for net in ("dlcr", "_ms5")):
num_layers = re.findall("dlcr([0-9]*)", net_type)[0]
if num_layers == "":
num_layers = 50
net_type = "resnet_{}".format(num_layers)
multi_stage = True
dataset_type = "multi-animal-imgaug"
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
if paf_graph is None: # Automatically form a complete PAF graph
partaffinityfield_graph = [
list(edge)
for edge in combinations(range(len(multianimalbodyparts)), 2)
]
else:
# Ignore possible connections between 'multi' and 'unique' body parts;
# one can never be too careful...
to_ignore = auxfun_multianimal.filter_unwanted_paf_connections(
cfg, paf_graph)
partaffinityfield_graph = [
edge for i, edge in enumerate(paf_graph) if i not in to_ignore
]
auxfun_multianimal.validate_paf_graph(cfg, partaffinityfield_graph)
print("Utilizing the following graph:", partaffinityfield_graph)
# Disable the prediction of PAFs if the graph is empty
partaffinityfield_predict = bool(partaffinityfield_graph)
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, Path(dlcparent_path), num_shuffles)
if Shuffles is None:
Shuffles = range(1, num_shuffles + 1, 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
# print(trainIndices,testIndices, Shuffles, augmenter_type,net_type)
if trainIndices is None and testIndices is None:
splits = []
for shuffle in Shuffles: # Creating shuffles starting from 1
for train_frac in cfg["TrainingFraction"]:
train_inds, test_inds = SplitTrials(range(len(Data)),
train_frac)
splits.append((train_frac, shuffle, (train_inds, test_inds)))
else:
if len(trainIndices) != len(testIndices) != len(Shuffles):
raise ValueError(
"Number of Shuffles and train and test indexes should be equal."
)
splits = []
for shuffle, (train_inds,
test_inds) in enumerate(zip(trainIndices, testIndices)):
trainFraction = round(
len(train_inds) * 1.0 / (len(train_inds) + len(test_inds)), 2)
print(
f"You passed a split with the following fraction: {int(100 * trainFraction)}%"
)
# Now that the training fraction is guaranteed to be correct,
# the values added to pad the indices are removed.
train_inds = np.asarray(train_inds)
train_inds = train_inds[train_inds != -1]
test_inds = np.asarray(test_inds)
test_inds = test_inds[test_inds != -1]
splits.append(
(trainFraction, Shuffles[shuffle], (train_inds, test_inds)))
for trainFraction, shuffle, (trainIndices, testIndices) in splits:
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
print(
"Creating training data for: Shuffle:",
shuffle,
"TrainFraction: ",
trainFraction,
)
# Make training file!
data = format_multianimal_training_data(
Data,
trainIndices,
cfg["project_path"],
numdigits,
)
if len(trainIndices) > 0:
(
datafilename,
metadatafilename,
) = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
################################################################################
# Saving metadata and data file (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename),
data,
trainIndices,
testIndices,
trainFraction,
)
datafilename = datafilename.split(".mat")[0] + ".pickle"
import pickle
with open(os.path.join(project_path, datafilename), "wb") as f:
# Pickle the 'labeled-data' dictionary using the highest protocol available.
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
auxiliaryfunctions.attempttomakefolder(Path(config).parents[0] /
modelfoldername,
recursive=True)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "train"))
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "test"))
path_train_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"train",
"pose_cfg.yaml",
))
path_test_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"pose_cfg.yaml",
))
path_inference_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"inference_cfg.yaml",
))
jointnames = [str(bpt) for bpt in multianimalbodyparts]
jointnames.extend([str(bpt) for bpt in uniquebodyparts])
items2change = {
"dataset":
datafilename,
"metadataset":
metadatafilename,
"num_joints":
len(multianimalbodyparts) +
len(uniquebodyparts), # cfg["uniquebodyparts"]),
"all_joints": [[i] for i in range(
len(multianimalbodyparts) + len(uniquebodyparts))
], # cfg["uniquebodyparts"]))],
"all_joints_names":
jointnames,
"init_weights":
model_path,
"project_path":
str(cfg["project_path"]),
"net_type":
net_type,
"multi_stage":
multi_stage,
"pairwise_loss_weight":
0.1,
"pafwidth":
20,
"partaffinityfield_graph":
partaffinityfield_graph,
"partaffinityfield_predict":
partaffinityfield_predict,
"weigh_only_present_joints":
False,
"num_limbs":
len(partaffinityfield_graph),
"dataset_type":
dataset_type,
"optimizer":
"adam",
"batch_size":
8,
"multi_step": [[1e-4, 7500], [5 * 1e-5, 12000], [1e-5,
200000]],
"save_iters":
10000,
"display_iters":
500,
"num_idchannel":
len(cfg["individuals"]) if cfg.get("identity", False) else 0,
"crop_size":
list(crop_size),
"crop_sampling":
crop_sampling,
}
trainingdata = MakeTrain_pose_yaml(items2change, path_train_config,
defaultconfigfile)
keys2save = [
"dataset",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"multi_stage",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
"dataset_type",
"partaffinityfield_predict",
"pairwise_predict",
"partaffinityfield_graph",
"num_limbs",
"dataset_type",
"num_idchannel",
]
MakeTest_pose_yaml(
trainingdata,
keys2save,
path_test_config,
nmsradius=5.0,
minconfidence=0.01,
sigma=1,
locref_smooth=False,
) # setting important def. values for inference
# Setting inference cfg file:
defaultinference_configfile = os.path.join(dlcparent_path,
"inference_cfg.yaml")
items2change = {
"minimalnumberofconnections":
int(len(cfg["multianimalbodyparts"]) / 2),
"topktoretain":
len(cfg["individuals"]) + 1 *
(len(cfg["uniquebodyparts"]) > 0),
"withid":
cfg.get("identity", False),
}
MakeInference_yaml(items2change, path_inference_config,
defaultinference_configfile)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
else:
pass
def create_multianimaltraining_dataset(
config,
num_shuffles=1,
Shuffles=None,
windows2linux=False,
net_type=None,
numdigits=2,
paf_graph=None,
):
"""
Creates a training dataset for multi-animal datasets. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Imporant differences to standard:
- stores coordinates with numdigits as many digits
- creates
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
net_type: string
Type of networks. Currently resnet_50, resnet_101, and resnet_152 are supported (not the MobileNets!)
numdigits: int, optional
paf_graph: list of lists, optional (default=None)
If not None, overwrite the default complete graph. This is useful for advanced users who
already know a good graph, or simply want to use a specific one. Note that, in that case,
the data-driven selection procedure upon model evaluation will be skipped.
Example
--------
>>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcut.create_multianimaltraining_dataset(r'C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
from skimage import io
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg["scorer"]
project_path = cfg["project_path"]
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
full_training_path = Path(project_path, trainingsetfolder)
auxiliaryfunctions.attempttomakefolder(full_training_path, recursive=True)
Data = trainingsetmanipulation.merge_annotateddatasets(
cfg, full_training_path, windows2linux
)
if Data is None:
return
Data = Data[scorer] # extract labeled data
# actualbpts=set(Data.columns.get_level_values(0))
def strip_cropped_image_name(path):
# utility function to split different crops from same image into either train or test!
filename = os.path.split(path)[1]
return filename.split("c")[0] if cfg["croppedtraining"] else filename
img_names = Data.index.map(strip_cropped_image_name).unique()
# loading & linking pretrained models
# CURRENTLY ONLY ResNet supported!
if net_type is None: # loading & linking pretrained models
net_type = cfg.get("default_net_type", "resnet_50")
elif not any(net in net_type for net in ("resnet", "eff", "dlc")):
raise ValueError(f"Unsupported network {net_type}.")
multi_stage = False
if net_type == "dlcrnet_ms5":
net_type = "resnet_50"
multi_stage = True
# multianimal case:
dataset_type = "multi-animal-imgaug"
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
if paf_graph is None: # Automatically form a complete PAF graph
partaffinityfield_graph = [
list(edge) for edge in combinations(range(len(multianimalbodyparts)), 2)
]
else:
# Ignore possible connections between 'multi' and 'unique' body parts;
# one can never be too careful...
to_ignore = auxfun_multianimal.filter_unwanted_paf_connections(
cfg, paf_graph
)
partaffinityfield_graph = [
edge for i, edge in enumerate(paf_graph) if i not in to_ignore
]
auxfun_multianimal.validate_paf_graph(cfg, partaffinityfield_graph)
print("Utilizing the following graph:", partaffinityfield_graph)
num_limbs = len(partaffinityfield_graph)
partaffinityfield_predict = True
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, Path(dlcparent_path), num_shuffles
)
if Shuffles == None:
Shuffles = range(1, num_shuffles + 1, 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
TrainingFraction = cfg["TrainingFraction"]
for shuffle in Shuffles: # Creating shuffles starting from 1
for trainFraction in TrainingFraction:
train_inds_temp, test_inds_temp = trainingsetmanipulation.SplitTrials(
range(len(img_names)), trainFraction
)
# Map back to the original indices.
temp = [name for i, name in enumerate(img_names) if i in test_inds_temp]
mask = Data.index.str.contains("|".join(temp))
testIndices = np.flatnonzero(mask)
trainIndices = np.flatnonzero(~mask)
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
# Make training file!
data = []
print(
"Creating training data for: Shuffle:",
shuffle,
"TrainFraction: ",
trainFraction,
)
print("This can take some time...")
for jj in tqdm(trainIndices):
jointsannotated = False
H = {}
# load image to get dimensions:
filename = Data.index[jj]
im = io.imread(os.path.join(cfg["project_path"], filename))
H["image"] = filename
try:
H["size"] = np.array(
[np.shape(im)[2], np.shape(im)[0], np.shape(im)[1]]
)
except:
# print "Grayscale!"
H["size"] = np.array([1, np.shape(im)[0], np.shape(im)[1]])
Joints = {}
for prfxindex, prefix in enumerate(individuals):
joints = (
np.zeros((len(uniquebodyparts) + len(multianimalbodyparts), 3))
* np.nan
)
if prefix != "single": # first ones are multianimalparts!
indexjoints = 0
for bpindex, bodypart in enumerate(multianimalbodyparts):
socialbdpt = bodypart # prefix+bodypart #build names!
# if socialbdpt in actualbpts:
try:
x, y = (
Data[prefix][socialbdpt]["x"][jj],
Data[prefix][socialbdpt]["y"][jj],
)
joints[indexjoints, 0] = int(bpindex)
joints[indexjoints, 1] = round(x, numdigits)
joints[indexjoints, 2] = round(y, numdigits)
indexjoints += 1
except:
pass
else:
indexjoints = len(multianimalbodyparts)
for bpindex, bodypart in enumerate(uniquebodyparts):
socialbdpt = bodypart # prefix+bodypart #build names!
# if socialbdpt in actualbpts:
try:
x, y = (
Data[prefix][socialbdpt]["x"][jj],
Data[prefix][socialbdpt]["y"][jj],
)
joints[indexjoints, 0] = len(
multianimalbodyparts
) + int(bpindex)
joints[indexjoints, 1] = round(x, 2)
joints[indexjoints, 2] = round(y, 2)
indexjoints += 1
except:
pass
# Drop missing body parts
joints = joints[~np.isnan(joints).any(axis=1)]
# Drop points lying outside the image
inside = np.logical_and.reduce(
(
joints[:, 1] < im.shape[1],
joints[:, 1] > 0,
joints[:, 2] < im.shape[0],
joints[:, 2] > 0,
)
)
joints = joints[inside]
if np.size(joints) > 0: # exclude images without labels
jointsannotated = True
Joints[prfxindex] = joints # np.array(joints, dtype=int)
H["joints"] = Joints
if jointsannotated: # exclude images without labels
data.append(H)
if len(trainIndices) > 0:
(
datafilename,
metadatafilename,
) = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg
)
################################################################################
# Saving metadata and data file (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename),
data,
trainIndices,
testIndices,
trainFraction,
)
datafilename = datafilename.split(".mat")[0] + ".pickle"
import pickle
with open(os.path.join(project_path, datafilename), "wb") as f:
# Pickle the 'labeled-data' dictionary using the highest protocol available.
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg
)
auxiliaryfunctions.attempttomakefolder(
Path(config).parents[0] / modelfoldername, recursive=True
)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "train")
)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "test")
)
path_train_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"train",
"pose_cfg.yaml",
)
)
path_test_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"pose_cfg.yaml",
)
)
path_inference_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"inference_cfg.yaml",
)
)
jointnames = [str(bpt) for bpt in multianimalbodyparts]
jointnames.extend([str(bpt) for bpt in uniquebodyparts])
items2change = {
"dataset": datafilename,
"metadataset": metadatafilename,
"num_joints": len(multianimalbodyparts)
+ len(uniquebodyparts), # cfg["uniquebodyparts"]),
"all_joints": [
[i]
for i in range(len(multianimalbodyparts) + len(uniquebodyparts))
], # cfg["uniquebodyparts"]))],
"all_joints_names": jointnames,
"init_weights": model_path,
"project_path": str(cfg["project_path"]),
"net_type": net_type,
"multi_stage": multi_stage,
"pairwise_loss_weight": 0.1,
"pafwidth": 20,
"partaffinityfield_graph": partaffinityfield_graph,
"partaffinityfield_predict": partaffinityfield_predict,
"weigh_only_present_joints": False,
"num_limbs": len(partaffinityfield_graph),
"dataset_type": dataset_type,
"optimizer": "adam",
"batch_size": 8,
"multi_step": [[1e-4, 7500], [5 * 1e-5, 12000], [1e-5, 200000]],
"save_iters": 10000,
"display_iters": 500,
"num_idchannel": len(cfg["individuals"])
if cfg.get("identity", False)
else 0,
}
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
trainingdata = trainingsetmanipulation.MakeTrain_pose_yaml(
items2change, path_train_config, defaultconfigfile
)
keys2save = [
"dataset",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"multi_stage",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
"dataset_type",
"partaffinityfield_predict",
"pairwise_predict",
"partaffinityfield_graph",
"num_limbs",
"dataset_type",
"num_idchannel",
]
trainingsetmanipulation.MakeTest_pose_yaml(
trainingdata,
keys2save,
path_test_config,
nmsradius=5.0,
minconfidence=0.01,
) # setting important def. values for inference
# Setting inference cfg file:
defaultinference_configfile = os.path.join(
dlcparent_path, "inference_cfg.yaml"
)
items2change = {
"minimalnumberofconnections": int(
len(cfg["multianimalbodyparts"]) / 2
),
"topktoretain": len(cfg["individuals"])
+ 1 * (len(cfg["uniquebodyparts"]) > 0),
"withid": cfg.get("identity", False),
}
trainingsetmanipulation.MakeInference_yaml(
items2change, path_inference_config, defaultinference_configfile
)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
else:
pass
def create_multianimaltraining_dataset(
config,
num_shuffles=1,
Shuffles=None,
windows2linux=False,
net_type=None,
numdigits=2,
paf_graph=None,
):
"""
Creates a training dataset for multi-animal datasets. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Imporant differences to standard:
- stores coordinates with numdigits as many digits
- creates
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
net_type: string
Type of networks. Currently resnet_50, resnet_101, and resnet_152, efficientnet-b0, efficientnet-b1, efficientnet-b2, efficientnet-b3,
efficientnet-b4, efficientnet-b5, and efficientnet-b6 as well as dlcrnet_ms5 are supported (not the MobileNets!).
See Lauer et al. 2021 https://www.biorxiv.org/content/10.1101/2021.04.30.442096v1
numdigits: int, optional
paf_graph: list of lists, optional (default=None)
If not None, overwrite the default complete graph. This is useful for advanced users who
already know a good graph, or simply want to use a specific one. Note that, in that case,
the data-driven selection procedure upon model evaluation will be skipped.
Example
--------
>>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcut.create_multianimaltraining_dataset(r'C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg["scorer"]
project_path = cfg["project_path"]
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
full_training_path = Path(project_path, trainingsetfolder)
auxiliaryfunctions.attempttomakefolder(full_training_path, recursive=True)
Data = merge_annotateddatasets(cfg, full_training_path, windows2linux)
if Data is None:
return
Data = Data[scorer]
def strip_cropped_image_name(path):
# utility function to split different crops from same image into either train or test!
head, filename = os.path.split(path)
if cfg["croppedtraining"]:
filename = filename.split("c")[0]
return os.path.join(head, filename)
img_names = Data.index.map(strip_cropped_image_name).unique()
if net_type is None: # loading & linking pretrained models
net_type = cfg.get("default_net_type", "dlcrnet_ms5")
elif not any(net in net_type for net in ("resnet", "eff", "dlc")):
raise ValueError(f"Unsupported network {net_type}.")
multi_stage = False
if net_type == "dlcrnet_ms5":
net_type = "resnet_50"
multi_stage = True
dataset_type = "multi-animal-imgaug"
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
if paf_graph is None: # Automatically form a complete PAF graph
partaffinityfield_graph = [
list(edge)
for edge in combinations(range(len(multianimalbodyparts)), 2)
]
else:
# Ignore possible connections between 'multi' and 'unique' body parts;
# one can never be too careful...
to_ignore = auxfun_multianimal.filter_unwanted_paf_connections(
cfg, paf_graph)
partaffinityfield_graph = [
edge for i, edge in enumerate(paf_graph) if i not in to_ignore
]
auxfun_multianimal.validate_paf_graph(cfg, partaffinityfield_graph)
print("Utilizing the following graph:", partaffinityfield_graph)
partaffinityfield_predict = True
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, Path(dlcparent_path), num_shuffles)
if Shuffles is None:
Shuffles = range(1, num_shuffles + 1, 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
TrainingFraction = cfg["TrainingFraction"]
for shuffle in Shuffles: # Creating shuffles starting from 1
for trainFraction in TrainingFraction:
train_inds_temp, test_inds_temp = SplitTrials(
range(len(img_names)), trainFraction)
# Map back to the original indices.
temp = [
re.escape(name) for i, name in enumerate(img_names)
if i in test_inds_temp
]
mask = Data.index.str.contains("|".join(temp))
testIndices = np.flatnonzero(mask)
trainIndices = np.flatnonzero(~mask)
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
print(
"Creating training data for: Shuffle:",
shuffle,
"TrainFraction: ",
trainFraction,
)
# Make training file!
data = format_multianimal_training_data(
Data,
trainIndices,
cfg["project_path"],
numdigits,
)
if len(trainIndices) > 0:
(
datafilename,
metadatafilename,
) = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
################################################################################
# Saving metadata and data file (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename),
data,
trainIndices,
testIndices,
trainFraction,
)
datafilename = datafilename.split(".mat")[0] + ".pickle"
import pickle
with open(os.path.join(project_path, datafilename), "wb") as f:
# Pickle the 'labeled-data' dictionary using the highest protocol available.
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
auxiliaryfunctions.attempttomakefolder(
Path(config).parents[0] / modelfoldername, recursive=True)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "train"))
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "test"))
path_train_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"train",
"pose_cfg.yaml",
))
path_test_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"pose_cfg.yaml",
))
path_inference_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"inference_cfg.yaml",
))
jointnames = [str(bpt) for bpt in multianimalbodyparts]
jointnames.extend([str(bpt) for bpt in uniquebodyparts])
items2change = {
"dataset":
datafilename,
"metadataset":
metadatafilename,
"num_joints":
len(multianimalbodyparts) +
len(uniquebodyparts), # cfg["uniquebodyparts"]),
"all_joints": [[i] for i in range(
len(multianimalbodyparts) + len(uniquebodyparts))
], # cfg["uniquebodyparts"]))],
"all_joints_names":
jointnames,
"init_weights":
model_path,
"project_path":
str(cfg["project_path"]),
"net_type":
net_type,
"multi_stage":
multi_stage,
"pairwise_loss_weight":
0.1,
"pafwidth":
20,
"partaffinityfield_graph":
partaffinityfield_graph,
"partaffinityfield_predict":
partaffinityfield_predict,
"weigh_only_present_joints":
False,
"num_limbs":
len(partaffinityfield_graph),
"dataset_type":
dataset_type,
"optimizer":
"adam",
"batch_size":
8,
"multi_step": [[1e-4, 7500], [5 * 1e-5, 12000],
[1e-5, 200000]],
"save_iters":
10000,
"display_iters":
500,
"num_idchannel":
len(cfg["individuals"])
if cfg.get("identity", False) else 0,
}
trainingdata = MakeTrain_pose_yaml(items2change,
path_train_config,
defaultconfigfile)
keys2save = [
"dataset",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"multi_stage",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
"dataset_type",
"partaffinityfield_predict",
"pairwise_predict",
"partaffinityfield_graph",
"num_limbs",
"dataset_type",
"num_idchannel",
]
MakeTest_pose_yaml(
trainingdata,
keys2save,
path_test_config,
nmsradius=5.0,
minconfidence=0.01,
) # setting important def. values for inference
# Setting inference cfg file:
defaultinference_configfile = os.path.join(
dlcparent_path, "inference_cfg.yaml")
items2change = {
"minimalnumberofconnections":
int(len(cfg["multianimalbodyparts"]) / 2),
"topktoretain":
len(cfg["individuals"]) + 1 *
(len(cfg["uniquebodyparts"]) > 0),
"withid":
cfg.get("identity", False),
}
MakeInference_yaml(items2change, path_inference_config,
defaultinference_configfile)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
else:
pass
dataset_type = "multi-animal-imgaug"
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
if paf_graph is None: # Automatically form a complete PAF graph
partaffinityfield_graph = [
list(edge) for edge in combinations(range(len(multianimalbodyparts)), 2)
]
else:
# Ignore possible connections between 'multi' and 'unique' body parts;
# one can never be too careful...
to_ignore = auxfun_multianimal.filter_unwanted_paf_connections(
cfg, paf_graph
)
partaffinityfield_graph = [
edge for i, edge in enumerate(paf_graph) if i not in to_ignore
]
auxfun_multianimal.validate_paf_graph(cfg, partaffinityfield_graph)
print("Utilizing the following graph:", partaffinityfield_graph)
partaffinityfield_predict = True
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, Path(dlcparent_path), num_shuffles