def create_multianimaltraining_dataset(
config,
num_shuffles=1,
Shuffles=None,
windows2linux=False,
net_type=None,
numdigits=2,
crop_size=(400, 400),
crop_sampling="hybrid",
paf_graph=None,
trainIndices=None,
testIndices=None,
):
"""
Creates a training dataset for multi-animal datasets. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Imporant differences to standard:
- stores coordinates with numdigits as many digits
- creates
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
net_type: string
Type of networks. Currently resnet_50, resnet_101, and resnet_152, efficientnet-b0, efficientnet-b1, efficientnet-b2, efficientnet-b3,
efficientnet-b4, efficientnet-b5, and efficientnet-b6 as well as dlcrnet_ms5 are supported (not the MobileNets!).
See Lauer et al. 2021 https://www.biorxiv.org/content/10.1101/2021.04.30.442096v1
numdigits: int, optional
crop_size: tuple of int, optional
Dimensions (width, height) of the crops for data augmentation.
Default is 400x400.
crop_sampling: str, optional
Crop centers sampling method. Must be either:
"uniform" (randomly over the image),
"keypoints" (randomly over the annotated keypoints),
"density" (weighing preferentially dense regions of keypoints),
or "hybrid" (alternating randomly between "uniform" and "density").
Default is "hybrid".
paf_graph: list of lists, optional (default=None)
If not None, overwrite the default complete graph. This is useful for advanced users who
already know a good graph, or simply want to use a specific one. Note that, in that case,
the data-driven selection procedure upon model evaluation will be skipped.
trainIndices: list of lists, optional (default=None)
List of one or multiple lists containing train indexes.
A list containing two lists of training indexes will produce two splits.
testIndices: list of lists, optional (default=None)
List of one or multiple lists containing test indexes.
Example
--------
>>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
>>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml', Shuffles=[0,1,2], trainIndices=[trainInd1, trainInd2, trainInd3], testIndices=[testInd1, testInd2, testInd3])
Windows:
>>> deeplabcut.create_multianimaltraining_dataset(r'C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
if windows2linux:
warnings.warn(
"`windows2linux` has no effect since 2.2.0.4 and will be removed in 2.2.1.",
FutureWarning,
)
if len(crop_size) != 2 or not all(isinstance(v, int) for v in crop_size):
raise ValueError(
"Crop size must be a tuple of two integers (width, height).")
if crop_sampling not in ("uniform", "keypoints", "density", "hybrid"):
raise ValueError(
f"Invalid sampling {crop_sampling}. Must be "
f"either 'uniform', 'keypoints', 'density', or 'hybrid.")
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg["scorer"]
project_path = cfg["project_path"]
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
full_training_path = Path(project_path, trainingsetfolder)
auxiliaryfunctions.attempttomakefolder(full_training_path, recursive=True)
Data = merge_annotateddatasets(cfg, full_training_path)
if Data is None:
return
Data = Data[scorer]
if net_type is None: # loading & linking pretrained models
net_type = cfg.get("default_net_type", "dlcrnet_ms5")
elif not any(net in net_type for net in ("resnet", "eff", "dlc", "mob")):
raise ValueError(f"Unsupported network {net_type}.")
multi_stage = False
### dlcnet_ms5: backbone resnet50 + multi-fusion & multi-stage module
### dlcr101_ms5/dlcr152_ms5: backbone resnet101/152 + multi-fusion & multi-stage module
if all(net in net_type for net in ("dlcr", "_ms5")):
num_layers = re.findall("dlcr([0-9]*)", net_type)[0]
if num_layers == "":
num_layers = 50
net_type = "resnet_{}".format(num_layers)
multi_stage = True
dataset_type = "multi-animal-imgaug"
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
if paf_graph is None: # Automatically form a complete PAF graph
partaffinityfield_graph = [
list(edge)
for edge in combinations(range(len(multianimalbodyparts)), 2)
]
else:
# Ignore possible connections between 'multi' and 'unique' body parts;
# one can never be too careful...
to_ignore = auxfun_multianimal.filter_unwanted_paf_connections(
cfg, paf_graph)
partaffinityfield_graph = [
edge for i, edge in enumerate(paf_graph) if i not in to_ignore
]
auxfun_multianimal.validate_paf_graph(cfg, partaffinityfield_graph)
print("Utilizing the following graph:", partaffinityfield_graph)
# Disable the prediction of PAFs if the graph is empty
partaffinityfield_predict = bool(partaffinityfield_graph)
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, Path(dlcparent_path), num_shuffles)
if Shuffles is None:
Shuffles = range(1, num_shuffles + 1, 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
# print(trainIndices,testIndices, Shuffles, augmenter_type,net_type)
if trainIndices is None and testIndices is None:
splits = []
for shuffle in Shuffles: # Creating shuffles starting from 1
for train_frac in cfg["TrainingFraction"]:
train_inds, test_inds = SplitTrials(range(len(Data)),
train_frac)
splits.append((train_frac, shuffle, (train_inds, test_inds)))
else:
if len(trainIndices) != len(testIndices) != len(Shuffles):
raise ValueError(
"Number of Shuffles and train and test indexes should be equal."
)
splits = []
for shuffle, (train_inds,
test_inds) in enumerate(zip(trainIndices, testIndices)):
trainFraction = round(
len(train_inds) * 1.0 / (len(train_inds) + len(test_inds)), 2)
print(
f"You passed a split with the following fraction: {int(100 * trainFraction)}%"
)
# Now that the training fraction is guaranteed to be correct,
# the values added to pad the indices are removed.
train_inds = np.asarray(train_inds)
train_inds = train_inds[train_inds != -1]
test_inds = np.asarray(test_inds)
test_inds = test_inds[test_inds != -1]
splits.append(
(trainFraction, Shuffles[shuffle], (train_inds, test_inds)))
for trainFraction, shuffle, (trainIndices, testIndices) in splits:
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
print(
"Creating training data for: Shuffle:",
shuffle,
"TrainFraction: ",
trainFraction,
)
# Make training file!
data = format_multianimal_training_data(
Data,
trainIndices,
cfg["project_path"],
numdigits,
)
if len(trainIndices) > 0:
(
datafilename,
metadatafilename,
) = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
################################################################################
# Saving metadata and data file (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename),
data,
trainIndices,
testIndices,
trainFraction,
)
datafilename = datafilename.split(".mat")[0] + ".pickle"
import pickle
with open(os.path.join(project_path, datafilename), "wb") as f:
# Pickle the 'labeled-data' dictionary using the highest protocol available.
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
auxiliaryfunctions.attempttomakefolder(Path(config).parents[0] /
modelfoldername,
recursive=True)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "train"))
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "test"))
path_train_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"train",
"pose_cfg.yaml",
))
path_test_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"pose_cfg.yaml",
))
path_inference_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"inference_cfg.yaml",
))
jointnames = [str(bpt) for bpt in multianimalbodyparts]
jointnames.extend([str(bpt) for bpt in uniquebodyparts])
items2change = {
"dataset":
datafilename,
"metadataset":
metadatafilename,
"num_joints":
len(multianimalbodyparts) +
len(uniquebodyparts), # cfg["uniquebodyparts"]),
"all_joints": [[i] for i in range(
len(multianimalbodyparts) + len(uniquebodyparts))
], # cfg["uniquebodyparts"]))],
"all_joints_names":
jointnames,
"init_weights":
model_path,
"project_path":
str(cfg["project_path"]),
"net_type":
net_type,
"multi_stage":
multi_stage,
"pairwise_loss_weight":
0.1,
"pafwidth":
20,
"partaffinityfield_graph":
partaffinityfield_graph,
"partaffinityfield_predict":
partaffinityfield_predict,
"weigh_only_present_joints":
False,
"num_limbs":
len(partaffinityfield_graph),
"dataset_type":
dataset_type,
"optimizer":
"adam",
"batch_size":
8,
"multi_step": [[1e-4, 7500], [5 * 1e-5, 12000], [1e-5,
200000]],
"save_iters":
10000,
"display_iters":
500,
"num_idchannel":
len(cfg["individuals"]) if cfg.get("identity", False) else 0,
"crop_size":
list(crop_size),
"crop_sampling":
crop_sampling,
}
trainingdata = MakeTrain_pose_yaml(items2change, path_train_config,
defaultconfigfile)
keys2save = [
"dataset",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"multi_stage",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
"dataset_type",
"partaffinityfield_predict",
"pairwise_predict",
"partaffinityfield_graph",
"num_limbs",
"dataset_type",
"num_idchannel",
]
MakeTest_pose_yaml(
trainingdata,
keys2save,
path_test_config,
nmsradius=5.0,
minconfidence=0.01,
sigma=1,
locref_smooth=False,
) # setting important def. values for inference
# Setting inference cfg file:
defaultinference_configfile = os.path.join(dlcparent_path,
"inference_cfg.yaml")
items2change = {
"minimalnumberofconnections":
int(len(cfg["multianimalbodyparts"]) / 2),
"topktoretain":
len(cfg["individuals"]) + 1 *
(len(cfg["uniquebodyparts"]) > 0),
"withid":
cfg.get("identity", False),
}
MakeInference_yaml(items2change, path_inference_config,
defaultinference_configfile)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
else:
pass
def create_multianimaltraining_dataset(
config,
num_shuffles=1,
Shuffles=None,
windows2linux=False,
net_type=None,
numdigits=2,
paf_graph=None,
):
"""
Creates a training dataset for multi-animal datasets. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Imporant differences to standard:
- stores coordinates with numdigits as many digits
- creates
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
net_type: string
Type of networks. Currently resnet_50, resnet_101, and resnet_152, efficientnet-b0, efficientnet-b1, efficientnet-b2, efficientnet-b3,
efficientnet-b4, efficientnet-b5, and efficientnet-b6 as well as dlcrnet_ms5 are supported (not the MobileNets!).
See Lauer et al. 2021 https://www.biorxiv.org/content/10.1101/2021.04.30.442096v1
numdigits: int, optional
paf_graph: list of lists, optional (default=None)
If not None, overwrite the default complete graph. This is useful for advanced users who
already know a good graph, or simply want to use a specific one. Note that, in that case,
the data-driven selection procedure upon model evaluation will be skipped.
Example
--------
>>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcut.create_multianimaltraining_dataset(r'C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg["scorer"]
project_path = cfg["project_path"]
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
full_training_path = Path(project_path, trainingsetfolder)
auxiliaryfunctions.attempttomakefolder(full_training_path, recursive=True)
Data = merge_annotateddatasets(cfg, full_training_path, windows2linux)
if Data is None:
return
Data = Data[scorer]
def strip_cropped_image_name(path):
# utility function to split different crops from same image into either train or test!
head, filename = os.path.split(path)
if cfg["croppedtraining"]:
filename = filename.split("c")[0]
return os.path.join(head, filename)
img_names = Data.index.map(strip_cropped_image_name).unique()
if net_type is None: # loading & linking pretrained models
net_type = cfg.get("default_net_type", "dlcrnet_ms5")
elif not any(net in net_type for net in ("resnet", "eff", "dlc")):
raise ValueError(f"Unsupported network {net_type}.")
multi_stage = False
if net_type == "dlcrnet_ms5":
net_type = "resnet_50"
multi_stage = True
dataset_type = "multi-animal-imgaug"
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
if paf_graph is None: # Automatically form a complete PAF graph
partaffinityfield_graph = [
list(edge)
for edge in combinations(range(len(multianimalbodyparts)), 2)
]
else:
# Ignore possible connections between 'multi' and 'unique' body parts;
# one can never be too careful...
to_ignore = auxfun_multianimal.filter_unwanted_paf_connections(
cfg, paf_graph)
partaffinityfield_graph = [
edge for i, edge in enumerate(paf_graph) if i not in to_ignore
]
auxfun_multianimal.validate_paf_graph(cfg, partaffinityfield_graph)
print("Utilizing the following graph:", partaffinityfield_graph)
partaffinityfield_predict = True
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, Path(dlcparent_path), num_shuffles)
if Shuffles is None:
Shuffles = range(1, num_shuffles + 1, 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
TrainingFraction = cfg["TrainingFraction"]
for shuffle in Shuffles: # Creating shuffles starting from 1
for trainFraction in TrainingFraction:
train_inds_temp, test_inds_temp = SplitTrials(
range(len(img_names)), trainFraction)
# Map back to the original indices.
temp = [
re.escape(name) for i, name in enumerate(img_names)
if i in test_inds_temp
]
mask = Data.index.str.contains("|".join(temp))
testIndices = np.flatnonzero(mask)
trainIndices = np.flatnonzero(~mask)
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
print(
"Creating training data for: Shuffle:",
shuffle,
"TrainFraction: ",
trainFraction,
)
# Make training file!
data = format_multianimal_training_data(
Data,
trainIndices,
cfg["project_path"],
numdigits,
)
if len(trainIndices) > 0:
(
datafilename,
metadatafilename,
) = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
################################################################################
# Saving metadata and data file (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename),
data,
trainIndices,
testIndices,
trainFraction,
)
datafilename = datafilename.split(".mat")[0] + ".pickle"
import pickle
with open(os.path.join(project_path, datafilename), "wb") as f:
# Pickle the 'labeled-data' dictionary using the highest protocol available.
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
auxiliaryfunctions.attempttomakefolder(
Path(config).parents[0] / modelfoldername, recursive=True)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "train"))
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "test"))
path_train_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"train",
"pose_cfg.yaml",
))
path_test_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"pose_cfg.yaml",
))
path_inference_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"inference_cfg.yaml",
))
jointnames = [str(bpt) for bpt in multianimalbodyparts]
jointnames.extend([str(bpt) for bpt in uniquebodyparts])
items2change = {
"dataset":
datafilename,
"metadataset":
metadatafilename,
"num_joints":
len(multianimalbodyparts) +
len(uniquebodyparts), # cfg["uniquebodyparts"]),
"all_joints": [[i] for i in range(
len(multianimalbodyparts) + len(uniquebodyparts))
], # cfg["uniquebodyparts"]))],
"all_joints_names":
jointnames,
"init_weights":
model_path,
"project_path":
str(cfg["project_path"]),
"net_type":
net_type,
"multi_stage":
multi_stage,
"pairwise_loss_weight":
0.1,
"pafwidth":
20,
"partaffinityfield_graph":
partaffinityfield_graph,
"partaffinityfield_predict":
partaffinityfield_predict,
"weigh_only_present_joints":
False,
"num_limbs":
len(partaffinityfield_graph),
"dataset_type":
dataset_type,
"optimizer":
"adam",
"batch_size":
8,
"multi_step": [[1e-4, 7500], [5 * 1e-5, 12000],
[1e-5, 200000]],
"save_iters":
10000,
"display_iters":
500,
"num_idchannel":
len(cfg["individuals"])
if cfg.get("identity", False) else 0,
}
trainingdata = MakeTrain_pose_yaml(items2change,
path_train_config,
defaultconfigfile)
keys2save = [
"dataset",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"multi_stage",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
"dataset_type",
"partaffinityfield_predict",
"pairwise_predict",
"partaffinityfield_graph",
"num_limbs",
"dataset_type",
"num_idchannel",
]
MakeTest_pose_yaml(
trainingdata,
keys2save,
path_test_config,
nmsradius=5.0,
minconfidence=0.01,
) # setting important def. values for inference
# Setting inference cfg file:
defaultinference_configfile = os.path.join(
dlcparent_path, "inference_cfg.yaml")
items2change = {
"minimalnumberofconnections":
int(len(cfg["multianimalbodyparts"]) / 2),
"topktoretain":
len(cfg["individuals"]) + 1 *
(len(cfg["uniquebodyparts"]) > 0),
"withid":
cfg.get("identity", False),
}
MakeInference_yaml(items2change, path_inference_config,
defaultinference_configfile)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
else:
pass