def create_multianimaltraining_dataset(
config,
num_shuffles=1,
Shuffles=None,
windows2linux=False,
net_type=None,
numdigits=2,
crop_size=(400, 400),
crop_sampling="hybrid",
paf_graph=None,
trainIndices=None,
testIndices=None,
):
"""
Creates a training dataset for multi-animal datasets. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Imporant differences to standard:
- stores coordinates with numdigits as many digits
- creates
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
net_type: string
Type of networks. Currently resnet_50, resnet_101, and resnet_152, efficientnet-b0, efficientnet-b1, efficientnet-b2, efficientnet-b3,
efficientnet-b4, efficientnet-b5, and efficientnet-b6 as well as dlcrnet_ms5 are supported (not the MobileNets!).
See Lauer et al. 2021 https://www.biorxiv.org/content/10.1101/2021.04.30.442096v1
numdigits: int, optional
crop_size: tuple of int, optional
Dimensions (width, height) of the crops for data augmentation.
Default is 400x400.
crop_sampling: str, optional
Crop centers sampling method. Must be either:
"uniform" (randomly over the image),
"keypoints" (randomly over the annotated keypoints),
"density" (weighing preferentially dense regions of keypoints),
or "hybrid" (alternating randomly between "uniform" and "density").
Default is "hybrid".
paf_graph: list of lists, optional (default=None)
If not None, overwrite the default complete graph. This is useful for advanced users who
already know a good graph, or simply want to use a specific one. Note that, in that case,
the data-driven selection procedure upon model evaluation will be skipped.
trainIndices: list of lists, optional (default=None)
List of one or multiple lists containing train indexes.
A list containing two lists of training indexes will produce two splits.
testIndices: list of lists, optional (default=None)
List of one or multiple lists containing test indexes.
Example
--------
>>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
>>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml', Shuffles=[0,1,2], trainIndices=[trainInd1, trainInd2, trainInd3], testIndices=[testInd1, testInd2, testInd3])
Windows:
>>> deeplabcut.create_multianimaltraining_dataset(r'C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
if windows2linux:
warnings.warn(
"`windows2linux` has no effect since 2.2.0.4 and will be removed in 2.2.1.",
FutureWarning,
)
if len(crop_size) != 2 or not all(isinstance(v, int) for v in crop_size):
raise ValueError(
"Crop size must be a tuple of two integers (width, height).")
if crop_sampling not in ("uniform", "keypoints", "density", "hybrid"):
raise ValueError(
f"Invalid sampling {crop_sampling}. Must be "
f"either 'uniform', 'keypoints', 'density', or 'hybrid.")
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg["scorer"]
project_path = cfg["project_path"]
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
full_training_path = Path(project_path, trainingsetfolder)
auxiliaryfunctions.attempttomakefolder(full_training_path, recursive=True)
Data = merge_annotateddatasets(cfg, full_training_path)
if Data is None:
return
Data = Data[scorer]
if net_type is None: # loading & linking pretrained models
net_type = cfg.get("default_net_type", "dlcrnet_ms5")
elif not any(net in net_type for net in ("resnet", "eff", "dlc", "mob")):
raise ValueError(f"Unsupported network {net_type}.")
multi_stage = False
### dlcnet_ms5: backbone resnet50 + multi-fusion & multi-stage module
### dlcr101_ms5/dlcr152_ms5: backbone resnet101/152 + multi-fusion & multi-stage module
if all(net in net_type for net in ("dlcr", "_ms5")):
num_layers = re.findall("dlcr([0-9]*)", net_type)[0]
if num_layers == "":
num_layers = 50
net_type = "resnet_{}".format(num_layers)
multi_stage = True
dataset_type = "multi-animal-imgaug"
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
if paf_graph is None: # Automatically form a complete PAF graph
partaffinityfield_graph = [
list(edge)
for edge in combinations(range(len(multianimalbodyparts)), 2)
]
else:
# Ignore possible connections between 'multi' and 'unique' body parts;
# one can never be too careful...
to_ignore = auxfun_multianimal.filter_unwanted_paf_connections(
cfg, paf_graph)
partaffinityfield_graph = [
edge for i, edge in enumerate(paf_graph) if i not in to_ignore
]
auxfun_multianimal.validate_paf_graph(cfg, partaffinityfield_graph)
print("Utilizing the following graph:", partaffinityfield_graph)
# Disable the prediction of PAFs if the graph is empty
partaffinityfield_predict = bool(partaffinityfield_graph)
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, Path(dlcparent_path), num_shuffles)
if Shuffles is None:
Shuffles = range(1, num_shuffles + 1, 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
# print(trainIndices,testIndices, Shuffles, augmenter_type,net_type)
if trainIndices is None and testIndices is None:
splits = []
for shuffle in Shuffles: # Creating shuffles starting from 1
for train_frac in cfg["TrainingFraction"]:
train_inds, test_inds = SplitTrials(range(len(Data)),
train_frac)
splits.append((train_frac, shuffle, (train_inds, test_inds)))
else:
if len(trainIndices) != len(testIndices) != len(Shuffles):
raise ValueError(
"Number of Shuffles and train and test indexes should be equal."
)
splits = []
for shuffle, (train_inds,
test_inds) in enumerate(zip(trainIndices, testIndices)):
trainFraction = round(
len(train_inds) * 1.0 / (len(train_inds) + len(test_inds)), 2)
print(
f"You passed a split with the following fraction: {int(100 * trainFraction)}%"
)
# Now that the training fraction is guaranteed to be correct,
# the values added to pad the indices are removed.
train_inds = np.asarray(train_inds)
train_inds = train_inds[train_inds != -1]
test_inds = np.asarray(test_inds)
test_inds = test_inds[test_inds != -1]
splits.append(
(trainFraction, Shuffles[shuffle], (train_inds, test_inds)))
for trainFraction, shuffle, (trainIndices, testIndices) in splits:
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
print(
"Creating training data for: Shuffle:",
shuffle,
"TrainFraction: ",
trainFraction,
)
# Make training file!
data = format_multianimal_training_data(
Data,
trainIndices,
cfg["project_path"],
numdigits,
)
if len(trainIndices) > 0:
(
datafilename,
metadatafilename,
) = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
################################################################################
# Saving metadata and data file (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename),
data,
trainIndices,
testIndices,
trainFraction,
)
datafilename = datafilename.split(".mat")[0] + ".pickle"
import pickle
with open(os.path.join(project_path, datafilename), "wb") as f:
# Pickle the 'labeled-data' dictionary using the highest protocol available.
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
auxiliaryfunctions.attempttomakefolder(Path(config).parents[0] /
modelfoldername,
recursive=True)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "train"))
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "test"))
path_train_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"train",
"pose_cfg.yaml",
))
path_test_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"pose_cfg.yaml",
))
path_inference_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"inference_cfg.yaml",
))
jointnames = [str(bpt) for bpt in multianimalbodyparts]
jointnames.extend([str(bpt) for bpt in uniquebodyparts])
items2change = {
"dataset":
datafilename,
"metadataset":
metadatafilename,
"num_joints":
len(multianimalbodyparts) +
len(uniquebodyparts), # cfg["uniquebodyparts"]),
"all_joints": [[i] for i in range(
len(multianimalbodyparts) + len(uniquebodyparts))
], # cfg["uniquebodyparts"]))],
"all_joints_names":
jointnames,
"init_weights":
model_path,
"project_path":
str(cfg["project_path"]),
"net_type":
net_type,
"multi_stage":
multi_stage,
"pairwise_loss_weight":
0.1,
"pafwidth":
20,
"partaffinityfield_graph":
partaffinityfield_graph,
"partaffinityfield_predict":
partaffinityfield_predict,
"weigh_only_present_joints":
False,
"num_limbs":
len(partaffinityfield_graph),
"dataset_type":
dataset_type,
"optimizer":
"adam",
"batch_size":
8,
"multi_step": [[1e-4, 7500], [5 * 1e-5, 12000], [1e-5,
200000]],
"save_iters":
10000,
"display_iters":
500,
"num_idchannel":
len(cfg["individuals"]) if cfg.get("identity", False) else 0,
"crop_size":
list(crop_size),
"crop_sampling":
crop_sampling,
}
trainingdata = MakeTrain_pose_yaml(items2change, path_train_config,
defaultconfigfile)
keys2save = [
"dataset",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"multi_stage",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
"dataset_type",
"partaffinityfield_predict",
"pairwise_predict",
"partaffinityfield_graph",
"num_limbs",
"dataset_type",
"num_idchannel",
]
MakeTest_pose_yaml(
trainingdata,
keys2save,
path_test_config,
nmsradius=5.0,
minconfidence=0.01,
sigma=1,
locref_smooth=False,
) # setting important def. values for inference
# Setting inference cfg file:
defaultinference_configfile = os.path.join(dlcparent_path,
"inference_cfg.yaml")
items2change = {
"minimalnumberofconnections":
int(len(cfg["multianimalbodyparts"]) / 2),
"topktoretain":
len(cfg["individuals"]) + 1 *
(len(cfg["uniquebodyparts"]) > 0),
"withid":
cfg.get("identity", False),
}
MakeInference_yaml(items2change, path_inference_config,
defaultinference_configfile)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
else:
pass
def create_training_dataset(
config,
num_shuffles=1,
Shuffles=None,
windows2linux=False,
userfeedback=False,
trainIndices=None,
testIndices=None,
net_type=None,
augmenter_type=None,
):
"""
Creates a training dataset. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
userfeedback: bool, optional
If this is set to false, then all requested train/test splits are created (no matter if they already exist). If you
want to assure that previous splits etc. are not overwritten, then set this to True and you will be asked for each split.
trainIndices: list of lists, optional (default=None)
List of one or multiple lists containing train indexes.
A list containing two lists of training indexes will produce two splits.
testIndices: list of lists, optional (default=None)
List of one or multiple lists containing test indexes.
net_type: string
Type of networks. Currently resnet_50, resnet_101, resnet_152, mobilenet_v2_1.0,mobilenet_v2_0.75, mobilenet_v2_0.5, and mobilenet_v2_0.35 are supported.
augmenter_type: string
Type of augmenter. Currently default, imgaug, tensorpack, and deterministic are supported.
Example
--------
>>> deeplabcut.create_training_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcut.create_training_dataset('C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
import scipy.io as sio
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
if cfg.get("multianimalproject", False):
from deeplabcut.generate_training_dataset.multiple_individuals_trainingsetmanipulation import (
create_multianimaltraining_dataset, )
create_multianimaltraining_dataset(config, num_shuffles, Shuffles,
windows2linux, net_type)
else:
scorer = cfg["scorer"]
project_path = cfg["project_path"]
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(
cfg) # Path concatenation OS platform independent
auxiliaryfunctions.attempttomakefolder(Path(
os.path.join(project_path, str(trainingsetfolder))),
recursive=True)
Data = merge_annotateddatasets(
cfg, Path(os.path.join(project_path, trainingsetfolder)),
windows2linux)
if Data is None:
return
Data = Data[scorer] # extract labeled data
# loading & linking pretrained models
if net_type is None: # loading & linking pretrained models
net_type = cfg.get("default_net_type", "resnet_50")
else:
if "resnet" in net_type or "mobilenet" in net_type:
pass
else:
raise ValueError("Invalid network type:", net_type)
if augmenter_type is None:
augmenter_type = cfg.get("default_augmenter", "imgaug")
if augmenter_type is None: # this could be in config.yaml for old projects!
# updating variable if null/None! #backwardscompatability
auxiliaryfunctions.edit_config(config,
{"default_augmenter": "imgaug"})
augmenter_type = "imgaug"
else:
if augmenter_type in [
"default",
"scalecrop",
"imgaug",
"tensorpack",
"deterministic",
]:
pass
else:
raise ValueError("Invalid augmenter type:", augmenter_type)
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, Path(dlcparent_path), num_shuffles)
if Shuffles is None:
Shuffles = range(1, num_shuffles + 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
# print(trainIndices,testIndices, Shuffles, augmenter_type,net_type)
if trainIndices is None and testIndices is None:
splits = [(
trainFraction,
shuffle,
SplitTrials(range(len(Data.index)), trainFraction),
) for trainFraction in cfg["TrainingFraction"]
for shuffle in Shuffles]
else:
if len(trainIndices) != len(testIndices) != len(Shuffles):
raise ValueError(
"Number of Shuffles and train and test indexes should be equal."
)
splits = []
for shuffle, (train_inds, test_inds) in enumerate(
zip(trainIndices, testIndices)):
trainFraction = round(
len(train_inds) * 1.0 / (len(train_inds) + len(test_inds)),
2)
print(
f"You passed a split with the following fraction: {int(100 * trainFraction)}%"
)
splits.append((trainFraction, Shuffles[shuffle], (train_inds,
test_inds)))
bodyparts = cfg["bodyparts"]
nbodyparts = len(bodyparts)
for trainFraction, shuffle, (trainIndices, testIndices) in splits:
if len(trainIndices) > 0:
if userfeedback:
trainposeconfigfile, _, _ = training.return_train_network_path(
config,
shuffle=shuffle,
trainingsetindex=cfg["TrainingFraction"].index(
trainFraction),
)
if trainposeconfigfile.is_file():
askuser = input(
"The model folder is already present. If you continue, it will overwrite the existing model (split). Do you want to continue?(yes/no): "
)
if (askuser == "no" or askuser == "No"
or askuser == "N" or askuser == "No"):
raise Exception(
"Use the Shuffles argument as a list to specify a different shuffle index. Check out the help for more details."
)
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
# Make training file!
(
datafilename,
metadatafilename,
) = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
################################################################################
# Saving data file (convert to training file for deeper cut (*.mat))
################################################################################
data, MatlabData = format_training_data(
Data, trainIndices, nbodyparts, project_path)
sio.savemat(os.path.join(project_path, datafilename),
{"dataset": MatlabData})
################################################################################
# Saving metadata (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename),
data,
trainIndices,
testIndices,
trainFraction,
)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
auxiliaryfunctions.attempttomakefolder(
Path(config).parents[0] / modelfoldername, recursive=True)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername) + "/train")
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername) + "/test")
path_train_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"train",
"pose_cfg.yaml",
))
path_test_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"pose_cfg.yaml",
))
# str(cfg['proj_path']+'/'+Path(modelfoldername) / 'test' / 'pose_cfg.yaml')
items2change = {
"dataset": datafilename,
"metadataset": metadatafilename,
"num_joints": len(bodyparts),
"all_joints": [[i] for i in range(len(bodyparts))],
"all_joints_names": [str(bpt) for bpt in bodyparts],
"init_weights": model_path,
"project_path": str(cfg["project_path"]),
"net_type": net_type,
"dataset_type": augmenter_type,
}
items2drop = {}
if augmenter_type == "scalecrop":
# these values are dropped as scalecrop
# doesn't have rotation implemented
items2drop = {"rotation": 0, "rotratio": 0.0}
trainingdata = MakeTrain_pose_yaml(items2change,
path_train_config,
defaultconfigfile,
items2drop)
keys2save = [
"dataset",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
]
MakeTest_pose_yaml(trainingdata, keys2save, path_test_config)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
return splits
def create_training_dataset(config,
num_shuffles=1,
Shuffles=None,
windows2linux=False,
trainIndexes=None,
testIndexes=None):
"""
Creates a training dataset. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
Example
--------
>>> deeplabcut.create_training_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcut.create_training_dataset('C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
from skimage import io
import scipy.io as sio
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg['scorer']
project_path = cfg['project_path']
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(
cfg) #Path concatenation OS platform independent
auxiliaryfunctions.attempttomakefolder(Path(
os.path.join(project_path, str(trainingsetfolder))),
recursive=True)
Data = merge_annotateddatasets(
cfg, project_path, Path(os.path.join(project_path, trainingsetfolder)),
windows2linux)
Data = Data[scorer] #extract labeled data
#loading & linking pretrained models
net_type = 'resnet_' + str(cfg['resnet'])
import deeplabcut
parent_path = Path(os.path.dirname(deeplabcut.__file__))
defaultconfigfile = str(parent_path / 'pose_cfg.yaml')
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, parent_path, num_shuffles)
if Shuffles == None:
Shuffles = range(1, num_shuffles + 1, 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
bodyparts = cfg['bodyparts']
TrainingFraction = cfg['TrainingFraction']
for shuffle in Shuffles: # Creating shuffles starting from 1
for trainFraction in TrainingFraction:
#trainIndexes, testIndexes = SplitTrials(range(len(Data.index)), trainFraction)
if trainIndexes is None and testIndexes is None:
trainIndexes, testIndexes = SplitTrials(
range(len(Data.index)), trainFraction)
else:
print(
"You passed a split with the following fraction:",
len(trainIndexes) * 1. /
(len(testIndexes) + len(trainIndexes)) * 100)
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
# Make training file!
data = []
for jj in trainIndexes:
H = {}
# load image to get dimensions:
filename = Data.index[jj]
im = io.imread(os.path.join(cfg['project_path'], filename))
H['image'] = filename
if np.ndim(im) == 3:
H['size'] = np.array(
[np.shape(im)[2],
np.shape(im)[0],
np.shape(im)[1]])
else:
# print "Grayscale!"
H['size'] = np.array([1, np.shape(im)[0], np.shape(im)[1]])
indexjoints = 0
joints = np.zeros((len(bodyparts), 3)) * np.nan
for bpindex, bodypart in enumerate(bodyparts):
if Data[bodypart]['x'][jj] < np.shape(
im)[1] and Data[bodypart]['y'][jj] < np.shape(
im)[0]: #are labels in image?
joints[indexjoints, 0] = int(bpindex)
joints[indexjoints, 1] = Data[bodypart]['x'][jj]
joints[indexjoints, 2] = Data[bodypart]['y'][jj]
indexjoints += 1
joints = joints[np.where(np.prod(
np.isfinite(joints),
1))[0], :] # drop NaN, i.e. lines for missing body parts
assert (np.prod(np.array(joints[:, 2]) < np.shape(im)[0])
) # y coordinate within image?
assert (np.prod(np.array(joints[:, 1]) < np.shape(im)[1])
) # x coordinate within image?
H['joints'] = np.array(joints, dtype=int)
if np.size(joints) > 0: #exclude images without labels
data.append(H)
if len(trainIndexes) > 0:
datafilename, metadatafilename = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
################################################################################
# Saving metadata (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename), data,
trainIndexes, testIndexes, trainFraction)
################################################################################
# Saving data file (convert to training file for deeper cut (*.mat))
################################################################################
DTYPE = [('image', 'O'), ('size', 'O'), ('joints', 'O')]
MatlabData = np.array(
[(np.array([data[item]['image']],
dtype='U'), np.array([data[item]['size']]),
boxitintoacell(data[item]['joints']))
for item in range(len(data))],
dtype=DTYPE)
sio.savemat(os.path.join(project_path, datafilename),
{'dataset': MatlabData})
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
auxiliaryfunctions.attempttomakefolder(
Path(config).parents[0] / modelfoldername, recursive=True)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername) + '/' +
'/train')
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername) + '/' +
'/test')
path_train_config = str(
os.path.join(cfg['project_path'], Path(modelfoldername),
'train', 'pose_cfg.yaml'))
path_test_config = str(
os.path.join(cfg['project_path'], Path(modelfoldername),
'test', 'pose_cfg.yaml'))
#str(cfg['proj_path']+'/'+Path(modelfoldername) / 'test' / 'pose_cfg.yaml')
items2change = {
"dataset": datafilename,
"metadataset": metadatafilename,
"num_joints": len(bodyparts),
"all_joints": [[i] for i in range(len(bodyparts))],
"all_joints_names": [str(bpt) for bpt in bodyparts],
"init_weights": model_path,
"project_path": str(cfg['project_path']),
"net_type": net_type
}
trainingdata = MakeTrain_pose_yaml(items2change,
path_train_config,
defaultconfigfile)
keys2save = [
"dataset", "num_joints", "all_joints", "all_joints_names",
"net_type", 'init_weights', 'global_scale',
'location_refinement', 'locref_stdev'
]
MakeTest_pose_yaml(trainingdata, keys2save, path_test_config)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
def create_multianimaltraining_dataset(
config,
num_shuffles=1,
Shuffles=None,
windows2linux=False,
net_type=None,
numdigits=2,
paf_graph=None,
):
"""
Creates a training dataset for multi-animal datasets. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Imporant differences to standard:
- stores coordinates with numdigits as many digits
- creates
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
net_type: string
Type of networks. Currently resnet_50, resnet_101, and resnet_152, efficientnet-b0, efficientnet-b1, efficientnet-b2, efficientnet-b3,
efficientnet-b4, efficientnet-b5, and efficientnet-b6 as well as dlcrnet_ms5 are supported (not the MobileNets!).
See Lauer et al. 2021 https://www.biorxiv.org/content/10.1101/2021.04.30.442096v1
numdigits: int, optional
paf_graph: list of lists, optional (default=None)
If not None, overwrite the default complete graph. This is useful for advanced users who
already know a good graph, or simply want to use a specific one. Note that, in that case,
the data-driven selection procedure upon model evaluation will be skipped.
Example
--------
>>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcut.create_multianimaltraining_dataset(r'C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg["scorer"]
project_path = cfg["project_path"]
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
full_training_path = Path(project_path, trainingsetfolder)
auxiliaryfunctions.attempttomakefolder(full_training_path, recursive=True)
Data = merge_annotateddatasets(cfg, full_training_path, windows2linux)
if Data is None:
return
Data = Data[scorer]
def strip_cropped_image_name(path):
# utility function to split different crops from same image into either train or test!
head, filename = os.path.split(path)
if cfg["croppedtraining"]:
filename = filename.split("c")[0]
return os.path.join(head, filename)
img_names = Data.index.map(strip_cropped_image_name).unique()
if net_type is None: # loading & linking pretrained models
net_type = cfg.get("default_net_type", "dlcrnet_ms5")
elif not any(net in net_type for net in ("resnet", "eff", "dlc")):
raise ValueError(f"Unsupported network {net_type}.")
multi_stage = False
if net_type == "dlcrnet_ms5":
net_type = "resnet_50"
multi_stage = True
dataset_type = "multi-animal-imgaug"
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
if paf_graph is None: # Automatically form a complete PAF graph
partaffinityfield_graph = [
list(edge)
for edge in combinations(range(len(multianimalbodyparts)), 2)
]
else:
# Ignore possible connections between 'multi' and 'unique' body parts;
# one can never be too careful...
to_ignore = auxfun_multianimal.filter_unwanted_paf_connections(
cfg, paf_graph)
partaffinityfield_graph = [
edge for i, edge in enumerate(paf_graph) if i not in to_ignore
]
auxfun_multianimal.validate_paf_graph(cfg, partaffinityfield_graph)
print("Utilizing the following graph:", partaffinityfield_graph)
partaffinityfield_predict = True
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, Path(dlcparent_path), num_shuffles)
if Shuffles is None:
Shuffles = range(1, num_shuffles + 1, 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
TrainingFraction = cfg["TrainingFraction"]
for shuffle in Shuffles: # Creating shuffles starting from 1
for trainFraction in TrainingFraction:
train_inds_temp, test_inds_temp = SplitTrials(
range(len(img_names)), trainFraction)
# Map back to the original indices.
temp = [
re.escape(name) for i, name in enumerate(img_names)
if i in test_inds_temp
]
mask = Data.index.str.contains("|".join(temp))
testIndices = np.flatnonzero(mask)
trainIndices = np.flatnonzero(~mask)
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
print(
"Creating training data for: Shuffle:",
shuffle,
"TrainFraction: ",
trainFraction,
)
# Make training file!
data = format_multianimal_training_data(
Data,
trainIndices,
cfg["project_path"],
numdigits,
)
if len(trainIndices) > 0:
(
datafilename,
metadatafilename,
) = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
################################################################################
# Saving metadata and data file (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename),
data,
trainIndices,
testIndices,
trainFraction,
)
datafilename = datafilename.split(".mat")[0] + ".pickle"
import pickle
with open(os.path.join(project_path, datafilename), "wb") as f:
# Pickle the 'labeled-data' dictionary using the highest protocol available.
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
auxiliaryfunctions.attempttomakefolder(
Path(config).parents[0] / modelfoldername, recursive=True)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "train"))
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername / "test"))
path_train_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"train",
"pose_cfg.yaml",
))
path_test_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"pose_cfg.yaml",
))
path_inference_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"inference_cfg.yaml",
))
jointnames = [str(bpt) for bpt in multianimalbodyparts]
jointnames.extend([str(bpt) for bpt in uniquebodyparts])
items2change = {
"dataset":
datafilename,
"metadataset":
metadatafilename,
"num_joints":
len(multianimalbodyparts) +
len(uniquebodyparts), # cfg["uniquebodyparts"]),
"all_joints": [[i] for i in range(
len(multianimalbodyparts) + len(uniquebodyparts))
], # cfg["uniquebodyparts"]))],
"all_joints_names":
jointnames,
"init_weights":
model_path,
"project_path":
str(cfg["project_path"]),
"net_type":
net_type,
"multi_stage":
multi_stage,
"pairwise_loss_weight":
0.1,
"pafwidth":
20,
"partaffinityfield_graph":
partaffinityfield_graph,
"partaffinityfield_predict":
partaffinityfield_predict,
"weigh_only_present_joints":
False,
"num_limbs":
len(partaffinityfield_graph),
"dataset_type":
dataset_type,
"optimizer":
"adam",
"batch_size":
8,
"multi_step": [[1e-4, 7500], [5 * 1e-5, 12000],
[1e-5, 200000]],
"save_iters":
10000,
"display_iters":
500,
"num_idchannel":
len(cfg["individuals"])
if cfg.get("identity", False) else 0,
}
trainingdata = MakeTrain_pose_yaml(items2change,
path_train_config,
defaultconfigfile)
keys2save = [
"dataset",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"multi_stage",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
"dataset_type",
"partaffinityfield_predict",
"pairwise_predict",
"partaffinityfield_graph",
"num_limbs",
"dataset_type",
"num_idchannel",
]
MakeTest_pose_yaml(
trainingdata,
keys2save,
path_test_config,
nmsradius=5.0,
minconfidence=0.01,
) # setting important def. values for inference
# Setting inference cfg file:
defaultinference_configfile = os.path.join(
dlcparent_path, "inference_cfg.yaml")
items2change = {
"minimalnumberofconnections":
int(len(cfg["multianimalbodyparts"]) / 2),
"topktoretain":
len(cfg["individuals"]) + 1 *
(len(cfg["uniquebodyparts"]) > 0),
"withid":
cfg.get("identity", False),
}
MakeInference_yaml(items2change, path_inference_config,
defaultinference_configfile)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
else:
pass
def create_pretrained_human_project(project,
experimenter,
videos,
working_directory=None,
copy_videos=False,
videotype='.avi',
createlabeledvideo=True,
analyzevideo=True):
"""
Creates a demo human project and analyzes a video with ResNet 101 weights pretrained on MPII Human Pose. This is from the DeeperCut paper by Insafutdinov et al. https://arxiv.org/abs/1605.03170 Please make sure to cite it too if you use this code!
Parameters
----------
project : string
String containing the name of the project.
experimenter : string
String containing the name of the experimenter.
videos : list
A list of string containing the full paths of the videos to include in the project.
working_directory : string, optional
The directory where the project will be created. The default is the ``current working directory``; if provided, it must be a string.
copy_videos : bool, optional
If this is set to True, the videos are copied to the ``videos`` directory. If it is False,symlink of the videos are copied to the project/videos directory. The default is ``False``; if provided it must be either
``True`` or ``False``.
analyzevideo " bool, optional
If true, then the video is analzyed and a labeled video is created. If false, then only the project will be created and the weights downloaded. You can then access them
Example
--------
Linux/MacOs
>>> deeplabcut.create_pretrained_human_project('human','Linus',['/data/videos/mouse1.avi'],'/analysis/project/',copy_videos=False)
Windows:
>>> deeplabcut.create_pretrained_human_project('human','Bill',[r'C:\yourusername\rig-95\Videos\reachingvideo1.avi'],r'C:\yourusername\analysis\project' copy_videos=False)
Users must format paths with either: r'C:\ OR 'C:\\ <- i.e. a double backslash \ \ )
--------
"""
cfg = deeplabcut.create_new_project(project, experimenter, videos,
working_directory, copy_videos,
videotype)
config = auxiliaryfunctions.read_config(cfg)
config['bodyparts'] = [
'ankle1', 'knee1', 'hip1', 'hip2', 'knee2', 'ankle2', 'wrist1',
'elbow1', 'shoulder1', 'shoulder2', 'elbow2', 'wrist2', 'chin',
'forehead'
]
config['skeleton'] = [['ankle1', 'knee1'], ['ankle2', 'knee2'],
['knee1', 'hip1'], ['knee2', 'hip2'],
['hip1', 'hip2'], ['shoulder1', 'shoulder2'],
['shoulder1', 'hip1'], ['shoulder2', 'hip2'],
['shoulder1', 'elbow1'], ['shoulder2', 'elbow2'],
['chin', 'forehead'], ['elbow1', 'wrist1'],
['elbow2', 'wrist2']]
config['default_net_type'] = 'resnet_101'
auxiliaryfunctions.write_config(cfg, config)
config = auxiliaryfunctions.read_config(cfg)
train_dir = Path(
os.path.join(
config['project_path'],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction=config['TrainingFraction'][0],
shuffle=1,
cfg=config)), 'train'))
test_dir = Path(
os.path.join(
config['project_path'],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction=config['TrainingFraction'][0],
shuffle=1,
cfg=config)), 'test'))
# Create the model directory
train_dir.mkdir(parents=True, exist_ok=True)
test_dir.mkdir(parents=True, exist_ok=True)
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction=config['TrainingFraction'][0], shuffle=1, cfg=config)
path_train_config = str(
os.path.join(config['project_path'], Path(modelfoldername), 'train',
'pose_cfg.yaml'))
path_test_config = str(
os.path.join(config['project_path'], Path(modelfoldername), 'test',
'pose_cfg.yaml'))
# Download the weights and put then in appropriate directory
cwd = os.getcwd()
os.chdir(train_dir)
print(
"Checking if the weights are already available, otherwise I will download them!"
)
weightfilename = auxfun_models.download_mpii_weigths(train_dir)
os.chdir(cwd)
# Create the pose_config.yaml files
parent_path = Path(os.path.dirname(deeplabcut.__file__))
defaultconfigfile = str(parent_path / 'pose_cfg.yaml')
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(config)
datafilename, metadatafilename = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder,
trainFraction=config['TrainingFraction'][0],
shuffle=1,
cfg=config)
bodyparts = config['bodyparts']
net_type = 'resnet_101'
num_shuffles = 1
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, parent_path, num_shuffles)
items2change = {
"dataset": 'dataset-test.mat', #datafilename,
"metadataset": metadatafilename,
"num_joints": len(bodyparts),
"all_joints": [[i] for i in range(len(bodyparts))],
"all_joints_names": [str(bpt) for bpt in bodyparts],
"init_weights":
weightfilename.split('.index')[0], #'models/mpii/snapshot-1030000',
"project_path": str(config['project_path']),
"net_type": net_type,
"dataset_type": "default"
}
trainingdata = MakeTrain_pose_yaml(items2change, path_train_config,
defaultconfigfile)
keys2save = [
"dataset", "dataset_type", "num_joints", "all_joints",
"all_joints_names", "net_type", 'init_weights', 'global_scale',
'location_refinement', 'locref_stdev'
]
MakeTest_pose_yaml(trainingdata, keys2save, path_test_config)
video_dir = os.path.join(config['project_path'], 'videos')
if analyzevideo == True:
# Analyze the videos
deeplabcut.analyze_videos(cfg, [video_dir],
videotype,
save_as_csv=True)
if createlabeledvideo == True:
deeplabcut.create_labeled_video(cfg, [video_dir],
videotype,
draw_skeleton=True)
deeplabcut.plot_trajectories(cfg, [video_dir], videotype)
return cfg, path_train_config
def create_training_dataset(config,num_shuffles=1,Shuffles=None,windows2linux=False,userfeedback=False,
trainIndexes=None,testIndexes=None,
net_type=None,augmenter_type=None,defaultconfigfile=None,items2change_pose={}):
"""
Creates a training dataset. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
userfeedback: bool, optional
If this is set to false, then all requested train/test splits are created (no matter if they already exist). If you
want to assure that previous splits etc. are not overwritten, then set this to True and you will be asked for each split.
net_type: string
Type of networks. Currently resnet_50, resnet_101, resnet_152, mobilenet_v2_1.0,mobilenet_v2_0.75, mobilenet_v2_0.5, and mobilenet_v2_0.35 are supported.
augmenter_type: string
Type of augmenter. Currently default, imgaug, tensorpack, and deterministic are supported.
Example
--------
>>> deeplabcut.create_training_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcut.create_training_dataset('C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
from skimage import io
import scipy.io as sio
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg['scorer']
project_path = cfg['project_path']
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) #Path concatenation OS platform independent
auxiliaryfunctions.attempttomakefolder(Path(os.path.join(project_path,str(trainingsetfolder))),recursive=True)
Data = merge_annotateddatasets(cfg,project_path,Path(os.path.join(project_path,trainingsetfolder)),windows2linux)
Data = Data[scorer] #extract labeled data
#%%
# check if we need to update the netwoek
items2change_flag = bool(items2change_pose)
if items2change_flag: # not empty
net_type = items2change_pose.get('net_type', None)
#loading & linking pretrained models
if net_type is None: #loading & linking pretrained models
net_type =cfg.get('default_net_type', 'resnet_50')
else:
if 'resnet' in net_type or 'mobilenet' in net_type:
pass
else:
raise ValueError('Invalid network type:', net_type)
if augmenter_type is None:
augmenter_type=cfg.get('default_augmenter', 'default')
else:
if augmenter_type in ['default','imgaug','tensorpack','deterministic']:
pass
else:
raise ValueError('Invalid augmenter type:', augmenter_type)
import deeplabcut
if defaultconfigfile is None:
parent_path = Path(os.path.dirname(deeplabcut.__file__))
defaultconfigfile = str(parent_path / 'pose_cfg.yaml')
model_path,num_shuffles=auxfun_models.Check4weights(net_type,parent_path,num_shuffles) #if the model does not exist >> throws error!
if Shuffles==None:
Shuffles=range(1,num_shuffles+1,1)
else:
Shuffles=[i for i in Shuffles if isinstance(i,int)]
bodyparts = cfg['bodyparts']
TrainingFraction = cfg['TrainingFraction']
for shuffle in Shuffles: # Creating shuffles starting from 1
for trainingsetindex,trainFraction in enumerate(TrainingFraction):
if userfeedback:
trainposeconfigfile,testposeconfigfile,snapshotfolder = training.return_train_network_path(config,shuffle=shuffle,trainingsetindex=trainingsetindex)
if os.path.isfile(trainposeconfigfile):
askuser=input ("The model folder is already present. If you continue, it will overwrite the existing model (split). Do you want to continue?(yes/no): ")
if askuser=='no'or askuser=='No' or askuser=='N' or askuser=='No':
raise Exception("Use the Shuffles argument as a list to specify a different shuffle index. Check out the help for more details.")
else:
pass
#trainIndexes, testIndexes = SplitTrials(range(len(Data.index)), trainFraction)
if trainIndexes is None and testIndexes is None:
trainIndexes, testIndexes = SplitTrials(range(len(Data.index)), trainFraction)
else:
print("You passed a split with the following fraction:", len(trainIndexes)*1./(len(testIndexes)+len(trainIndexes))*100)
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
# Make training file!
data = []
for jj in trainIndexes:
H = {}
# load image to get dimensions:
filename = Data.index[jj]
im = io.imread(os.path.join(cfg['project_path'],filename))
H['image'] = filename
if np.ndim(im)==3:
H['size'] = np.array(
[np.shape(im)[2],
np.shape(im)[0],
np.shape(im)[1]])
else:
# print "Grayscale!"
H['size'] = np.array([1, np.shape(im)[0], np.shape(im)[1]])
indexjoints=0
joints=np.zeros((len(bodyparts),3))*np.nan
for bpindex,bodypart in enumerate(bodyparts):
# check whether the labels are positive and inside the img
x_pos_n_inside = 0 <= Data[bodypart]['x'][jj] < np.shape(im)[1]
y_pos_n_inside = 0 <= Data[bodypart]['y'][jj] < np.shape(im)[0]
if x_pos_n_inside and y_pos_n_inside:
joints[indexjoints,0]=int(bpindex)
joints[indexjoints,1]=Data[bodypart]['x'][jj]
joints[indexjoints,2]=Data[bodypart]['y'][jj]
indexjoints+=1
joints = joints[np.where(
np.prod(np.isfinite(joints),
1))[0], :] # drop NaN, i.e. lines for missing body parts
assert (np.prod(np.array(joints[:, 2]) < np.shape(im)[0])
) # y coordinate within image?
assert (np.prod(np.array(joints[:, 1]) < np.shape(im)[1])
) # x coordinate within image?
H['joints'] = np.array(joints, dtype=int)
if np.size(joints)>0: #exclude images without labels
data.append(H)
if len(trainIndexes)>0:
datafilename,metadatafilename=auxiliaryfunctions.GetDataandMetaDataFilenames(trainingsetfolder,trainFraction,shuffle,cfg)
################################################################################
# Saving metadata (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(os.path.join(project_path,metadatafilename),data, trainIndexes, testIndexes, trainFraction)
################################################################################
# Saving data file (convert to training file for deeper cut (*.mat))
################################################################################
DTYPE = [('image', 'O'), ('size', 'O'), ('joints', 'O')]
MatlabData = np.array(
[(np.array([data[item]['image']], dtype='U'),
np.array([data[item]['size']]),
boxitintoacell(data[item]['joints']))
for item in range(len(data))],
dtype=DTYPE)
sio.savemat(os.path.join(project_path,datafilename), {'dataset': MatlabData})
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername=auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg)
auxiliaryfunctions.attempttomakefolder(Path(config).parents[0] / modelfoldername,recursive=True)
auxiliaryfunctions.attempttomakefolder(str(Path(config).parents[0] / modelfoldername)+ '/'+ '/train')
auxiliaryfunctions.attempttomakefolder(str(Path(config).parents[0] / modelfoldername)+ '/'+ '/test')
path_train_config = str(os.path.join(cfg['project_path'],Path(modelfoldername),'train','pose_cfg.yaml'))
path_test_config = str(os.path.join(cfg['project_path'],Path(modelfoldername),'test','pose_cfg.yaml'))
#str(cfg['proj_path']+'/'+Path(modelfoldername) / 'test' / 'pose_cfg.yaml')
items2change = {
"dataset": datafilename,
"metadataset": metadatafilename,
"num_joints": len(bodyparts),
"all_joints": [[i] for i in range(len(bodyparts))],
"all_joints_names": [str(bpt) for bpt in bodyparts],
"init_weights": model_path,
"project_path": str(cfg['project_path']),
"net_type": net_type,
"dataset_type": augmenter_type
}
items2change.update(items2change_pose)
trainingdata = MakeTrain_pose_yaml(items2change,path_train_config,defaultconfigfile)
keys2save = [
"dataset", "num_joints", "all_joints", "all_joints_names",
"net_type", 'init_weights', 'global_scale', 'location_refinement',
'locref_stdev'
]
MakeTest_pose_yaml(trainingdata, keys2save,path_test_config)
print("The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!")
return
def create_training_dataset(config,num_shuffles=1,Shuffles=None,windows2linux=False,userfeedback=False,
trainIndexes=None,testIndexes=None,
net_type=None,augmenter_type=None):
"""
Creates a training dataset. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
userfeedback: bool, optional
If this is set to false, then all requested train/test splits are created (no matter if they already exist). If you
want to assure that previous splits etc. are not overwritten, then set this to True and you will be asked for each split.
trainIndexes: list of lists, optional (default=None)
List of one or multiple lists containing train indexes.
A list containing two lists of training indexes will produce two splits.
testIndexes: list of lists, optional (default=None)
List of test indexes.
net_type: string
Type of networks. Currently resnet_50, resnet_101, resnet_152, mobilenet_v2_1.0,mobilenet_v2_0.75, mobilenet_v2_0.5, and mobilenet_v2_0.35 are supported.
augmenter_type: string
Type of augmenter. Currently default, imgaug, tensorpack, and deterministic are supported.
Example
--------
>>> deeplabcut.create_training_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcut.create_training_dataset('C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
import scipy.io as sio
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg['scorer']
project_path = cfg['project_path']
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) #Path concatenation OS platform independent
auxiliaryfunctions.attempttomakefolder(Path(os.path.join(project_path,str(trainingsetfolder))),recursive=True)
Data = merge_annotateddatasets(cfg,project_path,Path(os.path.join(project_path,trainingsetfolder)),windows2linux)
Data = Data[scorer] #extract labeled data
#loading & linking pretrained models
if net_type is None: #loading & linking pretrained models
net_type =cfg.get('default_net_type', 'resnet_50')
else:
if 'resnet' in net_type or 'mobilenet' in net_type:
pass
else:
raise ValueError('Invalid network type:', net_type)
if augmenter_type is None:
augmenter_type=cfg.get('default_augmenter', 'default')
else:
if augmenter_type in ['default','imgaug','tensorpack','deterministic']:
pass
else:
raise ValueError('Invalid augmenter type:', augmenter_type)
import deeplabcut
parent_path = Path(os.path.dirname(deeplabcut.__file__))
defaultconfigfile = str(parent_path / 'pose_cfg.yaml')
model_path,num_shuffles=auxfun_models.Check4weights(net_type,parent_path,num_shuffles) #if the model does not exist >> throws error!
if Shuffles is None:
Shuffles = range(1, num_shuffles + 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
if trainIndexes is None and testIndexes is None:
splits = [(trainFraction, shuffle, SplitTrials(range(len(Data.index)), trainFraction))
for trainFraction in cfg['TrainingFraction'] for shuffle in Shuffles]
else:
if len(trainIndexes) != len(testIndexes):
raise ValueError('Number of train and test indexes should be equal.')
splits = []
for shuffle, (train_inds, test_inds) in enumerate(zip(trainIndexes, testIndexes)):
trainFraction = len(train_inds) / (len(train_inds) + len(test_inds))
print(f"You passed a split with the following fraction: {int(100 * trainFraction)}%")
splits.append((trainFraction, shuffle, (train_inds, test_inds)))
bodyparts = cfg['bodyparts']
nbodyparts = len(bodyparts)
for trainFraction, shuffle, (trainIndexes, testIndexes) in splits:
if len(trainIndexes)>0:
if userfeedback:
trainposeconfigfile, _, _ = training.return_train_network_path(config, shuffle=shuffle, trainFraction=trainFraction)
if trainposeconfigfile.is_file():
askuser=input ("The model folder is already present. If you continue, it will overwrite the existing model (split). Do you want to continue?(yes/no): ")
if askuser=='no'or askuser=='No' or askuser=='N' or askuser=='No':
raise Exception("Use the Shuffles argument as a list to specify a different shuffle index. Check out the help for more details.")
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
# Make training file!
datafilename, metadatafilename = auxiliaryfunctions.GetDataandMetaDataFilenames(trainingsetfolder,
trainFraction, shuffle, cfg)
################################################################################
# Saving data file (convert to training file for deeper cut (*.mat))
################################################################################
data, MatlabData = format_training_data(Data, trainIndexes, nbodyparts, project_path)
sio.savemat(os.path.join(project_path,datafilename), {'dataset': MatlabData})
################################################################################
# Saving metadata (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(os.path.join(project_path,metadatafilename),data, trainIndexes, testIndexes, trainFraction)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername=auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg)
auxiliaryfunctions.attempttomakefolder(Path(config).parents[0] / modelfoldername,recursive=True)
auxiliaryfunctions.attempttomakefolder(str(Path(config).parents[0] / modelfoldername)+ '/'+ '/train')
auxiliaryfunctions.attempttomakefolder(str(Path(config).parents[0] / modelfoldername)+ '/'+ '/test')
path_train_config = str(os.path.join(cfg['project_path'],Path(modelfoldername),'train','pose_cfg.yaml'))
path_test_config = str(os.path.join(cfg['project_path'],Path(modelfoldername),'test','pose_cfg.yaml'))
#str(cfg['proj_path']+'/'+Path(modelfoldername) / 'test' / 'pose_cfg.yaml')
items2change = {
"dataset": datafilename,
"metadataset": metadatafilename,
"num_joints": len(bodyparts),
"all_joints": [[i] for i in range(len(bodyparts))],
"all_joints_names": [str(bpt) for bpt in bodyparts],
"init_weights": model_path,
"project_path": str(cfg['project_path']),
"net_type": net_type,
"dataset_type": augmenter_type,
}
trainingdata = MakeTrain_pose_yaml(items2change,path_train_config,defaultconfigfile)
keys2save = [
"dataset", "num_joints", "all_joints", "all_joints_names",
"net_type", 'init_weights', 'global_scale', 'location_refinement',
'locref_stdev'
]
MakeTest_pose_yaml(trainingdata, keys2save,path_test_config)
print("The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!")
def create_multianimaltraining_dataset(
config,
num_shuffles=1,
Shuffles=None,
windows2linux=False,
net_type=None,
numdigits=2,
):
"""
Creates a training dataset for multi-animal datasets. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Imporant differences to standard:
- stores coordinates with numdigits as many digits
- creates
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
net_type: string
Type of networks. Currently resnet_50, resnet_101, and resnet_152 are supported (not the MobileNets!)
numdigits: int, optional
Example
--------
>>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcut.create_multianimaltraining_dataset(r'C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
from skimage import io
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg["scorer"]
project_path = cfg["project_path"]
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(
cfg) # Path concatenatn OS platform independent
auxiliaryfunctions.attempttomakefolder(Path(
os.path.join(project_path, str(trainingsetfolder))),
recursive=True)
Data = trainingsetmanipulation.merge_annotateddatasets(
cfg, Path(os.path.join(project_path, trainingsetfolder)),
windows2linux)
if Data is None:
return
Data = Data[scorer] # extract labeled data
# actualbpts=set(Data.columns.get_level_values(0))
def strip_cropped_image_name(path):
# utility function to split different crops from same image into either train or test!
filename = os.path.split(path)[1]
return filename.split("c")[0]
img_names = Data.index.map(strip_cropped_image_name).unique()
# loading & linking pretrained models
# CURRENTLY ONLY ResNet supported!
if net_type is None: # loading & linking pretrained models
net_type = cfg.get("default_net_type", "resnet_50")
else:
if "resnet" in net_type: # or 'mobilenet' in net_type:
pass
else:
raise ValueError("Currently only resnet is supported.")
# multianimal case:
dataset_type = "multi-animal-imgaug"
partaffinityfield_graph = auxfun_multianimal.getpafgraph(cfg,
printnames=False)
# ATTENTION: order has to be multibodyparts, then uniquebodyparts (for indexing)
print("Utilizing the following graph:", partaffinityfield_graph)
num_limbs = len(partaffinityfield_graph)
partaffinityfield_predict = True
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, Path(dlcparent_path), num_shuffles)
if Shuffles == None:
Shuffles = range(1, num_shuffles + 1, 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
TrainingFraction = cfg["TrainingFraction"]
for shuffle in Shuffles: # Creating shuffles starting from 1
for trainFraction in TrainingFraction:
train_inds_temp, test_inds_temp = trainingsetmanipulation.SplitTrials(
range(len(img_names)), trainFraction)
# Map back to the original indices.
temp = [
name for i, name in enumerate(img_names) if i in test_inds_temp
]
mask = Data.index.str.contains("|".join(temp))
testIndices = np.flatnonzero(mask)
trainIndices = np.flatnonzero(~mask)
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
# Make training file!
data = []
print("Creating training data for ", shuffle, trainFraction)
print("This can take some time...")
for jj in tqdm(trainIndices):
jointsannotated = False
H = {}
# load image to get dimensions:
filename = Data.index[jj]
im = io.imread(os.path.join(cfg["project_path"], filename))
H["image"] = filename
try:
H["size"] = np.array(
[np.shape(im)[2],
np.shape(im)[0],
np.shape(im)[1]])
except:
# print "Grayscale!"
H["size"] = np.array([1, np.shape(im)[0], np.shape(im)[1]])
Joints = {}
for prfxindex, prefix in enumerate(individuals):
joints = (np.zeros(
(len(uniquebodyparts) + len(multianimalbodyparts), 3))
* np.nan)
if prefix != "single": # first ones are multianimalparts!
indexjoints = 0
for bpindex, bodypart in enumerate(
multianimalbodyparts):
socialbdpt = bodypart # prefix+bodypart #build names!
# if socialbdpt in actualbpts:
try:
x, y = (
Data[prefix][socialbdpt]["x"][jj],
Data[prefix][socialbdpt]["y"][jj],
)
joints[indexjoints, 0] = int(bpindex)
joints[indexjoints, 1] = round(x, numdigits)
joints[indexjoints, 2] = round(y, numdigits)
indexjoints += 1
except:
pass
else:
indexjoints = len(multianimalbodyparts)
for bpindex, bodypart in enumerate(uniquebodyparts):
socialbdpt = bodypart # prefix+bodypart #build names!
# if socialbdpt in actualbpts:
try:
x, y = (
Data[prefix][socialbdpt]["x"][jj],
Data[prefix][socialbdpt]["y"][jj],
)
joints[indexjoints, 0] = len(
multianimalbodyparts) + int(bpindex)
joints[indexjoints, 1] = round(x, 2)
joints[indexjoints, 2] = round(y, 2)
indexjoints += 1
except:
pass
# Drop missing body parts
joints = joints[~np.isnan(joints).any(axis=1)]
# Drop points lying outside the image
inside = np.logical_and.reduce((
joints[:, 1] < im.shape[1],
joints[:, 1] > 0,
joints[:, 2] < im.shape[0],
joints[:, 2] > 0,
))
joints = joints[inside]
if np.size(joints) > 0: # exclude images without labels
jointsannotated = True
Joints[prfxindex] = joints # np.array(joints, dtype=int)
H["joints"] = Joints
if jointsannotated: # exclude images without labels
data.append(H)
if len(trainIndices) > 0:
(
datafilename,
metadatafilename,
) = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
################################################################################
# Saving metadata and data file (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename),
data,
trainIndices,
testIndices,
trainFraction,
)
datafilename = datafilename.split(".mat")[0] + ".pickle"
import pickle
with open(os.path.join(project_path, datafilename), "wb") as f:
# Pickle the 'labeled-data' dictionary using the highest protocol available.
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
auxiliaryfunctions.attempttomakefolder(
Path(config).parents[0] / modelfoldername, recursive=True)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername) + "/" +
"/train")
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername) + "/" +
"/test")
path_train_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"train",
"pose_cfg.yaml",
))
path_test_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"pose_cfg.yaml",
))
path_inference_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"inference_cfg.yaml",
))
jointnames = [str(bpt) for bpt in multianimalbodyparts]
jointnames.extend([str(bpt) for bpt in uniquebodyparts])
items2change = {
"dataset":
datafilename,
"metadataset":
metadatafilename,
"num_joints":
len(multianimalbodyparts) +
len(uniquebodyparts), # cfg["uniquebodyparts"]),
"all_joints": [[i] for i in range(
len(multianimalbodyparts) + len(uniquebodyparts))
], # cfg["uniquebodyparts"]))],
"all_joints_names":
jointnames,
"init_weights":
model_path,
"project_path":
str(cfg["project_path"]),
"net_type":
net_type,
"pairwise_loss_weight":
0.1,
"pafwidth":
20,
"partaffinityfield_graph":
partaffinityfield_graph,
"partaffinityfield_predict":
partaffinityfield_predict,
"weigh_only_present_joints":
False,
"num_limbs":
len(partaffinityfield_graph),
"dataset_type":
dataset_type,
"optimizer":
"adam",
"batch_size":
8,
"multi_step": [[1e-4, 7500], [5 * 1e-5, 12000],
[1e-5, 200000]],
"save_iters":
10000,
"display_iters":
500,
}
defaultconfigfile = os.path.join(dlcparent_path,
"pose_cfg.yaml")
trainingdata = trainingsetmanipulation.MakeTrain_pose_yaml(
items2change, path_train_config, defaultconfigfile)
keys2save = [
"dataset",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
"dataset_type",
"partaffinityfield_predict",
"pairwise_predict",
"partaffinityfield_graph",
"num_limbs",
"dataset_type",
]
trainingsetmanipulation.MakeTest_pose_yaml(
trainingdata,
keys2save,
path_test_config,
nmsradius=5.0,
minconfidence=0.01,
) # setting important def. values for inference
# Setting inference cfg file:
defaultinference_configfile = os.path.join(
dlcparent_path, "inference_cfg.yaml")
items2change = {
"minimalnumberofconnections":
int(len(cfg["multianimalbodyparts"]) / 2),
"topktoretain":
len(cfg["individuals"]) + 1 *
(len(cfg["uniquebodyparts"]) > 0),
}
# TODO: "distnormalization": could be calculated here based on data and set
# >> now we calculate this during evaluation (which is a good spot...)
trainingsetmanipulation.MakeInference_yaml(
items2change, path_inference_config,
defaultinference_configfile)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
else:
pass
cfg, paf_graph
)
partaffinityfield_graph = [
edge for i, edge in enumerate(paf_graph) if i not in to_ignore
]
auxfun_multianimal.validate_paf_graph(cfg, partaffinityfield_graph)
print("Utilizing the following graph:", partaffinityfield_graph)
partaffinityfield_predict = True
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, Path(dlcparent_path), num_shuffles
)
if Shuffles is None:
Shuffles = range(1, num_shuffles + 1, 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
TrainingFraction = cfg["TrainingFraction"]
for shuffle in Shuffles: # Creating shuffles starting from 1
for trainFraction in TrainingFraction:
train_inds_temp, test_inds_temp = SplitTrials(
range(len(img_names)), trainFraction
)
# Map back to the original indices.
temp = [re.escape(name) for i, name in enumerate(img_names)
