def undistort_points(config, dataframe, camera_pair, destfolder):
cfg_3d = auxiliaryfunctions.read_config(config)
(
img_path,
path_corners,
path_camera_matrix,
path_undistort,
) = auxiliaryfunctions_3d.Foldernames3Dproject(cfg_3d)
"""
path_undistort = destfolder
filename_cam1 = Path(dataframe[0]).stem
filename_cam2 = Path(dataframe[1]).stem
#currently no interm. saving of this due to high speed.
# check if the undistorted files are already present
if os.path.exists(os.path.join(path_undistort,filename_cam1 + '_undistort.h5')) and os.path.exists(os.path.join(path_undistort,filename_cam2 + '_undistort.h5')):
print("The undistorted files are already present at %s" % os.path.join(path_undistort,filename_cam1))
dataFrame_cam1_undistort = pd.read_hdf(os.path.join(path_undistort,filename_cam1 + '_undistort.h5'))
dataFrame_cam2_undistort = pd.read_hdf(os.path.join(path_undistort,filename_cam2 + '_undistort.h5'))
else:
"""
if True:
# Create an empty dataFrame to store the undistorted 2d coordinates and likelihood
dataframe_cam1 = pd.read_hdf(dataframe[0])
dataframe_cam2 = pd.read_hdf(dataframe[1])
scorer_cam1 = dataframe_cam1.columns.get_level_values(0)[0]
scorer_cam2 = dataframe_cam2.columns.get_level_values(0)[0]
stereo_file = auxiliaryfunctions.read_pickle(
os.path.join(path_camera_matrix, "stereo_params.pickle"))
path_stereo_file = os.path.join(path_camera_matrix,
"stereo_params.pickle")
stereo_file = auxiliaryfunctions.read_pickle(path_stereo_file)
mtx_l = stereo_file[camera_pair]["cameraMatrix1"]
dist_l = stereo_file[camera_pair]["distCoeffs1"]
mtx_r = stereo_file[camera_pair]["cameraMatrix2"]
dist_r = stereo_file[camera_pair]["distCoeffs2"]
R1 = stereo_file[camera_pair]["R1"]
P1 = stereo_file[camera_pair]["P1"]
R2 = stereo_file[camera_pair]["R2"]
P2 = stereo_file[camera_pair]["P2"]
# Create an empty dataFrame to store the undistorted 2d coordinates and likelihood
(
dataFrame_cam1_undistort,
scorer_cam1,
bodyparts,
) = auxiliaryfunctions_3d.create_empty_df(dataframe_cam1,
scorer_cam1,
flag="2d")
(
dataFrame_cam2_undistort,
scorer_cam2,
bodyparts,
) = auxiliaryfunctions_3d.create_empty_df(dataframe_cam2,
scorer_cam2,
flag="2d")
for bpindex, bp in tqdm(enumerate(bodyparts)):
# Undistorting the points from cam1 camera
points_cam1 = np.array([
dataframe_cam1[scorer_cam1][bp]["x"].values[:],
dataframe_cam1[scorer_cam1][bp]["y"].values[:],
])
points_cam1 = points_cam1.T
points_cam1 = np.expand_dims(points_cam1, axis=1)
points_cam1_remapped = cv2.undistortPoints(src=points_cam1,
cameraMatrix=mtx_l,
distCoeffs=dist_l,
P=P1,
R=R1)
dataFrame_cam1_undistort.iloc[:][scorer_cam1, bp,
"x"] = points_cam1_remapped[:, 0,
0]
dataFrame_cam1_undistort.iloc[:][scorer_cam1, bp,
"y"] = points_cam1_remapped[:, 0,
1]
dataFrame_cam1_undistort.iloc[:][
scorer_cam1, bp, "likelihood"] = dataframe_cam1[scorer_cam1][
bp]["likelihood"].values[:]
# Undistorting the points from cam2 camera
points_cam2 = np.array([
dataframe_cam2[scorer_cam2][bp]["x"].values[:],
dataframe_cam2[scorer_cam2][bp]["y"].values[:],
])
points_cam2 = points_cam2.T
points_cam2 = np.expand_dims(points_cam2, axis=1)
points_cam2_remapped = cv2.undistortPoints(src=points_cam2,
cameraMatrix=mtx_r,
distCoeffs=dist_r,
P=P2,
R=R2)
dataFrame_cam2_undistort.iloc[:][scorer_cam2, bp,
"x"] = points_cam2_remapped[:, 0,
0]
dataFrame_cam2_undistort.iloc[:][scorer_cam2, bp,
"y"] = points_cam2_remapped[:, 0,
1]
dataFrame_cam2_undistort.iloc[:][
scorer_cam2, bp, "likelihood"] = dataframe_cam2[scorer_cam2][
bp]["likelihood"].values[:]
# Save the undistorted files
dataFrame_cam1_undistort.sort_index(inplace=True)
dataFrame_cam2_undistort.sort_index(inplace=True)
return (
dataFrame_cam1_undistort,
dataFrame_cam2_undistort,
stereo_file[camera_pair],
path_stereo_file,
)
def create_multianimaltraining_dataset(
config,
num_shuffles=1,
Shuffles=None,
windows2linux=False,
net_type=None,
numdigits=2,
):
"""
Creates a training dataset for multi-animal datasets. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Imporant differences to standard:
- stores coordinates with numdigits as many digits
- creates
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
net_type: string
Type of networks. Currently resnet_50, resnet_101, and resnet_152 are supported (not the MobileNets!)
numdigits: int, optional
Example
--------
>>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcut.create_multianimaltraining_dataset(r'C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
from skimage import io
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg["scorer"]
project_path = cfg["project_path"]
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(
cfg) # Path concatenatn OS platform independent
auxiliaryfunctions.attempttomakefolder(Path(
os.path.join(project_path, str(trainingsetfolder))),
recursive=True)
Data = trainingsetmanipulation.merge_annotateddatasets(
cfg, Path(os.path.join(project_path, trainingsetfolder)),
windows2linux)
if Data is None:
return
Data = Data[scorer] # extract labeled data
# actualbpts=set(Data.columns.get_level_values(0))
def strip_cropped_image_name(path):
# utility function to split different crops from same image into either train or test!
filename = os.path.split(path)[1]
return filename.split("c")[0]
img_names = Data.index.map(strip_cropped_image_name).unique()
# loading & linking pretrained models
# CURRENTLY ONLY ResNet supported!
if net_type is None: # loading & linking pretrained models
net_type = cfg.get("default_net_type", "resnet_50")
else:
if "resnet" in net_type: # or 'mobilenet' in net_type:
pass
else:
raise ValueError("Currently only resnet is supported.")
# multianimal case:
dataset_type = "multi-animal-imgaug"
partaffinityfield_graph = auxfun_multianimal.getpafgraph(cfg)
# ATTENTION: order has to be multibodyparts, then uniquebodyparts (for indexing)
print("Utilizing the following graph:", partaffinityfield_graph)
num_limbs = len(partaffinityfield_graph)
partaffinityfield_predict = True
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
model_path, num_shuffles = auxfun_models.Check4weights(
net_type, Path(dlcparent_path), num_shuffles)
if Shuffles == None:
Shuffles = range(1, num_shuffles + 1, 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
(
individuals,
uniquebodyparts,
multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
TrainingFraction = cfg["TrainingFraction"]
for shuffle in Shuffles: # Creating shuffles starting from 1
for trainFraction in TrainingFraction:
train_inds_temp, test_inds_temp = trainingsetmanipulation.SplitTrials(
range(len(img_names)), trainFraction)
# Map back to the original indices.
temp = [
name for i, name in enumerate(img_names) if i in test_inds_temp
]
mask = Data.index.str.contains("|".join(temp))
testIndexes = np.flatnonzero(mask)
trainIndexes = np.flatnonzero(~mask)
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
# Make training file!
data = []
print("Creating training data for ", shuffle, trainFraction)
print("This can take some time...")
for jj in tqdm(trainIndexes):
jointsannotated = False
H = {}
# load image to get dimensions:
filename = Data.index[jj]
im = io.imread(os.path.join(cfg["project_path"], filename))
H["image"] = filename
try:
H["size"] = np.array(
[np.shape(im)[2],
np.shape(im)[0],
np.shape(im)[1]])
except:
# print "Grayscale!"
H["size"] = np.array([1, np.shape(im)[0], np.shape(im)[1]])
Joints = {}
for prfxindex, prefix in enumerate(individuals):
joints = (np.zeros(
(len(uniquebodyparts) + len(multianimalbodyparts), 3))
* np.nan)
if prefix != "single": # first ones are multianimalparts!
indexjoints = 0
for bpindex, bodypart in enumerate(
multianimalbodyparts):
socialbdpt = bodypart # prefix+bodypart #build names!
# if socialbdpt in actualbpts:
try:
x, y = (
Data[prefix][socialbdpt]["x"][jj],
Data[prefix][socialbdpt]["y"][jj],
)
joints[indexjoints, 0] = int(bpindex)
joints[indexjoints, 1] = round(x, numdigits)
joints[indexjoints, 2] = round(y, numdigits)
indexjoints += 1
except:
pass
else:
indexjoints = len(multianimalbodyparts)
for bpindex, bodypart in enumerate(uniquebodyparts):
socialbdpt = bodypart # prefix+bodypart #build names!
# if socialbdpt in actualbpts:
try:
x, y = (
Data[prefix][socialbdpt]["x"][jj],
Data[prefix][socialbdpt]["y"][jj],
)
joints[indexjoints, 0] = len(
multianimalbodyparts) + int(bpindex)
joints[indexjoints, 1] = round(x, 2)
joints[indexjoints, 2] = round(y, 2)
indexjoints += 1
except:
pass
# Drop missing body parts
joints = joints[~np.isnan(joints).any(axis=1)]
# Drop points lying outside the image
inside = np.logical_and.reduce((
joints[:, 1] < im.shape[1],
joints[:, 1] > 0,
joints[:, 2] < im.shape[0],
joints[:, 2] > 0,
))
joints = joints[inside]
if np.size(joints) > 0: # exclude images without labels
jointsannotated = True
Joints[prfxindex] = joints # np.array(joints, dtype=int)
H["joints"] = Joints
if jointsannotated: # exclude images without labels
data.append(H)
if len(trainIndexes) > 0:
(
datafilename,
metadatafilename,
) = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
################################################################################
# Saving metadata and data file (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename),
data,
trainIndexes,
testIndexes,
trainFraction,
)
datafilename = datafilename.split(".mat")[0] + ".pickle"
import pickle
with open(os.path.join(project_path, datafilename), "wb") as f:
# Pickle the 'labeled-data' dictionary using the highest protocol available.
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
auxiliaryfunctions.attempttomakefolder(
Path(config).parents[0] / modelfoldername, recursive=True)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername) + "/" +
"/train")
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername) + "/" +
"/test")
path_train_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"train",
"pose_cfg.yaml",
))
path_test_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"pose_cfg.yaml",
))
path_inference_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"inference_cfg.yaml",
))
jointnames = [str(bpt) for bpt in multianimalbodyparts]
jointnames.extend([str(bpt) for bpt in uniquebodyparts])
items2change = {
"dataset":
datafilename,
"metadataset":
metadatafilename,
"num_joints":
len(multianimalbodyparts) +
len(uniquebodyparts), # cfg["uniquebodyparts"]),
"all_joints": [[i] for i in range(
len(multianimalbodyparts) + len(uniquebodyparts))
], # cfg["uniquebodyparts"]))],
"all_joints_names":
jointnames,
"init_weights":
model_path,
"project_path":
str(cfg["project_path"]),
"net_type":
net_type,
"pairwise_loss_weight":
0.1,
"pafwidth":
20,
"partaffinityfield_graph":
partaffinityfield_graph,
"partaffinityfield_predict":
partaffinityfield_predict,
"weigh_only_present_joints":
False,
"num_limbs":
len(partaffinityfield_graph),
"dataset_type":
dataset_type,
"optimizer":
"adam",
"batch_size":
8,
"multi_step": [[1e-4, 7500], [5 * 1e-5, 12000],
[1e-5, 200000]],
"save_iters":
10000,
"display_iters":
500,
}
defaultconfigfile = os.path.join(dlcparent_path,
"pose_cfg.yaml")
trainingdata = trainingsetmanipulation.MakeTrain_pose_yaml(
items2change, path_train_config, defaultconfigfile)
keys2save = [
"dataset",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
"dataset_type",
"partaffinityfield_predict",
"pairwise_predict",
"partaffinityfield_graph",
"num_limbs",
"dataset_type",
]
trainingsetmanipulation.MakeTest_pose_yaml(
trainingdata,
keys2save,
path_test_config,
nmsradius=5.0,
minconfidence=0.01,
) # setting important def. values for inference
# Setting inference cfg file:
defaultinference_configfile = os.path.join(
dlcparent_path, "inference_cfg.yaml")
items2change = {
"minimalnumberofconnections":
int(len(cfg["multianimalbodyparts"]) / 2),
"topktoretain":
len(cfg["individuals"]) + 1 *
(len(cfg["uniquebodyparts"]) > 0),
}
# TODO: "distnormalization": could be calculated here based on data and set
# >> now we calculate this during evaluation (which is a good spot...)
trainingsetmanipulation.MakeInference_yaml(
items2change, path_inference_config,
defaultinference_configfile)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
else:
pass
def create_labeled_video_3d(
config,
path,
videofolder=None,
start=0,
end=None,
trailpoints=0,
videotype="avi",
view=(-113, -270),
xlim=None,
ylim=None,
zlim=None,
draw_skeleton=True,
color_by="bodypart",
figsize=(20, 8),
fps=30,
dpi=300,
):
"""
Creates a video with views from the two cameras and the 3d reconstruction for a selected number of frames.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
path : list
A list of strings containing the full paths to triangulated files for analysis or a path to the directory, where all the triangulated files are stored.
videofolder: string
Full path of the folder where the videos are stored. Use this if the vidoes are stored in a different location other than where the triangulation files are stored. By default is ``None`` and therefore looks for video files in the directory where the triangulation file is stored.
start: int
Integer specifying the start of frame index to select. Default is set to 0.
end: int
Integer specifying the end of frame index to select. Default is set to None, where all the frames of the video are used for creating the labeled video.
trailpoints: int
Number of revious frames whose body parts are plotted in a frame (for displaying history). Default is set to 0.
videotype: string, optional
Checks for the extension of the video in case the input is a directory.\nOnly videos with this extension are analyzed. The default is ``.avi``
view: list
A list that sets the elevation angle in z plane and azimuthal angle in x,y plane of 3d view. Useful for rotating the axis for 3d view
xlim: list
A list of integers specifying the limits for xaxis of 3d view. By default it is set to [None,None], where the x limit is set by taking the minimum and maximum value of the x coordinates for all the bodyparts.
ylim: list
A list of integers specifying the limits for yaxis of 3d view. By default it is set to [None,None], where the y limit is set by taking the minimum and maximum value of the y coordinates for all the bodyparts.
zlim: list
A list of integers specifying the limits for zaxis of 3d view. By default it is set to [None,None], where the z limit is set by taking the minimum and maximum value of the z coordinates for all the bodyparts.
draw_skeleton: bool
If ``True`` adds a line connecting the body parts making a skeleton on on each frame. The body parts to be connected and the color of these connecting lines are specified in the config file. By default: ``True``
color_by : string, optional (default='bodypart')
Coloring rule. By default, each bodypart is colored differently.
If set to 'individual', points belonging to a single individual are colored the same.
Example
-------
Linux/MacOs
>>> deeplabcut.create_labeled_video_3d(config,['/data/project1/videos/3d.h5'],start=100, end=500)
To create labeled videos for all the triangulated files in the folder
>>> deeplabcut.create_labeled_video_3d(config,['/data/project1/videos'],start=100, end=500)
To set the xlim, ylim, zlim and rotate the view of the 3d axis
>>> deeplabcut.create_labeled_video_3d(config,['/data/project1/videos'],start=100, end=500,view=[30,90],xlim=[-12,12],ylim=[15,25],zlim=[20,30])
"""
start_path = os.getcwd()
# Read the config file and related variables
cfg_3d = auxiliaryfunctions.read_config(config)
cam_names = cfg_3d["camera_names"]
pcutoff = cfg_3d["pcutoff"]
markerSize = cfg_3d["dotsize"]
alphaValue = cfg_3d["alphaValue"]
cmap = cfg_3d["colormap"]
bodyparts2connect = cfg_3d["skeleton"]
skeleton_color = cfg_3d["skeleton_color"]
scorer_3d = cfg_3d["scorername_3d"]
if color_by not in ("bodypart", "individual"):
raise ValueError(f"Invalid color_by={color_by}")
file_list = auxiliaryfunctions_3d.Get_list_of_triangulated_and_videoFiles(
path, videotype, scorer_3d, cam_names, videofolder)
print(file_list)
if file_list == []:
raise Exception(
"No corresponding video file(s) found for the specified triangulated file or folder. Did you specify the video file type? If videos are stored in a different location, please use the ``videofolder`` argument to specify their path."
)
for file in file_list:
path_h5_file = Path(file[0]).parents[0]
triangulate_file = file[0]
# triangulated file is a list which is always sorted as [triangulated.h5,camera-1.videotype,camera-2.videotype]
# name for output video
file_name = str(Path(triangulate_file).stem)
videooutname = os.path.join(path_h5_file, file_name + ".mp4")
if os.path.isfile(videooutname):
print("Video already created...")
else:
string_to_remove = str(Path(triangulate_file).suffix)
pickle_file = triangulate_file.replace(string_to_remove,
"_meta.pickle")
metadata_ = auxiliaryfunctions_3d.LoadMetadata3d(pickle_file)
base_filename_cam1 = str(Path(file[1]).stem).split(videotype)[
0] # required for searching the filtered file
base_filename_cam2 = str(Path(file[2]).stem).split(videotype)[
0] # required for searching the filtered file
cam1_view_video = file[1]
cam2_view_video = file[2]
cam1_scorer = metadata_["scorer_name"][cam_names[0]]
cam2_scorer = metadata_["scorer_name"][cam_names[1]]
print("Creating 3D video from %s and %s using %s" % (
Path(cam1_view_video).name,
Path(cam2_view_video).name,
Path(triangulate_file).name,
))
# Read the video files and corresponfing h5 files
vid_cam1 = VideoReader(cam1_view_video)
vid_cam2 = VideoReader(cam2_view_video)
# Look for the filtered predictions file
try:
print("Looking for filtered predictions...")
df_cam1 = pd.read_hdf(
glob.glob(
os.path.join(
path_h5_file,
str("*" + base_filename_cam1 + cam1_scorer +
"*filtered.h5"),
))[0])
df_cam2 = pd.read_hdf(
glob.glob(
os.path.join(
path_h5_file,
str("*" + base_filename_cam2 + cam2_scorer +
"*filtered.h5"),
))[0])
# print("Found filtered predictions, will be use these for triangulation.")
print(
"Found the following filtered data: ",
os.path.join(
path_h5_file,
str("*" + base_filename_cam1 + cam1_scorer +
"*filtered.h5"),
),
os.path.join(
path_h5_file,
str("*" + base_filename_cam2 + cam2_scorer +
"*filtered.h5"),
),
)
except FileNotFoundError:
print(
"No filtered predictions found, the unfiltered predictions will be used instead."
)
df_cam1 = pd.read_hdf(
glob.glob(
os.path.join(
path_h5_file,
str(base_filename_cam1 + cam1_scorer +
"*.h5")))[0])
df_cam2 = pd.read_hdf(
glob.glob(
os.path.join(
path_h5_file,
str(base_filename_cam2 + cam2_scorer +
"*.h5")))[0])
df_3d = pd.read_hdf(triangulate_file)
try:
num_animals = df_3d.columns.get_level_values(
"individuals").unique().size
except KeyError:
num_animals = 1
if end is None:
end = len(df_3d) # All the frames
end = min(end, min(len(vid_cam1), len(vid_cam2)))
frames = list(range(start, end))
output_folder = Path(
os.path.join(path_h5_file, "temp_" + file_name))
output_folder.mkdir(parents=True, exist_ok=True)
# Flatten the list of bodyparts to connect
bodyparts2plot = list(
np.unique(
[val for sublist in bodyparts2connect for val in sublist]))
# Format data
mask2d = df_cam1.columns.get_level_values('bodyparts').isin(
bodyparts2plot)
xy1 = df_cam1.loc[:, mask2d].to_numpy().reshape(
(len(df_cam1), -1, 3))
visible1 = xy1[..., 2] >= pcutoff
xy1[~visible1] = np.nan
xy2 = df_cam2.loc[:, mask2d].to_numpy().reshape(
(len(df_cam1), -1, 3))
visible2 = xy2[..., 2] >= pcutoff
xy2[~visible2] = np.nan
mask = df_3d.columns.get_level_values('bodyparts').isin(
bodyparts2plot)
xyz = df_3d.loc[:, mask].to_numpy().reshape((len(df_3d), -1, 3))
xyz[~(visible1 & visible2)] = np.nan
bpts = df_3d.columns.get_level_values('bodyparts')[mask][::3]
links = make_labeled_video.get_segment_indices(
bodyparts2connect,
bpts,
)
ind_links = tuple(zip(*links))
if color_by == "bodypart":
color = plt.cm.get_cmap(cmap, len(bodyparts2plot))
colors_ = color(range(len(bodyparts2plot)))
colors = np.tile(colors_, (num_animals, 1))
elif color_by == "individual":
color = plt.cm.get_cmap(cmap, num_animals)
colors_ = color(range(num_animals))
colors = np.repeat(colors_, len(bodyparts2plot), axis=0)
# Trick to force equal aspect ratio of 3D plots
minmax = np.nanpercentile(xyz[frames], q=[25, 75], axis=(0, 1)).T
minmax *= 1.1
minmax_range = (minmax[:, 1] - minmax[:, 0]).max() / 2
if xlim is None:
mid_x = np.mean(minmax[0])
xlim = mid_x - minmax_range, mid_x + minmax_range
if ylim is None:
mid_y = np.mean(minmax[1])
ylim = mid_y - minmax_range, mid_y + minmax_range
if zlim is None:
mid_z = np.mean(minmax[2])
zlim = mid_z - minmax_range, mid_z + minmax_range
# Set up the matplotlib figure beforehand
fig, axes1, axes2, axes3 = set_up_grid(figsize, xlim, ylim, zlim,
view)
points_2d1 = axes1.scatter(
*np.zeros((2, len(bodyparts2plot))),
s=markerSize,
alpha=alphaValue,
)
im1 = axes1.imshow(np.zeros((vid_cam1.height, vid_cam1.width)))
points_2d2 = axes2.scatter(
*np.zeros((2, len(bodyparts2plot))),
s=markerSize,
alpha=alphaValue,
)
im2 = axes2.imshow(np.zeros((vid_cam2.height, vid_cam2.width)))
points_3d = axes3.scatter(
*np.zeros((3, len(bodyparts2plot))),
s=markerSize,
alpha=alphaValue,
)
if draw_skeleton:
# Set up skeleton LineCollections
segs = np.zeros((2, len(ind_links), 2))
coll1 = LineCollection(segs, colors=skeleton_color)
coll2 = LineCollection(segs, colors=skeleton_color)
axes1.add_collection(coll1)
axes2.add_collection(coll2)
segs = np.zeros((2, len(ind_links), 3))
coll_3d = Line3DCollection(segs, colors=skeleton_color)
axes3.add_collection(coll_3d)
writer = FFMpegWriter(fps=fps)
with writer.saving(fig, videooutname, dpi=dpi):
for k in tqdm(frames):
vid_cam1.set_to_frame(k)
vid_cam2.set_to_frame(k)
frame_cam1 = vid_cam1.read_frame()
frame_cam2 = vid_cam2.read_frame()
if frame_cam1 is None or frame_cam2 is None:
raise IOError("A video frame is empty.")
im1.set_data(frame_cam1)
im2.set_data(frame_cam2)
sl = slice(max(0, k - trailpoints), k + 1)
coords3d = xyz[sl]
coords1 = xy1[sl, :, :2]
coords2 = xy2[sl, :, :2]
points_3d._offsets3d = coords3d.reshape((-1, 3)).T
points_3d.set_color(colors)
points_2d1.set_offsets(coords1.reshape((-1, 2)))
points_2d1.set_color(colors)
points_2d2.set_offsets(coords2.reshape((-1, 2)))
points_2d2.set_color(colors)
if draw_skeleton:
segs3d = xyz[k][tuple([ind_links])].swapaxes(0, 1)
coll_3d.set_segments(segs3d)
segs1 = xy1[k, :, :2][tuple([ind_links
])].swapaxes(0, 1)
coll1.set_segments(segs1)
segs2 = xy2[k, :, :2][tuple([ind_links
])].swapaxes(0, 1)
coll2.set_segments(segs2)
writer.grab_frame()
fullpath to cropping_config.yaml file
case: string
case to plot
bodyparts: list
A list that contains bodyparts to plot. Each bodypart is in a string format
shuffle: int, optional
Integer value specifying the shuffle index to select for training. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use.
By default the first (note that TrainingFraction is a list in config.yaml).
"""
self.path_to_config = path_to_config
self.path_to_cropping_config = path_to_cropping_config
self.config = auxiliaryfunctions.read_config(self.path_to_config)
self.cropping_config = auxiliaryfunctions.read_config(
self.path_to_cropping_config)
self.case = case
self.bodyparts = bodyparts # make it as a property and cascade down all the others
self.shuffle = shuffle
self.trainingsetindex = trainingsetindex
self._case_full_name = case + '_beh'
self.project_path = self.config['project_path']
self.path_to_video = os.path.join(
self.project_path, 'videos', self._case_full_name + '.avi')
self.path_to_analysis = os.path.join(
self.project_path, 'analysis', self._case_full_name)
# inference_case_list = [os.path.basename(os.path.normpath(video_path)).split(
def extract_maps(
config,
shuffle=0,
trainingsetindex=0,
gputouse=None,
rescale=False,
Indices=None,
modelprefix="",
):
"""
Extracts the scoremap, locref, partaffinityfields (if available).
Returns a dictionary indexed by: trainingsetfraction, snapshotindex, and imageindex
for those keys, each item contains: (image,scmap,locref,paf,bpt names,partaffinity graph, imagename, True/False if this image was in trainingset)
----------
config : string
Full path of the config.yaml file as a string.
shuffle: integer
integers specifying shuffle index of the training dataset. The default is 0.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This
variable can also be set to "all".
rescale: bool, default False
Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every
image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported
in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated
on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the
original size!
Examples
--------
If you want to extract the data for image 0 and 103 (of the training set) for model trained with shuffle 0.
>>> deeplabcut.extract_maps(configfile,0,Indices=[0,103])
"""
from deeplabcut.utils.auxfun_videos import imread, imresize
from deeplabcut.pose_estimation_tensorflow.nnet import predict
from deeplabcut.pose_estimation_tensorflow.nnet import (
predict_multianimal as predictma, )
from deeplabcut.pose_estimation_tensorflow.config import load_config
from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input
from deeplabcut.utils import auxiliaryfunctions
from tqdm import tqdm
import tensorflow as tf
vers = (tf.__version__).split(".")
if int(vers[0]) == 1 and int(vers[1]) > 12:
TF = tf.compat.v1
else:
TF = tf
import pandas as pd
from pathlib import Path
import numpy as np
TF.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
# tf.logging.set_verbosity(tf.logging.WARN)
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
if trainingsetindex < len(
cfg["TrainingFraction"]) and trainingsetindex >= 0:
TrainingFractions = [
cfg["TrainingFraction"][int(trainingsetindex)]
]
else:
raise Exception(
"Please check the trainingsetindex! ",
trainingsetindex,
" should be an integer from 0 .. ",
int(len(cfg["TrainingFraction"]) - 1),
)
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
),
"df_with_missing",
)
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
Maps = {}
for trainFraction in TrainingFractions:
Maps[trainFraction] = {}
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(trainFraction,
shuffle,
cfg,
modelprefix=modelprefix)),
)
path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
# Load meta data
(
data,
trainIndices,
testIndices,
trainFraction,
) = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# change batch size, if it was edited during analysis!
dlc_cfg["batch_size"] = 1 # in case this was edited for analysis.
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg, modelprefix=modelprefix)),
)
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
# path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
])
try: # check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
########################### RESCALING (to global scale)
scale = dlc_cfg["global_scale"] if rescale else 1
Data *= scale
bptnames = [
dlc_cfg["all_joints_names"][i]
for i in range(len(dlc_cfg["all_joints"]))
]
for snapindex in snapindices:
dlc_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg["init_weights"].split(os.sep)[-1]
).split("-")[
-1] # read how many training siterations that corresponds to.
# Name for deeplabcut net (based on its parameters)
# DLCscorer,DLCscorerlegacy = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations)
# notanalyzed, resultsfilename, DLCscorer=auxiliaryfunctions.CheckifNotEvaluated(str(evaluationfolder),DLCscorer,DLCscorerlegacy,Snapshots[snapindex])
# print("Extracting maps for ", DLCscorer, " with # of trainingiterations:", trainingsiterations)
# if notanalyzed: #this only applies to ask if h5 exists...
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
Numimages = len(Data.index)
PredicteData = np.zeros(
(Numimages, 3 * len(dlc_cfg["all_joints_names"])))
print("Analyzing data...")
if Indices is None:
Indices = enumerate(Data.index)
else:
Ind = [Data.index[j] for j in Indices]
Indices = enumerate(Ind)
DATA = {}
for imageindex, imagename in tqdm(Indices):
image = imread(os.path.join(cfg["project_path"], imagename),
mode="RGB")
if scale != 1:
image = imresize(image, scale)
image_batch = data_to_input(image)
# Compute prediction with the CNN
outputs_np = sess.run(outputs, feed_dict={inputs: image_batch})
if cfg.get("multianimalproject", False):
scmap, locref, paf = predictma.extract_cnn_output(
outputs_np, dlc_cfg)
pagraph = dlc_cfg["partaffinityfield_graph"]
else:
scmap, locref = predict.extract_cnn_output(
outputs_np, dlc_cfg)
paf = None
pagraph = []
if imageindex in testIndices:
trainingfram = False
else:
trainingfram = True
DATA[imageindex] = [
image,
scmap,
locref,
paf,
bptnames,
pagraph,
imagename,
trainingfram,
]
Maps[trainFraction][Snapshots[snapindex]] = DATA
os.chdir(str(start_path))
return Maps
def plot_trajectories(config, videos, videotype='.avi', shuffle=1, trainingsetindex=0, filtered=False,
displayedbodyparts='all', showfigures=False, destfolder=None, outformat = '.pdf'):
"""
Plots the trajectories of various bodyparts across the video.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
displayedbodyparts: list of strings, optional
This select the body parts that are plotted in the video.
Either ``all``, then all body parts from config.yaml are used,
or a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
showfigures: bool, default false
If true then plots are also displayed.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
Example
--------
for labeling the frames
>>> deeplabcut.plot_trajectories('home/alex/analysis/project/reaching-task/config.yaml',['/home/alex/analysis/project/videos/reachingvideo1.avi'])
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
DLCscorer,DLCscorerlegacy = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction) #automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(cfg, displayedbodyparts)
Videos=auxiliaryfunctions.Getlistofvideos(videos,videotype)
for video in Videos:
print(video)
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder=destfolder
vname = str(Path(video).stem)
print("Starting % ", videofolder, video)
print("Video Name is: " + vname)
notanalyzed, dataname, DLCscorer=auxiliaryfunctions.CheckifNotAnalyzed(videofolder,vname,DLCscorer,DLCscorerlegacy,flag='checking')
if notanalyzed:
print("The video was not analyzed with this scorer:", DLCscorer)
else:
#LoadData
print("Loading ", video, "and data.")
datafound,metadata,Dataframe,DLCscorer,suffix=auxiliaryfunctions.LoadAnalyzedData(str(videofolder),vname,DLCscorer,filtered) #returns boolean variable if data was found and metadata + pandas array
if datafound:
basefolder=videofolder
auxiliaryfunctions.attempttomakefolder(basefolder)
auxiliaryfunctions.attempttomakefolder(os.path.join(basefolder,'plot-poses'))
tmpfolder = os.path.join(basefolder,'plot-poses', vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder)
PlottingResults(tmpfolder, Dataframe, DLCscorer, cfg, bodyparts, showfigures, suffix+outformat, vname = vname)
print('Plots created! Please check the directory "plot-poses" within the video directory')
def analyze_time_lapse_frames(config,
directory,
frametype='.png',
shuffle=1,
trainingsetindex=0,
gputouse=None,
save_as_csv=False,
rgb=True):
"""
Analyzed all images (of type = frametype) in a folder and stores the output in one file.
You can crop the frames (before analysis), by changing 'cropping'=True and setting 'x1','x2','y1','y2' in the config file.
Output: The labels are stored as MultiIndex Pandas Array, which contains the name of the network, body part name, (x, y) label position \n
in pixels, and the likelihood for each frame per body part. These arrays are stored in an efficient Hierarchical Data Format (HDF) \n
in the same directory, where the video is stored. However, if the flag save_as_csv is set to True, the data can also be exported in \n
comma-separated values format (.csv), which in turn can be imported in many programs, such as MATLAB, R, Prism, etc.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
directory: string
Full path to directory containing the frames that shall be analyzed
frametype: string, optional
Checks for the file extension of the frames. Only images with this extension are analyzed. The default is ``.png``
shuffle: int, optional
An integer specifying the shuffle index of the training dataset used for training the network. The default is 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
rbg: bool, optional.
Whether to load image as rgb; Note e.g. some tiffs do not alow that option in imread, then just set this to false.
Examples
--------
If you want to analyze all frames in /analysis/project/timelapseexperiment1
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml','/analysis/project/timelapseexperiment1')
--------
If you want to analyze all frames in /analysis/project/timelapseexperiment1
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml','/analysis/project/timelapseexperiment1')
--------
Note: for test purposes one can extract all frames from a video with ffmeg, e.g. ffmpeg -i testvideo.avi thumb%04d.png
"""
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ[
'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
if gputouse is not None: # gpu selection
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
vers = (tf.__version__).split('.')
if int(vers[0]) == 1 and int(vers[1]) > 12:
TF = tf.compat.v1
else:
TF = tf
TF.reset_default_graph()
start_path = os.getcwd(
) #record cwd to return to this directory in the end
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
modelfolder = os.path.join(
cfg["project_path"],
str(auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# Check which snapshots are available and sort them by # iterations
try:
Snapshots = np.array([
fn.split('.')[0]
for fn in os.listdir(os.path.join(modelfolder, 'train'))
if "index" in fn
])
except FileNotFoundError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s."
% (shuffle, shuffle))
if cfg['snapshotindex'] == 'all':
print(
"Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!"
)
snapshotindex = -1
else:
snapshotindex = cfg['snapshotindex']
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder)
##################################################
# Load and setup CNN part detector
##################################################
# Check if data already was generated:
dlc_cfg['init_weights'] = os.path.join(modelfolder, 'train',
Snapshots[snapshotindex])
trainingsiterations = (dlc_cfg['init_weights'].split(
os.sep)[-1]).split('-')[-1]
#update batchsize (based on parameters in config.yaml)
dlc_cfg['batch_size'] = cfg['batch_size']
# Name for scorer:
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, trainingsiterations=trainingsiterations)
sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
# update number of outputs and adjust pandas indices
dlc_cfg['num_outputs'] = cfg.get('num_outputs', 1)
xyz_labs_orig = ['x', 'y', 'likelihood']
suffix = [str(s + 1) for s in range(dlc_cfg['num_outputs'])]
suffix[0] = '' # first one has empty suffix for backwards compatibility
xyz_labs = [x + s for s in suffix for x in xyz_labs_orig]
pdindex = pd.MultiIndex.from_product(
[[DLCscorer], dlc_cfg['all_joints_names'], xyz_labs],
names=['scorer', 'bodyparts', 'coords'])
if gputouse is not None: #gpu selectinon
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
##################################################
# Loading the images
##################################################
#checks if input is a directory
if os.path.isdir(directory) == True:
"""
Analyzes all the frames in the directory.
"""
print("Analyzing all frames in the directory: ", directory)
os.chdir(directory)
framelist = np.sort(
[fn for fn in os.listdir(os.curdir) if (frametype in fn)])
vname = Path(directory).stem
notanalyzed, dataname, DLCscorer = auxiliaryfunctions.CheckifNotAnalyzed(
directory, vname, DLCscorer, DLCscorerlegacy, flag='framestack')
if notanalyzed:
nframes = len(framelist)
if nframes > 0:
start = time.time()
PredictedData, nframes, nx, ny = GetPosesofFrames(
cfg, dlc_cfg, sess, inputs, outputs, directory, framelist,
nframes, dlc_cfg['batch_size'], rgb)
stop = time.time()
if cfg['cropping'] == True:
coords = [cfg['x1'], cfg['x2'], cfg['y1'], cfg['y2']]
else:
coords = [0, nx, 0, ny]
dictionary = {
"start": start,
"stop": stop,
"run_duration": stop - start,
"Scorer": DLCscorer,
"config file": dlc_cfg,
"batch_size": dlc_cfg["batch_size"],
"num_outputs": dlc_cfg["num_outputs"],
"frame_dimensions": (ny, nx),
"nframes": nframes,
"cropping": cfg['cropping'],
"cropping_parameters": coords
}
metadata = {'data': dictionary}
print("Saving results in %s..." % (directory))
auxiliaryfunctions.SaveData(PredictedData[:nframes, :],
metadata, dataname, pdindex,
framelist, save_as_csv)
print(
"The folder was analyzed. Now your research can truly start!"
)
print(
"If the tracking is not satisfactory for some frome, consider expanding the training set."
)
else:
print(
"No frames were found. Consider changing the path or the frametype."
)
os.chdir(str(start_path))
def create_training_dataset(
config,
num_shuffles=1,
Shuffles=None,
windows2linux=False,
userfeedback=False,
trainIndices=None,
testIndices=None,
net_type=None,
augmenter_type=None,
posecfg_template=None,
):
"""Creates a training dataset.
Labels from all the extracted frames are merged into a single .h5 file.
Only the videos included in the config file are used to create this dataset.
Parameters
----------
config : string
Full path of the ``config.yaml`` file as a string.
num_shuffles : int, optional, default=1
Number of shuffles of training dataset to create, i.e. ``[1,2,3]`` for
``num_shuffles=3``.
Shuffles: list[int], optional
Alternatively the user can also give a list of shuffles.
userfeedback: bool, optional, default=False
If ``False``, all requested train/test splits are created (no matter if they
already exist). If you want to assure that previous splits etc. are not
overwritten, set this to ``True`` and you will be asked for each split.
trainIndices: list of lists, optional, default=None
List of one or multiple lists containing train indexes.
A list containing two lists of training indexes will produce two splits.
testIndices: list of lists, optional, default=None
List of one or multiple lists containing test indexes.
net_type: list, optional, default=None
Type of networks. Currently supported options are
* ``resnet_50``
* ``resnet_101``
* ``resnet_152``
* ``mobilenet_v2_1.0``
* ``mobilenet_v2_0.75``
* ``mobilenet_v2_0.5``
* ``mobilenet_v2_0.35``
* ``efficientnet-b0``
* ``efficientnet-b1``
* ``efficientnet-b2``
* ``efficientnet-b3``
* ``efficientnet-b4``
* ``efficientnet-b5``
* ``efficientnet-b6``
augmenter_type: string, optional, default=None
Type of augmenter. Currently supported augmenters are
* ``default``
* ``scalecrop``
* ``imgaug``
* ``tensorpack``
* ``deterministic``
posecfg_template: string, optional, default=None
Path to a ``pose_cfg.yaml`` file to use as a template for generating the new
one for the current iteration. Useful if you would like to start with the same
parameters a previous training iteration. None uses the default
``pose_cfg.yaml``.
Returns
-------
list(tuple) or None
If training dataset was successfully created, a list of tuples is returned.
The first two elements in each tuple represent the training fraction and the
shuffle value. The last two elements in each tuple are arrays of integers
representing the training and test indices.
Returns None if training dataset could not be created.
Notes
-----
Use the function ``add_new_videos`` at any stage of the project to add more videos
to the project.
Examples
--------
Linux/MacOS
>>> deeplabcut.create_training_dataset(
'/analysis/project/reaching-task/config.yaml', num_shuffles=1,
)
Windows
>>> deeplabcut.create_training_dataset(
'C:\\Users\\Ulf\\looming-task\\config.yaml', Shuffles=[3,17,5],
)
"""
import scipy.io as sio
if windows2linux:
# DeprecationWarnings are silenced since Python 3.2 unless triggered in __main__
warnings.warn(
"`windows2linux` has no effect since 2.2.0.4 and will be removed in 2.2.1.",
FutureWarning,
)
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
if posecfg_template:
if not posecfg_template.endswith("pose_cfg.yaml"):
raise ValueError(
"posecfg_template argument must contain path to a pose_cfg.yaml file"
)
else:
print("Reloading pose_cfg parameters from " + posecfg_template +
'\n')
from deeplabcut.utils.auxiliaryfunctions import read_plainconfig
prior_cfg = read_plainconfig(posecfg_template)
if cfg.get("multianimalproject", False):
from deeplabcut.generate_training_dataset.multiple_individuals_trainingsetmanipulation import (
create_multianimaltraining_dataset, )
create_multianimaltraining_dataset(config,
num_shuffles,
Shuffles,
net_type=net_type)
else:
scorer = cfg["scorer"]
project_path = cfg["project_path"]
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(
cfg) # Path concatenation OS platform independent
auxiliaryfunctions.attempttomakefolder(Path(
os.path.join(project_path, str(trainingsetfolder))),
recursive=True)
Data = merge_annotateddatasets(
cfg,
Path(os.path.join(project_path, trainingsetfolder)),
)
if Data is None:
return
Data = Data[scorer] # extract labeled data
# loading & linking pretrained models
if net_type is None: # loading & linking pretrained models
net_type = cfg.get("default_net_type", "resnet_50")
else:
if ("resnet" in net_type or "mobilenet" in net_type
or "efficientnet" in net_type):
pass
else:
raise ValueError("Invalid network type:", net_type)
if augmenter_type is None:
augmenter_type = cfg.get("default_augmenter", "imgaug")
if augmenter_type is None: # this could be in config.yaml for old projects!
# updating variable if null/None! #backwardscompatability
auxiliaryfunctions.edit_config(config,
{"default_augmenter": "imgaug"})
augmenter_type = "imgaug"
elif augmenter_type not in [
"default",
"scalecrop",
"imgaug",
"tensorpack",
"deterministic",
]:
raise ValueError("Invalid augmenter type:", augmenter_type)
if posecfg_template:
if net_type != prior_cfg["net_type"]:
print(
"WARNING: Specified net_type does not match net_type from posecfg_template path entered. Proceed with caution."
)
if augmenter_type != prior_cfg["dataset_type"]:
print(
"WARNING: Specified augmenter_type does not match dataset_type from posecfg_template path entered. Proceed with caution."
)
# Loading the encoder (if necessary downloading from TF)
dlcparent_path = auxiliaryfunctions.get_deeplabcut_path()
if not posecfg_template:
defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml")
elif posecfg_template:
defaultconfigfile = posecfg_template
model_path, num_shuffles = auxfun_models.check_for_weights(
net_type, Path(dlcparent_path), num_shuffles)
if Shuffles is None:
Shuffles = range(1, num_shuffles + 1)
else:
Shuffles = [i for i in Shuffles if isinstance(i, int)]
# print(trainIndices,testIndices, Shuffles, augmenter_type,net_type)
if trainIndices is None and testIndices is None:
splits = [(
trainFraction,
shuffle,
SplitTrials(range(len(Data.index)), trainFraction),
) for trainFraction in cfg["TrainingFraction"]
for shuffle in Shuffles]
else:
if len(trainIndices) != len(testIndices) != len(Shuffles):
raise ValueError(
"Number of Shuffles and train and test indexes should be equal."
)
splits = []
for shuffle, (train_inds, test_inds) in enumerate(
zip(trainIndices, testIndices)):
trainFraction = round(
len(train_inds) * 1.0 / (len(train_inds) + len(test_inds)),
2)
print(
f"You passed a split with the following fraction: {int(100 * trainFraction)}%"
)
# Now that the training fraction is guaranteed to be correct,
# the values added to pad the indices are removed.
train_inds = np.asarray(train_inds)
train_inds = train_inds[train_inds != -1]
test_inds = np.asarray(test_inds)
test_inds = test_inds[test_inds != -1]
splits.append((trainFraction, Shuffles[shuffle], (train_inds,
test_inds)))
bodyparts = cfg["bodyparts"]
nbodyparts = len(bodyparts)
for trainFraction, shuffle, (trainIndices, testIndices) in splits:
if len(trainIndices) > 0:
if userfeedback:
trainposeconfigfile, _, _ = training.return_train_network_path(
config,
shuffle=shuffle,
trainingsetindex=cfg["TrainingFraction"].index(
trainFraction),
)
if trainposeconfigfile.is_file():
askuser = input(
"The model folder is already present. If you continue, it will overwrite the existing model (split). Do you want to continue?(yes/no): "
)
if (askuser == "no" or askuser == "No"
or askuser == "N" or askuser == "No"):
raise Exception(
"Use the Shuffles argument as a list to specify a different shuffle index. Check out the help for more details."
)
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
# Make training file!
(
datafilename,
metadatafilename,
) = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
################################################################################
# Saving data file (convert to training file for deeper cut (*.mat))
################################################################################
data, MatlabData = format_training_data(
Data, trainIndices, nbodyparts, project_path)
sio.savemat(os.path.join(project_path, datafilename),
{"dataset": MatlabData})
################################################################################
# Saving metadata (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(
os.path.join(project_path, metadatafilename),
data,
trainIndices,
testIndices,
trainFraction,
)
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername = auxiliaryfunctions.get_model_folder(
trainFraction, shuffle, cfg)
auxiliaryfunctions.attempttomakefolder(
Path(config).parents[0] / modelfoldername, recursive=True)
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername) + "/train")
auxiliaryfunctions.attempttomakefolder(
str(Path(config).parents[0] / modelfoldername) + "/test")
path_train_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"train",
"pose_cfg.yaml",
))
path_test_config = str(
os.path.join(
cfg["project_path"],
Path(modelfoldername),
"test",
"pose_cfg.yaml",
))
# str(cfg['proj_path']+'/'+Path(modelfoldername) / 'test' / 'pose_cfg.yaml')
items2change = {
"dataset": datafilename,
"metadataset": metadatafilename,
"num_joints": len(bodyparts),
"all_joints": [[i] for i in range(len(bodyparts))],
"all_joints_names": [str(bpt) for bpt in bodyparts],
"init_weights": model_path,
"project_path": str(cfg["project_path"]),
"net_type": net_type,
"dataset_type": augmenter_type,
}
items2drop = {}
if augmenter_type == "scalecrop":
# these values are dropped as scalecrop
# doesn't have rotation implemented
items2drop = {"rotation": 0, "rotratio": 0.0}
# Also drop maDLC smart cropping augmentation parameters
for key in [
"pre_resize", "crop_size", "max_shift", "crop_sampling"
]:
items2drop[key] = None
trainingdata = MakeTrain_pose_yaml(items2change,
path_train_config,
defaultconfigfile,
items2drop)
keys2save = [
"dataset",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
]
MakeTest_pose_yaml(trainingdata, keys2save, path_test_config)
print(
"The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!"
)
return splits
def create_labeled_video(
config,
videos,
videotype="avi",
shuffle=1,
trainingsetindex=0,
filtered=False,
fastmode=True,
save_frames=False,
Frames2plot=None,
displayedbodyparts="all",
displayedindividuals="all",
codec="mp4v",
outputframerate=None,
destfolder=None,
draw_skeleton=False,
trailpoints=0,
displaycropped=False,
color_by="bodypart",
modelprefix="",
track_method="",
):
"""
Labels the bodyparts in a video. Make sure the video is already analyzed by the function 'analyze_video'
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
Number of shuffles of training dataset. Default is set to 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
fastmode: bool
If true uses openCV (much faster but less customization of video) vs matplotlib (if false). You can also
"save_frames" individually or not in the matplotlib mode (if you set the "save_frames" variable accordingly).
However, using matplotlib to create the frames it therefore allows much more flexible (one can set transparency of markers, crop, and easily customize).
save_frames: bool
If true creates each frame individual and then combines into a video. This variant is relatively slow as
it stores all individual frames.
Frames2plot: List of indices
If not None & save_frames=True then the frames corresponding to the index will be plotted. For example, Frames2plot=[0,11] will plot the first and the 12th frame.
displayedbodyparts: list of strings, optional
This selects the body parts that are plotted in the video. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
displayedindividuals: list of strings, optional
Individuals plotted in the video. By default, all individuals present in the config will be showed.
codec: codec for labeled video. Options see http://www.fourcc.org/codecs.php [depends on your ffmpeg installation.]
outputframerate: positive number, output frame rate for labeled video (only available for the mode with saving frames.) By default: None, which results in the original video rate.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
draw_skeleton: bool
If ``True`` adds a line connecting the body parts making a skeleton on on each frame. The body parts to be connected and the color of these connecting lines are specified in the config file. By default: ``False``
trailpoints: int
Number of revious frames whose body parts are plotted in a frame (for displaying history). Default is set to 0.
displaycropped: bool, optional
Specifies whether only cropped frame is displayed (with labels analyzed therein), or the original frame with the labels analyzed in the cropped subset.
color_by : string, optional (default='bodypart')
Coloring rule. By default, each bodypart is colored differently.
If set to 'individual', points belonging to a single individual are colored the same.
Examples
--------
If you want to create the labeled video for only 1 video
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to create the labeled video for only 1 video and store the individual frames
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'],fastmode=True, save_frames=True)
--------
If you want to create the labeled video for multiple videos
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to create the labeled video for all the videos (as .avi extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'])
--------
If you want to create the labeled video for all the videos (as .mp4 extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'],videotype='mp4')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, modelprefix=modelprefix
) # automatically loads corresponding model (even training iteration based on snapshot index)
if save_frames:
fastmode = False # otherwise one cannot save frames
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts
)
individuals = auxfun_multianimal.IntersectionofIndividualsandOnesGivenbyUser(
cfg, displayedindividuals
)
if draw_skeleton:
bodyparts2connect = cfg["skeleton"]
skeleton_color = cfg["skeleton_color"]
else:
bodyparts2connect = None
skeleton_color = None
start_path = os.getcwd()
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
if not Videos:
print("No video(s) were found. Please check your paths and/or 'video_type'.")
return
func = partial(
proc_video,
videos,
destfolder,
filtered,
DLCscorer,
DLCscorerlegacy,
track_method,
cfg,
individuals,
color_by,
bodyparts,
codec,
bodyparts2connect,
trailpoints,
save_frames,
outputframerate,
Frames2plot,
draw_skeleton,
skeleton_color,
displaycropped,
fastmode,
)
with Pool(min(os.cpu_count(), len(Videos))) as pool:
pool.map(func, Videos)
os.chdir(start_path)
def check_labels(
config,
Labels=["+", ".", "x"],
scale=1,
dpi=100,
draw_skeleton=True,
visualizeindividuals=True,
):
"""Check the labeled frames.
Double check if the labels were at the correct locations and stored in the proper
file format.
This creates a new subdirectory for each video under the 'labeled-data' and all the
frames are plotted with the labels.
Make sure that these labels are fine.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
Labels: list, default='+'
List of at least 3 matplotlib markers. The first one will be used to indicate
the human ground truth location (Default: +)
scale : float, default=1
Change the relative size of the output images.
dpi : int, optional, default=100
Output resolution in dpi.
draw_skeleton: bool, default=True
Plot skeleton overlaid over body parts.
visualizeindividuals: bool, default: True.
For a multianimal project, if True, the different individuals have different
colors (and all bodyparts the same). If False, the colors change over bodyparts
rather than individuals.
Returns
-------
None
Examples
--------
>>> deeplabcut.check_labels('/analysis/project/reaching-task/config.yaml')
"""
from deeplabcut.utils import visualization
cfg = auxiliaryfunctions.read_config(config)
videos = cfg["video_sets"].keys()
video_names = [_robust_path_split(video)[1] for video in videos]
folders = [
os.path.join(cfg["project_path"], "labeled-data", str(Path(i)))
for i in video_names
]
print("Creating images with labels by %s." % cfg["scorer"])
for folder in folders:
try:
DataCombined = pd.read_hdf(
os.path.join(str(folder),
"CollectedData_" + cfg["scorer"] + ".h5"))
conversioncode.guarantee_multiindex_rows(DataCombined)
if cfg.get("multianimalproject", False):
color_by = "individual" if visualizeindividuals else "bodypart"
else: # for single animal projects
color_by = "bodypart"
visualization.make_labeled_images_from_dataframe(
DataCombined,
cfg,
folder,
scale,
dpi=dpi,
keypoint=Labels[0],
draw_skeleton=draw_skeleton,
color_by=color_by,
)
except FileNotFoundError:
print("Attention:", folder,
"does not appear to have labeled data!")
print(
"If all the labels are ok, then use the function 'create_training_dataset' to create the training dataset!"
)
def adddatasetstovideolistandviceversa(config):
"""
First run comparevideolistsanddatafolders(config) to compare the folders in labeled-data and the ones listed under video_sets (in the config file).
If you detect differences this function can be used to maker sure each folder has a video entry & vice versa.
It corrects this problem in the following way:
If a video entry in the config file does not contain a folder in labeled-data, then the entry is removed.
If a folder in labeled-data does not contain a video entry in the config file then the prefix path will be added in front of the name of the labeled-data folder and combined
with the suffix variable as an ending. Width and height will be added as cropping variables as passed on.
Handle with care!
Parameter
----------
config : string
String containing the full path of the config file in the project.
"""
cfg = auxiliaryfunctions.read_config(config)
videos = cfg["video_sets"]
video_names = [Path(i).stem for i in videos]
alldatafolders = [
fn for fn in os.listdir(Path(config).parent / "labeled-data")
if "_labeled" not in fn and not fn.startswith(".")
]
print("Config file contains:", len(video_names))
print("Labeled-data contains:", len(alldatafolders))
toberemoved = []
for vn in video_names:
if vn not in alldatafolders:
print(vn, " is missing as a labeled folder >> removing key!")
for fullvideo in videos:
if vn in fullvideo:
toberemoved.append(fullvideo)
for vid in toberemoved:
del videos[vid]
# Load updated lists:
video_names = [Path(i).stem for i in videos]
for vn in alldatafolders:
if vn not in video_names:
print(vn, " is missing in config file >> adding it!")
# Find the corresponding video file
found = False
for file in os.listdir(os.path.join(cfg["project_path"],
"videos")):
if os.path.splitext(file)[0] == vn:
found = True
break
if found:
video_path = os.path.join(cfg["project_path"], "videos", file)
clip = VideoReader(video_path)
videos.update({
video_path: {
"crop": ", ".join(map(str, clip.get_bbox()))
}
})
auxiliaryfunctions.write_config(config, cfg)
def create_training_model_comparison(
config,
trainindex=0,
num_shuffles=1,
net_types=["resnet_50"],
augmenter_types=["imgaug"],
userfeedback=False,
windows2linux=False,
):
"""Creates a training dataset to compare networks and augmentation types.
The datasets are created such that the shuffles have same training and testing
indices. Therefore, this function is useful for benchmarking the performance of
different network and augmentation types on the same training/testdata.
Parameters
----------
config: str
Full path of the config.yaml file.
trainindex: int, optional, default=0
Either (in case uniform = True) indexes which element of TrainingFraction in
the config file should be used (note it is a list!).
Alternatively (uniform = False) indexes which folder is dropped, i.e. the first
if trainindex=0, the second if trainindex=1, etc.
num_shuffles : int, optional, default=1
Number of shuffles of training dataset to create,
i.e. [1,2,3] for num_shuffles=3.
net_types: list[str], optional, default=["resnet_50"]
Currently supported networks are
* ``"resnet_50"``
* ``"resnet_101"``
* ``"resnet_152"``
* ``"mobilenet_v2_1.0"``
* ``"mobilenet_v2_0.75"``
* ``"mobilenet_v2_0.5"``
* ``"mobilenet_v2_0.35"``
* ``"efficientnet-b0"``
* ``"efficientnet-b1"``
* ``"efficientnet-b2"``
* ``"efficientnet-b3"``
* ``"efficientnet-b4"``
* ``"efficientnet-b5"``
* ``"efficientnet-b6"``
augmenter_types: list[str], optional, default=["imgaug"]
Currently supported augmenters are
* ``"default"``
* ``"imgaug"``
* ``"tensorpack"``
* ``"deterministic"``
userfeedback: bool, optional, default=False
If ``False``, then all requested train/test splits are created, no matter if
they already exist. If you want to assure that previous splits etc. are not
overwritten, then set this to True and you will be asked for each split.
windows2linux
..deprecated::
Has no effect since 2.2.0.4 and will be removed in 2.2.1.
Returns
-------
shuffle_list: list
List of indices corresponding to the trainingsplits/models that were created.
Examples
--------
On Linux/MacOS
>>> shuffle_list = deeplabcut.create_training_model_comparison(
'/analysis/project/reaching-task/config.yaml',
num_shuffles=1,
net_types=['resnet_50','resnet_152'],
augmenter_types=['tensorpack','deterministic'],
)
On Windows
>>> shuffle_list = deeplabcut.create_training_model_comparison(
'C:\\Users\\Ulf\\looming-task\\config.yaml',
num_shuffles=1,
net_types=['resnet_50','resnet_152'],
augmenter_types=['tensorpack','deterministic'],
)
See ``examples/testscript_openfielddata_augmentationcomparison.py`` for an example
of how to use ``shuffle_list``.
"""
# read cfg file
cfg = auxiliaryfunctions.read_config(config)
if windows2linux:
warnings.warn(
"`windows2linux` has no effect since 2.2.0.4 and will be removed in 2.2.1.",
FutureWarning,
)
# create log file
log_file_name = os.path.join(cfg["project_path"],
"training_model_comparison.log")
logger = logging.getLogger("training_model_comparison")
if not logger.handlers:
logger = logging.getLogger("training_model_comparison")
hdlr = logging.FileHandler(log_file_name)
formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.INFO)
else:
pass
largestshuffleindex = get_largestshuffle_index(config)
shuffle_list = []
for shuffle in range(num_shuffles):
trainIndices, testIndices = mergeandsplit(config,
trainindex=trainindex,
uniform=True)
for idx_net, net in enumerate(net_types):
for idx_aug, aug in enumerate(augmenter_types):
get_max_shuffle_idx = (
largestshuffleindex + idx_aug +
idx_net * len(augmenter_types) +
shuffle * len(augmenter_types) * len(net_types))
shuffle_list.append(get_max_shuffle_idx)
log_info = str("Shuffle index:" + str(get_max_shuffle_idx) +
", net_type:" + net + ", augmenter_type:" +
aug + ", trainsetindex:" + str(trainindex) +
", frozen shuffle ID:" + str(shuffle))
create_training_dataset(
config,
Shuffles=[get_max_shuffle_idx],
net_type=net,
trainIndices=[trainIndices],
testIndices=[testIndices],
augmenter_type=aug,
userfeedback=userfeedback,
)
logger.info(log_info)
return shuffle_list
def __init__(self, parent, config, video, shuffle, Dataframe, savelabeled,
multianimal):
super(MainFrame, self).__init__(
"DeepLabCut2.0 - Manual Outlier Frame Extraction",
parent,
)
###################################################################################################################################################
# Spliting the frame into top and bottom panels. Bottom panels contains the widgets. The top panel is for showing images and plotting!
# topSplitter = wx.SplitterWindow(self)
#
# self.image_panel = ImagePanel(topSplitter, config,video,shuffle,Dataframe,self.gui_size)
# self.widget_panel = WidgetPanel(topSplitter)
#
# topSplitter.SplitHorizontally(self.image_panel, self.widget_panel,sashPosition=self.gui_size[1]*0.83)#0.9
# topSplitter.SetSashGravity(1)
# sizer = wx.BoxSizer(wx.VERTICAL)
# sizer.Add(topSplitter, 1, wx.EXPAND)
# self.SetSizer(sizer)
# Spliting the frame into top and bottom panels. Bottom panels contains the widgets. The top panel is for showing images and plotting!
topSplitter = wx.SplitterWindow(self)
vSplitter = wx.SplitterWindow(topSplitter)
self.image_panel = ImagePanel(vSplitter, self.gui_size)
self.choice_panel = ScrollPanel(vSplitter)
vSplitter.SplitVertically(self.image_panel,
self.choice_panel,
sashPosition=self.gui_size[0] * 0.8)
vSplitter.SetSashGravity(1)
self.widget_panel = WidgetPanel(topSplitter)
topSplitter.SplitHorizontally(vSplitter,
self.widget_panel,
sashPosition=self.gui_size[1] *
0.83) # 0.9
topSplitter.SetSashGravity(1)
sizer = wx.BoxSizer(wx.VERTICAL)
sizer.Add(topSplitter, 1, wx.EXPAND)
self.SetSizer(sizer)
###################################################################################################################################################
# Add Buttons to the WidgetPanel and bind them to their respective functions.
widgetsizer = wx.WrapSizer(orient=wx.HORIZONTAL)
self.load_button_sizer = wx.BoxSizer(wx.VERTICAL)
self.help_button_sizer = wx.BoxSizer(wx.VERTICAL)
self.help = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Help")
self.help_button_sizer.Add(self.help, 1, wx.ALL, 15)
# widgetsizer.Add(self.help , 1, wx.ALL, 15)
self.help.Bind(wx.EVT_BUTTON, self.helpButton)
widgetsizer.Add(self.help_button_sizer, 1, wx.ALL, 0)
self.grab = wx.Button(self.widget_panel,
id=wx.ID_ANY,
label="Grab Frames")
widgetsizer.Add(self.grab, 1, wx.ALL, 15)
self.grab.Bind(wx.EVT_BUTTON, self.grabFrame)
self.grab.Enable(True)
widgetsizer.AddStretchSpacer(5)
self.slider = wx.Slider(
self.widget_panel,
id=wx.ID_ANY,
value=0,
minValue=0,
maxValue=1,
size=(200, -1),
style=wx.SL_HORIZONTAL | wx.SL_AUTOTICKS | wx.SL_LABELS,
)
widgetsizer.Add(self.slider, 1, wx.ALL, 5)
self.slider.Bind(wx.EVT_SLIDER, self.OnSliderScroll)
widgetsizer.AddStretchSpacer(5)
self.start_frames_sizer = wx.BoxSizer(wx.VERTICAL)
self.end_frames_sizer = wx.BoxSizer(wx.VERTICAL)
self.start_frames_sizer.AddSpacer(15)
# self.startFrame = wx.SpinCtrl(self.widget_panel, value='0', size=(100, -1), min=0, max=120)
self.startFrame = wx.SpinCtrl(self.widget_panel,
value="0",
size=(100, -1)) # ,style=wx.SP_VERTICAL)
self.startFrame.Enable(False)
self.start_frames_sizer.Add(self.startFrame, 1,
wx.EXPAND | wx.ALIGN_LEFT, 15)
start_text = wx.StaticText(self.widget_panel,
label="Start Frame Index")
self.start_frames_sizer.Add(start_text, 1, wx.EXPAND | wx.ALIGN_LEFT,
15)
self.checkBox = wx.CheckBox(self.widget_panel,
id=wx.ID_ANY,
label="Range of frames")
self.checkBox.Bind(wx.EVT_CHECKBOX, self.activate_frame_range)
self.start_frames_sizer.Add(self.checkBox, 1,
wx.EXPAND | wx.ALIGN_LEFT, 15)
#
self.end_frames_sizer.AddSpacer(15)
self.endFrame = wx.SpinCtrl(self.widget_panel,
value="1",
size=(160, -1)) # , min=1, max=120)
self.endFrame.Enable(False)
self.end_frames_sizer.Add(self.endFrame, 1, wx.EXPAND | wx.ALIGN_LEFT,
15)
end_text = wx.StaticText(self.widget_panel, label="Number of Frames")
self.end_frames_sizer.Add(end_text, 1, wx.EXPAND | wx.ALIGN_LEFT, 15)
self.updateFrame = wx.Button(self.widget_panel,
id=wx.ID_ANY,
label="Update")
self.end_frames_sizer.Add(self.updateFrame, 1,
wx.EXPAND | wx.ALIGN_LEFT, 15)
self.updateFrame.Bind(wx.EVT_BUTTON, self.updateSlider)
self.updateFrame.Enable(False)
widgetsizer.Add(self.start_frames_sizer, 1, wx.ALL, 0)
widgetsizer.AddStretchSpacer(5)
widgetsizer.Add(self.end_frames_sizer, 1, wx.ALL, 0)
widgetsizer.AddStretchSpacer(15)
self.quit = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Quit")
widgetsizer.Add(self.quit, 1, wx.ALL, 15)
self.quit.Bind(wx.EVT_BUTTON, self.quitButton)
self.quit.Enable(True)
self.widget_panel.SetSizer(widgetsizer)
self.widget_panel.SetSizerAndFit(widgetsizer)
# Variables initialization
self.numberFrames = 0
self.currFrame = 0
self.figure = Figure()
self.axes = self.figure.add_subplot(111)
self.drs = []
self.extract_range_frame = False
self.firstFrame = 0
self.Colorscheme = []
# Read confing file
self.cfg = auxiliaryfunctions.read_config(config)
self.Task = self.cfg["Task"]
self.start = self.cfg["start"]
self.stop = self.cfg["stop"]
self.date = self.cfg["date"]
self.trainFraction = self.cfg["TrainingFraction"]
self.trainFraction = self.trainFraction[0]
self.videos = self.cfg["video_sets"].keys()
self.bodyparts = self.cfg["bodyparts"]
self.colormap = plt.get_cmap(self.cfg["colormap"])
self.colormap = self.colormap.reversed()
self.markerSize = self.cfg["dotsize"]
self.alpha = self.cfg["alphavalue"]
self.iterationindex = self.cfg["iteration"]
self.cropping = self.cfg["cropping"]
self.video_names = [Path(i).stem for i in self.videos]
self.config_path = Path(config)
self.video_source = Path(video).resolve()
self.shuffle = shuffle
self.Dataframe = Dataframe
self.savelabeled = savelabeled
self.multianimal = multianimal
if self.multianimal:
from deeplabcut.utils import auxfun_multianimal
(
self.individual_names,
self.uniquebodyparts,
self.multianimalbodyparts,
) = auxfun_multianimal.extractindividualsandbodyparts(self.cfg)
self.choiceBox, self.visualization_rdb = self.choice_panel.addRadioButtons(
)
self.Colorscheme = visualization.get_cmap(
len(self.individual_names), self.cfg["colormap"])
self.visualization_rdb.Bind(wx.EVT_RADIOBOX, self.clear_plot)
# Read the video file
self.vid = VideoWriter(str(self.video_source))
if self.cropping:
self.vid.set_bbox(self.cfg["x1"], self.cfg["x2"], self.cfg["y1"],
self.cfg["y2"])
self.filename = Path(self.video_source).name
self.numberFrames = len(self.vid)
self.strwidth = int(np.ceil(np.log10(self.numberFrames)))
# Set the values of slider and range of frames
self.startFrame.SetMax(self.numberFrames - 1)
self.slider.SetMax(self.numberFrames - 1)
self.endFrame.SetMax(self.numberFrames - 1)
self.startFrame.Bind(wx.EVT_SPINCTRL, self.updateSlider) # wx.EVT_SPIN
# Set the status bar
self.statusbar.SetStatusText("Working on video: {}".format(
self.filename))
# Adding the video file to the config file.
if self.vid.name not in self.video_names:
add.add_new_videos(self.config_path, [self.video_source])
self.update()
self.plot_labels()
self.widget_panel.Layout()
def triangulate(dlc_3d_config, video_cfg, origins, table_corners, tocchini, video_path, filterpredictions=True, destfolder=None,
save_as_csv=False):
cfg_3d = auxiliaryfunctions.read_config(dlc_3d_config)
img_path, path_corners, path_camera_matrix, path_undistort = auxiliaryfunctions_3d.Foldernames3Dproject(cfg_3d)
cam_names = cfg_3d['camera_names']
pcutoff = cfg_3d['pcutoff']
scorer_3d = cfg_3d['scorername_3d']
snapshots = {}
for cam in cam_names:
snapshots[cam] = cfg_3d[str('config_file_' + cam)]
# Check if the config file exists
if not os.path.exists(snapshots[cam]):
raise Exception(str("It seems the file specified in the variable config_file_" + str(cam)) +
" does not exist. Please edit the config file with correct file path and retry.")
path_stereo_file = os.path.join(path_camera_matrix, 'stereo_params.pickle')
stereo_file = auxiliaryfunctions.read_pickle(path_stereo_file)
position=np.array([0.0,0.0,0.0])
rotation=np.array([[1.0, 0.0, 0.0],[0.0, 1.0, 0.0], [0.0, 0.0, 1.0]])
rotations=[np.linalg.inv(rotation)]
translations=[np.matmul(-rotation, position)]
projections=[]
for cam_idx in range(1,len(cam_names)):
pair_rotation=stereo_file[cam_names[0] + '-' + cam_names[cam_idx]]['R']
pair_translation = stereo_file[cam_names[0] + '-' + cam_names[cam_idx]]['T']
rotations.append(np.matmul(pair_rotation,rotations[0]))
translations.append(np.matmul(pair_rotation, translations[0])+np.transpose(pair_translation))
for cam_idx in range(len(cam_names)):
path_camera_file = os.path.join(path_camera_matrix, '%s_intrinsic_params.pickle' % cam_names[cam_idx])
intrinsic_file = auxiliaryfunctions.read_pickle(path_camera_file)
projection=np.zeros((3,4))
projection[:,0:3]=rotations[cam_idx]
projection[:,3]=translations[cam_idx]
projection=np.matmul(intrinsic_file[cam_names[cam_idx]]['mtx'], projection)
projections.append(projection)
for view in cam_names:
origins[view]=np.array(origins[view])
origins[view][0]=origins[view][0]+video_cfg['crop_limits'][view][0]
origins[view][1]=origins[view][1]+video_cfg['crop_limits'][view][2]
[origin, pairs_used]=locate(cam_names,{'front':1,'side':1,'top':1}, origins, pcutoff, projections, reconMode='all')
table_coords_3d=[]
for corner_idx in range(len(table_corners['front'])):
corner={}
for view in cam_names:
corner[view]=table_corners[view][corner_idx]
corner[view] =np.array(corner[view])
corner[view][0]=corner[view][0]+video_cfg['crop_limits'][view][0]
corner[view][1]=corner[view][1]+video_cfg['crop_limits'][view][2]
[coord,pairs_used]=locate(cam_names,{'front':1,'side':1,'top':1},corner,pcutoff,projections, reconMode='all')
table_coords_3d.append(coord)
xy_norm=np.array([[0,0,1]])
table_center=np.mean(np.array(table_coords_3d),axis=0)
table_vec1=table_coords_3d[0]-table_center
table_vec2=table_coords_3d[1]-table_center
table_norm = unit_vector(np.cross(np.transpose(table_vec1), np.transpose(table_vec2)))
rot_vec=unit_vector(np.cross(xy_norm, table_norm), axis=1)
rot_angle=-np.arccos(np.abs(np.sum(xy_norm*table_norm))/np.sqrt(np.sum(table_norm**2)))
rot_mat=rotation_matrix(rot_angle, np.transpose(rot_vec))
rot_mat=np.matmul(rotation_matrix(np.pi, [1, 0, 0]), rot_mat)
rot_mat=rot_mat[0:3,0:3]
origin=np.matmul(rot_mat, origin)
for idx, coord in enumerate(table_coords_3d):
coord=np.matmul(rot_mat, coord)-origin
table_coords_3d[idx]=coord
tocchini_coords_3d=[]
for tocchino_idx in range(len(tocchini['front'])):
tocchino={}
for view in cam_names:
tocchino[view]=tocchini[view][tocchino_idx]
tocchino[view]=np.array(tocchino[view])
tocchino[view][0]=tocchino[view][0]+video_cfg['crop_limits'][view][0]
tocchino[view][1]=tocchino[view][1]+video_cfg['crop_limits'][view][2]
[coord,pairs_used]=locate(cam_names,{'front':1,'side':1,'top':1},tocchino,pcutoff,projections, reconMode='all')
tocchino_coord=np.matmul(rot_mat,coord)-origin
tocchini_coords_3d.append(tocchino_coord)
file_name_3d_scorer = []
dataname = []
for cam_name in cam_names:
dlc_3d_config = snapshots[cam_name]
cfg = auxiliaryfunctions.read_config(dlc_3d_config)
shuffle = cfg_3d[str('shuffle_' + cam_name)]
trainingsetindex = cfg_3d[str('trainingsetindex_' + cam_name)]
video=video_path[cam_name]
vname = Path(video).stem
trainFraction = cfg['TrainingFraction'][trainingsetindex]
modelfolder = os.path.join(cfg["project_path"],
str(auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
dlc_cfg = load_config(str(path_test_config))
Snapshots = np.array([fn.split('.')[0] for fn in os.listdir(os.path.join(modelfolder, 'train')) if "index" in fn])
snapshotindex = cfg['snapshotindex']
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
dlc_cfg['init_weights'] = os.path.join(modelfolder, 'train', Snapshots[snapshotindex])
trainingsiterations = (dlc_cfg['init_weights'].split(os.sep)[-1]).split('-')[-1]
DLCscorer = auxiliaryfunctions.GetScorerName(cfg, shuffle, trainFraction,
trainingsiterations=trainingsiterations)
file_name_3d_scorer.append(DLCscorer)
if filterpredictions:
dataname.append(os.path.join(destfolder, cam_name, vname + DLCscorer + 'filtered.h5'))
else:
dataname.append(os.path.join(destfolder, cam_name, vname + DLCscorer + '.h5'))
output_filename = os.path.join(destfolder, vname + '_' + scorer_3d)
if os.path.isfile(output_filename + '.h5'): # TODO: don't check twice and load the pickle file to check if the same snapshots + camera matrices were used.
print("Already analyzed...", output_filename+'.h5')
else:
if len(dataname) > 0:
df = pd.read_hdf(dataname[0])
df_3d, scorer_3d, bodyparts = auxiliaryfunctions_3d.create_empty_df(df, scorer_3d, flag='3d')
for bpindex, bp in enumerate(bodyparts):
bp_coords=np.zeros((3,len(df_3d)))
for f_idx in range(len(df_3d)):
likelihoods={}
coords={}
for cam_idx,cam_name in enumerate(cam_names):
dataframe_cam = pd.read_hdf(dataname[cam_idx])
scorer_cam = dataframe_cam.columns.get_level_values(0)[0]
likelihoods[cam_name]=dataframe_cam[scorer_cam][bp]['likelihood'].values[f_idx]
coords[cam_name]=np.array([dataframe_cam[scorer_cam][bp]['x'].values[f_idx], dataframe_cam[scorer_cam][bp]['y'].values[f_idx]])
coords[cam_name][0]=coords[cam_name][0]+video_cfg['crop_limits'][cam_name][0]
coords[cam_name][1]=coords[cam_name][1]+video_cfg['crop_limits'][cam_name][2]
[coord, pairs_used] = locate(cam_names, likelihoods, coords, pcutoff, projections, reconMode='bestpossible')
coord=np.matmul(rot_mat, coord)-origin
if pairs_used < 3:
coord[0]=np.nan
coord[1]=np.nan
coord[2]=np.nan
bp_coords[:,f_idx]=np.squeeze(coord)
df_3d.iloc[:][scorer_3d, bp, 'x'] = bp_coords[0,:]
df_3d.iloc[:][scorer_3d, bp, 'y'] = bp_coords[1,:]
df_3d.iloc[:][scorer_3d, bp, 'z'] = bp_coords[2,:]
df_3d.to_hdf(str(output_filename + '.h5'), 'df_with_missing', format='table', mode='w')
if save_as_csv:
df_3d.to_csv(str(output_filename + '.csv'))
print("Triangulated data for video", vname)
print("Results are saved under: ", destfolder)
return str(output_filename + '.h5'), table_coords_3d, tocchini_coords_3d
def create_training_model_comparison(config,trainindex=0,num_shuffles=1,net_types=['resnet_50'],augmenter_types=['default'],userfeedback=False,windows2linux=False):
"""
Creates a training dataset with different networks and augmentation types (dataset_loader) so that the shuffles
have same training and testing indices.
Therefore, this function is useful for benchmarking the performance of different network and augmentation types on the same training/testdata.\n
Parameter
----------
config : string
Full path of the config.yaml file as a string.
trainindex: int, optional
Either (in case uniform = True) indexes which element of TrainingFraction in the config file should be used (note it is a list!).
Alternatively (uniform = False) indexes which folder is dropped, i.e. the first if trainindex=0, the second if trainindex =1, etc.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
net_types: list
Type of networks. Currently resnet_50, resnet_101, resnet_152, mobilenet_v2_1.0,mobilenet_v2_0.75, mobilenet_v2_0.5, and mobilenet_v2_0.35 are supported.
augmenter_types: list
Type of augmenters. Currently "default", "imgaug", "tensorpack", and "deterministic" are supported.
userfeedback: bool, optional
If this is set to false, then all requested train/test splits are created (no matter if they already exist). If you
want to assure that previous splits etc. are not overwritten, then set this to True and you will be asked for each split.
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
Example
--------
>>> deeplabcut.create_training_model_comparison('/analysis/project/reaching-task/config.yaml',num_shuffles=1,net_types=['resnet_50','resnet_152'],augmenter_types=['tensorpack','deterministic'])
Windows:
>>> deeplabcut.create_training_model_comparison('C:\\Users\\Ulf\\looming-task\\config.yaml',num_shuffles=1,net_types=['resnet_50','resnet_152'],augmenter_types=['tensorpack','deterministic'])
--------
"""
# read cfg file
cfg = auxiliaryfunctions.read_config(config)
# create log file
log_file_name = os.path.join(cfg['project_path'],'training_model_comparison.log')
logger = logging.getLogger('training_model_comparison')
if not logger.handlers:
logger = logging.getLogger('training_model_comparison')
hdlr = logging.FileHandler(log_file_name)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.INFO)
else:
pass
largestshuffleindex = get_largestshuffle_index(config)
for shuffle in range(num_shuffles):
trainIndexes, testIndexes=mergeandsplit(config,trainindex=trainindex,uniform=True)
for idx_net,net in enumerate(net_types):
for idx_aug,aug in enumerate(augmenter_types):
get_max_shuffle_idx=(largestshuffleindex+idx_aug+idx_net*len(augmenter_types)+shuffle*len(augmenter_types)*len(net_types))+1 #get shuffle index; starts ith 0 so added 1
log_info = str("Shuffle index:" + str(get_max_shuffle_idx) + ", net_type:"+net +", augmenter_type:"+aug + ", trainsetindex:" +str(trainindex))
create_training_dataset(config,Shuffles=[get_max_shuffle_idx],net_type=net,trainIndexes=trainIndexes,testIndexes=testIndexes,augmenter_type=aug,userfeedback=userfeedback,windows2linux=windows2linux)
logger.info(log_info)
def create_video_with_all_detections(
config, videos, DLCscorername, displayedbodyparts="all", destfolder=None
):
"""
Create a video labeled with all the detections stored in a '*_full.pickle' file.
Parameters
----------
config : str
Absolute path to the config.yaml file
videos : list of str
A list of strings containing the full paths to videos for analysis or a path to the directory,
where all the videos with same extension are stored.
DLCscorername: str
Name of network. E.g. 'DLC_resnet50_project_userMar23shuffle1_50000
displayedbodyparts: list of strings, optional
This selects the body parts that are plotted in the video. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
"""
from deeplabcut.pose_estimation_tensorflow.lib.inferenceutils import (
convertdetectiondict2listoflist,
)
import pickle, re
cfg = auxiliaryfunctions.read_config(config)
for video in videos:
videofolder = os.path.splitext(video)[0]
if destfolder is None:
outputname = "{}_full.mp4".format(videofolder + DLCscorername)
full_pickle = os.path.join(videofolder + DLCscorername + "_full.pickle")
else:
auxiliaryfunctions.attempttomakefolder(destfolder)
outputname = os.path.join(
destfolder, str(Path(video).stem) + DLCscorername + "_full.mp4"
)
full_pickle = os.path.join(
destfolder, str(Path(video).stem) + DLCscorername + "_full.pickle"
)
if not (os.path.isfile(outputname)):
print("Creating labeled video for ", str(Path(video).stem))
with open(full_pickle, "rb") as file:
data = pickle.load(file)
header = data.pop("metadata")
all_jointnames = header["all_joints_names"]
if displayedbodyparts == "all":
numjoints = len(all_jointnames)
bpts = range(numjoints)
else: # select only "displayedbodyparts"
bpts = []
for bptindex, bp in enumerate(all_jointnames):
if bp in displayedbodyparts:
bpts.append(bptindex)
numjoints = len(bpts)
frame_names = list(data)
frames = [int(re.findall(r"\d+", name)[0]) for name in frame_names]
colorclass = plt.cm.ScalarMappable(cmap=cfg["colormap"])
C = colorclass.to_rgba(np.linspace(0, 1, numjoints))
colors = (C[:, :3] * 255).astype(np.uint8)
pcutoff = cfg["pcutoff"]
dotsize = cfg["dotsize"]
clip = vp(fname=video, sname=outputname, codec="mp4v")
ny, nx = clip.height(), clip.width()
for n in trange(clip.nframes):
frame = clip.load_frame()
try:
ind = frames.index(n)
dets = convertdetectiondict2listoflist(data[frame_names[ind]], bpts)
for i, det in enumerate(dets):
color = colors[i]
for x, y, p, _ in det:
if p > pcutoff:
rr, cc = circle(y, x, dotsize, shape=(ny, nx))
frame[rr, cc] = color
except ValueError: # No data stored for that particular frame
print(n, "no data")
pass
try:
clip.save_frame(frame)
except:
print(n, "frame writing error.")
pass
clip.close()
else:
print("Detections already plotted, ", outputname)
def analyzeskeleton(
config,
videos,
videotype="avi",
shuffle=1,
trainingsetindex=0,
filtered=False,
save_as_csv=False,
destfolder=None,
modelprefix="",
track_method="",
):
"""
Extracts length and orientation of each "bone" of the skeleton as defined in the config file.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
shuffle : int, optional
The shufle index of training dataset. The extracted frames will be stored in the labeled-dataset for
the corresponding shuffle of training dataset. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this
folder also needs to be passed.
track_method: string, optional
Specifies the tracker used to generate the data. Empty by default (corresponding to a single animal project).
For multiple animals, must be either 'box', 'skeleton', or 'ellipse' and will be taken from the config.yaml file if none is given.
"""
# Load config file, scorer and videos
cfg = auxiliaryfunctions.read_config(config)
if not cfg["skeleton"]:
raise ValueError("No skeleton defined in the config.yaml.")
track_method = auxfun_multianimal.get_track_method(
cfg, track_method=track_method)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction=cfg["TrainingFraction"][trainingsetindex],
modelprefix=modelprefix,
)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
print("Processing %s" % (video))
if destfolder is None:
destfolder = str(Path(video).parents[0])
vname = Path(video).stem
try:
df, filepath, scorer, _ = auxiliaryfunctions.load_analyzed_data(
destfolder, vname, DLCscorer, filtered, track_method)
output_name = filepath.replace(".h5", f"_skeleton.h5")
if os.path.isfile(output_name):
print(
f"Skeleton in video {vname} already processed. Skipping..."
)
continue
bones = {}
if "individuals" in df.columns.names:
for animal_name, df_ in df.groupby(level="individuals",
axis=1):
temp = df_.droplevel(["scorer", "individuals"], axis=1)
if animal_name != "single":
for bp1, bp2 in cfg["skeleton"]:
name = "{}_{}_{}".format(animal_name, bp1, bp2)
bones[name] = analyzebone(temp[bp1], temp[bp2])
else:
for bp1, bp2 in cfg["skeleton"]:
name = "{}_{}".format(bp1, bp2)
bones[name] = analyzebone(df[scorer][bp1], df[scorer][bp2])
skeleton = pd.concat(bones, axis=1)
skeleton.to_hdf(output_name,
"df_with_missing",
format="table",
mode="w")
if save_as_csv:
skeleton.to_csv(output_name.replace(".h5", ".csv"))
except FileNotFoundError as e:
print(e)
continue
def create_pretrained_project(
project,
experimenter,
videos,
model="full_human",
working_directory=None,
copy_videos=False,
videotype=None,
analyzevideo=True,
filtered=True,
createlabeledvideo=True,
trainFraction=None,
):
"""
Creates a new project directory, sub-directories and a basic configuration file.
Change its parameters to your projects need.
The project will also be initialized with a pre-trained model from the DeepLabCut model zoo!
http://modelzoo.deeplabcut.org
Parameters
----------
project : string
String containing the name of the project.
experimenter : string
String containing the name of the experimenter.
model: string, options see http://www.mousemotorlab.org/dlc-modelzoo
Current option and default: 'full_human' Creates a demo human project and analyzes a video with ResNet 101 weights pretrained on MPII Human Pose. This is from the DeeperCut paper
by Insafutdinov et al. https://arxiv.org/abs/1605.03170 Please make sure to cite it too if you use this code!
videos : list
A list of string containing the full paths of the videos to include in the project.
working_directory : string, optional
The directory where the project will be created. The default is the ``current working directory``; if provided, it must be a string.
copy_videos : bool, optional ON WINDOWS: TRUE is often necessary!
If this is set to True, the videos are copied to the ``videos`` directory. If it is False,symlink of the videos are copied to the project/videos directory. The default is ``False``; if provided it must be either
``True`` or ``False``.
analyzevideo " bool, optional
If true, then the video is analzyed and a labeled video is created. If false, then only the project will be created and the weights downloaded. You can then access them
filtered: bool, default false
Boolean variable indicating if filtered pose data output should be plotted rather than frame-by-frame predictions.
Filtered version can be calculated with deeplabcut.filterpredictions
trainFraction: By default value from *new* projects. (0.95)
Fraction that will be used in dlc-model/trainingset folder name.
Example
--------
Linux/MacOs loading full_human model and analzying video /homosapiens1.avi
>>> deeplabcut.create_pretrained_project('humanstrokestudy','Linus',['/data/videos/homosapiens1.avi'], copy_videos=False)
Loading full_cat model and analzying video "felixfeliscatus3.avi"
>>> deeplabcut.create_pretrained_project('humanstrokestudy','Linus',['/data/videos/felixfeliscatus3.avi'], model='full_cat')
Windows:
>>> deeplabcut.create_pretrained_project('humanstrokestudy','Bill',[r'C:\yourusername\rig-95\Videos\reachingvideo1.avi'],r'C:\yourusername\analysis\project' copy_videos=True)
Users must format paths with either: r'C:\ OR 'C:\\ <- i.e. a double backslash \ \ )
"""
if model in globals()["Modeloptions"]:
cwd = os.getcwd()
cfg = deeplabcut.create_new_project(project, experimenter, videos,
working_directory, copy_videos,
videotype)
if trainFraction is not None:
auxiliaryfunctions.edit_config(
cfg, {"TrainingFraction": [trainFraction]})
config = auxiliaryfunctions.read_config(cfg)
if model == "full_human":
config["bodyparts"] = [
"ankle1",
"knee1",
"hip1",
"hip2",
"knee2",
"ankle2",
"wrist1",
"elbow1",
"shoulder1",
"shoulder2",
"elbow2",
"wrist2",
"chin",
"forehead",
]
config["skeleton"] = [
["ankle1", "knee1"],
["ankle2", "knee2"],
["knee1", "hip1"],
["knee2", "hip2"],
["hip1", "hip2"],
["shoulder1", "shoulder2"],
["shoulder1", "hip1"],
["shoulder2", "hip2"],
["shoulder1", "elbow1"],
["shoulder2", "elbow2"],
["chin", "forehead"],
["elbow1", "wrist1"],
["elbow2", "wrist2"],
]
config["default_net_type"] = "resnet_101"
else: # just make a case and put the stuff you want.
# TBD: 'partaffinityfield_graph' >> use to set skeleton!
pass
auxiliaryfunctions.write_config(cfg, config)
config = auxiliaryfunctions.read_config(cfg)
train_dir = Path(
os.path.join(
config["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction=config["TrainingFraction"][0],
shuffle=1,
cfg=config,
)),
"train",
))
test_dir = Path(
os.path.join(
config["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction=config["TrainingFraction"][0],
shuffle=1,
cfg=config,
)),
"test",
))
# Create the model directory
train_dir.mkdir(parents=True, exist_ok=True)
test_dir.mkdir(parents=True, exist_ok=True)
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction=config["TrainingFraction"][0], shuffle=1, cfg=config)
path_train_config = str(
os.path.join(config["project_path"], Path(modelfoldername),
"train", "pose_cfg.yaml"))
path_test_config = str(
os.path.join(config["project_path"], Path(modelfoldername), "test",
"pose_cfg.yaml"))
# Download the weights and put then in appropriate directory
print("Dowloading weights...")
auxfun_models.DownloadModel(model, train_dir)
pose_cfg = deeplabcut.auxiliaryfunctions.read_plainconfig(
path_train_config)
print(path_train_config)
# Updating config file:
dict_ = {
"default_net_type": pose_cfg["net_type"],
"default_augmenter": pose_cfg["dataset_type"],
"bodyparts": pose_cfg["all_joints_names"],
"dotsize": 6,
}
auxiliaryfunctions.edit_config(cfg, dict_)
# downloading base encoder / not required unless on re-trains (but when a training set is created this happens anyway)
# model_path, num_shuffles=auxfun_models.Check4weights(pose_cfg['net_type'], parent_path, num_shuffles= 1)
# Updating training and test pose_cfg:
snapshotname = [fn for fn in os.listdir(train_dir)
if ".meta" in fn][0].split(".meta")[0]
dict2change = {
"init_weights": str(os.path.join(train_dir, snapshotname)),
"project_path": str(config["project_path"]),
}
UpdateTrain_pose_yaml(pose_cfg, dict2change, path_train_config)
keys2save = [
"dataset",
"dataset_type",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
]
MakeTest_pose_yaml(pose_cfg, keys2save, path_test_config)
video_dir = os.path.join(config["project_path"], "videos")
if analyzevideo == True:
print("Analyzing video...")
deeplabcut.analyze_videos(cfg, [video_dir],
videotype,
save_as_csv=True)
if createlabeledvideo == True:
if filtered:
deeplabcut.filterpredictions(cfg, [video_dir], videotype)
print("Plotting results...")
deeplabcut.create_labeled_video(cfg, [video_dir],
videotype,
draw_skeleton=True,
filtered=filtered)
deeplabcut.plot_trajectories(cfg, [video_dir],
videotype,
filtered=filtered)
os.chdir(cwd)
return cfg, path_train_config
else:
return "N/A", "N/A"
def analyze_videos(config,
videos,
videotype='avi',
shuffle=1,
trainingsetindex=0,
gputouse=None,
save_as_csv=False,
destfolder=None,
batchsize=None,
crop=None,
get_nframesfrommetadata=True,
TFGPUinference=True,
dynamic=(False, .5, 10)):
"""
Makes prediction based on a trained network. The index of the trained network is specified by parameters in the config file (in particular the variable 'snapshotindex')
Output: The labels are stored as MultiIndex Pandas Array, which contains the name of the network, body part name, (x, y) label position \n
in pixels, and the likelihood for each frame per body part. These arrays are stored in an efficient Hierarchical Data Format (HDF) \n
in the same directory, where the video is stored. However, if the flag save_as_csv is set to True, the data can also be exported in \n
comma-separated values format (.csv), which in turn can be imported in many programs, such as MATLAB, R, Prism, etc.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: int, optional
An integer specifying the shuffle index of the training dataset used for training the network. The default is 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this
folder also needs to be passed.
batchsize: int, default from pose_cfg.yaml
Change batch size for inference; if given overwrites value in pose_cfg.yaml
crop: list, optional (default=None)
List of cropping coordinates as [x1, x2, y1, y2].
Note that the same cropping parameters will then be used for all videos.
If different video crops are desired, run 'analyze_videos' on individual videos
with the corresponding cropping coordinates.
TFGPUinference: bool, default: True
Perform inference on GPU with Tensorflow code. Introduced in "Pretraining boosts out-of-domain robustness for pose estimation" by
Alexander Mathis, Mert Yüksekgönül, Byron Rogers, Matthias Bethge, Mackenzie W. Mathis Source: https://arxiv.org/abs/1909.11229
dynamic: triple containing (state,detectiontreshold,margin)
If the state is true, then dynamic cropping will be performed. That means that if an object is detected (i.e. any body part > detectiontreshold),
then object boundaries are computed according to the smallest/largest x position and smallest/largest y position of all body parts. This window is
expanded by the margin and from then on only the posture within this crop is analyzed (until the object is lost, i.e. <detectiontreshold). The
current position is utilized for updating the crop window for the next frame (this is why the margin is important and should be set large
enough given the movement of the animal).
Examples
--------
Windows example for analyzing 1 video
>>> deeplabcut.analyze_videos('C:\\myproject\\reaching-task\\config.yaml',['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi'])
--------
If you want to analyze only 1 video
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to analyze all videos of type avi in a folder:
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos'],videotype='.avi')
--------
If you want to analyze multiple videos
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to analyze multiple videos with shuffle = 2
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'],shuffle=2)
--------
If you want to analyze multiple videos with shuffle = 2 and save results as an additional csv file too
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'],shuffle=2,save_as_csv=True)
--------
"""
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ[
'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
if gputouse is not None: #gpu selection
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
tf.reset_default_graph()
start_path = os.getcwd(
) #record cwd to return to this directory in the end
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
if crop is not None:
cfg['cropping'] = True
cfg['x1'], cfg['x2'], cfg['y1'], cfg['y2'] = crop
print("Overwriting cropping parameters:", crop)
print(
"These are used for all videos, but won't be save to the cfg file."
)
modelfolder = os.path.join(
cfg["project_path"],
str(auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# Check which snapshots are available and sort them by # iterations
try:
Snapshots = np.array([
fn.split('.')[0]
for fn in os.listdir(os.path.join(modelfolder, 'train'))
if "index" in fn
])
except FileNotFoundError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s."
% (shuffle, shuffle))
if cfg['snapshotindex'] == 'all':
print(
"Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!"
)
snapshotindex = -1
else:
snapshotindex = cfg['snapshotindex']
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder)
##################################################
# Load and setup CNN part detector
##################################################
# Check if data already was generated:
dlc_cfg['init_weights'] = os.path.join(modelfolder, 'train',
Snapshots[snapshotindex])
trainingsiterations = (dlc_cfg['init_weights'].split(
os.sep)[-1]).split('-')[-1]
# Update number of output and batchsize
dlc_cfg['num_outputs'] = cfg.get('num_outputs',
dlc_cfg.get('num_outputs', 1))
if batchsize == None:
#update batchsize (based on parameters in config.yaml)
dlc_cfg['batch_size'] = cfg['batch_size']
else:
dlc_cfg['batch_size'] = batchsize
cfg['batch_size'] = batchsize
if dynamic[0]: #state=true
#(state,detectiontreshold,margin)=dynamic
print("Starting analysis in dynamic cropping mode with parameters:",
dynamic)
dlc_cfg['num_outputs'] = 1
TFGPUinference = False
dlc_cfg['batch_size'] = 1
print(
"Switching batchsize to 1, num_outputs (per animal) to 1 and TFGPUinference to False (all these features are not supported in this mode)."
)
# Name for scorer:
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, trainingsiterations=trainingsiterations)
if dlc_cfg['num_outputs'] > 1:
if TFGPUinference:
print(
"Switching to numpy-based keypoint extraction code, as multiple point extraction is not supported by TF code currently."
)
TFGPUinference = False
print("Extracting ", dlc_cfg['num_outputs'], "instances per bodypart")
xyz_labs_orig = ['x', 'y', 'likelihood']
suffix = [str(s + 1) for s in range(dlc_cfg['num_outputs'])]
suffix[
0] = '' # first one has empty suffix for backwards compatibility
xyz_labs = [x + s for s in suffix for x in xyz_labs_orig]
else:
xyz_labs = ['x', 'y', 'likelihood']
#sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
if TFGPUinference:
sess, inputs, outputs = predict.setup_GPUpose_prediction(dlc_cfg)
else:
sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
pdindex = pd.MultiIndex.from_product(
[[DLCscorer], dlc_cfg['all_joints_names'], xyz_labs],
names=['scorer', 'bodyparts', 'coords'])
##################################################
# Datafolder
##################################################
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
if len(Videos) > 0:
#looping over videos
for video in Videos:
DLCscorer = AnalyzeVideo(video, DLCscorer, DLCscorerlegacy,
trainFraction, cfg, dlc_cfg, sess, inputs,
outputs, pdindex, save_as_csv, destfolder,
TFGPUinference, dynamic)
os.chdir(str(start_path))
print(
"The videos are analyzed. Now your research can truly start! \n You can create labeled videos with 'create_labeled_video'."
)
print(
"If the tracking is not satisfactory for some videos, consider expanding the training set. You can use the function 'extract_outlier_frames' to extract any outlier frames!"
)
return DLCscorer #note: this is either DLCscorer or DLCscorerlegacy depending on what was used!
else:
os.chdir(str(start_path))
print("No video/s found. Please check your path!")
return DLCscorer
def create_training_dataset(config,num_shuffles=1,Shuffles=None,windows2linux=False):
"""
Creates a training dataset. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL]Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
Example
--------
>>> deeplabcut.create_training_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcut.create_training_dataset('C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
from skimage import io
import scipy.io as sio
import deeplabcut
import subprocess
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg['scorer']
project_path = cfg['project_path']
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) #Path concatenation OS platform independent
auxiliaryfunctions.attempttomakefolder(Path(os.path.join(project_path,str(trainingsetfolder))),recursive=True)
Data = merge_annotateddatasets(cfg,project_path,Path(os.path.join(project_path,trainingsetfolder)),windows2linux)
Data = Data[scorer] #extract labeled data
#set model type. we will allow more in the future.
if cfg['resnet']==50:
net_type ='resnet_'+str(cfg['resnet'])
resnet_path = str(Path(deeplabcut.__file__).parents[0] / 'pose_estimation_tensorflow/models/pretrained/resnet_v1_50.ckpt')
elif cfg['resnet']==101:
net_type ='resnet_'+str(cfg['resnet'])
resnet_path = str(Path(deeplabcut.__file__).parents[0] / 'Pose_Estimation_Tensorflow/models/pretrained/resnet_v1_101.ckpt')
else:
print("Currently only ResNet 50 or 101 supported, please change 'resnet' entry in config.yaml!")
num_shuffles=-1 #thus the loop below is empty...
if not Path(resnet_path).is_file():
"""
Downloads the ImageNet pretrained weights for ResNet.
"""
start = os.getcwd()
os.chdir(str(Path(resnet_path).parents[0]))
print("Downloading the pretrained model (ResNets)....")
subprocess.call("download.sh", shell=True)
os.chdir(start)
if Shuffles==None:
Shuffles=range(1,num_shuffles+1,1)
else:
Shuffles=[i for i in Shuffles if isinstance(i,int)]
bodyparts = cfg['bodyparts']
TrainingFraction = cfg['TrainingFraction']
for shuffle in Shuffles: # Creating shuffles starting from 1
for trainFraction in TrainingFraction:
trainIndexes, testIndexes = SplitTrials(range(len(Data.index)), trainFraction)
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
# Make matlab train file!
data = []
for jj in trainIndexes:
H = {}
# load image to get dimensions:
filename = Data.index[jj]
im = io.imread(os.path.join(cfg['project_path'],filename))
H['image'] = filename
try:
H['size'] = np.array(
[np.shape(im)[2],
np.shape(im)[0],
np.shape(im)[1]])
except:
# print "Grayscale!"
H['size'] = np.array([1, np.shape(im)[0], np.shape(im)[1]])
indexjoints=0
joints=np.zeros((len(bodyparts),3))*np.nan
for bpindex,bodypart in enumerate(bodyparts):
if Data[bodypart]['x'][jj]<np.shape(im)[1] and Data[bodypart]['y'][jj]<np.shape(im)[0]: #are labels in image?
joints[indexjoints,0]=int(bpindex)
joints[indexjoints,1]=Data[bodypart]['x'][jj]
joints[indexjoints,2]=Data[bodypart]['y'][jj]
indexjoints+=1
joints = joints[np.where(
np.prod(np.isfinite(joints),
1))[0], :] # drop NaN, i.e. lines for missing body parts
assert (np.prod(np.array(joints[:, 2]) < np.shape(im)[0])
) # y coordinate within image?
assert (np.prod(np.array(joints[:, 1]) < np.shape(im)[1])
) # x coordinate within image?
H['joints'] = np.array(joints, dtype=int)
if np.size(joints)>0: #exclude images without labels
data.append(H)
if len(trainIndexes)>0:
datafilename,metadatafilename=auxiliaryfunctions.GetDataandMetaDataFilenames(trainingsetfolder,trainFraction,shuffle,cfg)
################################################################################
# Saving metadata (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(os.path.join(project_path,metadatafilename),data, trainIndexes, testIndexes, trainFraction)
################################################################################
# Saving data file (convert to training file for deeper cut (*.mat))
################################################################################
DTYPE = [('image', 'O'), ('size', 'O'), ('joints', 'O')]
MatlabData = np.array(
[(np.array([data[item]['image']], dtype='U'),
np.array([data[item]['size']]),
boxitintoacell(data[item]['joints']))
for item in range(len(data))],
dtype=DTYPE)
sio.savemat(os.path.join(project_path,datafilename), {'dataset': MatlabData})
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername=auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg)
auxiliaryfunctions.attempttomakefolder(Path(config).parents[0] / modelfoldername,recursive=True)
auxiliaryfunctions.attempttomakefolder(str(Path(config).parents[0] / modelfoldername)+ '/'+ '/train')
auxiliaryfunctions.attempttomakefolder(str(Path(config).parents[0] / modelfoldername)+ '/'+ '/test')
path_train_config = str(os.path.join(cfg['project_path'],Path(modelfoldername),'train','pose_cfg.yaml'))
path_test_config = str(os.path.join(cfg['project_path'],Path(modelfoldername),'test','pose_cfg.yaml'))
#str(cfg['proj_path']+'/'+Path(modelfoldername) / 'test' / 'pose_cfg.yaml')
items2change = {
"dataset": datafilename,
"metadataset": metadatafilename,
"num_joints": len(bodyparts),
"all_joints": [[i] for i in range(len(bodyparts))],
"all_joints_names": bodyparts,
"init_weights": resnet_path,
"project_path": cfg['project_path'],
"net_type": net_type
}
defaultconfigfile = str(Path(deeplabcut.__file__).parents[0] / 'pose_cfg.yaml')
trainingdata = MakeTrain_pose_yaml(items2change,path_train_config,defaultconfigfile)
keys2save = [
"dataset", "num_joints", "all_joints", "all_joints_names",
"net_type", 'init_weights', 'global_scale', 'location_refinement',
'locref_stdev'
]
MakeTest_pose_yaml(trainingdata, keys2save,path_test_config)
print("The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!")
else:
pass
def __init__(self, parent, gui_size, cfg):
"""Constructor"""
wx.Panel.__init__(self, parent=parent)
# variable initialization
self.config = cfg
self.bodyparts = []
# design the panel
self.sizer = wx.GridBagSizer(5, 5)
text = wx.StaticText(self,
label="DeepLabCut - Step 6. Evaluate Network")
self.sizer.Add(text,
pos=(0, 0),
flag=wx.TOP | wx.LEFT | wx.BOTTOM,
border=15)
# Add logo of DLC
icon = wx.StaticBitmap(self, bitmap=wx.Bitmap(LOGO_PATH))
self.sizer.Add(icon,
pos=(0, 4),
flag=wx.TOP | wx.RIGHT | wx.ALIGN_RIGHT,
border=5)
line1 = wx.StaticLine(self)
self.sizer.Add(line1,
pos=(1, 0),
span=(1, 5),
flag=wx.EXPAND | wx.BOTTOM,
border=10)
self.cfg_text = wx.StaticText(self, label="Select the config file")
self.sizer.Add(self.cfg_text,
pos=(2, 0),
flag=wx.TOP | wx.LEFT,
border=5)
if sys.platform == "darwin":
self.sel_config = wx.FilePickerCtrl(
self,
path="",
style=wx.FLP_USE_TEXTCTRL,
message="Choose the config.yaml file",
wildcard="*.yaml",
)
else:
self.sel_config = wx.FilePickerCtrl(
self,
path="",
style=wx.FLP_USE_TEXTCTRL,
message="Choose the config.yaml file",
wildcard="config.yaml",
)
# self.sel_config = wx.FilePickerCtrl(self, path="",style=wx.FLP_USE_TEXTCTRL,message="Choose the config.yaml file", wildcard="config.yaml")
self.sizer.Add(self.sel_config,
pos=(2, 1),
span=(1, 3),
flag=wx.TOP | wx.EXPAND,
border=5)
self.sel_config.SetPath(self.config)
self.sel_config.Bind(wx.EVT_FILEPICKER_CHANGED, self.select_config)
sb = wx.StaticBox(self, label="Attributes")
boxsizer = wx.StaticBoxSizer(sb, wx.VERTICAL)
self.hbox1 = wx.BoxSizer(wx.HORIZONTAL)
self.hbox2 = wx.BoxSizer(wx.HORIZONTAL)
self.hbox3 = wx.BoxSizer(wx.HORIZONTAL)
config_file = auxiliaryfunctions.read_config(self.config)
shuffles_text = wx.StaticBox(self, label="Specify the shuffle")
shuffles_text_boxsizer = wx.StaticBoxSizer(shuffles_text, wx.VERTICAL)
self.shuffles = wx.SpinCtrl(self, value="1", min=0, max=100)
shuffles_text_boxsizer.Add(self.shuffles, 1,
wx.EXPAND | wx.TOP | wx.BOTTOM, 10)
#trainingset = wx.StaticBox(self, label="Specify the trainingset index")
#trainingset_boxsizer = wx.StaticBoxSizer(trainingset, wx.VERTICAL)
#self.trainingset = wx.SpinCtrl(self, value="0", min=0, max=100)
#trainingset_boxsizer.Add(
# self.trainingset, 1, wx.EXPAND | wx.TOP | wx.BOTTOM, 10
#)
self.plot_choice = wx.RadioBox(
self,
label="Want to plot predictions (as in standard DLC projects)?",
choices=["Yes", "No"],
majorDimension=1,
style=wx.RA_SPECIFY_COLS,
)
self.plot_choice.SetSelection(0)
self.plot_scoremaps = wx.RadioBox(
self,
label="Want to plot maps (ALL images): scoremaps, PAFs, locrefs?",
choices=["Yes", "No"],
majorDimension=1,
style=wx.RA_SPECIFY_COLS,
)
self.plot_scoremaps.SetSelection(1)
self.bodypart_choice = wx.RadioBox(
self,
label="Compare all bodyparts?",
choices=["Yes", "No"],
majorDimension=1,
style=wx.RA_SPECIFY_COLS,
)
self.bodypart_choice.Bind(wx.EVT_RADIOBOX, self.chooseOption)
if config_file.get("multianimalproject", False):
bodyparts = config_file["multianimalbodyparts"]
else:
bodyparts = config_file["bodyparts"]
self.bodyparts_to_compare = wx.CheckListBox(
self, choices=bodyparts, style=0, name="Select the bodyparts")
self.bodyparts_to_compare.Bind(wx.EVT_CHECKLISTBOX, self.getbp)
self.bodyparts_to_compare.Hide()
self.hbox1.Add(shuffles_text_boxsizer, 5,
wx.EXPAND | wx.TOP | wx.BOTTOM, 5)
#self.hbox1.Add(trainingset_boxsizer, 5, wx.EXPAND | wx.TOP | wx.BOTTOM, 5)
self.hbox1.Add(self.plot_scoremaps, 5, wx.EXPAND | wx.TOP | wx.BOTTOM,
5)
self.hbox2.Add(self.plot_choice, 5, wx.EXPAND | wx.TOP | wx.BOTTOM, 5)
self.hbox2.Add(self.bodypart_choice, 5, wx.EXPAND | wx.TOP | wx.BOTTOM,
5)
self.hbox2.Add(self.bodyparts_to_compare, 10,
wx.EXPAND | wx.TOP | wx.BOTTOM, 5)
boxsizer.Add(self.hbox1, 0, wx.EXPAND | wx.TOP | wx.BOTTOM, 10)
boxsizer.Add(self.hbox2, 5, wx.EXPAND | wx.TOP | wx.BOTTOM, 10)
self.sizer.Add(
boxsizer,
pos=(3, 0),
span=(1, 5),
flag=wx.EXPAND | wx.TOP | wx.LEFT | wx.RIGHT,
border=10,
)
self.help_button = wx.Button(self, label="Help")
self.sizer.Add(self.help_button, pos=(4, 0), flag=wx.LEFT, border=10)
self.help_button.Bind(wx.EVT_BUTTON, self.help_function)
self.ok = wx.Button(self, label="RUN: Evaluate Network")
self.sizer.Add(self.ok, pos=(4, 3))
self.ok.Bind(wx.EVT_BUTTON, self.evaluate_network)
self.ok = wx.Button(self, label="Optional: Plot 3 test maps")
self.sizer.Add(self.ok, pos=(5, 3))
self.ok.Bind(wx.EVT_BUTTON, self.plot_maps)
self.cancel = wx.Button(self, label="Reset")
self.sizer.Add(self.cancel,
pos=(4, 1),
span=(1, 1),
flag=wx.BOTTOM | wx.RIGHT,
border=10)
self.cancel.Bind(wx.EVT_BUTTON, self.cancel_evaluate_network)
if config_file.get("multianimalproject", False):
self.inf_cfg_text = wx.Button(
self, label="Edit the inference_config.yaml")
self.inf_cfg_text.Bind(wx.EVT_BUTTON, self.edit_inf_config)
self.sizer.Add(
self.inf_cfg_text,
pos=(4, 2),
span=(1, 1),
flag=wx.BOTTOM | wx.RIGHT,
border=10,
)
self.sizer.AddGrowableCol(2)
self.SetSizer(self.sizer)
self.sizer.Fit(self)
def extract_frames(
config,
mode="automatic",
algo="kmeans",
crop=False,
userfeedback=True,
cluster_step=1,
cluster_resizewidth=30,
cluster_color=False,
opencv=True,
slider_width=25,
):
"""
Extracts frames from the videos in the config.yaml file. Only the videos in the config.yaml will be used to select the frames.\n
Use the function ``add_new_video`` at any stage of the project to add new videos to the config file and extract their frames.
The provided function either selects frames from the videos in a randomly and temporally uniformly distributed way (uniform), \n
by clustering based on visual appearance (k-means), or by manual selection.
Three important parameters for automatic extraction: numframes2pick, start and stop are set in the config file.
Please refer to the user guide for more details on methods and parameters https://www.nature.com/articles/s41596-019-0176-0
or the preprint: https://www.biorxiv.org/content/biorxiv/early/2018/11/24/476531.full.pdf
Parameters
----------
config : string
Full path of the config.yaml file as a string.
mode : string
String containing the mode of extraction. It must be either ``automatic`` or ``manual``.
algo : string
String specifying the algorithm to use for selecting the frames. Currently, deeplabcut supports either ``kmeans`` or ``uniform`` based selection. This flag is
only required for ``automatic`` mode and the default is ``uniform``. For uniform, frames are picked in temporally uniform way, kmeans performs clustering on downsampled frames (see user guide for details).
Note: color information is discarded for kmeans, thus e.g. for camouflaged octopus clustering one might want to change this.
crop : bool, optional
If True, video frames are cropped according to the corresponding coordinates stored in the config.yaml.
Alternatively, if cropping coordinates are not known yet, crop='GUI' triggers a user interface
where the cropping area can be manually drawn and saved.
userfeedback: bool, optional
If this is set to false during automatic mode then frames for all videos are extracted. The user can set this to true, which will result in a dialog,
where the user is asked for each video if (additional/any) frames from this video should be extracted. Use this, e.g. if you have already labeled
some folders and want to extract data for new videos.
cluster_resizewidth: number, default: 30
For k-means one can change the width to which the images are downsampled (aspect ratio is fixed).
cluster_step: number, default: 1
By default each frame is used for clustering, but for long videos one could only use every nth frame (set by: cluster_step). This saves memory before clustering can start, however,
reading the individual frames takes longer due to the skipping.
cluster_color: bool, default: False
If false then each downsampled image is treated as a grayscale vector (discarding color information). If true, then the color channels are considered. This increases
the computational complexity.
opencv: bool, default: True
Uses openCV for loading & extractiong (otherwise moviepy (legacy))
slider_width: number, default: 25
Width of the video frames slider, in percent of window
Examples
--------
for selecting frames automatically with 'kmeans' and want to crop the frames.
>>> deeplabcut.extract_frames('/analysis/project/reaching-task/config.yaml','automatic','kmeans',True)
--------
for selecting frames automatically with 'kmeans' and defining the cropping area at runtime.
>>> deeplabcut.extract_frames('/analysis/project/reaching-task/config.yaml','automatic','kmeans','GUI')
--------
for selecting frames automatically with 'kmeans' and considering the color information.
>>> deeplabcut.extract_frames('/analysis/project/reaching-task/config.yaml','automatic','kmeans',cluster_color=True)
--------
for selecting frames automatically with 'uniform' and want to crop the frames.
>>> deeplabcut.extract_frames('/analysis/project/reaching-task/config.yaml','automatic',crop=True)
--------
for selecting frames manually,
>>> deeplabcut.extract_frames('/analysis/project/reaching-task/config.yaml','manual')
--------
for selecting frames manually, with a 60% wide frames slider
>>> deeplabcut.extract_frames('/analysis/project/reaching-task/config.yaml','manual', slider_width=60)
While selecting the frames manually, you do not need to specify the ``crop`` parameter in the command. Rather, you will get a prompt in the graphic user interface to choose
if you need to crop or not.
--------
"""
import os
import sys
import numpy as np
from pathlib import Path
from skimage import io
from skimage.util import img_as_ubyte
from deeplabcut.utils import frameselectiontools
from deeplabcut.utils import auxiliaryfunctions
if mode == "manual":
wd = Path(config).resolve().parents[0]
os.chdir(str(wd))
from deeplabcut.generate_training_dataset import frame_extraction_toolbox
from deeplabcut.utils import select_crop_parameters
frame_extraction_toolbox.show(config, slider_width)
elif mode == "automatic":
config_file = Path(config).resolve()
cfg = auxiliaryfunctions.read_config(config_file)
print("Config file read successfully.")
numframes2pick = cfg["numframes2pick"]
start = cfg["start"]
stop = cfg["stop"]
# Check for variable correctness
if start > 1 or stop > 1 or start < 0 or stop < 0 or start >= stop:
raise Exception(
"Erroneous start or stop values. Please correct it in the config file."
)
if numframes2pick < 1 and not int(numframes2pick):
raise Exception(
"Perhaps consider extracting more, or a natural number of frames."
)
videos = cfg["video_sets"].keys()
if opencv:
import cv2
else:
from moviepy.editor import VideoFileClip
has_failed = []
for vindex, video in enumerate(videos):
if userfeedback:
print(
"Do you want to extract (perhaps additional) frames for video:",
video,
"?",
)
askuser = input("yes/no")
else:
askuser = "******"
if (askuser == "y" or askuser == "yes" or askuser == "Ja"
or askuser == "ha" or askuser == "oui"
or askuser == "ouais"): # multilanguage support :)
if opencv:
cap = cv2.VideoCapture(video)
fps = cap.get(
5
) # https://docs.opencv.org/2.4/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get
nframes = int(cap.get(7))
else:
# Moviepy:
clip = VideoFileClip(video)
fps = clip.fps
nframes = int(np.ceil(clip.duration * 1.0 / fps))
if not nframes:
print("Video could not be opened. Skipping...")
continue
indexlength = int(np.ceil(np.log10(nframes)))
fname = Path(video)
output_path = Path(
config).parents[0] / "labeled-data" / fname.stem
if output_path.exists():
if len(os.listdir(output_path)):
if userfeedback:
askuser = input(
"The directory already contains some frames. Do you want to add to it?(yes/no): "
)
if not (askuser == "y" or askuser == "yes"
or askuser == "Y" or askuser == "Yes"):
sys.exit("Delete the frames and try again later!")
if crop == "GUI":
cfg = select_cropping_area(config, [video])
coords = cfg["video_sets"][video]["crop"].split(",")
if crop and not opencv:
clip = clip.crop(
y1=int(coords[2]),
y2=int(coords[3]),
x1=int(coords[0]),
x2=int(coords[1]),
)
elif not crop:
coords = None
print("Extracting frames based on %s ..." % algo)
if algo == "uniform":
if opencv:
frames2pick = frameselectiontools.UniformFramescv2(
cap, numframes2pick, start, stop)
else:
frames2pick = frameselectiontools.UniformFrames(
clip, numframes2pick, start, stop)
elif algo == "kmeans":
if opencv:
frames2pick = frameselectiontools.KmeansbasedFrameselectioncv2(
cap,
numframes2pick,
start,
stop,
crop,
coords,
step=cluster_step,
resizewidth=cluster_resizewidth,
color=cluster_color,
)
else:
frames2pick = frameselectiontools.KmeansbasedFrameselection(
clip,
numframes2pick,
start,
stop,
step=cluster_step,
resizewidth=cluster_resizewidth,
color=cluster_color,
)
else:
print(
"Please implement this method yourself and send us a pull request! Otherwise, choose 'uniform' or 'kmeans'."
)
frames2pick = []
if not len(frames2pick):
print("Frame selection failed...")
return
output_path = (Path(config).parents[0] / "labeled-data" /
Path(video).stem)
is_valid = []
if opencv:
for index in frames2pick:
cap.set(1, index) # extract a particular frame
ret, frame = cap.read()
if ret:
image = img_as_ubyte(
cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
img_name = (str(output_path) + "/img" +
str(index).zfill(indexlength) + ".png")
if crop:
io.imsave(
img_name,
image[int(coords[2]):int(coords[3]),
int(coords[0]):int(coords[1]), :, ],
) # y1 = int(coords[2]),y2 = int(coords[3]),x1 = int(coords[0]), x2 = int(coords[1]
else:
io.imsave(img_name, image)
is_valid.append(True)
else:
print("Frame", index, " not found!")
is_valid.append(False)
cap.release()
else:
for index in frames2pick:
try:
image = img_as_ubyte(
clip.get_frame(index * 1.0 / clip.fps))
img_name = (str(output_path) + "/img" +
str(index).zfill(indexlength) + ".png")
io.imsave(img_name, image)
if np.var(image) == 0: # constant image
print(
"Seems like black/constant images are extracted from your video. Perhaps consider using opencv under the hood, by setting: opencv=True"
)
is_valid.append(True)
except FileNotFoundError:
print("Frame # ", index, " does not exist.")
is_valid.append(False)
clip.close()
del clip
if not any(is_valid):
has_failed.append(True)
else:
has_failed.append(False)
if all(has_failed):
print("Frame extraction failed. Video files must be corrupted.")
return
elif any(has_failed):
print("Although most frames were extracted, some were invalid.")
else:
print("Frames were successfully extracted.")
print(
"\nYou can now label the frames using the function 'label_frames' "
"(if you extracted enough frames for all videos).")
else:
print(
"Invalid MODE. Choose either 'manual' or 'automatic'. Check ``help(deeplabcut.extract_frames)`` on python and ``deeplabcut.extract_frames?`` \
for ipython/jupyter notebook for more details.")
def create_labeled_video(config,
videos,
shuffle=1,
trainingsetindex=0,
videotype='avi',
save_frames=False,
delete=False,
displayedbodyparts='all',
codec='mp4v'):
"""
Labels the bodyparts in a video. Make sure the video is already analyzed by the function 'analyze_video'
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of string containing the full paths of the videos to analyze.
shuffle : int, optional
Number of shuffles of training dataset. Default is set to 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
videotype: string, optional
Checks for the extension of the video in case the input is a directory.\nOnly videos with this extension are analyzed. The default is ``.avi``
save_frames: bool
If true creates each frame individual and then combines into a video. This variant is relatively slow as
it stores all individual frames. However, it uses matplotlib to create the frames and is therefore much more flexible (one can set transparency of markers, crop, and easily customize).
delete: bool
If true then the individual frames created during the video generation will be deleted.
displayedbodyparts: list of strings, optional
This select the body parts that are plotted in the video. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
codec: codec for labeled video. Options see http://www.fourcc.org/codecs.php [depends on your ffmpeg installation.]
Examples
--------
If you want to create the labeled video for only 1 video
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to create the labeled video for only 1 video and store the individual frames
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'],save_frames=True)
--------
If you want to create the labeled video for multiple videos
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to create the labeled video for all the videos (as .avi extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'])
--------
If you want to create the labeled video for all the videos (as .mp4 extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'],videotype='mp4')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
DLCscorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction
) #automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
if [os.path.isdir(i) for i in videos] == [True]:
print("Analyzing all the videos in the directory")
videofolder = videos[0]
os.chdir(videofolder)
Videos = np.sort(
[fn for fn in os.listdir(os.curdir) if (videotype in fn)])
print("Starting ", videofolder, Videos)
else:
Videos = videos
for video in Videos:
videofolder = Path(video).parents[
0] #where your folder with videos is.
#os.chdir(str(videofolder))
videotype = Path(video).suffix
print("Starting % ", videofolder, videos)
vname = str(Path(video).stem)
if os.path.isfile(
os.path.join(str(videofolder),
vname + DLCscorer + '_labeled.mp4')):
print("Labeled video already created.")
else:
print("Loading ", video, "and data.")
dataname = os.path.join(str(videofolder),
vname + DLCscorer + '.h5')
try:
Dataframe = pd.read_hdf(dataname)
metadata = auxiliaryfunctions.LoadVideoMetadata(dataname)
#print(metadata)
datanames = [dataname]
except FileNotFoundError:
datanames = [
fn for fn in os.listdir(os.curdir)
if (vname in fn) and (".h5" in fn) and "resnet" in fn
]
if len(datanames) == 0:
print("The video was not analyzed with this scorer:",
DLCscorer)
print(
"No other scorers were found, please use the function 'analyze_videos' first."
)
elif len(datanames) > 0:
print("The video was not analyzed with this scorer:",
DLCscorer)
print("Other scorers were found, however:", datanames)
DLCscorer = 'DeepCut' + (
datanames[0].split('DeepCut')[1]).split('.h5')[0]
print("Creating labeled video for:", DLCscorer,
" instead.")
Dataframe = pd.read_hdf(datanames[0])
metadata = auxiliaryfunctions.LoadVideoMetadata(
datanames[0])
if len(datanames) > 0:
#Loading cropping data used during analysis
cropping = metadata['data']["cropping"]
[x1, x2, y1, y2] = metadata['data']["cropping_parameters"]
print(cropping, x1, x2, y1, y2)
if save_frames == True:
tmpfolder = os.path.join(str(videofolder), 'temp-' + vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder)
clip = vp(video)
#CreateVideoSlow(clip,Dataframe,tmpfolder,cfg["dotsize"],cfg["colormap"],cfg["alphavalue"],cfg["pcutoff"],cfg["cropping"],cfg["x1"],cfg["x2"],cfg["y1"],cfg["y2"],delete,DLCscorer,bodyparts)
CreateVideoSlow(clip, Dataframe, tmpfolder, cfg["dotsize"],
cfg["colormap"], cfg["alphavalue"],
cfg["pcutoff"], cropping, x1, x2, y1, y2,
delete, DLCscorer, bodyparts)
else:
clip = vp(fname=video,
sname=os.path.join(vname + DLCscorer +
'_labeled.mp4'),
codec=codec)
if cropping:
print(
"Fast video creation has currently not been implemented for cropped videos. Please use 'save_frames=True' to get the video."
)
else:
CreateVideo(clip, Dataframe, cfg["pcutoff"],
cfg["dotsize"], cfg["colormap"], DLCscorer,
bodyparts, cropping, x1, x2, y1,
y2) #NEED TO ADD CROPPING!
def triangulate(
config,
video_path,
videotype="avi",
filterpredictions=True,
filtertype="median",
gputouse=None,
destfolder=None,
save_as_csv=False,
):
"""
This function triangulates the detected DLC-keypoints from the two camera views
using the camera matrices (derived from calibration) to calculate 3D predictions.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
video_path : string/list of list
Full path of the directory where videos are saved. If the user wants to analyze
only a pair of videos, the user needs to pass them as a list of list of videos,
i.e. [['video1-camera-1.avi','video1-camera-2.avi']]
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n
Only videos with this extension are analyzed. The default is ``.avi``
filterpredictions: Bool, optional
Filter the predictions with filter specified by "filtertype". If specified it
should be either ``True`` or ``False``.
filtertype: string
Select which filter, 'arima' or 'median' filter (currently supported).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi).
If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video)
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``
Example
-------
Linux/MacOS
To analyze all the videos in the directory:
>>> deeplabcut.triangulate(config,'/data/project1/videos/')
To analyze only a few pairs of videos:
>>> deeplabcut.triangulate(config,[['/data/project1/videos/video1-camera-1.avi','/data/project1/videos/video1-camera-2.avi'],['/data/project1/videos/video2-camera-1.avi','/data/project1/videos/video2-camera-2.avi']])
Windows
To analyze all the videos in the directory:
>>> deeplabcut.triangulate(config,'C:\\yourusername\\rig-95\\Videos')
To analyze only a few pair of videos:
>>> deeplabcut.triangulate(config,[['C:\\yourusername\\rig-95\\Videos\\video1-camera-1.avi','C:\\yourusername\\rig-95\\Videos\\video1-camera-2.avi'],['C:\\yourusername\\rig-95\\Videos\\video2-camera-1.avi','C:\\yourusername\\rig-95\\Videos\\video2-camera-2.avi']])
"""
from deeplabcut.pose_estimation_tensorflow import predict_videos
from deeplabcut.post_processing import filtering
cfg_3d = auxiliaryfunctions.read_config(config)
cam_names = cfg_3d["camera_names"]
pcutoff = cfg_3d["pcutoff"]
scorer_3d = cfg_3d["scorername_3d"]
snapshots = {}
for cam in cam_names:
snapshots[cam] = cfg_3d[str("config_file_" + cam)]
# Check if the config file exists
if not os.path.exists(snapshots[cam]):
raise Exception(
str("It seems the file specified in the variable config_file_"
+ str(cam)) +
" does not exist. Please edit the config file with correct file path and retry."
)
# flag to check if the video_path variable is a string or a list of list
flag = False # assumes that video path is a list
if isinstance(video_path, str) == True:
flag = True
video_list = auxiliaryfunctions_3d.get_camerawise_videos(
video_path, cam_names, videotype=videotype)
else:
video_list = video_path
if video_list == []:
print("No videos found in the specified video path.", video_path)
print(
"Please make sure that the video names are specified with correct camera names as entered in the config file or"
)
print(
"perhaps the videotype is distinct from the videos in the path, I was looking for:",
videotype,
)
print("List of pairs:", video_list)
scorer_name = {}
run_triangulate = False
for i in range(len(video_list)):
dataname = []
for j in range(len(video_list[i])): # looping over cameras
if cam_names[j] in video_list[i][j]:
print("Analyzing video %s using %s" %
(video_list[i][j], str("config_file_" + cam_names[j])))
config_2d = snapshots[cam_names[j]]
cfg = auxiliaryfunctions.read_config(config_2d)
shuffle = cfg_3d[str("shuffle_" + cam_names[j])]
trainingsetindex = cfg_3d[str("trainingsetindex_" +
cam_names[j])]
trainFraction = cfg["TrainingFraction"][trainingsetindex]
if flag == True:
video = os.path.join(video_path, video_list[i][j])
else:
video_path = str(Path(video_list[i][j]).parents[0])
video = os.path.join(video_path, video_list[i][j])
if destfolder is None:
destfolder = str(Path(video).parents[0])
vname = Path(video).stem
prefix = str(vname).split(cam_names[j])[0]
suffix = str(vname).split(cam_names[j])[-1]
if prefix == "":
pass
elif prefix[-1] == "_" or prefix[-1] == "-":
prefix = prefix[:-1]
if suffix == "":
pass
elif suffix[0] == "_" or suffix[0] == "-":
suffix = suffix[1:]
if prefix == "":
output_file = os.path.join(destfolder, suffix)
else:
if suffix == "":
output_file = os.path.join(destfolder, prefix)
else:
output_file = os.path.join(destfolder,
prefix + "_" + suffix)
output_filename = os.path.join(
output_file + "_" + scorer_3d
) # Check if the videos are already analyzed for 3d
if os.path.isfile(output_filename + ".h5"):
if save_as_csv is True and not os.path.exists(
output_filename + ".csv"):
# In case user adds save_as_csv is True after triangulating
pd.read_hdf(output_filename + ".h5").to_csv(
str(output_filename + ".csv"))
print(
"Already analyzed...Checking the meta data for any change in the camera matrices and/or scorer names",
vname,
)
pickle_file = str(output_filename + "_meta.pickle")
metadata_ = auxiliaryfunctions_3d.LoadMetadata3d(
pickle_file)
(
img_path,
path_corners,
path_camera_matrix,
path_undistort,
_,
) = auxiliaryfunctions_3d.Foldernames3Dproject(cfg_3d)
path_stereo_file = os.path.join(path_camera_matrix,
"stereo_params.pickle")
stereo_file = auxiliaryfunctions.read_pickle(
path_stereo_file)
cam_pair = str(cam_names[0] + "-" + cam_names[1])
if_video_analyzed = False # variable to keep track if the video was already analyzed
# Check for the camera matrix
for k in metadata_["stereo_matrix"].keys():
if np.all(metadata_["stereo_matrix"][k] ==
stereo_file[cam_pair][k]):
pass
else:
run_triangulate = True
# Check for scorer names in the pickle file of 3d output
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction,
trainingsiterations="unknown")
if (metadata_["scorer_name"][cam_names[j]] == DLCscorer
): # TODO: CHECK FOR BOTH?
if_video_analyzed = True
elif metadata_["scorer_name"][
cam_names[j]] == DLCscorerlegacy:
if_video_analyzed = True
else:
if_video_analyzed = False
run_triangulate = True
if if_video_analyzed:
print("This file is already analyzed!")
dataname.append(
os.path.join(destfolder,
vname + DLCscorer + ".h5"))
scorer_name[cam_names[j]] = DLCscorer
else:
# Analyze video if score name is different
DLCscorer = predict_videos.analyze_videos(
config_2d,
[video],
videotype=videotype,
shuffle=shuffle,
trainingsetindex=trainingsetindex,
gputouse=gputouse,
destfolder=destfolder,
)
scorer_name[cam_names[j]] = DLCscorer
if_video_analyzed = False
run_triangulate = True
if filterpredictions:
filtering.filterpredictions(
config_2d,
[video],
videotype=videotype,
shuffle=shuffle,
trainingsetindex=trainingsetindex,
filtertype=filtertype,
destfolder=destfolder,
)
dataname.append(
os.path.join(destfolder,
vname + DLCscorer + ".h5"))
else: # need to do the whole jam.
DLCscorer = predict_videos.analyze_videos(
config_2d,
[video],
videotype=videotype,
shuffle=shuffle,
trainingsetindex=trainingsetindex,
gputouse=gputouse,
destfolder=destfolder,
)
scorer_name[cam_names[j]] = DLCscorer
run_triangulate = True
print(destfolder, vname, DLCscorer)
if filterpredictions:
filtering.filterpredictions(
config_2d,
[video],
videotype=videotype,
shuffle=shuffle,
trainingsetindex=trainingsetindex,
filtertype=filtertype,
destfolder=destfolder,
)
dataname.append(
os.path.join(destfolder,
vname + DLCscorer + ".h5"))
if run_triangulate:
# if len(dataname)>0:
# undistort points for this pair
print("Undistorting...")
(
dataFrame_camera1_undistort,
dataFrame_camera2_undistort,
stereomatrix,
path_stereo_file,
) = undistort_points(config, dataname,
str(cam_names[0] + "-" + cam_names[1]))
if len(dataFrame_camera1_undistort) != len(
dataFrame_camera2_undistort):
import warnings
warnings.warn(
"The number of frames do not match in the two videos. Please make sure that your videos have same number of frames and then retry! Excluding the extra frames from the longer video."
)
if len(dataFrame_camera1_undistort) > len(
dataFrame_camera2_undistort):
dataFrame_camera1_undistort = dataFrame_camera1_undistort[:len(
dataFrame_camera2_undistort)]
if len(dataFrame_camera2_undistort) > len(
dataFrame_camera1_undistort):
dataFrame_camera2_undistort = dataFrame_camera2_undistort[:len(
dataFrame_camera1_undistort)]
# raise Exception("The number of frames do not match in the two videos. Please make sure that your videos have same number of frames and then retry!")
X_final = []
triangulate = []
scorer_cam1 = dataFrame_camera1_undistort.columns.get_level_values(
0)[0]
scorer_cam2 = dataFrame_camera2_undistort.columns.get_level_values(
0)[0]
df_3d, scorer_3d, bodyparts = auxiliaryfunctions_3d.create_empty_df(
dataFrame_camera1_undistort, scorer_3d, flag="3d")
P1 = stereomatrix["P1"]
P2 = stereomatrix["P2"]
print("Computing the triangulation...")
for bpindex, bp in enumerate(bodyparts):
# Extract the indices of frames where the likelihood of a bodypart for both cameras are less than pvalue
likelihoods = np.array([
dataFrame_camera1_undistort[scorer_cam1][bp]
["likelihood"].values[:],
dataFrame_camera2_undistort[scorer_cam2][bp]
["likelihood"].values[:],
])
likelihoods = likelihoods.T
# Extract frames where likelihood for both the views is less than the pcutoff
low_likelihood_frames = np.any(likelihoods < pcutoff, axis=1)
# low_likelihood_frames = np.all(likelihoods < pcutoff, axis=1)
low_likelihood_frames = np.where(
low_likelihood_frames == True)[0]
points_cam1_undistort = np.array([
dataFrame_camera1_undistort[scorer_cam1][bp]
["x"].values[:],
dataFrame_camera1_undistort[scorer_cam1][bp]
["y"].values[:],
])
points_cam1_undistort = points_cam1_undistort.T
# For cam1 camera: Assign nans to x and y values of a bodypart where the likelihood for is less than pvalue
points_cam1_undistort[low_likelihood_frames] = np.nan, np.nan
points_cam1_undistort = np.expand_dims(points_cam1_undistort,
axis=1)
points_cam2_undistort = np.array([
dataFrame_camera2_undistort[scorer_cam2][bp]
["x"].values[:],
dataFrame_camera2_undistort[scorer_cam2][bp]
["y"].values[:],
])
points_cam2_undistort = points_cam2_undistort.T
# For cam2 camera: Assign nans to x and y values of a bodypart where the likelihood is less than pvalue
points_cam2_undistort[low_likelihood_frames] = np.nan, np.nan
points_cam2_undistort = np.expand_dims(points_cam2_undistort,
axis=1)
X_l = auxiliaryfunctions_3d.triangulatePoints(
P1, P2, points_cam1_undistort, points_cam2_undistort)
# ToDo: speed up func. below by saving in numpy.array
X_final.append(X_l)
triangulate.append(X_final)
triangulate = np.asanyarray(triangulate)
metadata = {}
metadata["stereo_matrix"] = stereomatrix
metadata["stereo_matrix_file"] = path_stereo_file
metadata["scorer_name"] = {
cam_names[0]: scorer_name[cam_names[0]],
cam_names[1]: scorer_name[cam_names[1]],
}
# Create an empty dataframe to store x,y,z of 3d data
for bpindex, bp in enumerate(bodyparts):
df_3d.iloc[:][scorer_3d, bp, "x"] = triangulate[0, bpindex,
0, :]
df_3d.iloc[:][scorer_3d, bp, "y"] = triangulate[0, bpindex,
1, :]
df_3d.iloc[:][scorer_3d, bp, "z"] = triangulate[0, bpindex,
2, :]
df_3d.to_hdf(
str(output_filename + ".h5"),
"df_with_missing",
format="table",
mode="w",
)
auxiliaryfunctions_3d.SaveMetadata3d(
str(output_filename + "_meta.pickle"), metadata)
if save_as_csv:
df_3d.to_csv(str(output_filename + ".csv"))
print("Triangulated data for video", video_list[i])
print("Results are saved under: ", destfolder)
# have to make the dest folder none so that it can be updated for a new pair of videos
if destfolder == str(Path(video).parents[0]):
destfolder = None
if len(video_list) > 0:
print("All videos were analyzed...")
print(
"Now you can create 3D video(s) using deeplabcut.create_labeled_video_3d"
)
def extract_save_all_maps(
config,
shuffle=1,
trainingsetindex=0,
comparisonbodyparts="all",
all_paf_in_one=True,
gputouse=None,
rescale=False,
Indices=None,
modelprefix="",
dest_folder=None,
):
"""
Extracts the scoremap, location refinement field and part affinity field prediction of the model. The maps
will be rescaled to the size of the input image and stored in the corresponding model folder in /evaluation-results.
----------
config : string
Full path of the config.yaml file as a string.
shuffle: integer
integers specifying shuffle index of the training dataset. The default is 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This
variable can also be set to "all".
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
all_paf_in_one : bool
By default, all part affinity fields are displayed on a single frame.
If false, individual fields are shown on separate frames.
Indices: default None
For which images shall the scmap/locref and paf be computed? Give a list of images
nplots_per_row: int, optional (default=None)
Number of plots per row in grid plots. By default, calculated to approximate a squared grid of plots
Examples
--------
Calculated maps for images 0, 1 and 33.
>>> deeplabcut.extract_save_all_maps('/analysis/project/reaching-task/config.yaml', shuffle=1,Indices=[0,1,33])
"""
from deeplabcut.utils.auxiliaryfunctions import (
read_config,
attempttomakefolder,
GetEvaluationFolder,
IntersectionofBodyPartsandOnesGivenbyUser,
)
from tqdm import tqdm
cfg = read_config(config)
data = extract_maps(config, shuffle, trainingsetindex, gputouse, rescale,
Indices, modelprefix)
comparisonbodyparts = IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
print("Saving plots...")
for frac, values in data.items():
if not dest_folder:
dest_folder = os.path.join(
cfg["project_path"],
str(
GetEvaluationFolder(frac,
shuffle,
cfg,
modelprefix=modelprefix)),
"maps",
)
attempttomakefolder(dest_folder)
filepath = "{imname}_{map}_{label}_{shuffle}_{frac}_{snap}.png"
dest_path = os.path.join(dest_folder, filepath)
for snap, maps in values.items():
for imagenr in tqdm(maps):
(
image,
scmap,
locref,
paf,
bptnames,
pafgraph,
impath,
trainingframe,
) = maps[imagenr]
label = "train" if trainingframe else "test"
imname = os.path.split(os.path.splitext(impath)[0])[1]
scmap, (locref_x, locref_y), paf = resize_all_maps(
image, scmap, locref, paf)
to_plot = [
i for i, bpt in enumerate(bptnames)
if bpt in comparisonbodyparts
]
list_of_inds = []
for n, edge in enumerate(pafgraph):
if any(ind in to_plot for ind in edge):
list_of_inds.append([(2 * n, 2 * n + 1),
(bptnames[edge[0]],
bptnames[edge[1]])])
if len(to_plot) > 1:
map_ = scmap[:, :, to_plot].sum(axis=2)
locref_x_ = locref_x[:, :, to_plot].sum(axis=2)
locref_y_ = locref_y[:, :, to_plot].sum(axis=2)
elif len(to_plot) == 1 and len(bptnames) > 1:
map_ = scmap[:, :, to_plot]
locref_x_ = locref_x[:, :, to_plot]
locref_y_ = locref_y[:, :, to_plot]
else:
map_ = scmap[:, :]
locref_x_ = locref_x[:, :]
locref_y_ = locref_y[:, :]
fig1, _ = visualize_scoremaps(image, map_)
temp = dest_path.format(
imname=imname,
map="scmap",
label=label,
shuffle=shuffle,
frac=frac,
snap=snap,
)
fig1.savefig(temp)
fig2, _ = visualize_locrefs(image, map_, locref_x_, locref_y_)
temp = dest_path.format(
imname=imname,
map="locref",
label=label,
shuffle=shuffle,
frac=frac,
snap=snap,
)
fig2.savefig(temp)
if paf is not None:
if not all_paf_in_one:
for inds, names in list_of_inds:
fig3, _ = visualize_paf(image, paf[:, :, [inds]])
temp = dest_path.format(
imname=imname,
map=f'paf_{"_".join(names)}',
label=label,
shuffle=shuffle,
frac=frac,
snap=snap,
)
fig3.savefig(temp)
else:
inds = [elem[0] for elem in list_of_inds]
n_inds = len(inds)
cmap = plt.cm.get_cmap(cfg["colormap"], n_inds)
colors = cmap(range(n_inds))
fig3, _ = visualize_paf(image,
paf[:, :, inds],
colors=colors)
temp = dest_path.format(
imname=imname,
map=f"paf",
label=label,
shuffle=shuffle,
frac=frac,
snap=snap,
)
fig3.savefig(temp)
plt.close("all")
def undistort_points(config, dataframe, camera_pair):
cfg_3d = auxiliaryfunctions.read_config(config)
(
img_path,
path_corners,
path_camera_matrix,
path_undistort,
_,
) = auxiliaryfunctions_3d.Foldernames3Dproject(cfg_3d)
"""
path_undistort = destfolder
filename_cam1 = Path(dataframe[0]).stem
filename_cam2 = Path(dataframe[1]).stem
#currently no interm. saving of this due to high speed.
# check if the undistorted files are already present
if os.path.exists(os.path.join(path_undistort,filename_cam1 + '_undistort.h5')) and os.path.exists(os.path.join(path_undistort,filename_cam2 + '_undistort.h5')):
print("The undistorted files are already present at %s" % os.path.join(path_undistort,filename_cam1))
dataFrame_cam1_undistort = pd.read_hdf(os.path.join(path_undistort,filename_cam1 + '_undistort.h5'))
dataFrame_cam2_undistort = pd.read_hdf(os.path.join(path_undistort,filename_cam2 + '_undistort.h5'))
else:
"""
if True:
# Create an empty dataFrame to store the undistorted 2d coordinates and likelihood
dataframe_cam1 = pd.read_hdf(dataframe[0])
dataframe_cam2 = pd.read_hdf(dataframe[1])
scorer_cam1 = dataframe_cam1.columns.get_level_values(0)[0]
scorer_cam2 = dataframe_cam2.columns.get_level_values(0)[0]
path_stereo_file = os.path.join(path_camera_matrix,
"stereo_params.pickle")
stereo_file = auxiliaryfunctions.read_pickle(path_stereo_file)
mtx_l = stereo_file[camera_pair]["cameraMatrix1"]
dist_l = stereo_file[camera_pair]["distCoeffs1"]
mtx_r = stereo_file[camera_pair]["cameraMatrix2"]
dist_r = stereo_file[camera_pair]["distCoeffs2"]
R1 = stereo_file[camera_pair]["R1"]
P1 = stereo_file[camera_pair]["P1"]
R2 = stereo_file[camera_pair]["R2"]
P2 = stereo_file[camera_pair]["P2"]
# Create an empty dataFrame to store the undistorted 2d coordinates and likelihood
(
dataFrame_cam1_undistort,
scorer_cam1,
bodyparts,
) = auxiliaryfunctions_3d.create_empty_df(dataframe_cam1,
scorer_cam1,
flag="2d")
(
dataFrame_cam2_undistort,
scorer_cam2,
bodyparts,
) = auxiliaryfunctions_3d.create_empty_df(dataframe_cam2,
scorer_cam2,
flag="2d")
for bpindex, bp in tqdm(enumerate(bodyparts)):
points_cam1 = dataframe_cam1.xs(bp, level="bodyparts",
axis=1).values[:, :2]
points_cam1_remapped = cv2.undistortPoints(
src=points_cam1.astype(np.float32),
cameraMatrix=mtx_l,
distCoeffs=dist_l,
P=P1,
R=R1,
)
dataFrame_cam1_undistort.loc(
axis=1)[scorer_cam1, bp,
["x", "y"]] = points_cam1_remapped.squeeze()
dataFrame_cam1_undistort.loc(
axis=1)[scorer_cam1, bp, "likelihood"] = dataframe_cam1.xs(
[bp, "likelihood"], level=["bodyparts",
"coords"], axis=1).values
# Undistorting the points from cam2 camera
points_cam2 = dataframe_cam2.xs(bp, level="bodyparts",
axis=1).values[:, :2]
points_cam2_remapped = cv2.undistortPoints(
src=points_cam2.astype(np.float32),
cameraMatrix=mtx_r,
distCoeffs=dist_r,
P=P2,
R=R2,
)
dataFrame_cam2_undistort.loc(
axis=1)[scorer_cam2, bp,
["x", "y"]] = points_cam2_remapped.squeeze()
dataFrame_cam2_undistort.loc(
axis=1)[scorer_cam2, bp, "likelihood"] = dataframe_cam2.xs(
[bp, "likelihood"], level=["bodyparts",
"coords"], axis=1).values
# Save the undistorted files
dataFrame_cam1_undistort.sort_index(inplace=True)
dataFrame_cam2_undistort.sort_index(inplace=True)
return (
dataFrame_cam1_undistort,
dataFrame_cam2_undistort,
stereo_file[camera_pair],
path_stereo_file,
)
def extract_outlier_frames(config,videos,videotype='avi',shuffle=1,trainingsetindex=0,outlieralgorithm='jump',comparisonbodyparts='all',epsilon=20,p_bound=.01,ARdegree=3,MAdegree=1,alpha=.01,extractionalgorithm='kmeans',automatic=False,cluster_resizewidth=30,cluster_color=False,opencv=True,savelabeled=True, destfolder=None):
"""
Extracts the outlier frames in case, the predictions are not correct for a certain video from the cropped video running from
start to stop as defined in config.yaml.
Another crucial parameter in config.yaml is how many frames to extract 'numframes2extract'.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
The shufle index of training dataset. The extracted frames will be stored in the labeled-dataset for
the corresponding shuffle of training dataset. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
outlieralgorithm: 'fitting', 'jump', 'uncertain', or 'manual'
String specifying the algorithm used to detect the outliers. Currently, deeplabcut supports three methods + a manual GUI option. 'Fitting'
fits a Auto Regressive Integrated Moving Average model to the data and computes the distance to the estimated data. Larger distances than
epsilon are then potentially identified as outliers. The methods 'jump' identifies larger jumps than 'epsilon' in any body part; and 'uncertain'
looks for frames with confidence below p_bound. The default is set to ``jump``.
comparisonbodyparts: list of strings, optional
This select the body parts for which the comparisons with the outliers are carried out. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
p_bound: float between 0 and 1, optional
For outlieralgorithm 'uncertain' this parameter defines the likelihood below, below which a body part will be flagged as a putative outlier.
epsilon; float,optional
Meaning depends on outlieralgoritm. The default is set to 20 pixels.
For outlieralgorithm 'fitting': Float bound according to which frames are picked when the (average) body part estimate deviates from model fit
For outlieralgorithm 'jump': Float bound specifying the distance by which body points jump from one frame to next (Euclidean distance)
ARdegree: int, optional
For outlieralgorithm 'fitting': Autoregressive degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part)
see https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
MAdegree: int
For outlieralgorithm 'fitting': MovingAvarage degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part)
See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
alpha: float
Significance level for detecting outliers based on confidence interval of fitted ARIMA model. Only the distance is used however.
extractionalgorithm : string, optional
String specifying the algorithm to use for selecting the frames from the identified putatative outlier frames. Currently, deeplabcut
supports either ``kmeans`` or ``uniform`` based selection (same logic as for extract_frames).
The default is set to``uniform``, if provided it must be either ``uniform`` or ``kmeans``.
automatic : bool, optional
Set it to True, if you want to extract outliers without being asked for user feedback.
cluster_resizewidth: number, default: 30
For k-means one can change the width to which the images are downsampled (aspect ratio is fixed).
cluster_color: bool, default: False
If false then each downsampled image is treated as a grayscale vector (discarding color information). If true, then the color channels are considered. This increases
the computational complexity.
opencv: bool, default: True
Uses openCV for loading & extractiong (otherwise moviepy (legacy))
savelabeled: bool, default: True
If true also saves frame with predicted labels in each folder.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
Examples
Windows example for extracting the frames with default settings
>>> deeplabcut.extract_outlier_frames('C:\\myproject\\reaching-task\\config.yaml',['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi'])
--------
for extracting the frames with default settings
>>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'])
--------
for extracting the frames with kmeans
>>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'],extractionalgorithm='kmeans')
--------
for extracting the frames with kmeans and epsilon = 5 pixels.
>>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'],epsilon = 5,extractionalgorithm='kmeans')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
DLCscorer,DLCscorerlegacy=auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction = cfg['TrainingFraction'][trainingsetindex])
Videos=auxiliaryfunctions.Getlistofvideos(videos,videotype)
for video in Videos:
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder=destfolder
notanalyzed,dataname,DLCscorer=auxiliaryfunctions.CheckifNotAnalyzed(videofolder,str(Path(video).stem),DLCscorer,DLCscorerlegacy,flag='checking')
if notanalyzed:
print("It seems the video has not been analyzed yet, or the video is not found! You can only refine the labels after the a video is analyzed. Please run 'analyze_video' first. Or, please double check your video file path")
else:
Dataframe = pd.read_hdf(dataname,'df_with_missing')
scorer=Dataframe.columns.get_level_values(0)[0] #reading scorer from
nframes=np.size(Dataframe.index)
# extract min and max index based on start stop interval.
startindex=max([int(np.floor(nframes*cfg['start'])),0])
stopindex=min([int(np.ceil(nframes*cfg['stop'])),nframes])
Index=np.arange(stopindex-startindex)+startindex
#figure out body part list:
bodyparts=auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(cfg,comparisonbodyparts)
Indices=[]
if outlieralgorithm=='uncertain': #necessary parameters: considered body parts and
for bpindex,bp in enumerate(bodyparts):
if bp in cfg['bodyparts']: #filter [who knows what users put in...]
p=Dataframe[scorer][bp]['likelihood'].values[Index]
Indices.extend(np.where(p<p_bound)[0]+startindex) # all indices between start and stop that are below p_bound.
elif outlieralgorithm=='jump':
for bpindex,bp in enumerate(bodyparts):
if bp in cfg['bodyparts']: #filter [who knows what users put in...]
dx=np.diff(Dataframe[scorer][bp]['x'].values[Index])
dy=np.diff(Dataframe[scorer][bp]['y'].values[Index])
# all indices between start and stop with jump larger than epsilon (leading up to this point!)
Indices.extend(np.where((dx**2+dy**2)>epsilon**2)[0]+startindex+1)
elif outlieralgorithm=='fitting':
#deviation_dataname = str(Path(videofolder)/Path(dataname))
# Calculate deviatons for video
[d,o] = ComputeDeviations(Dataframe,cfg,bodyparts,scorer,dataname,p_bound,alpha,ARdegree,MAdegree)
#Some heuristics for extracting frames based on distance:
Indices=np.where(d>epsilon)[0] # time points with at least average difference of epsilon
if len(Index)<cfg['numframes2pick']*2 and len(d)>cfg['numframes2pick']*2: # if too few points qualify, extract the most distant ones.
Indices=np.argsort(d)[::-1][:cfg['numframes2pick']*2]
elif outlieralgorithm=='manual':
wd = Path(config).resolve().parents[0]
os.chdir(str(wd))
from deeplabcut.refine_training_dataset import outlier_frame_extraction_toolbox
outlier_frame_extraction_toolbox.show(config,video,shuffle,Dataframe,scorer,savelabeled)
# Run always except when the outlieralgorithm == manual.
if not outlieralgorithm=='manual':
Indices=np.sort(list(set(Indices))) #remove repetitions.
print("Method ", outlieralgorithm, " found ", len(Indices)," putative outlier frames.")
print("Do you want to proceed with extracting ", cfg['numframes2pick'], " of those?")
if outlieralgorithm=='uncertain':
print("If this list is very large, perhaps consider changing the paramters (start, stop, p_bound, comparisonbodyparts) or use a different method.")
elif outlieralgorithm=='jump':
print("If this list is very large, perhaps consider changing the paramters (start, stop, epsilon, comparisonbodyparts) or use a different method.")
elif outlieralgorithm=='fitting':
print("If this list is very large, perhaps consider changing the paramters (start, stop, epsilon, ARdegree, MAdegree, alpha, comparisonbodyparts) or use a different method.")
if automatic==False:
askuser = input("yes/no")
else:
askuser='******'
if askuser=='y' or askuser=='yes' or askuser=='Ja' or askuser=='ha': # multilanguage support :)
#Now extract from those Indices!
ExtractFramesbasedonPreselection(Indices,extractionalgorithm,Dataframe,dataname,scorer,video,cfg,config,opencv,cluster_resizewidth,cluster_color,savelabeled)
else:
print("Nothing extracted, please change the parameters and start again...")
def create_training_dataset(config,num_shuffles=1,Shuffles=None,windows2linux=False,userfeedback=False,
trainIndexes=None,testIndexes=None,
net_type=None,augmenter_type=None):
"""
Creates a training dataset. Labels from all the extracted frames are merged into a single .h5 file.\n
Only the videos included in the config file are used to create this dataset.\n
[OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
num_shuffles : int, optional
Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1.
Shuffles: list of shuffles.
Alternatively the user can also give a list of shuffles (integers!).
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
userfeedback: bool, optional
If this is set to false, then all requested train/test splits are created (no matter if they already exist). If you
want to assure that previous splits etc. are not overwritten, then set this to True and you will be asked for each split.
net_type: string
Type of networks. Currently resnet_50, resnet_101, resnet_152, mobilenet_v2_1.0,mobilenet_v2_0.75, mobilenet_v2_0.5, and mobilenet_v2_0.35 are supported.
augmenter_type: string
Type of augmenter. Currently default, imgaug, tensorpack, and deterministic are supported.
Example
--------
>>> deeplabcut.create_training_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1)
Windows:
>>> deeplabcut.create_training_dataset('C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5])
--------
"""
from skimage import io
import scipy.io as sio
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg['scorer']
project_path = cfg['project_path']
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) #Path concatenation OS platform independent
auxiliaryfunctions.attempttomakefolder(Path(os.path.join(project_path,str(trainingsetfolder))),recursive=True)
Data = merge_annotateddatasets(cfg,project_path,Path(os.path.join(project_path,trainingsetfolder)),windows2linux)
Data = Data[scorer] #extract labeled data
#loading & linking pretrained models
if net_type is None: #loading & linking pretrained models
net_type =cfg.get('default_net_type', 'resnet_50')
else:
if 'resnet' in net_type or 'mobilenet' in net_type:
pass
else:
raise ValueError('Invalid network type:', net_type)
if augmenter_type is None:
augmenter_type=cfg.get('default_augmenter', 'default')
else:
if augmenter_type in ['default','imgaug','tensorpack','deterministic']:
pass
else:
raise ValueError('Invalid augmenter type:', augmenter_type)
import deeplabcut
parent_path = Path(os.path.dirname(deeplabcut.__file__))
defaultconfigfile = str(parent_path / 'pose_cfg.yaml')
model_path,num_shuffles=auxfun_models.Check4weights(net_type,parent_path,num_shuffles) #if the model does not exist >> throws error!
if Shuffles==None:
Shuffles=range(1,num_shuffles+1,1)
else:
Shuffles=[i for i in Shuffles if isinstance(i,int)]
bodyparts = cfg['bodyparts']
TrainingFraction = cfg['TrainingFraction']
for shuffle in Shuffles: # Creating shuffles starting from 1
for trainingsetindex,trainFraction in enumerate(TrainingFraction):
if userfeedback:
trainposeconfigfile,testposeconfigfile,snapshotfolder = training.return_train_network_path(config,shuffle=shuffle,trainingsetindex=trainingsetindex)
if os.path.isfile(trainposeconfigfile):
askuser=input ("The model folder is already present. If you continue, it will overwrite the existing model (split). Do you want to continue?(yes/no): ")
if askuser=='no'or askuser=='No' or askuser=='N' or askuser=='No':
raise Exception("Use the Shuffles argument as a list to specify a different shuffle index. Check out the help for more details.")
else:
pass
#trainIndexes, testIndexes = SplitTrials(range(len(Data.index)), trainFraction)
if trainIndexes is None and testIndexes is None:
trainIndexes, testIndexes = SplitTrials(range(len(Data.index)), trainFraction)
else:
print("You passed a split with the following fraction:", len(trainIndexes)*1./(len(testIndexes)+len(trainIndexes))*100)
####################################################
# Generating data structure with labeled information & frame metadata (for deep cut)
####################################################
# Make training file!
data = []
for jj in trainIndexes:
H = {}
# load image to get dimensions:
filename = Data.index[jj]
im = io.imread(os.path.join(cfg['project_path'],filename))
H['image'] = filename
if np.ndim(im)==3:
H['size'] = np.array(
[np.shape(im)[2],
np.shape(im)[0],
np.shape(im)[1]])
else:
# print "Grayscale!"
H['size'] = np.array([1, np.shape(im)[0], np.shape(im)[1]])
indexjoints=0
joints=np.zeros((len(bodyparts),3))*np.nan
for bpindex,bodypart in enumerate(bodyparts):
# check whether the labels are positive and inside the img
x_pos_n_inside = 0 <= Data[bodypart]['x'][jj] < np.shape(im)[1]
y_pos_n_inside = 0 <= Data[bodypart]['y'][jj] < np.shape(im)[0]
if x_pos_n_inside and y_pos_n_inside:
joints[indexjoints,0]=int(bpindex)
joints[indexjoints,1]=Data[bodypart]['x'][jj]
joints[indexjoints,2]=Data[bodypart]['y'][jj]
indexjoints+=1
joints = joints[np.where(
np.prod(np.isfinite(joints),
1))[0], :] # drop NaN, i.e. lines for missing body parts
assert (np.prod(np.array(joints[:, 2]) < np.shape(im)[0])
) # y coordinate within image?
assert (np.prod(np.array(joints[:, 1]) < np.shape(im)[1])
) # x coordinate within image?
H['joints'] = np.array(joints, dtype=int)
if np.size(joints)>0: #exclude images without labels
data.append(H)
if len(trainIndexes)>0:
datafilename,metadatafilename=auxiliaryfunctions.GetDataandMetaDataFilenames(trainingsetfolder,trainFraction,shuffle,cfg)
################################################################################
# Saving metadata (Pickle file)
################################################################################
auxiliaryfunctions.SaveMetadata(os.path.join(project_path,metadatafilename),data, trainIndexes, testIndexes, trainFraction)
################################################################################
# Saving data file (convert to training file for deeper cut (*.mat))
################################################################################
DTYPE = [('image', 'O'), ('size', 'O'), ('joints', 'O')]
MatlabData = np.array(
[(np.array([data[item]['image']], dtype='U'),
np.array([data[item]['size']]),
boxitintoacell(data[item]['joints']))
for item in range(len(data))],
dtype=DTYPE)
sio.savemat(os.path.join(project_path,datafilename), {'dataset': MatlabData})
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
modelfoldername=auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg)
auxiliaryfunctions.attempttomakefolder(Path(config).parents[0] / modelfoldername,recursive=True)
auxiliaryfunctions.attempttomakefolder(str(Path(config).parents[0] / modelfoldername)+ '/'+ '/train')
auxiliaryfunctions.attempttomakefolder(str(Path(config).parents[0] / modelfoldername)+ '/'+ '/test')
path_train_config = str(os.path.join(cfg['project_path'],Path(modelfoldername),'train','pose_cfg.yaml'))
path_test_config = str(os.path.join(cfg['project_path'],Path(modelfoldername),'test','pose_cfg.yaml'))
#str(cfg['proj_path']+'/'+Path(modelfoldername) / 'test' / 'pose_cfg.yaml')
items2change = {
"dataset": datafilename,
"metadataset": metadatafilename,
"num_joints": len(bodyparts),
"all_joints": [[i] for i in range(len(bodyparts))],
"all_joints_names": [str(bpt) for bpt in bodyparts],
"init_weights": model_path,
"project_path": str(cfg['project_path']),
"net_type": net_type,
"dataset_type": augmenter_type
}
trainingdata = MakeTrain_pose_yaml(items2change,path_train_config,defaultconfigfile)
keys2save = [
"dataset", "num_joints", "all_joints", "all_joints_names",
"net_type", 'init_weights', 'global_scale', 'location_refinement',
'locref_stdev'
]
MakeTest_pose_yaml(trainingdata, keys2save,path_test_config)
print("The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!")
def mergeandsplit(config, trainindex=0, uniform=True, windows2linux=False):
"""
This function allows additional control over "create_training_dataset".
Merge annotated data sets (from different folders) and split data in a specific way, returns the split variables (train/test indices).
Importantly, this allows one to freeze a split.
One can also either create a uniform split (uniform = True; thereby indexing TrainingFraction in config file) or leave-one-folder out split
by passing the index of the corrensponding video from the config.yaml file as variable trainindex.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
trainindex: int, optional
Either (in case uniform = True) indexes which element of TrainingFraction in the config file should be used (note it is a list!).
Alternatively (uniform = False) indexes which folder is dropped, i.e. the first if trainindex=0, the second if trainindex =1, etc.
uniform: bool, optional
Perform uniform split (disregarding folder structure in labeled data), or (if False) leave one folder out.
windows2linux: bool.
The annotation files contain path formated according to your operating system. If you label on windows
but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths.
Examples
--------
To create a leave-one-folder-out model:
>>> trainIndexes, testIndexes=deeplabcut.mergeandsplit(config,trainindex=0,uniform=False)
returns the indices for the first video folder (as defined in config file) as testIndexes and all others as trainIndexes.
You can then create the training set by calling (e.g. defining it as Shuffle 3):
>>> deeplabcut.create_training_dataset(config,Shuffles=[3],trainIndexes=trainIndexes,testIndexes=testIndexes)
To freeze a (uniform) split:
>>> trainIndexes, testIndexes=deeplabcut.mergeandsplit(config,trainindex=0,uniform=True)
You can then create two model instances that have the identical trainingset. Thereby you can assess the role of various parameters on the performance of DLC.
>>> deeplabcut.create_training_dataset(config,Shuffles=[0],trainIndexes=trainIndexes,testIndexes=testIndexes)
>>> deeplabcut.create_training_dataset(config,Shuffles=[1],trainIndexes=trainIndexes,testIndexes=testIndexes)
--------
"""
# Loading metadata from config file:
cfg = auxiliaryfunctions.read_config(config)
scorer = cfg['scorer']
project_path = cfg['project_path']
# Create path for training sets & store data there
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(
cfg) #Path concatenation OS platform independent
auxiliaryfunctions.attempttomakefolder(Path(
os.path.join(project_path, str(trainingsetfolder))),
recursive=True)
fn = os.path.join(project_path, trainingsetfolder,
'CollectedData_' + cfg['scorer'])
try:
Data = pd.read_hdf(fn + '.h5', 'df_with_missing')
except FileNotFoundError:
Data = merge_annotateddatasets(
cfg,
project_path,
Path(os.path.join(project_path, trainingsetfolder)),
windows2linux=windows2linux)
Data = Data[scorer] #extract labeled data
if uniform == True:
TrainingFraction = cfg['TrainingFraction']
trainFraction = TrainingFraction[trainindex]
trainIndexes, testIndexes = SplitTrials(range(len(Data.index)),
trainFraction)
else: #leave one folder out split
videos = cfg['video_sets'].keys()
test_video_name = [Path(i).stem for i in videos][trainindex]
print("Excluding the following folder (from training):",
test_video_name)
trainIndexes, testIndexes = [], []
for index, name in enumerate(Data.index):
#print(index,name.split(os.sep)[1])
if test_video_name == name.split(
os.sep)[1]: #this is the video name
#print(name,test_video_name)
testIndexes.append(index)
else:
trainIndexes.append(index)
return trainIndexes, testIndexes
def convert_single2multiplelegacyAM(config, userfeedback=True, target=None):
""" Convert multi animal to single animal code and vice versa. Note that by providing target='single'/'multi' this will be target! """
cfg = auxiliaryfunctions.read_config(config)
videos = cfg["video_sets"].keys()
video_names = [Path(i).stem for i in videos]
folders = [
Path(config).parent / "labeled-data" / Path(i) for i in video_names
]
prefixes, uniquebodyparts, multianimalbodyparts = extractindividualsandbodyparts(
cfg)
for folder in folders:
if userfeedback == True:
print("Do you want to convert the annotation file in folder:",
folder, "?")
askuser = input("yes/no")
else:
askuser = "******"
if (askuser == "y" or askuser == "yes" or askuser == "Ja"
or askuser == "ha"): # multilanguage support :)
fn = os.path.join(str(folder), "CollectedData_" + cfg["scorer"])
Data = pd.read_hdf(fn + ".h5")
imindex = Data.index
if "individuals" in Data.columns.names and (target == None
or target == "single"):
print(
"This is a multianimal data set, converting to single...",
folder)
for prfxindex, prefix in enumerate(prefixes):
if prefix == "single":
for j, bpt in enumerate(uniquebodyparts):
index = pd.MultiIndex.from_product(
[[cfg["scorer"]], [bpt], ["x", "y"]],
names=["scorer", "bodyparts", "coords"],
)
frame = pd.DataFrame(
Data[cfg["scorer"]][prefix][bpt].values,
columns=index,
index=imindex,
)
if j == 0:
dataFrame = frame
else:
dataFrame = pd.concat([dataFrame, frame],
axis=1)
else:
for j, bpt in enumerate(multianimalbodyparts):
index = pd.MultiIndex.from_product(
[[cfg["scorer"]], [prefix + bpt], ["x", "y"]],
names=["scorer", "bodyparts", "coords"],
)
frame = pd.DataFrame(
Data[cfg["scorer"]][prefix][bpt].values,
columns=index,
index=imindex,
)
if j == 0:
dataFrame = frame
else:
dataFrame = pd.concat([dataFrame, frame],
axis=1)
if prfxindex == 0:
DataFrame = dataFrame
else:
DataFrame = pd.concat([DataFrame, dataFrame], axis=1)
Data.to_hdf(fn + "multianimal.h5",
"df_with_missing",
format="table",
mode="w")
Data.to_csv(fn + "multianimal.csv")
DataFrame.to_hdf(fn + ".h5",
"df_with_missing",
format="table",
mode="w")
DataFrame.to_csv(fn + ".csv")
elif target == None or target == "multi":
print(
"This is a single animal data set, converting to multi...",
folder)
for prfxindex, prefix in enumerate(prefixes):
if prefix == "single":
if cfg["uniquebodyparts"] != [None]:
for j, bpt in enumerate(uniquebodyparts):
index = pd.MultiIndex.from_arrays(
np.array([
2 * [cfg["scorer"]],
2 * [prefix],
2 * [bpt],
["x", "y"],
]),
names=[
"scorer",
"individuals",
"bodyparts",
"coords",
],
)
if bpt in Data[cfg["scorer"]].keys():
frame = pd.DataFrame(
Data[cfg["scorer"]][bpt].values,
columns=index,
index=imindex,
)
else: # fill with nans...
frame = pd.DataFrame(
np.ones((len(imindex), 2)) * np.nan,
columns=index,
index=imindex,
)
if j == 0:
dataFrame = frame
else:
dataFrame = pd.concat([dataFrame, frame],
axis=1)
else:
dataFrame = None
else:
for j, bpt in enumerate(multianimalbodyparts):
index = pd.MultiIndex.from_arrays(
np.array([
2 * [cfg["scorer"]],
2 * [prefix],
2 * [bpt],
["x", "y"],
]),
names=[
"scorer", "individuals", "bodyparts",
"coords"
],
)
if prefix + "_" + bpt in Data[
cfg["scorer"]].keys():
frame = pd.DataFrame(
Data[cfg["scorer"]][prefix + "_" +
bpt].values,
columns=index,
index=imindex,
)
else:
frame = pd.DataFrame(
np.ones((len(imindex), 2)) * np.nan,
columns=index,
index=imindex,
)
if j == 0:
dataFrame = frame
else:
dataFrame = pd.concat([dataFrame, frame],
axis=1)
if prfxindex == 0:
DataFrame = dataFrame
else:
DataFrame = pd.concat([DataFrame, dataFrame], axis=1)
Data.to_hdf(fn + "singleanimal.h5",
"df_with_missing",
format="table",
mode="w")
Data.to_csv(fn + "singleanimal.csv")
DataFrame.to_hdf(fn + ".h5",
"df_with_missing",
format="table",
mode="w")
DataFrame.to_csv(fn + ".csv")