def train_tracking_transformer(
path_config_file,
dlcscorer,
videos,
videotype="",
train_frac=0.8,
modelprefix="",
train_epochs=100,
batch_size=64,
ckpt_folder="",
destfolder=None,
):
npy_list = []
videos = auxiliaryfunctions.get_list_of_videos(videos, videotype)
for video in videos:
videofolder = str(Path(video).parents[0])
if destfolder is None:
destfolder = videofolder
video_name = Path(video).stem
# video_name = '.'.join(video.split("/")[-1].split(".")[:-1])
files = glob.glob(
os.path.join(destfolder, video_name + dlcscorer + "*.npy"))
# assuming there is only one match
npy_list.append(files[0])
train_list, test_list = split_train_test(npy_list, train_frac)
train_loader, val_loader = make_dlc_dataloader(train_list, test_list,
batch_size)
# make my own model factory
num_kpts = train_list.shape[2]
feature_dim = train_list.shape[-1]
model = make_dlc_model(cfg, feature_dim, num_kpts)
# make my own loss factory
triplet_loss = easy_triplet_loss()
optimizer = make_easy_optimizer(cfg, model)
scheduler = create_scheduler(cfg, optimizer)
num_query = 1
do_dlc_train(
cfg,
model,
triplet_loss,
train_loader,
val_loader,
optimizer,
scheduler,
num_kpts,
feature_dim,
num_query,
total_epochs=train_epochs,
ckpt_folder=ckpt_folder,
)
def analyzeskeleton(
config,
videos,
videotype="",
shuffle=1,
trainingsetindex=0,
filtered=False,
save_as_csv=False,
destfolder=None,
modelprefix="",
track_method="",
):
"""Extracts length and orientation of each "bone" of the skeleton.
The bone and skeleton information is defined in the config file.
Parameters
----------
config: str
Full path of the config.yaml file.
videos: list[str]
The full paths to videos for analysis or a path to the directory, where all the
videos with same extension are stored.
videotype: str, optional, default=""
Checks for the extension of the video in case the input to the video is a
directory. Only videos with this extension are analyzed.
If left unspecified, videos with common extensions
('avi', 'mp4', 'mov', 'mpeg', 'mkv') are kept.
shuffle : int, optional, default=1
The shuffle index of training dataset. The extracted frames will be stored in
the labeled-dataset for the corresponding shuffle of training dataset.
trainingsetindex: int, optional, default=0
Integer specifying which TrainingsetFraction to use.
Note that TrainingFraction is a list in config.yaml.
filtered: bool, optional, default=False
Boolean variable indicating if filtered output should be plotted rather than
frame-by-frame predictions. Filtered version can be calculated with
``deeplabcut.filterpredictions``.
save_as_csv: bool, optional, default=False
Saves the predictions in a .csv file.
destfolder: string or None, optional, default=None
Specifies the destination folder for analysis data. If ``None``, the path of
the video is used. Note that for subsequent analysis this folder also needs to
be passed.
modelprefix: str, optional, default=""
Directory containing the deeplabcut models to use when evaluating the network.
By default, the models are assumed to exist in the project folder.
track_method: string, optional, default=""
Specifies the tracker used to generate the data.
Empty by default (corresponding to a single animal project).
For multiple animals, must be either 'box', 'skeleton', or 'ellipse' and will
be taken from the config.yaml file if none is given.
Returns
-------
None
"""
# Load config file, scorer and videos
cfg = auxiliaryfunctions.read_config(config)
if not cfg["skeleton"]:
raise ValueError("No skeleton defined in the config.yaml.")
track_method = auxfun_multianimal.get_track_method(
cfg, track_method=track_method)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction=cfg["TrainingFraction"][trainingsetindex],
modelprefix=modelprefix,
)
Videos = auxiliaryfunctions.get_list_of_videos(videos, videotype)
for video in Videos:
print("Processing %s" % (video))
if destfolder is None:
destfolder = str(Path(video).parents[0])
vname = Path(video).stem
try:
df, filepath, scorer, _ = auxiliaryfunctions.load_analyzed_data(
destfolder, vname, DLCscorer, filtered, track_method)
output_name = filepath.replace(".h5", f"_skeleton.h5")
if os.path.isfile(output_name):
print(
f"Skeleton in video {vname} already processed. Skipping..."
)
continue
bones = {}
if "individuals" in df.columns.names:
for animal_name, df_ in df.groupby(level="individuals",
axis=1):
temp = df_.droplevel(["scorer", "individuals"], axis=1)
if animal_name != "single":
for bp1, bp2 in cfg["skeleton"]:
name = "{}_{}_{}".format(animal_name, bp1, bp2)
bones[name] = analyzebone(temp[bp1], temp[bp2])
else:
for bp1, bp2 in cfg["skeleton"]:
name = "{}_{}".format(bp1, bp2)
bones[name] = analyzebone(df[scorer][bp1], df[scorer][bp2])
skeleton = pd.concat(bones, axis=1)
skeleton.to_hdf(output_name,
"df_with_missing",
format="table",
mode="w")
if save_as_csv:
skeleton.to_csv(output_name.replace(".h5", ".csv"))
except FileNotFoundError as e:
print(e)
continue
def extract_outlier_frames(
config,
videos,
videotype="",
shuffle=1,
trainingsetindex=0,
outlieralgorithm="jump",
comparisonbodyparts="all",
epsilon=20,
p_bound=0.01,
ARdegree=3,
MAdegree=1,
alpha=0.01,
extractionalgorithm="kmeans",
automatic=False,
cluster_resizewidth=30,
cluster_color=False,
opencv=True,
savelabeled=False,
destfolder=None,
modelprefix="",
track_method="",
):
"""Extracts the outlier frames.
Extracts the outlier frames if the predictions are not correct for a certain video
from the cropped video running from start to stop as defined in config.yaml.
Another crucial parameter in config.yaml is how many frames to extract
``numframes2extract``.
Parameters
----------
config: str
Full path of the config.yaml file.
videos : list[str]
The full paths to videos for analysis or a path to the directory, where all the
videos with same extension are stored.
videotype: str, optional, default=""
Checks for the extension of the video in case the input to the video is a
directory. Only videos with this extension are analyzed.
If left unspecified, videos with common extensions
('avi', 'mp4', 'mov', 'mpeg', 'mkv') are kept.
shuffle : int, optional, default=1
The shuffle index of training dataset. The extracted frames will be stored in
the labeled-dataset for the corresponding shuffle of training dataset.
trainingsetindex: int, optional, default=0
Integer specifying which TrainingsetFraction to use.
Note that TrainingFraction is a list in config.yaml.
outlieralgorithm: str, optional, default="jump".
String specifying the algorithm used to detect the outliers.
* ``'Fitting'`` fits a Auto Regressive Integrated Moving Average model to the
data and computes the distance to the estimated data. Larger distances than
epsilon are then potentially identified as outliers
* ``'jump'`` identifies larger jumps than 'epsilon' in any body part
* ``'uncertain'`` looks for frames with confidence below p_bound
* ``'manual'`` launches a GUI from which the user can choose the frames
comparisonbodyparts: list[str] or str, optional, default="all"
This selects the body parts for which the comparisons with the outliers are
carried out. If ``"all"``, then all body parts from config.yaml are used. If a
list of strings that are a subset of the full list E.g. ['hand','Joystick'] for
the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these body
parts.
p_bound: float between 0 and 1, optional, default=0.01
For outlieralgorithm ``'uncertain'`` this parameter defines the likelihood
below which a body part will be flagged as a putative outlier.
epsilon: float, optional, default=20
If ``'outlieralgorithm'`` is ``'fitting'``, this is the float bound according
to which frames are picked when the (average) body part estimate deviates from
model fit.
If ``'outlieralgorithm'`` is ``'jump'``, this is the float bound specifying the
distance by which body points jump from one frame to next (Euclidean distance).
ARdegree: int, optional, default=3
For outlieralgorithm ``'fitting'``: Autoregressive degree of ARIMA model degree.
(Note we use SARIMAX without exogeneous and seasonal part)
See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
MAdegree: int, optional, default=1
For outlieralgorithm ``'fitting'``: MovingAvarage degree of ARIMA model degree.
(Note we use SARIMAX without exogeneous and seasonal part)
See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
alpha: float, optional, default=0.01
Significance level for detecting outliers based on confidence interval of
fitted ARIMA model. Only the distance is used however.
extractionalgorithm : str, optional, default="kmeans"
String specifying the algorithm to use for selecting the frames from the
identified putatative outlier frames. Currently, deeplabcut supports either
``kmeans`` or ``uniform`` based selection (same logic as for extract_frames).
automatic : bool, optional, default=False
If ``True``, extract outliers without being asked for user feedback.
cluster_resizewidth: number, default=30
If ``"extractionalgorithm"`` is ``"kmeans"``, one can change the width to which
the images are downsampled (aspect ratio is fixed).
cluster_color: bool, optional, default=False
If ``False``, each downsampled image is treated as a grayscale vector
(discarding color information). If ``True``, then the color channels are
considered. This increases the computational complexity.
opencv: bool, optional, default=True
Uses openCV for loading & extractiong (otherwise moviepy (legacy)).
savelabeled: bool, optional, default=False
If ``True``, frame are saved with predicted labels in each folder.
destfolder: str or None, optional, default=None
Specifies the destination folder that was used for storing analysis data. If
``None``, the path of the video is used.
modelprefix: str, optional, default=""
Directory containing the deeplabcut models to use when evaluating the network.
By default, the models are assumed to exist in the project folder.
track_method: str, optional, default=""
Specifies the tracker used to generate the data.
Empty by default (corresponding to a single animal project).
For multiple animals, must be either 'box', 'skeleton', or 'ellipse' and will
be taken from the config.yaml file if none is given.
Returns
-------
None
Examples
--------
Extract the frames with default settings on Windows.
>>> deeplabcut.extract_outlier_frames(
'C:\\myproject\\reaching-task\\config.yaml',
['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi'],
)
Extract the frames with default settings on Linux/MacOS.
>>> deeplabcut.extract_outlier_frames(
'/analysis/project/reaching-task/config.yaml',
['/analysis/project/video/reachinvideo1.avi'],
)
Extract the frames using the "kmeans" algorithm.
>>> deeplabcut.extract_outlier_frames(
'/analysis/project/reaching-task/config.yaml',
['/analysis/project/video/reachinvideo1.avi'],
extractionalgorithm='kmeans',
)
Extract the frames using the "kmeans" algorithm and ``"epsilon=5"`` pixels.
>>> deeplabcut.extract_outlier_frames(
'/analysis/project/reaching-task/config.yaml',
['/analysis/project/video/reachinvideo1.avi'],
epsilon=5,
extractionalgorithm='kmeans',
)
"""
cfg = auxiliaryfunctions.read_config(config)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts
)
if not len(bodyparts):
raise ValueError("No valid bodyparts were selected.")
track_method = auxfun_multianimal.get_track_method(cfg, track_method=track_method)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction=cfg["TrainingFraction"][trainingsetindex],
modelprefix=modelprefix,
)
Videos = auxiliaryfunctions.get_list_of_videos(videos, videotype)
if len(Videos) == 0:
print("No suitable videos found in", videos)
for video in Videos:
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
vname = os.path.splitext(os.path.basename(video))[0]
try:
df, dataname, _, _ = auxiliaryfunctions.load_analyzed_data(
videofolder, vname, DLCscorer, track_method=track_method
)
nframes = len(df)
startindex = max([int(np.floor(nframes * cfg["start"])), 0])
stopindex = min([int(np.ceil(nframes * cfg["stop"])), nframes])
Index = np.arange(stopindex - startindex) + startindex
df = df.iloc[Index]
mask = df.columns.get_level_values("bodyparts").isin(bodyparts)
df_temp = df.loc[:, mask]
Indices = []
if outlieralgorithm == "uncertain":
p = df_temp.xs("likelihood", level="coords", axis=1)
ind = df_temp.index[(p < p_bound).any(axis=1)].tolist()
Indices.extend(ind)
elif outlieralgorithm == "jump":
temp_dt = df_temp.diff(axis=0) ** 2
temp_dt.drop("likelihood", axis=1, level="coords", inplace=True)
sum_ = temp_dt.sum(axis=1, level=1)
ind = df_temp.index[(sum_ > epsilon ** 2).any(axis=1)].tolist()
Indices.extend(ind)
elif outlieralgorithm == "fitting":
d, o = compute_deviations(
df_temp, dataname, p_bound, alpha, ARdegree, MAdegree
)
# Some heuristics for extracting frames based on distance:
ind = np.flatnonzero(
d > epsilon
) # time points with at least average difference of epsilon
if (
len(ind) < cfg["numframes2pick"] * 2
and len(d) > cfg["numframes2pick"] * 2
): # if too few points qualify, extract the most distant ones.
ind = np.argsort(d)[::-1][: cfg["numframes2pick"] * 2]
Indices.extend(ind)
elif outlieralgorithm == "manual":
wd = Path(config).resolve().parents[0]
os.chdir(str(wd))
from deeplabcut.gui import outlier_frame_extraction_toolbox
outlier_frame_extraction_toolbox.show(
config,
video,
shuffle,
df,
savelabeled,
cfg.get("multianimalproject", False),
)
# Run always except when the outlieralgorithm == manual.
if not outlieralgorithm == "manual":
Indices = np.sort(list(set(Indices))) # remove repetitions.
print(
"Method ",
outlieralgorithm,
" found ",
len(Indices),
" putative outlier frames.",
)
print(
"Do you want to proceed with extracting ",
cfg["numframes2pick"],
" of those?",
)
if outlieralgorithm == "uncertain" or outlieralgorithm == "jump":
print(
"If this list is very large, perhaps consider changing the parameters "
"(start, stop, p_bound, comparisonbodyparts) or use a different method."
)
elif outlieralgorithm == "fitting":
print(
"If this list is very large, perhaps consider changing the parameters "
"(start, stop, epsilon, ARdegree, MAdegree, alpha, comparisonbodyparts) "
"or use a different method."
)
if not automatic:
askuser = input("yes/no")
else:
askuser = "******"
if (
askuser == "y"
or askuser == "yes"
or askuser == "Ja"
or askuser == "ha"
): # multilanguage support :)
# Now extract from those Indices!
ExtractFramesbasedonPreselection(
Indices,
extractionalgorithm,
df,
video,
cfg,
config,
opencv,
cluster_resizewidth,
cluster_color,
savelabeled,
)
else:
print(
"Nothing extracted, please change the parameters and start again..."
)
except FileNotFoundError as e:
print(e)
print(
"It seems the video has not been analyzed yet, or the video is not found! "
"You can only refine the labels after the a video is analyzed. Please run 'analyze_video' first. "
"Or, please double check your video file path"
)
def filterpredictions(
config,
video,
videotype="",
shuffle=1,
trainingsetindex=0,
filtertype="median",
windowlength=5,
p_bound=0.001,
ARdegree=3,
MAdegree=1,
alpha=0.01,
save_as_csv=True,
destfolder=None,
modelprefix="",
track_method="",
):
"""Fits frame-by-frame pose predictions.
The pose predictions are fitted with ARIMA model (filtertype='arima') or median
filter (default).
Parameters
----------
config : string
Full path of the config.yaml file.
video : string
Full path of the video to extract the frame from. Make sure that this video is
already analyzed.
shuffle : int, optional, default=1
The shuffle index of training dataset. The extracted frames will be stored in
the labeled-dataset for the corresponding shuffle of training dataset.
trainingsetindex: int, optional, default=0
Integer specifying which TrainingsetFraction to use.
Note that TrainingFraction is a list in config.yaml.
filtertype: string, optional, default="median".
The filter type - 'arima', 'median' or 'spline'.
windowlength: int, optional, default=5
For filtertype='median' filters the input array using a local window-size given
by windowlength. The array will automatically be zero-padded.
https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.medfilt.html.
The windowlenght should be an odd number.
If filtertype='spline', windowlength is the maximal gap size to fill.
p_bound: float between 0 and 1, optional, default=0.001
For filtertype 'arima' this parameter defines the likelihood below,
below which a body part will be consided as missing data for filtering purposes.
ARdegree: int, optional, default=3
For filtertype 'arima' Autoregressive degree of Sarimax model degree.
see https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
MAdegree: int, optional, default=1
For filtertype 'arima' Moving Average degree of Sarimax model degree.
See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
alpha: float, optional, default=0.01
Significance level for detecting outliers based on confidence interval of fitted SARIMAX model.
save_as_csv: bool, optional, default=True
Saves the predictions in a .csv file.
destfolder: string, optional, default=None
Specifies the destination folder for analysis data. If ``None``, the path of
the video is used by default. Note that for subsequent analysis this folder
also needs to be passed.
modelprefix: str, optional, default=""
Directory containing the deeplabcut models to use when evaluating the network.
By default, the models are assumed to exist in the project folder.
track_method: string, optional, default=""
Specifies the tracker used to generate the data.
Empty by default (corresponding to a single animal project).
For multiple animals, must be either 'box', 'skeleton', or 'ellipse' and will
be taken from the config.yaml file if none is given.
Returns
-------
None
Examples
--------
Arima model:
>>> deeplabcut.filterpredictions(
'C:\\myproject\\reaching-task\\config.yaml',
['C:\\myproject\\trailtracking-task\\test.mp4'],
shuffle=3,
filterype='arima',
ARdegree=5,
MAdegree=2,
)
Use median filter over 10 bins:
>>> deeplabcut.filterpredictions(
'C:\\myproject\\reaching-task\\config.yaml',
['C:\\myproject\\trailtracking-task\\test.mp4'],
shuffle=3,
windowlength=10,
)
One can then use the filtered rather than the frame-by-frame predictions by calling:
>>> deeplabcut.plot_trajectories(
'C:\\myproject\\reaching-task\\config.yaml',
['C:\\myproject\\trailtracking-task\\test.mp4'],
shuffle=3,
filtered=True,
)
>>> deeplabcut.create_labeled_video(
'C:\\myproject\\reaching-task\\config.yaml',
['C:\\myproject\\trailtracking-task\\test.mp4'],
shuffle=3,
filtered=True,
)
"""
cfg = auxiliaryfunctions.read_config(config)
track_method = auxfun_multianimal.get_track_method(cfg, track_method=track_method)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction=cfg["TrainingFraction"][trainingsetindex],
modelprefix=modelprefix,
)
Videos = auxiliaryfunctions.get_list_of_videos(video, videotype)
if not len(Videos):
print("No video(s) were found. Please check your paths and/or 'videotype'.")
return
for video in Videos:
if destfolder is None:
destfolder = str(Path(video).parents[0])
print("Filtering with %s model %s" % (filtertype, video))
vname = Path(video).stem
try:
_ = auxiliaryfunctions.load_analyzed_data(
destfolder, vname, DLCscorer, True, track_method
)
print(f"Data from {vname} were already filtered. Skipping...")
except FileNotFoundError: # Data haven't been filtered yet
try:
df, filepath, _, _ = auxiliaryfunctions.load_analyzed_data(
destfolder, vname, DLCscorer, track_method=track_method
)
nrows = df.shape[0]
if filtertype == "arima":
temp = df.values.reshape((nrows, -1, 3))
placeholder = np.empty_like(temp)
for i in range(temp.shape[1]):
x, y, p = temp[:, i].T
meanx, _ = FitSARIMAXModel(
x, p, p_bound, alpha, ARdegree, MAdegree, False
)
meany, _ = FitSARIMAXModel(
y, p, p_bound, alpha, ARdegree, MAdegree, False
)
meanx[0] = x[0]
meany[0] = y[0]
placeholder[:, i] = np.c_[meanx, meany, p]
data = pd.DataFrame(
placeholder.reshape((nrows, -1)),
columns=df.columns,
index=df.index,
)
elif filtertype == "median":
data = df.copy()
mask = data.columns.get_level_values("coords") != "likelihood"
data.loc[:, mask] = df.loc[:, mask].apply(
signal.medfilt, args=(windowlength,), axis=0
)
elif filtertype == "spline":
data = df.copy()
mask_data = data.columns.get_level_values("coords").isin(("x", "y"))
xy = data.loc[:, mask_data].values
prob = data.loc[:, ~mask_data].values
missing = np.isnan(xy)
xy_filled = columnwise_spline_interp(xy, windowlength)
filled = ~np.isnan(xy_filled)
xy[filled] = xy_filled[filled]
inds = np.argwhere(missing & filled)
if inds.size:
# Retrieve original individual label indices
inds[:, 1] //= 2
inds = np.unique(inds, axis=0)
prob[inds[:, 0], inds[:, 1]] = 0.01
data.loc[:, ~mask_data] = prob
data.loc[:, mask_data] = xy
else:
raise ValueError(f"Unknown filter type {filtertype}")
outdataname = filepath.replace(".h5", "_filtered.h5")
data.to_hdf(outdataname, "df_with_missing", format="table", mode="w")
if save_as_csv:
print("Saving filtered csv poses!")
data.to_csv(outdataname.split(".h5")[0] + ".csv")
except FileNotFoundError as e:
print(e)
continue
def test_get_list_of_videos(tmpdir_factory):
fake_folder = tmpdir_factory.mktemp('videos')
n_ext = len(SUPPORTED_VIDEOS)
def _create_fake_file(filename):
path = str(fake_folder.join(filename))
with open(path, "w") as f:
f.write("")
return path
fake_videos = []
for ext in SUPPORTED_VIDEOS:
path = _create_fake_file(f"fake.{ext}")
fake_videos.append(path)
# Add some other office files:
path = _create_fake_file("fake.xls")
path = _create_fake_file("fake.pptx")
# Add a .pickle and .h5 files
_ = _create_fake_file("fake.pickle")
_ = _create_fake_file("fake.h5")
# By default, all videos with common extensions are taken from a directory
videos = auxiliaryfunctions.get_list_of_videos(
str(fake_folder),
videotype="",
)
assert len(videos) == n_ext
# A list of extensions can also be passed in
videos = auxiliaryfunctions.get_list_of_videos(
str(fake_folder),
videotype=SUPPORTED_VIDEOS,
)
assert len(videos) == n_ext
for ext in SUPPORTED_VIDEOS:
videos = auxiliaryfunctions.get_list_of_videos(
str(fake_folder),
videotype=ext,
)
assert len(videos) == 1
videos = auxiliaryfunctions.get_list_of_videos(
str(fake_folder),
videotype="unknown",
)
assert not len(videos)
videos = auxiliaryfunctions.get_list_of_videos(
fake_videos,
videotype="",
)
assert len(videos) == n_ext
for video in fake_videos:
videos = auxiliaryfunctions.get_list_of_videos([video], videotype="")
assert len(videos) == 1
for ext in SUPPORTED_VIDEOS:
videos = auxiliaryfunctions.get_list_of_videos(
fake_videos,
videotype=ext,
)
assert len(videos) == 1
def create_video_with_all_detections(
config,
videos,
videotype="",
shuffle=1,
trainingsetindex=0,
displayedbodyparts="all",
destfolder=None,
modelprefix="",
):
"""
Create a video labeled with all the detections stored in a '*_full.pickle' file.
Parameters
----------
config : str
Absolute path to the config.yaml file
videos : list of str
A list of strings containing the full paths to videos for analysis or a path to the directory,
where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed.
If left unspecified, videos with common extensions ('avi', 'mp4', 'mov', 'mpeg', 'mkv') are kept.
shuffle : int, optional
Number of shuffles of training dataset. Default is set to 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
displayedbodyparts: list of strings, optional
This selects the body parts that are plotted in the video. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
"""
from deeplabcut.pose_estimation_tensorflow.lib.inferenceutils import Assembler
import re
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
DLCscorername, _ = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, modelprefix=modelprefix)
videos = auxiliaryfunctions.get_list_of_videos(videos, videotype)
if not videos:
return
for video in videos:
videofolder = os.path.splitext(video)[0]
if destfolder is None:
outputname = "{}_full.mp4".format(videofolder + DLCscorername)
full_pickle = os.path.join(videofolder + DLCscorername +
"_full.pickle")
else:
auxiliaryfunctions.attempttomakefolder(destfolder)
outputname = os.path.join(
destfolder,
str(Path(video).stem) + DLCscorername + "_full.mp4")
full_pickle = os.path.join(
destfolder,
str(Path(video).stem) + DLCscorername + "_full.pickle")
if not (os.path.isfile(outputname)):
print("Creating labeled video for ", str(Path(video).stem))
h5file = full_pickle.replace("_full.pickle", ".h5")
data, _ = auxfun_multianimal.LoadFullMultiAnimalData(h5file)
data = dict(
data
) # Cast to dict (making a copy) so items can safely be popped
header = data.pop("metadata")
all_jointnames = header["all_joints_names"]
if displayedbodyparts == "all":
numjoints = len(all_jointnames)
bpts = range(numjoints)
else: # select only "displayedbodyparts"
bpts = []
for bptindex, bp in enumerate(all_jointnames):
if bp in displayedbodyparts:
bpts.append(bptindex)
numjoints = len(bpts)
frame_names = list(data)
frames = [int(re.findall(r"\d+", name)[0]) for name in frame_names]
colorclass = plt.cm.ScalarMappable(cmap=cfg["colormap"])
C = colorclass.to_rgba(np.linspace(0, 1, numjoints))
colors = (C[:, :3] * 255).astype(np.uint8)
pcutoff = cfg["pcutoff"]
dotsize = cfg["dotsize"]
clip = vp(fname=video, sname=outputname, codec="mp4v")
ny, nx = clip.height(), clip.width()
for n in trange(clip.nframes):
frame = clip.load_frame()
if frame is None:
continue
try:
ind = frames.index(n)
dets = Assembler._flatten_detections(
data[frame_names[ind]])
for det in dets:
if det.label not in bpts or det.confidence < pcutoff:
continue
x, y = det.pos
rr, cc = disk((y, x), dotsize, shape=(ny, nx))
frame[rr, cc] = colors[bpts.index(det.label)]
except ValueError: # No data stored for that particular frame
print(n, "no data")
pass
try:
clip.save_frame(frame)
except:
print(n, "frame writing error.")
pass
clip.close()
else:
print("Detections already plotted, ", outputname)
def create_labeled_video(
config,
videos,
videotype="",
shuffle=1,
trainingsetindex=0,
filtered=False,
fastmode=True,
save_frames=False,
keypoints_only=False,
Frames2plot=None,
displayedbodyparts="all",
displayedindividuals="all",
codec="mp4v",
outputframerate=None,
destfolder=None,
draw_skeleton=False,
trailpoints=0,
displaycropped=False,
color_by="bodypart",
modelprefix="",
track_method="",
):
"""Labels the bodyparts in a video.
Make sure the video is already analyzed by the function
``deeplabcut.analyze_videos``.
Parameters
----------
config : string
Full path of the config.yaml file.
videos : list[str]
A list of strings containing the full paths to videos for analysis or a path
to the directory, where all the videos with same extension are stored.
videotype: str, optional, default=""
Checks for the extension of the video in case the input to the video is a
directory. Only videos with this extension are analyzed.
If left unspecified, videos with common extensions
('avi', 'mp4', 'mov', 'mpeg', 'mkv') are kept.
shuffle : int, optional, default=1
Number of shuffles of training dataset.
trainingsetindex: int, optional, default=0
Integer specifying which TrainingsetFraction to use.
Note that TrainingFraction is a list in config.yaml.
filtered: bool, optional, default=False
Boolean variable indicating if filtered output should be plotted rather than
frame-by-frame predictions. Filtered version can be calculated with
``deeplabcut.filterpredictions``.
fastmode: bool, optional, default=True
If ``True``, uses openCV (much faster but less customization of video) instead
of matplotlib if ``False``. You can also "save_frames" individually or not in
the matplotlib mode (if you set the "save_frames" variable accordingly).
However, using matplotlib to create the frames it therefore allows much more
flexible (one can set transparency of markers, crop, and easily customize).
save_frames: bool, optional, default=False
If ``True``, creates each frame individual and then combines into a video.
Setting this to ``True`` is relatively slow as it stores all individual frames.
keypoints_only: bool, optional, default=False
By default, both video frames and keypoints are visible. If ``True``, only the
keypoints are shown. These clips are an hommage to Johansson movies,
see https://www.youtube.com/watch?v=1F5ICP9SYLU and of course his seminal
paper: "Visual perception of biological motion and a model for its analysis"
by Gunnar Johansson in Perception & Psychophysics 1973.
Frames2plot: List[int] or None, optional, default=None
If not ``None`` and ``save_frames=True`` then the frames corresponding to the
index will be plotted. For example, ``Frames2plot=[0,11]`` will plot the first
and the 12th frame.
displayedbodyparts: list[str] or str, optional, default="all"
This selects the body parts that are plotted in the video. If ``all``, then all
body parts from config.yaml are used. If a list of strings that are a subset of
the full list. E.g. ['hand','Joystick'] for the demo
Reaching-Mackenzie-2018-08-30/config.yaml to select only these body parts.
displayedindividuals: list[str] or str, optional, default="all"
Individuals plotted in the video.
By default, all individuals present in the config will be showed.
codec: str, optional, default="mp4v"
Codec for labeled video. For available options, see
http://www.fourcc.org/codecs.php. Note that this depends on your ffmpeg
installation.
outputframerate: int or None, optional, default=None
Positive number, output frame rate for labeled video (only available for the
mode with saving frames.) If ``None``, which results in the original video
rate.
destfolder: string or None, optional, default=None
Specifies the destination folder that was used for storing analysis data. If
``None``, the path of the video file is used.
draw_skeleton: bool, optional, default=False
If ``True`` adds a line connecting the body parts making a skeleton on each
frame. The body parts to be connected and the color of these connecting lines
are specified in the config file.
trailpoints: int, optional, default=0
Number of previous frames whose body parts are plotted in a frame
(for displaying history).
displaycropped: bool, optional, default=False
Specifies whether only cropped frame is displayed (with labels analyzed
therein), or the original frame with the labels analyzed in the cropped subset.
color_by : string, optional, default='bodypart'
Coloring rule. By default, each bodypart is colored differently.
If set to 'individual', points belonging to a single individual are colored the
same.
modelprefix: str, optional, default=""
Directory containing the deeplabcut models to use when evaluating the network.
By default, the models are assumed to exist in the project folder.
track_method: string, optional, default=""
Specifies the tracker used to generate the data.
Empty by default (corresponding to a single animal project).
For multiple animals, must be either 'box', 'skeleton', or 'ellipse' and will
be taken from the config.yaml file if none is given.
Returns
-------
None
Examples
--------
Create the labeled video for a single video
>>> deeplabcut.create_labeled_video(
'/analysis/project/reaching-task/config.yaml',
['/analysis/project/videos/reachingvideo1.avi'],
)
Create the labeled video for a single video and store the individual frames
>>> deeplabcut.create_labeled_video(
'/analysis/project/reaching-task/config.yaml',
['/analysis/project/videos/reachingvideo1.avi'],
fastmode=True,
save_frames=True,
)
Create the labeled video for multiple videos
>>> deeplabcut.create_labeled_video(
'/analysis/project/reaching-task/config.yaml',
[
'/analysis/project/videos/reachingvideo1.avi',
'/analysis/project/videos/reachingvideo2.avi',
],
)
Create the labeled video for all the videos with an .avi extension in a directory.
>>> deeplabcut.create_labeled_video(
'/analysis/project/reaching-task/config.yaml',
['/analysis/project/videos/'],
)
Create the labeled video for all the videos with an .mp4 extension in a directory.
>>> deeplabcut.create_labeled_video(
'/analysis/project/reaching-task/config.yaml',
['/analysis/project/videos/'],
videotype='mp4',
)
"""
cfg = auxiliaryfunctions.read_config(config)
track_method = auxfun_multianimal.get_track_method(
cfg, track_method=track_method)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, modelprefix=modelprefix
) # automatically loads corresponding model (even training iteration based on snapshot index)
if save_frames:
fastmode = False # otherwise one cannot save frames
keypoints_only = False
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
individuals = auxfun_multianimal.IntersectionofIndividualsandOnesGivenbyUser(
cfg, displayedindividuals)
if draw_skeleton:
bodyparts2connect = cfg["skeleton"]
skeleton_color = cfg["skeleton_color"]
else:
bodyparts2connect = None
skeleton_color = None
start_path = os.getcwd()
Videos = auxiliaryfunctions.get_list_of_videos(videos, videotype)
if not Videos:
return
func = partial(
proc_video,
videos,
destfolder,
filtered,
DLCscorer,
DLCscorerlegacy,
track_method,
cfg,
individuals,
color_by,
bodyparts,
codec,
bodyparts2connect,
trailpoints,
save_frames,
outputframerate,
Frames2plot,
draw_skeleton,
skeleton_color,
displaycropped,
fastmode,
keypoints_only,
)
with Pool(min(os.cpu_count(), len(Videos))) as pool:
pool.map(func, Videos)
os.chdir(start_path)
def plot_trajectories(
config,
videos,
videotype="",
shuffle=1,
trainingsetindex=0,
filtered=False,
displayedbodyparts="all",
displayedindividuals="all",
showfigures=False,
destfolder=None,
modelprefix="",
imagetype=".png",
resolution=100,
linewidth=1.0,
track_method="",
):
"""Plots the trajectories of various bodyparts across the video.
Parameters
----------
config: str
Full path of the config.yaml file.
videos: list[str]
Full paths to videos for analysis or a path to the directory, where all the
videos with same extension are stored.
videotype: str, optional, default=""
Checks for the extension of the video in case the input to the video is a
directory. Only videos with this extension are analyzed.
If left unspecified, videos with common extensions
('avi', 'mp4', 'mov', 'mpeg', 'mkv') are kept.
shuffle: int, optional, default=1
Integer specifying the shuffle index of the training dataset.
trainingsetindex: int, optional, default=0
Integer specifying which TrainingsetFraction to use.
Note that TrainingFraction is a list in config.yaml.
filtered: bool, optional, default=False
Boolean variable indicating if filtered output should be plotted rather than
frame-by-frame predictions. Filtered version can be calculated with
``deeplabcut.filterpredictions``.
displayedbodyparts: list[str] or str, optional, default="all"
This select the body parts that are plotted in the video.
Either ``all``, then all body parts from config.yaml are used,
or a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml
to select only these two body parts.
showfigures: bool, optional, default=False
If ``True`` then plots are also displayed.
destfolder: string or None, optional, default=None
Specifies the destination folder that was used for storing analysis data. If
``None``, the path of the video is used.
modelprefix: str, optional, default=""
Directory containing the deeplabcut models to use when evaluating the network.
By default, the models are assumed to exist in the project folder.
imagetype: string, optional, default=".png"
Specifies the output image format - '.tif', '.jpg', '.svg' and ".png".
resolution: int, optional, default=100
Specifies the resolution (in dpi) of saved figures.
Note higher resolution figures take longer to generate.
linewidth: float, optional, default=1.0
Specifies width of line for line and histogram plots.
track_method: string, optional, default=""
Specifies the tracker used to generate the data.
Empty by default (corresponding to a single animal project).
For multiple animals, must be either 'box', 'skeleton', or 'ellipse' and will
be taken from the config.yaml file if none is given.
Returns
-------
None
Examples
--------
To label the frames
>>> deeplabcut.plot_trajectories(
'home/alex/analysis/project/reaching-task/config.yaml',
['/home/alex/analysis/project/videos/reachingvideo1.avi'],
)
"""
cfg = auxiliaryfunctions.read_config(config)
track_method = auxfun_multianimal.get_track_method(cfg, track_method=track_method)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, modelprefix=modelprefix
) # automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts
)
individuals = auxfun_multianimal.IntersectionofIndividualsandOnesGivenbyUser(
cfg, displayedindividuals
)
Videos = auxiliaryfunctions.get_list_of_videos(videos, videotype)
if not len(Videos):
print(
"No videos found. Make sure you passed a list of videos and that *videotype* is right."
)
return
failed = []
for video in Videos:
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
vname = str(Path(video).stem)
print("Loading ", video, "and data.")
try:
df, _, _, suffix = auxiliaryfunctions.load_analyzed_data(
videofolder, vname, DLCscorer, filtered, track_method
)
failed.append(False)
tmpfolder = os.path.join(videofolder, "plot-poses", vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder, recursive=True)
# Keep only the individuals and bodyparts that were labeled
labeled_bpts = [
bp
for bp in df.columns.get_level_values("bodyparts").unique()
if bp in bodyparts
]
# Either display the animals defined in the config if they are found
# in the dataframe, or all the trajectories regardless of their names
try:
animals = set(df.columns.get_level_values("individuals"))
except KeyError:
animals = {""}
for animal in animals.intersection(individuals) or animals:
PlottingResults(
tmpfolder,
df,
cfg,
labeled_bpts,
animal,
showfigures,
suffix + animal + imagetype,
resolution=resolution,
linewidth=linewidth,
)
except FileNotFoundError as e:
failed.append(True)
print(e)
try:
_ = auxiliaryfunctions.load_detection_data(
video, DLCscorer, track_method
)
print(
'Call "deeplabcut.stitch_tracklets()"'
" prior to plotting the trajectories."
)
except FileNotFoundError as e:
print(e)
print(
f"Make sure {video} was previously analyzed, and that "
f'detections were successively converted to tracklets using "deeplabcut.convert_detections2tracklets()" '
f'and "deeplabcut.stitch_tracklets()".'
)
if not all(failed):
print(
'Plots created! Please check the directory "plot-poses" within the video directory'
)
else:
print(
f"Plots could not be created! "
f"Videos were not evaluated with the current scorer {DLCscorer}."
)