def plot_trajectories(config,
videos,
videotype='.avi',
shuffle=1,
trainingsetindex=0,
showfigures=False):
"""
Plots the trajectories of various bodyparts across the video.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
showfigures: bool, default false
If true then plots are also displayed.
Example
--------
for labeling the frames
>>> deeplabcut.plot_trajectories('home/alex/analysis/project/reaching-task/config.yaml',['/home/alex/analysis/project/videos/reachingvideo1.avi'])
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
DLCscorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction
) #automatically loads corresponding model (even training iteration based on snapshot index)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
print(video)
videofolder = str(
Path(video).parents[0]) #where your folder with videos is.
videotype = str(Path(video).suffix)
print("Starting % ", videofolder, videos)
basefolder = videofolder
auxiliaryfunctions.attempttomakefolder(basefolder)
RunTrajectoryAnalysis(video, basefolder, DLCscorer, videofolder, cfg,
showfigures)
print(
'Plots created! Please check the directory "plot-poses" within the video directory'
)
def create_labeled_video(
config,
videos,
videotype="avi",
shuffle=1,
trainingsetindex=0,
filtered=False,
fastmode=True,
save_frames=False,
keypoints_only=False,
Frames2plot=None,
displayedbodyparts="all",
displayedindividuals="all",
codec="mp4v",
outputframerate=None,
destfolder=None,
draw_skeleton=False,
trailpoints=0,
displaycropped=False,
color_by="bodypart",
modelprefix="",
track_method="",
):
"""
Labels the bodyparts in a video. Make sure the video is already analyzed by the function 'analyze_video'
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
Number of shuffles of training dataset. Default is set to 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
fastmode: bool
If true uses openCV (much faster but less customization of video) vs matplotlib (if false). You can also
"save_frames" individually or not in the matplotlib mode (if you set the "save_frames" variable accordingly).
However, using matplotlib to create the frames it therefore allows much more flexible (one can set transparency of markers, crop, and easily customize).
save_frames: bool
If true creates each frame individual and then combines into a video. This variant is relatively slow as
it stores all individual frames.
keypoints_only: bool, optional
By default, both video frames and keypoints are visible. If true, only the keypoints are shown. These clips are an hommage to Johansson movies,
see https://www.youtube.com/watch?v=1F5ICP9SYLU and of course his seminal paper: "Visual perception of biological motion and a model for its analysis"
by Gunnar Johansson in Perception & Psychophysics 1973.
Frames2plot: List of indices
If not None & save_frames=True then the frames corresponding to the index will be plotted. For example, Frames2plot=[0,11] will plot the first and the 12th frame.
displayedbodyparts: list of strings, optional
This selects the body parts that are plotted in the video. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
displayedindividuals: list of strings, optional
Individuals plotted in the video. By default, all individuals present in the config will be showed.
codec: codec for labeled video. Options see http://www.fourcc.org/codecs.php [depends on your ffmpeg installation.]
outputframerate: positive number, output frame rate for labeled video (only available for the mode with saving frames.) By default: None, which results in the original video rate.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
draw_skeleton: bool
If ``True`` adds a line connecting the body parts making a skeleton on on each frame. The body parts to be connected and the color of these connecting lines are specified in the config file. By default: ``False``
trailpoints: int
Number of revious frames whose body parts are plotted in a frame (for displaying history). Default is set to 0.
displaycropped: bool, optional
Specifies whether only cropped frame is displayed (with labels analyzed therein), or the original frame with the labels analyzed in the cropped subset.
color_by : string, optional (default='bodypart')
Coloring rule. By default, each bodypart is colored differently.
If set to 'individual', points belonging to a single individual are colored the same.
Examples
--------
If you want to create the labeled video for only 1 video
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to create the labeled video for only 1 video and store the individual frames
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'],fastmode=True, save_frames=True)
--------
If you want to create the labeled video for multiple videos
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to create the labeled video for all the videos (as .avi extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'])
--------
If you want to create the labeled video for all the videos (as .mp4 extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'],videotype='mp4')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, modelprefix=modelprefix
) # automatically loads corresponding model (even training iteration based on snapshot index)
if save_frames:
fastmode = False # otherwise one cannot save frames
keypoints_only = False
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
individuals = auxfun_multianimal.IntersectionofIndividualsandOnesGivenbyUser(
cfg, displayedindividuals)
if draw_skeleton:
bodyparts2connect = cfg["skeleton"]
skeleton_color = cfg["skeleton_color"]
else:
bodyparts2connect = None
skeleton_color = None
start_path = os.getcwd()
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
if not Videos:
print(
"No video(s) were found. Please check your paths and/or 'video_type'."
)
return
func = partial(
proc_video,
videos,
destfolder,
filtered,
DLCscorer,
DLCscorerlegacy,
track_method,
cfg,
individuals,
color_by,
bodyparts,
codec,
bodyparts2connect,
trailpoints,
save_frames,
outputframerate,
Frames2plot,
draw_skeleton,
skeleton_color,
displaycropped,
fastmode,
keypoints_only,
)
with Pool(min(os.cpu_count(), len(Videos))) as pool:
pool.map(func, Videos)
os.chdir(start_path)
def extract_outlier_frames(
config,
videos,
videotype=".avi",
shuffle=1,
trainingsetindex=0,
outlieralgorithm="jump",
comparisonbodyparts="all",
epsilon=20,
p_bound=0.01,
ARdegree=3,
MAdegree=1,
alpha=0.01,
extractionalgorithm="kmeans",
automatic=False,
cluster_resizewidth=30,
cluster_color=False,
opencv=True,
savelabeled=False,
destfolder=None,
modelprefix="",
track_method="",
):
"""
Extracts the outlier frames in case, the predictions are not correct for a certain video from the cropped video running from
start to stop as defined in config.yaml.
Another crucial parameter in config.yaml is how many frames to extract 'numframes2extract'.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
The shuffle index of training dataset. The extracted frames will be stored in the labeled-dataset for
the corresponding shuffle of training dataset. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
outlieralgorithm: 'fitting', 'jump', 'uncertain', or 'manual'
String specifying the algorithm used to detect the outliers. Currently, deeplabcut supports three methods + a manual GUI option. 'Fitting'
fits a Auto Regressive Integrated Moving Average model to the data and computes the distance to the estimated data. Larger distances than
epsilon are then potentially identified as outliers. The methods 'jump' identifies larger jumps than 'epsilon' in any body part; and 'uncertain'
looks for frames with confidence below p_bound. The default is set to ``jump``.
comparisonbodyparts: list of strings, optional
This selects the body parts for which the comparisons with the outliers are carried out. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
p_bound: float between 0 and 1, optional
For outlieralgorithm 'uncertain' this parameter defines the likelihood below, below which a body part will be flagged as a putative outlier.
epsilon; float,optional
Meaning depends on outlieralgoritm. The default is set to 20 pixels.
For outlieralgorithm 'fitting': Float bound according to which frames are picked when the (average) body part estimate deviates from model fit
For outlieralgorithm 'jump': Float bound specifying the distance by which body points jump from one frame to next (Euclidean distance)
ARdegree: int, optional
For outlieralgorithm 'fitting': Autoregressive degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part)
see https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
MAdegree: int
For outlieralgorithm 'fitting': MovingAvarage degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part)
See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
alpha: float
Significance level for detecting outliers based on confidence interval of fitted ARIMA model. Only the distance is used however.
extractionalgorithm : string, optional
String specifying the algorithm to use for selecting the frames from the identified putatative outlier frames. Currently, deeplabcut
supports either ``kmeans`` or ``uniform`` based selection (same logic as for extract_frames).
The default is set to``uniform``, if provided it must be either ``uniform`` or ``kmeans``.
automatic : bool, optional
Set it to True, if you want to extract outliers without being asked for user feedback.
cluster_resizewidth: number, default: 30
For k-means one can change the width to which the images are downsampled (aspect ratio is fixed).
cluster_color: bool, default: False
If false then each downsampled image is treated as a grayscale vector (discarding color information). If true, then the color channels are considered. This increases
the computational complexity.
opencv: bool, default: True
Uses openCV for loading & extractiong (otherwise moviepy (legacy))
savelabeled: bool, default: False
If true also saves frame with predicted labels in each folder.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
track_method: string, optional
Specifies the tracker used to generate the data. Empty by default (corresponding to a single animal project).
For multiple animals, must be either 'box', 'skeleton', or 'ellipse' and will be taken from the config.yaml file if none is given.
Examples
Windows example for extracting the frames with default settings
>>> deeplabcut.extract_outlier_frames('C:\\myproject\\reaching-task\\config.yaml',['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi'])
--------
for extracting the frames with default settings
>>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'])
--------
for extracting the frames with kmeans
>>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'],extractionalgorithm='kmeans')
--------
for extracting the frames with kmeans and epsilon = 5 pixels.
>>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'],epsilon = 5,extractionalgorithm='kmeans')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
if not len(bodyparts):
raise ValueError("No valid bodyparts were selected.")
track_method = auxfun_multianimal.get_track_method(
cfg, track_method=track_method)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction=cfg["TrainingFraction"][trainingsetindex],
modelprefix=modelprefix,
)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
if len(Videos) == 0:
print("No suitable videos found in", videos)
for video in Videos:
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
vname = os.path.splitext(os.path.basename(video))[0]
try:
df, dataname, _, _ = auxiliaryfunctions.load_analyzed_data(
videofolder, vname, DLCscorer, track_method=track_method)
nframes = len(df)
startindex = max([int(np.floor(nframes * cfg["start"])), 0])
stopindex = min([int(np.ceil(nframes * cfg["stop"])), nframes])
Index = np.arange(stopindex - startindex) + startindex
df = df.iloc[Index]
mask = df.columns.get_level_values("bodyparts").isin(bodyparts)
df_temp = df.loc[:, mask]
Indices = []
if outlieralgorithm == "uncertain":
p = df_temp.xs("likelihood", level="coords", axis=1)
ind = df_temp.index[(p < p_bound).any(axis=1)].tolist()
Indices.extend(ind)
elif outlieralgorithm == "jump":
temp_dt = df_temp.diff(axis=0)**2
temp_dt.drop("likelihood",
axis=1,
level="coords",
inplace=True)
sum_ = temp_dt.sum(axis=1, level=1)
ind = df_temp.index[(sum_ > epsilon**2).any(axis=1)].tolist()
Indices.extend(ind)
elif outlieralgorithm == "fitting":
d, o = compute_deviations(df_temp, dataname, p_bound, alpha,
ARdegree, MAdegree)
# Some heuristics for extracting frames based on distance:
ind = np.flatnonzero(
d > epsilon
) # time points with at least average difference of epsilon
if (
len(ind) < cfg["numframes2pick"] * 2
and len(d) > cfg["numframes2pick"] * 2
): # if too few points qualify, extract the most distant ones.
ind = np.argsort(d)[::-1][:cfg["numframes2pick"] * 2]
Indices.extend(ind)
elif outlieralgorithm == "manual":
wd = Path(config).resolve().parents[0]
os.chdir(str(wd))
from deeplabcut.gui import outlier_frame_extraction_toolbox
outlier_frame_extraction_toolbox.show(
config,
video,
shuffle,
df,
savelabeled,
cfg.get("multianimalproject", False),
)
# Run always except when the outlieralgorithm == manual.
if not outlieralgorithm == "manual":
Indices = np.sort(list(set(Indices))) # remove repetitions.
print(
"Method ",
outlieralgorithm,
" found ",
len(Indices),
" putative outlier frames.",
)
print(
"Do you want to proceed with extracting ",
cfg["numframes2pick"],
" of those?",
)
if outlieralgorithm == "uncertain" or outlieralgorithm == "jump":
print(
"If this list is very large, perhaps consider changing the parameters "
"(start, stop, p_bound, comparisonbodyparts) or use a different method."
)
elif outlieralgorithm == "fitting":
print(
"If this list is very large, perhaps consider changing the parameters "
"(start, stop, epsilon, ARdegree, MAdegree, alpha, comparisonbodyparts) "
"or use a different method.")
if not automatic:
askuser = input("yes/no")
else:
askuser = "******"
if (askuser == "y" or askuser == "yes" or askuser == "Ja"
or askuser == "ha"): # multilanguage support :)
# Now extract from those Indices!
ExtractFramesbasedonPreselection(
Indices,
extractionalgorithm,
df,
video,
cfg,
config,
opencv,
cluster_resizewidth,
cluster_color,
savelabeled,
)
else:
print(
"Nothing extracted, please change the parameters and start again..."
)
except FileNotFoundError as e:
print(e)
print(
"It seems the video has not been analyzed yet, or the video is not found! "
"You can only refine the labels after the a video is analyzed. Please run 'analyze_video' first. "
"Or, please double check your video file path")
def analyzeskeleton(
config,
videos,
videotype="avi",
shuffle=1,
trainingsetindex=0,
filtered=False,
save_as_csv=False,
destfolder=None,
modelprefix="",
track_method="",
):
"""
Extracts length and orientation of each "bone" of the skeleton as defined in the config file.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
shuffle : int, optional
The shufle index of training dataset. The extracted frames will be stored in the labeled-dataset for
the corresponding shuffle of training dataset. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this
folder also needs to be passed.
track_method: string, optional
Specifies the tracker used to generate the data. Empty by default (corresponding to
a single animal project). For multiple animals, must be either 'box', 'skeleton', or 'ellipse'.
"""
# Load config file, scorer and videos
cfg = auxiliaryfunctions.read_config(config)
if not cfg["skeleton"]:
raise ValueError("No skeleton defined in the config.yaml.")
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction=cfg["TrainingFraction"][trainingsetindex],
modelprefix=modelprefix,
)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
print("Processing %s" % (video))
if destfolder is None:
destfolder = str(Path(video).parents[0])
vname = Path(video).stem
try:
df, filepath, scorer, _ = auxiliaryfunctions.load_analyzed_data(
destfolder, vname, DLCscorer, filtered, track_method)
output_name = filepath.replace(".h5", f"_skeleton.h5")
if os.path.isfile(output_name):
print(
f"Skeleton in video {vname} already processed. Skipping..."
)
continue
bones = {}
if "individuals" in df.columns.names:
for animal_name, df_ in df.groupby(level="individuals",
axis=1):
temp = df_.droplevel(["scorer", "individuals"], axis=1)
if animal_name != "single":
for bp1, bp2 in cfg["skeleton"]:
name = "{}_{}_{}".format(animal_name, bp1, bp2)
bones[name] = analyzebone(temp[bp1], temp[bp2])
else:
for bp1, bp2 in cfg["skeleton"]:
name = "{}_{}".format(bp1, bp2)
bones[name] = analyzebone(df[scorer][bp1], df[scorer][bp2])
skeleton = pd.concat(bones, axis=1)
skeleton.to_hdf(output_name,
"df_with_missing",
format="table",
mode="w")
if save_as_csv:
skeleton.to_csv(output_name.replace(".h5", ".csv"))
except FileNotFoundError as e:
print(e)
continue
def create_video_with_all_detections(
config,
videos,
videotype="avi",
shuffle=1,
trainingsetindex=0,
displayedbodyparts="all",
destfolder=None,
modelprefix="",
):
"""
Create a video labeled with all the detections stored in a '*_full.pickle' file.
Parameters
----------
config : str
Absolute path to the config.yaml file
videos : list of str
A list of strings containing the full paths to videos for analysis or a path to the directory,
where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
Number of shuffles of training dataset. Default is set to 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
displayedbodyparts: list of strings, optional
This selects the body parts that are plotted in the video. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
"""
from deeplabcut.pose_estimation_tensorflow.lib.inferenceutils import Assembler
import pickle, re
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
DLCscorername, _ = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, modelprefix=modelprefix
)
videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
if not videos:
print("No video(s) were found. Please check your paths and/or 'video_type'.")
return
for video in videos:
videofolder = os.path.splitext(video)[0]
if destfolder is None:
outputname = "{}_full.mp4".format(videofolder + DLCscorername)
full_pickle = os.path.join(videofolder + DLCscorername + "_full.pickle")
else:
auxiliaryfunctions.attempttomakefolder(destfolder)
outputname = os.path.join(
destfolder, str(Path(video).stem) + DLCscorername + "_full.mp4"
)
full_pickle = os.path.join(
destfolder, str(Path(video).stem) + DLCscorername + "_full.pickle"
)
if not (os.path.isfile(outputname)):
print("Creating labeled video for ", str(Path(video).stem))
h5file = full_pickle.replace("_full.pickle", ".h5")
data, _ = auxfun_multianimal.LoadFullMultiAnimalData(h5file)
data = dict(data) # Cast to dict (making a copy) so items can safely be popped
header = data.pop("metadata")
all_jointnames = header["all_joints_names"]
if displayedbodyparts == "all":
numjoints = len(all_jointnames)
bpts = range(numjoints)
else: # select only "displayedbodyparts"
bpts = []
for bptindex, bp in enumerate(all_jointnames):
if bp in displayedbodyparts:
bpts.append(bptindex)
numjoints = len(bpts)
frame_names = list(data)
frames = [int(re.findall(r"\d+", name)[0]) for name in frame_names]
colorclass = plt.cm.ScalarMappable(cmap=cfg["colormap"])
C = colorclass.to_rgba(np.linspace(0, 1, numjoints))
colors = (C[:, :3] * 255).astype(np.uint8)
pcutoff = cfg["pcutoff"]
dotsize = cfg["dotsize"]
clip = vp(fname=video, sname=outputname, codec="mp4v")
ny, nx = clip.height(), clip.width()
for n in trange(clip.nframes):
frame = clip.load_frame()
if frame is None:
continue
try:
ind = frames.index(n)
dets = Assembler._flatten_detections(data[frame_names[ind]])
for det in dets:
if det.label not in bpts or det.confidence < pcutoff:
continue
x, y = det.pos
rr, cc = disk((y, x), dotsize, shape=(ny, nx))
frame[rr, cc] = colors[bpts.index(det.label)]
except ValueError: # No data stored for that particular frame
print(n, "no data")
pass
try:
clip.save_frame(frame)
except:
print(n, "frame writing error.")
pass
clip.close()
else:
print("Detections already plotted, ", outputname)
def analyze_videos(config,
videos,
videotype='avi',
shuffle=1,
trainingsetindex=0,
gputouse=None,
save_as_csv=False,
destfolder=None,
cropping=None):
"""
Makes prediction based on a trained network. The index of the trained network is specified by parameters in the config file (in particular the variable 'snapshotindex')
You can crop the video (before analysis), by changing 'cropping'=True and setting 'x1','x2','y1','y2' in the config file. The same cropping parameters will then be used for creating the video.
Note: you can also pass cropping = [x1,x2,y1,y2] coordinates directly, that then will be used for all videos. You can of course loop over videos & pass specific coordinates for each case.
Output: The labels are stored as MultiIndex Pandas Array, which contains the name of the network, body part name, (x, y) label position \n
in pixels, and the likelihood for each frame per body part. These arrays are stored in an efficient Hierarchical Data Format (HDF) \n
in the same directory, where the video is stored. However, if the flag save_as_csv is set to True, the data can also be exported in \n
comma-separated values format (.csv), which in turn can be imported in many programs, such as MATLAB, R, Prism, etc.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: int, optional
An integer specifying the shuffle index of the training dataset used for training the network. The default is 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this
folder also needs to be passed.
Examples
--------
Windows example for analyzing 1 video
>>> deeplabcut.analyze_videos('C:\\myproject\\reaching-task\\config.yaml',['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi'])
--------
If you want to analyze only 1 video
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to analyze all videos of type avi in a folder:
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos'],videotype='.avi')
--------
If you want to analyze multiple videos
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to analyze multiple videos with shuffle = 2
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'], shuffle=2)
--------
If you want to analyze multiple videos with shuffle = 2 and save results as an additional csv file too
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'], shuffle=2,save_as_csv=True)
--------
"""
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ[
'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
if gputouse is not None: #gpu selection
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
vers = (tf.__version__).split('.')
if int(vers[0]) == 1 and int(vers[1]) > 12:
TF = tf.compat.v1
else:
TF = tf
TF.reset_default_graph()
start_path = os.getcwd(
) #record cwd to return to this directory in the end
cfg = auxiliaryfunctions.read_config(config)
if cropping is not None:
cfg['cropping'] = True
cfg['x1'], cfg['x2'], cfg['y1'], cfg['y2'] = cropping
print("Overwriting cropping parameters:", cropping)
print(
"These are used for all videos, but won't be save to the cfg file."
)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
modelfolder = os.path.join(
cfg["project_path"],
str(auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# Check which snapshots are available and sort them by # iterations
try:
Snapshots = np.array([
fn.split('.')[0]
for fn in os.listdir(os.path.join(modelfolder, 'train'))
if "index" in fn
])
except FileNotFoundError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s."
% (shuffle, shuffle))
if cfg['snapshotindex'] == 'all':
print(
"Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!"
)
snapshotindex = -1
else:
snapshotindex = cfg['snapshotindex']
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder)
##################################################
# Load and setup CNN part detector
##################################################
# Check if data already was generated:
dlc_cfg['init_weights'] = os.path.join(modelfolder, 'train',
Snapshots[snapshotindex])
trainingsiterations = (dlc_cfg['init_weights'].split(
os.sep)[-1]).split('-')[-1]
#update batchsize (based on parameters in config.yaml)
dlc_cfg['batch_size'] = cfg['batch_size']
# update number of outputs
dlc_cfg['num_outputs'] = cfg.get('num_outputs', 1)
print('num_outputs = ', dlc_cfg['num_outputs'])
# Name for scorer:
DLCscorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, trainingsiterations=trainingsiterations)
sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
xyz_labs_orig = ['x', 'y', 'likelihood']
suffix = [str(s + 1) for s in range(dlc_cfg['num_outputs'])]
suffix[0] = '' # first one has empty suffix for backwards compatibility
xyz_labs = [x + s for s in suffix for x in xyz_labs_orig]
pdindex = pd.MultiIndex.from_product(
[[DLCscorer], dlc_cfg['all_joints_names'], xyz_labs],
names=['scorer', 'bodyparts', 'coords'])
##################################################
# Datafolder
##################################################
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
if len(Videos) > 0:
#looping over videos
for video in Videos:
AnalyzeVideo(video, DLCscorer, trainFraction, cfg, dlc_cfg, sess,
inputs, outputs, pdindex, save_as_csv, destfolder)
os.chdir(str(start_path))
print(
"The videos are analyzed. Now your research can truly start! \n You can create labeled videos with 'create_labeled_video'."
)
print(
"If the tracking is not satisfactory for some videos, consider expanding the training set. You can use the function 'extract_outlier_frames' to extract any outlier frames!"
)
else:
print("No video was found in the path/ or single video with path:",
videos)
print(
"Perhaps the videotype is distinct from the videos in the path, I was looking for:",
videotype)
return DLCscorer
def create_labeled_video(config,
videos,
videotype='avi',
shuffle=1,
trainingsetindex=0,
filtered=False,
save_frames=False,
Frames2plot=None,
delete=False,
displayedbodyparts='all',
codec='mp4v',
outputframerate=None,
destfolder=None,
draw_skeleton=False,
trailpoints=0,
displaycropped=False):
"""
Labels the bodyparts in a video. Make sure the video is already analyzed by the function 'analyze_video'
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
Number of shuffles of training dataset. Default is set to 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
videotype: string, optional
Checks for the extension of the video in case the input is a directory.\nOnly videos with this extension are analyzed. The default is ``.avi``
save_frames: bool
If true creates each frame individual and then combines into a video. This variant is relatively slow as
it stores all individual frames. However, it uses matplotlib to create the frames and is therefore much more flexible (one can set transparency of markers, crop, and easily customize).
Frames2plot: List of indices
If not None & save_frames=True then the frames corresponding to the index will be plotted. For example, Frames2plot=[0,11] will plot the first and the 12th frame.
delete: bool
If true then the individual frames created during the video generation will be deleted.
displayedbodyparts: list of strings, optional
This select the body parts that are plotted in the video. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
codec: codec for labeled video. Options see http://www.fourcc.org/codecs.php [depends on your ffmpeg installation.]
outputframerate: positive number, output frame rate for labeled video (only available for the mode with saving frames.) By default: None, which results in the original video rate.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
draw_skeleton: bool
If ``True`` adds a line connecting the body parts making a skeleton on on each frame. The body parts to be connected and the color of these connecting lines are specified in the config file. By default: ``False``
trailpoints: int
Number of revious frames whose body parts are plotted in a frame (for displaying history). Default is set to 0.
displaycropped: bool, optional
Specifies whether only cropped frame is displayed (with labels analyzed therein), or the original frame with the labels analyzed in the cropped subset.
Examples
--------
If you want to create the labeled video for only 1 video
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to create the labeled video for only 1 video and store the individual frames
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'],save_frames=True)
--------
If you want to create the labeled video for multiple videos
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to create the labeled video for all the videos (as .avi extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'])
--------
If you want to create the labeled video for all the videos (as .mp4 extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'],videotype='mp4')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction
) #automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
if draw_skeleton:
bodyparts2connect = cfg['skeleton']
skeleton_color = cfg['skeleton_color']
else:
bodyparts2connect = None
skeleton_color = None
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
if destfolder is None:
videofolder = Path(video).parents[
0] #where your folder with videos is.
else:
videofolder = destfolder
os.chdir(str(videofolder))
videotype = Path(video).suffix
print("Starting % ", videofolder, videos)
vname = str(Path(video).stem)
#if notanalyzed:
#notanalyzed,outdataname,sourcedataname,DLCscorer=auxiliaryfunctions.CheckifPostProcessing(folder,vname,DLCscorer,DLCscorerlegacy,suffix='checking')
if filtered == True:
videooutname1 = os.path.join(vname + DLCscorer +
'filtered_labeled.mp4')
videooutname2 = os.path.join(vname + DLCscorerlegacy +
'filtered_labeled.mp4')
else:
videooutname1 = os.path.join(vname + DLCscorer + '_labeled.mp4')
videooutname2 = os.path.join(vname + DLCscorerlegacy +
'_labeled.mp4')
if os.path.isfile(videooutname1) or os.path.isfile(videooutname2):
print("Labeled video already created.")
else:
print("Loading ", video, "and data.")
datafound, metadata, Dataframe, DLCscorer, suffix = auxiliaryfunctions.LoadAnalyzedData(
str(videofolder), vname, DLCscorer, filtered
) #returns boolean variable if data was found and metadata + pandas array
videooutname = os.path.join(vname + DLCscorer + suffix +
'_labeled.mp4')
if datafound and not os.path.isfile(
videooutname
): #checking again, for this loader video could exist
#Loading cropping data used during analysis
cropping = metadata['data']["cropping"]
[x1, x2, y1, y2] = metadata['data']["cropping_parameters"]
if save_frames == True:
tmpfolder = os.path.join(str(videofolder), 'temp-' + vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder)
clip = vp(video)
CreateVideoSlow(videooutname, clip, Dataframe, tmpfolder,
cfg["dotsize"], cfg["colormap"],
cfg["alphavalue"], cfg["pcutoff"],
trailpoints, cropping, x1, x2, y1, y2,
delete, DLCscorer, bodyparts,
outputframerate, Frames2plot,
bodyparts2connect, skeleton_color,
draw_skeleton, displaycropped)
else:
if displaycropped: #then the cropped video + the labels is depicted
clip = vp(fname=video,
sname=videooutname,
codec=codec,
sw=x2 - x1,
sh=y2 - y1)
CreateVideo(clip, Dataframe, cfg["pcutoff"],
cfg["dotsize"], cfg["colormap"], DLCscorer,
bodyparts, trailpoints, cropping, x1, x2,
y1, y2, bodyparts2connect, skeleton_color,
draw_skeleton, displaycropped)
else: #then the full video + the (perhaps in cropped mode analyzed labels) are depicted
clip = vp(fname=video, sname=videooutname, codec=codec)
CreateVideo(clip, Dataframe, cfg["pcutoff"],
cfg["dotsize"], cfg["colormap"], DLCscorer,
bodyparts, trailpoints, cropping, x1, x2,
y1, y2, bodyparts2connect, skeleton_color,
draw_skeleton, displaycropped)
def extract_outlier_frames(config,
videos,
videotype='avi',
shuffle=1,
trainingsetindex=0,
outlieralgorithm='jump',
comparisonbodyparts='all',
epsilon=20,
p_bound=.01,
ARdegree=3,
MAdegree=1,
alpha=.01,
extractionalgorithm='kmeans',
automatic=False,
cluster_resizewidth=30,
cluster_color=False,
opencv=True,
savelabeled=True,
destfolder=None):
"""
Extracts the outlier frames in case, the predictions are not correct for a certain video from the cropped video running from
start to stop as defined in config.yaml.
Another crucial parameter in config.yaml is how many frames to extract 'numframes2extract'.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
The shufle index of training dataset. The extracted frames will be stored in the labeled-dataset for
the corresponding shuffle of training dataset. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
outlieralgorithm: 'fitting', 'jump', 'uncertain', or 'manual'
String specifying the algorithm used to detect the outliers. Currently, deeplabcut supports three methods + a manual GUI option. 'Fitting'
fits a Auto Regressive Integrated Moving Average model to the data and computes the distance to the estimated data. Larger distances than
epsilon are then potentially identified as outliers. The methods 'jump' identifies larger jumps than 'epsilon' in any body part; and 'uncertain'
looks for frames with confidence below p_bound. The default is set to ``jump``.
comparisonbodyparts: list of strings, optional
This select the body parts for which the comparisons with the outliers are carried out. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
p_bound: float between 0 and 1, optional
For outlieralgorithm 'uncertain' this parameter defines the likelihood below, below which a body part will be flagged as a putative outlier.
epsilon; float,optional
Meaning depends on outlieralgoritm. The default is set to 20 pixels.
For outlieralgorithm 'fitting': Float bound according to which frames are picked when the (average) body part estimate deviates from model fit
For outlieralgorithm 'jump': Float bound specifying the distance by which body points jump from one frame to next (Euclidean distance)
ARdegree: int, optional
For outlieralgorithm 'fitting': Autoregressive degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part)
see https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
MAdegree: int
For outlieralgorithm 'fitting': MovingAvarage degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part)
See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
alpha: float
Significance level for detecting outliers based on confidence interval of fitted ARIMA model. Only the distance is used however.
extractionalgorithm : string, optional
String specifying the algorithm to use for selecting the frames from the identified putatative outlier frames. Currently, deeplabcut
supports either ``kmeans`` or ``uniform`` based selection (same logic as for extract_frames).
The default is set to``uniform``, if provided it must be either ``uniform`` or ``kmeans``.
automatic : bool, optional
Set it to True, if you want to extract outliers without being asked for user feedback.
cluster_resizewidth: number, default: 30
For k-means one can change the width to which the images are downsampled (aspect ratio is fixed).
cluster_color: bool, default: False
If false then each downsampled image is treated as a grayscale vector (discarding color information). If true, then the color channels are considered. This increases
the computational complexity.
opencv: bool, default: True
Uses openCV for loading & extractiong (otherwise moviepy (legacy))
savelabeled: bool, default: True
If true also saves frame with predicted labels in each folder.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
Examples
Windows example for extracting the frames with default settings
>>> deeplabcut.extract_outlier_frames('C:\\myproject\\reaching-task\\config.yaml',['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi'])
--------
for extracting the frames with default settings
>>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'])
--------
for extracting the frames with kmeans
>>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'],extractionalgorithm='kmeans')
--------
for extracting the frames with kmeans and epsilon = 5 pixels.
>>> deeplabcut.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'],epsilon = 5,extractionalgorithm='kmeans')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction=cfg['TrainingFraction'][trainingsetindex])
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
notanalyzed, dataname, DLCscorer = auxiliaryfunctions.CheckifNotAnalyzed(
videofolder,
str(Path(video).stem),
DLCscorer,
DLCscorerlegacy,
flag='checking')
if notanalyzed:
print(
"It seems the video has not been analyzed yet, or the video is not found! You can only refine the labels after the a video is analyzed. Please run 'analyze_video' first. Or, please double check your video file path"
)
else:
Dataframe = pd.read_hdf(dataname, 'df_with_missing')
scorer = Dataframe.columns.get_level_values(0)[
0] #reading scorer from
nframes = np.size(Dataframe.index)
# extract min and max index based on start stop interval.
startindex = max([int(np.floor(nframes * cfg['start'])), 0])
stopindex = min([int(np.ceil(nframes * cfg['stop'])), nframes])
Index = np.arange(stopindex - startindex) + startindex
#figure out body part list:
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
Indices = []
if outlieralgorithm == 'uncertain': #necessary parameters: considered body parts and
for bpindex, bp in enumerate(bodyparts):
if bp in cfg[
'bodyparts']: #filter [who knows what users put in...]
p = Dataframe[scorer][bp]['likelihood'].values[Index]
Indices.extend(
np.where(p < p_bound)[0] + startindex
) # all indices between start and stop that are below p_bound.
elif outlieralgorithm == 'jump':
for bpindex, bp in enumerate(bodyparts):
if bp in cfg[
'bodyparts']: #filter [who knows what users put in...]
dx = np.diff(Dataframe[scorer][bp]['x'].values[Index])
dy = np.diff(Dataframe[scorer][bp]['y'].values[Index])
# all indices between start and stop with jump larger than epsilon (leading up to this point!)
Indices.extend(
np.where((dx**2 + dy**2) > epsilon**2)[0] +
startindex + 1)
elif outlieralgorithm == 'fitting':
#deviation_dataname = str(Path(videofolder)/Path(dataname))
# Calculate deviatons for video
[d, o] = ComputeDeviations(Dataframe, cfg, bodyparts, scorer,
dataname, p_bound, alpha, ARdegree,
MAdegree)
#Some heuristics for extracting frames based on distance:
Indices = np.where(
d > epsilon
)[0] # time points with at least average difference of epsilon
if len(Index) < cfg['numframes2pick'] * 2 and len(d) > cfg[
'numframes2pick'] * 2: # if too few points qualify, extract the most distant ones.
Indices = np.argsort(d)[::-1][:cfg['numframes2pick'] * 2]
elif outlieralgorithm == 'manual':
wd = Path(config).resolve().parents[0]
os.chdir(str(wd))
from deeplabcut.refine_training_dataset import outlier_frame_extraction_toolbox
outlier_frame_extraction_toolbox.show(config, video, shuffle,
Dataframe, scorer,
savelabeled)
# Run always except when the outlieralgorithm == manual.
if not outlieralgorithm == 'manual':
Indices = np.sort(list(set(Indices))) #remove repetitions.
print("Method ", outlieralgorithm, " found ", len(Indices),
" putative outlier frames.")
print("Do you want to proceed with extracting ",
cfg['numframes2pick'], " of those?")
if outlieralgorithm == 'uncertain':
print(
"If this list is very large, perhaps consider changing the paramters (start, stop, p_bound, comparisonbodyparts) or use a different method."
)
elif outlieralgorithm == 'jump':
print(
"If this list is very large, perhaps consider changing the paramters (start, stop, epsilon, comparisonbodyparts) or use a different method."
)
elif outlieralgorithm == 'fitting':
print(
"If this list is very large, perhaps consider changing the paramters (start, stop, epsilon, ARdegree, MAdegree, alpha, comparisonbodyparts) or use a different method."
)
if automatic == False:
askuser = input("yes/no")
else:
askuser = '******'
if askuser == 'y' or askuser == 'yes' or askuser == 'Ja' or askuser == 'ha': # multilanguage support :)
#Now extract from those Indices!
ExtractFramesbasedonPreselection(
Indices, extractionalgorithm, Dataframe, dataname,
scorer, video, cfg, config, opencv,
cluster_resizewidth, cluster_color, savelabeled)
else:
print(
"Nothing extracted, please change the parameters and start again..."
)
def analyzeskeleton(config,
videos,
videotype='avi',
shuffle=1,
trainingsetindex=0,
save_as_csv=False,
destfolder=None):
"""
Extracts length and orientation of each "bone" of the skeleton as defined in the config file.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
shuffle : int, optional
The shufle index of training dataset. The extracted frames will be stored in the labeled-dataset for
the corresponding shuffle of training dataset. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this
folder also needs to be passed.
"""
# Load config file, scorer and videos
cfg = auxiliaryfunctions.read_config(config)
scorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction=cfg['TrainingFraction'][trainingsetindex])
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
print(video)
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
print("Processing %s" % (video))
dataname = str(Path(video).stem) + scorer
processedname = dataname.split('.h5')[0] + '_skeleton.h5'
try:
# See if file was already proccessed
Dataframe = pd.read_hdf(os.path.join(videofolder, processedname))
print("Video already processed...")
except FileNotFoundError:
try:
Dataframe = pd.read_hdf(
os.path.join(videofolder, dataname + '.h5'))
# Process skeleton
bones = {}
for bp1, bp2 in cfg['skeleton']:
name = "{}_{}".format(bp1, bp2)
bones[name] = analyzebone(Dataframe[scorer][bp1],
Dataframe[scorer][bp2])
skeleton = pd.concat(bones, axis=1)
# save
skeleton.to_hdf(processedname,
'df_with_missing',
format='table',
mode='w')
if save_as_csv:
skeleton.to_csv(processedname.split('.h5')[0] + '.csv')
except FileNotFoundError:
print("Video not analyzed -- Run analyze_videos first.")
def create_labeled_video(config,
videos,
videotype='avi',
shuffle=1,
trainingsetindex=0,
save_frames=False,
Frames2plot=None,
delete=False,
displayedbodyparts='all',
codec='mp4v',
outputframerate=None,
destfolder=None):
"""
Labels the bodyparts in a video. Make sure the video is already analyzed by the function 'analyze_video'
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
Number of shuffles of training dataset. Default is set to 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
videotype: string, optional
Checks for the extension of the video in case the input is a directory.\nOnly videos with this extension are analyzed. The default is ``.avi``
save_frames: bool
If true creates each frame individual and then combines into a video. This variant is relatively slow as
it stores all individual frames. However, it uses matplotlib to create the frames and is therefore much more flexible (one can set transparency of markers, crop, and easily customize).
Frames2plot: List of indices
If not None & save_frames=True then the frames corresponding to the index will be plotted. For example, Frames2plot=[0,11] will plot the first and the 12th frame.
delete: bool
If true then the individual frames created during the video generation will be deleted.
displayedbodyparts: list of strings, optional
This select the body parts that are plotted in the video. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
codec: codec for labeled video. Options see http://www.fourcc.org/codecs.php [depends on your ffmpeg installation.]
outputframerate: positive number, output frame rate for labeled video (only available for the mode with saving frames.) By default: None, which results in the original video rate.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
Examples
--------
If you want to create the labeled video for only 1 video
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to create the labeled video for only 1 video and store the individual frames
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'],save_frames=True)
--------
If you want to create the labeled video for multiple videos
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to create the labeled video for all the videos (as .avi extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'])
--------
If you want to create the labeled video for all the videos (as .mp4 extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'],videotype='mp4')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
DLCscorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction
) #automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
if destfolder is None:
#videofolder = str(Path(video).parents[0])
videofolder = Path(video).parents[
0] #where your folder with videos is.
else:
videofolder = destfolder
os.chdir(str(videofolder))
videotype = Path(video).suffix
print("Starting % ", videofolder, videos)
vname = str(Path(video).stem)
if os.path.isfile(
os.path.join(str(videofolder),
vname + DLCscorer + '_labeled.mp4')):
print("Labeled video already created.")
else:
print("Loading ", video, "and data.")
dataname = os.path.join(str(videofolder),
vname + DLCscorer + '.h5')
try:
Dataframe = pd.read_hdf(dataname)
metadata = auxiliaryfunctions.LoadVideoMetadata(dataname)
#print(metadata)
datanames = [dataname]
except FileNotFoundError:
datanames = [
fn for fn in os.listdir(os.curdir)
if (vname in fn) and (".h5" in fn) and "resnet" in fn
]
if len(datanames) == 0:
print("The video was not analyzed with this scorer:",
DLCscorer)
print(
"No other scorers were found, please use the function 'analyze_videos' first."
)
elif len(datanames) > 0:
print("The video was not analyzed with this scorer:",
DLCscorer)
print("Other scorers were found, however:", datanames)
DLCscorer = 'DeepCut' + (
datanames[0].split('DeepCut')[1]).split('.h5')[0]
print("Creating labeled video for:", DLCscorer,
" instead.")
Dataframe = pd.read_hdf(datanames[0])
metadata = auxiliaryfunctions.LoadVideoMetadata(
datanames[0])
if len(datanames) > 0:
#Loading cropping data used during analysis
cropping = metadata['data']["cropping"]
[x1, x2, y1, y2] = metadata['data']["cropping_parameters"]
print(cropping, x1, x2, y1, y2)
if save_frames == True:
tmpfolder = os.path.join(str(videofolder), 'temp-' + vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder)
clip = vp(video)
#CreateVideoSlow(clip,Dataframe,tmpfolder,cfg["dotsize"],cfg["colormap"],cfg["alphavalue"],cfg["pcutoff"],cfg["cropping"],cfg["x1"],cfg["x2"],cfg["y1"],cfg["y2"],delete,DLCscorer,bodyparts)
CreateVideoSlow(clip, Dataframe, tmpfolder, cfg["dotsize"],
cfg["colormap"], cfg["alphavalue"],
cfg["pcutoff"], cropping, x1, x2, y1, y2,
delete, DLCscorer, bodyparts,
outputframerate, Frames2plot)
else:
clip = vp(fname=video,
sname=os.path.join(vname + DLCscorer +
'_labeled.mp4'),
codec=codec)
if cropping:
print(
"Fast video creation has currently not been implemented for cropped videos. Please use 'save_frames=True' to get the video."
)
else:
CreateVideo(clip, Dataframe, cfg["pcutoff"],
cfg["dotsize"], cfg["colormap"], DLCscorer,
bodyparts, cropping, x1, x2, y1,
y2) #NEED TO ADD CROPPING!
def stitch_tracklets(
config_path,
videos,
videotype="avi",
shuffle=1,
trainingsetindex=0,
n_tracks=None,
min_length=10,
split_tracklets=True,
prestitch_residuals=True,
max_gap=None,
weight_func=None,
track_method="ellipse",
destfolder=None,
modelprefix="",
output_name="",
):
"""
Stitch sparse tracklets into full tracks via a graph-based,
minimum-cost flow optimization problem.
Parameters
----------
config_path : str
Path to the main project config.yaml file.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: int, optional
An integer specifying the shuffle index of the training dataset used for training the network. The default is 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
n_tracks : int, optional
Number of tracks to reconstruct. By default, taken as the number
of individuals defined in the config.yaml. Another number can be
passed if the number of animals in the video is different from
the number of animals the model was trained on.
min_length : int, optional
Tracklets less than `min_length` frames of length
are considered to be residuals; i.e., they do not participate
in building the graph and finding the solution to the
optimization problem, but are rather added last after
"almost-complete" tracks are formed. The higher the value,
the lesser the computational cost, but the higher the chance of
discarding relatively long and reliable tracklets that are
essential to solving the stitching task.
Default is 10, and must be 3 at least.
split_tracklets : bool, optional
By default, tracklets whose time indices are not consecutive integers
are split in shorter tracklets whose time continuity is guaranteed.
This is for example very powerful to get rid of tracking errors
(e.g., identity switches) which are often signaled by a missing
time frame at the moment they occur. Note though that for long
occlusions where tracker re-identification capability can be trusted,
setting `split_tracklets` to False is preferable.
prestitch_residuals : bool, optional
Residuals will by default be grouped together according to their
temporal proximity prior to being added back to the tracks.
This is done to improve robustness and simultaneously reduce complexity.
max_gap : int, optional
Maximal temporal gap to allow between a pair of tracklets.
This is automatically determined by the TrackletStitcher by default.
weight_func : callable, optional
Function accepting two tracklets as arguments and returning a scalar
that must be inversely proportional to the likelihood that the tracklets
belong to the same track; i.e., the higher the confidence that the
tracklets should be stitched together, the lower the returned value.
track_method: str, optional
Method used to track animals, either 'box', 'skeleton', or 'ellipse' (default).
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this
folder also needs to be passed.
output_name : str, optional
Name of the output h5 file.
By default, tracks are automatically stored into the same directory
as the pickle file and with its name.
Returns
-------
A TrackletStitcher object
"""
vids = auxiliaryfunctions.Getlistofvideos(videos, videotype)
if not vids:
print("No video(s) found. Please check your path!")
return
cfg = read_config(config_path)
animal_names = cfg["individuals"]
if n_tracks is None:
n_tracks = len(animal_names)
DLCscorer, _ = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
cfg["TrainingFraction"][trainingsetindex],
modelprefix=modelprefix,
)
for video in vids:
print("Processing... ", video)
videofolder = str(Path(video).parents[0])
dest = destfolder or videofolder
auxiliaryfunctions.attempttomakefolder(dest)
vname = Path(video).stem
dataname = os.path.join(dest, vname + DLCscorer + ".h5")
if track_method == "ellipse":
method = "el"
elif track_method == "box":
method = "bx"
else:
method = "sk"
pickle_file = dataname.split(".h5")[0] + f"_{method}.pickle"
try:
stitcher = TrackletStitcher.from_pickle(pickle_file, n_tracks,
min_length,
split_tracklets,
prestitch_residuals)
with_id = any(tracklet.identity != -1 for tracklet in stitcher)
if with_id and weight_func is None:
# Add in identity weighing before building the graph
def weight_func(t1, t2):
w = 0.01 if t1.identity == t2.identity else 1
return w * stitcher.calculate_edge_weight(t1, t2)
stitcher.build_graph(max_gap=max_gap, weight_func=weight_func)
stitcher.stitch()
stitcher.write_tracks(output_name, animal_names)
except FileNotFoundError as e:
print(e, "\nSkipping...")
def analyze_videos(config,videos, videotype='avi', shuffle=1, trainingsetindex=0,
gputouse=None, save_as_csv=False, destfolder=None, batchsize=None,
cropping=None,get_nframesfrommetadata=True, TFGPUinference=True,dynamic=(False,.5,10)):
"""
Makes prediction based on a trained network. The index of the trained network is specified by parameters in the config file (in particular the variable 'snapshotindex')
You can crop the video (before analysis), by changing 'cropping'=True and setting 'x1','x2','y1','y2' in the config file. The same cropping parameters will then be used for creating the video.
Output: The labels are stored as MultiIndex Pandas Array, which contains the name of the network, body part name, (x, y) label position \n
in pixels, and the likelihood for each frame per body part. These arrays are stored in an efficient Hierarchical Data Format (HDF) \n
in the same directory, where the video is stored. However, if the flag save_as_csv is set to True, the data can also be exported in \n
comma-separated values format (.csv), which in turn can be imported in many programs, such as MATLAB, R, Prism, etc.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: int, optional
An integer specifying the shuffle index of the training dataset used for training the network. The default is 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this
folder also needs to be passed.
batchsize: int, default from pose_cfg.yaml
Change batch size for inference; if given overwrites value in pose_cfg.yaml
TFGPUinference: bool, default: True
Perform inference on GPU with Tensorflow code. Introduced in "Pretraining boosts out-of-domain robustness for pose estimation" by
Alexander Mathis, Mert Yüksekgönül, Byron Rogers, Matthias Bethge, Mackenzie W. Mathis Source: https://arxiv.org/abs/1909.11229
dynamic: triple containing (state,detectiontreshold,margin)
If the state is true, then dynamic cropping will be performed. That means that if an object is detected (i.e. any body part > detectiontreshold),
then object boundaries are computed according to the smallest/largest x position and smallest/largest y position of all body parts. This window is
expanded by the margin and from then on only the posture within this crop is analyzed (until the object is lost, i.e. <detectiontreshold). The
current position is utilized for updating the crop window for the next frame (this is why the margin is important and should be set large
enough given the movement of the animal).
Examples
--------
Windows example for analyzing 1 video
>>> deeplabcut.analyze_videos('C:\\myproject\\reaching-task\\config.yaml',['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi'])
--------
If you want to analyze only 1 video
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to analyze all videos of type avi in a folder:
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos'],videotype='.avi')
--------
If you want to analyze multiple videos
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to analyze multiple videos with shuffle = 2
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'], shuffle=2)
--------
If you want to analyze multiple videos with shuffle = 2 and save results as an additional csv file too
>>> deeplabcut.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'], shuffle=2,save_as_csv=True)
--------
"""
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ['TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
if gputouse is not None: #gpu selection
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
tf.reset_default_graph()
start_path=os.getcwd() #record cwd to return to this directory in the end
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
if cropping is not None:
cfg['cropping']=True
cfg['x1'],cfg['x2'],cfg['y1'],cfg['y2']=cropping
print("Overwriting cropping parameters:", cropping)
print("These are used for all videos, but won't be save to the cfg file.")
modelfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError("It seems the model for shuffle %s and trainFraction %s does not exist."%(shuffle,trainFraction))
# Check which snapshots are available and sort them by # iterations
try:
Snapshots = np.array([fn.split('.')[0]for fn in os.listdir(os.path.join(modelfolder , 'train'))if "index" in fn])
except FileNotFoundError:
raise FileNotFoundError("Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s."%(shuffle,shuffle))
if cfg['snapshotindex'] == 'all':
print("Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!")
snapshotindex = -1
else:
snapshotindex=cfg['snapshotindex']
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder)
##################################################
# Load and setup CNN part detector
##################################################
# Check if data already was generated:
dlc_cfg['init_weights'] = os.path.join(modelfolder , 'train', Snapshots[snapshotindex])
trainingsiterations = (dlc_cfg['init_weights'].split(os.sep)[-1]).split('-')[-1]
# Update number of output and batchsize
dlc_cfg['num_outputs'] = cfg.get('num_outputs', dlc_cfg.get('num_outputs', 1))
if batchsize==None:
#update batchsize (based on parameters in config.yaml)
dlc_cfg['batch_size']=cfg['batch_size']
else:
dlc_cfg['batch_size']=batchsize
cfg['batch_size']=batchsize
if dynamic[0]: #state=true
#(state,detectiontreshold,margin)=dynamic
print("Starting analysis in dynamic cropping mode with parameters:", dynamic)
dlc_cfg['num_outputs']=1
TFGPUinference=False
dlc_cfg['batch_size']=1
print("Switching batchsize to 1, num_outputs (per animal) to 1 and TFGPUinference to False (all these features are not supported in this mode).")
# Name for scorer:
DLCscorer,DLCscorerlegacy = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations=trainingsiterations)
if dlc_cfg['num_outputs']>1:
if TFGPUinference:
print("Switching to numpy-based keypoint extraction code, as multiple point extraction is not supported by TF code currently.")
TFGPUinference=False
print("Extracting ", dlc_cfg['num_outputs'], "instances per bodypart")
xyz_labs_orig = ['x', 'y', 'likelihood']
suffix = [str(s+1) for s in range(dlc_cfg['num_outputs'])]
suffix[0] = '' # first one has empty suffix for backwards compatibility
xyz_labs = [x+s for s in suffix for x in xyz_labs_orig]
else:
xyz_labs = ['x', 'y', 'likelihood']
#sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
if TFGPUinference:
sess, inputs, outputs = predict.setup_GPUpose_prediction(dlc_cfg)
else:
sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
pdindex = pd.MultiIndex.from_product([[DLCscorer],
dlc_cfg['all_joints_names'],
xyz_labs],
names=['scorer', 'bodyparts', 'coords'])
##################################################
# Datafolder
##################################################
Videos=auxiliaryfunctions.Getlistofvideos(videos,videotype)
if len(Videos)>0:
#looping over videos
for video in Videos:
DLCscorer=AnalyzeVideo(video,DLCscorer,DLCscorerlegacy,trainFraction,cfg,dlc_cfg,sess,inputs, outputs,pdindex,save_as_csv, destfolder,TFGPUinference,dynamic)
os.chdir(str(start_path))
print("The videos are analyzed. Now your research can truly start! \n You can create labeled videos with 'create_labeled_video'.")
print("If the tracking is not satisfactory for some videos, consider expanding the training set. You can use the function 'extract_outlier_frames' to extract any outlier frames!")
return DLCscorer #note: this is either DLCscorer or DLCscorerlegacy depending on what was used!
else:
print("No video/s found. Please check your path!")
return DLCscorer
def plot_trajectories(config,
videos,
videotype='.avi',
shuffle=1,
trainingsetindex=0,
filtered=False,
showfigures=False,
displayedbodyparts='all',
destfolder=None):
"""
Plots the trajectories of various bodyparts across the video.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
showfigures: bool, default false
If true then plots are also displayed.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
Example
--------
for labeling the frames
>>> deeplabcut.plot_trajectories('home/alex/analysis/project/reaching-task/config.yaml',['/home/alex/analysis/project/videos/reachingvideo1.avi'])
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
DLCscorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction
) #automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
print(video)
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
videotype = str(Path(video).suffix)
print("Starting % ", videofolder, videos)
basefolder = videofolder
auxiliaryfunctions.attempttomakefolder(basefolder)
RunTrajectoryAnalysis(video, basefolder, DLCscorer, videofolder, cfg,
showfigures, bodyparts, filtered)
print(
'Plots created! Please check the directory "plot-poses" within the video directory'
)
def analyzeskeleton(
config,
videos,
videotype="avi",
shuffle=1,
trainingsetindex=0,
save_as_csv=False,
destfolder=None,
modelprefix="",
):
"""
Extracts length and orientation of each "bone" of the skeleton as defined in the config file.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
shuffle : int, optional
The shufle index of training dataset. The extracted frames will be stored in the labeled-dataset for
the corresponding shuffle of training dataset. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this
folder also needs to be passed.
"""
# Load config file, scorer and videos
cfg = auxiliaryfunctions.read_config(config)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction=cfg["TrainingFraction"][trainingsetindex],
modelprefix=modelprefix,
)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
print("Processing %s" % (video))
if destfolder is None:
destfolder = str(Path(video).parents[0])
vname = Path(video).stem
(
notanalyzed,
outdataname,
sourcedataname,
scorer,
) = auxiliaryfunctions.CheckifPostProcessing(destfolder,
vname,
DLCscorer,
DLCscorerlegacy,
suffix="_skeleton")
if notanalyzed:
Dataframe = pd.read_hdf(sourcedataname, "df_with_missing")
# Process skeleton
bones = {}
for bp1, bp2 in cfg["skeleton"]:
name = "{}_{}".format(bp1, bp2)
bones[name] = analyzebone(Dataframe[scorer][bp1],
Dataframe[scorer][bp2])
skeleton = pd.concat(bones, axis=1)
# save
skeleton.to_hdf(outdataname,
"df_with_missing",
format="table",
mode="w")
if save_as_csv:
skeleton.to_csv(outdataname.split(".h5")[0] + ".csv")
def plot_trajectories(
config,
videos,
videotype=".avi",
shuffle=1,
trainingsetindex=0,
filtered=False,
displayedbodyparts="all",
displayedindividuals="all",
showfigures=False,
destfolder=None,
modelprefix="",
imagetype=".png",
resolution=100,
linewidth=1.0,
track_method="",
):
"""
Plots the trajectories of various bodyparts across the video.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
displayedbodyparts: list of strings, optional
This select the body parts that are plotted in the video.
Either ``all``, then all body parts from config.yaml are used,
or a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
showfigures: bool, default false
If true then plots are also displayed.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
imagetype: string, default ".png"
Specifies the output image format, tested '.tif', '.jpg', '.svg' and ".png".
resolution: int, default 100
Specifies the resolution (in dpi) of saved figures. Note higher resolution figures take longer to generate.
linewidth: float, default 1.0
Specifies width of line for line and histogram plots.
track_method: string, optional
Specifies the tracker used to generate the data. Empty by default (corresponding to a single animal project).
For multiple animals, must be either 'box', 'skeleton', or 'ellipse' and will be taken from the config.yaml file if none is given.
Example
--------
for labeling the frames
>>> deeplabcut.plot_trajectories('home/alex/analysis/project/reaching-task/config.yaml',['/home/alex/analysis/project/videos/reachingvideo1.avi'])
--------
"""
cfg = auxiliaryfunctions.read_config(config)
track_method = auxfun_multianimal.get_track_method(
cfg, track_method=track_method)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, modelprefix=modelprefix
) # automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
individuals = auxfun_multianimal.IntersectionofIndividualsandOnesGivenbyUser(
cfg, displayedindividuals)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
if not len(Videos):
print(
"No videos found. Make sure you passed a list of videos and that *videotype* is right."
)
return
failed = []
for video in Videos:
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
vname = str(Path(video).stem)
print("Loading ", video, "and data.")
try:
df, _, _, suffix = auxiliaryfunctions.load_analyzed_data(
videofolder, vname, DLCscorer, filtered, track_method)
failed.append(False)
tmpfolder = os.path.join(videofolder, "plot-poses", vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder, recursive=True)
# Keep only the individuals and bodyparts that were labeled
labeled_bpts = [
bp for bp in df.columns.get_level_values("bodyparts").unique()
if bp in bodyparts
]
# Either display the animals defined in the config if they are found
# in the dataframe, or all the trajectories regardless of their names
try:
animals = set(df.columns.get_level_values("individuals"))
except KeyError:
animals = {""}
for animal in animals.intersection(individuals) or animals:
PlottingResults(
tmpfolder,
df,
cfg,
labeled_bpts,
animal,
showfigures,
suffix + animal + imagetype,
resolution=resolution,
linewidth=linewidth,
)
except FileNotFoundError as e:
failed.append(True)
print(e)
try:
_ = auxiliaryfunctions.load_detection_data(
video, DLCscorer, track_method)
print('Call "deeplabcut.stitch_tracklets()"'
" prior to plotting the trajectories.")
except FileNotFoundError as e:
print(e)
print(
f"Make sure {video} was previously analyzed, and that "
f'detections were successively converted to tracklets using "deeplabcut.convert_detections2tracklets()" '
f'and "deeplabcut.stitch_tracklets()".')
if not all(failed):
print(
'Plots created! Please check the directory "plot-poses" within the video directory'
)
else:
print(
f"Plots could not be created! "
f"Videos were not evaluated with the current scorer {DLCscorer}.")
def filterpredictions(
config,
video,
videotype="avi",
shuffle=1,
trainingsetindex=0,
filtertype="median",
windowlength=5,
p_bound=0.001,
ARdegree=3,
MAdegree=1,
alpha=0.01,
save_as_csv=True,
destfolder=None,
modelprefix="",
track_method="",
):
"""
Fits frame-by-frame pose predictions with ARIMA model (filtertype='arima') or median filter (default).
Parameter
----------
config : string
Full path of the config.yaml file as a string.
video : string
Full path of the video to extract the frame from. Make sure that this video is already analyzed.
shuffle : int, optional
The shufle index of training dataset. The extracted frames will be stored in the labeled-dataset for
the corresponding shuffle of training dataset. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtertype: string
Select which filter, 'arima', 'median' or 'spline'.
windowlength: int
For filtertype='median' filters the input array using a local window-size given by windowlength. The array will automatically be zero-padded.
https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.medfilt.html The windowlenght should be an odd number.
If filtertype='spline', windowlength is the maximal gap size to fill.
p_bound: float between 0 and 1, optional
For filtertype 'arima' this parameter defines the likelihood below,
below which a body part will be consided as missing data for filtering purposes.
ARdegree: int, optional
For filtertype 'arima' Autoregressive degree of Sarimax model degree.
see https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
MAdegree: int
For filtertype 'arima' Moving Avarage degree of Sarimax model degree.
See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
alpha: float
Significance level for detecting outliers based on confidence interval of fitted SARIMAX model.
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this
folder also needs to be passed.
Example
--------
Arima model:
deeplabcut.filterpredictions('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,filterype='arima',ARdegree=5,MAdegree=2)
Use median filter over 10bins:
deeplabcut.filterpredictions('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,windowlength=10)
One can then use the filtered rather than the frame-by-frame predictions by calling:
deeplabcut.plot_trajectories('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,filtered=True)
deeplabcut.create_labeled_video('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,filtered=True)
--------
Returns filtered pandas array with the same structure as normal output of network.
"""
cfg = auxiliaryfunctions.read_config(config)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
trainFraction=cfg["TrainingFraction"][trainingsetindex],
modelprefix=modelprefix,
)
Videos = auxiliaryfunctions.Getlistofvideos(video, videotype)
if not len(Videos):
print(
"No video(s) were found. Please check your paths and/or 'video_type'."
)
return
for video in Videos:
if destfolder is None:
destfolder = str(Path(video).parents[0])
print("Filtering with %s model %s" % (filtertype, video))
vname = Path(video).stem
try:
_ = auxiliaryfunctions.load_analyzed_data(destfolder, vname,
DLCscorer, True,
track_method)
print(f"Data from {vname} were already filtered. Skipping...")
except FileNotFoundError: # Data haven't been filtered yet
try:
df, filepath, _, _ = auxiliaryfunctions.load_analyzed_data(
destfolder, vname, DLCscorer, track_method=track_method)
nrows = df.shape[0]
if filtertype == "arima":
temp = df.values.reshape((nrows, -1, 3))
placeholder = np.empty_like(temp)
for i in range(temp.shape[1]):
x, y, p = temp[:, i].T
meanx, _ = FitSARIMAXModel(x, p, p_bound, alpha,
ARdegree, MAdegree, False)
meany, _ = FitSARIMAXModel(y, p, p_bound, alpha,
ARdegree, MAdegree, False)
meanx[0] = x[0]
meany[0] = y[0]
placeholder[:, i] = np.c_[meanx, meany, p]
data = pd.DataFrame(
placeholder.reshape((nrows, -1)),
columns=df.columns,
index=df.index,
)
elif filtertype == "median":
data = df.copy()
mask = data.columns.get_level_values(
"coords") != "likelihood"
data.loc[:,
mask] = df.loc[:,
mask].apply(signal.medfilt,
args=(windowlength, ),
axis=0)
elif filtertype == "spline":
data = df.copy()
mask_data = data.columns.get_level_values("coords").isin(
("x", "y"))
xy = data.loc[:, mask_data].values
prob = data.loc[:, ~mask_data].values
missing = np.isnan(xy)
xy_filled = columnwise_spline_interp(xy, windowlength)
filled = ~np.isnan(xy_filled)
xy[filled] = xy_filled[filled]
inds = np.argwhere(missing & filled)
if inds.size:
# Retrieve original individual label indices
inds[:, 1] //= 2
inds = np.unique(inds, axis=0)
prob[inds[:, 0], inds[:, 1]] = 0.01
data.loc[:, ~mask_data] = prob
data.loc[:, mask_data] = xy
else:
raise ValueError(f"Unknown filter type {filtertype}")
outdataname = filepath.replace(".h5", "_filtered.h5")
data.to_hdf(outdataname,
"df_with_missing",
format="table",
mode="w")
if save_as_csv:
print("Saving filtered csv poses!")
data.to_csv(outdataname.split(".h5")[0] + ".csv")
except FileNotFoundError as e:
print(e)
continue
def filterpredictions(config,
video,
videotype='avi',
shuffle=1,
trainingsetindex=0,
filtertype='median',
windowlength=5,
p_bound=.001,
ARdegree=3,
MAdegree=1,
alpha=.01,
save_as_csv=True,
destfolder=None):
"""
Fits frame-by-frame pose predictions with ARIMA model (filtertype='arima') or median filter (default).
Parameter
----------
config : string
Full path of the config.yaml file as a string.
video : string
Full path of the video to extract the frame from. Make sure that this video is already analyzed.
shuffle : int, optional
The shufle index of training dataset. The extracted frames will be stored in the labeled-dataset for
the corresponding shuffle of training dataset. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtertype: string
Select which filter, 'arima' or 'median' filter.
windowlength: int
For filtertype='median' filters the input array using a local window-size given by windowlength. The array will automatically be zero-padded.
https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.medfilt.html The windowlenght should be an odd number.
p_bound: float between 0 and 1, optional
For filtertype 'arima' this parameter defines the likelihood below,
below which a body part will be consided as missing data for filtering purposes.
ARdegree: int, optional
For filtertype 'arima' Autoregressive degree of Sarimax model degree.
see https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
MAdegree: int
For filtertype 'arima' Moving Avarage degree of Sarimax model degree.
See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
alpha: float
Significance level for detecting outliers based on confidence interval of fitted SARIMAX model.
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this
folder also needs to be passed.
Example
--------
Arima model:
deeplabcut.filterpredictions('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,filterype='arima',ARdegree=5,MAdegree=2)
Use median filter over 10bins:
deeplabcut.filterpredictions('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,windowlength=10)
One can then use the filtered rather than the frame-by-frame predictions by calling:
deeplabcut.plot_trajectories('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,filtered=True)
deeplabcut.create_labeled_video('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,filtered=True)
--------
Returns filtered pandas array with the same structure as normal output of network.
"""
cfg = auxiliaryfunctions.read_config(config)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction=cfg['TrainingFraction'][trainingsetindex])
Videos = auxiliaryfunctions.Getlistofvideos(video, videotype)
if len(Videos) > 0:
for video in Videos:
if destfolder is None:
destfolder = str(Path(video).parents[0])
print("Filtering with %s model %s" % (filtertype, video))
videofolder = destfolder
vname = Path(video).stem
notanalyzed, outdataname, sourcedataname, scorer = auxiliaryfunctions.CheckifPostProcessing(
destfolder,
vname,
DLCscorer,
DLCscorerlegacy,
suffix='filtered')
if notanalyzed:
Dataframe = pd.read_hdf(sourcedataname, 'df_with_missing')
for bpindex, bp in tqdm(enumerate(cfg['bodyparts'])):
pdindex = pd.MultiIndex.from_product(
[[scorer], [bp], ['x', 'y', 'likelihood']],
names=['scorer', 'bodyparts', 'coords'])
x, y, p = Dataframe[scorer][bp]['x'].values, Dataframe[
scorer][bp]['y'].values, Dataframe[scorer][bp][
'likelihood'].values
if filtertype == 'arima':
meanx, CIx = FitSARIMAXModel(x, p, p_bound, alpha,
ARdegree, MAdegree, False)
meany, CIy = FitSARIMAXModel(y, p, p_bound, alpha,
ARdegree, MAdegree, False)
meanx[0] = x[0]
meany[0] = y[0]
else:
meanx = signal.medfilt(x, kernel_size=windowlength)
meany = signal.medfilt(y, kernel_size=windowlength)
if bpindex == 0:
data = pd.DataFrame(np.hstack([
np.expand_dims(meanx, axis=1),
np.expand_dims(meany, axis=1),
np.expand_dims(p, axis=1)
]),
columns=pdindex)
else:
item = pd.DataFrame(np.hstack([
np.expand_dims(meanx, axis=1),
np.expand_dims(meany, axis=1),
np.expand_dims(p, axis=1)
]),
columns=pdindex)
data = pd.concat([data.T, item.T]).T
data.to_hdf(outdataname,
'df_with_missing',
format='table',
mode='w')
if save_as_csv:
print("Saving filtered csv poses!")
data.to_csv(outdataname.split('.h5')[0] + '.csv')
def filterpredictions(config,
video,
videotype='avi',
shuffle=1,
trainingsetindex=0,
p_bound=.001,
ARdegree=3,
MAdegree=1,
alpha=.01,
save_as_csv=True,
destfolder=None):
"""
Fits frame-by-frame pose predictions with SARIMAX model.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
video : string
Full path of the video to extract the frame from. Make sure that this video is already analyzed.
shuffle : int, optional
The shufle index of training dataset. The extracted frames will be stored in the labeled-dataset for
the corresponding shuffle of training dataset. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
comparisonbodyparts: list of strings, optional
This select the body parts for which SARIMAX models are fit. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
p_bound: float between 0 and 1, optional
For outlieralgorithm 'uncertain' this parameter defines the likelihood below,
below which a body part will be consided as missing data for filtering purposes.
ARdegree: int, optional
For outlieralgorithm 'fitting': Autoregressive degree of Sarimax model degree.
see https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
MAdegree: int
For outlieralgorithm 'fitting': Moving Avarage degree of Sarimax model degree.
See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
alpha: float
Significance level for detecting outliers based on confidence interval of fitted SARIMAX model.
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this
folder also needs to be passed.
Example
--------
deeplabcut.filterpredictions('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,ARdegree=5,MAdegree=2)
One can then use the filtered rather than the frame-by-frame predictions by calling:
deeplabcut.plot_trajectories('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,filtered=True)
deeplabcut.create_labeled_video('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,filtered=True)
--------
Returns filtered pandas array with the same structure as normal output of network.
"""
cfg = auxiliaryfunctions.read_config(config)
scorer = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction=cfg['TrainingFraction'][trainingsetindex])
Videos = auxiliaryfunctions.Getlistofvideos(video, videotype)
if len(Videos) > 0:
for video in Videos:
if destfolder is None:
destfolder = str(Path(video).parents[0])
print("Filtering with ARIMA model %s", video)
videofolder = str(Path(video).parents[0])
dataname = str(Path(video).stem) + scorer
filteredname = dataname.split('.h5')[0] + 'filtered.h5'
try:
Dataframe = pd.read_hdf(os.path.join(videofolder,
filteredname))
print("Video already filtered...")
except FileNotFoundError:
try:
Dataframe = pd.read_hdf(
os.path.join(videofolder, dataname + '.h5'))
for bpindex, bp in tqdm(enumerate(cfg['bodyparts'])):
pdindex = pd.MultiIndex.from_product(
[[scorer], [bp], ['x', 'y', 'likelihood']],
names=['scorer', 'bodyparts', 'coords'])
x, y, p = Dataframe[scorer][bp]['x'].values, Dataframe[
scorer][bp]['y'].values, Dataframe[scorer][bp][
'likelihood'].values
meanx, CIx = FitSARIMAXModel(x, p, p_bound, alpha,
ARdegree, MAdegree, False)
meany, CIy = FitSARIMAXModel(y, p, p_bound, alpha,
ARdegree, MAdegree, False)
meanx[0] = x[0]
meany[0] = y[0]
if bpindex == 0:
data = pd.DataFrame(np.hstack([
np.expand_dims(meanx, axis=1),
np.expand_dims(meany, axis=1),
np.expand_dims(p, axis=1)
]),
columns=pdindex)
else:
item = pd.DataFrame(np.hstack([
np.expand_dims(meanx, axis=1),
np.expand_dims(meany, axis=1),
np.expand_dims(p, axis=1)
]),
columns=pdindex)
data = pd.concat([data.T, item.T]).T
data.to_hdf(os.path.join(videofolder, filteredname),
'df_with_missing',
format='table',
mode='w')
if save_as_csv:
print("Saving filtered csv poses!")
data.to_csv(
os.path.join(videofolder,
filteredname.split('.h5')[0] +
'.csv'))
except FileNotFoundError:
print("Video not analyzed -- Run analyze_videos first.")
def plot_trajectories(config,
videos,
videotype='.avi',
shuffle=1,
trainingsetindex=0,
filtered=False,
displayedbodyparts='all',
showfigures=False,
destfolder=None):
"""
Plots the trajectories of various bodyparts across the video.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
displayedbodyparts: list of strings, optional
This select the body parts that are plotted in the video.
Either ``all``, then all body parts from config.yaml are used,
or a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
showfigures: bool, default false
If true then plots are also displayed.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
Example
--------
for labeling the frames
>>> deeplabcut.plot_trajectories('home/alex/analysis/project/reaching-task/config.yaml',['/home/alex/analysis/project/videos/reachingvideo1.avi'])
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction
) #automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
print(video)
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
vname = str(Path(video).stem)
print("Starting % ", videofolder, video)
notanalyzed, dataname, DLCscorer = auxiliaryfunctions.CheckifNotAnalyzed(
videofolder, vname, DLCscorer, DLCscorerlegacy, flag='checking')
if notanalyzed:
print("The video was not analyzed with this scorer:", DLCscorer)
else:
#LoadData
print("Loading ", video, "and data.")
datafound, metadata, Dataframe, DLCscorer, suffix = auxiliaryfunctions.LoadAnalyzedData(
str(videofolder), vname, DLCscorer, filtered
) #returns boolean variable if data was found and metadata + pandas array
if datafound:
basefolder = videofolder
auxiliaryfunctions.attempttomakefolder(basefolder)
auxiliaryfunctions.attempttomakefolder(
os.path.join(basefolder, 'plot-poses'))
tmpfolder = os.path.join(basefolder, 'plot-poses', vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder)
PlottingResults(tmpfolder, Dataframe, DLCscorer, cfg,
bodyparts, showfigures, suffix + '.png')
print(
'Plots created! Please check the directory "plot-poses" within the video directory'
)
def create_labeled_video(
config,
videos,
videotype="avi",
shuffle=1,
trainingsetindex=0,
filtered=False,
fastmode=True,
save_frames=False,
Frames2plot=None,
displayedbodyparts="all",
displayedindividuals="all",
codec="mp4v",
outputframerate=None,
destfolder=None,
draw_skeleton=False,
trailpoints=0,
displaycropped=False,
color_by="bodypart",
modelprefix="",
track_method="",
):
"""
Labels the bodyparts in a video. Make sure the video is already analyzed by the function 'analyze_video'
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
Number of shuffles of training dataset. Default is set to 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions
videotype: string, optional
Checks for the extension of the video in case the input is a directory.\nOnly videos with this extension are analyzed. The default is ``.avi``
fastmode: bool
If true uses openCV (much faster but less customization of video) vs matplotlib (if false). You can also
"save_frames" individually or not in the matplotlib mode (if you set the "save_frames" variable accordingly).
However, using matplotlib to create the frames it therefore allows much more flexible (one can set transparency of markers, crop, and easily customize).
save_frames: bool
If true creates each frame individual and then combines into a video. This variant is relatively slow as
it stores all individual frames.
Frames2plot: List of indices
If not None & save_frames=True then the frames corresponding to the index will be plotted. For example, Frames2plot=[0,11] will plot the first and the 12th frame.
displayedbodyparts: list of strings, optional
This selects the body parts that are plotted in the video. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
displayedindividuals: list of strings, optional
Individuals plotted in the video. By default, all individuals present in the config will be showed.
codec: codec for labeled video. Options see http://www.fourcc.org/codecs.php [depends on your ffmpeg installation.]
outputframerate: positive number, output frame rate for labeled video (only available for the mode with saving frames.) By default: None, which results in the original video rate.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
draw_skeleton: bool
If ``True`` adds a line connecting the body parts making a skeleton on on each frame. The body parts to be connected and the color of these connecting lines are specified in the config file. By default: ``False``
trailpoints: int
Number of revious frames whose body parts are plotted in a frame (for displaying history). Default is set to 0.
displaycropped: bool, optional
Specifies whether only cropped frame is displayed (with labels analyzed therein), or the original frame with the labels analyzed in the cropped subset.
color_by : string, optional (default='bodypart')
Coloring rule. By default, each bodypart is colored differently.
If set to 'individual', points belonging to a single individual are colored the same.
Examples
--------
If you want to create the labeled video for only 1 video
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to create the labeled video for only 1 video and store the individual frames
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'],fastmode=True, save_frames=True)
--------
If you want to create the labeled video for multiple videos
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to create the labeled video for all the videos (as .avi extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'])
--------
If you want to create the labeled video for all the videos (as .mp4 extension) in a directory.
>>> deeplabcut.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'],videotype='mp4')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg["TrainingFraction"][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, modelprefix=modelprefix
) # automatically loads corresponding model (even training iteration based on snapshot index)
if save_frames:
fastmode = False # otherwise one cannot save frames
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
individuals = auxfun_multianimal.IntersectionofIndividualsandOnesGivenbyUser(
cfg, displayedindividuals)
if draw_skeleton:
bodyparts2connect = cfg["skeleton"]
skeleton_color = cfg["skeleton_color"]
else:
bodyparts2connect = None
skeleton_color = None
start_path = os.getcwd()
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
if not len(Videos):
print(
"No video(s) were found. Please check your paths and/or 'video_type'."
)
return
for video in Videos:
videofolder = Path(video).parents[0]
if destfolder is None:
destfolder = videofolder # where your folder with videos is.
auxiliaryfunctions.attempttomakefolder(destfolder)
os.chdir(destfolder) # THE VIDEO IS STILL IN THE VIDEO FOLDER
videotype = Path(video).suffix
print("Starting % ", destfolder, videos)
vname = str(Path(video).stem)
# if notanalyzed:
# notanalyzed,outdataname,sourcedataname,DLCscorer=auxiliaryfunctions.CheckifPostProcessing(folder,vname,DLCscorer,DLCscorerlegacy,suffix='checking')
if filtered == True:
videooutname1 = os.path.join(vname + DLCscorer +
"filtered_labeled.mp4")
videooutname2 = os.path.join(vname + DLCscorerlegacy +
"filtered_labeled.mp4")
else:
videooutname1 = os.path.join(vname + DLCscorer + "_labeled.mp4")
videooutname2 = os.path.join(vname + DLCscorerlegacy +
"_labeled.mp4")
if os.path.isfile(videooutname1) or os.path.isfile(videooutname2):
print("Labeled video already created.")
else:
print("Loading ", video, "and data.")
try:
df, filepath, _, _ = auxiliaryfunctions.load_analyzed_data(
destfolder, vname, DLCscorer, filtered, track_method)
metadata = auxiliaryfunctions.load_video_metadata(
destfolder, vname, DLCscorer)
if cfg.get("multianimalproject", False):
s = "_id" if color_by == "individual" else "_bp"
else:
s = ""
videooutname = filepath.replace(".h5", f"{s}_labeled.mp4")
if os.path.isfile(videooutname):
print("Labeled video already created. Skipping...")
continue
if all(individuals):
df = df.loc(axis=1)[:, individuals]
cropping = metadata["data"]["cropping"]
[x1, x2, y1, y2] = metadata["data"]["cropping_parameters"]
labeled_bpts = [
bp for bp in df.columns.get_level_values(
"bodyparts").unique() if bp in bodyparts
]
if not fastmode:
tmpfolder = os.path.join(str(videofolder), "temp-" + vname)
if save_frames:
auxiliaryfunctions.attempttomakefolder(tmpfolder)
clip = vp(video)
CreateVideoSlow(
videooutname,
clip,
df,
tmpfolder,
cfg["dotsize"],
cfg["colormap"],
cfg["alphavalue"],
cfg["pcutoff"],
trailpoints,
cropping,
x1,
x2,
y1,
y2,
save_frames,
labeled_bpts,
outputframerate,
Frames2plot,
bodyparts2connect,
skeleton_color,
draw_skeleton,
displaycropped,
color_by,
)
else:
if (displaycropped
): # then the cropped video + the labels is depicted
clip = vp(
fname=video,
sname=videooutname,
codec=codec,
sw=x2 - x1,
sh=y2 - y1,
)
else: # then the full video + the (perhaps in cropped mode analyzed labels) are depicted
clip = vp(fname=video, sname=videooutname, codec=codec)
CreateVideo(
clip,
df,
cfg["pcutoff"],
cfg["dotsize"],
cfg["colormap"],
labeled_bpts,
trailpoints,
cropping,
x1,
x2,
y1,
y2,
bodyparts2connect,
skeleton_color,
draw_skeleton,
displaycropped,
color_by,
)
except FileNotFoundError as e:
print(e)
continue
os.chdir(start_path)
