def comparevideolistsanddatafolders(config):
"""
Auxiliary function that compares the folders in labeled-data and the ones listed under video_sets (in the config file).
Parameter
----------
config : string
String containing the full path of the config file in the project.
"""
cfg = auxiliaryfunctions.read_config(config)
videos = cfg['video_sets'].keys()
video_names = [Path(i).stem for i in videos]
alldatafolders = [fn for fn in os.listdir(Path(config).parent / 'labeled-data') if '_labeled' not in fn]
print("Config file contains:", len(video_names))
print("Labeled-data contains:", len(alldatafolders))
for vn in video_names:
if vn in alldatafolders:
pass
else:
print(vn, " is missing as a folder!")
for vn in alldatafolders:
if vn in video_names:
pass
else:
print(vn, " is missing in config file!")
def return_train_network_path(config, shuffle, trainFraction):
''' Returns the training and test pose config file names as well as the folder where the snapshot is
Parameter
----------
config : string
Full path of the config.yaml file as a string.
shuffle: int
Integer value specifying the shuffle index to select for training.
trainFraction: float
Float specifying which training set fraction to use.
Returns the triple: trainposeconfigfile, testposeconfigfile, snapshotfolder
'''
from deeplabcutcore.utils import auxiliaryfunctions
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction, shuffle, cfg)
trainposeconfigfile = Path(
os.path.join(cfg['project_path'], str(modelfoldername), "train",
"pose_cfg.yaml"))
testposeconfigfile = Path(
os.path.join(cfg['project_path'], str(modelfoldername), "test",
"pose_cfg.yaml"))
snapshotfolder = Path(
os.path.join(cfg['project_path'], str(modelfoldername), 'train'))
return trainposeconfigfile, testposeconfigfile, snapshotfolder
def dropimagesduetolackofannotation(config):
"""
Drop images from corresponding folder for not annotated images: /labeled-data/*folder*/CollectedData_*scorer*.h5
Will be carried out iteratively for all *folders* in labeled-data.
Parameter
----------
config : string
String containing the full path of the config file in the project.
"""
cfg = auxiliaryfunctions.read_config(config)
videos = cfg['video_sets'].keys()
video_names = [Path(i).stem for i in videos]
folders = [Path(config).parent / 'labeled-data' /Path(i) for i in video_names]
for folder in folders:
fn=os.path.join(str(folder),'CollectedData_' + cfg['scorer'] + '.h5')
DC = pd.read_hdf(fn, 'df_with_missing')
dropped=False
annotatedimages=[fn.split(os.sep)[-1] for fn in DC.index]
imagelist=[fns for fns in os.listdir(str(folder)) if '.png' in fns]
print("Annotated images: ", len(annotatedimages)," In folder:", len(imagelist))
for imagename in imagelist:
if imagename in annotatedimages:
pass
else:
fullpath=os.path.join(cfg['project_path'],'labeled-data',folder,imagename)
if os.path.isfile(fullpath):
print("Deleting", fullpath)
os.remove(fullpath)
annotatedimages=[fn.split(os.sep)[-1] for fn in DC.index]
imagelist=[fns for fns in os.listdir(str(folder)) if '.png' in fns]
print("PROCESSED:", folder, " now # of annotated images: ", len(annotatedimages)," in folder:", len(imagelist))
def dropannotationfileentriesduetodeletedimages(config):
"""
Drop entries for all deleted images in annotation files, i.e. for folders of the type: /labeled-data/*folder*/CollectedData_*scorer*.h5
Will be carried out iteratively for all *folders* in labeled-data.
Parameter
----------
config : string
String containing the full path of the config file in the project.
"""
cfg = auxiliaryfunctions.read_config(config)
videos = cfg['video_sets'].keys()
video_names = [Path(i).stem for i in videos]
folders = [Path(config).parent / 'labeled-data' /Path(i) for i in video_names]
for folder in folders:
fn=os.path.join(str(folder),'CollectedData_' + cfg['scorer'] + '.h5')
DC = pd.read_hdf(fn, 'df_with_missing')
dropped=False
for imagename in DC.index:
if os.path.isfile(os.path.join(cfg['project_path'],imagename)):
pass
else:
print("Dropping...", imagename)
DC = DC.drop(imagename)
dropped=True
if dropped==True:
DC.to_hdf(fn, key='df_with_missing', mode='w')
DC.to_csv(os.path.join(str(folder),'CollectedData_'+ cfg['scorer']+".csv"))
def dropduplicatesinannotatinfiles(config):
"""
Drop duplicate entries (of images) in annotation files (this should no longer happen, but might be useful).
Parameter
----------
config : string
String containing the full path of the config file in the project.
"""
cfg = auxiliaryfunctions.read_config(config)
videos = cfg['video_sets'].keys()
video_names = [Path(i).stem for i in videos]
folders = [Path(config).parent / 'labeled-data' /Path(i) for i in video_names]
for folder in folders:
try:
fn=os.path.join(str(folder),'CollectedData_' + cfg['scorer'] + '.h5')
DC = pd.read_hdf(fn, 'df_with_missing')
numimages=len(DC.index)
DC = DC[~DC.index.duplicated(keep='first')]
if len(DC.index)<numimages:
print("Dropped",numimages-len(DC.index))
DC.to_hdf(fn, key='df_with_missing', mode='w')
DC.to_csv(os.path.join(str(folder),'CollectedData_'+ cfg['scorer']+".csv"))
except FileNotFoundError:
print("Attention:", folder, "does not appear to have labeled data!")
def convertannotationdata_fromwindows2unixstyle(config, userfeedback=True):
"""
Converts paths in annotation file (CollectedData_*user*.h5) in labeled-data/videofolder
from windows to linux format. This is important when one e.g. labeling on Windows, but
wants to re-label/check_labels/ on a Linux computer.
Note for training data annotated on Windows in Linux this is not necessary, as the data
gets converted during training set creation.
config : string
Full path of the config.yaml file as a string.
userfeedback: bool, optional
If true the user will be asked specifically for each folder in labeled-data if the containing csv shall be converted to hdf format.
"""
cfg = auxiliaryfunctions.read_config(config)
videos = cfg["video_sets"].keys()
video_names = [Path(i).stem for i in videos]
folders = [
Path(config).parent / "labeled-data" / Path(i) for i in video_names
]
for folder in folders:
if userfeedback == True:
print("Do you want to convert the annotationdata in folder:",
folder, "?")
askuser = input("yes/no")
else:
askuser = "******"
if askuser == "y" or askuser == "yes" or askuser == "Ja" or askuser == "ha":
fn = os.path.join(str(folder), "CollectedData_" + cfg["scorer"])
Data = pd.read_hdf(fn + ".h5", "df_with_missing")
convertpaths_to_unixstyle(Data, fn, cfg)
def adddatasetstovideolistandviceversa(config,prefix,width,height,suffix='.mp4'):
"""
First run comparevideolistsanddatafolders(config) to compare the folders in labeled-data and the ones listed under video_sets (in the config file).
If you detect differences this function can be used to maker sure each folder has a video entry & vice versa.
It corrects this problem in the following way:
If a folder in labeled-data does not contain a video entry in the config file then the prefix path will be added in front of the name of the labeled-data folder and combined
with the suffix variable as an ending. Width and height will be added as cropping variables as passed on. TODO: This should be written from the actual images!
If a video entry in the config file does not contain a folder in labeled-data, then the entry is removed.
Handle with care!
Parameter
----------
config : string
String containing the full path of the config file in the project.
"""
cfg = auxiliaryfunctions.read_config(config)
videos = cfg['video_sets'].keys()
video_names = [Path(i).stem for i in videos]
alldatafolders = [fn for fn in os.listdir(Path(config).parent / 'labeled-data') if '_labeled' not in fn]
print("Config file contains:", len(video_names))
print("Labeled-data contains:", len(alldatafolders))
toberemoved=[]
for vn in video_names:
if vn in alldatafolders:
pass
else:
print(vn, " is missing as a labeled folder >> removing key!")
for fullvideo in cfg['video_sets'].keys():
if vn in fullvideo:
toberemoved.append(fullvideo)
for vid in toberemoved:
del cfg['video_sets'][vid]
#Load updated lists:
videos = cfg['video_sets'].keys()
video_names = [Path(i).stem for i in videos]
for vn in alldatafolders:
if vn in video_names:
pass
else:
print(vn, " is missing in config file >> adding it!")
#cfg['video_sets'][vn]
cfg['video_sets'].update({os.path.join(prefix,vn+suffix) : {'crop': ', '.join(map(str, [0, width, 0, height]))}})
auxiliaryfunctions.write_config(config,cfg)
def merge_datasets(config,forceiterate=None):
"""
Checks if the original training dataset can be merged with the newly refined training dataset. To do so it will check
if the frames in all extracted video sets were relabeled. If this is the case then the iterate variable is advanced by 1.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
forceiterate: int, optional
If an integer is given the iteration variable is set to this value (this is only done if all datasets were labeled or refined)
Example
--------
>>> deeplabcutcore.merge_datasets('/analysis/project/reaching-task/config.yaml')
--------
"""
import yaml
cfg = auxiliaryfunctions.read_config(config)
config_path = Path(config).parents[0]
bf=Path(str(config_path/'labeled-data'))
allfolders = [os.path.join(bf,fn) for fn in os.listdir(bf) if "_labeled" not in fn] #exclude labeled data folders!
flagged=False
for findex,folder in enumerate(allfolders):
if os.path.isfile(os.path.join(folder,'MachineLabelsRefine.h5')): #Folder that was manually refine...
pass
elif os.path.isfile(os.path.join(folder,'CollectedData_'+cfg['scorer']+'.h5')): #Folder that contains human data set...
pass
else:
print("The following folder was not manually refined,...",folder)
flagged=True
pass #this folder does not contain a MachineLabelsRefine file (not updated...)
if flagged==False:
# updates iteration by 1
iter_prev=cfg['iteration']
if not forceiterate:
cfg['iteration']=int(iter_prev+1)
else:
cfg['iteration']=forceiterate
auxiliaryfunctions.write_config(config,cfg)
print("Merged data sets and updated refinement iteration to "+str(cfg['iteration'])+".")
print("Now you can create a new training set for the expanded annotated images (use create_training_dataset).")
else:
print("Please label, or remove the un-corrected folders.")
def check_labels(config, Labels=["+", ".", "x"], scale=1):
"""
Double check if the labels were at correct locations and stored in a proper file format.\n
This creates a new subdirectory for each video under the 'labeled-data' and all the frames are plotted with the labels.\n
Make sure that these labels are fine.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
Labels: List of at least 3 matplotlib markers. The first one will be used to indicate the human ground truth location (Default: +)
scale : float, default =1
Change the relative size of the output images.
Example
--------
for labeling the frames
>>> deeplabcutcore.check_labels('/analysis/project/reaching-task/config.yaml')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
videos = cfg["video_sets"].keys()
video_names = [Path(i).stem for i in videos]
# plotting parameters:
cc = 0 # label index / here only 0, for human labeler
Colorscheme = get_cmap(len(cfg["bodyparts"]), cfg["colormap"])
# folders = [Path(config).parent / 'labeled-data' /Path(i) for i in video_names]
folders = [
os.path.join(cfg["project_path"], "labeled-data", str(Path(i)))
for i in video_names
]
print("Creating images with labels by %s." % cfg["scorer"])
for folder in folders:
try:
DataCombined = pd.read_hdf(
os.path.join(str(folder), "CollectedData_" + cfg["scorer"] + ".h5"),
"df_with_missing",
)
MakeLabeledPlots(folder, DataCombined, cfg, Labels, Colorscheme, cc, scale)
except FileNotFoundError:
print("Attention:", folder, "does not appear to have labeled data!")
print(
"If all the labels are ok, then use the function 'create_training_dataset' to create the training dataset!"
)
def get_largestshuffle_index(config):
""" Returns the largest shuffle for all dlc-models in the current iteration."""
cfg = auxiliaryfunctions.read_config(config)
project_path = cfg["project_path"]
iterate = "iteration-" + str(cfg["iteration"])
dlc_model_path = os.path.join(project_path, "dlc-models", iterate)
if os.path.isdir(dlc_model_path):
models = os.listdir(dlc_model_path)
# sort the models directories
models.sort(key=lambda f: int("".join(filter(str.isdigit, f))))
# get the shuffle index
max_shuffle_index = int(models[-1].split("shuffle")[-1])
else:
max_shuffle_index = 0
return max_shuffle_index
def get_largestshuffle_index(config):
''' Returns the largest shuffle for all dlc-models in the current iteration.'''
cfg = auxiliaryfunctions.read_config(config)
project_path = cfg['project_path']
iterate = 'iteration-'+str(cfg['iteration'])
dlc_model_path = os.path.join(project_path,'dlc-models',iterate)
if os.path.isdir(dlc_model_path):
models = os.listdir(dlc_model_path)
# sort the models directories
models.sort(key=lambda f: int(''.join(filter(str.isdigit, f))))
# get the shuffle index
max_shuffle_index = int(models[-1].split('shuffle')[-1])
else:
max_shuffle_index = 0
return(max_shuffle_index)
def select_cropping_area(config, videos=None):
"""
Interactively select the cropping area of all videos in the config.
A user interface pops up with a frame to select the cropping parameters.
Use the left click to draw a box and hit the button 'set cropping parameters'
to store the cropping parameters for a video in the config.yaml file.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : optional (default=None)
List of videos whose cropping areas are to be defined. Note that full paths are required.
By default, all videos in the config are successively loaded.
Returns
-------
cfg : dict
Updated project configuration
"""
from deeplabcutcore.utils import auxiliaryfunctions, auxfun_videos
cfg = auxiliaryfunctions.read_config(config)
if videos is None:
videos = cfg["video_sets"]
for video in videos:
coords = auxfun_videos.draw_bbox(video)
if coords:
cfg["video_sets"][video] = {
"crop": ", ".join(
map(
str,
[
int(coords[0]),
int(coords[2]),
int(coords[1]),
int(coords[3]),
],
)
)
}
auxiliaryfunctions.write_config(config, cfg)
return cfg
def refine_labels(config, multianimal=False):
"""
Refines the labels of the outlier frames extracted from the analyzed videos.\n Helps in augmenting the training dataset.
Use the function ``analyze_video`` to analyze a video and extracts the outlier frames using the function
``extract_outlier_frames`` before refining the labels.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
Screens : int value of the number of Screens in landscape mode, i.e. if you have 2 screens, enter 2. Default is 1.
scale_h & scale_w : you can modify how much of the screen the GUI should occupy. The default is .9 and .8, respectively.
img_scale : if you want to make the plot of the frame larger, consider changing this to .008 or more. Be careful though, too large and you will not see the buttons fully!
Examples
--------
>>> deeplabcutcore.refine_labels('/analysis/project/reaching-task/config.yaml', Screens=2, imag_scale=.0075)
--------
"""
startpath = os.getcwd()
wd = Path(config).resolve().parents[0]
os.chdir(str(wd))
cfg = auxiliaryfunctions.read_config(config)
if multianimal == False and not cfg.get("multianimalproject", False):
from deeplabcutcore.refine_training_dataset import refinement
refinement.show(config)
else: # loading multianimal labeling GUI
from deeplabcutcore.refine_training_dataset import (
multiple_individuals_refinement_toolbox,
)
multiple_individuals_refinement_toolbox.show(config)
os.chdir(startpath)
def transform_data(config):
"""
This function adds the full path to labeling dataset.
It also adds the correct path to the video file in the config file.
"""
import pandas as pd
cfg = auxiliaryfunctions.read_config(config)
project_path = str(Path(config).parents[0])
cfg["project_path"] = project_path
if "Reaching" in project_path:
video_file = os.path.join(project_path, "videos", "reachingvideo1.avi")
elif "openfield" in project_path:
video_file = os.path.join(project_path, "videos", "m4s1.mp4")
else:
print("This is not an offical demo dataset.")
if "WILL BE AUTOMATICALLY UPDATED BY DEMO CODE" in cfg["video_sets"].keys(
):
cfg["video_sets"][str(video_file)] = cfg["video_sets"].pop(
"WILL BE AUTOMATICALLY UPDATED BY DEMO CODE")
auxiliaryfunctions.write_config(config, cfg)
def transform_data(config):
"""
This function adds the full path to labeling dataset.
It also adds the correct path to the video file in the config file.
"""
import pandas as pd
cfg = auxiliaryfunctions.read_config(config)
project_path = str(Path(config).parents[0])
cfg['project_path'] = project_path
if 'Reaching' in project_path:
video_file = os.path.join(project_path, 'videos', 'reachingvideo1.avi')
elif 'openfield' in project_path:
video_file = os.path.join(project_path, 'videos', 'm4s1.mp4')
else:
print("This is not an offical demo dataset.")
if 'WILL BE AUTOMATICALLY UPDATED BY DEMO CODE' in cfg['video_sets'].keys(
):
cfg['video_sets'][str(video_file)] = cfg['video_sets'].pop(
'WILL BE AUTOMATICALLY UPDATED BY DEMO CODE')
auxiliaryfunctions.write_config(config, cfg)
def return_evaluate_network_data(config,
shuffle=0,
trainingsetindex=0,
comparisonbodyparts="all",
Snapindex=None,
rescale=False,
fulldata=False,
show_errors=True):
"""
Returns the results for (previously evaluated) network. deeplabcutcore.evaluate_network(..)
Returns list of (per model): [trainingsiterations,trainfraction,shuffle,trainerror,testerror,pcutoff,trainerrorpcutoff,testerrorpcutoff,Snapshots[snapindex],scale,net_type]
If fulldata=True, also returns (the complete annotation and prediction array)
Returns list of: (DataMachine, Data, data, trainIndices, testIndices, trainFraction, DLCscorer,comparisonbodyparts, cfg, Snapshots[snapindex])
----------
config : string
Full path of the config.yaml file as a string.
shuffle: integer
integers specifying shuffle index of the training dataset. The default is 0.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This
variable can also be set to "all".
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
rescale: bool, default False
Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every
image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported
in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated
on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the
original size!
Examples
--------
If you do not want to plot
>>> deeplabcutcore._evaluate_network_data('/analysis/project/reaching-task/config.yaml', shuffle=[1])
--------
If you want to plot
>>> deeplabcutcore.evaluate_network('/analysis/project/reaching-task/config.yaml',shuffle=[1],True)
"""
import os
from skimage import io
import skimage.color
from deeplabcutcore.pose_estimation_tensorflow.config import load_config
from deeplabcutcore.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input
from deeplabcutcore.utils import auxiliaryfunctions, visualization
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
#Data=pd.read_hdf(os.path.join(cfg["project_path"],str(trainingsetfolder),'CollectedData_' + cfg["scorer"] + '.h5'),'df_with_missing')
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
##################################################
# Load data...
##################################################
trainFraction = cfg["TrainingFraction"][trainingsetindex]
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
# Load meta data
data, trainIndices, testIndices, trainFraction = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
########################### RESCALING (to global scale)
if rescale == True:
scale = dlc_cfg['global_scale']
print("Rescaling Data to ", scale)
Data = pd.read_hdf(
os.path.join(cfg["project_path"], str(trainingsetfolder),
'CollectedData_' + cfg["scorer"] + '.h5'),
'df_with_missing') * scale
else:
scale = 1
Data = pd.read_hdf(
os.path.join(cfg["project_path"], str(trainingsetfolder),
'CollectedData_' + cfg["scorer"] + '.h5'),
'df_with_missing')
evaluationfolder = os.path.join(
cfg["project_path"],
str(auxiliaryfunctions.GetEvaluationFolder(trainFraction, shuffle,
cfg)))
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split('.')[0]
for fn in os.listdir(os.path.join(str(modelfolder), 'train'))
if "index" in fn
])
if len(Snapshots) == 0:
print(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
snapindices = []
else:
increasing_indices = np.argsort(
[int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if Snapindex == None:
Snapindex = cfg["snapshotindex"]
if Snapindex == -1:
snapindices = [-1]
elif Snapindex == "all":
snapindices = range(len(Snapshots))
elif Snapindex < len(Snapshots):
snapindices = [Snapindex]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
DATA = []
results = []
for snapindex in snapindices:
dlc_cfg['init_weights'] = os.path.join(
str(modelfolder), 'train',
Snapshots[snapindex]) #setting weights to corresponding snapshot.
trainingsiterations = (dlc_cfg['init_weights'].split(
os.sep)[-1]).split('-')[
-1] #read how many training siterations that corresponds to.
#name for deeplabcut net (based on its parameters)
#DLCscorer = auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, trainingsiterations)
print("Retrieving ", DLCscorer, " with # of trainingiterations:",
trainingsiterations)
notanalyzed, resultsfilename, DLCscorer = auxiliaryfunctions.CheckifNotEvaluated(
str(evaluationfolder), DLCscorer, DLCscorerlegacy,
Snapshots[snapindex])
#resultsfilename=os.path.join(str(evaluationfolder),DLCscorer + '-' + str(Snapshots[snapindex])+ '.h5') # + '-' + str(snapshot)+ ' #'-' + Snapshots[snapindex]+ '.h5')
print(resultsfilename)
if not notanalyzed and os.path.isfile(resultsfilename): #data exists..
DataMachine = pd.read_hdf(resultsfilename, 'df_with_missing')
DataCombined = pd.concat([Data.T, DataMachine.T], axis=0).T
RMSE, RMSEpcutoff = pairwisedistances(DataCombined, cfg["scorer"],
DLCscorer, cfg["pcutoff"],
comparisonbodyparts)
testerror = np.nanmean(RMSE.iloc[testIndices].values.flatten())
trainerror = np.nanmean(RMSE.iloc[trainIndices].values.flatten())
testerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[testIndices].values.flatten())
trainerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[trainIndices].values.flatten())
if show_errors == True:
print("Results for",
trainingsiterations, " training iterations:",
int(100 * trainFraction), shuffle, "train error:",
np.round(trainerror, 2), "pixels. Test error:",
np.round(testerror, 2), " pixels.")
print("With pcutoff of", cfg["pcutoff"], " train error:",
np.round(trainerrorpcutoff, 2), "pixels. Test error:",
np.round(testerrorpcutoff, 2), "pixels")
print("Snapshot", Snapshots[snapindex])
r = [
trainingsiterations,
int(100 * trainFraction), shuffle,
np.round(trainerror, 2),
np.round(testerror, 2), cfg["pcutoff"],
np.round(trainerrorpcutoff, 2),
np.round(testerrorpcutoff, 2), Snapshots[snapindex], scale,
dlc_cfg['net_type']
]
results.append(r)
else:
print("Model not trained/evaluated!")
if fulldata == True:
DATA.append([
DataMachine, Data, data, trainIndices, testIndices,
trainFraction, DLCscorer, comparisonbodyparts, cfg,
evaluationfolder, Snapshots[snapindex]
])
os.chdir(start_path)
if fulldata == True:
return DATA, results
else:
return results
def evaluate_network(config,
Shuffles=[1],
trainingsetindex=0,
plotting=None,
show_errors=True,
comparisonbodyparts="all",
gputouse=None,
rescale=False):
"""
Evaluates the network based on the saved models at different stages of the training network.\n
The evaluation results are stored in the .h5 and .csv file under the subdirectory 'evaluation_results'.
Change the snapshotindex parameter in the config file to 'all' in order to evaluate all the saved models.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
Shuffles: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This
variable can also be set to "all".
plotting: bool, optional
Plots the predictions on the train and test images. The default is ``False``; if provided it must be either ``True`` or ``False``
show_errors: bool, optional
Display train and test errors. The default is `True``
comparisonbodyparts: list of bodyparts, Default is "all".
The average error will be computed for those body parts only (Has to be a subset of the body parts).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
rescale: bool, default False
Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every
image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported
in pixels at rescaled to the *original* size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated
on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the
original size!
Examples
--------
If you do not want to plot
>>> deeplabcutcore.evaluate_network('/analysis/project/reaching-task/config.yaml', Shuffles=[1])
--------
If you want to plot
>>> deeplabcutcore.evaluate_network('/analysis/project/reaching-task/config.yaml',Shuffles=[1],True)
"""
import os
#import skimage.color
#from skimage.io import imread
from deeplabcutcore.utils.auxfun_videos import imread, imresize
from deeplabcutcore.pose_estimation_tensorflow.nnet import predict
from deeplabcutcore.pose_estimation_tensorflow.config import load_config
from deeplabcutcore.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input
from deeplabcutcore.utils import auxiliaryfunctions
import tensorflow as tf
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ[
'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
tf.compat.v1.reset_default_graph()
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' #
# tf.logging.set_verbosity(tf.logging.WARN)
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: #gpu selectinon
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
if trainingsetindex == 'all':
TrainingFractions = cfg["TrainingFraction"]
else:
if trainingsetindex < len(
cfg["TrainingFraction"]) and trainingsetindex >= 0:
TrainingFractions = [
cfg["TrainingFraction"][int(trainingsetindex)]
]
else:
raise Exception('Please check the trainingsetindex! ',
trainingsetindex,
' should be an integer from 0 .. ',
int(len(cfg["TrainingFraction"]) - 1))
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg)
Data = pd.read_hdf(
os.path.join(cfg["project_path"], str(trainingsetfolder),
'CollectedData_' + cfg["scorer"] + '.h5'),
'df_with_missing')
# Get list of body parts to evaluate network for
comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts)
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(
str(cfg["project_path"] + "/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder, trainFraction, shuffle, cfg)
modelfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetModelFolder(trainFraction, shuffle,
cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
# Load meta data
data, trainIndices, testIndices, trainFraction = auxiliaryfunctions.LoadMetadata(
os.path.join(cfg["project_path"], metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
#change batch size, if it was edited during analysis!
dlc_cfg['batch_size'] = 1 #in case this was edited for analysis.
#Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
trainFraction, shuffle, cfg)))
auxiliaryfunctions.attempttomakefolder(evaluationfolder,
recursive=True)
#path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([
fn.split('.')[0]
for fn in os.listdir(os.path.join(str(modelfolder), 'train'))
if "index" in fn
])
try: #check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction))
increasing_indices = np.argsort(
[int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
raise ValueError(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
final_result = []
########################### RESCALING (to global scale)
if rescale == True:
scale = dlc_cfg['global_scale']
Data = pd.read_hdf(
os.path.join(cfg["project_path"], str(trainingsetfolder),
'CollectedData_' + cfg["scorer"] + '.h5'),
'df_with_missing') * scale
else:
scale = 1
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
dlc_cfg['init_weights'] = os.path.join(
str(modelfolder), 'train', Snapshots[snapindex]
) #setting weights to corresponding snapshot.
trainingsiterations = (
dlc_cfg['init_weights'].split(os.sep)[-1]
).split(
'-'
)[-1] #read how many training siterations that corresponds to.
# Name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, trainingsiterations)
print("Running ", DLCscorer, " with # of trainingiterations:",
trainingsiterations)
notanalyzed, resultsfilename, DLCscorer = auxiliaryfunctions.CheckifNotEvaluated(
str(evaluationfolder), DLCscorer, DLCscorerlegacy,
Snapshots[snapindex])
if notanalyzed:
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = predict.setup_pose_prediction(
dlc_cfg)
Numimages = len(Data.index)
PredicteData = np.zeros(
(Numimages, 3 * len(dlc_cfg['all_joints_names'])))
print("Analyzing data...")
for imageindex, imagename in tqdm(enumerate(Data.index)):
image = imread(os.path.join(cfg['project_path'],
imagename),
mode='RGB')
if scale != 1:
image = imresize(image, scale)
#image = skimage.color.gray2rgb(image)
image_batch = data_to_input(image)
# Compute prediction with the CNN
outputs_np = sess.run(outputs,
feed_dict={inputs: image_batch})
scmap, locref = predict.extract_cnn_output(
outputs_np, dlc_cfg)
# Extract maximum scoring location from the heatmap, assume 1 person
pose = predict.argmax_pose_predict(
scmap, locref, dlc_cfg.stride)
PredicteData[imageindex, :] = pose.flatten(
) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts!
sess.close() #closes the current tf session
index = pd.MultiIndex.from_product(
[[DLCscorer], dlc_cfg['all_joints_names'],
['x', 'y', 'likelihood']],
names=['scorer', 'bodyparts', 'coords'])
# Saving results
DataMachine = pd.DataFrame(PredicteData,
columns=index,
index=Data.index.values)
DataMachine.to_hdf(resultsfilename,
'df_with_missing',
format='table',
mode='w')
print("Done and results stored for snapshot: ",
Snapshots[snapindex])
DataCombined = pd.concat([Data.T, DataMachine.T],
axis=0,
sort=False).T
RMSE, RMSEpcutoff = pairwisedistances(
DataCombined, cfg["scorer"], DLCscorer, cfg["pcutoff"],
comparisonbodyparts)
testerror = np.nanmean(
RMSE.iloc[testIndices].values.flatten())
trainerror = np.nanmean(
RMSE.iloc[trainIndices].values.flatten())
testerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[testIndices].values.flatten())
trainerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[trainIndices].values.flatten())
results = [
trainingsiterations,
int(100 * trainFraction), shuffle,
np.round(trainerror, 2),
np.round(testerror, 2), cfg["pcutoff"],
np.round(trainerrorpcutoff, 2),
np.round(testerrorpcutoff, 2)
]
final_result.append(results)
if show_errors == True:
print("Results for",
trainingsiterations, " training iterations:",
int(100 * trainFraction), shuffle,
"train error:",
np.round(trainerror, 2), "pixels. Test error:",
np.round(testerror, 2), " pixels.")
print("With pcutoff of",
cfg["pcutoff"], " train error:",
np.round(trainerrorpcutoff,
2), "pixels. Test error:",
np.round(testerrorpcutoff, 2), "pixels")
if scale != 1:
print(
"The predictions have been calculated for rescaled images (and rescaled ground truth). Scale:",
scale)
print(
"Thereby, the errors are given by the average distances between the labels by DLC and the scorer."
)
if plotting == True:
print("Plotting...")
foldername = os.path.join(
str(evaluationfolder), 'LabeledImages_' +
DLCscorer + '_' + Snapshots[snapindex])
auxiliaryfunctions.attempttomakefolder(foldername)
Plotting(
cfg, comparisonbodyparts, DLCscorer, trainIndices,
DataCombined * 1. / scale, foldername
) #Rescaling coordinates to have figure in original size!
tf.compat.v1.reset_default_graph()
#print(final_result)
else:
DataMachine = pd.read_hdf(resultsfilename,
'df_with_missing')
if plotting == True:
DataCombined = pd.concat([Data.T, DataMachine.T],
axis=0,
sort=False).T
print(
"Plotting...(attention scale might be inconsistent in comparison to when data was analyzed; i.e. if you used rescale)"
)
foldername = os.path.join(
str(evaluationfolder), 'LabeledImages_' +
DLCscorer + '_' + Snapshots[snapindex])
auxiliaryfunctions.attempttomakefolder(foldername)
Plotting(cfg, comparisonbodyparts, DLCscorer,
trainIndices, DataCombined * 1. / scale,
foldername)
if len(final_result) > 0: #Only append if results were calculated
make_results_file(final_result, evaluationfolder, DLCscorer)
print(
"The network is evaluated and the results are stored in the subdirectory 'evaluation_results'."
)
print(
"If it generalizes well, choose the best model for prediction and update the config file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos."
)
print(
"Otherwise consider retraining the network (see DeepLabCut workflow Fig 2)"
)
#returning to intial folder
os.chdir(str(start_path))
def extract_frames(config,
mode='automatic',
algo='kmeans',
crop=False,
userfeedback=True,
cluster_step=1,
cluster_resizewidth=30,
cluster_color=False,
opencv=True,
slider_width=25):
"""
Extracts frames from the videos in the config.yaml file. Only the videos in the config.yaml will be used to select the frames.\n
Use the function ``add_new_video`` at any stage of the project to add new videos to the config file and extract their frames.
The provided function either selects frames from the videos in a randomly and temporally uniformly distributed way (uniform), \n
by clustering based on visual appearance (k-means), or by manual selection.
Three important parameters for automatic extraction: numframes2pick, start and stop are set in the config file.
Please refer to the user guide for more details on methods and parameters https://www.nature.com/articles/s41596-019-0176-0
or the preprint: https://www.biorxiv.org/content/biorxiv/early/2018/11/24/476531.full.pdf
Parameters
----------
config : string
Full path of the config.yaml file as a string.
mode : string
String containing the mode of extraction. It must be either ``automatic`` or ``manual``.
algo : string
String specifying the algorithm to use for selecting the frames. Currently, deeplabcut supports either ``kmeans`` or ``uniform`` based selection. This flag is
only required for ``automatic`` mode and the default is ``uniform``. For uniform, frames are picked in temporally uniform way, kmeans performs clustering on downsampled frames (see user guide for details).
Note: color information is discarded for kmeans, thus e.g. for camouflaged octopus clustering one might want to change this.
crop : bool, optional
If True, video frames are cropped according to the corresponding coordinates stored in the config.yaml.
Alternatively, if cropping coordinates are not known yet, crop='GUI' triggers a user interface
where the cropping area can be manually drawn and saved.
userfeedback: bool, optional
If this is set to false during automatic mode then frames for all videos are extracted. The user can set this to true, which will result in a dialog,
where the user is asked for each video if (additional/any) frames from this video should be extracted. Use this, e.g. if you have already labeled
some folders and want to extract data for new videos.
cluster_resizewidth: number, default: 30
For k-means one can change the width to which the images are downsampled (aspect ratio is fixed).
cluster_step: number, default: 1
By default each frame is used for clustering, but for long videos one could only use every nth frame (set by: cluster_step). This saves memory before clustering can start, however,
reading the individual frames takes longer due to the skipping.
cluster_color: bool, default: False
If false then each downsampled image is treated as a grayscale vector (discarding color information). If true, then the color channels are considered. This increases
the computational complexity.
opencv: bool, default: True
Uses openCV for loading & extractiong (otherwise moviepy (legacy))
slider_width: number, default: 25
Width of the video frames slider, in percent of window
Examples
--------
for selecting frames automatically with 'kmeans' and want to crop the frames.
>>> deeplabcutcore.extract_frames('/analysis/project/reaching-task/config.yaml','automatic','kmeans',True)
--------
for selecting frames automatically with 'kmeans' and defining the cropping area at runtime.
>>> deeplabcutcore.extract_frames('/analysis/project/reaching-task/config.yaml','automatic','kmeans','GUI')
--------
for selecting frames automatically with 'kmeans' and considering the color information.
>>> deeplabcutcore.extract_frames('/analysis/project/reaching-task/config.yaml','automatic','kmeans',cluster_color=True)
--------
for selecting frames automatically with 'uniform' and want to crop the frames.
>>> deeplabcutcore.extract_frames('/analysis/project/reaching-task/config.yaml','automatic',crop=True)
--------
for selecting frames manually,
>>> deeplabcutcore.extract_frames('/analysis/project/reaching-task/config.yaml','manual')
--------
for selecting frames manually, with a 60% wide frames slider
>>> deeplabcutcore.extract_frames('/analysis/project/reaching-task/config.yaml','manual', slider_width=60)
While selecting the frames manually, you do not need to specify the ``crop`` parameter in the command. Rather, you will get a prompt in the graphic user interface to choose
if you need to crop or not.
--------
"""
import os
import sys
import numpy as np
from pathlib import Path
from skimage import io
from skimage.util import img_as_ubyte
from deeplabcutcore.utils import frameselectiontools
from deeplabcutcore.utils import auxiliaryfunctions
if mode == "manual":
wd = Path(config).resolve().parents[0]
os.chdir(str(wd))
from deeplabcutcore.generate_training_dataset import frame_extraction_toolbox
from deeplabcutcore.utils import select_crop_parameters
frame_extraction_toolbox.show(config, slider_width)
elif mode == "automatic":
config_file = Path(config).resolve()
cfg = auxiliaryfunctions.read_config(config_file)
print("Config file read successfully.")
numframes2pick = cfg['numframes2pick']
start = cfg['start']
stop = cfg['stop']
# Check for variable correctness
if start > 1 or stop > 1 or start < 0 or stop < 0 or start >= stop:
raise Exception(
"Erroneous start or stop values. Please correct it in the config file."
)
if numframes2pick < 1 and not int(numframes2pick):
raise Exception(
"Perhaps consider extracting more, or a natural number of frames."
)
videos = cfg['video_sets'].keys()
if opencv:
import cv2
else:
from moviepy.editor import VideoFileClip
has_failed = []
for vindex, video in enumerate(videos):
if userfeedback:
print(
"Do you want to extract (perhaps additional) frames for video:",
video, "?")
askuser = input("yes/no")
else:
askuser = "******"
if askuser == 'y' or askuser == 'yes' or askuser == 'Ja' or askuser == 'ha'\
or askuser == 'oui' or askuser == 'ouais': # multilanguage support :)
if opencv:
cap = cv2.VideoCapture(video)
fps = cap.get(
5
) # https://docs.opencv.org/2.4/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get
nframes = int(cap.get(7))
else:
# Moviepy:
clip = VideoFileClip(video)
fps = clip.fps
nframes = int(np.ceil(clip.duration * 1. / fps))
if not nframes:
print('Video could not be opened. Skipping...')
continue
indexlength = int(np.ceil(np.log10(nframes)))
fname = Path(video)
output_path = Path(
config).parents[0] / 'labeled-data' / fname.stem
if output_path.exists():
if len(os.listdir(output_path)):
if userfeedback:
askuser = input(
"The directory already contains some frames. Do you want to add to it?(yes/no): "
)
if not (askuser == 'y' or askuser == 'yes'
or askuser == 'Y' or askuser == 'Yes'):
sys.exit("Delete the frames and try again later!")
if crop == 'GUI':
cfg = select_cropping_area(config, [video])
coords = cfg['video_sets'][video]['crop'].split(',')
if crop and not opencv:
clip = clip.crop(y1=int(coords[2]),
y2=int(coords[3]),
x1=int(coords[0]),
x2=int(coords[1]))
elif not crop:
coords = None
print("Extracting frames based on %s ..." % algo)
if algo == 'uniform':
if opencv:
frames2pick = frameselectiontools.UniformFramescv2(
cap, numframes2pick, start, stop)
else:
frames2pick = frameselectiontools.UniformFrames(
clip, numframes2pick, start, stop)
elif algo == 'kmeans':
if opencv:
frames2pick = frameselectiontools.KmeansbasedFrameselectioncv2(
cap,
numframes2pick,
start,
stop,
crop,
coords,
step=cluster_step,
resizewidth=cluster_resizewidth,
color=cluster_color)
else:
frames2pick = frameselectiontools.KmeansbasedFrameselection(
clip,
numframes2pick,
start,
stop,
step=cluster_step,
resizewidth=cluster_resizewidth,
color=cluster_color)
else:
print(
"Please implement this method yourself and send us a pull request! Otherwise, choose 'uniform' or 'kmeans'."
)
frames2pick = []
if not len(frames2pick):
print('Frame selection failed...')
return
output_path = Path(config).parents[0] / 'labeled-data' / Path(
video).stem
is_valid = []
if opencv:
for index in frames2pick:
cap.set(1, index) # extract a particular frame
ret, frame = cap.read()
if ret:
image = img_as_ubyte(
cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
img_name = str(output_path) + '/img' + str(
index).zfill(indexlength) + ".png"
if crop:
io.imsave(
img_name,
image[int(coords[2]):int(coords[3]),
int(coords[0]):int(coords[1]), :]
) # y1 = int(coords[2]),y2 = int(coords[3]),x1 = int(coords[0]), x2 = int(coords[1]
else:
io.imsave(img_name, image)
is_valid.append(True)
else:
print("Frame", index, " not found!")
is_valid.append(False)
cap.release()
else:
for index in frames2pick:
try:
image = img_as_ubyte(
clip.get_frame(index * 1. / clip.fps))
img_name = str(output_path) + '/img' + str(
index).zfill(indexlength) + ".png"
io.imsave(img_name, image)
if np.var(image) == 0: # constant image
print(
"Seems like black/constant images are extracted from your video. Perhaps consider using opencv under the hood, by setting: opencv=True"
)
is_valid.append(True)
except FileNotFoundError:
print("Frame # ", index, " does not exist.")
is_valid.append(False)
clip.close()
del clip
if not any(is_valid):
has_failed.append(True)
else:
has_failed.append(False)
if all(has_failed):
print('Frame extraction failed. Video files must be corrupted.')
return
elif any(has_failed):
print('Although most frames were extracted, some were invalid.')
else:
print("Frames were successfully extracted.")
print(
"\nYou can now label the frames using the function 'label_frames' "
"(if you extracted enough frames for all videos).")
else:
print(
"Invalid MODE. Choose either 'manual' or 'automatic'. Check ``help(deeplabcutcore.extract_frames)`` on python and ``deeplabcutcore.extract_frames?`` \
for ipython/jupyter notebook for more details.")
def train_network(config,
shuffle=1,
trainingsetindex=0,
max_snapshots_to_keep=5,
displayiters=None,
saveiters=None,
maxiters=None,
allow_growth=False,
gputouse=None,
autotune=False,
keepdeconvweights=True):
"""Trains the network with the labels in the training dataset.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
shuffle: int, optional
Integer value specifying the shuffle index to select for training. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
Additional parameters:
max_snapshots_to_keep: int, or None. Sets how many snapshots are kept, i.e. states of the trained network. Every savinginteration many times
a snapshot is stored, however only the last max_snapshots_to_keep many are kept! If you change this to None, then all are kept.
See: https://github.com/AlexEMG/DeepLabCut/issues/8#issuecomment-387404835
displayiters: this variable is actually set in pose_config.yaml. However, you can overwrite it with this hack. Don't use this regularly, just if you are too lazy to dig out
the pose_config.yaml file for the corresponding project. If None, the value from there is used, otherwise it is overwritten! Default: None
saveiters: this variable is actually set in pose_config.yaml. However, you can overwrite it with this hack. Don't use this regularly, just if you are too lazy to dig out
the pose_config.yaml file for the corresponding project. If None, the value from there is used, otherwise it is overwritten! Default: None
maxiters: this variable is actually set in pose_config.yaml. However, you can overwrite it with this hack. Don't use this regularly, just if you are too lazy to dig out
the pose_config.yaml file for the corresponding project. If None, the value from there is used, otherwise it is overwritten! Default: None
allow_groth: bool, default false.
For some smaller GPUs the memory issues happen. If true, the memory allocator does not pre-allocate the entire specified
GPU memory region, instead starting small and growing as needed. See issue: https://forum.image.sc/t/how-to-stop-running-out-of-vram/30551/2
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
autotune: property of TensorFlow, somehow faster if 'false' (as Eldar found out, see https://github.com/tensorflow/tensorflow/issues/13317). Default: False
keepdeconvweights: bool, default: true
Also restores the weights of the deconvolution layers (and the backbone) when training from a snapshot. Note that if you change the number of bodyparts, you need to
set this to false for re-training.
Example
--------
for training the network for first shuffle of the training dataset.
>>> deeplabcutcore.train_network('/analysis/project/reaching-task/config.yaml')
--------
for training the network for second shuffle of the training dataset.
>>> deeplabcutcore.train_network('/analysis/project/reaching-task/config.yaml',shuffle=2,keepdeconvweights=True)
--------
"""
import tensorflow as tf
vers = (tf.__version__).split('.')
if int(vers[0]) == 1 and int(vers[1]) > 12:
TF = tf.compat.v1
else:
TF = tf
#reload logger.
import importlib
import logging
importlib.reload(logging)
logging.shutdown()
from deeplabcutcore.pose_estimation_tensorflow.train import train
from deeplabcutcore.utils import auxiliaryfunctions
TF.reset_default_graph()
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
modelfoldername = auxiliaryfunctions.GetModelFolder(
cfg["TrainingFraction"][trainingsetindex], shuffle, cfg)
poseconfigfile = Path(
os.path.join(cfg['project_path'], str(modelfoldername), "train",
"pose_cfg.yaml"))
if not poseconfigfile.is_file():
print("The training datafile ", poseconfigfile, " is not present.")
print(
"Probably, the training dataset for this specific shuffle index was not created."
)
print(
"Try with a different shuffle/trainingsetfraction or use function 'create_training_dataset' to create a new trainingdataset with this shuffle index."
)
else:
# Set environment variables
if autotune is not False: #see: https://github.com/tensorflow/tensorflow/issues/13317
os.environ['TF_CUDNN_USE_AUTOTUNE'] = '0'
if gputouse is not None:
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
try:
train(str(poseconfigfile),
displayiters,
saveiters,
maxiters,
max_to_keep=max_snapshots_to_keep,
keepdeconvweights=keepdeconvweights,
allow_growth=allow_growth
) #pass on path and file name for pose_cfg.yaml!
except BaseException as e:
raise e
finally:
os.chdir(str(start_path))
print(
"The network is now trained and ready to evaluate. Use the function 'evaluate_network' to evaluate the network."
)
def create_labeled_video(
config,
videos,
videotype="avi",
shuffle=1,
trainingsetindex=0,
filtered=False,
save_frames=False,
Frames2plot=None,
delete=False,
displayedbodyparts="all",
codec="mp4v",
outputframerate=None,
destfolder=None,
draw_skeleton=False,
trailpoints=0,
displaycropped=False,
):
"""
Labels the bodyparts in a video. Make sure the video is already analyzed by the function 'analyze_video'
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
Number of shuffles of training dataset. Default is set to 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcutcore.filterpredictions
videotype: string, optional
Checks for the extension of the video in case the input is a directory.\nOnly videos with this extension are analyzed. The default is ``.avi``
save_frames: bool
If true creates each frame individual and then combines into a video. This variant is relatively slow as
it stores all individual frames. However, it uses matplotlib to create the frames and is therefore much more flexible (one can set transparency of markers, crop, and easily customize).
Frames2plot: List of indices
If not None & save_frames=True then the frames corresponding to the index will be plotted. For example, Frames2plot=[0,11] will plot the first and the 12th frame.
delete: bool
If true then the individual frames created during the video generation will be deleted.
displayedbodyparts: list of strings, optional
This select the body parts that are plotted in the video. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
codec: codec for labeled video. Options see http://www.fourcc.org/codecs.php [depends on your ffmpeg installation.]
outputframerate: positive number, output frame rate for labeled video (only available for the mode with saving frames.) By default: None, which results in the original video rate.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
draw_skeleton: bool
If ``True`` adds a line connecting the body parts making a skeleton on on each frame. The body parts to be connected and the color of these connecting lines are specified in the config file. By default: ``False``
trailpoints: int
Number of revious frames whose body parts are plotted in a frame (for displaying history). Default is set to 0.
displaycropped: bool, optional
Specifies whether only cropped frame is displayed (with labels analyzed therein), or the original frame with the labels analyzed in the cropped subset.
Examples
--------
If you want to create the labeled video for only 1 video
>>> deeplabcutcore.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to create the labeled video for only 1 video and store the individual frames
>>> deeplabcutcore.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'],save_frames=True)
--------
If you want to create the labeled video for multiple videos
>>> deeplabcutcore.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to create the labeled video for all the videos (as .avi extension) in a directory.
>>> deeplabcutcore.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'])
--------
If you want to create the labeled video for all the videos (as .mp4 extension) in a directory.
>>> deeplabcutcore.create_labeled_video('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/'],videotype='mp4')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
start_path = os.getcwd() # record cwd to return to this directory in the end
trainFraction = cfg["TrainingFraction"][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction
) # automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts
)
if draw_skeleton:
bodyparts2connect = cfg["skeleton"]
skeleton_color = cfg["skeleton_color"]
else:
bodyparts2connect = None
skeleton_color = None
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
if destfolder is None:
videofolder = Path(video).parents[0] # where your folder with videos is.
else:
videofolder = destfolder
os.chdir(str(videofolder))
videotype = Path(video).suffix
print("Starting % ", videofolder, videos)
vname = str(Path(video).stem)
# if notanalyzed:
# notanalyzed,outdataname,sourcedataname,DLCscorer=auxiliaryfunctions.CheckifPostProcessing(folder,vname,DLCscorer,DLCscorerlegacy,suffix='checking')
if filtered == True:
videooutname1 = os.path.join(vname + DLCscorer + "filtered_labeled.mp4")
videooutname2 = os.path.join(
vname + DLCscorerlegacy + "filtered_labeled.mp4"
)
else:
videooutname1 = os.path.join(vname + DLCscorer + "_labeled.mp4")
videooutname2 = os.path.join(vname + DLCscorerlegacy + "_labeled.mp4")
if os.path.isfile(videooutname1) or os.path.isfile(videooutname2):
print("Labeled video already created.")
else:
print("Loading ", video, "and data.")
datafound, metadata, Dataframe, DLCscorer, suffix = auxiliaryfunctions.LoadAnalyzedData(
str(videofolder), vname, DLCscorer, filtered
) # returns boolean variable if data was found and metadata + pandas array
videooutname = os.path.join(vname + DLCscorer + suffix + "_labeled.mp4")
if datafound and not os.path.isfile(
videooutname
): # checking again, for this loader video could exist
# Loading cropping data used during analysis
cropping = metadata["data"]["cropping"]
[x1, x2, y1, y2] = metadata["data"]["cropping_parameters"]
if save_frames == True:
tmpfolder = os.path.join(str(videofolder), "temp-" + vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder)
clip = vp(video)
CreateVideoSlow(
videooutname,
clip,
Dataframe,
tmpfolder,
cfg["dotsize"],
cfg["colormap"],
cfg["alphavalue"],
cfg["pcutoff"],
trailpoints,
cropping,
x1,
x2,
y1,
y2,
delete,
DLCscorer,
bodyparts,
outputframerate,
Frames2plot,
bodyparts2connect,
skeleton_color,
draw_skeleton,
displaycropped,
)
else:
if (
displaycropped
): # then the cropped video + the labels is depicted
clip = vp(
fname=video,
sname=videooutname,
codec=codec,
sw=x2 - x1,
sh=y2 - y1,
)
CreateVideo(
clip,
Dataframe,
cfg["pcutoff"],
cfg["dotsize"],
cfg["colormap"],
DLCscorer,
bodyparts,
trailpoints,
cropping,
x1,
x2,
y1,
y2,
bodyparts2connect,
skeleton_color,
draw_skeleton,
displaycropped,
)
else: # then the full video + the (perhaps in cropped mode analyzed labels) are depicted
clip = vp(fname=video, sname=videooutname, codec=codec)
CreateVideo(
clip,
Dataframe,
cfg["pcutoff"],
cfg["dotsize"],
cfg["colormap"],
DLCscorer,
bodyparts,
trailpoints,
cropping,
x1,
x2,
y1,
y2,
bodyparts2connect,
skeleton_color,
draw_skeleton,
displaycropped,
)
os.chdir(str(start_path))
def create_labeled_video_3d(config,
path,
videofolder=None,
start=0,
end=None,
trailpoints=0,
videotype='avi',
view=[-113, -270],
xlim=[None, None],
ylim=[None, None],
zlim=[None, None],
draw_skeleton=True):
"""
Creates a video with views from the two cameras and the 3d reconstruction for a selected number of frames.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
path : list
A list of strings containing the full paths to triangulated files for analysis or a path to the directory, where all the triangulated files are stored.
videofolder: string
Full path of the folder where the videos are stored. Use this if the vidoes are stored in a different location other than where the triangulation files are stored. By default is ``None`` and therefore looks for video files in the directory where the triangulation file is stored.
start: int
Integer specifying the start of frame index to select. Default is set to 0.
end: int
Integer specifying the end of frame index to select. Default is set to None, where all the frames of the video are used for creating the labeled video.
trailpoints: int
Number of revious frames whose body parts are plotted in a frame (for displaying history). Default is set to 0.
videotype: string, optional
Checks for the extension of the video in case the input is a directory.\nOnly videos with this extension are analyzed. The default is ``.avi``
view: list
A list that sets the elevation angle in z plane and azimuthal angle in x,y plane of 3d view. Useful for rotating the axis for 3d view
xlim: list
A list of integers specifying the limits for xaxis of 3d view. By default it is set to [None,None], where the x limit is set by taking the minimum and maximum value of the x coordinates for all the bodyparts.
ylim: list
A list of integers specifying the limits for yaxis of 3d view. By default it is set to [None,None], where the y limit is set by taking the minimum and maximum value of the y coordinates for all the bodyparts.
zlim: list
A list of integers specifying the limits for zaxis of 3d view. By default it is set to [None,None], where the z limit is set by taking the minimum and maximum value of the z coordinates for all the bodyparts.
draw_skeleton: bool
If ``True`` adds a line connecting the body parts making a skeleton on on each frame. The body parts to be connected and the color of these connecting lines are specified in the config file. By default: ``True``
Example
-------
Linux/MacOs
>>> deeplabcutcore.create_labeled_video_3d(config,['/data/project1/videos/3d.h5'],start=100, end=500)
To create labeled videos for all the triangulated files in the folder
>>> deeplabcutcore.create_labeled_video_3d(config,['/data/project1/videos'],start=100, end=500)
To set the xlim, ylim, zlim and rotate the view of the 3d axis
>>> deeplabcutcore.create_labeled_video_3d(config,['/data/project1/videos'],start=100, end=500,view=[30,90],xlim=[-12,12],ylim=[15,25],zlim=[20,30])
"""
start_path = os.getcwd()
# Read the config file and related variables
cfg_3d = auxiliaryfunctions.read_config(config)
cam_names = cfg_3d['camera_names']
pcutoff = cfg_3d['pcutoff']
markerSize = cfg_3d['dotsize']
alphaValue = cfg_3d['alphaValue']
cmap = cfg_3d['colormap']
bodyparts2connect = cfg_3d['skeleton']
skeleton_color = cfg_3d['skeleton_color']
scorer_3d = cfg_3d['scorername_3d']
# Flatten the list of bodyparts to connect
bodyparts2plot = list(
np.unique([val for sublist in bodyparts2connect for val in sublist]))
color = plt.cm.get_cmap(cmap, len(bodyparts2plot))
file_list = auxiliaryfunctions_3d.Get_list_of_triangulated_and_videoFiles(
path, videotype, scorer_3d, cam_names, videofolder)
print(file_list)
if file_list == []:
raise Exception(
"No corresponding video file(s) found for the specified triangulated file or folder. Did you specify the video file type? If videos are stored in a different location, please use the ``videofolder`` argument to specify their path."
)
for file in file_list:
path_h5_file = Path(file[0]).parents[0]
triangulate_file = file[0]
# triangulated file is a list which is always sorted as [triangulated.h5,camera-1.videotype,camera-2.videotype]
#name for output video
file_name = str(Path(triangulate_file).stem)
if os.path.isfile(os.path.join(path_h5_file, file_name + '.mpg')):
print("Video already created...")
else:
string_to_remove = str(Path(triangulate_file).suffix)
pickle_file = triangulate_file.replace(
string_to_remove, '_includingmetadata.pickle')
metadata_ = auxiliaryfunctions_3d.LoadMetadata3d(pickle_file)
base_filename_cam1 = str(Path(file[1]).stem).split(videotype)[
0] # required for searching the filtered file
base_filename_cam2 = str(Path(file[2]).stem).split(videotype)[
0] # required for searching the filtered file
cam1_view_video = file[1]
cam2_view_video = file[2]
cam1_scorer = metadata_['scorer_name'][cam_names[0]]
cam2_scorer = metadata_['scorer_name'][cam_names[1]]
print("Creating 3D video from %s and %s using %s" %
(Path(cam1_view_video).name, Path(cam2_view_video).name,
Path(triangulate_file).name))
# Read the video files and corresponfing h5 files
vid_cam1 = cv2.VideoCapture(cam1_view_video)
vid_cam2 = cv2.VideoCapture(cam2_view_video)
# Look for the filtered predictions file
try:
print("Looking for filtered predictions...")
df_cam1 = pd.read_hdf(
glob.glob(
os.path.join(
path_h5_file,
str('*' + base_filename_cam1 + cam1_scorer +
'*filtered.h5')))[0])
df_cam2 = pd.read_hdf(
glob.glob(
os.path.join(
path_h5_file,
str('*' + base_filename_cam2 + cam2_scorer +
'*filtered.h5')))[0])
#print("Found filtered predictions, will be use these for triangulation.")
print(
"Found the following filtered data: ",
os.path.join(
path_h5_file,
str('*' + base_filename_cam1 + cam1_scorer +
'*filtered.h5')),
os.path.join(
path_h5_file,
str('*' + base_filename_cam2 + cam2_scorer +
'*filtered.h5')))
except FileNotFoundError:
print(
"No filtered predictions found, the unfiltered predictions will be used instead."
)
df_cam1 = pd.read_hdf(
glob.glob(
os.path.join(
path_h5_file,
str(base_filename_cam1 + cam1_scorer +
'*.h5')))[0])
df_cam2 = pd.read_hdf(
glob.glob(
os.path.join(
path_h5_file,
str(base_filename_cam2 + cam2_scorer +
'*.h5')))[0])
df_3d = pd.read_hdf(triangulate_file, 'df_with_missing')
plt.rcParams.update({'figure.max_open_warning': 0})
if end == None:
end = len(df_3d) # All the frames
frames = list(range(start, end, 1))
# Start plotting for every frame
for k in tqdm(frames):
output_folder, num_frames = plot2D(
cfg_3d, k, bodyparts2plot, vid_cam1, vid_cam2,
bodyparts2connect, df_cam1, df_cam2, df_3d, pcutoff,
markerSize, alphaValue, color, path_h5_file, file_name,
skeleton_color, view, draw_skeleton, trailpoints, xlim,
ylim, zlim)
# Once all the frames are saved, then make a movie using ffmpeg.
cwd = os.getcwd()
os.chdir(str(output_folder))
subprocess.call([
'ffmpeg', '-start_number',
str(start), '-framerate',
str(30), '-i',
str('img%0' + str(num_frames) + 'd.png'), '-r',
str(30), '-vb', '20M',
os.path.join(output_folder, str('../' + file_name + '.mpg'))
])
os.chdir(cwd)
os.chdir(start_path)
def analyze_time_lapse_frames(config,
directory,
frametype='.png',
shuffle=1,
trainingsetindex=0,
gputouse=None,
save_as_csv=False,
rgb=True):
"""
Analyzed all images (of type = frametype) in a folder and stores the output in one file.
You can crop the frames (before analysis), by changing 'cropping'=True and setting 'x1','x2','y1','y2' in the config file.
Output: The labels are stored as MultiIndex Pandas Array, which contains the name of the network, body part name, (x, y) label position \n
in pixels, and the likelihood for each frame per body part. These arrays are stored in an efficient Hierarchical Data Format (HDF) \n
in the same directory, where the video is stored. However, if the flag save_as_csv is set to True, the data can also be exported in \n
comma-separated values format (.csv), which in turn can be imported in many programs, such as MATLAB, R, Prism, etc.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
directory: string
Full path to directory containing the frames that shall be analyzed
frametype: string, optional
Checks for the file extension of the frames. Only images with this extension are analyzed. The default is ``.png``
shuffle: int, optional
An integer specifying the shuffle index of the training dataset used for training the network. The default is 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
rbg: bool, optional.
Whether to load image as rgb; Note e.g. some tiffs do not alow that option in imread, then just set this to false.
Examples
--------
If you want to analyze all frames in /analysis/project/timelapseexperiment1
>>> deeplabcutcore.analyze_videos('/analysis/project/reaching-task/config.yaml','/analysis/project/timelapseexperiment1')
--------
If you want to analyze all frames in /analysis/project/timelapseexperiment1
>>> deeplabcutcore.analyze_videos('/analysis/project/reaching-task/config.yaml','/analysis/project/timelapseexperiment1')
--------
Note: for test purposes one can extract all frames from a video with ffmeg, e.g. ffmpeg -i testvideo.avi thumb%04d.png
"""
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ[
'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
if gputouse is not None: #gpu selection
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
tf.compat.v1.reset_default_graph()
start_path = os.getcwd(
) #record cwd to return to this directory in the end
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
modelfolder = os.path.join(
cfg["project_path"],
str(auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# Check which snapshots are available and sort them by # iterations
try:
Snapshots = np.array([
fn.split('.')[0]
for fn in os.listdir(os.path.join(modelfolder, 'train'))
if "index" in fn
])
except FileNotFoundError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s."
% (shuffle, shuffle))
if cfg['snapshotindex'] == 'all':
print(
"Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!"
)
snapshotindex = -1
else:
snapshotindex = cfg['snapshotindex']
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder)
##################################################
# Load and setup CNN part detector
##################################################
# Check if data already was generated:
dlc_cfg['init_weights'] = os.path.join(modelfolder, 'train',
Snapshots[snapshotindex])
trainingsiterations = (dlc_cfg['init_weights'].split(
os.sep)[-1]).split('-')[-1]
#update batchsize (based on parameters in config.yaml)
dlc_cfg['batch_size'] = cfg['batch_size']
# Name for scorer:
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, trainingsiterations=trainingsiterations)
sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
# update number of outputs and adjust pandas indices
dlc_cfg['num_outputs'] = cfg.get('num_outputs', 1)
xyz_labs_orig = ['x', 'y', 'likelihood']
suffix = [str(s + 1) for s in range(dlc_cfg['num_outputs'])]
suffix[0] = '' # first one has empty suffix for backwards compatibility
xyz_labs = [x + s for s in suffix for x in xyz_labs_orig]
pdindex = pd.MultiIndex.from_product(
[[DLCscorer], dlc_cfg['all_joints_names'], xyz_labs],
names=['scorer', 'bodyparts', 'coords'])
if gputouse is not None: #gpu selectinon
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
##################################################
# Loading the images
##################################################
#checks if input is a directory
if os.path.isdir(directory) == True:
"""
Analyzes all the frames in the directory.
"""
print("Analyzing all frames in the directory: ", directory)
os.chdir(directory)
framelist = np.sort(
[fn for fn in os.listdir(os.curdir) if (frametype in fn)])
vname = Path(directory).stem
notanalyzed, dataname, DLCscorer = auxiliaryfunctions.CheckifNotAnalyzed(
directory, vname, DLCscorer, DLCscorerlegacy, flag='framestack')
if notanalyzed:
nframes = len(framelist)
if nframes > 0:
start = time.time()
PredictedData, nframes, nx, ny = GetPosesofFrames(
cfg, dlc_cfg, sess, inputs, outputs, directory, framelist,
nframes, dlc_cfg['batch_size'], rgb)
stop = time.time()
if cfg['cropping'] == True:
coords = [cfg['x1'], cfg['x2'], cfg['y1'], cfg['y2']]
else:
coords = [0, nx, 0, ny]
dictionary = {
"start": start,
"stop": stop,
"run_duration": stop - start,
"Scorer": DLCscorer,
"config file": dlc_cfg,
"batch_size": dlc_cfg["batch_size"],
"num_outputs": dlc_cfg["num_outputs"],
"frame_dimensions": (ny, nx),
"nframes": nframes,
"cropping": cfg['cropping'],
"cropping_parameters": coords
}
metadata = {'data': dictionary}
print("Saving results in %s..." % (directory))
auxiliaryfunctions.SaveData(PredictedData[:nframes, :],
metadata, dataname, pdindex,
framelist, save_as_csv)
print(
"The folder was analyzed. Now your research can truly start!"
)
print(
"If the tracking is not satisfactory for some frome, consider expanding the training set."
)
else:
print(
"No frames were found. Consider changing the path or the frametype."
)
os.chdir(str(start_path))
def extract_outlier_frames(
config,
videos,
videotype="avi",
shuffle=1,
trainingsetindex=0,
outlieralgorithm="jump",
comparisonbodyparts="all",
epsilon=20,
p_bound=0.01,
ARdegree=3,
MAdegree=1,
alpha=0.01,
extractionalgorithm="kmeans",
automatic=False,
cluster_resizewidth=30,
cluster_color=False,
opencv=True,
savelabeled=True,
destfolder=None,
):
"""
Extracts the outlier frames in case, the predictions are not correct for a certain video from the cropped video running from
start to stop as defined in config.yaml.
Another crucial parameter in config.yaml is how many frames to extract 'numframes2extract'.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle : int, optional
The shufle index of training dataset. The extracted frames will be stored in the labeled-dataset for
the corresponding shuffle of training dataset. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
outlieralgorithm: 'fitting', 'jump', 'uncertain', or 'manual'
String specifying the algorithm used to detect the outliers. Currently, deeplabcut supports three methods + a manual GUI option. 'Fitting'
fits a Auto Regressive Integrated Moving Average model to the data and computes the distance to the estimated data. Larger distances than
epsilon are then potentially identified as outliers. The methods 'jump' identifies larger jumps than 'epsilon' in any body part; and 'uncertain'
looks for frames with confidence below p_bound. The default is set to ``jump``.
comparisonbodyparts: list of strings, optional
This select the body parts for which the comparisons with the outliers are carried out. Either ``all``, then all body parts
from config.yaml are used orr a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
p_bound: float between 0 and 1, optional
For outlieralgorithm 'uncertain' this parameter defines the likelihood below, below which a body part will be flagged as a putative outlier.
epsilon; float,optional
Meaning depends on outlieralgoritm. The default is set to 20 pixels.
For outlieralgorithm 'fitting': Float bound according to which frames are picked when the (average) body part estimate deviates from model fit
For outlieralgorithm 'jump': Float bound specifying the distance by which body points jump from one frame to next (Euclidean distance)
ARdegree: int, optional
For outlieralgorithm 'fitting': Autoregressive degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part)
see https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
MAdegree: int
For outlieralgorithm 'fitting': MovingAvarage degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part)
See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
alpha: float
Significance level for detecting outliers based on confidence interval of fitted ARIMA model. Only the distance is used however.
extractionalgorithm : string, optional
String specifying the algorithm to use for selecting the frames from the identified putatative outlier frames. Currently, deeplabcut
supports either ``kmeans`` or ``uniform`` based selection (same logic as for extract_frames).
The default is set to``uniform``, if provided it must be either ``uniform`` or ``kmeans``.
automatic : bool, optional
Set it to True, if you want to extract outliers without being asked for user feedback.
cluster_resizewidth: number, default: 30
For k-means one can change the width to which the images are downsampled (aspect ratio is fixed).
cluster_color: bool, default: False
If false then each downsampled image is treated as a grayscale vector (discarding color information). If true, then the color channels are considered. This increases
the computational complexity.
opencv: bool, default: True
Uses openCV for loading & extractiong (otherwise moviepy (legacy))
savelabeled: bool, default: True
If true also saves frame with predicted labels in each folder.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
Examples
Windows example for extracting the frames with default settings
>>> deeplabcutcore.extract_outlier_frames('C:\\myproject\\reaching-task\\config.yaml',['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi'])
--------
for extracting the frames with default settings
>>> deeplabcutcore.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'])
--------
for extracting the frames with kmeans
>>> deeplabcutcore.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'],extractionalgorithm='kmeans')
--------
for extracting the frames with kmeans and epsilon = 5 pixels.
>>> deeplabcutcore.extract_outlier_frames('/analysis/project/reaching-task/config.yaml',['/analysis/project/video/reachinvideo1.avi'],epsilon = 5,extractionalgorithm='kmeans')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction=cfg["TrainingFraction"][trainingsetindex]
)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
notanalyzed, dataname, DLCscorer = auxiliaryfunctions.CheckifNotAnalyzed(
videofolder,
str(Path(video).stem),
DLCscorer,
DLCscorerlegacy,
flag="checking",
)
if notanalyzed:
print(
"It seems the video has not been analyzed yet, or the video is not found! You can only refine the labels after the a video is analyzed. Please run 'analyze_video' first. Or, please double check your video file path"
)
else:
Dataframe = pd.read_hdf(dataname, "df_with_missing")
scorer = Dataframe.columns.get_level_values(0)[0] # reading scorer from
nframes = np.size(Dataframe.index)
# extract min and max index based on start stop interval.
startindex = max([int(np.floor(nframes * cfg["start"])), 0])
stopindex = min([int(np.ceil(nframes * cfg["stop"])), nframes])
Index = np.arange(stopindex - startindex) + startindex
# figure out body part list:
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, comparisonbodyparts
)
Indices = []
if (
outlieralgorithm == "uncertain"
): # necessary parameters: considered body parts and
for bpindex, bp in enumerate(bodyparts):
if (
bp in cfg["bodyparts"]
): # filter [who knows what users put in...]
p = Dataframe[scorer][bp]["likelihood"].values[Index]
Indices.extend(
np.where(p < p_bound)[0] + startindex
) # all indices between start and stop that are below p_bound.
elif outlieralgorithm == "jump":
for bpindex, bp in enumerate(bodyparts):
if (
bp in cfg["bodyparts"]
): # filter [who knows what users put in...]
dx = np.diff(Dataframe[scorer][bp]["x"].values[Index])
dy = np.diff(Dataframe[scorer][bp]["y"].values[Index])
# all indices between start and stop with jump larger than epsilon (leading up to this point!)
Indices.extend(
np.where((dx ** 2 + dy ** 2) > epsilon ** 2)[0]
+ startindex
+ 1
)
elif outlieralgorithm == "fitting":
# deviation_dataname = str(Path(videofolder)/Path(dataname))
# Calculate deviatons for video
[d, o] = ComputeDeviations(
Dataframe,
cfg,
bodyparts,
scorer,
dataname,
p_bound,
alpha,
ARdegree,
MAdegree,
)
# Some heuristics for extracting frames based on distance:
Indices = np.where(d > epsilon)[
0
] # time points with at least average difference of epsilon
if (
len(Index) < cfg["numframes2pick"] * 2
and len(d) > cfg["numframes2pick"] * 2
): # if too few points qualify, extract the most distant ones.
Indices = np.argsort(d)[::-1][: cfg["numframes2pick"] * 2]
elif outlieralgorithm == "manual":
wd = Path(config).resolve().parents[0]
os.chdir(str(wd))
from deeplabcutcore.refine_training_dataset import (
outlier_frame_extraction_toolbox,
)
outlier_frame_extraction_toolbox.show(
config, video, shuffle, Dataframe, scorer, savelabeled
)
# Run always except when the outlieralgorithm == manual.
if not outlieralgorithm == "manual":
Indices = np.sort(list(set(Indices))) # remove repetitions.
print(
"Method ",
outlieralgorithm,
" found ",
len(Indices),
" putative outlier frames.",
)
print(
"Do you want to proceed with extracting ",
cfg["numframes2pick"],
" of those?",
)
if outlieralgorithm == "uncertain":
print(
"If this list is very large, perhaps consider changing the paramters (start, stop, p_bound, comparisonbodyparts) or use a different method."
)
elif outlieralgorithm == "jump":
print(
"If this list is very large, perhaps consider changing the paramters (start, stop, epsilon, comparisonbodyparts) or use a different method."
)
elif outlieralgorithm == "fitting":
print(
"If this list is very large, perhaps consider changing the paramters (start, stop, epsilon, ARdegree, MAdegree, alpha, comparisonbodyparts) or use a different method."
)
if automatic == False:
askuser = input("yes/no")
else:
askuser = "******"
if (
askuser == "y"
or askuser == "yes"
or askuser == "Ja"
or askuser == "ha"
): # multilanguage support :)
# Now extract from those Indices!
ExtractFramesbasedonPreselection(
Indices,
extractionalgorithm,
Dataframe,
dataname,
scorer,
video,
cfg,
config,
opencv,
cluster_resizewidth,
cluster_color,
savelabeled,
)
else:
print(
"Nothing extracted, please change the parameters and start again..."
)
def filterpredictions(config,video,videotype='avi',shuffle=1,trainingsetindex=0,
filtertype='median',windowlength=5,
p_bound=.001,ARdegree=3,MAdegree=1,alpha=.01,save_as_csv=True,destfolder=None):
"""
Fits frame-by-frame pose predictions with ARIMA model (filtertype='arima') or median filter (default).
Parameter
----------
config : string
Full path of the config.yaml file as a string.
video : string
Full path of the video to extract the frame from. Make sure that this video is already analyzed.
shuffle : int, optional
The shufle index of training dataset. The extracted frames will be stored in the labeled-dataset for
the corresponding shuffle of training dataset. Default is set to 1
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtertype: string
Select which filter, 'arima' or 'median' filter.
windowlength: int
For filtertype='median' filters the input array using a local window-size given by windowlength. The array will automatically be zero-padded.
https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.medfilt.html The windowlenght should be an odd number.
p_bound: float between 0 and 1, optional
For filtertype 'arima' this parameter defines the likelihood below,
below which a body part will be consided as missing data for filtering purposes.
ARdegree: int, optional
For filtertype 'arima' Autoregressive degree of Sarimax model degree.
see https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
MAdegree: int
For filtertype 'arima' Moving Avarage degree of Sarimax model degree.
See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
alpha: float
Significance level for detecting outliers based on confidence interval of fitted SARIMAX model.
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this
folder also needs to be passed.
Example
--------
Arima model:
deeplabcutcore.filterpredictions('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,filterype='arima',ARdegree=5,MAdegree=2)
Use median filter over 10bins:
deeplabcutcore.filterpredictions('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,windowlength=10)
One can then use the filtered rather than the frame-by-frame predictions by calling:
deeplabcutcore.plot_trajectories('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,filtered=True)
deeplabcutcore.create_labeled_video('C:\\myproject\\reaching-task\\config.yaml',['C:\\myproject\\trailtracking-task\\test.mp4'],shuffle=3,filtered=True)
--------
Returns filtered pandas array with the same structure as normal output of network.
"""
cfg = auxiliaryfunctions.read_config(config)
DLCscorer,DLCscorerlegacy=auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction = cfg['TrainingFraction'][trainingsetindex])
Videos=auxiliaryfunctions.Getlistofvideos(video,videotype)
if len(Videos)>0:
for video in Videos:
if destfolder is None:
destfolder = str(Path(video).parents[0])
print("Filtering with %s model %s"%(filtertype,video))
videofolder = destfolder
vname=Path(video).stem
notanalyzed,outdataname,sourcedataname,scorer=auxiliaryfunctions.CheckifPostProcessing(destfolder,vname,DLCscorer,DLCscorerlegacy,suffix='filtered')
if notanalyzed:
Dataframe = pd.read_hdf(sourcedataname,'df_with_missing')
for bpindex,bp in tqdm(enumerate(cfg['bodyparts'])):
pdindex = pd.MultiIndex.from_product([[scorer], [bp], ['x', 'y','likelihood']],names=['scorer', 'bodyparts', 'coords'])
x,y,p=Dataframe[scorer][bp]['x'].values,Dataframe[scorer][bp]['y'].values,Dataframe[scorer][bp]['likelihood'].values
if filtertype=='arima':
meanx,CIx=FitSARIMAXModel(x,p,p_bound,alpha,ARdegree,MAdegree,False)
meany,CIy=FitSARIMAXModel(y,p,p_bound,alpha,ARdegree,MAdegree,False)
meanx[0]=x[0]
meany[0]=y[0]
else:
meanx=signal.medfilt(x,kernel_size=windowlength)
meany=signal.medfilt(y,kernel_size=windowlength)
if bpindex==0:
data = pd.DataFrame(np.hstack([np.expand_dims(meanx,axis=1),np.expand_dims(meany,axis=1),np.expand_dims(p,axis=1)]), columns=pdindex)
else:
item=pd.DataFrame(np.hstack([np.expand_dims(meanx,axis=1),np.expand_dims(meany,axis=1),np.expand_dims(p,axis=1)]), columns=pdindex)
data=pd.concat([data.T, item.T]).T
data.to_hdf(outdataname, 'df_with_missing', format='table', mode='w')
if save_as_csv:
print("Saving filtered csv poses!")
data.to_csv(outdataname.split('.h5')[0]+'.csv')
def add_new_videos(config, videos, copy_videos=False, coords=None):
"""
Add new videos to the config file at any stage of the project.
Parameters
----------
config : string
String containing the full path of the config file in the project.
videos : list
A list of string containing the full paths of the videos to include in the project.
copy_videos : bool, optional
If this is set to True, the symlink of the videos are copied to the project/videos directory. The default is
``False``; if provided it must be either ``True`` or ``False``.
coords: list, optional
A list containing the list of cropping coordinates of the video. The default is set to None.
Examples
--------
Video will be added, with cropping dimenions according to the frame dimensinos of mouse5.avi
>>> deeplabcutcore.add_new_videos('/home/project/reaching-task-Tanmay-2018-08-23/config.yaml',['/data/videos/mouse5.avi'])
Video will be added, with cropping dimenions [0,100,0,200]
>>> deeplabcutcore.add_new_videos('/home/project/reaching-task-Tanmay-2018-08-23/config.yaml',['/data/videos/mouse5.avi'],copy_videos=False,coords=[[0,100,0,200]])
Two videos will be added, with cropping dimenions [0,100,0,200] and [0,100,0,250], respectively.
>>> deeplabcutcore.add_new_videos('/home/project/reaching-task-Tanmay-2018-08-23/config.yaml',['/data/videos/mouse5.avi','/data/videos/mouse6.avi'],copy_videos=False,coords=[[0,100,0,200],[0,100,0,250]])
"""
import os
import shutil
from pathlib import Path
from deeplabcutcore import DEBUG
from deeplabcutcore.utils import auxiliaryfunctions
import cv2
# Read the config file
cfg = auxiliaryfunctions.read_config(config)
video_path = Path(config).parents[0] / 'videos'
data_path = Path(config).parents[0] / 'labeled-data'
videos = [Path(vp) for vp in videos]
dirs = [data_path / Path(i.stem) for i in videos]
for p in dirs:
"""
Creates directory under data & perhaps copies videos (to /video)
"""
p.mkdir(parents=True, exist_ok=True)
destinations = [video_path.joinpath(vp.name) for vp in videos]
if copy_videos == True:
for src, dst in zip(videos, destinations):
if dst.exists():
pass
else:
print("Copying the videos")
shutil.copy(os.fspath(src), os.fspath(dst))
else:
for src, dst in zip(videos, destinations):
if dst.exists():
pass
else:
print("Creating the symbolic link of the video")
src = str(src)
dst = str(dst)
os.symlink(src, dst)
if copy_videos == True:
videos = destinations # in this case the *new* location should be added to the config file
# adds the video list to the config.yaml file
for idx, video in enumerate(videos):
try:
# For windows os.path.realpath does not work and does not link to the real video.
video_path = str(Path.resolve(Path(video)))
# video_path = os.path.realpath(video)
except:
video_path = os.readlink(video)
vcap = cv2.VideoCapture(video_path)
if vcap.isOpened():
# get vcap property
width = int(vcap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(vcap.get(cv2.CAP_PROP_FRAME_HEIGHT))
if coords == None:
cfg['video_sets'].update({
video_path: {
'crop': ', '.join(map(str, [0, width, 0, height]))
}
})
else:
c = coords[idx]
cfg['video_sets'].update(
{video_path: {
'crop': ', '.join(map(str, c))
}})
else:
print("Cannot open the video file!")
auxiliaryfunctions.write_config(config, cfg)
print(
"New video was added to the project! Use the function 'extract_frames' to select frames for labeling."
)
def check_undistortion(config, cbrow=8, cbcol=6, plot=True):
"""
This function undistorts the calibration images based on the camera matrices and stores them in the project folder(defined in the config file)
to visually check if the camera matrices are correct.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
cbrow : int
Int specifying the number of rows in the calibration image.
cbcol : int
Int specifying the number of columns in the calibration image.
plot : bool
If this is set to True, the results of undistortion are saved as plots. The default is ``True``; if provided it must be either ``True`` or ``False``.
Example
--------
Linux/MacOs/Windows
>>> deeplabcutcore.check_undistortion(config, cbrow = 8,cbcol = 6)
"""
# Read the config file
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
cfg_3d = auxiliaryfunctions.read_config(config)
img_path, path_corners, path_camera_matrix, path_undistort = auxiliaryfunctions_3d.Foldernames3Dproject(
cfg_3d
)
# colormap = plt.get_cmap(cfg_3d['colormap'])
markerSize = cfg_3d["dotsize"]
alphaValue = cfg_3d["alphaValue"]
markerType = cfg_3d["markerType"]
markerColor = cfg_3d["markerColor"]
cam_names = cfg_3d["camera_names"]
images = glob.glob(os.path.join(img_path, "*.jpg"))
# Sort the images
images.sort(key=lambda f: int("".join(filter(str.isdigit, f))))
"""
for fname in images:
for cam in cam_names:
if cam in fname:
filename = Path(fname).stem
ext = Path(fname).suffix
img = cv2.imread(fname)
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
"""
camera_pair = [[cam_names[0], cam_names[1]]]
stereo_params = auxiliaryfunctions.read_pickle(
os.path.join(path_camera_matrix, "stereo_params.pickle")
)
for pair in camera_pair:
map1_x, map1_y = cv2.initUndistortRectifyMap(
stereo_params[pair[0] + "-" + pair[1]]["cameraMatrix1"],
stereo_params[pair[0] + "-" + pair[1]]["distCoeffs1"],
stereo_params[pair[0] + "-" + pair[1]]["R1"],
stereo_params[pair[0] + "-" + pair[1]]["P1"],
(stereo_params[pair[0] + "-" + pair[1]]["image_shape"][0]),
cv2.CV_16SC2,
)
map2_x, map2_y = cv2.initUndistortRectifyMap(
stereo_params[pair[0] + "-" + pair[1]]["cameraMatrix2"],
stereo_params[pair[0] + "-" + pair[1]]["distCoeffs2"],
stereo_params[pair[0] + "-" + pair[1]]["R2"],
stereo_params[pair[0] + "-" + pair[1]]["P2"],
(stereo_params[pair[0] + "-" + pair[1]]["image_shape"][1]),
cv2.CV_16SC2,
)
cam1_undistort = []
cam2_undistort = []
for fname in images:
if pair[0] in fname:
filename = Path(fname).stem
img1 = cv2.imread(fname)
gray1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
h, w = img1.shape[:2]
_, corners1 = cv2.findChessboardCorners(gray1, (cbcol, cbrow), None)
corners_origin1 = cv2.cornerSubPix(
gray1, corners1, (11, 11), (-1, -1), criteria
)
# Remapping dataFrame_camera1_undistort
im_remapped1 = cv2.remap(img1, map1_x, map1_y, cv2.INTER_LANCZOS4)
imgpoints_proj_undistort = cv2.undistortPoints(
src=corners_origin1,
cameraMatrix=stereo_params[pair[0] + "-" + pair[1]][
"cameraMatrix1"
],
distCoeffs=stereo_params[pair[0] + "-" + pair[1]]["distCoeffs1"],
P=stereo_params[pair[0] + "-" + pair[1]]["P1"],
R=stereo_params[pair[0] + "-" + pair[1]]["R1"],
)
cam1_undistort.append(imgpoints_proj_undistort)
cv2.imwrite(
os.path.join(str(path_undistort), filename + "_undistort.jpg"),
im_remapped1,
)
imgpoints_proj_undistort = []
elif pair[1] in fname:
filename = Path(fname).stem
img2 = cv2.imread(fname)
gray2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)
h, w = img2.shape[:2]
_, corners2 = cv2.findChessboardCorners(gray2, (cbcol, cbrow), None)
corners_origin2 = cv2.cornerSubPix(
gray2, corners2, (11, 11), (-1, -1), criteria
)
# Remapping
im_remapped2 = cv2.remap(img2, map2_x, map2_y, cv2.INTER_LANCZOS4)
imgpoints_proj_undistort2 = cv2.undistortPoints(
src=corners_origin2,
cameraMatrix=stereo_params[pair[0] + "-" + pair[1]][
"cameraMatrix2"
],
distCoeffs=stereo_params[pair[0] + "-" + pair[1]]["distCoeffs2"],
P=stereo_params[pair[0] + "-" + pair[1]]["P2"],
R=stereo_params[pair[0] + "-" + pair[1]]["R2"],
)
cam2_undistort.append(imgpoints_proj_undistort2)
cv2.imwrite(
os.path.join(str(path_undistort), filename + "_undistort.jpg"),
im_remapped2,
)
imgpoints_proj_undistort2 = []
cam1_undistort = np.array(cam1_undistort)
cam2_undistort = np.array(cam2_undistort)
print("All images are undistorted and stored in %s" % str(path_undistort))
print(
"Use the function ``triangulate`` to undistort the dataframes and compute the triangulation"
)
if plot == True:
f1, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 10))
f1.suptitle(
str("Original Image: Views from " + pair[0] + " and " + pair[1]),
fontsize=25,
)
# Display images in RGB
ax1.imshow(cv2.cvtColor(img1, cv2.COLOR_BGR2RGB))
ax2.imshow(cv2.cvtColor(img2, cv2.COLOR_BGR2RGB))
norm = mcolors.Normalize(vmin=0.0, vmax=cam1_undistort.shape[1])
plt.savefig(os.path.join(str(path_undistort), "Original_Image.png"))
# Plot the undistorted corner points
f2, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 10))
f2.suptitle(
"Undistorted corner points on camera-1 and camera-2", fontsize=25
)
ax1.imshow(cv2.cvtColor(im_remapped1, cv2.COLOR_BGR2RGB))
ax2.imshow(cv2.cvtColor(im_remapped2, cv2.COLOR_BGR2RGB))
for i in range(0, cam1_undistort.shape[1]):
ax1.scatter(
[cam1_undistort[-1][i, 0, 0]],
[cam1_undistort[-1][i, 0, 1]],
marker=markerType,
s=markerSize,
color=markerColor,
alpha=alphaValue,
)
ax2.scatter(
[cam2_undistort[-1][i, 0, 0]],
[cam2_undistort[-1][i, 0, 1]],
marker=markerType,
s=markerSize,
color=markerColor,
alpha=alphaValue,
)
plt.savefig(os.path.join(str(path_undistort), "undistorted_points.png"))
# Triangulate
triangulate = auxiliaryfunctions_3d.compute_triangulation_calibration_images(
stereo_params[pair[0] + "-" + pair[1]],
cam1_undistort,
cam2_undistort,
path_undistort,
cfg_3d,
plot=True,
)
auxiliaryfunctions.write_pickle("triangulate.pickle", triangulate)
def plot_trajectories(config,
videos,
videotype='.avi',
shuffle=1,
trainingsetindex=0,
filtered=False,
displayedbodyparts='all',
showfigures=False,
destfolder=None):
"""
Plots the trajectories of various bodyparts across the video.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: list, optional
List of integers specifying the shuffle indices of the training dataset. The default is [1]
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
filtered: bool, default false
Boolean variable indicating if filtered output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcutcore.filterpredictions
displayedbodyparts: list of strings, optional
This select the body parts that are plotted in the video.
Either ``all``, then all body parts from config.yaml are used,
or a list of strings that are a subset of the full list.
E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these two body parts.
showfigures: bool, default false
If true then plots are also displayed.
destfolder: string, optional
Specifies the destination folder that was used for storing analysis data (default is the path of the video).
Example
--------
for labeling the frames
>>> deeplabcutcore.plot_trajectories('home/alex/analysis/project/reaching-task/config.yaml',['/home/alex/analysis/project/videos/reachingvideo1.avi'])
--------
"""
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction
) #automatically loads corresponding model (even training iteration based on snapshot index)
bodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(
cfg, displayedbodyparts)
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
for video in Videos:
print(video)
if destfolder is None:
videofolder = str(Path(video).parents[0])
else:
videofolder = destfolder
vname = str(Path(video).stem)
print("Starting % ", videofolder, video)
notanalyzed, dataname, DLCscorer = auxiliaryfunctions.CheckifNotAnalyzed(
videofolder, vname, DLCscorer, DLCscorerlegacy, flag='checking')
if notanalyzed:
print("The video was not analyzed with this scorer:", DLCscorer)
else:
#LoadData
print("Loading ", video, "and data.")
datafound, metadata, Dataframe, DLCscorer, suffix = auxiliaryfunctions.LoadAnalyzedData(
str(videofolder), vname, DLCscorer, filtered
) #returns boolean variable if data was found and metadata + pandas array
if datafound:
basefolder = videofolder
auxiliaryfunctions.attempttomakefolder(basefolder)
auxiliaryfunctions.attempttomakefolder(
os.path.join(basefolder, 'plot-poses'))
tmpfolder = os.path.join(basefolder, 'plot-poses', vname)
auxiliaryfunctions.attempttomakefolder(tmpfolder)
PlottingResults(tmpfolder, Dataframe, DLCscorer, cfg,
bodyparts, showfigures, suffix + '.png')
print(
'Plots created! Please check the directory "plot-poses" within the video directory'
)
def calibrate_cameras(config, cbrow=8, cbcol=6, calibrate=False, alpha=0.4):
"""This function extracts the corners points from the calibration images, calibrates the camera and stores the calibration files in the project folder (defined in the config file).
Make sure you have around 20-60 pairs of calibration images. The function should be used iteratively to select the right set of calibration images.
A pair of calibration image is considered "correct", if the corners are detected correctly in both the images. It may happen that during the first run of this function,
the extracted corners are incorrect or the order of detected corners does not align for the corresponding views (i.e. camera-1 and camera-2 images).
In such a case, remove those pairs of images and re-run this function. Once the right number of calibration images are selected,
use the parameter ``calibrate=True`` to calibrate the cameras.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
cbrow : int
Integer specifying the number of rows in the calibration image.
cbcol : int
Integer specifying the number of columns in the calibration image.
calibrate : bool
If this is set to True, the cameras are calibrated with the current set of calibration images. The default is ``False``
Set it to True, only after checking the results of the corner detection method and removing dysfunctional images!
alpha: float
Floating point number between 0 and 1 specifying the free scaling parameter. When alpha = 0, the rectified images with only valid pixels are stored
i.e. the rectified images are zoomed in. When alpha = 1, all the pixels from the original images are retained.
For more details: https://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html
Example
--------
Linux/MacOs/Windows
>>> deeplabcutcore.calibrate_camera(config)
Once the right set of calibration images are selected,
>>> deeplabcutcore.calibrate_camera(config,calibrate=True)
"""
# Termination criteria
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
# Prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
objp = np.zeros((cbrow * cbcol, 3), np.float32)
objp[:, :2] = np.mgrid[0:cbcol, 0:cbrow].T.reshape(-1, 2)
# Read the config file
cfg_3d = auxiliaryfunctions.read_config(config)
img_path, path_corners, path_camera_matrix, path_undistort = auxiliaryfunctions_3d.Foldernames3Dproject(
cfg_3d
)
images = glob.glob(os.path.join(img_path, "*.jpg"))
cam_names = cfg_3d["camera_names"]
# update the variable snapshot* in config file according to the name of the cameras
try:
for i in range(len(cam_names)):
cfg_3d[str("config_file_" + cam_names[i])] = cfg_3d.pop(
str("config_file_camera-" + str(i + 1))
)
for i in range(len(cam_names)):
cfg_3d[str("shuffle_" + cam_names[i])] = cfg_3d.pop(
str("shuffle_camera-" + str(i + 1))
)
except:
pass
project_path = cfg_3d["project_path"]
projconfigfile = os.path.join(str(project_path), "config.yaml")
auxiliaryfunctions.write_config_3d(projconfigfile, cfg_3d)
# Initialize the dictionary
img_shape = {}
objpoints = {} # 3d point in real world space
imgpoints = {} # 2d points in image plane.
dist_pickle = {}
stereo_params = {}
for cam in cam_names:
objpoints.setdefault(cam, [])
imgpoints.setdefault(cam, [])
dist_pickle.setdefault(cam, [])
# Sort the images.
images.sort(key=lambda f: int("".join(filter(str.isdigit, f))))
if len(images) == 0:
raise Exception(
"No calibration images found. Make sure the calibration images are saved as .jpg and with prefix as the camera name as specified in the config.yaml file."
)
for fname in images:
for cam in cam_names:
if cam in fname:
filename = Path(fname).stem
img = cv2.imread(fname)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# Find the chess board corners
ret, corners = cv2.findChessboardCorners(
gray, (cbcol, cbrow), None
) # (8,6) pattern (dimensions = common points of black squares)
# If found, add object points, image points (after refining them)
if ret == True:
img_shape[cam] = gray.shape[::-1]
objpoints[cam].append(objp)
corners = cv2.cornerSubPix(
gray, corners, (11, 11), (-1, -1), criteria
)
imgpoints[cam].append(corners)
# Draw the corners and store the images
img = cv2.drawChessboardCorners(img, (cbcol, cbrow), corners, ret)
cv2.imwrite(
os.path.join(str(path_corners), filename + "_corner.jpg"), img
)
else:
print("Corners not found for the image %s" % Path(fname).name)
try:
h, w = img.shape[:2]
except:
raise Exception(
"It seems that the name of calibration images does not match with the camera names in the config file. Please make sure that the calibration images are named with camera names as specified in the config.yaml file."
)
# Perform calibration for each cameras and store the matrices as a pickle file
if calibrate == True:
# Calibrating each camera
for cam in cam_names:
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(
objpoints[cam], imgpoints[cam], img_shape[cam], None, None
)
# Save the camera calibration result for later use (we won't use rvecs / tvecs)
dist_pickle[cam] = {
"mtx": mtx,
"dist": dist,
"objpoints": objpoints[cam],
"imgpoints": imgpoints[cam],
}
pickle.dump(
dist_pickle,
open(
os.path.join(path_camera_matrix, cam + "_intrinsic_params.pickle"),
"wb",
),
)
print(
"Saving intrinsic camera calibration matrices for %s as a pickle file in %s"
% (cam, os.path.join(path_camera_matrix))
)
# Compute mean re-projection errors for individual cameras
mean_error = 0
for i in range(len(objpoints[cam])):
imgpoints_proj, _ = cv2.projectPoints(
objpoints[cam][i], rvecs[i], tvecs[i], mtx, dist
)
error = cv2.norm(imgpoints[cam][i], imgpoints_proj, cv2.NORM_L2) / len(
imgpoints_proj
)
mean_error += error
print(
"Mean re-projection error for %s images: %.3f pixels "
% (cam, mean_error / len(objpoints[cam]))
)
# Compute stereo calibration for each pair of cameras
camera_pair = [[cam_names[0], cam_names[1]]]
for pair in camera_pair:
print("Computing stereo calibration for " % pair)
retval, cameraMatrix1, distCoeffs1, cameraMatrix2, distCoeffs2, R, T, E, F = cv2.stereoCalibrate(
objpoints[pair[0]],
imgpoints[pair[0]],
imgpoints[pair[1]],
dist_pickle[pair[0]]["mtx"],
dist_pickle[pair[0]]["dist"],
dist_pickle[pair[1]]["mtx"],
dist_pickle[pair[1]]["dist"],
(h, w),
flags=cv2.CALIB_FIX_INTRINSIC,
)
# Stereo Rectification
rectify_scale = (
alpha
) # Free scaling parameter check this https://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html#fisheye-stereorectify
R1, R2, P1, P2, Q, roi1, roi2 = cv2.stereoRectify(
cameraMatrix1,
distCoeffs1,
cameraMatrix2,
distCoeffs2,
(h, w),
R,
T,
alpha=rectify_scale,
)
stereo_params[pair[0] + "-" + pair[1]] = {
"cameraMatrix1": cameraMatrix1,
"cameraMatrix2": cameraMatrix2,
"distCoeffs1": distCoeffs1,
"distCoeffs2": distCoeffs2,
"R": R,
"T": T,
"E": E,
"F": F,
"R1": R1,
"R2": R2,
"P1": P1,
"P2": P2,
"roi1": roi1,
"roi2": roi2,
"Q": Q,
"image_shape": [img_shape[pair[0]], img_shape[pair[1]]],
}
print(
"Saving the stereo parameters for every pair of cameras as a pickle file in %s"
% str(os.path.join(path_camera_matrix))
)
auxiliaryfunctions.write_pickle(
os.path.join(path_camera_matrix, "stereo_params.pickle"), stereo_params
)
print(
"Camera calibration done! Use the function ``check_undistortion`` to check the check the calibration"
)
else:
print(
"Corners extracted! You may check for the extracted corners in the directory %s and remove the pair of images where the corners are incorrectly detected. If all the corners are detected correctly with right order, then re-run the same function and use the flag ``calibrate=True``, to calbrate the camera."
% str(path_corners)
)
def convertcsv2h5(config, userfeedback=True, scorer=None):
"""
Convert (image) annotation files in folder labeled-data from csv to h5.
This function allows the user to manually edit the csv (e.g. to correct the scorer name and then convert it into hdf format).
WARNING: conversion might corrupt the data.
config : string
Full path of the config.yaml file as a string.
userfeedback: bool, optional
If true the user will be asked specifically for each folder in labeled-data if the containing csv shall be converted to hdf format.
scorer: string, optional
If a string is given, then the scorer/annotator in all csv and hdf files that are changed, will be overwritten with this name.
Examples
--------
Convert csv annotation files for reaching-task project into hdf.
>>> deeplabcutcore.convertcsv2h5('/analysis/project/reaching-task/config.yaml')
--------
Convert csv annotation files for reaching-task project into hdf while changing the scorer/annotator in all annotation files to Albert!
>>> deeplabcutcore.convertcsv2h5('/analysis/project/reaching-task/config.yaml',scorer='Albert')
--------
"""
cfg = auxiliaryfunctions.read_config(config)
videos = cfg["video_sets"].keys()
video_names = [Path(i).stem for i in videos]
folders = [
Path(config).parent / "labeled-data" / Path(i) for i in video_names
]
if scorer == None:
scorer = cfg["scorer"]
for folder in folders:
try:
if userfeedback == True:
print("Do you want to convert the csv file in folder:", folder,
"?")
askuser = input("yes/no")
else:
askuser = "******"
if (askuser == "y" or askuser == "yes" or askuser == "Ja"
or askuser == "ha"): # multilanguage support :)
fn = os.path.join(str(folder),
"CollectedData_" + cfg["scorer"] + ".csv")
data = pd.read_csv(fn)
# nlines,numcolumns=data.shape
orderofbpincsv = list(data.values[0, 1:-1:2])
imageindex = list(data.values[2:, 0])
# assert(len(orderofbpincsv)==len(cfg['bodyparts']))
print(orderofbpincsv)
print(cfg["bodyparts"])
# TODO: test len of images vs. len of imagenames for another sanity check
index = pd.MultiIndex.from_product(
[[scorer], orderofbpincsv, ["x", "y"]],
names=["scorer", "bodyparts", "coords"],
)
frame = pd.DataFrame(
np.array(data.values[2:, 1:], dtype=float),
columns=index,
index=imageindex,
)
frame.to_hdf(
os.path.join(str(folder),
"CollectedData_" + cfg["scorer"] + ".h5"),
key="df_with_missing",
mode="w",
)
frame.to_csv(fn)
except FileNotFoundError:
print("Attention:", folder,
"does not appear to have labeled data!")
def analyze_videos(config,
videos,
videotype='avi',
shuffle=1,
trainingsetindex=0,
gputouse=None,
save_as_csv=False,
destfolder=None,
batchsize=None,
crop=None,
get_nframesfrommetadata=True,
TFGPUinference=True,
dynamic=(False, .5, 10)):
"""
Makes prediction based on a trained network. The index of the trained network is specified by parameters in the config file (in particular the variable 'snapshotindex')
Output: The labels are stored as MultiIndex Pandas Array, which contains the name of the network, body part name, (x, y) label position \n
in pixels, and the likelihood for each frame per body part. These arrays are stored in an efficient Hierarchical Data Format (HDF) \n
in the same directory, where the video is stored. However, if the flag save_as_csv is set to True, the data can also be exported in \n
comma-separated values format (.csv), which in turn can be imported in many programs, such as MATLAB, R, Prism, etc.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : list
A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.
videotype: string, optional
Checks for the extension of the video in case the input to the video is a directory.\n Only videos with this extension are analyzed. The default is ``.avi``
shuffle: int, optional
An integer specifying the shuffle index of the training dataset used for training the network. The default is 1.
trainingsetindex: int, optional
Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml).
gputouse: int, optional. Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not have a GPU put None.
See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries
save_as_csv: bool, optional
Saves the predictions in a .csv file. The default is ``False``; if provided it must be either ``True`` or ``False``
destfolder: string, optional
Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this
folder also needs to be passed.
batchsize: int, default from pose_cfg.yaml
Change batch size for inference; if given overwrites value in pose_cfg.yaml
crop: list, optional (default=None)
List of cropping coordinates as [x1, x2, y1, y2].
Note that the same cropping parameters will then be used for all videos.
If different video crops are desired, run 'analyze_videos' on individual videos
with the corresponding cropping coordinates.
TFGPUinference: bool, default: True
Perform inference on GPU with Tensorflow code. Introduced in "Pretraining boosts out-of-domain robustness for pose estimation" by
Alexander Mathis, Mert Yüksekgönül, Byron Rogers, Matthias Bethge, Mackenzie W. Mathis Source: https://arxiv.org/abs/1909.11229
dynamic: triple containing (state,detectiontreshold,margin)
If the state is true, then dynamic cropping will be performed. That means that if an object is detected (i.e. any body part > detectiontreshold),
then object boundaries are computed according to the smallest/largest x position and smallest/largest y position of all body parts. This window is
expanded by the margin and from then on only the posture within this crop is analyzed (until the object is lost, i.e. <detectiontreshold). The
current position is utilized for updating the crop window for the next frame (this is why the margin is important and should be set large
enough given the movement of the animal).
Examples
--------
Windows example for analyzing 1 video
>>> deeplabcutcore.analyze_videos('C:\\myproject\\reaching-task\\config.yaml',['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi'])
--------
If you want to analyze only 1 video
>>> deeplabcutcore.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi'])
--------
If you want to analyze all videos of type avi in a folder:
>>> deeplabcutcore.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos'],videotype='.avi')
--------
If you want to analyze multiple videos
>>> deeplabcutcore.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'])
--------
If you want to analyze multiple videos with shuffle = 2
>>> deeplabcutcore.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'],shuffle=2)
--------
If you want to analyze multiple videos with shuffle = 2 and save results as an additional csv file too
>>> deeplabcutcore.analyze_videos('/analysis/project/reaching-task/config.yaml',['/analysis/project/videos/reachingvideo1.avi','/analysis/project/videos/reachingvideo2.avi'],shuffle=2,save_as_csv=True)
--------
"""
if 'TF_CUDNN_USE_AUTOTUNE' in os.environ:
del os.environ[
'TF_CUDNN_USE_AUTOTUNE'] #was potentially set during training
if gputouse is not None: #gpu selection
os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
tf.compat.v1.reset_default_graph()
start_path = os.getcwd(
) #record cwd to return to this directory in the end
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
if crop is not None:
cfg['cropping'] = True
cfg['x1'], cfg['x2'], cfg['y1'], cfg['y2'] = crop
print("Overwriting cropping parameters:", crop)
print(
"These are used for all videos, but won't be save to the cfg file."
)
modelfolder = os.path.join(
cfg["project_path"],
str(auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError(
"It seems the model for shuffle %s and trainFraction %s does not exist."
% (shuffle, trainFraction))
# Check which snapshots are available and sort them by # iterations
try:
Snapshots = np.array([
fn.split('.')[0]
for fn in os.listdir(os.path.join(modelfolder, 'train'))
if "index" in fn
])
except FileNotFoundError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s."
% (shuffle, shuffle))
if cfg['snapshotindex'] == 'all':
print(
"Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!"
)
snapshotindex = -1
else:
snapshotindex = cfg['snapshotindex']
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
print("Using %s" % Snapshots[snapshotindex], "for model", modelfolder)
##################################################
# Load and setup CNN part detector
##################################################
# Check if data already was generated:
dlc_cfg['init_weights'] = os.path.join(modelfolder, 'train',
Snapshots[snapshotindex])
trainingsiterations = (dlc_cfg['init_weights'].split(
os.sep)[-1]).split('-')[-1]
# Update number of output and batchsize
dlc_cfg['num_outputs'] = cfg.get('num_outputs',
dlc_cfg.get('num_outputs', 1))
if batchsize == None:
#update batchsize (based on parameters in config.yaml)
dlc_cfg['batch_size'] = cfg['batch_size']
else:
dlc_cfg['batch_size'] = batchsize
cfg['batch_size'] = batchsize
if dynamic[0]: #state=true
#(state,detectiontreshold,margin)=dynamic
print("Starting analysis in dynamic cropping mode with parameters:",
dynamic)
dlc_cfg['num_outputs'] = 1
TFGPUinference = False
dlc_cfg['batch_size'] = 1
print(
"Switching batchsize to 1, num_outputs (per animal) to 1 and TFGPUinference to False (all these features are not supported in this mode)."
)
# Name for scorer:
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg, shuffle, trainFraction, trainingsiterations=trainingsiterations)
if dlc_cfg['num_outputs'] > 1:
if TFGPUinference:
print(
"Switching to numpy-based keypoint extraction code, as multiple point extraction is not supported by TF code currently."
)
TFGPUinference = False
print("Extracting ", dlc_cfg['num_outputs'], "instances per bodypart")
xyz_labs_orig = ['x', 'y', 'likelihood']
suffix = [str(s + 1) for s in range(dlc_cfg['num_outputs'])]
suffix[
0] = '' # first one has empty suffix for backwards compatibility
xyz_labs = [x + s for s in suffix for x in xyz_labs_orig]
else:
xyz_labs = ['x', 'y', 'likelihood']
#sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
if TFGPUinference:
sess, inputs, outputs = predict.setup_GPUpose_prediction(dlc_cfg)
else:
sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
pdindex = pd.MultiIndex.from_product(
[[DLCscorer], dlc_cfg['all_joints_names'], xyz_labs],
names=['scorer', 'bodyparts', 'coords'])
##################################################
# Datafolder
##################################################
Videos = auxiliaryfunctions.Getlistofvideos(videos, videotype)
if len(Videos) > 0:
#looping over videos
for video in Videos:
DLCscorer = AnalyzeVideo(video, DLCscorer, DLCscorerlegacy,
trainFraction, cfg, dlc_cfg, sess, inputs,
outputs, pdindex, save_as_csv, destfolder,
TFGPUinference, dynamic)
os.chdir(str(start_path))
print(
"The videos are analyzed. Now your research can truly start! \n You can create labeled videos with 'create_labeled_video'."
)
print(
"If the tracking is not satisfactory for some videos, consider expanding the training set. You can use the function 'extract_outlier_frames' to extract any outlier frames!"
)
return DLCscorer #note: this is either DLCscorer or DLCscorerlegacy depending on what was used!
else:
os.chdir(str(start_path))
print("No video/s found. Please check your path!")
return DLCscorer