def adddatasetstovideolistandviceversa(config,prefix,width,height,suffix='.mp4'):
"""
First run comparevideolistsanddatafolders(config) to compare the folders in labeled-data and the ones listed under video_sets (in the config file).
If you detect differences this function can be used to maker sure each folder has a video entry & vice versa.
It corrects this problem in the following way:
If a folder in labeled-data does not contain a video entry in the config file then the prefix path will be added in front of the name of the labeled-data folder and combined
with the suffix variable as an ending. Width and height will be added as cropping variables as passed on. TODO: This should be written from the actual images!
If a video entry in the config file does not contain a folder in labeled-data, then the entry is removed.
Handle with care!
Parameter
----------
config : string
String containing the full path of the config file in the project.
"""
cfg = auxiliaryfunctions.read_config(config)
videos = cfg['video_sets'].keys()
video_names = [Path(i).stem for i in videos]
alldatafolders = [fn for fn in os.listdir(Path(config).parent / 'labeled-data') if '_labeled' not in fn]
print("Config file contains:", len(video_names))
print("Labeled-data contains:", len(alldatafolders))
toberemoved=[]
for vn in video_names:
if vn in alldatafolders:
pass
else:
print(vn, " is missing as a labeled folder >> removing key!")
for fullvideo in cfg['video_sets'].keys():
if vn in fullvideo:
toberemoved.append(fullvideo)
for vid in toberemoved:
del cfg['video_sets'][vid]
#Load updated lists:
videos = cfg['video_sets'].keys()
video_names = [Path(i).stem for i in videos]
for vn in alldatafolders:
if vn in video_names:
pass
else:
print(vn, " is missing in config file >> adding it!")
#cfg['video_sets'][vn]
cfg['video_sets'].update({os.path.join(prefix,vn+suffix) : {'crop': ', '.join(map(str, [0, width, 0, height]))}})
auxiliaryfunctions.write_config(config,cfg)
def merge_datasets(config,forceiterate=None):
"""
Checks if the original training dataset can be merged with the newly refined training dataset. To do so it will check
if the frames in all extracted video sets were relabeled. If this is the case then the iterate variable is advanced by 1.
Parameter
----------
config : string
Full path of the config.yaml file as a string.
forceiterate: int, optional
If an integer is given the iteration variable is set to this value (this is only done if all datasets were labeled or refined)
Example
--------
>>> deeplabcutcore.merge_datasets('/analysis/project/reaching-task/config.yaml')
--------
"""
import yaml
cfg = auxiliaryfunctions.read_config(config)
config_path = Path(config).parents[0]
bf=Path(str(config_path/'labeled-data'))
allfolders = [os.path.join(bf,fn) for fn in os.listdir(bf) if "_labeled" not in fn] #exclude labeled data folders!
flagged=False
for findex,folder in enumerate(allfolders):
if os.path.isfile(os.path.join(folder,'MachineLabelsRefine.h5')): #Folder that was manually refine...
pass
elif os.path.isfile(os.path.join(folder,'CollectedData_'+cfg['scorer']+'.h5')): #Folder that contains human data set...
pass
else:
print("The following folder was not manually refined,...",folder)
flagged=True
pass #this folder does not contain a MachineLabelsRefine file (not updated...)
if flagged==False:
# updates iteration by 1
iter_prev=cfg['iteration']
if not forceiterate:
cfg['iteration']=int(iter_prev+1)
else:
cfg['iteration']=forceiterate
auxiliaryfunctions.write_config(config,cfg)
print("Merged data sets and updated refinement iteration to "+str(cfg['iteration'])+".")
print("Now you can create a new training set for the expanded annotated images (use create_training_dataset).")
else:
print("Please label, or remove the un-corrected folders.")
def select_cropping_area(config, videos=None):
"""
Interactively select the cropping area of all videos in the config.
A user interface pops up with a frame to select the cropping parameters.
Use the left click to draw a box and hit the button 'set cropping parameters'
to store the cropping parameters for a video in the config.yaml file.
Parameters
----------
config : string
Full path of the config.yaml file as a string.
videos : optional (default=None)
List of videos whose cropping areas are to be defined. Note that full paths are required.
By default, all videos in the config are successively loaded.
Returns
-------
cfg : dict
Updated project configuration
"""
from deeplabcutcore.utils import auxiliaryfunctions, auxfun_videos
cfg = auxiliaryfunctions.read_config(config)
if videos is None:
videos = cfg["video_sets"]
for video in videos:
coords = auxfun_videos.draw_bbox(video)
if coords:
cfg["video_sets"][video] = {
"crop": ", ".join(
map(
str,
[
int(coords[0]),
int(coords[2]),
int(coords[1]),
int(coords[3]),
],
)
)
}
auxiliaryfunctions.write_config(config, cfg)
return cfg
def transform_data(config):
"""
This function adds the full path to labeling dataset.
It also adds the correct path to the video file in the config file.
"""
import pandas as pd
cfg = auxiliaryfunctions.read_config(config)
project_path = str(Path(config).parents[0])
cfg['project_path'] = project_path
if 'Reaching' in project_path:
video_file = os.path.join(project_path, 'videos', 'reachingvideo1.avi')
elif 'openfield' in project_path:
video_file = os.path.join(project_path, 'videos', 'm4s1.mp4')
else:
print("This is not an offical demo dataset.")
if 'WILL BE AUTOMATICALLY UPDATED BY DEMO CODE' in cfg['video_sets'].keys(
):
cfg['video_sets'][str(video_file)] = cfg['video_sets'].pop(
'WILL BE AUTOMATICALLY UPDATED BY DEMO CODE')
auxiliaryfunctions.write_config(config, cfg)
def transform_data(config):
"""
This function adds the full path to labeling dataset.
It also adds the correct path to the video file in the config file.
"""
import pandas as pd
cfg = auxiliaryfunctions.read_config(config)
project_path = str(Path(config).parents[0])
cfg["project_path"] = project_path
if "Reaching" in project_path:
video_file = os.path.join(project_path, "videos", "reachingvideo1.avi")
elif "openfield" in project_path:
video_file = os.path.join(project_path, "videos", "m4s1.mp4")
else:
print("This is not an offical demo dataset.")
if "WILL BE AUTOMATICALLY UPDATED BY DEMO CODE" in cfg["video_sets"].keys(
):
cfg["video_sets"][str(video_file)] = cfg["video_sets"].pop(
"WILL BE AUTOMATICALLY UPDATED BY DEMO CODE")
auxiliaryfunctions.write_config(config, cfg)
def add_new_videos(config, videos, copy_videos=False, coords=None):
"""
Add new videos to the config file at any stage of the project.
Parameters
----------
config : string
String containing the full path of the config file in the project.
videos : list
A list of string containing the full paths of the videos to include in the project.
copy_videos : bool, optional
If this is set to True, the symlink of the videos are copied to the project/videos directory. The default is
``False``; if provided it must be either ``True`` or ``False``.
coords: list, optional
A list containing the list of cropping coordinates of the video. The default is set to None.
Examples
--------
Video will be added, with cropping dimenions according to the frame dimensinos of mouse5.avi
>>> deeplabcutcore.add_new_videos('/home/project/reaching-task-Tanmay-2018-08-23/config.yaml',['/data/videos/mouse5.avi'])
Video will be added, with cropping dimenions [0,100,0,200]
>>> deeplabcutcore.add_new_videos('/home/project/reaching-task-Tanmay-2018-08-23/config.yaml',['/data/videos/mouse5.avi'],copy_videos=False,coords=[[0,100,0,200]])
Two videos will be added, with cropping dimenions [0,100,0,200] and [0,100,0,250], respectively.
>>> deeplabcutcore.add_new_videos('/home/project/reaching-task-Tanmay-2018-08-23/config.yaml',['/data/videos/mouse5.avi','/data/videos/mouse6.avi'],copy_videos=False,coords=[[0,100,0,200],[0,100,0,250]])
"""
import os
import shutil
from pathlib import Path
from deeplabcutcore import DEBUG
from deeplabcutcore.utils import auxiliaryfunctions
import cv2
# Read the config file
cfg = auxiliaryfunctions.read_config(config)
video_path = Path(config).parents[0] / 'videos'
data_path = Path(config).parents[0] / 'labeled-data'
videos = [Path(vp) for vp in videos]
dirs = [data_path / Path(i.stem) for i in videos]
for p in dirs:
"""
Creates directory under data & perhaps copies videos (to /video)
"""
p.mkdir(parents=True, exist_ok=True)
destinations = [video_path.joinpath(vp.name) for vp in videos]
if copy_videos == True:
for src, dst in zip(videos, destinations):
if dst.exists():
pass
else:
print("Copying the videos")
shutil.copy(os.fspath(src), os.fspath(dst))
else:
for src, dst in zip(videos, destinations):
if dst.exists():
pass
else:
print("Creating the symbolic link of the video")
src = str(src)
dst = str(dst)
os.symlink(src, dst)
if copy_videos == True:
videos = destinations # in this case the *new* location should be added to the config file
# adds the video list to the config.yaml file
for idx, video in enumerate(videos):
try:
# For windows os.path.realpath does not work and does not link to the real video.
video_path = str(Path.resolve(Path(video)))
# video_path = os.path.realpath(video)
except:
video_path = os.readlink(video)
vcap = cv2.VideoCapture(video_path)
if vcap.isOpened():
# get vcap property
width = int(vcap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(vcap.get(cv2.CAP_PROP_FRAME_HEIGHT))
if coords == None:
cfg['video_sets'].update({
video_path: {
'crop': ', '.join(map(str, [0, width, 0, height]))
}
})
else:
c = coords[idx]
cfg['video_sets'].update(
{video_path: {
'crop': ', '.join(map(str, c))
}})
else:
print("Cannot open the video file!")
auxiliaryfunctions.write_config(config, cfg)
print(
"New video was added to the project! Use the function 'extract_frames' to select frames for labeling."
)
def create_new_project(
project,
experimenter,
videos,
working_directory=None,
copy_videos=False,
videotype=".avi",
):
"""Creates a new project directory, sub-directories and a basic configuration file. The configuration file is loaded with the default values. Change its parameters to your projects need.
Parameters
----------
project : string
String containing the name of the project.
experimenter : string
String containing the name of the experimenter.
videos : list
A list of string containing the full paths of the videos to include in the project.
Attention: Can also be a directory, then all videos of videotype will be imported.
working_directory : string, optional
The directory where the project will be created. The default is the ``current working directory``; if provided, it must be a string.
copy_videos : bool, optional
If this is set to True, the videos are copied to the ``videos`` directory. If it is False,symlink of the videos are copied to the project/videos directory. The default is ``False``; if provided it must be either
``True`` or ``False``.
Example
--------
Linux/MacOs
>>> deeplabcutcore.create_new_project('reaching-task','Linus',['/data/videos/mouse1.avi','/data/videos/mouse2.avi','/data/videos/mouse3.avi'],'/analysis/project/')
>>> deeplabcutcore.create_new_project('reaching-task','Linus',['/data/videos'],videotype='.mp4')
Windows:
>>> deeplabcutcore.create_new_project('reaching-task','Bill',[r'C:\yourusername\rig-95\Videos\reachingvideo1.avi'], copy_videos=True)
Users must format paths with either: r'C:\ OR 'C:\\ <- i.e. a double backslash \ \ )
"""
from datetime import datetime as dt
from deeplabcutcore.utils import auxiliaryfunctions
date = dt.today()
month = date.strftime("%B")
day = date.day
d = str(month[0:3] + str(day))
date = dt.today().strftime("%Y-%m-%d")
if working_directory == None:
working_directory = "."
wd = Path(working_directory).resolve()
project_name = "{pn}-{exp}-{date}".format(pn=project,
exp=experimenter,
date=date)
project_path = wd / project_name
# Create project and sub-directories
if not DEBUG and project_path.exists():
print('Project "{}" already exists!'.format(project_path))
return
video_path = project_path / "videos"
data_path = project_path / "labeled-data"
shuffles_path = project_path / "training-datasets"
results_path = project_path / "dlc-models"
for p in [video_path, data_path, shuffles_path, results_path]:
p.mkdir(parents=True, exist_ok=DEBUG)
print('Created "{}"'.format(p))
# Add all videos in the folder. Multiple folders can be passed in a list, similar to the video files. Folders and video files can also be passed!
vids = []
for i in videos:
# Check if it is a folder
if os.path.isdir(i):
vids_in_dir = [
os.path.join(i, vp) for vp in os.listdir(i) if videotype in vp
]
vids = vids + vids_in_dir
if len(vids_in_dir) == 0:
print("No videos found in", i)
print(
"Perhaps change the videotype, which is currently set to:",
videotype,
)
else:
videos = vids
print(
len(vids_in_dir),
" videos from the directory",
i,
"were added to the project.",
)
else:
if os.path.isfile(i):
vids = vids + [i]
videos = vids
videos = [Path(vp) for vp in videos]
dirs = [data_path / Path(i.stem) for i in videos]
for p in dirs:
"""
Creates directory under data
"""
p.mkdir(parents=True, exist_ok=True)
destinations = [video_path.joinpath(vp.name) for vp in videos]
if copy_videos == True:
print("Copying the videos")
for src, dst in zip(videos, destinations):
shutil.copy(
os.fspath(src),
os.fspath(dst)) # https://www.python.org/dev/peps/pep-0519/
else:
# creates the symlinks of the video and puts it in the videos directory.
print("Attempting to create a symbolic link of the video ...")
for src, dst in zip(videos, destinations):
if dst.exists() and not DEBUG:
raise FileExistsError("Video {} exists already!".format(dst))
try:
src = str(src)
dst = str(dst)
os.symlink(src, dst)
except OSError:
import subprocess
subprocess.check_call("mklink %s %s" % (dst, src), shell=True)
print("Created the symlink of {} to {}".format(src, dst))
videos = destinations
if copy_videos == True:
videos = (
destinations
) # in this case the *new* location should be added to the config file
# adds the video list to the config.yaml file
video_sets = {}
for video in videos:
print(video)
try:
# For windows os.path.realpath does not work and does not link to the real video. [old: rel_video_path = os.path.realpath(video)]
rel_video_path = str(Path.resolve(Path(video)))
except:
rel_video_path = os.readlink(str(video))
vcap = cv2.VideoCapture(rel_video_path)
if vcap.isOpened():
width = int(vcap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(vcap.get(cv2.CAP_PROP_FRAME_HEIGHT))
video_sets[rel_video_path] = {
"crop": ", ".join(map(str, [0, width, 0, height]))
}
else:
print("Cannot open the video file! Skipping to the next one...")
os.remove(video) # Removing the video or link from the project
if not len(video_sets):
# Silently sweep the files that were already written.
shutil.rmtree(project_path, ignore_errors=True)
print(
"WARNING: No valid videos were found. The project was not created ..."
)
print("Verify the video files and re-create the project.")
return "nothingcreated"
# Set values to config file:
cfg_file, ruamelFile = auxiliaryfunctions.create_config_template()
cfg_file["Task"] = project
cfg_file["scorer"] = experimenter
cfg_file["video_sets"] = video_sets
cfg_file["project_path"] = str(project_path)
cfg_file["date"] = d
cfg_file["bodyparts"] = ["bodypart1", "bodypart2", "bodypart3", "objectA"]
cfg_file["cropping"] = False
cfg_file["start"] = 0
cfg_file["stop"] = 1
cfg_file["numframes2pick"] = 20
cfg_file["TrainingFraction"] = [0.95]
cfg_file["iteration"] = 0
# cfg_file['resnet']=50
cfg_file["default_net_type"] = "resnet_50"
cfg_file["default_augmenter"] = "default"
cfg_file["snapshotindex"] = -1
cfg_file["x1"] = 0
cfg_file["x2"] = 640
cfg_file["y1"] = 277
cfg_file["y2"] = 624
cfg_file["batch_size"] = (
8
) # batch size during inference (video - analysis); see https://www.biorxiv.org/content/early/2018/10/30/457242
cfg_file["corner2move2"] = (50, 50)
cfg_file["move2corner"] = True
cfg_file["skeleton"] = [["bodypart1", "bodypart2"],
["objectA", "bodypart3"]]
cfg_file["skeleton_color"] = "black"
cfg_file["pcutoff"] = 0.6
cfg_file["dotsize"] = 12 # for plots size of dots
cfg_file["alphavalue"] = 0.7 # for plots transparency of markers
cfg_file["colormap"] = "jet" # for plots type of colormap
projconfigfile = os.path.join(str(project_path), "config.yaml")
# Write dictionary to yaml config file
auxiliaryfunctions.write_config(projconfigfile, cfg_file)
print('Generated "{}"'.format(project_path / "config.yaml"))
print(
"\nA new project with name %s is created at %s and a configurable file (config.yaml) is stored there. Change the parameters in this file to adapt to your project's needs.\n Once you have changed the configuration file, use the function 'extract_frames' to select frames for labeling.\n. [OPTIONAL] Use the function 'add_new_videos' to add new videos to your project (at any stage)."
% (project_name, str(wd)))
return projconfigfile
def create_pretrained_project(
project,
experimenter,
videos,
model="full_human",
working_directory=None,
copy_videos=False,
videotype=None,
analyzevideo=True,
filtered=True,
createlabeledvideo=True,
trainFraction=None,
):
"""
Creates a new project directory, sub-directories and a basic configuration file.
Change its parameters to your projects need.
The project will also be initialized with a pre-trained model from the DeepLabCut model zoo!
http://www.mousemotorlab.org/dlc-modelzoo
Parameters
----------
project : string
String containing the name of the project.
experimenter : string
String containing the name of the experimenter.
model: string, options see http://www.mousemotorlab.org/dlc-modelzoo
Current option and default: 'full_human' Creates a demo human project and analyzes a video with ResNet 101 weights pretrained on MPII Human Pose. This is from the DeeperCut paper
by Insafutdinov et al. https://arxiv.org/abs/1605.03170 Please make sure to cite it too if you use this code!
videos : list
A list of string containing the full paths of the videos to include in the project.
working_directory : string, optional
The directory where the project will be created. The default is the ``current working directory``; if provided, it must be a string.
copy_videos : bool, optional ON WINDOWS: TRUE is often necessary!
If this is set to True, the videos are copied to the ``videos`` directory. If it is False,symlink of the videos are copied to the project/videos directory. The default is ``False``; if provided it must be either
``True`` or ``False``.
analyzevideo " bool, optional
If true, then the video is analzyed and a labeled video is created. If false, then only the project will be created and the weights downloaded. You can then access them
filtered: bool, default false
Boolean variable indicating if filtered pose data output should be plotted rather than frame-by-frame predictions.
Filtered version can be calculated with deeplabcutcore.filterpredictions
trainFraction: By default value from *new* projects. (0.95)
Fraction that will be used in dlc-model/trainingset folder name.
Example
--------
Linux/MacOs loading full_human model and analzying video /homosapiens1.avi
>>> deeplabcutcore.create_pretrained_project('humanstrokestudy','Linus',['/data/videos/homosapiens1.avi'], copy_videos=False)
Loading full_cat model and analzying video "felixfeliscatus3.avi"
>>> deeplabcutcore.create_pretrained_project('humanstrokestudy','Linus',['/data/videos/felixfeliscatus3.avi'], model='full_cat')
Windows:
>>> deeplabcutcore.create_pretrained_project('humanstrokestudy','Bill',[r'C:\yourusername\rig-95\Videos\reachingvideo1.avi'],r'C:\yourusername\analysis\project' copy_videos=True)
Users must format paths with either: r'C:\ OR 'C:\\ <- i.e. a double backslash \ \ )
"""
if model in globals()["Modeloptions"]:
cwd = os.getcwd()
cfg = deeplabcutcore.create_new_project(project, experimenter, videos,
working_directory, copy_videos,
videotype)
if trainFraction is not None:
auxiliaryfunctions.edit_config(
cfg, {"TrainingFraction": [trainFraction]})
config = auxiliaryfunctions.read_config(cfg)
if model == "full_human":
config["bodyparts"] = [
"ankle1",
"knee1",
"hip1",
"hip2",
"knee2",
"ankle2",
"wrist1",
"elbow1",
"shoulder1",
"shoulder2",
"elbow2",
"wrist2",
"chin",
"forehead",
]
config["skeleton"] = [
["ankle1", "knee1"],
["ankle2", "knee2"],
["knee1", "hip1"],
["knee2", "hip2"],
["hip1", "hip2"],
["shoulder1", "shoulder2"],
["shoulder1", "hip1"],
["shoulder2", "hip2"],
["shoulder1", "elbow1"],
["shoulder2", "elbow2"],
["chin", "forehead"],
["elbow1", "wrist1"],
["elbow2", "wrist2"],
]
config["default_net_type"] = "resnet_101"
else: # just make a case and put the stuff you want.
# TBD: 'partaffinityfield_graph' >> use to set skeleton!
pass
auxiliaryfunctions.write_config(cfg, config)
config = auxiliaryfunctions.read_config(cfg)
train_dir = Path(
os.path.join(
config["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction=config["TrainingFraction"][0],
shuffle=1,
cfg=config,
)),
"train",
))
test_dir = Path(
os.path.join(
config["project_path"],
str(
auxiliaryfunctions.GetModelFolder(
trainFraction=config["TrainingFraction"][0],
shuffle=1,
cfg=config,
)),
"test",
))
# Create the model directory
train_dir.mkdir(parents=True, exist_ok=True)
test_dir.mkdir(parents=True, exist_ok=True)
modelfoldername = auxiliaryfunctions.GetModelFolder(
trainFraction=config["TrainingFraction"][0], shuffle=1, cfg=config)
path_train_config = str(
os.path.join(config["project_path"], Path(modelfoldername),
"train", "pose_cfg.yaml"))
path_test_config = str(
os.path.join(config["project_path"], Path(modelfoldername), "test",
"pose_cfg.yaml"))
# Download the weights and put then in appropriate directory
print("Dowloading weights...")
auxfun_models.DownloadModel(model, train_dir)
pose_cfg = deeplabcutcore.auxiliaryfunctions.read_plainconfig(
path_train_config)
print(path_train_config)
# Updating config file:
dict = {
"default_net_type": pose_cfg["net_type"],
"default_augmenter": pose_cfg["dataset_type"],
"bodyparts": pose_cfg["all_joints_names"],
"skeleton": [], # TODO: update with paf_graph
"dotsize": 6,
}
auxiliaryfunctions.edit_config(cfg, dict)
# Create the pose_config.yaml files
parent_path = Path(os.path.dirname(deeplabcutcore.__file__))
defaultconfigfile = str(parent_path / "pose_cfg.yaml")
trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(config)
datafilename, metadatafilename = auxiliaryfunctions.GetDataandMetaDataFilenames(
trainingsetfolder,
trainFraction=config["TrainingFraction"][0],
shuffle=1,
cfg=config,
)
# downloading base encoder / not required unless on re-trains (but when a training set is created this happens anyway)
# model_path, num_shuffles=auxfun_models.Check4weights(pose_cfg['net_type'], parent_path, num_shuffles= 1)
# Updating training and test pose_cfg:
snapshotname = [fn for fn in os.listdir(train_dir)
if ".meta" in fn][0].split(".meta")[0]
dict2change = {
"init_weights": str(os.path.join(train_dir, snapshotname)),
"project_path": str(config["project_path"]),
}
UpdateTrain_pose_yaml(pose_cfg, dict2change, path_train_config)
keys2save = [
"dataset",
"dataset_type",
"num_joints",
"all_joints",
"all_joints_names",
"net_type",
"init_weights",
"global_scale",
"location_refinement",
"locref_stdev",
]
MakeTest_pose_yaml(pose_cfg, keys2save, path_test_config)
video_dir = os.path.join(config["project_path"], "videos")
if analyzevideo == True:
print("Analyzing video...")
deeplabcutcore.analyze_videos(cfg, [video_dir],
videotype,
save_as_csv=True)
if createlabeledvideo == True:
if filtered:
deeplabcutcore.filterpredictions(cfg, [video_dir], videotype)
print("Plotting results...")
deeplabcutcore.create_labeled_video(cfg, [video_dir],
videotype,
draw_skeleton=True,
filtered=filtered)
deeplabcutcore.plot_trajectories(cfg, [video_dir],
videotype,
filtered=filtered)
os.chdir(cwd)
return cfg, path_train_config
else:
return "N/A", "N/A"