def SelectFrames(videopath, filename, x1, x2, y1, y2, cropping, videotype,
start, stop, Task, selectionalgorithm):
''' Selecting frames from videos for labeling.'''
if start > 1.0 or stop > 1.0 or start < 0 or stop < 0 or start >= stop:
print(
"Please change start & stop, they should form a normalized interval with 0<= start < stop<=1."
)
else:
basefolder = 'data-' + Task + '/'
auxiliaryfunctions.attempttomakefolder(basefolder)
videos = auxiliaryfunctions.GetVideoList(filename, videopath,
videotype)
for vindex, video in enumerate(videos):
print("Loading ", video)
clip = VideoFileClip(os.path.join(videopath, video))
print("Duration of video [s], ", clip.duration, "fps, ", clip.fps,
"Cropped frame dimensions: ", clip.size)
####################################################
# Creating folder with name of experiment and extract random frames
####################################################
folder = video.split('.')[0]
auxiliaryfunctions.attempttomakefolder(
os.path.join(basefolder, folder))
indexlength = int(np.ceil(np.log10(clip.duration * clip.fps)))
# Extract the first frame (not cropped!) - useful for data augmentation
index = 0
image = img_as_ubyte(clip.get_frame(index * 1. / clip.fps))
io.imsave(
os.path.join(basefolder, folder,
"img" + str(index).zfill(indexlength) + ".png"),
image)
if cropping == True:
# Select ROI of interest by adjusting values in myconfig.py
clip = clip.crop(y1=y1, y2=y2, x1=x1, x2=x2)
print("Extracting frames ...")
if selectionalgorithm == 'uniform':
frames2pick = frameselectiontools.UniformFrames(
clip, numframes2pick, start, stop)
elif selectionalgorithm == 'kmeans':
frames2pick = frameselectiontools.KmeansbasedFrameselection(
clip, numframes2pick, start, stop)
else:
print(
"Please implement this method yourself and send us a pull request!"
)
frames2pick = []
indexlength = int(np.ceil(np.log10(clip.duration * clip.fps)))
for index in frames2pick:
try:
image = img_as_ubyte(clip.get_frame(index * 1. / clip.fps))
io.imsave(
os.path.join(
basefolder, folder,
"img" + str(index).zfill(indexlength) + ".png"),
image)
except FileNotFoundError:
print("Frame # ", index, " does not exist.")
image = io.imread(os.path.join(models_folder_path,
unaugmented_folder_name, 'data-' + Task,
imagename),
mode='RGB')
image = skimage.color.gray2rgb(image)
image_batch = data_to_input(image)
# Compute prediction with the CNN
outputs_np = sess.run(outputs, feed_dict={inputs: image_batch})
scmap, locref = predict.extract_cnn_output(outputs_np, cfg)
# Extract maximum scoring location from the heatmap, assume 1 person
pose = predict.argmax_pose_predict(scmap, locref, cfg.stride)
PredicteData[imageindex, :] = pose.flatten(
) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts!
index = pd.MultiIndex.from_product(
[[DLCscorer], cfg['all_joints_names'], ['x', 'y', 'likelihood']],
names=['scorer', 'bodyparts', 'coords'])
# Saving results:
auxiliaryfunctions.attempttomakefolder("Results")
DataMachine = pd.DataFrame(PredicteData,
columns=index,
index=Data.index.values)
DataMachine.to_hdf(os.path.join("Results", DLCscorer + '.h5'),
'df_with_missing',
format='table',
mode='w')
print("Done and results stored for snapshot: ", Snapshots[snapshotIndex])
date) + 'shuffle' + str(shuffle) + '_' + str(trainingsiterations)
##################################################
# Datafolder
##################################################
# videofolder='../videos/' #where your folder with videos is.
os.chdir(videofolder)
videos = np.sort([fn for fn in os.listdir(os.curdir) if (".mov" in fn)])
print("Starting ", videofolder, videos)
for video in videos:
vname = video.split('.')[0]
tmpfolder = 'temp' + vname
auxiliaryfunctions.attempttomakefolder(tmpfolder)
if os.path.isfile(tmpfolder + '/' + vname + '.mp4'):
print("Labeled video already created.")
else:
print("Loading ", video, "and data.")
dataname = video.split('.')[0] + scorer + '.h5'
try:
Dataframe = pd.read_hdf(dataname)
clip = VideoFileClip(video)
except FileNotFoundError:
print("Data was not analyzed (run AnalysisVideos.py first).")
ny, nx = clip.size # dimensions of frame (height, width)
fps = clip.fps
nframes = np.sum(1 for j in clip.iter_frames())
def MakingLabeledVideoTrial(video_file):
"""
DeepLabCut Toolbox
https://github.com/AlexEMG/DeepLabCut
A Mathis, [email protected]
M Mathis, [email protected]
This script labels the bodyparts in videos as analzyed by "AnalyzeVideos.py". This code is relatively slow as
it stores all individual frames. Use MakingLabeledVideo_fast.py instead for faster (and slightly different) version (frames are not stored).
python3 MakingLabeledVideoTrial.py video_file
Note: run python3 AnalyzeVideos.py first.
"Functionalized by Adam S. Lowet, 10/28/19, to run in parallel"
"""
####################################################
# Dependencies
####################################################
import os.path
import sys
subfolder = os.getcwd().split('analysis-tools')[0]
sys.path.append(subfolder)
# add parent directory: (where nnet & config are!)
sys.path.append(os.path.join(subfolder, "pose-tensorflow"))
sys.path.append(os.path.join(subfolder, "config"))
# Dependencies for video:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import imageio
from skimage.util import img_as_ubyte
from moviepy.editor import VideoFileClip
import subprocess
import pandas as pd
import numpy as np
from tqdm import tqdm
import os
import glob
import auxiliaryfunctions
####################################################
# Loading descriptors of model
####################################################
from myconfig_analysis import cropping, Task, date, \
resnet, shuffle, trainingsiterations, pcutoff, deleteindividualframes, x1, x2, y1, y2, videotype, alphavalue, dotsize, colormap
from myconfig_analysis import scorer as humanscorer
# loading meta data / i.e. training & test files
#basefolder = os.path.join('..','pose-tensorflow','models')
#datafolder = os.path.join(basefolder , "UnaugmentedDataSet_" + Task + date)
#Data = pd.read_hdf(os.path.join(datafolder , 'data-' + Task , 'CollectedData_' + humanscorer + '.h5'),'df_with_missing')
# Name for scorer based on passed on parameters from myconfig_analysis. Make sure they refer to the network of interest.
scorer = 'DeepCut' + "_resnet" + str(resnet) + "_" + Task + str(
date) + 'shuffle' + str(shuffle) + '_' + str(trainingsiterations)
####################################################
# Auxiliary function
####################################################
# https://stackoverflow.com/questions/14720331/how-to-generate-random-colors-in-matplotlib
def get_cmap(n, name=colormap):
'''Returns a function that maps each index in 0, 1, ..., n-1 to a distinct
RGB color; the keyword argument name must be a standard mpl colormap name.'''
return plt.cm.get_cmap(name, n)
def CreateVideo(clip, Dataframe):
''' Creating individual frames with labeled body parts and making a video'''
scorer = np.unique(Dataframe.columns.get_level_values(0))[0]
bodyparts2plot = list(np.unique(Dataframe.columns.get_level_values(1)))
colors = get_cmap(len(bodyparts2plot))
ny, nx = clip.size # dimensions of frame (height, width)
fps = clip.fps
nframes = len(Dataframe.index)
if cropping:
# one might want to adjust
clip = clip.crop(y1=y1, y2=y2, x1=x1, x2=x2)
clip.reader.initialize()
print("Duration of video [s]: ", clip.duration, ", recorded with ",
fps, "fps!")
print("Overall # of frames: ", nframes,
"with cropped frame dimensions: ", clip.size)
print("Generating frames")
for index in tqdm(range(nframes)):
imagename = tmpfolder + "/file%04d.png" % index
if os.path.isfile(tmpfolder + "/file%04d.png" % index):
pass
else:
plt.axis('off')
image = img_as_ubyte(clip.reader.read_frame())
#image = img_as_ubyte(clip.get_frame(index * 1. / clip.fps))
if np.ndim(image) > 2:
h, w, nc = np.shape(image)
else:
h, w = np.shape(image)
plt.figure(frameon=False, figsize=(w * 1. / 100, h * 1. / 100))
plt.subplots_adjust(left=0,
bottom=0,
right=1,
top=1,
wspace=0,
hspace=0)
plt.imshow(image)
for bpindex, bp in enumerate(bodyparts2plot):
if Dataframe[scorer][bp]['likelihood'].values[
index] > pcutoff:
plt.scatter(Dataframe[scorer][bp]['x'].values[index],
Dataframe[scorer][bp]['y'].values[index],
s=dotsize**2,
color=colors(bpindex),
alpha=alphavalue)
plt.xlim(0, w)
plt.ylim(0, h)
plt.axis('off')
plt.subplots_adjust(left=0,
bottom=0,
right=1,
top=1,
wspace=0,
hspace=0)
plt.gca().invert_yaxis()
plt.savefig(imagename)
plt.close("all")
os.chdir(tmpfolder)
print("Generating video")
subprocess.call([
'ffmpeg', '-framerate',
str(clip.fps), '-i', 'file%04d.png', '-r', '30',
'../' + vname + '_DeepLabCutlabeled.mp4'
])
if deleteindividualframes:
for file_name in glob.glob("*.png"):
os.remove(file_name)
os.chdir("../")
##################################################
# Datafolder
##################################################
# video_dir='../videos/' #where your folder with videos is.
video_dir = os.path.dirname(video_file)
os.chdir(video_dir)
#videos = np.sort([fn for fn in os.listdir(os.curdir) if (videotype in fn) and ("labeled" not in fn)])
#print("Starting ", video_dir, videos)
#for video in videos:
video = os.path.basename(video_file)
vname = video.split('.')[0]
# fails here as originally written because vname is full path. Get basename!
tmpfolder = 'temp' + vname
auxiliaryfunctions.attempttomakefolder(tmpfolder)
if os.path.isfile(os.path.join(tmpfolder,
vname + '_DeepLabCutlabeled.mp4')):
print("Labeled video already created.")
else:
print("Loading ", video, "and data.")
dataname = video.split('.')[0] + scorer + '.h5'
try: # to load data for this video + scorer
Dataframe = pd.read_hdf(dataname)
clip = VideoFileClip(video)
CreateVideo(clip, Dataframe)
except FileNotFoundError:
datanames = [
fn for fn in os.listdir(os.curdir)
if (vname in fn) and (".h5" in fn) and "resnet" in fn
]
if len(datanames) == 0:
print("The video was not analyzed with this scorer:", scorer)
print(
"No other scorers were found, please run AnalysisVideos.py first."
)
elif len(datanames) > 0:
print("The video was not analyzed with this scorer:", scorer)
print("Other scorers were found, however:", datanames)
print("Creating labeled video for:", datanames[0], " instead.")
Dataframe = pd.read_hdf(datanames[0])
clip = VideoFileClip(video)
CreateVideo(clip, Dataframe)
import sys
sys.path.append(os.getcwd().split('Generating_a_Training_Set')[0])
from myconfig import Task, vidpath, filename, x1, x2, y1, y2, portion, \
Scorers, bodyparts, date, cropping
import auxiliaryfunctions
# check global variables:
print(Task)
print(filename)
print(bodyparts)
print(Scorers)
print(date)
basefolder = 'data-' + Task + '/'
auxiliaryfunctions.attempttomakefolder(basefolder)
#####################################################################
# First load the image and crop (if necessary).
#####################################################################
# Number of frames to pick (set this to 0 until you found right cropping)
numframes2pick = 10
clip = VideoFileClip(vidpath + '/' + filename)
print("Duration of video [s], ", clip.duration, "fps, ", clip.fps,
"Cropped frame dimensions: ", clip.size)
ny, nx = clip.size # dimensions of frame (width, height)
# Select ROI of interest by adjusting values in myconfig.py
MatlabData = np.array([(np.array(
[data[item]['image']], dtype='U'), np.array(
[data[item]['size']]), boxitintoacell(data[item]['joints']))
for item in range(len(data))],
dtype=DTYPE)
sio.savemat(bf + filename_matfile + '.mat', {'dataset': MatlabData})
################################################################################
# Creating file structure for training &
# Test files as well as pose_yaml files (containing training and testing information)
#################################################################################
experimentname = Task + date + '-trainset' + str(
int(trainFraction * 100)) + 'shuffle' + str(shuffle)
auxiliaryfunctions.attempttomakefolder(experimentname)
auxiliaryfunctions.attempttomakefolder(experimentname + '/train')
auxiliaryfunctions.attempttomakefolder(experimentname + '/test')
items2change = {
"dataset": basefolder + filename_matfile + '.mat',
"num_joints": len(bodyparts),
"all_joints": [[i] for i in range(len(bodyparts))],
"all_joints_names": bodyparts
}
trainingdata = MakeTrain_pose_yaml(items2change,
experimentname + '/train/' +
'pose_cfg.yaml',
filename='pose_cfg.yaml')
keys2save = [
import sys
import cv2
sys.path.append(os.getcwd().split('Generating_a_Training_Set')[0])
from myconfig import Task, vidpath, filename, x1, x2, y1, y2, portion, \
Scorers, bodyparts, date, cropping
import auxiliaryfunctions
# check global variables:
print(Task)
print(filename)
print(bodyparts)
print(Scorers)
print(date)
basefolder = 'data-' + Task
auxiliaryfunctions.attempttomakefolder(basefolder)
#####################################################################
# First load the image and crop (if necessary).
#####################################################################
# Number of frames to pick (set this to 0 until you found right cropping)
numframes2pick = 100
clip = VideoFileClip(os.path.join(vidpath, filename))
print("Duration of video [s], ", clip.duration, "fps, ", clip.fps,
"Cropped frame dimensions: ", clip.size)
ny, nx = clip.size # dimensions of frame (width, height)
# Select ROI of interest by adjusting values in myconfig.py
trainerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[trainIndexes].values.flatten())
print("Results for", trainingiterations, "training iterations:",
int(100 * trainFraction), shuffle, "train error:",
np.round(trainerror, 2), "pixels. Test error:",
np.round(testerror, 2), " pixels.")
print("With pcutoff of", pcutoff, " train error:",
np.round(trainerrorpcutoff, 2), "pixels. Test error:",
np.round(testerrorpcutoff, 2), "pixels")
print(
"Thereby, the errors are given by the average distances between the labels by DLC and the scorer."
)
if plotting == True:
foldername = os.path.join('LabeledImages_' + scorer_machine)
auxiliaryfunctions.attempttomakefolder(foldername)
NumFrames = np.size(DataCombined.index)
for ind in np.arange(NumFrames):
fn = DataCombined.index[ind]
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
MakeLabeledImage(DataCombined, ind,
os.path.join(datafolder, 'data-' + Task),
[scorer, scorer_machine],
comparisonbodyparts, colors)
if ind in trainIndexes:
plt.savefig(
os.path.join(
foldername, 'TrainingImg' + str(ind) + '_' +
fn.split('/')[0] + '_' + fn.split('/')[1]))
