def SelectFrames(videopath, filename, x1, x2, y1, y2, cropping, videotype, start, stop, Task, selectionalgorithm): ''' Selecting frames from videos for labeling.''' if start > 1.0 or stop > 1.0 or start < 0 or stop < 0 or start >= stop: print( "Please change start & stop, they should form a normalized interval with 0<= start < stop<=1." ) else: basefolder = 'data-' + Task + '/' auxiliaryfunctions.attempttomakefolder(basefolder) videos = auxiliaryfunctions.GetVideoList(filename, videopath, videotype) for vindex, video in enumerate(videos): print("Loading ", video) clip = VideoFileClip(os.path.join(videopath, video)) print("Duration of video [s], ", clip.duration, "fps, ", clip.fps, "Cropped frame dimensions: ", clip.size) #################################################### # Creating folder with name of experiment and extract random frames #################################################### folder = video.split('.')[0] auxiliaryfunctions.attempttomakefolder( os.path.join(basefolder, folder)) indexlength = int(np.ceil(np.log10(clip.duration * clip.fps))) # Extract the first frame (not cropped!) - useful for data augmentation index = 0 image = img_as_ubyte(clip.get_frame(index * 1. / clip.fps)) io.imsave( os.path.join(basefolder, folder, "img" + str(index).zfill(indexlength) + ".png"), image) if cropping == True: # Select ROI of interest by adjusting values in myconfig.py clip = clip.crop(y1=y1, y2=y2, x1=x1, x2=x2) print("Extracting frames ...") if selectionalgorithm == 'uniform': frames2pick = frameselectiontools.UniformFrames( clip, numframes2pick, start, stop) elif selectionalgorithm == 'kmeans': frames2pick = frameselectiontools.KmeansbasedFrameselection( clip, numframes2pick, start, stop) else: print( "Please implement this method yourself and send us a pull request!" ) frames2pick = [] indexlength = int(np.ceil(np.log10(clip.duration * clip.fps))) for index in frames2pick: try: image = img_as_ubyte(clip.get_frame(index * 1. / clip.fps)) io.imsave( os.path.join( basefolder, folder, "img" + str(index).zfill(indexlength) + ".png"), image) except FileNotFoundError: print("Frame # ", index, " does not exist.")
image = io.imread(os.path.join(models_folder_path, unaugmented_folder_name, 'data-' + Task, imagename), mode='RGB') image = skimage.color.gray2rgb(image) image_batch = data_to_input(image) # Compute prediction with the CNN outputs_np = sess.run(outputs, feed_dict={inputs: image_batch}) scmap, locref = predict.extract_cnn_output(outputs_np, cfg) # Extract maximum scoring location from the heatmap, assume 1 person pose = predict.argmax_pose_predict(scmap, locref, cfg.stride) PredicteData[imageindex, :] = pose.flatten( ) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts! index = pd.MultiIndex.from_product( [[DLCscorer], cfg['all_joints_names'], ['x', 'y', 'likelihood']], names=['scorer', 'bodyparts', 'coords']) # Saving results: auxiliaryfunctions.attempttomakefolder("Results") DataMachine = pd.DataFrame(PredicteData, columns=index, index=Data.index.values) DataMachine.to_hdf(os.path.join("Results", DLCscorer + '.h5'), 'df_with_missing', format='table', mode='w') print("Done and results stored for snapshot: ", Snapshots[snapshotIndex])
date) + 'shuffle' + str(shuffle) + '_' + str(trainingsiterations) ################################################## # Datafolder ################################################## # videofolder='../videos/' #where your folder with videos is. os.chdir(videofolder) videos = np.sort([fn for fn in os.listdir(os.curdir) if (".mov" in fn)]) print("Starting ", videofolder, videos) for video in videos: vname = video.split('.')[0] tmpfolder = 'temp' + vname auxiliaryfunctions.attempttomakefolder(tmpfolder) if os.path.isfile(tmpfolder + '/' + vname + '.mp4'): print("Labeled video already created.") else: print("Loading ", video, "and data.") dataname = video.split('.')[0] + scorer + '.h5' try: Dataframe = pd.read_hdf(dataname) clip = VideoFileClip(video) except FileNotFoundError: print("Data was not analyzed (run AnalysisVideos.py first).") ny, nx = clip.size # dimensions of frame (height, width) fps = clip.fps nframes = np.sum(1 for j in clip.iter_frames())
def MakingLabeledVideoTrial(video_file): """ DeepLabCut Toolbox https://github.com/AlexEMG/DeepLabCut A Mathis, [email protected] M Mathis, [email protected] This script labels the bodyparts in videos as analzyed by "AnalyzeVideos.py". This code is relatively slow as it stores all individual frames. Use MakingLabeledVideo_fast.py instead for faster (and slightly different) version (frames are not stored). python3 MakingLabeledVideoTrial.py video_file Note: run python3 AnalyzeVideos.py first. "Functionalized by Adam S. Lowet, 10/28/19, to run in parallel" """ #################################################### # Dependencies #################################################### import os.path import sys subfolder = os.getcwd().split('analysis-tools')[0] sys.path.append(subfolder) # add parent directory: (where nnet & config are!) sys.path.append(os.path.join(subfolder, "pose-tensorflow")) sys.path.append(os.path.join(subfolder, "config")) # Dependencies for video: import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import imageio from skimage.util import img_as_ubyte from moviepy.editor import VideoFileClip import subprocess import pandas as pd import numpy as np from tqdm import tqdm import os import glob import auxiliaryfunctions #################################################### # Loading descriptors of model #################################################### from myconfig_analysis import cropping, Task, date, \ resnet, shuffle, trainingsiterations, pcutoff, deleteindividualframes, x1, x2, y1, y2, videotype, alphavalue, dotsize, colormap from myconfig_analysis import scorer as humanscorer # loading meta data / i.e. training & test files #basefolder = os.path.join('..','pose-tensorflow','models') #datafolder = os.path.join(basefolder , "UnaugmentedDataSet_" + Task + date) #Data = pd.read_hdf(os.path.join(datafolder , 'data-' + Task , 'CollectedData_' + humanscorer + '.h5'),'df_with_missing') # Name for scorer based on passed on parameters from myconfig_analysis. Make sure they refer to the network of interest. scorer = 'DeepCut' + "_resnet" + str(resnet) + "_" + Task + str( date) + 'shuffle' + str(shuffle) + '_' + str(trainingsiterations) #################################################### # Auxiliary function #################################################### # https://stackoverflow.com/questions/14720331/how-to-generate-random-colors-in-matplotlib def get_cmap(n, name=colormap): '''Returns a function that maps each index in 0, 1, ..., n-1 to a distinct RGB color; the keyword argument name must be a standard mpl colormap name.''' return plt.cm.get_cmap(name, n) def CreateVideo(clip, Dataframe): ''' Creating individual frames with labeled body parts and making a video''' scorer = np.unique(Dataframe.columns.get_level_values(0))[0] bodyparts2plot = list(np.unique(Dataframe.columns.get_level_values(1))) colors = get_cmap(len(bodyparts2plot)) ny, nx = clip.size # dimensions of frame (height, width) fps = clip.fps nframes = len(Dataframe.index) if cropping: # one might want to adjust clip = clip.crop(y1=y1, y2=y2, x1=x1, x2=x2) clip.reader.initialize() print("Duration of video [s]: ", clip.duration, ", recorded with ", fps, "fps!") print("Overall # of frames: ", nframes, "with cropped frame dimensions: ", clip.size) print("Generating frames") for index in tqdm(range(nframes)): imagename = tmpfolder + "/file%04d.png" % index if os.path.isfile(tmpfolder + "/file%04d.png" % index): pass else: plt.axis('off') image = img_as_ubyte(clip.reader.read_frame()) #image = img_as_ubyte(clip.get_frame(index * 1. / clip.fps)) if np.ndim(image) > 2: h, w, nc = np.shape(image) else: h, w = np.shape(image) plt.figure(frameon=False, figsize=(w * 1. / 100, h * 1. / 100)) plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0) plt.imshow(image) for bpindex, bp in enumerate(bodyparts2plot): if Dataframe[scorer][bp]['likelihood'].values[ index] > pcutoff: plt.scatter(Dataframe[scorer][bp]['x'].values[index], Dataframe[scorer][bp]['y'].values[index], s=dotsize**2, color=colors(bpindex), alpha=alphavalue) plt.xlim(0, w) plt.ylim(0, h) plt.axis('off') plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0) plt.gca().invert_yaxis() plt.savefig(imagename) plt.close("all") os.chdir(tmpfolder) print("Generating video") subprocess.call([ 'ffmpeg', '-framerate', str(clip.fps), '-i', 'file%04d.png', '-r', '30', '../' + vname + '_DeepLabCutlabeled.mp4' ]) if deleteindividualframes: for file_name in glob.glob("*.png"): os.remove(file_name) os.chdir("../") ################################################## # Datafolder ################################################## # video_dir='../videos/' #where your folder with videos is. video_dir = os.path.dirname(video_file) os.chdir(video_dir) #videos = np.sort([fn for fn in os.listdir(os.curdir) if (videotype in fn) and ("labeled" not in fn)]) #print("Starting ", video_dir, videos) #for video in videos: video = os.path.basename(video_file) vname = video.split('.')[0] # fails here as originally written because vname is full path. Get basename! tmpfolder = 'temp' + vname auxiliaryfunctions.attempttomakefolder(tmpfolder) if os.path.isfile(os.path.join(tmpfolder, vname + '_DeepLabCutlabeled.mp4')): print("Labeled video already created.") else: print("Loading ", video, "and data.") dataname = video.split('.')[0] + scorer + '.h5' try: # to load data for this video + scorer Dataframe = pd.read_hdf(dataname) clip = VideoFileClip(video) CreateVideo(clip, Dataframe) except FileNotFoundError: datanames = [ fn for fn in os.listdir(os.curdir) if (vname in fn) and (".h5" in fn) and "resnet" in fn ] if len(datanames) == 0: print("The video was not analyzed with this scorer:", scorer) print( "No other scorers were found, please run AnalysisVideos.py first." ) elif len(datanames) > 0: print("The video was not analyzed with this scorer:", scorer) print("Other scorers were found, however:", datanames) print("Creating labeled video for:", datanames[0], " instead.") Dataframe = pd.read_hdf(datanames[0]) clip = VideoFileClip(video) CreateVideo(clip, Dataframe)
import sys sys.path.append(os.getcwd().split('Generating_a_Training_Set')[0]) from myconfig import Task, vidpath, filename, x1, x2, y1, y2, portion, \ Scorers, bodyparts, date, cropping import auxiliaryfunctions # check global variables: print(Task) print(filename) print(bodyparts) print(Scorers) print(date) basefolder = 'data-' + Task + '/' auxiliaryfunctions.attempttomakefolder(basefolder) ##################################################################### # First load the image and crop (if necessary). ##################################################################### # Number of frames to pick (set this to 0 until you found right cropping) numframes2pick = 10 clip = VideoFileClip(vidpath + '/' + filename) print("Duration of video [s], ", clip.duration, "fps, ", clip.fps, "Cropped frame dimensions: ", clip.size) ny, nx = clip.size # dimensions of frame (width, height) # Select ROI of interest by adjusting values in myconfig.py
MatlabData = np.array([(np.array( [data[item]['image']], dtype='U'), np.array( [data[item]['size']]), boxitintoacell(data[item]['joints'])) for item in range(len(data))], dtype=DTYPE) sio.savemat(bf + filename_matfile + '.mat', {'dataset': MatlabData}) ################################################################################ # Creating file structure for training & # Test files as well as pose_yaml files (containing training and testing information) ################################################################################# experimentname = Task + date + '-trainset' + str( int(trainFraction * 100)) + 'shuffle' + str(shuffle) auxiliaryfunctions.attempttomakefolder(experimentname) auxiliaryfunctions.attempttomakefolder(experimentname + '/train') auxiliaryfunctions.attempttomakefolder(experimentname + '/test') items2change = { "dataset": basefolder + filename_matfile + '.mat', "num_joints": len(bodyparts), "all_joints": [[i] for i in range(len(bodyparts))], "all_joints_names": bodyparts } trainingdata = MakeTrain_pose_yaml(items2change, experimentname + '/train/' + 'pose_cfg.yaml', filename='pose_cfg.yaml') keys2save = [
import sys import cv2 sys.path.append(os.getcwd().split('Generating_a_Training_Set')[0]) from myconfig import Task, vidpath, filename, x1, x2, y1, y2, portion, \ Scorers, bodyparts, date, cropping import auxiliaryfunctions # check global variables: print(Task) print(filename) print(bodyparts) print(Scorers) print(date) basefolder = 'data-' + Task auxiliaryfunctions.attempttomakefolder(basefolder) ##################################################################### # First load the image and crop (if necessary). ##################################################################### # Number of frames to pick (set this to 0 until you found right cropping) numframes2pick = 100 clip = VideoFileClip(os.path.join(vidpath, filename)) print("Duration of video [s], ", clip.duration, "fps, ", clip.fps, "Cropped frame dimensions: ", clip.size) ny, nx = clip.size # dimensions of frame (width, height) # Select ROI of interest by adjusting values in myconfig.py
trainerrorpcutoff = np.nanmean( RMSEpcutoff.iloc[trainIndexes].values.flatten()) print("Results for", trainingiterations, "training iterations:", int(100 * trainFraction), shuffle, "train error:", np.round(trainerror, 2), "pixels. Test error:", np.round(testerror, 2), " pixels.") print("With pcutoff of", pcutoff, " train error:", np.round(trainerrorpcutoff, 2), "pixels. Test error:", np.round(testerrorpcutoff, 2), "pixels") print( "Thereby, the errors are given by the average distances between the labels by DLC and the scorer." ) if plotting == True: foldername = os.path.join('LabeledImages_' + scorer_machine) auxiliaryfunctions.attempttomakefolder(foldername) NumFrames = np.size(DataCombined.index) for ind in np.arange(NumFrames): fn = DataCombined.index[ind] fig = plt.figure() ax = fig.add_subplot(1, 1, 1) MakeLabeledImage(DataCombined, ind, os.path.join(datafolder, 'data-' + Task), [scorer, scorer_machine], comparisonbodyparts, colors) if ind in trainIndexes: plt.savefig( os.path.join( foldername, 'TrainingImg' + str(ind) + '_' + fn.split('/')[0] + '_' + fn.split('/')[1]))