def process_motion_tracking(self, config, create_labelled_video=False): """ Run DeepLabCut motion tracking on behavioural videos. """ # bloated so imported when needed import deeplabcut # pylint: disable=import-error self.extract_videos() config = Path(config).expanduser() if not config.exists(): raise PixelsError(f"Config at {config} not found.") for recording in self.files: if 'camera_data' in recording: video = self.interim / recording['camera_data'].with_suffix( '.avi') if not video.exists(): raise PixelsError( f"Path {video} should exist but doesn't... discuss.") deeplabcut.analyze_videos(config, [video]) deeplabcut.plot_trajectories(config, [video]) if create_labelled_video: deeplabcut.create_labeled_video(config, [video])
def generate_track_h5(config_path,untracked_videos,suffix=".mp4"): os.environ["DLClight"]='True' # all the child process have this env but not its father process if (platform.system()=="Linux"): import deeplabcut deeplabcut.analyze_videos(config_path,untracked_videos,shuffle=1,save_as_csv=True,videotype=suffix) deeplabcut.plot_trajectories(config_path,untracked_videos) deeplabcut.create_labeled_video(config_path,untracked_videos)
def analyze_videos(self,event): shuffle = self.shuffle.GetValue() trainingsetindex = self.trainingset.GetValue() if self.csv.GetStringSelection() == "Yes": save_as_csv = True else: save_as_csv = False if self.cfg['cropping']: crop = self.cfg['x1'], self.cfg['x2'], self.cfg['y1'], self.cfg['y2'] else: crop = None if self.dynamic.GetStringSelection() == "No": dynamic = (False, .5, 10) else: dynamic = (True, .5, 10) if self.filter.GetStringSelection() == "No": filter = None else: filter = True # if self.draw_skeleton.GetStringSelection() == "Yes": # self.draw = True # else: # self.draw = True # print(self.config,self.filelist,self.videotype.GetValue(),shuffle,trainingsetindex,gputouse=None,save_as_csv=save_as_csv,destfolder=self.destfolder,cropping=cropping) deeplabcut.analyze_videos(self.config, self.filelist, videotype=self.videotype.GetValue(), shuffle=shuffle, trainingsetindex=trainingsetindex, gputouse=None, save_as_csv=save_as_csv, destfolder=self.destfolder, crop=crop, dynamic=dynamic) if self.filter.GetStringSelection() == "Yes": deeplabcut.filterpredictions(self.config, self.filelist, videotype=self.videotype.GetValue(), shuffle=shuffle, trainingsetindex=trainingsetindex, filtertype='median', windowlength=5, p_bound=0.001, ARdegree=3, MAdegree=1, alpha=0.01, save_as_csv=True, destfolder=self.destfolder) if self.create_labeled_videos.GetStringSelection() == "Yes": deeplabcut.create_labeled_video(self.config,self.filelist,self.videotype.GetValue(),shuffle=shuffle, trainingsetindex=trainingsetindex, draw_skeleton= self.draw,trailpoints = self.trail_points.GetValue(), filtered=True) if self.trajectory.GetStringSelection() == "Yes": deeplabcut.plot_trajectories(self.config, self.filelist, displayedbodyparts=self.bodyparts, videotype=self.videotype.GetValue(), shuffle=shuffle, trainingsetindex=trainingsetindex, filtered=True, showfigures=False, destfolder=self.destfolder)
def runDeepLabCut(Inputfilepath,OutputFilepath): '''Function inputs are filepath to videos to be tracked by DLC and the folder to save the output to Videos are copied to output folder, than processed in DLC based on the dlc config path DLC output is saved in outputfilepath and the output is also converted to npy and saved as well ''' #####################Copy Videos to DLC Folder############ for dir in [Inputfilepath]:#Iterates through input folder for video in os.listdir(dir):#Iterates through each video in folder #ffmpeg call to copy videos to dlc folder subprocess.call(['ffmpeg', '-i', Inputfilepath+'/'+video, OutputFilepath+'/'+video]) #################### DeepLabCut ############################ for dir in [OutputFilepath]:# Loop through dlc folder for video in os.listdir(dir): #Analyze the videos through deeplabcut deeplabcut.analyze_videos(baseProjectPath+'/'+DLCconfigPath, [OutputFilepath +'/'+ video], save_as_csv=True) deeplabcut.plot_trajectories(baseProjectPath+'/'+DLCconfigPath,[OutputFilepath +'/'+ video]) for dir in [OutputFilepath]:#Loop through dlc folder for video in dir:# for each video in folder #Create a DLC video deeplabcut.create_labeled_video(baseProjectPath+'/'+DLCconfigPath, glob.glob(os.path.join(OutputFilepath ,'*mp4'))) #If there is not a folder for dlc npy output, create one if not os.path.exists(OutputFilepath + 'DLCnpy'): os.mkdir(OutputFilepath+ 'DLCnpy') #Load all dlc csv output files csvfiles = glob.glob(OutputFilepath+'/*csv') #For loop gets csv data from all cameras j=0 for data in csvfiles: datapoints = pd.read_csv(data) # read in the csv data print(datapoints) parsedDlcData = datapoints.iloc[3:,7:10].values#the last element in the array is the P value #print(parsedDlcData) print(parsedDlcData) np.save(OutputFilepath+'/DLCnpy/dlc_'+cam_names[j]+'.npy',parsedDlcData)#Save data j+=1
print("<<<<<<<") import concurrent.futures #videolists = glob.glob("/gpfsdata/home/sqiu/data/video/novel_context/20180704/asf/*asf") #videolists = glob.glob("/gpfsdata/home/sqiu/data/video/novel_context/20180706_chs/asf/*asf") videolists = glob.glob( "/gpfsdata/home/sqiu/data/video/stage_1_training/*/*.mp4") if len(videolists) == 0: print("there is no video selected") sys.exit() else: print(videolists) config_path = r'/gpfsdata/home/sqiu/job/dlc/linear_track_40cm-QS-2019-09-06/config.yaml' def task(gpuNo): deeplabcut.analyze_videos(config_path, videolists, shuffle=1, save_as_csv=True, videotype=".mp4", gputouse=gpuNo) with concurrent.futures.ProcessPoolExecutor() as executor: for i, _ in enumerate(executor.map(task, [0, 1, 2]), 1): print(i, len(videolists)) #deeplabcut.analyze_videos(config_path,videolists,shuffle=1,save_as_csv=True,videotype=".mp4",gputouse=0) deeplabcut.plot_trajectories(config_path, videolists) deeplabcut.create_labeled_video(config_path, videolists)
DLC_config = deeplabcut.auxiliaryfunctions.edit_config(posefile, edits) print("TRAIN NETWORK", shuffle) deeplabcut.train_network( path_config_file, shuffle=shuffle, saveiters=10000, displayiters=200, maxiters=maxiters, max_snapshots_to_keep=11, ) print("EVALUATE") deeplabcut.evaluate_network(path_config_file, Shuffles=[shuffle], plotting=True) print("Analyze Video") videofile_path = os.path.join(os.getcwd(), "openfield-Pranav-2018-10-30", "videos", "m3v1mp4.mp4") deeplabcut.analyze_videos(path_config_file, [videofile_path], shuffle=shuffle) print("Create Labeled Video and plot") deeplabcut.create_labeled_video(path_config_file, [videofile_path], shuffle=shuffle) deeplabcut.plot_trajectories(path_config_file, [videofile_path], shuffle=shuffle)
outpath=os.path.join( cfg['project_path'], 'videos')) vname = Path(newvideo).stem deeplabcut.analyze_videos(path_config_file, [newvideo], shuffle=shuffle, save_as_csv=True, destfolder=dfolder) print("CREATE VIDEO") deeplabcut.create_labeled_video(path_config_file, [newvideo], shuffle=shuffle, destfolder=dfolder) print("Making plots") deeplabcut.plot_trajectories(path_config_file, [newvideo], shuffle=shuffle, destfolder=dfolder) print("EXTRACT OUTLIERS") deeplabcut.extract_outlier_frames(path_config_file, [newvideo], shuffle=shuffle, outlieralgorithm='jump', epsilon=0, automatic=True, destfolder=dfolder) file = os.path.join(cfg['project_path'], 'labeled-data', vname, "machinelabels-iter" + str(cfg['iteration']) + '.h5') print("RELABELING") DF = pd.read_hdf(file, 'df_with_missing') DLCscorer = np.unique(DF.columns.get_level_values(0))[0]
def analyze_videos(self, event): shuffle = self.shuffle.GetValue() trainingsetindex = self.trainingset.GetValue() if self.cfg.get("multianimalproject", False): print("Analyzing ... ") else: if self.csv.GetStringSelection() == "Yes": save_as_csv = True else: save_as_csv = False if self.dynamic.GetStringSelection() == "No": dynamic = (False, 0.5, 10) else: dynamic = (True, 0.5, 10) if self.filter.GetStringSelection() == "No": filter = None else: filter = True if self.cfg["cropping"] == "True": crop = self.cfg["x1"], self.cfg["x2"], self.cfg["y1"], self.cfg[ "y2"] else: crop = None if self.cfg.get("multianimalproject", False): if self.robust.GetStringSelection() == "No": robust = False else: robust = True scorername = deeplabcut.analyze_videos( self.config, self.filelist, videotype=self.videotype.GetValue(), shuffle=shuffle, trainingsetindex=trainingsetindex, gputouse=None, cropping=crop, robust_nframes=robust, ) if self.create_video_with_all_detections.GetStringSelection( ) == "Yes": trainFrac = self.cfg["TrainingFraction"][trainingsetindex] scorername, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( self.cfg, shuffle, trainFraction=trainFrac) print(scorername) deeplabcut.create_video_with_all_detections( self.config, self.filelist, DLCscorername=scorername) else: scorername = deeplabcut.analyze_videos( self.config, self.filelist, videotype=self.videotype.GetValue(), shuffle=shuffle, trainingsetindex=trainingsetindex, gputouse=None, save_as_csv=save_as_csv, cropping=crop, dynamic=dynamic, ) if self.filter.GetStringSelection() == "Yes": deeplabcut.filterpredictions( self.config, self.filelist, videotype=self.videotype.GetValue(), shuffle=shuffle, trainingsetindex=trainingsetindex, filtertype="median", windowlength=5, save_as_csv=True, ) if self.trajectory.GetStringSelection() == "Yes": if self.showfigs.GetStringSelection() == "No": showfig = False else: showfig = True deeplabcut.plot_trajectories( self.config, self.filelist, displayedbodyparts=self.bodyparts, videotype=self.videotype.GetValue(), shuffle=shuffle, trainingsetindex=trainingsetindex, filtered=True, showfigures=showfig, )
def runOPandDLC(): #Set directory, and original filepath os.chdir("/Windows/system32") origfilepath = baseProjectPath + '/' + subject + '/' + sessionID rawfilepath = baseProjectPath + '/' + subject + '/' + sessionID + '/Raw' #Create main filepath for Intermediate processed if not os.path.exists(origfilepath + '/Intermediate'): os.mkdir(origfilepath + '/Intermediate') interfilepath = origfilepath + '/Intermediate' if not os.path.exists(origfilepath + '/Processed'): os.mkdir(origfilepath + '/Processed') datadir1 = [rawfilepath + '/' + rawVideoFolder] #Create a folder for the resized videos if not os.path.exists(interfilepath + '/Resized'): os.mkdir(interfilepath + '/Resized') filepath1 = interfilepath + '/Resized' datadir2 = [filepath1] #Create a folder for the resized videos if not os.path.exists(interfilepath + '/Undistorted'): os.mkdir(interfilepath + '/Undistorted') filepath2 = interfilepath + '/Undistorted' datadir3 = [filepath2] #Create a folder for the deeplabcut output if not os.path.exists(interfilepath + '/DeepLabCut'): os.mkdir(interfilepath + '/DeepLabCut') filepath3 = interfilepath + '/DeepLabCut' datadir4 = [filepath3] #Create a folder for the openpose output if not os.path.exists(interfilepath + '/OpenPose'): os.mkdir(interfilepath + '/OpenPose') filepath4 = interfilepath + '/OpenPose' #Create a folder for videos if not os.path.exists(interfilepath + '/VideoOutput'): os.mkdir(interfilepath + '/VideoOutput') filepath5 = interfilepath + '/VideoOutput' for dir in datadir1: for video in os.listdir(dir): subprocess.call([ 'ffmpeg', '-i', rawfilepath + '/' + rawVideoFolder + '/' + video, '-vf', 'scale=1280:960', filepath1 + '/' + video ]) #Use ffmpeg to resize videos and save them in the just created resized video folder #camBvids = open(filepath1+'/camBvids.txt','a') #camCvids = open(filepath1+'/camCvids.txt','a') #camDvids = open(filepath1+'/camDvids.txt','a') #if num_of_Video_parts >1 : # for dir in datadir1: # for video in os.listdir(dir): # if video[:4] == 'CamA': # camAvids = open(filepath1+'/camAvids.txt','a') # camAvids.write('CamA') # if video[:4] == 'CamB': # camBvids.write(filepath1+'/'+video) # if video[:4] == 'CamC': # camCvids.write(filepath1+'/'+video) # if video[:4] == 'CamD': # camDvids.write(filepath1+'/'+video) # subprocess.call(['ffmpeg', '-f', 'concat', '-safe', '0', '-i', filepath1+'/camAvids.txt', '-c' ,'copy' ,filepath1+'/CamA.mp4']) #subprocess.call(['ffmpeg', '-f', 'concat', '-safe', '0', '-i', filepath1+'/camBvids.txt', '-c' ,'copy' ,filepath1+'/CamB.mp4']) #subprocess.call(['ffmpeg', '-f', 'concat', '-safe', '0', '-i', filepath1+'/camCvids.txt', '-c' ,'copy' ,filepath1+'/CamC.mp4']) #subprocess.call(['ffmpeg', '-f', 'concat', '-safe', '0', '-i', filepath1+'/camDvids.txt', '-c' ,'copy' ,filepath1+'/CamD.mp4']) #Use ffmpeg to Undistort videos for dir in datadir2: for video in os.listdir(dir): subprocess.call([ 'ffmpeg', '-i', filepath1 + '/' + video, '-vf', "lenscorrection=cx=0.5:cy=0.5:k1=-.115:k2=-0.022", filepath2 + '/' + video ]) #Use deeplabcut to analyze videos and save the results to the folder for processed videos for dir in datadir3: for video in os.listdir(dir): deeplabcut.analyze_videos(baseProjectPath + '/' + DLCconfigPath, [filepath2 + '/' + video], videotype='mp4', destfolder=filepath3, save_as_csv=True) deeplabcut.plot_trajectories(baseProjectPath + '/' + DLCconfigPath, [filepath2 + '/' + video], videotype='mp4', destfolder=filepath3) # deeplabcut.create_labeled_video(DLCconfigPath,[filepath2 +'/'+ video],videotype = 'mp4', destfolder = filepath5) #Change directory to openpose and run openpose on the videos then save the results to the processed video folder os.chdir("/Users/MatthisLab/openpose") for dir in datadir3: for video in os.listdir(dir): videoName = video[:4] subprocess.call([ 'bin/OpenPoseDemo.exe', '--video', filepath2 + '/' + video, '--hand', '--face', '--write_video', filepath5 + '/OpenPose' + videoName + '.avi', '--write_json', filepath4 + '/' + videoName ])
# In[14]: deeplabcut.evaluate_network(path_config) # In[15]: deeplabcut.analyze_videos(path_config, ['/home/amr/Trial_DeepLabCut/mouse.avi'], save_as_csv=True) # In[26]: deeplabcut.create_labeled_video(path_config, ['/home/amr/Trial_DeepLabCut/mouse.avi']) # In[27]: deeplabcut.create_labeled_video(path_config, ['/home/amr/Trial_DeepLabCut/mouse.avi'], save_frames=True) # In[29]: deeplabcut.plot_trajectories(path_config, ['/home/amr/Trial_DeepLabCut/mouse.avi'])
def plot_trajectories(projectId): config_path = projectRepository.find_one({'_id': ObjectId(projectId) })['config_path'] deeplabcut.plot_trajectories(config_path, [], filtered=True) return "Not Implemented", 501
destfolder=DESTFOLDER, dynamic=(True, 0.1, 5), ) print("analyze again...") deeplabcut.analyze_videos(path_config_file, [newvideo], save_as_csv=True, destfolder=DESTFOLDER) print("CREATE VIDEO") deeplabcut.create_labeled_video(path_config_file, [newvideo], destfolder=DESTFOLDER, save_frames=True) print("Making plots") deeplabcut.plot_trajectories(path_config_file, [newvideo], destfolder=DESTFOLDER) print("EXTRACT OUTLIERS") deeplabcut.extract_outlier_frames( path_config_file, [newvideo], outlieralgorithm="jump", epsilon=0, automatic=True, destfolder=DESTFOLDER, ) deeplabcut.extract_outlier_frames( path_config_file, [newvideo], outlieralgorithm="fitting",
print("\n") print("\n") print("'config_path' is:", config_path) print("'dlc.__version__' is:'", dlc.__version__) print("\n") print("This is the name of the program:", sys.argv[0]) print("str(sys.argv):", str(sys.argv), "\n") print("\n") print("\n") edits = {'snapshotindex': snapshotindex} dlc.auxiliaryfunctions.edit_config(config_path, edits) print('\nediting the config file... ') for item in edits.items(): print(item) print('edit completed!') dlc.plot_trajectories(config=config_path, videos=videos_path_list, videotype='.mp4', shuffle=shuffleindex, trainingsetindex=0, filtered=False, showfigures=False, destfolder=None) print("dlc_plot_trajectories.py with the call", str(sys.argv), "is done!") print("returning snapshotindex back to 'all'...!") edits = {'snapshotindex': 'all'} dlc.auxiliaryfunctions.edit_config(config_path, edits) print("snapshotindex is set back to 'all'")
/February/February1 /February/February2 etc. ''' ''' subfolders=getsubfolders(basepath) for subfolder in subfolders: #this would be January, February etc. in the upper example print("Starting analyze data in:", subfolder) subsubfolders=getsubfolders(subfolder) for subsubfolder in subsubfolders: #this would be Febuary1, etc. in the upper example...''' subfolders = getsubfolders(basepath) '''for subfolder in subfolders: print("Starting analyze data in:", subfolder) subsubfolders=getsubfolders(subfolder) for subsubfolder in subsubfolders: print("Starting analyze data in:", subsubfolder)''' for path in videopath: for vtype in ['.mp4']: deeplabcut.analyze_videos(config, [path], shuffle=shuffle, videotype=vtype, save_as_csv=True) deeplabcut.filterpredictions(config, [path], videotype=vtype, shuffle=shuffle) deeplabcut.plot_trajectories(config, [path], videotype=vtype) deeplabcut.create_labeled_video(config, [path], videotype=vtype, filtered=True)
def create_videos(self, event): shuffle = self.shuffle.GetValue() trainingsetindex = self.trainingset.GetValue() # self.filelist = self.filelist + self.vids if self.filter.GetStringSelection() == "No": filtered = False else: filtered = True if self.video_slow.GetStringSelection() == "Yes": self.slow = True else: self.slow = False if len(self.bodyparts) == 0: self.bodyparts = "all" config_file = auxiliaryfunctions.read_config(self.config) if config_file.get("multianimalproject", False): print("Creating a video with the " + self.trackertypes.GetValue() + " tracker method!") if self.plot_idv.GetStringSelection() == "Yes": color_by = "individual" else: color_by = "bodypart" deeplabcut.create_labeled_video( self.config, self.filelist, self.videotype.GetValue(), shuffle=shuffle, trainingsetindex=trainingsetindex, save_frames=self.slow, draw_skeleton=self.draw, displayedbodyparts=self.bodyparts, trailpoints=self.trail_points.GetValue(), filtered=filtered, color_by=color_by, track_method=self.trackertypes.GetValue(), ) if self.trajectory.GetStringSelection() == "Yes": deeplabcut.plot_trajectories( self.config, self.filelist, displayedbodyparts=self.bodyparts, videotype=self.videotype.GetValue(), shuffle=shuffle, trainingsetindex=trainingsetindex, filtered=filtered, showfigures=False, track_method=self.trackertypes.GetValue(), ) else: deeplabcut.create_labeled_video( self.config, self.filelist, self.videotype.GetValue(), shuffle=shuffle, trainingsetindex=trainingsetindex, save_frames=self.slow, draw_skeleton=self.draw, displayedbodyparts=self.bodyparts, trailpoints=self.trail_points.GetValue(), filtered=filtered, )
df.to_hdf( picklefile.replace("pickle", "h5"), "df_with_missing", format="table", mode="w", ) df.to_hdf( picklefile.replace("sk", "bx").replace("pickle", "h5"), "df_with_missing", format="table", mode="w", ) print("Plotting trajectories...") deeplabcut.plot_trajectories(config_path, [new_video_path], "mp4", track_method="box") deeplabcut.plot_trajectories(config_path, [new_video_path], "mp4", track_method="skeleton") print("Trajectory plotted.") print("Creating labeled video...") deeplabcut.create_labeled_video( config_path, [new_video_path], "mp4", save_frames=False, color_by="individual", track_method="box", )
def create_pretrained_project( project, experimenter, videos, model="full_human", working_directory=None, copy_videos=False, videotype=None, analyzevideo=True, filtered=True, createlabeledvideo=True, trainFraction=None, ): """ Creates a new project directory, sub-directories and a basic configuration file. Change its parameters to your projects need. The project will also be initialized with a pre-trained model from the DeepLabCut model zoo! http://www.mousemotorlab.org/dlc-modelzoo Parameters ---------- project : string String containing the name of the project. experimenter : string String containing the name of the experimenter. model: string, options see http://www.mousemotorlab.org/dlc-modelzoo Current option and default: 'full_human' Creates a demo human project and analyzes a video with ResNet 101 weights pretrained on MPII Human Pose. This is from the DeeperCut paper by Insafutdinov et al. https://arxiv.org/abs/1605.03170 Please make sure to cite it too if you use this code! videos : list A list of string containing the full paths of the videos to include in the project. working_directory : string, optional The directory where the project will be created. The default is the ``current working directory``; if provided, it must be a string. copy_videos : bool, optional ON WINDOWS: TRUE is often necessary! If this is set to True, the videos are copied to the ``videos`` directory. If it is False,symlink of the videos are copied to the project/videos directory. The default is ``False``; if provided it must be either ``True`` or ``False``. analyzevideo " bool, optional If true, then the video is analzyed and a labeled video is created. If false, then only the project will be created and the weights downloaded. You can then access them filtered: bool, default false Boolean variable indicating if filtered pose data output should be plotted rather than frame-by-frame predictions. Filtered version can be calculated with deeplabcut.filterpredictions trainFraction: By default value from *new* projects. (0.95) Fraction that will be used in dlc-model/trainingset folder name. Example -------- Linux/MacOs loading full_human model and analzying video /homosapiens1.avi >>> deeplabcut.create_pretrained_project('humanstrokestudy','Linus',['/data/videos/homosapiens1.avi'], copy_videos=False) Loading full_cat model and analzying video "felixfeliscatus3.avi" >>> deeplabcut.create_pretrained_project('humanstrokestudy','Linus',['/data/videos/felixfeliscatus3.avi'], model='full_cat') Windows: >>> deeplabcut.create_pretrained_project('humanstrokestudy','Bill',[r'C:\yourusername\rig-95\Videos\reachingvideo1.avi'],r'C:\yourusername\analysis\project' copy_videos=True) Users must format paths with either: r'C:\ OR 'C:\\ <- i.e. a double backslash \ \ ) """ if model in globals()["Modeloptions"]: cwd = os.getcwd() cfg = deeplabcut.create_new_project(project, experimenter, videos, working_directory, copy_videos, videotype) if trainFraction is not None: auxiliaryfunctions.edit_config( cfg, {"TrainingFraction": [tranFraction]}) config = auxiliaryfunctions.read_config(cfg) if model == "full_human": config["bodyparts"] = [ "ankle1", "knee1", "hip1", "hip2", "knee2", "ankle2", "wrist1", "elbow1", "shoulder1", "shoulder2", "elbow2", "wrist2", "chin", "forehead", ] config["skeleton"] = [ ["ankle1", "knee1"], ["ankle2", "knee2"], ["knee1", "hip1"], ["knee2", "hip2"], ["hip1", "hip2"], ["shoulder1", "shoulder2"], ["shoulder1", "hip1"], ["shoulder2", "hip2"], ["shoulder1", "elbow1"], ["shoulder2", "elbow2"], ["chin", "forehead"], ["elbow1", "wrist1"], ["elbow2", "wrist2"], ] config["default_net_type"] = "resnet_101" else: # just make a case and put the stuff you want. # TBD: 'partaffinityfield_graph' >> use to set skeleton! pass auxiliaryfunctions.write_config(cfg, config) config = auxiliaryfunctions.read_config(cfg) train_dir = Path( os.path.join( config["project_path"], str( auxiliaryfunctions.GetModelFolder( trainFraction=config["TrainingFraction"][0], shuffle=1, cfg=config, )), "train", )) test_dir = Path( os.path.join( config["project_path"], str( auxiliaryfunctions.GetModelFolder( trainFraction=config["TrainingFraction"][0], shuffle=1, cfg=config, )), "test", )) # Create the model directory train_dir.mkdir(parents=True, exist_ok=True) test_dir.mkdir(parents=True, exist_ok=True) modelfoldername = auxiliaryfunctions.GetModelFolder( trainFraction=config["TrainingFraction"][0], shuffle=1, cfg=config) path_train_config = str( os.path.join(config["project_path"], Path(modelfoldername), "train", "pose_cfg.yaml")) path_test_config = str( os.path.join(config["project_path"], Path(modelfoldername), "test", "pose_cfg.yaml")) # Download the weights and put then in appropriate directory print("Dowloading weights...") auxfun_models.DownloadModel(model, train_dir) pose_cfg = deeplabcut.auxiliaryfunctions.read_plainconfig( path_train_config) print(path_train_config) # Updating config file: dict = { "default_net_type": pose_cfg["net_type"], "default_augmenter": pose_cfg["dataset_type"], "bodyparts": pose_cfg["all_joints_names"], "skeleton": [], # TODO: update with paf_graph "dotsize": 6, } auxiliaryfunctions.edit_config(cfg, dict) # Create the pose_config.yaml files parent_path = Path(os.path.dirname(deeplabcut.__file__)) defaultconfigfile = str(parent_path / "pose_cfg.yaml") trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(config) datafilename, metadatafilename = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction=config["TrainingFraction"][0], shuffle=1, cfg=config, ) # downloading base encoder / not required unless on re-trains (but when a training set is created this happens anyway) # model_path, num_shuffles=auxfun_models.Check4weights(pose_cfg['net_type'], parent_path, num_shuffles= 1) # Updating training and test pose_cfg: snapshotname = [fn for fn in os.listdir(train_dir) if ".meta" in fn][0].split(".meta")[0] dict2change = { "init_weights": str(os.path.join(train_dir, snapshotname)), "project_path": str(config["project_path"]), } UpdateTrain_pose_yaml(pose_cfg, dict2change, path_train_config) keys2save = [ "dataset", "dataset_type", "num_joints", "all_joints", "all_joints_names", "net_type", "init_weights", "global_scale", "location_refinement", "locref_stdev", ] MakeTest_pose_yaml(pose_cfg, keys2save, path_test_config) video_dir = os.path.join(config["project_path"], "videos") if analyzevideo == True: print("Analyzing video...") deeplabcut.analyze_videos(cfg, [video_dir], videotype, save_as_csv=True) if createlabeledvideo == True: if filtered: deeplabcut.filterpredictions(cfg, [video_dir], videotype) print("Plotting results...") deeplabcut.create_labeled_video(cfg, [video_dir], videotype, draw_skeleton=True, filtered=filtered) deeplabcut.plot_trajectories(cfg, [video_dir], videotype, filtered=filtered) os.chdir(cwd) return cfg, path_train_config else: return "N/A", "N/A"
def usingDeep(path_config, project): exit = False while exit == False: video_path = path_config.split("/") video_path = '/' + video_path[1] + '/' + video_path[ 2] + '/' + video_path[3] + '/' + video_path[4] + '/videos/' print_usage("project") action = input() while action not in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]: try: action = int(action) if action not in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]: print("please type number between 0 and 12") action = input() except ValueError: print("Please enter number") action = input() print("been here") if action == 0: return False elif action == 1: print("do you want to crop the video? yes/no") if input() == "yes" or "y": print( "how many videos you you want to crop? (use number: 1,2,3 etc)" ) crops = int(input()) print("only crop all the video's and than exit") for loop in range(0, crops): deeplabcut.extract_frames(path_config, 'manual', 'kmeans', crop=True) deeplabcut.extract_frames(path_config, 'automatic', 'kmeans', crop=True) elif action == 2: deeplabcut.label_frames(path_config) elif action == 3: deeplabcut.check_labels(path_config) elif action == 4: deeplabcut.create_training_dataset(path_config) elif action == 5: with open("training_network.py") as fp: lines = fp.readlines() lines[3] = lines[3].split("=") lines[3] = lines[3][0] + "= '" + path_config + "'\n" with open("training_network.py", "w") as fp: for line in lines: fp.writelines(line) print("run: sbatch slurm.sh") return True elif action == 6: try: deeplabcut.evaluate_network(path_config, Shuffles=[1], trainingsetindex=0, plotting=None, show_errors=True, comparisonbodyparts='all', gputouse=None, rescale=False) except OSError as e: print("file does not exist") elif action == 7: print("\nType video name in project/videos you want to analyze") video_path = video_path + create_dict_list( path_config[:-11] + "videos/", 1)[0] with open("training_network.py") as fp: lines = fp.readlines() lines[3] = lines[3].split("=") lines[3] = lines[3][0] + "= '" + path_config + "'\n" lines[4] = lines[4].split("=") lines[4] = lines[4][0] + "= '" + video_path + "'\n" with open("training_network.py", "w") as fp: for line in lines: fp.writelines(line) print( "run: sbatch slurm.sh after changing the command in training_network.py" ) return True elif action == 8: print( "\nChoose the video in project/videos you want to plot trajectories from" ) video_path = video_path + create_dict_list( path_config[:-11] + "videos/", 1)[0] print(video_path) deeplabcut.plot_trajectories(path_config, [video_path], filtered=True) elif action == 9: print( "\nChoose the video in project/videos you want to make a labeled video from" ) video_path = video_path + create_dict_list( path_config[:-11] + "videos/", 1)[0] deeplabcut.create_labeled_video(path_config, [video_path], videotype='.mp4', draw_skeleton=True) elif action == 10: print("\nChoose where to upload the video from") video_path = '/data/11012579/videos/' + create_dict_list( '/data/11012579/videos/', 0)[0] print("\nChoose which video to upload") video_path_list = [ video_path + "/" + create_dict_list(video_path, 1)[0] ] while True: print("\nDo you want to add more videos?\nType yes or no") if input() == 'yes': video_path_list.append(video_path + "/" + create_dict_list(video_path, 1)[0]) else: deeplabcut.add_new_videos(path_config, video_path_list, copy_videos=False) break elif action == 11: print("also here") Dlc_results2 = pd.read_hdf( '/data/11012579/videos/vidDLC_resnet50_demo_project_grab2Feb7shuffle1_11500.h5' ) Dlc_results2.plot() else: print_usage("error") print("klaar")
def analyze_and_clean_arm_dat(dat_mantle_dlc, adir, pcutoff, body_part_line_list, play_back_speed): #gets a path to the raw video that the previous script created when making the above csv vid = glob.glob(adir + '\\*.mp4')[0] #gets the crop parameters from mantle dlc as a string list print('getting crop parameters') crop_parameters = dat_mantle_dlc['crop_parameters'] #gets the (x1,y1) crop parameters as a list of 1x2 floats crop_x1y1_parameters = [ from_crop_parameters_get_x1y1(s) for s in crop_parameters ] #gets a list of the paths to the original images cropped from #img_paths_list = dat_mantle_dlc['img_paths'].to_list() #path to were the next step of dlc analysis will go path_arm_dir = adir + '\\arm_dlc_data' print('checking if: ' + path_arm_dir + 'exists') #if the directory hasent been made before we will make it and then run the dlc analysis if os.path.exists(path_arm_dir) != True: print('making: ' + path_arm_dir) os.mkdir(path_arm_dir) path_config = 'C:\\Users\\wlwee\\Documents\\python\\fhl_three_target_experiment\\MODEL\\arms-weert-2020-10-18\\config.yaml' deeplabcut.analyze_videos(path_config, [vid], videotype='mp4', save_as_csv=True, destfolder=path_arm_dir) deeplabcut.create_labeled_video(path_config, [vid], videotype='mp4', draw_skeleton='True', destfolder=path_arm_dir) deeplabcut.plot_trajectories(path_config, [vid], videotype='mp4', destfolder=path_arm_dir) else: print(path_arm_dir + ' already exists, skipping dlc arm model analysis') #opens the dlc arm model data print('getting dlc data') path_arm_dlc = glob.glob(path_arm_dir + '\\*.h5')[0] print('opening: ' + path_arm_dlc) dat_arm_dlc = pd.read_hdf(path_arm_dlc) #converts the arm model data frame into a numpy array which is easier to work with arm_dlc_numpy = dat_arm_dlc.to_numpy() #makes a path to were we will be storing the cleaned arm model data path_clean_arm_data_dir = path_arm_dir + '//clean_arm_data' #if the directory hasent been made before this makes it print('checking if: ' + path_clean_arm_data_dir + 'exists') if os.path.exists(path_clean_arm_data_dir) != True: os.mkdir(path_clean_arm_data_dir) else: print(path_clean_arm_data_dir + ' already exists, not making it again') #makes four empty dataframes which we will use to store the results from the below script as csvs dat_arm_dlc_clean = pd.DataFrame() dat_arm_dlc_clean_real = pd.DataFrame() dat_arm_speeds = pd.DataFrame() dat_arm_speeds_real = pd.DataFrame() print() for ii in range( 0, len(arm_dlc_numpy[0]) - 2, 3): #steps left to right through the dataframe by body part body_part = dat_arm_dlc.columns[ii][1] print(body_part) print(ii, ii + 1, ii + 2) x = [] y = [] p = [] s = 0 for iii in range( 0, len(arm_dlc_numpy) ): #steps top to bottom through the dataframe for a body part x_a = arm_dlc_numpy[iii][ii] #grab the dlc x value y_a = arm_dlc_numpy[iii][ii + 1] #grab the dlc y value p_a = arm_dlc_numpy[iii][ii + 2] #grab the probability value if p_a < p_cutoff: #if p_a is less than p_cutoff we set values to be the last sig detection x_a = arm_dlc_numpy[s][ii] y_a = arm_dlc_numpy[s][ii + 1] p_a = arm_dlc_numpy[s][ii + 2] else: #p_a > p_cutoff we save the index into a variable s for calling above s = iii p.append(p_a) x.append(x_a) y.append(y_a) d_x = [] d_y = [] abs_d_x = [] abs_d_y = [] for n in range( 1, len(x) - 1 ): #time steps through the extracted values from the dataframe, drops 1st and last values speed_x = (x[n] - x[n - 1]) speed_y = (y[n] - y[n - 1]) ''' #doing a speed replacement to the last known location ruins it #this is because if it jumps a short distance it will get stuck if speed_x > 120: x[n] = x[n-1] speed_x = abs(x[n] - x[n-1]) if speed_y > 120: y[n] = y[n-1] speed_y = abs(y[n] - y[n-1])''' d_x.append(speed_x) d_y.append(speed_y) abs_d_x.append(abs(speed_x)) abs_d_y.append(abs(speed_y)) #checking what the speeds are like before filtering mean_abs_dx, st_abs_x = st.mean(abs_d_x), st.stdev(abs_d_x) mean_abs_dy, st_abs_y = st.mean(abs_d_y), st.stdev(abs_d_y) mean_dx, st_x = st.mean(d_x), st.stdev(d_x) mean_dy, st_y = st.mean(d_y), st.stdev(d_y) ## some speed filtering step should be here ## pure linear interpolation does not fix jumps it 'smoothes' them # finds indexes where the speeds are outside of mean + 2*stdev d_x # does not filter slow speeds only fast speeds index_to_interpolate = [] for i in range(0, len(d_x)): ''' #mantle if ii <= 6: if d_x[i] > mean_dx + 2 * st_x or d_y[i] > mean_dy + 2 * st_y: index_to_interpolate.append(i + 1) ''' #arms if ii > 6: #if abs(d_x[i]) > mean_abs_dx + 1.5 * st_abs_x or abs(d_y[i]) > mean_abs_dy + 1.5 * st_abs_y or abs(d_x[i]) > 80 or abs(d_y[i]) > 80: if abs(d_x[i]) > 50 or abs(d_y[i]) > 50: index_to_interpolate.append(i + 1) interpolate_index_tuple = [] x_i, y_i = x, y #creates two new lists so we can manipulate x, y lists w/o changing originals for i in range(0, len(index_to_interpolate)): more, less = False, False #two booleans so we can loop until they are both True #grabs the first before and after indexs x1 = x_i[index_to_interpolate[i] - 1] y1 = y_i[index_to_interpolate[i] - 1] x2 = x_i[index_to_interpolate[i] + 1] y2 = y_i[index_to_interpolate[i] + 1] itt_1 = index_to_interpolate[i] - 1 itt_2 = index_to_interpolate[i] + 1 n, m = 1, 1 #grabs starting number of steps and step count while more == False and less == False: if i + n < len( index_to_interpolate ): #we iterate n forward until we get to a frame that does not need to be iterated if index_to_interpolate[i] + n == index_to_interpolate[i + n]: #if we have a list iterate 16,17,18,20 and we are at index 0 #we step till index 2 then stop there #we interpolate till the x2 index 18 + 1 x2 = x_i[index_to_interpolate[i + n] + 1] y2 = y_i[index_to_interpolate[i + n] + 1] itt_2 = index_to_interpolate[i + n] + 1 n = n + 1 else: more = True else: more = True if i - m > 0: #same as above but going backwards if index_to_interpolate[i] - m == index_to_interpolate[i - m]: x1 = x_i[index_to_interpolate[i - m] - 1] y1 = y_i[index_to_interpolate[i - m] - 1] itt_1 = index_to_interpolate[i - m] - 1 m = m + 1 else: less = True else: less = True interpolate_index_tuple.append([itt_1, itt_2, i - m, i + n]) #n equals steps forward from the bad x value to the x value we want to iterate to #m equals steps backwards from the bad x value to the x value we want to iterate to #steps = n + m at = m / (n + m) ''' #incase of errors uncomment this section print() print(ii, iii) print(x1,y1) print(x2,y2) print(n+m,m,n,at) ''' #see interpolate def above for behavior x_inter, y_inter = interpolate(float(x1), float(x2), float(y1), float(y2), at) #rewrite x_i for the interpolate index to x_inter x_i[index_to_interpolate[i]] = x_inter y_i[index_to_interpolate[i]] = y_inter ''' print(x_inter, y_inter) print() ''' d_xi, d_yi = [], [ ] #new variables to hold interapolation speeds so we can investigate them for n in range(1, len(x_i) - 1): speed_xi = (x_i[n] - x_i[n - 1]) speed_yi = (y_i[n] - y_i[n - 1]) d_xi.append(speed_xi) d_yi.append(speed_yi) x_ri, y_ri = [], [] for i in range( 0, len(x_i) ): #get the 'real' x, y values by shifting relative to the x1, y1 crop parameters x_ri_a = x_i[i] + crop_x1y1_parameters[i][0] y_ri_a = y_i[i] + crop_x1y1_parameters[i][1] x_ri.append(x_ri_a) y_ri.append(y_ri_a) d_xi_r, d_yi_r = [], [] for n in range(1, len(x_ri) - 1): speed_xi_r = (x_ri[n] - x_ri[n - 1]) speed_yi_r = (y_ri[n] - y_ri[n - 1]) d_xi_r.append(speed_xi_r) d_yi_r.append(speed_yi_r) #here we calculate speed means and standard deviations for the filtered data before and after it is returned to the 'real' reference frame mean_dxi, st_xi = st.mean(d_xi), st.stdev(d_xi) mean_dyi, st_yi = st.mean(d_yi), st.stdev(d_yi) mean_dxi_r, st_xi_r = st.mean(d_xi_r), st.stdev(d_xi_r) mean_dyi_r, st_yi_r = st.mean(d_yi_r), st.stdev(d_yi_r) print('number interpolated frames ' + str(len(index_to_interpolate))) print('unfiltered speed ' + body_part + '_x: ' + str(round(mean_dx, 3)) + ' +- ' + str(round(st_x, 3))) print('filtered speed ' + body_part + '_x: ' + str(round(mean_dxi, 3)) + ' +- ' + str(round(st_xi, 3))) print('filtered real speed ' + body_part + '_x: ' + str(round(mean_dxi_r, 3)) + ' +- ' + str(round(st_xi_r, 3))) print('unfiltered speed ' + body_part + '_y: ' + str(round(mean_dy, 3)) + ' +- ' + str(round(st_y, 3))) print('filtered speed ' + body_part + '_y: ' + str(round(mean_dyi, 3)) + ' +- ' + str(round(st_yi, 3))) print('filtered real speed ' + body_part + '_y: ' + str(round(mean_dyi_r, 3)) + ' +- ' + str(round(st_yi_r, 3))) p_test = p[2:] print('p min ' + str(round(min(p_test), 5))) print() #creates the dataframes that will store our filtered positional data #octopus reference frame and real reference frame #column names are tuples dat_arm_dlc_clean[dat_arm_dlc.columns[ii]] = x_i dat_arm_dlc_clean[dat_arm_dlc.columns[ii + 1]] = y_i dat_arm_dlc_clean[dat_arm_dlc.columns[ii + 2]] = p dat_arm_dlc_clean_real[dat_arm_dlc.columns[ii]] = x_ri dat_arm_dlc_clean_real[dat_arm_dlc.columns[ii + 1]] = y_ri dat_arm_dlc_clean_real[dat_arm_dlc.columns[ii + 2]] = p #creates the dataframes that will store our filtered speed data #octopus reference frame and real reference frame #column names are tuples dat_arm_speeds[dat_arm_dlc.columns[ii]] = d_xi dat_arm_speeds[dat_arm_dlc.columns[ii + 1]] = d_yi dat_arm_speeds_real[dat_arm_dlc.columns[ii]] = d_xi_r dat_arm_speeds_real[dat_arm_dlc.columns[ii + 1]] = d_yi_r path_dat_arm_dlc_clean_csv = path_clean_arm_data_dir + '\\octo_ref_clean_arm_data.csv' dat_arm_dlc_clean.to_csv(path_dat_arm_dlc_clean_csv) path_dat_arm_dlc_clean_real_csv = path_clean_arm_data_dir + '\\octo_real_ref_clean_arm_data.csv' dat_arm_dlc_clean_real.to_csv(path_dat_arm_dlc_clean_real_csv) path_dat_arm_speeds_csv = path_clean_arm_data_dir + '\\octo_ref_speed.csv' dat_arm_speeds.to_csv(path_dat_arm_speeds_csv) path_dat_arm_speeds_real_csv = path_clean_arm_data_dir + '\\octo_ref_speed_real.csv' dat_arm_speeds_real.to_csv(path_dat_arm_speeds_real_csv) ## below we are going to make a video using our filtered data to draw points and lines upon it ## we do this to investigate what our filtered predictions look like numpy_arm_dlc_clean = dat_arm_dlc_clean.to_numpy() numpy_arm_dlc_clean_real = dat_arm_dlc_clean_real.to_numpy() numpy_arm_speed = dat_arm_speeds.to_numpy() numpy_arm_speed_real = dat_arm_speeds_real.to_numpy() vid_dlc_labeled = glob.glob(path_arm_dir + '\\*.mp4')[0] video_clean_labels_name = path_clean_arm_data_dir + '\\clean_labels_vid.mp4' video_clean_labels_hull_name = path_clean_arm_data_dir + '\\clean_labels_hull_vid.mp4' #open the three target location .csv target_location_path = os.path.dirname( os.path.dirname(adir)) + '\\three_target_location.csv' if os.path.exists(target_location_path) != True: print(target_location_path + ' does not exist!') else: print('opening ' + target_location_path) dat_target = pd.read_csv(target_location_path) #turn the dat_target dataframe into a numpy array numpy_target = dat_target.to_numpy() food_location_xy = [] #loop left to right through the first row of the array and find which target has food for i in range(0, len(numpy_target[0]), 3): print(numpy_target[0][i], numpy_target[0][i + 1], numpy_target[0][i + 2]) if numpy_target[0][i + 2] == 1: food_location_xy.append( [numpy_target[0][i], numpy_target[0][i + 1]]) food_location_xy = food_location_xy[0] cap = cv2.VideoCapture(vid) cap_dlc = cv2.VideoCapture(vid_dlc_labeled) frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) fs = 0 width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) video = cv2.VideoWriter(video_clean_labels_name, cv2.VideoWriter_fourcc(*'mp4v'), 7.0, (width, height)) video_hull = cv2.VideoWriter(video_clean_labels_hull_name, cv2.VideoWriter_fourcc(*'mp4v'), 7.0, (width, height)) font = cv2.FONT_HERSHEY_SIMPLEX hull_list = [] hull_list_rotated = [] l_eye_list, r_eye_list, mean_eye_list = [], [], [] while (cap.isOpened()): ret, frame = cap.read() ret_dlc, frame_dlc = cap_dlc.read() frame_edit = frame xy_lst = numpy_arm_dlc_clean[fs].tolist() #grab list of coordinates del xy_lst[2::3] #reshape the list to remove p-values hull_points = [] for i in range(0, len(xy_lst), 2): hull_points.append([ xy_lst[i], xy_lst[i + 1] ]) #turn the list of body parts into a coordinate list #give the coordinate list to the convexHull function, saves a list of coordinates hull = convexHull(hull_points, len(hull_points)) #make new pts that is in the format opencv fillPoly wants pts = np.array(hull, np.int32) pts = pts.reshape((-1, 1, 2)) #create a copy of frame_edit to overlay on frame edit overlay = frame_edit.copy() alpha = 0.3 #transparency for the frame_edit overlay cv2.fillPoly(overlay, [pts], (100, 255, 255)) #put a polygon on the overlay frame_edit = cv2.addWeighted(overlay, alpha, frame_edit, 1 - alpha, 0) #put the overlay on frame_edit #save the hull hull_list.append(hull) #convex hull octopus frame of view re-orientation #save the eye coordinates le_xy = [numpy_arm_dlc_clean[fs][0], numpy_arm_dlc_clean[fs][1]] l_eye_list.append(le_xy) re_xy = [numpy_arm_dlc_clean[fs][3], numpy_arm_dlc_clean[fs][4]] r_eye_list.append(re_xy) #get mean eye coordinates mean_e_xy = [ st.mean([le_xy[0], re_xy[0]]), st.mean([le_xy[1], re_xy[1]]) ] mean_eye_list.append(mean_e_xy) #move left eye and right eye to the origin as we can get angle to rotate the hull l_xy_o = [le_xy[0] - mean_e_xy[0], le_xy[1] - mean_e_xy[1]] r_xy_o = [re_xy[0] - mean_e_xy[0], re_xy[1] - mean_e_xy[1]] #atan2 gives negative and positive radian values relative to the positive x-axis theta = math.atan2(l_xy_o[1], l_xy_o[0]) #theta_r = math.atan2(r_xy_o[1],r_xy_o[0]) #we will rotate the right eye to the positive x-axis and left eye to the negative x-axis le_re_reorientated_list = [] le_xy_r = rotate([0, 0], l_xy_o, (math.pi - theta)) re_xy_r = rotate([0, 0], r_xy_o, (math.pi - theta)) le_re_reorientated_list.append([le_xy_r, re_xy_r]) #since we are dealing with big floats for my sake I set the eyes to zero #the eyes so far are not helpful if round(le_xy_r[1], 5) > 0 or round(re_xy_r[1], 5) > 0: print( str(fs, le_xy_r[1], re_xy_r[1]) + ' eye rotate error theta_l < 0 and l_xy_o[0] < r_xy_o[0]') else: le_xy_r[1] = 0.0 re_xy_r[1] = 0.0 #here we rotate the hull rotated_hull = [] for hull_cord in hull: ho_xy = [hull_cord[0] - mean_e_xy[0], hull_cord[1] - mean_e_xy[1]] ho_xy_r = rotate([0, 0], ho_xy, (math.pi - theta)) ho_xy_r = [ho_xy_r[0] + width / 2, ho_xy_r[1] + height / 2] rotated_hull.append(ho_xy_r) hull_list_rotated.append(rotated_hull) ''' frame_hull_unrotated = np.zeros((width,height,3),np.uint8) pts = np.array(hull, np.int32) pts = pts.reshape((-1,1,2)) cv2.fillPoly(frame_hull_unrotated,[pts],(255,255,255))''' frame_hull_rotated = np.zeros((width, height, 3), np.uint8) pts = np.array(rotated_hull, np.int32) pts = pts.reshape((-1, 1, 2)) cv2.fillPoly(frame_hull_rotated, [pts], (255, 255, 255)) cv2.circle(frame_hull_rotated, (int(width / 2), int(height / 2)), 1, (0, 0, 0)) ##get the real eye coordinates so we can point at the food #save the eye coordinates le_xy_real = [ numpy_arm_dlc_clean_real[fs][0], numpy_arm_dlc_clean_real[fs][1] ] re_xy_real = [ numpy_arm_dlc_clean_real[fs][3], numpy_arm_dlc_clean_real[fs][4] ] #get real mean eye coordinates mean_e_xy_r = [ st.mean([le_xy_real[0], re_xy_real[0]]), st.mean([le_xy_real[1], re_xy_real[1]]) ] food_location_xy_r = [ food_location_xy[0] - mean_e_xy_r[0], food_location_xy[1] - mean_e_xy_r[1] ] food_location_xy_r_r = rotate([0, 0], food_location_xy_r, (math.pi - theta)) food_location_xy_r_r_o = [ food_location_xy_r_r[0] + width / 2, food_location_xy_r_r[1] + height / 2 ] #get the mean speed of the eyes if fs > 0 and fs < frame_count - 2: speed_le_xy = [ numpy_arm_speed_real[fs][0], numpy_arm_speed_real[fs][1] ] speed_re_xy = [ numpy_arm_speed_real[fs][2], numpy_arm_speed_real[fs][3] ] speed_em_xy = [ st.mean([speed_le_xy[0], speed_re_xy[0]]), st.mean([speed_le_xy[1], speed_re_xy[1]]) ] speed_em_xy_rotated = rotate([0, 0], speed_em_xy, (math.pi - theta)) sxy = [ int(10 * speed_em_xy_rotated[0] + width / 2), int(10 * speed_em_xy_rotated[1] + height / 2) ] print(food_location_xy_r_r) cv2.arrowedLine(frame_hull_rotated, (int(width / 2), int(height / 2)), (int(food_location_xy_r_r_o[0]), int(food_location_xy_r_r_o[1])), (200, 100, 0), thickness=2) cv2.arrowedLine(frame_hull_rotated, (int(width / 2), int(height / 2)), (sxy[0], sxy[1]), (100, 200, 0), thickness=3) speed_text = '(' + str(round( speed_em_xy_rotated[0], 2)) + ',' + str( round(speed_em_xy_rotated[1], 2)) + ')' cv2.putText(frame_hull_rotated, speed_text, (sxy[0], sxy[1]), font, 0.25, (100, 200, 0)) #draw whitish grey lines using body_part_line_list for part in body_part_line_list: xy_l0 = int(numpy_arm_dlc_clean[fs][part[0] * 3]), int( numpy_arm_dlc_clean[fs][part[0] * 3 + 1]) xy_l1 = int(numpy_arm_dlc_clean[fs][part[1] * 3]), int( numpy_arm_dlc_clean[fs][part[1] * 3 + 1]) cv2.line(frame_edit, xy_l0, xy_l1, (255 / 2, 255 / 2, 255 / 2), 1) #draw color coded body parts #goes left to right through the numpy array by steps of 3 ss = 0 for i in range(0, len(numpy_arm_dlc_clean[fs]), 3): xy_c = int(numpy_arm_dlc_clean[fs][i]), int( numpy_arm_dlc_clean[fs][i + 1]) col1 = int(255 - 255 / len(numpy_arm_dlc_clean[fs]) * i) col2 = int(255 / len(numpy_arm_dlc_clean[fs]) * i) cv2.circle(frame_edit, xy_c, 3, (col1, 0, col2), -1) if fs > 0 and fs < frame_count - 2: speed_xt = numpy_arm_speed[fs][i - ss] speed_yt = numpy_arm_speed[fs][i - ss + 1] speed_xy = str(round(math.sqrt(speed_xt**2 + speed_yt**2), 1)) cv2.putText(frame_edit, speed_xy, xy_c, font, 0.25, (255 / 2, 255 / 2, 255 / 2)) ss = ss + 1 '''cv2.putText(frame_edit, 'speed ')''' #cv2.imshow('FrameClear',frame) cv2.imshow('FrameEdit', frame_edit) cv2.imshow('FrameDLC', frame_dlc) #cv2.imshow('FrameUnrotatedHull',frame_hull_unrotated) cv2.imshow('FrameRotatedHull', frame_hull_rotated) video.write(frame_edit) video_hull.write(frame_hull_rotated) cv2.waitKey(25) fs = fs + 1 if fs == frame_count: break video_hull.release() video.release() cap_dlc.release() cap.release() cv2.destroyAllWindows()
def create_pretrained_human_project(project, experimenter, videos, working_directory=None, copy_videos=False, videotype='.avi', createlabeledvideo=True, analyzevideo=True): """ Creates a demo human project and analyzes a video with ResNet 101 weights pretrained on MPII Human Pose. This is from the DeeperCut paper by Insafutdinov et al. https://arxiv.org/abs/1605.03170 Please make sure to cite it too if you use this code! Parameters ---------- project : string String containing the name of the project. experimenter : string String containing the name of the experimenter. videos : list A list of string containing the full paths of the videos to include in the project. working_directory : string, optional The directory where the project will be created. The default is the ``current working directory``; if provided, it must be a string. copy_videos : bool, optional If this is set to True, the videos are copied to the ``videos`` directory. If it is False,symlink of the videos are copied to the project/videos directory. The default is ``False``; if provided it must be either ``True`` or ``False``. analyzevideo " bool, optional If true, then the video is analzyed and a labeled video is created. If false, then only the project will be created and the weights downloaded. You can then access them Example -------- Linux/MacOs >>> deeplabcut.create_pretrained_human_project('human','Linus',['/data/videos/mouse1.avi'],'/analysis/project/',copy_videos=False) Windows: >>> deeplabcut.create_pretrained_human_project('human','Bill',[r'C:\yourusername\rig-95\Videos\reachingvideo1.avi'],r'C:\yourusername\analysis\project' copy_videos=False) Users must format paths with either: r'C:\ OR 'C:\\ <- i.e. a double backslash \ \ ) -------- """ cfg = deeplabcut.create_new_project(project, experimenter, videos, working_directory, copy_videos, videotype) config = auxiliaryfunctions.read_config(cfg) config['bodyparts'] = [ 'ankle1', 'knee1', 'hip1', 'hip2', 'knee2', 'ankle2', 'wrist1', 'elbow1', 'shoulder1', 'shoulder2', 'elbow2', 'wrist2', 'chin', 'forehead' ] config['skeleton'] = [['ankle1', 'knee1'], ['ankle2', 'knee2'], ['knee1', 'hip1'], ['knee2', 'hip2'], ['hip1', 'hip2'], ['shoulder1', 'shoulder2'], ['shoulder1', 'hip1'], ['shoulder2', 'hip2'], ['shoulder1', 'elbow1'], ['shoulder2', 'elbow2'], ['chin', 'forehead'], ['elbow1', 'wrist1'], ['elbow2', 'wrist2']] config['default_net_type'] = 'resnet_101' auxiliaryfunctions.write_config(cfg, config) config = auxiliaryfunctions.read_config(cfg) train_dir = Path( os.path.join( config['project_path'], str( auxiliaryfunctions.GetModelFolder( trainFraction=config['TrainingFraction'][0], shuffle=1, cfg=config)), 'train')) test_dir = Path( os.path.join( config['project_path'], str( auxiliaryfunctions.GetModelFolder( trainFraction=config['TrainingFraction'][0], shuffle=1, cfg=config)), 'test')) # Create the model directory train_dir.mkdir(parents=True, exist_ok=True) test_dir.mkdir(parents=True, exist_ok=True) modelfoldername = auxiliaryfunctions.GetModelFolder( trainFraction=config['TrainingFraction'][0], shuffle=1, cfg=config) path_train_config = str( os.path.join(config['project_path'], Path(modelfoldername), 'train', 'pose_cfg.yaml')) path_test_config = str( os.path.join(config['project_path'], Path(modelfoldername), 'test', 'pose_cfg.yaml')) # Download the weights and put then in appropriate directory cwd = os.getcwd() os.chdir(train_dir) print( "Checking if the weights are already available, otherwise I will download them!" ) weightfilename = auxfun_models.download_mpii_weigths(train_dir) os.chdir(cwd) # Create the pose_config.yaml files parent_path = Path(os.path.dirname(deeplabcut.__file__)) defaultconfigfile = str(parent_path / 'pose_cfg.yaml') trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(config) datafilename, metadatafilename = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction=config['TrainingFraction'][0], shuffle=1, cfg=config) bodyparts = config['bodyparts'] net_type = 'resnet_101' num_shuffles = 1 model_path, num_shuffles = auxfun_models.Check4weights( net_type, parent_path, num_shuffles) items2change = { "dataset": 'dataset-test.mat', #datafilename, "metadataset": metadatafilename, "num_joints": len(bodyparts), "all_joints": [[i] for i in range(len(bodyparts))], "all_joints_names": [str(bpt) for bpt in bodyparts], "init_weights": weightfilename.split('.index')[0], #'models/mpii/snapshot-1030000', "project_path": str(config['project_path']), "net_type": net_type, "dataset_type": "default" } trainingdata = MakeTrain_pose_yaml(items2change, path_train_config, defaultconfigfile) keys2save = [ "dataset", "dataset_type", "num_joints", "all_joints", "all_joints_names", "net_type", 'init_weights', 'global_scale', 'location_refinement', 'locref_stdev' ] MakeTest_pose_yaml(trainingdata, keys2save, path_test_config) video_dir = os.path.join(config['project_path'], 'videos') if analyzevideo == True: # Analyze the videos deeplabcut.analyze_videos(cfg, [video_dir], videotype, save_as_csv=True) if createlabeledvideo == True: deeplabcut.create_labeled_video(cfg, [video_dir], videotype, draw_skeleton=True) deeplabcut.plot_trajectories(cfg, [video_dir], videotype) return cfg, path_train_config
We also provide a function to plot the data overtime and pixels in frames. The provided plotting function in this toolbox utilizes matplotlib 44 ; therefore, these plots can easily be customized. To call this function, type the following: The ouput files can also be easily imported into many programs for further behavioral analysis (see Stage XI and ‘Anticipated results’). """ if "filterpredictions" in stages_to_run: deeplabcut.filterpredictions(config_path, videos_to_filter_predictions, videotype='avi', shuffle=1, trainingsetindex=0, filtertype='median', windowlength=5, p_bound=.001, ARdegree=3, MAdegree=1, alpha=.01, save_as_csv=True, destfolder=None) if "step_12" in stages_to_run or "plot_trajectories" in stages_to_run: deeplabcut.plot_trajectories(config_path, videos_to_plot_trajectories, videotype='.avi', shuffle=1, trainingsetindex=0, filtered=False, showfigures=False, destfolder=None) if "step_12_bis" in stages_to_run or "analyzeskeleton" in stages_to_run: deeplabcut.analyzeskeleton(config_path, videos_to_plot_trajectories, videotype='avi', shuffle=1, trainingsetindex=0, save_as_csv=True, destfolder=None) """ In addition, the toolbox provides a function to create labeled videos based on the extracted poses by plotting the labels on top of the frame and creating a video. To use it to create multiple labeled videos (provided either as each video path or as a folder path), type the following: >> deeplabcut.create_labeled_video(config_path,[‘Full path of video 1’, ‘Full path of video 2’]) This function has various parameters; in particular, the user can set the colormap, the dotsize, and the alphavalue of the labels in the config.yaml file, and can pass a variable called displayedbodyparts to select only a subset of parts to be plotted. The user can also save
track_method=TESTTRACKER ) print("Tracklets created...") deeplabcut.stitch_tracklets( config_path, [new_video_path], "mp4", output_name=os.path.splitext(new_video_path)[0] + scorer + "_el.h5", track_method=TESTTRACKER ) print("Plotting trajectories...") deeplabcut.plot_trajectories( config_path, [new_video_path], "mp4", track_method=TESTTRACKER ) print("Trajectory plotted.") print("Creating labeled video...") deeplabcut.create_labeled_video( config_path, [new_video_path], "mp4", save_frames=False, color_by="individual", track_method=TESTTRACKER ) print("Labeled video created.")
) deeplabcut.convert_detections2tracklets( config_path, [new_video_path], "mp4", track_method="ellipse" ) print("Tracklets created...") deeplabcut.stitch_tracklets( config_path, [new_video_path], "mp4", output_name=os.path.splitext(new_video_path)[0] + scorer + "_el.h5", ) print("Plotting trajectories...") deeplabcut.plot_trajectories( config_path, [new_video_path], "mp4", track_method="ellipse" ) print("Trajectory plotted.") print("Creating labeled video...") deeplabcut.create_labeled_video( config_path, [new_video_path], "mp4", save_frames=False, color_by="individual", track_method="ellipse", ) print("Labeled video created.") print("Filtering predictions...")
from moviepy.editor import VideoFileClip,VideoClip clip = VideoFileClip(video[0]) clip.reader.initialize() def make_frame(t): return clip.get_frame(1) newclip = VideoClip(make_frame, duration=1) newclip.write_videofile(newvideo,fps=30) deeplabcut.analyze_videos(path_config_file,[newvideo],save_as_csv=True, destfolder=dfolder) print("CREATE VIDEO") deeplabcut.create_labeled_video(path_config_file,[newvideo], destfolder=dfolder) print("Making plots") deeplabcut.plot_trajectories(path_config_file,[newvideo], destfolder=dfolder) print("EXTRACT OUTLIERS") deeplabcut.extract_outlier_frames(path_config_file,[newvideo],outlieralgorithm='jump',epsilon=0,automatic=True, destfolder=dfolder) file=os.path.join(cfg['project_path'],'labeled-data',vname,"machinelabels-iter"+ str(cfg['iteration']) + '.h5') print("RELABELING") DF=pd.read_hdf(file,'df_with_missing') DLCscorer=np.unique(DF.columns.get_level_values(0))[0] DF.columns.set_levels([scorer.replace(DLCscorer,scorer)],level=0,inplace=True) DF =DF.drop('likelihood',axis=1,level=2) DF.to_csv(os.path.join(cfg['project_path'],'labeled-data',vname,"CollectedData_" + scorer + ".csv")) DF.to_hdf(os.path.join(cfg['project_path'],'labeled-data',vname,"CollectedData_" + scorer + '.h5'),'df_with_missing',format='table', mode='w')
def analyze_videos(self, event): shuffle = self.shuffle.GetValue() videotype = self.videotype.GetValue(), if self.cfg.get("multianimalproject", False): print("DLC network loading and video analysis starting ... ") auto_track = True else: if self.csv.GetStringSelection() == "Yes": save_as_csv = True else: save_as_csv = False if self.dynamic.GetStringSelection() == "No": dynamic = (False, 0.5, 10) else: dynamic = (True, 0.5, 10) if self.filter.GetStringSelection() == "No": _filter = False else: _filter = True if self.cfg["cropping"] == "True": crop = self.cfg["x1"], self.cfg["x2"], self.cfg["y1"], self.cfg[ "y2"] else: crop = None if self.cfg.get("multianimalproject", False): if self.robust.GetStringSelection() == "No": robust = False else: robust = True scorername = deeplabcut.analyze_videos( self.config, self.filelist, self.videotype.GetValue(), shuffle=shuffle, gputouse=None, cropping=crop, robust_nframes=robust, auto_track=True, n_tracks=self.ntracks.GetValue(), calibrate=self.calibrate.GetStringSelection() == "Yes", identity_only=self.identity_toggle.GetStringSelection() == "Yes", ) if self.create_video_with_all_detections.GetStringSelection( ) == "Yes": deeplabcut.create_video_with_all_detections( self.config, self.filelist, self.videotype.GetValue(), shuffle=shuffle, ) if self.filter.GetStringSelection() == "Yes": deeplabcut.filterpredictions( self.config, self.filelist, self.videotype.GetValue(), ) if self.csv.GetStringSelection() == "Yes": deeplabcut.analyze_videos_converth5_to_csv(self.filelist, listofvideos=True) if self.nwb.GetStringSelection() == "Yes": deeplabcut.analyze_videos_converth5_to_nwb( self.config, self.filelist, listofvideos=True, ) if self.trajectory.GetStringSelection() == "Yes": if self.showfigs.GetStringSelection() == "No": showfig = False else: showfig = True deeplabcut.plot_trajectories(self.config, self.filelist, self.videotype.GetValue(), showfigures=showfig) else: scorername = deeplabcut.analyze_videos( self.config, self.filelist, self.videotype.GetValue(), shuffle=shuffle, gputouse=None, save_as_csv=save_as_csv, cropping=crop, dynamic=dynamic, ) if _filter: deeplabcut.filterpredictions( self.config, self.filelist, self.videotype.GetValue(), shuffle=shuffle, filtertype="median", windowlength=5, save_as_csv=save_as_csv, ) if self.trajectory.GetStringSelection() == "Yes": if self.showfigs.GetStringSelection() == "No": showfig = False else: showfig = True deeplabcut.plot_trajectories( self.config, self.filelist, self.videotype.GetValue(), displayedbodyparts=self.bodyparts, shuffle=shuffle, filtered=_filter, showfigures=showfig, )
deeplabcut.check_labels(path_config_file) #this creates a subdirectory with the frames + your labels # Reviewed the labels, the seem to be ok # Downloading the ResNets dataset: deeplabcut.create_training_dataset(path_config_file) # Training the dataset deeplabcut.train_network(path_config_file) # Evaluating the results deeplabcut.evaluate_network(path_config_file) # Analyzing video videofile_path = ['dlc-blinking/an3_vid2_full/eyes_only-Guy-2019-01-25/videos/animal_3_video_2_150fps_correct.mp4', 'dlc-blinking/an3_vid2_full/eyes_only-Guy-2019-01-25/videos/march_8_animal_1_video_150fps_correct.mp4'] #Enter the list of videos to analyze. videofile_path = ['dlc-blinking/whisk/whisk_only-Guy-2019-02-01/videos/crush_19_01_07_animal_3.mp4', 'dlc-blinking/whisk/whisk_only-Guy-2019-02-01/videos/animal_3_video_2_150fps_correct.mp4'] deeplabcut.analyze_videos(path_config_file,videofile_path,save_as_csv=True) deeplabcut.create_labeled_video(path_config_file, ['D:\\dlc-blinking\\an3_vid2_full\\whisk_only-Guy-2019-02-01\\videos\\crush_19_01_07_animal_3.mp4'], save_frames=True) deeplabcut.create_labeled_video(path_config_file, ['D:\\dlc-blinking\\whisk\\whisk_only-Guy-2019-02-01\\videos\\animal_3_video_2_150fps_correct.mp4'], save_frames=True) %matplotlib notebook #for making interactive plots. deeplabcut.plot_trajectories(path_config_file,videofile_path) # TICTOCS: # training - up to 72/96 hours # analyzing - 45 minutes and 1.5 hours # labeling - 25 minutes and 50 minutes
for video in os.listdir(dir): subprocess.call([ 'ffmpeg', '-i', filepath1 + '/' + video, '-vf', "lenscorrection=cx=0.5:cy=0.5:k1=-.115:k2=-0.022", filepath2 + '/' + video ]) #Use deeplabcut to analyze videos and save the results to the folder for processed videos for dir in datadir3: for video in os.listdir(dir): print(video) deeplabcut.analyze_videos(configPath, [filepath2 + '/' + video], destfolder=filepath3, save_as_csv=True) deeplabcut.plot_trajectories(configPath, [filepath2 + '/' + video], destfolder=filepath3) #deeplabcut.create_labeled_video(configPath,[filepath2 +'/'+ video],videotype = 'mp4', destfolder = filepath3) #Change directory to openpose and run openpose on the videos then save the results to the processed video folder os.chdir("/Users/MatthisLab/openpose") for dir in datadir3: for video in os.listdir(dir): videoName = video[:-4] subprocess.call([ 'bin/OpenPoseDemo.exe', '--video', filepath2 + '/' + video, '--hand', '--face', '--write_video', filepath4 + '/' + videoName, '--write_json', filepath4 + '/' + videoName ])