def calculatepafdistancebounds(config, shuffle=0, trainingsetindex=0, modelprefix="", numdigits=0, onlytrain=False): """ Returns distances along paf edges in train/test data ---------- config : string Full path of the config.yaml file as a string. shuffle: integer integers specifying shuffle index of the training dataset. The default is 0. trainingsetindex: int, optional Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This variable can also be set to "all". numdigits: number of digits to round for distances. """ import os from deeplabcut.utils import auxiliaryfunctions, auxfun_multianimal from deeplabcut.pose_estimation_tensorflow.config import load_config # Read file path for pose_config file. >> pass it on cfg = auxiliaryfunctions.read_config(config) if cfg["multianimalproject"] == True: ( individuals, uniquebodyparts, multianimalbodyparts, ) = auxfun_multianimal.extractindividualsandbodyparts(cfg) # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) trainFraction = cfg["TrainingFraction"][trainingsetindex] datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) modelfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetModelFolder(trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) # Load meta data & annotations ( data, trainIndices, testIndices, trainFraction, ) = auxiliaryfunctions.LoadMetadata( os.path.join(cfg["project_path"], metadatafn)) Data = pd.read_hdf( os.path.join( cfg["project_path"], str(trainingsetfolder), "CollectedData_" + cfg["scorer"] + ".h5", ), "df_with_missing", )[cfg["scorer"]] path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" dlc_cfg = load_config(str(path_test_config)) # get the graph! partaffinityfield_graph = dlc_cfg.partaffinityfield_graph jointnames = [ dlc_cfg.all_joints_names[i] for i in range(len(dlc_cfg.all_joints)) ] # plt.figure() Cutoff = {} path_inferencebounds_config = (Path(modelfolder) / "test" / "inferencebounds.yaml") inferenceboundscfg = {} for pi, edge in enumerate(partaffinityfield_graph): j1, j2 = jointnames[edge[0]], jointnames[edge[1]] ds_within = [] ds_across = [] for ind in individuals: for ind2 in individuals: if ind != "single" and ind2 != "single": if (ind, j1, "x") in Data.keys() and ( ind2, j2, "y", ) in Data.keys(): distances = np.sqrt( (Data[ind, j1, "x"] - Data[ind2, j1, "x"])**2 + (Data[ind, j1, "y"] - Data[ind2, j2, "y"])**2) else: distances = None if distances is not None: if onlytrain: # extracts only distances on training data. distances = distances.iloc[trainIndices] # source=np.array(Data[ind,j1,'x'][jj],Data[ind,j1,'y'][jj]) # target=np.array(Data[ind2,j1,'x'][jj],Data[ind2,j2,'y'][jj]) if ind == ind2: ds_within.extend(distances.values.flatten()) else: ds_across.extend(distances.values.flatten()) edgeencoding = str(edge[0]) + "_" + str(edge[1]) inferenceboundscfg[edgeencoding] = {} if len(ds_within) > 0: inferenceboundscfg[edgeencoding]["intra_max"] = str( round(np.nanmax(ds_within), numdigits)) inferenceboundscfg[edgeencoding]["intra_min"] = str( round(np.nanmin(ds_within), numdigits)) else: inferenceboundscfg[edgeencoding]["intra_max"] = str( 1e5) # large number (larger than any image diameter) inferenceboundscfg[edgeencoding]["intra_min"] = str(1e5) # NOTE: the inter-animal distances are currently not used, but are interesting to compare to intra_* if len(ds_across) > 0: inferenceboundscfg[edgeencoding]["inter_max"] = str( round(np.nanmax(ds_across), numdigits)) inferenceboundscfg[edgeencoding]["inter_min"] = str( round(np.nanmin(ds_across), numdigits)) else: inferenceboundscfg[edgeencoding]["inter_max"] = str( 1e5 ) # large number (larger than image diameters in typical experiments) inferenceboundscfg[edgeencoding]["inter_min"] = str(1e5) # print(inferenceboundscfg) # plt.subplot(len(partaffinityfield_graph),1,pi+1) # plt.hist(ds_within,bins=np.linspace(0,100,21),color='red') # plt.hist(ds_across,bins=np.linspace(0,100,21),color='blue') auxiliaryfunctions.write_plainconfig(str(path_inferencebounds_config), dict(inferenceboundscfg)) return inferenceboundscfg else: print("You might as well bring owls to Athens.") return {}
def merge_annotateddatasets(cfg, trainingsetfolder_full, windows2linux): """ Merges all the h5 files for all labeled-datasets (from individual videos). This is a bit of a mess because of cross platform compatibility. Within platform comp. is straightforward. But if someone labels on windows and wants to train on a unix cluster or colab... """ AnnotationData = [] data_path = Path(os.path.join(cfg["project_path"], "labeled-data")) videos = cfg["video_sets"].keys() for video in videos: _, filename, _ = _robust_path_split(video) file_path = os.path.join(data_path / filename, f'CollectedData_{cfg["scorer"]}.h5') try: data = pd.read_hdf(file_path, "df_with_missing") AnnotationData.append(data) except FileNotFoundError: print( file_path, " not found (perhaps not annotated). If training on cropped data, " "make sure to call `cropimagesandlabels` prior to creating the dataset.", ) if not len(AnnotationData): print( "Annotation data was not found by splitting video paths (from config['video_sets']). An alternative route is taken..." ) AnnotationData = conversioncode.merge_windowsannotationdataONlinuxsystem( cfg) if not len(AnnotationData): print("No data was found!") return AnnotationData = pd.concat(AnnotationData).sort_index() # When concatenating DataFrames with misaligned column labels, # all sorts of reordering may happen (mainly depending on 'sort' and 'join') # Ensure the 'bodyparts' level agrees with the order in the config file. if cfg.get("multianimalproject", False): ( _, uniquebodyparts, multianimalbodyparts, ) = auxfun_multianimal.extractindividualsandbodyparts(cfg) bodyparts = multianimalbodyparts + uniquebodyparts else: bodyparts = cfg["bodyparts"] AnnotationData = AnnotationData.reindex( bodyparts, axis=1, level=AnnotationData.columns.names.index("bodyparts")) # Let's check if the code is *not* run on windows (Source: #https://stackoverflow.com/questions/1325581/how-do-i-check-if-im-running-on-windows-in-python) # but the paths are in windows format... windowspath = "\\" in AnnotationData.index[0] if os.name != "nt" and windowspath and not windows2linux: print( "It appears that the images were labeled on a Windows system, but you are currently trying to create a training set on a Unix system. \n In this case the paths should be converted. Do you want to proceed with the conversion?" ) askuser = input("yes/no") else: askuser = "******" filename = os.path.join(trainingsetfolder_full, f'CollectedData_{cfg["scorer"]}') if (windows2linux or askuser == "yes" or askuser == "y" or askuser == "Ja"): # convert windows path in pandas array \\ to unix / ! AnnotationData = conversioncode.convertpaths_to_unixstyle( AnnotationData, filename) print("Annotation data converted to unix format...") else: # store as is AnnotationData.to_hdf(filename + ".h5", key="df_with_missing", mode="w") AnnotationData.to_csv(filename + ".csv") # human readable. return AnnotationData
) print("Config edited.") print("Extracting frames...") deeplabcut.extract_frames(config_path, mode="automatic", userfeedback=False) print("Frames extracted.") print("Creating artificial data...") rel_folder = os.path.join("labeled-data", os.path.splitext(video)[0]) image_folder = os.path.join(cfg["project_path"], rel_folder) n_animals = len(cfg["individuals"]) ( animals, bodyparts_single, bodyparts_multi, ) = auxfun_multianimal.extractindividualsandbodyparts(cfg) animals_id = [i for i in range(n_animals) for _ in bodyparts_multi] + [ n_animals ] * len(bodyparts_single) map_ = dict(zip(range(len(animals)), animals)) individuals = [map_[ind] for ind in animals_id for _ in range(2)] scorer = [SCORER] * len(individuals) coords = ["x", "y"] * len(animals_id) bodyparts = [ bp for _ in range(n_animals) for bp in bodyparts_multi for _ in range(2) ] bodyparts += [bp for bp in bodyparts_single for _ in range(2)] columns = pd.MultiIndex.from_arrays( [scorer, individuals, bodyparts, coords], names=["scorer", "individuals", "bodyparts", "coords"], )
def create_multianimaltraining_dataset( config, num_shuffles=1, Shuffles=None, windows2linux=False, net_type=None, numdigits=2, paf_graph=None, ): """ Creates a training dataset for multi-animal datasets. Labels from all the extracted frames are merged into a single .h5 file.\n Only the videos included in the config file are used to create this dataset.\n [OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project. Imporant differences to standard: - stores coordinates with numdigits as many digits - creates Parameter ---------- config : string Full path of the config.yaml file as a string. num_shuffles : int, optional Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1. Shuffles: list of shuffles. Alternatively the user can also give a list of shuffles (integers!). windows2linux: bool. The annotation files contain path formated according to your operating system. If you label on windows but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths. net_type: string Type of networks. Currently resnet_50, resnet_101, and resnet_152, efficientnet-b0, efficientnet-b1, efficientnet-b2, efficientnet-b3, efficientnet-b4, efficientnet-b5, and efficientnet-b6 as well as dlcrnet_ms5 are supported (not the MobileNets!). See Lauer et al. 2021 https://www.biorxiv.org/content/10.1101/2021.04.30.442096v1 numdigits: int, optional paf_graph: list of lists, optional (default=None) If not None, overwrite the default complete graph. This is useful for advanced users who already know a good graph, or simply want to use a specific one. Note that, in that case, the data-driven selection procedure upon model evaluation will be skipped. Example -------- >>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1) Windows: >>> deeplabcut.create_multianimaltraining_dataset(r'C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5]) -------- """ # Loading metadata from config file: cfg = auxiliaryfunctions.read_config(config) scorer = cfg["scorer"] project_path = cfg["project_path"] # Create path for training sets & store data there trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) full_training_path = Path(project_path, trainingsetfolder) auxiliaryfunctions.attempttomakefolder(full_training_path, recursive=True) Data = merge_annotateddatasets(cfg, full_training_path, windows2linux) if Data is None: return Data = Data[scorer] def strip_cropped_image_name(path): # utility function to split different crops from same image into either train or test! head, filename = os.path.split(path) if cfg["croppedtraining"]: filename = filename.split("c")[0] return os.path.join(head, filename) img_names = Data.index.map(strip_cropped_image_name).unique() if net_type is None: # loading & linking pretrained models net_type = cfg.get("default_net_type", "dlcrnet_ms5") elif not any(net in net_type for net in ("resnet", "eff", "dlc")): raise ValueError(f"Unsupported network {net_type}.") multi_stage = False if net_type == "dlcrnet_ms5": net_type = "resnet_50" multi_stage = True dataset_type = "multi-animal-imgaug" ( individuals, uniquebodyparts, multianimalbodyparts, ) = auxfun_multianimal.extractindividualsandbodyparts(cfg) if paf_graph is None: # Automatically form a complete PAF graph partaffinityfield_graph = [ list(edge) for edge in combinations(range(len(multianimalbodyparts)), 2) ] else: # Ignore possible connections between 'multi' and 'unique' body parts; # one can never be too careful... to_ignore = auxfun_multianimal.filter_unwanted_paf_connections( cfg, paf_graph) partaffinityfield_graph = [ edge for i, edge in enumerate(paf_graph) if i not in to_ignore ] auxfun_multianimal.validate_paf_graph(cfg, partaffinityfield_graph) print("Utilizing the following graph:", partaffinityfield_graph) partaffinityfield_predict = True # Loading the encoder (if necessary downloading from TF) dlcparent_path = auxiliaryfunctions.get_deeplabcut_path() defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml") model_path, num_shuffles = auxfun_models.Check4weights( net_type, Path(dlcparent_path), num_shuffles) if Shuffles is None: Shuffles = range(1, num_shuffles + 1, 1) else: Shuffles = [i for i in Shuffles if isinstance(i, int)] TrainingFraction = cfg["TrainingFraction"] for shuffle in Shuffles: # Creating shuffles starting from 1 for trainFraction in TrainingFraction: train_inds_temp, test_inds_temp = SplitTrials( range(len(img_names)), trainFraction) # Map back to the original indices. temp = [ re.escape(name) for i, name in enumerate(img_names) if i in test_inds_temp ] mask = Data.index.str.contains("|".join(temp)) testIndices = np.flatnonzero(mask) trainIndices = np.flatnonzero(~mask) #################################################### # Generating data structure with labeled information & frame metadata (for deep cut) #################################################### print( "Creating training data for: Shuffle:", shuffle, "TrainFraction: ", trainFraction, ) # Make training file! data = format_multianimal_training_data( Data, trainIndices, cfg["project_path"], numdigits, ) if len(trainIndices) > 0: ( datafilename, metadatafilename, ) = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) ################################################################################ # Saving metadata and data file (Pickle file) ################################################################################ auxiliaryfunctions.SaveMetadata( os.path.join(project_path, metadatafilename), data, trainIndices, testIndices, trainFraction, ) datafilename = datafilename.split(".mat")[0] + ".pickle" import pickle with open(os.path.join(project_path, datafilename), "wb") as f: # Pickle the 'labeled-data' dictionary using the highest protocol available. pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) ################################################################################ # Creating file structure for training & # Test files as well as pose_yaml files (containing training and testing information) ################################################################################# modelfoldername = auxiliaryfunctions.GetModelFolder( trainFraction, shuffle, cfg) auxiliaryfunctions.attempttomakefolder( Path(config).parents[0] / modelfoldername, recursive=True) auxiliaryfunctions.attempttomakefolder( str(Path(config).parents[0] / modelfoldername / "train")) auxiliaryfunctions.attempttomakefolder( str(Path(config).parents[0] / modelfoldername / "test")) path_train_config = str( os.path.join( cfg["project_path"], Path(modelfoldername), "train", "pose_cfg.yaml", )) path_test_config = str( os.path.join( cfg["project_path"], Path(modelfoldername), "test", "pose_cfg.yaml", )) path_inference_config = str( os.path.join( cfg["project_path"], Path(modelfoldername), "test", "inference_cfg.yaml", )) jointnames = [str(bpt) for bpt in multianimalbodyparts] jointnames.extend([str(bpt) for bpt in uniquebodyparts]) items2change = { "dataset": datafilename, "metadataset": metadatafilename, "num_joints": len(multianimalbodyparts) + len(uniquebodyparts), # cfg["uniquebodyparts"]), "all_joints": [[i] for i in range( len(multianimalbodyparts) + len(uniquebodyparts)) ], # cfg["uniquebodyparts"]))], "all_joints_names": jointnames, "init_weights": model_path, "project_path": str(cfg["project_path"]), "net_type": net_type, "multi_stage": multi_stage, "pairwise_loss_weight": 0.1, "pafwidth": 20, "partaffinityfield_graph": partaffinityfield_graph, "partaffinityfield_predict": partaffinityfield_predict, "weigh_only_present_joints": False, "num_limbs": len(partaffinityfield_graph), "dataset_type": dataset_type, "optimizer": "adam", "batch_size": 8, "multi_step": [[1e-4, 7500], [5 * 1e-5, 12000], [1e-5, 200000]], "save_iters": 10000, "display_iters": 500, "num_idchannel": len(cfg["individuals"]) if cfg.get("identity", False) else 0, } trainingdata = MakeTrain_pose_yaml(items2change, path_train_config, defaultconfigfile) keys2save = [ "dataset", "num_joints", "all_joints", "all_joints_names", "net_type", "multi_stage", "init_weights", "global_scale", "location_refinement", "locref_stdev", "dataset_type", "partaffinityfield_predict", "pairwise_predict", "partaffinityfield_graph", "num_limbs", "dataset_type", "num_idchannel", ] MakeTest_pose_yaml( trainingdata, keys2save, path_test_config, nmsradius=5.0, minconfidence=0.01, ) # setting important def. values for inference # Setting inference cfg file: defaultinference_configfile = os.path.join( dlcparent_path, "inference_cfg.yaml") items2change = { "minimalnumberofconnections": int(len(cfg["multianimalbodyparts"]) / 2), "topktoretain": len(cfg["individuals"]) + 1 * (len(cfg["uniquebodyparts"]) > 0), "withid": cfg.get("identity", False), } MakeInference_yaml(items2change, path_inference_config, defaultinference_configfile) print( "The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!" ) else: pass
def create_multianimaltraining_dataset( config, num_shuffles=1, Shuffles=None, windows2linux=False, net_type=None, numdigits=2, crop_size=(400, 400), crop_sampling="hybrid", paf_graph=None, trainIndices=None, testIndices=None, n_edges_threshold=105, paf_graph_degree=6, ): """ Creates a training dataset for multi-animal datasets. Labels from all the extracted frames are merged into a single .h5 file.\n Only the videos included in the config file are used to create this dataset.\n [OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project. Imporant differences to standard: - stores coordinates with numdigits as many digits - creates Parameter ---------- config : string Full path of the config.yaml file as a string. num_shuffles : int, optional Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1. Shuffles: list of shuffles. Alternatively the user can also give a list of shuffles (integers!). net_type: string Type of networks. Currently resnet_50, resnet_101, and resnet_152, efficientnet-b0, efficientnet-b1, efficientnet-b2, efficientnet-b3, efficientnet-b4, efficientnet-b5, and efficientnet-b6 as well as dlcrnet_ms5 are supported (not the MobileNets!). See Lauer et al. 2021 https://www.biorxiv.org/content/10.1101/2021.04.30.442096v1 numdigits: int, optional crop_size: tuple of int, optional Dimensions (width, height) of the crops for data augmentation. Default is 400x400. crop_sampling: str, optional Crop centers sampling method. Must be either: "uniform" (randomly over the image), "keypoints" (randomly over the annotated keypoints), "density" (weighing preferentially dense regions of keypoints), or "hybrid" (alternating randomly between "uniform" and "density"). Default is "hybrid". paf_graph: list of lists, or "config" optional (default=None) If not None, overwrite the default complete graph. This is useful for advanced users who already know a good graph, or simply want to use a specific one. Note that, in that case, the data-driven selection procedure upon model evaluation will be skipped. "config" will use the skeleton defined in the config file. trainIndices: list of lists, optional (default=None) List of one or multiple lists containing train indexes. A list containing two lists of training indexes will produce two splits. testIndices: list of lists, optional (default=None) List of one or multiple lists containing test indexes. n_edges_threshold: int, optional (default=105) Number of edges above which the graph is automatically pruned. paf_graph_degree: int, optional (default=6) Degree of paf_graph when automatically pruning it (before training). Example -------- >>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1) >>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml', Shuffles=[0,1,2], trainIndices=[trainInd1, trainInd2, trainInd3], testIndices=[testInd1, testInd2, testInd3]) Windows: >>> deeplabcut.create_multianimaltraining_dataset(r'C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5]) -------- """ if windows2linux: warnings.warn( "`windows2linux` has no effect since 2.2.0.4 and will be removed in 2.2.1.", FutureWarning, ) if len(crop_size) != 2 or not all(isinstance(v, int) for v in crop_size): raise ValueError( "Crop size must be a tuple of two integers (width, height).") if crop_sampling not in ("uniform", "keypoints", "density", "hybrid"): raise ValueError( f"Invalid sampling {crop_sampling}. Must be " f"either 'uniform', 'keypoints', 'density', or 'hybrid.") # Loading metadata from config file: cfg = auxiliaryfunctions.read_config(config) scorer = cfg["scorer"] project_path = cfg["project_path"] # Create path for training sets & store data there trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) full_training_path = Path(project_path, trainingsetfolder) auxiliaryfunctions.attempttomakefolder(full_training_path, recursive=True) Data = merge_annotateddatasets(cfg, full_training_path) if Data is None: return Data = Data[scorer] if net_type is None: # loading & linking pretrained models net_type = cfg.get("default_net_type", "dlcrnet_ms5") elif not any(net in net_type for net in ("resnet", "eff", "dlc", "mob")): raise ValueError(f"Unsupported network {net_type}.") multi_stage = False ### dlcnet_ms5: backbone resnet50 + multi-fusion & multi-stage module ### dlcr101_ms5/dlcr152_ms5: backbone resnet101/152 + multi-fusion & multi-stage module if all(net in net_type for net in ("dlcr", "_ms5")): num_layers = re.findall("dlcr([0-9]*)", net_type)[0] if num_layers == "": num_layers = 50 net_type = "resnet_{}".format(num_layers) multi_stage = True dataset_type = "multi-animal-imgaug" ( individuals, uniquebodyparts, multianimalbodyparts, ) = auxfun_multianimal.extractindividualsandbodyparts(cfg) if paf_graph is None: # Automatically form a complete PAF graph n_bpts = len(multianimalbodyparts) partaffinityfield_graph = [ list(edge) for edge in combinations(range(n_bpts), 2) ] n_edges_orig = len(partaffinityfield_graph) # If the graph is unnecessarily large (with 15+ keypoints by default), # we randomly prune it to a size guaranteeing an average node degree of 6; # see Suppl. Fig S9c in Lauer et al., 2022. if n_edges_orig >= n_edges_threshold: partaffinityfield_graph = auxfun_multianimal.prune_paf_graph( partaffinityfield_graph, average_degree=paf_graph_degree, ) else: if paf_graph == "config": # Use the skeleton defined in the config file skeleton = cfg["skeleton"] paf_graph = [ sorted((multianimalbodyparts.index(bpt1), multianimalbodyparts.index(bpt2))) for bpt1, bpt2 in skeleton ] print( "Using `skeleton` from the config file as a paf_graph. Data-driven skeleton will not be computed." ) # Ignore possible connections between 'multi' and 'unique' body parts; # one can never be too careful... to_ignore = auxfun_multianimal.filter_unwanted_paf_connections( cfg, paf_graph) partaffinityfield_graph = [ edge for i, edge in enumerate(paf_graph) if i not in to_ignore ] auxfun_multianimal.validate_paf_graph(cfg, partaffinityfield_graph) print("Utilizing the following graph:", partaffinityfield_graph) # Disable the prediction of PAFs if the graph is empty partaffinityfield_predict = bool(partaffinityfield_graph) # Loading the encoder (if necessary downloading from TF) dlcparent_path = auxiliaryfunctions.get_deeplabcut_path() defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml") model_path, num_shuffles = auxfun_models.Check4weights( net_type, Path(dlcparent_path), num_shuffles) if Shuffles is None: Shuffles = range(1, num_shuffles + 1, 1) else: Shuffles = [i for i in Shuffles if isinstance(i, int)] # print(trainIndices,testIndices, Shuffles, augmenter_type,net_type) if trainIndices is None and testIndices is None: splits = [] for shuffle in Shuffles: # Creating shuffles starting from 1 for train_frac in cfg["TrainingFraction"]: train_inds, test_inds = SplitTrials(range(len(Data)), train_frac) splits.append((train_frac, shuffle, (train_inds, test_inds))) else: if len(trainIndices) != len(testIndices) != len(Shuffles): raise ValueError( "Number of Shuffles and train and test indexes should be equal." ) splits = [] for shuffle, (train_inds, test_inds) in enumerate(zip(trainIndices, testIndices)): trainFraction = round( len(train_inds) * 1.0 / (len(train_inds) + len(test_inds)), 2) print( f"You passed a split with the following fraction: {int(100 * trainFraction)}%" ) # Now that the training fraction is guaranteed to be correct, # the values added to pad the indices are removed. train_inds = np.asarray(train_inds) train_inds = train_inds[train_inds != -1] test_inds = np.asarray(test_inds) test_inds = test_inds[test_inds != -1] splits.append( (trainFraction, Shuffles[shuffle], (train_inds, test_inds))) for trainFraction, shuffle, (trainIndices, testIndices) in splits: #################################################### # Generating data structure with labeled information & frame metadata (for deep cut) #################################################### print( "Creating training data for: Shuffle:", shuffle, "TrainFraction: ", trainFraction, ) # Make training file! data = format_multianimal_training_data( Data, trainIndices, cfg["project_path"], numdigits, ) if len(trainIndices) > 0: ( datafilename, metadatafilename, ) = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) ################################################################################ # Saving metadata and data file (Pickle file) ################################################################################ auxiliaryfunctions.SaveMetadata( os.path.join(project_path, metadatafilename), data, trainIndices, testIndices, trainFraction, ) datafilename = datafilename.split(".mat")[0] + ".pickle" import pickle with open(os.path.join(project_path, datafilename), "wb") as f: # Pickle the 'labeled-data' dictionary using the highest protocol available. pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) ################################################################################ # Creating file structure for training & # Test files as well as pose_yaml files (containing training and testing information) ################################################################################# modelfoldername = auxiliaryfunctions.GetModelFolder( trainFraction, shuffle, cfg) auxiliaryfunctions.attempttomakefolder(Path(config).parents[0] / modelfoldername, recursive=True) auxiliaryfunctions.attempttomakefolder( str(Path(config).parents[0] / modelfoldername / "train")) auxiliaryfunctions.attempttomakefolder( str(Path(config).parents[0] / modelfoldername / "test")) path_train_config = str( os.path.join( cfg["project_path"], Path(modelfoldername), "train", "pose_cfg.yaml", )) path_test_config = str( os.path.join( cfg["project_path"], Path(modelfoldername), "test", "pose_cfg.yaml", )) path_inference_config = str( os.path.join( cfg["project_path"], Path(modelfoldername), "test", "inference_cfg.yaml", )) jointnames = [str(bpt) for bpt in multianimalbodyparts] jointnames.extend([str(bpt) for bpt in uniquebodyparts]) items2change = { "dataset": datafilename, "metadataset": metadatafilename, "num_joints": len(multianimalbodyparts) + len(uniquebodyparts), # cfg["uniquebodyparts"]), "all_joints": [[i] for i in range( len(multianimalbodyparts) + len(uniquebodyparts)) ], # cfg["uniquebodyparts"]))], "all_joints_names": jointnames, "init_weights": model_path, "project_path": str(cfg["project_path"]), "net_type": net_type, "multi_stage": multi_stage, "pairwise_loss_weight": 0.1, "pafwidth": 20, "partaffinityfield_graph": partaffinityfield_graph, "partaffinityfield_predict": partaffinityfield_predict, "weigh_only_present_joints": False, "num_limbs": len(partaffinityfield_graph), "dataset_type": dataset_type, "optimizer": "adam", "batch_size": 8, "multi_step": [[1e-4, 7500], [5 * 1e-5, 12000], [1e-5, 200000]], "save_iters": 10000, "display_iters": 500, "num_idchannel": len(cfg["individuals"]) if cfg.get("identity", False) else 0, "crop_size": list(crop_size), "crop_sampling": crop_sampling, } trainingdata = MakeTrain_pose_yaml(items2change, path_train_config, defaultconfigfile) keys2save = [ "dataset", "num_joints", "all_joints", "all_joints_names", "net_type", "multi_stage", "init_weights", "global_scale", "location_refinement", "locref_stdev", "dataset_type", "partaffinityfield_predict", "pairwise_predict", "partaffinityfield_graph", "num_limbs", "dataset_type", "num_idchannel", ] MakeTest_pose_yaml( trainingdata, keys2save, path_test_config, nmsradius=5.0, minconfidence=0.01, sigma=1, locref_smooth=False, ) # setting important def. values for inference # Setting inference cfg file: defaultinference_configfile = os.path.join(dlcparent_path, "inference_cfg.yaml") items2change = { "minimalnumberofconnections": int(len(cfg["multianimalbodyparts"]) / 2), "topktoretain": len(cfg["individuals"]) + 1 * (len(cfg["uniquebodyparts"]) > 0), "withid": cfg.get("identity", False), } MakeInference_yaml(items2change, path_inference_config, defaultinference_configfile) print( "The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!" ) else: pass
def __init__(self, parent, config, video, shuffle, Dataframe, savelabeled, multianimal): super(MainFrame, self).__init__("DeepLabCut2.0 - Manual Outlier Frame Extraction", parent) ################################################################################################################################################### # Spliting the frame into top and bottom panels. Bottom panels contains the widgets. The top panel is for showing images and plotting! # topSplitter = wx.SplitterWindow(self) # # self.image_panel = ImagePanel(topSplitter, config,video,shuffle,Dataframe,self.gui_size) # self.widget_panel = WidgetPanel(topSplitter) # # topSplitter.SplitHorizontally(self.image_panel, self.widget_panel,sashPosition=self.gui_size[1]*0.83)#0.9 # topSplitter.SetSashGravity(1) # sizer = wx.BoxSizer(wx.VERTICAL) # sizer.Add(topSplitter, 1, wx.EXPAND) # self.SetSizer(sizer) # Spliting the frame into top and bottom panels. Bottom panels contains the widgets. The top panel is for showing images and plotting! topSplitter = wx.SplitterWindow(self) vSplitter = wx.SplitterWindow(topSplitter) self.image_panel = ImagePanel(vSplitter, config, self.gui_size) self.choice_panel = ScrollPanel(vSplitter) vSplitter.SplitVertically(self.image_panel, self.choice_panel, sashPosition=self.gui_size[0] * 0.8) vSplitter.SetSashGravity(1) self.widget_panel = WidgetPanel(topSplitter) topSplitter.SplitHorizontally(vSplitter, self.widget_panel, sashPosition=self.gui_size[1] * 0.83) # 0.9 topSplitter.SetSashGravity(1) sizer = wx.BoxSizer(wx.VERTICAL) sizer.Add(topSplitter, 1, wx.EXPAND) self.SetSizer(sizer) ################################################################################################################################################### # Add Buttons to the WidgetPanel and bind them to their respective functions. widgetsizer = wx.WrapSizer(orient=wx.HORIZONTAL) self.load_button_sizer = wx.BoxSizer(wx.VERTICAL) self.help_button_sizer = wx.BoxSizer(wx.VERTICAL) self.help = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Help") self.help_button_sizer.Add(self.help, 1, wx.ALL, 15) # widgetsizer.Add(self.help , 1, wx.ALL, 15) self.help.Bind(wx.EVT_BUTTON, self.helpButton) widgetsizer.Add(self.help_button_sizer, 1, wx.ALL, 0) self.grab = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Grab Frames") widgetsizer.Add(self.grab, 1, wx.ALL, 15) self.grab.Bind(wx.EVT_BUTTON, self.grabFrame) self.grab.Enable(True) widgetsizer.AddStretchSpacer(5) self.slider = wx.Slider( self.widget_panel, id=wx.ID_ANY, value=0, minValue=0, maxValue=1, size=(200, -1), style=wx.SL_HORIZONTAL | wx.SL_AUTOTICKS | wx.SL_LABELS, ) widgetsizer.Add(self.slider, 1, wx.ALL, 5) self.slider.Bind(wx.EVT_SLIDER, self.OnSliderScroll) widgetsizer.AddStretchSpacer(5) self.start_frames_sizer = wx.BoxSizer(wx.VERTICAL) self.end_frames_sizer = wx.BoxSizer(wx.VERTICAL) self.start_frames_sizer.AddSpacer(15) # self.startFrame = wx.SpinCtrl(self.widget_panel, value='0', size=(100, -1), min=0, max=120) self.startFrame = wx.SpinCtrl(self.widget_panel, value="0", size=(100, -1)) # ,style=wx.SP_VERTICAL) self.startFrame.Enable(False) self.start_frames_sizer.Add(self.startFrame, 1, wx.EXPAND | wx.ALIGN_LEFT, 15) start_text = wx.StaticText(self.widget_panel, label="Start Frame Index") self.start_frames_sizer.Add(start_text, 1, wx.EXPAND | wx.ALIGN_LEFT, 15) self.checkBox = wx.CheckBox(self.widget_panel, id=wx.ID_ANY, label="Range of frames") self.checkBox.Bind(wx.EVT_CHECKBOX, self.activate_frame_range) self.start_frames_sizer.Add(self.checkBox, 1, wx.EXPAND | wx.ALIGN_LEFT, 15) # self.end_frames_sizer.AddSpacer(15) self.endFrame = wx.SpinCtrl(self.widget_panel, value="1", size=(160, -1)) # , min=1, max=120) self.endFrame.Enable(False) self.end_frames_sizer.Add(self.endFrame, 1, wx.EXPAND | wx.ALIGN_LEFT, 15) end_text = wx.StaticText(self.widget_panel, label="Number of Frames") self.end_frames_sizer.Add(end_text, 1, wx.EXPAND | wx.ALIGN_LEFT, 15) self.updateFrame = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Update") self.end_frames_sizer.Add(self.updateFrame, 1, wx.EXPAND | wx.ALIGN_LEFT, 15) self.updateFrame.Bind(wx.EVT_BUTTON, self.updateSlider) self.updateFrame.Enable(False) widgetsizer.Add(self.start_frames_sizer, 1, wx.ALL, 0) widgetsizer.AddStretchSpacer(5) widgetsizer.Add(self.end_frames_sizer, 1, wx.ALL, 0) widgetsizer.AddStretchSpacer(15) self.quit = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Quit") widgetsizer.Add(self.quit, 1, wx.ALL, 15) self.quit.Bind(wx.EVT_BUTTON, self.quitButton) self.quit.Enable(True) self.widget_panel.SetSizer(widgetsizer) self.widget_panel.SetSizerAndFit(widgetsizer) # Variables initialization self.numberFrames = 0 self.currFrame = 0 self.figure = Figure() self.axes = self.figure.add_subplot(111) self.drs = [] self.extract_range_frame = False self.firstFrame = 0 self.Colorscheme = [] # Read confing file self.cfg = auxiliaryfunctions.read_config(config) self.Task = self.cfg["Task"] self.start = self.cfg["start"] self.stop = self.cfg["stop"] self.date = self.cfg["date"] self.trainFraction = self.cfg["TrainingFraction"] self.trainFraction = self.trainFraction[0] self.videos = self.cfg["video_sets"].keys() self.bodyparts = self.cfg["bodyparts"] self.colormap = plt.get_cmap(self.cfg["colormap"]) self.colormap = self.colormap.reversed() self.markerSize = self.cfg["dotsize"] self.alpha = self.cfg["alphavalue"] self.iterationindex = self.cfg["iteration"] self.cropping = self.cfg["cropping"] self.video_names = [Path(i).stem for i in self.videos] self.config_path = Path(config) self.video_source = Path(video).resolve() self.shuffle = shuffle self.Dataframe = Dataframe conversioncode.guarantee_multiindex_rows(self.Dataframe) self.savelabeled = savelabeled self.multianimal = multianimal if self.multianimal: from deeplabcut.utils import auxfun_multianimal ( self.individual_names, self.uniquebodyparts, self.multianimalbodyparts, ) = auxfun_multianimal.extractindividualsandbodyparts(self.cfg) self.choiceBox, self.visualization_rdb = self.choice_panel.addRadioButtons( ) self.Colorscheme = visualization.get_cmap( len(self.individual_names), self.cfg["colormap"]) self.visualization_rdb.Bind(wx.EVT_RADIOBOX, self.clear_plot) # Read the video file self.vid = VideoWriter(str(self.video_source)) if self.cropping: self.vid.set_bbox(self.cfg["x1"], self.cfg["x2"], self.cfg["y1"], self.cfg["y2"]) self.filename = Path(self.video_source).name self.numberFrames = len(self.vid) self.strwidth = int(np.ceil(np.log10(self.numberFrames))) # Set the values of slider and range of frames self.startFrame.SetMax(self.numberFrames - 1) self.slider.SetMax(self.numberFrames - 1) self.endFrame.SetMax(self.numberFrames - 1) self.startFrame.Bind(wx.EVT_SPINCTRL, self.updateSlider) # wx.EVT_SPIN # Set the status bar self.statusbar.SetStatusText("Working on video: {}".format( self.filename)) # Adding the video file to the config file. if self.vid.name not in self.video_names: add.add_new_videos(self.config_path, [self.video_source]) self.update() self.plot_labels() self.widget_panel.Layout()
def browseDir(self, event): """ Show the DirDialog and ask the user to change the directory where machine labels are stored """ self.statusbar.SetStatusText( "Looking for a folder to start labeling...") cwd = os.path.join(os.getcwd(), "labeled-data") dlg = wx.DirDialog( self, "Choose the directory where your extracted frames are saved:", cwd, style=wx.DD_DEFAULT_STYLE, ) if dlg.ShowModal() == wx.ID_OK: self.dir = dlg.GetPath() self.load.Enable(False) self.next.Enable(True) self.save.Enable(True) else: dlg.Destroy() self.Close(True) return dlg.Destroy() # Enabling the zoom, pan and home buttons self.zoom.Enable(True) self.home.Enable(True) self.pan.Enable(True) self.lock.Enable(True) # Reading config file and its variables self.cfg = auxiliaryfunctions.read_config(self.config_file) self.scorer = self.cfg["scorer"] ( individuals, uniquebodyparts, multianimalbodyparts, ) = auxfun_multianimal.extractindividualsandbodyparts(self.cfg) self.multibodyparts = multianimalbodyparts self.uniquebodyparts = uniquebodyparts self.individual_names = individuals self.videos = self.cfg["video_sets"].keys() self.markerSize = self.cfg["dotsize"] self.edgewidth = self.markerSize // 3 self.alpha = self.cfg["alphavalue"] self.colormap = plt.get_cmap(self.cfg["colormap"]) self.colormap = self.colormap.reversed() self.idmap = plt.cm.get_cmap("Set1", len(individuals)) self.project_path = self.cfg["project_path"] if self.uniquebodyparts == []: self.are_unique_bodyparts_present = False self.buttonCounter = {i: [] for i in self.individual_names} self.index = np.sort([ fn for fn in glob.glob(os.path.join(self.dir, "*.png")) if ("labeled.png" not in fn) ]) self.statusbar.SetStatusText("Working on folder: {}".format( os.path.split(str(self.dir))[-1])) self.relativeimagenames = [ "labeled" + n.split("labeled")[1] for n in self.index ] # [n.split(self.project_path+'/')[1] for n in self.index] # Reading the existing dataset,if already present try: self.dataFrame = pd.read_hdf( os.path.join(self.dir, "CollectedData_" + self.scorer + ".h5"), "df_with_missing", ) self.dataFrame.sort_index(inplace=True) self.prev.Enable(True) # Finds the first empty row in the dataframe and sets the iteration to that index for idx, j in enumerate(self.dataFrame.index): values = self.dataFrame.loc[j, :].values if np.prod(np.isnan(values)) == 1: self.iter = idx break else: self.iter = idx except: # Create an empty data frame self.dataFrame = MainFrame.create_dataframe( self, self.dataFrame, self.relativeimagenames, self.individual_names, self.uniquebodyparts, self.multibodyparts, ) self.iter = 0 # Reading the image name self.img = self.index[self.iter] img_name = Path(self.index[self.iter]).name # Checking for new frames and adding them to the existing dataframe old_imgs = np.sort(list(self.dataFrame.index)) self.newimages = list(set(self.relativeimagenames) - set(old_imgs)) if self.newimages == []: pass else: print("Found new frames..") # Create an empty dataframe with all the new images and then merge this to the existing dataframe. self.df = MainFrame.create_dataframe( self, None, self.newimages, self.individual_names, self.uniquebodyparts, self.multibodyparts, ) self.dataFrame = pd.concat([self.dataFrame, self.df], axis=0) # Sort it by the index values self.dataFrame.sort_index(inplace=True) # checks for unique bodyparts if len(self.multibodyparts) != len(set(self.multibodyparts)): print( "Error - bodyparts must have unique labels! Please choose unique bodyparts in config.yaml file and try again. Quitting for now!" ) self.Close(True) # Extracting the list of old labels oldMultiBodyParts = self.dataFrame.iloc[:, self.dataFrame.columns. get_level_values(1) == [ i for i in self. individual_names if i != "single" ][0], ].columns.get_level_values(2) _, idx = np.unique(oldMultiBodyParts, return_index=True) oldMultiBodyparts2plot = list(oldMultiBodyParts[np.sort(idx)]) # Extracting the list of old unique labels if self.are_unique_bodyparts_present: oldUniqueBodyParts = self.dataFrame.iloc[:, self.dataFrame.columns. get_level_values(1) == "single"].columns.get_level_values( 2) _, idx_unique = np.unique(oldUniqueBodyParts, return_index=True) oldUniqueBodyparts2plot = list( oldUniqueBodyParts[np.sort(idx_unique)]) self.new_UniqueBodyparts = [ x for x in self.uniquebodyparts if x not in oldUniqueBodyparts2plot ] else: self.new_UniqueBodyparts = [] # Checking if new labels are added or not self.new_Multibodyparts = [ x for x in self.multibodyparts if x not in oldMultiBodyparts2plot ] # Checking if user added a new label if (self.new_Multibodyparts == [] and self.new_UniqueBodyparts == []): # i.e. no new labels ( self.figure, self.axes, self.canvas, self.toolbar, self.image_axis, ) = self.image_panel.drawplot( self.img, img_name, self.iter, self.index, self.multibodyparts, self.colormap, keep_view=self.view_locked, ) else: # Found new labels in either multiple bodyparts or unique bodyparts dlg = wx.MessageDialog( None, "New label found in the config file. Do you want to see all the other labels?", "New label found", wx.YES_NO | wx.ICON_WARNING, ) result = dlg.ShowModal() if result == wx.ID_NO: if self.new_Multibodyparts != []: self.multibodyparts = self.new_Multibodyparts if self.new_UniqueBodyparts != []: self.uniquebodyparts = self.new_UniqueBodyparts self.dataFrame = MainFrame.create_dataframe( self, self.dataFrame, self.relativeimagenames, self.individual_names, self.new_UniqueBodyparts, self.new_Multibodyparts, ) ( self.figure, self.axes, self.canvas, self.toolbar, self.image_axis, ) = self.image_panel.drawplot( self.img, img_name, self.iter, self.index, self.multibodyparts, self.colormap, keep_view=self.view_locked, ) self.axes.callbacks.connect("xlim_changed", self.onZoom) self.axes.callbacks.connect("ylim_changed", self.onZoom) if self.individual_names[0] == "single": ( self.choiceBox, self.individualrdb, self.rdb, self.change_marker_size, self.checkBox, ) = self.choice_panel.addRadioButtons(self.uniquebodyparts, self.individual_names, self.file, self.markerSize) self.image_panel.addcolorbar( self.img, self.image_axis, self.iter, self.uniquebodyparts, self.colormap, ) else: ( self.choiceBox, self.individualrdb, self.rdb, self.change_marker_size, self.checkBox, ) = self.choice_panel.addRadioButtons(self.multibodyparts, self.individual_names, self.file, self.markerSize) self.image_panel.addcolorbar(self.img, self.image_axis, self.iter, self.multibodyparts, self.colormap) self.individualrdb.Bind(wx.EVT_RADIOBOX, self.select_individual) # check if single is slected when radio buttons are changed if self.rdb.GetStringSelection == "single": self.norm, self.colorIndex = self.image_panel.getColorIndices( self.img, self.uniquebodyparts) else: self.norm, self.colorIndex = self.image_panel.getColorIndices( self.img, self.multibodyparts) self.buttonCounter = MainFrame.plot(self, self.img) self.cidClick = self.canvas.mpl_connect("button_press_event", self.onClick) self.cidClick = self.canvas.mpl_connect("button_press_event", self.onClick) self.checkBox.Bind(wx.EVT_CHECKBOX, self.activateSlider) self.change_marker_size.Bind(wx.EVT_SLIDER, self.OnSliderScroll)
def __init__(self, parent, config, video, shuffle, Dataframe, savelabeled, multianimal): # Settting the GUI size and panels design displays = (wx.Display(i) for i in range(wx.Display.GetCount()) ) # Gets the number of displays screenSizes = [ display.GetGeometry().GetSize() for display in displays ] # Gets the size of each display index = 0 # For display 1. screenWidth = screenSizes[index][0] screenHeight = screenSizes[index][1] self.gui_size = (screenWidth * 0.7, screenHeight * 0.85) wx.Frame.__init__( self, parent, id=wx.ID_ANY, title="DeepLabCut2.0 - Manual Outlier Frame Extraction", size=wx.Size(self.gui_size), pos=wx.DefaultPosition, style=wx.RESIZE_BORDER | wx.DEFAULT_FRAME_STYLE | wx.TAB_TRAVERSAL, ) self.statusbar = self.CreateStatusBar() self.statusbar.SetStatusText("") self.SetSizeHints( wx.Size(self.gui_size) ) # This sets the minimum size of the GUI. It can scale now! ################################################################################################################################################### # Spliting the frame into top and bottom panels. Bottom panels contains the widgets. The top panel is for showing images and plotting! # topSplitter = wx.SplitterWindow(self) # # self.image_panel = ImagePanel(topSplitter, config,video,shuffle,Dataframe,self.gui_size) # self.widget_panel = WidgetPanel(topSplitter) # # topSplitter.SplitHorizontally(self.image_panel, self.widget_panel,sashPosition=self.gui_size[1]*0.83)#0.9 # topSplitter.SetSashGravity(1) # sizer = wx.BoxSizer(wx.VERTICAL) # sizer.Add(topSplitter, 1, wx.EXPAND) # self.SetSizer(sizer) # Spliting the frame into top and bottom panels. Bottom panels contains the widgets. The top panel is for showing images and plotting! topSplitter = wx.SplitterWindow(self) vSplitter = wx.SplitterWindow(topSplitter) self.image_panel = ImagePanel(vSplitter, self.gui_size) self.choice_panel = ScrollPanel(vSplitter) vSplitter.SplitVertically(self.image_panel, self.choice_panel, sashPosition=self.gui_size[0] * 0.8) vSplitter.SetSashGravity(1) self.widget_panel = WidgetPanel(topSplitter) topSplitter.SplitHorizontally(vSplitter, self.widget_panel, sashPosition=self.gui_size[1] * 0.83) # 0.9 topSplitter.SetSashGravity(1) sizer = wx.BoxSizer(wx.VERTICAL) sizer.Add(topSplitter, 1, wx.EXPAND) self.SetSizer(sizer) ################################################################################################################################################### # Add Buttons to the WidgetPanel and bind them to their respective functions. widgetsizer = wx.WrapSizer(orient=wx.HORIZONTAL) self.load_button_sizer = wx.BoxSizer(wx.VERTICAL) self.help_button_sizer = wx.BoxSizer(wx.VERTICAL) self.help = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Help") self.help_button_sizer.Add(self.help, 1, wx.ALL, 15) # widgetsizer.Add(self.help , 1, wx.ALL, 15) self.help.Bind(wx.EVT_BUTTON, self.helpButton) widgetsizer.Add(self.help_button_sizer, 1, wx.ALL, 0) self.grab = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Grab Frames") widgetsizer.Add(self.grab, 1, wx.ALL, 15) self.grab.Bind(wx.EVT_BUTTON, self.grabFrame) self.grab.Enable(True) widgetsizer.AddStretchSpacer(5) self.slider = wx.Slider( self.widget_panel, id=wx.ID_ANY, value=0, minValue=0, maxValue=1, size=(200, -1), style=wx.SL_HORIZONTAL | wx.SL_AUTOTICKS | wx.SL_LABELS, ) widgetsizer.Add(self.slider, 1, wx.ALL, 5) self.slider.Bind(wx.EVT_SLIDER, self.OnSliderScroll) widgetsizer.AddStretchSpacer(5) self.start_frames_sizer = wx.BoxSizer(wx.VERTICAL) self.end_frames_sizer = wx.BoxSizer(wx.VERTICAL) self.start_frames_sizer.AddSpacer(15) # self.startFrame = wx.SpinCtrl(self.widget_panel, value='0', size=(100, -1), min=0, max=120) self.startFrame = wx.SpinCtrl(self.widget_panel, value="0", size=(100, -1)) # ,style=wx.SP_VERTICAL) self.startFrame.Enable(False) self.start_frames_sizer.Add(self.startFrame, 1, wx.EXPAND | wx.ALIGN_LEFT, 15) start_text = wx.StaticText(self.widget_panel, label="Start Frame Index") self.start_frames_sizer.Add(start_text, 1, wx.EXPAND | wx.ALIGN_LEFT, 15) self.checkBox = wx.CheckBox(self.widget_panel, id=wx.ID_ANY, label="Range of frames") self.checkBox.Bind(wx.EVT_CHECKBOX, self.activate_frame_range) self.start_frames_sizer.Add(self.checkBox, 1, wx.EXPAND | wx.ALIGN_LEFT, 15) # self.end_frames_sizer.AddSpacer(15) self.endFrame = wx.SpinCtrl(self.widget_panel, value="1", size=(160, -1)) # , min=1, max=120) self.endFrame.Enable(False) self.end_frames_sizer.Add(self.endFrame, 1, wx.EXPAND | wx.ALIGN_LEFT, 15) end_text = wx.StaticText(self.widget_panel, label="Number of Frames") self.end_frames_sizer.Add(end_text, 1, wx.EXPAND | wx.ALIGN_LEFT, 15) self.updateFrame = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Update") self.end_frames_sizer.Add(self.updateFrame, 1, wx.EXPAND | wx.ALIGN_LEFT, 15) self.updateFrame.Bind(wx.EVT_BUTTON, self.updateSlider) self.updateFrame.Enable(False) widgetsizer.Add(self.start_frames_sizer, 1, wx.ALL, 0) widgetsizer.AddStretchSpacer(5) widgetsizer.Add(self.end_frames_sizer, 1, wx.ALL, 0) widgetsizer.AddStretchSpacer(15) self.quit = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Quit") widgetsizer.Add(self.quit, 1, wx.ALL, 15) self.quit.Bind(wx.EVT_BUTTON, self.quitButton) self.quit.Enable(True) self.widget_panel.SetSizer(widgetsizer) self.widget_panel.SetSizerAndFit(widgetsizer) # Variables initialization self.numberFrames = 0 self.currFrame = 0 self.figure = Figure() self.axes = self.figure.add_subplot(111) self.drs = [] self.extract_range_frame = False self.firstFrame = 0 self.Colorscheme = [] # self.cropping = False # Read confing file self.cfg = auxiliaryfunctions.read_config(config) self.Task = self.cfg["Task"] self.start = self.cfg["start"] self.stop = self.cfg["stop"] self.date = self.cfg["date"] self.trainFraction = self.cfg["TrainingFraction"] self.trainFraction = self.trainFraction[0] self.videos = self.cfg["video_sets"].keys() self.bodyparts = self.cfg["bodyparts"] self.colormap = plt.get_cmap(self.cfg["colormap"]) self.colormap = self.colormap.reversed() self.markerSize = self.cfg["dotsize"] self.alpha = self.cfg["alphavalue"] self.iterationindex = self.cfg["iteration"] self.cropping = self.cfg["cropping"] self.video_names = [Path(i).stem for i in self.videos] self.config_path = Path(config) self.video_source = Path(video).resolve() self.shuffle = shuffle self.Dataframe = Dataframe self.savelabeled = savelabeled self.multianimal = multianimal if self.multianimal: from deeplabcut.utils import auxfun_multianimal ( self.individual_names, self.uniquebodyparts, self.multianimalbodyparts, ) = auxfun_multianimal.extractindividualsandbodyparts(self.cfg) self.choiceBox, self.visualization_rdb = self.choice_panel.addRadioButtons( ) self.Colorscheme = visualization.get_cmap( len(self.individual_names), self.cfg["colormap"]) self.visualization_rdb.Bind(wx.EVT_RADIOBOX, self.clear_plot) # Read the video file self.vid = cv2.VideoCapture(str(self.video_source)) self.videoPath = os.path.dirname(self.video_source) self.filename = Path(self.video_source).name self.numberFrames = int(self.vid.get(cv2.CAP_PROP_FRAME_COUNT)) self.strwidth = int(np.ceil(np.log10(self.numberFrames))) # Set the values of slider and range of frames self.startFrame.SetMax(self.numberFrames - 1) self.slider.SetMax(self.numberFrames - 1) self.endFrame.SetMax(self.numberFrames - 1) self.startFrame.Bind(wx.EVT_SPINCTRL, self.updateSlider) # wx.EVT_SPIN # Set the status bar self.statusbar.SetStatusText("Working on video: {}".format( os.path.split(str(self.video_source))[-1])) # Adding the video file to the config file. if not (str(self.video_source.stem) in self.video_names): add.add_new_videos(self.config_path, [self.video_source]) self.filename = Path(self.video_source).name self.update() self.plot_labels() self.widget_panel.Layout()
def _benchmark_paf_graphs( config, inference_cfg, data, paf_inds, greedy=False, add_discarded=True, identity_only=False, calibration_file="", oks_sigma=0.1, ): n_multi = len(auxfun_multianimal.extractindividualsandbodyparts(config)[2]) data_ = {"metadata": data.pop("metadata")} for k, v in data.items(): data_[k] = v["prediction"] ass = Assembler( data_, max_n_individuals=inference_cfg["topktoretain"], n_multibodyparts=n_multi, greedy=greedy, pcutoff=inference_cfg.get("pcutoff", 0.1), min_affinity=inference_cfg.get("pafthreshold", 0.1), add_discarded=add_discarded, identity_only=identity_only, ) if calibration_file: ass.calibrate(calibration_file) params = ass.metadata image_paths = params["imnames"] bodyparts = params["joint_names"] idx = (data[image_paths[0]]["groundtruth"][2].unstack("coords").reindex( bodyparts, level="bodyparts").index) individuals = idx.get_level_values("individuals").unique() n_individuals = len(individuals) map_ = dict(zip(individuals, range(n_individuals))) # Form ground truth beforehand ground_truth = [] for i, imname in enumerate(image_paths): temp = data[imname]["groundtruth"][2] ground_truth.append(temp.to_numpy().reshape((-1, 2))) ground_truth = np.stack(ground_truth) temp = np.ones((*ground_truth.shape[:2], 3)) temp[..., :2] = ground_truth temp = temp.reshape((temp.shape[0], n_individuals, -1, 3)) ass_true_dict = _parse_ground_truth_data(temp) ids = np.vectorize(map_.get)( idx.get_level_values("individuals").to_numpy()) ground_truth = np.insert(ground_truth, 2, ids, axis=2) # Assemble animals on the full set of detections paf_inds = sorted(paf_inds, key=len) paf_graph = ass.graph n_graphs = len(paf_inds) all_scores = [] all_metrics = [] for j, paf in enumerate(paf_inds, start=1): print(f"Graph {j}|{n_graphs}") ass.paf_inds = paf ass.assemble() oks = evaluate_assembly(ass.assemblies, ass_true_dict, oks_sigma) all_metrics.append(oks) scores = np.full((len(image_paths), 2), np.nan) for i, imname in enumerate(tqdm(image_paths)): gt = ground_truth[i] gt = gt[~np.isnan(gt).any(axis=1)] if len(np.unique( gt[:, 2])) < 2: # Only consider frames with 2+ animals continue # Count the number of unassembled bodyparts n_dets = len(gt) animals = ass.assemblies.get(i) if animals is None: if n_dets: scores[i, 0] = 1 else: animals = [ np.c_[animal.data, np.ones(animal.data.shape[0]) * n] for n, animal in enumerate(animals) ] hyp = np.concatenate(animals) hyp = hyp[~np.isnan(hyp).any(axis=1)] scores[i, 0] = (n_dets - hyp.shape[0]) / n_dets neighbors = _find_closest_neighbors(gt[:, :2], hyp[:, :2]) valid = neighbors != -1 id_gt = gt[valid, 2] id_hyp = hyp[neighbors[valid], -1] mat = contingency_matrix(id_gt, id_hyp) purity = mat.max(axis=0).sum() / mat.sum() scores[i, 1] = purity all_scores.append((scores, paf)) dfs = [] for score, inds in all_scores: df = pd.DataFrame(score, columns=["miss", "purity"]) df["ngraph"] = len(inds) dfs.append(df) big_df = pd.concat(dfs) group = big_df.groupby("ngraph") return (all_scores, group.agg(["mean", "std"]).T, all_metrics)
def __init__(self, parent, config): super(MainFrame, self).__init__( "DeepLabCut - Refinement ToolBox", parent, ) self.Bind(wx.EVT_CHAR_HOOK, self.OnKeyPressed) ################################################################################################################################################### # Spliting the frame into top and bottom panels. Bottom panels contains the widgets. The top panel is for showing images and plotting! topSplitter = wx.SplitterWindow(self) vSplitter = wx.SplitterWindow(topSplitter) self.image_panel = ImagePanel(vSplitter, config, self.gui_size) self.choice_panel = ScrollPanel(vSplitter) # self.choice_panel.SetupScrolling(scroll_x=True, scroll_y=True, scrollToTop=False) # self.choice_panel.SetupScrolling(scroll_x=True, scrollToTop=False) vSplitter.SplitVertically(self.image_panel, self.choice_panel, sashPosition=self.gui_size[0] * 0.8) vSplitter.SetSashGravity(1) self.widget_panel = WidgetPanel(topSplitter) topSplitter.SplitHorizontally(vSplitter, self.widget_panel, sashPosition=self.gui_size[1] * 0.83) # 0.9 topSplitter.SetSashGravity(1) sizer = wx.BoxSizer(wx.VERTICAL) sizer.Add(topSplitter, 1, wx.EXPAND) self.SetSizer(sizer) ################################################################################################################################################### # Add Buttons to the WidgetPanel and bind them to their respective functions. widgetsizer = wx.WrapSizer(orient=wx.HORIZONTAL) self.load = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Load labels") widgetsizer.Add(self.load, 1, wx.ALL, 15) self.load.Bind(wx.EVT_BUTTON, self.browseDir) self.prev = wx.Button(self.widget_panel, id=wx.ID_ANY, label="<<Previous") widgetsizer.Add(self.prev, 1, wx.ALL, 15) self.prev.Bind(wx.EVT_BUTTON, self.prevImage) self.prev.Enable(False) self.next = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Next>>") widgetsizer.Add(self.next, 1, wx.ALL, 15) self.next.Bind(wx.EVT_BUTTON, self.nextImage) self.next.Enable(False) self.help = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Help") widgetsizer.Add(self.help, 1, wx.ALL, 15) self.help.Bind(wx.EVT_BUTTON, self.helpButton) self.help.Enable(True) self.zoom = wx.ToggleButton(self.widget_panel, label="Zoom") widgetsizer.Add(self.zoom, 1, wx.ALL, 15) self.zoom.Bind(wx.EVT_TOGGLEBUTTON, self.zoomButton) self.widget_panel.SetSizer(widgetsizer) self.zoom.Enable(False) self.home = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Home") widgetsizer.Add(self.home, 1, wx.ALL, 15) self.home.Bind(wx.EVT_BUTTON, self.homeButton) self.widget_panel.SetSizer(widgetsizer) self.home.Enable(False) self.pan = wx.ToggleButton(self.widget_panel, id=wx.ID_ANY, label="Pan") widgetsizer.Add(self.pan, 1, wx.ALL, 15) self.pan.Bind(wx.EVT_TOGGLEBUTTON, self.panButton) self.widget_panel.SetSizer(widgetsizer) self.pan.Enable(False) self.lock = wx.CheckBox(self.widget_panel, id=wx.ID_ANY, label="Lock View") widgetsizer.Add(self.lock, 1, wx.ALL, 15) self.lock.Bind(wx.EVT_CHECKBOX, self.lockChecked) self.widget_panel.SetSizer(widgetsizer) self.lock.Enable(False) self.save = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Save") widgetsizer.Add(self.save, 1, wx.ALL, 15) self.save.Bind(wx.EVT_BUTTON, self.saveDataSet) self.save.Enable(False) widgetsizer.AddStretchSpacer(15) self.quit = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Quit") widgetsizer.Add(self.quit, 1, wx.ALL, 15) self.quit.Bind(wx.EVT_BUTTON, self.quitButton) self.widget_panel.SetSizer(widgetsizer) self.widget_panel.SetSizerAndFit(widgetsizer) self.widget_panel.Layout() ############################################################################################################################### # Variable initialization self.currentDirectory = os.getcwd() self.index = [] self.iter = [] self.threshold = [] self.file = 0 self.updatedCoords = [] self.drs = [] self.cfg = auxiliaryfunctions.read_config(config) self.humanscorer = self.cfg["scorer"] self.move2corner = self.cfg["move2corner"] self.center = self.cfg["corner2move2"] self.colormap = plt.get_cmap(self.cfg["colormap"]) self.colormap = self.colormap.reversed() self.markerSize = self.cfg["dotsize"] self.alpha = self.cfg["alphavalue"] self.iterationindex = self.cfg["iteration"] self.project_path = self.cfg["project_path"] self.bodyparts = self.cfg["bodyparts"] self.threshold = 0.1 self.img_size = (10, 6) # (imgW, imgH) # width, height in inches. self.preview = False self.view_locked = False # Workaround for MAC - xlim and ylim changed events seem to be triggered too often so need to make sure that the # xlim and ylim have actually changed before turning zoom off self.prezoom_xlim = [] self.prezoom_ylim = [] from deeplabcut.utils import auxfun_multianimal ( self.individual_names, self.uniquebodyparts, self.multianimalbodyparts, ) = auxfun_multianimal.extractindividualsandbodyparts(self.cfg) # self.choiceBox,self.visualization_rdb = self.choice_panel.addRadioButtons() self.Colorscheme = visualization.get_cmap(len(self.individual_names), self.cfg["colormap"])
def evaluate_multianimal_full( config, Shuffles=[1], trainingsetindex=0, plotting=None, show_errors=True, comparisonbodyparts="all", gputouse=None, modelprefix="", c_engine=False, ): """ WIP multi animal project. """ import os from deeplabcut.pose_estimation_tensorflow.nnet import predict from deeplabcut.pose_estimation_tensorflow.nnet import ( predict_multianimal as predictma, ) from deeplabcut.utils import auxiliaryfunctions, auxfun_multianimal from deeplabcut.pose_estimation_tensorflow.dataset.pose_dataset import data_to_input import tensorflow as tf if "TF_CUDNN_USE_AUTOTUNE" in os.environ: del os.environ[ "TF_CUDNN_USE_AUTOTUNE"] # was potentially set during training tf.reset_default_graph() os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" # if gputouse is not None: # gpu selectinon os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse) start_path = os.getcwd() ################################################## # Load data... ################################################## cfg = auxiliaryfunctions.read_config(config) if trainingsetindex == "all": TrainingFractions = cfg["TrainingFraction"] else: TrainingFractions = [cfg["TrainingFraction"][trainingsetindex]] # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder(cfg) Data = pd.read_hdf( os.path.join( cfg["project_path"], str(trainingsetfolder), "CollectedData_" + cfg["scorer"] + ".h5", ), "df_with_missing", ) # Get list of body parts to evaluate network for comparisonbodyparts = auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser( cfg, comparisonbodyparts) colors = visualization.get_cmap(len(comparisonbodyparts), name=cfg["colormap"]) # Make folder for evaluation auxiliaryfunctions.attempttomakefolder( str(cfg["project_path"] + "/evaluation-results/")) for shuffle in Shuffles: for trainFraction in TrainingFractions: ################################################## # Load and setup CNN part detector ################################################## datafn, metadatafn = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) modelfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetModelFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" # Load meta data ( data, trainIndices, testIndices, trainFraction, ) = auxiliaryfunctions.LoadMetadata( os.path.join(cfg["project_path"], metadatafn)) try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError: raise FileNotFoundError( "It seems the model for shuffle %s and trainFraction %s does not exist." % (shuffle, trainFraction)) # TODO: IMPLEMENT for different batch sizes? dlc_cfg["batch_size"] = 1 # due to differently sized images!!! # Create folder structure to store results. evaluationfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetEvaluationFolder( trainFraction, shuffle, cfg, modelprefix=modelprefix)), ) auxiliaryfunctions.attempttomakefolder(evaluationfolder, recursive=True) # path_train_config = modelfolder / 'train' / 'pose_cfg.yaml' # Check which snapshots are available and sort them by # iterations Snapshots = np.array([ fn.split(".")[0] for fn in os.listdir(os.path.join(str(modelfolder), "train")) if "index" in fn ]) if len(Snapshots) == 0: print( "Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so." % (shuffle, trainFraction)) else: increasing_indices = np.argsort( [int(m.split("-")[1]) for m in Snapshots]) Snapshots = Snapshots[increasing_indices] if cfg["snapshotindex"] == -1: snapindices = [-1] elif cfg["snapshotindex"] == "all": snapindices = range(len(Snapshots)) elif cfg["snapshotindex"] < len(Snapshots): snapindices = [cfg["snapshotindex"]] else: print( "Invalid choice, only -1 (last), any integer up to last, or all (as string)!" ) ( individuals, uniquebodyparts, multianimalbodyparts, ) = auxfun_multianimal.extractindividualsandbodyparts(cfg) final_result = [] ################################################## # Compute predictions over images ################################################## for snapindex in snapindices: dlc_cfg["init_weights"] = os.path.join( str(modelfolder), "train", Snapshots[snapindex] ) # setting weights to corresponding snapshot. trainingsiterations = ( dlc_cfg["init_weights"].split(os.sep)[-1] ).split( "-" )[-1] # read how many training siterations that corresponds to. # name for deeplabcut net (based on its parameters) DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, trainFraction, trainingsiterations, modelprefix=modelprefix, ) print( "Running ", DLCscorer, " with # of trainingiterations:", trainingsiterations, ) ( notanalyzed, resultsfilename, DLCscorer, ) = auxiliaryfunctions.CheckifNotEvaluated( str(evaluationfolder), DLCscorer, DLCscorerlegacy, Snapshots[snapindex], ) if os.path.isfile( resultsfilename.split(".h5")[0] + "_full.pickle"): print("Model already evaluated.", resultsfilename) else: if plotting: foldername = os.path.join( str(evaluationfolder), "LabeledImages_" + DLCscorer + "_" + Snapshots[snapindex], ) auxiliaryfunctions.attempttomakefolder(foldername) # print(dlc_cfg) # Specifying state of model (snapshot / training state) sess, inputs, outputs = predict.setup_pose_prediction( dlc_cfg) PredicteData = {} print("Analyzing data...") for imageindex, imagename in tqdm(enumerate( Data.index)): image_path = os.path.join(cfg["project_path"], imagename) image = io.imread(image_path) frame = img_as_ubyte(skimage.color.gray2rgb(image)) GT = Data.iloc[imageindex] # Storing GT data as dictionary, so it can be used for calculating connection costs groundtruthcoordinates = [] groundtruthidentity = [] for bptindex, bpt in enumerate( dlc_cfg["all_joints_names"]): coords = np.zeros([len(individuals), 2 ]) * np.nan identity = [] for prfxindex, prefix in enumerate( individuals): if bpt in uniquebodyparts and prefix == "single": coords[prfxindex, :] = np.array([ GT[cfg["scorer"]][prefix][bpt] ["x"], GT[cfg["scorer"]][prefix][bpt] ["y"], ]) identity.append(prefix) elif (bpt in multianimalbodyparts and prefix != "single"): coords[prfxindex, :] = np.array([ GT[cfg["scorer"]][prefix][bpt] ["x"], GT[cfg["scorer"]][prefix][bpt] ["y"], ]) identity.append(prefix) else: identity.append("nix") groundtruthcoordinates.append( coords[np.isfinite(coords[:, 0]), :]) groundtruthidentity.append( np.array(identity)[np.isfinite(coords[:, 0])]) PredicteData[imagename] = {} PredicteData[imagename]["index"] = imageindex pred = predictma.get_detectionswithcostsandGT( frame, groundtruthcoordinates, dlc_cfg, sess, inputs, outputs, outall=False, nms_radius=dlc_cfg.nmsradius, det_min_score=dlc_cfg.minconfidence, c_engine=c_engine, ) PredicteData[imagename]["prediction"] = pred PredicteData[imagename]["groundtruth"] = [ groundtruthidentity, groundtruthcoordinates, GT, ] if plotting: coords_pred = pred["coordinates"][0] probs_pred = pred["confidence"] fig = visualization.make_multianimal_labeled_image( frame, groundtruthcoordinates, coords_pred, probs_pred, colors, cfg["dotsize"], cfg["alphavalue"], cfg["pcutoff"], ) visualization.save_labeled_frame( fig, image_path, foldername, imageindex in trainIndices, ) sess.close() # closes the current tf session PredicteData["metadata"] = { "nms radius": dlc_cfg.nmsradius, "minimal confidence": dlc_cfg.minconfidence, "PAFgraph": dlc_cfg.partaffinityfield_graph, "all_joints": [[i] for i in range(len(dlc_cfg.all_joints))], "all_joints_names": [ dlc_cfg.all_joints_names[i] for i in range(len(dlc_cfg.all_joints)) ], "stride": dlc_cfg.get("stride", 8), } print( "Done and results stored for snapshot: ", Snapshots[snapindex], ) dictionary = { "Scorer": DLCscorer, "DLC-model-config file": dlc_cfg, "trainIndices": trainIndices, "testIndices": testIndices, "trainFraction": trainFraction, } metadata = {"data": dictionary} auxfun_multianimal.SaveFullMultiAnimalData( PredicteData, metadata, resultsfilename) tf.reset_default_graph() # returning to intial folder os.chdir(str(start_path))
def create_multianimaltraining_dataset( config, num_shuffles=1, Shuffles=None, windows2linux=False, net_type=None, numdigits=2, ): """ Creates a training dataset for multi-animal datasets. Labels from all the extracted frames are merged into a single .h5 file.\n Only the videos included in the config file are used to create this dataset.\n [OPTIONAL] Use the function 'add_new_video' at any stage of the project to add more videos to the project. Imporant differences to standard: - stores coordinates with numdigits as many digits - creates Parameter ---------- config : string Full path of the config.yaml file as a string. num_shuffles : int, optional Number of shuffles of training dataset to create, i.e. [1,2,3] for num_shuffles=3. Default is set to 1. Shuffles: list of shuffles. Alternatively the user can also give a list of shuffles (integers!). windows2linux: bool. The annotation files contain path formated according to your operating system. If you label on windows but train & evaluate on a unix system (e.g. ubunt, colab, Mac) set this variable to True to convert the paths. net_type: string Type of networks. Currently resnet_50, resnet_101, and resnet_152 are supported (not the MobileNets!) numdigits: int, optional Example -------- >>> deeplabcut.create_multianimaltraining_dataset('/analysis/project/reaching-task/config.yaml',num_shuffles=1) Windows: >>> deeplabcut.create_multianimaltraining_dataset(r'C:\\Users\\Ulf\\looming-task\\config.yaml',Shuffles=[3,17,5]) -------- """ from skimage import io # Loading metadata from config file: cfg = auxiliaryfunctions.read_config(config) scorer = cfg["scorer"] project_path = cfg["project_path"] # Create path for training sets & store data there trainingsetfolder = auxiliaryfunctions.GetTrainingSetFolder( cfg) # Path concatenatn OS platform independent auxiliaryfunctions.attempttomakefolder(Path( os.path.join(project_path, str(trainingsetfolder))), recursive=True) Data = trainingsetmanipulation.merge_annotateddatasets( cfg, Path(os.path.join(project_path, trainingsetfolder)), windows2linux) if Data is None: return Data = Data[scorer] # extract labeled data # actualbpts=set(Data.columns.get_level_values(0)) def strip_cropped_image_name(path): # utility function to split different crops from same image into either train or test! filename = os.path.split(path)[1] return filename.split("c")[0] img_names = Data.index.map(strip_cropped_image_name).unique() # loading & linking pretrained models # CURRENTLY ONLY ResNet supported! if net_type is None: # loading & linking pretrained models net_type = cfg.get("default_net_type", "resnet_50") else: if "resnet" in net_type: # or 'mobilenet' in net_type: pass else: raise ValueError("Currently only resnet is supported.") # multianimal case: dataset_type = "multi-animal-imgaug" partaffinityfield_graph = auxfun_multianimal.getpafgraph(cfg, printnames=False) # ATTENTION: order has to be multibodyparts, then uniquebodyparts (for indexing) print("Utilizing the following graph:", partaffinityfield_graph) num_limbs = len(partaffinityfield_graph) partaffinityfield_predict = True # Loading the encoder (if necessary downloading from TF) dlcparent_path = auxiliaryfunctions.get_deeplabcut_path() defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml") model_path, num_shuffles = auxfun_models.Check4weights( net_type, Path(dlcparent_path), num_shuffles) if Shuffles == None: Shuffles = range(1, num_shuffles + 1, 1) else: Shuffles = [i for i in Shuffles if isinstance(i, int)] ( individuals, uniquebodyparts, multianimalbodyparts, ) = auxfun_multianimal.extractindividualsandbodyparts(cfg) TrainingFraction = cfg["TrainingFraction"] for shuffle in Shuffles: # Creating shuffles starting from 1 for trainFraction in TrainingFraction: train_inds_temp, test_inds_temp = trainingsetmanipulation.SplitTrials( range(len(img_names)), trainFraction) # Map back to the original indices. temp = [ name for i, name in enumerate(img_names) if i in test_inds_temp ] mask = Data.index.str.contains("|".join(temp)) testIndices = np.flatnonzero(mask) trainIndices = np.flatnonzero(~mask) #################################################### # Generating data structure with labeled information & frame metadata (for deep cut) #################################################### # Make training file! data = [] print("Creating training data for ", shuffle, trainFraction) print("This can take some time...") for jj in tqdm(trainIndices): jointsannotated = False H = {} # load image to get dimensions: filename = Data.index[jj] im = io.imread(os.path.join(cfg["project_path"], filename)) H["image"] = filename try: H["size"] = np.array( [np.shape(im)[2], np.shape(im)[0], np.shape(im)[1]]) except: # print "Grayscale!" H["size"] = np.array([1, np.shape(im)[0], np.shape(im)[1]]) Joints = {} for prfxindex, prefix in enumerate(individuals): joints = (np.zeros( (len(uniquebodyparts) + len(multianimalbodyparts), 3)) * np.nan) if prefix != "single": # first ones are multianimalparts! indexjoints = 0 for bpindex, bodypart in enumerate( multianimalbodyparts): socialbdpt = bodypart # prefix+bodypart #build names! # if socialbdpt in actualbpts: try: x, y = ( Data[prefix][socialbdpt]["x"][jj], Data[prefix][socialbdpt]["y"][jj], ) joints[indexjoints, 0] = int(bpindex) joints[indexjoints, 1] = round(x, numdigits) joints[indexjoints, 2] = round(y, numdigits) indexjoints += 1 except: pass else: indexjoints = len(multianimalbodyparts) for bpindex, bodypart in enumerate(uniquebodyparts): socialbdpt = bodypart # prefix+bodypart #build names! # if socialbdpt in actualbpts: try: x, y = ( Data[prefix][socialbdpt]["x"][jj], Data[prefix][socialbdpt]["y"][jj], ) joints[indexjoints, 0] = len( multianimalbodyparts) + int(bpindex) joints[indexjoints, 1] = round(x, 2) joints[indexjoints, 2] = round(y, 2) indexjoints += 1 except: pass # Drop missing body parts joints = joints[~np.isnan(joints).any(axis=1)] # Drop points lying outside the image inside = np.logical_and.reduce(( joints[:, 1] < im.shape[1], joints[:, 1] > 0, joints[:, 2] < im.shape[0], joints[:, 2] > 0, )) joints = joints[inside] if np.size(joints) > 0: # exclude images without labels jointsannotated = True Joints[prfxindex] = joints # np.array(joints, dtype=int) H["joints"] = Joints if jointsannotated: # exclude images without labels data.append(H) if len(trainIndices) > 0: ( datafilename, metadatafilename, ) = auxiliaryfunctions.GetDataandMetaDataFilenames( trainingsetfolder, trainFraction, shuffle, cfg) ################################################################################ # Saving metadata and data file (Pickle file) ################################################################################ auxiliaryfunctions.SaveMetadata( os.path.join(project_path, metadatafilename), data, trainIndices, testIndices, trainFraction, ) datafilename = datafilename.split(".mat")[0] + ".pickle" import pickle with open(os.path.join(project_path, datafilename), "wb") as f: # Pickle the 'labeled-data' dictionary using the highest protocol available. pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) ################################################################################ # Creating file structure for training & # Test files as well as pose_yaml files (containing training and testing information) ################################################################################# modelfoldername = auxiliaryfunctions.GetModelFolder( trainFraction, shuffle, cfg) auxiliaryfunctions.attempttomakefolder( Path(config).parents[0] / modelfoldername, recursive=True) auxiliaryfunctions.attempttomakefolder( str(Path(config).parents[0] / modelfoldername) + "/" + "/train") auxiliaryfunctions.attempttomakefolder( str(Path(config).parents[0] / modelfoldername) + "/" + "/test") path_train_config = str( os.path.join( cfg["project_path"], Path(modelfoldername), "train", "pose_cfg.yaml", )) path_test_config = str( os.path.join( cfg["project_path"], Path(modelfoldername), "test", "pose_cfg.yaml", )) path_inference_config = str( os.path.join( cfg["project_path"], Path(modelfoldername), "test", "inference_cfg.yaml", )) jointnames = [str(bpt) for bpt in multianimalbodyparts] jointnames.extend([str(bpt) for bpt in uniquebodyparts]) items2change = { "dataset": datafilename, "metadataset": metadatafilename, "num_joints": len(multianimalbodyparts) + len(uniquebodyparts), # cfg["uniquebodyparts"]), "all_joints": [[i] for i in range( len(multianimalbodyparts) + len(uniquebodyparts)) ], # cfg["uniquebodyparts"]))], "all_joints_names": jointnames, "init_weights": model_path, "project_path": str(cfg["project_path"]), "net_type": net_type, "pairwise_loss_weight": 0.1, "pafwidth": 20, "partaffinityfield_graph": partaffinityfield_graph, "partaffinityfield_predict": partaffinityfield_predict, "weigh_only_present_joints": False, "num_limbs": len(partaffinityfield_graph), "dataset_type": dataset_type, "optimizer": "adam", "batch_size": 8, "multi_step": [[1e-4, 7500], [5 * 1e-5, 12000], [1e-5, 200000]], "save_iters": 10000, "display_iters": 500, } defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml") trainingdata = trainingsetmanipulation.MakeTrain_pose_yaml( items2change, path_train_config, defaultconfigfile) keys2save = [ "dataset", "num_joints", "all_joints", "all_joints_names", "net_type", "init_weights", "global_scale", "location_refinement", "locref_stdev", "dataset_type", "partaffinityfield_predict", "pairwise_predict", "partaffinityfield_graph", "num_limbs", "dataset_type", ] trainingsetmanipulation.MakeTest_pose_yaml( trainingdata, keys2save, path_test_config, nmsradius=5.0, minconfidence=0.01, ) # setting important def. values for inference # Setting inference cfg file: defaultinference_configfile = os.path.join( dlcparent_path, "inference_cfg.yaml") items2change = { "minimalnumberofconnections": int(len(cfg["multianimalbodyparts"]) / 2), "topktoretain": len(cfg["individuals"]) + 1 * (len(cfg["uniquebodyparts"]) > 0), } # TODO: "distnormalization": could be calculated here based on data and set # >> now we calculate this during evaluation (which is a good spot...) trainingsetmanipulation.MakeInference_yaml( items2change, path_inference_config, defaultinference_configfile) print( "The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!" ) else: pass
def __init__(self, parent,config): # Settting the GUI size and panels design displays = (wx.Display(i) for i in range(wx.Display.GetCount())) # Gets the number of displays screenSizes = [display.GetGeometry().GetSize() for display in displays] # Gets the size of each display index = 0 # For display 1. screenWidth = screenSizes[index][0] screenHeight = screenSizes[index][1] self.gui_size = (screenWidth*0.7,screenHeight*0.85) wx.Frame.__init__ ( self, parent, id = wx.ID_ANY, title = 'DeepLabCut2.0 - Refinement ToolBox', size = wx.Size(self.gui_size), pos = wx.DefaultPosition, style = wx.RESIZE_BORDER|wx.DEFAULT_FRAME_STYLE|wx.TAB_TRAVERSAL ) self.statusbar = self.CreateStatusBar() self.statusbar.SetStatusText("") self.Bind(wx.EVT_CHAR_HOOK, self.OnKeyPressed) self.SetSizeHints(wx.Size(self.gui_size)) # This sets the minimum size of the GUI. It can scale now! ################################################################################################################################################### # Spliting the frame into top and bottom panels. Bottom panels contains the widgets. The top panel is for showing images and plotting! topSplitter = wx.SplitterWindow(self) vSplitter = wx.SplitterWindow(topSplitter) self.image_panel = ImagePanel(vSplitter, config,self.gui_size) self.choice_panel = ScrollPanel(vSplitter) # self.choice_panel.SetupScrolling(scroll_x=True, scroll_y=True, scrollToTop=False) # self.choice_panel.SetupScrolling(scroll_x=True, scrollToTop=False) vSplitter.SplitVertically(self.image_panel,self.choice_panel, sashPosition=self.gui_size[0]*0.8) vSplitter.SetSashGravity(1) self.widget_panel = WidgetPanel(topSplitter) topSplitter.SplitHorizontally(vSplitter, self.widget_panel,sashPosition=self.gui_size[1]*0.83)#0.9 topSplitter.SetSashGravity(1) sizer = wx.BoxSizer(wx.VERTICAL) sizer.Add(topSplitter, 1, wx.EXPAND) self.SetSizer(sizer) ################################################################################################################################################### # Add Buttons to the WidgetPanel and bind them to their respective functions. widgetsizer = wx.WrapSizer(orient=wx.HORIZONTAL) self.load = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Load labels") widgetsizer.Add(self.load , 1, wx.ALL, 15) self.load.Bind(wx.EVT_BUTTON, self.browseDir) self.prev = wx.Button(self.widget_panel, id=wx.ID_ANY, label="<<Previous") widgetsizer.Add(self.prev , 1, wx.ALL, 15) self.prev.Bind(wx.EVT_BUTTON, self.prevImage) self.prev.Enable(False) self.next = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Next>>") widgetsizer.Add(self.next , 1, wx.ALL, 15) self.next.Bind(wx.EVT_BUTTON, self.nextImage) self.next.Enable(False) self.help = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Help") widgetsizer.Add(self.help , 1, wx.ALL, 15) self.help.Bind(wx.EVT_BUTTON, self.helpButton) self.help.Enable(True) self.zoom = wx.ToggleButton(self.widget_panel, label="Zoom") widgetsizer.Add(self.zoom , 1, wx.ALL, 15) self.zoom.Bind(wx.EVT_TOGGLEBUTTON, self.zoomButton) self.widget_panel.SetSizer(widgetsizer) self.zoom.Enable(False) self.home = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Home") widgetsizer.Add(self.home , 1, wx.ALL,15) self.home.Bind(wx.EVT_BUTTON, self.homeButton) self.widget_panel.SetSizer(widgetsizer) self.home.Enable(False) self.pan = wx.ToggleButton(self.widget_panel, id=wx.ID_ANY, label="Pan") widgetsizer.Add(self.pan , 1, wx.ALL, 15) self.pan.Bind(wx.EVT_TOGGLEBUTTON, self.panButton) self.widget_panel.SetSizer(widgetsizer) self.pan.Enable(False) self.lock = wx.CheckBox(self.widget_panel, id=wx.ID_ANY, label="Lock View") widgetsizer.Add(self.lock, 1, wx.ALL, 15) self.lock.Bind(wx.EVT_CHECKBOX, self.lockChecked) self.widget_panel.SetSizer(widgetsizer) self.lock.Enable(False) self.save = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Save") widgetsizer.Add(self.save , 1, wx.ALL, 15) self.save.Bind(wx.EVT_BUTTON, self.saveDataSet) self.save.Enable(False) widgetsizer.AddStretchSpacer(15) self.quit = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Quit") widgetsizer.Add(self.quit , 1, wx.ALL|wx.ALIGN_RIGHT, 15) self.quit.Bind(wx.EVT_BUTTON, self.quitButton) self.widget_panel.SetSizer(widgetsizer) self.widget_panel.SetSizerAndFit(widgetsizer) self.widget_panel.Layout() ############################################################################################################################### # Variable initialization self.currentDirectory = os.getcwd() self.index = [] self.iter = [] self.threshold = [] self.file = 0 self.updatedCoords = [] self.drs = [] self.cfg = auxiliaryfunctions.read_config(config) self.humanscorer = self.cfg['scorer'] self.move2corner = self.cfg['move2corner'] self.center = self.cfg['corner2move2'] self.colormap = plt.get_cmap(self.cfg['colormap']) self.colormap = self.colormap.reversed() self.markerSize = self.cfg['dotsize'] self.alpha = self.cfg['alphavalue'] self.iterationindex = self.cfg['iteration'] self.project_path=self.cfg['project_path'] self.bodyparts = self.cfg['bodyparts'] self.threshold = 0.1 self.img_size = (10,6)# (imgW, imgH) # width, height in inches. self.preview = False self.view_locked=False # Workaround for MAC - xlim and ylim changed events seem to be triggered too often so need to make sure that the # xlim and ylim have actually changed before turning zoom off self.prezoom_xlim=[] self.prezoom_ylim=[] from deeplabcut.utils import auxfun_multianimal self.individual_names,self.uniquebodyparts,self.multianimalbodyparts = auxfun_multianimal.extractindividualsandbodyparts(self.cfg) # self.choiceBox,self.visualization_rdb = self.choice_panel.addRadioButtons() self.Colorscheme = visualization.get_cmap(len(self.individual_names),self.cfg['colormap'])