def test_frame_merge_between_predicted_and_user(skeleton, centered_pair_vid): user_inst = Instance(skeleton=skeleton, points={skeleton.nodes[0]: Point(1, 2)},) user_labels = Labels( [LabeledFrame(video=centered_pair_vid, frame_idx=0, instances=[user_inst],)] ) pred_inst = PredictedInstance( skeleton=skeleton, points={skeleton.nodes[0]: PredictedPoint(1, 2, score=1.0)}, score=1.0, ) pred_labels = Labels( [LabeledFrame(video=centered_pair_vid, frame_idx=0, instances=[pred_inst],)] ) # Merge predictions into current labels dataset _, _, new_conflicts = Labels.complex_merge_between( user_labels, new_labels=pred_labels, unify=False, # since we used match_to when loading predictions file ) # new predictions should replace old ones Labels.finish_complex_merge(user_labels, new_conflicts) # We should be able to cleanly merge the user and the predicted instance, # and we want to retain both even though they perfectly match. assert user_inst in user_labels[0].instances assert pred_inst in user_labels[0].instances assert len(user_labels[0].instances) == 2
def fit_tracks(filename: Text, instance_count: int): """Wraps `TrackCleaner` for easier cli api.""" labels = Labels.load_file(filename) video = labels.videos[0] frames = labels.find(video) TrackCleaner(instance_count=instance_count).run(frames=frames) # Rebuild list of tracks labels.tracks = list({ instance.track for frame in labels for instance in frame.instances if instance.track }) labels.tracks.sort(key=operator.attrgetter("spawned_on", "name")) # Save new file save_filename = filename save_filename = save_filename.replace(".slp", ".cleaned.slp") save_filename = save_filename.replace(".h5", ".cleaned.h5") save_filename = save_filename.replace(".json", ".cleaned.json") Labels.save_file(labels, save_filename) print(f"Saved: {save_filename}")
def retrack(): import argparse import operator import os import time from sleap import Labels parser = argparse.ArgumentParser() parser.add_argument("data_path", help="Path to SLEAP project file") parser.add_argument( "-o", "--output", type=str, default=None, help="The output filename to use for the predicted data.", ) Tracker.add_cli_parser_args(parser) args = parser.parse_args() tracker_args = { key: val for key, val in vars(args).items() if val is not None } tracker = Tracker.make_tracker_by_name(**tracker_args) print(tracker) print("Loading predictions...") t0 = time.time() labels = Labels.load_file(args.data_path, args.data_path) frames = sorted(labels.labeled_frames, key=operator.attrgetter("frame_idx")) frames = frames # [:1000] print(f"Done loading predictions in {time.time() - t0} seconds.") print("Starting tracker...") frames = run_tracker(frames=frames, tracker=tracker) tracker.final_pass(frames) new_labels = Labels(labeled_frames=frames) if args.output: output_path = args.output else: out_dir = os.path.dirname(args.data_path) out_name = os.path.basename( args.data_path) + f".{tracker.get_name()}.slp" output_path = os.path.join(out_dir, out_name) print(f"Saving: {output_path}") Labels.save_file(new_labels, output_path)
def find_frame_pairs(labels_gt: Labels, labels_pr: Labels, user_labels_only: bool = True ) -> List[Tuple[LabeledFrame, LabeledFrame]]: """Find corresponding frames across two sets of labels. Args: labels_gt: A `sleap.Labels` instance with ground truth instances. labels_pr: A `sleap.Labels` instance with predicted instances. user_labels_only: If False, frames with predicted instances in `labels_gt` will also be considered for matching. Returns: A list of pairs of `sleap.LabeledFrame`s in the form `(frame_gt, frame_pr)`. """ frame_pairs = [] for video_gt in labels_gt.videos: # Find matching video instance in predictions. video_pr = None for video in labels_pr.videos: if isinstance(video.backend, type( video_gt.backend)) and video.matches(video_gt): video_pr = video break if video_pr is None: continue # Find labeled frames in this video. labeled_frames_gt = labels_gt.find(video_gt) if user_labels_only: labeled_frames_gt = [ lf for lf in labeled_frames_gt if lf.has_user_instances ] # Attempt to match each labeled frame in the ground truth. for labeled_frame_gt in labeled_frames_gt: labeled_frames_pr = labels_pr.find( video_pr, frame_idx=labeled_frame_gt.frame_idx) if not labeled_frames_pr: # No match continue elif len(labeled_frames_pr) == 1: # Match! frame_pairs.append((labeled_frame_gt, labeled_frames_pr[0])) else: # Too many matches. raise ValueError( "More than one labeled frame found in predictions.") return frame_pairs
def merge_results(self): """Merges result frames into labels dataset.""" # Remove any frames without instances new_lfs = list(filter(lambda lf: len(lf.instances), self.results)) # Merge predictions into current labels dataset _, _, new_conflicts = Labels.complex_merge_between( self.labels, new_labels=Labels(new_lfs), unify=False, # since we used match_to when loading predictions file ) # new predictions should replace old ones Labels.finish_complex_merge(self.labels, new_conflicts)
def demo_gui(): from sleap.gui.dialogs.formbuilder import YamlFormWidget from sleap import Labels from PySide2.QtWidgets import QApplication labels = Labels.load_file( "tests/data/json_format_v2/centered_pair_predictions.json" ) options_lists = dict(node=labels.skeletons[0].node_names) app = QApplication() win = YamlFormWidget.from_name( "suggestions", title="Generate Suggestions", field_options_lists=options_lists ) def demo_suggestions(params): print(params) x = VideoFrameSuggestions.suggest(params=params, labels=labels) for suggested_frame in x: print( suggested_frame.video.backend.filename, suggested_frame.frame_idx, suggested_frame.group, ) win.mainAction.connect(demo_suggestions) win.show() app.exec_()
def predict_subprocess( self, item_for_inference: ItemForInference, append_results: bool = False, waiting_callback: Optional[Callable] = None, ) -> Tuple[Text, bool]: """Runs inference in a subprocess.""" cli_args, output_path = self.make_predict_cli_call(item_for_inference) print("Command line call:") print(" \\\n".join(cli_args)) print() with sub.Popen(cli_args) as proc: while proc.poll() is None: if waiting_callback is not None: if waiting_callback() == -1: # -1 signals user cancellation return "", False time.sleep(0.1) print(f"Process return code: {proc.returncode}") success = proc.returncode == 0 if success and append_results: # Load frames from inference into results list new_inference_labels = Labels.load_file(output_path, match_to=self.labels) self.results.extend(new_inference_labels.labeled_frames) return output_path, success
def read(cls, file: FileHandle, *args, **kwargs,) -> Labels: filename = file.filename # Load data from the YAML file project_data = yaml.load(file.text, Loader=yaml.SafeLoader) # Create skeleton which we'll use for each video skeleton = Skeleton() skeleton.add_nodes(project_data["bodyparts"]) # Get subdirectories of videos and labeled data root_dir = os.path.dirname(filename) videos_dir = os.path.join(root_dir, "videos") labeled_data_dir = os.path.join(root_dir, "labeled-data") with os.scandir(labeled_data_dir) as file_iterator: data_subdirs = [file.path for file in file_iterator if file.is_dir()] labeled_frames = [] # Each subdirectory of labeled data corresponds to a video. # We'll go through each and import the labeled frames. for data_subdir in data_subdirs: csv_files = find_files_by_suffix( data_subdir, prefix="CollectedData", suffix=".csv" ) if csv_files: csv_path = csv_files[0] # Try to find a full video corresponding to this subdir. # If subdirectory is foo, we look for foo.mp4 in videos dir. shortname = os.path.split(data_subdir)[-1] video_path = os.path.join(videos_dir, f"{shortname}.mp4") if os.path.exists(video_path): video = Video.from_filename(video_path) else: # When no video is found, the individual frame images # stored in the labeled data subdir will be used. print( f"Unable to find {video_path} so using individual frame images." ) video = None # Import the labeled fraems labeled_frames.extend( LabelsDeepLabCutCsvAdaptor.read_frames( FileHandle(csv_path), full_video=video, skeleton=skeleton ) ) else: print(f"No csv data file found in {data_subdir}") return Labels(labeled_frames=labeled_frames)
def read( cls, file: FileHandle, full_video: Optional[Video] = None, *args, **kwargs, ) -> Labels: return Labels( labeled_frames=cls.read_frames(file, full_video, *args, **kwargs))
def from_unlabeled_suggestions(cls, labels: sleap.Labels) -> "LabelsReader": """Create a `LabelsReader` using the unlabeled suggestions in a `Labels` set. Args: labels: A `sleap.Labels` instance containing unlabeled suggestions. Returns: A `LabelsReader` instance that can create a dataset for pipelining. """ inds = labels.get_unlabeled_suggestion_inds() return cls(labels=labels, example_indices=inds)
def read_headers( cls, file: format.filehandle.FileHandle, video_search: Union[Callable, List[Text], None] = None, match_to: Optional[Labels] = None, ): f = file.file # Extract the Labels JSON metadata and create Labels object with just this # metadata. dicts = json_loads( f.require_group("metadata").attrs["json"].tostring().decode() ) # These items are stored in separate lists because the metadata group got to be # too big. for key in ("videos", "tracks", "suggestions"): hdf5_key = f"{key}_json" if hdf5_key in f: items = [json_loads(item_json) for item_json in f[hdf5_key]] dicts[key] = items # Video path "." means the video is saved in same file as labels, so replace # these paths. for video_item in dicts["videos"]: if video_item["backend"]["filename"] == ".": video_item["backend"]["filename"] = file.filename # Use the video_callback for finding videos with broken paths: # 1. Accept single string as video search path if isinstance(video_search, str): video_search = [video_search] # 2. Accept list of strings as video search paths if hasattr(video_search, "__iter__"): # If the callback is an iterable, then we'll expect it to be a list of # strings and build a non-gui callback with those as the search paths. search_paths = [ # os.path.dirname(path) if os.path.isfile(path) else path path for path in video_search ] # Make the search function from list of paths video_search = Labels.make_video_callback(search_paths) # 3. Use the callback function (either given as arg or build from paths) if callable(video_search): video_search(dicts["videos"]) # Create the Labels object with the header data we've loaded labels = labels_json.LabelsJsonAdaptor.from_json_data(dicts, match_to=match_to) return labels
def predict_subprocess( self, item_for_inference: ItemForInference, append_results: bool = False, waiting_callback: Optional[Callable] = None, gui: bool = True, ) -> Tuple[Text, bool]: """Runs inference in a subprocess.""" cli_args, output_path = self.make_predict_cli_call(item_for_inference, gui=gui) print("Command line call:") print(" ".join(cli_args)) print() # Run inference CLI capturing output. with subprocess.Popen(cli_args, stdout=subprocess.PIPE) as proc: # Poll until finished. while proc.poll() is None: # Read line. line = proc.stdout.readline() line = line.decode().rstrip() if line.startswith("{"): # Parse line. line_data = json.loads(line) else: # Pass through non-json output. print(line) line_data = {} if waiting_callback is not None: # Pass line data to callback. ret = waiting_callback(**line_data) if ret == "cancel": # Stop if callback returned cancel signal. kill_process(proc.pid) print(f"Killed PID: {proc.pid}") return "", "canceled" time.sleep(0.05) print(f"Process return code: {proc.returncode}") success = proc.returncode == 0 if success and append_results: # Load frames from inference into results list new_inference_labels = Labels.load_file(output_path, match_to=self.labels) self.results.extend(new_inference_labels.labeled_frames) # Return "success" or return code if failed. ret = "success" if success else proc.returncode return output_path, ret
def save_predictions_from_cli(args, predicted_frames, prediction_metadata=None): from sleap import Labels if args.output: output_path = args.output elif args.video_path: out_dir = os.path.dirname(args.video_path) out_name = os.path.basename(args.video_path) + ".predictions.slp" output_path = os.path.join(out_dir, out_name) elif args.labels: out_dir = os.path.dirname(args.labels) out_name = os.path.basename(args.labels) + ".predictions.slp" output_path = os.path.join(out_dir, out_name) else: # We shouldn't ever get here but if we do, just save in working dir. output_path = "predictions.slp" labels = Labels(labeled_frames=predicted_frames, provenance=prediction_metadata) print(f"Saving: {output_path}") Labels.save_file(labels, output_path)
def read( cls, file: FileHandle, video_path: str, skeleton_path: str, *args, **kwargs, ) -> Labels: f = file.file video = Video.from_filename(video_path) skeleton_data = pd.read_csv(skeleton_path, header=0) skeleton = Skeleton() skeleton.add_nodes(skeleton_data["name"]) nodes = skeleton.nodes for name, parent, swap in skeleton_data.itertuples(index=False, name=None): if parent is not np.nan: skeleton.add_edge(parent, name) lfs = [] pose_matrix = f["pose"][:] track_count, frame_count, node_count, _ = pose_matrix.shape tracks = [Track(0, f"Track {i}") for i in range(track_count)] for frame_idx in range(frame_count): lf_instances = [] for track_idx in range(track_count): points_array = pose_matrix[track_idx, frame_idx, :, :] points = dict() for p in range(len(points_array)): x, y, score = points_array[p] points[nodes[p]] = Point(x, y) # TODO: score inst = Instance(skeleton=skeleton, track=tracks[track_idx], points=points) lf_instances.append(inst) lfs.append( LabeledFrame(video, frame_idx=frame_idx, instances=lf_instances)) return Labels(labeled_frames=lfs)
def demo_training_dialog(): app = QtWidgets.QApplication([]) filename = "tests/data/json_format_v1/centered_pair.json" labels = Labels.load_file(filename) win = LearningDialog("inference", labels_filename=filename, labels=labels) win.frame_selection = {"clip": {labels.videos[0]: (1, 2, 3, 4)}} # win.training_editor_widget.set_fields_from_key_val_dict({ # "_backbone_name": "unet", # "_heads_name": "centered_instance", # }) # # win.training_editor_widget.form_widgets["model"].set_field_enabled("_heads_name", False) win.show() app.exec_()
def test_inference_merging(): skeleton = Skeleton() video = Video(backend=MediaVideo) lf_user_only = LabeledFrame(video=video, frame_idx=0, instances=[Instance(skeleton=skeleton)]) lf_pred_only = LabeledFrame( video=video, frame_idx=1, instances=[PredictedInstance(skeleton=skeleton)]) lf_both = LabeledFrame( video=video, frame_idx=2, instances=[ Instance(skeleton=skeleton), PredictedInstance(skeleton=skeleton) ], ) labels = Labels([lf_user_only, lf_pred_only, lf_both]) task = runners.InferenceTask( trained_job_paths=None, inference_params=None, labels=labels, results=[ LabeledFrame( video=labels.video, frame_idx=2, instances=[ PredictedInstance(skeleton=skeleton), PredictedInstance(skeleton=skeleton), ], ) ], ) task.merge_results() assert len(labels) == 3 assert labels[0].frame_idx == 0 assert labels[0].has_user_instances assert labels[1].frame_idx == 1 assert labels[1].has_predicted_instances assert labels[2].frame_idx == 2 assert len(labels[2].user_instances) == 1 assert len(labels[2].predicted_instances) == 2
def skeleton(self): # cache skeleton so we only search once if self._skeleton is None and not self._tried_finding_skeleton: # if skeleton was saved in config, great! if self.config.data.labels.skeletons: self._skeleton = self.config.data.labels.skeletons[0] # otherwise try loading it from validation labels (much slower!) else: filename = self._get_file_path(f"labels_gt.val.slp") if filename is not None: val_labels = Labels.load_file(filename) if val_labels.skeletons: self._skeleton = val_labels.skeletons[0] # don't try loading again (needed in case it's still None) self._tried_finding_skeleton = True return self._skeleton
def __init__( self, mode: Text, labels_filename: Text, labels: Optional[Labels] = None, skeleton: Optional["Skeleton"] = None, *args, **kwargs, ): super(LearningDialog, self).__init__() if labels is None: labels = Labels.load_file(labels_filename) if skeleton is None and labels.skeletons: skeleton = labels.skeletons[0] self.mode = mode self.labels_filename = labels_filename self.labels = labels self.skeleton = skeleton self._frame_selection = None self.current_pipeline = "" self.tabs = dict() self.shown_tab_names = [] self._cfg_getter = configs.TrainingConfigsGetter.make_from_labels_filename( labels_filename=self.labels_filename) # Layout for buttons buttons = QtWidgets.QDialogButtonBox() self.cancel_button = buttons.addButton( QtWidgets.QDialogButtonBox.Cancel) self.save_button = buttons.addButton( "Save configuration files...", QtWidgets.QDialogButtonBox.ApplyRole) self.run_button = buttons.addButton( "Run", QtWidgets.QDialogButtonBox.AcceptRole) buttons_layout = QtWidgets.QHBoxLayout() buttons_layout.addWidget(buttons, alignment=QtCore.Qt.AlignTop) buttons_layout_widget = QtWidgets.QWidget() buttons_layout_widget.setLayout(buttons_layout) self.pipeline_form_widget = TrainingPipelineWidget(mode=mode, skeleton=skeleton) if mode == "training": tab_label = "Training Pipeline" elif mode == "inference": # self.pipeline_form_widget = InferencePipelineWidget() tab_label = "Inference Pipeline" else: raise ValueError(f"Invalid LearningDialog mode: {mode}") self.tab_widget = QtWidgets.QTabWidget() self.tab_widget.addTab(self.pipeline_form_widget, tab_label) self.make_tabs() self.message_widget = QtWidgets.QLabel("") # Layout for entire dialog layout = QtWidgets.QVBoxLayout() layout.addWidget(self.tab_widget) layout.addWidget(self.message_widget) layout.addWidget(buttons_layout_widget) self.setLayout(layout) # Default to most recently trained pipeline (if there is one) self.set_pipeline_from_most_recent() # Connect functions to update pipeline tabs when pipeline changes self.pipeline_form_widget.updatePipeline.connect(self.set_pipeline) self.pipeline_form_widget.emitPipeline() self.connect_signals() # Connect actions for buttons buttons.accepted.connect(self.run) buttons.rejected.connect(self.reject) buttons.clicked.connect(self.on_button_click) # Connect button for previewing the training data if "_view_datagen" in self.pipeline_form_widget.buttons: self.pipeline_form_widget.buttons["_view_datagen"].clicked.connect( self.view_datagen)
def read( cls, file: FileHandle, video: Union[Video, str], *args, **kwargs, ) -> Labels: connect_adj_nodes = False if video is None: raise ValueError( "Cannot read analysis hdf5 if no video specified.") if not isinstance(video, Video): video = Video.from_filename(video) f = file.file tracks_matrix = f["tracks"][:].T track_names_list = f["track_names"][:].T node_names_list = f["node_names"][:].T # shape: frames * nodes * 2 * tracks frame_count, node_count, _, track_count = tracks_matrix.shape tracks = [ Track(0, track_name.decode()) for track_name in track_names_list ] skeleton = Skeleton() last_node_name = None for node_name in node_names_list: node_name = node_name.decode() skeleton.add_node(node_name) if connect_adj_nodes and last_node_name: skeleton.add_edge(last_node_name, node_name) last_node_name = node_name frames = [] for frame_idx in range(frame_count): instances = [] for track_idx in range(track_count): points = tracks_matrix[frame_idx, ..., track_idx] if not np.all(np.isnan(points)): point_scores = np.ones(len(points)) # make everything a PredictedInstance since the usual use # case is to export predictions for analysis instances.append( PredictedInstance.from_arrays( points=points, point_confidences=point_scores, skeleton=skeleton, track=tracks[track_idx], instance_score=1, )) if instances: frames.append( LabeledFrame(video=video, frame_idx=frame_idx, instances=instances)) return Labels(labeled_frames=frames)
self.context.labels.remove_instance(lf, inst, in_transaction=True) if not lf.instances: self.context.labels.remove(lf) # Update caches since we skipped doing this after each deletion self.context.labels.update_cache() # Log update self.context.changestack_push("delete instances") if __name__ == "__main__": app = QtWidgets.QApplication([]) from sleap import Labels from sleap.gui.commands import CommandContext labels = Labels.load_file( "tests/data/json_format_v2/centered_pair_predictions.json") context = CommandContext.from_labels(labels) context.state["frame_idx"] = 123 context.state["video"] = labels.videos[0] context.state["has_frame_range"] = True context.state["frame_range"] = (10, 20) win = DeleteDialog(context=context) win.show() app.exec_()
def read( cls, file: FileHandle, video_search: Union[Callable, List[Text], None] = None, match_to: Optional[Labels] = None, *args, **kwargs, ) -> Labels: pass """ Deserialize JSON file as new :class:`Labels` instance. Args: filename: Path to JSON file. video_callback: A callback function that which can modify video paths before we try to create the corresponding :class:`Video` objects. Usually you'll want to pass a callback created by :meth:`make_video_callback` or :meth:`make_gui_video_callback`. Alternately, if you pass a list of strings we'll construct a non-gui callback with those strings as the search paths. match_to: If given, we'll replace particular objects in the data dictionary with *matching* objects in the match_to :class:`Labels` object. This ensures that the newly instantiated :class:`Labels` can be merged without duplicate matching objects (e.g., :class:`Video` objects ). Returns: A new :class:`Labels` object. """ tmp_dir = None filename = file.filename # Check if the file is a zipfile for not. if zipfile.is_zipfile(filename): # Make a tmpdir, located in the directory that the file exists, to unzip # its contents. tmp_dir = os.path.join( os.path.dirname(filename), f"tmp_{os.getpid()}_{os.path.basename(filename)}", ) if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir, ignore_errors=True) try: os.mkdir(tmp_dir) except FileExistsError: pass # tmp_dir = tempfile.mkdtemp(dir=os.path.dirname(filename)) try: # Register a cleanup routine that deletes the tmpdir on program exit # if something goes wrong. The True is for ignore_errors atexit.register(shutil.rmtree, tmp_dir, True) # Uncompress the data into the directory shutil.unpack_archive(filename, extract_dir=tmp_dir) # We can now open the JSON file, save the zip file and # replace file with the first JSON file we find in the archive. json_files = [ os.path.join(tmp_dir, file) for file in os.listdir(tmp_dir) if file.endswith(".json") ] if len(json_files) == 0: raise ValueError( f"No JSON file found inside {filename}. Are you sure this is a valid sLEAP dataset." ) filename = json_files[0] except Exception as ex: # If we had problems, delete the temp directory and reraise the exception. shutil.rmtree(tmp_dir, ignore_errors=True) raise # Open and parse the JSON in filename with open(filename, "r") as file: # FIXME: Peek into the json to see if there is version string. # We do this to tell apart old JSON data from leap_dev vs the # newer format for sLEAP. json_str = file.read() dicts = json_loads(json_str) # If we have a version number, then it is new sLEAP format if "version" in dicts: # Cache the working directory. cwd = os.getcwd() # Replace local video paths (for imagestore) if tmp_dir: for vid in dicts["videos"]: vid["backend"]["filename"] = os.path.join( tmp_dir, vid["backend"]["filename"]) # Use the video_callback for finding videos with broken paths: # 1. Accept single string as video search path if isinstance(video_search, str): video_search = [video_search] # 2. Accept list of strings as video search paths if hasattr(video_search, "__iter__"): # If the callback is an iterable, then we'll expect it to be a # list of strings and build a non-gui callback with those as # the search paths. # When path is to a file, use the path of parent directory. search_paths = [ os.path.dirname(path) if os.path.isfile(path) else path for path in video_search ] # Make the search function from list of paths video_search = Labels.make_video_callback(search_paths) # 3. Use the callback function (either given as arg or build from paths) if callable(video_search): abort = video_search(dicts["videos"]) if abort: raise FileNotFoundError # Try to load the labels filename. try: labels = cls.from_json_data(dicts, match_to=match_to) except FileNotFoundError: # FIXME: We are going to the labels JSON that has references to # video files. Lets change directory to the dirname of the json file # so that relative paths will be from this directory. Maybe # it is better to feed the dataset dirname all the way down to # the Video object. This seems like less coupling between classes # though. if os.path.dirname(filename) != "": os.chdir(os.path.dirname(filename)) # Try again labels = cls.from_json_data(dicts, match_to=match_to) except Exception as ex: # Ok, we give up, where the hell are these videos! raise # Re-raise. finally: os.chdir( cwd) # Make sure to change back if we have problems. return labels else: frames = load_labels_json_old(data_path=filename, parsed_json=dicts) return Labels(frames)
def from_json_data(cls, data: Union[str, dict], match_to: Optional["Labels"] = None) -> "Labels": """ Create instance of class from data in dictionary. Method is used by other methods that load from JSON. Args: data: Dictionary, deserialized from JSON. match_to: If given, we'll replace particular objects in the data dictionary with *matching* objects in the match_to :class:`Labels` object. This ensures that the newly instantiated :class:`Labels` can be merged without duplicate matching objects (e.g., :class:`Video` objects ). Returns: A new :class:`Labels` object. """ # Parse the json string if needed. if type(data) is str: dicts = json_loads(data) else: dicts = data dicts["tracks"] = dicts.get( "tracks", []) # don't break if json doesn't include tracks # First, deserialize the skeletons, videos, and nodes lists. # The labels reference these so we will need them while deserializing. nodes = cattr.structure(dicts["nodes"], List[Node]) idx_to_node = {i: nodes[i] for i in range(len(nodes))} skeletons = Skeleton.make_cattr(idx_to_node).structure( dicts["skeletons"], List[Skeleton]) videos = Video.cattr().structure(dicts["videos"], List[Video]) try: # First try unstructuring tuple (newer format) track_cattr = cattr.Converter( unstruct_strat=cattr.UnstructureStrategy.AS_TUPLE) tracks = track_cattr.structure(dicts["tracks"], List[Track]) except: # Then try unstructuring dict (older format) try: tracks = cattr.structure(dicts["tracks"], List[Track]) except: raise ValueError("Unable to load tracks as tuple or dict!") # if we're given a Labels object to match, use its objects when they match if match_to is not None: for idx, sk in enumerate(skeletons): for old_sk in match_to.skeletons: if sk.matches(old_sk): # use nodes from matched skeleton for (node, match_node) in zip(sk.nodes, old_sk.nodes): node_idx = nodes.index(node) nodes[node_idx] = match_node # use skeleton from match skeletons[idx] = old_sk break for idx, vid in enumerate(videos): for old_vid in match_to.videos: # Try to match videos using either their current or source filename # if available. old_vid_paths = [old_vid.filename] if getattr(old_vid.backend, "has_embedded_images", False): old_vid_paths.append( old_vid.backend._source_video.filename) new_vid_paths = [vid.filename] if getattr(vid.backend, "has_embedded_images", False): new_vid_paths.append( vid.backend._source_video.filename) is_match = False for old_vid_path in old_vid_paths: for new_vid_path in new_vid_paths: if old_vid_path == new_vid_path or weak_filename_match( old_vid_path, new_vid_path): is_match = True videos[idx] = old_vid break if is_match: break if is_match: break suggestions = [] if "suggestions" in dicts: suggestions_cattr = cattr.Converter() suggestions_cattr.register_structure_hook( Video, lambda x, type: videos[int(x)]) try: suggestions = suggestions_cattr.structure( dicts["suggestions"], List[SuggestionFrame]) except Exception as e: print("Error while loading suggestions (1)") print(e) try: # Convert old suggestion format to new format. # Old format: {video: list of frame indices} # New format: [SuggestionFrames] old_suggestions = suggestions_cattr.structure( dicts["suggestions"], Dict[Video, List]) for video in old_suggestions.keys(): suggestions.extend([ SuggestionFrame(video, idx) for idx in old_suggestions[video] ]) except Exception as e: print("Error while loading suggestions (2)") print(e) pass if "negative_anchors" in dicts: negative_anchors_cattr = cattr.Converter() negative_anchors_cattr.register_structure_hook( Video, lambda x, type: videos[int(x)]) negative_anchors = negative_anchors_cattr.structure( dicts["negative_anchors"], Dict[Video, List]) else: negative_anchors = dict() if "provenance" in dicts: provenance = dicts["provenance"] else: provenance = dict() # If there is actual labels data, get it. if "labels" in dicts: label_cattr = make_instance_cattr() label_cattr.register_structure_hook( Skeleton, lambda x, type: skeletons[int(x)]) label_cattr.register_structure_hook(Video, lambda x, type: videos[int(x)]) label_cattr.register_structure_hook( Node, lambda x, type: x if isinstance(x, Node) else nodes[int(x)]) label_cattr.register_structure_hook( Track, lambda x, type: None if x is None else tracks[int(x)]) labels = label_cattr.structure(dicts["labels"], List[LabeledFrame]) else: labels = [] return Labels( labeled_frames=labels, videos=videos, skeletons=skeletons, nodes=nodes, suggestions=suggestions, negative_anchors=negative_anchors, tracks=tracks, provenance=provenance, )
def evaluate_model( cfg: TrainingJobConfig, labels_reader: LabelsReader, model: Model, save: bool = True, split_name: Text = "test", ) -> Tuple[Labels, Dict[Text, Any]]: """Evaluate a trained model and save metrics and predictions. Args: cfg: The `TrainingJobConfig` associated with the model. labels_reader: A `LabelsReader` pipeline generator that reads the ground truth data to evaluate. model: The `sleap.nn.model.Model` instance to evaluate. save: If True, save the predictions and metrics to the model folder. split_name: String name to append to the saved filenames. Returns: A tuple of `(labels_pr, metrics)`. `labels_pr` will contain the predicted labels. `metrics` will contain the evaluated metrics given the predictions, or None if the metrics failed to be computed. """ # Setup predictor for evaluation. head_config = cfg.model.heads.which_oneof() if isinstance(head_config, CentroidsHeadConfig): predictor = TopDownPredictor( centroid_config=cfg, centroid_model=model, confmap_config=None, confmap_model=None, ) elif isinstance(head_config, CenteredInstanceConfmapsHeadConfig): predictor = TopDownPredictor( centroid_config=None, centroid_model=None, confmap_config=cfg, confmap_model=model, ) elif isinstance(head_config, MultiInstanceConfig): predictor = sleap.nn.inference.BottomUpPredictor(bottomup_config=cfg, bottomup_model=model) elif isinstance(head_config, SingleInstanceConfmapsHeadConfig): predictor = sleap.nn.inference.SingleInstancePredictor( confmap_config=cfg, confmap_model=model) else: raise ValueError("Unrecognized model type:", head_config) # Predict. labels_pr = predictor.predict(labels_reader, make_labels=True) # Compute metrics. try: metrics = evaluate(labels_reader.labels, labels_pr) except: logger.warning("Failed to compute metrics.") metrics = None # Save. if save: labels_pr_path = os.path.join(cfg.outputs.run_path, f"labels_pr.{split_name}.slp") Labels.save_file(labels_pr, labels_pr_path) logger.info("Saved predictions: %s", labels_pr_path) if metrics is not None: metrics_path = os.path.join(cfg.outputs.run_path, f"metrics.{split_name}.npz") np.savez_compressed(metrics_path, **{"metrics": metrics}) logger.info("Saved metrics: %s", metrics_path) if metrics is not None: logger.info("OKS mAP: %f", metrics["oks_voc.mAP"]) return labels_pr, metrics
# This script automatically deletes predictions with scores below 10%. # It also names each track after the uploaded file, but may need to be altered based on circumstances (lines 36-37). from sleap import Labels import sys SCORE_THRESHOLD = .1 filename = sys.argv[1] out_filename = str(filename)+'_highscores_trackName.h5' labels = Labels.load_file(filename) lf_inst_list = [] # Find the (frame, instance) pairs with score below threshold for frame in labels: for instance in frame: if hasattr(instance, "score"): if instance.score is not None: if instance.score < SCORE_THRESHOLD: lf_inst_list.append((frame, instance)) if lf_inst_list: print(f"Removing {len(lf_inst_list)} instances...") # Remove each of the instances for frame, instance in lf_inst_list: labels.remove_instance(frame, instance, in_transaction=True)
def read( cls, file: FileHandle, img_dir: str, use_missing_gui: bool = False, *args, **kwargs, ) -> Labels: dicts = file.json # Make skeletons from "categories" skeleton_map = dict() for category in dicts["categories"]: skeleton = Skeleton(name=category["name"]) skeleton_id = category["id"] node_names = category["keypoints"] skeleton.add_nodes(node_names) try: for src_idx, dst_idx in category["skeleton"]: skeleton.add_edge(node_names[src_idx], node_names[dst_idx]) except IndexError as e: # According to the COCO data format specifications[^1], the edges # are supposed to be 1-indexed. But in some of their own # dataset the edges are 1-indexed! So we'll try. # [1]: http://cocodataset.org/#format-data # Clear any edges we already created using 0-indexing skeleton.clear_edges() # Add edges for src_idx, dst_idx in category["skeleton"]: skeleton.add_edge(node_names[src_idx - 1], node_names[dst_idx - 1]) skeleton_map[skeleton_id] = skeleton # Make videos from "images" # Remove images that aren't referenced in the annotations img_refs = [annotation["image_id"] for annotation in dicts["annotations"]] dicts["images"] = list(filter(lambda im: im["id"] in img_refs, dicts["images"])) # Key in JSON file should be "file_name", but sometimes it's "filename", # so we have to check both. img_filename_key = "file_name" if img_filename_key not in dicts["images"][0].keys(): img_filename_key = "filename" # First add the img_dir to each image filename img_paths = [ os.path.join(img_dir, image[img_filename_key]) for image in dicts["images"] ] # See if there are any missing files img_missing = [not os.path.exists(path) for path in img_paths] if sum(img_missing): if use_missing_gui: okay = MissingFilesDialog(img_paths, img_missing).exec_() if not okay: return None else: raise FileNotFoundError( f"Images for COCO dataset could not be found in {img_dir}." ) # Update the image paths (with img_dir or user selected path) for image, path in zip(dicts["images"], img_paths): image[img_filename_key] = path # Create the video objects for the image files image_video_map = dict() vid_id_video_map = dict() for image in dicts["images"]: image_id = image["id"] image_filename = image[img_filename_key] # Sometimes images have a vid_id which links multiple images # together as one video. If so, we'll use that as the video key. # But if there isn't a vid_id, we'll treat each images as a # distinct video and use the image id as the video id. vid_id = image.get("vid_id", image_id) if vid_id not in vid_id_video_map: kwargs = dict(filenames=[image_filename]) for key in ("width", "height"): if key in image: kwargs[key] = image[key] video = Video.from_image_filenames(**kwargs) vid_id_video_map[vid_id] = video frame_idx = 0 else: video = vid_id_video_map[vid_id] frame_idx = video.num_frames video.backend.filenames.append(image_filename) image_video_map[image_id] = (video, frame_idx) # Make instances from "annotations" lf_map = dict() track_map = dict() for annotation in dicts["annotations"]: skeleton = skeleton_map[annotation["category_id"]] image_id = annotation["image_id"] video, frame_idx = image_video_map[image_id] keypoints = np.array(annotation["keypoints"], dtype="int").reshape(-1, 3) track = None if "track_id" in annotation: track_id = annotation["track_id"] if track_id not in track_map: track_map[track_id] = Track(frame_idx, str(track_id)) track = track_map[track_id] points = dict() any_visible = False for i in range(len(keypoints)): node = skeleton.nodes[i] x, y, flag = keypoints[i] if flag == 0: # node not labeled for this instance continue is_visible = flag == 2 any_visible = any_visible or is_visible points[node] = Point(x, y, is_visible) if points: # If none of the points had 2 has the "visible" flag, we'll # assume this incorrect and just mark all as visible. if not any_visible: for point in points.values(): point.visible = True inst = Instance(skeleton=skeleton, points=points, track=track) if image_id not in lf_map: lf_map[image_id] = LabeledFrame(video, frame_idx) lf_map[image_id].insert(0, inst) return Labels(labeled_frames=list(lf_map.values()))
def read( cls, file: FileHandle, gui: bool = True, *args, **kwargs, ): filename = file.filename mat_contents = sio.loadmat(filename) box_path = cls._unwrap_mat_scalar(mat_contents["boxPath"]) # If the video file isn't found, try in the same dir as the mat file if not os.path.exists(box_path): file_dir = os.path.dirname(filename) box_path_name = box_path.split("\\")[-1] # assume windows path box_path = os.path.join(file_dir, box_path_name) if not os.path.exists(box_path): if gui: video_paths = [box_path] missing = [True] okay = MissingFilesDialog(video_paths, missing).exec_() if not okay or missing[0]: return box_path = video_paths[0] else: # Ignore missing videos if not loading from gui box_path = "" if os.path.exists(box_path): vid = Video.from_hdf5( dataset="box", filename=box_path, input_format="channels_first" ) else: vid = None nodes_ = mat_contents["skeleton"]["nodes"] edges_ = mat_contents["skeleton"]["edges"] points_ = mat_contents["positions"] edges_ = edges_ - 1 # convert matlab 1-indexing to python 0-indexing nodes = cls._unwrap_mat_array(nodes_) edges = cls._unwrap_mat_array(edges_) nodes = list(map(str, nodes)) # convert np._str to str sk = Skeleton(name=filename) sk.add_nodes(nodes) for edge in edges: sk.add_edge(source=nodes[edge[0]], destination=nodes[edge[1]]) labeled_frames = [] node_count, _, frame_count = points_.shape for i in range(frame_count): new_inst = Instance(skeleton=sk) for node_idx, node in enumerate(nodes): x = points_[node_idx][0][i] y = points_[node_idx][1][i] new_inst[node] = Point(x, y) if len(new_inst.points): new_frame = LabeledFrame(video=vid, frame_idx=i) new_frame.instances = (new_inst,) labeled_frames.append(new_frame) labels = Labels(labeled_frames=labeled_frames, videos=[vid], skeletons=[sk]) return labels
"""Returns mean of aligned points for instances.""" points = get_instances_points(instances) node_a, node_b = get_most_stable_node_pair(points, min_dist=4.0) aligned = align_instances(points, node_a=node_a, node_b=node_b) points_mean, points_std = get_mean_and_std_for_points(aligned) return points_mean if __name__ == "__main__": # filename = "tests/data/json_format_v2/centered_pair_predictions.json" # filename = "/Volumes/fileset-mmurthy/shruthi/code/sleap_expts/preds/screen_all.5pts_tmp_augment_200122/191210_102108_18159112_rig3_2.preds.h5" filename = "/Volumes/fileset-mmurthy/talmo/wt_gold_labeling/100919.sleap_wt_gold.13pt_init.n=288.junyu.h5" labels = Labels.load_file(filename) points = get_instances_points(labels.instances()) get_stable_node_pairs(points, np.array(labels.skeletons[0].node_names)) # import time # # t0 = time.time() # labels.add_instance( # frame=labels.find_first(video=labels.videos[0]), # instance=make_mean_instance(align_instances(points, 12, 0)) # ) # print(labels.find_first(video=labels.videos[0])) # print("time", time.time() - t0) # # Labels.save_file(labels, "mean.h5")
def main(): parser = argparse.ArgumentParser() parser.add_argument("input_path", help="Path to input file.") parser.add_argument( "-o", "--output", default="", help="Path to output file (optional)." ) parser.add_argument( "--format", default="slp", help="Output format. Default ('slp') is SLEAP dataset; " "'analysis' results in analysis.h5 file; " "'h5' or 'json' results in SLEAP dataset " "with specified file format.", ) parser.add_argument( "--video", default="", help="Path to video (if needed for conversion)." ) args = parser.parse_args() video_callback = Labels.make_video_callback([os.path.dirname(args.input_path)]) try: labels = Labels.load_file(args.input_path, video_search=video_callback) except TypeError: print("Input file isn't SLEAP dataset so attempting other importers...") from sleap.io.format import read video_path = args.video if args.video else None labels = read( args.input_path, for_object="labels", as_format="*", video_search=video_callback, video=video_path, ) if args.format == "analysis": from sleap.info.write_tracking_h5 import main as write_analysis if args.output: output_path = args.output else: output_path = args.input_path output_path = re.sub("(\.json(\.zip)?|\.h5|\.slp)$", "", output_path) output_path = output_path + ".analysis.h5" write_analysis(labels, output_path=output_path, all_frames=True) elif args.output: print(f"Output SLEAP dataset: {args.output}") Labels.save_file(labels, args.output) elif args.format in ("slp", "h5", "json"): output_path = f"{args.input_path}.{args.format}" print(f"Output SLEAP dataset: {output_path}") Labels.save_file(labels, output_path) else: print("You didn't specify how to convert the file.") print(args)