def multi_skel_instances(skeleton, stickman): """ Setup some instances that reference multiple skeletons """ # Generate some instances NUM_INSTANCES = 500 instances = [] for i in range(NUM_INSTANCES): instance = Instance(skeleton=skeleton, video=None, frame_idx=i) instance["head"] = Point(i * 1, i * 2) instance["left-wing"] = Point(10 + i * 1, 10 + i * 2) instance["right-wing"] = Point(20 + i * 1, 20 + i * 2) # Lets make an NaN entry to test skip_nan as well instance["thorax"] instances.append(instance) # Setup some instances of the stick man on the same frames for i in range(NUM_INSTANCES): instance = Instance(skeleton=stickman, video=None, frame_idx=i) instance["head"] = Point(i * 10, i * 20) instance["body"] = Point(100 + i * 1, 100 + i * 2) instance["left-arm"] = Point(200 + i * 1, 200 + i * 2) instances.append(instance) return instances
def test_instance_access(): labels = Labels() dummy_skeleton = Skeleton() dummy_video = Video(backend=MediaVideo) dummy_video2 = Video(backend=MediaVideo) for i in range(10): labels.append( LabeledFrame( dummy_video, frame_idx=i, instances=[Instance(dummy_skeleton), Instance(dummy_skeleton)], )) for i in range(10): labels.append( LabeledFrame( dummy_video2, frame_idx=i, instances=[ Instance(dummy_skeleton), Instance(dummy_skeleton), Instance(dummy_skeleton), ], )) assert len(labels.all_instances) == 50 assert len(list(labels.instances(video=dummy_video))) == 20 assert len(list(labels.instances(video=dummy_video2))) == 30
def test_label_mutability(): dummy_video = Video(backend=MediaVideo) dummy_skeleton = Skeleton() dummy_instance = Instance(dummy_skeleton) dummy_frame = LabeledFrame(dummy_video, frame_idx=0, instances=[dummy_instance]) labels = Labels() labels.append(dummy_frame) assert dummy_video in labels.videos assert dummy_video in labels assert dummy_skeleton in labels.skeletons assert dummy_skeleton in labels assert dummy_frame in labels.labeled_frames assert dummy_frame in labels assert (dummy_video, 0) in labels assert (dummy_video, 1) not in labels dummy_video2 = Video(backend=MediaVideo) dummy_skeleton2 = Skeleton(name="dummy2") dummy_instance2 = Instance(dummy_skeleton2) dummy_frame2 = LabeledFrame(dummy_video2, frame_idx=0, instances=[dummy_instance2]) assert dummy_video2 not in labels assert dummy_skeleton2 not in labels assert dummy_frame2 not in labels labels.append(dummy_frame2) assert dummy_video2 in labels assert dummy_frame2 in labels labels.remove_video(dummy_video2) assert dummy_video2 not in labels assert dummy_frame2 not in labels assert len(labels.find(dummy_video2)) == 0 assert len(labels) == 1 labels.append(LabeledFrame(dummy_video, frame_idx=0)) assert len(labels) == 1 dummy_frames = [LabeledFrame(dummy_video, frame_idx=i) for i in range(10)] dummy_frames2 = [LabeledFrame(dummy_video2, frame_idx=i) for i in range(10)] for f in dummy_frames + dummy_frames2: labels.append(f) assert len(labels) == 20 labels.remove_video(dummy_video2) assert len(labels) == 10 assert len(labels.find(dummy_video)) == 10 assert dummy_frame in labels assert all([label in labels for label in dummy_frames[1:]]) assert dummy_video2 not in labels assert len(labels.find(dummy_video2)) == 0 assert all([label not in labels for label in dummy_frames2]) labels.remove_video(dummy_video) assert len(labels.find(dummy_video)) == 0
def test_non_exist_node(skeleton): """ Test is instances throw key errors for nodes that don't exist in the skeleton. """ instance = Instance(skeleton=skeleton) with pytest.raises(KeyError): instance["non-existent-node"].x = 1 with pytest.raises(KeyError): instance = Instance(skeleton=skeleton, points={"non-exist": Point()})
def test_duplicate_skeletons_serializing(): vid = Video.from_filename("foo.mp4") skeleton_a = Skeleton.load_json("tests/data/skeleton/fly_skeleton_legs.json") skeleton_b = Skeleton.load_json("tests/data/skeleton/fly_skeleton_legs.json") lf_a = LabeledFrame(vid, frame_idx=2, instances=[Instance(skeleton_a)]) lf_b = LabeledFrame(vid, frame_idx=3, instances=[Instance(skeleton_b)]) new_labels = Labels(labeled_frames=[lf_a, lf_b]) new_labels_json = new_labels.to_dict()
def test_scalar_properties(): # Scalar dummy_video = Video(backend=MediaVideo) dummy_skeleton = Skeleton() dummy_instance = Instance(dummy_skeleton) dummy_frame = LabeledFrame(dummy_video, frame_idx=0, instances=[dummy_instance]) labels = Labels() labels.append(dummy_frame) assert labels.video == dummy_video assert labels.skeleton == dummy_skeleton # Empty labels = Labels() with pytest.raises(ValueError): labels.video with pytest.raises(ValueError): labels.skeleton # More than one video dummy_skeleton = Skeleton() labels = Labels() labels.append( LabeledFrame(Video(backend=MediaVideo), frame_idx=0, instances=[Instance(dummy_skeleton)])) labels.append( LabeledFrame(Video(backend=MediaVideo), frame_idx=0, instances=[Instance(dummy_skeleton)])) assert labels.skeleton == dummy_skeleton with pytest.raises(ValueError): labels.video # More than one skeleton dummy_video = Video(backend=MediaVideo) labels = Labels() labels.append( LabeledFrame(dummy_video, frame_idx=0, instances=[Instance(Skeleton())])) labels.append( LabeledFrame(dummy_video, frame_idx=1, instances=[Instance(Skeleton())])) assert labels.video == dummy_video with pytest.raises(ValueError): labels.skeleton
def test_distinct_skeletons_serializing(): vid = Video.from_filename("foo.mp4") skeleton_a = Skeleton.load_json("tests/data/skeleton/fly_skeleton_legs.json") skeleton_b = Skeleton.load_json("tests/data/skeleton/fly_skeleton_legs.json") skeleton_b.add_node("foo") lf_a = LabeledFrame(vid, frame_idx=2, instances=[Instance(skeleton_a)]) lf_b = LabeledFrame(vid, frame_idx=3, instances=[Instance(skeleton_b)]) new_labels = Labels(labeled_frames=[lf_a, lf_b]) # Make sure we can serialize this new_labels_json = new_labels.to_dict()
def multi_skel_vid_labels(hdf5_vid, small_robot_mp4_vid, skeleton, stickman): """ Build a big list of LabeledFrame objects and wrap it in Labels class. Args: hdf5_vid: An HDF5 video fixture small_robot_mp4_vid: An MP4 video fixture skeleton: A fly skeleton. stickman: A stickman skeleton Returns: The Labels object containing all the labeled frames """ labels = [] stick_tracks = [ Track(spawned_on=0, name=f"Stickman {i}") for i in range(6) ] fly_tracks = [Track(spawned_on=0, name=f"Fly {i}") for i in range(6)] # Make some tracks None to test that fly_tracks[3] = None stick_tracks[2] = None for f in range(500): vid = [hdf5_vid, small_robot_mp4_vid][f % 2] label = LabeledFrame(video=vid, frame_idx=f % vid.frames) fly_instances = [] for i in range(6): fly_instances.append( Instance(skeleton=skeleton, track=fly_tracks[i])) for node in skeleton.nodes: fly_instances[i][node] = Point(x=i % vid.width, y=i % vid.height) stickman_instances = [] for i in range(6): stickman_instances.append( Instance(skeleton=stickman, track=stick_tracks[i])) for node in stickman.nodes: stickman_instances[i][node] = Point(x=i % vid.width, y=i % vid.height) label.instances = stickman_instances + fly_instances labels.append(label) labels = Labels(labels) return labels
def test_frame_merge_predicted_and_user(skeleton, centered_pair_vid): user_inst = Instance( skeleton=skeleton, points={skeleton.nodes[0]: Point(1, 2)}, ) user_frame = LabeledFrame( video=centered_pair_vid, frame_idx=0, instances=[user_inst], ) pred_inst = PredictedInstance( skeleton=skeleton, points={skeleton.nodes[0]: PredictedPoint(1, 2, score=1.0)}, score=1.0, ) pred_frame = LabeledFrame( video=centered_pair_vid, frame_idx=0, instances=[pred_inst], ) LabeledFrame.complex_frame_merge(user_frame, pred_frame) # We should be able to cleanly merge the user and the predicted instance, # and we want to retain both even though they perfectly match. assert user_inst in user_frame.instances assert pred_inst in user_frame.instances assert len(user_frame.instances) == 2
def removal_test_labels(): skeleton = Skeleton() video = Video(backend=MediaVideo) lf_user_only = LabeledFrame( video=video, frame_idx=0, instances=[Instance(skeleton=skeleton)] ) lf_pred_only = LabeledFrame( video=video, frame_idx=1, instances=[PredictedInstance(skeleton=skeleton)] ) lf_both = LabeledFrame( video=video, frame_idx=2, instances=[Instance(skeleton=skeleton), PredictedInstance(skeleton=skeleton)], ) labels = Labels([lf_user_only, lf_pred_only, lf_both]) return labels
def test_instance_node_get_set_item(skeleton): """ Test basic get item and set item functionality of instances. """ instance = Instance(skeleton=skeleton) instance["head"].x = 20 instance["head"].y = 50 instance["left-wing"] = Point(x=30, y=40, visible=False) assert instance["head"].x == 20 assert instance["head"].y == 50 assert instance["left-wing"] == Point(x=30, y=40, visible=False) thorax_point = instance["thorax"] assert math.isnan(thorax_point.x) and math.isnan(thorax_point.y) instance[0] = [-20, -50] assert instance["head"].x == -20 assert instance["head"].y == -50 instance[0] = np.array([-21, -51]) assert instance["head"].x == -21 assert instance["head"].y == -51
def test_frame_merge_between_predicted_and_user(skeleton, centered_pair_vid): user_inst = Instance(skeleton=skeleton, points={skeleton.nodes[0]: Point(1, 2)},) user_labels = Labels( [LabeledFrame(video=centered_pair_vid, frame_idx=0, instances=[user_inst],)] ) pred_inst = PredictedInstance( skeleton=skeleton, points={skeleton.nodes[0]: PredictedPoint(1, 2, score=1.0)}, score=1.0, ) pred_labels = Labels( [LabeledFrame(video=centered_pair_vid, frame_idx=0, instances=[pred_inst],)] ) # Merge predictions into current labels dataset _, _, new_conflicts = Labels.complex_merge_between( user_labels, new_labels=pred_labels, unify=False, # since we used match_to when loading predictions file ) # new predictions should replace old ones Labels.finish_complex_merge(user_labels, new_conflicts) # We should be able to cleanly merge the user and the predicted instance, # and we want to retain both even though they perfectly match. assert user_inst in user_labels[0].instances assert pred_inst in user_labels[0].instances assert len(user_labels[0].instances) == 2
def test_points_array(skeleton): """ Test conversion of instances to points array""" node_names = ["left-wing", "head", "right-wing"] points = {"head": Point(1, 4), "left-wing": Point(2, 5), "right-wing": Point(3, 6)} instance1 = Instance(skeleton=skeleton, points=points) pts = instance1.get_points_array() assert pts.shape == (len(skeleton.nodes), 2) assert np.allclose(pts[skeleton.node_to_index("left-wing"), :], [2, 5]) assert np.allclose(pts[skeleton.node_to_index("head"), :], [1, 4]) assert np.allclose(pts[skeleton.node_to_index("right-wing"), :], [3, 6]) assert np.isnan(pts[skeleton.node_to_index("thorax"), :]).all() # Now change a point, make sure it is reflected instance1["head"].x = 0 instance1["thorax"] = Point(1, 2) pts = instance1.get_points_array() assert np.allclose(pts[skeleton.node_to_index("head"), :], [0, 4]) assert np.allclose(pts[skeleton.node_to_index("thorax"), :], [1, 2]) # Make sure that invisible points are nan iff invisible_as_nan=True instance1["thorax"] = Point(1, 2, visible=False) pts = instance1.get_points_array() assert not np.isnan(pts[skeleton.node_to_index("thorax"), :]).all() pts = instance1.points_array assert np.isnan(pts[skeleton.node_to_index("thorax"), :]).all()
def test_points_array_copying(skeleton): node_names = ["left-wing", "head", "right-wing"] points = {"head": Point(1, 4), "left-wing": Point(2, 5), "right-wing": Point(3, 6)} instance1 = Instance(skeleton=skeleton, points=points) first_node = skeleton.nodes[0] # Make sure that changing *uncopied* points array does change instance. pts = instance1.get_points_array(copy=False) assert pts[0]["x"] == instance1[first_node].x pts[0]["x"] = 123 assert pts[0]["x"] == instance1[first_node].x # these should match # Make sure that changing copied points array doesn't change instance. pts = instance1.get_points_array(copy=True) assert pts[0][0] == instance1[first_node].x pts[0][0] = 456 assert pts[0][0] != instance1[first_node].x # these shouldn't match # Make sure we can get full copy pts = instance1.get_points_array(copy=True, full=True) assert pts.shape[1] == 4 # x, y, visible, complete # Make sure we can get copy with just coordinates pts = instance1.get_points_array(copy=True, full=False) assert pts.shape[1] == 2 # x, y
def test_merge_predictions(): dummy_video_a = Video.from_filename("foo.mp4") dummy_video_b = Video.from_filename("foo.mp4") dummy_skeleton_a = Skeleton() dummy_skeleton_a.add_node("node") dummy_skeleton_b = Skeleton() dummy_skeleton_b.add_node("node") dummy_instances_a = [] dummy_instances_a.append( Instance(skeleton=dummy_skeleton_a, points=dict(node=Point(1, 1))) ) dummy_instances_a.append( Instance(skeleton=dummy_skeleton_a, points=dict(node=Point(2, 2))) ) labels_a = Labels() labels_a.append( LabeledFrame(dummy_video_a, frame_idx=0, instances=dummy_instances_a) ) dummy_instances_b = [] dummy_instances_b.append( Instance(skeleton=dummy_skeleton_b, points=dict(node=Point(1, 1))) ) dummy_instances_b.append( PredictedInstance( skeleton=dummy_skeleton_b, points=dict(node=Point(3, 3)), score=1 ) ) labels_b = Labels() labels_b.append( LabeledFrame(dummy_video_b, frame_idx=0, instances=dummy_instances_b) ) # Frames have one redundant instance (perfect match) and all the # non-matching instances are different types (one predicted, one not). merged, extra_a, extra_b = Labels.complex_merge_between(labels_a, labels_b) assert len(merged[dummy_video_a]) == 1 assert len(merged[dummy_video_a][0]) == 1 # the predicted instance was merged assert not extra_a assert not extra_b
def test_dont_unify_skeletons(): vid = Video.from_filename("foo.mp4") skeleton_a = Skeleton.load_json("tests/data/skeleton/fly_skeleton_legs.json") skeleton_b = Skeleton.load_json("tests/data/skeleton/fly_skeleton_legs.json") lf_a = LabeledFrame(vid, frame_idx=2, instances=[Instance(skeleton_a)]) lf_b = LabeledFrame(vid, frame_idx=3, instances=[Instance(skeleton_b)]) labels = Labels(labeled_frames=[lf_a]) labels.extend_from([lf_b], unify=False) ids = skeleton_ids_from_label_instances(labels) # Make sure we still have two distinct skeleton objects assert len(set(ids)) == 2 # Make sure we can serialize this labels.to_dict()
def test_instance_labeled_frame_ref(skeleton, centered_pair_vid): """ Test whether links between labeled frames and instances are kept """ instances = [Instance(skeleton=skeleton) for i in range(3)] frame = LabeledFrame(video=centered_pair_vid, frame_idx=0, instances=instances) assert frame.instances[0].frame == frame assert frame[0].frame == frame assert frame[0].frame_idx == 0
def test_unify_skeletons(): vid = Video.from_filename("foo.mp4") skeleton_a = Skeleton.load_json("tests/data/skeleton/fly_skeleton_legs.json") skeleton_b = Skeleton.load_json("tests/data/skeleton/fly_skeleton_legs.json") lf_a = LabeledFrame(vid, frame_idx=2, instances=[Instance(skeleton_a)]) lf_b = LabeledFrame(vid, frame_idx=3, instances=[Instance(skeleton_b)]) labels = Labels() labels.extend_from([lf_a], unify=True) labels.extend_from([lf_b], unify=True) ids = skeleton_ids_from_label_instances(labels) # Make sure that skeleton_b got replaced with skeleton_a when we # added the frame with "unify" set assert len(set(ids)) == 1 # Make sure we can serialize this labels.to_dict()
def test_multivideo_tracks(): vid_a = Video.from_filename("foo.mp4") vid_b = Video.from_filename("bar.mp4") skeleton = Skeleton.load_json("tests/data/skeleton/fly_skeleton_legs.json") track_a = Track(spawned_on=2, name="A") track_b = Track(spawned_on=3, name="B") inst_a = Instance(track=track_a, skeleton=skeleton) inst_b = Instance(track=track_b, skeleton=skeleton) lf_a = LabeledFrame(vid_a, frame_idx=2, instances=[inst_a]) lf_b = LabeledFrame(vid_b, frame_idx=3, instances=[inst_b]) labels = Labels(labeled_frames=[lf_a, lf_b]) # Try setting video B instance to track used in video A labels.track_swap(vid_b, new_track=track_a, old_track=track_b, frame_range=(3, 4)) assert inst_b.track == track_a
def test_instance_rotation(skeleton): instance = Instance(skeleton=skeleton) instance["head"].x = 20 instance["head"].y = 50 # affine transformation matrix w/ rotation and translation # cv2.getRotationMatrix2D((10, 10), 45, 1) mat = np.array([[0.70710678, 0.70710678, -4.14213562], [-0.70710678, 0.70710678, 10.0]]) instance.transform_points(mat) assert int(instance["head"].x) == 45 assert int(instance["head"].y) == 31
def test_basic_suggestions(small_robot_mp4_vid): dummy_video = small_robot_mp4_vid dummy_skeleton = Skeleton() dummy_instance = Instance(dummy_skeleton) dummy_frame = LabeledFrame(dummy_video, frame_idx=0, instances=[dummy_instance]) labels = Labels() labels.append(dummy_frame) suggestions = VideoFrameSuggestions.suggest( labels=labels, params=dict(method="sample", per_video=13) ) labels.set_suggestions(suggestions) assert len(labels.get_video_suggestions(dummy_video)) == 13
def read( cls, file: FileHandle, video_path: str, skeleton_path: str, *args, **kwargs, ) -> Labels: f = file.file video = Video.from_filename(video_path) skeleton_data = pd.read_csv(skeleton_path, header=0) skeleton = Skeleton() skeleton.add_nodes(skeleton_data["name"]) nodes = skeleton.nodes for name, parent, swap in skeleton_data.itertuples(index=False, name=None): if parent is not np.nan: skeleton.add_edge(parent, name) lfs = [] pose_matrix = f["pose"][:] track_count, frame_count, node_count, _ = pose_matrix.shape tracks = [Track(0, f"Track {i}") for i in range(track_count)] for frame_idx in range(frame_count): lf_instances = [] for track_idx in range(track_count): points_array = pose_matrix[track_idx, frame_idx, :, :] points = dict() for p in range(len(points_array)): x, y, score = points_array[p] points[nodes[p]] = Point(x, y) # TODO: score inst = Instance(skeleton=skeleton, track=tracks[track_idx], points=points) lf_instances.append(inst) lfs.append( LabeledFrame(video, frame_idx=frame_idx, instances=lf_instances)) return Labels(labeled_frames=lfs)
def test_instance_node_multi_get_set_item(skeleton): """ Test basic get item and set item functionality of instances. """ node_names = ["left-wing", "head", "right-wing"] points = {"head": Point(1, 4), "left-wing": Point(2, 5), "right-wing": Point(3, 6)} instance1 = Instance(skeleton=skeleton, points=points) instance1[node_names] = list(points.values()) x_values = [p.x for p in instance1[node_names]] y_values = [p.y for p in instance1[node_names]] assert np.allclose(x_values, [1, 2, 3]) assert np.allclose(y_values, [4, 5, 6])
def test_instance_point_iter(skeleton): """ Test iteration methods over instances. """ node_names = ["left-wing", "head", "right-wing"] points = {"head": Point(1, 4), "left-wing": Point(2, 5), "right-wing": Point(3, 6)} instance = Instance(skeleton=skeleton, points=points) assert [node.name for node in instance.nodes] == ["head", "left-wing", "right-wing"] assert np.allclose([p.x for p in instance.points], [1, 2, 3]) assert np.allclose([p.y for p in instance.points], [4, 5, 6]) # Make sure we can iterate over tuples for (node, point) in instance.nodes_points: assert points[node.name] == point
def test_modifying_skeleton(skeleton): node_names = ["left-wing", "head", "right-wing"] points = {"head": Point(1, 4), "left-wing": Point(2, 5), "right-wing": Point(3, 6)} instance1 = Instance(skeleton=skeleton, points=points) assert len(instance1.points) == 3 skeleton.add_node("new test node") instance1.points # this updates instance with changes from skeleton instance1["new test node"] = Point(7, 8) assert len(instance1.points) == 4 skeleton.delete_node("head") assert len(instance1.points) == 3
def instances(skeleton): # Generate some instances NUM_INSTANCES = 500 instances = [] for i in range(NUM_INSTANCES): instance = Instance(skeleton=skeleton) instance["head"] = Point(i * 1, i * 2) instance["left-wing"] = Point(10 + i * 1, 10 + i * 2) instance["right-wing"] = Point(20 + i * 1, 20 + i * 2) # Lets make an NaN entry to test skip_nan as well instance["thorax"] instances.append(instance) return instances
def test_remove_user_instances_with_new_labels(removal_test_labels): labels = removal_test_labels assert len(labels) == 3 new_labels = Labels([ LabeledFrame( video=labels.video, frame_idx=0, instances=[Instance(skeleton=labels.skeleton)], ) ]) labels.remove_user_instances(new_labels=new_labels) assert len(labels) == 2 assert labels[0].frame_idx == 1 assert not labels[0].has_user_instances assert labels[0].has_predicted_instances assert labels[1].frame_idx == 2 assert labels[1].has_user_instances assert labels[1].has_predicted_instances
def test_instance_comparison(skeleton): node_names = ["left-wing", "head", "right-wing"] points = {"head": Point(1, 4), "left-wing": Point(2, 5), "right-wing": Point(3, 6)} instance1 = Instance(skeleton=skeleton, points=points) instance2 = copy.deepcopy(instance1) assert instance1.matches(instance1) assert instance1 != instance2 assert instance1.matches(instance2) instance2["head"].x = 42 assert not instance1.matches(instance2) instance2 = copy.deepcopy(instance1) instance2.skeleton.add_node("extra_node") assert not instance1.matches(instance2)
def test_labels_merge(): dummy_video = Video(backend=MediaVideo) dummy_skeleton = Skeleton() dummy_skeleton.add_node("node") labels = Labels() dummy_frames = [] # Add 10 instances with different points (so they aren't "redundant") for i in range(10): instance = Instance(skeleton=dummy_skeleton, points=dict(node=Point(i, i))) dummy_frame = LabeledFrame(dummy_video, frame_idx=0, instances=[instance]) dummy_frames.append(dummy_frame) labels.labeled_frames.extend(dummy_frames) assert len(labels) == 10 assert len(labels.labeled_frames[0].instances) == 1 labels.merge_matching_frames() assert len(labels) == 1 assert len(labels.labeled_frames[0].instances) == 10
def test_deserialize_suggestions(small_robot_mp4_vid, tmpdir): dummy_video = small_robot_mp4_vid dummy_skeleton = Skeleton() dummy_instance = Instance(dummy_skeleton) dummy_frame = LabeledFrame(dummy_video, frame_idx=0, instances=[dummy_instance]) labels = Labels() labels.append(dummy_frame) suggestions = VideoFrameSuggestions.suggest( labels=labels, params=dict(method="sample", per_video=13) ) labels.set_suggestions(suggestions) filename = os.path.join(tmpdir, "new_suggestions.h5") Labels.save_file(filename=filename, labels=labels) new_suggestion_labels = Labels.load_file(filename) assert len(suggestions) == len(new_suggestion_labels.suggestions) assert [frame.frame_idx for frame in suggestions] == [ frame.frame_idx for frame in new_suggestion_labels.suggestions ]