def test_should_return_batches_equivalent_to_number_of_frames(self):
video_info = DataFrameMetadata("dataset_1", 'dummy.avi')
video_loader = VideoLoader(video_info)
batches = list(video_loader.load())
dummy_frames = list(self.create_dummy_frames())
self.assertEqual(len(batches), NUM_FRAMES)
self.assertEqual(dummy_frames, [batch.frames[0] for batch in batches])
def test_should_return_batches_equivalent_to_number_of_frames(self):
video_info = VideoMetaInfo('dummy.avi', 10, VideoFormat.MPEG)
video_loader = VideoLoader(video_info)
batches = list(video_loader.load())
dummy_frames = list(self.create_dummy_frames())
self.assertEqual(len(batches), NUM_FRAMES)
self.assertEqual(dummy_frames, [batch.frames[0] for batch in batches])
def test_should_return_batches_equivalent_to_number_of_frames(self):
video_info = DataFrameMetadata("dataset_1", 'dummy.avi')
video_loader = VideoLoader(video_info)
batches = list(video_loader.load())
dummy_frames = list(self.create_dummy_frames())
self.assertEqual(len(batches), NUM_FRAMES)
expected = [batch.frames.to_dict('records')[0] for batch in batches]
self.assertTrue(custom_list_of_dicts_equal(dummy_frames, expected))
def test_should_return_single_batch_if_batch_size_equal_to_no_of_frames(
self):
video_info = DataFrameMetadata("dataset_1", 'dummy.avi')
video_loader = VideoLoader(video_info, batch_size=NUM_FRAMES)
dummy_frames = list(
self.create_dummy_frames(filters=[i for i in range(NUM_FRAMES)]))
batches = list(video_loader.load())
self.assertEqual(1, len(batches))
self.assertEqual(dummy_frames, list(batches[0].frames))
def test_should_return_only_few_frames_when_limit_is_specified(self):
video_info = DataFrameMetadata("dataset_1", 'dummy.avi')
limit = 4
video_loader = VideoLoader(video_info, limit=limit)
dummy_frames = list(
self.create_dummy_frames(filters=[i for i in range(limit)]))
batches = list(video_loader.load())
self.assertEqual(limit, len(batches))
self.assertEqual(dummy_frames, [batch.frames[0] for batch in batches])
def test_should_skip_first_two_frames_with_offset_two(self):
video_info = DataFrameMetadata("dataset_1", 'dummy.avi')
video_loader = VideoLoader(video_info, offset=2)
dummy_frames = list(
self.create_dummy_frames(filters=[i
for i in range(2, NUM_FRAMES)]))
batches = list(video_loader.load())
self.assertEqual(NUM_FRAMES - 2, len(batches))
self.assertEqual(dummy_frames, [batch.frames[0] for batch in batches])
def test_should_return_half_then_number_of_batches_with_skip_of_two(self):
video_info = DataFrameMetadata("dataset_1", 'dummy.avi')
video_loader = VideoLoader(video_info, skip_frames=2)
batches = list(video_loader.load())
dummy_frames = list(
self.create_dummy_frames(
filters=[i * 2 for i in range(NUM_FRAMES // 2)]))
self.assertEqual(len(batches), NUM_FRAMES / 2)
self.assertEqual(dummy_frames, [batch.frames[0] for batch in batches])
def test_should_return_single_batch_if_batch_size_equal_to_no_of_frames(
self):
video_info = VideoMetaInfo('dummy.avi', 10, VideoFormat.MPEG)
video_loader = VideoLoader(video_info, batch_size=NUM_FRAMES)
dummy_frames = list(
self.create_dummy_frames(filters=[i for i in range(NUM_FRAMES)]))
batches = list(video_loader.load())
self.assertEqual(1, len(batches))
self.assertEqual(dummy_frames, list(batches[0].frames))
def test_should_return_only_few_frames_when_limit_is_specified(self):
video_info = DataFrameMetadata("dataset_1", 'dummy.avi')
limit = 4
video_loader = VideoLoader(video_info, limit=limit)
dummy_frames = list(
self.create_dummy_frames(filters=[i for i in range(limit)]))
batches = list(video_loader.load())
self.assertEqual(limit, len(batches))
expected = [batch.frames.to_dict('records')[0] for batch in batches]
self.assertTrue(custom_list_of_dicts_equal(dummy_frames, expected))
def test_should_skip_first_two_frames_with_offset_two(self):
video_info = DataFrameMetadata("dataset_1", 'dummy.avi')
video_loader = VideoLoader(video_info, offset=2)
dummy_frames = list(
self.create_dummy_frames(
filters=[i for i in range(2, NUM_FRAMES)]))
batches = list(video_loader.load())
self.assertEqual(NUM_FRAMES - 2, len(batches))
expected = [batch.frames.to_dict('records')[0] for batch in batches]
self.assertTrue(custom_list_of_dicts_equal(dummy_frames, expected))
def test_should_return_half_then_number_of_batches_with_skip_of_two(self):
video_info = DataFrameMetadata("dataset_1", 'dummy.avi')
video_loader = VideoLoader(video_info, skip_frames=2)
batches = list(video_loader.load())
dummy_frames = list(
self.create_dummy_frames(
filters=[i * 2 for i in range(NUM_FRAMES // 2)]))
self.assertEqual(len(batches), NUM_FRAMES / 2)
expected = [batch.frames.to_dict('records')[0] for batch in batches]
self.assertTrue(custom_list_of_dicts_equal(dummy_frames, expected))
class DiskStorageExecutor(AbstractStorageExecutor):
"""
This is a simple disk based executor. It assumes that frames are
directly being read from the disk(video file).
Note: For a full fledged deployment this might be replaced with a
Transaction manager which keeps track of the frames.
"""
def __init__(self, node: StoragePlan):
super().__init__(node)
self.storage = VideoLoader(node.video,
batch_size=node.batch_size,
skip_frames=node.skip_frames,
limit=node.limit,
offset=node.offset)
def validate(self):
pass
def next(self) -> Iterator[FrameBatch]:
for batch in self.storage.load():
yield batch