def test_round_trip(self):
# first, write some garbage in
adapter = RoundTripAdapter()
output_directory = os.path.join(self.temp_dir, "ANY_OUTPUT_DIR")
ingestor = GulpIngestor(adapter, output_directory, 2, 1)
ingestor()
# then, read it and make sure the garbage came back out
gulp_directory = GulpDirectory(output_directory)
gulp_chunk = next(gulp_directory.chunks())
expected_output_shapes = [[(4, 1, 3),
(3, 1, 3),
(2, 1, 3),
(1, 1, 3)]
]
expected_meta = [{'name': 'bunch of numpy arrays'}]
for i, (frames, meta) in enumerate(gulp_chunk):
self.assertEqual(expected_meta[i], meta)
self.assertEqual(expected_output_shapes[i],
[np.array(f).shape for f in frames])
# check that random_access works
expected_frames = [
np.ones((4, 1, 3), dtype='uint8'),
np.ones((3, 1, 3), dtype='uint8'),
np.ones((2, 1, 3), dtype='uint8'),
np.ones((1, 1, 3), dtype='uint8'),
]
received_frames, received_meta = gulp_directory[1]
for ef, rf in zip(expected_frames, received_frames):
npt.assert_array_equal(ef, np.array(rf))
self.assertEqual(expected_meta[0], received_meta)
# now append/extend the gulps
GulpIngestor(RoundTripAdapter(ids=[3, 4, 5]),
output_directory, 2, 1)()
# then, read it again, using __iter__
gulp_directory = GulpDirectory(output_directory)
gulp_chunk = next(iter(gulp_directory))
expected_output_shapes = [(4, 1, 3),
(3, 1, 3),
(2, 1, 3),
(1, 1, 3)]
expected_meta = {'name': 'bunch of numpy arrays'}
for frames, meta in gulp_chunk:
self.assertEqual(expected_meta, meta)
self.assertEqual(expected_output_shapes,
[np.array(f).shape for f in frames])
def test_init(self):
adapter = RoundTripAdapter()
output_directory = os.path.join(self.temp_dir, "ANY_OUTPUT_DIR")
ingestor = GulpIngestor(adapter, output_directory, 2, 1)
ingestor()
gulp_directory = GulpDirectory(output_directory)
self.assertEqual(gulp_directory.output_dir, output_directory)
expected_all_meta_dicts = [
OrderedDict([('1',
OrderedDict([('frame_info',
[[0, 1, 632],
[632, 1, 632],
[1264, 1, 632],
[1896, 1, 632]]),
('meta_data',
[OrderedDict(
[('name',
'bunch of numpy arrays')])])]))]),
OrderedDict([('2',
OrderedDict([('frame_info',
[[0, 1, 632],
[632, 1, 632]]),
('meta_data',
[OrderedDict(
[('name', 'shorter_video')])])]))])]
self.assertEqual(gulp_directory.all_meta_dicts,
expected_all_meta_dicts)
self.assertEqual(gulp_directory.chunk_lookup, {'1': 0, '2': 1})
expected_merged_meta_dict = {
'1': OrderedDict([('frame_info',
[[0, 1, 632],
[632, 1, 632],
[1264, 1, 632],
[1896, 1, 632]]),
('meta_data',
[OrderedDict(
[('name',
'bunch of numpy arrays')])])]),
'2': OrderedDict([('frame_info',
[[0, 1, 632],
[632, 1, 632]]),
('meta_data',
[OrderedDict([('name',
'shorter_video')])])])}
self.assertEqual(gulp_directory.merged_meta_dict,
expected_merged_meta_dict)
def remove_entries_with_duplicate_ids(output_directory, meta_dict):
meta_dict = _remove_duplicates_in_metadict(meta_dict)
gulp_directory = GulpDirectory(output_directory)
existing_ids = list(gulp_directory.merged_meta_dict.keys())
# this assumes no duplicates in existing_ids
new_meta = []
for meta_info in meta_dict:
if str(meta_info['id']) in existing_ids:
print('Id {} already in GulpDirectory, I skip it!'.format(
meta_info['id']))
else:
new_meta.append(meta_info)
if len(new_meta) == 0:
print("no items to add... Abort")
raise DuplicateIdException
return new_meta
def test_random_access(self):
# ingest dummy videos
adapter = DummyVideosAdapter(num_videos=25)
output_directory = os.path.join(self.temp_dir, "ANY_OUTPUT_DIR")
ingestor = GulpIngestor(adapter, output_directory, 2, 1)
ingestor()
# create gulp directory
gulp_directory = GulpDirectory(output_directory)
# check all videos can be accessed
for id_ in adapter.ids:
with self.subTest(id_=id_):
# check img id is in the lookup table
self.assertTrue(id_ in gulp_directory.chunk_lookup)
# check the img can be accessed
img, meta = gulp_directory[id_]
# check the meta id match
self.assertEqual(meta['id'], id_)