def test_id_and_url_lists(self):
metadata1 = MetaDataItem(title="title",
url="fake url 1",
download_src="youtube")
metadata2 = MetaDataItem(title="title",
url="fake url 2",
download_src="youtube")
try:
self.storage.publish_new_metadata(metadata1)
self.storage.publish_new_metadata(metadata2)
actual_id_list = self.storage.fetch_video_id_list()
self.assertTrue(
all(
map(lambda id: id in set(actual_id_list),
{metadata1.id, metadata2.id})))
actual_url_list = self.storage.fetch_video_url_list()
self.assertTrue(
all(
map(lambda id: id in set(actual_url_list),
{metadata1.url, metadata2.url})))
# Clean up after test
finally:
self.storage.delete_metadata(metadata1.id)
self.storage.delete_metadata(metadata2.id)
self.assertFalse(self.metadata_exists(metadata1))
self.assertFalse(self.metadata_exists(metadata2))
def test_unique_urls(self):
upd = DataUpdater()
mockdb = MockDataAccessor()
upd.set_database(mockdb)
upd.run(
MetaDataItem(title="unique",
url="fake url 1",
download_src="youtube"))
upd.run(
MetaDataItem(title="repeat1",
url="fake url 2",
download_src="youtube",
is_split_url=True))
try:
upd.run(
MetaDataItem(title="not allowed",
url="fake url 1",
download_src="youtube"))
self.assertTrue(False)
except StopSignal:
# expected path
pass
upd.run(
MetaDataItem(title="repeat2",
url="fake url 2",
download_src="youtube",
is_split_url=True))
def test_clone(self):
json = {
"id": "id",
"title": "blah",
"url": "www.ca",
"download_src": "www.website.ca",
"accident_locations": [2, 5, 7],
"bb_fields": {
"objects": [{
"id": 1,
"has_collision": True,
"class": "car",
"bboxes": [(0, 1, 2, 3, 4), (1, 5, 6, 7, 8)],
}, {
"id":
2,
"has_collision":
False,
"class":
"truck",
"bboxes": [(0, 11, 12, 13, 14), (1, 15, 16, 17, 18)],
}],
"collision_locations": [1, 2, 3],
"resolution": (360, 360)
},
}
mdi = MetaDataItem(**json)
clone = mdi.clone()
clone.bb_fields.objects[1].bboxes[1].x1 = 1
self.assertEqual(mdi.bb_fields.objects[1].bboxes[1].x1, 5)
self.assertEqual(clone.bb_fields.objects[1].bboxes[1].x1, 1)
mdi.bb_fields.objects[1].bboxes[1].x1 = 2
self.assertEqual(mdi.bb_fields.objects[1].bboxes[1].x1, 2)
self.assertEqual(clone.bb_fields.objects[1].bboxes[1].x1, 1)
def test_id_and_url_lists(self):
metadata1 = MetaDataItem(title="title",
url="fake url 1",
download_src="youtube")
metadata2 = MetaDataItem(title="title",
url="fake url 2",
download_src="youtube")
try:
self.storage.publish_new_metadata(metadata1)
self.storage.publish_new_metadata(metadata2)
actual_id_list = self.storage.fetch_video_id_list()
self.assertEqual(actual_id_list, [metadata1.id, metadata2.id])
actual_url_list = self.storage.fetch_video_url_list()
self.assertEqual(actual_url_list, [metadata1.url, metadata2.url])
# Clean up after test
finally:
self.storage.delete_metadata(metadata1.id)
self.storage.delete_metadata(metadata2.id)
self.assertFalse(
os.path.exists(
os.path.join(self.storage_loc, gen_filename(metadata1.id))))
self.assertFalse(
os.path.exists(
os.path.join(self.storage_loc, gen_filename(metadata2.id))))
def test_id_and_url_lists_generated_correctly_on_startup(self):
metadata1 = MetaDataItem(title="title",
url="fake url 1",
download_src="youtube")
metadata2 = MetaDataItem(title="title",
url="fake url 2",
download_src="youtube")
try:
self.storage.publish_new_metadata(metadata1)
self.storage.publish_new_metadata(metadata2)
# Create a second storage system to run the initialization code again with existing files
storage2 = LocalStorageAccessor(self.storage_loc)
actual_id_list = storage2.fetch_video_id_list()
self.assertEqual(set(actual_id_list), {metadata1.id, metadata2.id})
actual_url_list = storage2.fetch_video_url_list()
self.assertEqual(set(actual_url_list),
{metadata1.url, metadata2.url})
# Clean up after test
finally:
self.storage.delete_metadata(metadata1.id)
self.storage.delete_metadata(metadata2.id)
self.assertFalse(
os.path.exists(
os.path.join(self.storage_loc, gen_filename(metadata1.id))))
self.assertFalse(
os.path.exists(
os.path.join(self.storage_loc, gen_filename(metadata2.id))))
def test_metadata_update(self):
metadata = MetaDataItem(title="title",
url="fake url 1",
download_src="youtube")
try:
self.storage.publish_new_metadata(metadata)
self.assertTrue(
os.path.exists(
os.path.join(self.storage_loc, gen_filename(metadata.id))))
metadata.title = "new title"
self.storage.update_metadata(metadata)
# Ensure file is still there
self.assertTrue(
os.path.exists(
os.path.join(self.storage_loc, gen_filename(metadata.id))))
# Check that update was successful by fetching the file
metadata = self.storage.fetch_metadata(metadata.id)
self.assertEqual(metadata.title, "new title")
finally:
self.storage.delete_metadata(metadata.id)
self.assertFalse(
os.path.exists(
os.path.join(self.storage_loc, gen_filename(metadata.id))))
def test_fetch_metadata_with_tags(self):
metadata = MetaDataItem(title="title",
url="fake url 1",
download_src="youtube")
metadata.add_tag("hello", "world")
try:
self.storage.publish_new_metadata(metadata)
fetched_metadata = self.storage.fetch_metadata(metadata.id)
self.assertEqual(fetched_metadata.to_json(), metadata.to_json())
finally:
self.storage.delete_metadata(metadata.id)
self.assertFalse(self.metadata_exists(metadata))
def test_filter_condition_query(self):
metadata1 = MetaDataItem(title="title",
url="fake url 1",
download_src="youtube",
collision_type="car",
location="Canada")
metadata2 = MetaDataItem(title="title",
url="fake url 2",
download_src="youtube",
collision_type="human",
location="Canada")
metadata3 = MetaDataItem(title="title",
url="fake url 3",
download_src="youtube",
collision_type="human",
location="America")
try:
self.storage.publish_new_metadata(metadata1)
self.storage.publish_new_metadata(metadata2)
self.storage.publish_new_metadata(metadata3)
condition = FilterCondition(
"title == 'title' and location == 'Canada' and collision_type != 'car'"
)
print(
condition.tokenize(
"title == 'title' and location == 'Canada' and collision_type != 'car'"
))
metadata_list = self.storage.fetch_newest_videos(
filter_cond=condition)
self.assertEqual(
set(map(lambda metadata: metadata.url, metadata_list)),
{metadata2.url})
# Clean up after test
finally:
self.storage.delete_metadata(metadata1.id)
self.storage.delete_metadata(metadata2.id)
self.storage.delete_metadata(metadata3.id)
self.assertFalse(
os.path.exists(
os.path.join(self.storage_loc, gen_filename(metadata1.id))))
self.assertFalse(
os.path.exists(
os.path.join(self.storage_loc, gen_filename(metadata2.id))))
self.assertFalse(
os.path.exists(
os.path.join(self.storage_loc, gen_filename(metadata3.id))))
def test_from_json_accident_in_root(self):
json = {
"id": "id",
"title": "blah",
"url": "www.ca",
"download_src": "www.website.ca",
"accident_locations": [1, 2, 3]
}
mdi = MetaDataItem(**json)
self.assertListEqual(mdi.bb_fields.collision_locations, [1, 2, 3])
def test_stateful(self):
src = Source()
mockdb = MockDataAccessor()
mockdb.publish_new_metadata(
MetaDataItem(title="first",
url="fake url 1",
download_src="youtube"))
mockdb.publish_new_metadata(
MetaDataItem(title="second",
url="fake url 2",
download_src="youtube"))
src.set_database(mockdb)
item = src.run()
self.assertEqual("first", item.title)
item = src.run()
self.assertEqual("second", item.title)
def test_filtering(self):
mdi_list = [
MetaDataItem(title="Vid1", url="c.com", download_src="youtube", id=1, collision_type="bike", location="China"),
MetaDataItem(title="Vid2", url="c.com", download_src="youtube", id=2, collision_type="car", location="Canada"),
MetaDataItem(title="Vid3", url="c.com", download_src="youtube", id=3, collision_type="bike", location="Canada"),
MetaDataItem(title="Vid4", url="c.com", download_src="youtube", id=4, collision_type="car", location="China"),
MetaDataItem(title="Vid5", url="c.com", download_src="youtube", id=5, collision_type="walking", location="Canada")
]
fc1 = FilterCondition("location == 'Canada' and collision_type != 'car'")
lst1 = fc1.filter(mdi_list)
self.assertEqual(len(lst1), 2)
self.assertEqual(lst1[0].id, 3)
self.assertEqual(lst1[1].id, 5)
fc2 = FilterCondition("title == 'Vid1' or title == 'Vid5'")
lst2 = fc2.filter(mdi_list)
self.assertEqual(len(lst2), 2)
self.assertEqual(lst2[0].id, 1)
self.assertEqual(lst2[1].id, 5)
def test_filter_condition_query(self):
metadata1 = MetaDataItem(title="title",
url="fake url 1",
download_src="youtube",
collision_type="car",
location="Canada")
metadata2 = MetaDataItem(title="title",
url="fake url 2",
download_src="youtube",
collision_type="human",
location="Canada")
metadata3 = MetaDataItem(title="title",
url="fake url 3",
download_src="youtube",
collision_type="human",
location="America")
try:
self.storage.publish_new_metadata(metadata1)
self.storage.publish_new_metadata(metadata2)
self.storage.publish_new_metadata(metadata3)
self.assertTrue(self.metadata_exists(metadata1))
self.assertTrue(self.metadata_exists(metadata2))
self.assertTrue(self.metadata_exists(metadata3))
condition = FilterCondition(
"title == 'title' and location == 'Canada' and collision_type != 'car'"
)
metadata_list = self.storage.fetch_newest_videos(
filter_cond=condition)
self.assertEqual(
set(map(lambda metadata: metadata.url, metadata_list)),
{metadata2.url})
finally:
self.storage.delete_metadata(metadata1.id)
self.storage.delete_metadata(metadata2.id)
self.storage.delete_metadata(metadata3.id)
self.assertFalse(self.metadata_exists(metadata1))
self.assertFalse(self.metadata_exists(metadata2))
self.assertFalse(self.metadata_exists(metadata3))
def test_tag_overwrite(self):
metadata = MetaDataItem(title="title",
url="fake url 1",
download_src="youtube",
collision_type="car-v-car",
description="desc",
location="loc")
simple_tag = {"hello": "world", "number": 1}
metadata.add_tag("simple", simple_tag)
self.assertEqual(metadata.tags,
{"simple": {
"hello": "world",
"number": 1
}})
updated_value_tag = {"number": 2}
metadata.add_tag("simple", updated_value_tag)
self.assertEqual(metadata.tags,
{"simple": {
"hello": "world",
"number": 2
}})
def test_metadata_file_creation_and_deletion(self):
metadata = MetaDataItem(title="title",
url="fake url 1",
download_src="youtube")
try:
self.storage.publish_new_metadata(metadata)
self.assertTrue(
metadata.id is not ""
) # metadata id successfully updated by the publish call
self.assertTrue(self.metadata_exists(metadata))
finally:
self.storage.delete_metadata(metadata.id)
self.assertFalse(self.metadata_exists(metadata))
def test_fetch_newest_videos(self):
metadata1 = MetaDataItem(title="title",
url="fake url 1",
download_src="youtube")
metadata2 = MetaDataItem(title="title",
url="fake url 2",
download_src="youtube")
metadata3 = MetaDataItem(title="title",
url="fake url 3",
download_src="youtube")
try:
self.storage.publish_new_metadata(metadata1)
self.storage.publish_new_metadata(metadata2)
self.storage.publish_new_metadata(metadata3)
self.assertTrue(self.metadata_exists(metadata1))
self.assertTrue(self.metadata_exists(metadata2))
self.assertTrue(self.metadata_exists(metadata3))
metadata_list = self.storage.fetch_newest_videos(metadata1.id)
self.assertEqual(
set(map(lambda metadata: metadata.url, metadata_list)),
{metadata2.url, metadata3.url})
metadata_list = self.storage.fetch_newest_videos(metadata2.id)
self.assertEqual(
set(map(lambda metadata: metadata.url, metadata_list)),
{metadata3.url})
metadata_list = self.storage.fetch_newest_videos(metadata3.id)
self.assertEqual(metadata_list, [])
finally:
self.storage.delete_metadata(metadata1.id)
self.storage.delete_metadata(metadata2.id)
self.storage.delete_metadata(metadata3.id)
self.assertFalse(self.metadata_exists(metadata1))
self.assertFalse(self.metadata_exists(metadata2))
self.assertFalse(self.metadata_exists(metadata3))
def test_download_video_in_album(self):
test_url = "https://imgur.com/r/carcrash/EN6aTa6"
expected_title = hashlib.sha224(test_url.encode()).hexdigest() + ".mp4"
file_path = TestImgurDownloader.download_location
vid_str = VideoStorageService(file_path)
downloader = ImgurDownloader()
downloader.set_video_storage(vid_str)
video_item = downloader.run(
MetaDataItem(title="title", url=test_url, download_src="imgur"))
vid_filename = os.path.join(file_path, expected_title)
self.assertTrue(os.path.exists(vid_filename))
os.remove(vid_filename)
def test_metadata_file_fetching(self):
metadata = MetaDataItem(title="title",
url="fake url 1",
download_src="youtube")
try:
self.storage.publish_new_metadata(metadata)
self.assertTrue(self.metadata_exists(metadata))
metadata2 = self.storage.fetch_metadata(metadata.id)
self.assertEqual(str(metadata), str(metadata2))
finally:
self.storage.delete_metadata(metadata.id)
# Ensure deletion worked so that data is not polluted
self.assertFalse(self.metadata_exists(metadata))
def test_store_load_delete_video(self):
metaData = MetaDataItem(title="title",
url="url",
download_src="youtube",
collision_type="car-v-car",
description="desc",
location="loc")
vidItem = VideoItem(metadata=metaData)
self.storage.store_video(vidItem)
self.assertTrue(os.path.exists(self.storage.get_file(vidItem)))
vidItem2 = self.storage.load_video(metaData)
self.assertEqual(vidItem.encode(), vidItem2.encode())
self.storage.delete_video(vidItem)
self.assertFalse(os.path.exists(self.storage.get_file(vidItem)))
def test_update_state(self):
m = MetaDataItem(id="Hello",
title="hello",
url="world",
download_src="youtube")
v = VideoItem(m)
al_prog = AutoLabeler(val="in-progress")
v = al_prog.run(v)
self.assertEqual(v.metadata.get_tag('state'), "in-progress")
al_comp = AutoLabeler()
v = al_comp.run(v)
self.assertEqual(v.metadata.get_tag('state'), "processed")
def test_default_config(self):
f = Filterer()
m = MetaDataItem(id="Hello",
title="hello",
url="world",
download_src="youtube")
v = VideoItem(m)
res_v = f.run(v)
self.assertEqual(v, res_v)
al = AutoLabeler()
v = al.run(v)
ret = f.run(v)
self.assertIsNone(ret, msg="Filterer should return None if failed")
def test_list_and_delete_all(self):
videos = ["vid1", "vid2", "vid3"]
for vid in videos:
self.storage.store_video(
VideoItem(metadata=MetaDataItem(title="title",
url=vid,
download_src="youtube",
collision_type="comp",
description="desc",
location="loc"),
filepath=None))
for vid in self.storage.list_videos():
videos.remove(vid.metadata.url)
self.assertTrue(len(videos) == 0, "Not all videos listed")
self.storage.delete_all()
self.assertTrue(
len([x for x in self.storage.list_videos()]) == 0,
"Not all videos deleted")
def run(self, obj):
return [
MetaDataItem(title=None, url=None, download_src=None)
for _ in range(self.iters)
]
def run(self, obj):
print(f"run {self.event_num}")
events.put(self.event_num)
return MetaDataItem(title=None, url=None, download_src=None)
def test_old_to_new_metadata(self):
self.maxDiff = None
old_mdi = {
"accident_locations": [1, 10, 100],
"title": "title",
"url": "www.website.ca",
"download_src": "website",
"id": "5",
'collision_type': None,
'description': None,
'start_i': 0,
'end_i': 100,
'enum_tags': ["enum_tag1", "enum_tag2"],
'is_cancelled': False,
'is_split_url': False,
'location': None,
'tags': {
"tag1": "val1"
},
"bb_fields": {
"frames": ["0001", "0001"],
"ids": ["1", "2"],
"clss": ["car", "truck"],
"x1s": [1, 5],
"y1s": [2, 6],
"x2s": [3, 7],
"y2s": [4, 8],
"has_collision": [1, 0],
}
}
expected_new_mdi = {
"title": "title",
"url": "www.website.ca",
"download_src": "website",
"id": "5",
'collision_type': None,
'description': None,
'start_i': 0,
'end_i': 100,
'enum_tags': ["enum_tag1", "enum_tag2"],
'is_cancelled': False,
'is_split_url': False,
'location': None,
'tags': {
"tag1": "val1"
},
"bb_fields": {
"accident_locations": [1, 10, 100],
"objects": [{
"id": 1,
"has_collision": True,
"class": "car",
"bboxes": [(0, 1, 2, 3, 4)],
}, {
"id": 2,
"has_collision": False,
"class": "truck",
"bboxes": [(0, 5, 6, 7, 8)],
}]
}
}
mdi = MetaDataItem(**old_mdi)
self.assertEqual(expected_new_mdi, mdi.to_json())
def sample_mdi():
return MetaDataItem(**sample_mdi_dict())
def next_downloadable(self):
return MetaDataItem(title="title",
url="fake url 1",
download_src="youtube")
def publish_new_metadata(self, metadata: MetaDataItem) -> str:
id = str(len(self.metadata_list))
metadata.id = id
self.metadata_list.append(metadata)
return id
