def test_id_and_url_lists(self): metadata1 = MetaDataItem(title="title", url="fake url 1", download_src="youtube") metadata2 = MetaDataItem(title="title", url="fake url 2", download_src="youtube") try: self.storage.publish_new_metadata(metadata1) self.storage.publish_new_metadata(metadata2) actual_id_list = self.storage.fetch_video_id_list() self.assertTrue( all( map(lambda id: id in set(actual_id_list), {metadata1.id, metadata2.id}))) actual_url_list = self.storage.fetch_video_url_list() self.assertTrue( all( map(lambda id: id in set(actual_url_list), {metadata1.url, metadata2.url}))) # Clean up after test finally: self.storage.delete_metadata(metadata1.id) self.storage.delete_metadata(metadata2.id) self.assertFalse(self.metadata_exists(metadata1)) self.assertFalse(self.metadata_exists(metadata2))
def test_unique_urls(self): upd = DataUpdater() mockdb = MockDataAccessor() upd.set_database(mockdb) upd.run( MetaDataItem(title="unique", url="fake url 1", download_src="youtube")) upd.run( MetaDataItem(title="repeat1", url="fake url 2", download_src="youtube", is_split_url=True)) try: upd.run( MetaDataItem(title="not allowed", url="fake url 1", download_src="youtube")) self.assertTrue(False) except StopSignal: # expected path pass upd.run( MetaDataItem(title="repeat2", url="fake url 2", download_src="youtube", is_split_url=True))
def test_clone(self): json = { "id": "id", "title": "blah", "url": "www.ca", "download_src": "www.website.ca", "accident_locations": [2, 5, 7], "bb_fields": { "objects": [{ "id": 1, "has_collision": True, "class": "car", "bboxes": [(0, 1, 2, 3, 4), (1, 5, 6, 7, 8)], }, { "id": 2, "has_collision": False, "class": "truck", "bboxes": [(0, 11, 12, 13, 14), (1, 15, 16, 17, 18)], }], "collision_locations": [1, 2, 3], "resolution": (360, 360) }, } mdi = MetaDataItem(**json) clone = mdi.clone() clone.bb_fields.objects[1].bboxes[1].x1 = 1 self.assertEqual(mdi.bb_fields.objects[1].bboxes[1].x1, 5) self.assertEqual(clone.bb_fields.objects[1].bboxes[1].x1, 1) mdi.bb_fields.objects[1].bboxes[1].x1 = 2 self.assertEqual(mdi.bb_fields.objects[1].bboxes[1].x1, 2) self.assertEqual(clone.bb_fields.objects[1].bboxes[1].x1, 1)
def test_id_and_url_lists(self): metadata1 = MetaDataItem(title="title", url="fake url 1", download_src="youtube") metadata2 = MetaDataItem(title="title", url="fake url 2", download_src="youtube") try: self.storage.publish_new_metadata(metadata1) self.storage.publish_new_metadata(metadata2) actual_id_list = self.storage.fetch_video_id_list() self.assertEqual(actual_id_list, [metadata1.id, metadata2.id]) actual_url_list = self.storage.fetch_video_url_list() self.assertEqual(actual_url_list, [metadata1.url, metadata2.url]) # Clean up after test finally: self.storage.delete_metadata(metadata1.id) self.storage.delete_metadata(metadata2.id) self.assertFalse( os.path.exists( os.path.join(self.storage_loc, gen_filename(metadata1.id)))) self.assertFalse( os.path.exists( os.path.join(self.storage_loc, gen_filename(metadata2.id))))
def test_id_and_url_lists_generated_correctly_on_startup(self): metadata1 = MetaDataItem(title="title", url="fake url 1", download_src="youtube") metadata2 = MetaDataItem(title="title", url="fake url 2", download_src="youtube") try: self.storage.publish_new_metadata(metadata1) self.storage.publish_new_metadata(metadata2) # Create a second storage system to run the initialization code again with existing files storage2 = LocalStorageAccessor(self.storage_loc) actual_id_list = storage2.fetch_video_id_list() self.assertEqual(set(actual_id_list), {metadata1.id, metadata2.id}) actual_url_list = storage2.fetch_video_url_list() self.assertEqual(set(actual_url_list), {metadata1.url, metadata2.url}) # Clean up after test finally: self.storage.delete_metadata(metadata1.id) self.storage.delete_metadata(metadata2.id) self.assertFalse( os.path.exists( os.path.join(self.storage_loc, gen_filename(metadata1.id)))) self.assertFalse( os.path.exists( os.path.join(self.storage_loc, gen_filename(metadata2.id))))
def test_metadata_update(self): metadata = MetaDataItem(title="title", url="fake url 1", download_src="youtube") try: self.storage.publish_new_metadata(metadata) self.assertTrue( os.path.exists( os.path.join(self.storage_loc, gen_filename(metadata.id)))) metadata.title = "new title" self.storage.update_metadata(metadata) # Ensure file is still there self.assertTrue( os.path.exists( os.path.join(self.storage_loc, gen_filename(metadata.id)))) # Check that update was successful by fetching the file metadata = self.storage.fetch_metadata(metadata.id) self.assertEqual(metadata.title, "new title") finally: self.storage.delete_metadata(metadata.id) self.assertFalse( os.path.exists( os.path.join(self.storage_loc, gen_filename(metadata.id))))
def test_fetch_metadata_with_tags(self): metadata = MetaDataItem(title="title", url="fake url 1", download_src="youtube") metadata.add_tag("hello", "world") try: self.storage.publish_new_metadata(metadata) fetched_metadata = self.storage.fetch_metadata(metadata.id) self.assertEqual(fetched_metadata.to_json(), metadata.to_json()) finally: self.storage.delete_metadata(metadata.id) self.assertFalse(self.metadata_exists(metadata))
def test_filter_condition_query(self): metadata1 = MetaDataItem(title="title", url="fake url 1", download_src="youtube", collision_type="car", location="Canada") metadata2 = MetaDataItem(title="title", url="fake url 2", download_src="youtube", collision_type="human", location="Canada") metadata3 = MetaDataItem(title="title", url="fake url 3", download_src="youtube", collision_type="human", location="America") try: self.storage.publish_new_metadata(metadata1) self.storage.publish_new_metadata(metadata2) self.storage.publish_new_metadata(metadata3) condition = FilterCondition( "title == 'title' and location == 'Canada' and collision_type != 'car'" ) print( condition.tokenize( "title == 'title' and location == 'Canada' and collision_type != 'car'" )) metadata_list = self.storage.fetch_newest_videos( filter_cond=condition) self.assertEqual( set(map(lambda metadata: metadata.url, metadata_list)), {metadata2.url}) # Clean up after test finally: self.storage.delete_metadata(metadata1.id) self.storage.delete_metadata(metadata2.id) self.storage.delete_metadata(metadata3.id) self.assertFalse( os.path.exists( os.path.join(self.storage_loc, gen_filename(metadata1.id)))) self.assertFalse( os.path.exists( os.path.join(self.storage_loc, gen_filename(metadata2.id)))) self.assertFalse( os.path.exists( os.path.join(self.storage_loc, gen_filename(metadata3.id))))
def test_from_json_accident_in_root(self): json = { "id": "id", "title": "blah", "url": "www.ca", "download_src": "www.website.ca", "accident_locations": [1, 2, 3] } mdi = MetaDataItem(**json) self.assertListEqual(mdi.bb_fields.collision_locations, [1, 2, 3])
def test_stateful(self): src = Source() mockdb = MockDataAccessor() mockdb.publish_new_metadata( MetaDataItem(title="first", url="fake url 1", download_src="youtube")) mockdb.publish_new_metadata( MetaDataItem(title="second", url="fake url 2", download_src="youtube")) src.set_database(mockdb) item = src.run() self.assertEqual("first", item.title) item = src.run() self.assertEqual("second", item.title)
def test_filtering(self): mdi_list = [ MetaDataItem(title="Vid1", url="c.com", download_src="youtube", id=1, collision_type="bike", location="China"), MetaDataItem(title="Vid2", url="c.com", download_src="youtube", id=2, collision_type="car", location="Canada"), MetaDataItem(title="Vid3", url="c.com", download_src="youtube", id=3, collision_type="bike", location="Canada"), MetaDataItem(title="Vid4", url="c.com", download_src="youtube", id=4, collision_type="car", location="China"), MetaDataItem(title="Vid5", url="c.com", download_src="youtube", id=5, collision_type="walking", location="Canada") ] fc1 = FilterCondition("location == 'Canada' and collision_type != 'car'") lst1 = fc1.filter(mdi_list) self.assertEqual(len(lst1), 2) self.assertEqual(lst1[0].id, 3) self.assertEqual(lst1[1].id, 5) fc2 = FilterCondition("title == 'Vid1' or title == 'Vid5'") lst2 = fc2.filter(mdi_list) self.assertEqual(len(lst2), 2) self.assertEqual(lst2[0].id, 1) self.assertEqual(lst2[1].id, 5)
def test_filter_condition_query(self): metadata1 = MetaDataItem(title="title", url="fake url 1", download_src="youtube", collision_type="car", location="Canada") metadata2 = MetaDataItem(title="title", url="fake url 2", download_src="youtube", collision_type="human", location="Canada") metadata3 = MetaDataItem(title="title", url="fake url 3", download_src="youtube", collision_type="human", location="America") try: self.storage.publish_new_metadata(metadata1) self.storage.publish_new_metadata(metadata2) self.storage.publish_new_metadata(metadata3) self.assertTrue(self.metadata_exists(metadata1)) self.assertTrue(self.metadata_exists(metadata2)) self.assertTrue(self.metadata_exists(metadata3)) condition = FilterCondition( "title == 'title' and location == 'Canada' and collision_type != 'car'" ) metadata_list = self.storage.fetch_newest_videos( filter_cond=condition) self.assertEqual( set(map(lambda metadata: metadata.url, metadata_list)), {metadata2.url}) finally: self.storage.delete_metadata(metadata1.id) self.storage.delete_metadata(metadata2.id) self.storage.delete_metadata(metadata3.id) self.assertFalse(self.metadata_exists(metadata1)) self.assertFalse(self.metadata_exists(metadata2)) self.assertFalse(self.metadata_exists(metadata3))
def test_tag_overwrite(self): metadata = MetaDataItem(title="title", url="fake url 1", download_src="youtube", collision_type="car-v-car", description="desc", location="loc") simple_tag = {"hello": "world", "number": 1} metadata.add_tag("simple", simple_tag) self.assertEqual(metadata.tags, {"simple": { "hello": "world", "number": 1 }}) updated_value_tag = {"number": 2} metadata.add_tag("simple", updated_value_tag) self.assertEqual(metadata.tags, {"simple": { "hello": "world", "number": 2 }})
def test_metadata_file_creation_and_deletion(self): metadata = MetaDataItem(title="title", url="fake url 1", download_src="youtube") try: self.storage.publish_new_metadata(metadata) self.assertTrue( metadata.id is not "" ) # metadata id successfully updated by the publish call self.assertTrue(self.metadata_exists(metadata)) finally: self.storage.delete_metadata(metadata.id) self.assertFalse(self.metadata_exists(metadata))
def test_fetch_newest_videos(self): metadata1 = MetaDataItem(title="title", url="fake url 1", download_src="youtube") metadata2 = MetaDataItem(title="title", url="fake url 2", download_src="youtube") metadata3 = MetaDataItem(title="title", url="fake url 3", download_src="youtube") try: self.storage.publish_new_metadata(metadata1) self.storage.publish_new_metadata(metadata2) self.storage.publish_new_metadata(metadata3) self.assertTrue(self.metadata_exists(metadata1)) self.assertTrue(self.metadata_exists(metadata2)) self.assertTrue(self.metadata_exists(metadata3)) metadata_list = self.storage.fetch_newest_videos(metadata1.id) self.assertEqual( set(map(lambda metadata: metadata.url, metadata_list)), {metadata2.url, metadata3.url}) metadata_list = self.storage.fetch_newest_videos(metadata2.id) self.assertEqual( set(map(lambda metadata: metadata.url, metadata_list)), {metadata3.url}) metadata_list = self.storage.fetch_newest_videos(metadata3.id) self.assertEqual(metadata_list, []) finally: self.storage.delete_metadata(metadata1.id) self.storage.delete_metadata(metadata2.id) self.storage.delete_metadata(metadata3.id) self.assertFalse(self.metadata_exists(metadata1)) self.assertFalse(self.metadata_exists(metadata2)) self.assertFalse(self.metadata_exists(metadata3))
def test_download_video_in_album(self): test_url = "https://imgur.com/r/carcrash/EN6aTa6" expected_title = hashlib.sha224(test_url.encode()).hexdigest() + ".mp4" file_path = TestImgurDownloader.download_location vid_str = VideoStorageService(file_path) downloader = ImgurDownloader() downloader.set_video_storage(vid_str) video_item = downloader.run( MetaDataItem(title="title", url=test_url, download_src="imgur")) vid_filename = os.path.join(file_path, expected_title) self.assertTrue(os.path.exists(vid_filename)) os.remove(vid_filename)
def test_metadata_file_fetching(self): metadata = MetaDataItem(title="title", url="fake url 1", download_src="youtube") try: self.storage.publish_new_metadata(metadata) self.assertTrue(self.metadata_exists(metadata)) metadata2 = self.storage.fetch_metadata(metadata.id) self.assertEqual(str(metadata), str(metadata2)) finally: self.storage.delete_metadata(metadata.id) # Ensure deletion worked so that data is not polluted self.assertFalse(self.metadata_exists(metadata))
def test_store_load_delete_video(self): metaData = MetaDataItem(title="title", url="url", download_src="youtube", collision_type="car-v-car", description="desc", location="loc") vidItem = VideoItem(metadata=metaData) self.storage.store_video(vidItem) self.assertTrue(os.path.exists(self.storage.get_file(vidItem))) vidItem2 = self.storage.load_video(metaData) self.assertEqual(vidItem.encode(), vidItem2.encode()) self.storage.delete_video(vidItem) self.assertFalse(os.path.exists(self.storage.get_file(vidItem)))
def test_update_state(self): m = MetaDataItem(id="Hello", title="hello", url="world", download_src="youtube") v = VideoItem(m) al_prog = AutoLabeler(val="in-progress") v = al_prog.run(v) self.assertEqual(v.metadata.get_tag('state'), "in-progress") al_comp = AutoLabeler() v = al_comp.run(v) self.assertEqual(v.metadata.get_tag('state'), "processed")
def test_default_config(self): f = Filterer() m = MetaDataItem(id="Hello", title="hello", url="world", download_src="youtube") v = VideoItem(m) res_v = f.run(v) self.assertEqual(v, res_v) al = AutoLabeler() v = al.run(v) ret = f.run(v) self.assertIsNone(ret, msg="Filterer should return None if failed")
def test_list_and_delete_all(self): videos = ["vid1", "vid2", "vid3"] for vid in videos: self.storage.store_video( VideoItem(metadata=MetaDataItem(title="title", url=vid, download_src="youtube", collision_type="comp", description="desc", location="loc"), filepath=None)) for vid in self.storage.list_videos(): videos.remove(vid.metadata.url) self.assertTrue(len(videos) == 0, "Not all videos listed") self.storage.delete_all() self.assertTrue( len([x for x in self.storage.list_videos()]) == 0, "Not all videos deleted")
def run(self, obj): return [ MetaDataItem(title=None, url=None, download_src=None) for _ in range(self.iters) ]
def run(self, obj): print(f"run {self.event_num}") events.put(self.event_num) return MetaDataItem(title=None, url=None, download_src=None)
def test_old_to_new_metadata(self): self.maxDiff = None old_mdi = { "accident_locations": [1, 10, 100], "title": "title", "url": "www.website.ca", "download_src": "website", "id": "5", 'collision_type': None, 'description': None, 'start_i': 0, 'end_i': 100, 'enum_tags': ["enum_tag1", "enum_tag2"], 'is_cancelled': False, 'is_split_url': False, 'location': None, 'tags': { "tag1": "val1" }, "bb_fields": { "frames": ["0001", "0001"], "ids": ["1", "2"], "clss": ["car", "truck"], "x1s": [1, 5], "y1s": [2, 6], "x2s": [3, 7], "y2s": [4, 8], "has_collision": [1, 0], } } expected_new_mdi = { "title": "title", "url": "www.website.ca", "download_src": "website", "id": "5", 'collision_type': None, 'description': None, 'start_i': 0, 'end_i': 100, 'enum_tags': ["enum_tag1", "enum_tag2"], 'is_cancelled': False, 'is_split_url': False, 'location': None, 'tags': { "tag1": "val1" }, "bb_fields": { "accident_locations": [1, 10, 100], "objects": [{ "id": 1, "has_collision": True, "class": "car", "bboxes": [(0, 1, 2, 3, 4)], }, { "id": 2, "has_collision": False, "class": "truck", "bboxes": [(0, 5, 6, 7, 8)], }] } } mdi = MetaDataItem(**old_mdi) self.assertEqual(expected_new_mdi, mdi.to_json())
def sample_mdi(): return MetaDataItem(**sample_mdi_dict())
def next_downloadable(self): return MetaDataItem(title="title", url="fake url 1", download_src="youtube")
def publish_new_metadata(self, metadata: MetaDataItem) -> str: id = str(len(self.metadata_list)) metadata.id = id self.metadata_list.append(metadata) return id