def test_validation(self):
FilterCondition("location == 'Canada' and collision_type != 'car'")
FilterCondition("title < 'a' or url > 'Z' or description == 'hello'")
FilterCondition("1 >= 0 and (2 <= 0 or 7 != 2)")
self.check_failed_syntax_validation("vacation == 'Canada'")
self.check_failed_syntax_validation("(location == 'Canada'")
self.check_failed_syntax_validation("location == 'Canada')")
self.check_failed_syntax_validation(")location = 'Canada'(")
self.check_failed_syntax_validation("location = 'Cana'da'")
self.check_failed_syntax_validation("location == 'Cana'da' !=")
self.check_failed_type_validation("location >= 4")
def test_filter_condition_query(self):
metadata1 = MetaDataItem(title="title",
url="fake url 1",
download_src="youtube",
collision_type="car",
location="Canada")
metadata2 = MetaDataItem(title="title",
url="fake url 2",
download_src="youtube",
collision_type="human",
location="Canada")
metadata3 = MetaDataItem(title="title",
url="fake url 3",
download_src="youtube",
collision_type="human",
location="America")
try:
self.storage.publish_new_metadata(metadata1)
self.storage.publish_new_metadata(metadata2)
self.storage.publish_new_metadata(metadata3)
condition = FilterCondition(
"title == 'title' and location == 'Canada' and collision_type != 'car'"
)
print(
condition.tokenize(
"title == 'title' and location == 'Canada' and collision_type != 'car'"
))
metadata_list = self.storage.fetch_newest_videos(
filter_cond=condition)
self.assertEqual(
set(map(lambda metadata: metadata.url, metadata_list)),
{metadata2.url})
# Clean up after test
finally:
self.storage.delete_metadata(metadata1.id)
self.storage.delete_metadata(metadata2.id)
self.storage.delete_metadata(metadata3.id)
self.assertFalse(
os.path.exists(
os.path.join(self.storage_loc, gen_filename(metadata1.id))))
self.assertFalse(
os.path.exists(
os.path.join(self.storage_loc, gen_filename(metadata2.id))))
self.assertFalse(
os.path.exists(
os.path.join(self.storage_loc, gen_filename(metadata3.id))))
def fetch_newest_videos(
self,
last_id: str = None,
filter_cond: FilterCondition = None) -> Iterator[MetaDataItem]:
if last_id is not None:
metadata_list = self.metadata_list[int(last_id):]
else:
metadata_list = self.metadata_list
metadata_list = list(filter(None, metadata_list))
if filter_cond is not None:
return iter(filter_cond.filter(metadata_list))
else:
return iter(metadata_list)
def test_filter_condition_query(self):
metadata1 = MetaDataItem(title="title",
url="fake url 1",
download_src="youtube",
collision_type="car",
location="Canada")
metadata2 = MetaDataItem(title="title",
url="fake url 2",
download_src="youtube",
collision_type="human",
location="Canada")
metadata3 = MetaDataItem(title="title",
url="fake url 3",
download_src="youtube",
collision_type="human",
location="America")
try:
self.storage.publish_new_metadata(metadata1)
self.storage.publish_new_metadata(metadata2)
self.storage.publish_new_metadata(metadata3)
self.assertTrue(self.metadata_exists(metadata1))
self.assertTrue(self.metadata_exists(metadata2))
self.assertTrue(self.metadata_exists(metadata3))
condition = FilterCondition(
"title == 'title' and location == 'Canada' and collision_type != 'car'"
)
metadata_list = self.storage.fetch_newest_videos(
filter_cond=condition)
self.assertEqual(
set(map(lambda metadata: metadata.url, metadata_list)),
{metadata2.url})
finally:
self.storage.delete_metadata(metadata1.id)
self.storage.delete_metadata(metadata2.id)
self.storage.delete_metadata(metadata3.id)
self.assertFalse(self.metadata_exists(metadata1))
self.assertFalse(self.metadata_exists(metadata2))
self.assertFalse(self.metadata_exists(metadata3))
def update_states_of_in_progress_metadata():
f = FirebaseAccessor()
fu = FirebaseUpdater()
fc = FilterCondition( "tags['state'] == 'in-progress'")
all_videos = f.fetch_newest_videos(filter_cond=fc)
print("Num videos to go: " + str(len(all_videos)))
for video in all_videos:
print(video)
inp = input()
if inp == "c":
# clear state
video.add_tag('state', '')
fu.run(video)
elif inp == "f":
# Finished
video.add_tag('state', 'processed')
fu.run(video)
def test_filtering(self):
mdi_list = [
MetaDataItem(title="Vid1", url="c.com", download_src="youtube", id=1, collision_type="bike", location="China"),
MetaDataItem(title="Vid2", url="c.com", download_src="youtube", id=2, collision_type="car", location="Canada"),
MetaDataItem(title="Vid3", url="c.com", download_src="youtube", id=3, collision_type="bike", location="Canada"),
MetaDataItem(title="Vid4", url="c.com", download_src="youtube", id=4, collision_type="car", location="China"),
MetaDataItem(title="Vid5", url="c.com", download_src="youtube", id=5, collision_type="walking", location="Canada")
]
fc1 = FilterCondition("location == 'Canada' and collision_type != 'car'")
lst1 = fc1.filter(mdi_list)
self.assertEqual(len(lst1), 2)
self.assertEqual(lst1[0].id, 3)
self.assertEqual(lst1[1].id, 5)
fc2 = FilterCondition("title == 'Vid1' or title == 'Vid5'")
lst2 = fc2.filter(mdi_list)
self.assertEqual(len(lst2), 2)
self.assertEqual(lst2[0].id, 1)
self.assertEqual(lst2[1].id, 5)
def get_metadata_with_url(url):
f = FirebaseAccessor()
fc = FilterCondition("url == '{}'".format(url))
data = f.fetch_newest_videos(filter_cond=fc)
return data
def get_metadata_with_url(url):
l = LocalStorageAccessor()
fc = FilterCondition("url == '{}'".format(url))
data = l.fetch_newest_videos(filter_cond=fc)
return data
from src.database.LocalStorageAccessor import LocalStorageAccessor
from src.data.FilterCondition import FilterCondition
def get_num_metadata_item():
l = LocalStorageAccessor()
ids = l.fetch_video_id_list()
return len(ids)
def get_metadata_with_id(id):
l = LocalStorageAccessor()
fc = FilterCondition("id == '{}'".format(id))
data = l.fetch_newest_videos(filter_cond=fc)
return data
def get_metadata_with_fc(fc):
l = LocalStorageAccessor()
data = l.fetch_newest_videos(filter_cond=fc)
return data
def get_metadata_with_url(url):
l = LocalStorageAccessor()
fc = FilterCondition("url == '{}'".format(url))
data = l.fetch_newest_videos(filter_cond=fc)
return data
fc = FilterCondition("url == '{}' and start_i == {}".format('https://www.youtube.com/watch?v=vhACO_m5pH0', 71))
print(get_metadata_with_fc(fc))
def check_failed_type_validation(self, filter_string):
try:
FilterCondition(filter_string)
self.assertFalse(True)
except TypeError:
pass
def check_failed_syntax_validation(self, filter_string):
try:
FilterCondition(filter_string)
self.assertFalse(True)
except SyntaxError:
pass