def check_privacy_status(video_id1,
is_private,
check_db=True,
check_api=False):
with django_db_blocker.unblock():
Video.recompute_computed_properties(only_pending=True)
priv_public = 'private' if is_private else 'public'
WebDriverWait(driver, TIME_WAIT).until(
EC.presence_of_element_located(
(By.ID, f'id_video_{video_id1}_{priv_public}')))
if check_db:
with django_db_blocker.unblock():
get_object_with_timeout(VideoRatingPrivacy,
video__video_id=video_id1,
user=up,
is_public=not is_private)
if check_api:
pe = do_api_call_v2(driver, '/videos/?video_id=' +
video_id1)['results'][0]['public_experts']
if is_private:
assert len(pe) == 0
else:
assert len(pe) == 1, pe
assert pe[0]['username'] == test_username
def handle(self, **options):
if options['set_all_pending']:
Video.objects.all().update(is_update_pending=True)
elif options['cron']:
# only update videos
Video.recompute_computed_properties(only_pending=True)
else:
print("Non-null ratings...")
nonnull_rating()
print("Email domains...")
fill_email_domains()
print("Creating emails...")
create_emails()
print("Email verif..." "")
recompute_property_verif_email()
print("Ratings...")
recompute_property_expertrating()
print("Avatar hashes...")
recompute_property_avatar_hash()
print("Demo accounts...")
demo_account()
def test_first_rate_later(self):
"""Testing with I RATE LATER (sampling only from rate_later)."""
u = self.create_user()
# creating videos
n_videos = 50
video_ids = [str(uuid1()) for _ in range(n_videos)]
videos_lst = Video.objects.bulk_create(
[Video(video_id=vid) for vid in video_ids])
videos_qs = Video.objects.filter(video_id__in=video_ids)
assert len(videos_qs) == len(videos_lst)
# randomly adding some to RATE LATER
vrl = [
VideoRateLater(video=v, user=u) for v in videos_qs
if np.random.rand() > 0.5
]
vrl_ids = [x.video.video_id for x in vrl]
VideoRateLater.objects.bulk_create(vrl)
for _ in range(self.n_tests_probabilistic):
assert sample_first_video_helper(
u, do_log=False, base_qs=videos_qs).video_id in vrl_ids
def add_inconsistency(user, feature):
"""Create an inconsistency of 3 videos were v0 better v1 better v2 better v0."""
video_ids = [
"%09d-%s-inc" % (i, random_alphanumeric()) for i in range(3)
]
objs = [Video(video_id=vid) for vid in video_ids]
_ = Video.objects.bulk_create(objs)
videos = [Video.objects.get(video_id=vid) for vid in video_ids]
# create rating where 1 is better than 2
# reverse -- create in other order (but same meaning!)
def create_rating(idx1, idx2, reverse=False):
if reverse:
ExpertRating.objects.create(user=user,
**{feature: MAX_VALUE},
video_1=videos[idx2],
video_2=videos[idx1])
else:
ExpertRating.objects.create(user=user,
**{feature: 0},
video_1=videos[idx1],
video_2=videos[idx2])
create_rating(0, 1, reverse=np.random.rand() < 0.5)
create_rating(1, 2, reverse=np.random.rand() < 0.5)
create_rating(2, 0, reverse=np.random.rand() < 0.5)
return tuple([x.id for x in videos])
def only_download_to_db(self):
"""Create videos from the only_download list."""
if not self.only_download:
return None
return Video.objects.bulk_create(
[Video(video_id=vid) for vid in self.only_download],
ignore_conflicts=True)
def video_get_or_create_verify_raise(field=None, **kwargs):
"""Either get a video, or create one. On errors, raise serializers.ValidationError."""
try:
v = Video.get_or_create_with_validation(**kwargs)
return v
except ModelValidationError as e:
if field is None:
raise serializers.ValidationError(e.message_dict)
else:
raise serializers.ValidationError({field: e.message_dict})
def loadYTVideoToDB(metadata, creator, series):
"""
Creates a video and saves it's tags, unless it's a duplicate, in which case None is returned
"""
# TODO: see if YouTube video blocks embedding
videos = Video.objects.filter(source='youtube', vid_id=metadata['id'])
# if this yt video is alread in the series, then dont add it again
if SeriesVideo.objects.filter(video__in=videos, series=series).count():
return None
video = Video(source='youtube',
vid_id=metadata['id'],
duration=metadata['duration'],
name=metadata['name'],
description=metadata['description'],
thumbnail=metadata['thumbnail'],
creator=creator)
video.save()
tags = metadata['tags']
tag_objs = []
# find or create a tag so we can link it to this video
for tag_name in tags:
tag, _ = Tag.objects.get_or_create(name=tag_name)
tag_objs.append(tag)
video.save()
# store or update tags, linking to this video
for tag_obj in tag_objs:
video_tag = VideoTag(video=video, tag=tag_obj)
video_tag.save()
return video
def test_set_privacy_settings(driver, django_db_blocker):
cleanup_rate_later(django_db_blocker)
with django_db_blocker.unblock():
video_id1 = create_test_video()
video_id2 = create_test_video()
u = DjangoUser.objects.get(username=test_username)
ui, _ = UserInformation.objects.get_or_create(user=u)
ui.show_my_profile = True
ui.save()
up, _ = UserPreferences.objects.get_or_create(user=u)
EmailDomain.objects.create(domain="@tournesol.app",
status=EmailDomain.STATUS_ACCEPTED)
VerifiableEmail.objects.create(user=ui,
email="*****@*****.**",
is_verified=True)
ExpertRating.objects.create(
user=up,
video_1=Video.objects.get(video_id=video_id1),
video_2=Video.objects.get(video_id=video_id2))
Video.recompute_computed_properties(only_pending=True)
login(driver)
fill_rate_later(django_db_blocker)
set_all_features_enabled(django_db_blocker)
driver.get(web_url + '/rate/' + video_id1 + '/' + video_id2)
WebDriverWait(driver, TIME_WAIT).until(
EC.presence_of_element_located((By.ID, 'id_expert_rating_page')))
def check_privacy_status(video_id1,
is_private,
check_db=True,
check_api=False):
with django_db_blocker.unblock():
Video.recompute_computed_properties(only_pending=True)
priv_public = 'private' if is_private else 'public'
WebDriverWait(driver, TIME_WAIT).until(
EC.presence_of_element_located(
(By.ID, f'id_video_{video_id1}_{priv_public}')))
if check_db:
with django_db_blocker.unblock():
get_object_with_timeout(VideoRatingPrivacy,
video__video_id=video_id1,
user=up,
is_public=not is_private)
if check_api:
pe = do_api_call_v2(driver, '/videos/?video_id=' +
video_id1)['results'][0]['public_experts']
if is_private:
assert len(pe) == 0
else:
assert len(pe) == 1, pe
assert pe[0]['username'] == test_username
def set_privacy(video_id1, is_private):
# opening the privacy menu...
WebDriverWait(driver, TIME_WAIT).until(
EC.presence_of_element_located(
(By.ID, f'id_open_privacy_menu_{video_id1}')))
driver.find_element_by_id(f'id_open_privacy_menu_{video_id1}').click()
priv_true_false = 'true' if is_private else 'false'
WebDriverWait(driver, TIME_WAIT).until(
EC.presence_of_element_located(
(By.ID, f'menu_set_private_{priv_true_false}_{video_id1}')))
driver.find_element_by_id(
f'menu_set_private_{priv_true_false}_{video_id1}').click()
WebDriverWait(driver, TIME_WAIT).until(
EC.element_to_be_clickable(
(By.ID, f'id_open_privacy_menu_{video_id1}')))
original_is_private = not VideoRatingPrivacy.DEFAULT_VALUE_IS_PUBLIC
check_privacy_status(video_id1, original_is_private, check_db=False)
set_privacy(video_id1, not original_is_private)
check_privacy_status(video_id1, not original_is_private, check_api=True)
set_privacy(video_id1, original_is_private)
check_privacy_status(video_id1, original_is_private, check_api=True)
logout(driver)
cleanup_rate_later(django_db_blocker)
def add_videos_in_folder_to_db(self):
"""Take videos from folder and load meta-data to db."""
with Path(self.video_dir):
# list of video IDs
suffix = '.description'
self.ids_in_folder = [x[:-len(suffix)]
for x in os.listdir() if x.endswith(suffix)]
ids_in_folder_set = set(self.ids_in_folder)
# list of .dump files for a particular video ID
self.dump_files = {
id_in_folder: [x for x in os.listdir() if x.startswith(id_in_folder)
and x.endswith('.dump')]
for id_in_folder in self.ids_in_folder
}
# is video unlisted? might break if youtube changes
UNLISTED_STR_HTML = '<meta itemprop="unlisted" content="True">'
is_unlisted = {
video_id: any([UNLISTED_STR_HTML in open(dump, 'r').read()
for dump in dumps])
for video_id, dumps in self.dump_files.items()
}
# adding video IDs in case if they don't exist
Video.objects.bulk_create([
Video(video_id=vid)
for vid in self.ids_in_folder
], ignore_conflicts=True)
for video_ in self.get_queryset():
wrong_url = 'is not a valid URL' in self.last_fill_info[video_.video_id]
with transaction.atomic():
video = Video.objects.get(pk=video_.pk)
video.wrong_url = wrong_url
video.download_failed = video.video_id not in ids_in_folder_set
video.is_unlisted = is_unlisted.get(video.video_id, False)
video.last_download_time = make_aware(datetime.datetime.now())
video.download_attempts += 1
video.save()
if video.wrong_url:
logging.warning(f"Video {video.video_id} has a wrong url")
if video.download_failed:
logging.warning(f"Metadata download for {video.video_id} failed")
if video.is_unlisted:
logging.warning(f"Video {video.video_id} is unlisted")
# saving descriptions and info
for v in tqdm(self.get_queryset()):
vid = v.video_id
if vid not in self.ids_in_folder:
continue
info = json.load(open('%s.info.json' % vid, 'r'))
description = open('%s.description' % vid, 'r').read()
v.description = description
v.name = info['title']
v.info = info
v.save()
try:
video_fill_info(v)
except Exception:
print("Info fill failure for %s" % vid)
def test_rate_later_top_k(driver, django_db_blocker):
# creating popular videos...
with django_db_blocker.unblock():
# determining current max views...
max_views = 0
if Video.objects.all().count() > 0:
max_views = Video.objects.all().aggregate(Max(F('views')))['views__max']
video_ids = [random_alphanumeric() for _ in range(n_top_popular)]
Video.objects.bulk_create([Video(video_id=vid, views=max_views + 1)
for vid in video_ids])
VideoRateLater.objects.filter(user__user__username=test_username).delete()
video_ids_set = set(video_ids)
login(driver)
WebDriverWait(driver, TIME_WAIT).until(
EC.presence_of_element_located((By.ID, 'rate_later_menu')))
print("Going to the rate later page")
expert_interface_btn = driver.find_element_by_id('rate_later_menu')
expert_interface_btn.click()
assert not element_present_now(driver, 'id_controls_prev_next')
assert not element_present_now(driver, 'id_big_expert_interface')
def add_from_topk():
# requesting new video
WebDriverWait(driver, TIME_WAIT).until(
EC.presence_of_element_located((By.CLASS_NAME, 'new_video_button')))
elems = driver.find_elements_by_class_name('new_video_button')
assert len(elems) == 1
elems[0].click()
WebDriverWait(driver, TIME_WAIT).until(
EC.presence_of_element_located((By.ID, 'id_top_video_videocard')))
WebDriverWait(driver, TIME_WAIT).until(
EC.presence_of_element_located((By.ID, 'id_big_add_rate_later')))
driver.find_element_by_id('id_big_add_rate_later').click()
WebDriverWait(driver, TIME_WAIT).until(
EC.presence_of_element_located((By.ID, 'id_rate_later_submit_ok')))
with django_db_blocker.unblock():
ids_vrl = list(VideoRateLater.objects.filter(
user__user__username=test_username).values('video__video_id'))
ids_vrl = [x['video__video_id'] for x in ids_vrl]
assert set(ids_vrl).issubset(video_ids_set)
return len(ids_vrl)
for i in range(1, minNumRateLater + 2):
assert add_from_topk() == i
if i >= minNumRateLater:
WebDriverWait(driver, TIME_WAIT).until(
EC.presence_of_element_located((By.ID, 'id_big_expert_interface')))
else:
WebDriverWait(driver, TIME_WAIT).until(
EC.presence_of_element_located((By.ID, 'id_recommend_add_more')))
# trying to add manually now...
elem = driver.find_element_by_class_name('video_id_text_field')
elem = elem.find_element_by_tag_name('input')
elem.clear()
elem.send_keys('abacaba')
WebDriverWait(driver, TIME_WAIT).until(
EC.presence_of_element_located((By.ID, 'id_big_add_rate_later')))
driver.find_element_by_id('id_big_add_rate_later').click()
WebDriverWait(driver, TIME_WAIT).until(
EC.presence_of_element_located((By.ID, 'id_rate_later_submit_ok')))
with django_db_blocker.unblock():
get_object_with_timeout(VideoRateLater, user__user__username=test_username,
video__video_id='abacaba')
logout(driver)
def handle(self, **options):
print("Computing quantiles...")
Video.recompute_quantiles()
print("Computing pareto optimality...")
Video.recompute_pareto()
def test_inc(self, n_inc=10):
user = self.create_user()
n_videos, n_ratings = self.n_videos, self.n_ratings
video_ids = [
"%09d-%s-inc" % (i, random_alphanumeric()) for i in range(n_videos)
]
objs = [Video(video_id=vid) for vid in video_ids]
_ = Video.objects.bulk_create(objs)
videos_lst = list(Video.objects.filter(video_id__in=video_ids))
videos_dct = {}
for v in videos_lst:
videos_dct[v.video_id] = v
videos = [videos_dct[k] for k in video_ids]
# creating random ratings
video_firsts = np.random.choice(n_videos - 1, n_ratings,
replace=True) + 1
video_seconds = []
for i in video_firsts:
video_seconds.append(np.random.choice(i))
assert len(video_firsts) == len(video_seconds)
pairs = list(zip(video_firsts, video_seconds))
assert all([x > y] for x, y in pairs)
assert (len(pairs) - len(set(pairs))) / len(pairs) < 0.05
pairs = set(pairs)
ratings = [
ExpertRating(
user=user,
video_1=videos[int(i)],
video_2=videos[int(j)],
**{f: np.random.rand() * MAX_VALUE
for f in VIDEO_FIELDS}) for i, j in pairs
]
_ = ExpertRating.objects.bulk_create(ratings)
ground_truth_inconsistencies = [
TestInconsistencies3.add_inconsistency(user, feature='reliability')
for _ in range(n_inc)
]
qs = ExpertRating.objects.filter(user=user)
edges = get_edges_list_db(qs, 'reliability')
cycles_3, weights_3 = get_cycles_weights_3(edges)
assert set(ground_truth_inconsistencies).issubset(cycles_3)
inc_founds = inconsistencies_3_for_queryset(user)
for (v1id, v2id, v3id) in ground_truth_inconsistencies:
v1 = Video.objects.get(pk=v1id)
v2 = Video.objects.get(pk=v2id)
v3 = Video.objects.get(pk=v3id)
yt_ids = [v1.video_id, v2.video_id, v3.video_id]
founds = []
for inc_found in inc_founds:
if inc_found['feature'] != 'reliability':
continue
if set(yt_ids) != set(inc_found['videos']):
continue
founds.append(inc_found)
assert len(founds) == 1, (founds, yt_ids)
inc_found = founds[0]
cmp = inc_found['comparisons']
assert cmp[0]['videoA'] == yt_ids[0], (cmp, yt_ids)
assert cmp[0]['videoB'] == yt_ids[1], (cmp, yt_ids)
assert cmp[0]['score'] == 0.0
assert cmp[1]['videoA'] == yt_ids[1], (cmp, yt_ids)
assert cmp[1]['videoB'] == yt_ids[2], (cmp, yt_ids)
assert cmp[1]['score'] == 0.0
assert cmp[2]['videoA'] == yt_ids[2], (cmp, yt_ids)
assert cmp[2]['videoB'] == yt_ids[0], (cmp, yt_ids)
assert cmp[2]['score'] == 0.0
assert inc_found['videos'] == yt_ids + [yt_ids[0]]
assert inc_found['weight'] == 0.5
def test_sample_with_other(self):
u = self.create_user()
n_videos = 4
video_ids = [f"{i}-{str(uuid1())}" for i in range(n_videos)]
Video.objects.bulk_create([Video(video_id=vid) for vid in video_ids])
videos_qs = Video.objects.filter(video_id__in=video_ids)
videos_lst = [videos_qs.get(video_id=vid) for vid in video_ids]
v_first = videos_lst[0]
base_qs = videos_qs
n_tests_probabilistic = self.n_tests_probabilistic
# test no rate later, no ratings -> all exceptions
for _ in range(n_tests_probabilistic):
with random_active_features(u):
with pytest.raises(ActiveLearningException) as excinfo:
sample_video_with_other_helper(v_first, u, base_qs=base_qs)
assert excinfo.value.reason.name == 'FIRST_RATED_AGAINST_ALL_RATED',\
excinfo.value.reason.name
with all_active_features(u):
with pytest.raises(ActiveLearningException) as excinfo:
sample_video_with_other_helper(v_first, u, base_qs=base_qs)
assert excinfo.value.reason.name == 'FIRST_RATED_AGAINST_ALL_RATED', \
excinfo.value.reason.name
with no_active_features(u):
with pytest.raises(ActiveLearningException) as excinfo:
sample_video_with_other_helper(v_first, u, base_qs=base_qs)
assert excinfo.value.reason.name == 'NO_FEATURES', \
excinfo.value.reason.name
# same for NO VIDEOS
for _ in range(n_tests_probabilistic):
with random_active_features(u):
with pytest.raises(ActiveLearningException) as excinfo:
sample_video_with_other_helper(v_first,
u,
base_qs=base_qs.none())
assert excinfo.value.reason.name == 'FIRST_RATED_AGAINST_ALL_RATED', \
excinfo.value.reason.name
with all_active_features(u):
with pytest.raises(ActiveLearningException) as excinfo:
sample_video_with_other_helper(v_first,
u,
base_qs=base_qs.none())
assert excinfo.value.reason.name == 'FIRST_RATED_AGAINST_ALL_RATED', \
excinfo.value.reason.name
with no_active_features(u):
with pytest.raises(ActiveLearningException) as excinfo:
sample_video_with_other_helper(v_first,
u,
base_qs=base_qs.none())
assert excinfo.value.reason.name == 'NO_FEATURES', \
excinfo.value.reason.name
# adding some to RATE LATER
VideoRateLater.objects.create(user=u, video=videos_lst[0])
VideoRateLater.objects.create(user=u, video=videos_lst[1])
# only sampling the [1] video, as the [0] is equal to v_first
for _ in range(n_tests_probabilistic):
with random_active_features(u):
assert sample_video_with_other_helper(v_first, u, base_qs=base_qs).video_id == \
video_ids[1]
with all_active_features(u):
assert sample_video_with_other_helper(v_first, u, base_qs=base_qs).video_id == \
video_ids[1]
with no_active_features(u):
assert sample_video_with_other_helper(v_first, u, base_qs=base_qs).video_id == \
video_ids[1]
VideoRateLater.objects.filter(user=u).delete()
# looking for videos with zero rating on active features
r = ExpertRating.objects.create(user=u,
video_1=videos_lst[0],
video_2=videos_lst[1])
# have v_rated=0, 1, but 1 doesn't have any valid ratings
for _ in range(n_tests_probabilistic):
with random_active_features(u):
assert sample_video_with_other_helper(v_first, u, base_qs=base_qs).video_id == \
video_ids[1]
with all_active_features(u):
assert sample_video_with_other_helper(v_first, u, base_qs=base_qs).video_id == \
video_ids[1]
with no_active_features(u):
with pytest.raises(ActiveLearningException) as excinfo:
sample_video_with_other_helper(v_first,
u,
base_qs=base_qs.none())
assert excinfo.value.reason.name == 'NO_FEATURES', \
excinfo.value.reason.name
# setting value for reliability
r.reliability = 1.0
r.save()
# should have an error, because there are no videos in v_rated we
# could compare v_first to (v[1] is already taken)
with set_active_features(u, ['reliability']):
with pytest.raises(ActiveLearningException) as excinfo:
sample_video_with_other_helper(v_first, u, base_qs=base_qs)
assert excinfo.value.reason.name == 'FIRST_RATED_AGAINST_ALL_RATED', \
excinfo.value.reason.name
# for another feature, we still need to rate them
with set_active_features(u, ['backfire_risk']):
assert sample_video_with_other_helper(v_first, u, base_qs=base_qs).video_id == \
video_ids[1]
# rating 0-2
# this way, v_rated=0,1,2, and 2 has no ratings on backfire_risk (but 0, 1 have)
r2 = ExpertRating.objects.create(user=u,
video_1=videos_lst[0],
video_2=videos_lst[2])
with set_active_features(u, ['reliability']):
assert sample_video_with_other_helper(v_first, u, base_qs=base_qs).video_id == \
video_ids[2]
# adding some value. this way, all are rated 0-1, 0-2, so there should be an exception
r2.reliability = 0.5
r2.save()
with set_active_features(u, ['reliability']):
with pytest.raises(ActiveLearningException) as excinfo:
assert sample_video_with_other_helper(v_first, u, base_qs=base_qs).video_id == \
video_ids[2]
assert excinfo.value.reason.name == 'FIRST_RATED_AGAINST_ALL_RATED', \
excinfo.value.reason.name
# deleting r2 and adding r2
r2.delete()
# rating 1-2
# this way, v_rated=0,1,2, (0-1, 1-2), but 2 is unrated, so it will be returned from case 2
r3 = ExpertRating.objects.create(user=u,
video_1=videos_lst[1],
video_2=videos_lst[2])
with set_active_features(u, ['reliability']):
assert sample_video_with_other_helper(v_first, u, base_qs=base_qs).video_id == \
video_ids[2]
# now, setting some value. this way, all videos have _some_ values on all quality features
r3.reliability = 0.5
r3.save()
# gin.bind_parameter('sample_video_with_other_helper.c_noise', 0.0)
gin.bind_parameter(
'annotate_active_score_with_other_nonoise.c_exploration', 0.0)
gin.bind_parameter(
'annotate_active_score_with_other_nonoise.c_skipped', 0.0)
# testing theta part...
r4 = ExpertRating.objects.create(video_1=videos_lst[0],
video_2=videos_lst[3],
user=u)
r4.reliability = 0.5
r4.save()
VideoRating.objects.filter(user=u).delete()
VideoRating.objects.create(video=videos_lst[0], reliability=1, user=u)
VideoRating.objects.create(video=videos_lst[1], reliability=2, user=u)
VideoRating.objects.create(video=videos_lst[2], reliability=3, user=u)
VideoRating.objects.create(video=videos_lst[3], reliability=4, user=u)
# checking THETA part computation
for _ in range(n_tests_probabilistic):
with set_active_features(u, ['reliability', 'engaging']):
qs = sample_video_with_other_helper(v_first,
u,
base_qs=base_qs,
c_noise=0,
debug_return_qs=True)
assert len(qs) == 3
entries = [
qs.get(video_id=video_ids[id_])._theta_part
for id_ in [1, 2, 3]
]
assert np.allclose(entries, [0.5, 1, 1.5])
vid_return = sample_video_with_other_helper(v_first,
u,
base_qs=base_qs,
c_noise=0).video_id
assert vid_return == video_ids[2], (vid_return, video_ids)
# correct 1, but returning not rated 2
# gin.bind_parameter('sample_video_with_other_helper.c_noise', 0.0)
gin.bind_parameter(
'annotate_active_score_with_other_nonoise.c_exploration', 1.0)
gin.bind_parameter(
'annotate_active_score_with_other_nonoise.c_skipped', 0.0)
gin.bind_parameter('annotate_active_score_with_other_nonoise.c_theta',
0.0)
test_v_ids = [video_ids[z] for z in [1, 2, 3]]
for _ in range(n_tests_probabilistic):
with set_active_features(u, ['reliability', 'engaging']):
qs = sample_video_with_other_helper(v_first,
u,
base_qs=base_qs,
c_noise=0,
debug_return_qs=True)
assert len(qs) == 3
got_vals = [
qs.get(video_id=vid)._exploration_part
for vid in test_v_ids
]
correct_vals = [
2. / ExpertRating.objects.filter(
Q(user=u) & (Q(video_1__video_id=v)
| Q(video_2__video_id=v))).count()
for v in test_v_ids
]
assert np.allclose(got_vals, correct_vals)
assert sample_video_with_other_helper(
v_first, u, base_qs=base_qs,
c_noise=0).video_id == video_ids[2]
# correct 1, but returning not rated 2
# gin.bind_parameter('sample_video_with_other_helper.c_noise', 0.0)
gin.bind_parameter(
'annotate_active_score_with_other_nonoise.c_exploration', 0.0)
gin.bind_parameter(
'annotate_active_score_with_other_nonoise.c_skipped', 1.0)
gin.bind_parameter('annotate_active_score_with_other_nonoise.c_theta',
0.0)
# testing SKIPS part
VideoSelectorSkips.objects.create(video=videos_lst[1], user=u)
VideoSelectorSkips.objects.create(video=videos_lst[1], user=u)
VideoSelectorSkips.objects.create(video=videos_lst[2], user=u)
test_v_ids = [video_ids[z] for z in [1, 2, 3]]
for _ in range(n_tests_probabilistic):
with set_active_features(u, ['reliability', 'engaging']):
qs = sample_video_with_other_helper(v_first,
u,
base_qs=base_qs,
c_noise=0,
debug_return_qs=True)
assert len(qs) == 3
correct_vals = [2, 1, 0]
got_vals = [
qs.get(video_id=vid)._skipped_part for vid in test_v_ids
]
assert np.allclose(correct_vals, got_vals)
assert sample_video_with_other_helper(
v_first, u, base_qs=base_qs,
c_noise=0).video_id == video_ids[2]
def test_first_no_rating_active_f(self):
"""II. No active features -> exception, active features, othr -> sampling from empty."""
u = self.create_user()
n_videos = 50
video_ids = [f"{i}-{str(uuid1())}" for i in range(n_videos)]
Video.objects.bulk_create([Video(video_id=vid) for vid in video_ids])
videos_qs = Video.objects.filter(video_id__in=video_ids)
videos_lst = [videos_qs.get(video_id=vid) for vid in video_ids]
# creating a rating with null values
r = ExpertRating.objects.create(video_1=videos_lst[0],
video_2=videos_lst[1],
user=u)
# rating has all empty values -> no rating -> sampling these two videos
for _ in range(self.n_tests_probabilistic):
with all_active_features(u):
v = sample_first_video_helper(u,
do_log=False,
base_qs=videos_qs)
assert v.video_id in video_ids[:2]
# no active features -> exception
with no_active_features(u):
with pytest.raises(ActiveLearningException) as excinfo:
sample_first_video_helper(u, do_log=False, base_qs=videos_qs)
assert excinfo.value.reason.name == 'NO_FEATURES', \
excinfo.value.reason.name
# adding value for reliability for the rating
r.reliability = 0.5
r.save()
# print('line 584')
# active reliability -> complete graph
with set_active_features(u, ['reliability']):
with pytest.raises(ActiveLearningException) as excinfo:
sample_first_video_helper(u, do_log=True, base_qs=videos_qs)
assert excinfo.value.reason.name == 'NO_RATE_LATER_ALL_RATED', \
excinfo.value.reason.name
# other feature -> sampling from empty
with set_active_features(u, ['backfire_risk']):
v = sample_first_video_helper(u, do_log=False, base_qs=videos_qs)
assert v.video_id in video_ids[:2]
with no_active_features(u):
with pytest.raises(ActiveLearningException) as excinfo:
sample_first_video_helper(u, do_log=False, base_qs=videos_qs)
assert excinfo.value.reason.name == 'NO_FEATURES', \
excinfo.value.reason.name
# rating two other videos
# 1-2, 1-3, but not 2-3. should only sample from 2-3
r2 = ExpertRating.objects.create(video_1=videos_lst[0],
video_2=videos_lst[2],
user=u)
# graph: 0-1, reliability=set
# 0-2, reliability=none
# 1, 2 are in zero-rating
# since we didn't have set the value, we sample from v_zero_rating
with set_active_features(u, ['reliability']):
v = sample_first_video_helper(u, do_log=True, base_qs=videos_qs)
print("SAMPLED:", v)
assert v.video_id in [video_ids[1], video_ids[2]], v
r2.reliability = 1.0
r2.save()
# setting uncertainty for videos[1]
VideoRating.objects.create(user=u,
video=videos_lst[1],
reliability_uncertainty=10)
# since we didn't have set the value, we sample from v_zero_rating
for _ in range(self.n_tests_probabilistic):
with set_active_features(u, ['reliability']):
v = sample_first_video_helper(u,
do_log=True,
base_qs=videos_qs)
assert v.video_id == video_ids[1]
def test_wrong_video_id_protection(self, client, current_user_preferences):
wrong_id = '$$$videoAAA'
good_id = 'abacaba'
good_id1 = 'abacabaaa'
assert good_id != good_id1
assert not re.match(youtubeVideoIdRegex, wrong_id)
assert re.match(youtubeVideoIdRegex, good_id)
assert re.match(youtubeVideoIdRegex, good_id1)
# List of endpoints where videos can be created:
# Video POST
# Expert Ratings POST
# Expert Rating Set by Video IDS
# Expert Ratings Slider Value Change
# Rate later POST
# VIDEO POST
r = client.post('/api/v2/videos/', data={'video_id': wrong_id})
print(r.status_code, r.json())
assert r.status_code == 400
assert r.json()['video_id'][0].startswith('Video ID must match')
# EXPERT RATING, two ways
r = client.post('/api/v2/expert_ratings/', data={'video_1': wrong_id,
'video_2': good_id})
print(r.status_code, r.json())
assert r.status_code == 400
assert r.json()['video_1']['video_id'][0].startswith('Video ID must match')
r = client.post('/api/v2/expert_ratings/', data={'video_1': good_id,
'video_2': wrong_id})
print(r.status_code, r.json())
assert r.status_code == 400
assert r.json()['video_2']['video_id'][0].startswith('Video ID must match')
# cleaning videos
Video.objects.all().delete()
# creating some good videos
v1 = Video(video_id=good_id)
v1.full_clean()
v1.save()
v2 = Video(video_id=good_id1)
v2.full_clean()
v2.save()
# creating the rating
rating = ExpertRating.objects.create(user=current_user_preferences, video_1=v1,
video_2=v2)
# EXPERT RATING PATCH
for method in [client.put, client.patch]:
r = method(f'/api/v2/expert_ratings/{rating.id}/',
data={'video_1': wrong_id, 'video_2': good_id1})
print(r.status_code, r.json())
assert r.status_code == 400
assert r.json()['video_1']['video_id'][0].startswith('Video ID must match')
for method in [client.put, client.patch]:
r = method(f'/api/v2/expert_ratings/{rating.id}/',
data={'video_1': good_id, 'video_2': wrong_id})
print(r.status_code, r.json())
assert r.status_code == 400
assert r.json()['video_2']['video_id'][0].startswith('Video ID must match')
# EXPERT RATING SET BY VIDEO IDS
r = client.patch('/api/v2/expert_ratings/by_video_ids/?'
f'video_left={good_id}&video_right={good_id1}&force_set_ids=true',
data={'video_1': wrong_id, 'video_2': good_id1})
print(r.status_code, r.json())
assert r.status_code == 400
assert r.json()['video_1']['video_id'][0].startswith('Video ID must match')
r = client.patch('/api/v2/expert_ratings/by_video_ids/?'
f'video_left={good_id}&video_right={good_id1}&force_set_ids=true',
data={'video_1': good_id, 'video_2': wrong_id})
print(r.status_code, r.json())
assert r.status_code == 400
assert r.json()['video_2']['video_id'][0].startswith('Video ID must match')
# SLIDER VALUE CHANGE
r = client.post('/api/v2/expert_ratings/register_slider_change/',
data={'video_left': wrong_id, 'video_right': good_id1})
print(r.status_code, r.json())
assert r.status_code == 400
assert r.json()['video_left']['video_id'][0].startswith('Video ID must match')
r = client.post('/api/v2/expert_ratings/register_slider_change/',
data={'video_left': good_id, 'video_right': wrong_id})
print(r.status_code, r.json())
assert r.status_code == 400
assert r.json()['video_right']['video_id'][0].startswith('Video ID must match')
# RATELATER
r = client.post('/api/v2/rate_later/',
data={'video': wrong_id})
print(r.status_code, r.json())
assert r.status_code == 400
assert r.json()['video']['video_id'][0].startswith('Video ID must match')