def test_simple(self):
load_gin_config('backend/ml_model/config/featureless_config.gin')
username = '******'
dj_user = DjangoUser.objects.create(username=username)
up = UserPreferences.objects.create(user=dj_user)
v1 = Video.objects.create(video_id="online-1", reliability=0)
v2 = Video.objects.create(video_id="online-2", reliability=1.0)
VideoRating.objects.create(video=v1, user=up, reliability=0)
VideoRating.objects.create(video=v2, user=up, reliability=1.0)
r = ExpertRating.objects.create(user=up,
video_1=v1,
video_2=v2,
reliability=100)
field = 'reliability'
update_context = OnlineRatingUpdateContext(r, field)
def get_scores_online(rating):
result = compute_online_update(
rating_value=rating,
idx_set=update_context.idx_set,
model_tensor_orig=update_context.model_tensor,
mb_np_orig=update_context.mb_np)
update_context.write_updates_to_db(result['new_model_tensor'])
s1 = VideoRating.objects.get(video=v1, user=up).reliability
s2 = VideoRating.objects.get(video=v2, user=up).reliability
return s1, s2
s1, s2 = get_scores_online(-1)
assert s1 > s2
s1, s2 = get_scores_online(1)
assert s1 < s2
def test_gin_files(self):
for root, dirs, files in os.walk(BASE_DIR):
for file in files:
path = os.path.join(root, file)
if file.endswith('.gin') and os.path.isfile(path):
load_gin_config(path)
gin.clear_config()
def setUp(self):
load_gin_config("backend/ml_model/config/common.gin")
load_gin_config("backend/ml_model/config/featureless_config.gin")
self.v1 = Video.objects.create(video_id="a")
self.v2 = Video.objects.create(video_id="b")
self.videos = [self.v1, self.v2]
for v in self.videos:
v.set_embedding(np.random.randn(Video.EMBEDDING_LEN))
v.save()
self.djangouser = DjangoUser.objects.create_user(username="******",
password="******")
self.prefs = UserPreferences.objects.create(user=self.djangouser)
# making the user verified
self.ui = UserInformation.objects.create(user=self.djangouser)
accepted_domain = create_accepted_domain()
self.vemail = VerifiableEmail.objects.create(
user=self.ui,
email=f"{uuid1()}{accepted_domain}",
is_verified=True)
data = {k: np.random.rand() * 100 for k in VIDEO_FIELDS}
self.rating = ExpertRating.objects.create(video_1=self.v1,
video_2=self.v2,
user=self.prefs,
**data)
def cached_class_instance(rating_pk, field, timeout_sec=100):
"""Get a cached instance or create one"""
cache = current_cache()
current_time = time()
# key to search for in the cache
key = (rating_pk, field)
# invalidating old caches...
# TODO: replace with a faster implementation
for existing_key in deepcopy(list(cache.keys())):
if current_time - cache[existing_key][0] >= timeout_sec:
del cache[existing_key]
# need to add a new element, and there are too many already...
if key not in cache and len(cache) >= MAX_CACHE_PER_THREAD:
raise ValueError(
"Too many elements in cache, please wait and try again...")
# creating an item if it doesn't exist or if it's too old
if key not in cache or current_time - cache[key][0] >= timeout_sec:
load_gin_config('backend/ml_model/config/featureless_config.gin')
cache[key] = (current_time,
OnlineRatingUpdateContext(
expert_rating=ExpertRating.objects.get(pk=rating_pk),
feature=field))
# returning the object
return {'cache': cache[key][1], 'cache_created': cache[key][0]}
def handle(self, **options):
load_gin_config(options['config'])
try:
vm = VideoManager(only_download=options['only_download'])
except Exception as e:
print("Arguments are wrong, initialization failed")
print(e)
return None
if options['console']:
commands = [
x for x in dir(vm)
if not x.startswith('_') and callable(getattr(vm, x))
]
print("Available commands: ", commands)
print("Type exit to stop")
while True:
print("> ", end="")
inp = input().strip()
if inp in commands:
f = getattr(vm, inp)
print(signature(f))
f()
elif inp == 'exit':
return None
else:
print("Unknown command")
print("Available commands: ", commands)
print("Type exit to stop")
else:
print("Doing default import")
vm.fill_info()
vm.add_videos_in_folder_to_db()
vm.clear_info()
def handle(self, **options):
print_memory(stage="Command init")
features = options["features"]
if features is None:
features = VIDEO_FIELDS
for f in features:
assert f in VIDEO_FIELDS, f"Feature {f} not recognized, {VIDEO_FIELDS}"
print(f"Using features {', '.join(features)}")
for config in options["config"]:
print("Loading config", config)
load_gin_config(config)
# running parallel hparam tuning with Ray
if options["tune"]:
def pre_parse():
"""Load django before reading configuration (otherwise have import error)."""
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE",
"django_react.settings")
import django
django.setup()
if options["tune_resume"]:
gin.bind_parameter('tune_run.resume', True)
tune_gin(experiment, pre_parse=pre_parse)
# regular training
else:
print_memory(stage="pre-learner init")
learner_obj = learner()(features=features)
print_memory(stage="learner created")
# print("pre-fit reached... entering infinite loop")
# from time import sleep
# while True:
# sleep(1)
#
# print_mem_epoch = partial(print_memory, stage='EPOCH')
learner_obj.fit(epochs=options["epochs_override"])
print_memory(stage="post train")
learner_obj.update_features()
print_memory(stage="post update")
def setUp(self):
load_gin_config("backend/ml_model/config/featureless_config.gin")
gin.bind_parameter(
"FeaturelessMedianPreferenceAverageRegularizationAggregator.epochs",
1000)
# creating videos
self.videos = [
Video.objects.create(video_id=f"video{i}") for i in tqdm(range(2))
]
# creating users
self.djangousers = [
DjangoUser.objects.create_user(username=f"rater{i}",
password=f"1234{i}")
for i in tqdm(range(2))
]
self.userprefs = [
UserPreferences.objects.create(user=u) for u in self.djangousers
]
# making the user verified
self.userinfos = [
UserInformation.objects.create(user=u) for u in self.djangousers
]
self.verify = [False, True]
accepted_domain = create_accepted_domain()
self.vemails = [
VerifiableEmail.objects.create(user=ui,
email=f"{uuid1()}{accepted_domain}",
is_verified=verify)
for ui, verify in zip(self.userinfos, self.verify)
]
data_rest = {k: 50 for k in VIDEO_FIELDS[1:]}
self.f = VIDEO_FIELDS[0]
# rater0 likes video0, rater1 likes video1
ExpertRating.objects.create(user=self.userprefs[0],
video_1=self.videos[0],
video_2=self.videos[1],
**data_rest,
**{self.f: 0})
ExpertRating.objects.create(user=self.userprefs[1],
video_1=self.videos[0],
video_2=self.videos[1],
**data_rest,
**{self.f: 100})
def setUp(self):
load_gin_config("backend/ml_model/config/featureless_config.gin")
gin.bind_parameter(
"FeaturelessMedianPreferenceAverageRegularizationAggregator.epochs",
1000)
# creating videos
self.videos = [
Video.objects.create(video_id=f"video{i}")
for i in tqdm(range(200))
]
# creating users
self.djangousers = [
DjangoUser.objects.create_user(username=f"rater{i}",
password=f"1234{i}")
for i in tqdm(range(10))
]
self.userprefs = [
UserPreferences.objects.create(user=u) for u in self.djangousers
]
# making the user verified
self.userinfos = [
UserInformation.objects.create(user=u) for u in self.djangousers
]
accepted_domain = create_accepted_domain()
self.vemails = [
VerifiableEmail.objects.create(user=ui,
email=f"{uuid1()}{accepted_domain}",
is_verified=True)
for ui in self.userinfos
]
# creating expert ratings
ratings = []
for _ in tqdm(range(1000)):
v1 = np.random.choice(self.videos)
v2 = np.random.choice(self.videos)
u = np.random.choice(self.userprefs)
data = {k: np.random.rand() * 100 for k in VIDEO_FIELDS}
ratings.append(ExpertRating(video_1=v1, video_2=v2, user=u,
**data))
ExpertRating.objects.bulk_create(ratings, ignore_conflicts=True)
def load_videos_thread(config):
"""Download metadata for a few videos."""
to_download = qs_videos_to_download()
to_download_subsample = qs_subsample_to_download(to_download)
ids = [x.video_id for x in to_download_subsample]
if not ids:
print("No videos, exiting!")
return
print(f"Total to-download: {to_download.count()} subsampled: {len(ids)}")
load_gin_config(config)
vm = VideoManager(only_download=ids)
vm.fill_info()
vm.add_videos_in_folder_to_db()
vm.clear_info()
def test_download_metadata(self):
CharField.register_lookup(Length, 'length')
TextField.register_lookup(Length, 'length')
load_gin_config('backend/add_videos.gin')
example_unlisted_video = 'dHFYikxUatY'
example_correct_video = '9bZkp7q19f0'
example_wrong_url_video = 'w$url'
example_notfound_video = 'notfoundvid'
test_videos = [
example_unlisted_video, example_correct_video,
example_wrong_url_video, example_notfound_video
]
test_responses = [{
'name__length__gt': 5,
'is_unlisted': True,
'description__length__gt': 1,
'publication_date__isnull': False,
'language__isnull': False,
'views__isnull': False,
'metadata_timestamp__isnull': False,
'uploader__isnull': False,
'duration__isnull': False,
'last_download_time__isnull': False,
'download_attempts__gt': 0,
'add_time__isnull': False,
'download_failed': False
}, {
'name__length__gt': 5,
'is_unlisted': False,
'views__isnull': False,
'description__length__gt': 1,
'publication_date__isnull': False,
'uploader__isnull': False,
'duration__isnull': False,
'language__isnull': False,
'metadata_timestamp__isnull': False,
'wrong_url': False,
'last_download_time__isnull': False,
'download_attempts__gt': 0,
'add_time__isnull': False,
'download_failed': False
}, {
'name': "",
'is_unlisted': False,
'metadata_timestamp__isnull': True,
'wrong_url': True,
'last_download_time__isnull': False,
'download_attempts__gt': 0,
'add_time__isnull': False,
'download_failed': True
}, {
'name': "",
'is_unlisted': False,
'metadata_timestamp__isnull': True,
'wrong_url': False,
'last_download_time__isnull': False,
'download_attempts__gt': 0,
'add_time__isnull': False,
'download_failed': True
}]
for v, resp in zip(test_videos, test_responses):
Video.objects.filter(video_id=v).delete()
vm = VideoManager(only_download=[v])
vm.fill_info()
vm.add_videos_in_folder_to_db()
qs = Video.objects.filter(video_id=v)
assert qs.count() == 1
# format: key, val
errors = []
for key, val in resp.items():
sub_qs = qs.filter(**{key: val})
if sub_qs.count() != 1:
errors.append((key, val))
assert not errors, (v, resp, errors)
o = qs.get()
print(o)