def test_hardcoded_dataset(mode):
assert mode in ['sparse', 'dense']
dataset = ToyHardcodedDataset()
dataset._generate_many(100)
all_ratings = AllRatingsWithCommon(
experts=dataset.users,
objects=dataset.objects,
output_features=dataset.fields,
name="tst",
var_init_cls=VariableIndexLayer
if mode == 'dense' else SparseVariableIndexLayer,
)
# creating models
models = [
FeaturelessPreferenceLearningModel(expert=user,
all_ratings=all_ratings)
for user in dataset.users
]
for r in dataset.ratings:
u_idx = dataset.users.index(r["user"])
ratings_as_vector = np.array([r["ratings"][k]
for k in dataset.fields]) / 100.0
models[u_idx].register_preference(
o1=r["o1"],
o2=r["o2"],
p1_vs_p2=ratings_as_vector,
weights=np.ones(len(ratings_as_vector)),
)
call_on_dataset_end(models)
# aggregating models
aggregator = FeaturelessMedianPreferenceAverageRegularizationAggregator(
models=models,
loss_fcn=loss_fcn_dense if mode == 'dense' else loss_fcn_sparse,
hypers={
"C": 1.0,
"mu": 1.0,
"lambda_": 1.0,
"default_score_value": 1.0,
"sample_every": 100
},
batch_params=dict(
sample_experts=5000,
sample_ratings_per_expert=5000,
sample_objects_per_expert=5000,
),
)
aggregator.fit(epochs=1000)
result = aggregator.models[0](["trump_video"])[0]
assert isinstance(result, np.ndarray), "Wrong output"
result = aggregator(["trump_video"])[0]
assert isinstance(result, np.ndarray), "Wrong output"
aggregator.plot_loss()
plt.savefig("_test_plot.png")
def validate_order(dataset, aggregator):
"""Test that downvoted videos have smaller ratings."""
for user_id, user in enumerate(dataset.users):
got_scores = aggregator.models[user_id](dataset.objects)
expect_scores = dataset.scores_dict[user]
errors = 0
for i, feature in enumerate(dataset.fields):
for i1, o1 in enumerate(dataset.objects):
for i2, o2 in enumerate(dataset.objects):
if o1 == o2:
continue
delta1 = got_scores[i2][i] - got_scores[i1][i]
if (o1, o2) in expect_scores[feature]:
delta2 = expect_scores[feature][(o1, o2)]
else:
delta2 = 100 - expect_scores[feature][(o2, o1)]
delta2 = (delta2 - 50) / 50.0
if delta1 * delta2 <= 0:
print(
f"Invalid result: {user} {feature} {o1} {o2} got"
f" {got_scores[i1][i]} {got_scores[i2][i]} rating {delta2}"
)
errors += 1
else:
print("Valid result")
assert not errors, "There were %s errors" % errors
validate_order(dataset, aggregator)
class DatabasePreferenceLearnerFeatureless(DatabasePreferenceLearner):
"""Learn models from the database, save/restore."""
def create_models(self):
"""Create learning models and the aggregator."""
self.all_ratings = AllRatingsWithCommon(
experts=self.users,
objects=self.videos,
output_features=self.features,
name="prod",
)
print_memory(stage="DPLF:ratings_nodata_created")
# creating models
self.user_to_model = {
user: FeaturelessPreferenceLearningModel(
expert=user, all_ratings=self.all_ratings
)
for user in self.users
}
print_memory(stage="DPLF:models_created")
# before creating the aggregator, filling models with data
self.user_to_size = {
user: self.fill_model_data(self.user_to_model[user], user)
for user in tqdmem(self.users, desc="fill_data")
}
# virtual 'common' data
fplm_common = FeaturelessPreferenceLearningModel(
expert=AllRatingsWithCommon.COMMON_EXPERT, all_ratings=self.all_ratings
)
fplm_common.on_dataset_end()
print_memory(stage="DPLF:data_filled")
# resetting the model given the data
self.all_ratings.reset_model()
print_memory(stage="DPLF:model_reset_ok")
# aggregating models
self.aggregator = FeaturelessMedianPreferenceAverageRegularizationAggregator(
models=[self.user_to_model[u] for u in self.users]
)
self.aggregator.certification_status = self.user_certified
print_memory(stage="DPLF:aggregator_created")
def visualize(self):
"""Plot model predictions and losses."""
self.aggregator.plot_loss()
self.save_figure()
def predict_user(self, user, videos):
# @todo: use vectorized operations
assert isinstance(user, UserPreferences)
model = self.user_to_model[user.id]
result = list(model([v.video_id for v in videos]))
for i, video in enumerate(videos):
if not model.ratings_with_object(video.video_id):
result[i] = None
return result
def predict_aggregated(self, videos):
# @todo: use vectorized operations
return self.aggregator([v.video_id for v in videos])
def fit(self, **kwargs):
"""Fit on latest database records."""
self.stats["dataset_size"] = self.user_to_size
super(DatabasePreferenceLearnerFeatureless, self).fit(**kwargs)
def fill_model_data(self, model, user):
"""Populate model data from db."""
n = 0
for dct in self.get_dataset(user=user):
v1, v2, res, w = [
dct[key] for key in ["video_1", "video_2", "cmp", "weights"]
]
model.register_preference(v1, v2, res, w)
n += 1
model.on_dataset_end()
return n