def test_serializer_deletes_removed_variants(self):
experiment = ExperimentFactory.create(
type=ExperimentConstants.TYPE_GENERIC)
control_variant = ExperimentVariantFactory.create(
experiment=experiment, is_control=True)
ExperimentVariantFactory.create(experiment=experiment,
is_control=False)
treatment2_variant = ExperimentVariantFactory.create(
experiment=experiment, is_control=False)
self.assertEqual(experiment.variants.all().count(), 3)
self.control_variant_data["id"] = control_variant.id
self.treatment_variant_data["id"] = treatment2_variant.id
data = {
"variants":
[self.control_variant_data, self.treatment_variant_data]
}
serializer = ExperimentDesignBaseSerializer(
instance=experiment, data=data, context={"request": self.request})
self.assertTrue(serializer.is_valid())
experiment = serializer.save()
self.assertEqual(experiment.variants.all().count(), 2)
self.assertEqual(set(experiment.variants.all()),
set([control_variant, treatment2_variant]))
def test_serializer_puts_control_branch_first_and_sorts_rest_by_id(self):
ExperimentVariantFactory.create(is_control=True)
sorted_treatment_ids = sorted([
ExperimentVariantFactory.create(is_control=False).id
for i in range(3)
])
serializer = ExperimentDesignVariantBaseSerializer(
ExperimentVariant.objects.all().order_by("-id"), many=True)
self.assertTrue(serializer.data[0]["is_control"])
self.assertFalse(any([b["is_control"] for b in serializer.data[1:]]))
self.assertEqual(sorted_treatment_ids,
[b["id"] for b in serializer.data[1:]])
def test_update_changelog_creates_no_log_when_no_change(self):
experiment = ExperimentFactory.create_with_status(
target_status=Experiment.STATUS_DRAFT, num_variants=0)
variant = ExperimentVariantFactory.create(
ratio=100,
name="variant name",
experiment=experiment,
value=None,
addon_release_url=None,
)
data = {
"variants": [{
"id": variant.id,
"ratio": variant.ratio,
"description": variant.description,
"name": variant.name,
"is_control": variant.is_control,
}]
}
self.assertEqual(experiment.changes.count(), 1)
serializer = ExperimentDesignBaseSerializer(
instance=experiment, data=data, context={"request": self.request})
self.assertTrue(serializer.is_valid())
experiment = serializer.save()
self.assertEqual(experiment.changes.count(), 1)
def test_serializer_outputs_expected_schema_non_multi_pref_format(self):
experiment = ExperimentFactory.create(
normandy_slug="normandy-slug",
pref_branch=Experiment.PREF_BRANCH_DEFAULT,
pref_type=Experiment.PREF_TYPE_JSON_STR,
pref_name="browser.pref",
firefox_min_version="55.0",
)
variant = ExperimentVariantFactory.create(slug="control",
ratio=25,
experiment=experiment,
value='{"some": "json"}')
serializer = ExperimentRecipeMultiPrefVariantSerializer(variant)
expected_data = {
"preferences": {
"browser.pref": {
"preferenceBranchType": "default",
"preferenceType": "string",
"preferenceValue": '{"some": "json"}',
}
},
"ratio": 25,
"slug": "control",
}
self.assertDictEqual(expected_data, serializer.data)
def test_serializer_outputs_expected_schema(self):
variant = ExperimentVariantFactory.create()
vp = VariantPreferencesFactory.create(variant=variant)
serializer = ExperimentDesignBranchMultiPrefSerializer(variant)
self.assertEqual(
serializer.data,
{
"id":
variant.id,
"description":
variant.description,
"ratio":
variant.ratio,
"is_control":
False,
"name":
variant.name,
"preferences": [{
"id": vp.id,
"pref_name": vp.pref_name,
"pref_value": vp.pref_value,
"pref_branch": vp.pref_branch,
"pref_type": vp.pref_type,
}],
},
)
def test_serializer_updates_existing_variant_pref(self):
variant = ExperimentVariantFactory.create(experiment=self.experiment)
variant_pref = VariantPreferencesFactory.create(variant=variant)
self.pref1["id"] = variant_pref.id
self.control_variant["id"] = variant.id
self.control_variant["ratio"] = 100
self.assertEqual(variant.preferences.all().count(), 1)
self.assertTrue(variant.preferences.get(id=variant_pref.id))
data = {"is_multi_pref": True, "variants": [self.control_variant]}
serializer = ExperimentDesignMultiPrefSerializer(
instance=self.experiment,
data=data,
context={"request": self.request})
self.assertTrue(serializer.is_valid())
experiment = serializer.save()
variant = ExperimentVariant.objects.get(id=variant.id)
self.assertEqual(variant.preferences.all().count(), 2)
self.assertEqual(
variant.preferences.filter(id=variant_pref.id).count(), 1)
self.assertEqual(experiment.changes.count(), 1)
def test_serializer_outputs_expected_schema(self):
experiment = ExperimentFactory.create()
variant = ExperimentVariantFactory.create(experiment=experiment,
is_control=True)
vp = VariantPreferencesFactory.create(variant=variant)
serializer = ExperimentDesignMultiPrefSerializer(experiment)
serializer.data["variants"][0]["preferences"][0].pop("id")
self.assertCountEqual(
serializer.data["variants"][0],
{
"id":
vp.id,
"description":
variant.description,
"is_control":
variant.is_control,
"name":
variant.name,
"ratio":
variant.ratio,
"preferences": [{
"pref_name": vp.pref_name,
"pref_value": vp.pref_value,
"pref_branch": vp.pref_branch,
"pref_type": vp.pref_type,
}],
},
)
def setUp(self):
self.user_email = "*****@*****.**"
self.experiment = ExperimentFactory.create(type="pref")
self.variant = ExperimentVariantFactory.create(
experiment=self.experiment, is_control=True)
self.preference = VariantPreferencesFactory.create(
variant=self.variant)
def test_variant_pref_value_returns_bool_when_type_is_bool(self):
experiment = ExperimentFactory.create(pref_type=Experiment.PREF_TYPE_BOOL)
variant = ExperimentVariantFactory.create(experiment=experiment, value="false")
value = PrefValueField(
type_field="experiment__pref_type", value_field="value"
).to_representation(variant)
self.assertEqual(value, False)
def test_serializer_outputs_expected_schema(self):
experiment = ExperimentFactory()
variant = ExperimentVariantFactory.create(experiment=experiment)
serializer = ExperimentRecipeMessageVariantSerializer(variant)
self.assertDictEqual(
serializer.data,
{"ratio": variant.ratio, "slug": variant.slug, "value": {}, "groups": []},
)
def test_variant_pref_value_str_returns_bool_when_type_is_str(self):
experiment = ExperimentFactory.create(
pref_type=Experiment.PREF_TYPE_STR)
variant = ExperimentVariantFactory.create(experiment=experiment,
value="it's a string")
value = PrefValueField(type_field="experiment__pref_type",
value_field="value").to_representation(variant)
self.assertEqual(value, "it's a string")
def test_serializer_outputs_expected_schema(self):
experiment = ExperimentFactory(pref_type=Experiment.PREF_TYPE_STR)
variant = ExperimentVariantFactory.create(experiment=experiment)
serializer = ExperimentRecipeVariantSerializer(variant)
self.assertDictEqual(
serializer.data,
{"ratio": variant.ratio, "slug": variant.slug, "value": variant.value},
)
def test_serializer_adds_new_variant(self):
experiment = ExperimentFactory.create(
type=ExperimentConstants.TYPE_GENERIC)
control_variant = ExperimentVariantFactory.create(
experiment=experiment, is_control=True)
treatment1_variant = ExperimentVariantFactory.create(
experiment=experiment, is_control=False)
self.assertEqual(experiment.variants.all().count(), 2)
self.control_variant_data["id"] = control_variant.id
self.control_variant_data["ratio"] = 33
self.treatment_variant_data["id"] = treatment1_variant.id
self.treatment_variant_data["ratio"] = 33
treatment2_variant_data = {
"name": "New Branch",
"ratio": 34,
"description": "New Branch",
"is_control": False,
}
data = {
"variants": [
self.control_variant_data,
self.treatment_variant_data,
treatment2_variant_data,
]
}
serializer = ExperimentDesignBaseSerializer(
instance=experiment, data=data, context={"request": self.request})
self.assertTrue(serializer.is_valid())
experiment = serializer.save()
self.assertEqual(experiment.variants.all().count(), 3)
new_variant = ExperimentVariant.objects.get(
name=treatment2_variant_data["name"])
self.assertEqual(
set(experiment.variants.all()),
set([control_variant, treatment1_variant, new_variant]),
)
def test_serializer_outputs_expected_int_val(self):
experiment = ExperimentFactory(pref_type=Experiment.PREF_TYPE_INT)
variant = ExperimentVariantFactory.create(experiment=experiment, value="28")
serializer = ExperimentRecipeVariantSerializer(variant)
self.assertEqual(type(serializer.data["value"]), int)
self.assertEqual(
serializer.data, {"ratio": variant.ratio, "slug": variant.slug, "value": 28}
)
def test_serializer_updates_existing_variants(self):
experiment = ExperimentFactory.create(
type=ExperimentConstants.TYPE_GENERIC)
control_variant = ExperimentVariantFactory.create(
experiment=experiment, is_control=True)
treatment_variant = ExperimentVariantFactory.create(
experiment=experiment, is_control=False)
self.assertEqual(experiment.variants.all().count(), 2)
self.control_variant_data["id"] = control_variant.id
self.treatment_variant_data["id"] = treatment_variant.id
data = {
"variants":
[self.control_variant_data, self.treatment_variant_data]
}
serializer = ExperimentDesignBaseSerializer(
instance=experiment, data=data, context={"request": self.request})
self.assertTrue(serializer.is_valid())
experiment = serializer.save()
self.assertEqual(experiment.variants.all().count(), 2)
control_variant = ExperimentVariant.objects.get(id=control_variant.id)
self.assertEqual(control_variant.name,
self.control_variant_data["name"])
self.assertEqual(control_variant.ratio,
self.control_variant_data["ratio"])
self.assertEqual(control_variant.description,
self.control_variant_data["description"])
treatment_variant = ExperimentVariant.objects.get(
id=treatment_variant.id)
self.assertEqual(treatment_variant.name,
self.treatment_variant_data["name"])
self.assertEqual(treatment_variant.ratio,
self.treatment_variant_data["ratio"])
self.assertEqual(treatment_variant.description,
self.treatment_variant_data["description"])
def test_serializer_outputs_expected_schema(self):
variant = ExperimentVariantFactory.create()
serialized = ExperimentVariantSerializer(variant)
self.assertEqual(
serialized.data, {
'description': variant.description,
'name': variant.name,
'ratio': variant.ratio,
'slug': variant.slug,
'value': variant.value,
})
def test_serializer_outputs_expected_bool(self):
experiment = ExperimentFactory(pref_type=Experiment.PREF_TYPE_BOOL)
variant = ExperimentVariantFactory.create(
experiment=experiment, value="true"
)
serializer = ExperimentRecipeVariantSerializer(variant)
self.assertEqual(type(serializer.data["value"]), bool)
self.assertEqual(
serializer.data,
{"ratio": variant.ratio, "slug": variant.slug, "value": True},
)
def test_serializer_outputs_expected_schema(self):
variant = ExperimentVariantFactory.create()
serialized = ExperimentVariantSerializer(variant)
self.assertEqual(
serialized.data,
{
"description": variant.description,
"is_control": variant.is_control,
"name": variant.name,
"ratio": variant.ratio,
"slug": variant.slug,
"value": variant.value,
},
)
def test_serializer_outputs_expected_schema(self):
variant = ExperimentVariantFactory.create()
vp = VariantPreferencesFactory.create(variant=variant)
serializer = ExperimentDesignBranchVariantPreferencesSerializer(vp)
self.assertEqual(
serializer.data,
{
"id": vp.id,
"pref_name": vp.pref_name,
"pref_branch": vp.pref_branch,
"pref_type": vp.pref_type,
"pref_value": vp.pref_value,
},
)
def test_serializer_outputs_expected_schema(self):
variant = ExperimentVariantFactory.create()
serializer_data = ExperimentRapidVariantSerializer(instance=variant).data
self.assertDictEqual(
{
"id": variant.id,
"name": variant.name,
"ratio": variant.ratio,
"description": variant.description,
"is_control": variant.is_control,
"value": variant.value,
},
serializer_data,
)
def test_seriailzer_outputs_expected_schema_for_single_pref_experiment(self):
experiment = ExperimentFactory.create(
pref_type=Experiment.PREF_TYPE_JSON_STR, firefox_max_version="70.0"
)
variant = ExperimentVariantFactory.create(experiment=experiment)
serializer = ExperimentRecipeMultiPrefVariantSerializer(variant)
self.assertEqual(serializer.data["ratio"], variant.ratio)
self.assertEqual(serializer.data["slug"], variant.slug)
serialized_preferences = serializer.data["preferences"]
self.assertDictEqual(
serialized_preferences[experiment.pref_name],
{
"preferenceBranchType": experiment.pref_branch,
"preferenceType": PrefTypeField().to_representation(experiment.pref_type),
"preferenceValue": variant.value,
},
)
def test_seriailzer_outputs_expected_schema_for_multi_pref_variant(self):
experiment = ExperimentFactory.create(
pref_type=Experiment.PREF_TYPE_JSON_STR, is_multi_pref=True
)
variant = ExperimentVariantFactory.create(experiment=experiment)
preference = VariantPreferencesFactory.create(variant=variant)
serializer = ExperimentRecipeMultiPrefVariantSerializer(variant)
self.assertEqual(serializer.data["ratio"], variant.ratio)
self.assertEqual(serializer.data["slug"], variant.slug)
serialized_preferences = serializer.data["preferences"]
self.assertDictEqual(
serialized_preferences[preference.pref_name],
{
"preferenceBranchType": preference.pref_branch,
"preferenceType": PrefTypeField().to_representation(preference.pref_type),
"preferenceValue": preference.pref_value,
},
)
self.assertEqual(serializer.data["ratio"], variant.ratio)
self.assertEqual(serializer.data["slug"], variant.slug)
def test_serializer_outputs_expected_schema_for_multi_pref_format(self):
experiment = ExperimentFactory.create(normandy_slug="normandy-slug",
firefox_min_version="55.0",
is_multi_pref=True)
variant = ExperimentVariantFactory.create(slug="control",
ratio=25,
experiment=experiment)
preference = VariantPreferencesFactory.create(variant=variant)
serializer = ExperimentRecipeMultiPrefVariantSerializer(variant)
self.assertEqual(serializer.data["ratio"], 25)
self.assertEqual(serializer.data["slug"], "control")
serialized_preferences = serializer.data["preferences"]
self.assertDictEqual(
serialized_preferences[preference.pref_name],
{
"preferenceBranchType": preference.pref_branch,
"preferenceType": preference.pref_type,
"preferenceValue": preference.pref_value,
},
)
def test_serializer_outputs_expected_schema_for_accepted(self):
audience = "us_only"
features = ["pinned_tabs", "picture_in_picture"]
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_ACCEPTED,
audience=audience,
features=features,
firefox_channel=Experiment.CHANNEL_RELEASE,
firefox_min_version="80.0",
proposed_start_date=None,
proposed_duration=28,
proposed_enrollment=7,
rapid_type=Experiment.RAPID_AA,
type=Experiment.TYPE_RAPID,
)
experiment.variants.all().delete()
ExperimentVariantFactory.create(experiment=experiment,
ratio=1,
slug="control",
is_control=True)
ExperimentVariantFactory.create(experiment=experiment,
ratio=1,
slug="treatment",
is_control=False)
ExperimentBucketNamespace.request_namespace_buckets(
experiment.recipe_slug, experiment, 100)
serializer = ExperimentRapidRecipeSerializer(experiment)
data = serializer.data
arguments = data.pop("arguments")
branches = arguments.pop("branches")
self.assertDictEqual(
data,
{
"id":
experiment.recipe_slug,
"filter_expression":
"env.version|versionCompare('80.!') >= 0",
"targeting":
f'[userId, "{experiment.recipe_slug}"]'
"|bucketSample(0, 100, 10000) "
"&& localeLanguageCode == 'en' && region == 'US' "
"&& browserSettings.update.channel == 'release'",
"enabled":
True,
},
)
self.assertDictEqual(
dict(arguments),
{
"userFacingName": experiment.name,
"userFacingDescription": experiment.public_description,
"slug": experiment.recipe_slug,
"active": True,
"isEnrollmentPaused": False,
"endDate": None,
"proposedEnrollment": experiment.proposed_enrollment,
"features": features,
"referenceBranch": "control",
"startDate": None,
"bucketConfig": {
"count": experiment.bucket.count,
"namespace": experiment.bucket.namespace.name,
"randomizationUnit": "userId",
"start": experiment.bucket.start,
"total": experiment.bucket.namespace.total,
},
},
)
converted_branches = [dict(branch) for branch in branches]
self.assertEqual(
converted_branches,
[
{
"ratio": 1,
"slug": "treatment",
"value": None
},
{
"ratio": 1,
"slug": "control",
"value": None
},
],
)
def test_update_change_log_creates_log_with_correct_change(self):
experiment = ExperimentFactory.create()
variant = ExperimentVariantFactory.create(
experiment=experiment,
ratio=100,
description="it's a description",
name="variant name",
)
variant_data = {
"ratio": 100,
"description": variant.description,
"name": variant.name,
}
changed_values = {
"variants": {
"new_value": {
"variants": [variant_data]
},
"old_value": None,
"display_name": "Branches",
}
}
ExperimentChangeLog.objects.create(
experiment=experiment,
changed_by=UserFactory(),
old_status=Experiment.STATUS_DRAFT,
new_status=Experiment.STATUS_DRAFT,
changed_values=changed_values,
message="",
)
self.assertEqual(experiment.changes.count(), 1)
change_data = {
"variants": [{
"id": variant.id,
"ratio": 100,
"description": "some other description",
"name": "some other name",
"is_control": False,
}]
}
serializer = ExperimentDesignBaseSerializer(
instance=experiment,
data=change_data,
context={"request": self.request})
self.assertTrue(serializer.is_valid())
experiment = serializer.save()
serializer_variant_data = ExperimentVariantSerializer(variant).data
self.assertEqual(experiment.changes.count(), 2)
changed_values = experiment.changes.latest().changed_values
variant = ExperimentVariant.objects.get(id=variant.id)
changed_serializer_variant_data = ExperimentVariantSerializer(
variant).data
self.assertIn("variants", changed_values)
self.assertEqual(changed_values["variants"]["old_value"],
[serializer_variant_data])
self.assertEqual(changed_values["variants"]["new_value"],
[changed_serializer_variant_data])
def test_generate_change_log_gives_correct_output(self):
experiment = ExperimentFactory.create_with_status(
target_status=Experiment.STATUS_REVIEW,
num_variants=0,
short_description="description",
qa_status="pretty good",
firefox_min_version="55.0",
)
variant1 = ExperimentVariantFactory.create(
experiment=experiment,
ratio=75,
description="variant1 description",
name="variant1",
slug="variant1-slug",
)
variant1.save()
old_serialized_val = ChangeLogSerializer(experiment).data
experiment.short_description = "changing the description"
experiment.qa_status = "good"
experiment.firefox_min_version = "56.0"
variant2 = ExperimentVariantFactory.create(
experiment=experiment,
ratio=25,
description="variant2 description",
name="variant2",
slug="variant2-slug",
)
variant2.save()
VariantPreferencesFactory.create(
variant=variant2,
pref_name="p1",
pref_type=Experiment.PREF_TYPE_INT,
pref_branch=Experiment.PREF_BRANCH_DEFAULT,
pref_value="5",
)
experiment.save()
new_serialized_val = ChangeLogSerializer(experiment).data
changed_variant_pref = {
"pref_name": "p1",
"pref_type": "integer",
"pref_branch": "default",
"pref_value": "5",
}
changed_data = {
"short_description":
"changing the description",
"qa_status":
"good",
"firefox_min_version":
"56.0",
"variants": [{
"ratio": 25,
"description": "variant2 description",
"name": "variant2",
"slug": "variant2-slug",
"preferences": [changed_variant_pref],
}],
}
user = UserFactory.create()
generate_change_log(old_serialized_val, new_serialized_val, experiment,
changed_data, user)
changed_value = experiment.changes.latest().changed_values
expected_changed_value = {
"firefox_min_version": {
"display_name": "Firefox Min Version",
"new_value": "56.0",
"old_value": "55.0",
},
"qa_status": {
"display_name": "Qa Status",
"new_value": "good",
"old_value": "pretty good",
},
"short_description": {
"display_name": "Short Description",
"new_value": "changing the description",
"old_value": "description",
},
"variants": {
"display_name":
"Branches",
"new_value": [
{
"ratio": 25,
"description": "variant2 description",
"name": "variant2",
"slug": "variant2-slug",
},
{
"ratio": 75,
"description": "variant1 description",
"name": "variant1",
"slug": "variant1-slug",
},
],
"old_value": [{
"ratio": 75,
"description": "variant1 description",
"name": "variant1",
"slug": "variant1-slug",
}],
},
}
self.assertEqual(
expected_changed_value["firefox_min_version"],
changed_value["firefox_min_version"],
)
self.assertEqual(expected_changed_value["qa_status"],
changed_value["qa_status"])
self.assertEqual(
expected_changed_value["short_description"],
changed_value["short_description"],
)
self.assertCountEqual(expected_changed_value["variants"],
changed_value["variants"])
def test_serializer_updates_experiment_and_creates_changelog(self):
owner = UserFactory(email="*****@*****.**")
experiment = ExperimentRapidFactory.create_with_status(
Experiment.STATUS_DRAFT,
owner=owner,
name="rapid experiment",
slug="rapid-experiment",
objectives="gotta go fast",
audience="us_only",
features=["picture_in_picture", "pinned_tabs"],
firefox_channel=Experiment.CHANNEL_RELEASE,
firefox_min_version="79.0",
)
experiment.variants.all().delete()
variant = ExperimentVariantFactory.create(experiment=experiment)
self.assertEqual(experiment.changes.count(), 1)
data = {
"name": "changing the name",
"objectives": "changing objectives",
"audience": "all_english",
"features": ["pinned_tabs"],
"firefox_channel": Experiment.CHANNEL_NIGHTLY,
"firefox_min_version": "80.0",
"variants": [
{
"id": variant.id,
"name": "something else",
"description": "something",
"is_control": True,
"ratio": 50,
"value": "something",
},
{
"name": "variant1",
"description": "variant1 description",
"is_control": False,
"ratio": 50,
"value": "variant value",
},
],
}
serializer = ExperimentRapidSerializer(
instance=experiment, data=data, context={"request": self.request}
)
self.assertTrue(serializer.is_valid())
experiment = serializer.save()
self.assertEqual(experiment.changes.count(), 2)
changelog = experiment.changes.latest()
self.assertEqual(changelog.old_status, Experiment.STATUS_DRAFT)
self.assertEqual(changelog.new_status, Experiment.STATUS_DRAFT)
self.assertEqual(
set(changelog.changed_values.keys()),
set(
[
"audience",
"features",
"firefox_channel",
"firefox_min_version",
"name",
"objectives",
"public_description",
"variants",
]
),
)
