def test_message_experiment_sets_default_locales_countries(self):
[
LocaleFactory.create(code=locale)
for locale in Experiment.MESSAGE_DEFAULT_LOCALES
]
[
CountryFactory.create(code=country)
for country in Experiment.MESSAGE_DEFAULT_COUNTRIES
]
self.data["type"] = Experiment.TYPE_MESSAGE
form = ExperimentOverviewForm(request=self.request, data=self.data)
self.assertTrue(form.is_valid())
experiment = form.save()
self.assertEqual(
set(experiment.locales.values_list("code", flat=True)),
set(Experiment.MESSAGE_DEFAULT_LOCALES),
)
self.assertEqual(
set(experiment.countries.values_list("code", flat=True)),
set(Experiment.MESSAGE_DEFAULT_COUNTRIES),
)
def test_serializer_outputs_expected_schema_for_addon_experiment(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_SHIP,
firefox_min_version="63.0",
type=Experiment.TYPE_ADDON,
locales=[LocaleFactory.create()],
countries=[CountryFactory.create()],
platforms=[Experiment.PLATFORM_WINDOWS],
windows_versions=[Experiment.VERSION_WINDOWS_8],
)
serializer = ExperimentRecipeSerializer(experiment)
self.assertEqual(serializer.data["action_name"], "opt-out-study")
self.assertEqual(serializer.data["name"], experiment.name)
expected_comment = (
f"{experiment.client_matching}\n"
f"Platform: ['All Windows']\n"
f"Windows Versions: ['Windows 8']\n"
)
self.assertEqual(serializer.data["comment"], expected_comment)
self.assertEqual(
serializer.data["filter_object"],
[
FilterObjectBucketSampleSerializer(experiment).data,
FilterObjectChannelSerializer(experiment).data,
FilterObjectVersionsSerializer(experiment).data,
FilterObjectLocaleSerializer(experiment).data,
FilterObjectCountrySerializer(experiment).data,
],
)
self.assertDictEqual(
serializer.data["arguments"],
ExperimentRecipeAddonArgumentsSerializer(experiment).data,
)
self.assertEqual(serializer.data["experimenter_slug"], experiment.slug)
def test_serializer_outputs_expected_schema(self):
country1 = CountryFactory(code="CA", name="Canada")
locale1 = LocaleFactory(code="da", name="Danish")
project = ProjectFactory.create()
experiment = ExperimentFactory.create(locales=[locale1],
countries=[country1],
projects=[project])
related_exp = ExperimentFactory.create()
experiment.related_to.add(related_exp)
serializer = ChangeLogSerializer(experiment)
risk_tech_description = experiment.risk_technical_description
expected_data = {
"type":
experiment.type,
"owner":
experiment.owner.id,
"name":
experiment.name,
"short_description":
experiment.short_description,
"related_work":
experiment.related_work,
"related_to": [related_exp.id],
"proposed_start_date":
str(experiment.proposed_start_date),
"proposed_duration":
experiment.proposed_duration,
"proposed_enrollment":
experiment.proposed_enrollment,
"design":
experiment.design,
"addon_experiment_id":
experiment.addon_experiment_id,
"addon_release_url":
experiment.addon_release_url,
"pref_name":
experiment.pref_name,
"pref_type":
experiment.pref_type,
"pref_branch":
experiment.pref_branch,
"public_description":
experiment.public_description,
"population_percent":
"{0:.4f}".format(experiment.population_percent),
"firefox_min_version":
experiment.firefox_min_version,
"firefox_max_version":
experiment.firefox_max_version,
"firefox_channel":
experiment.firefox_channel,
"client_matching":
experiment.client_matching,
"locales": [{
"code": "da",
"name": "Danish"
}],
"countries": [{
"code": "CA",
"name": "Canada"
}],
"projects": [{
"slug": project.slug
}],
"platforms":
experiment.platforms,
"windows_versions":
experiment.windows_versions,
"profile_age":
experiment.profile_age,
"objectives":
experiment.objectives,
"audience":
experiment.audience,
"features":
experiment.features,
"total_enrolled_clients":
experiment.total_enrolled_clients,
"analysis":
experiment.analysis,
"analysis_owner":
experiment.analysis_owner.id,
"survey_required":
experiment.survey_required,
"survey_urls":
experiment.survey_urls,
"survey_instructions":
experiment.survey_instructions,
"engineering_owner":
experiment.engineering_owner,
"bugzilla_id":
experiment.bugzilla_id,
"recipe_slug":
experiment.recipe_slug,
"normandy_id":
experiment.normandy_id,
"other_normandy_ids":
experiment.other_normandy_ids,
"data_science_issue_url":
experiment.data_science_issue_url,
"feature_bugzilla_url":
experiment.feature_bugzilla_url,
"risk_partner_related":
experiment.risk_partner_related,
"risk_brand":
experiment.risk_brand,
"risk_fast_shipped":
experiment.risk_fast_shipped,
"risk_confidential":
experiment.risk_confidential,
"risk_release_population":
experiment.risk_release_population,
"risk_revenue":
experiment.risk_revenue,
"risk_data_category":
experiment.risk_data_category,
"risk_external_team_impact":
experiment.risk_external_team_impact,
"risk_telemetry_data":
experiment.risk_telemetry_data,
"risk_ux":
experiment.risk_ux,
"risk_security":
experiment.risk_security,
"risk_revision":
experiment.risk_revision,
"risk_technical":
experiment.risk_technical,
"risk_technical_description":
risk_tech_description,
"risks":
experiment.risks,
"testing":
experiment.testing,
"test_builds":
experiment.test_builds,
"qa_status":
experiment.qa_status,
"review_science":
experiment.review_science,
"review_engineering":
experiment.review_engineering,
"review_qa_requested":
experiment.review_qa_requested,
"review_intent_to_ship":
experiment.review_intent_to_ship,
"review_bugzilla":
experiment.review_bugzilla,
"review_qa":
experiment.review_qa,
"review_relman":
experiment.review_relman,
"review_advisory":
experiment.review_advisory,
"review_legal":
experiment.review_legal,
"review_ux":
experiment.review_ux,
"review_security":
experiment.review_security,
"review_vp":
experiment.review_vp,
"review_data_steward":
experiment.review_data_steward,
"review_comms":
experiment.review_comms,
"review_impacted_teams":
experiment.review_impacted_teams,
"variants": [{
"description":
variant.description,
"is_control":
variant.is_control,
"name":
variant.name,
"ratio":
variant.ratio,
"slug":
variant.slug,
"value":
variant.value,
"addon_release_url":
variant.addon_release_url,
"preferences": [{
"pref_name": preference.pref_name,
"pref_type": preference.pref_type,
"pref_branch": preference.pref_branch,
"pref_value": preference.pref_value,
} for preference in variant.preferences.all()],
"message_targeting":
variant.message_targeting,
"message_threshold":
variant.message_threshold,
"message_triggers":
variant.message_triggers,
} for variant in experiment.variants.all()],
"results_url":
experiment.results_url,
"results_initial":
experiment.results_initial,
"results_lessons_learned":
experiment.results_lessons_learned,
"results_fail_to_launch":
experiment.results_fail_to_launch,
"results_recipe_errors":
experiment.results_recipe_errors,
"results_restarts":
experiment.results_restarts,
"results_low_enrollment":
experiment.results_low_enrollment,
"results_early_end":
experiment.results_early_end,
"results_no_usable_data":
experiment.results_no_usable_data,
"results_failures_notes":
experiment.results_failures_notes,
"results_changes_to_firefox":
experiment.results_changes_to_firefox,
"results_data_for_hypothesis":
experiment.results_data_for_hypothesis,
"results_confidence":
experiment.results_confidence,
"results_measure_impact":
experiment.results_measure_impact,
"results_impact_notes":
experiment.results_impact_notes,
"rollout_playbook":
experiment.rollout_playbook,
"rollout_type":
experiment.rollout_type,
"message_type":
experiment.message_type,
"message_template":
experiment.message_template,
}
self.assertEqual(set(serializer.data.keys()),
set(expected_data.keys()))
self.assertEqual(serializer.data, expected_data)
def test_view_saves_experiment(self):
user_email = "*****@*****.**"
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT)
locale = LocaleFactory()
country = CountryFactory()
data = {
"population_percent": "11",
"firefox_version": Experiment.VERSION_CHOICES[-1][0],
"firefox_channel": Experiment.CHANNEL_NIGHTLY,
"client_matching": "New matching!",
"platform": Experiment.PLATFORM_WINDOWS,
"locales": [locale.code],
"countries": [country.code],
"public_name": "hello",
"public_description": "description",
"pref_key": "browser.test.example",
"pref_type": Experiment.PREF_TYPE_STR,
"pref_branch": Experiment.PREF_BRANCH_DEFAULT,
"variants-TOTAL_FORMS": "3",
"variants-INITIAL_FORMS": "0",
"variants-MIN_NUM_FORMS": "0",
"variants-MAX_NUM_FORMS": "1000",
"variants-0-is_control": True,
"variants-0-ratio": "34",
"variants-0-name": "control name",
"variants-0-description": "control desc",
"variants-0-value": '"control value"',
"variants-1-is_control": False,
"variants-1-ratio": "33",
"variants-1-name": "branch 1 name",
"variants-1-description": "branch 1 desc",
"variants-1-value": '"branch 1 value"',
"variants-2-is_control": False,
"variants-2-ratio": "33",
"variants-2-name": "branch 2 name",
"variants-2-description": "branch 2 desc",
"variants-2-value": '"branch 2 value"',
}
response = self.client.post(
reverse("experiments-variants-update",
kwargs={"slug": experiment.slug}),
data,
**{settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 302)
experiment = Experiment.objects.get()
self.assertEqual(
experiment.population_percent,
decimal.Decimal(data["population_percent"]),
)
self.assertEqual(experiment.firefox_version, data["firefox_version"])
self.assertEqual(experiment.firefox_channel, data["firefox_channel"])
self.assertEqual(experiment.platform, data["platform"])
self.assertEqual(experiment.pref_key, data["pref_key"])
self.assertEqual(experiment.pref_type, data["pref_type"])
self.assertEqual(experiment.pref_branch, data["pref_branch"])
self.assertTrue(locale in experiment.locales.all())
self.assertTrue(country in experiment.countries.all())
self.assertEqual(experiment.changes.count(), 2)
change = experiment.changes.latest()
self.assertEqual(change.changed_by.email, user_email)
self.assertEqual(change.old_status, experiment.STATUS_DRAFT)
self.assertEqual(change.new_status, experiment.STATUS_DRAFT)
def test_serializer_outputs_expected_schema_for_multipref(self):
experiment = ExperimentFactory.create(
name="public name",
firefox_min_version="70.0",
locales=[LocaleFactory.create()],
countries=[CountryFactory.create()],
public_description="this is my public description!",
normandy_slug="some-random-slug",
platforms=[
Experiment.PLATFORM_WINDOWS,
Experiment.PLATFORM_MAC,
Experiment.PLATFORM_LINUX,
],
is_multi_pref=True,
)
variant = ExperimentVariant(slug="slug-value",
ratio=25,
experiment=experiment,
is_control=True)
variant.save()
pref = VariantPreferencesFactory.create(variant=variant)
expected_comment = expected_comment = f"{experiment.client_matching}\n"
serializer = ExperimentRecipeSerializer(experiment)
self.assertEqual(serializer.data["action_name"],
"multi-preference-experiment")
self.assertEqual(serializer.data["name"], experiment.name)
self.assertEqual(serializer.data["comment"], expected_comment)
self.assertEqual(
serializer.data["filter_object"],
[
FilterObjectBucketSampleSerializer(experiment).data,
FilterObjectChannelSerializer(experiment).data,
FilterObjectVersionsSerializer(experiment).data,
FilterObjectLocaleSerializer(experiment).data,
FilterObjectCountrySerializer(experiment).data,
],
)
expected_data = {
"slug":
"some-random-slug",
"experimentDocumentUrl":
experiment.experiment_url,
"userFacingName":
"public name",
"userFacingDescription":
"this is my public description!",
"branches": [{
"preferences": {
pref.pref_name: {
"preferenceBranchType": pref.pref_branch,
"preferenceType": pref.pref_type,
"preferenceValue": pref.pref_value,
}
},
"ratio": 25,
"slug": "slug-value",
}],
}
self.assertDictEqual(serializer.data["arguments"], expected_data)
def test_serializer_outputs_expected_multipref_schema_for_singularpref(
self):
experiment = ExperimentFactory.create(
name="public name",
pref_type=Experiment.PREF_TYPE_INT,
pref_branch=Experiment.PREF_BRANCH_DEFAULT,
firefox_min_version="70.0",
locales=[LocaleFactory.create()],
countries=[CountryFactory.create()],
public_description="this is my public description!",
normandy_slug="some-random-slug",
platforms=[Experiment.PLATFORM_WINDOWS],
)
variant = ExperimentVariant(slug="slug-value",
ratio=25,
experiment=experiment,
value=5)
variant.save()
expected_comment = expected_comment = (
f"{experiment.client_matching}\n"
f"Platform: ['All Windows']\n")
serializer = ExperimentRecipeSerializer(experiment)
self.assertEqual(serializer.data["action_name"],
"multi-preference-experiment")
self.assertEqual(serializer.data["name"], experiment.name)
self.assertEqual(serializer.data["comment"], expected_comment)
self.assertEqual(
serializer.data["filter_object"],
[
FilterObjectBucketSampleSerializer(experiment).data,
FilterObjectChannelSerializer(experiment).data,
FilterObjectVersionsSerializer(experiment).data,
FilterObjectLocaleSerializer(experiment).data,
FilterObjectCountrySerializer(experiment).data,
],
)
expected_data = {
"slug":
"some-random-slug",
"experimentDocumentUrl":
experiment.experiment_url,
"userFacingName":
"public name",
"userFacingDescription":
"this is my public description!",
"branches": [{
"preferences": {
experiment.pref_name: {
"preferenceBranchType": "default",
"preferenceType": Experiment.PREF_TYPE_INT,
"preferenceValue": 5,
}
},
"ratio": 25,
"slug": "slug-value",
}],
}
self.assertDictEqual(serializer.data["arguments"], expected_data)