def test_control_property_returns_experiment_control(self):
experiment = ExperimentFactory.create_with_variants()
control = ExperimentVariant.objects.get(
experiment=experiment, is_control=True
)
self.assertEqual(experiment.control, control)
def test_serializer_outputs_expected_schema(self):
country1 = CountryFactory(code="CA", name="Canada")
locale1 = LocaleFactory(code="da", name="Danish")
project = ProjectFactory.create()
experiment = ExperimentFactory.create(
locales=[locale1], countries=[country1], projects=[project]
)
related_exp = ExperimentFactory.create()
experiment.related_to.add(related_exp)
serializer = ChangeLogSerializer(experiment)
risk_tech_description = experiment.risk_technical_description
expected_data = {
"type": experiment.type,
"owner": experiment.owner.id,
"name": experiment.name,
"short_description": experiment.short_description,
"related_work": experiment.related_work,
"related_to": [related_exp.id],
"proposed_start_date": str(experiment.proposed_start_date),
"proposed_duration": experiment.proposed_duration,
"proposed_enrollment": experiment.proposed_enrollment,
"design": experiment.design,
"addon_experiment_id": experiment.addon_experiment_id,
"addon_release_url": experiment.addon_release_url,
"pref_name": experiment.pref_name,
"pref_type": experiment.pref_type,
"pref_branch": experiment.pref_branch,
"public_description": experiment.public_description,
"population_percent": "{0:.4f}".format(experiment.population_percent),
"firefox_min_version": experiment.firefox_min_version,
"firefox_max_version": experiment.firefox_max_version,
"firefox_channel": experiment.firefox_channel,
"client_matching": experiment.client_matching,
"locales": [{"code": "da", "name": "Danish"}],
"countries": [{"code": "CA", "name": "Canada"}],
"projects": [{"slug": project.slug}],
"platforms": experiment.platforms,
"windows_versions": experiment.windows_versions,
"profile_age": experiment.profile_age,
"objectives": experiment.objectives,
"total_enrolled_clients": experiment.total_enrolled_clients,
"analysis": experiment.analysis,
"analysis_owner": experiment.analysis_owner.id,
"survey_required": experiment.survey_required,
"survey_urls": experiment.survey_urls,
"survey_instructions": experiment.survey_instructions,
"engineering_owner": experiment.engineering_owner,
"bugzilla_id": experiment.bugzilla_id,
"recipe_slug": experiment.recipe_slug,
"normandy_id": experiment.normandy_id,
"other_normandy_ids": experiment.other_normandy_ids,
"data_science_issue_url": experiment.data_science_issue_url,
"feature_bugzilla_url": experiment.feature_bugzilla_url,
"risk_partner_related": experiment.risk_partner_related,
"risk_brand": experiment.risk_brand,
"risk_fast_shipped": experiment.risk_fast_shipped,
"risk_confidential": experiment.risk_confidential,
"risk_release_population": experiment.risk_release_population,
"risk_revenue": experiment.risk_revenue,
"risk_data_category": experiment.risk_data_category,
"risk_external_team_impact": experiment.risk_external_team_impact,
"risk_telemetry_data": experiment.risk_telemetry_data,
"risk_ux": experiment.risk_ux,
"risk_security": experiment.risk_security,
"risk_revision": experiment.risk_revision,
"risk_technical": experiment.risk_technical,
"risk_technical_description": risk_tech_description,
"risks": experiment.risks,
"testing": experiment.testing,
"test_builds": experiment.test_builds,
"qa_status": experiment.qa_status,
"review_science": experiment.review_science,
"review_engineering": experiment.review_engineering,
"review_qa_requested": experiment.review_qa_requested,
"review_intent_to_ship": experiment.review_intent_to_ship,
"review_bugzilla": experiment.review_bugzilla,
"review_qa": experiment.review_qa,
"review_relman": experiment.review_relman,
"review_advisory": experiment.review_advisory,
"review_legal": experiment.review_legal,
"review_ux": experiment.review_ux,
"review_security": experiment.review_security,
"review_vp": experiment.review_vp,
"review_data_steward": experiment.review_data_steward,
"review_comms": experiment.review_comms,
"review_impacted_teams": experiment.review_impacted_teams,
"variants": [
{
"description": variant.description,
"is_control": variant.is_control,
"name": variant.name,
"ratio": variant.ratio,
"slug": variant.slug,
"value": variant.value,
"addon_release_url": variant.addon_release_url,
"preferences": [
{
"pref_name": preference.pref_name,
"pref_type": preference.pref_type,
"pref_branch": preference.pref_branch,
"pref_value": preference.pref_value,
}
for preference in variant.preferences.all()
],
"message_targeting": variant.message_targeting,
"message_threshold": variant.message_threshold,
"message_triggers": variant.message_triggers,
}
for variant in experiment.variants.all()
],
"results_url": experiment.results_url,
"results_initial": experiment.results_initial,
"results_lessons_learned": experiment.results_lessons_learned,
"results_fail_to_launch": experiment.results_fail_to_launch,
"results_recipe_errors": experiment.results_recipe_errors,
"results_restarts": experiment.results_restarts,
"results_low_enrollment": experiment.results_low_enrollment,
"results_early_end": experiment.results_early_end,
"results_no_usable_data": experiment.results_no_usable_data,
"results_failures_notes": experiment.results_failures_notes,
"results_changes_to_firefox": experiment.results_changes_to_firefox,
"results_data_for_hypothesis": experiment.results_data_for_hypothesis,
"results_confidence": experiment.results_confidence,
"results_measure_impact": experiment.results_measure_impact,
"results_impact_notes": experiment.results_impact_notes,
"rollout_playbook": experiment.rollout_playbook,
"rollout_type": experiment.rollout_type,
"message_type": experiment.message_type,
"message_template": experiment.message_template,
"preferences": [
{
"pref_name": preference.pref_name,
"pref_type": preference.pref_type,
"pref_value": preference.pref_value,
}
for preference in experiment.preferences.all()
],
}
self.assertEqual(set(serializer.data.keys()), set(expected_data.keys()))
self.maxDiff = None
self.assertEqual(serializer.data, expected_data)
def setUp(self):
super().setUp()
self.experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT)
def test_bugzilla_url_returns_none_if_bugzilla_id_not_set(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT
)
self.assertIsNone(experiment.bugzilla_url)
def test_view_saves_experiment(self):
user_email = "*****@*****.**"
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT)
locale = LocaleFactory()
country = CountryFactory()
data = {
"population_percent": "11",
"firefox_min_version": Experiment.VERSION_CHOICES[-2][0],
"firefox_max_version": Experiment.VERSION_CHOICES[-1][0],
"firefox_channel": Experiment.CHANNEL_NIGHTLY,
"client_matching": "New matching!",
"platform": Experiment.PLATFORM_WINDOWS,
"locales": [locale.code],
"countries": [country.code],
"public_name": "hello",
"public_description": "description",
"pref_key": "browser.test.example",
"pref_type": Experiment.PREF_TYPE_STR,
"pref_branch": Experiment.PREF_BRANCH_DEFAULT,
"variants-TOTAL_FORMS": "3",
"variants-INITIAL_FORMS": "0",
"variants-MIN_NUM_FORMS": "0",
"variants-MAX_NUM_FORMS": "1000",
"variants-0-is_control": True,
"variants-0-ratio": "34",
"variants-0-name": "control name",
"variants-0-description": "control desc",
"variants-0-value": '"control value"',
"variants-1-is_control": False,
"variants-1-ratio": "33",
"variants-1-name": "branch 1 name",
"variants-1-description": "branch 1 desc",
"variants-1-value": '"branch 1 value"',
"variants-2-is_control": False,
"variants-2-ratio": "33",
"variants-2-name": "branch 2 name",
"variants-2-description": "branch 2 desc",
"variants-2-value": '"branch 2 value"',
}
response = self.client.post(
reverse("experiments-variants-update",
kwargs={"slug": experiment.slug}),
data,
**{settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 302)
experiment = Experiment.objects.get()
self.assertEqual(
experiment.population_percent,
decimal.Decimal(data["population_percent"]),
)
self.assertEqual(experiment.firefox_min_version,
data["firefox_min_version"])
self.assertEqual(experiment.firefox_max_version,
data["firefox_max_version"])
self.assertEqual(experiment.firefox_channel, data["firefox_channel"])
self.assertEqual(experiment.platform, data["platform"])
self.assertEqual(experiment.pref_key, data["pref_key"])
self.assertEqual(experiment.pref_type, data["pref_type"])
self.assertEqual(experiment.pref_branch, data["pref_branch"])
self.assertTrue(locale in experiment.locales.all())
self.assertTrue(country in experiment.countries.all())
self.assertEqual(experiment.changes.count(), 2)
change = experiment.changes.latest()
self.assertEqual(change.changed_by.email, user_email)
self.assertEqual(change.old_status, experiment.STATUS_DRAFT)
self.assertEqual(change.new_status, experiment.STATUS_DRAFT)
def test_completed_all_sections_true_when_complete(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_REVIEW
)
self.assertTrue(experiment.completed_all_sections)
def test_experiment_url(self):
experiment = ExperimentFactory.create(slug="experiment-slug")
self.assertEqual(
experiment.experiment_url,
"https://localhost/experiments/experiment-slug/",
)
def test_experiment_is_begun(self):
for status in Experiment.STATUS_LIVE, Experiment.STATUS_COMPLETE:
experiment = ExperimentFactory.create_with_status(status)
self.assertTrue(experiment.is_begun)
def test_overview_is_not_complete_when_not_saved(self):
experiment = ExperimentFactory.build()
self.assertFalse(experiment.completed_overview)
def test_experiment_is_editable_as_review(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_REVIEW
)
self.assertTrue(experiment.is_editable)
def test_experient_is_not_editable_after_review(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_SHIP
)
self.assertFalse(experiment.is_editable)
def test_experiment_is_editable_as_draft(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT
)
self.assertTrue(experiment.is_editable)
def test_ordered_changes_orders_by_date(self):
experiment = ExperimentFactory.create()
date1 = datetime.datetime.now() - datetime.timedelta(days=2)
date2 = datetime.datetime.now() - datetime.timedelta(days=1)
date3 = datetime.datetime.now()
user1 = UserFactory.create()
user2 = UserFactory.create()
user3 = UserFactory.create()
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user1,
changed_on=date1,
message="a",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user1,
changed_on=date1,
message="b",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user1,
changed_on=date1,
message="b",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user2,
changed_on=date1,
message="c",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user2,
changed_on=date2,
message="d",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user3,
changed_on=date2,
message="e",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user3,
changed_on=date2,
message="f",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user1,
changed_on=date3,
message="g",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user1,
changed_on=date3,
message="h",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user2,
changed_on=date3,
message="i",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user3,
changed_on=date3,
message="j",
)
expected_changes = {
date1.date(): {user1: set(["a", "b"]), user2: set(["c"])},
date2.date(): {user2: set(["d"]), user3: set(["e", "f"])},
date3.date(): {
user1: set(["g", "h"]),
user2: set(["i"]),
user3: set(["j"]),
},
}
ordered_dates = [date for date, changes in experiment.ordered_changes]
self.assertEqual(
ordered_dates, [date3.date(), date2.date(), date1.date()]
)
day3_users = [
user for user, user_changes in experiment.ordered_changes[0][1]
]
self.assertEqual(set(day3_users), set([user1, user2, user3]))
day2_users = [
user for user, user_changes in experiment.ordered_changes[1][1]
]
self.assertEqual(set(day2_users), set([user2, user3]))
day1_users = [
user for user, user_changes in experiment.ordered_changes[2][1]
]
self.assertEqual(set(day1_users), set([user1, user2]))
for date, date_changes in experiment.ordered_changes:
for user, user_changes in date_changes:
self.assertEqual(user_changes, expected_changes[date][user])
def test_grouped_changes_groups_by_date_then_user(self):
experiment = ExperimentFactory.create()
date1 = datetime.datetime.now() - datetime.timedelta(days=2)
date2 = datetime.datetime.now() - datetime.timedelta(days=1)
date3 = datetime.datetime.now()
user1 = UserFactory.create()
user2 = UserFactory.create()
user3 = UserFactory.create()
change1 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user1, changed_on=date1
)
change2 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user1, changed_on=date1
)
change3 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user1, changed_on=date1
)
change4 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user2, changed_on=date1
)
change5 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user2, changed_on=date2
)
change6 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user3, changed_on=date2
)
change7 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user3, changed_on=date2
)
change8 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user1, changed_on=date3
)
change9 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user1, changed_on=date3
)
change10 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user2, changed_on=date3
)
change11 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user3, changed_on=date3
)
self.assertEqual(
set(experiment.grouped_changes.keys()),
set([date1.date(), date2.date(), date3.date()]),
)
self.assertEqual(
set(experiment.grouped_changes[date1.date()].keys()),
set([user1, user2]),
)
self.assertEqual(
set(experiment.grouped_changes[date2.date()].keys()),
set([user2, user3]),
)
self.assertEqual(
set(experiment.grouped_changes[date3.date()].keys()),
set([user1, user2, user3]),
)
self.assertEqual(
experiment.grouped_changes[date1.date()][user1],
set([change1, change2, change3]),
)
self.assertEqual(
experiment.grouped_changes[date1.date()][user2], set([change4])
)
self.assertEqual(
experiment.grouped_changes[date2.date()][user2], set([change5])
)
self.assertEqual(
experiment.grouped_changes[date2.date()][user3],
set([change6, change7]),
)
self.assertEqual(
experiment.grouped_changes[date3.date()][user1],
set([change8, change9]),
)
self.assertEqual(
experiment.grouped_changes[date3.date()][user2], set([change10])
)
self.assertEqual(
experiment.grouped_changes[date3.date()][user3], set([change11])
)
def test_completed_required_reviews_false_when_reviews_not_complete(self):
experiment = ExperimentFactory.create()
self.assertFalse(experiment.completed_required_reviews)
def test_overview_is_complete_when_saved(self):
experiment = ExperimentFactory.create()
self.assertTrue(experiment.completed_overview)
def test_completed_all_sections_false_when_incomplete(self):
experiment = ExperimentFactory.create()
self.assertFalse(experiment.completed_all_sections)
def test_population_is_not_complete_when_defaults_set(self):
experiment = ExperimentFactory.create(
population_percent=0.0, firefox_version="", firefox_channel=""
)
self.assertFalse(experiment.completed_population)
def test_get_absolute_url(self):
experiment = ExperimentFactory.create(slug="experiment-slug")
self.assertEqual(
experiment.get_absolute_url(), "/experiments/experiment-slug/"
)
def test_population_is_complete_when_values_set(self):
experiment = ExperimentFactory.create()
self.assertTrue(experiment.completed_population)
def test_reject_url(self):
experiment = ExperimentFactory.create(slug="experiment")
self.assertEqual(
experiment.reject_url,
"https://localhost/api/v1/experiments/experiment/reject/",
)
def test_variants_is_not_complete_when_no_variants_saved(self):
experiment = ExperimentFactory.create()
self.assertFalse(experiment.completed_variants)
def test_serializer_outputs_expected_schema(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_COMPLETE, countries=[], locales=[])
serialized = ExperimentSerializer(experiment)
expected_data = {
"analysis":
experiment.analysis,
"analysis_owner":
experiment.analysis_owner,
"client_matching":
experiment.client_matching,
"platform":
experiment.platform,
"end_date":
JSTimestampField().to_representation(experiment.end_date),
"experiment_url":
experiment.experiment_url,
"firefox_channel":
experiment.firefox_channel,
"firefox_version":
experiment.firefox_version,
"name":
experiment.name,
"objectives":
experiment.objectives,
"population":
experiment.population,
"population_percent":
"{0:.4f}".format(experiment.population_percent),
"pref_branch":
experiment.pref_branch,
"pref_key":
experiment.pref_key,
"pref_type":
experiment.pref_type,
"addon_name":
experiment.addon_name,
"addon_experiment_id":
experiment.addon_experiment_id,
"addon_testing_url":
experiment.addon_testing_url,
"addon_release_url":
experiment.addon_release_url,
"proposed_start_date":
JSTimestampField().to_representation(
experiment.proposed_start_date),
"proposed_enrollment":
experiment.proposed_enrollment,
"proposed_duration":
experiment.proposed_duration,
"short_description":
experiment.short_description,
"slug":
experiment.slug,
"start_date":
JSTimestampField().to_representation(experiment.start_date),
"type":
experiment.type,
"variants": [
ExperimentVariantSerializer(variant).data
for variant in experiment.variants.all()
],
"locales": [],
"countries": [],
}
self.assertEqual(set(serialized.data.keys()),
set(expected_data.keys()))
self.assertEqual(serialized.data, expected_data)
def test_variants_is_complete_when_variants_saved(self):
experiment = ExperimentFactory.create_with_variants()
self.assertTrue(experiment.completed_variants)
def test_generate_change_log_gives_correct_output(self):
experiment = ExperimentFactory.create_with_status(
target_status=Experiment.STATUS_REVIEW,
num_variants=0,
short_description="description",
qa_status="pretty good",
firefox_min_version="55.0",
)
variant1 = ExperimentVariantFactory.create(
experiment=experiment,
ratio=75,
description="variant1 description",
name="variant1",
slug="variant1-slug",
)
variant1.save()
old_serialized_val = ChangeLogSerializer(experiment).data
experiment.short_description = "changing the description"
experiment.qa_status = "good"
experiment.firefox_min_version = "56.0"
variant2 = ExperimentVariantFactory.create(
experiment=experiment,
ratio=25,
description="variant2 description",
name="variant2",
slug="variant2-slug",
)
variant2.save()
VariantPreferencesFactory.create(
variant=variant2,
pref_name="p1",
pref_type=Experiment.PREF_TYPE_INT,
pref_branch=Experiment.PREF_BRANCH_DEFAULT,
pref_value="5",
)
experiment.save()
new_serialized_val = ChangeLogSerializer(experiment).data
changed_variant_pref = {
"pref_name": "p1",
"pref_type": "integer",
"pref_branch": "default",
"pref_value": "5",
}
changed_data = {
"short_description": "changing the description",
"qa_status": "good",
"firefox_min_version": "56.0",
"variants": [
{
"ratio": 25,
"description": "variant2 description",
"name": "variant2",
"slug": "variant2-slug",
"preferences": [changed_variant_pref],
}
],
}
user = UserFactory.create()
generate_change_log(
old_serialized_val, new_serialized_val, experiment, changed_data, user
)
changed_value = experiment.changes.latest().changed_values
expected_changed_value = {
"firefox_min_version": {
"display_name": "Firefox Min Version",
"new_value": "56.0",
"old_value": "55.0",
},
"qa_status": {
"display_name": "Qa Status",
"new_value": "good",
"old_value": "pretty good",
},
"short_description": {
"display_name": "Short Description",
"new_value": "changing the description",
"old_value": "description",
},
"variants": {
"display_name": "Branches",
"new_value": [
{
"ratio": 25,
"description": "variant2 description",
"name": "variant2",
"slug": "variant2-slug",
},
{
"ratio": 75,
"description": "variant1 description",
"name": "variant1",
"slug": "variant1-slug",
},
],
"old_value": [
{
"ratio": 75,
"description": "variant1 description",
"name": "variant1",
"slug": "variant1-slug",
}
],
},
}
self.assertEqual(
expected_changed_value["firefox_min_version"],
changed_value["firefox_min_version"],
)
self.assertEqual(expected_changed_value["qa_status"], changed_value["qa_status"])
self.assertEqual(
expected_changed_value["short_description"],
changed_value["short_description"],
)
self.assertCountEqual(
expected_changed_value["variants"], changed_value["variants"]
)
def test_objectives_is_not_complete_with_still_default(self):
experiment = ExperimentFactory.create(
objectives=Experiment.OBJECTIVES_DEFAULT,
analysis=Experiment.ANALYSIS_DEFAULT,
)
self.assertFalse(experiment.completed_objectives)
def test_add_start_date_comment_task_failure(self):
experiment = ExperimentFactory.create(normandy_id=12345)
self.mock_bugzilla_requests_post.side_effect = RequestException
with self.assertRaises(bugzilla.BugzillaError):
tasks.add_start_date_comment_task(experiment.id)
def test_objectives_is_complete_with_non_defaults(self):
experiment = ExperimentFactory.create(
objectives="Some objectives!", analysis="Some analysis!"
)
self.assertTrue(experiment.completed_objectives)
def test_list_filters_by_search_text(self):
user_email = "*****@*****.**"
exp_1 = ExperimentFactory.create_with_status(
random.choice(Experiment.STATUS_CHOICES)[0],
name="Experiment One Cat",
short_description="",
slug="exp-1",
related_work="",
addon_experiment_id="1",
pref_name="",
public_description="",
objectives="",
analysis="",
engineering_owner="",
bugzilla_id="4",
recipe_slug="",
)
exp_2 = ExperimentFactory.create_with_status(
random.choice(Experiment.STATUS_CHOICES)[0],
name="Experiment Two Cat",
short_description="",
slug="exp-2",
related_work="",
addon_experiment_id="2",
pref_name="",
public_description="",
objectives="",
analysis="",
engineering_owner="",
bugzilla_id="5",
recipe_slug="",
)
exp_3 = ExperimentFactory.create_with_status(
random.choice(Experiment.STATUS_CHOICES)[0],
name="Experiment Three Dog",
short_description="",
slug="exp-3",
related_work="",
addon_experiment_id="3",
pref_name="",
public_description="",
objectives="",
analysis="",
engineering_owner="",
bugzilla_id="6",
recipe_slug="",
)
first_response_context = self.client.get(
"{url}?{params}".format(url=reverse("home"),
params=urlencode({"search": "Cat"})),
**{
settings.OPENIDC_EMAIL_HEADER: user_email
},
).context[0]
second_response_context = self.client.get(
"{url}?{params}".format(url=reverse("home"),
params=urlencode({"search": "Dog"})),
**{
settings.OPENIDC_EMAIL_HEADER: user_email
},
).context[0]
self.assertEqual(set(first_response_context["experiments"]),
set([exp_1, exp_2]))
self.assertEqual(set(second_response_context["experiments"]),
set([exp_3]))
def test_end_date_returns_proposed_end_date_if_change_is_missing(self):
experiment = ExperimentFactory.create_with_variants()
self.assertEqual(experiment.end_date, experiment.proposed_end_date)