def test_update_bugzilla_pref_experiment(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT,
name="An Experiment",
bugzilla_id="123",
type=Experiment.TYPE_PREF,
firefox_min_version="55.0",
firefox_max_version="56.0",
firefox_channel="Beta",
)
update_experiment_bug(experiment)
summary = "[Experiment] Pref-Flip Experiment: An Experiment Fx 55.0 to 56.0 Beta"
self.mock_bugzilla_requests_put.assert_called_with(
settings.BUGZILLA_UPDATE_URL.format(id=experiment.bugzilla_id),
{
"summary": summary,
"cf_user_story": format_bug_body(experiment)
},
)
def test_form_saves_results(self):
created_experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_COMPLETE)
self.assertEqual(created_experiment.changes.count(), 6)
data = self.valid_data.copy()
form = ExperimentResultsForm(request=self.request,
data=data,
instance=created_experiment)
self.assertTrue(form.is_valid())
experiment = form.save()
self.assertEqual(experiment.results_url, "https://example.com")
self.assertEqual(experiment.results_initial,
"Initially, all went well.")
self.assertTrue(experiment.results_recipe_errors)
self.assertTrue(experiment.results_confidence)
self.assertEqual(experiment.results_failures_notes, "Bad.")
self.assertEqual(experiment.changes.count(), 7)
def test_view_updates_status_and_redirects(self):
user_email = "*****@*****.**"
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT)
new_status = experiment.STATUS_REVIEW
response = self.client.post(
reverse("experiments-status-update",
kwargs={"slug": experiment.slug}),
{"status": new_status},
**{settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertRedirects(
response,
reverse("experiments-detail", kwargs={"slug": experiment.slug}),
fetch_redirect_response=False,
)
updated_experiment = Experiment.objects.get(slug=experiment.slug)
self.assertEqual(updated_experiment.status, new_status)
def test_form_saves_objectives(self):
created_experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT
)
data = {
"objectives": "The objective is to experiment!",
"analysis_owner": "Jim Bob The Data Scientist",
"analysis": "Lets analyze the results!",
}
form = ExperimentObjectivesForm(
request=self.request, data=data, instance=created_experiment
)
self.assertTrue(form.is_valid())
experiment = form.save()
self.assertEqual(experiment.objectives, data["objectives"])
self.assertEqual(experiment.analysis, data["analysis"])
def test_update_bugzilla_addon_experiment(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT,
name="An Experiment",
bugzilla_id="123",
type=Experiment.TYPE_ADDON,
firefox_min_version="56.0",
firefox_max_version="",
firefox_channel="Nightly",
)
update_experiment_bug(experiment)
summary = (
"[Experiment] Add-On Experiment: An Experiment Fx 56.0 Nightly")
self.mock_bugzilla_requests_put.assert_called_with(
settings.BUGZILLA_UPDATE_URL.format(id=experiment.bugzilla_id),
{
"summary": summary,
"cf_user_story": format_bug_body(experiment)
},
)
def test_optional_reviews(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_REVIEW, review_relman=True, review_science=True
)
form = ExperimentReviewForm(
request=self.request, data={}, instance=experiment
)
self.assertEqual(
form.optional_reviews,
(
form["review_legal"],
form["review_ux"],
form["review_security"],
form["review_vp"],
form["review_data_steward"],
form["review_comms"],
form["review_impacted_teams"],
),
)
def test_valid_recipe_id_updates_experiment_status(self):
user_email = "*****@*****.**"
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_SHIP)
normandy_id = 123
response = self.client.post(
reverse("experiments-normandy-update",
kwargs={"slug": experiment.slug}),
{"normandy_id": normandy_id},
**{settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertRedirects(
response,
reverse("experiments-detail", kwargs={"slug": experiment.slug}),
fetch_redirect_response=False,
)
experiment = Experiment.objects.get(id=experiment.id)
self.assertEqual(experiment.normandy_id, normandy_id)
self.assertEqual(experiment.status, Experiment.STATUS_ACCEPTED)
def test_creating_pref_bugzilla_ticket_returns_ticket_id(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT,
name="An Experiment",
firefox_min_version="56.0",
firefox_max_version="57.0",
)
response_data = create_experiment_bug(experiment)
self.assertEqual(response_data, self.bugzilla_id)
self.mock_bugzilla_requests_post.assert_called_with(
settings.BUGZILLA_CREATE_URL,
{
"product":
"Shield",
"component":
"Shield Study",
"version":
"unspecified",
"summary":
"[Experiment]: {experiment}".format(experiment=experiment),
"description":
experiment.BUGZILLA_OVERVIEW_TEMPLATE.format(
experiment=experiment),
"cc":
settings.BUGZILLA_CC_LIST,
"type":
"task",
"priority":
"P3",
"assigned_to":
experiment.owner.email,
"see_also": [12345],
"blocks": [12345],
"url":
experiment.experiment_url,
},
)
def test_create_bugzilla_ticket_creation_with_bad_assigned_to_val(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT, name="An Experiment")
self.mock_bugzilla_requests_get.side_effect = [
self.buildMockFailureResponse(),
self.buildMockSuccessBugResponse(),
]
response_data = create_experiment_bug(experiment)
self.assertEqual(response_data, self.bugzilla_id)
expected_call_data = {
"product":
"Shield",
"component":
"Shield Study",
"version":
"unspecified",
"summary":
"[Experiment]: {experiment}".format(experiment=experiment),
"description":
experiment.BUGZILLA_OVERVIEW_TEMPLATE.format(
experiment=experiment),
"cc":
settings.BUGZILLA_CC_LIST,
"type":
"task",
"priority":
"P3",
"assigned_to":
None,
"blocks": [12345],
"url":
experiment.experiment_url,
}
self.mock_bugzilla_requests_post.assert_called_with(
settings.BUGZILLA_CREATE_URL, expected_call_data)
def test_view_saves_experiment(self):
user_email = "*****@*****.**"
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT)
new_start_date = datetime.date.today() + datetime.timedelta(
days=random.randint(1, 100))
new_end_date = new_start_date + datetime.timedelta(
days=random.randint(1, 100))
data = {
"type": Experiment.TYPE_PREF,
"name": "A new name!",
"short_description": "A new description!",
"proposed_start_date": new_start_date,
"proposed_end_date": new_end_date,
}
response = self.client.post(
reverse("experiments-overview-update",
kwargs={"slug": experiment.slug}),
data,
**{settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 302)
experiment = Experiment.objects.get()
self.assertEqual(experiment.name, data["name"])
self.assertEqual(experiment.short_description,
data["short_description"])
self.assertEqual(experiment.proposed_start_date, new_start_date)
self.assertEqual(experiment.proposed_end_date, new_end_date)
self.assertEqual(experiment.changes.count(), 2)
change = experiment.changes.latest()
self.assertEqual(change.changed_by.email, user_email)
self.assertEqual(change.old_status, experiment.STATUS_DRAFT)
self.assertEqual(change.new_status, experiment.STATUS_DRAFT)
def test_form_saves_risks(self):
created_experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT
)
data = {
"risk_partner_related": True,
"risk_brand": True,
"risk_fast_shipped": True,
"risk_confidential": True,
"risk_release_population": True,
"risk_technical": True,
"risk_technical_description": "It's complicated",
"risks": "There are some risks",
"testing": "Always be sure to test!",
"test_builds": "Latest build",
"qa_status": "It ain't easy being green",
}
form = ExperimentRisksForm(
request=self.request, data=data, instance=created_experiment
)
self.assertTrue(form.is_valid())
experiment = form.save()
self.assertTrue(experiment.risk_partner_related)
self.assertTrue(experiment.risk_brand)
self.assertTrue(experiment.risk_fast_shipped)
self.assertTrue(experiment.risk_confidential)
self.assertTrue(experiment.risk_release_population)
self.assertTrue(experiment.risk_technical)
self.assertEqual(
experiment.risk_technical_description,
data["risk_technical_description"],
)
self.assertEqual(experiment.risks, data["risks"])
self.assertEqual(experiment.testing, data["testing"])
self.assertEqual(experiment.test_builds, data["test_builds"])
self.assertEqual(experiment.qa_status, data["qa_status"])
def test_required_reviews(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_REVIEW, review_relman=True, review_science=True
)
form = ExperimentReviewForm(
request=self.request, data={}, instance=experiment
)
self.assertEqual(
form.required_reviews,
(
form["review_advisory"],
form["review_science"],
form["review_engineering"],
form["review_qa_requested"],
form["review_intent_to_ship"],
form["review_bugzilla"],
form["review_qa"],
form["review_relman"],
),
)
def test_view_redirects_to_detail_page_when_form_is_invalid(self):
user_email = "*****@*****.**"
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT
)
section = "invalid section"
text = ""
response = self.client.post(
reverse(
"experiments-comment-create", kwargs={"slug": experiment.slug}
),
{"experiment": experiment.id, "section": section, "text": text},
**{settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertRedirects(
response,
reverse("experiments-detail", kwargs={"slug": experiment.slug}),
fetch_redirect_response=False,
)
def test_serializer_outputs_expected_schema_for_windows_versions(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_SHIP,
firefox_min_version="65.0",
type=Experiment.TYPE_PREF,
locales=[LocaleFactory.create()],
countries=[CountryFactory.create()],
windows_versions=[Experiment.VERSION_WINDOWS_8],
)
serializer = ExperimentRecipeSerializer(experiment)
self.assertEqual(
serializer.data["filter_object"],
[
FilterObjectBucketSampleSerializer(experiment).data,
FilterObjectChannelSerializer(experiment).data,
FilterObjectVersionsSerializer(experiment).data,
FilterObjectLocaleSerializer(experiment).data,
FilterObjectCountrySerializer(experiment).data,
],
)
def test_form_saves_variants(self):
created_experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT, num_variants=0
)
form = ExperimentVariantsPrefForm(
request=self.request, data=self.data, instance=created_experiment
)
self.assertTrue(form.is_valid())
experiment = form.save()
self.assertEqual(experiment.variants.count(), 3)
branch0 = experiment.variants.get(name=self.branch0_name)
self.assertTrue(branch0.is_control)
self.assertEqual(branch0.ratio, 34)
self.assertEqual(
branch0.description, self.data["variants-0-description"]
)
self.assertEqual(branch0.value, self.data["variants-0-value"])
branch1 = experiment.variants.get(name=self.branch1_name)
self.assertFalse(branch1.is_control)
self.assertEqual(branch1.ratio, 33)
self.assertEqual(
branch1.description, self.data["variants-1-description"]
)
self.assertEqual(branch1.value, self.data["variants-1-value"])
branch2 = experiment.variants.get(name=self.branch2_name)
self.assertFalse(branch2.is_control)
self.assertEqual(branch2.ratio, 33)
self.assertEqual(
branch2.description, self.data["variants-2-description"]
)
self.assertEqual(branch2.value, self.data["variants-2-value"])
def test_view_saves_experiment(self):
user_email = "*****@*****.**"
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_COMPLETE)
data = {
"results_url": "https://example.com",
"results_lessons_learned": "Many lessons were learned.",
}
response = self.client.post(
reverse("experiments-results-update",
kwargs={"slug": experiment.slug}),
data,
**{settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 302)
experiment = Experiment.objects.get()
self.assertEqual(experiment.results_url, "https://example.com")
self.assertEqual(experiment.results_lessons_learned,
"Many lessons were learned.")
def test_serializer_outputs_expected_schema(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_SHIP)
serializer = ExperimentRecipePrefArgumentsSerializer(experiment)
self.assertEqual(
serializer.data,
{
"preferenceBranchType":
experiment.pref_branch,
"slug":
experiment.normandy_slug,
"experimentDocumentUrl":
experiment.experiment_url,
"preferenceName":
experiment.pref_key,
"preferenceType":
experiment.pref_type,
"branches": [
ExperimentRecipeVariantSerializer(variant).data
for variant in experiment.variants.all()
],
},
)
def test_ship_experiment_not_updated(self):
experiment = ExperimentFactory.create_with_status(
target_status=Experiment.STATUS_SHIP)
mock_response_data = {}
mock_response = mock.Mock()
mock_response.json = mock.Mock()
mock_response.json.return_value = mock_response_data
mock_response.raise_for_status.side_effect = None
mock_response.status_code = 404
self.mock_normandy_requests_get.return_value = mock_response
tasks.update_recipe_ids_to_experiments()
self.assertEqual(experiment.status, Experiment.STATUS_SHIP)
self.assertIsNone(experiment.normandy_id)
self.assertIsNone(experiment.other_normandy_ids)
self.assertFalse(
experiment.changes.filter(
old_status=Experiment.STATUS_SHIP,
new_status=Experiment.STATUS_ACCEPTED,
).exists())
def test_manager_returns_sections(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT)
risk_comment = ExperimentCommentFactory.create(
experiment=experiment, section=Experiment.SECTION_RISKS)
testing_comment = ExperimentCommentFactory.create(
experiment=experiment, section=Experiment.SECTION_TESTING)
self.assertIn(
risk_comment,
experiment.comments.sections[experiment.SECTION_RISKS],
)
self.assertIn(
testing_comment,
experiment.comments.sections[experiment.SECTION_TESTING],
)
self.assertNotIn(
risk_comment,
experiment.comments.sections[experiment.SECTION_TESTING],
)
self.assertNotIn(
testing_comment,
experiment.comments.sections[experiment.SECTION_RISKS],
)
def test_serializer_outputs_expected_schema_for_addon_experiment(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_SHIP,
firefox_min_version="63.0",
type=Experiment.TYPE_ADDON,
locales=[LocaleFactory.create()],
countries=[CountryFactory.create()],
platforms=[Experiment.PLATFORM_WINDOWS],
windows_versions=[Experiment.VERSION_WINDOWS_8],
)
serializer = ExperimentRecipeSerializer(experiment)
self.assertEqual(serializer.data["action_name"], "opt-out-study")
self.assertEqual(serializer.data["name"], experiment.name)
expected_comment = (
f"{experiment.client_matching}\n"
f"Platform: ['All Windows']\n"
f"Windows Versions: ['Windows 8']\n"
)
self.assertEqual(serializer.data["comment"], expected_comment)
self.assertEqual(
serializer.data["filter_object"],
[
FilterObjectBucketSampleSerializer(experiment).data,
FilterObjectChannelSerializer(experiment).data,
FilterObjectVersionsSerializer(experiment).data,
FilterObjectLocaleSerializer(experiment).data,
FilterObjectCountrySerializer(experiment).data,
],
)
self.assertDictEqual(
serializer.data["arguments"],
ExperimentRecipeAddonArgumentsSerializer(experiment).data,
)
self.assertEqual(serializer.data["experimenter_slug"], experiment.slug)
def test_create_bugzilla_ticket_retries_with_no_assignee(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT, name="An Experiment")
self.setUpMockBugzillaInvalidUser()
response_data = create_experiment_bug(experiment)
self.assertEqual(response_data, self.bugzilla_id)
self.assertEqual(self.mock_bugzilla_requests_post.call_count, 2)
expected_call_data = {
"product":
"Shield",
"component":
"Shield Experiment",
"version":
"unspecified",
"summary":
"[Shield] {experiment}".format(experiment=experiment),
"description":
experiment.BUGZILLA_OVERVIEW_TEMPLATE.format(
experiment=experiment),
"assigned_to":
experiment.owner.email,
"cc":
settings.BUGZILLA_CC_LIST,
}
self.mock_bugzilla_requests_post.assert_any_call(
settings.BUGZILLA_CREATE_URL, expected_call_data)
del expected_call_data["assigned_to"]
self.mock_bugzilla_requests_post.assert_any_call(
settings.BUGZILLA_CREATE_URL, expected_call_data)
def test_view_creates_comment_redirects_to_detail_page(self):
user_email = "*****@*****.**"
experiment = ExperimentFactory.create_with_status(Experiment.STATUS_DRAFT)
section = experiment.SECTION_OBJECTIVES
text = "Hello!"
response = self.client.post(
reverse("experiments-comment-create", kwargs={"slug": experiment.slug}),
{"experiment": experiment.id, "section": section, "text": text},
**{settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertRedirects(
response,
"{url}#{section}-comments".format(
url=reverse("experiments-detail", kwargs={"slug": experiment.slug}),
section=section,
),
fetch_redirect_response=False,
)
comment = experiment.comments.sections[section][0]
self.assertEqual(comment.text, text)
self.assertEqual(comment.created_by.email, user_email)
def test_serializer_outputs_expected_schema(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_COMPLETE,
countries=[],
locales=[],
normandy_slug="a-normandy-slug",
normandy_id=123,
other_normandy_ids=[],
results_fail_to_launch=False,
results_failures_notes="failure notes",
platforms=[Experiment.PLATFORM_LINUX],
)
# ensure expected_data has "string" if pref_type is json string
pref_type = PrefTypeField().to_representation(experiment.pref_type)
serializer = ExperimentSerializer(experiment)
expected_data = {
"client_matching":
experiment.client_matching,
"platforms":
experiment.platforms,
"end_date":
JSTimestampField().to_representation(experiment.end_date),
"experiment_url":
experiment.experiment_url,
"firefox_channel":
experiment.firefox_channel,
"firefox_min_version":
experiment.firefox_min_version,
"firefox_max_version":
experiment.firefox_max_version,
"name":
experiment.name,
"population":
experiment.population,
"population_percent":
"{0:.4f}".format(experiment.population_percent),
"pref_branch":
experiment.pref_branch,
"pref_name":
experiment.pref_name,
"pref_type":
pref_type,
"addon_experiment_id":
experiment.addon_experiment_id,
"addon_release_url":
experiment.addon_release_url,
"proposed_start_date":
JSTimestampField().to_representation(
experiment.proposed_start_date),
"proposed_enrollment":
experiment.proposed_enrollment,
"proposed_duration":
experiment.proposed_duration,
"public_description":
experiment.public_description,
"slug":
experiment.slug,
"start_date":
JSTimestampField().to_representation(experiment.start_date),
"status":
Experiment.STATUS_COMPLETE,
"type":
experiment.type,
"normandy_slug":
experiment.normandy_slug,
"normandy_id":
experiment.normandy_id,
"other_normandy_ids":
experiment.other_normandy_ids,
"variants": [
ExperimentVariantSerializer(variant).data
for variant in experiment.variants.all()
],
"locales": [],
"countries": [],
"changes": [
ExperimentChangeLogSerializer(change).data
for change in experiment.changes.all()
],
"results": {
"results_url": None,
"results_initial": None,
"results_lessons_learned": None,
"results_fail_to_launch": False,
"results_recipe_errors": None,
"results_restarts": None,
"results_low_enrollment": None,
"results_early_end": None,
"results_no_usable_data": None,
"results_failures_notes": "failure notes",
"results_changes_to_firefox": None,
"results_data_for_hypothesis": None,
"results_confidence": None,
"results_measure_impact": None,
"results_impact_notes": None,
},
"telemetry_event_category":
experiment.telemetry_event_category,
"telemetry_event_method":
experiment.telemetry_event_method,
"telemetry_event_object":
experiment.telemetry_event_object,
"telemetry_event_value":
experiment.telemetry_event_value,
}
self.assertEqual(set(serializer.data.keys()),
set(expected_data.keys()))
self.assertEqual(serializer.data, expected_data)
def test_create_bugzilla_ticket_creation_failure(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT, name="An Experiment")
self.setupMockBugzillaCreationFailure()
self.assertRaises(BugzillaError, create_experiment_bug, experiment)
def test_experiment_is_begun(self):
for status in Experiment.STATUS_LIVE, Experiment.STATUS_COMPLETE:
experiment = ExperimentFactory.create_with_status(status)
self.assertTrue(experiment.is_begun)
def test_serializer_outputs_expected_schema_for_accepted(self):
audience = "us_only"
features = ["pinned_tabs", "picture_in_picture"]
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_ACCEPTED,
audience=audience,
features=features,
firefox_channel=Experiment.CHANNEL_RELEASE,
firefox_min_version="80.0",
proposed_start_date=None,
proposed_duration=28,
proposed_enrollment=7,
rapid_type=Experiment.RAPID_AA,
type=Experiment.TYPE_RAPID,
)
experiment.variants.all().delete()
ExperimentVariantFactory.create(experiment=experiment,
ratio=1,
slug="control",
is_control=True)
ExperimentVariantFactory.create(experiment=experiment,
ratio=1,
slug="treatment",
is_control=False)
ExperimentBucketNamespace.request_namespace_buckets(
experiment.recipe_slug, experiment, 100)
serializer = ExperimentRapidRecipeSerializer(experiment)
data = serializer.data
arguments = data.pop("arguments")
branches = arguments.pop("branches")
self.assertDictEqual(
data,
{
"id":
experiment.recipe_slug,
"filter_expression":
"env.version|versionCompare('80.!') >= 0",
"targeting":
f'[userId, "{experiment.recipe_slug}"]'
"|bucketSample(0, 100, 10000) "
"&& localeLanguageCode == 'en' && region == 'US' "
"&& browserSettings.update.channel == 'release'",
"enabled":
True,
},
)
self.assertDictEqual(
dict(arguments),
{
"userFacingName": experiment.name,
"userFacingDescription": experiment.public_description,
"slug": experiment.recipe_slug,
"active": True,
"isEnrollmentPaused": False,
"endDate": None,
"proposedEnrollment": experiment.proposed_enrollment,
"features": features,
"referenceBranch": "control",
"startDate": None,
"bucketConfig": {
"count": experiment.bucket.count,
"namespace": experiment.bucket.namespace.name,
"randomizationUnit": "userId",
"start": experiment.bucket.start,
"total": experiment.bucket.namespace.total,
},
},
)
converted_branches = [dict(branch) for branch in branches]
self.assertEqual(
converted_branches,
[
{
"ratio": 1,
"slug": "treatment",
"value": None
},
{
"ratio": 1,
"slug": "control",
"value": None
},
],
)
def test_experiment_without_normandy_id(self):
ExperimentFactory.create_with_status(
target_status=Experiment.STATUS_ACCEPTED, normandy_id=None)
tasks.update_experiment_info()
self.mock_normandy_requests_get.assert_not_called()
def setUp(self):
super().setUp()
self.experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT, bugzilla_id=None)
def test_bugzilla_url_returns_none_if_bugzilla_id_not_set(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_DRAFT)
self.assertIsNone(experiment.bugzilla_url)
def test_completed_all_sections_true_when_complete(self):
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_REVIEW)
self.assertTrue(experiment.completed_all_sections)