def test_control_property_returns_experiment_control(self): experiment = ExperimentFactory.create_with_variants() control = ExperimentVariant.objects.get( experiment=experiment, is_control=True ) self.assertEqual(experiment.control, control)
def test_serializer_outputs_expected_schema(self): country1 = CountryFactory(code="CA", name="Canada") locale1 = LocaleFactory(code="da", name="Danish") project = ProjectFactory.create() experiment = ExperimentFactory.create( locales=[locale1], countries=[country1], projects=[project] ) related_exp = ExperimentFactory.create() experiment.related_to.add(related_exp) serializer = ChangeLogSerializer(experiment) risk_tech_description = experiment.risk_technical_description expected_data = { "type": experiment.type, "owner": experiment.owner.id, "name": experiment.name, "short_description": experiment.short_description, "related_work": experiment.related_work, "related_to": [related_exp.id], "proposed_start_date": str(experiment.proposed_start_date), "proposed_duration": experiment.proposed_duration, "proposed_enrollment": experiment.proposed_enrollment, "design": experiment.design, "addon_experiment_id": experiment.addon_experiment_id, "addon_release_url": experiment.addon_release_url, "pref_name": experiment.pref_name, "pref_type": experiment.pref_type, "pref_branch": experiment.pref_branch, "public_description": experiment.public_description, "population_percent": "{0:.4f}".format(experiment.population_percent), "firefox_min_version": experiment.firefox_min_version, "firefox_max_version": experiment.firefox_max_version, "firefox_channel": experiment.firefox_channel, "client_matching": experiment.client_matching, "locales": [{"code": "da", "name": "Danish"}], "countries": [{"code": "CA", "name": "Canada"}], "projects": [{"slug": project.slug}], "platforms": experiment.platforms, "windows_versions": experiment.windows_versions, "profile_age": experiment.profile_age, "objectives": experiment.objectives, "total_enrolled_clients": experiment.total_enrolled_clients, "analysis": experiment.analysis, "analysis_owner": experiment.analysis_owner.id, "survey_required": experiment.survey_required, "survey_urls": experiment.survey_urls, "survey_instructions": experiment.survey_instructions, "engineering_owner": experiment.engineering_owner, "bugzilla_id": experiment.bugzilla_id, "recipe_slug": experiment.recipe_slug, "normandy_id": experiment.normandy_id, "other_normandy_ids": experiment.other_normandy_ids, "data_science_issue_url": experiment.data_science_issue_url, "feature_bugzilla_url": experiment.feature_bugzilla_url, "risk_partner_related": experiment.risk_partner_related, "risk_brand": experiment.risk_brand, "risk_fast_shipped": experiment.risk_fast_shipped, "risk_confidential": experiment.risk_confidential, "risk_release_population": experiment.risk_release_population, "risk_revenue": experiment.risk_revenue, "risk_data_category": experiment.risk_data_category, "risk_external_team_impact": experiment.risk_external_team_impact, "risk_telemetry_data": experiment.risk_telemetry_data, "risk_ux": experiment.risk_ux, "risk_security": experiment.risk_security, "risk_revision": experiment.risk_revision, "risk_technical": experiment.risk_technical, "risk_technical_description": risk_tech_description, "risks": experiment.risks, "testing": experiment.testing, "test_builds": experiment.test_builds, "qa_status": experiment.qa_status, "review_science": experiment.review_science, "review_engineering": experiment.review_engineering, "review_qa_requested": experiment.review_qa_requested, "review_intent_to_ship": experiment.review_intent_to_ship, "review_bugzilla": experiment.review_bugzilla, "review_qa": experiment.review_qa, "review_relman": experiment.review_relman, "review_advisory": experiment.review_advisory, "review_legal": experiment.review_legal, "review_ux": experiment.review_ux, "review_security": experiment.review_security, "review_vp": experiment.review_vp, "review_data_steward": experiment.review_data_steward, "review_comms": experiment.review_comms, "review_impacted_teams": experiment.review_impacted_teams, "variants": [ { "description": variant.description, "is_control": variant.is_control, "name": variant.name, "ratio": variant.ratio, "slug": variant.slug, "value": variant.value, "addon_release_url": variant.addon_release_url, "preferences": [ { "pref_name": preference.pref_name, "pref_type": preference.pref_type, "pref_branch": preference.pref_branch, "pref_value": preference.pref_value, } for preference in variant.preferences.all() ], "message_targeting": variant.message_targeting, "message_threshold": variant.message_threshold, "message_triggers": variant.message_triggers, } for variant in experiment.variants.all() ], "results_url": experiment.results_url, "results_initial": experiment.results_initial, "results_lessons_learned": experiment.results_lessons_learned, "results_fail_to_launch": experiment.results_fail_to_launch, "results_recipe_errors": experiment.results_recipe_errors, "results_restarts": experiment.results_restarts, "results_low_enrollment": experiment.results_low_enrollment, "results_early_end": experiment.results_early_end, "results_no_usable_data": experiment.results_no_usable_data, "results_failures_notes": experiment.results_failures_notes, "results_changes_to_firefox": experiment.results_changes_to_firefox, "results_data_for_hypothesis": experiment.results_data_for_hypothesis, "results_confidence": experiment.results_confidence, "results_measure_impact": experiment.results_measure_impact, "results_impact_notes": experiment.results_impact_notes, "rollout_playbook": experiment.rollout_playbook, "rollout_type": experiment.rollout_type, "message_type": experiment.message_type, "message_template": experiment.message_template, "preferences": [ { "pref_name": preference.pref_name, "pref_type": preference.pref_type, "pref_value": preference.pref_value, } for preference in experiment.preferences.all() ], } self.assertEqual(set(serializer.data.keys()), set(expected_data.keys())) self.maxDiff = None self.assertEqual(serializer.data, expected_data)
def setUp(self): super().setUp() self.experiment = ExperimentFactory.create_with_status( Experiment.STATUS_DRAFT)
def test_bugzilla_url_returns_none_if_bugzilla_id_not_set(self): experiment = ExperimentFactory.create_with_status( Experiment.STATUS_DRAFT ) self.assertIsNone(experiment.bugzilla_url)
def test_view_saves_experiment(self): user_email = "*****@*****.**" experiment = ExperimentFactory.create_with_status( Experiment.STATUS_DRAFT) locale = LocaleFactory() country = CountryFactory() data = { "population_percent": "11", "firefox_min_version": Experiment.VERSION_CHOICES[-2][0], "firefox_max_version": Experiment.VERSION_CHOICES[-1][0], "firefox_channel": Experiment.CHANNEL_NIGHTLY, "client_matching": "New matching!", "platform": Experiment.PLATFORM_WINDOWS, "locales": [locale.code], "countries": [country.code], "public_name": "hello", "public_description": "description", "pref_key": "browser.test.example", "pref_type": Experiment.PREF_TYPE_STR, "pref_branch": Experiment.PREF_BRANCH_DEFAULT, "variants-TOTAL_FORMS": "3", "variants-INITIAL_FORMS": "0", "variants-MIN_NUM_FORMS": "0", "variants-MAX_NUM_FORMS": "1000", "variants-0-is_control": True, "variants-0-ratio": "34", "variants-0-name": "control name", "variants-0-description": "control desc", "variants-0-value": '"control value"', "variants-1-is_control": False, "variants-1-ratio": "33", "variants-1-name": "branch 1 name", "variants-1-description": "branch 1 desc", "variants-1-value": '"branch 1 value"', "variants-2-is_control": False, "variants-2-ratio": "33", "variants-2-name": "branch 2 name", "variants-2-description": "branch 2 desc", "variants-2-value": '"branch 2 value"', } response = self.client.post( reverse("experiments-variants-update", kwargs={"slug": experiment.slug}), data, **{settings.OPENIDC_EMAIL_HEADER: user_email}, ) self.assertEqual(response.status_code, 302) experiment = Experiment.objects.get() self.assertEqual( experiment.population_percent, decimal.Decimal(data["population_percent"]), ) self.assertEqual(experiment.firefox_min_version, data["firefox_min_version"]) self.assertEqual(experiment.firefox_max_version, data["firefox_max_version"]) self.assertEqual(experiment.firefox_channel, data["firefox_channel"]) self.assertEqual(experiment.platform, data["platform"]) self.assertEqual(experiment.pref_key, data["pref_key"]) self.assertEqual(experiment.pref_type, data["pref_type"]) self.assertEqual(experiment.pref_branch, data["pref_branch"]) self.assertTrue(locale in experiment.locales.all()) self.assertTrue(country in experiment.countries.all()) self.assertEqual(experiment.changes.count(), 2) change = experiment.changes.latest() self.assertEqual(change.changed_by.email, user_email) self.assertEqual(change.old_status, experiment.STATUS_DRAFT) self.assertEqual(change.new_status, experiment.STATUS_DRAFT)
def test_completed_all_sections_true_when_complete(self): experiment = ExperimentFactory.create_with_status( Experiment.STATUS_REVIEW ) self.assertTrue(experiment.completed_all_sections)
def test_experiment_url(self): experiment = ExperimentFactory.create(slug="experiment-slug") self.assertEqual( experiment.experiment_url, "https://localhost/experiments/experiment-slug/", )
def test_experiment_is_begun(self): for status in Experiment.STATUS_LIVE, Experiment.STATUS_COMPLETE: experiment = ExperimentFactory.create_with_status(status) self.assertTrue(experiment.is_begun)
def test_overview_is_not_complete_when_not_saved(self): experiment = ExperimentFactory.build() self.assertFalse(experiment.completed_overview)
def test_experiment_is_editable_as_review(self): experiment = ExperimentFactory.create_with_status( Experiment.STATUS_REVIEW ) self.assertTrue(experiment.is_editable)
def test_experient_is_not_editable_after_review(self): experiment = ExperimentFactory.create_with_status( Experiment.STATUS_SHIP ) self.assertFalse(experiment.is_editable)
def test_experiment_is_editable_as_draft(self): experiment = ExperimentFactory.create_with_status( Experiment.STATUS_DRAFT ) self.assertTrue(experiment.is_editable)
def test_ordered_changes_orders_by_date(self): experiment = ExperimentFactory.create() date1 = datetime.datetime.now() - datetime.timedelta(days=2) date2 = datetime.datetime.now() - datetime.timedelta(days=1) date3 = datetime.datetime.now() user1 = UserFactory.create() user2 = UserFactory.create() user3 = UserFactory.create() ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user1, changed_on=date1, message="a", ) ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user1, changed_on=date1, message="b", ) ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user1, changed_on=date1, message="b", ) ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user2, changed_on=date1, message="c", ) ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user2, changed_on=date2, message="d", ) ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user3, changed_on=date2, message="e", ) ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user3, changed_on=date2, message="f", ) ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user1, changed_on=date3, message="g", ) ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user1, changed_on=date3, message="h", ) ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user2, changed_on=date3, message="i", ) ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user3, changed_on=date3, message="j", ) expected_changes = { date1.date(): {user1: set(["a", "b"]), user2: set(["c"])}, date2.date(): {user2: set(["d"]), user3: set(["e", "f"])}, date3.date(): { user1: set(["g", "h"]), user2: set(["i"]), user3: set(["j"]), }, } ordered_dates = [date for date, changes in experiment.ordered_changes] self.assertEqual( ordered_dates, [date3.date(), date2.date(), date1.date()] ) day3_users = [ user for user, user_changes in experiment.ordered_changes[0][1] ] self.assertEqual(set(day3_users), set([user1, user2, user3])) day2_users = [ user for user, user_changes in experiment.ordered_changes[1][1] ] self.assertEqual(set(day2_users), set([user2, user3])) day1_users = [ user for user, user_changes in experiment.ordered_changes[2][1] ] self.assertEqual(set(day1_users), set([user1, user2])) for date, date_changes in experiment.ordered_changes: for user, user_changes in date_changes: self.assertEqual(user_changes, expected_changes[date][user])
def test_grouped_changes_groups_by_date_then_user(self): experiment = ExperimentFactory.create() date1 = datetime.datetime.now() - datetime.timedelta(days=2) date2 = datetime.datetime.now() - datetime.timedelta(days=1) date3 = datetime.datetime.now() user1 = UserFactory.create() user2 = UserFactory.create() user3 = UserFactory.create() change1 = ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user1, changed_on=date1 ) change2 = ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user1, changed_on=date1 ) change3 = ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user1, changed_on=date1 ) change4 = ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user2, changed_on=date1 ) change5 = ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user2, changed_on=date2 ) change6 = ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user3, changed_on=date2 ) change7 = ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user3, changed_on=date2 ) change8 = ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user1, changed_on=date3 ) change9 = ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user1, changed_on=date3 ) change10 = ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user2, changed_on=date3 ) change11 = ExperimentChangeLogFactory.create( experiment=experiment, changed_by=user3, changed_on=date3 ) self.assertEqual( set(experiment.grouped_changes.keys()), set([date1.date(), date2.date(), date3.date()]), ) self.assertEqual( set(experiment.grouped_changes[date1.date()].keys()), set([user1, user2]), ) self.assertEqual( set(experiment.grouped_changes[date2.date()].keys()), set([user2, user3]), ) self.assertEqual( set(experiment.grouped_changes[date3.date()].keys()), set([user1, user2, user3]), ) self.assertEqual( experiment.grouped_changes[date1.date()][user1], set([change1, change2, change3]), ) self.assertEqual( experiment.grouped_changes[date1.date()][user2], set([change4]) ) self.assertEqual( experiment.grouped_changes[date2.date()][user2], set([change5]) ) self.assertEqual( experiment.grouped_changes[date2.date()][user3], set([change6, change7]), ) self.assertEqual( experiment.grouped_changes[date3.date()][user1], set([change8, change9]), ) self.assertEqual( experiment.grouped_changes[date3.date()][user2], set([change10]) ) self.assertEqual( experiment.grouped_changes[date3.date()][user3], set([change11]) )
def test_completed_required_reviews_false_when_reviews_not_complete(self): experiment = ExperimentFactory.create() self.assertFalse(experiment.completed_required_reviews)
def test_overview_is_complete_when_saved(self): experiment = ExperimentFactory.create() self.assertTrue(experiment.completed_overview)
def test_completed_all_sections_false_when_incomplete(self): experiment = ExperimentFactory.create() self.assertFalse(experiment.completed_all_sections)
def test_population_is_not_complete_when_defaults_set(self): experiment = ExperimentFactory.create( population_percent=0.0, firefox_version="", firefox_channel="" ) self.assertFalse(experiment.completed_population)
def test_get_absolute_url(self): experiment = ExperimentFactory.create(slug="experiment-slug") self.assertEqual( experiment.get_absolute_url(), "/experiments/experiment-slug/" )
def test_population_is_complete_when_values_set(self): experiment = ExperimentFactory.create() self.assertTrue(experiment.completed_population)
def test_reject_url(self): experiment = ExperimentFactory.create(slug="experiment") self.assertEqual( experiment.reject_url, "https://localhost/api/v1/experiments/experiment/reject/", )
def test_variants_is_not_complete_when_no_variants_saved(self): experiment = ExperimentFactory.create() self.assertFalse(experiment.completed_variants)
def test_serializer_outputs_expected_schema(self): experiment = ExperimentFactory.create_with_status( Experiment.STATUS_COMPLETE, countries=[], locales=[]) serialized = ExperimentSerializer(experiment) expected_data = { "analysis": experiment.analysis, "analysis_owner": experiment.analysis_owner, "client_matching": experiment.client_matching, "platform": experiment.platform, "end_date": JSTimestampField().to_representation(experiment.end_date), "experiment_url": experiment.experiment_url, "firefox_channel": experiment.firefox_channel, "firefox_version": experiment.firefox_version, "name": experiment.name, "objectives": experiment.objectives, "population": experiment.population, "population_percent": "{0:.4f}".format(experiment.population_percent), "pref_branch": experiment.pref_branch, "pref_key": experiment.pref_key, "pref_type": experiment.pref_type, "addon_name": experiment.addon_name, "addon_experiment_id": experiment.addon_experiment_id, "addon_testing_url": experiment.addon_testing_url, "addon_release_url": experiment.addon_release_url, "proposed_start_date": JSTimestampField().to_representation( experiment.proposed_start_date), "proposed_enrollment": experiment.proposed_enrollment, "proposed_duration": experiment.proposed_duration, "short_description": experiment.short_description, "slug": experiment.slug, "start_date": JSTimestampField().to_representation(experiment.start_date), "type": experiment.type, "variants": [ ExperimentVariantSerializer(variant).data for variant in experiment.variants.all() ], "locales": [], "countries": [], } self.assertEqual(set(serialized.data.keys()), set(expected_data.keys())) self.assertEqual(serialized.data, expected_data)
def test_variants_is_complete_when_variants_saved(self): experiment = ExperimentFactory.create_with_variants() self.assertTrue(experiment.completed_variants)
def test_generate_change_log_gives_correct_output(self): experiment = ExperimentFactory.create_with_status( target_status=Experiment.STATUS_REVIEW, num_variants=0, short_description="description", qa_status="pretty good", firefox_min_version="55.0", ) variant1 = ExperimentVariantFactory.create( experiment=experiment, ratio=75, description="variant1 description", name="variant1", slug="variant1-slug", ) variant1.save() old_serialized_val = ChangeLogSerializer(experiment).data experiment.short_description = "changing the description" experiment.qa_status = "good" experiment.firefox_min_version = "56.0" variant2 = ExperimentVariantFactory.create( experiment=experiment, ratio=25, description="variant2 description", name="variant2", slug="variant2-slug", ) variant2.save() VariantPreferencesFactory.create( variant=variant2, pref_name="p1", pref_type=Experiment.PREF_TYPE_INT, pref_branch=Experiment.PREF_BRANCH_DEFAULT, pref_value="5", ) experiment.save() new_serialized_val = ChangeLogSerializer(experiment).data changed_variant_pref = { "pref_name": "p1", "pref_type": "integer", "pref_branch": "default", "pref_value": "5", } changed_data = { "short_description": "changing the description", "qa_status": "good", "firefox_min_version": "56.0", "variants": [ { "ratio": 25, "description": "variant2 description", "name": "variant2", "slug": "variant2-slug", "preferences": [changed_variant_pref], } ], } user = UserFactory.create() generate_change_log( old_serialized_val, new_serialized_val, experiment, changed_data, user ) changed_value = experiment.changes.latest().changed_values expected_changed_value = { "firefox_min_version": { "display_name": "Firefox Min Version", "new_value": "56.0", "old_value": "55.0", }, "qa_status": { "display_name": "Qa Status", "new_value": "good", "old_value": "pretty good", }, "short_description": { "display_name": "Short Description", "new_value": "changing the description", "old_value": "description", }, "variants": { "display_name": "Branches", "new_value": [ { "ratio": 25, "description": "variant2 description", "name": "variant2", "slug": "variant2-slug", }, { "ratio": 75, "description": "variant1 description", "name": "variant1", "slug": "variant1-slug", }, ], "old_value": [ { "ratio": 75, "description": "variant1 description", "name": "variant1", "slug": "variant1-slug", } ], }, } self.assertEqual( expected_changed_value["firefox_min_version"], changed_value["firefox_min_version"], ) self.assertEqual(expected_changed_value["qa_status"], changed_value["qa_status"]) self.assertEqual( expected_changed_value["short_description"], changed_value["short_description"], ) self.assertCountEqual( expected_changed_value["variants"], changed_value["variants"] )
def test_objectives_is_not_complete_with_still_default(self): experiment = ExperimentFactory.create( objectives=Experiment.OBJECTIVES_DEFAULT, analysis=Experiment.ANALYSIS_DEFAULT, ) self.assertFalse(experiment.completed_objectives)
def test_add_start_date_comment_task_failure(self): experiment = ExperimentFactory.create(normandy_id=12345) self.mock_bugzilla_requests_post.side_effect = RequestException with self.assertRaises(bugzilla.BugzillaError): tasks.add_start_date_comment_task(experiment.id)
def test_objectives_is_complete_with_non_defaults(self): experiment = ExperimentFactory.create( objectives="Some objectives!", analysis="Some analysis!" ) self.assertTrue(experiment.completed_objectives)
def test_list_filters_by_search_text(self): user_email = "*****@*****.**" exp_1 = ExperimentFactory.create_with_status( random.choice(Experiment.STATUS_CHOICES)[0], name="Experiment One Cat", short_description="", slug="exp-1", related_work="", addon_experiment_id="1", pref_name="", public_description="", objectives="", analysis="", engineering_owner="", bugzilla_id="4", recipe_slug="", ) exp_2 = ExperimentFactory.create_with_status( random.choice(Experiment.STATUS_CHOICES)[0], name="Experiment Two Cat", short_description="", slug="exp-2", related_work="", addon_experiment_id="2", pref_name="", public_description="", objectives="", analysis="", engineering_owner="", bugzilla_id="5", recipe_slug="", ) exp_3 = ExperimentFactory.create_with_status( random.choice(Experiment.STATUS_CHOICES)[0], name="Experiment Three Dog", short_description="", slug="exp-3", related_work="", addon_experiment_id="3", pref_name="", public_description="", objectives="", analysis="", engineering_owner="", bugzilla_id="6", recipe_slug="", ) first_response_context = self.client.get( "{url}?{params}".format(url=reverse("home"), params=urlencode({"search": "Cat"})), **{ settings.OPENIDC_EMAIL_HEADER: user_email }, ).context[0] second_response_context = self.client.get( "{url}?{params}".format(url=reverse("home"), params=urlencode({"search": "Dog"})), **{ settings.OPENIDC_EMAIL_HEADER: user_email }, ).context[0] self.assertEqual(set(first_response_context["experiments"]), set([exp_1, exp_2])) self.assertEqual(set(second_response_context["experiments"]), set([exp_3]))
def test_end_date_returns_proposed_end_date_if_change_is_missing(self): experiment = ExperimentFactory.create_with_variants() self.assertEqual(experiment.end_date, experiment.proposed_end_date)