def test_invalid_experiment_treatment_branch_requires_description(self): experiment = NimbusExperimentFactory.create_with_status( NimbusExperiment.Status.DRAFT, application=NimbusExperiment.Application.DESKTOP.value, feature_config=NimbusFeatureConfigFactory( application=NimbusExperiment.Application.DESKTOP.value), ) treatment_branch = NimbusBranchFactory.create(experiment=experiment, description="") experiment.branches.add(treatment_branch) experiment.save() serializer = NimbusReadyForReviewSerializer( experiment, data=NimbusReadyForReviewSerializer( experiment, context={ "user": self.user }, ).data, context={"user": self.user}, ) self.assertFalse(serializer.is_valid()) self.assertEqual( serializer.errors["treatment_branches"][1], ["Description cannot be blank."], )
def test_update_experiment_branches_with_feature_config(self): user_email = "*****@*****.**" feature = NimbusFeatureConfigFactory( schema="{}", application=NimbusExperiment.Application.FENIX ) experiment = NimbusExperimentFactory.create( status=NimbusExperiment.Status.DRAFT, application=NimbusExperiment.Application.FENIX, ) experiment_id = experiment.id reference_branch = {"name": "control", "description": "a control", "ratio": 1} treatment_branches = [{"name": "treatment1", "description": "desc1", "ratio": 1}] response = self.query( UPDATE_EXPERIMENT_MUTATION, variables={ "input": { "id": experiment.id, "featureConfigId": feature.id, "referenceBranch": reference_branch, "treatmentBranches": treatment_branches, "changelogMessage": "test changelog message", } }, headers={settings.OPENIDC_EMAIL_HEADER: user_email}, ) self.assertEqual(response.status_code, 200) experiment = NimbusExperiment.objects.get(id=experiment_id) self.assertEqual(experiment.feature_config, feature) self.assertEqual(experiment.branches.count(), 2) self.assertEqual(experiment.reference_branch.name, reference_branch["name"]) treatment_branch = experiment.treatment_branches[0] self.assertEqual(treatment_branch.name, treatment_branches[0]["name"])
def test_update_experiment_branches_with_feature_config(self): user_email = "*****@*****.**" feature = NimbusFeatureConfigFactory(schema="{}") experiment = NimbusExperimentFactory.create( status=NimbusExperiment.Status.DRAFT) experiment_id = experiment.id reference_branch = { "name": "control", "description": "a control", "ratio": 1 } treatment_branches = [{ "name": "treatment1", "description": "desc1", "ratio": 1 }] response = self.query( UPDATE_EXPERIMENT_BRANCHES_MUTATION, variables={ "input": { "nimbusExperimentId": experiment.id, "featureConfigId": feature.id, "clientMutationId": "randomid", "referenceBranch": reference_branch, "treatmentBranches": treatment_branches, } }, headers={settings.OPENIDC_EMAIL_HEADER: user_email}, ) self.assertEqual(response.status_code, 200) content = json.loads(response.content) result = content["data"]["updateExperimentBranches"] self.assertEqual( result["nimbusExperiment"], { "id": str(experiment.id), "featureConfig": { "name": feature.name }, "status": experiment.status.upper(), "name": experiment.name, "slug": experiment.slug, "referenceBranch": reference_branch, "treatmentBranches": treatment_branches, }, ) experiment = NimbusExperiment.objects.get(id=experiment_id) self.assertEqual(experiment.feature_config, feature) self.assertEqual(experiment.branches.count(), 2) self.assertEqual(experiment.reference_branch.name, reference_branch["name"]) treatment_branch = experiment.treatment_branches[0] self.assertEqual(treatment_branch.name, treatment_branches[0]["name"])
def test_valid_experiment(self): experiment = NimbusExperimentFactory.create_with_status( NimbusExperiment.Status.DRAFT, application=NimbusExperiment.Application.DESKTOP.value, feature_config=NimbusFeatureConfigFactory( application=NimbusExperiment.Application.DESKTOP.value), ) serializer = NimbusReadyForReviewSerializer( experiment, data=NimbusReadyForReviewSerializer( experiment, context={ "user": self.user }, ).data, context={"user": self.user}, ) self.assertTrue(serializer.is_valid())
def test_invalid_experiment_reference_branch_requires_description(self): experiment = NimbusExperimentFactory.create_with_status( NimbusExperiment.Status.DRAFT, application=NimbusExperiment.Application.DESKTOP.value, feature_config=NimbusFeatureConfigFactory( application=NimbusExperiment.Application.DESKTOP.value), ) experiment.reference_branch.description = "" experiment.save() serializer = NimbusReadyForReviewSerializer( experiment, data=NimbusReadyForReviewSerializer( experiment, context={ "user": self.user }, ).data, context={"user": self.user}, ) self.assertFalse(serializer.is_valid()) self.assertEqual( serializer.errors, {"reference_branch": ["Description cannot be blank."]}, )
def test_invalid_experiment_default_hypothesis(self): experiment = NimbusExperimentFactory.create_with_status( NimbusExperiment.Status.DRAFT, application=NimbusExperiment.Application.DESKTOP.value, feature_config=NimbusFeatureConfigFactory( application=NimbusExperiment.Application.DESKTOP.value), ) experiment.hypothesis = NimbusExperiment.HYPOTHESIS_DEFAULT experiment.save() serializer = NimbusReadyForReviewSerializer( experiment, data=NimbusReadyForReviewSerializer( experiment, context={ "user": self.user }, ).data, context={"user": self.user}, ) self.assertFalse(serializer.is_valid()) self.assertEqual( serializer.errors, {"hypothesis": ["Hypothesis cannot be the default value."]}, )
def test_outputs_expected_schema_for_complete_experiment(self): application = NimbusExperiment.Application.DESKTOP feature_config = NimbusFeatureConfigFactory.create() project = ProjectFactory.create() primary_outcome = Outcomes.by_application(application)[0].slug secondary_outcome = Outcomes.by_application(application)[1].slug experiment = NimbusExperimentFactory.create_with_lifecycle( NimbusExperimentFactory.Lifecycles.ENDING_APPROVE_APPROVE, application=application, feature_config=feature_config, projects=[project], primary_outcomes=[primary_outcome], secondary_outcomes=[secondary_outcome], ) data = dict(NimbusExperimentChangeLogSerializer(experiment).data) branches_data = [dict(b) for b in data.pop("branches")] control_branch_data = dict(data.pop("reference_branch")) locales_data = data.pop("locales") countries_data = data.pop("countries") feature_config_data = data.pop("feature_config") published_dto_data = data.pop("published_dto") self.assertEqual( data, { "application": experiment.application, "channel": experiment.channel, "firefox_min_version": experiment.firefox_min_version, "hypothesis": experiment.hypothesis, "is_paused": experiment.is_paused, "name": experiment.name, "owner": experiment.owner.email, "population_percent": str(experiment.population_percent), "primary_outcomes": [primary_outcome], "projects": [project.slug], "proposed_duration": experiment.proposed_duration, "proposed_enrollment": experiment.proposed_enrollment, "public_description": experiment.public_description, "publish_status": experiment.publish_status, "results_data": None, "risk_brand": experiment.risk_brand, "risk_mitigation_link": experiment.risk_mitigation_link, "risk_partner_related": experiment.risk_partner_related, "risk_revenue": experiment.risk_revenue, "secondary_outcomes": [secondary_outcome], "slug": experiment.slug, "status": experiment.status, "status_next": experiment.status_next, "targeting_config_slug": experiment.targeting_config_slug, "total_enrolled_clients": experiment.total_enrolled_clients, }, ) self.assertEqual( published_dto_data.keys(), dict(NimbusExperimentSerializer(experiment).data).keys(), ) self.assertEqual( feature_config_data, { "name": feature_config.name, "slug": feature_config.slug, "description": feature_config.description, "application": feature_config.application, "owner_email": feature_config.owner_email, "schema": feature_config.schema, }, ) self.assertEqual( set(locales_data), set(experiment.locales.all().values_list("code", flat=True)), ) self.assertEqual( set(countries_data), set(experiment.countries.all().values_list("code", flat=True)), ) self.assertEqual( control_branch_data, { "description": experiment.reference_branch.description, "feature_enabled": experiment.reference_branch.feature_enabled, "feature_value": experiment.reference_branch.feature_value, "name": experiment.reference_branch.name, "ratio": experiment.reference_branch.ratio, "slug": experiment.reference_branch.slug, }, ) for branch in experiment.branches.all(): self.assertIn( { "description": branch.description, "feature_enabled": branch.feature_enabled, "feature_value": branch.feature_value, "name": branch.name, "ratio": branch.ratio, "slug": branch.slug, }, branches_data, )
def test_nimbus_config(self): user_email = "*****@*****.**" feature_configs = NimbusFeatureConfigFactory.create_batch(10) response = self.query( """ query{ nimbusConfig{ application { label value } channel { label value } firefoxMinVersion { label value } featureConfig { name slug id description } outcomes { friendlyName slug application description } targetingConfigSlug { label value applicationValues } documentationLink { label value } hypothesisDefault maxPrimaryOutcomes locales { code name } countries { code name } } } """, headers={settings.OPENIDC_EMAIL_HEADER: user_email}, ) self.assertEqual(response.status_code, 200) content = json.loads(response.content) config = content["data"]["nimbusConfig"] def assertChoices(data, text_choices): self.assertEqual(len(data), len(text_choices.names)) for index, name in enumerate(text_choices.names): self.assertEqual(data[index]["label"], text_choices[name].label) self.assertEqual(data[index]["value"], name) assertChoices(config["application"], NimbusExperiment.Application) assertChoices(config["channel"], NimbusExperiment.Channel) assertChoices(config["firefoxMinVersion"], NimbusExperiment.Version) assertChoices(config["documentationLink"], NimbusExperiment.DocumentationLink) self.assertEqual(len(config["featureConfig"]), 13) for outcome in Outcomes.all(): self.assertIn( { "slug": outcome.slug, "friendlyName": outcome.friendly_name, "application": NimbusExperiment.Application(outcome.application).name, "description": outcome.description, }, config["outcomes"], ) for feature_config in feature_configs: config_feature_config = next( filter(lambda f: f["id"] == feature_config.id, config["featureConfig"]) ) self.assertEqual(config_feature_config["id"], feature_config.id) self.assertEqual(config_feature_config["name"], feature_config.name) self.assertEqual(config_feature_config["slug"], feature_config.slug) self.assertEqual( config_feature_config["description"], feature_config.description ) for choice in NimbusExperiment.TargetingConfig: self.assertIn( { "label": choice.label, "value": choice.name, "applicationValues": list( NimbusExperiment.TARGETING_CONFIGS[ choice.value ].application_choice_names ), }, config["targetingConfigSlug"], ) self.assertEqual(config["hypothesisDefault"], NimbusExperiment.HYPOTHESIS_DEFAULT) self.assertEqual( config["maxPrimaryOutcomes"], NimbusExperiment.MAX_PRIMARY_OUTCOMES ) for locale in Locale.objects.all(): self.assertIn({"code": locale.code, "name": locale.name}, config["locales"]) for country in Country.objects.all(): self.assertIn( {"code": country.code, "name": country.name}, config["countries"] )
def test_outputs_expected_schema_for_complete_experiment(self): application = NimbusExperiment.Application.DESKTOP probe_set = NimbusProbeSetFactory.create() feature_config = NimbusFeatureConfigFactory.create() project = ProjectFactory.create() experiment = NimbusExperimentFactory.create_with_status( NimbusExperiment.Status.COMPLETE, application=application, probe_sets=[probe_set], feature_config=feature_config, projects=[project], ) data = dict(NimbusExperimentChangeLogSerializer(experiment).data) branches_data = [dict(b) for b in data.pop("branches")] control_branch_data = dict(data.pop("reference_branch")) self.assertEqual( data, { "application": experiment.application, "channel": experiment.channel, "feature_config": { "name": feature_config.name, "slug": feature_config.slug, "description": feature_config.description, "application": feature_config.application, "owner_email": feature_config.owner_email, "schema": None, }, "firefox_min_version": experiment.firefox_min_version, "hypothesis": experiment.hypothesis, "is_paused": experiment.is_paused, "name": experiment.name, "owner": experiment.owner.email, "population_percent": str(experiment.population_percent), "probe_sets": [probe_set.slug], "projects": [project.slug], "proposed_duration": experiment.proposed_duration, "proposed_enrollment": experiment.proposed_enrollment, "public_description": experiment.public_description, "slug": experiment.slug, "status": experiment.status, "targeting_config_slug": experiment.targeting_config_slug, "total_enrolled_clients": experiment.total_enrolled_clients, }, ) self.assertEqual( control_branch_data, { "description": experiment.reference_branch.description, "feature_enabled": experiment.reference_branch.feature_enabled, "feature_value": experiment.reference_branch.feature_value, "name": experiment.reference_branch.name, "ratio": experiment.reference_branch.ratio, "slug": experiment.reference_branch.slug, }, ) for branch in experiment.branches.all(): self.assertIn( { "description": branch.description, "feature_enabled": branch.feature_enabled, "feature_value": branch.feature_value, "name": branch.name, "ratio": branch.ratio, "slug": branch.slug, }, branches_data, )
def test_nimbus_config(self): user_email = "*****@*****.**" # Create some probes and feature configs feature_configs = NimbusFeatureConfigFactory.create_batch(10) probe_sets = NimbusProbeSetFactory.create_batch(10) response = self.query( """ query{ nimbusConfig{ applicationChannels { label channels } application { label value } channel { label value } firefoxMinVersion { label value } featureConfig { name slug id description } probeSets { id name } targetingConfigSlug { label value } hypothesisDefault maxPrimaryProbeSets } } """, headers={settings.OPENIDC_EMAIL_HEADER: user_email}, ) self.assertEqual(response.status_code, 200) content = json.loads(response.content) config = content["data"]["nimbusConfig"] def assertChoices(data, text_choices): self.assertEqual(len(data), len(text_choices.names)) for index, name in enumerate(text_choices.names): self.assertEqual(data[index]["label"], text_choices[name].label) self.assertEqual(data[index]["value"], name) assertChoices(config["application"], NimbusExperiment.Application) assertChoices(config["channel"], NimbusExperiment.Channel) assertChoices(config["firefoxMinVersion"], NimbusExperiment.Version) assertChoices(config["targetingConfigSlug"], NimbusExperiment.TargetingConfig) self.assertEqual(len(config["featureConfig"]), 10) self.assertEqual(len(config["probeSets"]), 10) app_channels = config["applicationChannels"] self.assertEqual(len(app_channels), len(NimbusExperiment.ApplicationChannels)) # Transform list to dict for easier comparison app_channel_dict = {ac["label"]: ac["channels"] for ac in app_channels} for app_label, channel_names in NimbusExperiment.ApplicationChannels.items( ): app_compare_channels = app_channel_dict[app_label.label] self.assertEqual( set(app_compare_channels), set(channel.label for channel in channel_names), ) for probe_set in probe_sets: config_probe_set = next( filter(lambda p: p["id"] == str(probe_set.id), config["probeSets"])) self.assertEqual(config_probe_set["id"], str(probe_set.id)) self.assertEqual(config_probe_set["name"], probe_set.name) for feature_config in feature_configs: config_feature_config = next( filter(lambda f: f["id"] == str(feature_config.id), config["featureConfig"])) self.assertEqual(config_feature_config["id"], str(feature_config.id)) self.assertEqual(config_feature_config["name"], feature_config.name) self.assertEqual(config_feature_config["slug"], feature_config.slug) self.assertEqual(config_feature_config["description"], feature_config.description) self.assertEqual(config["hypothesisDefault"], NimbusExperiment.HYPOTHESIS_DEFAULT) self.assertEqual(config["maxPrimaryProbeSets"], NimbusExperiment.MAX_PRIMARY_PROBE_SETS)