def test_invalid_experiment_treatment_branch_requires_description(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT,
application=NimbusExperiment.Application.DESKTOP.value,
feature_config=NimbusFeatureConfigFactory(
application=NimbusExperiment.Application.DESKTOP.value),
)
treatment_branch = NimbusBranchFactory.create(experiment=experiment,
description="")
experiment.branches.add(treatment_branch)
experiment.save()
serializer = NimbusReadyForReviewSerializer(
experiment,
data=NimbusReadyForReviewSerializer(
experiment,
context={
"user": self.user
},
).data,
context={"user": self.user},
)
self.assertFalse(serializer.is_valid())
self.assertEqual(
serializer.errors["treatment_branches"][1],
["Description cannot be blank."],
)
def test_update_experiment_branches_with_feature_config(self):
user_email = "*****@*****.**"
feature = NimbusFeatureConfigFactory(
schema="{}", application=NimbusExperiment.Application.FENIX
)
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT,
application=NimbusExperiment.Application.FENIX,
)
experiment_id = experiment.id
reference_branch = {"name": "control", "description": "a control", "ratio": 1}
treatment_branches = [{"name": "treatment1", "description": "desc1", "ratio": 1}]
response = self.query(
UPDATE_EXPERIMENT_MUTATION,
variables={
"input": {
"id": experiment.id,
"featureConfigId": feature.id,
"referenceBranch": reference_branch,
"treatmentBranches": treatment_branches,
"changelogMessage": "test changelog message",
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
experiment = NimbusExperiment.objects.get(id=experiment_id)
self.assertEqual(experiment.feature_config, feature)
self.assertEqual(experiment.branches.count(), 2)
self.assertEqual(experiment.reference_branch.name, reference_branch["name"])
treatment_branch = experiment.treatment_branches[0]
self.assertEqual(treatment_branch.name, treatment_branches[0]["name"])
def test_update_experiment_branches_with_feature_config(self):
user_email = "*****@*****.**"
feature = NimbusFeatureConfigFactory(schema="{}")
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT)
experiment_id = experiment.id
reference_branch = {
"name": "control",
"description": "a control",
"ratio": 1
}
treatment_branches = [{
"name": "treatment1",
"description": "desc1",
"ratio": 1
}]
response = self.query(
UPDATE_EXPERIMENT_BRANCHES_MUTATION,
variables={
"input": {
"nimbusExperimentId": experiment.id,
"featureConfigId": feature.id,
"clientMutationId": "randomid",
"referenceBranch": reference_branch,
"treatmentBranches": treatment_branches,
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
result = content["data"]["updateExperimentBranches"]
self.assertEqual(
result["nimbusExperiment"],
{
"id": str(experiment.id),
"featureConfig": {
"name": feature.name
},
"status": experiment.status.upper(),
"name": experiment.name,
"slug": experiment.slug,
"referenceBranch": reference_branch,
"treatmentBranches": treatment_branches,
},
)
experiment = NimbusExperiment.objects.get(id=experiment_id)
self.assertEqual(experiment.feature_config, feature)
self.assertEqual(experiment.branches.count(), 2)
self.assertEqual(experiment.reference_branch.name,
reference_branch["name"])
treatment_branch = experiment.treatment_branches[0]
self.assertEqual(treatment_branch.name, treatment_branches[0]["name"])
def test_valid_experiment(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT,
application=NimbusExperiment.Application.DESKTOP.value,
feature_config=NimbusFeatureConfigFactory(
application=NimbusExperiment.Application.DESKTOP.value),
)
serializer = NimbusReadyForReviewSerializer(
experiment,
data=NimbusReadyForReviewSerializer(
experiment,
context={
"user": self.user
},
).data,
context={"user": self.user},
)
self.assertTrue(serializer.is_valid())
def test_invalid_experiment_reference_branch_requires_description(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT,
application=NimbusExperiment.Application.DESKTOP.value,
feature_config=NimbusFeatureConfigFactory(
application=NimbusExperiment.Application.DESKTOP.value),
)
experiment.reference_branch.description = ""
experiment.save()
serializer = NimbusReadyForReviewSerializer(
experiment,
data=NimbusReadyForReviewSerializer(
experiment,
context={
"user": self.user
},
).data,
context={"user": self.user},
)
self.assertFalse(serializer.is_valid())
self.assertEqual(
serializer.errors,
{"reference_branch": ["Description cannot be blank."]},
)
def test_invalid_experiment_default_hypothesis(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT,
application=NimbusExperiment.Application.DESKTOP.value,
feature_config=NimbusFeatureConfigFactory(
application=NimbusExperiment.Application.DESKTOP.value),
)
experiment.hypothesis = NimbusExperiment.HYPOTHESIS_DEFAULT
experiment.save()
serializer = NimbusReadyForReviewSerializer(
experiment,
data=NimbusReadyForReviewSerializer(
experiment,
context={
"user": self.user
},
).data,
context={"user": self.user},
)
self.assertFalse(serializer.is_valid())
self.assertEqual(
serializer.errors,
{"hypothesis": ["Hypothesis cannot be the default value."]},
)
def test_outputs_expected_schema_for_complete_experiment(self):
application = NimbusExperiment.Application.DESKTOP
feature_config = NimbusFeatureConfigFactory.create()
project = ProjectFactory.create()
primary_outcome = Outcomes.by_application(application)[0].slug
secondary_outcome = Outcomes.by_application(application)[1].slug
experiment = NimbusExperimentFactory.create_with_lifecycle(
NimbusExperimentFactory.Lifecycles.ENDING_APPROVE_APPROVE,
application=application,
feature_config=feature_config,
projects=[project],
primary_outcomes=[primary_outcome],
secondary_outcomes=[secondary_outcome],
)
data = dict(NimbusExperimentChangeLogSerializer(experiment).data)
branches_data = [dict(b) for b in data.pop("branches")]
control_branch_data = dict(data.pop("reference_branch"))
locales_data = data.pop("locales")
countries_data = data.pop("countries")
feature_config_data = data.pop("feature_config")
published_dto_data = data.pop("published_dto")
self.assertEqual(
data,
{
"application": experiment.application,
"channel": experiment.channel,
"firefox_min_version": experiment.firefox_min_version,
"hypothesis": experiment.hypothesis,
"is_paused": experiment.is_paused,
"name": experiment.name,
"owner": experiment.owner.email,
"population_percent": str(experiment.population_percent),
"primary_outcomes": [primary_outcome],
"projects": [project.slug],
"proposed_duration": experiment.proposed_duration,
"proposed_enrollment": experiment.proposed_enrollment,
"public_description": experiment.public_description,
"publish_status": experiment.publish_status,
"results_data": None,
"risk_brand": experiment.risk_brand,
"risk_mitigation_link": experiment.risk_mitigation_link,
"risk_partner_related": experiment.risk_partner_related,
"risk_revenue": experiment.risk_revenue,
"secondary_outcomes": [secondary_outcome],
"slug": experiment.slug,
"status": experiment.status,
"status_next": experiment.status_next,
"targeting_config_slug": experiment.targeting_config_slug,
"total_enrolled_clients": experiment.total_enrolled_clients,
},
)
self.assertEqual(
published_dto_data.keys(),
dict(NimbusExperimentSerializer(experiment).data).keys(),
)
self.assertEqual(
feature_config_data,
{
"name": feature_config.name,
"slug": feature_config.slug,
"description": feature_config.description,
"application": feature_config.application,
"owner_email": feature_config.owner_email,
"schema": feature_config.schema,
},
)
self.assertEqual(
set(locales_data),
set(experiment.locales.all().values_list("code", flat=True)),
)
self.assertEqual(
set(countries_data),
set(experiment.countries.all().values_list("code", flat=True)),
)
self.assertEqual(
control_branch_data,
{
"description": experiment.reference_branch.description,
"feature_enabled": experiment.reference_branch.feature_enabled,
"feature_value": experiment.reference_branch.feature_value,
"name": experiment.reference_branch.name,
"ratio": experiment.reference_branch.ratio,
"slug": experiment.reference_branch.slug,
},
)
for branch in experiment.branches.all():
self.assertIn(
{
"description": branch.description,
"feature_enabled": branch.feature_enabled,
"feature_value": branch.feature_value,
"name": branch.name,
"ratio": branch.ratio,
"slug": branch.slug,
},
branches_data,
)
def test_nimbus_config(self):
user_email = "*****@*****.**"
feature_configs = NimbusFeatureConfigFactory.create_batch(10)
response = self.query(
"""
query{
nimbusConfig{
application {
label
value
}
channel {
label
value
}
firefoxMinVersion {
label
value
}
featureConfig {
name
slug
id
description
}
outcomes {
friendlyName
slug
application
description
}
targetingConfigSlug {
label
value
applicationValues
}
documentationLink {
label
value
}
hypothesisDefault
maxPrimaryOutcomes
locales {
code
name
}
countries {
code
name
}
}
}
""",
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
config = content["data"]["nimbusConfig"]
def assertChoices(data, text_choices):
self.assertEqual(len(data), len(text_choices.names))
for index, name in enumerate(text_choices.names):
self.assertEqual(data[index]["label"], text_choices[name].label)
self.assertEqual(data[index]["value"], name)
assertChoices(config["application"], NimbusExperiment.Application)
assertChoices(config["channel"], NimbusExperiment.Channel)
assertChoices(config["firefoxMinVersion"], NimbusExperiment.Version)
assertChoices(config["documentationLink"], NimbusExperiment.DocumentationLink)
self.assertEqual(len(config["featureConfig"]), 13)
for outcome in Outcomes.all():
self.assertIn(
{
"slug": outcome.slug,
"friendlyName": outcome.friendly_name,
"application": NimbusExperiment.Application(outcome.application).name,
"description": outcome.description,
},
config["outcomes"],
)
for feature_config in feature_configs:
config_feature_config = next(
filter(lambda f: f["id"] == feature_config.id, config["featureConfig"])
)
self.assertEqual(config_feature_config["id"], feature_config.id)
self.assertEqual(config_feature_config["name"], feature_config.name)
self.assertEqual(config_feature_config["slug"], feature_config.slug)
self.assertEqual(
config_feature_config["description"], feature_config.description
)
for choice in NimbusExperiment.TargetingConfig:
self.assertIn(
{
"label": choice.label,
"value": choice.name,
"applicationValues": list(
NimbusExperiment.TARGETING_CONFIGS[
choice.value
].application_choice_names
),
},
config["targetingConfigSlug"],
)
self.assertEqual(config["hypothesisDefault"], NimbusExperiment.HYPOTHESIS_DEFAULT)
self.assertEqual(
config["maxPrimaryOutcomes"], NimbusExperiment.MAX_PRIMARY_OUTCOMES
)
for locale in Locale.objects.all():
self.assertIn({"code": locale.code, "name": locale.name}, config["locales"])
for country in Country.objects.all():
self.assertIn(
{"code": country.code, "name": country.name}, config["countries"]
)
def test_outputs_expected_schema_for_complete_experiment(self):
application = NimbusExperiment.Application.DESKTOP
probe_set = NimbusProbeSetFactory.create()
feature_config = NimbusFeatureConfigFactory.create()
project = ProjectFactory.create()
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.COMPLETE,
application=application,
probe_sets=[probe_set],
feature_config=feature_config,
projects=[project],
)
data = dict(NimbusExperimentChangeLogSerializer(experiment).data)
branches_data = [dict(b) for b in data.pop("branches")]
control_branch_data = dict(data.pop("reference_branch"))
self.assertEqual(
data,
{
"application": experiment.application,
"channel": experiment.channel,
"feature_config": {
"name": feature_config.name,
"slug": feature_config.slug,
"description": feature_config.description,
"application": feature_config.application,
"owner_email": feature_config.owner_email,
"schema": None,
},
"firefox_min_version": experiment.firefox_min_version,
"hypothesis": experiment.hypothesis,
"is_paused": experiment.is_paused,
"name": experiment.name,
"owner": experiment.owner.email,
"population_percent": str(experiment.population_percent),
"probe_sets": [probe_set.slug],
"projects": [project.slug],
"proposed_duration": experiment.proposed_duration,
"proposed_enrollment": experiment.proposed_enrollment,
"public_description": experiment.public_description,
"slug": experiment.slug,
"status": experiment.status,
"targeting_config_slug": experiment.targeting_config_slug,
"total_enrolled_clients": experiment.total_enrolled_clients,
},
)
self.assertEqual(
control_branch_data,
{
"description": experiment.reference_branch.description,
"feature_enabled": experiment.reference_branch.feature_enabled,
"feature_value": experiment.reference_branch.feature_value,
"name": experiment.reference_branch.name,
"ratio": experiment.reference_branch.ratio,
"slug": experiment.reference_branch.slug,
},
)
for branch in experiment.branches.all():
self.assertIn(
{
"description": branch.description,
"feature_enabled": branch.feature_enabled,
"feature_value": branch.feature_value,
"name": branch.name,
"ratio": branch.ratio,
"slug": branch.slug,
},
branches_data,
)
def test_nimbus_config(self):
user_email = "*****@*****.**"
# Create some probes and feature configs
feature_configs = NimbusFeatureConfigFactory.create_batch(10)
probe_sets = NimbusProbeSetFactory.create_batch(10)
response = self.query(
"""
query{
nimbusConfig{
applicationChannels {
label
channels
}
application {
label
value
}
channel {
label
value
}
firefoxMinVersion {
label
value
}
featureConfig {
name
slug
id
description
}
probeSets {
id
name
}
targetingConfigSlug {
label
value
}
hypothesisDefault
maxPrimaryProbeSets
}
}
""",
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
config = content["data"]["nimbusConfig"]
def assertChoices(data, text_choices):
self.assertEqual(len(data), len(text_choices.names))
for index, name in enumerate(text_choices.names):
self.assertEqual(data[index]["label"],
text_choices[name].label)
self.assertEqual(data[index]["value"], name)
assertChoices(config["application"], NimbusExperiment.Application)
assertChoices(config["channel"], NimbusExperiment.Channel)
assertChoices(config["firefoxMinVersion"], NimbusExperiment.Version)
assertChoices(config["targetingConfigSlug"],
NimbusExperiment.TargetingConfig)
self.assertEqual(len(config["featureConfig"]), 10)
self.assertEqual(len(config["probeSets"]), 10)
app_channels = config["applicationChannels"]
self.assertEqual(len(app_channels),
len(NimbusExperiment.ApplicationChannels))
# Transform list to dict for easier comparison
app_channel_dict = {ac["label"]: ac["channels"] for ac in app_channels}
for app_label, channel_names in NimbusExperiment.ApplicationChannels.items(
):
app_compare_channels = app_channel_dict[app_label.label]
self.assertEqual(
set(app_compare_channels),
set(channel.label for channel in channel_names),
)
for probe_set in probe_sets:
config_probe_set = next(
filter(lambda p: p["id"] == str(probe_set.id),
config["probeSets"]))
self.assertEqual(config_probe_set["id"], str(probe_set.id))
self.assertEqual(config_probe_set["name"], probe_set.name)
for feature_config in feature_configs:
config_feature_config = next(
filter(lambda f: f["id"] == str(feature_config.id),
config["featureConfig"]))
self.assertEqual(config_feature_config["id"],
str(feature_config.id))
self.assertEqual(config_feature_config["name"],
feature_config.name)
self.assertEqual(config_feature_config["slug"],
feature_config.slug)
self.assertEqual(config_feature_config["description"],
feature_config.description)
self.assertEqual(config["hypothesisDefault"],
NimbusExperiment.HYPOTHESIS_DEFAULT)
self.assertEqual(config["maxPrimaryProbeSets"],
NimbusExperiment.MAX_PRIMARY_PROBE_SETS)