def test_update_experiment_outcomes(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT,
application=NimbusExperiment.Application.DESKTOP,
primary_outcomes=[],
secondary_outcomes=[],
)
outcomes = [
o.slug for o in Outcomes.by_application(NimbusExperiment.Application.DESKTOP)
]
primary_outcomes = outcomes[: NimbusExperiment.MAX_PRIMARY_OUTCOMES]
secondary_outcomes = outcomes[NimbusExperiment.MAX_PRIMARY_OUTCOMES :]
response = self.query(
UPDATE_EXPERIMENT_MUTATION,
variables={
"input": {
"id": experiment.id,
"primaryOutcomes": primary_outcomes,
"secondaryOutcomes": secondary_outcomes,
"changelogMessage": "test changelog message",
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200, response.content)
experiment = NimbusExperiment.objects.get(slug=experiment.slug)
self.assertEqual(experiment.primary_outcomes, primary_outcomes)
self.assertEqual(experiment.secondary_outcomes, secondary_outcomes)
def validate_secondary_outcomes(self, value):
value_set = set(value)
valid_outcomes = set([
o.slug for o in Outcomes.by_application(self.instance.application)
])
if valid_outcomes.intersection(value_set) != value_set:
invalid_outcomes = value_set - valid_outcomes
raise serializers.ValidationError(
f"Invalid choices for secondary outcomes: {invalid_outcomes}")
return value
def validate_primary_outcomes(self, value):
value_set = set(value)
if len(value) > NimbusExperiment.MAX_PRIMARY_OUTCOMES:
raise serializers.ValidationError(
"Exceeded maximum primary outcome limit of "
f"{NimbusExperiment.MAX_PRIMARY_OUTCOMES}.")
valid_outcomes = set([
o.slug for o in Outcomes.by_application(self.instance.application)
])
if valid_outcomes.intersection(value_set) != value_set:
invalid_outcomes = value_set - valid_outcomes
raise serializers.ValidationError(
f"Invalid choices for primary outcomes: {invalid_outcomes}")
return value
def test_outputs_expected_schema_for_complete_experiment(self):
application = NimbusExperiment.Application.DESKTOP
feature_config = NimbusFeatureConfigFactory.create()
project = ProjectFactory.create()
primary_outcome = Outcomes.by_application(application)[0].slug
secondary_outcome = Outcomes.by_application(application)[1].slug
experiment = NimbusExperimentFactory.create_with_lifecycle(
NimbusExperimentFactory.Lifecycles.ENDING_APPROVE_APPROVE,
application=application,
feature_config=feature_config,
projects=[project],
primary_outcomes=[primary_outcome],
secondary_outcomes=[secondary_outcome],
)
data = dict(NimbusExperimentChangeLogSerializer(experiment).data)
branches_data = [dict(b) for b in data.pop("branches")]
control_branch_data = dict(data.pop("reference_branch"))
locales_data = data.pop("locales")
countries_data = data.pop("countries")
feature_config_data = data.pop("feature_config")
published_dto_data = data.pop("published_dto")
self.assertEqual(
data,
{
"application": experiment.application,
"channel": experiment.channel,
"firefox_min_version": experiment.firefox_min_version,
"hypothesis": experiment.hypothesis,
"is_paused": experiment.is_paused,
"name": experiment.name,
"owner": experiment.owner.email,
"population_percent": str(experiment.population_percent),
"primary_outcomes": [primary_outcome],
"projects": [project.slug],
"proposed_duration": experiment.proposed_duration,
"proposed_enrollment": experiment.proposed_enrollment,
"public_description": experiment.public_description,
"publish_status": experiment.publish_status,
"results_data": None,
"risk_brand": experiment.risk_brand,
"risk_mitigation_link": experiment.risk_mitigation_link,
"risk_partner_related": experiment.risk_partner_related,
"risk_revenue": experiment.risk_revenue,
"secondary_outcomes": [secondary_outcome],
"slug": experiment.slug,
"status": experiment.status,
"status_next": experiment.status_next,
"targeting_config_slug": experiment.targeting_config_slug,
"total_enrolled_clients": experiment.total_enrolled_clients,
},
)
self.assertEqual(
published_dto_data.keys(),
dict(NimbusExperimentSerializer(experiment).data).keys(),
)
self.assertEqual(
feature_config_data,
{
"name": feature_config.name,
"slug": feature_config.slug,
"description": feature_config.description,
"application": feature_config.application,
"owner_email": feature_config.owner_email,
"schema": feature_config.schema,
},
)
self.assertEqual(
set(locales_data),
set(experiment.locales.all().values_list("code", flat=True)),
)
self.assertEqual(
set(countries_data),
set(experiment.countries.all().values_list("code", flat=True)),
)
self.assertEqual(
control_branch_data,
{
"description": experiment.reference_branch.description,
"feature_enabled": experiment.reference_branch.feature_enabled,
"feature_value": experiment.reference_branch.feature_value,
"name": experiment.reference_branch.name,
"ratio": experiment.reference_branch.ratio,
"slug": experiment.reference_branch.slug,
},
)
for branch in experiment.branches.all():
self.assertIn(
{
"description": branch.description,
"feature_enabled": branch.feature_enabled,
"feature_value": branch.feature_value,
"name": branch.name,
"ratio": branch.ratio,
"slug": branch.slug,
},
branches_data,
)
def setUpClass(cls):
super().setUpClass()
Outcomes.clear_cache()
class NimbusExperimentFactory(factory.django.DjangoModelFactory):
publish_status = NimbusExperiment.PublishStatus.IDLE
owner = factory.SubFactory(UserFactory)
name = factory.LazyAttribute(lambda o: faker.catch_phrase())
slug = factory.LazyAttribute(
lambda o: slugify(o.name)[:NimbusExperiment.MAX_SLUG_LEN])
public_description = factory.LazyAttribute(lambda o: faker.text(200))
risk_mitigation_link = factory.LazyAttribute(lambda o: faker.uri())
proposed_duration = factory.LazyAttribute(lambda o: random.randint(10, 60))
proposed_enrollment = factory.LazyAttribute(
lambda o: random.randint(2, o.proposed_duration))
population_percent = factory.LazyAttribute(
lambda o: decimal.Decimal(random.randint(1, 10) * 10))
total_enrolled_clients = factory.LazyAttribute(
lambda o: random.randint(1, 100) * 1000)
firefox_min_version = factory.LazyAttribute(
lambda o: random.choice(list(NimbusExperiment.Version)).value)
application = factory.LazyAttribute(
lambda o: random.choice(list(NimbusExperiment.Application)).value)
channel = factory.LazyAttribute(
lambda o: random.choice(list(NimbusExperiment.Channel)).value)
hypothesis = factory.LazyAttribute(lambda o: faker.text(1000))
feature_config = factory.SubFactory(
"experimenter.experiments.tests.factories.NimbusFeatureConfigFactory")
targeting_config_slug = factory.LazyAttribute(
lambda o: random.choice(list(NimbusExperiment.TargetingConfig)).value)
primary_outcomes = factory.LazyAttribute(
lambda o: [oc.slug for oc in Outcomes.all()[:2]])
secondary_outcomes = factory.LazyAttribute(
lambda o: [oc.slug for oc in Outcomes.all()[2:]])
risk_partner_related = factory.LazyAttribute(
lambda o: random.choice([True, False]))
risk_revenue = factory.LazyAttribute(
lambda o: random.choice([True, False]))
risk_brand = factory.LazyAttribute(lambda o: random.choice([True, False]))
class Meta:
model = NimbusExperiment
exclude = ("Lifecycles", "LifecycleStates")
Lifecycles = Lifecycles
LifecycleStates = LifecycleStates
@factory.post_generation
def projects(self, create, extracted, **kwargs):
if not create:
# Simple build, do nothing.
return
if isinstance(extracted, Iterable):
# A list of groups were passed in, use them
for project in extracted:
self.projects.add(project)
else:
for i in range(3):
self.projects.add(ProjectFactory.create())
@factory.post_generation
def branches(self, create, extracted, **kwargs):
if not create:
# Simple build, do nothing.
return
if isinstance(extracted, Iterable):
# A list of groups were passed in, use them
for branch in extracted:
self.branches.add(branch)
else:
NimbusBranchFactory.create(experiment=self)
self.reference_branch = NimbusBranchFactory.create(experiment=self)
self.save()
@factory.post_generation
def document_links(self, create, extracted, **kwargs):
if not create:
# Simple build, do nothing.
return
if isinstance(extracted, Iterable):
# A list of links were passed in, use them
for link in extracted:
self.documentation_links.add(link)
else:
for title, _ in NimbusExperiment.DocumentationLink.choices:
self.documentation_links.add(
NimbusDocumentationLinkFactory.create_with_title(
experiment=self, title=title))
@factory.post_generation
def locales(self, create, extracted, **kwargs):
if not create:
# Simple build, do nothing.
return
if extracted is None and Locale.objects.exists():
extracted = Locale.objects.all()[:3]
if extracted:
self.locales.add(*extracted)
@factory.post_generation
def countries(self, create, extracted, **kwargs):
if not create:
# Simple build, do nothing.
return
if extracted is None and Country.objects.exists():
extracted = Country.objects.all()[:3]
if extracted:
self.countries.add(*extracted)
@classmethod
def create_with_lifecycle(cls,
lifecycle,
with_random_timespan=False,
**kwargs):
experiment = cls.create(**kwargs)
now = timezone.now() - datetime.timedelta(
days=random.randint(100, 200))
for state in lifecycle.value:
experiment.apply_lifecycle_state(state)
if (experiment.status == experiment.Status.LIVE
and experiment.status_next is None
and "published_dto" not in kwargs):
experiment.published_dto = NimbusExperimentSerializer(
experiment).data
experiment.save()
if experiment.has_filter(
experiment.Filters.SHOULD_ALLOCATE_BUCKETS):
experiment.allocate_bucket_range()
change = generate_nimbus_changelog(
experiment,
experiment.owner,
f"set lifecycle {lifecycle} state {state}",
)
if with_random_timespan:
change.changed_on = now
change.save()
now += datetime.timedelta(days=random.randint(5, 20))
return NimbusExperiment.objects.get(id=experiment.id)
def test_nimbus_config(self):
user_email = "*****@*****.**"
feature_configs = NimbusFeatureConfigFactory.create_batch(10)
response = self.query(
"""
query{
nimbusConfig{
application {
label
value
}
channel {
label
value
}
firefoxMinVersion {
label
value
}
featureConfig {
name
slug
id
description
}
outcomes {
friendlyName
slug
application
description
}
targetingConfigSlug {
label
value
applicationValues
}
documentationLink {
label
value
}
hypothesisDefault
maxPrimaryOutcomes
locales {
code
name
}
countries {
code
name
}
}
}
""",
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
config = content["data"]["nimbusConfig"]
def assertChoices(data, text_choices):
self.assertEqual(len(data), len(text_choices.names))
for index, name in enumerate(text_choices.names):
self.assertEqual(data[index]["label"], text_choices[name].label)
self.assertEqual(data[index]["value"], name)
assertChoices(config["application"], NimbusExperiment.Application)
assertChoices(config["channel"], NimbusExperiment.Channel)
assertChoices(config["firefoxMinVersion"], NimbusExperiment.Version)
assertChoices(config["documentationLink"], NimbusExperiment.DocumentationLink)
self.assertEqual(len(config["featureConfig"]), 13)
for outcome in Outcomes.all():
self.assertIn(
{
"slug": outcome.slug,
"friendlyName": outcome.friendly_name,
"application": NimbusExperiment.Application(outcome.application).name,
"description": outcome.description,
},
config["outcomes"],
)
for feature_config in feature_configs:
config_feature_config = next(
filter(lambda f: f["id"] == feature_config.id, config["featureConfig"])
)
self.assertEqual(config_feature_config["id"], feature_config.id)
self.assertEqual(config_feature_config["name"], feature_config.name)
self.assertEqual(config_feature_config["slug"], feature_config.slug)
self.assertEqual(
config_feature_config["description"], feature_config.description
)
for choice in NimbusExperiment.TargetingConfig:
self.assertIn(
{
"label": choice.label,
"value": choice.name,
"applicationValues": list(
NimbusExperiment.TARGETING_CONFIGS[
choice.value
].application_choice_names
),
},
config["targetingConfigSlug"],
)
self.assertEqual(config["hypothesisDefault"], NimbusExperiment.HYPOTHESIS_DEFAULT)
self.assertEqual(
config["maxPrimaryOutcomes"], NimbusExperiment.MAX_PRIMARY_OUTCOMES
)
for locale in Locale.objects.all():
self.assertIn({"code": locale.code, "name": locale.name}, config["locales"])
for country in Country.objects.all():
self.assertIn(
{"code": country.code, "name": country.name}, config["countries"]
)
def resolve_outcomes(root, info):
return Outcomes.all()