def test_experiment_returns_country_and_locale(self):
user_email = "*****@*****.**"
NimbusExperimentFactory.create(publish_status=NimbusExperiment.PublishStatus.IDLE)
response = self.query(
"""
query {
experiments {
countries {
code
name
}
locales {
code
name
}
}
}
""",
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
experiment_data = content["data"]["experiments"][0]
for locale in Locale.objects.all():
self.assertIn(
{"code": locale.code, "name": locale.name}, experiment_data["locales"]
)
for country in Country.objects.all():
self.assertIn(
{"code": country.code, "name": country.name}, experiment_data["countries"]
)
def test_check_experiment_with_no_review_status_pushes_nothing(self):
for status in [
NimbusExperiment.Status.DRAFT,
NimbusExperiment.Status.ACCEPTED,
NimbusExperiment.Status.LIVE,
NimbusExperiment.Status.COMPLETE,
]:
NimbusExperimentFactory.create(status=status)
self.setup_kinto_no_pending_review()
tasks.nimbus_check_kinto_push_queue()
self.mock_push_task.assert_not_called()
def test_push_experiment_to_kinto_reraises_exception(self):
experiment = NimbusExperimentFactory.create(
publish_status=NimbusExperiment.PublishStatus.APPROVED, )
self.mock_kinto_client.create_record.side_effect = Exception
with self.assertRaises(Exception):
tasks.nimbus_push_experiment_to_kinto(
settings.KINTO_COLLECTION_NIMBUS_DESKTOP, experiment.id)
def test_generate_nimbus_changelog_without_prior_change(self):
experiment = NimbusExperimentFactory.create()
self.assertEqual(experiment.changes.count(), 0)
generate_nimbus_changelog(experiment, self.user, "test message")
self.assertEqual(experiment.changes.count(), 1)
change = experiment.changes.get()
self.assertEqual(change.experiment, experiment)
self.assertEqual(change.message, "test message")
self.assertEqual(change.changed_by, self.user)
self.assertEqual(change.old_status, None)
self.assertEqual(change.old_status_next, None)
self.assertEqual(change.old_publish_status, None)
self.assertEqual(change.new_status, NimbusExperiment.Status.DRAFT)
self.assertEqual(change.new_status_next, None)
self.assertEqual(change.new_publish_status,
NimbusExperiment.PublishStatus.IDLE)
self.assertEqual(
change.experiment_data,
dict(NimbusExperimentChangeLogSerializer(experiment).data),
)
def test_monitoring_dashboard_url_returns_url_when_experiment_is_complete(
self):
experiment = NimbusExperimentFactory.create(
slug="experiment",
status=NimbusExperiment.Status.COMPLETE,
)
NimbusChangeLogFactory.create(
experiment=experiment,
old_status=NimbusExperiment.Status.ACCEPTED,
new_status=NimbusExperiment.Status.LIVE,
changed_on=datetime.date(2019, 5, 1),
)
NimbusChangeLogFactory.create(
experiment=experiment,
old_status=NimbusExperiment.Status.LIVE,
new_status=NimbusExperiment.Status.COMPLETE,
changed_on=datetime.date(2019, 5, 10),
)
self.assertEqual(
experiment.monitoring_dashboard_url,
settings.MONITORING_URL.format(
slug=experiment.slug,
from_date=1556582400000,
to_date=1557619200000,
),
)
def test_full_isolation_group_creates_next_isolation_group_adds_bucket_range(
self, ):
"""
Rare case: An isolation group with experiment bucket allocations exists, and the
next requested bucket allocation would overflow its total bucket range, and so a
an isolation group with the same name but subsequent instance ID is created.
This is currently treated naively, ie does not account for possible collisions and
overlaps. When this case becomes more common this will likely need to be given
more thought.
"""
experiment = NimbusExperimentFactory.create(
application=self.application)
isolation_group = NimbusIsolationGroupFactory.create(
name=experiment.slug, application=self.application, total=100)
NimbusBucketRangeFactory(isolation_group=isolation_group, count=100)
bucket = NimbusIsolationGroup.request_isolation_group_buckets(
experiment.slug, experiment, 100)
self.assertEqual(bucket.start, 0)
self.assertEqual(bucket.end, 99)
self.assertEqual(bucket.count, 100)
self.assertEqual(bucket.isolation_group.name, isolation_group.name)
self.assertEqual(bucket.isolation_group.instance,
isolation_group.instance + 1)
self.assertEqual(
bucket.isolation_group.randomization_unit,
self.randomization_unit,
)
def test_targeting_config_set(self):
experiment = NimbusExperimentFactory.create(
targeting_config_slug=NimbusExperiment.TargetingConfig.ALL_ENGLISH)
self.assertEqual(
experiment.targeting_config,
NimbusExperiment.TARGETING_CONFIGS[
NimbusExperiment.TargetingConfig.ALL_ENGLISH],
)
def test_start_date_returns_datetime_for_started_experiment(self):
experiment = NimbusExperimentFactory.create()
start_change = NimbusChangeLogFactory(
experiment=experiment,
old_status=NimbusExperiment.Status.ACCEPTED,
new_status=NimbusExperiment.Status.LIVE,
)
self.assertEqual(experiment.start_date, start_change.changed_on)
def test_end_date_returns_datetime_for_ended_experiment(self):
experiment = NimbusExperimentFactory.create()
end_change = NimbusChangeLogFactory(
experiment=experiment,
old_status=NimbusExperiment.Status.LIVE,
new_status=NimbusExperiment.Status.COMPLETE,
)
self.assertEqual(experiment.end_date, end_change.changed_on)
def test_form_rejects_reference_branch_from_other_experiment(self):
experiment = NimbusExperimentFactory.create()
branch = NimbusBranchFactory.create()
form = NimbusExperimentAdminForm(instance=experiment,
data={"reference_branch": branch.id})
self.assertFalse(form.is_valid())
self.assertIn("reference_branch", form.errors)
def test_targeting_for_mobile(self):
experiment = NimbusExperimentFactory.create(
firefox_min_version=NimbusExperiment.Version.FIREFOX_83,
targeting_config_slug=NimbusExperiment.TargetingConfig.ALL_ENGLISH,
application=NimbusExperiment.Application.FENIX,
channel=NimbusExperiment.Channel.NO_CHANNEL,
)
self.assertEqual(experiment.targeting, "localeLanguageCode == 'en'")
def test_results_ready_false(self):
experiment = NimbusExperimentFactory.create()
NimbusChangeLogFactory.create(
experiment=experiment,
old_status=NimbusExperiment.Status.DRAFT,
new_status=NimbusExperiment.Status.LIVE,
changed_on=datetime.date.today() - datetime.timedelta(days=2),
)
self.assertFalse(experiment.results_ready)
def test_results_ready_true(self):
experiment = NimbusExperimentFactory.create()
NimbusChangeLogFactory.create(
experiment=experiment,
old_status=NimbusExperiment.Status.DRAFT,
new_status=NimbusExperiment.Status.LIVE,
changed_on=datetime.date(2019, 5, 1),
)
self.assertTrue(experiment.results_ready)
def test_form_rejects_invalid_json(self):
feature_config = NimbusFeatureConfigFactory(schema=self.schema)
experiment = NimbusExperimentFactory.create(feature_config=feature_config)
branch = NimbusBranchFactory.create(experiment=experiment)
form = NimbusBranchAdminForm(
instance=branch,
data={"experiment": experiment, "feature_value": "{invalid"},
)
self.assertFalse(form.is_valid())
self.assertIn("feature_value", form.errors)
def test_start_date_uses_most_recent_start_change(self):
experiment = NimbusExperimentFactory.create()
NimbusChangeLogFactory(
experiment=experiment,
old_status=NimbusExperiment.Status.DRAFT,
new_status=NimbusExperiment.Status.LIVE,
)
start_change = NimbusChangeLogFactory(
experiment=experiment,
old_status=NimbusExperiment.Status.DRAFT,
new_status=NimbusExperiment.Status.LIVE,
)
self.assertEqual(experiment.start_date, start_change.changed_on.date())
def test_end_date_uses_most_recent_end_change(self):
experiment = NimbusExperimentFactory.create()
NimbusChangeLogFactory(
experiment=experiment,
old_status=NimbusExperiment.Status.LIVE,
new_status=NimbusExperiment.Status.COMPLETE,
)
end_change = NimbusChangeLogFactory(
experiment=experiment,
old_status=NimbusExperiment.Status.LIVE,
new_status=NimbusExperiment.Status.COMPLETE,
)
self.assertEqual(experiment.end_date, end_change.changed_on.date())
def test_monitoring_dashboard_url_is_when_experiment_not_begun(self):
experiment = NimbusExperimentFactory.create(
slug="experiment",
status=NimbusExperiment.Status.DRAFT,
)
self.assertEqual(
experiment.monitoring_dashboard_url,
settings.MONITORING_URL.format(
slug=experiment.slug,
from_date="",
to_date="",
),
)
def test_targeting_for_experiment_without_channels(self):
experiment = NimbusExperimentFactory.create(
firefox_min_version=NimbusExperiment.Version.FIREFOX_83,
targeting_config_slug=NimbusExperiment.TargetingConfig.ALL_ENGLISH,
application=NimbusExperiment.Application.DESKTOP,
channel=NimbusExperiment.Channel.NO_CHANNEL,
)
self.assertEqual(
experiment.targeting,
("version|versionCompare('83.!') >= 0 "
"&& 'app.shield.optoutstudies.enabled'|preferenceValue "
"&& localeLanguageCode == 'en'"),
)
def test_analysis_results_view_no_data(self, status, mock_exists):
user_email = "*****@*****.**"
mock_exists.return_value = False
experiment = NimbusExperimentFactory.create(status=status)
response = self.client.get(
reverse("visualization-analysis-data",
kwargs={"slug": experiment.slug}),
**{settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
json_data = json.loads(response.content)
self.assertEqual({}, json_data)
def test_existing_isolation_group_adds_bucket_range(self):
"""
Rare case: An isolation group with no buckets allocated already exists.
This may become common when users can create their own isolation groups
and then later assign experiments to them.
"""
experiment = NimbusExperimentFactory.create()
isolation_group = NimbusIsolationGroupFactory.create(
name=experiment.slug)
bucket = NimbusIsolationGroup.request_isolation_group_buckets(
experiment.slug, experiment, 100)
self.assertEqual(bucket.start, 0)
self.assertEqual(bucket.end, 99)
self.assertEqual(bucket.count, 100)
self.assertEqual(bucket.isolation_group, isolation_group)
def test_signoff_recommendations(
self,
risk_brand,
risk_revenue,
risk_partner_related,
vp_recommended,
legal_recommended,
):
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT,
risk_brand=risk_brand,
risk_revenue=risk_revenue,
risk_partner_related=risk_partner_related,
)
self.assertEqual(experiment.signoff_recommendations["qa_signoff"], True)
self.assertEqual(experiment.signoff_recommendations["vp_signoff"], vp_recommended)
self.assertEqual(
experiment.signoff_recommendations["legal_signoff"], legal_recommended
)
def test_existing_isolation_group_with_matching_name_but_not_application_is_filtered(
self, ):
"""
Now that isolation groups are bound to applications, we have to check for the
case where isolation groups with the same name but different applications are
treated separately.
"""
name = "isolation group name"
NimbusIsolationGroupFactory.create(
name=name, application=NimbusExperiment.Application.DESKTOP)
experiment = NimbusExperimentFactory.create(
name=name,
slug=name,
application=NimbusExperiment.Application.FENIX)
bucket = NimbusIsolationGroup.request_isolation_group_buckets(
name, experiment, 100)
self.assertEqual(bucket.isolation_group.name, name)
self.assertEqual(bucket.isolation_group.application,
NimbusExperiment.Application.FENIX)
def test_existing_isolation_group_with_buckets_adds_next_bucket_range(
self):
"""
Common case: An isolation group with experiment bucket allocations exists,
and a subsequent bucket allocation is requested. This will be the common case
for any experiments that share an isolation group.
"""
experiment = NimbusExperimentFactory.create()
isolation_group = NimbusIsolationGroupFactory.create(
name=experiment.slug)
NimbusBucketRangeFactory.create(isolation_group=isolation_group,
start=0,
count=100)
bucket = NimbusIsolationGroup.request_isolation_group_buckets(
experiment.slug, experiment, 100)
self.assertEqual(bucket.start, 100)
self.assertEqual(bucket.end, 199)
self.assertEqual(bucket.count, 100)
self.assertEqual(bucket.isolation_group, isolation_group)
def test_serializer_update_branches(self):
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT, )
reference_branch = {
"name": "control",
"description": "a control",
"ratio": 1
}
treatment_branches = [
{
"name": "treatment1",
"description": "desc1",
"ratio": 1
},
{
"name": "testment2",
"description": "desc2",
"ratio": 1
},
]
input = {
"feature_config": None,
"reference_branch": reference_branch,
"treatment_branches": treatment_branches,
}
serializer = NimbusBranchUpdateSerializer(experiment,
data=input,
partial=True,
context={"user": self.user})
self.assertTrue(serializer.is_valid())
experiment = serializer.save()
for key, val in reference_branch.items():
self.assertEqual(getattr(experiment.reference_branch, key), val)
experiment_treatment_branches = experiment.branches.exclude(
id=experiment.reference_branch.id).order_by("id")
for index, branch in enumerate(treatment_branches):
for key, val in branch.items():
self.assertEqual(
getattr(experiment_treatment_branches[index], key), val)
def test_empty_isolation_group_creates_isolation_group_and_bucket_range(self):
"""
Common case: A new empty isolation group for an experiment
that is orthogonal to all other current experiments. This will
likely describe most experiment launches.
"""
experiment = NimbusExperimentFactory.create(application=self.application)
bucket = NimbusIsolationGroup.request_isolation_group_buckets(
experiment.slug, experiment, 100
)
self.assertEqual(bucket.start, 0)
self.assertEqual(bucket.end, 99)
self.assertEqual(bucket.count, 100)
self.assertEqual(bucket.isolation_group.name, experiment.slug)
self.assertEqual(bucket.isolation_group.instance, 1)
self.assertEqual(bucket.isolation_group.total, NimbusExperiment.BUCKET_TOTAL)
self.assertEqual(
bucket.isolation_group.randomization_unit,
experiment.application_config.randomization_unit,
)
def test_form_accepts_valid_feature_value(self):
feature_config = NimbusFeatureConfigFactory(schema=self.schema)
experiment = NimbusExperimentFactory.create(feature_config=feature_config)
branch = NimbusBranchFactory.create(experiment=experiment)
form = NimbusBranchAdminForm(
instance=branch,
data={
"experiment": experiment,
"feature_value": json.dumps(
{
"firstName": "John",
"lastName": "Jacob",
"age": 3,
}
),
},
)
self.assertFalse(form.is_valid())
self.assertNotIn("feature_value", form.errors)
def test_monitoring_dashboard_url_returns_url_when_experiment_has_begun(self):
experiment = NimbusExperimentFactory.create(
slug="experiment",
status=NimbusExperiment.Status.LIVE,
)
NimbusChangeLogFactory.create(
experiment=experiment,
old_status=NimbusExperiment.Status.DRAFT,
new_status=NimbusExperiment.Status.LIVE,
changed_on=datetime.date(2019, 5, 1),
)
self.assertEqual(
experiment.monitoring_dashboard_url,
settings.MONITORING_URL.format(
slug=experiment.slug,
from_date=1556582400000,
to_date="",
),
)
def test_analysis_results_view_data(self, status, mock_exists, mock_open):
user_email = "*****@*****.**"
DATA_ROW = {"point": 12, "upper": 13, "lower": 10}
FULL_DATA = {"daily": DATA_ROW, "weekly": DATA_ROW}
class File:
def read(self):
return json.dumps(DATA_ROW)
mock_open.return_value = File()
mock_exists.return_value = True
experiment = NimbusExperimentFactory.create(status=status)
response = self.client.get(
reverse("visualization-analysis-data",
kwargs={"slug": experiment.slug}),
**{settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
json_data = json.loads(response.content)
self.assertEqual(FULL_DATA, json_data)
def test_experiment_returns_publish_status(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(
publish_status=NimbusExperiment.PublishStatus.IDLE
)
response = self.query(
"""
query {
experiments {
publishStatus
}
}
""",
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
experiment_data = content["data"]["experiments"][0]
self.assertEqual(
experiment_data["publishStatus"],
experiment.publish_status.name,
)
def test_check_experiment_with_review_and_kinto_pending_pushes_nothing(
self):
NimbusExperimentFactory.create(status=NimbusExperiment.Status.REVIEW, )
self.setup_kinto_pending_review()
tasks.nimbus_check_kinto_push_queue()
self.mock_push_task.assert_not_called()