def test_experiments_by_status(self):
user_email = "*****@*****.**"
draft_exp = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT)
NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.ACCEPTED)
response = self.query(
"""
query {
experiments(status: Draft) {
name
slug
publicDescription
}
}
""",
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
experiments = content["data"]["experiments"]
self.assertEqual(len(experiments), 1)
for key in experiments[0]:
self.assertEqual(experiments[0][key],
str(getattr(draft_exp, to_snake_case(key))))
def test_experiment_updates_when_record_is_in_main(self):
experiment1 = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.ACCEPTED, )
experiment2 = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.ACCEPTED, )
experiment3 = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT, )
self.assertEqual(experiment1.changes.count(), 3)
self.assertEqual(experiment2.changes.count(), 3)
self.assertEqual(experiment3.changes.count(), 1)
self.setup_kinto_get_main_records([experiment1.slug])
tasks.nimbus_check_experiments_are_live()
self.assertEqual(experiment3.changes.count(), 1)
self.assertTrue(
experiment1.changes.filter(
changed_by__email=settings.KINTO_DEFAULT_CHANGELOG_USER,
old_status=NimbusExperiment.Status.ACCEPTED,
new_status=NimbusExperiment.Status.LIVE,
).exists())
self.assertFalse(
experiment2.changes.filter(
changed_by__email=settings.KINTO_DEFAULT_CHANGELOG_USER,
old_status=NimbusExperiment.Status.ACCEPTED,
new_status=NimbusExperiment.Status.LIVE,
).exists())
def test_only_completes_experiments_with_matching_application_collection(
self):
desktop_experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.LIVE,
application=NimbusExperiment.Application.DESKTOP,
)
fenix_experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.LIVE,
application=NimbusExperiment.Application.FENIX,
)
def get_records(bucket, collection):
if collection == settings.KINTO_COLLECTION_NIMBUS_DESKTOP:
return [{"id": desktop_experiment.slug}]
if collection == settings.KINTO_COLLECTION_NIMBUS_MOBILE:
return [{"id": fenix_experiment.slug}]
self.mock_kinto_client.get_records.side_effect = get_records
tasks.nimbus_check_experiments_are_complete()
self.assertTrue(
NimbusExperiment.objects.filter(
id=desktop_experiment.id,
status=NimbusExperiment.Status.LIVE).exists())
self.assertTrue(
NimbusExperiment.objects.filter(
id=fenix_experiment.id,
status=NimbusExperiment.Status.LIVE).exists())
def test_invalid_experiment_treatment_branch_requires_description(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT,
application=NimbusExperiment.Application.DESKTOP.value,
feature_config=NimbusFeatureConfigFactory(
application=NimbusExperiment.Application.DESKTOP.value),
)
treatment_branch = NimbusBranchFactory.create(experiment=experiment,
description="")
experiment.branches.add(treatment_branch)
experiment.save()
serializer = NimbusReadyForReviewSerializer(
experiment,
data=NimbusReadyForReviewSerializer(
experiment,
context={
"user": self.user
},
).data,
context={"user": self.user},
)
self.assertFalse(serializer.is_valid())
self.assertEqual(
serializer.errors["treatment_branches"][1],
["Description cannot be blank."],
)
def test_send_experiment_ending_email(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.LIVE,
proposed_duration=10,
)
experiment.changes.filter(
old_status=NimbusExperiment.Status.ACCEPTED,
new_status=NimbusExperiment.Status.LIVE,
).update(changed_on=datetime.datetime.now() -
datetime.timedelta(days=10))
nimbus_send_experiment_ending_email(experiment)
sent_email = mail.outbox[-1]
self.assertEqual(
sent_email.subject,
NimbusExperiment.EMAIL_EXPERIMENT_END_SUBJECT,
)
self.assertEqual(sent_email.content_subtype, "html")
self.assertTrue(
experiment.emails.filter(
type=NimbusExperiment.EmailType.EXPERIMENT_END).exists())
self.assertEqual(
sent_email.recipients(),
[experiment.owner.email],
)
self.assertIn(experiment.experiment_url, sent_email.body)
def test_experiments_with_branches_returns_branch_data(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT, )
response = self.query(
"""
query {
experiments {
referenceBranch {
slug
}
treatmentBranches {
slug
}
}
}
""",
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
experiment_data = content["data"]["experiments"][0]
self.assertEqual(
experiment_data["referenceBranch"],
{"slug": experiment.reference_branch.slug},
)
self.assertEqual(
{b["slug"]
for b in experiment_data["treatmentBranches"]},
{b.slug
for b in experiment.treatment_branches},
)
def test_experiment_by_slug_not_ready_for_review(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT,
hypothesis=NimbusExperiment.HYPOTHESIS_DEFAULT)
response = self.query(
"""
query experimentBySlug($slug: String!) {
experimentBySlug(slug: $slug) {
readyForReview {
message
ready
}
}
}
""",
variables={"slug": experiment.slug},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200, response.content)
content = json.loads(response.content)
experiment_data = content["data"]["experimentBySlug"]
self.assertEqual(
experiment_data["readyForReview"],
{
"message": {
"hypothesis": ["Hypothesis cannot be the default value."]
},
"ready": False,
},
)
def test_experiment_by_slug_ready_for_review(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT)
response = self.query(
"""
query experimentBySlug($slug: String!) {
experimentBySlug(slug: $slug) {
name
slug
publicDescription
readyForReview {
message
ready
}
}
}
""",
variables={"slug": experiment.slug},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200, response.content)
content = json.loads(response.content)
experiment_data = content["data"]["experimentBySlug"]
self.assertEqual(experiment_data["name"], experiment.name)
self.assertEqual(experiment_data["slug"], experiment.slug)
self.assertEqual(experiment_data["publicDescription"],
experiment.public_description)
self.assertEqual(experiment_data["readyForReview"], {
"message": {},
"ready": True
})
def test_push_experiment_to_kinto_sends_fenix_experiment_data(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT,
application=NimbusExperiment.Application.FENIX,
population_percent=Decimal("50.0"),
)
tasks.nimbus_push_experiment_to_kinto(experiment.id)
self.assertEqual(experiment.bucket_range.start, 0)
self.assertEqual(experiment.bucket_range.count, 5000)
data = NimbusExperimentSerializer(experiment).data
self.mock_kinto_client.create_record.assert_called_with(
data=data,
collection=settings.KINTO_COLLECTION_NIMBUS_MOBILE,
bucket=settings.KINTO_BUCKET,
if_not_exists=True,
)
self.assertTrue(
NimbusChangeLog.objects.filter(
experiment=experiment,
changed_by__email=settings.KINTO_DEFAULT_CHANGELOG_USER,
old_status=NimbusExperiment.Status.DRAFT,
new_status=NimbusExperiment.Status.ACCEPTED,
).exists())
def test_saves_existing_experiment_with_changelog(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT,
application=NimbusExperiment.Application.FENIX,
hypothesis="Existing hypothesis",
name="Existing Name",
slug="existing-name",
public_description="Existing public description",
)
self.assertEqual(experiment.changes.count(), 1)
data = {
"application": NimbusExperiment.Application.DESKTOP,
"hypothesis": "New Hypothesis",
"name": "New Name",
"public_description": "New public description",
}
serializer = NimbusExperimentOverviewSerializer(
experiment, data=data, context={"user": self.user})
self.assertTrue(serializer.is_valid())
experiment = serializer.save()
self.assertEqual(experiment.changes.count(), 2)
self.assertEqual(experiment.application,
NimbusExperiment.Application.DESKTOP)
self.assertEqual(experiment.hypothesis, "New Hypothesis")
self.assertEqual(experiment.name, "New Name")
self.assertEqual(experiment.slug, "existing-name")
self.assertEqual(experiment.public_description,
"New public description")
def test_experiments_with_no_branches_returns_empty_values(self):
user_email = "*****@*****.**"
NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT, branches=[])
response = self.query(
"""
query {
experiments {
referenceBranch {
name
slug
description
ratio
}
treatmentBranches {
name
slug
description
ratio
}
}
}
""",
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
experiment_data = content["data"]["experiments"][0]
self.assertEqual(
experiment_data["referenceBranch"],
{
"name": "",
"slug": "",
"description": "",
"ratio": 1
},
)
self.assertEqual(
experiment_data["treatmentBranches"],
[{
"name": "",
"slug": "",
"description": "",
"ratio": 1
}],
)
def test_proposed_end_date_returns_start_date_plus_duration(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.LIVE,
proposed_duration=10,
)
self.assertEqual(
experiment.proposed_end_date,
datetime.date.today() + datetime.timedelta(days=10),
)
def test_checkexperiment_with_review_and_no_kinto_pending_pushes_experiment(
self, ):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.REVIEW)
self.assertEqual(experiment.changes.count(), 2)
self.setup_kinto_no_pending_review()
tasks.nimbus_check_kinto_push_queue()
self.mock_push_task.assert_called_with(experiment.id)
def handle(self, *args, **options):
for status, _ in Experiment.STATUS_CHOICES:
random_type = random.choice(Experiment.TYPE_CHOICES)[0]
experiment = ExperimentFactory.create_with_status(status, type=random_type)
logger.info("Created {}: {}".format(experiment, status))
for status, _ in NimbusExperiment.Status.choices:
experiment = NimbusExperimentFactory.create_with_status(status)
logger.info("Created {}: {}".format(experiment, status))
def test_experiment_ending_email_not_sent_for_experiments_before_proposed_end_date(
self, ):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.LIVE,
proposed_duration=10,
)
self.assertEqual(experiment.emails.count(), 0)
self.setup_kinto_get_main_records([experiment.slug])
tasks.nimbus_check_experiments_are_complete()
self.assertEqual(experiment.emails.count(), 0)
def test_should_end_returns_True_after_proposed_end_date(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.LIVE,
proposed_duration=10,
)
experiment.changes.filter(
old_status=NimbusExperiment.Status.ACCEPTED,
new_status=NimbusExperiment.Status.LIVE,
).update(changed_on=datetime.datetime.now() -
datetime.timedelta(days=10))
self.assertTrue(experiment.should_end)
def test_serializers_with_feature_value_None(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.ACCEPTED,
branches=[],
)
experiment.reference_branch = NimbusBranchFactory(
experiment=experiment, feature_value=None)
experiment.save()
serializer = NimbusExperimentSerializer(experiment)
self.assertIsNone(serializer.data["branches"][0]["feature"]["value"])
check_schema("experiments/NimbusExperiment", serializer.data)
def test_sets_application_channel_for_fenix_experiment(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.ACCEPTED,
application=NimbusExperiment.Application.FENIX,
channel=NimbusExperiment.Channel.FENIX_NIGHTLY,
)
serializer = NimbusExperimentSerializer(experiment)
self.assertEqual(serializer.data["application"],
NimbusExperiment.Channel.FENIX_NIGHTLY)
self.assertEqual(serializer.data["channel"],
NimbusExperiment.Channel.FENIX_NIGHTLY)
check_schema("experiments/NimbusExperiment", serializer.data)
def test_experiment_by_slug_not_found(self):
user_email = "*****@*****.**"
NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT)
response = self.query(
"""
query experimentBySlug($slug: String!) {
experimentBySlug(slug: $slug) {
name
slug
publicDescription
}
}
""",
variables={"slug": "nope"},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
experiment = content["data"]["experimentBySlug"]
self.assertIsNone(experiment)
def test_serializer_returns_error_for_non_unique_slug(self):
NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.ACCEPTED,
name="non unique slug",
slug="non-unique-slug",
)
data = {
"name": "non-unique slug",
"hypothesis": "Test hypothesis",
"application": NimbusExperiment.Application.DESKTOP.value,
"public_description": "Test description",
}
serializer = NimbusExperimentOverviewSerializer(
data=data, context={"user": self.user})
self.assertFalse(serializer.is_valid())
self.assertIn(
"Name maps to a pre-existing slug, please choose another name",
serializer.errors["name"],
)
def test_get_nimbus_experiment_returns_expected_data(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.LIVE)
response = self.client.get(
reverse(
"nimbus-experiment-rest-detail",
kwargs={"slug": experiment.slug},
), )
self.assertEqual(response.status_code, 200)
json_data = json.loads(response.content)
self.assertEqual(
NimbusExperimentSerializer(experiment).data, json_data)
def test_serializer_outputs_targeting_for_experiment_without_firefox_min_version(
self, ):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.ACCEPTED,
firefox_min_version=None,
targeting_config_slug=NimbusExperiment.TargetingConfig.ALL_ENGLISH,
channel=NimbusExperiment.Channel.DESKTOP_NIGHTLY,
)
serializer = NimbusExperimentSerializer(experiment)
self.assertEqual(
serializer.data["targeting"],
('browserSettings.update.channel == "nightly" '
"&& localeLanguageCode == 'en' "
"&& 'app.shield.optoutstudies.enabled'|preferenceValue"),
)
check_schema("experiments/NimbusExperiment", serializer.data)
def test_serializer_outputs_targeting_for_experiment_without_channels(
self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.ACCEPTED,
firefox_min_version=NimbusExperiment.Version.FIREFOX_80,
targeting_config_slug=NimbusExperiment.TargetingConfig.ALL_ENGLISH,
application=NimbusExperiment.Application.DESKTOP,
channel=None,
)
serializer = NimbusExperimentSerializer(experiment)
self.assertEqual(
serializer.data["targeting"],
("version|versionCompare('80.!') >= 0 "
"&& localeLanguageCode == 'en' "
"&& 'app.shield.optoutstudies.enabled'|preferenceValue"),
)
check_schema("experiments/NimbusExperiment", serializer.data)
def test_valid_experiment(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT,
application=NimbusExperiment.Application.DESKTOP.value,
feature_config=NimbusFeatureConfigFactory(
application=NimbusExperiment.Application.DESKTOP.value),
)
serializer = NimbusReadyForReviewSerializer(
experiment,
data=NimbusReadyForReviewSerializer(
experiment,
context={
"user": self.user
},
).data,
context={"user": self.user},
)
self.assertTrue(serializer.is_valid())
def test_list_view_serializes_experiments(self):
experiments = []
for status in NimbusExperiment.Status:
if status not in [
NimbusExperiment.Status.DRAFT,
NimbusExperiment.Status.REVIEW,
]:
experiments.append(
NimbusExperimentFactory.create_with_status(status.value,
slug=status))
response = self.client.get(reverse("nimbus-experiment-rest-list"), )
self.assertEqual(response.status_code, 200)
json_data = json.loads(response.content)
json_slugs = set([d["id"] for d in json_data])
expected_slugs = set(e.slug for e in experiments)
self.assertEqual(json_slugs, expected_slugs)
def test_serializer_outputs_expected_schema_without_feature(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.ACCEPTED,
feature_config=None,
)
serializer = NimbusExperimentSerializer(experiment)
experiment_data = serializer.data.copy()
branches_data = [dict(b) for b in experiment_data.pop("branches")]
self.assertEqual(len(branches_data), 2)
for branch in experiment.branches.all():
self.assertIn(
{
"slug": branch.slug,
"ratio": branch.ratio
},
branches_data,
)
check_schema("experiments/NimbusExperiment", serializer.data)
def test_generate_nimbus_changelog_with_prior_change(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT)
self.assertEqual(experiment.changes.count(), 1)
generate_nimbus_changelog(experiment, self.user)
self.assertEqual(experiment.changes.count(), 2)
change = experiment.latest_change()
self.assertEqual(change.experiment, experiment)
self.assertEqual(change.changed_by, self.user)
self.assertEqual(change.old_status, NimbusExperiment.Status.DRAFT)
self.assertEqual(change.new_status, NimbusExperiment.Status.DRAFT)
self.assertEqual(
change.experiment_data,
dict(NimbusExperimentChangeLogSerializer(experiment).data),
)
def test_invalid_experiment_requires_non_zero_population_percent(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT,
population_percent=0.0,
)
serializer = NimbusReadyForReviewSerializer(
experiment,
data=NimbusReadyForReviewSerializer(
experiment,
context={
"user": self.user
},
).data,
context={"user": self.user},
)
self.assertFalse(serializer.is_valid())
self.assertEqual(
str(serializer.errors["population_percent"][0]),
"Ensure this value is greater than or equal to 0.0001.",
)
def test_experiment_ending_email_sent_for_experiments_past_proposed_end_date(
self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.LIVE,
proposed_duration=10,
)
experiment.changes.filter(
old_status=NimbusExperiment.Status.ACCEPTED,
new_status=NimbusExperiment.Status.LIVE,
).update(changed_on=datetime.datetime.now() -
datetime.timedelta(days=10))
self.assertEqual(experiment.emails.count(), 0)
self.setup_kinto_get_main_records([experiment.slug])
tasks.nimbus_check_experiments_are_complete()
self.assertTrue(
experiment.emails.filter(
type=NimbusExperiment.EmailType.EXPERIMENT_END).exists())
self.assertEqual(len(mail.outbox), 1)
def test_invalid_experiment_reference_branch_requires_description(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT,
application=NimbusExperiment.Application.DESKTOP.value,
feature_config=NimbusFeatureConfigFactory(
application=NimbusExperiment.Application.DESKTOP.value),
)
experiment.reference_branch.description = ""
experiment.save()
serializer = NimbusReadyForReviewSerializer(
experiment,
data=NimbusReadyForReviewSerializer(
experiment,
context={
"user": self.user
},
).data,
context={"user": self.user},
)
self.assertFalse(serializer.is_valid())
self.assertEqual(
serializer.errors,
{"reference_branch": ["Description cannot be blank."]},
)