def test_cannot_check_review_qa_without_permissions(self):
user_1 = UserFactory.create()
user_2 = UserFactory.create()
content_type = ContentType.objects.get_for_model(Experiment)
permission = Permission.objects.get(content_type=content_type,
codename="can_check_QA_signoff")
user_1.user_permissions.add(permission)
self.assertTrue(user_1.has_perm("experiments.can_check_QA_signoff"))
self.assertFalse(user_2.has_perm("experiments.can_check_QA_signoff"))
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_REVIEW)
self.request.user = user_2
form = ExperimentReviewForm(request=self.request,
data={"review_qa": True},
instance=experiment)
self.assertTrue(form.is_valid())
experiment = form.save()
self.assertFalse(experiment.review_qa)
self.request.user = user_1
form = ExperimentReviewForm(request=self.request,
data={"review_qa": True},
instance=experiment)
self.assertTrue(form.is_valid())
experiment = form.save()
self.assertTrue(experiment.review_qa)
def setUp(self):
self.experiment = ExperimentFactory.create_with_variants(
name="Greatest Experiment",
slug="greatest-experiment",
firefox_min_version="68.0",
firefox_max_version="69.0",
firefox_channel="Nightly",
proposed_start_date=date(2019, 5, 1),
proposed_enrollment=5,
proposed_duration=10,
)
self.experiment.analysis_owner = UserFactory.create()
self.subscribing_user = UserFactory.create()
self.experiment.subscribers.add(self.subscribing_user)
def test_accepted_experiment_becomes_live_if_normandy_enabled(self):
exp_1 = ExperimentFactory.create_with_status(
target_status=Experiment.STATUS_ACCEPTED, normandy_id=1234)
subscribing_user = UserFactory.create()
exp_1.subscribers.add(subscribing_user)
tasks.update_experiment_info()
experiment = Experiment.objects.get(normandy_id=1234)
self.assertEqual(experiment.status, Experiment.STATUS_LIVE)
self.assertTrue(
experiment.changes.filter(
changed_by__email="*****@*****.**",
old_status=Experiment.STATUS_ACCEPTED,
new_status=Experiment.STATUS_LIVE,
).exists())
# status is live so bugzilla should be called
experiment2 = Experiment.objects.get(id=experiment.id)
comment = "Start Date: {} End Date: {}".format(experiment2.start_date,
experiment2.end_date)
self.mock_bugzilla_requests_post.assert_called_with(
settings.BUGZILLA_COMMENT_URL.format(id=experiment2.bugzilla_id),
{"comment": comment},
)
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].cc,
[experiment.owner.email, subscribing_user.email])
def test_generate_change_log_is_empty_when_no_change(self):
experiment = ExperimentFactory.create()
old_serialized_val = ChangeLogSerializer(experiment).data
new_serialized_val = ChangeLogSerializer(experiment).data
changed_data = {}
user = UserFactory.create()
generate_change_log(old_serialized_val, new_serialized_val, experiment,
changed_data, user)
changed_values = experiment.changes.latest().changed_values
self.assertEqual(changed_values, {})
def test_send_intent_to_ship_email_with_risk_fields(self):
experiment = ExperimentFactory.create(
name="Experiment",
slug="experiment",
risks="Fictitious risk",
risk_technical_description="Fictitious technical challenge",
population_percent=10.0,
firefox_min_version="56.0",
firefox_max_version="",
firefox_channel="Nightly",
)
sender = "*****@*****.**"
release_drivers = "*****@*****.**"
user = UserFactory.create(email="*****@*****.**")
experiment.subscribers.add(user)
with self.settings(EMAIL_SENDER=sender,
EMAIL_RELEASE_DRIVERS=release_drivers):
send_intent_to_ship_email(experiment.id)
bug_url = settings.BUGZILLA_DETAIL_URL.format(
id=experiment.bugzilla_id)
expected_locales = self.format_locales(experiment)
expected_countries = self.format_countries(experiment)
expected_version_channel = self.format_version_and_channel(experiment)
sent_email = mail.outbox[-1]
self.assertEqual(sent_email.subject,
"Delivery Intent to ship: Experiment 56.0 Nightly")
self.assertEqual(sent_email.from_email, sender)
self.assertEqual(
set(sent_email.recipients()),
set([
release_drivers,
experiment.owner.email,
experiment.analysis_owner,
"*****@*****.**",
]),
)
self.assertTrue(
experiment.emails.filter(
type=ExperimentConstants.INTENT_TO_SHIP_EMAIL_LABEL).exists())
self.assertIn(f"Experimenter Bug: {bug_url}", sent_email.body)
self.assertIn(f"Locales: {expected_locales}; {expected_countries}",
sent_email.body)
self.assertIn(f"Timeline & Channel: {expected_version_channel}",
sent_email.body)
self.assertIn("Fictitious risk", sent_email.body)
self.assertIn("Fictitious technical challenge", sent_email.body)
def test_experiment_status_updates_by_existing_user(self):
User = get_user_model()
user = UserFactory(email="*****@*****.**")
self.assertTrue(User.objects.filter(email="*****@*****.**").exists())
ExperimentFactory.create_with_status(
target_status=Experiment.STATUS_ACCEPTED, normandy_id=1234)
tasks.update_experiment_info()
experiment = Experiment.objects.get(normandy_id=1234)
self.assertEqual(experiment.status, Experiment.STATUS_LIVE)
self.assertTrue(
experiment.changes.filter(
changed_by=user,
old_status=Experiment.STATUS_ACCEPTED,
new_status=Experiment.STATUS_LIVE,
).exists())
def test_send_intent_to_ship_email_without_risk_fields(self):
experiment = ExperimentFactory.create(
name="Experiment",
slug="experiment",
risks="",
risk_technical_description="",
population_percent=10.0,
firefox_min_version="56.0",
firefox_max_version="",
firefox_channel="Nightly",
)
user = UserFactory.create(email="*****@*****.**")
experiment.subscribers.add(user)
sender = "*****@*****.**"
release_drivers = "*****@*****.**"
with self.settings(EMAIL_SENDER=sender,
EMAIL_RELEASE_DRIVERS=release_drivers):
send_intent_to_ship_email(experiment.id)
bug_url = settings.BUGZILLA_DETAIL_URL.format(
id=experiment.bugzilla_id)
expected_locales = self.format_locales(experiment)
expected_countries = self.format_countries(experiment)
expected_version_channel = self.format_version_and_channel(experiment)
sent_email = mail.outbox[-1]
self.assertEqual(
sent_email.subject,
"SHIELD Study Intent to ship: Experiment 56.0 Nightly",
)
self.assertIn(f"Experimenter Bug: {bug_url}", sent_email.body)
self.assertIn(
f"Locales: {expected_locales}; {expected_countries}",
sent_email.body,
)
self.assertIn(f"Timeline & Channel: {expected_version_channel}",
sent_email.body)
self.assertEqual(sent_email.content_subtype, "html")
self.assertEqual(sent_email.from_email, sender)
self.assertEqual(
set(sent_email.recipients()),
set([release_drivers, experiment.owner.email,
"*****@*****.**"]),
)
def test_form_saves_reviews(self):
user = UserFactory.create()
content_type = ContentType.objects.get_for_model(Experiment)
experiment_model_permissions = Permission.objects.filter(
content_type=content_type, codename__startswith="can_check")
for permission in experiment_model_permissions:
user.user_permissions.add(permission)
self.request.user = user
experiment = ExperimentFactory.create_with_status(
Experiment.STATUS_REVIEW)
self.assertEqual(experiment.changes.count(), 2)
self.assertFalse(experiment.review_science)
self.assertFalse(experiment.review_engineering)
self.assertFalse(experiment.review_qa_requested)
self.assertFalse(experiment.review_intent_to_ship)
self.assertFalse(experiment.review_bugzilla)
self.assertFalse(experiment.review_qa)
self.assertFalse(experiment.review_relman)
self.assertFalse(experiment.review_advisory)
self.assertFalse(experiment.review_legal)
self.assertFalse(experiment.review_ux)
self.assertFalse(experiment.review_security)
self.assertFalse(experiment.review_vp)
self.assertFalse(experiment.review_data_steward)
self.assertFalse(experiment.review_comms)
self.assertFalse(experiment.review_impacted_teams)
data = {
"review_science": True,
"review_engineering": True,
"review_qa_requested": True,
"review_intent_to_ship": True,
"review_bugzilla": True,
"review_qa": True,
"review_relman": True,
"review_advisory": True,
"review_legal": True,
"review_ux": True,
"review_security": True,
"review_vp": True,
"review_data_steward": True,
"review_comms": True,
"review_impacted_teams": True,
}
form = ExperimentReviewForm(request=self.request,
data=data,
instance=experiment)
self.assertTrue(form.is_valid())
experiment = form.save()
self.assertTrue(experiment.review_science)
self.assertTrue(experiment.review_engineering)
self.assertTrue(experiment.review_qa_requested)
self.assertTrue(experiment.review_intent_to_ship)
self.assertTrue(experiment.review_bugzilla)
self.assertTrue(experiment.review_qa)
self.assertTrue(experiment.review_relman)
self.assertTrue(experiment.review_advisory)
self.assertTrue(experiment.review_legal)
self.assertTrue(experiment.review_ux)
self.assertTrue(experiment.review_security)
self.assertTrue(experiment.review_vp)
self.assertTrue(experiment.review_data_steward)
self.assertTrue(experiment.review_comms)
self.assertTrue(experiment.review_impacted_teams)
self.assertEqual(experiment.changes.count(), 3)
def test_generate_change_log_gives_correct_output(self):
experiment = ExperimentFactory.create_with_status(
target_status=Experiment.STATUS_REVIEW,
num_variants=0,
short_description="description",
qa_status="pretty good",
firefox_min_version="55.0",
)
variant1 = ExperimentVariantFactory.create(
experiment=experiment,
ratio=75,
description="variant1 description",
name="variant1",
slug="variant1-slug",
)
variant1.save()
old_serialized_val = ChangeLogSerializer(experiment).data
experiment.short_description = "changing the description"
experiment.qa_status = "good"
experiment.firefox_min_version = "56.0"
variant2 = ExperimentVariantFactory.create(
experiment=experiment,
ratio=25,
description="variant2 description",
name="variant2",
slug="variant2-slug",
)
variant2.save()
VariantPreferencesFactory.create(
variant=variant2,
pref_name="p1",
pref_type=Experiment.PREF_TYPE_INT,
pref_branch=Experiment.PREF_BRANCH_DEFAULT,
pref_value="5",
)
experiment.save()
new_serialized_val = ChangeLogSerializer(experiment).data
changed_variant_pref = {
"pref_name": "p1",
"pref_type": "integer",
"pref_branch": "default",
"pref_value": "5",
}
changed_data = {
"short_description":
"changing the description",
"qa_status":
"good",
"firefox_min_version":
"56.0",
"variants": [{
"ratio": 25,
"description": "variant2 description",
"name": "variant2",
"slug": "variant2-slug",
"preferences": [changed_variant_pref],
}],
}
user = UserFactory.create()
generate_change_log(old_serialized_val, new_serialized_val, experiment,
changed_data, user)
changed_value = experiment.changes.latest().changed_values
expected_changed_value = {
"firefox_min_version": {
"display_name": "Firefox Min Version",
"new_value": "56.0",
"old_value": "55.0",
},
"qa_status": {
"display_name": "Qa Status",
"new_value": "good",
"old_value": "pretty good",
},
"short_description": {
"display_name": "Short Description",
"new_value": "changing the description",
"old_value": "description",
},
"variants": {
"display_name":
"Branches",
"new_value": [
{
"ratio": 25,
"description": "variant2 description",
"name": "variant2",
"slug": "variant2-slug",
},
{
"ratio": 75,
"description": "variant1 description",
"name": "variant1",
"slug": "variant1-slug",
},
],
"old_value": [{
"ratio": 75,
"description": "variant1 description",
"name": "variant1",
"slug": "variant1-slug",
}],
},
}
self.assertEqual(
expected_changed_value["firefox_min_version"],
changed_value["firefox_min_version"],
)
self.assertEqual(expected_changed_value["qa_status"],
changed_value["qa_status"])
self.assertEqual(
expected_changed_value["short_description"],
changed_value["short_description"],
)
self.assertCountEqual(expected_changed_value["variants"],
changed_value["variants"])
def test_update_change_log_creates_log_with_correct_change(self):
experiment = ExperimentFactory.create()
variant = ExperimentVariantFactory.create(
experiment=experiment,
ratio=100,
description="it's a description",
name="variant name",
)
variant_data = {
"ratio": 100,
"description": variant.description,
"name": variant.name,
}
changed_values = {
"variants": {
"new_value": {
"variants": [variant_data]
},
"old_value": None,
"display_name": "Branches",
}
}
ExperimentChangeLog.objects.create(
experiment=experiment,
changed_by=UserFactory(),
old_status=Experiment.STATUS_DRAFT,
new_status=Experiment.STATUS_DRAFT,
changed_values=changed_values,
message="",
)
self.assertEqual(experiment.changes.count(), 1)
change_data = {
"variants": [{
"id": variant.id,
"ratio": 100,
"description": "some other description",
"name": "some other name",
"is_control": False,
}]
}
serializer = ExperimentDesignBaseSerializer(
instance=experiment,
data=change_data,
context={"request": self.request})
self.assertTrue(serializer.is_valid())
experiment = serializer.save()
serializer_variant_data = ExperimentVariantSerializer(variant).data
self.assertEqual(experiment.changes.count(), 2)
changed_values = experiment.changes.latest().changed_values
variant = ExperimentVariant.objects.get(id=variant.id)
changed_serializer_variant_data = ExperimentVariantSerializer(
variant).data
self.assertIn("variants", changed_values)
self.assertEqual(changed_values["variants"]["old_value"],
[serializer_variant_data])
self.assertEqual(changed_values["variants"]["new_value"],
[changed_serializer_variant_data])