def test_has_normandy_info_not_true_if_missing_normandy_info(self):
experiment = ExperimentFactory.create(
normandy_id=None, normandy_slug=None
)
self.assertFalse(experiment.has_normandy_info)
def test_overview_is_complete_when_saved(self):
experiment = ExperimentFactory.create()
self.assertTrue(experiment.completed_overview)
def test_population_is_complete_when_values_set(self):
experiment = ExperimentFactory.create()
self.assertTrue(experiment.completed_population)
def test_has_external_urls_is_true_when_test_tube_url_is_set(self):
experiment = ExperimentFactory.create(status=Experiment.STATUS_LIVE)
self.assertTrue(experiment.has_external_urls)
def test_grouped_changes_groups_by_date_then_user(self):
experiment = ExperimentFactory.create()
date1 = datetime.datetime.now() - datetime.timedelta(days=2)
date2 = datetime.datetime.now() - datetime.timedelta(days=1)
date3 = datetime.datetime.now()
user1 = UserFactory.create()
user2 = UserFactory.create()
user3 = UserFactory.create()
change1 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user1, changed_on=date1
)
change2 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user1, changed_on=date1
)
change3 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user1, changed_on=date1
)
change4 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user2, changed_on=date1
)
change5 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user2, changed_on=date2
)
change6 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user3, changed_on=date2
)
change7 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user3, changed_on=date2
)
change8 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user1, changed_on=date3
)
change9 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user1, changed_on=date3
)
change10 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user2, changed_on=date3
)
change11 = ExperimentChangeLogFactory.create(
experiment=experiment, changed_by=user3, changed_on=date3
)
self.assertEqual(
set(experiment.grouped_changes.keys()),
set([date1.date(), date2.date(), date3.date()]),
)
self.assertEqual(
set(experiment.grouped_changes[date1.date()].keys()),
set([user1, user2]),
)
self.assertEqual(
set(experiment.grouped_changes[date2.date()].keys()),
set([user2, user3]),
)
self.assertEqual(
set(experiment.grouped_changes[date3.date()].keys()),
set([user1, user2, user3]),
)
self.assertEqual(
experiment.grouped_changes[date1.date()][user1],
set([change1, change2, change3]),
)
self.assertEqual(
experiment.grouped_changes[date1.date()][user2], set([change4])
)
self.assertEqual(
experiment.grouped_changes[date2.date()][user2], set([change5])
)
self.assertEqual(
experiment.grouped_changes[date2.date()][user3],
set([change6, change7]),
)
self.assertEqual(
experiment.grouped_changes[date3.date()][user1],
set([change8, change9]),
)
self.assertEqual(
experiment.grouped_changes[date3.date()][user2], set([change10])
)
self.assertEqual(
experiment.grouped_changes[date3.date()][user3], set([change11])
)
def test_test_tube_url_is_none_when_experiment_not_begun(self):
experiment = ExperimentFactory.create(
slug="experiment", status=Experiment.STATUS_DRAFT
)
self.assertIsNone(experiment.test_tube_url)
def test_has_external_urls_is_true_when_feature_bugzilla_url_is_set(self):
experiment = ExperimentFactory.create(
feature_bugzilla_url="www.bugzilla.com/123/"
)
self.assertTrue(experiment.has_external_urls)
def test_timeline_is_not_complete_when_missing_dates(self):
experiment = ExperimentFactory.create(
proposed_start_date=None, proposed_duration=None
)
self.assertFalse(experiment.completed_timeline)
def test_timeline_is_complete_when_dates_set(self):
experiment = ExperimentFactory.create(
proposed_start_date=datetime.date.today(), proposed_duration=10
)
self.assertTrue(experiment.completed_timeline)
def test_experiment_url(self):
experiment = ExperimentFactory.create(slug="experiment-slug")
self.assertEqual(
experiment.experiment_url,
f"https://{settings.HOSTNAME}/experiments/experiment-slug/",
)
def test_ordered_changes_orders_by_date(self):
experiment = ExperimentFactory.create()
date1 = timezone.now() - datetime.timedelta(days=2)
date2 = timezone.now() - datetime.timedelta(days=1)
date3 = timezone.now()
user1 = UserFactory.create()
user2 = UserFactory.create()
user3 = UserFactory.create()
a = ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user1,
changed_on=date1,
message="a",
)
b = ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user1,
changed_on=date1,
message="b",
)
c = ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user2,
changed_on=date1,
message="c",
)
d = ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user2,
changed_on=date2,
message="d",
)
e = ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user3,
changed_on=date2,
message="e",
)
f = ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user3,
changed_on=date2,
message="f",
)
g = ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user1,
changed_on=date3,
message="g",
)
h = ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user1,
changed_on=date3,
message="h",
)
i = ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user2,
changed_on=date3,
message="i",
)
j = ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user3,
changed_on=date3,
message="j",
)
expected_changes = {
date1.date(): {user1: set([a, b]), user2: set([c])},
date2.date(): {user2: set([d]), user3: set([e, f])},
date3.date(): {
user1: set([g, h]),
user2: set([i]),
user3: set([j]),
},
}
ordered_dates = [date for date, changes in experiment.ordered_changes]
self.assertEqual(
ordered_dates, [date3.date(), date2.date(), date1.date()]
)
day3_users = [
user for user, user_changes in experiment.ordered_changes[0][1]
]
self.assertEqual(set(day3_users), set([user1, user2, user3]))
day2_users = [
user for user, user_changes in experiment.ordered_changes[1][1]
]
self.assertEqual(set(day2_users), set([user2, user3]))
day1_users = [
user for user, user_changes in experiment.ordered_changes[2][1]
]
self.assertEqual(set(day1_users), set([user1, user2]))
for date, date_changes in experiment.ordered_changes:
for user, user_changes in date_changes:
self.assertEqual(user_changes, expected_changes[date][user])
def test_get_absolute_url(self):
experiment = ExperimentFactory.create(slug="experiment-slug")
self.assertEqual(
experiment.get_absolute_url(), "/experiments/experiment-slug/"
)
def test_format_dc_normandy_urls_returns_empty_list_without_normandy_id(
self
):
experiment = ExperimentFactory.create(normandy_id=None)
self.assertEqual(len(experiment.format_dc_normandy_urls), 0)
def test_has_normandy_info_true_with_normandy_id(self):
experiment = ExperimentFactory.create(
normandy_id="123", normandy_slug=None
)
self.assertTrue(experiment.has_normandy_info)
def test_unique_name_with_same_slug_raises_error(self):
ExperimentFactory.create(slug="slug")
self.data["name"] = "slug#"
form = ExperimentOverviewForm(request=self.request, data=self.data)
self.assertFalse(form.is_valid())
def test_addons_is_not_complete_when_release_url_not_set(self):
experiment = ExperimentFactory.create(
addon_experiment_id=None, addon_release_url=None
)
self.assertFalse(experiment.completed_addon)
def test_no_fields_required(self):
experiment = ExperimentFactory.create()
form = ExperimentResultsForm(request=self.request,
data={},
instance=experiment)
self.assertTrue(form.is_valid())
def test_addons_is_complete_when_release_url_set(self):
experiment = ExperimentFactory.create(
addon_experiment_id="addon-experiment-id",
addon_release_url="https://www.example.com/release.xpi",
)
self.assertTrue(experiment.completed_addon)
def test_has_external_urls_is_false_when_no_external_urls(self):
experiment = ExperimentFactory.create(
data_science_bugzilla_url="", feature_bugzilla_url=""
)
self.assertFalse(experiment.has_external_urls)
def test_objectives_is_not_complete_with_still_default(self):
experiment = ExperimentFactory.create(
objectives=Experiment.OBJECTIVES_DEFAULT,
analysis=Experiment.ANALYSIS_DEFAULT,
)
self.assertFalse(experiment.completed_objectives)
def test_has_external_urls_is_true_when_bugzilla_url_is_set(self):
experiment = ExperimentFactory.create(bugzilla_id="1234")
self.assertTrue(experiment.has_external_urls)
def test_objectives_is_complete_with_non_defaults(self):
experiment = ExperimentFactory.create(
objectives="Some objectives!", analysis="Some analysis!"
)
self.assertTrue(experiment.completed_objectives)
def test_has_external_urls_is_true_when_bugzilla_and_test_tube_urls(self):
experiment = ExperimentFactory.create(
status=Experiment.STATUS_LIVE, bugzilla_id="1234"
)
self.assertTrue(experiment.has_external_urls)
def test_completed_required_reviews_false_when_reviews_not_complete(self):
experiment = ExperimentFactory.create()
self.assertFalse(experiment.completed_required_reviews)
def test_ordered_changes_orders_by_date(self):
experiment = ExperimentFactory.create()
date1 = datetime.datetime.now() - datetime.timedelta(days=2)
date2 = datetime.datetime.now() - datetime.timedelta(days=1)
date3 = datetime.datetime.now()
user1 = UserFactory.create()
user2 = UserFactory.create()
user3 = UserFactory.create()
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user1,
changed_on=date1,
message="a",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user1,
changed_on=date1,
message="b",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user1,
changed_on=date1,
message="b",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user2,
changed_on=date1,
message="c",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user2,
changed_on=date2,
message="d",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user3,
changed_on=date2,
message="e",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user3,
changed_on=date2,
message="f",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user1,
changed_on=date3,
message="g",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user1,
changed_on=date3,
message="h",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user2,
changed_on=date3,
message="i",
)
ExperimentChangeLogFactory.create(
experiment=experiment,
changed_by=user3,
changed_on=date3,
message="j",
)
expected_changes = {
date1.date(): {user1: set(["a", "b"]), user2: set(["c"])},
date2.date(): {user2: set(["d"]), user3: set(["e", "f"])},
date3.date(): {
user1: set(["g", "h"]),
user2: set(["i"]),
user3: set(["j"]),
},
}
ordered_dates = [date for date, changes in experiment.ordered_changes]
self.assertEqual(
ordered_dates, [date3.date(), date2.date(), date1.date()]
)
day3_users = [
user for user, user_changes in experiment.ordered_changes[0][1]
]
self.assertEqual(set(day3_users), set([user1, user2, user3]))
day2_users = [
user for user, user_changes in experiment.ordered_changes[1][1]
]
self.assertEqual(set(day2_users), set([user2, user3]))
day1_users = [
user for user, user_changes in experiment.ordered_changes[2][1]
]
self.assertEqual(set(day1_users), set([user1, user2]))
for date, date_changes in experiment.ordered_changes:
for user, user_changes in date_changes:
self.assertEqual(user_changes, expected_changes[date][user])
def test_serializer_outputs_expected_schema(self):
country1 = CountryFactory(code="CA", name="Canada")
locale1 = LocaleFactory(code="da", name="Danish")
experiment = ExperimentFactory.create(locales=[locale1],
countries=[country1])
related_exp = ExperimentFactory.create()
experiment.related_to.add(related_exp)
serializer = ChangeLogSerializer(experiment)
risk_tech_description = experiment.risk_technical_description
# ensure expected_data has "string" if pref_type is json string
pref_type = PrefTypeField().to_representation(experiment.pref_type)
expected_data = {
"type":
experiment.type,
"owner":
experiment.owner.id,
"name":
experiment.name,
"short_description":
experiment.short_description,
"related_work":
experiment.related_work,
"related_to": [related_exp.id],
"proposed_start_date":
str(experiment.proposed_start_date),
"proposed_duration":
experiment.proposed_duration,
"proposed_enrollment":
experiment.proposed_enrollment,
"design":
experiment.design,
"addon_experiment_id":
experiment.addon_experiment_id,
"addon_release_url":
experiment.addon_release_url,
"pref_key":
experiment.pref_key,
"pref_type":
pref_type,
"pref_branch":
experiment.pref_branch,
"public_name":
experiment.public_name,
"public_description":
experiment.public_description,
"population_percent":
"{0:.4f}".format(experiment.population_percent),
"firefox_min_version":
experiment.firefox_min_version,
"firefox_max_version":
experiment.firefox_max_version,
"firefox_channel":
experiment.firefox_channel,
"client_matching":
experiment.client_matching,
"locales": [{
"code": "da",
"name": "Danish"
}],
"countries": [{
"code": "CA",
"name": "Canada"
}],
"platform":
experiment.platform,
"objectives":
experiment.objectives,
"analysis":
experiment.analysis,
"analysis_owner":
experiment.analysis_owner.id,
"survey_required":
experiment.survey_required,
"survey_urls":
experiment.survey_urls,
"survey_instructions":
experiment.survey_instructions,
"engineering_owner":
experiment.engineering_owner,
"bugzilla_id":
experiment.bugzilla_id,
"normandy_slug":
experiment.normandy_slug,
"normandy_id":
experiment.normandy_id,
"data_science_bugzilla_url":
experiment.data_science_bugzilla_url,
"feature_bugzilla_url":
experiment.feature_bugzilla_url,
"risk_internal_only":
experiment.risk_internal_only,
"risk_partner_related":
experiment.risk_partner_related,
"risk_brand":
experiment.risk_brand,
"risk_fast_shipped":
experiment.risk_fast_shipped,
"risk_confidential":
experiment.risk_confidential,
"risk_release_population":
experiment.risk_release_population,
"risk_revenue":
experiment.risk_revenue,
"risk_data_category":
experiment.risk_data_category,
"risk_external_team_impact":
experiment.risk_external_team_impact,
"risk_telemetry_data":
experiment.risk_telemetry_data,
"risk_ux":
experiment.risk_ux,
"risk_security":
experiment.risk_security,
"risk_revision":
experiment.risk_revision,
"risk_technical":
experiment.risk_technical,
"risk_technical_description":
risk_tech_description,
"risks":
experiment.risks,
"testing":
experiment.testing,
"test_builds":
experiment.test_builds,
"qa_status":
experiment.qa_status,
"review_science":
experiment.review_science,
"review_engineering":
experiment.review_engineering,
"review_qa_requested":
experiment.review_qa_requested,
"review_intent_to_ship":
experiment.review_intent_to_ship,
"review_bugzilla":
experiment.review_bugzilla,
"review_qa":
experiment.review_qa,
"review_relman":
experiment.review_relman,
"review_advisory":
experiment.review_advisory,
"review_legal":
experiment.review_legal,
"review_ux":
experiment.review_ux,
"review_security":
experiment.review_security,
"review_vp":
experiment.review_vp,
"review_data_steward":
experiment.review_data_steward,
"review_comms":
experiment.review_comms,
"review_impacted_teams":
experiment.review_impacted_teams,
"variants": [
ExperimentVariantSerializer(variant).data
for variant in experiment.variants.all()
],
"results_url":
experiment.results_url,
"results_initial":
experiment.results_initial,
"results_lessons_learned":
experiment.results_lessons_learned,
}
self.assertEqual(set(serializer.data.keys()),
set(expected_data.keys()))
self.assertEqual(serializer.data, expected_data)
def test_population_is_not_complete_when_defaults_set(self):
experiment = ExperimentFactory.create(
population_percent=0.0, firefox_version="", firefox_channel=""
)
self.assertFalse(experiment.completed_population)
def test_changelog_values(self):
experiment = Experiment()
experiment.save()
ds_url = "{base}DS-123".format(base=settings.DS_ISSUE_HOST)
bug_url = "{base}show_bug.cgi?id=123".format(
base=settings.BUGZILLA_HOST)
related_exp = ExperimentFactory.create()
project = ProjectFactory.create()
data = {
"type": Experiment.TYPE_PREF,
"name": "A new experiment!",
"short_description": "Let us learn new things",
"data_science_issue_url": ds_url,
"owner": self.user.id,
"analysis_owner": self.user.id,
"engineering_owner": "Lisa the Engineer",
"public_description": "Let us learn new public things",
"related_to": [related_exp],
"feature_bugzilla_url": bug_url,
"projects": [project],
}
form = ExperimentOverviewForm(request=self.request,
data=data,
instance=experiment)
self.assertTrue(form.is_valid())
experiment = form.save()
latest_changes = experiment.changes.latest()
expected_data = {
"analysis_owner": {
"display_name": "Data Science Owner",
"new_value": experiment.analysis_owner.id,
"old_value": None,
},
"data_science_issue_url": {
"display_name": "Data Science Issue URL",
"new_value": "https://jira.example.com/browse/DS-123",
"old_value": None,
},
"engineering_owner": {
"display_name": "Engineering Owner",
"new_value": "Lisa the Engineer",
"old_value": None,
},
"feature_bugzilla_url": {
"display_name": "Feature Bugzilla URL",
"new_value":
"https://bugzilla.example.com/show_bug.cgi?id=123",
"old_value": None,
},
"name": {
"display_name": "Public Name",
"new_value": "A new experiment!",
"old_value": None,
},
"owner": {
"display_name": "Delivery Owner",
"new_value": experiment.owner.id,
"old_value": None,
},
"projects": {
"display_name": "Projects",
"new_value": [{
"slug": project.slug
}],
"old_value": None,
},
"public_description": {
"display_name": "Public Description",
"new_value": "Let us learn new public things",
"old_value": None,
},
"related_to": {
"display_name": "Related Deliveries",
"new_value": [related_exp.id],
"old_value": None,
},
"short_description": {
"display_name": "Internal Description",
"new_value": "Let us learn new things",
"old_value": None,
},
}
self.maxDiff = None
self.assertEqual(expected_data, latest_changes.changed_values)
def test_variants_is_not_complete_when_no_variants_saved(self):
experiment = ExperimentFactory.create()
self.assertFalse(experiment.completed_variants)
def test_completed_results_returns_true_if_any_results(self):
experiment = ExperimentFactory.create(
results_initial="The results here were great."
)
self.assertTrue(experiment.completed_results)