def test_update_experiment_status_error(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.ACCEPTED, )
response = self.query(
UPDATE_EXPERIMENT_STATUS_MUTATION,
variables={
"input": {
"nimbusExperimentId": experiment.id,
"clientMutationId": "randomid",
"status": NimbusExperiment.Status.REVIEW.name,
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
result = content["data"]["updateExperimentStatus"]
self.assertEqual(
result["message"],
{
"experiment": [
"Nimbus Experiment has status 'Accepted', but can only be "
"changed when set to 'Draft'."
]
},
)
def test_update_experiment_error(self):
user_email = "*****@*****.**"
long_name = "test" * 1000
experiment = NimbusExperimentFactory.create(status=NimbusExperiment.Status.DRAFT)
response = self.query(
UPDATE_EXPERIMENT_MUTATION,
variables={
"input": {
"id": experiment.id,
"name": long_name,
"hypothesis": "new hypothesis",
"changelogMessage": "test changelog message",
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
result = content["data"]["updateExperiment"]
self.assertEqual(
result["message"],
{
"name": ["Ensure this field has no more than 255 characters."],
},
)
def test_update_experiment_outcomes_error(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT,
application=NimbusExperiment.Application.DESKTOP,
primary_outcomes=[],
secondary_outcomes=[],
)
response = self.query(
UPDATE_EXPERIMENT_MUTATION,
variables={
"input": {
"id": experiment.id,
"primaryOutcomes": ["invalid-outcome"],
"secondaryOutcomes": ["invalid-outcome"],
"changelogMessage": "test changelog message",
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200, response.content)
content = json.loads(response.content)
result = content["data"]["updateExperiment"]
self.assertEqual(
result["message"],
{
"primary_outcomes": [
"Invalid choices for primary outcomes: {'invalid-outcome'}"
],
"secondary_outcomes": [
"Invalid choices for secondary outcomes: {'invalid-outcome'}"
],
},
)
def test_update_experiment_branches_with_feature_config_error(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(status=NimbusExperiment.Status.DRAFT)
reference_branch = {"name": "control", "description": "a control", "ratio": 1}
treatment_branches = [{"name": "treatment1", "description": "desc1", "ratio": 1}]
invalid_feature_config_id = (
NimbusFeatureConfig.objects.all().order_by("-id").first().id + 1
)
response = self.query(
UPDATE_EXPERIMENT_MUTATION,
variables={
"input": {
"id": experiment.id,
"featureConfigId": invalid_feature_config_id,
"referenceBranch": reference_branch,
"treatmentBranches": treatment_branches,
"changelogMessage": "test changelog message",
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
result = content["data"]["updateExperiment"]
self.assertEqual(
result["message"],
{
"feature_config": [
f'Invalid pk "{invalid_feature_config_id}" - object does not exist.'
]
},
)
def test_update_experiment_outcomes(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT,
application=NimbusExperiment.Application.DESKTOP,
primary_outcomes=[],
secondary_outcomes=[],
)
outcomes = [
o.slug for o in Outcomes.by_application(NimbusExperiment.Application.DESKTOP)
]
primary_outcomes = outcomes[: NimbusExperiment.MAX_PRIMARY_OUTCOMES]
secondary_outcomes = outcomes[NimbusExperiment.MAX_PRIMARY_OUTCOMES :]
response = self.query(
UPDATE_EXPERIMENT_MUTATION,
variables={
"input": {
"id": experiment.id,
"primaryOutcomes": primary_outcomes,
"secondaryOutcomes": secondary_outcomes,
"changelogMessage": "test changelog message",
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200, response.content)
experiment = NimbusExperiment.objects.get(slug=experiment.slug)
self.assertEqual(experiment.primary_outcomes, primary_outcomes)
self.assertEqual(experiment.secondary_outcomes, secondary_outcomes)
def test_update_experiment_status(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT, )
response = self.query(
UPDATE_EXPERIMENT_STATUS_MUTATION,
variables={
"input": {
"nimbusExperimentId": experiment.id,
"clientMutationId": "randomid",
"status": NimbusExperiment.Status.REVIEW.name,
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
result = content["data"]["updateExperimentStatus"]
self.assertEqual(
result["nimbusExperiment"],
{
"id": str(experiment.id),
"status": NimbusExperiment.Status.REVIEW.name,
},
)
experiment = NimbusExperiment.objects.get(id=experiment.id)
self.assertEqual(experiment.status, NimbusExperiment.Status.REVIEW)
def test_update_experiment_branches_with_feature_config(self):
user_email = "*****@*****.**"
feature = NimbusFeatureConfigFactory(
schema="{}", application=NimbusExperiment.Application.FENIX
)
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT,
application=NimbusExperiment.Application.FENIX,
)
experiment_id = experiment.id
reference_branch = {"name": "control", "description": "a control", "ratio": 1}
treatment_branches = [{"name": "treatment1", "description": "desc1", "ratio": 1}]
response = self.query(
UPDATE_EXPERIMENT_MUTATION,
variables={
"input": {
"id": experiment.id,
"featureConfigId": feature.id,
"referenceBranch": reference_branch,
"treatmentBranches": treatment_branches,
"changelogMessage": "test changelog message",
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
experiment = NimbusExperiment.objects.get(id=experiment_id)
self.assertEqual(experiment.feature_config, feature)
self.assertEqual(experiment.branches.count(), 2)
self.assertEqual(experiment.reference_branch.name, reference_branch["name"])
treatment_branch = experiment.treatment_branches[0]
self.assertEqual(treatment_branch.name, treatment_branches[0]["name"])
def test_update_experiment_audience_error(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT,
channel=NimbusExperiment.Channel.NO_CHANNEL,
firefox_min_version=NimbusExperiment.Channel.NO_CHANNEL,
population_percent=0.0,
proposed_duration=0,
proposed_enrollment=0,
targeting_config_slug=NimbusExperiment.TargetingConfig.NO_TARGETING,
total_enrolled_clients=0,
)
response = self.query(
UPDATE_EXPERIMENT_MUTATION,
variables={
"input": {
"id": experiment.id,
"populationPercent": "10.23471",
"changelogMessage": "test changelog message",
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
result = content["data"]["updateExperiment"]
self.assertEqual(
result["message"],
{
"population_percent": [
"Ensure that there are no more than 4 decimal places."
]
},
)
def test_update_experiment_probe_sets_error(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT, probe_sets=[])
response = self.query(
UPDATE_EXPERIMENT_PROBESETS_MUTATION,
variables={
"input": {
"nimbusExperimentId": experiment.id,
"clientMutationId": "randomid",
"primaryProbeSetIds": [123],
"secondaryProbeSetIds": [],
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
result = content["data"]["updateExperimentProbeSets"]
self.assertEqual(
result["message"],
{
"primary_probe_sets":
['Invalid pk "123" - object does not exist.']
},
)
def test_end_experiment_fails_with_nonlive_status(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT,
)
response = self.query(
UPDATE_EXPERIMENT_MUTATION,
variables={
"input": {
"id": experiment.id,
"publishStatus": NimbusExperiment.PublishStatus.REVIEW.name,
"statusNext": NimbusExperiment.Status.COMPLETE.name,
"changelogMessage": "test changelog message",
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200, response.content)
content = json.loads(response.content)
result = content["data"]["updateExperiment"]
self.assertEqual(
result["message"],
{
"status_next": [
"Invalid choice for status_next: 'Complete' - with status 'Draft',"
" the only valid choices are 'None, Live'"
]
},
)
def test_update_experiment_branches_with_feature_config(self):
user_email = "*****@*****.**"
feature = NimbusFeatureConfigFactory(schema="{}")
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT)
experiment_id = experiment.id
reference_branch = {
"name": "control",
"description": "a control",
"ratio": 1
}
treatment_branches = [{
"name": "treatment1",
"description": "desc1",
"ratio": 1
}]
response = self.query(
UPDATE_EXPERIMENT_BRANCHES_MUTATION,
variables={
"input": {
"nimbusExperimentId": experiment.id,
"featureConfigId": feature.id,
"clientMutationId": "randomid",
"referenceBranch": reference_branch,
"treatmentBranches": treatment_branches,
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
result = content["data"]["updateExperimentBranches"]
self.assertEqual(
result["nimbusExperiment"],
{
"id": str(experiment.id),
"featureConfig": {
"name": feature.name
},
"status": experiment.status.upper(),
"name": experiment.name,
"slug": experiment.slug,
"referenceBranch": reference_branch,
"treatmentBranches": treatment_branches,
},
)
experiment = NimbusExperiment.objects.get(id=experiment_id)
self.assertEqual(experiment.feature_config, feature)
self.assertEqual(experiment.branches.count(), 2)
self.assertEqual(experiment.reference_branch.name,
reference_branch["name"])
treatment_branch = experiment.treatment_branches[0]
self.assertEqual(treatment_branch.name, treatment_branches[0]["name"])
def test_update_experiment_audience(self):
user_email = "*****@*****.**"
country = CountryFactory.create()
locale = LocaleFactory.create()
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT,
channel=NimbusExperiment.Channel.NO_CHANNEL,
application=NimbusConstants.Application.DESKTOP,
firefox_min_version=NimbusExperiment.Version.NO_VERSION,
population_percent=0.0,
proposed_duration=0,
proposed_enrollment=0,
targeting_config_slug=NimbusExperiment.TargetingConfig.NO_TARGETING,
total_enrolled_clients=0,
)
self.query(
UPDATE_EXPERIMENT_MUTATION,
variables={
"input": {
"id": experiment.id,
"channel": NimbusConstants.Channel.BETA.name,
"firefoxMinVersion": NimbusConstants.Version.FIREFOX_83.name,
"populationPercent": "10",
"proposedDuration": 42,
"proposedEnrollment": 120,
"targetingConfigSlug": (
NimbusConstants.TargetingConfig.ALL_ENGLISH.name
),
"totalEnrolledClients": 100,
"changelogMessage": "test changelog message",
"countries": [country.id],
"locales": [locale.id],
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
experiment = NimbusExperiment.objects.get(id=experiment.id)
self.assertEqual(experiment.channel, NimbusConstants.Channel.BETA.value)
self.assertEqual(
experiment.firefox_min_version, NimbusConstants.Version.FIREFOX_83.value
)
self.assertEqual(experiment.population_percent, 10.0)
self.assertEqual(experiment.proposed_duration, 42)
self.assertEqual(experiment.proposed_enrollment, 120)
self.assertEqual(
experiment.targeting_config_slug,
NimbusConstants.TargetingConfig.ALL_ENGLISH.value,
)
self.assertEqual(experiment.total_enrolled_clients, 100)
self.assertEqual(list(experiment.countries.all()), [country])
self.assertEqual(list(experiment.locales.all()), [locale])
def test_update_experiment_overview(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT,
slug="old slug",
name="old name",
hypothesis="old hypothesis",
public_description="old public description",
)
response = self.query(
UPDATE_EXPERIMENT_OVERVIEW_MUTATION,
variables={
"input": {
"id": experiment.id,
"name": "new name",
"hypothesis": "new hypothesis",
"publicDescription": "new public description",
"clientMutationId": "randomid",
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200, response.content)
content = json.loads(response.content)
result = content["data"]["updateExperimentOverview"]
experiment_result = result["nimbusExperiment"]
self.assertEqual(
experiment_result,
{
"id": f"{experiment.id}",
"name": "new name",
"hypothesis": "new hypothesis",
"publicDescription": "new public description",
},
)
self.assertEqual(result["clientMutationId"], "randomid")
self.assertEqual(result["message"], "success")
self.assertEqual(result["status"], 200)
experiment = NimbusExperiment.objects.first()
self.assertEqual(experiment.slug, "old slug")
self.assertEqual(experiment.name, "new name")
self.assertEqual(experiment.hypothesis, "new hypothesis")
self.assertEqual(experiment.public_description,
"new public description")
def test_update_experiment_documentation_links(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(status=NimbusExperiment.Status.DRAFT)
experiment_id = experiment.id
documentation_links = [
{
"title": NimbusExperiment.DocumentationLink.DS_JIRA.value,
"link": "https://example.com/bar",
},
{
"title": NimbusExperiment.DocumentationLink.ENG_TICKET.value,
"link": "https://example.com/quux",
},
{
"title": NimbusExperiment.DocumentationLink.DESIGN_DOC.value,
"link": "https://example.com/plotz",
},
]
response = self.query(
UPDATE_EXPERIMENT_MUTATION,
variables={
"input": {
"id": experiment.id,
"documentationLinks": documentation_links,
"changelogMessage": "test changelog message",
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
experiment = NimbusExperiment.objects.get(id=experiment_id)
experiment_links = experiment.documentation_links.all()
for key in (
"title",
"link",
):
self.assertEqual(
{b[key] for b in documentation_links},
{getattr(b, key) for b in experiment_links},
)
def test_update_experiment_publish_status(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(
publish_status=NimbusExperiment.PublishStatus.IDLE,
)
response = self.query(
UPDATE_EXPERIMENT_MUTATION,
variables={
"input": {
"id": experiment.id,
"publishStatus": NimbusExperiment.PublishStatus.REVIEW.name,
"changelogMessage": "test changelog message",
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
experiment = NimbusExperiment.objects.get(id=experiment.id)
self.assertEqual(experiment.publish_status, NimbusExperiment.PublishStatus.REVIEW)
def test_update_experiment_overview(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT,
slug="old slug",
name="old name",
hypothesis="old hypothesis",
public_description="old public description",
risk_brand=False,
risk_revenue=False,
risk_partner_related=False,
)
response = self.query(
UPDATE_EXPERIMENT_MUTATION,
variables={
"input": {
"id": experiment.id,
"name": "new name",
"hypothesis": "new hypothesis",
"publicDescription": "new public description",
"changelogMessage": "test changelog message",
"riskBrand": True,
"riskRevenue": True,
"riskPartnerRelated": True,
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200, response.content)
content = json.loads(response.content)
result = content["data"]["updateExperiment"]
self.assertEqual(result["message"], "success")
experiment = NimbusExperiment.objects.first()
self.assertEqual(experiment.slug, "old slug")
self.assertEqual(experiment.name, "new name")
self.assertEqual(experiment.hypothesis, "new hypothesis")
self.assertEqual(experiment.public_description, "new public description")
self.assertEqual(experiment.risk_brand, True)
self.assertEqual(experiment.risk_revenue, True)
self.assertEqual(experiment.risk_partner_related, True)
def test_update_experiment_probe_sets(self):
user_email = "*****@*****.**"
probe_sets = [NimbusProbeSetFactory() for i in range(3)]
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT, probe_sets=[])
response = self.query(
UPDATE_EXPERIMENT_PROBESETS_MUTATION,
variables={
"input": {
"nimbusExperimentId": experiment.id,
"clientMutationId": "randomid",
"primaryProbeSetIds": [p.id for p in probe_sets[:2]],
"secondaryProbeSetIds": [p.id for p in probe_sets[2:]],
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
result = content["data"]["updateExperimentProbeSets"]
self.assertEqual(
result["nimbusExperiment"],
{
"id":
str(experiment.id),
"primaryProbeSets": [{
"name": probe_set.name,
"id": str(probe_set.id)
} for probe_set in probe_sets[:2]],
"secondaryProbeSets": [{
"name": probe_set.name,
"id": str(probe_set.id)
} for probe_set in probe_sets[2:]],
},
)
experiment = NimbusExperiment.objects.get(id=experiment.id)
self.assertEqual(set(experiment.probe_sets.all()), set(probe_sets))
# These must be compared as lists to ensure ordering is retained.
self.assertEqual(list(experiment.primary_probe_sets), probe_sets[:2])
self.assertEqual(list(experiment.secondary_probe_sets), probe_sets[2:])
def test_launch_experiment_valid_with_preview_status(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.PREVIEW,
)
response = self.query(
UPDATE_EXPERIMENT_MUTATION,
variables={
"input": {
"id": experiment.id,
"status": NimbusExperiment.Status.DRAFT.name,
"statusNext": NimbusExperiment.Status.LIVE.name,
"publishStatus": NimbusExperiment.PublishStatus.REVIEW.name,
"changelogMessage": "test changelog message",
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200, response.content)
content = json.loads(response.content)
result = content["data"]["updateExperiment"]
self.assertEqual(result["message"], "success")
def test_update_experiment_branches_with_feature_config_error(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT)
reference_branch = {
"name": "control",
"description": "a control",
"ratio": 1
}
treatment_branches = [{
"name": "treatment1",
"description": "desc1",
"ratio": 1
}]
# The NimbusExperimentFactory always creates a single feature config.
self.assertEqual(NimbusFeatureConfig.objects.count(), 1)
response = self.query(
UPDATE_EXPERIMENT_BRANCHES_MUTATION,
variables={
"input": {
"nimbusExperimentId": experiment.id,
"featureConfigId": 2,
"clientMutationId": "randomid",
"referenceBranch": reference_branch,
"treatmentBranches": treatment_branches,
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
result = content["data"]["updateExperimentBranches"]
self.assertEqual(
result["message"],
{"feature_config": ['Invalid pk "2" - object does not exist.']},
)
def test_get_experiment_type_formats_nimbus_exp_properly(self):
exp = NimbusExperimentFactory.create(
application=NimbusExperiment.Application.DESKTOP)
reportlog_type = get_experiment_type(exp)
self.assertEqual(reportlog_type, "Nimbus-Firefox-Desktop")
def test_update_experiment_audience(self):
user_email = "*****@*****.**"
experiment = NimbusExperimentFactory.create(
status=NimbusExperiment.Status.DRAFT,
channel=None,
application=NimbusConstants.Application.DESKTOP,
firefox_min_version=None,
population_percent=0.0,
proposed_duration=None,
proposed_enrollment=None,
targeting_config_slug=None,
total_enrolled_clients=0,
)
response = self.query(
UPDATE_EXPERIMENT_AUDIENCE_MUTATION,
variables={
"input": {
"nimbusExperimentId":
experiment.id,
"clientMutationId":
"randomid",
"channel":
NimbusConstants.Channel.DESKTOP_BETA.name,
"firefoxMinVersion":
NimbusConstants.Version.FIREFOX_80.name,
"populationPercent":
"10",
"proposedDuration":
42,
"proposedEnrollment":
120,
"targetingConfigSlug":
(NimbusConstants.TargetingConfig.ALL_ENGLISH.name),
"totalEnrolledClients":
100,
}
},
headers={settings.OPENIDC_EMAIL_HEADER: user_email},
)
content = json.loads(response.content)
result = content["data"]["updateExperimentAudience"]
self.assertEqual(
result["nimbusExperiment"],
{
"id":
str(experiment.id),
"channel":
NimbusConstants.Channel.DESKTOP_BETA.name,
"firefoxMinVersion":
NimbusConstants.Version.FIREFOX_80.name,
"populationPercent":
"10.0000",
"proposedDuration":
42,
"proposedEnrollment":
120,
"targetingConfigSlug":
NimbusConstants.TargetingConfig.ALL_ENGLISH.name,
"totalEnrolledClients":
100,
},
)
experiment = NimbusExperiment.objects.get(id=experiment.id)
self.assertEqual(experiment.channel,
NimbusConstants.Channel.DESKTOP_BETA.value)
self.assertEqual(experiment.firefox_min_version,
NimbusConstants.Version.FIREFOX_80.value)
self.assertEqual(experiment.population_percent, 10.0)
self.assertEqual(experiment.proposed_duration, 42)
self.assertEqual(experiment.proposed_enrollment, 120)
self.assertEqual(
experiment.targeting_config_slug,
NimbusConstants.TargetingConfig.ALL_ENGLISH.value,
)
self.assertEqual(experiment.total_enrolled_clients, 100)