def nimbus_check_experiments_are_live():
"""
A scheduled task that checks the kinto collection for any experiment slugs that are
present in the collection but are not yet marked as live in the database and marks
them as live.
"""
metrics.incr("check_experiments_are_live.started")
accepted_experiments = NimbusExperiment.objects.filter(
status=NimbusExperiment.Status.ACCEPTED)
for collection in NimbusExperiment.KINTO_APPLICATION_COLLECTION.values():
kinto_client = KintoClient(collection)
records = kinto_client.get_main_records()
record_ids = [r.get("id") for r in records]
for experiment in accepted_experiments:
if experiment.slug in record_ids:
logger.info(
f"{experiment} status is being updated to live".format(
experiment=experiment))
experiment.status = NimbusExperiment.Status.LIVE
experiment.save()
generate_nimbus_changelog(experiment, get_kinto_user())
logger.info(f"{experiment} status is set to Live")
metrics.incr("check_experiments_are_live.completed")
def nimbus_push_experiment_to_kinto(collection, experiment_id):
"""
An invoked task that given a single experiment id, query it in the db, serialize it,
and push its data to the configured collection. If it fails for any reason, log the
error and reraise it so it will be forwarded to sentry.
"""
metrics.incr("push_experiment_to_kinto.started")
try:
experiment = NimbusExperiment.objects.get(id=experiment_id)
logger.info(f"Pushing {experiment.slug} to Kinto")
kinto_client = KintoClient(collection)
data = NimbusExperimentSerializer(experiment).data
kinto_client.create_record(data)
experiment.publish_status = NimbusExperiment.PublishStatus.WAITING
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=NimbusChangeLog.Messages.LAUNCHING_TO_KINTO,
)
logger.info(f"{experiment.slug} pushed to Kinto")
metrics.incr("push_experiment_to_kinto.completed")
except Exception as e:
metrics.incr("push_experiment_to_kinto.failed")
logger.info(
f"Pushing experiment {experiment.slug} to Kinto failed: {e}")
raise e
def nimbus_check_experiments_are_complete():
"""
A scheduled task that checks the kinto collection for any experiment slugs that are
marked as live in the database but missing from the collection, indicating that they
are no longer live and can be marked as complete.
"""
metrics.incr("check_experiments_are_complete.started")
for (
collection,
applications,
) in NimbusExperiment.KINTO_COLLECTION_APPLICATIONS.items():
kinto_client = KintoClient(collection)
live_experiments = NimbusExperiment.objects.filter(
status=NimbusExperiment.Status.LIVE,
application__in=applications,
)
records = kinto_client.get_main_records()
for experiment in live_experiments:
if (experiment.should_end and not experiment.emails.filter(
type=NimbusExperiment.EmailType.EXPERIMENT_END).exists()):
nimbus_send_experiment_ending_email(experiment)
if experiment.slug not in records:
logger.info(
f"{experiment.slug} status is being updated to complete".
format(experiment=experiment))
experiment.status = NimbusExperiment.Status.COMPLETE
experiment.status_next = None
experiment.publish_status = NimbusExperiment.PublishStatus.IDLE
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=NimbusChangeLog.Messages.COMPLETED,
)
logger.info(f"{experiment.slug} status is set to Complete")
metrics.incr("check_experiments_are_complete.completed")
def test_update_record_updates_record_patches_collection(
self, review, status):
client = KintoClient(self.collection, review=review)
data = {"id": "my-record", "field": "value"}
client.update_record(data)
self.mock_kinto_client.update_record.assert_called_with(
data=data,
collection=self.collection,
bucket=settings.KINTO_BUCKET_WORKSPACE,
if_match='"0"',
)
self.mock_kinto_client.patch_collection.assert_called_with(
id=self.collection,
data={"status": status},
bucket=settings.KINTO_BUCKET_WORKSPACE,
)
def nimbus_push_experiment_to_kinto(experiment_id):
"""
An invoked task that given a single experiment id, query it in the db, serialize it,
and push its data to the configured collection. If it fails for any reason, log the
error and reraise it so it will be forwarded to sentry.
"""
metrics.incr("push_experiment_to_kinto.started")
try:
experiment = NimbusExperiment.objects.get(id=experiment_id)
logger.info(f"Pushing {experiment} to Kinto")
kinto_client = KintoClient(
NimbusExperiment.KINTO_APPLICATION_COLLECTION[
experiment.application])
if not NimbusBucketRange.objects.filter(
experiment=experiment).exists():
NimbusIsolationGroup.request_isolation_group_buckets(
experiment.slug,
experiment,
int(experiment.population_percent / Decimal("100.0") *
NimbusExperiment.BUCKET_TOTAL),
)
data = NimbusExperimentSerializer(experiment).data
kinto_client.push_to_kinto(data)
experiment.status = NimbusExperiment.Status.ACCEPTED
experiment.save()
generate_nimbus_changelog(experiment, get_kinto_user())
logger.info(f"{experiment} pushed to Kinto")
metrics.incr("push_experiment_to_kinto.completed")
except Exception as e:
metrics.incr("push_experiment_to_kinto.failed")
logger.info(
f"Pushing experiment id {experiment_id} to Kinto failed: {e}")
raise e
def nimbus_check_experiments_are_complete():
"""
A scheduled task that checks the kinto collection for any experiment slugs that are
marked as live in the database but missing from the collection, indicating that they
are no longer live and can be marked as complete.
"""
metrics.incr("check_experiments_are_complete.started")
for application, collection in NimbusExperiment.KINTO_APPLICATION_COLLECTION.items(
):
kinto_client = KintoClient(collection)
live_experiments = NimbusExperiment.objects.filter(
status=NimbusExperiment.Status.LIVE,
application=application,
)
records = kinto_client.get_main_records()
record_ids = [r.get("id") for r in records]
print("found record ids", record_ids)
for experiment in live_experiments:
print("checking", experiment)
if (experiment.should_end and not experiment.emails.filter(
type=NimbusExperiment.EmailType.EXPERIMENT_END).exists()):
nimbus_send_experiment_ending_email(experiment)
if experiment.slug not in record_ids:
logger.info(
f"{experiment} status is being updated to complete".format(
experiment=experiment))
experiment.status = NimbusExperiment.Status.COMPLETE
experiment.save()
generate_nimbus_changelog(experiment, get_kinto_user())
logger.info(f"{experiment} status is set to Complete")
metrics.incr("check_experiments_are_complete.completed")
def test_create_record_creates_record_patches_collection(
self, review, status):
client = KintoClient(self.collection, review=review)
client.create_record({"test": "data"})
self.mock_kinto_client_creator.assert_called_with(
server_url=settings.KINTO_HOST,
auth=(settings.KINTO_USER, settings.KINTO_PASS),
)
self.mock_kinto_client.create_record.assert_called_with(
data={"test": "data"},
collection=self.collection,
bucket=settings.KINTO_BUCKET_WORKSPACE,
if_not_exists=True,
)
self.mock_kinto_client.patch_collection.assert_called_with(
id=self.collection,
data={"status": status},
bucket=settings.KINTO_BUCKET_WORKSPACE,
)
def test_delete_record_deletes_record_patches_collection(
self, review, status):
client = KintoClient(self.collection, review=review)
record_id = "abc-123"
client.delete_record(record_id)
self.mock_kinto_client_creator.assert_called_with(
server_url=settings.KINTO_HOST,
auth=(settings.KINTO_USER, settings.KINTO_PASS),
)
self.mock_kinto_client.delete_record.assert_called_with(
id=record_id,
collection=self.collection,
bucket=settings.KINTO_BUCKET_WORKSPACE,
)
self.mock_kinto_client.patch_collection.assert_called_with(
id=self.collection,
data={"status": status},
bucket=settings.KINTO_BUCKET_WORKSPACE,
)
def nimbus_check_experiments_are_live():
"""
A scheduled task that checks the kinto collection for any experiment slugs that are
present in the collection but are not yet marked as live in the database and marks
them as live.
"""
metrics.incr("check_experiments_are_live.started")
for (
collection,
applications,
) in NimbusExperiment.KINTO_COLLECTION_APPLICATIONS.items():
kinto_client = KintoClient(collection)
records = kinto_client.get_main_records()
for experiment in NimbusExperiment.objects.waiting_to_launch_queue(
applications):
if experiment.slug in records:
logger.info(
f"{experiment} status is being updated to live".format(
experiment=experiment))
experiment.status = NimbusExperiment.Status.LIVE
experiment.status_next = None
experiment.publish_status = NimbusExperiment.PublishStatus.IDLE
experiment.published_dto = records[experiment.slug]
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=NimbusChangeLog.Messages.LIVE,
)
logger.info(f"{experiment.slug} status is set to Live")
metrics.incr("check_experiments_are_live.completed")
def nimbus_check_kinto_push_queue():
"""
Because kinto has a restriction that it can only have a single pending review, this
task brokers the queue of all experiments ready to be pushed to kinto and ensures
that only a single experiment is ever in review.
A scheduled task that
- Checks the kinto collection for a single rejected experiment from a previous push
- If one exists, pull it out of the collection and mark it as rejected
- Checks if there is still a pending review and if so, aborts
- Gets the list of all experiments ready to be pushed to kinto and pushes the first
one
"""
metrics.incr("check_kinto_push_queue.started")
for application, collection in NimbusExperiment.KINTO_APPLICATION_COLLECTION.items(
):
kinto_client = KintoClient(collection)
rejected_collection_data = kinto_client.get_rejected_collection_data()
if rejected_collection_data:
rejected_slug = kinto_client.get_rejected_record()
experiment = NimbusExperiment.objects.get(slug=rejected_slug)
experiment.status = NimbusExperiment.Status.DRAFT
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=
f'Rejected: {rejected_collection_data["last_reviewer_comment"]}',
)
kinto_client.rollback_changes()
if kinto_client.has_pending_review():
metrics.incr(f"check_kinto_push_queue.{collection}_pending_review")
return
queued_experiments = NimbusExperiment.objects.filter(
status=NimbusExperiment.Status.REVIEW, application=application)
if queued_experiments.exists():
nimbus_push_experiment_to_kinto.delay(
queued_experiments.first().id)
metrics.incr(
f"check_kinto_push_queue.{collection}_queued_experiment_selected"
)
else:
metrics.incr(
f"check_kinto_push_queue.{collection}_no_experiments_queued")
metrics.incr("check_kinto_push_queue.completed")
def nimbus_check_kinto_push_queue_by_collection(collection):
"""
Because kinto has a restriction that it can only have a single pending review, this
task brokers the queue of all experiments ready to be pushed to kinto and ensures
that only a single experiment is ever in review.
A scheduled task that
- Checks the kinto collection for a single rejected experiment from a previous push
- If one exists, pull it out of the collection and mark it as rejected
- Checks if there is still a pending review and if so, aborts
- Gets the list of all experiments ready to be pushed to kinto and pushes the first
one
- Checks for experiments that should be paused but are not paused in the kinto
collection and marks them as paused and updates the record in the collection.
"""
metrics.incr(f"check_kinto_push_queue_by_collection:{collection}.started")
applications = NimbusExperiment.KINTO_COLLECTION_APPLICATIONS[collection]
kinto_client = KintoClient(collection)
should_rollback = False
if kinto_client.has_pending_review():
logger.info(f"{collection} has pending review")
should_abort = handle_pending_review(applications)
if should_abort:
return
should_rollback = True
if kinto_client.has_rejection():
logger.info(f"{collection} has rejection")
handle_rejection(applications, kinto_client)
should_rollback = True
if should_rollback:
kinto_client.rollback_changes()
records = kinto_client.get_main_records()
handle_launching_experiments(applications, records)
handle_updating_experiments(applications, records)
handle_ending_experiments(applications, records)
handle_waiting_experiments(applications)
if queued_launch_experiment := NimbusExperiment.objects.launch_queue(
applications).first():
nimbus_push_experiment_to_kinto.delay(collection,
queued_launch_experiment.id)
def nimbus_synchronize_preview_experiments_in_kinto():
"""
A scheduled task that pushes any experiments with status PREVIEW to the preview
collection and removes any experiments not with status PREVIEW from the preview
collection.
"""
metrics.incr("nimbus_synchronize_preview_experiments_in_kinto.started")
kinto_client = KintoClient(settings.KINTO_COLLECTION_NIMBUS_PREVIEW,
review=False)
try:
published_preview_slugs = kinto_client.get_main_records().keys()
should_publish_experiments = NimbusExperiment.objects.filter(
status=NimbusExperiment.Status.PREVIEW).exclude(
slug__in=published_preview_slugs)
for experiment in should_publish_experiments:
data = NimbusExperimentSerializer(experiment).data
kinto_client.create_record(data)
logger.info(f"{experiment.slug} is being pushed to preview")
should_unpublish_experiments = NimbusExperiment.objects.filter(
slug__in=published_preview_slugs).exclude(
status=NimbusExperiment.Status.PREVIEW)
for experiment in should_unpublish_experiments:
kinto_client.delete_record(experiment.slug)
logger.info(f"{experiment.slug} is being removed from preview")
metrics.incr(
"nimbus_synchronize_preview_experiments_in_kinto.completed")
except Exception as e:
metrics.incr("nimbus_synchronize_preview_experiments_in_kinto.failed")
logger.info(f"Synchronizing preview experiments failed: {e}")
raise e
def setUp(self):
super().setUp()
self.collection = "test-collection"
self.client = KintoClient(self.collection)
class TestKintoClient(MockKintoClientMixin, TestCase):
def setUp(self):
super().setUp()
self.collection = "test-collection"
self.client = KintoClient(self.collection)
@parameterized.expand([
[False, KINTO_SIGN_STATUS],
[True, KINTO_REVIEW_STATUS],
])
def test_create_record_creates_record_patches_collection(
self, review, status):
client = KintoClient(self.collection, review=review)
client.create_record({"test": "data"})
self.mock_kinto_client_creator.assert_called_with(
server_url=settings.KINTO_HOST,
auth=(settings.KINTO_USER, settings.KINTO_PASS),
)
self.mock_kinto_client.create_record.assert_called_with(
data={"test": "data"},
collection=self.collection,
bucket=settings.KINTO_BUCKET_WORKSPACE,
if_not_exists=True,
)
self.mock_kinto_client.patch_collection.assert_called_with(
id=self.collection,
data={"status": status},
bucket=settings.KINTO_BUCKET_WORKSPACE,
)
@parameterized.expand([
[False, KINTO_SIGN_STATUS],
[True, KINTO_REVIEW_STATUS],
])
def test_update_record_updates_record_patches_collection(
self, review, status):
client = KintoClient(self.collection, review=review)
data = {"id": "my-record", "field": "value"}
client.update_record(data)
self.mock_kinto_client.update_record.assert_called_with(
data=data,
collection=self.collection,
bucket=settings.KINTO_BUCKET_WORKSPACE,
if_match='"0"',
)
self.mock_kinto_client.patch_collection.assert_called_with(
id=self.collection,
data={"status": status},
bucket=settings.KINTO_BUCKET_WORKSPACE,
)
@parameterized.expand([
[False, KINTO_SIGN_STATUS],
[True, KINTO_REVIEW_STATUS],
])
def test_delete_record_deletes_record_patches_collection(
self, review, status):
client = KintoClient(self.collection, review=review)
record_id = "abc-123"
client.delete_record(record_id)
self.mock_kinto_client_creator.assert_called_with(
server_url=settings.KINTO_HOST,
auth=(settings.KINTO_USER, settings.KINTO_PASS),
)
self.mock_kinto_client.delete_record.assert_called_with(
id=record_id,
collection=self.collection,
bucket=settings.KINTO_BUCKET_WORKSPACE,
)
self.mock_kinto_client.patch_collection.assert_called_with(
id=self.collection,
data={"status": status},
bucket=settings.KINTO_BUCKET_WORKSPACE,
)
def test_rollback_changes_patches_collection(self):
self.client.rollback_changes()
self.mock_kinto_client_creator.assert_called_with(
server_url=settings.KINTO_HOST,
auth=(settings.KINTO_USER, settings.KINTO_PASS),
)
self.mock_kinto_client.patch_collection.assert_called_with(
id=self.collection,
data={"status": KINTO_ROLLBACK_STATUS},
bucket=settings.KINTO_BUCKET_WORKSPACE,
)
def test_returns_true_for_pending_review(self):
self.setup_kinto_pending_review()
self.assertTrue(self.client.has_pending_review())
def test_returns_false_for_no_pending_review(self):
self.setup_kinto_no_pending_review()
self.assertFalse(self.client.has_pending_review())
def test_returns_records(self):
slug = "test-slug"
self.setup_kinto_get_main_records([slug])
self.assertEqual(self.client.get_main_records(),
{slug: {
"id": slug,
"last_modified": "0"
}})
def test_returns_no_records(self):
self.setup_kinto_get_main_records([])
self.assertEqual(self.client.get_main_records(), {})
def test_returns_nothing_when_not_rejects(self):
self.setup_kinto_no_pending_review()
self.assertIsNone(self.client.get_rejected_collection_data())
def test_returns_rejected_data(self):
self.setup_kinto_rejected_review()
self.assertTrue(self.client.get_rejected_collection_data())
class TestKintoClient(MockKintoClientMixin, TestCase):
def setUp(self):
super().setUp()
self.collection = "test-collection"
self.client = KintoClient(self.collection)
def test_push_to_kinto_sends_data_updates_collection(self):
self.client.push_to_kinto({"test": "data"})
self.mock_kinto_client_creator.assert_called_with(
server_url=settings.KINTO_HOST,
auth=(settings.KINTO_USER, settings.KINTO_PASS),
)
self.mock_kinto_client.create_record.assert_called_with(
data={"test": "data"},
collection=self.collection,
bucket=settings.KINTO_BUCKET,
if_not_exists=True,
)
self.mock_kinto_client.patch_collection.assert_called_with(
id=self.collection,
data={"status": KINTO_REVIEW_STATUS},
bucket=settings.KINTO_BUCKET,
)
def test_rollback_changes_patches_collection(self):
self.client.rollback_changes()
self.mock_kinto_client_creator.assert_called_with(
server_url=settings.KINTO_HOST,
auth=(settings.KINTO_USER, settings.KINTO_PASS),
)
self.mock_kinto_client.patch_collection.assert_called_with(
id=self.collection,
data={"status": KINTO_ROLLBACK_STATUS},
bucket=settings.KINTO_BUCKET,
)
def test_returns_true_for_pending_review(self):
self.setup_kinto_pending_review()
self.assertTrue(self.client.has_pending_review())
def test_returns_false_for_no_pending_review(self):
self.setup_kinto_no_pending_review()
self.assertFalse(self.client.has_pending_review())
def test_returns_records(self):
slug = "test-slug"
self.setup_kinto_get_main_records([slug])
self.assertEqual(self.client.get_main_records(), [{"id": slug}])
def test_returns_no_records(self):
self.setup_kinto_get_main_records([])
self.assertEqual(self.client.get_main_records(), [])
def test_returns_nothing_when_not_rejects(self):
self.setup_kinto_no_pending_review()
self.assertIsNone(self.client.get_rejected_collection_data())
def test_returns_rejected_data(self):
self.setup_kinto_rejected_review()
self.assertTrue(self.client.get_rejected_collection_data())
def test_returns_rejected_record(self):
self.mock_kinto_client.get_records.side_effect = [
[{
"id": "bug-12345-rapid-test-release-55"
}],
[
{
"id": "bug-12345-rapid-test-release-55"
},
{
"id": "bug-9999-rapid-test-release-55"
},
],
]
self.assertEqual(self.client.get_rejected_record(),
"bug-9999-rapid-test-release-55")