def nimbus_push_experiment_to_kinto(collection, experiment_id):
"""
An invoked task that given a single experiment id, query it in the db, serialize it,
and push its data to the configured collection. If it fails for any reason, log the
error and reraise it so it will be forwarded to sentry.
"""
metrics.incr("push_experiment_to_kinto.started")
try:
experiment = NimbusExperiment.objects.get(id=experiment_id)
logger.info(f"Pushing {experiment.slug} to Kinto")
kinto_client = KintoClient(collection)
data = NimbusExperimentSerializer(experiment).data
kinto_client.create_record(data)
experiment.publish_status = NimbusExperiment.PublishStatus.WAITING
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=NimbusChangeLog.Messages.PUSHED_TO_KINTO,
)
logger.info(f"{experiment.slug} pushed to Kinto")
metrics.incr("push_experiment_to_kinto.completed")
except Exception as e:
metrics.incr("push_experiment_to_kinto.failed")
logger.info(
f"Pushing experiment {experiment.slug} to Kinto failed: {e}")
raise e
def test_generate_nimbus_changelog_with_prior_change(self):
experiment = NimbusExperimentFactory.create_with_lifecycle(
NimbusExperimentFactory.Lifecycles.CREATED)
self.assertEqual(experiment.changes.count(), 1)
experiment.status = NimbusExperiment.Status.DRAFT
experiment.status_next = NimbusExperiment.Status.LIVE
experiment.publish_status = NimbusExperiment.PublishStatus.REVIEW
experiment.save()
generate_nimbus_changelog(experiment, self.user, "test message")
self.assertEqual(experiment.changes.count(), 2)
change = experiment.changes.latest_change()
self.assertEqual(change.experiment, experiment)
self.assertEqual(change.message, "test message")
self.assertEqual(change.changed_by, self.user)
self.assertEqual(change.old_status, NimbusExperiment.Status.DRAFT)
self.assertEqual(change.old_status_next, None)
self.assertEqual(change.old_publish_status,
NimbusExperiment.PublishStatus.IDLE)
self.assertEqual(change.new_status, NimbusExperiment.Status.DRAFT)
self.assertEqual(change.new_status_next, NimbusExperiment.Status.LIVE)
self.assertEqual(change.new_publish_status,
NimbusExperiment.PublishStatus.REVIEW)
self.assertEqual(
change.experiment_data,
dict(NimbusExperimentChangeLogSerializer(experiment).data),
)
def nimbus_end_experiment_in_kinto(collection, experiment_id):
"""
An invoked task that given a single experiment id, delete its data from
the configured collection. If it fails for any reason, log the error and
reraise it so it will be forwarded to sentry.
"""
metrics.incr("end_experiment_in_kinto.started")
try:
experiment = NimbusExperiment.objects.get(id=experiment_id)
logger.info(f"Deleting {experiment.slug} from Kinto")
kinto_client = KintoClient(collection)
kinto_client.delete_record(experiment.slug)
experiment.publish_status = NimbusExperiment.PublishStatus.WAITING
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=NimbusChangeLog.Messages.DELETED_FROM_KINTO,
)
logger.info(f"{experiment.slug} deleted from Kinto")
metrics.incr("end_experiment_in_kinto.completed")
except Exception as e:
metrics.incr("end_experiment_in_kinto.failed")
logger.info(
f"Deleting experiment {experiment.slug} from Kinto failed: {e}")
raise e
def test_generate_nimbus_changelog_without_prior_change(self):
experiment = NimbusExperimentFactory.create()
self.assertEqual(experiment.changes.count(), 0)
generate_nimbus_changelog(experiment, self.user, "test message")
self.assertEqual(experiment.changes.count(), 1)
change = experiment.changes.get()
self.assertEqual(change.experiment, experiment)
self.assertEqual(change.message, "test message")
self.assertEqual(change.changed_by, self.user)
self.assertEqual(change.old_status, None)
self.assertEqual(change.old_status_next, None)
self.assertEqual(change.old_publish_status, None)
self.assertEqual(change.new_status, NimbusExperiment.Status.DRAFT)
self.assertEqual(change.new_status_next, None)
self.assertEqual(change.new_publish_status,
NimbusExperiment.PublishStatus.IDLE)
self.assertEqual(
change.experiment_data,
dict(NimbusExperimentChangeLogSerializer(experiment).data),
)
def nimbus_check_experiments_are_live():
"""
A scheduled task that checks the kinto collection for any experiment slugs that are
present in the collection but are not yet marked as live in the database and marks
them as live.
"""
metrics.incr("check_experiments_are_live.started")
accepted_experiments = NimbusExperiment.objects.filter(
status=NimbusExperiment.Status.ACCEPTED)
for collection in NimbusExperiment.KINTO_APPLICATION_COLLECTION.values():
kinto_client = KintoClient(collection)
records = kinto_client.get_main_records()
record_ids = [r.get("id") for r in records]
for experiment in accepted_experiments:
if experiment.slug in record_ids:
logger.info(
f"{experiment} status is being updated to live".format(
experiment=experiment))
experiment.status = NimbusExperiment.Status.LIVE
experiment.save()
generate_nimbus_changelog(experiment, get_kinto_user())
logger.info(f"{experiment} status is set to Live")
metrics.incr("check_experiments_are_live.completed")
def nimbus_update_experiment_in_kinto(collection, experiment_id):
"""
An invoked task that given a single experiment id, reserializes
and updates the record. If it fails for any reason, log the error and
reraise it so it will be forwarded to sentry.
"""
metrics.incr("update_experiment_in_kinto.started")
try:
experiment = NimbusExperiment.objects.get(id=experiment_id)
logger.info(f"Updating {experiment.slug} in Kinto")
kinto_client = KintoClient(collection)
data = NimbusExperimentSerializer(experiment).data
kinto_client.update_record(data)
experiment.publish_status = NimbusExperiment.PublishStatus.WAITING
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=NimbusChangeLog.Messages.UPDATED_IN_KINTO,
)
logger.info(f"{experiment.slug} updated in Kinto")
metrics.incr("update_experiment_in_kinto.completed")
except Exception as e:
metrics.incr("update_experiment_in_kinto.failed")
logger.info(
f"Updating experiment {experiment.slug} in Kinto failed: {e}")
raise e
def nimbus_check_kinto_push_queue():
"""
Because kinto has a restriction that it can only have a single pending review, this
task brokers the queue of all experiments ready to be pushed to kinto and ensures
that only a single experiment is ever in review.
A scheduled task that
- Checks the kinto collection for a single rejected experiment from a previous push
- If one exists, pull it out of the collection and mark it as rejected
- Checks if there is still a pending review and if so, aborts
- Gets the list of all experiments ready to be pushed to kinto and pushes the first
one
"""
metrics.incr("check_kinto_push_queue.started")
for application, collection in NimbusExperiment.KINTO_APPLICATION_COLLECTION.items(
):
kinto_client = KintoClient(collection)
rejected_collection_data = kinto_client.get_rejected_collection_data()
if rejected_collection_data:
rejected_slug = kinto_client.get_rejected_record()
experiment = NimbusExperiment.objects.get(slug=rejected_slug)
experiment.status = NimbusExperiment.Status.DRAFT
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=
f'Rejected: {rejected_collection_data["last_reviewer_comment"]}',
)
kinto_client.rollback_changes()
if kinto_client.has_pending_review():
metrics.incr(f"check_kinto_push_queue.{collection}_pending_review")
return
queued_experiments = NimbusExperiment.objects.filter(
status=NimbusExperiment.Status.REVIEW, application=application)
if queued_experiments.exists():
nimbus_push_experiment_to_kinto.delay(
queued_experiments.first().id)
metrics.incr(
f"check_kinto_push_queue.{collection}_queued_experiment_selected"
)
else:
metrics.incr(
f"check_kinto_push_queue.{collection}_no_experiments_queued")
metrics.incr("check_kinto_push_queue.completed")
def create_with_lifecycle(cls,
lifecycle,
with_random_timespan=False,
**kwargs):
experiment = cls.create(**kwargs)
now = timezone.now() - datetime.timedelta(
days=random.randint(100, 200))
for state in lifecycle.value:
experiment.apply_lifecycle_state(state)
if (experiment.status == experiment.Status.LIVE
and experiment.status_next is None
and "published_dto" not in kwargs):
experiment.published_dto = NimbusExperimentSerializer(
experiment).data
experiment.save()
if experiment.has_filter(
experiment.Filters.SHOULD_ALLOCATE_BUCKETS):
experiment.allocate_bucket_range()
change = generate_nimbus_changelog(
experiment,
experiment.owner,
f"set lifecycle {lifecycle} state {state}",
)
if with_random_timespan:
change.changed_on = now
change.save()
now += datetime.timedelta(days=random.randint(5, 20))
return NimbusExperiment.objects.get(id=experiment.id)
def handle_rejection(applications, kinto_client):
collection_data = kinto_client.get_rejected_collection_data()
experiment = NimbusExperiment.objects.waiting(applications).first()
if experiment:
experiment.publish_status = NimbusExperiment.PublishStatus.IDLE
experiment.status_next = None
experiment.is_paused = False
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=collection_data["last_reviewer_comment"],
)
logger.info(f"{experiment} rejected")
def handle_waiting_experiments(applications):
waiting_experiments = NimbusExperiment.objects.filter(
publish_status=NimbusExperiment.PublishStatus.WAITING,
application__in=applications,
)
for experiment in waiting_experiments:
experiment.status_next = None
experiment.publish_status = NimbusExperiment.PublishStatus.IDLE
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=NimbusChangeLog.Messages.REJECTED_FROM_KINTO,
)
logger.info(f"{experiment.slug} rejected without reason(rollback)")
def nimbus_check_experiments_are_complete():
"""
A scheduled task that checks the kinto collection for any experiment slugs that are
marked as live in the database but missing from the collection, indicating that they
are no longer live and can be marked as complete.
"""
metrics.incr("check_experiments_are_complete.started")
for (
collection,
applications,
) in NimbusExperiment.KINTO_COLLECTION_APPLICATIONS.items():
kinto_client = KintoClient(collection)
live_experiments = NimbusExperiment.objects.filter(
status=NimbusExperiment.Status.LIVE,
application__in=applications,
)
records = kinto_client.get_main_records()
for experiment in live_experiments:
if (experiment.should_end and not experiment.emails.filter(
type=NimbusExperiment.EmailType.EXPERIMENT_END).exists()):
nimbus_send_experiment_ending_email(experiment)
if experiment.slug not in records:
logger.info(
f"{experiment.slug} status is being updated to complete".
format(experiment=experiment))
experiment.status = NimbusExperiment.Status.COMPLETE
experiment.status_next = None
experiment.publish_status = NimbusExperiment.PublishStatus.IDLE
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=NimbusChangeLog.Messages.COMPLETED,
)
logger.info(f"{experiment.slug} status is set to Complete")
metrics.incr("check_experiments_are_complete.completed")
def handle_pending_review(applications):
experiment = NimbusExperiment.objects.waiting(applications).first()
if experiment:
if experiment.should_timeout:
experiment.publish_status = NimbusExperiment.PublishStatus.REVIEW
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=NimbusChangeLog.Messages.TIMED_OUT_IN_KINTO,
)
logger.info(f"{experiment} timed out")
else:
# There is a pending review but it shouldn't time out
return True
def test_no_change_to_published_dto(self):
experiment = NimbusExperimentFactory.create_with_lifecycle(
NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING,
published_dto={"id": "experiment"},
)
change = generate_nimbus_changelog(experiment, self.user,
"test message")
self.assertFalse(change.published_dto_changed)
def nimbus_push_experiment_to_kinto(experiment_id):
"""
An invoked task that given a single experiment id, query it in the db, serialize it,
and push its data to the configured collection. If it fails for any reason, log the
error and reraise it so it will be forwarded to sentry.
"""
metrics.incr("push_experiment_to_kinto.started")
try:
experiment = NimbusExperiment.objects.get(id=experiment_id)
logger.info(f"Pushing {experiment} to Kinto")
kinto_client = KintoClient(
NimbusExperiment.KINTO_APPLICATION_COLLECTION[
experiment.application])
if not NimbusBucketRange.objects.filter(
experiment=experiment).exists():
NimbusIsolationGroup.request_isolation_group_buckets(
experiment.slug,
experiment,
int(experiment.population_percent / Decimal("100.0") *
NimbusExperiment.BUCKET_TOTAL),
)
data = NimbusExperimentSerializer(experiment).data
kinto_client.push_to_kinto(data)
experiment.status = NimbusExperiment.Status.ACCEPTED
experiment.save()
generate_nimbus_changelog(experiment, get_kinto_user())
logger.info(f"{experiment} pushed to Kinto")
metrics.incr("push_experiment_to_kinto.completed")
except Exception as e:
metrics.incr("push_experiment_to_kinto.failed")
logger.info(
f"Pushing experiment id {experiment_id} to Kinto failed: {e}")
raise e
def handle_ending_experiments(applications, records):
for experiment in NimbusExperiment.objects.waiting_to_end_queue(
applications):
if experiment.slug not in records:
logger.info(
f"{experiment.slug} status is being updated to complete".
format(experiment=experiment))
experiment.status = NimbusExperiment.Status.COMPLETE
experiment.status_next = None
experiment.publish_status = NimbusExperiment.PublishStatus.IDLE
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=NimbusChangeLog.Messages.COMPLETED,
)
logger.info(f"{experiment.slug} ended")
def test_generate_nimbus_changelog_with_prior_change(self):
experiment = NimbusExperimentFactory.create_with_status(
NimbusExperiment.Status.DRAFT)
self.assertEqual(experiment.changes.count(), 1)
generate_nimbus_changelog(experiment, self.user)
self.assertEqual(experiment.changes.count(), 2)
change = experiment.latest_change()
self.assertEqual(change.experiment, experiment)
self.assertEqual(change.changed_by, self.user)
self.assertEqual(change.old_status, NimbusExperiment.Status.DRAFT)
self.assertEqual(change.new_status, NimbusExperiment.Status.DRAFT)
self.assertEqual(
change.experiment_data,
dict(NimbusExperimentChangeLogSerializer(experiment).data),
)
def create_with_status(cls, target_status, **kwargs):
experiment = cls.create(**kwargs)
for status, _ in NimbusExperiment.Status.choices:
experiment.status = status
experiment.save()
generate_nimbus_changelog(experiment, experiment.owner)
if status == NimbusExperiment.Status.REVIEW.value:
NimbusIsolationGroup.request_isolation_group_buckets(
experiment.slug,
experiment,
100,
)
if status == target_status:
break
return NimbusExperiment.objects.get(id=experiment.id)
def handle_updating_experiments(applications, kinto_client):
updating_experiments = NimbusExperiment.objects.filter(
NimbusExperiment.Filters.IS_UPDATING,
application__in=applications,
)
if updating_experiments.exists():
records = kinto_client.get_main_records()
for experiment in updating_experiments:
experiment.publish_status = NimbusExperiment.PublishStatus.IDLE
experiment.status_next = None
experiment.published_dto = records.get(experiment.slug)
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=NimbusChangeLog.Messages.UPDATED_IN_KINTO,
)
logger.info(f"{experiment} updated")
def nimbus_check_experiments_are_complete():
"""
A scheduled task that checks the kinto collection for any experiment slugs that are
marked as live in the database but missing from the collection, indicating that they
are no longer live and can be marked as complete.
"""
metrics.incr("check_experiments_are_complete.started")
for application, collection in NimbusExperiment.KINTO_APPLICATION_COLLECTION.items(
):
kinto_client = KintoClient(collection)
live_experiments = NimbusExperiment.objects.filter(
status=NimbusExperiment.Status.LIVE,
application=application,
)
records = kinto_client.get_main_records()
record_ids = [r.get("id") for r in records]
print("found record ids", record_ids)
for experiment in live_experiments:
print("checking", experiment)
if (experiment.should_end and not experiment.emails.filter(
type=NimbusExperiment.EmailType.EXPERIMENT_END).exists()):
nimbus_send_experiment_ending_email(experiment)
if experiment.slug not in record_ids:
logger.info(
f"{experiment} status is being updated to complete".format(
experiment=experiment))
experiment.status = NimbusExperiment.Status.COMPLETE
experiment.save()
generate_nimbus_changelog(experiment, get_kinto_user())
logger.info(f"{experiment} status is set to Complete")
metrics.incr("check_experiments_are_complete.completed")
def save(self, *args, **kwargs):
with transaction.atomic():
experiment = super().save(*args, **kwargs)
if experiment.has_filter(
experiment.Filters.SHOULD_ALLOCATE_BUCKETS):
experiment.allocate_bucket_range()
if self.should_call_preview_task:
nimbus_synchronize_preview_experiments_in_kinto.apply_async(
countdown=5)
if self.should_call_push_task:
collection = NimbusExperiment.KINTO_APPLICATION_COLLECTION[
experiment.application]
nimbus_check_kinto_push_queue_by_collection.apply_async(
countdown=5, args=[collection])
generate_nimbus_changelog(experiment,
self.context["user"],
message=self.changelog_message)
return experiment
def handle_launching_experiments(applications, records):
for experiment in NimbusExperiment.objects.waiting_to_launch_queue(
applications):
if experiment.slug in records:
logger.info(f"{experiment} status is being updated to live".format(
experiment=experiment))
published_record = records[experiment.slug].copy()
published_record.pop("last_modified")
experiment.status = NimbusExperiment.Status.LIVE
experiment.status_next = None
experiment.publish_status = NimbusExperiment.PublishStatus.IDLE
experiment.published_dto = published_record
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=NimbusChangeLog.Messages.LIVE,
)
logger.info(f"{experiment.slug} launched")
def nimbus_check_experiments_are_live():
"""
A scheduled task that checks the kinto collection for any experiment slugs that are
present in the collection but are not yet marked as live in the database and marks
them as live.
"""
metrics.incr("check_experiments_are_live.started")
for (
collection,
applications,
) in NimbusExperiment.KINTO_COLLECTION_APPLICATIONS.items():
kinto_client = KintoClient(collection)
records = kinto_client.get_main_records()
for experiment in NimbusExperiment.objects.waiting_to_launch_queue(
applications):
if experiment.slug in records:
logger.info(
f"{experiment} status is being updated to live".format(
experiment=experiment))
experiment.status = NimbusExperiment.Status.LIVE
experiment.status_next = None
experiment.publish_status = NimbusExperiment.PublishStatus.IDLE
experiment.published_dto = records[experiment.slug]
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=NimbusChangeLog.Messages.LIVE,
)
logger.info(f"{experiment.slug} status is set to Live")
metrics.incr("check_experiments_are_live.completed")
def handle_updating_experiments(applications, records):
for experiment in NimbusExperiment.objects.waiting_to_update_queue(
applications):
published_record = records.get(experiment.slug).copy()
published_record.pop("last_modified")
stored_record = experiment.published_dto.copy()
stored_record.pop("last_modified", None)
if published_record != stored_record:
logger.info(f"{experiment} is updated in Kinto".format(
experiment=experiment))
experiment.publish_status = NimbusExperiment.PublishStatus.IDLE
experiment.status_next = None
experiment.published_dto = published_record
experiment.save()
generate_nimbus_changelog(
experiment,
get_kinto_user(),
message=NimbusChangeLog.Messages.UPDATED_IN_KINTO,
)
logger.info(f"{experiment.slug} updated")
def save(self, *args, **kwargs):
experiment = super().save(*args, **kwargs)
generate_nimbus_changelog(experiment, self.context["user"])
return experiment
