def nimbus_push_experiment_to_kinto(collection, experiment_id): """ An invoked task that given a single experiment id, query it in the db, serialize it, and push its data to the configured collection. If it fails for any reason, log the error and reraise it so it will be forwarded to sentry. """ metrics.incr("push_experiment_to_kinto.started") try: experiment = NimbusExperiment.objects.get(id=experiment_id) logger.info(f"Pushing {experiment.slug} to Kinto") kinto_client = KintoClient(collection) data = NimbusExperimentSerializer(experiment).data kinto_client.create_record(data) experiment.publish_status = NimbusExperiment.PublishStatus.WAITING experiment.save() generate_nimbus_changelog( experiment, get_kinto_user(), message=NimbusChangeLog.Messages.PUSHED_TO_KINTO, ) logger.info(f"{experiment.slug} pushed to Kinto") metrics.incr("push_experiment_to_kinto.completed") except Exception as e: metrics.incr("push_experiment_to_kinto.failed") logger.info( f"Pushing experiment {experiment.slug} to Kinto failed: {e}") raise e
def test_generate_nimbus_changelog_with_prior_change(self): experiment = NimbusExperimentFactory.create_with_lifecycle( NimbusExperimentFactory.Lifecycles.CREATED) self.assertEqual(experiment.changes.count(), 1) experiment.status = NimbusExperiment.Status.DRAFT experiment.status_next = NimbusExperiment.Status.LIVE experiment.publish_status = NimbusExperiment.PublishStatus.REVIEW experiment.save() generate_nimbus_changelog(experiment, self.user, "test message") self.assertEqual(experiment.changes.count(), 2) change = experiment.changes.latest_change() self.assertEqual(change.experiment, experiment) self.assertEqual(change.message, "test message") self.assertEqual(change.changed_by, self.user) self.assertEqual(change.old_status, NimbusExperiment.Status.DRAFT) self.assertEqual(change.old_status_next, None) self.assertEqual(change.old_publish_status, NimbusExperiment.PublishStatus.IDLE) self.assertEqual(change.new_status, NimbusExperiment.Status.DRAFT) self.assertEqual(change.new_status_next, NimbusExperiment.Status.LIVE) self.assertEqual(change.new_publish_status, NimbusExperiment.PublishStatus.REVIEW) self.assertEqual( change.experiment_data, dict(NimbusExperimentChangeLogSerializer(experiment).data), )
def nimbus_end_experiment_in_kinto(collection, experiment_id): """ An invoked task that given a single experiment id, delete its data from the configured collection. If it fails for any reason, log the error and reraise it so it will be forwarded to sentry. """ metrics.incr("end_experiment_in_kinto.started") try: experiment = NimbusExperiment.objects.get(id=experiment_id) logger.info(f"Deleting {experiment.slug} from Kinto") kinto_client = KintoClient(collection) kinto_client.delete_record(experiment.slug) experiment.publish_status = NimbusExperiment.PublishStatus.WAITING experiment.save() generate_nimbus_changelog( experiment, get_kinto_user(), message=NimbusChangeLog.Messages.DELETED_FROM_KINTO, ) logger.info(f"{experiment.slug} deleted from Kinto") metrics.incr("end_experiment_in_kinto.completed") except Exception as e: metrics.incr("end_experiment_in_kinto.failed") logger.info( f"Deleting experiment {experiment.slug} from Kinto failed: {e}") raise e
def test_generate_nimbus_changelog_without_prior_change(self): experiment = NimbusExperimentFactory.create() self.assertEqual(experiment.changes.count(), 0) generate_nimbus_changelog(experiment, self.user, "test message") self.assertEqual(experiment.changes.count(), 1) change = experiment.changes.get() self.assertEqual(change.experiment, experiment) self.assertEqual(change.message, "test message") self.assertEqual(change.changed_by, self.user) self.assertEqual(change.old_status, None) self.assertEqual(change.old_status_next, None) self.assertEqual(change.old_publish_status, None) self.assertEqual(change.new_status, NimbusExperiment.Status.DRAFT) self.assertEqual(change.new_status_next, None) self.assertEqual(change.new_publish_status, NimbusExperiment.PublishStatus.IDLE) self.assertEqual( change.experiment_data, dict(NimbusExperimentChangeLogSerializer(experiment).data), )
def nimbus_check_experiments_are_live(): """ A scheduled task that checks the kinto collection for any experiment slugs that are present in the collection but are not yet marked as live in the database and marks them as live. """ metrics.incr("check_experiments_are_live.started") accepted_experiments = NimbusExperiment.objects.filter( status=NimbusExperiment.Status.ACCEPTED) for collection in NimbusExperiment.KINTO_APPLICATION_COLLECTION.values(): kinto_client = KintoClient(collection) records = kinto_client.get_main_records() record_ids = [r.get("id") for r in records] for experiment in accepted_experiments: if experiment.slug in record_ids: logger.info( f"{experiment} status is being updated to live".format( experiment=experiment)) experiment.status = NimbusExperiment.Status.LIVE experiment.save() generate_nimbus_changelog(experiment, get_kinto_user()) logger.info(f"{experiment} status is set to Live") metrics.incr("check_experiments_are_live.completed")
def nimbus_update_experiment_in_kinto(collection, experiment_id): """ An invoked task that given a single experiment id, reserializes and updates the record. If it fails for any reason, log the error and reraise it so it will be forwarded to sentry. """ metrics.incr("update_experiment_in_kinto.started") try: experiment = NimbusExperiment.objects.get(id=experiment_id) logger.info(f"Updating {experiment.slug} in Kinto") kinto_client = KintoClient(collection) data = NimbusExperimentSerializer(experiment).data kinto_client.update_record(data) experiment.publish_status = NimbusExperiment.PublishStatus.WAITING experiment.save() generate_nimbus_changelog( experiment, get_kinto_user(), message=NimbusChangeLog.Messages.UPDATED_IN_KINTO, ) logger.info(f"{experiment.slug} updated in Kinto") metrics.incr("update_experiment_in_kinto.completed") except Exception as e: metrics.incr("update_experiment_in_kinto.failed") logger.info( f"Updating experiment {experiment.slug} in Kinto failed: {e}") raise e
def nimbus_check_kinto_push_queue(): """ Because kinto has a restriction that it can only have a single pending review, this task brokers the queue of all experiments ready to be pushed to kinto and ensures that only a single experiment is ever in review. A scheduled task that - Checks the kinto collection for a single rejected experiment from a previous push - If one exists, pull it out of the collection and mark it as rejected - Checks if there is still a pending review and if so, aborts - Gets the list of all experiments ready to be pushed to kinto and pushes the first one """ metrics.incr("check_kinto_push_queue.started") for application, collection in NimbusExperiment.KINTO_APPLICATION_COLLECTION.items( ): kinto_client = KintoClient(collection) rejected_collection_data = kinto_client.get_rejected_collection_data() if rejected_collection_data: rejected_slug = kinto_client.get_rejected_record() experiment = NimbusExperiment.objects.get(slug=rejected_slug) experiment.status = NimbusExperiment.Status.DRAFT experiment.save() generate_nimbus_changelog( experiment, get_kinto_user(), message= f'Rejected: {rejected_collection_data["last_reviewer_comment"]}', ) kinto_client.rollback_changes() if kinto_client.has_pending_review(): metrics.incr(f"check_kinto_push_queue.{collection}_pending_review") return queued_experiments = NimbusExperiment.objects.filter( status=NimbusExperiment.Status.REVIEW, application=application) if queued_experiments.exists(): nimbus_push_experiment_to_kinto.delay( queued_experiments.first().id) metrics.incr( f"check_kinto_push_queue.{collection}_queued_experiment_selected" ) else: metrics.incr( f"check_kinto_push_queue.{collection}_no_experiments_queued") metrics.incr("check_kinto_push_queue.completed")
def create_with_lifecycle(cls, lifecycle, with_random_timespan=False, **kwargs): experiment = cls.create(**kwargs) now = timezone.now() - datetime.timedelta( days=random.randint(100, 200)) for state in lifecycle.value: experiment.apply_lifecycle_state(state) if (experiment.status == experiment.Status.LIVE and experiment.status_next is None and "published_dto" not in kwargs): experiment.published_dto = NimbusExperimentSerializer( experiment).data experiment.save() if experiment.has_filter( experiment.Filters.SHOULD_ALLOCATE_BUCKETS): experiment.allocate_bucket_range() change = generate_nimbus_changelog( experiment, experiment.owner, f"set lifecycle {lifecycle} state {state}", ) if with_random_timespan: change.changed_on = now change.save() now += datetime.timedelta(days=random.randint(5, 20)) return NimbusExperiment.objects.get(id=experiment.id)
def handle_rejection(applications, kinto_client): collection_data = kinto_client.get_rejected_collection_data() experiment = NimbusExperiment.objects.waiting(applications).first() if experiment: experiment.publish_status = NimbusExperiment.PublishStatus.IDLE experiment.status_next = None experiment.is_paused = False experiment.save() generate_nimbus_changelog( experiment, get_kinto_user(), message=collection_data["last_reviewer_comment"], ) logger.info(f"{experiment} rejected")
def handle_waiting_experiments(applications): waiting_experiments = NimbusExperiment.objects.filter( publish_status=NimbusExperiment.PublishStatus.WAITING, application__in=applications, ) for experiment in waiting_experiments: experiment.status_next = None experiment.publish_status = NimbusExperiment.PublishStatus.IDLE experiment.save() generate_nimbus_changelog( experiment, get_kinto_user(), message=NimbusChangeLog.Messages.REJECTED_FROM_KINTO, ) logger.info(f"{experiment.slug} rejected without reason(rollback)")
def nimbus_check_experiments_are_complete(): """ A scheduled task that checks the kinto collection for any experiment slugs that are marked as live in the database but missing from the collection, indicating that they are no longer live and can be marked as complete. """ metrics.incr("check_experiments_are_complete.started") for ( collection, applications, ) in NimbusExperiment.KINTO_COLLECTION_APPLICATIONS.items(): kinto_client = KintoClient(collection) live_experiments = NimbusExperiment.objects.filter( status=NimbusExperiment.Status.LIVE, application__in=applications, ) records = kinto_client.get_main_records() for experiment in live_experiments: if (experiment.should_end and not experiment.emails.filter( type=NimbusExperiment.EmailType.EXPERIMENT_END).exists()): nimbus_send_experiment_ending_email(experiment) if experiment.slug not in records: logger.info( f"{experiment.slug} status is being updated to complete". format(experiment=experiment)) experiment.status = NimbusExperiment.Status.COMPLETE experiment.status_next = None experiment.publish_status = NimbusExperiment.PublishStatus.IDLE experiment.save() generate_nimbus_changelog( experiment, get_kinto_user(), message=NimbusChangeLog.Messages.COMPLETED, ) logger.info(f"{experiment.slug} status is set to Complete") metrics.incr("check_experiments_are_complete.completed")
def handle_pending_review(applications): experiment = NimbusExperiment.objects.waiting(applications).first() if experiment: if experiment.should_timeout: experiment.publish_status = NimbusExperiment.PublishStatus.REVIEW experiment.save() generate_nimbus_changelog( experiment, get_kinto_user(), message=NimbusChangeLog.Messages.TIMED_OUT_IN_KINTO, ) logger.info(f"{experiment} timed out") else: # There is a pending review but it shouldn't time out return True
def test_no_change_to_published_dto(self): experiment = NimbusExperimentFactory.create_with_lifecycle( NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, published_dto={"id": "experiment"}, ) change = generate_nimbus_changelog(experiment, self.user, "test message") self.assertFalse(change.published_dto_changed)
def nimbus_push_experiment_to_kinto(experiment_id): """ An invoked task that given a single experiment id, query it in the db, serialize it, and push its data to the configured collection. If it fails for any reason, log the error and reraise it so it will be forwarded to sentry. """ metrics.incr("push_experiment_to_kinto.started") try: experiment = NimbusExperiment.objects.get(id=experiment_id) logger.info(f"Pushing {experiment} to Kinto") kinto_client = KintoClient( NimbusExperiment.KINTO_APPLICATION_COLLECTION[ experiment.application]) if not NimbusBucketRange.objects.filter( experiment=experiment).exists(): NimbusIsolationGroup.request_isolation_group_buckets( experiment.slug, experiment, int(experiment.population_percent / Decimal("100.0") * NimbusExperiment.BUCKET_TOTAL), ) data = NimbusExperimentSerializer(experiment).data kinto_client.push_to_kinto(data) experiment.status = NimbusExperiment.Status.ACCEPTED experiment.save() generate_nimbus_changelog(experiment, get_kinto_user()) logger.info(f"{experiment} pushed to Kinto") metrics.incr("push_experiment_to_kinto.completed") except Exception as e: metrics.incr("push_experiment_to_kinto.failed") logger.info( f"Pushing experiment id {experiment_id} to Kinto failed: {e}") raise e
def handle_ending_experiments(applications, records): for experiment in NimbusExperiment.objects.waiting_to_end_queue( applications): if experiment.slug not in records: logger.info( f"{experiment.slug} status is being updated to complete". format(experiment=experiment)) experiment.status = NimbusExperiment.Status.COMPLETE experiment.status_next = None experiment.publish_status = NimbusExperiment.PublishStatus.IDLE experiment.save() generate_nimbus_changelog( experiment, get_kinto_user(), message=NimbusChangeLog.Messages.COMPLETED, ) logger.info(f"{experiment.slug} ended")
def test_generate_nimbus_changelog_with_prior_change(self): experiment = NimbusExperimentFactory.create_with_status( NimbusExperiment.Status.DRAFT) self.assertEqual(experiment.changes.count(), 1) generate_nimbus_changelog(experiment, self.user) self.assertEqual(experiment.changes.count(), 2) change = experiment.latest_change() self.assertEqual(change.experiment, experiment) self.assertEqual(change.changed_by, self.user) self.assertEqual(change.old_status, NimbusExperiment.Status.DRAFT) self.assertEqual(change.new_status, NimbusExperiment.Status.DRAFT) self.assertEqual( change.experiment_data, dict(NimbusExperimentChangeLogSerializer(experiment).data), )
def create_with_status(cls, target_status, **kwargs): experiment = cls.create(**kwargs) for status, _ in NimbusExperiment.Status.choices: experiment.status = status experiment.save() generate_nimbus_changelog(experiment, experiment.owner) if status == NimbusExperiment.Status.REVIEW.value: NimbusIsolationGroup.request_isolation_group_buckets( experiment.slug, experiment, 100, ) if status == target_status: break return NimbusExperiment.objects.get(id=experiment.id)
def handle_updating_experiments(applications, kinto_client): updating_experiments = NimbusExperiment.objects.filter( NimbusExperiment.Filters.IS_UPDATING, application__in=applications, ) if updating_experiments.exists(): records = kinto_client.get_main_records() for experiment in updating_experiments: experiment.publish_status = NimbusExperiment.PublishStatus.IDLE experiment.status_next = None experiment.published_dto = records.get(experiment.slug) experiment.save() generate_nimbus_changelog( experiment, get_kinto_user(), message=NimbusChangeLog.Messages.UPDATED_IN_KINTO, ) logger.info(f"{experiment} updated")
def nimbus_check_experiments_are_complete(): """ A scheduled task that checks the kinto collection for any experiment slugs that are marked as live in the database but missing from the collection, indicating that they are no longer live and can be marked as complete. """ metrics.incr("check_experiments_are_complete.started") for application, collection in NimbusExperiment.KINTO_APPLICATION_COLLECTION.items( ): kinto_client = KintoClient(collection) live_experiments = NimbusExperiment.objects.filter( status=NimbusExperiment.Status.LIVE, application=application, ) records = kinto_client.get_main_records() record_ids = [r.get("id") for r in records] print("found record ids", record_ids) for experiment in live_experiments: print("checking", experiment) if (experiment.should_end and not experiment.emails.filter( type=NimbusExperiment.EmailType.EXPERIMENT_END).exists()): nimbus_send_experiment_ending_email(experiment) if experiment.slug not in record_ids: logger.info( f"{experiment} status is being updated to complete".format( experiment=experiment)) experiment.status = NimbusExperiment.Status.COMPLETE experiment.save() generate_nimbus_changelog(experiment, get_kinto_user()) logger.info(f"{experiment} status is set to Complete") metrics.incr("check_experiments_are_complete.completed")
def save(self, *args, **kwargs): with transaction.atomic(): experiment = super().save(*args, **kwargs) if experiment.has_filter( experiment.Filters.SHOULD_ALLOCATE_BUCKETS): experiment.allocate_bucket_range() if self.should_call_preview_task: nimbus_synchronize_preview_experiments_in_kinto.apply_async( countdown=5) if self.should_call_push_task: collection = NimbusExperiment.KINTO_APPLICATION_COLLECTION[ experiment.application] nimbus_check_kinto_push_queue_by_collection.apply_async( countdown=5, args=[collection]) generate_nimbus_changelog(experiment, self.context["user"], message=self.changelog_message) return experiment
def handle_launching_experiments(applications, records): for experiment in NimbusExperiment.objects.waiting_to_launch_queue( applications): if experiment.slug in records: logger.info(f"{experiment} status is being updated to live".format( experiment=experiment)) published_record = records[experiment.slug].copy() published_record.pop("last_modified") experiment.status = NimbusExperiment.Status.LIVE experiment.status_next = None experiment.publish_status = NimbusExperiment.PublishStatus.IDLE experiment.published_dto = published_record experiment.save() generate_nimbus_changelog( experiment, get_kinto_user(), message=NimbusChangeLog.Messages.LIVE, ) logger.info(f"{experiment.slug} launched")
def nimbus_check_experiments_are_live(): """ A scheduled task that checks the kinto collection for any experiment slugs that are present in the collection but are not yet marked as live in the database and marks them as live. """ metrics.incr("check_experiments_are_live.started") for ( collection, applications, ) in NimbusExperiment.KINTO_COLLECTION_APPLICATIONS.items(): kinto_client = KintoClient(collection) records = kinto_client.get_main_records() for experiment in NimbusExperiment.objects.waiting_to_launch_queue( applications): if experiment.slug in records: logger.info( f"{experiment} status is being updated to live".format( experiment=experiment)) experiment.status = NimbusExperiment.Status.LIVE experiment.status_next = None experiment.publish_status = NimbusExperiment.PublishStatus.IDLE experiment.published_dto = records[experiment.slug] experiment.save() generate_nimbus_changelog( experiment, get_kinto_user(), message=NimbusChangeLog.Messages.LIVE, ) logger.info(f"{experiment.slug} status is set to Live") metrics.incr("check_experiments_are_live.completed")
def handle_updating_experiments(applications, records): for experiment in NimbusExperiment.objects.waiting_to_update_queue( applications): published_record = records.get(experiment.slug).copy() published_record.pop("last_modified") stored_record = experiment.published_dto.copy() stored_record.pop("last_modified", None) if published_record != stored_record: logger.info(f"{experiment} is updated in Kinto".format( experiment=experiment)) experiment.publish_status = NimbusExperiment.PublishStatus.IDLE experiment.status_next = None experiment.published_dto = published_record experiment.save() generate_nimbus_changelog( experiment, get_kinto_user(), message=NimbusChangeLog.Messages.UPDATED_IN_KINTO, ) logger.info(f"{experiment.slug} updated")
def save(self, *args, **kwargs): experiment = super().save(*args, **kwargs) generate_nimbus_changelog(experiment, self.context["user"]) return experiment