def test_peek_buried(c: Client) -> None: id = c.put(b"buried") job = c.reserve() c.bury(job) job = c.peek_buried() assert job.id == id assert job.body == b"buried"
def test_kick(c: Client) -> None: c.put(b"a delayed job", delay=30) c.put(b"another delayed job", delay=45) c.put(b"a ready job") job = c.reserve() c.bury(job) assert c.kick(10) == 1 assert c.kick(10) == 2
def test_delays(c: Client) -> None: with assert_seconds(1): c.put(b"delayed", delay=1) job = c.reserve() assert job.body == b"delayed" with assert_seconds(2): c.release(job, delay=2) with pytest.raises(TimedOutError): c.reserve(timeout=1) job = c.reserve(timeout=1) c.bury(job) with pytest.raises(TimedOutError): c.reserve(timeout=0)
def handle_creation(b: greenstalk.Client, job: greenstalk.Job): global logger data = json.loads(job.body) try: application = InstanceApplication.objects.get( id=data["application_id"]) except ObjectDoesNotExist: logger.warn("Unable to find application #%d, burying" % data["application_id"]) try_log( mail_admins, "Burying job #%d" % job.id, "Please inspect job #%d (application %d) manually" % (job.id, data["application_id"])) b.bury(job) close_old_connections() return finally: close_old_connections() logger.info("Handling %s (job: %d)", application.hostname, application.job_id) while True: sleep(15) logger.info("Checking %s (job: %d)", application.hostname, application.job_id) status = application.cluster.get_job_status(application.job_id) if status["end_ts"]: logger.info("%s (job: %d) done. Status: %s", application.hostname, application.job_id, status["status"]) if status["status"] == "error": application.status = STATUS_FAILED application.backend_message = smart_str(status["opresult"]) application.save() logger.warn("%s (job: %d) failed. Notifying admins", application.hostname, application.job_id) try_log( mail_admins, "Instance creation failure for %s on %s" % (application.hostname, application.cluster), json.dumps(status, indent=2)) else: application.status = STATUS_SUCCESS application.backend_message = None application.save() logger.info("Mailing %s about %s", application.applicant.email, application.hostname) fqdn = Site.objects.get_current().domain instance_url = "https://%s%s" % \ (fqdn, urlresolvers.reverse("instance-detail", args=(application.cluster.slug, application.hostname))) mail_body = render_to_string( "instances/emails/instance_created_mail.txt", { "application": application, "instance_url": instance_url, "BRANDING": settings.BRANDING }) mail_body_managers = render_to_string( "instances/emails/instance_created_mail.txt", { "application": application, "reviewer": application.reviewer, "instance_url": instance_url, "BRANDING": settings.BRANDING }) try_log( send_mail, settings.EMAIL_SUBJECT_PREFIX + "Instance %s is ready" % application.hostname, mail_body, settings.SERVER_EMAIL, [application.applicant.email]) logger.info("Mailing managers about %s" % application.hostname) try_log(mail_managers, "Instance %s is ready" % application.hostname, mail_body_managers) b.delete(job) close_old_connections() break b.delete(job)
def handle_job_lock(b: greenstalk.Client, job: greenstalk.Job): global logger data = json.loads(job.body) lock_key = data["lock_key"] instance = data["instance"] job_id = int(data["job_id"]) logger.info("Handling lock key %s (job %d)" % (lock_key, job_id)) try: cluster = Cluster.objects.get(slug=data["cluster"]) except ObjectDoesNotExist: logger.warn("Got lock key %s for unknown cluster %s, burying" % (data["lock_key"], data["cluster"])) b.bury(job) close_old_connections() return finally: close_old_connections() pi = next_poll_interval() while True: logger.debug("Checking lock key %s (job: %d)" % (lock_key, job_id)) reason = cache.get(lock_key) if reason is None: logger.info("Lock key %s vanished, forgetting it" % lock_key) b.delete(job) return logger.debug("Polling job %d" % job_id) try: status = cluster.get_job_status(job_id) except Exception as err: logger.warn("Error polling job: %s" % str(err)) close_old_connections() sleep(next(pi)) continue finally: close_old_connections() logger.debug("Done") if status["end_ts"]: logger.info("Job %d finished, removing lock %s" % (job_id, lock_key)) if "flush_keys" in data: for key in data["flush_keys"]: cache.delete(key) cache.delete(lock_key) locked_instances = cache.get('locked_instances') # This should contain at least 1 instance if locked_instances is not None: try: locked_instances.pop("%s" % instance) except KeyError: pass if len(locked_instances) == 0: cache.delete('locked_instances') else: cache.set('locked_instances', locked_instances, 90) else: # This could be due to a cache fail or restart. For the time log it logger.warn( "Unable to find instance %s in locked instances cache key" % instance) clear_cluster_users_cache(cluster.slug) b.delete(job) return # Touch the key cache.set(lock_key, reason, 30) b.touch(job) sleep(next(pi))