def test_ttr(c: Client) -> None: c.put(b"two second ttr", ttr=2) with assert_seconds(1): job = c.reserve() with pytest.raises(DeadlineSoonError): c.reserve() with assert_seconds(1): c.touch(job) with pytest.raises(DeadlineSoonError): c.reserve() c.release(job)
def handle_job_lock(b: greenstalk.Client, job: greenstalk.Job): global logger data = json.loads(job.body) lock_key = data["lock_key"] instance = data["instance"] job_id = int(data["job_id"]) logger.info("Handling lock key %s (job %d)" % (lock_key, job_id)) try: cluster = Cluster.objects.get(slug=data["cluster"]) except ObjectDoesNotExist: logger.warn("Got lock key %s for unknown cluster %s, burying" % (data["lock_key"], data["cluster"])) b.bury(job) close_old_connections() return finally: close_old_connections() pi = next_poll_interval() while True: logger.debug("Checking lock key %s (job: %d)" % (lock_key, job_id)) reason = cache.get(lock_key) if reason is None: logger.info("Lock key %s vanished, forgetting it" % lock_key) b.delete(job) return logger.debug("Polling job %d" % job_id) try: status = cluster.get_job_status(job_id) except Exception as err: logger.warn("Error polling job: %s" % str(err)) close_old_connections() sleep(next(pi)) continue finally: close_old_connections() logger.debug("Done") if status["end_ts"]: logger.info("Job %d finished, removing lock %s" % (job_id, lock_key)) if "flush_keys" in data: for key in data["flush_keys"]: cache.delete(key) cache.delete(lock_key) locked_instances = cache.get('locked_instances') # This should contain at least 1 instance if locked_instances is not None: try: locked_instances.pop("%s" % instance) except KeyError: pass if len(locked_instances) == 0: cache.delete('locked_instances') else: cache.set('locked_instances', locked_instances, 90) else: # This could be due to a cache fail or restart. For the time log it logger.warn( "Unable to find instance %s in locked instances cache key" % instance) clear_cluster_users_cache(cluster.slug) b.delete(job) return # Touch the key cache.set(lock_key, reason, 30) b.touch(job) sleep(next(pi))