def test_ttr(c: Client) -> None:
c.put(b"two second ttr", ttr=2)
with assert_seconds(1):
job = c.reserve()
with pytest.raises(DeadlineSoonError):
c.reserve()
with assert_seconds(1):
c.touch(job)
with pytest.raises(DeadlineSoonError):
c.reserve()
c.release(job)
def handle_job_lock(b: greenstalk.Client, job: greenstalk.Job):
global logger
data = json.loads(job.body)
lock_key = data["lock_key"]
instance = data["instance"]
job_id = int(data["job_id"])
logger.info("Handling lock key %s (job %d)" % (lock_key, job_id))
try:
cluster = Cluster.objects.get(slug=data["cluster"])
except ObjectDoesNotExist:
logger.warn("Got lock key %s for unknown cluster %s, burying" %
(data["lock_key"], data["cluster"]))
b.bury(job)
close_old_connections()
return
finally:
close_old_connections()
pi = next_poll_interval()
while True:
logger.debug("Checking lock key %s (job: %d)" % (lock_key, job_id))
reason = cache.get(lock_key)
if reason is None:
logger.info("Lock key %s vanished, forgetting it" % lock_key)
b.delete(job)
return
logger.debug("Polling job %d" % job_id)
try:
status = cluster.get_job_status(job_id)
except Exception as err:
logger.warn("Error polling job: %s" % str(err))
close_old_connections()
sleep(next(pi))
continue
finally:
close_old_connections()
logger.debug("Done")
if status["end_ts"]:
logger.info("Job %d finished, removing lock %s" %
(job_id, lock_key))
if "flush_keys" in data:
for key in data["flush_keys"]:
cache.delete(key)
cache.delete(lock_key)
locked_instances = cache.get('locked_instances')
# This should contain at least 1 instance
if locked_instances is not None:
try:
locked_instances.pop("%s" % instance)
except KeyError:
pass
if len(locked_instances) == 0:
cache.delete('locked_instances')
else:
cache.set('locked_instances', locked_instances, 90)
else:
# This could be due to a cache fail or restart. For the time log it
logger.warn(
"Unable to find instance %s in locked instances cache key"
% instance)
clear_cluster_users_cache(cluster.slug)
b.delete(job)
return
# Touch the key
cache.set(lock_key, reason, 30)
b.touch(job)
sleep(next(pi))
