def test_kills_stale_workers_after_timeout(self):
timeout = 1
worker = Worker(['basic'], timeout=timeout)
self.resq.enqueue(TimeoutJob, timeout + 1)
assert worker.get_failed() == 0
worker.fork_worker(worker.reserve())
assert worker.get_failed() == 1
def test_kills_stale_workers_after_timeout(self):
timeout = 1
worker = Worker(['basic'], timeout=timeout)
self.resq.enqueue(TimeoutJob, timeout + 1)
assert worker.get_failed() == 0
worker.fork_worker(worker.reserve())
assert worker.get_failed() == 1
def test_detect_code_0_os_exit_as_success(self):
worker = Worker(['basic'])
self.resq.enqueue(PrematureHardExitJob, 0)
assert worker.job() == {}
assert worker.get_failed() == 0
worker.fork_worker(worker.reserve())
assert worker.job() == {}
assert worker.get_failed() == 0
def test_detect_non_0_sys_exit_as_failure(self):
worker = Worker(['basic'])
self.resq.enqueue(PrematureExitJob, 9)
assert worker.job() == {}
assert worker.get_failed() == 0
worker.fork_worker(worker.reserve())
assert worker.job() == {}
assert worker.get_failed() == 1
def test_detect_crashed_workers_as_failures(self):
worker = Worker(['basic'])
self.resq.enqueue(CrashJob)
assert worker.job() == {}
assert worker.get_failed() == 0
worker.fork_worker(worker.reserve())
assert worker.job() == {}
assert worker.get_failed() == 1
def test_detect_code_0_os_exit_as_success(self):
worker = Worker(['basic'])
self.resq.enqueue(PrematureHardExitJob, 0)
assert worker.job() == {}
assert worker.get_failed() == 0
worker.fork_worker(worker.reserve())
assert worker.job() == {}
assert worker.get_failed() == 0
def test_detect_non_0_sys_exit_as_failure(self):
worker = Worker(['basic'])
self.resq.enqueue(PrematureExitJob, 9)
assert worker.job() == {}
assert worker.get_failed() == 0
worker.fork_worker(worker.reserve())
assert worker.job() == {}
assert worker.get_failed() == 1
def test_detect_crashed_workers_as_failures(self):
worker = Worker(['basic'])
self.resq.enqueue(CrashJob)
assert worker.job() == {}
assert worker.get_failed() == 0
worker.fork_worker(worker.reserve())
assert worker.job() == {}
assert worker.get_failed() == 1
def test_failed(self):
name = "%s:%s:%s" % (os.uname()[1],os.getpid(),'basic')
worker = Worker(['basic'])
worker.failed()
assert self.redis.exists("resque:stat:failed")
assert self.redis.exists("resque:stat:failed:%s" % name)
assert self.redis.get("resque:stat:failed") == str(1)
assert self.redis.get("resque:stat:failed:%s" % name) == str(1)
assert worker.get_failed() == 1
worker.failed()
assert self.redis.get("resque:stat:failed") == str(2)
assert self.redis.get("resque:stat:failed:%s" % name) == str(2)
assert worker.get_failed() == 2
def test_failed(self):
name = "%s:%s:%s" % (os.uname()[1],os.getpid(),'basic')
worker = Worker(['basic'])
worker.failed()
assert self.redis.exists("resque:stat:failed")
assert self.redis.exists("resque:stat:failed:%s" % name)
assert self.redis.get("resque:stat:failed") == str(1)
assert self.redis.get("resque:stat:failed:%s" % name) == str(1)
assert worker.get_failed() == 1
worker.failed()
assert self.redis.get("resque:stat:failed") == str(2)
assert self.redis.get("resque:stat:failed:%s" % name) == str(2)
assert worker.get_failed() == 2
def test_kills_stale_workers_after_timeout(self):
import signal
timeout = 1
worker = Worker(['basic'], timeout=timeout)
self.resq.enqueue(TimeoutJob, timeout + 1)
child = os.fork()
if child:
assert worker.get_failed() == 0
time.sleep(timeout + 2)
os.kill(child, signal.SIGKILL)
os.waitpid(-1, os.WNOHANG)
assert worker.get_failed() == 1
else:
worker.work()
def test_sigterm_worker_gets_requeued(self):
worker = Worker(['basic'])
self.resq.enqueue(SigtermJob, 'string argument')
assert worker.job() == {}
assert worker.get_failed() == 0
job = worker.reserve()
worker.fork_worker(job)
# our last job should land back in the queue
job2 = worker.reserve()
assert str(job2) == str(job)
# didn't run job2, so there shouldn't be more jobs
assert worker.reserve(timeout=1) == None
assert worker.job() == {}
assert worker.get_failed() == 0
def test_retries_give_up_eventually(self):
now = datetime.datetime.utcnow()
self.set_utcnow(now)
worker = Worker(['basic'])
scheduler = Scheduler()
# queue up a job that will fail for 60 seconds
self.resq.enqueue(RetryOnExceptionJob,
now + datetime.timedelta(seconds=60))
worker.process()
assert worker.get_failed() == 0
# check it retries the first time
self.set_utcnow(now + datetime.timedelta(seconds=5))
scheduler.handle_delayed_items()
assert None == worker.process()
assert worker.get_failed() == 0
# check it fails when we've been trying too long
self.set_utcnow(now + datetime.timedelta(seconds=20))
scheduler.handle_delayed_items()
assert None == worker.process()
assert worker.get_failed() == 1
def test_retry_on_exception(self):
now = datetime.datetime.utcnow()
self.set_utcnow(now)
worker = Worker(['basic'])
scheduler = Scheduler()
# queue up a job that will fail for 30 seconds
self.resq.enqueue(RetryOnExceptionJob,
now + datetime.timedelta(seconds=30))
worker.process()
assert worker.get_failed() == 0
# check it retries the first time
self.set_utcnow(now + datetime.timedelta(seconds=5))
scheduler.handle_delayed_items()
assert None == worker.process()
assert worker.get_failed() == 0
# check it runs fine when it's stopped crashing
self.set_utcnow(now + datetime.timedelta(seconds=60))
scheduler.handle_delayed_items()
assert True == worker.process()
assert worker.get_failed() == 0
def test_retries_give_up_eventually(self):
now = datetime.datetime.now()
self.set_current_time(now)
worker = Worker(['basic'])
scheduler = Scheduler()
# queue up a job that will fail for 60 seconds
self.resq.enqueue(RetryOnExceptionJob,
now + datetime.timedelta(seconds=60))
worker.process()
assert worker.get_failed() == 0
# check it retries the first time
self.set_current_time(now + datetime.timedelta(seconds=5))
scheduler.handle_delayed_items()
assert None == worker.process()
assert worker.get_failed() == 0
# check it fails when we've been trying too long
self.set_current_time(now + datetime.timedelta(seconds=20))
scheduler.handle_delayed_items()
assert None == worker.process()
assert worker.get_failed() == 1
def test_retry_on_exception(self):
now = datetime.datetime.now()
self.set_current_time(now)
worker = Worker(['basic'])
scheduler = Scheduler()
# queue up a job that will fail for 30 seconds
self.resq.enqueue(RetryOnExceptionJob,
now + datetime.timedelta(seconds=30))
worker.process()
assert worker.get_failed() == 0
# check it retries the first time
self.set_current_time(now + datetime.timedelta(seconds=5))
scheduler.handle_delayed_items()
assert None == worker.process()
assert worker.get_failed() == 0
# check it runs fine when it's stopped crashing
self.set_current_time(now + datetime.timedelta(seconds=60))
scheduler.handle_delayed_items()
assert True == worker.process()
assert worker.get_failed() == 0
