def test_retry_from_other_queue_stays_on_queue(worker):
worker.start(queues="default exec")
worker.send_task("tests.tasks.general.Retry", {"delay": 1},
queue="exec",
accept_statuses="retry")
time.sleep(2)
job_id = worker.mongodb_jobs.mrq_jobs.find()[0]["_id"]
job = Job(job_id).fetch()
assert job.data["status"] == "retry"
assert job.data["queue"] == "exec"
assert Queue("default").size() == 0
assert Queue("exec").size() == 0
worker.stop(deps=False)
worker.start(queues="default", deps=False)
# Should do nothing yet
worker.send_task("mrq.basetasks.cleaning.RequeueRetryJobs", {}, block=True)
assert Queue("default").size() == 0
assert Queue("exec").size() == 1
job = Job(job_id).fetch()
assert job.data["status"] == "queued"
assert job.data["queue"] == "exec"
def test_queue_notify(worker, worker2):
worker.start(flags="--max_latency 30 --config tests/fixtures/config-notify.py", queues="withnotify withoutnotify")
# Used to queue jobs in the same environment & config!
worker2.start(flags="--config tests/fixtures/config-notify.py")
id1 = worker2.send_task("tests.tasks.general.SendTask", {
"params": {"a": 42, "b": 1},
"path": "tests.tasks.general.Add",
"queue": "withnotify"
})
time.sleep(2)
assert Job(id1).fetch().data["status"] == "success"
assert Job(id1).fetch().data["result"] == 43
id2 = worker2.send_task("tests.tasks.general.SendTask", {
"params": {"a": 43, "b": 1},
"path": "tests.tasks.general.Add",
"queue": "withoutnotify"
})
time.sleep(2)
assert Job(id2).fetch().data["status"] == "queued"
def test_interrupt_maxconcurrency(worker):
# The worker will raise a maxconcurrency on the second job
worker.start(flags="--greenlets=2")
job_ids = worker.send_tasks("tests.tasks.concurrency.LockedAdd",
[{
"a": i,
"b": 1,
"sleep": 2
} for i in range(2)],
block=False)
worker.wait_for_tasks_results(
job_ids, accept_statuses=["success", "failed", "maxconcurrency"])
job_statuses = [Job(job_id).fetch().data["status"] for job_id in job_ids]
assert set(job_statuses) == set(["success", "maxconcurrency"])
# the job concurrency key must be equal to 0
last_job_id = worker.send_task("tests.tasks.concurrency.LockedAdd", {
"a": 1,
"b": 1,
"sleep": 2
},
block=False)
last_job = Job(last_job_id).wait(poll_interval=0.01)
assert last_job.get("status") == "success"
def test_interrupt_worker_gracefully(worker, p_flags):
""" Test what happens when we interrupt a running worker gracefully. """
worker.start(flags=p_flags)
job_id = worker.send_task(
"tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 5}, block=False)
time.sleep(2)
job = Job(job_id).fetch().data
assert job["status"] == "started"
# Stop the worker gracefully. first job should still finish!
worker.stop(block=False, deps=False)
time.sleep(1)
# Should not be accepting new jobs!
job_id2 = worker.send_task(
"tests.tasks.general.Add", {"a": 42, "b": 1, "sleep": 4}, block=False)
time.sleep(1)
job = Job(job_id2).fetch().data
assert job.get("status") == "queued"
time.sleep(4)
job = Job(job_id).fetch().data
assert job["status"] == "success"
assert job["result"] == 42
job = Job(job_id2).fetch().data
assert job.get("status") == "queued"
def test_retry_otherqueue_delay_nonzero(worker):
# delay = 0 should requeue right away.
worker.send_task("tests.tasks.general.Retry", {
"queue": "noexec",
"delay": 2
},
block=True,
accept_statuses=["retry"])
assert Queue("default").size() == 0
assert Queue("noexec").size() == 0
job_id = worker.mongodb_jobs.mrq_jobs.find()[0]["_id"]
job = Job(job_id).fetch()
assert job.data["status"] == "retry"
# Should do nothing yet
worker.send_task("mrq.basetasks.cleaning.RequeueRetryJobs", {}, block=True)
assert Queue("default").size() == 0
assert Queue("noexec").size() == 0
time.sleep(2)
# Should requeue
worker.send_task("mrq.basetasks.cleaning.RequeueRetryJobs", {}, block=True)
assert Queue("default").size() == 0
assert Queue("noexec").size() == 1
job = Job(job_id).fetch()
assert job.data["status"] == "queued"
def test_cancel_by_path(worker):
# Start the worker with only one greenlet so that tasks execute sequentially
worker.start(flags="--gevent 1")
job_id1 = worker.send_task("tests.tasks.general.MongoInsert", {"a": 41, "sleep": 2}, block=False)
worker.send_task("mrq.basetasks.utils.JobAction", {
"path": "tests.tasks.general.MongoInsert",
"status": "queued",
"action": "cancel"
}, block=False)
job_id2 = worker.send_task("tests.tasks.general.MongoInsert", {"a": 43}, block=False)
Job(job_id2).wait(poll_interval=0.01)
# Leave some time to unqueue job_id2 without executing.
time.sleep(1)
worker.stop(deps=False)
job1 = Job(job_id1).fetch().data
job2 = Job(job_id2).fetch().data
assert job1["status"] == "success"
assert job1["result"] == {"a": 41, "sleep": 2}
assert job2["status"] == "cancel"
assert job2["dateexpires"] > job2["dateupdated"]
assert job2.get("result") is None
assert worker.mongodb_logs.tests_inserts.count() == 1
assert Queue("default").size() == 0
def test_interrupt_worker_double_sigint(worker, p_flags):
""" Test what happens when we interrupt a running worker with 2 SIGINTs. """
start_time = time.time()
worker.start(flags=p_flags)
job_id = worker.send_task("tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 20}, block=False)
while Job(job_id).fetch().data["status"] == "queued":
time.sleep(0.1)
job = Job(job_id).fetch().data
assert job["status"] == "started"
# Stop the worker gracefully. first job should still finish!
worker.stop(block=False, deps=False)
time.sleep(1)
# Should not be accepting new jobs!
job_id2 = worker.send_task("tests.tasks.general.Add", {"a": 42, "b": 1, "sleep": 20}, block=False)
time.sleep(1)
job2 = Job(job_id2).fetch().data
assert job2.get("status") == "queued"
job = Job(job_id).fetch().data
assert job["status"] == "started"
# Sending a second kill -2 should make it stop
worker.stop(block=True, deps=False, force=True)
while Job(job_id).fetch().data["status"] == "started":
time.sleep(0.1)
job = Job(job_id).fetch().data
assert job["status"] == "interrupt"
assert time.time() - start_time < 15
# Then try the cleaning task that requeues interrupted jobs
assert Queue("default").size() == 1
worker.start(queues="cleaning", deps=False, flush=False)
res = worker.send_task("mrq.basetasks.cleaning.RequeueInterruptedJobs", {}, block=True, queue="cleaning")
assert res["requeued"] == 1
assert Queue("default").size() == 2
Queue("default").list_job_ids() == [str(job_id2), str(job_id)]
job = Job(job_id).fetch().data
assert job["status"] == "queued"
assert job["queue"] == "default"
def run(self, params):
# If there are more than this much items on the queue, we don't try to check if our mongodb
# jobs are still queued.
max_queue_items = params.get("max_queue_items", 1000)
stats = {"fetched": 0, "requeued": 0}
all_queues = Queue.all_known()
for queue_name in all_queues:
queue = Queue(queue_name)
queue_size = queue.size()
if queue.is_raw:
continue
log.info("Checking queue %s" % queue_name)
if queue_size > max_queue_items:
log.info("Stopping because queue %s has %s items" %
(queue_name, queue_size))
continue
queue_jobs_ids = set(queue.list_job_ids(limit=max_queue_items + 1))
if len(queue_jobs_ids) >= max_queue_items:
log.info(
"Stopping because queue %s actually had more than %s items"
% (queue_name, len(queue_jobs_ids)))
continue
for job_data in connections.mongodb_jobs.mrq_jobs.find(
{
"queue": queue_name,
"status": "queued"
},
projection={
"_id": 1
}).sort([["_id", 1]]):
stats["fetched"] += 1
if str(job_data["_id"]) in queue_jobs_ids:
log.info("Found job %s on queue %s. Stopping" %
(job_data["_id"], queue.id))
break
# At this point, this job is not on the queue and we're sure
# the queue is less than max_queue_items
# We can safely requeue the job.
log.info("Requeueing %s on %s" % (job_data["_id"], queue.id))
stats["requeued"] += 1
job = Job(job_data["_id"])
job.requeue(queue=queue_name)
return stats
def run(self, params):
# If there are more than this much items on the queue, we don't try to check if our mongodb
# jobs are still queued.
max_queue_items = params.get("max_queue_items", 1000)
stats = {"fetched": 0, "requeued": 0}
# This was only checking in Redis and wasn't resistant to a redis-wide flush.
# Doing Queue.all() is slower but covers more edge cases.
# all_queues = Queue.all_known()
all_queues = Queue.all()
log.info("Checking %s queues" % len(all_queues))
for queue_name in all_queues:
queue = Queue(queue_name)
queue_size = queue.size()
# If the queue is raw, the jobs were only stored in redis so they are lost for good.
if queue.is_raw:
continue
log.info("Checking queue %s" % queue_name)
if queue_size > max_queue_items:
log.info("Stopping because queue %s has %s items" % (queue_name, queue_size))
continue
queue_jobs_ids = set(queue.list_job_ids(limit=max_queue_items + 1))
if len(queue_jobs_ids) >= max_queue_items:
log.info(
"Stopping because queue %s actually had more than %s items" % (queue_name, len(queue_jobs_ids))
)
continue
for job_data in connections.mongodb_jobs.mrq_jobs.find(
{"queue": queue_name, "status": "queued"}, projection={"_id": 1}
).sort([["_id", 1]]):
stats["fetched"] += 1
if str(job_data["_id"]) in queue_jobs_ids:
log.info("Found job %s on queue %s. Stopping" % (job_data["_id"], queue.id))
break
# At this point, this job is not on the queue and we're sure
# the queue is less than max_queue_items
# We can safely requeue the job.
log.info("Requeueing %s on %s" % (job_data["_id"], queue.id))
stats["requeued"] += 1
job = Job(job_data["_id"])
job.requeue(queue=queue_name)
return stats
def test_retry(worker):
job_id = worker.send_task("mrq.basetasks.tests.general.Retry", {"queue": "noexec", "countdown": 60}, block=False)
job_data = Job(job_id).wait(poll_interval=0.01, full_data=True)
assert job_data["queue"] == "noexec"
assert job_data["status"] == "retry"
assert job_data["dateretry"] > datetime.datetime.utcnow()
assert job_data.get("result") is None
def get_job():
job_id = request.args['job_id']
job = Job(job_id)
job.fetch()
if job.data["params"].get("user"):
if not g.user.is_authenticated() or (job.data["params"].get("user") != str(g.user.id)):
return "Unauthorized."
return json.dumps({k: v for k, v in job.data.iteritems() if k in ("status", "result")})
def test_interrupt_worker_double_sigint(worker, p_flags):
""" Test what happens when we interrupt a running worker with 2 SIGINTs. """
start_time = time.time()
worker.start(flags=p_flags)
job_id = worker.send_task(
"tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 10}, block=False)
time.sleep(1)
job = Job(job_id).fetch().data
assert job["status"] == "started"
# Stop the worker gracefully. first job should still finish!
worker.stop(block=False, deps=False)
time.sleep(1)
# Should not be accepting new jobs!
job_id2 = worker.send_task(
"tests.tasks.general.Add", {"a": 42, "b": 1, "sleep": 10}, block=False)
time.sleep(1)
job = Job(job_id2).fetch().data
assert job.get("status") == "queued"
# Sending a second kill -2 should make it stop
worker.stop(block=True, deps=False, force=True)
time.sleep(1)
job = Job(job_id).fetch().data
assert job["status"] == "interrupt"
assert time.time() - start_time < 8
# Then try the cleaning task that requeues interrupted jobs
assert Queue("default").size() == 1
worker.start(queues="cleaning", deps=False, flush=False)
res = worker.send_task(
"mrq.basetasks.cleaning.RequeueInterruptedJobs", {}, block=True, queue="cleaning")
assert res["requeued"] == 1
assert Queue("default").size() == 2
Queue("default").list_job_ids() == [str(job_id2), str(job_id)]
job = Job(job_id).fetch().data
assert job["status"] == "queued"
assert job["queue"] == "default"
def test_retry_cancel_on_retry(worker):
job_id = worker.send_task("mrq.basetasks.tests.general.Retry", {
"queue": "noexec",
"countdown": 60,
"cancel_on_retry": True
}, block=False)
job_data = Job(job_id).wait(poll_interval=0.01, full_data=True)
assert job_data["status"] == "cancel"
assert job_data["queue"] == "default"
assert job_data.get("result") is None
def wait_for_tasks_results(self, job_ids, block=True, accept_statuses=["success"]):
if not block:
return job_ids
results = []
for job_id in job_ids:
job = Job(job_id).wait(poll_interval=0.01)
assert job.get("status") in accept_statuses, "Job had status %s, not in %s. Dump: %s" % (job.get("status"), accept_statuses, job)
results.append(job.get("result"))
return results
def test_retry(worker):
job_id = worker.send_task("tests.tasks.general.Retry", {
"queue": "noexec",
"delay": 60
},
block=False)
job_data = Job(job_id).wait(poll_interval=0.01, full_data=True)
assert job_data["queue"] == "noexec"
assert job_data["status"] == "retry"
assert job_data["dateretry"] > datetime.datetime.utcnow()
assert job_data.get("result") is None
def test_cli_run_nonblocking(worker):
worker.start_deps()
job_id1 = worker.send_task_cli("tests.tasks.general.Add", {"a": 41, "b": 1}, block=False)
job1 = Job(job_id1).fetch()
job1.wait(poll_interval=0.01)
job1.fetch()
assert job1.data["status"] == "success"
assert job1.data["result"] == 42
def run(self, params):
collection = connections.mongodb_jobs.mrq_jobs
# If there are more than this much items on the queue, we don't try to check if our mongodb
# jobs are still queued.
max_queue_items = params.get("max_queue_items", 1000)
stats = {"fetched": 0, "requeued": 0}
for job_data in collection.find({
"status": "queued"
},
fields={
"_id": 1,
"queue": 1
}).sort([("_id", 1)]):
stats["fetched"] += 1
queue = Queue(job_data["queue"])
queue_size = queue.size()
if queue_size > max_queue_items:
log.info("Stopping because queue %s has %s items" %
(queue, queue_size))
break
queue_jobs_ids = set(queue.list_job_ids(limit=max_queue_items + 1))
if len(queue_jobs_ids) >= max_queue_items:
log.info(
"Stopping because queue %s actually had more than %s items"
% (queue, len(queue_jobs_ids)))
break
if str(job_data["_id"]) in queue_jobs_ids:
log.info("Stopping because we found job %s in redis" %
job_data["_id"])
break
# At this point, this job is not on the queue and we're sure
# the queue is less than max_queue_items
# We can safely requeue the job.
log.info("Requeueing %s on %s" % (job_data["_id"], queue.id))
stats["requeued"] += 1
job = Job(job_data["_id"])
job.requeue(queue=job_data["queue"])
return stats
def wait_for_tasks_results(self, job_ids, block=True, accept_statuses=["success"]):
if not block:
return job_ids
results = []
for job_id in job_ids:
job = Job(job_id).wait(poll_interval=0.01)
assert job.get("status") in accept_statuses, "Job had status %s, not in %s. Dump: %s" % (
job.get("status"), accept_statuses, job)
results.append(job.get("result"))
return results
def get_job():
job_id = request.args['job_id']
job = Job(job_id)
job.fetch()
if job.data["params"].get("user"):
if not g.user.is_authenticated() or (job.data["params"].get("user") !=
str(g.user.id)):
return "Unauthorized."
return json.dumps(
{k: v
for k, v in job.data.iteritems() if k in ("status", "result")})
def test_interrupt_worker_sigterm(worker, p_flags):
""" Test what happens when we interrupt a running worker with 1 SIGTERM.
We should have had time to mark the task as 'interrupt' so that we can restart it somewhere else right away.
"""
start_time = time.time()
worker.start(flags=p_flags)
job_id = worker.send_task("tests.tasks.general.Add", {
"a": 41,
"b": 1,
"sleep": 10
},
block=False)
time.sleep(1)
worker.stop(block=True, sig=15, deps=False)
time.sleep(2)
job = Job(job_id).fetch().data
assert job["status"] == "interrupt"
assert time.time() - start_time < 6
worker.stop_deps()
def test_disconnects_service_during_task(worker, p_service):
""" Test what happens when mongodb disconnects during a job
"""
worker.start()
if p_service == "mongodb":
service = worker.fixture_mongodb
elif p_service == "redis":
service = worker.fixture_redis
service_pid = service.process.pid
job_id1 = worker.send_task("tests.tasks.general.Add", {
"a": 41,
"b": 1,
"sleep": 5
},
block=False,
queue="default")
time.sleep(2)
service.stop()
service.start()
service_pid2 = service.process.pid
# Make sure we did restart
assert service_pid != service_pid2
time.sleep(5)
# Result should be there without issues
assert Job(job_id1).fetch().data["result"] == 42
def test_cancel_by_path(worker):
# Start the worker with only one greenlet so that tasks execute
# sequentially
worker.start(flags="--greenlets 1")
job_id1 = worker.send_task("tests.tasks.general.MongoInsert", {
"a": 41,
"sleep": 2
},
block=False)
worker.wait_for_idle()
job_id2 = worker.send_task("tests.tasks.general.MongoInsert", {"a": 43},
block=False,
queue="testMrq")
worker.send_task("mrq.basetasks.utils.JobAction", {
"path": "tests.tasks.general.MongoInsert",
"status": "queued",
"action": "cancel"
},
block=False)
worker.wait_for_idle()
# Leave some time to unqueue job_id2 without executing.
time.sleep(1)
worker.stop(deps=False)
job1 = Job(job_id1).fetch().data
job2 = Job(job_id2).fetch().data
assert job1["status"] == "success"
assert job1["result"] == {"a": 41, "sleep": 2}
assert job2["status"] == "cancel"
assert job2["dateexpires"] > job2["dateupdated"]
assert job2.get("result") is None
assert worker.mongodb_jobs.tests_inserts.count() == 1
assert Queue("default").size() == 0
worker.stop_deps()
def test_custom_delimiters(worker, delimiter):
queue = "main" + delimiter
subqueue = queue + "subqueue"
worker.start(queues=queue, flags="--subqueues_refresh_interval=0.1 --subqueues_delimiter=%s" % delimiter)
job_id = worker.send_task("tests.tasks.general.GetTime", {}, queue=subqueue, block=False)
Job(job_id).wait(poll_interval=0.01)
worker.stop()
def test_interrupt_maxconcurrency(worker):
# The worker will raise a maxconcurrency on the second job
worker.start(flags="--greenlets=2")
job_ids = worker.send_tasks(
"tests.tasks.concurrency.LockedAdd", [{"a": i, "b": 1, "sleep": 2} for i in range(2)], block=False
)
worker.wait_for_tasks_results(job_ids, accept_statuses=["success", "failed", "maxconcurrency"])
job_statuses = [Job(job_id).fetch().data["status"] for job_id in job_ids]
assert set(job_statuses) == set(["success", "maxconcurrency"])
# the job concurrency key must be equal to 0
last_job_id = worker.send_task("tests.tasks.concurrency.LockedAdd", {"a": 1, "b": 1, "sleep": 2}, block=False)
last_job = Job(last_job_id).wait(poll_interval=0.01)
assert last_job.get("status") == "success"
def test_retry(worker):
worker.start(flags="--config tests/fixtures/config-scheduler8.py")
job_id = worker.send_task("tests.tasks.general.Retry", {
"queue": "noexec",
"delay": 60
},
block=False)
job_data = Job(job_id).wait(poll_interval=0.01, full_data=True)
assert job_data["queue"] == "noexec"
assert job_data["status"] == "retry"
assert job_data["dateretry"] > datetime.datetime.utcnow()
assert datetime.datetime.utcnow() + datetime.timedelta(
days=1) < job_data["dateexpires"] < datetime.datetime.utcnow(
) + datetime.timedelta(days=3)
assert job_data.get("result") is None
def run(self, params):
self.collection = connections.mongodb_jobs.mrq_jobs
# If there are more than this much items on the queue, we don't try to check if our mongodb
# jobs are still queued.
max_queue_items = params.get("max_queue_items", 1000)
stats = {
"fetched": 0,
"requeued": 0
}
for job_data in self.collection.find({
"status": "queued"
}, fields={"_id": 1, "queue": 1}).sort([("_id", 1)]):
stats["fetched"] += 1
queue = Queue(job_data["queue"])
queue_size = queue.size()
if queue_size > max_queue_items:
log.info("Stopping because queue %s has %s items" % (queue, queue_size))
break
queue_jobs_ids = set(queue.list_job_ids(limit=max_queue_items + 1))
if len(queue_jobs_ids) >= max_queue_items:
log.info("Stopping because queue %s actually had more than %s items" % (queue, len(queue_jobs_ids)))
break
if str(job_data["_id"]) in queue_jobs_ids:
log.info("Stopping because we found job %s in redis" % job_data["_id"])
break
# At this point, this job is not on the queue and we're sure the queue is less than max_queue_items
# We can safely requeue the job.
log.info("Requeueing %s on %s" % (job_data["_id"], queue.id))
stats["requeued"] += 1
job = Job(job_data["_id"])
job.requeue(queue=job_data["queue"])
return stats
def test_retry_max_retries(worker):
# Task has maxretries=1
worker.start(flags="--config tests/fixtures/config-retry1.py")
worker.send_task("tests.tasks.general.Retry", {},
block=True,
accept_statuses=["retry"])
assert Queue("default").size() == 0
job_id = worker.mongodb_jobs.mrq_jobs.find()[0]["_id"]
job = Job(job_id).fetch()
assert job.data["status"] == "retry"
assert job.data["retry_count"] == 1
time.sleep(2)
# Should requeue
worker.send_task("mrq.basetasks.cleaning.RequeueRetryJobs", {}, block=True)
time.sleep(2)
assert Queue("default").size() == 0
job = Job(job_id).fetch()
assert job.data["status"] == "maxretries"
assert job.data["retry_count"] == 1
# Then, manual requeue from the dashboard should reset the retry_count field.
params = {
"action": "requeue",
"status": "maxretries",
"destination_queue": "noexec"
}
worker.send_task("mrq.basetasks.utils.JobAction", params, block=True)
job = Job(job_id).fetch()
assert job.data["status"] == "queued"
assert job.data["queue"] == "noexec"
assert job.data["retry_count"] == 0
def test_matchable_subqueues(worker, queues, enqueue_on):
worker.start(queues=" ".join(queues), flags="--subqueues_refresh_interval=0.1")
job_ids = []
for subqueue in enqueue_on:
job_id = worker.send_task("tests.tasks.general.GetTime", {}, queue=subqueue, block=False)
job_ids.append(job_id)
assert all([Job(j).wait(poll_interval=0.01, timeout=3) for j in job_ids])
worker.stop()
def test_pause_resume(worker):
worker.start(flags="--paused_queues_refresh_interval=0.1")
Queue("high").pause()
assert Queue("high").is_paused()
# wait for the paused_queues list to be refreshed
time.sleep(2)
job_id1 = send_task("tests.tasks.general.MongoInsert", {"a": 41},
queue="high")
job_id2 = send_task("tests.tasks.general.MongoInsert", {"a": 43},
queue="low")
time.sleep(5)
job1 = Job(job_id1).fetch().data
job2 = Job(job_id2).fetch().data
assert job1["status"] == "queued"
assert job2["status"] == "success"
assert job2["result"] == {"a": 43}
assert worker.mongodb_jobs.tests_inserts.count() == 1
Queue("high").resume()
Job(job_id1).wait(poll_interval=0.01)
job1 = Job(job_id1).fetch().data
assert job1["status"] == "success"
assert job1["result"] == {"a": 41}
assert worker.mongodb_jobs.tests_inserts.count() == 2
worker.stop()
def test_job_requeue(worker):
from mrq.context import connections
from mrq.job import Job
worker.start()
job_id = worker.send_task("tests.tasks.general.RaiseException", {},
block=False)
worker.wait_for_idle()
assert int(connections.redis.get("queuesize:%s" % "default")) == 0
Job(job_id).requeue()
assert int(connections.redis.get("queuesize:%s" % "default")) == 1
def test_interrupt_worker_gracefully(worker, p_flags):
""" Test what happens when we interrupt a running worker gracefully. """
worker.start(flags=p_flags)
job_id = worker.send_task(
"tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 5}, block=False)
time.sleep(2)
job = Job(job_id).fetch().data
assert job["status"] == "started"
# Stop the worker gracefully. first job should still finish!
worker.stop(block=False, deps=False)
time.sleep(1)
# Should not be accepting new jobs!
job_id2 = worker.send_task(
"tests.tasks.general.Add", {"a": 42, "b": 1, "sleep": 4}, block=False)
time.sleep(1)
job = Job(job_id2).fetch().data
assert job.get("status") == "queued"
time.sleep(4)
job = Job(job_id).fetch().data
assert job["status"] == "success"
assert job["result"] == 42
job = Job(job_id2).fetch().data
assert job.get("status") == "queued"
def run(self, params):
additional_timeout = params.get("timeout", 300)
stats = {
"requeued": 0,
"started": 0
}
# There shouldn't be that much "started" jobs so we can quite safely
# iterate over them.
fields = {"_id": 1, "datestarted": 1, "queue": 1, "path": 1, "retry_count": 1}
for job_data in connections.mongodb_jobs.mrq_jobs.find(
{"status": "started"}, projection=fields):
job = Job(job_data["_id"])
job.set_data(job_data)
stats["started"] += 1
expire_date = datetime.datetime.utcnow(
) - datetime.timedelta(seconds=job.timeout + additional_timeout)
if job_data["datestarted"] < expire_date:
log.debug("Requeueing job %s" % job.id)
job.requeue()
stats["requeued"] += 1
return stats
def test_matchable_subqueues(worker, queues, enqueue_on):
worker.start(queues=" ".join(queues),
flags="--subqueues_refresh_interval=0.1")
job_ids = []
for subqueue in enqueue_on:
job_id = worker.send_task("tests.tasks.general.GetTime", {},
queue=subqueue,
block=False)
job_ids.append(job_id)
for i, job_id in enumerate(job_ids):
print("Checking queue %s" % enqueue_on[i])
assert Job(job_id).wait(poll_interval=0.01, timeout=5)
def test_retry_max_retries_zero(worker):
# Task has maxretries=1
worker.start(flags="--config tests/fixtures/config-retry1.py")
worker.send_task("tests.tasks.general.Retry", {"max_retries": 0},
block=True,
accept_statuses=["maxretries"])
assert Queue("default").size() == 0
job_id = worker.mongodb_jobs.mrq_jobs.find()[0]["_id"]
job = Job(job_id).fetch()
assert job.data["status"] == "maxretries"
def test_refresh_interval(worker):
""" Tests that a refresh interval of 0 disables the subqueue detection """
worker.start(queues="test/", flags="--subqueues_refresh_interval=0")
time.sleep(2)
job_id1 = worker.send_task("tests.tasks.general.GetTime", {"a": 41},
queue="test/subqueue",
block=False)
time.sleep(5)
job1 = Job(job_id1).fetch().data
assert job1["status"] == "queued"
def test_unmatchable_subqueues(worker, queue, enqueue_on):
worker.start(queues=queue, flags="--subqueues_refresh_interval=0.1")
job_ids = []
for subqueue in enqueue_on:
job_id = worker.send_task("tests.tasks.general.GetTime", {}, queue=subqueue, block=False)
job_ids.append(job_id)
time.sleep(2)
results = [Job(j).fetch().data.get("status") for j in job_ids]
# ensure tasks are not consumed by a worker
assert results == ["queued"] * len(results)
worker.stop()
def test_pause_subqueue(worker):
# set config in current context in order to have a subqueue delimiter
set_current_config(get_config(config_type="worker"))
worker.start(
queues="high high/",
flags=
"--subqueues_refresh_interval=1 --paused_queues_refresh_interval=1")
Queue("high").pause()
assert Queue("high/").is_paused()
# wait for the paused_queues list to be refreshed
time.sleep(2)
job_id1 = send_task("tests.tasks.general.MongoInsert", {"a": 41},
queue="high")
job_id2 = send_task("tests.tasks.general.MongoInsert", {"a": 43},
queue="high/subqueue")
# wait a bit to make sure the jobs status will still be queued
time.sleep(5)
job1 = Job(job_id1).fetch().data
job2 = Job(job_id2).fetch().data
assert job1["status"] == "queued"
assert job2["status"] == "queued"
assert worker.mongodb_jobs.tests_inserts.count() == 0
Queue("high/").resume()
Job(job_id1).wait(poll_interval=0.01)
Job(job_id2).wait(poll_interval=0.01)
job1 = Job(job_id1).fetch().data
job2 = Job(job_id2).fetch().data
assert job1["status"] == "success"
assert job1["result"] == {"a": 41}
assert job2["status"] == "success"
assert job2["result"] == {"a": 43}
assert worker.mongodb_jobs.tests_inserts.count() == 2
worker.stop()
def test_general_requeue_order(worker):
from mrq.job import Job
jobids = worker.send_tasks("tests.tasks.general.Add", [
{"a": 41, "b": 1, "sleep": 4},
{"a": 42, "b": 1, "sleep": 1},
{"a": 43, "b": 1, "sleep": 1}
], block=False)
time.sleep(2)
# We should be executing job1 now. Let's requeue job2, making it go to the end of the queue.
Job(jobids[1]).requeue()
worker.wait_for_idle()
assert [x["result"] for x in worker.mongodb_jobs.mrq_jobs.find().sort(
[["dateupdated", 1]])] == [42, 44, 43]
def test_cli_run_nonblocking(worker):
worker.start()
job_id1 = worker.send_task_cli(
"tests.tasks.general.Add", {"a": 41, "b": 1}, queue="default")
job1 = Job(job_id1).fetch()
job1.wait(poll_interval=0.01)
job1.fetch()
assert job1.data["status"] == "success"
assert job1.data["result"] == 42
def test_pause_refresh_interval(worker):
""" Tests that a refresh interval of 0 disables the pause functionnality """
worker.start(flags="--paused_queues_refresh_interval=0")
Queue("high").pause()
assert Queue("high").is_paused()
# wait for the paused_queues list to be refreshed
time.sleep(2)
job_id1 = send_task("tests.tasks.general.MongoInsert", {"a": 41},
queue="high")
time.sleep(5)
job1 = Job(job_id1).fetch().data
assert job1["status"] == "success"
assert job1["result"] == {"a": 41}
def run(self, params):
redis_key_started = Queue.redis_key_started()
stats = {
"fetched": 0,
"requeued": 0
}
# Fetch all the jobs started more than a minute ago - they should not
# be in redis:started anymore
job_ids = connections.redis.zrangebyscore(
redis_key_started, "-inf", time.time() - params.get("timeout", 60))
# TODO this should be wrapped inside Queue or Worker
# we shouldn't access these internals here
queue_obj = Queue("default")
unserialized_job_ids = queue_obj.unserialize_job_ids(job_ids)
for i, job_id in enumerate(job_ids):
queue = Job(unserialized_job_ids[i], start=False, fetch=False).fetch(
full_data=True).data["queue"]
queue_obj = Queue(queue)
stats["fetched"] += 1
log.info("Requeueing %s on %s" % (unserialized_job_ids[i], queue))
# TODO LUA script & don't rpush if not in zset anymore.
with connections.redis.pipeline(transaction=True) as pipeline:
pipeline.zrem(redis_key_started, job_id)
pipeline.rpush(queue_obj.redis_key, job_id)
pipeline.execute()
stats["requeued"] += 1
return stats
def test_raw_no_storage(worker):
""" Test tasks that don't store unless they go to error status like 'failed' """
worker.start(
flags="--config tests/fixtures/config-raw1.py",
queues="default testnostorage_raw"
)
jobs_collection = worker.mongodb_jobs.mrq_jobs
test_collection = worker.mongodb_logs.tests_inserts
worker.send_raw_tasks("testnostorage_raw", [
"tests.tasks.general.MongoInsert 3"
], block=False)
time.sleep(2)
# No started inserted.
assert jobs_collection.count() == 0
time.sleep(2)
# No success either, but we did insert
assert test_collection.count() == 1
assert jobs_collection.count() == 0
test_collection.remove({})
# However failed tasks get stored.
worker.send_raw_tasks("testnostorage_raw", [
"tests.tasks.general.RaiseException 0"
], block=False)
time.sleep(2)
# Failed was inserted.
assert jobs_collection.count({"status": "failed", "path": "tests.tasks.general.RaiseException"}) == 1
# If we requeue and don't raise, should be OK and inserted this time, even in success
# no_storage depends on a raw queue, not a task path.
_id = jobs_collection.find_one()["_id"]
jobs_collection.update({"_id": _id}, {"$set": {"path": "tests.tasks.general.MongoInsert"}})
job = Job(_id).fetch(full_data=True)
job.requeue(queue="default")
time.sleep(1)
assert test_collection.count() == 1
assert jobs_collection.count() == 1
assert jobs_collection.count({"status": "success"}) == 1
jobs_collection.remove({})
# Test with retry: should be inserted
worker.send_raw_tasks("testnostorage_raw", [
"tests.tasks.general.Retry 0"
], block=False)
assert jobs_collection.count({"status": "started"}) == 0
time.sleep(2)
assert jobs_collection.count({"status": "retry"}) == 1
