def test_interrupt_redis_started_jobs(worker): worker.start( queues="xxx", flags=" --config tests/fixtures/config-lostjobs.py") worker.send_task("tests.tasks.general.Add", { "a": 41, "b": 1, "sleep": 10}, block=False, queue="xxx") worker.send_task("tests.tasks.general.Add", { "a": 41, "b": 1, "sleep": 10}, block=False, queue="xxx") time.sleep(3) worker.stop(deps=False) assert Queue("xxx").size() == 0 assert connections.redis.zcard(Queue.redis_key_started()) == 2 worker.start(queues="default", start_deps=False, flush=False) assert connections.redis.zcard(Queue.redis_key_started()) == 2 res = worker.send_task("mrq.basetasks.cleaning.RequeueRedisStartedJobs", { "timeout": 0 }, block=True, queue="default") assert res["fetched"] == 2 assert res["requeued"] == 2 assert Queue("xxx").size() == 2 assert Queue("default").size() == 0 assert connections.redis.zcard(Queue.redis_key_started()) == 0
def test_known_queues_lifecycle(worker): worker.start(queues="default_reverse xtest test_timed_set", flags="--config tests/fixtures/config-raw1.py") time.sleep(1) # Test known queues from mrq.queue import Queue, send_task assert set(Queue.redis_known_queues().keys()) == set(["default", "xtest", "test_timed_set"]) # Try queueing a task send_task("tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 1}, queue="x") time.sleep(1) assert set(Queue.redis_known_queues().keys()) == set(["x", "default", "xtest", "test_timed_set"]) Queue("x").add_to_known_queues(timestamp=time.time() - (8 * 86400)) worker.send_task("mrq.basetasks.cleaning.CleanKnownQueues", {}, block=True) # Not removed - not empty yet. assert set(Queue.redis_known_queues().keys()) == set(["x", "default", "xtest", "test_timed_set"]) Queue("x").empty() # Still not removed. assert set(Queue.redis_known_queues().keys()) == set(["x", "default", "xtest", "test_timed_set"]) worker.send_task("mrq.basetasks.cleaning.CleanKnownQueues", {}, block=True) # Now we're good assert set(Queue.redis_known_queues().keys()) == set(["default", "xtest", "test_timed_set"])
def test_raw_set(worker, has_subqueue, p_queue, p_set): flags = "--greenlets 10 --config tests/fixtures/config-raw1.py" if has_subqueue: flags = "%s --subqueues_refresh_interval=0.1" % flags # worker should dequeue all subqueues p_queue = "%s/" % p_queue worker.start(flags=flags, queues=p_queue) if has_subqueue: # queue tasks in p_queue/subqueue p_queue = "%ssubqueue" % p_queue test_collection = worker.mongodb_logs.tests_inserts jobs_collection = worker.mongodb_jobs.mrq_jobs assert jobs_collection.count() == 0 assert Queue(p_queue).size() == 0 worker.send_raw_tasks(p_queue, ["aaa", "bbb", "ccc", "bbb"], block=True) assert Queue(p_queue).size() == 0 if p_set: assert jobs_collection.count() == 3 assert jobs_collection.count({"status": "success"}) == 3 assert test_collection.count() == 3 else: assert jobs_collection.count() == 4 assert jobs_collection.count({"status": "success"}) == 4 assert test_collection.count() == 4
def test_retry_from_other_queue_stays_on_queue(worker): worker.start(queues="default exec") worker.send_task("tests.tasks.general.Retry", {"delay": 1}, queue="exec", accept_statuses="retry") time.sleep(2) job_id = worker.mongodb_jobs.mrq_jobs.find()[0]["_id"] job = Job(job_id).fetch() assert job.data["status"] == "retry" assert job.data["queue"] == "exec" assert Queue("default").size() == 0 assert Queue("exec").size() == 0 worker.stop(deps=False) worker.start(queues="default", deps=False) # Should do nothing yet worker.send_task("mrq.basetasks.cleaning.RequeueRetryJobs", {}, block=True) assert Queue("default").size() == 0 assert Queue("exec").size() == 1 job = Job(job_id).fetch() assert job.data["status"] == "queued" assert job.data["queue"] == "exec"
def test_raw_mixed(worker, p_queue, p_greenlets): worker.start_deps() worker.send_raw_tasks( "test_raw", ["aaa", "bbb", "ccc"], start=False, block=False) worker.send_task("tests.tasks.general.MongoInsert", { "not_raw": "ddd" }, start=False, block=False) assert Queue("test_raw").size() == 3 assert Queue("default").size() == 1 worker.start(flags="--greenlets %s --config tests/fixtures/config-raw1.py" % p_greenlets, queues=p_queue, deps=False) test_collection = worker.mongodb_logs.tests_inserts jobs_collection = worker.mongodb_jobs.mrq_jobs time.sleep(1) assert Queue("test_raw").size() == 0 assert Queue("default").size() == 0 assert test_collection.count() == 4 assert jobs_collection.count() == 4 assert jobs_collection.find({"status": "success"}).count() == 4 assert list(jobs_collection.find({"status": "success"}))[0]["worker"]
def test_interrupt_worker_double_sigint(worker, p_flags): """ Test what happens when we interrupt a running worker with 2 SIGINTs. """ start_time = time.time() worker.start(flags=p_flags) job_id = worker.send_task( "tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 10}, block=False) time.sleep(1) job = Job(job_id).fetch().data assert job["status"] == "started" # Stop the worker gracefully. first job should still finish! worker.stop(block=False, deps=False) time.sleep(1) # Should not be accepting new jobs! job_id2 = worker.send_task( "tests.tasks.general.Add", {"a": 42, "b": 1, "sleep": 10}, block=False) time.sleep(1) job = Job(job_id2).fetch().data assert job.get("status") == "queued" # Sending a second kill -2 should make it stop worker.stop(block=True, deps=False, force=True) time.sleep(1) job = Job(job_id).fetch().data assert job["status"] == "interrupt" assert time.time() - start_time < 8 # Then try the cleaning task that requeues interrupted jobs assert Queue("default").size() == 1 worker.start(queues="cleaning", deps=False, flush=False) res = worker.send_task( "mrq.basetasks.cleaning.RequeueInterruptedJobs", {}, block=True, queue="cleaning") assert res["requeued"] == 1 assert Queue("default").size() == 2 Queue("default").list_job_ids() == [str(job_id2), str(job_id)] job = Job(job_id).fetch().data assert job["status"] == "queued" assert job["queue"] == "default"
def test_pause_subqueue(worker): # set config in current context in order to have a subqueue delimiter set_current_config(get_config(config_type="worker")) worker.start( queues="high high/", flags= "--subqueues_refresh_interval=1 --paused_queues_refresh_interval=1") Queue("high").pause() assert Queue("high/").is_paused() # wait for the paused_queues list to be refreshed time.sleep(2) job_id1 = send_task("tests.tasks.general.MongoInsert", {"a": 41}, queue="high") job_id2 = send_task("tests.tasks.general.MongoInsert", {"a": 43}, queue="high/subqueue") # wait a bit to make sure the jobs status will still be queued time.sleep(5) job1 = Job(job_id1).fetch().data job2 = Job(job_id2).fetch().data assert job1["status"] == "queued" assert job2["status"] == "queued" assert worker.mongodb_jobs.tests_inserts.count() == 0 Queue("high/").resume() Job(job_id1).wait(poll_interval=0.01) Job(job_id2).wait(poll_interval=0.01) job1 = Job(job_id1).fetch().data job2 = Job(job_id2).fetch().data assert job1["status"] == "success" assert job1["result"] == {"a": 41} assert job2["status"] == "success" assert job2["result"] == {"a": 43} assert worker.mongodb_jobs.tests_inserts.count() == 2 worker.stop()
def test_raw_remove(worker, p_queue): worker.start_deps() worker.send_raw_tasks(p_queue, ["aa", "bb", "cc"], block=False, start=False) assert Queue(p_queue).size() == 3 Queue(p_queue).remove_raw_jobs(["aa", "cc"]) assert Queue(p_queue).size() == 1
def test_retry_otherqueue_delay_zero(worker): # delay = 0 should requeue right away. job_id = worker.send_task("tests.tasks.general.Retry", { "queue": "noexec", "delay": 0 }, block=False) time.sleep(1) assert worker.mongodb_logs.tests_inserts.find().count() == 1 assert Queue("default").size() == 0 assert Queue("noexec").size() == 1 assert Queue("noexec").list_job_ids() == [str(job_id)]
def test_worker_crash(worker): """ Test that when a worker crashes its running jobs are requeued """ worker.start(queues="default") worker.send_task("tests.tasks.general.Add", { "a": 41, "b": 1, "sleep": 10 }, block=False, queue="default") time.sleep(5) worker.stop(block=True, sig=9, deps=False) time.sleep(1) # simulate worker crash worker.mongodb_jobs.mrq_workers.delete_many({}) worker.start(queues="cleaning", deps=False, flush=False) res = worker.send_task("mrq.basetasks.cleaning.RequeueStartedJobs", {"timeout": 90}, block=True, queue="cleaning") assert res["requeued"] == 1 assert res["started"] == 2 assert Queue("default").size() == 1
def run(self, params): collection = connections.mongodb_jobs.simple_crawler_urls collection.remove({}) Queue("crawl").empty()
def run(self, params): max_age = int(params.get("max_age") or (7 * 86400)) pretend = bool(params.get("pretend")) check_mongo = bool(params.get("check_mongo")) known_queues = Queue.redis_known_queues() removed_queues = [] queues_from_config = Queue.all_known_from_config() print "Found %s known queues & %s from config" % ( len(known_queues), len(queues_from_config)) # Only clean queues older than N days time_threshold = time.time() - max_age for queue, time_last_used in known_queues.iteritems(): if queue in queues_from_config: continue if time_last_used < time_threshold: q = Queue(queue, add_to_known_queues=False) size = q.size() if check_mongo: size += connections.mongodb_jobs.mrq_jobs.count( {"queue": queue}) if size == 0: removed_queues.append(queue) print "Removing empty queue '%s' from known queues ..." % queue if not pretend: q.remove_from_known_queues() print "Cleaned %s queues" % len(removed_queues) return removed_queues
def test_interrupt_redis_flush(worker): """ Test what happens when we flush redis after queueing jobs. The RequeueLostJobs task should put them back in redis. """ worker.start(queues="cleaning", deps=True, flush=True) job_id1 = worker.send_task("tests.tasks.general.Add", { "a": 41, "b": 1, "sleep": 10}, block=False, queue="default") job_id2 = worker.send_task("tests.tasks.general.Add", { "a": 41, "b": 1, "sleep": 10}, block=False, queue="default") job_id3 = worker.send_task("tests.tasks.general.Add", { "a": 41, "b": 1, "sleep": 10}, block=False, queue="otherq") assert Queue("default").size() == 2 assert Queue("otherq").size() == 1 res = worker.send_task( "mrq.basetasks.cleaning.RequeueLostJobs", {}, block=True, queue="cleaning") # We should try the first job only, and when seeing it's there we should # stop. assert res["fetched"] == 1 assert res["requeued"] == 0 assert Queue("default").size() == 2 assert Queue("otherq").size() == 1 # Then flush redis! worker.fixture_redis.flush() # Assert the queues are empty. assert Queue("default").size() == 0 assert Queue("otherq").size() == 0 res = worker.send_task( "mrq.basetasks.cleaning.RequeueLostJobs", {}, block=True, queue="cleaning") assert res["fetched"] == 3 assert res["requeued"] == 3 assert Queue("default").size() == 2 assert Queue("otherq").size() == 1 assert Queue("default").list_job_ids() == [str(job_id1), str(job_id2)] assert Queue("otherq").list_job_ids() == [str(job_id3)]
def build_api_datatables_query(req): query = {} if req.args.get("redisqueue"): query["_id"] = { "$in": [ ObjectId(x) for x in Queue(req.args.get("redisqueue")).list_job_ids( limit=1000) ] } else: for param in ["queue", "path", "exceptiontype"]: if req.args.get(param): if "*" in req.args[param]: regexp = "^%s$" % req.args[param].replace("*", ".*") query[param] = {"$regex": regexp} else: query[param] = req.args[param] if req.args.get("queue") and req.args["queue"].endswith("/"): subqueues = Queue(req.args["queue"]).get_known_subqueues() query["queue"] = {"$in": list(subqueues)} if req.args.get("status"): statuses = req.args["status"].split("-") if len(statuses) == 1: query["status"] = statuses[0] else: query["status"] = {"$in": statuses} if req.args.get("id"): query["_id"] = ObjectId(req.args.get("id")) if req.args.get("worker"): query["worker"] = ObjectId(req.args.get("worker")) if req.args.get("params"): try: params_dict = json.loads(req.args.get("params")) for key in params_dict: query["params.%s" % key] = params_dict[key] except Exception as e: # pylint: disable=broad-except print("Error will converting form JSON: %s" % e) return query
def run(self, params): # If there are more than this much items on the queue, we don't try to check if our mongodb # jobs are still queued. max_queue_items = params.get("max_queue_items", 1000) stats = {"fetched": 0, "requeued": 0} all_queues = Queue.all_known() for queue_name in all_queues: queue = Queue(queue_name) queue_size = queue.size() if queue.is_raw: continue log.info("Checking queue %s" % queue_name) if queue_size > max_queue_items: log.info("Stopping because queue %s has %s items" % (queue_name, queue_size)) continue queue_jobs_ids = set(queue.list_job_ids(limit=max_queue_items + 1)) if len(queue_jobs_ids) >= max_queue_items: log.info( "Stopping because queue %s actually had more than %s items" % (queue_name, len(queue_jobs_ids))) continue for job_data in connections.mongodb_jobs.mrq_jobs.find( { "queue": queue_name, "status": "queued" }, projection={ "_id": 1 }).sort([["_id", 1]]): stats["fetched"] += 1 if str(job_data["_id"]) in queue_jobs_ids: log.info("Found job %s on queue %s. Stopping" % (job_data["_id"], queue.id)) break # At this point, this job is not on the queue and we're sure # the queue is less than max_queue_items # We can safely requeue the job. log.info("Requeueing %s on %s" % (job_data["_id"], queue.id)) stats["requeued"] += 1 job = Job(job_data["_id"]) job.requeue(queue=queue_name) return stats
def test_raw_exception(worker): p_queue = "testexception_raw" worker.start( flags="--greenlets 10 --config tests/fixtures/config-raw1.py", queues=p_queue) jobs_collection = worker.mongodb_jobs.mrq_jobs assert jobs_collection.count() == 0 assert Queue(p_queue).size() == 0 worker.send_raw_tasks(p_queue, ["msg1"], block=True) failjob = list(jobs_collection.find())[0] assert Queue("default").size() == 0 assert Queue(p_queue).size() == 0 assert jobs_collection.count() == 1 assert failjob["status"] == "failed" worker.stop(deps=False) worker.start( deps=False, flags="--greenlets 10 --config tests/fixtures/config-raw1.py", queues="default") worker.send_task( "mrq.basetasks.utils.JobAction", { "id": failjob["_id"], "action": "requeue" }, block=True ) assert Queue("default").size() == 0 assert Queue(p_queue).size() == 0 assert Queue("testx").size() == 1 assert jobs_collection.count() == 2 assert list(jobs_collection.find({"_id": failjob["_id"]}))[ 0]["status"] == "queued" assert list(jobs_collection.find({"_id": {"$ne": failjob["_id"]}}))[ 0]["status"] == "success" worker.stop(deps=False) worker.start( deps=False, flags="--greenlets 10 --config tests/fixtures/config-raw1.py", queues="default testx") time.sleep(2) assert Queue(p_queue).size() == 0 assert jobs_collection.count() == 2 assert Queue("testx").size() == 0 assert list(jobs_collection.find({"_id": failjob["_id"]}))[ 0]["status"] == "failed"
def test_retry_max_retries(worker): # Task has maxretries=1 worker.start(flags="--config tests/fixtures/config-retry1.py") worker.send_task("tests.tasks.general.Retry", {}, block=True, accept_statuses=["retry"]) assert Queue("default").size() == 0 job_id = worker.mongodb_jobs.mrq_jobs.find()[0]["_id"] job = Job(job_id).fetch() assert job.data["status"] == "retry" assert job.data["retry_count"] == 1 time.sleep(2) # Should requeue worker.send_task("mrq.basetasks.cleaning.RequeueRetryJobs", {}, block=True) time.sleep(2) assert Queue("default").size() == 0 job = Job(job_id).fetch() assert job.data["status"] == "maxretries" assert job.data["retry_count"] == 1 # Then, manual requeue from the dashboard should reset the retry_count field. params = { "action": "requeue", "status": "maxretries", "destination_queue": "noexec" } worker.send_task("mrq.basetasks.utils.JobAction", params, block=True) job = Job(job_id).fetch() assert job.data["status"] == "queued" assert job.data["queue"] == "noexec" assert job.data["retry_count"] == 0
def test_pause_resume(worker): worker.start(flags="--paused_queues_refresh_interval=0.1") Queue("high").pause() assert Queue("high").is_paused() # wait for the paused_queues list to be refreshed time.sleep(2) job_id1 = send_task("tests.tasks.general.MongoInsert", {"a": 41}, queue="high") job_id2 = send_task("tests.tasks.general.MongoInsert", {"a": 43}, queue="low") time.sleep(5) job1 = Job(job_id1).fetch().data job2 = Job(job_id2).fetch().data assert job1["status"] == "queued" assert job2["status"] == "success" assert job2["result"] == {"a": 43} assert worker.mongodb_jobs.tests_inserts.count() == 1 Queue("high").resume() Job(job_id1).wait(poll_interval=0.01) job1 = Job(job_id1).fetch().data assert job1["status"] == "success" assert job1["result"] == {"a": 41} assert worker.mongodb_jobs.tests_inserts.count() == 2 worker.stop()
def test_pause_refresh_interval(worker): """ Tests that a refresh interval of 0 disables the pause functionnality """ worker.start(flags="--paused_queues_refresh_interval=0") Queue("high").pause() assert Queue("high").is_paused() # wait for the paused_queues list to be refreshed time.sleep(2) job_id1 = send_task("tests.tasks.general.MongoInsert", {"a": 41}, queue="high") time.sleep(5) job1 = Job(job_id1).fetch().data assert job1["status"] == "success" assert job1["result"] == {"a": 41}
def test_raw_retry(worker): p_queue = "testretry_raw" worker.start(flags="--greenlets 10 --config tests/fixtures/config-raw1.py", queues=p_queue) jobs_collection = worker.mongodb_jobs.mrq_jobs assert jobs_collection.count() == 0 assert Queue(p_queue).size() == 0 worker.send_raw_tasks(p_queue, [0], block=True) failjob = list(jobs_collection.find())[0] assert Queue("testx").size() == 1 assert Queue("default").size() == 0 assert Queue(p_queue).size() == 0 assert jobs_collection.count() == 1 assert failjob["status"] == "queued" assert failjob["queue"] == "testx"
def test_retry_otherqueue_delay_nonzero(worker): # delay = 0 should requeue right away. worker.send_task("tests.tasks.general.Retry", { "queue": "noexec", "delay": 2 }, block=True, accept_statuses=["retry"]) assert Queue("default").size() == 0 assert Queue("noexec").size() == 0 job_id = worker.mongodb_jobs.mrq_jobs.find()[0]["_id"] job = Job(job_id).fetch() assert job.data["status"] == "retry" # Should do nothing yet worker.send_task("mrq.basetasks.cleaning.RequeueRetryJobs", {}, block=True) assert Queue("default").size() == 0 assert Queue("noexec").size() == 0 time.sleep(2) # Should requeue worker.send_task("mrq.basetasks.cleaning.RequeueRetryJobs", {}, block=True) assert Queue("default").size() == 0 assert Queue("noexec").size() == 1 job = Job(job_id).fetch() assert job.data["status"] == "queued"
def run(self, params): redis_key_started = Queue.redis_key_started() stats = { "fetched": 0, "requeued": 0 } # Fetch all the jobs started more than a minute ago - they should not # be in redis:started anymore job_ids = connections.redis.zrangebyscore( redis_key_started, "-inf", time.time() - params.get("timeout", 60)) # TODO this should be wrapped inside Queue or Worker # we shouldn't access these internals here queue_obj = Queue("default") unserialized_job_ids = queue_obj.unserialize_job_ids(job_ids) for i, job_id in enumerate(job_ids): queue = Job(unserialized_job_ids[i], start=False, fetch=False).fetch( full_data=True).data["queue"] queue_obj = Queue(queue) stats["fetched"] += 1 log.info("Requeueing %s on %s" % (unserialized_job_ids[i], queue)) # TODO LUA script & don't rpush if not in zset anymore. with connections.redis.pipeline(transaction=True) as pipeline: pipeline.zrem(redis_key_started, job_id) pipeline.rpush(queue_obj.redis_key, job_id) pipeline.execute() stats["requeued"] += 1 return stats
def run(self, params): max_age = int(params.get("max_age") or (7 * 86400)) known_queues = Queue.redis_known_queues() # Only clean queues older than N days time_threshold = time.time() - max_age for queue, time_last_used in known_queues.iteritems(): if time_last_used < time_threshold: q = Queue(queue, add_to_known_queues=False) if q.size() == 0: q.remove_from_known_queues()
def test_abort(worker): worker.start() worker.send_task("tests.tasks.general.Abort", {"a": 41}, accept_statuses=["abort"]) assert Queue("default").size() == 0 db_jobs = list(worker.mongodb_jobs.mrq_jobs.find()) assert len(db_jobs) == 1 assert db_jobs[0]["status"] == "abort"
def test_interrupt_maxjobs(worker): # The worker will stop after doing 5 jobs worker.start(flags="--max_jobs 5 --greenlets 2", queues="test1 default") worker.send_tasks("tests.tasks.general.Add", [ {"a": i, "b": 1, "sleep": 0} for i in range(12) ], block=False) time.sleep(2) assert Queue("default").size() == 7
def run(self, params): collection = connections.mongodb_jobs.mrq_jobs # If there are more than this much items on the queue, we don't try to check if our mongodb # jobs are still queued. max_queue_items = params.get("max_queue_items", 1000) stats = {"fetched": 0, "requeued": 0} for job_data in collection.find({ "status": "queued" }, fields={ "_id": 1, "queue": 1 }).sort([("_id", 1)]): stats["fetched"] += 1 queue = Queue(job_data["queue"]) queue_size = queue.size() if queue_size > max_queue_items: log.info("Stopping because queue %s has %s items" % (queue, queue_size)) break queue_jobs_ids = set(queue.list_job_ids(limit=max_queue_items + 1)) if len(queue_jobs_ids) >= max_queue_items: log.info( "Stopping because queue %s actually had more than %s items" % (queue, len(queue_jobs_ids))) break if str(job_data["_id"]) in queue_jobs_ids: log.info("Stopping because we found job %s in redis" % job_data["_id"]) break # At this point, this job is not on the queue and we're sure # the queue is less than max_queue_items # We can safely requeue the job. log.info("Requeueing %s on %s" % (job_data["_id"], queue.id)) stats["requeued"] += 1 job = Job(job_data["_id"]) job.requeue(queue=job_data["queue"]) return stats
def test_sorted_graph(worker): p_queue = "test_sorted_set" worker.start_deps() assert Queue(p_queue).size() == 0 worker.send_raw_tasks(p_queue, { "000": -1, "aaa": 1, "aaa2": 1.5, "bbb": 2, "ccc": 4 }, start=False, block=False) time.sleep(0.5) assert Queue(p_queue).size() == 5 assert Queue(p_queue).get_sorted_graph( 1, 4, slices=3, include_inf=True) == [1, 2, 1, 0, 1] assert Queue(p_queue).get_sorted_graph( 1, 4, slices=3, include_inf=False) == [2, 1, 0] worker.stop_deps()
def test_abort(worker): worker.start() worker.send_task("tests.tasks.general.Abort", {"a": 41}, accept_statuses=["abort"]) assert Queue("default").size() == 0 db_jobs = list(worker.mongodb_jobs.mrq_jobs.find()) assert len(db_jobs) == 1 job = db_jobs[0] assert job["status"] == "abort" assert job.get("dateexpires") is not None assert job["dateexpires"] < datetime.utcnow() + timedelta(hours=24)
def send_raw_tasks(self, queue, params_list, start=True, block=True): if not self.started and start: self.start() queue_raw_jobs(queue, params_list) if block: # Wait for the queue to be empty. Might be error-prone when tasks # are in-memory between the 2 q = Queue(queue) while q.size() > 0 or self.mongodb_jobs.mrq_jobs.find({"status": "started"}).count() > 0: # print "S", q.size(), # self.mongodb_jobs.mrq_jobs.find({"status": # "started"}).count() time.sleep(0.1)
def test_retry_max_retries_zero(worker): # Task has maxretries=1 worker.start(flags="--config tests/fixtures/config-retry1.py") worker.send_task("tests.tasks.general.Retry", {"max_retries": 0}, block=True, accept_statuses=["maxretries"]) assert Queue("default").size() == 0 job_id = worker.mongodb_jobs.mrq_jobs.find()[0]["_id"] job = Job(job_id).fetch() assert job.data["status"] == "maxretries"