def test_known_queues_lifecycle(worker):
worker.start(queues="default_reverse xtest test_timed_set", flags="--config tests/fixtures/config-raw1.py")
time.sleep(1)
# Test known queues
from mrq.queue import Queue, send_task
assert set(Queue.redis_known_queues().keys()) == set(["default", "xtest", "test_timed_set"])
# Try queueing a task
send_task("tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 1}, queue="x")
time.sleep(1)
assert set(Queue.redis_known_queues().keys()) == set(["x", "default", "xtest", "test_timed_set"])
Queue("x").add_to_known_queues(timestamp=time.time() - (8 * 86400))
worker.send_task("mrq.basetasks.cleaning.CleanKnownQueues", {}, block=True)
# Not removed - not empty yet.
assert set(Queue.redis_known_queues().keys()) == set(["x", "default", "xtest", "test_timed_set"])
Queue("x").empty()
# Still not removed.
assert set(Queue.redis_known_queues().keys()) == set(["x", "default", "xtest", "test_timed_set"])
worker.send_task("mrq.basetasks.cleaning.CleanKnownQueues", {}, block=True)
# Now we're good
assert set(Queue.redis_known_queues().keys()) == set(["default", "xtest", "test_timed_set"])
def test_pause_subqueue(worker):
# set config in current context in order to have a subqueue delimiter
set_current_config(get_config(config_type="worker"))
worker.start(
queues="high high/",
flags=
"--subqueues_refresh_interval=1 --paused_queues_refresh_interval=1")
Queue("high").pause()
assert Queue("high/").is_paused()
# wait for the paused_queues list to be refreshed
time.sleep(2)
job_id1 = send_task("tests.tasks.general.MongoInsert", {"a": 41},
queue="high")
job_id2 = send_task("tests.tasks.general.MongoInsert", {"a": 43},
queue="high/subqueue")
# wait a bit to make sure the jobs status will still be queued
time.sleep(5)
job1 = Job(job_id1).fetch().data
job2 = Job(job_id2).fetch().data
assert job1["status"] == "queued"
assert job2["status"] == "queued"
assert worker.mongodb_jobs.tests_inserts.count() == 0
Queue("high/").resume()
Job(job_id1).wait(poll_interval=0.01)
Job(job_id2).wait(poll_interval=0.01)
job1 = Job(job_id1).fetch().data
job2 = Job(job_id2).fetch().data
assert job1["status"] == "success"
assert job1["result"] == {"a": 41}
assert job2["status"] == "success"
assert job2["result"] == {"a": 43}
assert worker.mongodb_jobs.tests_inserts.count() == 2
worker.stop()
def test_pause_subqueue(worker):
# set config in current context in order to have a subqueue delimiter
set_current_config(get_config(config_type="worker"))
worker.start(queues="high high/", flags="--subqueues_refresh_interval=1 --paused_queues_refresh_interval=1")
Queue("high").pause()
assert Queue("high/").is_paused()
# wait for the paused_queues list to be refreshed
time.sleep(2)
job_id1 = send_task(
"tests.tasks.general.MongoInsert", {"a": 41},
queue="high")
job_id2 = send_task(
"tests.tasks.general.MongoInsert", {"a": 43},
queue="high/subqueue")
# wait a bit to make sure the jobs status will still be queued
time.sleep(5)
job1 = Job(job_id1).fetch().data
job2 = Job(job_id2).fetch().data
assert job1["status"] == "queued"
assert job2["status"] == "queued"
assert worker.mongodb_jobs.tests_inserts.count() == 0
Queue("high/").resume()
Job(job_id1).wait(poll_interval=0.01)
Job(job_id2).wait(poll_interval=0.01)
job1 = Job(job_id1).fetch().data
job2 = Job(job_id2).fetch().data
assert job1["status"] == "success"
assert job1["result"] == {"a": 41}
assert job2["status"] == "success"
assert job2["result"] == {"a": 43}
assert worker.mongodb_jobs.tests_inserts.count() == 2
worker.stop()
def main():
parser = argparse.ArgumentParser(description='Runs a task')
cfg = config.get_config(parser=parser, config_type="run")
cfg["is_cli"] = True
set_current_config(cfg)
log.info(cfg)
if len(cfg["taskargs"]) == 1:
params = json.loads(cfg["taskargs"][0])
else:
params = {}
# mrq-run taskpath a 1 b 2 => {"a": "1", "b": "2"}
for group in utils.group_iter(cfg["taskargs"], n=2):
if len(group) != 2:
print "Number of arguments wasn't even"
sys.exit(1)
params[group[0]] = group[1]
if cfg["async"]:
ret = queue.send_task(cfg["taskpath"], params, sync=False, queue=cfg["queue"])
print ret
else:
worker_class = load_class_by_path(cfg["worker_class"])
job = worker_class.job_class(None)
job.data = {
"path": cfg["taskpath"],
"params": params,
"queue": cfg["queue"]
}
job.datestarted = datetime.datetime.utcnow()
set_current_job(job)
ret = job.perform()
print json.dumps(ret)
def test_pause_refresh_interval(worker):
""" Tests that a refresh interval of 0 disables the pause functionnality """
worker.start(flags="--paused_queues_refresh_interval=0")
Queue("high").pause()
assert Queue("high").is_paused()
# wait for the paused_queues list to be refreshed
time.sleep(2)
job_id1 = send_task(
"tests.tasks.general.MongoInsert", {"a": 41},
queue="high")
time.sleep(5)
job1 = Job(job_id1).fetch().data
assert job1["status"] == "success"
assert job1["result"] == {"a": 41}
worker.stop()
def test_pause_resume(worker):
worker.start(flags="--paused_queues_refresh_interval=0.1")
Queue("high").pause()
assert Queue("high").is_paused()
# wait for the paused_queues list to be refreshed
time.sleep(2)
job_id1 = send_task(
"tests.tasks.general.MongoInsert", {"a": 41},
queue="high")
job_id2 = send_task(
"tests.tasks.general.MongoInsert", {"a": 43},
queue="low")
time.sleep(5)
job1 = Job(job_id1).fetch().data
job2 = Job(job_id2).fetch().data
assert job1["status"] == "queued"
assert job2["status"] == "success"
assert job2["result"] == {"a": 43}
assert worker.mongodb_jobs.tests_inserts.count() == 1
Queue("high").resume()
Job(job_id1).wait(poll_interval=0.01)
job1 = Job(job_id1).fetch().data
assert job1["status"] == "success"
assert job1["result"] == {"a": 41}
assert worker.mongodb_jobs.tests_inserts.count() == 2
worker.stop()
def test_known_queues_lifecycle(worker):
worker.start(
queues="default_reverse xtest test_timed_set",
flags="--config tests/fixtures/config-raw1.py --subqueues_refresh_interval=0.1"
)
time.sleep(1)
worker.wait_for_idle()
# Test known queues
from mrq.queue import Queue, send_task, send_raw_tasks
# Just watching queues doesn't add them to known ones.
# BTW this doesn't read config from the worker, just db/redis.
assert set(Queue.all_known()) == set()
# Try queueing a task
send_task("tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 1}, queue="x")
jobs = list(worker.mongodb_jobs.mrq_jobs.find())
assert len(jobs) == 1
assert jobs[0]["queue"] == "x"
assert set(Queue.all_known()) == set(["x"])
Queue("x").empty()
jobs = list(worker.mongodb_jobs.mrq_jobs.find())
assert len(jobs) == 0
assert set(Queue.all_known()) == set()
all_known = worker.send_task("tests.tasks.general.QueueAllKnown", {}, queue="default")
# Will get all from config
assert len(all_known) > 0
# Now add a job on a raw queue
send_raw_tasks("test_raw/sub", ["a", "b", "c"])
time.sleep(1)
all_known_plus_sub = worker.send_task("tests.tasks.general.QueueAllKnown", {}, queue="default")
assert set(all_known_plus_sub) == set(all_known).union(set(["test_raw/sub"]))
def test_pause_resume(worker):
worker.start(flags="--paused_queues_refresh_interval=0.1")
Queue("high").pause()
assert Queue("high").is_paused()
# wait for the paused_queues list to be refreshed
time.sleep(2)
job_id1 = send_task("tests.tasks.general.MongoInsert", {"a": 41},
queue="high")
job_id2 = send_task("tests.tasks.general.MongoInsert", {"a": 43},
queue="low")
time.sleep(5)
job1 = Job(job_id1).fetch().data
job2 = Job(job_id2).fetch().data
assert job1["status"] == "queued"
assert job2["status"] == "success"
assert job2["result"] == {"a": 43}
assert worker.mongodb_jobs.tests_inserts.count() == 1
Queue("high").resume()
Job(job_id1).wait(poll_interval=0.01)
job1 = Job(job_id1).fetch().data
assert job1["status"] == "success"
assert job1["result"] == {"a": 41}
assert worker.mongodb_jobs.tests_inserts.count() == 2
worker.stop()
def test_general_simple_task_reverse(worker):
worker.start(queues="default_reverse xtest test_timed_set",
flags="--config tests/fixtures/config-raw1.py")
result = worker.send_tasks("tests.tasks.general.Add", [{
"a": 41,
"b": 1,
"sleep": 1
}, {
"a": 41,
"b": 1,
"sleep": 1
}, {
"a": 40,
"b": 1,
"sleep": 1
}])
assert result == [42, 42, 41]
assert [
x["result"]
for x in worker.mongodb_jobs.mrq_jobs.find().sort([["dateupdated", 1]])
] == [41, 42, 42]
# Test known queues
from mrq.queue import Queue, send_task
assert Queue.redis_known_queues() == set(
["default", "xtest", "test_timed_set"])
# Try queueing a task
send_task("tests.tasks.general.Add", {
"a": 41,
"b": 1,
"sleep": 1
},
queue="x")
time.sleep(1)
assert Queue.redis_known_queues() == set(
["x", "default", "xtest", "test_timed_set"])
def test_general_simple_task_reverse(worker):
worker.start(queues="default_reverse xtest test_timed_set", flags="--config tests/fixtures/config-raw1.py")
result = worker.send_tasks("tests.tasks.general.Add", [
{"a": 41, "b": 1, "sleep": 1},
{"a": 41, "b": 1, "sleep": 1},
{"a": 40, "b": 1, "sleep": 1}
])
assert result == [42, 42, 41]
assert [x["result"] for x in worker.mongodb_jobs.mrq_jobs.find().sort(
[["dateupdated", 1]])] == [41, 42, 42]
# Test known queues
from mrq.queue import Queue, send_task
assert Queue.redis_known_queues() == set(["default", "xtest", "test_timed_set"])
# Try queueing a task
send_task("tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 1}, queue="x")
time.sleep(1)
assert Queue.redis_known_queues() == set(["x", "default", "xtest", "test_timed_set"])
def test_pause_refresh_interval(worker):
""" Tests that a refresh interval of 0 disables the pause functionnality """
worker.start(flags="--paused_queues_refresh_interval=0")
Queue("high").pause()
assert Queue("high").is_paused()
# wait for the paused_queues list to be refreshed
time.sleep(2)
job_id1 = send_task("tests.tasks.general.MongoInsert", {"a": 41},
queue="high")
time.sleep(5)
job1 = Job(job_id1).fetch().data
assert job1["status"] == "success"
assert job1["result"] == {"a": 41}
def run(self, params): return send_task(params["path"], params["params"])
def run(self, params):
return send_task("mrq.basetasks.utils.JobAction", {
"status": "retry",
"dateretry": {"$lte": datetime.datetime.utcnow()},
"action": "requeue"
}, sync=True)
def run(self, params):
return send_task("mrq.basetasks.utils.JobAction", {
"status": "interrupt",
"action": "requeue"
}, sync=True)
def api_job_action():
return jsonify({
"job_id": send_task("mrq.basetasks.utils.JobAction", {k: v for k, v in request.form.iteritems()}, queue=get_current_config()["dashboard_queue"])
})