def test_known_queues_lifecycle(worker):
worker.start(queues="default_reverse xtest test_timed_set", flags="--config tests/fixtures/config-raw1.py")
time.sleep(1)
# Test known queues
from mrq.queue import Queue, send_task
assert set(Queue.redis_known_queues().keys()) == set(["default", "xtest", "test_timed_set"])
# Try queueing a task
send_task("tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 1}, queue="x")
time.sleep(1)
assert set(Queue.redis_known_queues().keys()) == set(["x", "default", "xtest", "test_timed_set"])
Queue("x").add_to_known_queues(timestamp=time.time() - (8 * 86400))
worker.send_task("mrq.basetasks.cleaning.CleanKnownQueues", {}, block=True)
# Not removed - not empty yet.
assert set(Queue.redis_known_queues().keys()) == set(["x", "default", "xtest", "test_timed_set"])
Queue("x").empty()
# Still not removed.
assert set(Queue.redis_known_queues().keys()) == set(["x", "default", "xtest", "test_timed_set"])
worker.send_task("mrq.basetasks.cleaning.CleanKnownQueues", {}, block=True)
# Now we're good
assert set(Queue.redis_known_queues().keys()) == set(["default", "xtest", "test_timed_set"])
def run(self, params):
max_age = int(params.get("max_age") or (7 * 86400))
pretend = bool(params.get("pretend"))
check_mongo = bool(params.get("check_mongo"))
known_queues = Queue.redis_known_queues()
removed_queues = []
queues_from_config = Queue.all_known_from_config()
print "Found %s known queues & %s from config" % (len(known_queues), len(queues_from_config))
# Only clean queues older than N days
time_threshold = time.time() - max_age
for queue, time_last_used in known_queues.iteritems():
if queue in queues_from_config:
continue
if time_last_used < time_threshold:
q = Queue(queue, add_to_known_queues=False)
size = q.size()
if check_mongo:
size += connections.mongodb_jobs.mrq_jobs.count({"queue": queue})
if size == 0:
removed_queues.append(queue)
print "Removing empty queue '%s' from known queues ..." % queue
if not pretend:
q.remove_from_known_queues()
print "Cleaned %s queues" % len(removed_queues)
return removed_queues
def run(self, params):
max_age = int(params.get("max_age") or (7 * 86400))
pretend = bool(params.get("pretend"))
check_mongo = bool(params.get("check_mongo"))
known_queues = Queue.redis_known_queues()
removed_queues = []
queues_from_config = Queue.all_known_from_config()
print "Found %s known queues & %s from config" % (
len(known_queues), len(queues_from_config))
# Only clean queues older than N days
time_threshold = time.time() - max_age
for queue, time_last_used in known_queues.iteritems():
if queue in queues_from_config:
continue
if time_last_used < time_threshold:
q = Queue(queue, add_to_known_queues=False)
size = q.size()
if check_mongo:
size += connections.mongodb_jobs.mrq_jobs.count(
{"queue": queue})
if size == 0:
removed_queues.append(queue)
print "Removing empty queue '%s' from known queues ..." % queue
if not pretend:
q.remove_from_known_queues()
print "Cleaned %s queues" % len(removed_queues)
return removed_queues
def test_general_simple_task_reverse(worker):
worker.start(queues="default_reverse xtest test_timed_set",
flags="--config tests/fixtures/config-raw1.py")
result = worker.send_tasks("tests.tasks.general.Add", [{
"a": 41,
"b": 1,
"sleep": 1
}, {
"a": 41,
"b": 1,
"sleep": 1
}, {
"a": 40,
"b": 1,
"sleep": 1
}])
assert result == [42, 42, 41]
assert [
x["result"]
for x in worker.mongodb_jobs.mrq_jobs.find().sort([["dateupdated", 1]])
] == [41, 42, 42]
# Test known queues
from mrq.queue import Queue, send_task
assert Queue.redis_known_queues() == set(
["default", "xtest", "test_timed_set"])
# Try queueing a task
send_task("tests.tasks.general.Add", {
"a": 41,
"b": 1,
"sleep": 1
},
queue="x")
time.sleep(1)
assert Queue.redis_known_queues() == set(
["x", "default", "xtest", "test_timed_set"])
def run(self, params):
max_age = int(params.get("max_age") or (7 * 86400))
known_queues = Queue.redis_known_queues()
# Only clean queues older than N days
time_threshold = time.time() - max_age
for queue, time_last_used in known_queues.iteritems():
if time_last_used < time_threshold:
q = Queue(queue, add_to_known_queues=False)
if q.size() == 0:
q.remove_from_known_queues()
def test_general_simple_task_reverse(worker):
worker.start(queues="default_reverse xtest test_timed_set", flags="--config tests/fixtures/config-raw1.py")
result = worker.send_tasks("tests.tasks.general.Add", [
{"a": 41, "b": 1, "sleep": 1},
{"a": 41, "b": 1, "sleep": 1},
{"a": 40, "b": 1, "sleep": 1}
])
assert result == [42, 42, 41]
assert [x["result"] for x in worker.mongodb_jobs.mrq_jobs.find().sort(
[["dateupdated", 1]])] == [41, 42, 42]
# Test known queues
from mrq.queue import Queue, send_task
assert Queue.redis_known_queues() == set(["default", "xtest", "test_timed_set"])
# Try queueing a task
send_task("tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 1}, queue="x")
time.sleep(1)
assert Queue.redis_known_queues() == set(["x", "default", "xtest", "test_timed_set"])
