def test_known_queues_lifecycle(worker): worker.start(queues="default_reverse xtest test_timed_set", flags="--config tests/fixtures/config-raw1.py") time.sleep(1) # Test known queues from mrq.queue import Queue, send_task assert set(Queue.redis_known_queues().keys()) == set(["default", "xtest", "test_timed_set"]) # Try queueing a task send_task("tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 1}, queue="x") time.sleep(1) assert set(Queue.redis_known_queues().keys()) == set(["x", "default", "xtest", "test_timed_set"]) Queue("x").add_to_known_queues(timestamp=time.time() - (8 * 86400)) worker.send_task("mrq.basetasks.cleaning.CleanKnownQueues", {}, block=True) # Not removed - not empty yet. assert set(Queue.redis_known_queues().keys()) == set(["x", "default", "xtest", "test_timed_set"]) Queue("x").empty() # Still not removed. assert set(Queue.redis_known_queues().keys()) == set(["x", "default", "xtest", "test_timed_set"]) worker.send_task("mrq.basetasks.cleaning.CleanKnownQueues", {}, block=True) # Now we're good assert set(Queue.redis_known_queues().keys()) == set(["default", "xtest", "test_timed_set"])
def run(self, params): max_age = int(params.get("max_age") or (7 * 86400)) pretend = bool(params.get("pretend")) check_mongo = bool(params.get("check_mongo")) known_queues = Queue.redis_known_queues() removed_queues = [] queues_from_config = Queue.all_known_from_config() print "Found %s known queues & %s from config" % (len(known_queues), len(queues_from_config)) # Only clean queues older than N days time_threshold = time.time() - max_age for queue, time_last_used in known_queues.iteritems(): if queue in queues_from_config: continue if time_last_used < time_threshold: q = Queue(queue, add_to_known_queues=False) size = q.size() if check_mongo: size += connections.mongodb_jobs.mrq_jobs.count({"queue": queue}) if size == 0: removed_queues.append(queue) print "Removing empty queue '%s' from known queues ..." % queue if not pretend: q.remove_from_known_queues() print "Cleaned %s queues" % len(removed_queues) return removed_queues
def run(self, params): max_age = int(params.get("max_age") or (7 * 86400)) pretend = bool(params.get("pretend")) check_mongo = bool(params.get("check_mongo")) known_queues = Queue.redis_known_queues() removed_queues = [] queues_from_config = Queue.all_known_from_config() print "Found %s known queues & %s from config" % ( len(known_queues), len(queues_from_config)) # Only clean queues older than N days time_threshold = time.time() - max_age for queue, time_last_used in known_queues.iteritems(): if queue in queues_from_config: continue if time_last_used < time_threshold: q = Queue(queue, add_to_known_queues=False) size = q.size() if check_mongo: size += connections.mongodb_jobs.mrq_jobs.count( {"queue": queue}) if size == 0: removed_queues.append(queue) print "Removing empty queue '%s' from known queues ..." % queue if not pretend: q.remove_from_known_queues() print "Cleaned %s queues" % len(removed_queues) return removed_queues
def test_general_simple_task_reverse(worker): worker.start(queues="default_reverse xtest test_timed_set", flags="--config tests/fixtures/config-raw1.py") result = worker.send_tasks("tests.tasks.general.Add", [{ "a": 41, "b": 1, "sleep": 1 }, { "a": 41, "b": 1, "sleep": 1 }, { "a": 40, "b": 1, "sleep": 1 }]) assert result == [42, 42, 41] assert [ x["result"] for x in worker.mongodb_jobs.mrq_jobs.find().sort([["dateupdated", 1]]) ] == [41, 42, 42] # Test known queues from mrq.queue import Queue, send_task assert Queue.redis_known_queues() == set( ["default", "xtest", "test_timed_set"]) # Try queueing a task send_task("tests.tasks.general.Add", { "a": 41, "b": 1, "sleep": 1 }, queue="x") time.sleep(1) assert Queue.redis_known_queues() == set( ["x", "default", "xtest", "test_timed_set"])
def run(self, params): max_age = int(params.get("max_age") or (7 * 86400)) known_queues = Queue.redis_known_queues() # Only clean queues older than N days time_threshold = time.time() - max_age for queue, time_last_used in known_queues.iteritems(): if time_last_used < time_threshold: q = Queue(queue, add_to_known_queues=False) if q.size() == 0: q.remove_from_known_queues()
def test_general_simple_task_reverse(worker): worker.start(queues="default_reverse xtest test_timed_set", flags="--config tests/fixtures/config-raw1.py") result = worker.send_tasks("tests.tasks.general.Add", [ {"a": 41, "b": 1, "sleep": 1}, {"a": 41, "b": 1, "sleep": 1}, {"a": 40, "b": 1, "sleep": 1} ]) assert result == [42, 42, 41] assert [x["result"] for x in worker.mongodb_jobs.mrq_jobs.find().sort( [["dateupdated", 1]])] == [41, 42, 42] # Test known queues from mrq.queue import Queue, send_task assert Queue.redis_known_queues() == set(["default", "xtest", "test_timed_set"]) # Try queueing a task send_task("tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 1}, queue="x") time.sleep(1) assert Queue.redis_known_queues() == set(["x", "default", "xtest", "test_timed_set"])