コード例 #1
0
def test_known_queues_lifecycle(worker):

    worker.start(queues="default_reverse xtest test_timed_set", flags="--config tests/fixtures/config-raw1.py")
    time.sleep(1)
    # Test known queues
    from mrq.queue import Queue, send_task
    assert set(Queue.redis_known_queues().keys()) == set(["default", "xtest", "test_timed_set"])

    # Try queueing a task
    send_task("tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 1}, queue="x")
    time.sleep(1)
    assert set(Queue.redis_known_queues().keys()) == set(["x", "default", "xtest", "test_timed_set"])

    Queue("x").add_to_known_queues(timestamp=time.time() - (8 * 86400))

    worker.send_task("mrq.basetasks.cleaning.CleanKnownQueues", {}, block=True)

    # Not removed - not empty yet.
    assert set(Queue.redis_known_queues().keys()) == set(["x", "default", "xtest", "test_timed_set"])

    Queue("x").empty()

    # Still not removed.
    assert set(Queue.redis_known_queues().keys()) == set(["x", "default", "xtest", "test_timed_set"])

    worker.send_task("mrq.basetasks.cleaning.CleanKnownQueues", {}, block=True)

    # Now we're good
    assert set(Queue.redis_known_queues().keys()) == set(["default", "xtest", "test_timed_set"])
コード例 #2
0
def test_pause_subqueue(worker):

    # set config in current context in order to have a subqueue delimiter
    set_current_config(get_config(config_type="worker"))

    worker.start(
        queues="high high/",
        flags=
        "--subqueues_refresh_interval=1 --paused_queues_refresh_interval=1")

    Queue("high").pause()

    assert Queue("high/").is_paused()

    # wait for the paused_queues list to be refreshed
    time.sleep(2)

    job_id1 = send_task("tests.tasks.general.MongoInsert", {"a": 41},
                        queue="high")

    job_id2 = send_task("tests.tasks.general.MongoInsert", {"a": 43},
                        queue="high/subqueue")

    # wait a bit to make sure the jobs status will still be queued
    time.sleep(5)

    job1 = Job(job_id1).fetch().data
    job2 = Job(job_id2).fetch().data

    assert job1["status"] == "queued"
    assert job2["status"] == "queued"

    assert worker.mongodb_jobs.tests_inserts.count() == 0

    Queue("high/").resume()

    Job(job_id1).wait(poll_interval=0.01)

    Job(job_id2).wait(poll_interval=0.01)

    job1 = Job(job_id1).fetch().data
    job2 = Job(job_id2).fetch().data

    assert job1["status"] == "success"
    assert job1["result"] == {"a": 41}

    assert job2["status"] == "success"
    assert job2["result"] == {"a": 43}

    assert worker.mongodb_jobs.tests_inserts.count() == 2

    worker.stop()
コード例 #3
0
ファイル: test_pause.py プロジェクト: ggueret/mrq
def test_pause_subqueue(worker):

    # set config in current context in order to have a subqueue delimiter
    set_current_config(get_config(config_type="worker"))

    worker.start(queues="high high/", flags="--subqueues_refresh_interval=1 --paused_queues_refresh_interval=1")

    Queue("high").pause()

    assert Queue("high/").is_paused()

    # wait for the paused_queues list to be refreshed
    time.sleep(2)

    job_id1 = send_task(
        "tests.tasks.general.MongoInsert", {"a": 41},
        queue="high")

    job_id2 = send_task(
        "tests.tasks.general.MongoInsert", {"a": 43},
        queue="high/subqueue")

    # wait a bit to make sure the jobs status will still be queued
    time.sleep(5)

    job1 = Job(job_id1).fetch().data
    job2 = Job(job_id2).fetch().data

    assert job1["status"] == "queued"
    assert job2["status"] == "queued"

    assert worker.mongodb_jobs.tests_inserts.count() == 0

    Queue("high/").resume()

    Job(job_id1).wait(poll_interval=0.01)

    Job(job_id2).wait(poll_interval=0.01)

    job1 = Job(job_id1).fetch().data
    job2 = Job(job_id2).fetch().data

    assert job1["status"] == "success"
    assert job1["result"] == {"a": 41}

    assert job2["status"] == "success"
    assert job2["result"] == {"a": 43}

    assert worker.mongodb_jobs.tests_inserts.count() == 2

    worker.stop()
コード例 #4
0
ファイル: mrq_run.py プロジェクト: matrixorz/mrq
def main():

  parser = argparse.ArgumentParser(description='Runs a task')

  cfg = config.get_config(parser=parser, config_type="run")
  cfg["is_cli"] = True
  set_current_config(cfg)
  log.info(cfg)
  if len(cfg["taskargs"]) == 1:
    params = json.loads(cfg["taskargs"][0])
  else:
    params = {}

    # mrq-run taskpath a 1 b 2 => {"a": "1", "b": "2"}
    for group in utils.group_iter(cfg["taskargs"], n=2):
      if len(group) != 2:
        print "Number of arguments wasn't even"
        sys.exit(1)
      params[group[0]] = group[1]

  if cfg["async"]:
    ret = queue.send_task(cfg["taskpath"], params, sync=False, queue=cfg["queue"])
    print ret
  else:
    worker_class = load_class_by_path(cfg["worker_class"])
    job = worker_class.job_class(None)
    job.data = {
      "path": cfg["taskpath"],
      "params": params,
      "queue": cfg["queue"]
    }
    job.datestarted = datetime.datetime.utcnow()
    set_current_job(job)
    ret = job.perform()
    print json.dumps(ret)
コード例 #5
0
ファイル: test_pause.py プロジェクト: ggueret/mrq
def test_pause_refresh_interval(worker):

    """ Tests that a refresh interval of 0 disables the pause functionnality """

    worker.start(flags="--paused_queues_refresh_interval=0")

    Queue("high").pause()

    assert Queue("high").is_paused()

    # wait for the paused_queues list to be refreshed
    time.sleep(2)

    job_id1 = send_task(
        "tests.tasks.general.MongoInsert", {"a": 41},
        queue="high")

    time.sleep(5)

    job1 = Job(job_id1).fetch().data

    assert job1["status"] == "success"
    assert job1["result"] == {"a": 41}

    worker.stop()
コード例 #6
0
ファイル: test_pause.py プロジェクト: ggueret/mrq
def test_pause_resume(worker):

    worker.start(flags="--paused_queues_refresh_interval=0.1")

    Queue("high").pause()

    assert Queue("high").is_paused()

    # wait for the paused_queues list to be refreshed
    time.sleep(2)

    job_id1 = send_task(
        "tests.tasks.general.MongoInsert", {"a": 41},
        queue="high")

    job_id2 = send_task(
        "tests.tasks.general.MongoInsert", {"a": 43},
        queue="low")

    time.sleep(5)

    job1 = Job(job_id1).fetch().data
    job2 = Job(job_id2).fetch().data

    assert job1["status"] == "queued"

    assert job2["status"] == "success"
    assert job2["result"] == {"a": 43}

    assert worker.mongodb_jobs.tests_inserts.count() == 1

    Queue("high").resume()

    Job(job_id1).wait(poll_interval=0.01)

    job1 = Job(job_id1).fetch().data

    assert job1["status"] == "success"
    assert job1["result"] == {"a": 41}

    assert worker.mongodb_jobs.tests_inserts.count() == 2

    worker.stop()
コード例 #7
0
ファイル: test_general.py プロジェクト: yjshen1982/mrq
def test_known_queues_lifecycle(worker):

    worker.start(
        queues="default_reverse xtest test_timed_set",
        flags="--config tests/fixtures/config-raw1.py --subqueues_refresh_interval=0.1"
    )
    time.sleep(1)
    worker.wait_for_idle()

    # Test known queues
    from mrq.queue import Queue, send_task, send_raw_tasks

    # Just watching queues doesn't add them to known ones.
    # BTW this doesn't read config from the worker, just db/redis.
    assert set(Queue.all_known()) == set()

    # Try queueing a task
    send_task("tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 1}, queue="x")

    jobs = list(worker.mongodb_jobs.mrq_jobs.find())
    assert len(jobs) == 1
    assert jobs[0]["queue"] == "x"

    assert set(Queue.all_known()) == set(["x"])

    Queue("x").empty()

    jobs = list(worker.mongodb_jobs.mrq_jobs.find())
    assert len(jobs) == 0
    assert set(Queue.all_known()) == set()

    all_known = worker.send_task("tests.tasks.general.QueueAllKnown", {}, queue="default")
    # Will get all from config
    assert len(all_known) > 0

    # Now add a job on a raw queue
    send_raw_tasks("test_raw/sub", ["a", "b", "c"])
    time.sleep(1)

    all_known_plus_sub = worker.send_task("tests.tasks.general.QueueAllKnown", {}, queue="default")
    assert set(all_known_plus_sub) == set(all_known).union(set(["test_raw/sub"]))
コード例 #8
0
def test_pause_resume(worker):

    worker.start(flags="--paused_queues_refresh_interval=0.1")

    Queue("high").pause()

    assert Queue("high").is_paused()

    # wait for the paused_queues list to be refreshed
    time.sleep(2)

    job_id1 = send_task("tests.tasks.general.MongoInsert", {"a": 41},
                        queue="high")

    job_id2 = send_task("tests.tasks.general.MongoInsert", {"a": 43},
                        queue="low")

    time.sleep(5)

    job1 = Job(job_id1).fetch().data
    job2 = Job(job_id2).fetch().data

    assert job1["status"] == "queued"

    assert job2["status"] == "success"
    assert job2["result"] == {"a": 43}

    assert worker.mongodb_jobs.tests_inserts.count() == 1

    Queue("high").resume()

    Job(job_id1).wait(poll_interval=0.01)

    job1 = Job(job_id1).fetch().data

    assert job1["status"] == "success"
    assert job1["result"] == {"a": 41}

    assert worker.mongodb_jobs.tests_inserts.count() == 2

    worker.stop()
コード例 #9
0
def test_general_simple_task_reverse(worker):

    worker.start(queues="default_reverse xtest test_timed_set",
                 flags="--config tests/fixtures/config-raw1.py")

    result = worker.send_tasks("tests.tasks.general.Add", [{
        "a": 41,
        "b": 1,
        "sleep": 1
    }, {
        "a": 41,
        "b": 1,
        "sleep": 1
    }, {
        "a": 40,
        "b": 1,
        "sleep": 1
    }])

    assert result == [42, 42, 41]

    assert [
        x["result"]
        for x in worker.mongodb_jobs.mrq_jobs.find().sort([["dateupdated", 1]])
    ] == [41, 42, 42]

    # Test known queues
    from mrq.queue import Queue, send_task
    assert Queue.redis_known_queues() == set(
        ["default", "xtest", "test_timed_set"])

    # Try queueing a task
    send_task("tests.tasks.general.Add", {
        "a": 41,
        "b": 1,
        "sleep": 1
    },
              queue="x")
    time.sleep(1)
    assert Queue.redis_known_queues() == set(
        ["x", "default", "xtest", "test_timed_set"])
コード例 #10
0
ファイル: test_general.py プロジェクト: CharleyFarley/mrq
def test_general_simple_task_reverse(worker):

    worker.start(queues="default_reverse xtest test_timed_set", flags="--config tests/fixtures/config-raw1.py")

    result = worker.send_tasks("tests.tasks.general.Add", [
        {"a": 41, "b": 1, "sleep": 1},
        {"a": 41, "b": 1, "sleep": 1},
        {"a": 40, "b": 1, "sleep": 1}
    ])

    assert result == [42, 42, 41]

    assert [x["result"] for x in worker.mongodb_jobs.mrq_jobs.find().sort(
        [["dateupdated", 1]])] == [41, 42, 42]

    # Test known queues
    from mrq.queue import Queue, send_task
    assert Queue.redis_known_queues() == set(["default", "xtest", "test_timed_set"])

    # Try queueing a task
    send_task("tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 1}, queue="x")
    time.sleep(1)
    assert Queue.redis_known_queues() == set(["x", "default", "xtest", "test_timed_set"])
コード例 #11
0
ファイル: test_pause.py プロジェクト: yjshen1982/mrq
def test_pause_refresh_interval(worker):
    """ Tests that a refresh interval of 0 disables the pause functionnality """

    worker.start(flags="--paused_queues_refresh_interval=0")

    Queue("high").pause()

    assert Queue("high").is_paused()

    # wait for the paused_queues list to be refreshed
    time.sleep(2)

    job_id1 = send_task("tests.tasks.general.MongoInsert", {"a": 41},
                        queue="high")

    time.sleep(5)

    job1 = Job(job_id1).fetch().data

    assert job1["status"] == "success"
    assert job1["result"] == {"a": 41}
コード例 #12
0
ファイル: general.py プロジェクト: bossjones/mrq
 def run(self, params):
   return send_task(params["path"], params["params"])
コード例 #13
0
ファイル: cleaning.py プロジェクト: nfredrik/mrq
 def run(self, params):
   return send_task("mrq.basetasks.utils.JobAction", {
     "status": "retry",
     "dateretry": {"$lte": datetime.datetime.utcnow()},
     "action": "requeue"
   }, sync=True)
コード例 #14
0
ファイル: cleaning.py プロジェクト: nfredrik/mrq
 def run(self, params):
   return send_task("mrq.basetasks.utils.JobAction", {
     "status": "interrupt",
     "action": "requeue"
   }, sync=True)
コード例 #15
0
ファイル: app.py プロジェクト: bossjones/mrq
def api_job_action():
  return jsonify({
    "job_id": send_task("mrq.basetasks.utils.JobAction", {k: v for k, v in request.form.iteritems()}, queue=get_current_config()["dashboard_queue"])
  })