def test_general_simple_task_one(worker):
result = worker.send_task(
"tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 1})
assert result == 42
time.sleep(0.1)
db_workers = list(worker.mongodb_jobs.mrq_workers.find())
assert len(db_workers) == 1
assert db_workers[0]["status"] in ["full", "wait"]
# Test the HTTP admin API
admin_worker = json.load(urllib2.urlopen("http://localhost:20020"))
assert admin_worker["_id"] == str(db_workers[0]["_id"])
assert admin_worker["status"] == "wait"
# Stop the worker gracefully
worker.stop(deps=False)
db_jobs = list(worker.mongodb_jobs.mrq_jobs.find())
assert len(db_jobs) == 1
assert db_jobs[0]["result"] == 42
assert db_jobs[0]["status"] == "success"
assert db_jobs[0]["queue"] == "default"
assert db_jobs[0]["worker"]
assert db_jobs[0]["datestarted"]
assert db_jobs[0]["dateupdated"]
assert db_jobs[0]["totaltime"] > 1
assert db_jobs[0]["_id"]
assert db_jobs[0]["params"] == {"a": 41, "b": 1, "sleep": 1}
assert db_jobs[0]["path"] == "tests.tasks.general.Add"
assert db_jobs[0]["time"] < 0.1
assert db_jobs[0]["switches"] >= 1
from mrq.job import get_job_result
assert get_job_result(db_jobs[0]["_id"]) == {"result": 42, "status": "success"}
db_workers = list(worker.mongodb_jobs.mrq_workers.find())
assert len(db_workers) == 1
assert db_workers[0]["_id"] == db_jobs[0]["worker"]
assert db_workers[0]["status"] == "stop"
assert db_workers[0]["jobs"] == []
assert db_workers[0]["done_jobs"] == 1
assert db_workers[0]["config"]
assert db_workers[0]["_id"]
# Job logs
db_logs = list(
worker.mongodb_logs.mrq_logs.find({"job": db_jobs[0]["_id"]}))
assert len(db_logs) == 1
assert "adding" in db_logs[0]["logs"]
# Worker logs
db_logs = list(
worker.mongodb_logs.mrq_logs.find({"worker": db_workers[0]["_id"]}))
assert len(db_logs) >= 1
def test_general_task_blacklist(worker):
worker.start(queues="default", flags="--task_blacklist tests.tasks.general.Add,tests.tasks.general.Square")
job1 = worker.send_task("tests.tasks.general.Add", {"a": 41, "b": 1}, block=False)
job2 = worker.send_task("tests.tasks.general.Square", {"n": 41}, block=False)
job3 = worker.send_task("tests.tasks.general.GetTime", {}, block=False)
time.sleep(3)
res1 = get_job_result(job1)
res2 = get_job_result(job2)
res3 = get_job_result(job3)
assert res1["status"] == "queued"
assert res2["status"] == "queued"
assert res3["status"] == "success"
def wait_for_job(path, params, **kwargs):
job_id = queue_job(path, params, **kwargs)
while True:
time.sleep(5)
res = get_job_result(job_id)
if res["status"] == "success":
return res.get("result")
elif res["status"] not in ["queued", "started", "interrupt"]:
raise Exception("Job %s was in status %s" %
(path, res.get("status")))
def wait_for_job(path, params, **kwargs):
job_id = queue_job(path, params, **kwargs)
while True:
time.sleep(5)
res = get_job_result(job_id)
if res["status"] == "success":
return res.get("result")
elif res["status"] not in ["queued", "started", "interrupt"]:
raise Exception("Job %s was in status %s" % (
path, res.get("status")
))
def test_worker_interrupt_after_max_time(worker):
worker.start(flags="--greenlets=2 --max_time=2", queues="test1 default")
task_ids = worker.send_tasks("tests.tasks.general.Add", [{"a": i, "b": 1, "sleep": 3} for i in range(5)],
block=False)
time.sleep(5)
results = [get_job_result(task_id) for task_id in task_ids]
queued_tasks = [result for result in results if result['status'] == "queued"]
successful_tasks = [(i, result) for i, result in enumerate(results) if result['status'] == "success"]
assert len(queued_tasks) == 3
assert len(successful_tasks) == 2
for i, result in successful_tasks:
assert result['result'] == i + 1
def test_general_simple_task_one(worker):
result = worker.send_task(
"tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 1})
assert result == 42
time.sleep(0.1)
db_workers = list(worker.mongodb_jobs.mrq_workers.find())
assert len(db_workers) == 1
worker_report = worker.get_report()
assert worker_report["status"] in ["full", "wait"]
assert worker_report["done_jobs"] == 1
# Test the HTTP admin API
admin_worker = json.load(urllib2.urlopen("http://localhost:20020"))
assert admin_worker["_id"] == str(db_workers[0]["_id"])
assert admin_worker["status"] == "wait"
# Stop the worker gracefully
worker.stop(deps=False)
db_jobs = list(worker.mongodb_jobs.mrq_jobs.find())
assert len(db_jobs) == 1
assert db_jobs[0]["result"] == 42
assert db_jobs[0]["status"] == "success"
assert db_jobs[0]["queue"] == "default"
assert db_jobs[0]["worker"]
assert db_jobs[0]["datestarted"]
assert db_jobs[0]["dateupdated"]
assert db_jobs[0]["totaltime"] > 1
assert db_jobs[0]["_id"]
assert db_jobs[0]["params"] == {"a": 41, "b": 1, "sleep": 1}
assert db_jobs[0]["path"] == "tests.tasks.general.Add"
assert db_jobs[0]["time"] < 0.5
assert db_jobs[0]["switches"] >= 1
from mrq.job import get_job_result
assert get_job_result(db_jobs[0]["_id"]) == {"result": 42, "status": "success"}
db_workers = list(worker.mongodb_jobs.mrq_workers.find())
assert len(db_workers) == 1
assert db_workers[0]["_id"] == db_jobs[0]["worker"]
assert db_workers[0]["status"] == "stop"
assert db_workers[0]["jobs"] == []
assert db_workers[0]["done_jobs"] == 1
assert db_workers[0]["config"]
assert db_workers[0]["_id"]
# Job logs
db_logs = list(
worker.mongodb_logs.mrq_logs.find({"job": db_jobs[0]["_id"]}))
assert len(db_logs) == 1
assert "adding" in db_logs[0]["logs"]
# Worker logs
db_logs = list(
worker.mongodb_logs.mrq_logs.find({"worker": db_workers[0]["_id"]}))
assert len(db_logs) >= 1
worker.stop_deps()
def test_agent_force_terminate(worker):
worker.start(agent=True, flags="--worker_group xxx --total_memory=500 --total_cpu=500 --orchestrate_interval=1 --report_interval=1")
#
# First, test interrupting a worker doing only a sleeping process.
#
connections.mongodb_jobs.mrq_workergroups.insert_one({
"_id": "xxx",
"commands": ["mrq-worker default --report_interval=60"],
"process_termination_timeout": 1
})
time.sleep(5)
job1 = worker.send_task("tests.tasks.general.Add", {"a": 41, "b": 2, "sleep": 100}, block=False)
time.sleep(3)
res1 = get_job_result(job1)
assert res1["status"] == "started"
connections.mongodb_jobs.mrq_workergroups.update_one(
{"_id": "xxx"},
{"$set": {"commands": ["mrq-worker otherqueue --report_interval=60"]
}})
time.sleep(5)
res1 = get_job_result(job1)
assert res1["status"] == "interrupt"
#
# Then, test interrupting an infinite-looping task
#
job2 = worker.send_task("tests.tasks.general.CPULoop", {}, block=False, queue="otherqueue")
time.sleep(3)
res2 = get_job_result(job2)
assert res2["status"] == "started"
pids_before_sigkill = psutil.pids()
connections.mongodb_jobs.mrq_workergroups.update_one(
{"_id": "xxx"},
{"$set": {"commands": ["mrq-worker otherqueue2 --report_interval=60"]
}})
# SIGKILL is sent after 5 seconds
time.sleep(10)
# Because we are SIGKILLing, the job won't even be marked as interrupt!
res2 = get_job_result(job2)
assert res2["status"] == "started"
# However, we make sure there is no leftover process and that the worker was replaced.
pids_after_sigkill = psutil.pids()
assert len(pids_after_sigkill) == len(pids_before_sigkill)
assert set(pids_after_sigkill) != set(pids_before_sigkill)