def test_general_simple_task_one(worker): result = worker.send_task( "tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 1}) assert result == 42 time.sleep(0.1) db_workers = list(worker.mongodb_jobs.mrq_workers.find()) assert len(db_workers) == 1 assert db_workers[0]["status"] in ["full", "wait"] # Test the HTTP admin API admin_worker = json.load(urllib2.urlopen("http://localhost:20020")) assert admin_worker["_id"] == str(db_workers[0]["_id"]) assert admin_worker["status"] == "wait" # Stop the worker gracefully worker.stop(deps=False) db_jobs = list(worker.mongodb_jobs.mrq_jobs.find()) assert len(db_jobs) == 1 assert db_jobs[0]["result"] == 42 assert db_jobs[0]["status"] == "success" assert db_jobs[0]["queue"] == "default" assert db_jobs[0]["worker"] assert db_jobs[0]["datestarted"] assert db_jobs[0]["dateupdated"] assert db_jobs[0]["totaltime"] > 1 assert db_jobs[0]["_id"] assert db_jobs[0]["params"] == {"a": 41, "b": 1, "sleep": 1} assert db_jobs[0]["path"] == "tests.tasks.general.Add" assert db_jobs[0]["time"] < 0.1 assert db_jobs[0]["switches"] >= 1 from mrq.job import get_job_result assert get_job_result(db_jobs[0]["_id"]) == {"result": 42, "status": "success"} db_workers = list(worker.mongodb_jobs.mrq_workers.find()) assert len(db_workers) == 1 assert db_workers[0]["_id"] == db_jobs[0]["worker"] assert db_workers[0]["status"] == "stop" assert db_workers[0]["jobs"] == [] assert db_workers[0]["done_jobs"] == 1 assert db_workers[0]["config"] assert db_workers[0]["_id"] # Job logs db_logs = list( worker.mongodb_logs.mrq_logs.find({"job": db_jobs[0]["_id"]})) assert len(db_logs) == 1 assert "adding" in db_logs[0]["logs"] # Worker logs db_logs = list( worker.mongodb_logs.mrq_logs.find({"worker": db_workers[0]["_id"]})) assert len(db_logs) >= 1
def test_general_task_blacklist(worker): worker.start(queues="default", flags="--task_blacklist tests.tasks.general.Add,tests.tasks.general.Square") job1 = worker.send_task("tests.tasks.general.Add", {"a": 41, "b": 1}, block=False) job2 = worker.send_task("tests.tasks.general.Square", {"n": 41}, block=False) job3 = worker.send_task("tests.tasks.general.GetTime", {}, block=False) time.sleep(3) res1 = get_job_result(job1) res2 = get_job_result(job2) res3 = get_job_result(job3) assert res1["status"] == "queued" assert res2["status"] == "queued" assert res3["status"] == "success"
def wait_for_job(path, params, **kwargs): job_id = queue_job(path, params, **kwargs) while True: time.sleep(5) res = get_job_result(job_id) if res["status"] == "success": return res.get("result") elif res["status"] not in ["queued", "started", "interrupt"]: raise Exception("Job %s was in status %s" % (path, res.get("status")))
def wait_for_job(path, params, **kwargs): job_id = queue_job(path, params, **kwargs) while True: time.sleep(5) res = get_job_result(job_id) if res["status"] == "success": return res.get("result") elif res["status"] not in ["queued", "started", "interrupt"]: raise Exception("Job %s was in status %s" % ( path, res.get("status") ))
def test_worker_interrupt_after_max_time(worker): worker.start(flags="--greenlets=2 --max_time=2", queues="test1 default") task_ids = worker.send_tasks("tests.tasks.general.Add", [{"a": i, "b": 1, "sleep": 3} for i in range(5)], block=False) time.sleep(5) results = [get_job_result(task_id) for task_id in task_ids] queued_tasks = [result for result in results if result['status'] == "queued"] successful_tasks = [(i, result) for i, result in enumerate(results) if result['status'] == "success"] assert len(queued_tasks) == 3 assert len(successful_tasks) == 2 for i, result in successful_tasks: assert result['result'] == i + 1
def test_general_simple_task_one(worker): result = worker.send_task( "tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 1}) assert result == 42 time.sleep(0.1) db_workers = list(worker.mongodb_jobs.mrq_workers.find()) assert len(db_workers) == 1 worker_report = worker.get_report() assert worker_report["status"] in ["full", "wait"] assert worker_report["done_jobs"] == 1 # Test the HTTP admin API admin_worker = json.load(urllib2.urlopen("http://localhost:20020")) assert admin_worker["_id"] == str(db_workers[0]["_id"]) assert admin_worker["status"] == "wait" # Stop the worker gracefully worker.stop(deps=False) db_jobs = list(worker.mongodb_jobs.mrq_jobs.find()) assert len(db_jobs) == 1 assert db_jobs[0]["result"] == 42 assert db_jobs[0]["status"] == "success" assert db_jobs[0]["queue"] == "default" assert db_jobs[0]["worker"] assert db_jobs[0]["datestarted"] assert db_jobs[0]["dateupdated"] assert db_jobs[0]["totaltime"] > 1 assert db_jobs[0]["_id"] assert db_jobs[0]["params"] == {"a": 41, "b": 1, "sleep": 1} assert db_jobs[0]["path"] == "tests.tasks.general.Add" assert db_jobs[0]["time"] < 0.5 assert db_jobs[0]["switches"] >= 1 from mrq.job import get_job_result assert get_job_result(db_jobs[0]["_id"]) == {"result": 42, "status": "success"} db_workers = list(worker.mongodb_jobs.mrq_workers.find()) assert len(db_workers) == 1 assert db_workers[0]["_id"] == db_jobs[0]["worker"] assert db_workers[0]["status"] == "stop" assert db_workers[0]["jobs"] == [] assert db_workers[0]["done_jobs"] == 1 assert db_workers[0]["config"] assert db_workers[0]["_id"] # Job logs db_logs = list( worker.mongodb_logs.mrq_logs.find({"job": db_jobs[0]["_id"]})) assert len(db_logs) == 1 assert "adding" in db_logs[0]["logs"] # Worker logs db_logs = list( worker.mongodb_logs.mrq_logs.find({"worker": db_workers[0]["_id"]})) assert len(db_logs) >= 1 worker.stop_deps()
def test_agent_force_terminate(worker): worker.start(agent=True, flags="--worker_group xxx --total_memory=500 --total_cpu=500 --orchestrate_interval=1 --report_interval=1") # # First, test interrupting a worker doing only a sleeping process. # connections.mongodb_jobs.mrq_workergroups.insert_one({ "_id": "xxx", "commands": ["mrq-worker default --report_interval=60"], "process_termination_timeout": 1 }) time.sleep(5) job1 = worker.send_task("tests.tasks.general.Add", {"a": 41, "b": 2, "sleep": 100}, block=False) time.sleep(3) res1 = get_job_result(job1) assert res1["status"] == "started" connections.mongodb_jobs.mrq_workergroups.update_one( {"_id": "xxx"}, {"$set": {"commands": ["mrq-worker otherqueue --report_interval=60"] }}) time.sleep(5) res1 = get_job_result(job1) assert res1["status"] == "interrupt" # # Then, test interrupting an infinite-looping task # job2 = worker.send_task("tests.tasks.general.CPULoop", {}, block=False, queue="otherqueue") time.sleep(3) res2 = get_job_result(job2) assert res2["status"] == "started" pids_before_sigkill = psutil.pids() connections.mongodb_jobs.mrq_workergroups.update_one( {"_id": "xxx"}, {"$set": {"commands": ["mrq-worker otherqueue2 --report_interval=60"] }}) # SIGKILL is sent after 5 seconds time.sleep(10) # Because we are SIGKILLing, the job won't even be marked as interrupt! res2 = get_job_result(job2) assert res2["status"] == "started" # However, we make sure there is no leftover process and that the worker was replaced. pids_after_sigkill = psutil.pids() assert len(pids_after_sigkill) == len(pids_before_sigkill) assert set(pids_after_sigkill) != set(pids_before_sigkill)