def test_progress2(queue, worker): queue.enqueue(ProgressJob, progress_interval=1) worker.start(1) worker.wait_queue() data = list(queue.config.sys.log.find()) assert sum([1 for d in data if "progress" in d["message"] and d["level"] == "DEBUG"]) >= 5
def test_binary_out(queue, worker, mongodb): job = queue.enqueue(BinaryOutputTestJob) worker.start(3) worker.wait_queue() assert mongodb.core4test.sys.stdout.count_documents({}) == 1 doc = mongodb.core4test.sys.stdout.find_one() assert doc["_id"] == job._id assert doc["stdout"] == b"evil payload \xDE\xAD\xBE\xEF."
def test_requires_arg(queue, worker): job = queue.enqueue(RequiresArgTest) worker.start(1) while True: j = queue.find_job(job._id) if j.state == "error": break worker.stop()
def test_restart_inactive(queue, worker): job = queue.enqueue(RestartInactiveTest) worker.start(1) while True: j = queue.find_job(job._id) if j.state == "inactive": break queue.restart_job(job._id) worker.wait_queue()
def test_fail2inactive(queue, worker, mongodb): import tests.project.work queue.enqueue(tests.project.work.ErrorJob, defer_max=5, attempts=10) worker.start(1) while queue.config.sys.queue.count_documents({}) > 0: time.sleep(1) if queue.config.sys.queue.count_documents({"state": "inactive"}) == 1: break worker.stop()
def test_progress3(queue, worker): queue.enqueue(ProgressJob) worker.start(1) worker.wait_queue() data = list(queue.config.sys.log.find()) print([(d["level"], d["message"]) for d in data]) assert sum([ 1 for d in data if "progress" in d["message"] and d["level"] == "DEBUG" ]) == 1
def test_success_after_failure(queue, worker): import tests.project.work queue.enqueue(tests.project.work.ErrorJob, success=True) worker.start(1) worker.wait_queue() data = list(queue.config.sys.log.find()) assert sum([1 for d in data if "done execution" in d["message"]]) == 3 assert sum([1 for d in data if "done execution with [failed]" in d["message"]]) == 2 assert sum([1 for d in data if "done execution with [complete]" in d["message"]]) == 1
def test_zombie(queue, worker): job = queue.enqueue(NoProgressJob, zombie_time=1) worker.start(1) worker.wait_queue() assert queue.config.sys.journal.count_documents({}) == 1 assert queue.config.sys.queue.count_documents({}) == 0 job = queue.find_job(job._id) assert job.zombie_at is not None data = list(queue.config.sys.log.find()) assert sum([1 for d in data if "successfully set zombie job" in d["message"]]) == 1
def test_mass_defer(queue, worker, mongodb): import tests.project.work for i in range(0, 10): queue.enqueue(tests.project.work.DeferJob, i=i, success=True, defer_time=1, defer_max=2) worker.start(4) while queue.config.sys.queue.count_documents({}) > 0: time.sleep(1) if queue.config.sys.queue.count_documents({"state": "inactive"}) == 10: break worker.stop()
def test_progress1(queue, worker): # fh = open("/tmp/test.txt", "w", encoding="utf-8") # fh.write("%s\n" %(os.path.abspath(os.curdir))) # fh.write("%s\n" %(sys.executable)) # fh.close() queue.enqueue(ProgressJob) worker.start(1) worker.wait_queue() data = list(queue.config.sys.log.find()) assert sum([1 for d in data if "progress" in d["message"] and d["level"] == "DEBUG"]) == 2
def test_remove_deferred(queue, worker, mongodb): import tests.project.work job = queue.enqueue(tests.project.work.DeferJob, defer_time=10) worker.start(1) while queue.config.sys.queue.count_documents({"state": "deferred"}) == 0: time.sleep(0.25) assert queue.remove_job(job._id) while queue.config.sys.queue.count_documents({}) > 0: time.sleep(0.25) worker.stop() assert queue.config.sys.journal.count_documents({}) == 1 assert queue.config.sys.queue.count_documents({}) == 0
def test_stdout(queue, worker, mongodb): job = queue.enqueue(OutputTestJob) worker.start(3) worker.wait_queue() assert mongodb.core4test.sys.stdout.count_documents({}) == 1 doc = mongodb.core4test.sys.stdout.find_one() print(doc) assert doc["_id"] == job._id assert ("this output comes from tests.be.test_worker.OutputTestJob" in doc["stdout"]) assert ("this comes from echo" in doc["stdout"]) assert ("this comes from C" in doc["stdout"])
def test_remove_failed(queue, worker, mongodb): import tests.project.work job = queue.enqueue(tests.project.work.ErrorJob, attempts=5, sleep=1) worker.start(1) while queue.config.sys.queue.count_documents({"state": "running"}) == 0: time.sleep(0.25) assert queue.remove_job(job._id) while queue.config.sys.queue.count_documents({}) > 0: time.sleep(0.25) worker.stop() assert queue.config.sys.journal.count_documents({}) == 1 assert queue.config.sys.queue.count_documents({}) == 0
def test_remove_complete(queue, worker, mongodb): job = queue.enqueue(core4.queue.helper.job.example.DummyJob, sleep=3) worker.start(1) while queue.config.sys.queue.count_documents({"state": "running"}) == 0: time.sleep(0.25) assert queue.remove_job(job._id) while queue.config.sys.queue.count_documents({}) > 0: time.sleep(0.25) worker.stop() assert queue.config.sys.journal.count_documents({}) == 1 assert queue.config.sys.queue.count_documents({}) == 0 job = queue.find_job(job._id) assert job.state == "complete"
def test_defer(queue, worker): import tests.project.work queue.enqueue(tests.project.work.DeferJob) worker.start(1) while queue.config.sys.queue.count_documents({}) > 0: time.sleep(1) if queue.config.sys.queue.count_documents({"state": "inactive"}) > 0: break worker.stop() data = list(queue.config.sys.log.find()) assert sum([1 for d in data if "done execution with [deferred]" in d["message"]]) > 2 assert sum([1 for d in data if "done execution with [inactive]" in d["message"]]) == 1
def test_failed_deferred(queue, worker): job = queue.enqueue(RestartFailedTest) worker.start(1) while True: j = queue.find_job(job._id) if j.state == "failed": break queue.restart_job(job._id) worker.wait_queue() assert queue.config.sys.journal.count_documents({}) == 1 assert queue.config.sys.queue.count_documents({}) == 0 job = queue.find_job(job._id) assert job.trial == 2 assert job.state == core4.queue.job.STATE_COMPLETE
def test_project_maintenance(queue, worker): job = queue.enqueue(core4.queue.helper.job.example.DummyJob) worker.start(1) while queue.config.sys.queue.count_documents({}) > 0: time.sleep(1) curr = worker.worker[0].cycle["total"] queue.enter_maintenance("core4") assert queue.config.sys.queue.count_documents({}) == 0 job = queue.enqueue(core4.queue.helper.job.example.DummyJob) assert queue.config.sys.queue.count_documents({}) == 1 while worker.worker[0].cycle["total"] < curr + 10: time.sleep(1) assert queue.config.sys.queue.count_documents({}) == 1 queue.leave_maintenance("core4") worker.wait_queue() assert queue.config.sys.queue.count_documents({}) == 0
def test_restart_killed(queue, worker): job = queue.enqueue(RestartKilledTest) worker.start(1) while True: j = queue.find_job(job._id) if j.state == "running": break queue.kill_job(job._id) while True: j = queue.find_job(job._id) if j.state == "killed": break new_id = queue.restart_job(job._id) queue.restart_job(new_id) # queue.remove_job(job._id) worker.wait_queue()
def test_remove_error(queue, worker): import tests.project.work job = queue.enqueue(tests.project.work.ErrorJob, attempts=1) worker.start(1) while queue.config.sys.queue.count_documents({}) > 0: time.sleep(1) if queue.config.sys.queue.count_documents({"state": "error"}) > 0: break assert queue.remove_job(job._id) while queue.config.sys.queue.count_documents({}) > 0: time.sleep(1) worker.stop() assert queue.config.sys.journal.count_documents({}) == 1 assert queue.config.sys.queue.count_documents({}) == 0 job = queue.find_job(job._id) assert job.state == "error"
def test_remove_inactive(queue, worker): import tests.project.work job = queue.enqueue(tests.project.work.DeferJob, defer_max=1) worker.start(1) while queue.config.sys.queue.count_documents({}) > 0: time.sleep(1) if queue.config.sys.queue.count_documents({"state": "inactive"}) > 0: break assert queue.remove_job(job._id) while queue.config.sys.queue.count_documents({}) > 0: time.sleep(1) worker.stop() assert queue.config.sys.journal.count_documents({}) == 1 assert queue.config.sys.queue.count_documents({}) == 0 job = queue.find_job(job._id) assert job.state == "inactive"
def test_error(queue, worker): import tests.project.work queue.enqueue(tests.project.work.ErrorJob) worker.start(1) while queue.config.sys.queue.count_documents({}) > 0: time.sleep(0.25) if queue.config.sys.queue.count_documents({"state": "error"}) > 0: break worker.stop() data = list(queue.config.sys.log.find()) from pprint import pprint pprint(data) assert sum([1 for d in data if "done execution" in d["message"]]) == 3 assert sum([1 for d in data if "start execution" in d["message"]]) == 3 delta = [d["created"] for d in data if "execution" in d["message"] and d["level"] == "INFO"] assert (delta[1] - delta[0]).total_seconds() >= 3 assert (delta[2] - delta[1]).total_seconds() >= 3
def test_nonstop(queue, worker): job = queue.enqueue(core4.queue.helper.job.example.DummyJob, sleep=5, wall_time=1) worker.start(1) while queue.config.sys.queue.count_documents({}) > 0: time.sleep(0.1) if queue.config.sys.queue.count_documents( {"wall_at": {"$ne": None}}) > 0: break while queue.config.sys.queue.count_documents({}) > 0: time.sleep(0.1) worker.stop() assert queue.config.sys.journal.count_documents({}) == 1 assert queue.config.sys.queue.count_documents({}) == 0 job = queue.find_job(job._id) assert job.wall_at is not None data = list(queue.config.sys.log.find()) assert sum([1 for d in data if "successfully set non-stop job" in d["message"]]) == 1
def test_kill(queue, worker): job = queue.enqueue(ForeverJob, zombie_time=2) worker.start(1) while True: job = queue.find_job(job._id) if job.locked and job.locked["pid"]: break queue.kill_job(job._id) while True: job = queue.find_job(job._id) if job.state == "killed": break queue.remove_job(job._id) worker.wait_queue() assert queue.config.sys.journal.count_documents({}) == 1 assert queue.config.sys.queue.count_documents({}) == 0 job = queue.find_job(job._id) assert job.state == "killed" assert job.killed_at is not None
def test_no_pid(queue, worker): job = queue.enqueue(ForeverJob) worker.start(1) while True: job = queue.find_job(job._id) if job.locked and job.locked["pid"]: job = queue.find_job(job._id) proc = psutil.Process(job.locked["pid"]) time.sleep(5) print("kill now", job.locked["pid"]) print("my pid", core4.util.node.get_pid()) proc.kill() break while True: job = queue.find_job(job._id) print(job.state) if job.state == "killed": break worker.stop()
def test_restart_error(queue, worker): job = queue.enqueue(RestartErrorTest) worker.start(1) while True: j = queue.find_job(job._id) if j.state == "error": break new_id = queue.restart_job(job._id) worker.wait_queue() assert queue.config.sys.journal.count_documents({}) == 2 assert queue.config.sys.queue.count_documents({}) == 0 parent = queue.find_job(job._id) assert parent.state == core4.queue.job.STATE_ERROR child = queue.find_job(new_id) assert child.state == core4.queue.job.STATE_COMPLETE assert parent.enqueued["child_id"] == child._id assert child.enqueued["parent_id"] == parent._id assert parent.enqueued["parent_id"] is None assert "child_id" not in child.enqueued assert queue.config.sys.lock.count_documents({}) == 0
def test_setup(): worker = core4.queue.worker.CoreWorker() worker.exit = True worker.start()
def test_ok(queue, worker): queue.enqueue(core4.queue.helper.job.example.DummyJob, sleep=0) worker.start(1) worker.wait_queue()