def test_progress2(queue, worker):
queue.enqueue(ProgressJob, progress_interval=1)
worker.start(1)
worker.wait_queue()
data = list(queue.config.sys.log.find())
assert sum([1 for d in data
if "progress" in d["message"] and d["level"] == "DEBUG"]) >= 5
def test_binary_out(queue, worker, mongodb):
job = queue.enqueue(BinaryOutputTestJob)
worker.start(3)
worker.wait_queue()
assert mongodb.core4test.sys.stdout.count_documents({}) == 1
doc = mongodb.core4test.sys.stdout.find_one()
assert doc["_id"] == job._id
assert doc["stdout"] == b"evil payload \xDE\xAD\xBE\xEF."
def test_requires_arg(queue, worker):
job = queue.enqueue(RequiresArgTest)
worker.start(1)
while True:
j = queue.find_job(job._id)
if j.state == "error":
break
worker.stop()
def test_restart_inactive(queue, worker):
job = queue.enqueue(RestartInactiveTest)
worker.start(1)
while True:
j = queue.find_job(job._id)
if j.state == "inactive":
break
queue.restart_job(job._id)
worker.wait_queue()
def test_fail2inactive(queue, worker, mongodb):
import tests.project.work
queue.enqueue(tests.project.work.ErrorJob, defer_max=5, attempts=10)
worker.start(1)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(1)
if queue.config.sys.queue.count_documents({"state": "inactive"}) == 1:
break
worker.stop()
def test_progress3(queue, worker):
queue.enqueue(ProgressJob)
worker.start(1)
worker.wait_queue()
data = list(queue.config.sys.log.find())
print([(d["level"], d["message"]) for d in data])
assert sum([
1 for d in data if "progress" in d["message"] and d["level"] == "DEBUG"
]) == 1
def test_success_after_failure(queue, worker):
import tests.project.work
queue.enqueue(tests.project.work.ErrorJob, success=True)
worker.start(1)
worker.wait_queue()
data = list(queue.config.sys.log.find())
assert sum([1 for d in data if "done execution" in d["message"]]) == 3
assert sum([1 for d in data if
"done execution with [failed]" in d["message"]]) == 2
assert sum([1 for d in data if
"done execution with [complete]" in d["message"]]) == 1
def test_zombie(queue, worker):
job = queue.enqueue(NoProgressJob, zombie_time=1)
worker.start(1)
worker.wait_queue()
assert queue.config.sys.journal.count_documents({}) == 1
assert queue.config.sys.queue.count_documents({}) == 0
job = queue.find_job(job._id)
assert job.zombie_at is not None
data = list(queue.config.sys.log.find())
assert sum([1 for d in data
if "successfully set zombie job" in d["message"]]) == 1
def test_mass_defer(queue, worker, mongodb):
import tests.project.work
for i in range(0, 10):
queue.enqueue(tests.project.work.DeferJob, i=i, success=True, defer_time=1,
defer_max=2)
worker.start(4)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(1)
if queue.config.sys.queue.count_documents({"state": "inactive"}) == 10:
break
worker.stop()
def test_progress1(queue, worker):
# fh = open("/tmp/test.txt", "w", encoding="utf-8")
# fh.write("%s\n" %(os.path.abspath(os.curdir)))
# fh.write("%s\n" %(sys.executable))
# fh.close()
queue.enqueue(ProgressJob)
worker.start(1)
worker.wait_queue()
data = list(queue.config.sys.log.find())
assert sum([1 for d in data
if "progress" in d["message"] and d["level"] == "DEBUG"]) == 2
def test_remove_deferred(queue, worker, mongodb):
import tests.project.work
job = queue.enqueue(tests.project.work.DeferJob, defer_time=10)
worker.start(1)
while queue.config.sys.queue.count_documents({"state": "deferred"}) == 0:
time.sleep(0.25)
assert queue.remove_job(job._id)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(0.25)
worker.stop()
assert queue.config.sys.journal.count_documents({}) == 1
assert queue.config.sys.queue.count_documents({}) == 0
def test_stdout(queue, worker, mongodb):
job = queue.enqueue(OutputTestJob)
worker.start(3)
worker.wait_queue()
assert mongodb.core4test.sys.stdout.count_documents({}) == 1
doc = mongodb.core4test.sys.stdout.find_one()
print(doc)
assert doc["_id"] == job._id
assert ("this output comes from tests.be.test_worker.OutputTestJob"
in doc["stdout"])
assert ("this comes from echo" in doc["stdout"])
assert ("this comes from C" in doc["stdout"])
def test_remove_failed(queue, worker, mongodb):
import tests.project.work
job = queue.enqueue(tests.project.work.ErrorJob, attempts=5, sleep=1)
worker.start(1)
while queue.config.sys.queue.count_documents({"state": "running"}) == 0:
time.sleep(0.25)
assert queue.remove_job(job._id)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(0.25)
worker.stop()
assert queue.config.sys.journal.count_documents({}) == 1
assert queue.config.sys.queue.count_documents({}) == 0
def test_remove_complete(queue, worker, mongodb):
job = queue.enqueue(core4.queue.helper.job.example.DummyJob, sleep=3)
worker.start(1)
while queue.config.sys.queue.count_documents({"state": "running"}) == 0:
time.sleep(0.25)
assert queue.remove_job(job._id)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(0.25)
worker.stop()
assert queue.config.sys.journal.count_documents({}) == 1
assert queue.config.sys.queue.count_documents({}) == 0
job = queue.find_job(job._id)
assert job.state == "complete"
def test_defer(queue, worker):
import tests.project.work
queue.enqueue(tests.project.work.DeferJob)
worker.start(1)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(1)
if queue.config.sys.queue.count_documents({"state": "inactive"}) > 0:
break
worker.stop()
data = list(queue.config.sys.log.find())
assert sum([1 for d in data if
"done execution with [deferred]" in d["message"]]) > 2
assert sum([1 for d in data if
"done execution with [inactive]" in d["message"]]) == 1
def test_failed_deferred(queue, worker):
job = queue.enqueue(RestartFailedTest)
worker.start(1)
while True:
j = queue.find_job(job._id)
if j.state == "failed":
break
queue.restart_job(job._id)
worker.wait_queue()
assert queue.config.sys.journal.count_documents({}) == 1
assert queue.config.sys.queue.count_documents({}) == 0
job = queue.find_job(job._id)
assert job.trial == 2
assert job.state == core4.queue.job.STATE_COMPLETE
def test_project_maintenance(queue, worker):
job = queue.enqueue(core4.queue.helper.job.example.DummyJob)
worker.start(1)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(1)
curr = worker.worker[0].cycle["total"]
queue.enter_maintenance("core4")
assert queue.config.sys.queue.count_documents({}) == 0
job = queue.enqueue(core4.queue.helper.job.example.DummyJob)
assert queue.config.sys.queue.count_documents({}) == 1
while worker.worker[0].cycle["total"] < curr + 10:
time.sleep(1)
assert queue.config.sys.queue.count_documents({}) == 1
queue.leave_maintenance("core4")
worker.wait_queue()
assert queue.config.sys.queue.count_documents({}) == 0
def test_restart_killed(queue, worker):
job = queue.enqueue(RestartKilledTest)
worker.start(1)
while True:
j = queue.find_job(job._id)
if j.state == "running":
break
queue.kill_job(job._id)
while True:
j = queue.find_job(job._id)
if j.state == "killed":
break
new_id = queue.restart_job(job._id)
queue.restart_job(new_id)
# queue.remove_job(job._id)
worker.wait_queue()
def test_remove_error(queue, worker):
import tests.project.work
job = queue.enqueue(tests.project.work.ErrorJob, attempts=1)
worker.start(1)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(1)
if queue.config.sys.queue.count_documents({"state": "error"}) > 0:
break
assert queue.remove_job(job._id)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(1)
worker.stop()
assert queue.config.sys.journal.count_documents({}) == 1
assert queue.config.sys.queue.count_documents({}) == 0
job = queue.find_job(job._id)
assert job.state == "error"
def test_remove_inactive(queue, worker):
import tests.project.work
job = queue.enqueue(tests.project.work.DeferJob, defer_max=1)
worker.start(1)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(1)
if queue.config.sys.queue.count_documents({"state": "inactive"}) > 0:
break
assert queue.remove_job(job._id)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(1)
worker.stop()
assert queue.config.sys.journal.count_documents({}) == 1
assert queue.config.sys.queue.count_documents({}) == 0
job = queue.find_job(job._id)
assert job.state == "inactive"
def test_error(queue, worker):
import tests.project.work
queue.enqueue(tests.project.work.ErrorJob)
worker.start(1)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(0.25)
if queue.config.sys.queue.count_documents({"state": "error"}) > 0:
break
worker.stop()
data = list(queue.config.sys.log.find())
from pprint import pprint
pprint(data)
assert sum([1 for d in data if "done execution" in d["message"]]) == 3
assert sum([1 for d in data if "start execution" in d["message"]]) == 3
delta = [d["created"] for d in data if
"execution" in d["message"] and d["level"] == "INFO"]
assert (delta[1] - delta[0]).total_seconds() >= 3
assert (delta[2] - delta[1]).total_seconds() >= 3
def test_nonstop(queue, worker):
job = queue.enqueue(core4.queue.helper.job.example.DummyJob, sleep=5,
wall_time=1)
worker.start(1)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(0.1)
if queue.config.sys.queue.count_documents(
{"wall_at": {"$ne": None}}) > 0:
break
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(0.1)
worker.stop()
assert queue.config.sys.journal.count_documents({}) == 1
assert queue.config.sys.queue.count_documents({}) == 0
job = queue.find_job(job._id)
assert job.wall_at is not None
data = list(queue.config.sys.log.find())
assert sum([1 for d in data
if "successfully set non-stop job" in d["message"]]) == 1
def test_kill(queue, worker):
job = queue.enqueue(ForeverJob, zombie_time=2)
worker.start(1)
while True:
job = queue.find_job(job._id)
if job.locked and job.locked["pid"]:
break
queue.kill_job(job._id)
while True:
job = queue.find_job(job._id)
if job.state == "killed":
break
queue.remove_job(job._id)
worker.wait_queue()
assert queue.config.sys.journal.count_documents({}) == 1
assert queue.config.sys.queue.count_documents({}) == 0
job = queue.find_job(job._id)
assert job.state == "killed"
assert job.killed_at is not None
def test_no_pid(queue, worker):
job = queue.enqueue(ForeverJob)
worker.start(1)
while True:
job = queue.find_job(job._id)
if job.locked and job.locked["pid"]:
job = queue.find_job(job._id)
proc = psutil.Process(job.locked["pid"])
time.sleep(5)
print("kill now", job.locked["pid"])
print("my pid", core4.util.node.get_pid())
proc.kill()
break
while True:
job = queue.find_job(job._id)
print(job.state)
if job.state == "killed":
break
worker.stop()
def test_restart_error(queue, worker):
job = queue.enqueue(RestartErrorTest)
worker.start(1)
while True:
j = queue.find_job(job._id)
if j.state == "error":
break
new_id = queue.restart_job(job._id)
worker.wait_queue()
assert queue.config.sys.journal.count_documents({}) == 2
assert queue.config.sys.queue.count_documents({}) == 0
parent = queue.find_job(job._id)
assert parent.state == core4.queue.job.STATE_ERROR
child = queue.find_job(new_id)
assert child.state == core4.queue.job.STATE_COMPLETE
assert parent.enqueued["child_id"] == child._id
assert child.enqueued["parent_id"] == parent._id
assert parent.enqueued["parent_id"] is None
assert "child_id" not in child.enqueued
assert queue.config.sys.lock.count_documents({}) == 0
def test_setup():
worker = core4.queue.worker.CoreWorker()
worker.exit = True
worker.start()
def test_ok(queue, worker):
queue.enqueue(core4.queue.helper.job.example.DummyJob, sleep=0)
worker.start(1)
worker.wait_queue()
