def test_remove_failed(queue, worker, mongodb):
import tests.project.work
job = queue.enqueue(tests.project.work.ErrorJob, attempts=5, sleep=1)
worker.start(1)
while queue.config.sys.queue.count_documents({"state": "running"}) == 0:
time.sleep(0.25)
assert queue.remove_job(job._id)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(0.25)
worker.stop()
assert queue.config.sys.journal.count_documents({}) == 1
assert queue.config.sys.queue.count_documents({}) == 0
def test_remove_complete(queue, worker, mongodb):
job = queue.enqueue(core4.queue.helper.job.example.DummyJob, sleep=3)
worker.start(1)
while queue.config.sys.queue.count_documents({"state": "running"}) == 0:
time.sleep(0.25)
assert queue.remove_job(job._id)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(0.25)
worker.stop()
assert queue.config.sys.journal.count_documents({}) == 1
assert queue.config.sys.queue.count_documents({}) == 0
job = queue.find_job(job._id)
assert job.state == "complete"
def test_mass_defer(queue, worker, mongodb):
import tests.project.work
for i in range(0, 10):
queue.enqueue(tests.project.work.DeferJob,
i=i,
success=True,
defer_time=1,
defer_max=2)
worker.start(4)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(1)
if queue.config.sys.queue.count_documents({"state": "inactive"}) == 10:
break
worker.stop()
def test_defer(queue, worker):
import tests.project.work
queue.enqueue(tests.project.work.DeferJob)
worker.start(1)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(1)
if queue.config.sys.queue.count_documents({"state": "inactive"}) > 0:
break
worker.stop()
data = list(queue.config.sys.log.find())
assert sum([1 for d in data if
"done execution with [deferred]" in d["message"]]) > 2
assert sum([1 for d in data if
"done execution with [inactive]" in d["message"]]) == 1
def test_remove_error(queue, worker):
import tests.project.work
job = queue.enqueue(tests.project.work.ErrorJob, attempts=1)
worker.start(1)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(1)
if queue.config.sys.queue.count_documents({"state": "error"}) > 0:
break
assert queue.remove_job(job._id)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(1)
worker.stop()
assert queue.config.sys.journal.count_documents({}) == 1
assert queue.config.sys.queue.count_documents({}) == 0
job = queue.find_job(job._id)
assert job.state == "error"
def test_remove_inactive(queue, worker):
import tests.project.work
job = queue.enqueue(tests.project.work.DeferJob, defer_max=1)
worker.start(1)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(1)
if queue.config.sys.queue.count_documents({"state": "inactive"}) > 0:
break
assert queue.remove_job(job._id)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(1)
worker.stop()
assert queue.config.sys.journal.count_documents({}) == 1
assert queue.config.sys.queue.count_documents({}) == 0
job = queue.find_job(job._id)
assert job.state == "inactive"
def test_error(queue, worker):
import tests.project.work
queue.enqueue(tests.project.work.ErrorJob)
worker.start(1)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(0.25)
if queue.config.sys.queue.count_documents({"state": "error"}) > 0:
break
worker.stop()
data = list(queue.config.sys.log.find())
from pprint import pprint
pprint(data)
assert sum([1 for d in data if "done execution" in d["message"]]) == 3
assert sum([1 for d in data if "start execution" in d["message"]]) == 3
delta = [d["created"] for d in data if
"execution" in d["message"] and d["level"] == "INFO"]
assert (delta[1] - delta[0]).total_seconds() >= 3
assert (delta[2] - delta[1]).total_seconds() >= 3
def test_no_pid(queue, worker):
job = queue.enqueue(ForeverJob)
worker.start(1)
while True:
job = queue.find_job(job._id)
if job.locked and job.locked["pid"]:
job = queue.find_job(job._id)
proc = psutil.Process(job.locked["pid"])
time.sleep(5)
print("kill now", job.locked["pid"])
print("my pid", core4.util.node.get_pid())
proc.kill()
break
while True:
job = queue.find_job(job._id)
print(job.state)
if job.state == "killed":
break
worker.stop()
def test_nonstop(queue, worker):
job = queue.enqueue(core4.queue.helper.job.example.DummyJob, sleep=5,
wall_time=1)
worker.start(1)
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(0.1)
if queue.config.sys.queue.count_documents(
{"wall_at": {"$ne": None}}) > 0:
break
while queue.config.sys.queue.count_documents({}) > 0:
time.sleep(0.1)
worker.stop()
assert queue.config.sys.journal.count_documents({}) == 1
assert queue.config.sys.queue.count_documents({}) == 0
job = queue.find_job(job._id)
assert job.wall_at is not None
data = list(queue.config.sys.log.find())
assert sum([1 for d in data
if "successfully set non-stop job" in d["message"]]) == 1
