def waitWorkDone(cls, txnCreator, reactor, timeout, workTypes):
"""
Wait for the specified job to complete. Only use this in tests
that need to wait for results from jobs.
"""
t = time.time()
while True:
count = [0]
@inlineCallbacks
def _countTypes(txn):
for t in workTypes:
work = yield t.all(txn)
count[0] += len(work)
yield inTransaction(txnCreator, _countTypes)
if count[0] == 0:
break
if time.time() - t > timeout:
returnValue(False)
d = Deferred()
reactor.callLater(0.1, lambda: d.callback(None))
yield d
returnValue(True)
def waitEmpty(cls, txnCreator, reactor, timeout):
"""
Wait for the job queue to drain. Only use this in tests
that need to wait for results from jobs.
"""
t = time.time()
while True:
work = yield inTransaction(txnCreator, cls.all)
if not work:
break
if time.time() - t > timeout:
returnValue(False)
d = Deferred()
reactor.callLater(0.1, lambda: d.callback(None))
yield d
returnValue(True)
def waitJobDone(cls, txnCreator, reactor, timeout, jobID):
"""
Wait for the specified job to complete. Only use this in tests
that need to wait for results from jobs.
"""
t = time.time()
while True:
work = yield inTransaction(txnCreator, cls.query, expr=(cls.jobID == jobID))
if not work:
break
if time.time() - t > timeout:
returnValue(False)
d = Deferred()
reactor.callLater(0.1, lambda: d.callback(None))
yield d
returnValue(True)
def _failureCleanUp(delay=None):
@inlineCallbacks
def _cleanUp2(txn2):
try:
job = yield cls.load(txn2, jobDescriptor.jobID)
except NoSuchRecord:
log.debug(
"JobItem: {workType} {jobid} disappeared t={tm}",
workType=jobDescriptor.workType,
jobid=jobDescriptor.jobID,
tm=_tm(),
)
else:
log.debug(
"JobItem: {workType} {jobid} marking as failed {count} t={tm}",
workType=jobDescriptor.workType,
jobid=jobDescriptor.jobID,
count=job.failed + 1,
tm=_tm(),
)
yield job.failedToRun(locked=isinstance(e, JobRunningError), delay=delay)
return inTransaction(txnFactory, _cleanUp2, "ultimatelyPerform._failureCleanUp")
