def test_monitor_step(engine):
mon = Monitor(engine)
api.schedule(engine,
'print_sleep_and_go_away',
21,
metadata={'user': '******'})
stats = mon.step()
assert len(stats.new) == 2
assert all(isinstance(w, int) for w in stats.new)
# simulate a hard crash
for _, proc in mon.workers.items():
if proc.poll() is None:
kill_process_tree(proc.pid)
dead = mon.reap_dead_workers()
assert dead == stats.new
stats2 = mon.step()
assert stats2.new != stats.new
assert len(stats2.new) == 2
assert all(isinstance(w, int) for w in stats2.new)
mon.killall()
def test_cleanup_unstarted(engine):
with engine.begin() as cn:
cn.execute('delete from rework.task')
cn.execute('delete from rework.worker')
mon = Monitor(engine, 'default', None, 1, 1, 0, False)
mon.register()
mon.ensure_workers()
with engine.begin() as cn:
insert('rework.worker').values(host='127.0.0.1', domain='default').do(
cn) # unborn worker
nworkers = engine.execute('select count(*) from rework.worker').scalar()
assert nworkers == 2
t = api.schedule(engine, 'raw_input', b'foo')
t.join()
mon.killall(msg=None)
deleted = mon.cleanup_unstarted()
assert deleted == 1
assert engine.execute('select count(*) from rework.worker').scalar() == 1
assert engine.execute('select count(*) from rework.task').scalar() == 1
deleted = mon.cleanup_unstarted()
assert deleted == 0
def workers(engine, domain='default',
minworkers=0, maxworkers=1,
maxruns=0, maxmem=0,
debug=False):
"""context manager to set up a test monitor for a given domain
It can be configured with `domain`, `maxworkers`, `minworkers`, `maxruns`,
`maxmem` and `debug`.
Here's an example usage:
.. code-block:: python
with workers(engine, numworkers=2) as monitor:
t1 = api.schedule(engine, 'compute_sum', [1, 2, 3, 4, 5])
t2 = api.schedule(engine, 'compute_sum', 'abcd')
t1.join()
assert t1.output == 15
t2.join()
assert t2.traceback.endswith(
"TypeError: unsupported operand type(s) "
"for +: 'int' and 'str'"
)
assert len(monitor.wids) == 2
At the end of the block, the monitor-controlled workers are
cleaned up.
However the database entries are still there.
"""
mon = Monitor(
engine, domain, minworkers, maxworkers, maxruns, maxmem, debug
)
mon.register()
mon.ensure_workers()
try:
yield mon
except:
mon.killall(
'Something killed the monitor',
traceback.format_exc()
)
finally:
mon.killall()
mon.unregister()