def test_cleanup_unstarted(engine):
with engine.begin() as cn:
cn.execute('delete from rework.task')
cn.execute('delete from rework.worker')
mon = Monitor(engine, 'default', None, 1, 1, 0, False)
mon.register()
mon.ensure_workers()
with engine.begin() as cn:
insert('rework.worker').values(host='127.0.0.1', domain='default').do(
cn) # unborn worker
nworkers = engine.execute('select count(*) from rework.worker').scalar()
assert nworkers == 2
t = api.schedule(engine, 'raw_input', b'foo')
t.join()
mon.killall(msg=None)
deleted = mon.cleanup_unstarted()
assert deleted == 1
assert engine.execute('select count(*) from rework.worker').scalar() == 1
assert engine.execute('select count(*) from rework.task').scalar() == 1
deleted = mon.cleanup_unstarted()
assert deleted == 0
def test_run_worker(engine):
mon = Monitor(engine, maxruns=1)
wid = mon.new_worker()
t = api.schedule(engine, 'print_sleep_and_go_away', 0)
worker = Worker(engine.url, wid, os.getppid(), maxruns=1)
worker.run()
assert t.state == 'done'
def test_basic_worker_operations(engine):
mon = Monitor(engine)
wid = mon.new_worker()
worker = Worker(engine.url, wid)
with worker.running_status():
assert engine.execute(
'select count(id) from rework.worker where running = true').scalar(
) == 1
assert engine.execute(
'select count(id) from rework.worker where running = true').scalar(
) == 0
def test_monitor_step(engine):
mon = Monitor(engine)
api.schedule(engine,
'print_sleep_and_go_away',
21,
metadata={'user': '******'})
stats = mon.step()
assert len(stats.new) == 2
assert all(isinstance(w, int) for w in stats.new)
# simulate a hard crash
for _, proc in mon.workers.items():
if proc.poll() is None:
kill_process_tree(proc.pid)
dead = mon.reap_dead_workers()
assert dead == stats.new
stats2 = mon.step()
assert stats2.new != stats.new
assert len(stats2.new) == 2
assert all(isinstance(w, int) for w in stats2.new)
mon.killall()
def test_basic_task_operations(engine):
api.schedule(engine,
'print_sleep_and_go_away',
21,
metadata={'user': '******'})
known = [(name, Path(path).name) for name, path in engine.execute(
'select name, path from rework.operation order by name').fetchall()]
assert [
('allocate_and_leak_mbytes', 'tasks.py'),
('capture_logs', 'tasks.py'),
('flush_captured_stdout', 'tasks.py'),
('infinite_loop', 'tasks.py'),
('infinite_loop_long_timeout', 'tasks.py'),
('infinite_loop_timeout', 'tasks.py'),
('log_swarm', 'tasks.py'),
('normal_exception', 'tasks.py'),
('print_sleep_and_go_away', 'tasks.py'),
('raw_input', 'tasks.py'),
('run_in_non_default_domain', 'tasks.py'),
('stderr_swarm', 'tasks.py'),
('unstopable_death', 'tasks.py'),
] == known
mon = Monitor(engine)
wid = mon.new_worker()
t = Task.fromqueue(engine, wid)
t.run()
assert t.output == 42
assert t.metadata == {'user': '******'}
cdate = t._propvalue('queued')
now = datetime.now()
assert now.year == cdate.year
assert now.month == cdate.month
t2 = Task.byid(engine, t.tid)
assert (t2.tid, t2.operation) == (t.tid, t.operation)
t3 = Task.byid(engine, 42000)
assert t3 is None
with pytest.raises(Exception) as err:
api.schedule(engine, 'no_such_task')
assert err.value.args[0] == 'No operation was found for these parameters'
def test_basic_worker_task_execution(engine):
t = api.schedule(engine, 'print_sleep_and_go_away', 21)
assert t.state == 'queued'
guard(engine, "select count(id) from rework.task where status = 'queued'",
lambda res: res.scalar() == 1)
guard(engine, 'select count(id) from rework.worker where running = true',
lambda res: res.scalar() == 0)
mon = Monitor(engine, 'default', maxworkers=1)
mon.ensure_workers()
guard(engine, 'select count(id) from rework.worker where running = true',
lambda res: res.scalar() == 1)
guard(engine, "select count(id) from rework.task where status = 'running'",
lambda res: res.scalar() == 1)
t.join()
assert t.output == 42
assert t.state == 'done'
guard(engine, "select count(id) from rework.task where status = 'running'",
lambda res: res.scalar() == 0)
def workers(engine, domain='default',
minworkers=0, maxworkers=1,
maxruns=0, maxmem=0,
debug=False):
"""context manager to set up a test monitor for a given domain
It can be configured with `domain`, `maxworkers`, `minworkers`, `maxruns`,
`maxmem` and `debug`.
Here's an example usage:
.. code-block:: python
with workers(engine, numworkers=2) as monitor:
t1 = api.schedule(engine, 'compute_sum', [1, 2, 3, 4, 5])
t2 = api.schedule(engine, 'compute_sum', 'abcd')
t1.join()
assert t1.output == 15
t2.join()
assert t2.traceback.endswith(
"TypeError: unsupported operand type(s) "
"for +: 'int' and 'str'"
)
assert len(monitor.wids) == 2
At the end of the block, the monitor-controlled workers are
cleaned up.
However the database entries are still there.
"""
mon = Monitor(
engine, domain, minworkers, maxworkers, maxruns, maxmem, debug
)
mon.register()
mon.ensure_workers()
try:
yield mon
except:
mon.killall(
'Something killed the monitor',
traceback.format_exc()
)
finally:
mon.killall()
mon.unregister()
def monitor(dburi, **config):
" start a monitor controlling min/max workers "
engine = create_engine(find_dburi(dburi))
monitor = Monitor(engine, **config)
monitor.run()