def test_remove_job(): m, s, socket = init() testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir) results = [] def cb(event, cmd): jobs = m.jobs() results.append((cmd, jobs)) socket.bind("command_success", cb) socket.send_command("load", config.to_dict(), start=False) socket.send_command("unload", "dummy") def stop(h): h.close() socket.close() m.stop() t = pyuv.Timer(m.loop) t.start(stop, 0.4, 0.0) m.run() assert len(results) == 2 cmd0, jobs0 = results[0] cmd1, jobs1 = results[1] assert cmd0.result()["ok"] == True assert cmd1.result()["ok"] == True assert len(jobs0) == 1 assert jobs0[0] == "default.dummy" assert len(jobs1) == 0
def test_simple_job(): m, s, socket = init() testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir) # send a command cmd0 = socket.send_command("load", config.to_dict(), start=False) cmd1 = socket.send_command("jobs") results = [] def do_events(h): results.append((len(m.jobs()), len(s.jobs()), s.jobs()[0])) def stop(h): h.close() socket.close() m.stop() t = pyuv.Timer(m.loop) t.start(do_events, 0.4, 0.0) t1 = pyuv.Timer(m.loop) t1.start(stop, 0.8, 0.0) m.run() assert cmd0.error() == None assert cmd1.error() == None assert results[0] == (1, 1, "default.dummy") assert cmd0.result() == {"ok": True} assert cmd1.result()["jobs"][0] == "default.dummy"
def test_sessions(): m = Manager() started = [] stopped = [] def cb(evtype, info): if evtype == "start": started.append(info['name']) elif evtype == "stop": stopped.append(info['name']) start_job = lambda mgr, label: mgr.start_job(label) stop_job = lambda mgr, label: mgr.stop_job(label) m.start() m.events.subscribe('start', cb) m.events.subscribe('stop', cb) testfile, cmd, args, wdir = dummy_cmd() a = ProcessConfig("a", cmd, args=args, cwd=wdir) b = ProcessConfig("b", cmd, args=args, cwd=wdir) # load process config in different sessions m.load(a, sessionid="ga", start=False) m.load(b, sessionid="ga", start=False) m.load(a, sessionid="gb", start=False) sessions = m.sessions ga1 = m.jobs('ga') gb1 = m.jobs('gb') m.jobs_walk(start_job, "ga") m.jobs_walk(start_job, "gb") ga2 = [] def rem_cb(h): m.unload("a", sessionid="ga") [ga2.append(name) for name in m.jobs('ga')] t0 = pyuv.Timer(m.loop) t0.start(rem_cb, 0.2, 0.0) m.jobs_walk(stop_job, "gb") def stop(handle): m.events.unsubscribe("start", cb) m.events.unsubscribe("stop", cb) m.stop() t = pyuv.Timer(m.loop) t.start(stop, 0.6, 0.0) m.run() assert len(sessions) == 2 assert sessions == ['ga', 'gb'] assert ga1 == ['ga.a', 'ga.b'] assert gb1 == ['gb.a'] assert started == ['ga.a', 'ga.b', 'gb.a'] assert stopped == ['gb.a', 'ga.a'] assert ga2 == ['ga.b']
def test_flapping(): m = Manager() m.start() states = [] cmd, args, wdir = crash_cmd() # create flapping info object flapping = FlappingInfo(attempts=1., window=1, retry_in=0.1, max_retry=1) # load conf conf1 = ProcessConfig("crashing", cmd, args=args, cwd=wdir, flapping=flapping) conf2 = ProcessConfig("crashing2", cmd, args=args, cwd=wdir) m.load(conf1) m.load(conf2) time.sleep(0.2) def cb(handle): state = m._get_locked_state("crashing") states.append(state.stopped) state2 = m._get_locked_state("crashing2") states.append(state2.stopped) m.stop() t = pyuv.Timer(m.loop) t.start(cb, 0.8, 0.0) m.run() assert states == [True, False]
def test_priority(): m = Manager() started = [] def cb(evtype, info): started.append(info['name']) m.start() m.events.subscribe('start', cb) testfile, cmd, args, wdir = dummy_cmd() a = ProcessConfig("a", cmd, args=args, cwd=wdir) b = ProcessConfig("b", cmd, args=args, cwd=wdir) c = ProcessConfig("c", cmd, args=args, cwd=wdir) m.load(a, start=False) m.load(b, start=False) m.load(c, start=False) # start all processes m.jobs_walk(lambda mgr, label: mgr.start_job(label)) def stop(handle): m.events.unsubscribe("start", cb) m.stop() t = pyuv.Timer(m.loop) t.start(stop, 0.4, 0.0) m.run() assert started == ["default.a", "default.b", "default.c"]
def test_processes_stats(): def collect_cb(inf, m, name): inf.append(m.stats(name)) m = Manager() m.start() testfile, cmd, args, wdir = dummy_cmd() testfile1, cmd1, args1, wdir1 = dummy_cmd() configa = ProcessConfig("a", cmd, args=args, cwd=wdir) m.load(configa) time.sleep(0.2) infos = [] infos2 = [] m.jobs_walk(partial(collect_cb, infos)) configb = ProcessConfig("b", cmd, args=args, cwd=wdir) m.load(configb) m.jobs_walk(partial(collect_cb, infos2)) m.stop() m.run() assert len(infos) == 1 assert len(infos2) == 2 assert infos[0]['name'] == "default.a" assert infos2[0]['name'] == "default.a" assert infos2[1]['name'] == "default.b"
def test_stdio(): m, s = init() emitted = [] def cb(ch, data): emitted.append(data) responses = [] def cb2(ch, result, error): responses.append((result, error)) if sys.platform == 'win32': config = ProcessConfig("echo", "cmd.exe", args=["/c", "proc_stdin_stdout.py"], redirect_output=["stdout"], redirect_input=True) else: config = ProcessConfig("echo", "./proc_stdin_stdout.py", cwd=os.path.dirname(__file__), redirect_output=["stdout"], redirect_input=True) # load the process m.load(config) time.sleep(0.2) # start a channel p = s.get_process(1) channel = p.socket() channel.start() # subscribe to remote events channel.start_read(cb) # write to STDIN channel.write(b"ECHO" + linesep, cb2) def stop(handle): channel.stop_read() channel.close() m.stop() t = pyuv.Timer(m.loop) t.start(stop, 0.3, 0.0) m.run() assert len(emitted) == 1 assert emitted == [b'ECHO\n\n'] assert responses == [(b"OK", None)]
def test_pids(): m, s = init() testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir) s.load(config) time.sleep(0.2) job = s.get_job("dummy") assert isinstance(job, Job) == True pid = s.get_process(1) assert isinstance(pid, Process) == True assert pid.pid == 1 assert pid.info.get('name') == "default.dummy" assert pid.name == "default.dummy" assert pid.os_pid == pid.info.get('os_pid') assert job.pids == [1] pid.stop() assert 1 not in m.running time.sleep(0.2) assert job.pids == [2] m.stop() m.run()
def test_stats(): m = Manager() monitored = [] def cb(info): monitored.append(info) m.start() testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("a", cmd, args=args, cwd=wdir) m.load(config) os_pid = m.running[1].os_pid chan = m.subscribe("STATS:default.a") chan.bind(cb) def stop(handle): chan.unbind(cb) m.stop() t = pyuv.Timer(m.loop) t.start(stop, 0.3, 0.0) m.run() assert len(monitored) >= 1 res = monitored[0] assert "cpu" in res assert res["os_pid"] == os_pid
def test_process_events(): emitted = [] m = Manager() m.start() def cb(ev, *args): emitted.append(ev) # subscribe to all events chan = m.subscribe("JOB:default.dummy") chan.bind_all(cb) testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir) m.load(config) m.unload(config) time.sleep(0.2) m.stop() m.run() assert 'start' in emitted assert 'spawn' in emitted assert 'stop' in emitted assert 'exit' in emitted
def test_process_exit_event(): emitted = [] m = Manager() m.start() def cb(ev, msg): emitted.append(msg) # subscribe to all events m.events.subscribe('job.default.dummy.exit', cb) testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir) m.load(config) m.unload("dummy") time.sleep(0.2) m.stop() m.run() assert len(emitted) == 1 assert len(emitted[0]) == 7 msg = emitted[0] assert "exit_status" in msg assert msg['once'] == False
def test_basic(): emitted = [] m = Manager() m.start() def cb(ev, msg): emitted.append((ev, msg['name'])) # subscribe to all events chan = m.subscribe("EVENTS") chan.bind('.', cb) testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir, numprocesses=4) m.load(config) m.scale("dummy", 1) m.unload("dummy") time.sleep(0.2) m.stop() m.run() assert ('load', 'default.dummy') in emitted assert ('start', 'default.dummy') in emitted assert ('update', 'default.dummy') in emitted assert ('stop', 'default.dummy') in emitted assert ('unload', 'default.dummy') in emitted
def test_send_signal(): m = Manager() m.start() testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir) m.load(config) time.sleep(0.2) m.killall("dummy", signal.SIGHUP) time.sleep(0.2) with pytest.raises(ProcessError) as e: m.killall("dummy1", signal.SIGHUP) assert e.errno == 404 m.stop_job("dummy") def stop(handle): handle.stop() m.stop() t = pyuv.Timer(m.loop) t.start(stop, 0.8, 0.0) m.run() with open(testfile, 'r') as f: res = f.read() assert res == 'STARTHUPQUITSTOP'
def test_scalein(): m = Manager() m.start() testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir, numprocesses=1) m.load(config) state = m._get_locked_state("dummy") assert len(state.running) == 1 ret = m.scale("dummy", 1) assert ret == 2 time.sleep(0.2) assert len(state.running) == 2 ret = m.scale("dummy", 1) assert ret == 3 time.sleep(0.2) assert len(state.running) == 3 ret = m.scale("dummy", 3) assert ret == 6 time.sleep(0.2) assert len(state.running) == 6 m.stop() m.run()
def test_monitor(): m = Manager() monitored = [] def cb(evtype, info): monitored.append((evtype, info)) m.start() testfile, cmd, args, wdir = dummy_cmd() configa = ProcessConfig("a", cmd, args=args, cwd=wdir) m.load(configa) time.sleep(0.2) os_pid = m.running[1].os_pid m.monitor(cb, "a") def stop(handle): m.unmonitor(cb, "a") m.stop() t = pyuv.Timer(m.loop) t.start(stop, 0.3, 0.0) m.run() assert len(monitored) >= 1 res = monitored[0] assert res[0] == "stat" assert "cpu" in res[1] assert res[1]["os_pid"] == os_pid
def test_custom_stream(): m, s = init() emitted = [] def cb(ch, data): emitted.append(data) responses = [] def cb2(ch, result, error): responses.append((result, error)) if sys.platform == 'win32': config = ProcessConfig("echo", "cmd.exe", args=["/c", "proc_custom_stream.py"], custom_streams=["ctrl"]) else: config = ProcessConfig("echo", "./proc_custom_stream.py", cwd=os.path.dirname(__file__), custom_streams=["ctrl"]) m.load(config) # start a channel p = s.get_process(1) channel = p.socket(stream="ctrl") channel.start() # subscribe to remote events channel.start_read(cb) # write to STDIN channel.write(b"ECHO" + linesep, cb2) def stop(handle): channel.stop_read() channel.close() m.stop() t = pyuv.Timer(m.loop) t.start(stop, 0.4, 0.0) m.loop.run() assert len(emitted) == 1 assert emitted == [b'ECHO\n'] assert responses == [(b"OK", None)]
def on_manager(h): # start the manager with the HTTP API m.start(apps=[http_handler]) testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir) m.load(config) m.stop_process(1) m.unload("dummy") h.start(wait, 0.3, 0.0)
def test_mode_writable(): m, s = init() responses = [] def cb2(ch, result, error): responses.append((result, error)) if sys.platform == 'win32': config = ProcessConfig("echo", "cmd.exe", args=["/c", "proc_stdin_stdout.py"], redirect_output=["stdout"], redirect_input=True) else: config = ProcessConfig("echo", "./proc_stdin_stdout.py", cwd=os.path.dirname(__file__), redirect_output=["stdout"], redirect_input=True) # load the process m.load(config) time.sleep(0.2) # start a channel p1 = s.get_process(1) channel = p1.socket(mode=pyuv.UV_WRITABLE) channel.start() # subscribe to remote events with pytest.raises(IOError): channel.start_read(lambda ch, d: None) channel.write(b"ECHO" + linesep, cb2) def stop(handle): channel.stop_read() channel.close() m.stop() t = pyuv.Timer(m.loop) t.start(stop, 0.3, 0.0) m.run() assert responses == [(b"OK", None)]
def test_start_multiple(): m = Manager() m.start() testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir, numprocesses=2) m.load(config) state = m._get_locked_state("dummy") assert len(state.running) == 2 m.stop() m.run()
def test_commit(): m, s, socket = init() testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir, numprocesses=0) # send a command cmd0 = socket.send_command("load", config.to_dict(), start=False) cmd1 = socket.send_command("commit", "dummy") def stop(c): socket.close() m.stop() cmd1.add_done_callback(stop) m.run() assert cmd0.error() == None assert cmd1.error() == None assert cmd0.result() == {"ok": True} assert cmd1.result()["pid"] == 1
def test_remove_job(): m, s = init() testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir) s.load(config, start=False) assert s.jobs()[0] == "default.dummy" s.unload("dummy") assert len(s.jobs()) == 0 assert len(m.jobs()) == 0 assert len(m.sessions) == 0 m.stop() m.run()
def test_simple_job(): m, s = init() assert s.jobs() == [] testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir) m.load(config, start=False) time.sleep(0.2) assert len(m.jobs()) == 1 assert len(s.jobs()) == 1 assert s.jobs()[0] == "default.dummy" m.stop() m.run()
def create_config(self, channels): """Create worker's process configuration.""" config = self.app.config return ProcessConfig( name=self.job_name, cmd=sys.executable, args=['-c', '{0}'.format(self.command)], env=dict(os.environ, IS_WORKER='1'), numprocesses=config.WORKERS, redirect_input=True, redirect_output=['out', 'err'], custom_streams=['handshake', 'incoming', 'outgoing'], custom_channels=channels, graceful_timeout=config.PROCESS_STOP_TIMEOUT, )
def test_start_stop_job(): res = [] m = Manager() m.start() testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir) m.load(config) state = m._get_locked_state("dummy") res.append(len(state.running)) m.stop_job("dummy") res.append(len(state.running)) m.stop() m.run() assert res == [1, 0]
def test_running(): m, s = init() testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir) job = s.load(config) time.sleep(0.2) assert len(m.running) == 1 assert len(s.running()) == 1 assert 1 in m.running assert s.running()[0] == 1 m.stop() m.run()
def test_numprocesses(): m = Manager() m.start() testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir, numprocesses=4) m.load(config) state = m._get_locked_state("dummy") assert len(state.running) == 4 state.numprocesses = 0 assert state.numprocesses == 0 m.manage("dummy") time.sleep(0.2) assert len(state.running) == 0 m.stop() m.run()
def test_process_commit(): emitted = [] m = Manager() m.start() def cb(ev, msg): emitted.append(msg) # subscribe to all events m.events.subscribe('job.default.dummy.exit', cb) testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir, numprocesses=0) m.load(config) pids0 = m.pids() pid = m.commit("dummy", env={"BLAH": "test"}) process = m.get_process(pid) pids1 = m.pids() state = m._get_locked_state("dummy") assert len(state.running) == 0 assert state.numprocesses == 0 assert len(state.running_out) == 1 m.unload("dummy") time.sleep(0.2) m.stop() m.run() assert pids0 == [] assert pid == 1 assert process.name == "default.dummy" assert process.pid == 1 assert "BLAH" in process.env assert pids1 == [1] assert len(emitted) == 1 assert len(emitted[0]) == 7 msg = emitted[0] assert "exit_status" in msg assert msg['once'] == True
def test_restart_manager(): results = [] m = Manager() m.start() testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir, numprocesses=4) m.load(config) results.append(m.pids("dummy")) def cb(manager): results.append(m.pids("dummy")) m.stop() m.restart() t = pyuv.Timer(m.loop) t.start(cb, 0.4, 0.0) m.run() assert results[0] != results[1]
def test_stats(): m, s = init() testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir) s.load(config) time.sleep(0.2) pid = s.get_process(1) assert isinstance(pid, Process) == True assert pid.pid == 1 stats = pid.stats assert isinstance(stats, dict) == True assert "cpu" in stats assert "mem_info1" in stats pid.stop() m.stop() m.run()
def test_process_stats(): m = Manager() m.start() testfile, cmd, args, wdir = dummy_cmd() config = ProcessConfig("dummy", cmd, args=args, cwd=wdir) m.load(config) time.sleep(0.2) info = m.stats("dummy") info_by_id = m.get_process(1).info os_pid = m.running[1].os_pid m.stop() m.run() assert isinstance(info, dict) assert isinstance(info_by_id, dict) assert "os_pid" in info_by_id assert info_by_id["os_pid"] == os_pid assert info['name'] == "default.dummy" assert len(info['stats']) == 1 assert info['stats'][0]['os_pid'] == info_by_id['os_pid']