def worker(*, uid: int, endpoint: str):
ctx = zmq.Context()
dispatcher = RPCDispatcher()
client = RPCClient(
JSONRPCProtocol(),
ZmqClientTransport.create(ctx, endpoint),
)
proxy = client.get_proxy()
@dispatcher.public
def do_task():
from time import sleep
for i in range(5):
proxy.notify(f"{uid} y{i} ")
sleep(0.1)
return f"OK {uid}"
@dispatcher.public
def exit():
proxy.exit(uid)
transport = ZmqServerTransport.create(
ctx, endpoint.replace(".manager", f".{uid}"))
rpc_server = RPCServer(transport, JSONRPCProtocol(), dispatcher)
rpc_server.trace = partial(print, file=sys.stderr)
rpc_server.serve_forever()
def client(*, endpoint: str, callback_endpoint: str):
ctx = zmq.Context()
dispatcher = RPCDispatcher()
@dispatcher.public
def notify(s: str):
print("** {s} **")
callback_server = RPCServer(
ZmqServerTransport.create(ctx, callback_endpoint), JSONRPCProtocol(),
dispatcher)
callback_server.trace = print
threading.Thread(target=callback_server.serve_forever, daemon=True).start()
rpc_client = RPCClient(JSONRPCProtocol(),
ZmqClientTransport.create(ctx, endpoint))
remote_server = rpc_client.get_proxy()
for i in range(7):
result = remote_server.act(i, "Hello, World!")
print("Server answered:", result)
result = remote_server.shutdown()
print("Send shutdown:", result)
def test_handle_message(transport, protocol, dispatcher):
server = RPCServer(transport, protocol, dispatcher)
server.receive_one_message()
transport.receive_message.assert_called()
protocol.parse_request.assert_called_with(RECMSG)
dispatcher.dispatch.assert_called_with(PARMSG)
dispatcher.dispatch().serialize.assert_called()
transport.send_reply.assert_called_with(CONTEXT, SERMSG)
def worker():
rpc_server = RPCServer(
SubprocessServerTransport(
input_port=sys.stdin.buffer, output_port=sys.stdout.buffer
),
JSONRPCProtocol(),
dispatcher,
)
rpc_server.trace = partial(print, file=sys.stderr)
rpc_server.serve_forever()
def test_handle_message_callback(transport, protocol, dispatcher):
server = RPCServer(transport, protocol, dispatcher)
server.trace = Mock(return_value=None)
server.receive_one_message()
assert server.trace.call_args_list == [
call('-->', CONTEXT, RECMSG),
call('<--', CONTEXT, SERMSG)
]
server.trace.assert_called()
def connectHandler(cmd, handler):
dispatcher = RPCDispatcher()
print("cmd: {}".format(" ".join(cmd)))
# line buffered
p = subprocess.Popen(cmd,
bufsize=1,
universal_newlines=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
transport = PipeTransport(p.stdout, p.stdin)
rpc_server = RPCServer(transport, JSONRPCProtocol(), dispatcher)
dispatcher.register_instance(handler, '')
return (rpc_server, p)
def main(args):
cmd = ["clef", "--stdio-ui"]
if len(args) > 0 and args[0] == "test":
cmd.extend(["--stdio-ui-test"])
print("cmd: {}".format(" ".join(cmd)))
dispatcher = RPCDispatcher()
dispatcher.register_instance(StdIOHandler(), '')
# line buffered
p = subprocess.Popen(cmd, bufsize=1, universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
rpc_server = RPCServer(
PipeTransport(p.stdout, p.stdin),
JSONRPCProtocol(),
dispatcher
)
rpc_server.serve_forever()
def manager():
ctx = zmq.Context()
transport = ZmqServerTransport.create(ctx, "tcp://127.0.0.1:5002")
rpc_server = RPCServer(transport, JSONRPCProtocol(), dispatcher)
# rpc_server.trace = print
th = threading.Thread(daemon=True, target=rpc_server.serve_forever)
th.start()
cmd = [sys.executable, __file__, "worker"]
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, text=True)
for i in range(10):
send(f"world{i}", port=p.stdin)
p.stdin.close()
th.join()
p.terminate()
def __init__(self, req_callback):
# print 'initializing Rpc'
self.ctx = zmq.Context()
self.dispatcher = RPCDispatcher()
self.transport = ZmqServerTransport.create(self.ctx, 'tcp://127.0.0.1:8000')
self.req_callback = req_callback
self.rpc_server = RPCServer(
self.transport,
JSONRPCProtocol(),
self.dispatcher
)
self.dispatcher.public(self.request) # register this function (replacing the decorator)
# print 'READYc: '+str(time.clock())
# sys.exit(0)
self.rpc_server.serve_forever()
def init_server(self, transport, logger):
'''
Internal function that should not be called explicitly.
'''
if isinstance(transport, ZmqServerTransport):
self.transport = transport
else:
# dict like {'receiver': transport, 'receiver': replier_endpoint}
for key in transport:
if 'tcp' not in str(transport[key]):
transport[key] = "tcp://" + str(transport[key])
self.transport = ZmqServerTransport.create(self.ctx, transport)
self.transport.publisher = self.publisher
self.rpc_server = RPCServer(self.transport, self.protocol,
self.dispatcher)
self.rpc_server.set_logger(logger)
self.register_instance({'server': self})
self.rpc_server.start()
self.logger.info('rpc server {} started.'.format(self.endpoint))
def run(*, port: int = 8888, sentinel: str = "") -> None:
if sentinel:
p = pathlib.Path(sentinel)
ok = False
for wait_time in [0.1, 0.2, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4]:
if not p.exists():
print(f" wait ... {wait_time}", file=sys.stderr)
time.sleep(wait_time)
continue
print(f"ack {sentinel}", file=sys.stderr)
p.unlink()
ok = True
break
if not ok:
raise RuntimeError(f"timeout, f{sentinel}, {time.time()-st}")
print(f"listen ... {port}", file=sys.stderr)
transport = ZmqServerTransport.create(ctx, f"tcp://127.0.0.1:{port}")
rpc_server = RPCServer(transport, JSONRPCProtocol(), dispatcher)
rpc_server.serve_forever()
def test_handle_message(transport, protocol, dispatcher):
server = RPCServer(transport, protocol, dispatcher)
server.process_one_message()
transport.receive_message.assert_called()
protocol.parse_request.assert_called_with(RECMSG)
import json
import zmq
from tinyrpc.protocols.jsonrpc import JSONRPCProtocol
from tinyrpc.transports.zmq import ZmqServerTransport
from tinyrpc.server import RPCServer
from tinyrpc.dispatch import RPCDispatcher
ctx = zmq.Context()
dispatcher = RPCDispatcher()
transport = ZmqServerTransport.create(ctx, 'tcp://127.0.0.1:8000')
rpc_server = RPCServer(transport, JSONRPCProtocol(), dispatcher)
@dispatcher.public
def request(s):
# return s[::-1]
rq = json.loads(s)
print 'REQUEST: '
print rq['meta']
print rq['body']
if 'OFPFlowMod' in rq['body']:
print rq['body']['OFPFlowMod']
return 'PERMIT'
rpc_server.serve_forever()
def manager():
@dataclasses.dataclass
class Handler:
uid: int
process: subprocess.Popen
remote_server: RPCProxy
ctx = zmq.Context()
dispatcher = RPCDispatcher()
living_process = {None} # None is semaphore
@dispatcher.public
def notify(msg: t.Any):
print("*", msg, "*")
@dispatcher.public
def exit(uid: int):
print("EXIT", living_process.remove(uid))
# use tmpfs?
with tempfile.TemporaryDirectory() as d:
endpoint = f"ipc://{d}/worker.manager"
transport = ZmqServerTransport.create(ctx, endpoint)
rpc_server = RPCServer(transport, JSONRPCProtocol(), dispatcher)
def server_loop():
while len(living_process) > 0:
rpc_server.receive_one_message()
th = threading.Thread(target=server_loop, daemon=True)
th.start()
handlers = []
n = 3
protocol = JSONRPCProtocol()
for uid in range(n):
living_process.add(uid)
cmd = [
sys.executable,
"-u",
__file__,
"worker",
"--uid",
str(uid),
"--endpoint",
endpoint,
]
p = subprocess.Popen(
cmd,
# stdout=subprocess.PIPE,
# stderr=subprocess.PIPE,
# text=False, # for read1
)
rpc_client = RPCClient(
protocol,
ZmqClientTransport.create(
ctx, endpoint.replace(".manager", f".{uid}")),
)
remote_server = rpc_client.get_proxy()
handlers.append(
Handler(uid=uid, process=p, remote_server=remote_server))
# 本当はguardなどを使う
living_process.remove(None) # None is semaphore
futs = []
from concurrent.futures import ThreadPoolExecutor
with ThreadPoolExecutor() as ex:
for h in handlers:
futs.append(ex.submit(h.remote_server.do_task))
futs.append(ex.submit(h.remote_server.do_task))
for fut in futs:
print(fut.result())
for h in handlers:
h.remote_server.exit()
print("END")
th.join()
def server(*, endpoint: str):
import queue
import threading
import time
from concurrent.futures import Future
from tinyrpc.transports.zmq import ZmqServerTransport
from tinyrpc.server import RPCServer
from tinyrpc.dispatch import RPCDispatcher
q = queue.Queue()
ev = threading.Event()
ctx = zmq.Context()
dispatcher = RPCDispatcher()
transport = ZmqServerTransport.create(ctx, endpoint)
rpc_server = RPCServer(transport, JSONRPCProtocol(), dispatcher)
rpc_server.trace = print
running = True
@dispatcher.public
def act(uid: int, s: str) -> str:
q.put(("act", (uid, s), {}))
ev.set()
return "ok"
@dispatcher.public
def shutdown() -> None:
nonlocal running
running = False
def do_act(uid: int, s: str) -> Future:
fut = Future()
def do():
for i in range(5):
print(f"{uid:02d}: {i} {s}")
time.sleep(0.1)
fut.set_result(("ok", uid))
threading.Thread(target=do, daemon=True).start()
return fut
def do_server():
nonlocal running
while running:
rpc_server.receive_one_message()
def do_loop(*, ns):
nonlocal running
while running:
fn, args, kwargs = q.get()
fut = ns[f"do_{fn}"](*args, **kwargs)
def cont(fut):
q.task_done()
fut.add_done_callback(cont)
ns = locals()
th = threading.Thread(target=do_loop, kwargs={"ns": ns}, daemon=True)
th.start()
th2 = threading.Thread(target=do_server, daemon=True)
th2.start()
ev.wait()
q.join()
return out_box.pop(0)
class CallbackServerTransport(ServerTransport):
def receive_message(self) -> t.Tuple[t.Any, bytes]:
context = "*context*"
return context, in_box.pop(0)
def send_reply(self, context: t.Any, reply: bytes):
print(context)
out_box.append(reply)
rpc_server = RPCServer(
CallbackServerTransport(),
JSONRPCProtocol(),
dispatcher,
)
rpc_server.trace = print
# 本来はこれを実行する
# rpc_server.serve_forever()
rpc_client = RPCClient(JSONRPCProtocol(), CallbackClientTransport())
proxy = rpc_client.get_proxy()
try:
result = proxy.add(10, 20)
print(f"result is {result}")
except Exception as e:
print(e)
def server(*, endpoint: str, callback_endpoint: str):
q = queue.Queue()
ev = threading.Event()
ctx = zmq.Context()
dispatcher = RPCDispatcher()
rpc_server = RPCServer(ZmqServerTransport.create(ctx, endpoint),
JSONRPCProtocol(), dispatcher)
rpc_server.trace = print
# client
callback_client = RPCClient(
JSONRPCProtocol(), ZmqClientTransport.create(ctx, callback_endpoint))
callback_remote_server = callback_client.get_proxy(one_way=True)
running = True
@dispatcher.public
def act(uid: int, s: str) -> str:
q.put(("act", (uid, s), {}))
ev.set()
return "ok"
@dispatcher.public
def shutdown() -> None:
nonlocal running
running = False
def do_act(uid: int, s: str) -> Future:
fut = Future()
def do():
for i in range(5):
callback_remote_server.notify(f"{uid:02d}: {i} {s}")
print(f"{uid:02d}: {i} {s}")
time.sleep(0.1)
fut.set_result(("ok", uid))
threading.Thread(target=do, daemon=True).start()
return fut
def do_server():
nonlocal running
while running:
rpc_server.receive_one_message()
def do_loop(*, ns):
nonlocal running
while running:
fn, args, kwargs = q.get()
fut = ns[f"do_{fn}"](*args, **kwargs)
def cont(fut):
q.task_done()
fut.add_done_callback(cont)
ns = locals()
th = threading.Thread(target=do_loop, kwargs={"ns": ns}, daemon=True)
th.start()
th2 = threading.Thread(target=do_server, daemon=True)
th2.start()
ev.wait()
q.join()