def client(*, endpoint: str, callback_endpoint: str):
ctx = zmq.Context()
dispatcher = RPCDispatcher()
@dispatcher.public
def notify(s: str):
print("** {s} **")
callback_server = RPCServer(
ZmqServerTransport.create(ctx, callback_endpoint), JSONRPCProtocol(),
dispatcher)
callback_server.trace = print
threading.Thread(target=callback_server.serve_forever, daemon=True).start()
rpc_client = RPCClient(JSONRPCProtocol(),
ZmqClientTransport.create(ctx, endpoint))
remote_server = rpc_client.get_proxy()
for i in range(7):
result = remote_server.act(i, "Hello, World!")
print("Server answered:", result)
result = remote_server.shutdown()
print("Send shutdown:", result)
def worker(*, uid: int, endpoint: str):
ctx = zmq.Context()
dispatcher = RPCDispatcher()
client = RPCClient(
JSONRPCProtocol(),
ZmqClientTransport.create(ctx, endpoint),
)
proxy = client.get_proxy()
@dispatcher.public
def do_task():
from time import sleep
for i in range(5):
proxy.notify(f"{uid} y{i} ")
sleep(0.1)
return f"OK {uid}"
@dispatcher.public
def exit():
proxy.exit(uid)
transport = ZmqServerTransport.create(
ctx, endpoint.replace(".manager", f".{uid}"))
rpc_server = RPCServer(transport, JSONRPCProtocol(), dispatcher)
rpc_server.trace = partial(print, file=sys.stderr)
rpc_server.serve_forever()
def __init__(self, ws, rpc_callback):
dispatcher = RPCDispatcher()
dispatcher.register_instance(rpc_callback)
super(WebSocketRPCServer, self).__init__(
WebSocketServerTransport(ws),
JSONRPCProtocol(),
dispatcher,
)
def __init__(self, function_lists, is_id_valid):
self._rpc_dispatcher = RPCDispatcher()
self._rpc_protocol = JSONRPCProtocol()
self._is_id_valid = is_id_valid
for f in function_lists:
self._rpc_dispatcher.add_method(f)
logger.info("RPC service initialized")
def _load_core_modules(app):
dispatcher = RPCDispatcher()
setattr(app, 'dispatcher', dispatcher)
for module in CORE_MODULES:
try:
module_imp = importlib.import_module(module)
app.register_blueprint(module_imp.bp)
except AttributeError as e:
app.logger.warning(str(e))
except ImportError as e:
app.logger.exception(str(e))
def WebSocketRPCHandler(trustlines: TrustlinesRelay):
dispatcher = RPCDispatcher()
dispatcher.add_method(partial(subscribe, trustlines), "subscribe")
protocol = JSONRPCProtocol()
def handle(ws):
app = RPCWebSocketApplication(protocol, dispatcher, ws)
app.handle()
return handle
def __init__(self, app):
log.debug('initializing JSONRPCServer')
BaseService.__init__(self, app)
self.app = app
self.dispatcher = RPCDispatcher()
transport = WsgiServerTransport(queue_class=gevent.queue.Queue)
# start wsgi server as a background-greenlet
self.wsgi_server = gevent.wsgi.WSGIServer(('127.0.0.1', 5000),
transport.handle)
self.rpc_server = RPCServerGreenlets(transport, JSONRPCProtocol(),
self.dispatcher)
def MessagingWebSocketRPCHandler(trustlines: TrustlinesRelay):
dispatcher = RPCDispatcher()
dispatcher.add_method(partial(messaging_subscribe, trustlines), "listen")
dispatcher.add_method(partial(get_missed_messages, trustlines), "getMissedMessages")
protocol = JSONRPCProtocol()
def handle(ws):
app = RPCWebSocketApplication(protocol, dispatcher, ws)
app.handle()
return handle
def connectHandler(cmd, handler):
dispatcher = RPCDispatcher()
print("cmd: {}".format(" ".join(cmd)))
# line buffered
p = subprocess.Popen(cmd,
bufsize=1,
universal_newlines=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
transport = PipeTransport(p.stdout, p.stdin)
rpc_server = RPCServer(transport, JSONRPCProtocol(), dispatcher)
dispatcher.register_instance(handler, '')
return (rpc_server, p)
def __init__(self,
transport,
publisher=None,
ctx=None,
protocol=None,
dispatcher=None):
self.ctx = ctx if ctx else zmq.Context().instance()
self.protocol = protocol if protocol else JSONRPCProtocol()
self.dispatcher = dispatcher if dispatcher else RPCDispatcher()
self.publisher = publisher if publisher else NoOpPublisher()
if isinstance(transport, dict):
# dictionary:
if 'receiver' in transport and 'replier' in transport:
self.endpoints = transport
else:
msg = 'endpoint dictionary {} should contains receiver and replier'
raise Exception(msg.format(transport))
self.endpoint = self.endpoints['receiver']
elif isinstance(transport, basestring):
# only 1 endpoint is provided; create endpoint for replier by adding port by 10000
pattern = '(tcp://)?((?P<ip>[0-9.*]+):)?(?P<port>[0-9]+)'
re_groups = re.match(pattern, transport.strip())
if not re_groups:
raise Exception(
'Invalid transport format {}; '
'expecting tcp://IP:PORT or IP:PORT'.format(transport))
replier_port = int(re_groups.group('port')) + 10000
ip = re_groups.group('ip') if re_groups.group('ip') else '*'
receiver_endpoint = 'tcp://{}:{}'.format(ip, replier_port)
replier_endpoint = 'tcp://{}:{}'.format(ip, replier_port)
self.endpoints = {
'receiver': transport,
'replier': replier_endpoint
}
self.endpoint = self.endpoints['receiver']
else:
# existing transport instance
self.endpoint = transport
logger_name = self.endpoint.replace('://', '')
logger_name = logger_name.replace(':', '_')
logger_name = logger_name.replace('tcp', '')
self.logger = RPCLogger(logger_name)
self.init_server(self.endpoints, self.logger)
self.server_mode = 'normal'
def main(args):
cmd = ["clef", "--stdio-ui"]
if len(args) > 0 and args[0] == "test":
cmd.extend(["--stdio-ui-test"])
print("cmd: {}".format(" ".join(cmd)))
dispatcher = RPCDispatcher()
dispatcher.register_instance(StdIOHandler(), '')
# line buffered
p = subprocess.Popen(cmd, bufsize=1, universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
rpc_server = RPCServer(
PipeTransport(p.stdout, p.stdin),
JSONRPCProtocol(),
dispatcher
)
rpc_server.serve_forever()
def nsRPCBringupServer(ns, path, host, port, maxconn):
name = os.path.basename(path)
dev_path = '/dev/rpc/{}'.format(name)
if nsRPCisServer(ns, path) is not True:
nsError(ns, "RPC service {} misconfigured".format(name))
return False
nsInfo(ns, "Configuring RPC server from {}".format(path))
nsMkdir(ns, dev_path)
nsMkdir(ns, "{}/root".format(dev_path))
_to_root = nsGet(ns, "{}/jail".format(path), [])
for j in _to_root:
_n = os.path.basename(j)
_dst = "{}/root/{}".format(dev_path, _n)
nsInfo(ns, "RPC.JAIL({}): {}".format(name, j))
nsLn(ns, j, _dst)
dispatcher = RPCDispatcher()
nsSet(ns, "{}/dispatcher".format(dev_path), dispatcher)
for h in nsLs(ns, "{}/handlers".format(path)):
nsInfo(ns, "Registering {}->{} ".format(name, h))
_fun = nsGet(ns, "{}/handlers/{}".format(path, h))
dispatcher.add_method(partial(_fun, dev_path), h)
transport = WsgiServerTransport(queue_class=gevent.queue.Queue)
nsSet(ns, "{}/transport".format(dev_path), transport)
nsSet(ns, "{}/listen".format(dev_path), host)
nsSet(ns, "{}/port".format(dev_path), host)
nsConsole(
ns, "RPC server {} will be listening on tcp://{}:{}".format(
name, host, port))
pool = gevent.pool.Pool(maxconn)
nsSet(ns, "{}/pool".format(dev_path), pool)
wsgi_server = gevent.pywsgi.WSGIServer((host, port),
transport.handle,
spawn=pool,
log=None)
nsSet(ns, "{}/wsgi".format(dev_path), wsgi_server)
nsDaemon(ns, "{}:WSGI".format(name), wsgi_server.serve_forever, _raw=True)
rpc_server = RPCServerGreenlets(transport, JSONRPCProtocol(), dispatcher)
if nsGet(ns, "/config/RPCCatchCalls") is True or nsGet(
ns, "/etc/flags/rpctrace", False) is True:
_q = gevent.queue.Queue()
nsSet(ns, "{}/trace".format(dev_path), _q)
rpc_server.trace = partial(nsRPCCatcher, ns, dev_path, _q)
nsSet(ns, "{}/rpc", rpc_server)
nsDaemon(ns, "{}:RPC".format(name), rpc_server.serve_forever, _raw=True)
nsInfo(ns, "RPC server {} is up".format(name))
return True
def main():
import argparse
parser = argparse.ArgumentParser(description="Wotabag rpi server daemon.")
parser.add_argument('config_file', type=argparse.FileType('r'),
help='YAML configuration file.')
args = parser.parse_args()
# Initialize dbus (this must be done before spawning any threads)
dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
dbus.mainloop.glib.threads_init()
bus = dbus.SystemBus()
mainloop = GObject.MainLoop()
with WotabagManager(args.config_file) as wotabag:
logger = logging.getLogger('wotabag')
# Register RPC endpoints
dispatcher = RPCDispatcher()
dispatcher.register_instance(wotabag, 'wotabag.')
# Configure BLE GATT server
sdp_transport = SDPServerTransport(queue_class=gevent.queue.Queue)
ble.advertising_main(mainloop, bus, '')
ble.gatt_server_main(mainloop, bus, '', sdp_transport)
gevent_thread = Thread(target=gevent_main, args=(wotabag, dispatcher, sdp_transport))
try:
# Start gevent (RPC server) thread
gevent_thread.start()
# Run GATT server
logger.info("Running BLE server")
mainloop.run()
except KeyboardInterrupt:
logger.info("Done, cleaning up...")
except Exception as e:
logger.exception(e)
finally:
server_done.set()
mainloop.quit()
logger.info("BLE server finished")
gevent_thread.join()
def __init__(self, req_callback):
# print 'initializing Rpc'
self.ctx = zmq.Context()
self.dispatcher = RPCDispatcher()
self.transport = ZmqServerTransport.create(self.ctx, 'tcp://127.0.0.1:8000')
self.req_callback = req_callback
self.rpc_server = RPCServer(
self.transport,
JSONRPCProtocol(),
self.dispatcher
)
self.dispatcher.public(self.request) # register this function (replacing the decorator)
# print 'READYc: '+str(time.clock())
# sys.exit(0)
self.rpc_server.serve_forever()
def subdispatch():
return RPCDispatcher()
def dispatch():
return RPCDispatcher()
def server(*, endpoint: str, callback_endpoint: str):
q = queue.Queue()
ev = threading.Event()
ctx = zmq.Context()
dispatcher = RPCDispatcher()
rpc_server = RPCServer(ZmqServerTransport.create(ctx, endpoint),
JSONRPCProtocol(), dispatcher)
rpc_server.trace = print
# client
callback_client = RPCClient(
JSONRPCProtocol(), ZmqClientTransport.create(ctx, callback_endpoint))
callback_remote_server = callback_client.get_proxy(one_way=True)
running = True
@dispatcher.public
def act(uid: int, s: str) -> str:
q.put(("act", (uid, s), {}))
ev.set()
return "ok"
@dispatcher.public
def shutdown() -> None:
nonlocal running
running = False
def do_act(uid: int, s: str) -> Future:
fut = Future()
def do():
for i in range(5):
callback_remote_server.notify(f"{uid:02d}: {i} {s}")
print(f"{uid:02d}: {i} {s}")
time.sleep(0.1)
fut.set_result(("ok", uid))
threading.Thread(target=do, daemon=True).start()
return fut
def do_server():
nonlocal running
while running:
rpc_server.receive_one_message()
def do_loop(*, ns):
nonlocal running
while running:
fn, args, kwargs = q.get()
fut = ns[f"do_{fn}"](*args, **kwargs)
def cont(fut):
q.task_done()
fut.add_done_callback(cont)
ns = locals()
th = threading.Thread(target=do_loop, kwargs={"ns": ns}, daemon=True)
th.start()
th2 = threading.Thread(target=do_server, daemon=True)
th2.start()
ev.wait()
q.join()
def __init__(self,
transport,
publisher=None,
ctx=None,
protocol=None,
dispatcher=None,
log_level=INFO,
log_folder_path=None,
name=None,
threadpool_size=None):
self.ctx = ctx if ctx else zmq.Context().instance()
self.protocol = protocol if protocol else JSONRPCProtocol()
self.dispatcher = dispatcher if dispatcher else RPCDispatcher()
self.publisher = publisher if publisher else NoOpPublisher()
threadpool_size = threadpool_size or THREAD_POOL_WORKERS
if isinstance(transport, dict):
# dictionary:
if 'receiver' in transport and 'replier' in transport:
self.endpoints = transport
else:
msg = 'endpoint dictionary {} should contains receiver and replier'
raise Exception(msg.format(transport))
self.endpoint = self.endpoints['receiver']
elif isinstance(transport, basestring):
# only 1 endpoint is provided; create endpoint for replier by adding port by 10000
pattern = '(tcp://)?((?P<ip>[0-9.*]+):)?(?P<port>[0-9]+)'
re_groups = re.match(pattern, transport.strip())
if not re_groups:
raise Exception(
'Invalid transport format {}; '
'expecting tcp://IP:PORT or IP:PORT'.format(transport))
replier_port = int(re_groups.group('port')) + 10000
ip = re_groups.group('ip') if re_groups.group('ip') else '*'
receiver_endpoint = 'tcp://{}:{}'.format(ip, replier_port)
replier_endpoint = 'tcp://{}:{}'.format(ip, replier_port)
self.endpoints = {
'receiver': transport,
'replier': replier_endpoint
}
self.endpoint = self.endpoints['receiver']
else:
# existing transport instance
self.endpoints = transport
self.endpoint = transport.endpoint['receiver']
if name:
# name should be string.
err_msg = 'RPC server name ({}) shall be string.'.format(name)
assert isinstance(name, basestring), err_msg
err_msg = 'RPC server name ({}) shall not contain .'.format(name)
assert '.' not in name, err_msg
err_msg = 'RPC server name ({}) shall not contain {}'.format(
name, os.sep)
assert os.sep not in name, err_msg
logger_name = name
else:
# use port as name.
pattern = 'tcp://(?P<ip>[0-9.*]+):(?P<port>[0-9]+)'
re_groups = re.match(pattern, self.endpoint)
logger_name = re_groups.group('port')
self.logger = RPCLogger(name=logger_name,
level=log_level,
log_folder_path=log_folder_path)
# logger for registered instance, like drivers and test functions
self.service_logger = RPCLogger(logger_name + '_service',
level=log_level,
log_folder_path=log_folder_path)
self.init_server(self.endpoints, threadpool_size)
self.server_mode = 'normal'
def manager():
@dataclasses.dataclass
class Handler:
uid: int
process: subprocess.Popen
remote_server: RPCProxy
ctx = zmq.Context()
dispatcher = RPCDispatcher()
living_process = {None} # None is semaphore
@dispatcher.public
def notify(msg: t.Any):
print("*", msg, "*")
@dispatcher.public
def exit(uid: int):
print("EXIT", living_process.remove(uid))
# use tmpfs?
with tempfile.TemporaryDirectory() as d:
endpoint = f"ipc://{d}/worker.manager"
transport = ZmqServerTransport.create(ctx, endpoint)
rpc_server = RPCServer(transport, JSONRPCProtocol(), dispatcher)
def server_loop():
while len(living_process) > 0:
rpc_server.receive_one_message()
th = threading.Thread(target=server_loop, daemon=True)
th.start()
handlers = []
n = 3
protocol = JSONRPCProtocol()
for uid in range(n):
living_process.add(uid)
cmd = [
sys.executable,
"-u",
__file__,
"worker",
"--uid",
str(uid),
"--endpoint",
endpoint,
]
p = subprocess.Popen(
cmd,
# stdout=subprocess.PIPE,
# stderr=subprocess.PIPE,
# text=False, # for read1
)
rpc_client = RPCClient(
protocol,
ZmqClientTransport.create(
ctx, endpoint.replace(".manager", f".{uid}")),
)
remote_server = rpc_client.get_proxy()
handlers.append(
Handler(uid=uid, process=p, remote_server=remote_server))
# 本当はguardなどを使う
living_process.remove(None) # None is semaphore
futs = []
from concurrent.futures import ThreadPoolExecutor
with ThreadPoolExecutor() as ex:
for h in handlers:
futs.append(ex.submit(h.remote_server.do_task))
futs.append(ex.submit(h.remote_server.do_task))
for fut in futs:
print(fut.result())
for h in handlers:
h.remote_server.exit()
print("END")
th.join()
def server(*, endpoint: str):
import queue
import threading
import time
from concurrent.futures import Future
from tinyrpc.transports.zmq import ZmqServerTransport
from tinyrpc.server import RPCServer
from tinyrpc.dispatch import RPCDispatcher
q = queue.Queue()
ev = threading.Event()
ctx = zmq.Context()
dispatcher = RPCDispatcher()
transport = ZmqServerTransport.create(ctx, endpoint)
rpc_server = RPCServer(transport, JSONRPCProtocol(), dispatcher)
rpc_server.trace = print
running = True
@dispatcher.public
def act(uid: int, s: str) -> str:
q.put(("act", (uid, s), {}))
ev.set()
return "ok"
@dispatcher.public
def shutdown() -> None:
nonlocal running
running = False
def do_act(uid: int, s: str) -> Future:
fut = Future()
def do():
for i in range(5):
print(f"{uid:02d}: {i} {s}")
time.sleep(0.1)
fut.set_result(("ok", uid))
threading.Thread(target=do, daemon=True).start()
return fut
def do_server():
nonlocal running
while running:
rpc_server.receive_one_message()
def do_loop(*, ns):
nonlocal running
while running:
fn, args, kwargs = q.get()
fut = ns[f"do_{fn}"](*args, **kwargs)
def cont(fut):
q.task_done()
fut.add_done_callback(cont)
ns = locals()
th = threading.Thread(target=do_loop, kwargs={"ns": ns}, daemon=True)
th.start()
th2 = threading.Thread(target=do_server, daemon=True)
th2.start()
ev.wait()
q.join()
import json
import zmq
from tinyrpc.protocols.jsonrpc import JSONRPCProtocol
from tinyrpc.transports.zmq import ZmqServerTransport
from tinyrpc.server import RPCServer
from tinyrpc.dispatch import RPCDispatcher
ctx = zmq.Context()
dispatcher = RPCDispatcher()
transport = ZmqServerTransport.create(ctx, 'tcp://127.0.0.1:8000')
rpc_server = RPCServer(transport, JSONRPCProtocol(), dispatcher)
@dispatcher.public
def request(s):
# return s[::-1]
rq = json.loads(s)
print 'REQUEST: '
print rq['meta']
print rq['body']
if 'OFPFlowMod' in rq['body']:
print rq['body']['OFPFlowMod']
return 'PERMIT'
rpc_server.serve_forever()
def listener(socket, address):
#$print('New connection from %s:%s' % address)
rfileobj = socket.makefile(mode='rb')
session = Session(rfileobj,
False,
keep_alive_interval=100,
keep_alive_timeout=100)
while not session.closed():
stream = session.accept_stream()
# print "accepted", stream
gevent.spawn(stream_handle, stream)
print "server: active strems", session.stream_count
# session.close()
dispatch = RPCDispatcher()
@dispatch.public
def foo(name):
return "hello %s" % name
@dispatch.public
def echo(name):
return name
@dispatch.public
def bar(sec):
gevent.sleep(sec)
