def __init__(self, name, endpoint="127.0.0.1", port=10053, init_args=sys.argv, **kwargs):
"""
It:
* goes to Locator and get service api (put into self._service_api)
* initializes event loop and all asio elements (write/read streams)
* initializes session counter
* registers callback on epoll READ event and\
binds callback to decoder (_on_message)
"""
if '--locator' in init_args:
try:
port = int(init_args[init_args.index('--locator') + 1])
except ValueError as err:
port = 10053
except IndexError as err:
port = 10053
self._try_reconnect = kwargs.get("reconnect_once", True)
self._counter = 1
self.loop = ev.Loop()
self._locator_host = endpoint
self._locator_port = port
self.servicename = name
self._init_endpoint() # initialize pipe
self.decoder = Decoder()
self.decoder.bind(self._on_message)
self.w_stream = WritableStream(self.loop, self.pipe)
self.r_stream = ReadableStream(self.loop, self.pipe)
self.r_stream.bind(self.decoder.decode)
self.loop.register_read_event(self.r_stream._on_event, self.pipe.fileno())
def __init__(self, init_args=None, disown_timeout=2, heartbeat_timeout=20):
self._logger = core_log
self._init_endpoint(init_args or sys.argv)
self.sessions = dict()
self.sandbox = Sandbox()
self.loop = ev.Loop()
self.disown_timer = ev.Timer(self.on_disown, disown_timeout, self.loop)
self.heartbeat_timer = ev.Timer(self.on_heartbeat, heartbeat_timeout, self.loop)
self.heartbeat_timer.start()
if isinstance(self.endpoint, types.TupleType) or isinstance(self.endpoint, types.ListType):
if len(self.endpoint) == 2:
socket_type = socket.AF_INET
elif len(self.endpoint) == 4:
socket_type = socket.AF_INET6
else:
raise ValueError('invalid endpoint')
elif isinstance(self.endpoint, types.StringType):
socket_type = socket.AF_UNIX
else:
raise ValueError('invalid endpoint')
sock = socket.socket(socket_type)
self.pipe = Pipe(sock)
self.pipe.connect(self.endpoint, blocking=True)
self.loop.bind_on_fd(self.pipe.fileno())
self.decoder = Decoder()
self.decoder.bind(self.on_message)
self.w_stream = WritableStream(self.loop, self.pipe)
self.r_stream = ReadableStream(self.loop, self.pipe)
self.r_stream.bind(self.decoder.decode)
self.loop.register_read_event(self.r_stream._on_event,
self.pipe.fileno())
self._logger.debug("Worker with %s send handshake" % self.id)
# Send both messages - to run timers properly. This messages will be sent
# only after all initialization, so they have same purpose.
self._send_handshake()
self._send_heartbeat()
def __init__(self, name, endpoint="localhost", port=10053, init_args=sys.argv):
def closure(number):
def wrapper(*args):
def register_callback(callback, errorback=None):
self.w_stream.write([number, self._counter, args])
self._subscribers[self._counter] = (callback, errorback)
# FIX: Move counter increment for supporting stream-loke services
self._counter += 1
return register_callback
return wrapper
service_endpoint, _, service_api = self._get_api(name, endpoint, port)
self._service_api = service_api
self.servicename = name
for number, methodname in service_api.iteritems():
setattr(self, methodname, closure(number))
self.service = ev.Service()
self._counter = 1
self._subscribers = dict()
self.pipe = ServicePipe(service_endpoint)
self.decoder = Decoder()
self.decoder.bind(self._on_message)
self.service.bind_on_fd(self.pipe.fileno())
self.w_stream = WritableStream(self.service, self.pipe)
self.r_stream = ReadableStream(self.service, self.pipe)
self.r_stream.bind(self.decoder.decode)
self.service.register_read_event(self.r_stream._on_event, self.pipe.fileno())
try:
self.app_name = init_args[init_args.index("--app") + 1]
except ValueError:
self.app_name = "standalone"
class BaseService(object):
""" Implements basic functional for services:
* all asio stuff
* perform_sync method for synchronous operations
You should reimplement _on_message function - this is callback for decoder,
so this function is called with every incoming decoded message
"""
def __init__(self, name, endpoint="127.0.0.1", port=10053, init_args=sys.argv, **kwargs):
"""
It:
* goes to Locator and get service api (put into self._service_api)
* initializes event loop and all asio elements (write/read streams)
* initializes session counter
* registers callback on epoll READ event and\
binds callback to decoder (_on_message)
"""
if '--locator' in init_args:
try:
port = int(init_args[init_args.index('--locator') + 1])
except ValueError as err:
port = 10053
except IndexError as err:
port = 10053
self._try_reconnect = kwargs.get("reconnect_once", True)
self._counter = 1
self.loop = ev.Loop()
self._locator_host = endpoint
self._locator_port = port
self.servicename = name
self._init_endpoint() # initialize pipe
self.decoder = Decoder()
self.decoder.bind(self._on_message)
self.w_stream = WritableStream(self.loop, self.pipe)
self.r_stream = ReadableStream(self.loop, self.pipe)
self.r_stream.bind(self.decoder.decode)
self.loop.register_read_event(self.r_stream._on_event, self.pipe.fileno())
def _init_endpoint(self):
locator = Locator()
self.service_endpoint, _, service_api = locator.resolve(self.servicename, self._locator_host, self._locator_port)
self._service_api = service_api
# msgpack convert in list or tuple depend on version - make it tuple
self.pipe = Pipe(tuple(self.service_endpoint), self.reconnect if self._try_reconnect else None)
self.loop.bind_on_fd(self.pipe.fileno())
def reconnect(self):
self.loop.stop_listening(self.pipe.fileno())
#try:
# self.pipe.sock.close()
#except Exception as err:
# print(str(err))
try:
self._init_endpoint()
except LocatorResolveError as err:
pass
else:
self.w_stream.reconnect(self.pipe)
self.r_stream.reconnect(self.pipe)
def perform_sync(self, method, *args, **kwargs):
""" Do not use the service synchronously after treatment to him asynchronously!
Use for these purposes the other instance of the service!
"""
timeout = kwargs.get("timeout", 5)
# Get number of current method
try:
number = (_num for _num, _name in self._service_api.iteritems() if _name == method).next()
except StopIteration as err:
raise ServiceError(self.servicename, "method %s is not available" % method, -100)
try:
# DO IT SYNC
self.pipe.settimeout(timeout)
self.pipe.writeall(packb([number, self._counter, args]))
self._counter += 1
u = Unpacker()
msg = None
# If we receive rpc::error, put ServiceError here,
# and raise this error instead of StopIteration on rpc::choke,
# because after rpc::error we always receive choke.
_error = None
while True:
response = self.pipe.recv(4096)
u.feed(response)
for _data in u:
msg = Message.initialize(_data)
if msg is None:
continue
if msg.id == message.RPC_CHUNK:
yield unpackb(msg.data)
elif msg.id == message.RPC_CHOKE:
raise _error or StopIteration
elif msg.id == message.RPC_ERROR:
_error = ServiceError(self.servicename, msg.message, msg.code)
finally:
self.pipe.settimeout(0) # return to non-blocking mode
def _on_message(self, args):
raise NotImplementedError()
@property
def connected(self):
return self.pipe.connected
class Worker(object):
def __init__(self, init_args=None, disown_timeout=2, heartbeat_timeout=20):
self._logger = core_log
self._init_endpoint(init_args or sys.argv)
self.sessions = dict()
self.sandbox = Sandbox()
self.loop = ev.Loop()
self.disown_timer = ev.Timer(self.on_disown, disown_timeout, self.loop)
self.heartbeat_timer = ev.Timer(self.on_heartbeat, heartbeat_timeout, self.loop)
self.heartbeat_timer.start()
if isinstance(self.endpoint, types.TupleType) or isinstance(self.endpoint, types.ListType):
if len(self.endpoint) == 2:
socket_type = socket.AF_INET
elif len(self.endpoint) == 4:
socket_type = socket.AF_INET6
else:
raise ValueError('invalid endpoint')
elif isinstance(self.endpoint, types.StringType):
socket_type = socket.AF_UNIX
else:
raise ValueError('invalid endpoint')
sock = socket.socket(socket_type)
self.pipe = Pipe(sock)
self.pipe.connect(self.endpoint, blocking=True)
self.loop.bind_on_fd(self.pipe.fileno())
self.decoder = Decoder()
self.decoder.bind(self.on_message)
self.w_stream = WritableStream(self.loop, self.pipe)
self.r_stream = ReadableStream(self.loop, self.pipe)
self.r_stream.bind(self.decoder.decode)
self.loop.register_read_event(self.r_stream._on_event,
self.pipe.fileno())
self._logger.debug("Worker with %s send handshake" % self.id)
# Send both messages - to run timers properly. This messages will be sent
# only after all initialization, so they have same purpose.
self._send_handshake()
def _init_endpoint(self, init_args):
try:
self.id = init_args[init_args.index("--uuid") + 1]
# app_name = init_args[init_args.index("--app") + 1]
self.endpoint = init_args[init_args.index("--endpoint") + 1]
except Exception as err:
self._logger.error("Wrong cmdline arguments: %s " % err)
raise RuntimeError("Wrong cmdline arguments")
def run(self, binds=None):
if not binds:
binds = {}
for event, name in binds.iteritems():
self.on(event, name)
self._send_heartbeat()
self.loop.run()
def terminate(self, reason, msg):
self.w_stream.write(Message(message.RPC_TERMINATE, 0, reason, msg).pack())
self.loop.stop()
exit(1)
# Event machine
def on(self, event, callback):
self.sandbox.on(event, callback)
# Events
def on_heartbeat(self):
self._send_heartbeat()
def on_message(self, args):
msg = Message.initialize(args)
if msg is None:
return
elif msg.id == message.RPC_INVOKE:
request = Request()
stream = Stream(msg.session, self, msg.event)
try:
self.sandbox.invoke(msg.event, request, stream)
self.sessions[msg.session] = request
except (ImportError, SyntaxError) as err:
stream.error(2, "unrecoverable error: %s " % str(err))
self.terminate(1, "Bad code")
except Exception as err:
self._logger.error("On invoke error: %s" % err)
traceback.print_stack()
stream.error(1, "Invocation error")
elif msg.id == message.RPC_CHUNK:
self._logger.debug("Receive chunk: %d" % msg.session)
try:
_session = self.sessions[msg.session]
_session.push(msg.data)
except Exception as err:
self._logger.error("On push error: %s" % str(err))
self.terminate(1, "Push error: %s" % str(err))
return
elif msg.id == message.RPC_CHOKE:
self._logger.debug("Receive choke: %d" % msg.session)
_session = self.sessions.get(msg.session, None)
if _session is not None:
_session.close()
self.sessions.pop(msg.session)
elif msg.id == message.RPC_HEARTBEAT:
self._logger.debug("Receive heartbeat. Stop disown timer")
self.disown_timer.stop()
elif msg.id == message.RPC_TERMINATE:
self._logger.debug("Receive terminate. %s, %s" % (msg.reason, msg.message))
self.terminate(msg.reason, msg.message)
elif msg.id == message.RPC_ERROR:
_session = self.sessions.get(msg.session, None)
if _session is not None:
_session.error(RequestError(msg.message))
def on_disown(self):
try:
self._logger.error("Disowned")
finally:
self.loop.stop()
# Private:
def _send_handshake(self):
self.w_stream.write(Message(message.RPC_HANDSHAKE, 0, self.id).pack())
def _send_heartbeat(self):
self.disown_timer.start()
self._logger.debug("Send heartbeat. Start disown timer")
self.w_stream.write(Message(message.RPC_HEARTBEAT, 0).pack())
def send_choke(self, session):
self.w_stream.write(Message(message.RPC_CHOKE, session).pack())
def send_chunk(self, session, data):
self.w_stream.write(Message(message.RPC_CHUNK, session, data).pack())
def send_error(self, session, code, msg):
self.w_stream.write(Message(message.RPC_ERROR, session, code, msg).pack())
class Service(object):
def __init__(self, name, endpoint="localhost", port=10053, init_args=sys.argv):
def closure(number):
def wrapper(*args):
def register_callback(callback, errorback=None):
self.w_stream.write([number, self._counter, args])
self._subscribers[self._counter] = (callback, errorback)
# FIX: Move counter increment for supporting stream-loke services
self._counter += 1
return register_callback
return wrapper
service_endpoint, _, service_api = self._get_api(name, endpoint, port)
self._service_api = service_api
self.servicename = name
for number, methodname in service_api.iteritems():
setattr(self, methodname, closure(number))
self.service = ev.Service()
self._counter = 1
self._subscribers = dict()
self.pipe = ServicePipe(service_endpoint)
self.decoder = Decoder()
self.decoder.bind(self._on_message)
self.service.bind_on_fd(self.pipe.fileno())
self.w_stream = WritableStream(self.service, self.pipe)
self.r_stream = ReadableStream(self.service, self.pipe)
self.r_stream.bind(self.decoder.decode)
self.service.register_read_event(self.r_stream._on_event, self.pipe.fileno())
try:
self.app_name = init_args[init_args.index("--app") + 1]
except ValueError:
self.app_name = "standalone"
def _get_api(self, name, endpoint, port):
locator_pipe = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
locator_pipe.settimeout(4.0)
locator_pipe.connect((endpoint, port))
locator_pipe.send(packb([0, 1, [name]]))
u = Unpacker()
msg = None
while msg is None:
response = locator_pipe.recv(80960)
u.feed(response)
msg = Message.initialize(u.next())
locator_pipe.close()
if msg.id == message.RPC_CHUNK: #PROTOCOL_LIST.index("rpc::chunk"):
return unpackb(msg.data)
if msg.id == message.RPC_ERROR: #PROTOCOL_LIST.index("rpc::error"):
raise Exception(msg.message)
def perform_sync(self, method, *args, **kwargs):
""" Do not use the service synchronously after treatment to him asynchronously!
Use for these purposes the other instance of the service!
"""
timeout = kwargs.get("timeout", 5)
# Get number of current method
try:
number = (_num for _num, _name in self._service_api.iteritems() if _name == method).next()
except StopIteration as err:
raise ServiceError(self.servicename, "method %s is not available" % method, -100)
try:
self.pipe.settimeout(timeout) # DO IT SYNC
self.pipe.writeall(packb([number, self._counter, args]))
self._counter += 1
u = Unpacker()
msg = None
# If we receive rpc::error, put ServiceError here,
# and raise this error instead of StopIteration on rpc::choke,
# because after rpc::error we always receive choke.
_error = None
while True:
response = self.pipe.recv(4096)
u.feed(response)
for _data in u:
msg = Message.initialize(_data)
if msg is None:
continue
if msg.id == message.RPC_CHUNK: #PROTOCOL_LIST.index("rpc::chunk"):
yield unpackb(msg.data)
elif msg.id == message.RPC_CHOKE: #PROTOCOL_LIST.index("rpc::choke"):
raise _error or StopIteration
elif msg.id == message.RPC_ERROR: #PROTOCOL_LIST.index("rpc::error"):
_error = ServiceError(self.servicename, msg.message, msg.code)
finally:
self.pipe.settimeout(0) #return to non-blocking mode
def _on_message(self, args):
msg = Message.initialize(args)
if msg is None:
return
try:
if msg.id == message.RPC_CHUNK: #PROTOCOL_LIST.index("rpc::chunk"):
self._subscribers[msg.session][0](unpackb(msg.data))
elif msg.id == message.RPC_CHOKE: #PROTOCOL_LIST.index("rpc::choke"):
self._subscribers.pop(msg.session, None)
elif msg.id == message.RPC_ERROR: #PROTOCOL_LIST.index("rpc::error"):
self._subscribers[msg.session][1](ServiceError(self.servicename, msg.message, msg.code))
except Exception as err:
print "Exception in _on_message: %s" % str(err)
class Worker(object):
def __init__(self, init_args=None, disown_timeout=2, heartbeat_timeout=20):
self._logger = core_log
self._init_endpoint(init_args or sys.argv)
self.sessions = dict()
self.sandbox = Sandbox()
self.loop = ev.Loop()
self.disown_timer = ev.Timer(self.on_disown, disown_timeout, self.loop)
self.heartbeat_timer = ev.Timer(self.on_heartbeat, heartbeat_timeout, self.loop)
self.heartbeat_timer.start()
if isinstance(self.endpoint, types.TupleType) or isinstance(self.endpoint, types.ListType):
if len(self.endpoint) == 2:
socket_type = socket.AF_INET
elif len(self.endpoint) == 4:
socket_type = socket.AF_INET6
else:
raise ValueError('invalid endpoint')
elif isinstance(self.endpoint, types.StringType):
socket_type = socket.AF_UNIX
else:
raise ValueError('invalid endpoint')
sock = socket.socket(socket_type)
self.pipe = Pipe(sock)
self.pipe.connect(self.endpoint, blocking=True)
self.loop.bind_on_fd(self.pipe.fileno())
self.decoder = Decoder()
self.decoder.bind(self.on_message)
self.w_stream = WritableStream(self.loop, self.pipe)
self.r_stream = ReadableStream(self.loop, self.pipe)
self.r_stream.bind(self.decoder.decode)
self.loop.register_read_event(self.r_stream._on_event,
self.pipe.fileno())
self._logger.debug("Worker with %s send handshake" % self.id)
# Send both messages - to run timers properly. This messages will be sent
# only after all initialization, so they have same purpose.
self._send_handshake()
self._send_heartbeat()
def _init_endpoint(self, init_args):
try:
self.id = init_args[init_args.index("--uuid") + 1]
# app_name = init_args[init_args.index("--app") + 1]
self.endpoint = init_args[init_args.index("--endpoint") + 1]
except Exception as err:
self._logger.error("Wrong cmdline arguments: %s " % err)
raise RuntimeError("Wrong cmdline arguments")
def run(self, binds=None):
if not binds:
binds = {}
for event, name in binds.iteritems():
self.on(event, name)
self.loop.run()
def terminate(self, reason, msg):
self.w_stream.write(Message(message.RPC_TERMINATE, 0, reason, msg).pack())
self.loop.stop()
exit(1)
# Event machine
def on(self, event, callback):
self.sandbox.on(event, callback)
# Events
def on_heartbeat(self):
self._send_heartbeat()
def on_message(self, args):
msg = Message.initialize(args)
if msg is None:
return
elif msg.id == message.RPC_INVOKE:
request = Request()
stream = Stream(msg.session, self, msg.event)
try:
self.sandbox.invoke(msg.event, request, stream)
self.sessions[msg.session] = request
except (ImportError, SyntaxError) as err:
stream.error(2, "unrecoverable error: %s " % str(err))
self.terminate(1, "Bad code")
except Exception as err:
self._logger.error("On invoke error: %s" % err)
traceback.print_stack()
stream.error(1, "Invocation error")
elif msg.id == message.RPC_CHUNK:
self._logger.debug("Receive chunk: %d" % msg.session)
try:
_session = self.sessions[msg.session]
_session.push(msg.data)
except Exception as err:
self._logger.error("On push error: %s" % str(err))
self.terminate(1, "Push error: %s" % str(err))
return
elif msg.id == message.RPC_CHOKE:
self._logger.debug("Receive choke: %d" % msg.session)
_session = self.sessions.get(msg.session, None)
if _session is not None:
_session.close()
self.sessions.pop(msg.session)
elif msg.id == message.RPC_HEARTBEAT:
self._logger.debug("Receive heartbeat. Stop disown timer")
self.disown_timer.stop()
elif msg.id == message.RPC_TERMINATE:
self._logger.debug("Receive terminate. %s, %s" % (msg.reason, msg.message))
self.terminate(msg.reason, msg.message)
elif msg.id == message.RPC_ERROR:
_session = self.sessions.get(msg.session, None)
if _session is not None:
_session.error(RequestError(msg.message))
def on_disown(self):
try:
self._logger.error("Disowned")
finally:
self.loop.stop()
# Private:
def _send_handshake(self):
self.w_stream.write(Message(message.RPC_HANDSHAKE, 0, self.id).pack())
def _send_heartbeat(self):
self.disown_timer.start()
self._logger.debug("Send heartbeat. Start disown timer")
self.w_stream.write(Message(message.RPC_HEARTBEAT, 0).pack())
def send_choke(self, session):
self.w_stream.write(Message(message.RPC_CHOKE, session).pack())
def send_chunk(self, session, data):
self.w_stream.write(Message(message.RPC_CHUNK, session, data).pack())
def send_error(self, session, code, msg):
self.w_stream.write(Message(message.RPC_ERROR, session, code, msg).pack())
