def __init__(self, name, endpoint="127.0.0.1", port=10053, init_args=sys.argv, **kwargs): """ It: * goes to Locator and get service api (put into self._service_api) * initializes event loop and all asio elements (write/read streams) * initializes session counter * registers callback on epoll READ event and\ binds callback to decoder (_on_message) """ if '--locator' in init_args: try: port = int(init_args[init_args.index('--locator') + 1]) except ValueError as err: port = 10053 except IndexError as err: port = 10053 self._try_reconnect = kwargs.get("reconnect_once", True) self._counter = 1 self.loop = ev.Loop() self._locator_host = endpoint self._locator_port = port self.servicename = name self._init_endpoint() # initialize pipe self.decoder = Decoder() self.decoder.bind(self._on_message) self.w_stream = WritableStream(self.loop, self.pipe) self.r_stream = ReadableStream(self.loop, self.pipe) self.r_stream.bind(self.decoder.decode) self.loop.register_read_event(self.r_stream._on_event, self.pipe.fileno())
def __init__(self, init_args=None, disown_timeout=2, heartbeat_timeout=20): self._logger = core_log self._init_endpoint(init_args or sys.argv) self.sessions = dict() self.sandbox = Sandbox() self.loop = ev.Loop() self.disown_timer = ev.Timer(self.on_disown, disown_timeout, self.loop) self.heartbeat_timer = ev.Timer(self.on_heartbeat, heartbeat_timeout, self.loop) self.heartbeat_timer.start() if isinstance(self.endpoint, types.TupleType) or isinstance(self.endpoint, types.ListType): if len(self.endpoint) == 2: socket_type = socket.AF_INET elif len(self.endpoint) == 4: socket_type = socket.AF_INET6 else: raise ValueError('invalid endpoint') elif isinstance(self.endpoint, types.StringType): socket_type = socket.AF_UNIX else: raise ValueError('invalid endpoint') sock = socket.socket(socket_type) self.pipe = Pipe(sock) self.pipe.connect(self.endpoint, blocking=True) self.loop.bind_on_fd(self.pipe.fileno()) self.decoder = Decoder() self.decoder.bind(self.on_message) self.w_stream = WritableStream(self.loop, self.pipe) self.r_stream = ReadableStream(self.loop, self.pipe) self.r_stream.bind(self.decoder.decode) self.loop.register_read_event(self.r_stream._on_event, self.pipe.fileno()) self._logger.debug("Worker with %s send handshake" % self.id) # Send both messages - to run timers properly. This messages will be sent # only after all initialization, so they have same purpose. self._send_handshake() self._send_heartbeat()
def __init__(self, name, endpoint="localhost", port=10053, init_args=sys.argv): def closure(number): def wrapper(*args): def register_callback(callback, errorback=None): self.w_stream.write([number, self._counter, args]) self._subscribers[self._counter] = (callback, errorback) # FIX: Move counter increment for supporting stream-loke services self._counter += 1 return register_callback return wrapper service_endpoint, _, service_api = self._get_api(name, endpoint, port) self._service_api = service_api self.servicename = name for number, methodname in service_api.iteritems(): setattr(self, methodname, closure(number)) self.service = ev.Service() self._counter = 1 self._subscribers = dict() self.pipe = ServicePipe(service_endpoint) self.decoder = Decoder() self.decoder.bind(self._on_message) self.service.bind_on_fd(self.pipe.fileno()) self.w_stream = WritableStream(self.service, self.pipe) self.r_stream = ReadableStream(self.service, self.pipe) self.r_stream.bind(self.decoder.decode) self.service.register_read_event(self.r_stream._on_event, self.pipe.fileno()) try: self.app_name = init_args[init_args.index("--app") + 1] except ValueError: self.app_name = "standalone"
class BaseService(object): """ Implements basic functional for services: * all asio stuff * perform_sync method for synchronous operations You should reimplement _on_message function - this is callback for decoder, so this function is called with every incoming decoded message """ def __init__(self, name, endpoint="127.0.0.1", port=10053, init_args=sys.argv, **kwargs): """ It: * goes to Locator and get service api (put into self._service_api) * initializes event loop and all asio elements (write/read streams) * initializes session counter * registers callback on epoll READ event and\ binds callback to decoder (_on_message) """ if '--locator' in init_args: try: port = int(init_args[init_args.index('--locator') + 1]) except ValueError as err: port = 10053 except IndexError as err: port = 10053 self._try_reconnect = kwargs.get("reconnect_once", True) self._counter = 1 self.loop = ev.Loop() self._locator_host = endpoint self._locator_port = port self.servicename = name self._init_endpoint() # initialize pipe self.decoder = Decoder() self.decoder.bind(self._on_message) self.w_stream = WritableStream(self.loop, self.pipe) self.r_stream = ReadableStream(self.loop, self.pipe) self.r_stream.bind(self.decoder.decode) self.loop.register_read_event(self.r_stream._on_event, self.pipe.fileno()) def _init_endpoint(self): locator = Locator() self.service_endpoint, _, service_api = locator.resolve(self.servicename, self._locator_host, self._locator_port) self._service_api = service_api # msgpack convert in list or tuple depend on version - make it tuple self.pipe = Pipe(tuple(self.service_endpoint), self.reconnect if self._try_reconnect else None) self.loop.bind_on_fd(self.pipe.fileno()) def reconnect(self): self.loop.stop_listening(self.pipe.fileno()) #try: # self.pipe.sock.close() #except Exception as err: # print(str(err)) try: self._init_endpoint() except LocatorResolveError as err: pass else: self.w_stream.reconnect(self.pipe) self.r_stream.reconnect(self.pipe) def perform_sync(self, method, *args, **kwargs): """ Do not use the service synchronously after treatment to him asynchronously! Use for these purposes the other instance of the service! """ timeout = kwargs.get("timeout", 5) # Get number of current method try: number = (_num for _num, _name in self._service_api.iteritems() if _name == method).next() except StopIteration as err: raise ServiceError(self.servicename, "method %s is not available" % method, -100) try: # DO IT SYNC self.pipe.settimeout(timeout) self.pipe.writeall(packb([number, self._counter, args])) self._counter += 1 u = Unpacker() msg = None # If we receive rpc::error, put ServiceError here, # and raise this error instead of StopIteration on rpc::choke, # because after rpc::error we always receive choke. _error = None while True: response = self.pipe.recv(4096) u.feed(response) for _data in u: msg = Message.initialize(_data) if msg is None: continue if msg.id == message.RPC_CHUNK: yield unpackb(msg.data) elif msg.id == message.RPC_CHOKE: raise _error or StopIteration elif msg.id == message.RPC_ERROR: _error = ServiceError(self.servicename, msg.message, msg.code) finally: self.pipe.settimeout(0) # return to non-blocking mode def _on_message(self, args): raise NotImplementedError() @property def connected(self): return self.pipe.connected
class Worker(object): def __init__(self, init_args=None, disown_timeout=2, heartbeat_timeout=20): self._logger = core_log self._init_endpoint(init_args or sys.argv) self.sessions = dict() self.sandbox = Sandbox() self.loop = ev.Loop() self.disown_timer = ev.Timer(self.on_disown, disown_timeout, self.loop) self.heartbeat_timer = ev.Timer(self.on_heartbeat, heartbeat_timeout, self.loop) self.heartbeat_timer.start() if isinstance(self.endpoint, types.TupleType) or isinstance(self.endpoint, types.ListType): if len(self.endpoint) == 2: socket_type = socket.AF_INET elif len(self.endpoint) == 4: socket_type = socket.AF_INET6 else: raise ValueError('invalid endpoint') elif isinstance(self.endpoint, types.StringType): socket_type = socket.AF_UNIX else: raise ValueError('invalid endpoint') sock = socket.socket(socket_type) self.pipe = Pipe(sock) self.pipe.connect(self.endpoint, blocking=True) self.loop.bind_on_fd(self.pipe.fileno()) self.decoder = Decoder() self.decoder.bind(self.on_message) self.w_stream = WritableStream(self.loop, self.pipe) self.r_stream = ReadableStream(self.loop, self.pipe) self.r_stream.bind(self.decoder.decode) self.loop.register_read_event(self.r_stream._on_event, self.pipe.fileno()) self._logger.debug("Worker with %s send handshake" % self.id) # Send both messages - to run timers properly. This messages will be sent # only after all initialization, so they have same purpose. self._send_handshake() def _init_endpoint(self, init_args): try: self.id = init_args[init_args.index("--uuid") + 1] # app_name = init_args[init_args.index("--app") + 1] self.endpoint = init_args[init_args.index("--endpoint") + 1] except Exception as err: self._logger.error("Wrong cmdline arguments: %s " % err) raise RuntimeError("Wrong cmdline arguments") def run(self, binds=None): if not binds: binds = {} for event, name in binds.iteritems(): self.on(event, name) self._send_heartbeat() self.loop.run() def terminate(self, reason, msg): self.w_stream.write(Message(message.RPC_TERMINATE, 0, reason, msg).pack()) self.loop.stop() exit(1) # Event machine def on(self, event, callback): self.sandbox.on(event, callback) # Events def on_heartbeat(self): self._send_heartbeat() def on_message(self, args): msg = Message.initialize(args) if msg is None: return elif msg.id == message.RPC_INVOKE: request = Request() stream = Stream(msg.session, self, msg.event) try: self.sandbox.invoke(msg.event, request, stream) self.sessions[msg.session] = request except (ImportError, SyntaxError) as err: stream.error(2, "unrecoverable error: %s " % str(err)) self.terminate(1, "Bad code") except Exception as err: self._logger.error("On invoke error: %s" % err) traceback.print_stack() stream.error(1, "Invocation error") elif msg.id == message.RPC_CHUNK: self._logger.debug("Receive chunk: %d" % msg.session) try: _session = self.sessions[msg.session] _session.push(msg.data) except Exception as err: self._logger.error("On push error: %s" % str(err)) self.terminate(1, "Push error: %s" % str(err)) return elif msg.id == message.RPC_CHOKE: self._logger.debug("Receive choke: %d" % msg.session) _session = self.sessions.get(msg.session, None) if _session is not None: _session.close() self.sessions.pop(msg.session) elif msg.id == message.RPC_HEARTBEAT: self._logger.debug("Receive heartbeat. Stop disown timer") self.disown_timer.stop() elif msg.id == message.RPC_TERMINATE: self._logger.debug("Receive terminate. %s, %s" % (msg.reason, msg.message)) self.terminate(msg.reason, msg.message) elif msg.id == message.RPC_ERROR: _session = self.sessions.get(msg.session, None) if _session is not None: _session.error(RequestError(msg.message)) def on_disown(self): try: self._logger.error("Disowned") finally: self.loop.stop() # Private: def _send_handshake(self): self.w_stream.write(Message(message.RPC_HANDSHAKE, 0, self.id).pack()) def _send_heartbeat(self): self.disown_timer.start() self._logger.debug("Send heartbeat. Start disown timer") self.w_stream.write(Message(message.RPC_HEARTBEAT, 0).pack()) def send_choke(self, session): self.w_stream.write(Message(message.RPC_CHOKE, session).pack()) def send_chunk(self, session, data): self.w_stream.write(Message(message.RPC_CHUNK, session, data).pack()) def send_error(self, session, code, msg): self.w_stream.write(Message(message.RPC_ERROR, session, code, msg).pack())
class Service(object): def __init__(self, name, endpoint="localhost", port=10053, init_args=sys.argv): def closure(number): def wrapper(*args): def register_callback(callback, errorback=None): self.w_stream.write([number, self._counter, args]) self._subscribers[self._counter] = (callback, errorback) # FIX: Move counter increment for supporting stream-loke services self._counter += 1 return register_callback return wrapper service_endpoint, _, service_api = self._get_api(name, endpoint, port) self._service_api = service_api self.servicename = name for number, methodname in service_api.iteritems(): setattr(self, methodname, closure(number)) self.service = ev.Service() self._counter = 1 self._subscribers = dict() self.pipe = ServicePipe(service_endpoint) self.decoder = Decoder() self.decoder.bind(self._on_message) self.service.bind_on_fd(self.pipe.fileno()) self.w_stream = WritableStream(self.service, self.pipe) self.r_stream = ReadableStream(self.service, self.pipe) self.r_stream.bind(self.decoder.decode) self.service.register_read_event(self.r_stream._on_event, self.pipe.fileno()) try: self.app_name = init_args[init_args.index("--app") + 1] except ValueError: self.app_name = "standalone" def _get_api(self, name, endpoint, port): locator_pipe = socket.socket(socket.AF_INET, socket.SOCK_STREAM) locator_pipe.settimeout(4.0) locator_pipe.connect((endpoint, port)) locator_pipe.send(packb([0, 1, [name]])) u = Unpacker() msg = None while msg is None: response = locator_pipe.recv(80960) u.feed(response) msg = Message.initialize(u.next()) locator_pipe.close() if msg.id == message.RPC_CHUNK: #PROTOCOL_LIST.index("rpc::chunk"): return unpackb(msg.data) if msg.id == message.RPC_ERROR: #PROTOCOL_LIST.index("rpc::error"): raise Exception(msg.message) def perform_sync(self, method, *args, **kwargs): """ Do not use the service synchronously after treatment to him asynchronously! Use for these purposes the other instance of the service! """ timeout = kwargs.get("timeout", 5) # Get number of current method try: number = (_num for _num, _name in self._service_api.iteritems() if _name == method).next() except StopIteration as err: raise ServiceError(self.servicename, "method %s is not available" % method, -100) try: self.pipe.settimeout(timeout) # DO IT SYNC self.pipe.writeall(packb([number, self._counter, args])) self._counter += 1 u = Unpacker() msg = None # If we receive rpc::error, put ServiceError here, # and raise this error instead of StopIteration on rpc::choke, # because after rpc::error we always receive choke. _error = None while True: response = self.pipe.recv(4096) u.feed(response) for _data in u: msg = Message.initialize(_data) if msg is None: continue if msg.id == message.RPC_CHUNK: #PROTOCOL_LIST.index("rpc::chunk"): yield unpackb(msg.data) elif msg.id == message.RPC_CHOKE: #PROTOCOL_LIST.index("rpc::choke"): raise _error or StopIteration elif msg.id == message.RPC_ERROR: #PROTOCOL_LIST.index("rpc::error"): _error = ServiceError(self.servicename, msg.message, msg.code) finally: self.pipe.settimeout(0) #return to non-blocking mode def _on_message(self, args): msg = Message.initialize(args) if msg is None: return try: if msg.id == message.RPC_CHUNK: #PROTOCOL_LIST.index("rpc::chunk"): self._subscribers[msg.session][0](unpackb(msg.data)) elif msg.id == message.RPC_CHOKE: #PROTOCOL_LIST.index("rpc::choke"): self._subscribers.pop(msg.session, None) elif msg.id == message.RPC_ERROR: #PROTOCOL_LIST.index("rpc::error"): self._subscribers[msg.session][1](ServiceError(self.servicename, msg.message, msg.code)) except Exception as err: print "Exception in _on_message: %s" % str(err)
class Worker(object): def __init__(self, init_args=None, disown_timeout=2, heartbeat_timeout=20): self._logger = core_log self._init_endpoint(init_args or sys.argv) self.sessions = dict() self.sandbox = Sandbox() self.loop = ev.Loop() self.disown_timer = ev.Timer(self.on_disown, disown_timeout, self.loop) self.heartbeat_timer = ev.Timer(self.on_heartbeat, heartbeat_timeout, self.loop) self.heartbeat_timer.start() if isinstance(self.endpoint, types.TupleType) or isinstance(self.endpoint, types.ListType): if len(self.endpoint) == 2: socket_type = socket.AF_INET elif len(self.endpoint) == 4: socket_type = socket.AF_INET6 else: raise ValueError('invalid endpoint') elif isinstance(self.endpoint, types.StringType): socket_type = socket.AF_UNIX else: raise ValueError('invalid endpoint') sock = socket.socket(socket_type) self.pipe = Pipe(sock) self.pipe.connect(self.endpoint, blocking=True) self.loop.bind_on_fd(self.pipe.fileno()) self.decoder = Decoder() self.decoder.bind(self.on_message) self.w_stream = WritableStream(self.loop, self.pipe) self.r_stream = ReadableStream(self.loop, self.pipe) self.r_stream.bind(self.decoder.decode) self.loop.register_read_event(self.r_stream._on_event, self.pipe.fileno()) self._logger.debug("Worker with %s send handshake" % self.id) # Send both messages - to run timers properly. This messages will be sent # only after all initialization, so they have same purpose. self._send_handshake() self._send_heartbeat() def _init_endpoint(self, init_args): try: self.id = init_args[init_args.index("--uuid") + 1] # app_name = init_args[init_args.index("--app") + 1] self.endpoint = init_args[init_args.index("--endpoint") + 1] except Exception as err: self._logger.error("Wrong cmdline arguments: %s " % err) raise RuntimeError("Wrong cmdline arguments") def run(self, binds=None): if not binds: binds = {} for event, name in binds.iteritems(): self.on(event, name) self.loop.run() def terminate(self, reason, msg): self.w_stream.write(Message(message.RPC_TERMINATE, 0, reason, msg).pack()) self.loop.stop() exit(1) # Event machine def on(self, event, callback): self.sandbox.on(event, callback) # Events def on_heartbeat(self): self._send_heartbeat() def on_message(self, args): msg = Message.initialize(args) if msg is None: return elif msg.id == message.RPC_INVOKE: request = Request() stream = Stream(msg.session, self, msg.event) try: self.sandbox.invoke(msg.event, request, stream) self.sessions[msg.session] = request except (ImportError, SyntaxError) as err: stream.error(2, "unrecoverable error: %s " % str(err)) self.terminate(1, "Bad code") except Exception as err: self._logger.error("On invoke error: %s" % err) traceback.print_stack() stream.error(1, "Invocation error") elif msg.id == message.RPC_CHUNK: self._logger.debug("Receive chunk: %d" % msg.session) try: _session = self.sessions[msg.session] _session.push(msg.data) except Exception as err: self._logger.error("On push error: %s" % str(err)) self.terminate(1, "Push error: %s" % str(err)) return elif msg.id == message.RPC_CHOKE: self._logger.debug("Receive choke: %d" % msg.session) _session = self.sessions.get(msg.session, None) if _session is not None: _session.close() self.sessions.pop(msg.session) elif msg.id == message.RPC_HEARTBEAT: self._logger.debug("Receive heartbeat. Stop disown timer") self.disown_timer.stop() elif msg.id == message.RPC_TERMINATE: self._logger.debug("Receive terminate. %s, %s" % (msg.reason, msg.message)) self.terminate(msg.reason, msg.message) elif msg.id == message.RPC_ERROR: _session = self.sessions.get(msg.session, None) if _session is not None: _session.error(RequestError(msg.message)) def on_disown(self): try: self._logger.error("Disowned") finally: self.loop.stop() # Private: def _send_handshake(self): self.w_stream.write(Message(message.RPC_HANDSHAKE, 0, self.id).pack()) def _send_heartbeat(self): self.disown_timer.start() self._logger.debug("Send heartbeat. Start disown timer") self.w_stream.write(Message(message.RPC_HEARTBEAT, 0).pack()) def send_choke(self, session): self.w_stream.write(Message(message.RPC_CHOKE, session).pack()) def send_chunk(self, session, data): self.w_stream.write(Message(message.RPC_CHUNK, session, data).pack()) def send_error(self, session, code, msg): self.w_stream.write(Message(message.RPC_ERROR, session, code, msg).pack())