class AbstractService(object):
"""Represents abstract cocaine service.
It provides basic service operations like getting its actual network address, determining if the service is
connecting or connected.
There is no other useful public methods, so the main aim of this class - is to provide superclass for inheriting
for actual services or service-like objects (i.e. Locator).
:ivar name: service name.
"""
def __init__(self, name):
self.name = name
self._pipe = None
self._ioLoop = None
self._writableStream = None
self._readableStream = None
self._subscribers = {}
self._session = 0
self.version = 0
self.api = {}
@property
def address(self):
"""Return actual network address (`sockaddr`) of the current service if it is connected.
Returned `sockaddr` is a tuple describing a socket address, whose format depends on the returned
family `(address, port)` 2-tuple for AF_INET, or `(address, port, flow info, scope id)` 4-tuple for AF_INET6),
and is meant to be passed to the socket.connect() method.
It the service is not connected this method returns tuple `('NOT_CONNECTED', 0)`.
"""
return self._pipe.address if self.isConnected() else ('NOT_CONNECTED', 0)
def isConnecting(self):
"""Return true if the service is in connecting state."""
return self._pipe is not None and self._pipe.isConnecting()
def isConnected(self):
"""Return true if the service is in connected state."""
return self._pipe is not None and self._pipe.isConnected()
def disconnect(self):
"""Disconnect service from its endpoint and destroys all communications between them.
.. note:: This method does nothing if the service is not connected.
"""
if not self._pipe:
return
self._pipe.close()
self._pipe = None
@strategy.coroutine
def _connectToEndpoint(self, host, port, timeout, blocking=False):
if self.isConnected():
raise IllegalStateError('service "{0}" is already connected'.format(self.name))
addressInfoList = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
if not addressInfoList:
raise ConnectionResolveError((host, port))
pipe_timeout = float(timeout) / len(addressInfoList) if timeout is not None else None
log.debug('Connecting to the service "{0}", candidates: {1}'.format(self.name, addressInfoList))
start = time()
errors = []
for family, socktype, proto, canonname, address in addressInfoList:
log.debug(' - connecting to "{0} {1}"'.format(proto, address))
sock = socket.socket(family=family, type=socktype, proto=proto)
try:
self._pipe = Pipe(sock)
yield self._pipe.connect(address, timeout=pipe_timeout, blocking=blocking)
log.debug(' - success')
except ConnectionError as err:
errors.append(err)
log.debug(' - failed - {0}'.format(err))
except Exception as err:
log.warn('Unexpected error caught while connecting to the "{0}" - {1}'.format(address, err))
else:
self._ioLoop = self._pipe._ioLoop
self._writableStream = WritableStream(self._ioLoop, self._pipe)
self._readableStream = ReadableStream(self._ioLoop, self._pipe)
self._ioLoop.bind_on_fd(self._pipe.fileno())
def decode_and_dispatch(on_event):
def dispatch(unpacker):
for chunk in unpacker:
on_event(chunk)
return dispatch
self._readableStream.bind(decode_and_dispatch(self._on_message))
return
if timeout is not None and time() - start > timeout:
raise ConnectionTimeoutError((host, port), timeout)
prefix = 'service resolving failed. Reason:'
reason = '{0} [{1}]'.format(prefix, ', '.join(str(err) for err in errors))
raise ConnectionError((host, port), reason)
def _on_message(self, args):
msg = Message.initialize(args)
if msg is None:
return
try:
if msg.id == message.RPC_CHUNK:
self._subscribers[msg.session].trigger(msgpack.unpackb(msg.data))
elif msg.id == message.RPC_CHOKE:
future = self._subscribers.pop(msg.session, None)
assert future is not None, 'one of subscribers has suddenly disappeared'
if future is not None:
future.close()
elif msg.id == message.RPC_ERROR:
self._subscribers[msg.session].error(ServiceError(self.name, msg.message, msg.code))
except Exception as err:
log.warning('"_on_message" method has caught an error - %s', err)
raise err
def _invoke(self, methodId):
def wrapper(*args, **kwargs):
future = Deferred()
timeout = kwargs.get('timeout', None)
if timeout is not None:
timeoutId = self._ioLoop.add_timeout(time() + timeout, lambda: future.error(TimeoutError(timeout)))
def timeoutRemover(func):
def wrapper(*args, **kwargs):
self._ioLoop.remove_timeout(timeoutId)
return func(*args, **kwargs)
return wrapper
future.close = timeoutRemover(future.close)
self._session += 1
self._writableStream.write([methodId, self._session, args])
self._subscribers[self._session] = future
return Chain([lambda: future], ioLoop=self._ioLoop)
return wrapper
def perform_sync(self, method, *args, **kwargs):
"""Performs synchronous method invocation via direct socket usage without the participation of the event loop.
Returns generator of chunks.
:param method: method name.
:param args: method arguments.
:param kwargs: method keyword arguments. You can specify `timeout` keyword to set socket timeout.
.. note:: Left for backward compatibility, tests and other stuff. Indiscriminate using of this method can lead
to the summoning of Satan.
.. warning:: Do not mix synchronous and asynchronous usage of service!
"""
if not self.isConnected():
raise IllegalStateError('service "{0}" is not connected'.format(self.name))
if method not in self.api:
raise ValueError('service "{0}" has no method named "{1}"'.format(self.name, method))
timeout = kwargs.get('timeout', None)
if timeout is not None and timeout <= 0:
raise ValueError('timeout must be positive number')
with scope.socket.timeout(self._pipe.sock, timeout) as sock:
self._session += 1
sock.send(msgpack.dumps([self.api[method], self._session, args]))
unpacker = msgpack.Unpacker()
error = None
while True:
data = sock.recv(4096)
unpacker.feed(data)
for chunk in unpacker:
msg = Message.initialize(chunk)
if msg is None:
continue
if msg.id == message.RPC_CHUNK:
yield msgpack.loads(msg.data)
elif msg.id == message.RPC_CHOKE:
raise error or StopIteration
elif msg.id == message.RPC_ERROR:
error = ServiceError(self.name, msg.message, msg.code)
class AbstractService(object):
"""Represents abstract cocaine service.
It provides basic service operations like getting its actual network address, determining if the service is
connecting or connected.
There is no other useful public methods, so the main aim of this class - is to provide superclass for inheriting
for actual services or service-like objects (i.e. Locator).
:ivar name: service name.
"""
def __init__(self, name):
self.name = name
self._pipe = None
self._ioLoop = None
self._writableStream = None
self._readableStream = None
self._subscribers = {}
self._session = 0
self.version = 0
self.api = {}
@property
def address(self):
"""Return actual network address (`sockaddr`) of the current service if it is connected.
Returned `sockaddr` is a tuple describing a socket address, whose format depends on the returned
family `(address, port)` 2-tuple for AF_INET, or `(address, port, flow info, scope id)` 4-tuple for AF_INET6),
and is meant to be passed to the socket.connect() method.
It the service is not connected this method returns tuple `('NOT_CONNECTED', 0)`.
"""
return self._pipe.address if self.isConnected() else ('NOT_CONNECTED',
0)
def isConnecting(self):
"""Return true if the service is in connecting state."""
return self._pipe is not None and self._pipe.isConnecting()
def isConnected(self):
"""Return true if the service is in connected state."""
return self._pipe is not None and self._pipe.isConnected()
def disconnect(self):
"""Disconnect service from its endpoint and destroys all communications between them.
.. note:: This method does nothing if the service is not connected.
"""
if not self._pipe:
return
self._pipe.close()
self._pipe = None
@strategy.coroutine
def _connectToEndpoint(self, host, port, timeout, blocking=False):
if self.isConnected():
raise IllegalStateError(
'service "{0}" is already connected'.format(self.name))
addressInfoList = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
if not addressInfoList:
raise ConnectionResolveError((host, port))
pipe_timeout = float(timeout) / len(
addressInfoList) if timeout is not None else None
log.debug('Connecting to the service "{0}", candidates: {1}'.format(
self.name, addressInfoList))
start = time()
errors = []
for family, socktype, proto, canonname, address in addressInfoList:
log.debug(' - connecting to "{0} {1}"'.format(proto, address))
sock = socket.socket(family=family, type=socktype, proto=proto)
try:
self._pipe = Pipe(sock)
yield self._pipe.connect(address,
timeout=pipe_timeout,
blocking=blocking)
log.debug(' - success')
except ConnectionError as err:
errors.append(err)
log.debug(' - failed - {0}'.format(err))
except Exception as err:
log.warn(
'Unexpected error caught while connecting to the "{0}" - {1}'
.format(address, err))
else:
self._ioLoop = self._pipe._ioLoop
self._writableStream = WritableStream(self._ioLoop, self._pipe)
self._readableStream = ReadableStream(self._ioLoop, self._pipe)
self._ioLoop.bind_on_fd(self._pipe.fileno())
def decode_and_dispatch(on_event):
def dispatch(unpacker):
for chunk in unpacker:
on_event(chunk)
return dispatch
self._readableStream.bind(decode_and_dispatch(
self._on_message))
return
if timeout is not None and time() - start > timeout:
raise ConnectionTimeoutError((host, port), timeout)
prefix = 'service resolving failed. Reason:'
reason = '{0} [{1}]'.format(prefix,
', '.join(str(err) for err in errors))
raise ConnectionError((host, port), reason)
def _on_message(self, args):
msg = Message.initialize(args)
if msg is None:
return
try:
if msg.id == message.RPC_CHUNK:
self._subscribers[msg.session].trigger(
msgpack.unpackb(msg.data))
elif msg.id == message.RPC_CHOKE:
future = self._subscribers.pop(msg.session, None)
assert future is not None, 'one of subscribers has suddenly disappeared'
if future is not None:
future.close()
elif msg.id == message.RPC_ERROR:
self._subscribers[msg.session].error(
ServiceError(self.name, msg.message, msg.code))
except Exception as err:
log.warning('"_on_message" method has caught an error - %s', err)
raise err
def _invoke(self, methodId):
def wrapper(*args, **kwargs):
future = Deferred()
timeout = kwargs.get('timeout', None)
if timeout is not None:
timeoutId = self._ioLoop.add_timeout(
time() + timeout,
lambda: future.error(TimeoutError(timeout)))
def timeoutRemover(func):
def wrapper(*args, **kwargs):
self._ioLoop.remove_timeout(timeoutId)
return func(*args, **kwargs)
return wrapper
future.close = timeoutRemover(future.close)
self._session += 1
self._writableStream.write([methodId, self._session, args])
self._subscribers[self._session] = future
return Chain([lambda: future], ioLoop=self._ioLoop)
return wrapper
def perform_sync(self, method, *args, **kwargs):
"""Performs synchronous method invocation via direct socket usage without the participation of the event loop.
Returns generator of chunks.
:param method: method name.
:param args: method arguments.
:param kwargs: method keyword arguments. You can specify `timeout` keyword to set socket timeout.
.. note:: Left for backward compatibility, tests and other stuff. Indiscriminate using of this method can lead
to the summoning of Satan.
.. warning:: Do not mix synchronous and asynchronous usage of service!
"""
if not self.isConnected():
raise IllegalStateError('service "{0}" is not connected'.format(
self.name))
if method not in self.api:
raise ValueError('service "{0}" has no method named "{1}"'.format(
self.name, method))
timeout = kwargs.get('timeout', None)
if timeout is not None and timeout <= 0:
raise ValueError('timeout must be positive number')
with scope.socket.timeout(self._pipe.sock, timeout) as sock:
self._session += 1
sock.send(msgpack.dumps([self.api[method], self._session, args]))
unpacker = msgpack.Unpacker()
error = None
while True:
data = sock.recv(4096)
unpacker.feed(data)
for chunk in unpacker:
msg = Message.initialize(chunk)
if msg is None:
continue
if msg.id == message.RPC_CHUNK:
yield msgpack.loads(msg.data)
elif msg.id == message.RPC_CHOKE:
raise error or StopIteration
elif msg.id == message.RPC_ERROR:
error = ServiceError(self.name, msg.message, msg.code)
class Worker(object):
def __init__(self, init_args=None, disown_timeout=2, heartbeat_timeout=20):
self._logger = core_log
self._init_endpoint(init_args or sys.argv)
self.sessions = dict()
self.sandbox = Sandbox()
self.loop = ev.Loop()
self.disown_timer = ev.Timer(self.on_disown, disown_timeout, self.loop)
self.heartbeat_timer = ev.Timer(self.on_heartbeat, heartbeat_timeout, self.loop)
self.heartbeat_timer.start()
if isinstance(self.endpoint, types.TupleType) or isinstance(self.endpoint, types.ListType):
if len(self.endpoint) == 2:
socket_type = socket.AF_INET
elif len(self.endpoint) == 4:
socket_type = socket.AF_INET6
else:
raise ValueError('invalid endpoint')
elif isinstance(self.endpoint, types.StringType):
socket_type = socket.AF_UNIX
else:
raise ValueError('invalid endpoint')
sock = socket.socket(socket_type)
self.pipe = Pipe(sock)
self.pipe.connect(self.endpoint, blocking=True)
self.loop.bind_on_fd(self.pipe.fileno())
self.decoder = Decoder()
self.decoder.bind(self.on_message)
self.w_stream = WritableStream(self.loop, self.pipe)
self.r_stream = ReadableStream(self.loop, self.pipe)
self.r_stream.bind(self.decoder.decode)
self.loop.register_read_event(self.r_stream._on_event,
self.pipe.fileno())
self._logger.debug("Worker with %s send handshake" % self.id)
# Send both messages - to run timers properly. This messages will be sent
# only after all initialization, so they have same purpose.
self._send_handshake()
def _init_endpoint(self, init_args):
try:
self.id = init_args[init_args.index("--uuid") + 1]
# app_name = init_args[init_args.index("--app") + 1]
self.endpoint = init_args[init_args.index("--endpoint") + 1]
except Exception as err:
self._logger.error("Wrong cmdline arguments: %s " % err)
raise RuntimeError("Wrong cmdline arguments")
def run(self, binds=None):
if not binds:
binds = {}
for event, name in binds.iteritems():
self.on(event, name)
self._send_heartbeat()
self.loop.run()
def terminate(self, reason, msg):
self.w_stream.write(Message(message.RPC_TERMINATE, 0, reason, msg).pack())
self.loop.stop()
exit(1)
# Event machine
def on(self, event, callback):
self.sandbox.on(event, callback)
# Events
def on_heartbeat(self):
self._send_heartbeat()
def on_message(self, args):
msg = Message.initialize(args)
if msg is None:
return
elif msg.id == message.RPC_INVOKE:
request = Request()
stream = Stream(msg.session, self, msg.event)
try:
self.sandbox.invoke(msg.event, request, stream)
self.sessions[msg.session] = request
except (ImportError, SyntaxError) as err:
stream.error(2, "unrecoverable error: %s " % str(err))
self.terminate(1, "Bad code")
except Exception as err:
self._logger.error("On invoke error: %s" % err)
traceback.print_stack()
stream.error(1, "Invocation error")
elif msg.id == message.RPC_CHUNK:
self._logger.debug("Receive chunk: %d" % msg.session)
try:
_session = self.sessions[msg.session]
_session.push(msg.data)
except Exception as err:
self._logger.error("On push error: %s" % str(err))
self.terminate(1, "Push error: %s" % str(err))
return
elif msg.id == message.RPC_CHOKE:
self._logger.debug("Receive choke: %d" % msg.session)
_session = self.sessions.get(msg.session, None)
if _session is not None:
_session.close()
self.sessions.pop(msg.session)
elif msg.id == message.RPC_HEARTBEAT:
self._logger.debug("Receive heartbeat. Stop disown timer")
self.disown_timer.stop()
elif msg.id == message.RPC_TERMINATE:
self._logger.debug("Receive terminate. %s, %s" % (msg.reason, msg.message))
self.terminate(msg.reason, msg.message)
elif msg.id == message.RPC_ERROR:
_session = self.sessions.get(msg.session, None)
if _session is not None:
_session.error(RequestError(msg.message))
def on_disown(self):
try:
self._logger.error("Disowned")
finally:
self.loop.stop()
# Private:
def _send_handshake(self):
self.w_stream.write(Message(message.RPC_HANDSHAKE, 0, self.id).pack())
def _send_heartbeat(self):
self.disown_timer.start()
self._logger.debug("Send heartbeat. Start disown timer")
self.w_stream.write(Message(message.RPC_HEARTBEAT, 0).pack())
def send_choke(self, session):
self.w_stream.write(Message(message.RPC_CHOKE, session).pack())
def send_chunk(self, session, data):
self.w_stream.write(Message(message.RPC_CHUNK, session, data).pack())
def send_error(self, session, code, msg):
self.w_stream.write(Message(message.RPC_ERROR, session, code, msg).pack())
class Worker(object):
def __init__(self, init_args=None, disown_timeout=2, heartbeat_timeout=20):
self._logger = core_log
self._init_endpoint(init_args or sys.argv)
self.sessions = dict()
self.sandbox = Sandbox()
self.loop = ev.Loop()
self.disown_timer = ev.Timer(self.on_disown, disown_timeout, self.loop)
self.heartbeat_timer = ev.Timer(self.on_heartbeat, heartbeat_timeout, self.loop)
self.heartbeat_timer.start()
if isinstance(self.endpoint, types.TupleType) or isinstance(self.endpoint, types.ListType):
if len(self.endpoint) == 2:
socket_type = socket.AF_INET
elif len(self.endpoint) == 4:
socket_type = socket.AF_INET6
else:
raise ValueError('invalid endpoint')
elif isinstance(self.endpoint, types.StringType):
socket_type = socket.AF_UNIX
else:
raise ValueError('invalid endpoint')
sock = socket.socket(socket_type)
self.pipe = Pipe(sock)
self.pipe.connect(self.endpoint, blocking=True)
self.loop.bind_on_fd(self.pipe.fileno())
self.decoder = Decoder()
self.decoder.bind(self.on_message)
self.w_stream = WritableStream(self.loop, self.pipe)
self.r_stream = ReadableStream(self.loop, self.pipe)
self.r_stream.bind(self.decoder.decode)
self.loop.register_read_event(self.r_stream._on_event,
self.pipe.fileno())
self._logger.debug("Worker with %s send handshake" % self.id)
# Send both messages - to run timers properly. This messages will be sent
# only after all initialization, so they have same purpose.
self._send_handshake()
self._send_heartbeat()
def _init_endpoint(self, init_args):
try:
self.id = init_args[init_args.index("--uuid") + 1]
# app_name = init_args[init_args.index("--app") + 1]
self.endpoint = init_args[init_args.index("--endpoint") + 1]
except Exception as err:
self._logger.error("Wrong cmdline arguments: %s " % err)
raise RuntimeError("Wrong cmdline arguments")
def run(self, binds=None):
if not binds:
binds = {}
for event, name in binds.iteritems():
self.on(event, name)
self.loop.run()
def terminate(self, reason, msg):
self.w_stream.write(Message(message.RPC_TERMINATE, 0, reason, msg).pack())
self.loop.stop()
exit(1)
# Event machine
def on(self, event, callback):
self.sandbox.on(event, callback)
# Events
def on_heartbeat(self):
self._send_heartbeat()
def on_message(self, args):
msg = Message.initialize(args)
if msg is None:
return
elif msg.id == message.RPC_INVOKE:
request = Request()
stream = Stream(msg.session, self, msg.event)
try:
self.sandbox.invoke(msg.event, request, stream)
self.sessions[msg.session] = request
except (ImportError, SyntaxError) as err:
stream.error(2, "unrecoverable error: %s " % str(err))
self.terminate(1, "Bad code")
except Exception as err:
self._logger.error("On invoke error: %s" % err)
traceback.print_stack()
stream.error(1, "Invocation error")
elif msg.id == message.RPC_CHUNK:
self._logger.debug("Receive chunk: %d" % msg.session)
try:
_session = self.sessions[msg.session]
_session.push(msg.data)
except Exception as err:
self._logger.error("On push error: %s" % str(err))
self.terminate(1, "Push error: %s" % str(err))
return
elif msg.id == message.RPC_CHOKE:
self._logger.debug("Receive choke: %d" % msg.session)
_session = self.sessions.get(msg.session, None)
if _session is not None:
_session.close()
self.sessions.pop(msg.session)
elif msg.id == message.RPC_HEARTBEAT:
self._logger.debug("Receive heartbeat. Stop disown timer")
self.disown_timer.stop()
elif msg.id == message.RPC_TERMINATE:
self._logger.debug("Receive terminate. %s, %s" % (msg.reason, msg.message))
self.terminate(msg.reason, msg.message)
elif msg.id == message.RPC_ERROR:
_session = self.sessions.get(msg.session, None)
if _session is not None:
_session.error(RequestError(msg.message))
def on_disown(self):
try:
self._logger.error("Disowned")
finally:
self.loop.stop()
# Private:
def _send_handshake(self):
self.w_stream.write(Message(message.RPC_HANDSHAKE, 0, self.id).pack())
def _send_heartbeat(self):
self.disown_timer.start()
self._logger.debug("Send heartbeat. Start disown timer")
self.w_stream.write(Message(message.RPC_HEARTBEAT, 0).pack())
def send_choke(self, session):
self.w_stream.write(Message(message.RPC_CHOKE, session).pack())
def send_chunk(self, session, data):
self.w_stream.write(Message(message.RPC_CHUNK, session, data).pack())
def send_error(self, session, code, msg):
self.w_stream.write(Message(message.RPC_ERROR, session, code, msg).pack())
