def kasaya_connection_broken(self, addr):
"""
Should be called when connection with kasaya is broken.
"""
LOG.debug("Connection closed with %s", addr)
if self.status < 3: # is worker is already stopping?
self.status = 1 #set status as 1 - waiting for start
def handle_request(self, message):
"""
All control requests are handled here.
message - message body
islocal - if true then request is from localhost
"""
method = message['method'] #".".join(message['method'])
LOG.debug("Management call [%s]" % method)
#LOG.debug(repr(message))
try:
func = self.__ctltasks[method] # get handler for method
except KeyError:
raise exceptions.MethodNotFound("Control method %s not exists" %
method)
# fill missing parameters
if 'args' not in message:
message['args'] = []
if 'kwargs' not in message:
message['kwargs'] = {}
try:
# call internal function
result = func(*message['args'], **message['kwargs'])
except RedirectRequiredToIP as e:
# redirect to IP
addr = self.ip_to_zmq_addr(e.remote_ip)
return self.redirect(addr, message)
except RedirectRequiredToAddr as e:
# redirect to address
return self.redirect(e.remote_addr, message)
return result
def run_task(self, funcname, args, kwargs):
# find task in worker db
try:
task = worker_methods_db[funcname]
except KeyError:
self._tasks_nonex += 1
LOG.info("Unknown worker task called [%s]" % funcname)
return exception_serialize_internal('Method %s not found' %
funcname)
# try to run function and catch exceptions
try:
LOG.debug("task %s, args %s, kwargs %s" %
(funcname, repr(args), repr(kwargs)))
func = task['func']
tout = task['timeout']
if tout is None:
# call task without timeout
result = func(*args, **kwargs)
else:
# call task with timeout
with gevent.Timeout(tout, TaskTimeout):
result = func(*args, **kwargs)
self._tasks_succes += 1
task['res_succ'] += 1
return {'message': messages.RESULT, 'result': result}
except TaskTimeout as e:
# timeout exceeded
self._tasks_error += 1
task['res_tout'] += 1
err = exception_serialize(e, internal=False)
LOG.info("Task [%s] timeout (after %i s)." % (funcname, tout))
return err
except Exception as e:
# exception occured
self._tasks_error += 1
task['res_err'] += 1
err = exception_serialize(e, internal=False)
LOG.info("Task [%s] exception [%s]. Message: %s" %
(funcname, err['name'], err['description']))
LOG.debug(err['traceback'])
return err
finally:
# close django connection
# if worker is using Django ORM we must close database connection manually,
# or each task will leave one unclosed connection. This is done automatically.
if task['close_djconn']:
try:
_close_dj_connection()
except Exception as e:
if e.__class__.__name__ == "ImproperlyConfigured":
# django connection is not required or diango orm is not used at all,
# because of that we replace _close_dj_connection function by empty lambda
global _close_dj_connection
_close_dj_connection = lambda: None
def stop(self):
self.__hbloop = False
self.status = 3
LOG.debug("Sending stop notification. Address [%s]" %
self.loop.address)
self.SYNC.notify_worker_stop()
self.loop.stop()
# killing greenlets
for g in self.__greens:
g.kill(block=True)
LOG.debug("Worker [%s] stopped" % self.servicename)
def register_raw_task(self, message_type, raw_responce, func):
"""
Raw task is used internally to enhance kasaya protocol
by handling special types of messages. It's used internally
by kasaya own daemons
"""
self._raw_tasks[message_type] = {
'func': func,
'raw_resp': raw_responce
}
LOG.debug("Registered raw task handler %s -> %s" %
(message_type, func.__name__))
def worker_prepare(self, worker_id):
"""
After start, worker is in offline state.
It need to be configured and after then it can be activated to be online.
This function make all required things and when worker is online it broadcast new worker in network.
"""
wrknfo = self.DB.worker_get(worker_id)
# all configuration of worker should be there
pass
# send information to worker to start processing tasks
msg = {'message': messages.CTL_CALL, 'method': 'start'}
res = send_and_receive_response(wrknfo['addr'], msg)
LOG.debug("Local worker [%s] on [%s] is now online" %
(wrknfo['service'], wrknfo['addr']))
# broadcast new worker state
self.DB.worker_set_state(worker_id, True)
def run(self):
if not self.__skip_loading_modules:
self.__load_modules()
self.status = 1
LOG.debug("Service [%s] starting." % self.servicename)
# before run...
self._worker_just_started()
self.__greens = []
self.__greens.append(gevent.spawn(self.loop.loop))
self.__greens.append(gevent.spawn(self.heartbeat_loop))
try:
gevent.joinall(self.__greens)
finally:
self.stop()
self.close()
# just finished working
self._worker_just_stopped()
def register_task(self, name, func, timeout, anonymous, permissions,
close_dj_conn):
# task name
if name is None:
name = func.__name__
if name in self.db:
if func == self.db[name]['func']:
return
c = "Task %s is already registered" % name
LOG.critical(c)
raise Exception(c)
# timeout
if not timeout is None:
if type(timeout) != int:
raise Exception("Timeout must be integer value")
if timeout < 0:
raise Exception("Timeout cannot be negative value")
# extra params for task
doc = func.__doc__
if not doc is None:
doc = doc.strip()
taskdata = {
'func': func,
'doc': doc, # docstring
'timeout': timeout, # timeout in seconds
'anon': anonymous, # can task be executed without authorisation
'perms': permissions, # permissions required to call task
'close_djconn': close_dj_conn, # close django connection on exit
'res_succ': 0, # successful calls
'res_err': 0, # error finishing calls
'res_tout': 0, # timed out calls
}
self.db[name] = taskdata
LOG.debug("Registered task %s" % name)
def loop(self):
while self.is_running:
# receive data
msgdata, addr = self.SOCK.recvfrom(4096)
# skip own broadcast messages
if addr[0] == self.own_ip:
continue
# deserialize
try:
msgdata, repreq = self.serializer.deserialize(msgdata)
except NotOurMessage:
continue
except Exception:
LOG.warning("Message from broadcast deserialisation error")
LOG.debug(
"Broken message body dump in hex (only first 1024 bytes):\n%s"
% msgdata[:1024].encode("hex"))
continue
# own broadcast from another interface
try:
if msgdata['__sid__'] == self.ID:
continue
except KeyError:
continue
# message type
try:
msg = msgdata['message']
except KeyError:
LOG.debug("Decoded message is incomplete. Message dump: %s" %
repr(msgdata))
continue
# find handler
try:
handler = self._msgdb[msg]
except KeyError:
# unknown messages are ignored silently
LOG.warning("Unknown message received [%s]" % msg)
LOG.debug("Message body dump:\n%s" % repr(msgdata))
continue
# run handler
try:
handler(msgdata)
except Exception as e:
# log exception details
excname = e.__class__.__name__
# traceback
tback = traceback.format_exc()
try:
tback = unicode(tback, "utf-8")
except:
tback = repr(tback)
# error message
errmsg = e.message
try:
errmsg = unicode(errmsg, "utf-8")
except:
errmsg = repr(errmsg)
# log & clean
LOG.error(
"Exception [%s] when processing message [%s]. Message: %s."
% (excname, msg, errmsg))
LOG.debug("Message dump: %s" % repr(msgdata))
LOG.debug(tback)
del excname, tback, errmsg
def __init__(self, silentinit=False):
# binary header
import struct
#
# header format
# ! big-endian
# 6s - 6 characters of header prefix
# h - protocol version (=1)
# L - unsigned long - data size
# 16s - 16 bytes of initial vector (used for encryption)
# ? - boolean - compression flag
# H - unsigned short - how many bytes trim from data
# ? - boolean - response required
#
self.header = struct.Struct(b"!6s h L 16s ? H ?")
self._version = 1
# encrypted or not...
if settings.ENCRYPTION:
# encrypter, decrypter
from kasaya.core.protocol.encryption import encrypt, decrypt
self.encrypt = encrypt
self.decrypt = decrypt
self.serialize = self._encrypted_serialize
self.deserialize = self._encrypted_deserialize
self._passwd = _load_passwd()
else:
self.serialize = self._plain_serialize
self.deserialize = self._plain_deserialize
# servicebus name
import sys
py3 = sys.version_info >= (3, 0)
if py3:
busname = bytes(settings.SERVICE_BUS_NAME, "ascii")
busname += b" " * (6 - len(busname))
else:
busname = str(settings.SERVICE_BUS_NAME)
busname += b" " * (6 - len(busname))
self._busname = busname
# transport protocol
if settings.TRANSPORT_PROTOCOL == "pickle":
from kasaya.core.protocol.transport.tr_pickle import bin_2_data, data_2_bin
elif settings.TRANSPORT_PROTOCOL == "bson":
if py3:
# python 3 bson
from kasaya.core.protocol.transport.tr_bson3 import bin_2_data, data_2_bin
else:
# python 2 bson
from kasaya.core.protocol.transport.tr_bson2 import bin_2_data, data_2_bin
elif settings.TRANSPORT_PROTOCOL == "msgpack":
from kasaya.core.protocol.transport.tr_msgpack import bin_2_data, data_2_bin
else:
raise Exception("Unsupported transport protocol %s" %
settings.TRANSPORT_PROTOCOL)
self.bin_2_data = bin_2_data
self.data_2_bin = data_2_bin
if silentinit:
return
LOG.debug(
"Service bus is configured to use %s as transport protocol." %
settings.TRANSPORT_PROTOCOL)
def __init__(self,
servicename=None,
load_config=True,
skip_loading_modules=False):
super(WorkerDaemon, self).__init__()
# config loader
if servicename is None:
load_config = True
if load_config:
LOG.info("Loading service.conf")
if servicename is None:
servicename = self.__load_config()
self.servicename = servicename
self.__skip_loading_modules = skip_loading_modules
# worker status
# 0 - initialized
# 1 - starting or waiting for reconnect to kasaya
# 2 - working
# 3 - stopping
# 4 - dead
self.status = 0
LOG.info("Starting worker daemon, service [%s], ID: [%s]" %
(self.servicename, self.ID))
adr = "tcp://%s:%i" % (settings.BIND_WORKER_TO,
settings.WORKER_MIN_PORT)
self.loop = MessageLoop(adr, settings.WORKER_MAX_PORT)
add_event_handler("sender-conn-closed", self.kasaya_connection_broken)
add_event_handler("sender-conn-started",
self.kasaya_connection_started)
self.SYNC = KasayaLocalClient(autoreconnect=True, sessionid=self.ID)
self.SYNC.setup(servicename, self.loop.address, self.ID, os.getpid())
LOG.debug("Binded to socket [%s]" %
(",".join(self.loop.binded_ip_list())))
# registering handlers
self.loop.register_message(messages.SYNC_CALL,
self.handle_sync_call,
raw_msg_response=True)
self.loop.register_message(messages.CTL_CALL,
self.handle_control_request)
# heartbeat
self.__hbloop = True
#exposing methods
self.exposed_methods = []
# control tasks
self.ctl = ControlTasks()
self.ctl.register_task("stop", self.CTL_stop)
self.ctl.register_task("start", self.CTL_start)
self.ctl.register_task("stats", self.CTL_stats)
self.ctl.register_task("tasks", self.CTL_methods)
# stats
#self._sb_errors = 0 # internal service bus errors
self._tasks_succes = 0 # succesfully processed tasks
self._tasks_error = 0 # task which triggered exceptions
self._tasks_nonex = 0 # non existing tasks called
self._tasks_control = 0 # control tasks received
self._start_time = datetime.datetime.now() # time of worker start
def kasaya_connection_started(self, addr):
"""
This will be called when connection with kasaya is started
"""
LOG.debug("Connected to %s", addr)
self.SYNC.notify_worker_live(self.status)
def connection_handler(self, SOCK, address):
ssid = None
while True:
try:
msgdata, resreq = _receive_and_deserialize(
SOCK, self.serializer)
except (NoData, ConnectionClosed):
return
try:
msg = msgdata['message']
except KeyError:
if resreq:
self._send_noop(SOCK)
LOG.debug("Decoded message is incomplete. Message dump: %s" %
repr(msgdata))
continue
# message SET_SESSION_ID is special message
# it never return reply and is not propagated to handlers
if msg == messages.SET_SESSION_ID:
try:
ssid = msgdata['id']
#print("conn session id" , address, ssid)
except KeyError:
pass
if resreq:
self._send_noop(SOCK)
continue
# find message handler
try:
handler, rawmsg = self._msgdb[msg]
except KeyError:
# unknown messages are ignored
if resreq:
self._send_noop(SOCK)
LOG.warning("Unknown message received [%s]" % msg)
LOG.debug("Message body dump:\n%s" % repr(msgdata))
continue
# run handler
try:
result = handler(msgdata)
except Exception as e:
result = exception_serialize(e, False)
LOG.info(
"Exception [%s] when processing message [%s]. Message: %s."
% (result['name'], msg, result['description']))
#LOG.debug("Message dump: %s" % repr(msgdata) )
#LOG.debug(result['traceback'])
if not resreq:
# if response is not required, then don't send exceptions
continue
_serialize_and_send(
SOCK,
self.serializer,
exception_serialize(e, False),
resreq=False, # response never require another response
)
continue
# response is not expected, throw result and back to loop
if not resreq:
continue
try:
# send result
if rawmsg:
_serialize_and_send(
SOCK,
self.serializer,
result,
resreq=False,
)
else:
_serialize_and_send(SOCK,
self.serializer, {
"message": messages.RESULT,
"result": result,
},
resreq=False)
except ConnectionClosed:
return