def __load_config(self):
"""
This function is used only if servicename is not given, and
daemon is not started by kasaya daemon.
"""
from kasaya.conf import load_worker_settings, set_value
try:
config = load_worker_settings("service.conf")
except IOError:
import sys
LOG.critical(
"File 'service.conf' not found, unable to start service.")
sys.exit(1)
# system settings overwriting
for k, v in config['config'].items():
set_value(k, v)
# worker environment
for k, v in config['env'].items():
os.environ[k.upper()] = v
# service name
svcc = config['service']
svname = svcc['name']
LOG.info("Service config loaded. Service name: %s" % svname)
# set flag to load tasks automatically
self.__auto_load_tasks_module = svcc['module']
return svname
def worker_start_remote(self, worker_id, host_id, address, service):
"""
Remote worker started
"""
self.DB.worker_register(host_id, worker_id, service, address)
LOG.info("Remote worker [%s] started, address [%s] [id:%s]" %
(service, address, worker_id))
def on_remote_kasayad_start(self, host_id, addr):
"""
Remote kasaya host started
"""
# register self in database
self.DB.host_register(host_id, addr)
LOG.info("Remote kasaya daemon started, address: %s [id:%s]" %
(addr, host_id))
def worker_start_local(self, worker_id, address, service, pid):
"""
Local worker started
"""
self.DB.worker_register(self.DAEMON.ID, worker_id, service, address,
pid, False)
LOG.info("Local worker [%s] started, address [%s] [id:%s]" %
(service, address, worker_id))
# emit signal
emit("worker-local-wait", worker_id)
def close(self):
"""
Notifies network about shutting down, closes database
and all used sockets.
"""
LOG.info("Stopping local kasaya daemon")
self.on_local_kasayad_stop(self.ID, local=True)
self.WORKER.close()
self.DB.close()
self.BC.close()
def CTL_start(self):
"""
Set status of worker as running. This allow to process tasks
"""
if self.status == 1:
self.status = 2
LOG.info("Received status: running.")
# call tasks after worker started listening
g = gevent.Greenlet(self._worker_listening)
g.start()
return True
return False
def __init__(self):
super(KasayaDaemon, self).__init__(is_host=True)
# event handlers
add_event_handler("host-join", self.on_remote_kasayad_start)
add_event_handler("host-leave", self.on_remote_kasayad_stop)
self.hostname = system.get_hostname()
LOG.info("Starting local kasaya daemon with ID: [%s]" % self.ID)
self.DB = NetworkStateDB() # database
self.BC = UDPBroadcast(self.ID) # broadcaster
self.SYNC = Synchronizer(self.DB, self.ID) # synchronisation
self.WORKER = SyncWorker(server=self, database=self.DB)
self.BC.set_own_ip(self.WORKER.own_ip)
def notify_kasayad_refresh(self, ID, services=None, local=False):
"""
Received information on host changes
"""
if services is not None:
slst = ", ".join(services)
if local:
# local changes require broadcast new service status
self.BC.send_host_refresh(self.ID, services=services)
LOG.info("Local service list changed [%s]" % slst)
else:
# remote host services requires daabase update
# local updates are entered to database
# before notify_kasayad_refresh is called
self.DB.service_update_list(self.ID, services)
LOG.info("Remote host service list changed [%s]" % slst)
def run_task(self, funcname, args, kwargs):
# find task in worker db
try:
task = worker_methods_db[funcname]
except KeyError:
self._tasks_nonex += 1
LOG.info("Unknown worker task called [%s]" % funcname)
return exception_serialize_internal('Method %s not found' %
funcname)
# try to run function and catch exceptions
try:
LOG.debug("task %s, args %s, kwargs %s" %
(funcname, repr(args), repr(kwargs)))
func = task['func']
tout = task['timeout']
if tout is None:
# call task without timeout
result = func(*args, **kwargs)
else:
# call task with timeout
with gevent.Timeout(tout, TaskTimeout):
result = func(*args, **kwargs)
self._tasks_succes += 1
task['res_succ'] += 1
return {'message': messages.RESULT, 'result': result}
except TaskTimeout as e:
# timeout exceeded
self._tasks_error += 1
task['res_tout'] += 1
err = exception_serialize(e, internal=False)
LOG.info("Task [%s] timeout (after %i s)." % (funcname, tout))
return err
except Exception as e:
# exception occured
self._tasks_error += 1
task['res_err'] += 1
err = exception_serialize(e, internal=False)
LOG.info("Task [%s] exception [%s]. Message: %s" %
(funcname, err['name'], err['description']))
LOG.debug(err['traceback'])
return err
finally:
# close django connection
# if worker is using Django ORM we must close database connection manually,
# or each task will leave one unclosed connection. This is done automatically.
if task['close_djconn']:
try:
_close_dj_connection()
except Exception as e:
if e.__class__.__name__ == "ImproperlyConfigured":
# django connection is not required or diango orm is not used at all,
# because of that we replace _close_dj_connection function by empty lambda
global _close_dj_connection
_close_dj_connection = lambda: None
def notify_kasayad_start(self, ID, hostname, ip, services, local=False):
"""
Send information about startup of host to all other hosts in network.
"""
isnew = self.DB.host_register(ID, hostname, ip, services)
if local:
# it is ourself starting, send broadcast to other kasaya daemons
self.BC.send_host_start(ID, hostname, ip, services)
if isnew:
# new kasayad
# send request to local workers to send immadiately ping broadcast
# to inform new kasaya daemon about self
#self.WORKER.request_workers_broadcast()
# it's remote host starting, information is from broadcast
LOG.info(
"Remote kasaya daemon [%s] started, address [%s], ID [%s]" %
(hostname, ip, ID))
# if registered new kasayad AND it's not local host, then
# it must be new host in network, which don't know other hosts.
# We send again registering information about self syncd instance.
gevent.sleep(0.5)
self.notify_kasayad_self_start()
def worker_stop_remote(self, worker_id):
"""
Remote worker stopped
"""
self.DB.worker_unregister(ID=worker_id)
LOG.info("Remote worker stopped [id:%s]" % worker_id)
def worker_stop_local(self, worker_id):
"""
Local worker stopped
"""
self.DB.worker_unregister(ID=worker_id)
LOG.info("Local worker stopped [id:%s]" % worker_id)
def __init__(self,
servicename=None,
load_config=True,
skip_loading_modules=False):
super(WorkerDaemon, self).__init__()
# config loader
if servicename is None:
load_config = True
if load_config:
LOG.info("Loading service.conf")
if servicename is None:
servicename = self.__load_config()
self.servicename = servicename
self.__skip_loading_modules = skip_loading_modules
# worker status
# 0 - initialized
# 1 - starting or waiting for reconnect to kasaya
# 2 - working
# 3 - stopping
# 4 - dead
self.status = 0
LOG.info("Starting worker daemon, service [%s], ID: [%s]" %
(self.servicename, self.ID))
adr = "tcp://%s:%i" % (settings.BIND_WORKER_TO,
settings.WORKER_MIN_PORT)
self.loop = MessageLoop(adr, settings.WORKER_MAX_PORT)
add_event_handler("sender-conn-closed", self.kasaya_connection_broken)
add_event_handler("sender-conn-started",
self.kasaya_connection_started)
self.SYNC = KasayaLocalClient(autoreconnect=True, sessionid=self.ID)
self.SYNC.setup(servicename, self.loop.address, self.ID, os.getpid())
LOG.debug("Binded to socket [%s]" %
(",".join(self.loop.binded_ip_list())))
# registering handlers
self.loop.register_message(messages.SYNC_CALL,
self.handle_sync_call,
raw_msg_response=True)
self.loop.register_message(messages.CTL_CALL,
self.handle_control_request)
# heartbeat
self.__hbloop = True
#exposing methods
self.exposed_methods = []
# control tasks
self.ctl = ControlTasks()
self.ctl.register_task("stop", self.CTL_stop)
self.ctl.register_task("start", self.CTL_start)
self.ctl.register_task("stats", self.CTL_stats)
self.ctl.register_task("tasks", self.CTL_methods)
# stats
#self._sb_errors = 0 # internal service bus errors
self._tasks_succes = 0 # succesfully processed tasks
self._tasks_error = 0 # task which triggered exceptions
self._tasks_nonex = 0 # non existing tasks called
self._tasks_control = 0 # control tasks received
self._start_time = datetime.datetime.now() # time of worker start
def connection_handler(self, SOCK, address):
ssid = None
while True:
try:
msgdata, resreq = _receive_and_deserialize(
SOCK, self.serializer)
except (NoData, ConnectionClosed):
return
try:
msg = msgdata['message']
except KeyError:
if resreq:
self._send_noop(SOCK)
LOG.debug("Decoded message is incomplete. Message dump: %s" %
repr(msgdata))
continue
# message SET_SESSION_ID is special message
# it never return reply and is not propagated to handlers
if msg == messages.SET_SESSION_ID:
try:
ssid = msgdata['id']
#print("conn session id" , address, ssid)
except KeyError:
pass
if resreq:
self._send_noop(SOCK)
continue
# find message handler
try:
handler, rawmsg = self._msgdb[msg]
except KeyError:
# unknown messages are ignored
if resreq:
self._send_noop(SOCK)
LOG.warning("Unknown message received [%s]" % msg)
LOG.debug("Message body dump:\n%s" % repr(msgdata))
continue
# run handler
try:
result = handler(msgdata)
except Exception as e:
result = exception_serialize(e, False)
LOG.info(
"Exception [%s] when processing message [%s]. Message: %s."
% (result['name'], msg, result['description']))
#LOG.debug("Message dump: %s" % repr(msgdata) )
#LOG.debug(result['traceback'])
if not resreq:
# if response is not required, then don't send exceptions
continue
_serialize_and_send(
SOCK,
self.serializer,
exception_serialize(e, False),
resreq=False, # response never require another response
)
continue
# response is not expected, throw result and back to loop
if not resreq:
continue
try:
# send result
if rawmsg:
_serialize_and_send(
SOCK,
self.serializer,
result,
resreq=False,
)
else:
_serialize_and_send(SOCK,
self.serializer, {
"message": messages.RESULT,
"result": result,
},
resreq=False)
except ConnectionClosed:
return
def on_remote_kasayad_stop(self, host_id):
"""
received information about kasaya host leaving network
"""
self.DB.host_unregister(self.ID)
LOG.info("Remote kasaya daemon stopped, [id:%s]" % host_id)