class Module(object):
""" Kervi application class that starts a kervi application and loads all needed modules.
"""
def __init__(self, user_config=None, **kwargs):
""" Settings is a dictionary with the following content
"""
from kervi.config import load
import inspect
import sys
import os
script_path = os.path.basename(os.path.abspath(inspect.stack()[1][1]))
script_name, script_ext = os.path.splitext(script_path)
config_files = []
import getopt
opts, args = getopt.getopt(sys.argv[1:], "c", [
"config_file=", "as-service", "install-service",
"uninstall-service", "start-service", "stop-service",
"restart-service", "status-service", "detect-devices"
])
for opt, arg in opts:
if opt in ("-c", "--config_file"):
if os.path.isfile(arg):
config_files += [arg]
else:
print("Specified config file not found:", arg)
config_files += [script_name + ".config.json"]
config_files += ["kervi.config.json"]
selected_config_file = None
for config_file in config_files:
if os.path.isfile(config_file):
selected_config_file = config_file
break
if selected_config_file:
print("using config file:", selected_config_file)
#else:
# print("no config file found, revert to defaults")
self.config = load(config_file=selected_config_file,
config_user=user_config,
config_base=get_default_config())
service_commands = []
detect_devices = None
self._as_service = False
for opt, arg in opts:
if opt in ("--as-service"):
self._as_service = True
if opt in ("--detect-devices"):
detect_devices = True
if opt in ("--install-service"):
service_commands += ["install"]
if opt in ("--uninstall-service"):
service_commands += ["uninstall"]
if opt in ("--start-service"):
service_commands += ["start"]
if opt in ("--stop-service"):
service_commands = ["stop"]
if opt in ("--restart-service"):
service_commands = ["restart"]
if opt in ("--service-status"):
service_commands = ["status"]
if service_commands:
import kervi.hal as hal
hal_driver = hal._load(self.config.platform.driver)
if hal_driver:
hal.service_commands(service_commands,
self.config.application.name,
self.config.application.id, script_path)
exit()
if detect_devices:
import kervi.hal as hal
hal_driver = hal._load(self.config.platform.driver)
if hal_driver:
devices = hal.detect_devices()
print("devices:")
_pretty_print(devices)
exit()
self._log_handler = KerviLogHandler(self.config)
self._log_queue = self._log_handler._log_queue
self._logger = KerviLog("module")
self._logger.info("Starting kervi module, please wait")
try:
from kervi.version import VERSION
except:
VERSION = "0.0.0"
self._logger.verbose("kervi version: %s", VERSION)
self.started = False
if self.config.module.app_connection_local and not self.config.network.ipc_root_address:
self._logger.verbose("Locating kervi application with id {0}...",
self.config.application.id)
from kervi.utility.discovery import find_kervi_app
address, port = find_kervi_app(self.config.application.id)
if address:
self.config.network.ipc_root_address = address
self.config.network.ipc_root_port = port
else:
self._logger.error("Locate kervi application failed")
exit()
self._root_address = "tcp://" + self.config.network.ipc_root_address + ":" + str(
self.config.network.ipc_root_port)
from kervi.plugin.message_bus.bus_manager import BusManager
self.bus_manager = BusManager()
#self.config.network.ipc_root_port = nethelper.get_free_port([self.config.network.ipc_root_port])
self._logger.verbose("ip: {0}", self.config.network.ip)
if self.config.module.app_connection_local:
self.bus_manager.load(
"kervi-module", self.config.network.ipc_module_port,
"tcp://" + self.config.network.ipc_root_address + ":" +
str(self.config.network.ipc_root_port), self.config.network.ip)
else:
self.bus_manager.load("kervi-main",
self.config.network.ipc_root_port, None,
self.config.network.ipc_root_address)
from kervi import spine
self.spine = spine.Spine()
self.spine.register_event_handler("processReady",
self._process_ready,
scope="app-" +
self.config.application.id)
self.spine = spine.Spine()
self._module_processes = []
self._process_info = []
self._process_info_lock = threading.Lock()
import kervi.hal as hal
hal_driver = hal._load()
if hal_driver:
self._logger.verbose("Using HAL driver: %s", hal_driver)
self._actions = _ModuleActions(self)
@property
def actions(self):
return self._actions
def _validateSettings(self):
pass
def get_process_info(self):
return {"id": "application"}
def _process_ready(self, scope, process_id, pid):
self._process_info_lock.acquire()
try:
for process in self._process_info:
if process["id"] == process_id:
process["ready"] = True
process["pid"] = pid
finally:
self._process_info_lock.release()
def _is_ready(self):
result = True
self._process_info_lock.acquire()
try:
for process in self._process_info:
if not process["ready"]:
result = False
finally:
self._process_info_lock.release()
return result
def _start(self):
self.started = True
if self._as_service:
import signal
signal.signal(signal.SIGINT, handler_stop_signals)
signal.signal(signal.SIGTERM, handler_stop_signals)
try:
import dashboards
except ImportError:
pass
#self.spine.run()
self.spine.send_command("startThreads", local_only=True)
time.sleep(.5)
module_port = self.config.network.ipc_module_port
pluginManager = PluginManager(self.config)
self._process_info_lock.acquire()
#self._process_info = [{"id":"IPC", "ready":False}]
plugin_modules = pluginManager.prepare_load()
for plugin_module in plugin_modules:
self._process_info.append({
"id": plugin_module,
"ready": False,
"pid": None
})
self._process_info_lock.release()
module_port = self.config.network.ipc_module_port
module_port = pluginManager.load_plugins(module_port + 1)
for module in self.config.modules:
module_port += 1
self._process_info_lock.acquire()
self._process_info += [{"id": module, "ready": False, "pid": None}]
self._process_info_lock.release()
self._module_processes += [
process._start_process("module-" + self.config.module.ip,
module,
self.config,
nethelper.get_free_port([module_port]),
app_helpers._KerviModuleLoader,
process_id=self.config.module.id + "-" +
module)
]
while not self._is_ready():
time.sleep(1)
self._logger.info("module connected to application at: %s",
self._root_address)
self._logger.info("Press ctrl + c to stop your module")
self.spine.trigger_event("moduleReady", self.config.module.id)
self.spine.send_command("kervi_action_module_main")
def _input_thread(self, list):
try:
raw_input()
list.append(None)
except KeyboardInterrupt:
pass
except:
pass
def run(self):
"""Run the application loop. The application will continue until termination with ctrl-c"""
if not self.started:
self._start()
try:
char_list = []
thread.start_new_thread(self._input_thread, (char_list, ))
while not char_list and _app_running:
self.spine.trigger_event("modulePing", self.config.module.id)
time.sleep(1)
except KeyboardInterrupt:
pass
self.stop()
def _xrun(self):
"""Used in tests"""
if not self.started:
self._start()
def stop(self, force_exit=True, restart=False):
self.spine.send_command("kervi_action_module_exit")
self.spine.trigger_event("moduleStopped", self.config.module.id)
self._logger.warn("stopping processes")
process._stop_processes("module-" + self.config.module.id)
self.spine.trigger_event("processTerminating", None, local_only=True)
time.sleep(1)
spine.Spine().stop()
#for thread in threading.enumerate():
# print("running thread",thread.name)
self._logger.info("module stopped")
if force_exit:
import os
os._exit(0)
def _stop_action(self):
self._logger.verbose("module action: stop")
self.stop()
def _restart_action(self):
self._logger.verbose("module action: restart")
self.stop(restart=True)
def _reboot_action(self):
self._logger.verbose("module action: reboot")
import kervi.hal as hal
hal.device_reboot()
def _shutdown_action(self):
self._logger.verbose("module action: shutdown")
import kervi.hal as hal
hal.device_shutdown()
class Application(object):
""" Kervi application class that starts a kervi application and loads all needed modules.
This class should be used by it self like:
"""
def __init__(self, user_config=None):
""" Settings is a dictionary with the following content
"""
global _console_log
print("\033[92mStarting kervi application \033[0m")
import inspect
import getopt
self.char_list = []
self._websocket_event = threading.Event()
self._in_stop = False
self._discovery_thread = None
config_files = []
self._webserver_info = None
opts, args = getopt.getopt(sys.argv[1:], "c", [
"config_file=", "as-service", "install-service",
"uninstall-service", "start-service", "stop-service",
"restart-service", "status-service", "detect-devices"
])
for opt, arg in opts:
if opt in ("-c", "--config_file"):
if os.path.isfile(arg):
config_files += [arg]
else:
print("Specified config file not found:", arg)
script_path = os.path.abspath(inspect.stack()[1][1])
script_name = os.path.basename(script_path)
script_name, script_ext = os.path.splitext(script_name)
config_files += [script_name + ".config.json"]
config_files += ["kervi.config.json"]
selected_config_file = None
for config_file in config_files:
if os.path.isfile(config_file):
selected_config_file = config_file
break
from kervi.config import load
self.config = load(config_file=selected_config_file,
config_user=user_config,
config_base=get_default_config())
service_commands = []
detect_devices = None
self._as_service = False
for opt, arg in opts:
if opt in ("--as-service"):
self._as_service = True
if opt in ("--detect-devices"):
detect_devices = True
if opt in ("--install-service"):
service_commands += ["install"]
if opt in ("--uninstall-service"):
service_commands += ["uninstall"]
if opt in ("--start-service"):
service_commands += ["start"]
if opt in ("--stop-service"):
service_commands = ["stop"]
if opt in ("--restart-service"):
service_commands = ["restart"]
if opt in ("--service-status"):
service_commands = ["status"]
if service_commands:
import kervi.hal as hal
hal_driver = hal._load(self.config.platform.driver)
if hal_driver:
hal.service_commands(service_commands,
self.config.application.name,
self.config.application.id, script_path)
exit()
if detect_devices:
import kervi.hal as hal
hal_driver = hal._load(self.config.platform.driver)
if hal_driver:
devices = hal.detect_devices()
print("devices:")
_pretty_print(devices)
exit()
self._log_handler = KerviLogHandler(self.config)
self._log_queue = self._log_handler._log_queue
self._logger = KerviLog("application")
#self._validateSettings()
self.started = False
self._webserver_port = None
try:
from kervi.version import VERSION
except:
VERSION = "0.0.0"
self._logger.verbose("kervi version: %s", VERSION)
import kervi.hal as hal
hal_driver = hal._load(self.config.platform.driver)
if hal_driver:
self._logger.verbose("platform driver: %s", hal_driver)
self._logger.verbose("board: %s", hal.get_board_name())
from kervi.plugin.message_bus.bus_manager import BusManager
self.bus_manager = BusManager(self._log_queue)
self.config.network.ipc_root_port = nethelper.get_free_port(
[self.config.network.ipc_root_port])
self.bus_manager.load("kervi-main", self.config.network.ipc_root_port,
None, self.config.network.ipc_root_address)
from kervi import spine
self.spine = spine.Spine()
self.spine.register_query_handler("GetApplicationInfo",
self._get_application_info)
self.spine.register_query_handler("getProcessInfo",
self.get_process_info)
self.spine.register_event_handler("modulePing", self._module_ping)
self.spine.register_event_handler("processReady",
self._process_ready,
scope="app-" +
self.config.application.id)
self.spine.register_event_handler("WebAppReady",
self._webapp_ready,
scope="app-" +
self.config.application.id)
self.spine.register_event_handler("websocketReady",
self._websocket_ready,
scope="app-" +
self.config.application.id)
self._module_processes = []
self._process_info = []
self._process_info_lock = threading.Lock()
self._kervi_modules = []
self._kervi_modules_lock = threading.Lock()
from kervi.storage.storage_manager import StorageManager
self._storage_manager = StorageManager(self._log_queue)
from kervi.utility.authorization_manager import AuthorizationManager
self._authorization_manager = AuthorizationManager(self._log_queue)
from kervi.messaging.message_manager import MessageManager
self._message_manager = MessageManager(self._log_queue)
self._app_actions = _AppActions(self)
@property
def actions(self):
return self._app_actions
def _validateSettings(self):
pass
def _get_application_info(self):
unit_system = self.config.unit_system
display = {}
display["unit_system"] = self.config.display.unit_systems.systems[
unit_system]
display["texts"] = self.config.texts.ui
return {
"name": self.config.application.name,
"id": self.config.application.id,
"display": display,
"unit_system": unit_system
}
def get_process_info(self):
return {"id": "application"}
def _module_ping(self, module_id):
with self._kervi_modules_lock:
for module in self._kervi_modules:
if module.module_id == module_id:
if not module.is_alive:
self._logger.verbose("module started: %s", module_id)
self.spine.trigger_event("moduleStarted", module_id)
module.ping()
break
else:
self._logger.verbose("module started: %s", module_id)
self._kervi_modules += [KerviModule(module_id)]
self.spine.trigger_event("moduleStarted", module_id)
def _check_kervi_modules(self):
with self._kervi_modules_lock:
for module in self._kervi_modules:
if not module.is_alive and module.connected:
self._logger.verbose("module stopped: %s",
module.module_id)
module.connected = False
self.spine.trigger_event("moduleStopped", module.module_id)
def _process_ready(self, scope, process_id, pid):
self._process_info_lock.acquire()
try:
for process in self._process_info:
if process["id"] == process_id:
process["ready"] = True
process["pid"] = pid
finally:
self._process_info_lock.release()
def _webapp_ready(self, scope, webserver_info):
self._webserver_info = webserver_info
ready_message = "Reach your application at http://" + self._webserver_info[
"ip"] + ":" + str(self._webserver_info["port"])
self.spine.log.info(ready_message)
self._websocket_event.wait()
self.spine.send_command("startWebSocket")
def _websocket_ready(self, scope, info):
self._websocket_event.set()
def _is_ready(self):
result = True
self._process_info_lock.acquire()
try:
for process in self._process_info:
if not process["ready"]:
result = False
finally:
self._process_info_lock.release()
return result
def _start(self):
self.started = True
try:
import dashboards
except ImportError:
pass
import kervi.core.utility.process as process
import kervi.utility.application_helpers as app_helpers
#if self._as_service:
signal.signal(signal.SIGINT, handler_stop_signals)
signal.signal(signal.SIGTERM, handler_stop_signals)
self.spine.send_command("startThreads", local_only=True)
time.sleep(.5)
module_port = self.config.network.ipc_root_port
pluginManager = PluginManager(self.config, log_queue=self._log_queue)
self._process_info_lock.acquire()
plugin_modules = pluginManager.prepare_load()
for plugin_module in plugin_modules:
self._process_info.append({
"id": plugin_module,
"ready": False,
"pid": None
})
self._process_info_lock.release()
module_port = pluginManager.load_plugins(module_port + 1)
for module in self.config.modules:
self._process_info_lock.acquire()
self._process_info += [{"id": module, "ready": False, "pid": None}]
self._process_info_lock.release()
module_port += 1
self._module_processes += [
process._start_process("app-" + self.config.application.id,
module,
self.config,
nethelper.get_free_port([module_port]),
app_helpers._KerviModuleLoader,
log_queue=self._log_queue)
]
#print("wait for ready")
try:
while not self.char_list and not self._is_ready():
#print(self.char_list)
time.sleep(1)
except KeyboardInterrupt:
pass
#print("pi", self._process_info)
if not self._in_stop:
from kervi.dashboards import Dashboard
Dashboard._add_default()
self.spine.send_command("kervi_action_app_main")
ready_message = "Your Kervi application is running"
self._logger.info(ready_message)
self._logger.info("Press ctrl + c to stop your application")
self.spine.trigger_event("appReady", self.config.application.id)
if self.config.discovery.enabled:
self._discovery_thread = KerviAppDiscovery(
self.config.network.ip,
self.config.network.ipc_root_port,
self.config.discovery.port,
self.config.application.id,
self.config.discovery.challenge,
self.config.application.name,
"http://" + self.config.network.
ip # + ":" + str(self.config.network.http_port)
)
self._discovery_thread.start()
else:
self._discovery_thread = None
def _input_thread(self, list):
try:
raw_input()
list.append(None)
except KeyboardInterrupt:
pass
except:
pass
def run(self):
"""Run the application loop. The application will continue until termination with ctrl-c"""
thread.start_new_thread(self._input_thread, (self.char_list, ))
if not self.started:
self._start()
try:
while not self.char_list and _app_running:
self._check_kervi_modules()
self.spine.trigger_event("appPing", self.config.application.id,
{"ts": int(time.time())})
time.sleep(1)
except KeyboardInterrupt:
pass
self.stop()
def _xrun(self):
"""Used in tests"""
if not self.started:
self._start()
def stop(self, force_exit=True, restart=False):
self._logger.warn("stopping processes")
self._in_stop = True
self.spine.send_command("kervi_action_app_exit")
if self._discovery_thread:
self._discovery_thread.terminate()
import kervi.core.utility.process as process
process._stop_processes("app-" + self.config.application.id)
self.spine.trigger_event("processTerminating", None, local_only=True)
time.sleep(1)
self.bus_manager.stop()
self._log_handler.stop()
if self.config.development.debug_threads:
time.sleep(1)
for thread in threading.enumerate():
print("running thread", thread.name)
self._logger.info("application stopped")
import psutil
current_process = psutil.Process()
children = current_process.children(recursive=True)
for child in children:
child.kill()
if force_exit:
import os
os._exit(0)
def _stop_action(self):
self._logger.verbose("stop action", "")
self.stop()
def _restart_action(self):
self._logger.verbose("restart action", "")
self.stop(restart=True)
def _reboot_action(self):
self._logger.verbose("reboot action", "")
import kervi.hal as hal
hal.device_reboot()
def _shutdown_action(self):
self._logger.verbose("shutdown action", "")
import kervi.hal as hal
hal.device_shutdown()
class ZMQBus():
_context = None
_handlers = NamedLists()
_connections = []
_event_sock = None
_command_sock = None
_query_sock = None
log = None
def __init__(self):
pass
def set_log(self, logName):
self.log = KerviLog(logName)
@property
def root_gone(self):
return not self._is_root and time.time() - self._last_ping > 10
def reset_bus(self, process_id, signal_port, ip=None, root_address=None, event_port=None):
self._handlers = NamedLists()
self._process_id = process_id
self._query_id_count = 0
self._uuid_handler = uuid.uuid4().hex
self._is_root = (root_address is None)
self._root_address = root_address
self._signal_address = "tcp://"+ ip +":" + str(signal_port)
self._context = zmq.Context()
self._response_events = []
self._last_ping = time.time()
self._connections_lock = threading.Lock()
self._linked_handlers = []
self._linked_response_handlers = []
self._message_threads = []
self._message_thread = 0
self._message_threads_lock = threading.Lock()
for i in range(5):
self._message_threads += [ZMQHandlerThread(self)]
self._root_event = None
self._event_socket = self._context.socket(zmq.PUB)
self._event_socket.bind(_KERVI_EVENT_ADDRESS)
self._stream_socket = self._context.socket(zmq.PUB)
self._stream_socket.bind(_KERVI_STREAM_ADDRESS)
self._command_socket = self._context.socket(zmq.PUB)
self._command_socket.bind(_KERVI_COMMAND_ADDRESS)
self._query_socket = self._context.socket(zmq.PUB)
self._query_socket.bind(_KERVI_QUERY_ADDRESS)
self._message_handler = ZMQMessageThread(self, self._signal_address, True)
self._query_handler = ZMQMessageThread(self, _KERVI_QUERY_ADDRESS)
self._event_handler = ZMQMessageThread(self, _KERVI_EVENT_ADDRESS)
self._stream_handler = ZMQMessageThread(self, _KERVI_STREAM_ADDRESS)
self._command_handler = ZMQMessageThread(self, _KERVI_COMMAND_ADDRESS)
self._register_handler("signal:ping", self._on_ping)
self._message_handler.register("signal:ping")
self._message_handler.register("signal:exit")
self._message_handler.register("queryResponse")
self._message_handler.register("query:")
self._query_handler.register("query:")
self._message_handler.register("signal:exit")
self._query_handler.register("signal:exit")
self._event_handler.register("signal:exit")
self._stream_handler.register("signal:exit")
self._command_handler.register("signal:exit")
self._ping_thread = ZMQPingThread(self)
self._command_lock = threading.Lock()
self._event_lock = threading.Lock()
self._stream_lock = threading.Lock()
self._query_lock = threading.Lock()
self._observed_streams = []
self.register_query_handler("GetRoutingInfo", self._get_routing_info)
def _get_routing_info(self):
result = []
ignore_topics = [
"GetRoutingInfo",
"ping",
"terminateProcess",
"getProcessInfo",
"processReady",
"appReady",
"authorizationAllowAnonymousUser",
"authorizationValidSessionHeader",
"authorizationRemoveSession",
"startWebSocket",
"stopThreads"
]
for tag in self._handlers.get_list_names():
tag_info = tag.split(":")
tag_id = None
if len(tag_info) == 3:
tag_id = tag_info[2]
if not tag_info[1] in ignore_topics:
result.append({
"id": tag_id,
"direction": "in",
"topic_type": tag_info[0],
"topic": tag_info[1]
})
return result
def _add_linked_handler(self, func, **kwargs):
groups = kwargs.get("groups", None)
scopes = kwargs.get("scopes", [])
argspec = inspect.getargspec(func)
self._linked_handlers.append((func, groups, scopes, argspec.keywords != None))
def _add_linked_response_handler(self, handler):
self._linked_response_handlers.append(handler)
def _register_handler(self, tag, func, **kwargs):
groups = kwargs.get("groups", None)
scopes = kwargs.get("scopes", [])
argspec = inspect.getargspec(func)
self._handlers.add(tag, (func, groups, scopes, argspec.keywords != None))
def _unregister_handler(self, tag, func, **kwargs):
groups = kwargs.get("groups", None)
scopes = kwargs.get("scopes", [])
argspec = inspect.getargspec(func)
self._handlers.remove(tag, (func, groups, scopes, argspec.keywords != None))
def get_handler_info(self):
return self._handlers.get_list_names()
def get_connection_info(self):
result = []
for connection in self._connections:
result += [{"process": connection.process_id, "address": connection.address}]
return result
def _add_message(self, tag, message, stream_data):
with self._message_threads_lock:
if tag.startswith("query:"):
query = ZMQQueryThread(self, tag, message)
query.start()
else:
self._message_threads[self._message_thread].add_message(tag, message, stream_data)
self._message_thread += 1
if self._message_thread >= len(self._message_threads):
self._message_thread = 0
def _handle_message(self, tag, message, stream_data=None):
func_list = []
func_list += self._linked_handlers
functions = self._handlers.get_list_data(tag)
if functions:
func_list += functions
if tag.startswith("event:"):
tag_parts = tag.split(":")
event = "event:" + tag_parts[1] +":"
if event!=tag:
functions = self._handlers.get_list_data(event)
if functions:
func_list += functions
stream_event = None
if tag.startswith("stream:"):
tag_parts = tag.split(":")
event = "stream:" + tag_parts[1] +":"
if len(tag_parts)>2:
stream_event = tag_parts[2]
try:
self._observed_streams.index(tag)
except ValueError:
#print("s", tag, event, stream_event, self._process_id)
self._observed_streams.append(tag)
if event!=tag:
functions = self._handlers.get_list_data(event)
if functions:
func_list += functions
result = []
session = None
session_groups = None
message_scopes = None
process_id = None
if "process_id" in message:
process_id = message["process_id"]
if "session" in message:
session = message["session"]
if session:
session_groups = session['groups']
else:
session_groups = None
if "scope" in message:
message_scopes = message["scope"]
if "injected" in message:
injected = message["injected"]
else:
injected = None
message_args = []
message_kwargs = dict()
if "kwargs" in message:
message_kwargs = message["kwargs"]
if "id" in message:
if tag.startswith("query:"):
message_kwargs["query_id"] = message["id"]
else:
message_args += [message["id"]]
if stream_event:
message_args += [stream_event]
if stream_data:
message_args += [stream_data]
response_address = None
if "responseAddress" in message:
response_address = message["responseAddress"]
if "address" in message:
message_args += [message["address"]]
if "processId" in message:
message_args += [message["processId"]]
if "processList" in message:
message_args += [message["processList"]]
if "args" in message:
message_args += message["args"]
message_kwargs = dict(message_kwargs, injected=injected, session=session, topic_tag=tag, response_address=response_address, process_id=process_id)
send_response = True
try:
if func_list:
for func, groups, handler_scopes, has_keywords in func_list:
authorized = True
if session_groups != None and groups and len(groups) > 0:
for group in groups:
if group in session_groups:
break
else:
authorized = False
if message_scopes:
for message_scope in message_scopes:
if message_scope in handler_scopes:
break
else:
authorized = False
if authorized:
if not has_keywords:
sub_result = func(*message_args)
if sub_result and sub_result == "****no_response****":
send_response = False
if sub_result:
result += [sub_result]
else:
sub_result = func(*message_args, **message_kwargs)
if sub_result and sub_result == "****no_response****":
send_response = False
elif sub_result:
result += [sub_result]
if len(result) == 1:
result = result[0]
except Exception as e:
raise e
if response_address and send_response:
self.send_query_response(response_address, message["id"], result)
return result
def send_query_response(self, response_address, query_id, result):
message = {"messageType":"queryResponse", "address": self._signal_address, "id":query_id, "response":result}
if response_address == "inproc_query":
self.resolve_response(message)
else:
self.send_connection_message(response_address, "queryResponse", message)
def run(self):
for message_thread in self._message_threads:
message_thread.start()
self._message_handler.connect()
self._message_handler.start()
#time.sleep(1)
self._event_handler.connect()
self._stream_handler.connect()
self._command_handler.connect()
self._query_handler.connect()
self._query_handler.start()
self._event_handler.start()
self._stream_handler.start()
self._command_handler.start()
if self._root_address:
self.connect_to_root()
self._ping_thread.start()
if self._is_root:
logger = logging.getLogger()
self.log.verbose("IPC address: %s", self._signal_address)
def stop(self):
exit_tag = "signal:exit"
package = [exit_tag.encode(), json.dumps({}, ensure_ascii=False, cls=_ObjectEncoder).encode('utf8')]
for connection in self._connections:
connection.send_package(package)
self._command_lock.acquire()
try:
self._command_socket.send_multipart(package)
finally:
self._command_lock.release()
self._query_lock.acquire()
try:
self._query_socket.send_multipart(package)
finally:
self._query_lock.release()
self._event_lock.acquire()
try:
self._event_socket.send_multipart(package)
finally:
self._event_lock.release()
self._stream_lock.acquire()
try:
self._stream_socket.send_multipart(package)
finally:
self._stream_lock.release()
time.sleep(1)
self._ping_thread.stop()
self._ping_thread.join()
self._message_handler.stop()
self._event_handler.stop()
self._stream_handler.stop()
self._command_handler.stop()
self._query_handler.stop()
for connection in self._connections:
connection.disconnect()
for message_thread in self._message_threads:
message_thread.stop()
def connect_to_root(self):
self._root_event = threading.Event()
connection = ProcessConnection(self, True)
connection.process_id = "kervi-main"
self._connections_lock.acquire()
self._connections += [connection]
connection.connect(self._root_address)
self._connections_lock.release()
def wait_for_root(self, timeout=20):
if self._root_event:
if self._root_event.wait(timeout):
pass
else:
self.log.error("root not found! Process= %s address= %s", self._process_id, self._root_address)
self._root_event = None
@property
def is_connected(self):
result = True
self._connections_lock.acquire()
try:
if not self._is_root and not self._connections:
result = False
if result:
for connection in self._connections:
if not connection.is_connected:
result = False
finally:
self._connections_lock.release()
return result
def send_connection_message(self, address, tag, message):
for connection in self._connections:
if connection.address == address:
p = json.dumps(message, ensure_ascii=False, cls=_ObjectEncoder).encode('utf8')
connection.send_package([tag.encode(), p])
return
self.log.warn("connection not found %s %s %s", address, tag, message)
def _on_ping(self, address, process_id, process_list):
#print("p", address, process_id)
self._connections_lock.acquire()
new_connection = True
connection_list = []
try:
is_connected = False
for process in process_list:
if process["processId"] == self._process_id:
is_connected = True
break
for connection in self._connections:
if connection.process_id == process_id:
if not connection.is_connected and is_connected:
connection.is_connected = True
connection.include_ping = True
if connection.is_root_connection and self._root_event:
self._root_event.set()
if connection.is_connected:
connection.ping()
new_connection = False
if new_connection:
connection = ProcessConnection(self)
connection.include_ping = True
self._connections += [connection]
connection.register(address, process_id)
if not self._is_root and address == self._root_address:
self._last_ping = time.time()
for process in process_list:
if process["processId"] == self._process_id:
break
for connection in self._connections:
if connection.process_id == process["processId"]:
break
else:
connection = ProcessConnection(self)
self._connections += [connection]
connection.register(process["address"], process["processId"])
finally:
self._connections_lock.release()
def _ping_connections(self):
self._connections_lock.acquire()
try:
connection_list = []
for connection in self._connections:
if connection.include_ping:
connection_list += [{"address":connection.address, "processId":connection.process_id}]
ping_message = {
'address':self._signal_address,
'processId':self._process_id,
'processList': connection_list
}
p = json.dumps(ping_message, ensure_ascii=False, cls=_ObjectEncoder).encode('utf8')
ping_tag = "signal:ping"
package = [ping_tag.encode(), p]
for connection in self._connections:
connection.send_package(package)
finally:
self._connections_lock.release()
def send_command(self, command, *args, **kwargs):
injected = kwargs.pop("injected", "")
scope = kwargs.pop("scope", None)
groups = kwargs.pop("groups", None)
session = kwargs.pop("session", None)
local_only = kwargs.pop("local_only", False)
command_message = {
"command":command,
"args":args,
"injected":injected,
"scope":scope,
"session":session,
"groups": groups,
"kwargs": kwargs
}
p = json.dumps(command_message, ensure_ascii=False, cls=_ObjectEncoder).encode('utf8')
command_tag = "command:" + command
package = [command_tag.encode(), p]
self._command_lock.acquire()
try:
self._command_socket.send_multipart(package)
finally:
self._command_lock.release()
if not local_only:
for connection in self._connections:
connection.send_package(package)
def register_command_handler(self, command, func, **kwargs):
tag = "command:"+command
self._register_handler(tag, func, **kwargs)
self._command_handler.register(tag)
self._message_handler.register(tag)
def unregister_command_handler(self, command, func, **kwargs):
tag = "command:"+command
self._unregister_handler(tag, func, **kwargs)
self._command_handler.unregister(tag)
self._message_handler.unregister(tag)
def trigger_event(self, event, id, *args, **kwargs):
injected = kwargs.pop("injected", "")
scope = kwargs.pop("scope", None)
groups = kwargs.pop("groups", None)
session = kwargs.pop("session", None)
local_only = kwargs.pop("local_only", False)
event_message = {
'event':event,
'id':id,
'args':args,
"injected":injected,
"scope":scope,
"groups":groups,
"session":session,
"kwargs": kwargs,
"process_id": self._process_id
}
p = json.dumps(event_message, ensure_ascii=False, cls=_ObjectEncoder).encode('utf8')
event_tag = "event:" + event + ":"
if id:
event_tag += id
package = [event_tag.encode(), p]
self._event_lock.acquire()
try:
self._event_socket.send_multipart(package)
finally:
self._event_lock.release()
if not local_only:
for connection in self._connections:
connection.send_package(package)
def register_event_handler(self, event, func, component_id=None, **kwargs):
tag = "event:"+event +":"
if component_id:
tag += component_id
if func:
self._register_handler(tag, func, **kwargs)
self._event_handler.register(tag)
self._message_handler.register(tag)
def unregister_event_handler(self, event, func, component_id=None, **kwargs):
tag = "event:"+event +":"
if component_id:
tag += component_id
if func:
self._unregister_handler(tag, func, **kwargs)
self._event_handler.unregister(tag)
self._message_handler.unregister(tag)
def stream_data(self, stream_id, stream_event, data, *args, **kwargs):
injected = kwargs.pop("injected", "")
scope = kwargs.pop("scope", None)
groups = kwargs.pop("groups", None)
session = kwargs.pop("session", None)
local_only = kwargs.pop("local_only", False)
event_message = {
'event':stream_event,
'id':stream_id,
#'data': data,
'args':args,
"injected":injected,
"scope":scope,
"groups":groups,
"session":session,
"kwargs": kwargs,
"process_id": self._process_id
}
p = json.dumps(event_message, ensure_ascii=False, cls=_ObjectEncoder).encode('utf8')
event_tag = "stream:" + stream_id + ":" + stream_event + ":"
package = [event_tag.encode(), p, data]
self._stream_lock.acquire()
try:
self._stream_socket.send_multipart(package)
finally:
self._stream_lock.release()
if not local_only:
for connection in self._connections:
connection.send_package(package)
def register_stream_handler(self, stream_id, func, stream_event=None, **kwargs):
tag = "stream:" + stream_id + ":"
if stream_event:
tag += stream_event + ":"
if func:
self._register_handler(tag, func, **kwargs)
#print("rsh", self._process_id, tag, func)
self._stream_handler.register(tag)
self._message_handler.register(tag)
def unregister_stream_handler(self, stream_id, func, stream_event=None, **kwargs):
tag = "stream:" + stream_id + ":"
if stream_event:
tag += stream_event + ":"
if func:
self._unregister_handler(tag, func, **kwargs)
self._stream_handler.unregister(tag)
self._message_handler.unregister(tag)
def resolve_response(self, message):
for event in self._response_events:
if event["id"] == message["id"] and not event["processed"]:
if message["response"]:
event["response"] += [message["response"]]
event["process_count"] = event["process_count"] - 1
event["handled_by"] += [message["address"]]
if event["process_count"] <= 0:
event["processed"] = True
event["eventSignal"].set()
for handler in self._linked_response_handlers:
handler(event)
def send_query(self, query, *args, **kwargs):
self._connections_lock.acquire()
try:
self._query_id_count += 1
result = []
injected = kwargs.pop("injected", "")
scope = kwargs.pop("scope", None)
groups = kwargs.pop("groups", None)
session = kwargs.pop("session", None)
processes = kwargs.pop("processes", None)
timeout = kwargs.pop("timeout", 10)
wait = kwargs.pop("wait", True)
headers = kwargs.pop("headers", None)
local_only = kwargs.pop("local_only", False)
query_id = self._uuid_handler + "-" + str(self._query_id_count)
process_count = 1
if not local_only:
for connection in self._connections:
if connection.is_alive and (not processes or (connection.process_id in processes)):
process_count += 1
event = threading.Event()
event_data = {
"id":query_id,
"eventSignal":event,
"response":[],
"processed":False,
"process_count": process_count,
"process_id": self._process_id,
"query": query,
"handled_by": [],
"headers": headers
}
self._response_events += [event_data]
query_message = {
'query':query,
"id":query_id,
"responseAddress":"inproc_query",
'args':args,
"injected":injected,
"scope":scope,
"groups":groups,
"session":session,
"kwargs": kwargs
}
p = json.dumps(query_message, ensure_ascii=False, cls=_ObjectEncoder).encode('utf8')
query_tag = "query:" + query
package = [query_tag.encode(), p]
self._query_lock.acquire()
try:
self._query_socket.send_multipart(package)
finally:
self._query_lock.release()
if not local_only and len(self._connections) > 0:
query_message["responseAddress"] = self._signal_address
p = json.dumps(query_message, ensure_ascii=False, cls=_ObjectEncoder).encode('utf8')
package = [query_tag.encode(), p]
for connection in self._connections:
if connection.is_alive and (not processes or (connection.process_id in processes)):
connection.send_package(package)
except Exception as ex:
self.log.exception("error send query %s", query)
finally:
self._connections_lock.release()
if wait:
event.wait(timeout)
if not event_data["processed"]:
self.log.warn("send query timeout %s %s %s %s", self._signal_address, query, event_data["handled_by"], event_data["id"])
result = event_data["response"]
self._response_events.remove(event_data)
if isinstance(result, list) and not isinstance(result, dict) and len(result) == 1:
return result[0]
else:
return result
return None
def register_query_handler(self, query, func, **kwargs):
self._register_handler("query:"+query, func, **kwargs)
def unregister_query_handler(self, query, func, **kwargs):
self._unregister_handler("query:"+query, func, **kwargs)
class __ConfigClass(_KerviConfig):
def __init__(self):
super()
self._keys = []
self._config_path = None
self._json_data = None
self._user_config = None
self._config = None
self._config_base = None
self._is_loaded = False
self._log = KerviLog("config")
def _load(self, **kwargs):
self._config_path = kwargs.get("config_file", None)
self._json_data = kwargs.get("json_data", None)
self._user_data = kwargs.get("config_user", {})
self._config_base = kwargs.get("config_base", {})
if self._json_data:
self._user_config = json.loads(self._json_data)
elif not self._user_config and self._config_path:
try:
if os.path.isfile(self._config_path):
self._log.verbose("use config: %s", self._config_path)
config_data = ""
with open(self._config_path, "r") as config_file:
for line in config_file:
line_trim = line.lstrip()
if not line_trim.startswith("#"):
config_data += line
else:
config_data += "\n"
if config_data:
self._user_config = json.loads(config_data)
except Exception as ex:
self._log.error("error in config file: %s", ex)
self._config = self._config_base
if self._user_config:
self._config = _deep_update(self._config, self._user_config)
if self._user_data:
self._config = _deep_update(self._config, self._user_data)
self._load_dict(self._config, top=self)
self._is_loaded = True
def _validate(self):
return True
def to_json(self):
return json.dumps(self._config)
def get(self, name, default_value=None):
if hasattr(self, name):
return getattr(self, name)
else:
return default_value
@property
def is_loaded(self):
return self._is_loaded
def _load_dict(self, d, **kwargs):
top = kwargs.get("top", _KerviConfig())
seqs = tuple, list, set, frozenset
for i, j in d.items():
top._keys += [i]
if isinstance(j, dict):
item = self._load_dict(j)
setattr(top, i, item)
elif isinstance(j, seqs):
item = type(j)(
self._load_dict(sj) if isinstance(sj, dict) else sj
for sj in j)
setattr(top, i, item)
else:
setattr(top, i, j)
#setattr(top, "keys", keys)
return top
def list(self):
print("cl", dir(self))
class _PluginInfo:
def __init__(self,
plugin_module,
config,
plugin_type,
load_config,
manager=None,
load_silent=False):
self._config = config
self._plugin_module = plugin_module
self._plugin_type = plugin_type
self._load_config = load_config
self._load_silent = load_silent
self.instance = None
self._manager = manager
self._first_process_step = True
self._log = KerviLog("PluginInfo")
def _start_plugin_process(self, module_port):
process._start_process("plugin-" +
self._manager._config.application.id + "." +
self._plugin_module,
"plugin_" + self._plugin_module,
self._manager._config,
nethelper.get_free_port([module_port]),
_KerviPluginProcess,
plugin_module=self._plugin_module,
plugin_config=self._config,
log_queue=self._manager._log_queue,
load_silent=self._manager._load_silent)
def _create_instance(self):
try:
if not self._load_silent:
self._log.verbose("load plugin: %s", self._plugin_module)
module = __import__(self._plugin_module, fromlist=[''])
self.instance = module.init_plugin(self._config, self._manager)
if self._manager and self._manager._plugin_classes:
valid = False
for plugin_class in self._manager._plugin_classes:
if isinstance(self.instance, plugin_class):
valid = True
break
if not valid:
self._log.error("Invalid plugin class: %s expected: %s",
self.instance,
self._manager._plugin_classes)
self.instance = None
except Exception as ex:
self._log.exception("Could not load plugin: %s",
self._plugin_module)
def load(self, module_port=None):
if self.own_process:
self._start_plugin_process(module_port)
else:
self._create_instance()
def process_step(self):
if self.instance:
if self._first_process_step:
self.instance.first_process_step()
self._first_process_step = False
else:
self.instance.process_step()
def terminate_process(self):
if self.instance:
self.instance.terminate_process()
@property
def plugin_module(self):
return self._plugin_module
@property
def own_process(self):
if self._config:
return self._config.get("own_process",
self._load_config.own_process)
elif self._load_config:
return self._load_config.own_process
else:
return False
@property
def managed(self):
if self._config:
return self._config.get("managed", self._load_config.managed)
elif self._load_config:
return self._load_config.managed
else:
return False