def start_stats(self):
""" Start Statistics manager
"""
print("Start Stats")
self.log.info(
"Starting statistics manager. Its logs will be in a dedicated log file"
)
self.stat_mgr = StatsManager(handler_params=[self], xpl=self.myxpl)
self.stat_mgr.load()
self.log.info("Stat manager started")
def start_stats(self):
""" Start Statistics manager
"""
print("Start Stats")
self.log.info("Starting statistics manager. Its logs will be in a dedicated log file")
self.stat_mgr = StatsManager(handler_params = [self], xpl = self.myxpl)
self.stat_mgr.load()
self.log.info("Stat manager started")
class Rest(XplPlugin):
""" REST Server
- create a HTTP server
- process REST requests
"""
def __init__(self, server_ip, server_port):
""" Initiate DbHelper, Logs and config
Then, start HTTP server and give it initialized data
@param server_ip : ip of HTTP server
@param server_port : port of HTTP server
"""
XplPlugin.__init__(self, name='rest')
# logging initialization
self.log.info("Rest Server initialisation...")
self.log.debug("locale : %s %s" % locale.getdefaultlocale())
# logging Queue activities
log_queue = logger.Logger('rest-queues')
self.log_queue = log_queue.get_logger('rest-queues')
self.log_queue.info("Rest's queues activities...")
# logging data manipulation initialization
log_dm = logger.Logger('rest-dm')
self.log_dm = log_dm.get_logger('rest-dm')
self.log_dm.info("Rest Server Data Manipulation...")
# API version
self._rest_api_version = REST_API_VERSION
# Hosts list
self._hosts_list = {
self.get_sanitized_hostname(): {
"id": self.get_sanitized_hostname(),
"status": "on",
"primary": True,
"last_seen": time.time(),
"ip": "",
"interval": "1"
}
}
try:
### Config
# directory data
cfg = Loader('domogik')
config = cfg.load()
conf = dict(config[1])
self.log_dir_path = conf['log_dir_path']
# plugin installation path
if conf.has_key('package_path'):
self._package_path = conf['package_path']
self._src_prefix = None
self.log.info("Set package path to '%s' " % self._package_path)
print("Set package path to '%s' " % self._package_path)
self._design_dir = "%s/domogik_packages/design/" % self._package_path
self.package_mode = True
else:
self.log.info("No package path defined in config file")
self._package_path = None
self._src_prefix = conf['src_prefix']
self._design_dir = "%s/share/domogik/design/" % conf[
'src_prefix']
self.package_mode = False
# HTTP server ip and port
try:
cfg_rest = Loader('rest')
config_rest = cfg_rest.load()
conf_rest = dict(config_rest[1])
self.server_ip = conf_rest['rest_server_ip']
self.server_port = conf_rest['rest_server_port']
except KeyError:
# default parameters
self.server_ip = server_ip
self.server_port = server_port
self.log.info("Configuration : ip:port = %s:%s" %
(self.server_ip, self.server_port))
# SSL configuration
try:
cfg_rest = Loader('rest')
config_rest = cfg_rest.load()
conf_rest = dict(config_rest[1])
self.use_ssl = conf_rest['rest_use_ssl']
if self.use_ssl == "True":
self.use_ssl = True
else:
self.use_ssl = False
self.ssl_certificate = conf_rest['rest_ssl_certificate']
except KeyError:
# default parameters
self.use_ssl = USE_SSL
self.ssl_certificate = SSL_CERTIFICATE
if self.use_ssl == True:
self.log.info(
"Configuration : SSL support activated (certificate : %s)"
% self.ssl_certificate)
else:
self.log.info("Configuration : SSL support not activated")
# File repository
try:
cfg_rest = Loader('rest')
config_rest = cfg_rest.load()
conf_rest = dict(config_rest[1])
self.repo_dir = conf_rest['rest_repository']
except KeyError:
# default parameters
self.repo_dir = DEFAULT_REPO_DIR
# Gloal Queues config
self.log.debug("Get queues configuration")
self._config = Query(self.myxpl, self.log)
self._queue_timeout = self._config.query('rest', 'q-timeout')
if self._queue_timeout == None:
self._queue_timeout = QUEUE_TIMEOUT
self._queue_timeout = float(self._queue_timeout)
self._queue_package_size = self._config.query('rest', 'q-pkg-size')
if self._queue_package_size == None:
self._queue_package_size = QUEUE_PACKAGE_SIZE
self._queue_package_size = float(self._queue_package_size)
self._queue_size = self._config.query('rest', 'q-size')
if self._queue_size == None:
self._queue_size = QUEUE_SIZE
self._queue_size = float(self._queue_size)
self._queue_life_expectancy = self._config.query(
'rest', 'q-life-exp')
if self._queue_life_expectancy == None:
self._queue_life_expectancy = QUEUE_LIFE_EXPECTANCY
self._queue_life_expectancy = float(self._queue_life_expectancy)
self._queue_sleep = self._config.query('rest', 'q-sleep')
if self._queue_sleep == None:
self._queue_sleep = QUEUE_SLEEP
self._queue_sleep = float(self._queue_sleep)
# /command Queues config
self._queue_command_size = self._config.query('rest', 'q-cmd-size')
if self._queue_command_size == None:
self._queue_command_size = QUEUE_COMMAND_SIZE
self._queue_command_size = float(self._queue_command_size)
# /event Queues config
self._event_timeout = self._config.query('rest', 'evt-timeout')
if self._event_timeout == None:
self._event_timeout = EVENT_TIMEOUT
self._event_timeout = float(self._event_timeout)
self._queue_event_size = self._config.query('rest', 'q-evt-size')
if self._queue_event_size == None:
self._queue_event_size = QUEUE_EVENT_SIZE
self._queue_event_size = float(self._queue_event_size)
self._queue_event_timeout = self._config.query(
'rest', 'q-evt-timeout')
if self._queue_event_timeout == None:
self._queue_event_timeout = QUEUE_EVENT_TIMEOUT
self._queue_event_timeout = float(self._queue_event_timeout)
self._queue_event_life_expectancy = self._config.query(
'rest', 'q-evt-life-exp')
if self._queue_event_life_expectancy == None:
self._queue_event_life_expectancy = QUEUE_EVENT_LIFE_EXPECTANCY
self._queue_event_life_expectancy = float(
self._queue_event_life_expectancy)
# Queues for xPL
# Queues for packages management
self._queue_package = Queue(self._queue_package_size)
# Queues for domogik system actions
self._queue_system_list = Queue(self._queue_size)
self._queue_system_detail = Queue(self._queue_size)
self._queue_system_start = Queue(self._queue_size)
self._queue_system_stop = Queue(self._queue_size)
# Queues for /command
self._queue_command = Queue(self._queue_command_size)
# Queues for /events/domogik
self._queue_event_dmg = Queue(self._queue_event_size)
# Queues for /events/request
# this queue will be fill by stat manager
self._event_requests = RequestEvents(
self.get_stop, self.log, self._event_timeout,
self._queue_event_size, self._queue_event_timeout,
self._queue_event_life_expectancy)
self.add_stop_cb(self._event_requests.set_stop_clean)
# Queues for /events/domogik
# this queue will be fill by stat manager
self._event_dmg = DmgEvents(self.get_stop, self.log,
self._event_timeout,
self._queue_event_size,
self._queue_event_timeout,
self._queue_event_life_expectancy)
# notice : adding data in queue is made in _add_to_queue_system_list
self.add_stop_cb(self._event_dmg.set_stop_clean)
# define listeners for queues
self.log.debug("Create listeners")
if self.package_mode == True:
Listener(self._list_installed_packages, self.myxpl, \
{'schema': 'domogik.package',
'xpltype': 'xpl-trig',
'command' : 'installed-packages-list'})
Listener(self._add_to_queue_package, self.myxpl, \
{'schema': 'domogik.package',
'xpltype': 'xpl-trig'})
Listener(self._add_to_queue_system_list, self.myxpl, \
{'schema': 'domogik.system',
'xpltype': 'xpl-trig',
'command' : 'list'})
Listener(self._add_to_queue_system_list, self.myxpl, \
{'schema': 'domogik.system',
'xpltype': 'xpl-trig',
'command' : 'enable'})
Listener(self._add_to_queue_system_list, self.myxpl, \
{'schema': 'domogik.system',
'xpltype': 'xpl-trig',
'command' : 'disable'})
Listener(self._add_to_queue_system_detail, self.myxpl, \
{'schema': 'domogik.system',
'xpltype': 'xpl-trig',
'command' : 'detail'})
Listener(self._add_to_queue_system_start, self.myxpl, \
{'schema': 'domogik.system',
'xpltype': 'xpl-trig',
'command' : 'start'})
Listener(self._add_to_queue_system_stop, self.myxpl, \
{'schema': 'domogik.system',
'xpltype': 'xpl-trig',
'command' : 'stop'})
Listener(self._add_to_queue_command, self.myxpl, \
{'xpltype': 'xpl-trig'})
# Listener for hosts list
Listener(self._list_hosts, self.myxpl, \
{'schema': 'hbeat.app',
'xpltype': 'xpl-stat'})
# Background process to check if hosts has disappeared
thr_hbeat = XplTimer(10, \
self._refresh_status_for_list_hosts, \
self.myxpl)
thr_hbeat.start()
self._discover_hosts()
# Enable hbeat
self.enable_hbeat()
# Ask for installed packages on all hosts
# Semaphore init for installed package list update
self.sema_installed = Semaphore(value=1)
self._installed_packages = {}
if self.package_mode == True:
self._get_installed_packages_from_manager()
# Launch server, stats
self.log.info("REST Initialisation OK")
self.add_stop_cb(self.stop_http)
self.server = None
self.start_stats()
self.start_http()
except:
self.log.error("%s" % self.get_exception())
def _add_to_queue_package(self, message):
""" Add data in a queue
"""
self._put_in_queue(self._queue_package, message)
def _add_to_queue_system_list(self, message):
""" Add data in a queue
"""
self._put_in_queue(self._queue_system_list, message)
current_date = calendar.timegm(time.gmtime())
self._event_dmg.add_in_queues({
"timestamp": current_date,
"data": "plugin-list-updated"
})
def _add_to_queue_system_detail(self, message):
""" Add data in a queue
"""
self._put_in_queue(self._queue_system_detail, message)
def _add_to_queue_system_start(self, message):
""" Add data in a queue
"""
self._put_in_queue(self._queue_system_start, message)
def _add_to_queue_system_stop(self, message):
""" Add data in a queue
"""
self._put_in_queue(self._queue_system_stop, message)
def _add_to_queue_command(self, message):
""" Add data in a queue
"""
self._put_in_queue(self._queue_command, message)
def _get_from_queue(self,
my_queue,
filter_type=None,
filter_schema=None,
filter_data=None,
nb_rec=0,
timeout=None):
""" Encapsulation for _get_from_queue_in
If timeout not elapsed and _get_from_queue didn't find a valid data
call again _get_from_queue until timeout
This encapsulation is used to process case where queue is not empty but there is
no valid data in it and we want to wait for timeout
"""
if timeout == None:
timeout = self._queue_timeout
start_time = time.time()
while time.time() - start_time < timeout:
try:
return self._get_from_queue_without_waiting(
my_queue, filter_type, filter_schema, filter_data, nb_rec,
timeout)
except Empty:
# no data in queue for us.... let's continue until time elapsed
# in order not rest not working so much, let it make a pause
time.sleep(self._queue_sleep)
# time elapsed... we can raise the Empty exception
raise Empty
def _get_from_queue_without_waiting(self,
my_queue,
filter_type=None,
filter_schema=None,
filter_data=None,
nb_rec=0,
timeout=None):
""" Get an item from queue (recursive function)
Checks are made on :
- life expectancy of message
- filter given
- size of queue
If necessary, each item of queue is read.
@param my_queue : queue to get data from
@param filter_type : filter on a schema type
@param filter_schema : filter on a specific schema
@param filter_data : dictionnay of filters. Examples :
- {"command" : "start", ...}
- {"plugin" : "wol%", ...} : here "%" indicate that we search for something starting with "wol"
@param nb_rec : internal parameter (do not use it for first call). Used to check recursivity VS queue size
@param timeout : to use a different timeout from default one
"""
if timeout == None:
timeout = self._queue_timeout
self.log_queue.debug("Get from queue : %s (recursivity deepth : %s)" %
(str(my_queue), nb_rec))
# check if recursivity doesn't exceed queue size
if nb_rec > my_queue.qsize():
self.log_queue.warning(
"Get from queue %s : number of call exceed queue size (%s) : return None"
% (str(my_queue), my_queue.qsize()))
# we raise an "Empty" exception because we consider that if we don't find
# the good data, it is as if it was "empty"
raise Empty
msg_time, message = my_queue.get(True, timeout)
# if message not too old, we process it
if time.time() - msg_time < self._queue_life_expectancy:
# no filter defined
if filter_type == None and filter_schema == None and filter_data == None:
self.log_queue.debug("Get from queue %s : return %s" %
(str(my_queue), str(message)))
return message
# we want to filter data
else:
keep_data = True
if filter_type != None and filter_type.lower(
) != message.type.lower():
keep_data = False
if filter_schema != None and filter_schema.lower(
) != message.schema.lower():
keep_data = False
if filter_data != None and keep_data == True:
# data
self.log_queue.debug("Filter on message %s WITH %s" %
(message.data, filter_data))
for key in filter_data:
# take care of final "%" in order to search data starting by filter_data[key]
if filter_data[key][-1] == "%":
if message.data.has_key(key):
msg_data = str(message.data[key])
my_filter_data = str(filter_data[key])
len_data = len(my_filter_data) - 1
if msg_data[0:len_data] != my_filter_data[0:-1]:
keep_data = False
else:
keep_data = False
# normal search
else:
if message.data.has_key(key):
if message.data[key].lower(
) != filter_data[key].lower():
keep_data = False
else:
keep_data = False
# if message is ok for us, return it
if keep_data == True:
self.log_queue.debug("Get from queue %s : return %s" %
(str(my_queue), str(message)))
return message
# else, message get back in queue and get another one
else:
self.log_queue.debug(
"Get from queue %s : bad data, check another one..." %
(str(my_queue)))
self._put_in_queue(my_queue, message)
return self._get_from_queue_without_waiting(
my_queue, filter_type, filter_schema, filter_data,
nb_rec + 1, timeout)
# if message too old : get an other message
else:
self.log_queue.debug(
"Get from queue %s : data too old, check another one..." %
(str(my_queue)))
return self._get_from_queue_without_waiting(
my_queue, filter_type, filter_schema, filter_data, nb_rec + 1,
timeout)
def _put_in_queue(self, my_queue, message):
""" put a message in a named queue
@param my_queue : queue
@param message : data to put in queue
"""
self.log_queue.debug("Put in queue %s : %s" %
(str(my_queue), str(message)))
try:
my_queue.put((time.time(), message), True, self._queue_timeout)
# Clean queue to make space
except Full:
msg = "Queue '%s' is full : cleaning it to make some space..." % my_queue
self.log_queue.debug(msg)
print(msg)
# queue is full : start cleaning it
nb_ck = 0
while nb_ck < my_queue.qsize():
(q_time, q_data) = my_queue.get()
# data to keep
if time.time() - self._queue_life_expectancy < q_time:
my_queue.put((q_time, q_data), True, self._queue_timeout)
nb_ck += 1
my_queue.put((time.time(), message), True, self._queue_timeout)
self.log_queue.debug("Cleaning finished")
def _discover_hosts(self):
""" Send a hbeat.request to discover managers
"""
mess = XplMessage()
mess.set_type('xpl-cmnd')
mess.set_target("*")
mess.set_schema('hbeat.request')
mess.add_data({'command': 'request'})
self.myxpl.send(mess)
def _list_hosts(self, message):
""" Maintain list of Domogik hosts
@param message : hbeat.app xpl message
"""
tmp1 = message.source.split(".")
tmp2 = tmp1[0].split("-")
vendor = tmp2[0]
device = tmp2[1]
instance = tmp1[1]
if vendor == "domogik" and device == "manager":
# host not in the list
if self._hosts_list.has_key(instance) == False:
self._hosts_list[instance] = {"primary": False}
self._hosts_list[instance]["status"] = "on"
self._hosts_list[instance]["last_seen"] = time.time()
self._hosts_list[instance]["interval"] = 60 * int(
message.data["interval"])
self._hosts_list[instance]["ip"] = message.data["remote-ip"]
self._hosts_list[instance]["id"] = instance
def _refresh_status_for_list_hosts(self):
""" Check if hosts has disappeared
"""
now = time.time()
for instance in self._hosts_list:
if (now - self._hosts_list[instance]["last_seen"] >
self._hosts_list[instance]["interval"]):
self._hosts_list[instance]["status"] = "off"
def _get_installed_packages_from_manager(self):
""" Send a xpl message to all managers to get installed packages list
"""
### Send xpl message to get list
message = XplMessage()
message.set_type("xpl-cmnd")
message.set_schema("domogik.package")
message.add_data({"command": "installed-packages-list"})
message.add_data({"host": "*"})
self.myxpl.send(message)
def get_installed_packages(self):
""" return list of installed packages
There is a semaphore in order not to return the list when it is
updated (may be incomplete)
"""
# developper mode : all plugins are installed
# TODO : remove log lines after tests
self.log.debug("*** get_installed_packages")
self.sema_installed.acquire()
self.log.debug("*** sema acquired")
ret = self._installed_packages
self.sema_installed.release()
self.log.debug("*** sema released")
return ret
def _list_installed_packages(self, message):
""" Called when a list of installed packages is received
@param host : host
@param pkg_type : type of package
"""
print("Get new installed packages list")
self.log.debug("Get new installed packages list")
self.sema_installed.acquire()
self.log.debug("*** sema acquired")
self.log.debug("*** msg = %s" % message)
# process message
host = message.data["host"]
self._installed_packages[host] = {}
pkg_mgr = PackageManager()
idx = 0
loop_again = True
self.log.debug("*** before while")
while loop_again:
try:
self.log.debug("*** in while : idx=%s" % idx)
pkg_type = message.data["type" + str(idx)]
if message.data["enabled" + str(idx)].lower() == "yes":
enabled = True
else:
enabled = False
data = {
"fullname": message.data["fullname" + str(idx)],
"id": message.data["id" + str(idx)],
"version": message.data["version" + str(idx)],
"type": message.data["type" + str(idx)],
#"source" : message.data["source"+str(idx)],
"enabled": enabled
}
self.log.debug("*** call get_available_updates(%s, %s, %s)" %
(data["type"], data["id"], data["version"]))
updates = pkg_mgr.get_available_updates(
data["type"], data["id"], data["version"])
self.log.debug("*** after get_available_updates")
data["updates"] = updates
if self._installed_packages[host].has_key(pkg_type) == False:
self._installed_packages[host][pkg_type] = []
self._installed_packages[host][pkg_type].append(data)
self.log.debug("*** before idx += 1")
idx += 1
except KeyError:
self.log.debug("*** except keyerror")
loop_again = False
except:
self.log.debug("*** except global")
self.log.error(
"Error while creating list of installed packages : %s" %
traceback.format_exc())
loop_again = False
self.log.debug("*** before release")
self.sema_installed.release()
self.log.debug("*** sema released")
def start_http(self):
""" Start HTTP Server
"""
# Start HTTP server
self.log.info("Start HTTP Server on %s:%s..." %
(self.server_ip, self.server_port))
if self.use_ssl:
self.server = HTTPSServerWithParam((self.server_ip, int(self.server_port)), RestHandler, \
handler_params = [self])
else:
self.server = HTTPServerWithParam((self.server_ip, int(self.server_port)), RestHandler, \
handler_params = [self])
self.server.serve_forever()
def stop_http(self):
""" Stop HTTP Server
"""
self.server.stop_handling()
def start_stats(self):
""" Start Statistics manager
"""
print("Start Stats")
self.log.info(
"Starting statistics manager. Its logs will be in a dedicated log file"
)
self.stat_mgr = StatsManager(handler_params=[self], xpl=self.myxpl)
self.stat_mgr.load()
self.log.info("Stat manager started")
def reload_stats(self):
""" Reload Statistics manager
"""
time.sleep(1)
print("Reload Stats")
self.log.info(
"Reloading statistics manager. Its logs will be in a dedicated log file"
)
self.stat_mgr.load()
def get_exception(self):
""" Get exception and display it on stdout
"""
my_exception = str(traceback.format_exc()).replace('"', "'")
print("==== Error in REST ====")
print(my_exception)
print("=======================")
return my_exception
class Rest(XplPlugin):
""" REST Server
- create a HTTP server
- process REST requests
"""
def __init__(self, server_ip, server_port):
""" Initiate DbHelper, Logs and config
Then, start HTTP server and give it initialized data
@param server_ip : ip of HTTP server
@param server_port : port of HTTP server
"""
XplPlugin.__init__(self, name = 'rest')
# logging initialization
self.log.info("Rest Server initialisation...")
self.log.debug("locale : %s %s" % locale.getdefaultlocale())
# logging Queue activities
log_queue = logger.Logger('rest-queues')
self.log_queue = log_queue.get_logger('rest-queues')
self.log_queue.info("Rest's queues activities...")
# logging data manipulation initialization
log_dm = logger.Logger('rest-dm')
self.log_dm = log_dm.get_logger('rest-dm')
self.log_dm.info("Rest Server Data Manipulation...")
# API version
self._rest_api_version = REST_API_VERSION
# Hosts list
self._hosts_list = {self.get_sanitized_hostname() :
{"id" : self.get_sanitized_hostname(),
"status" : "on",
"primary" : True,
"last_seen" : time.time(),
"ip" : "",
"interval" : "1"}}
try:
### Config
# directory data
cfg = Loader('domogik')
config = cfg.load()
conf = dict(config[1])
self.log_dir_path = conf['log_dir_path']
# plugin installation path
if conf.has_key('package_path'):
self._package_path = conf['package_path']
self._src_prefix = None
self.log.info("Set package path to '%s' " % self._package_path)
print("Set package path to '%s' " % self._package_path)
self._design_dir = "%s/domogik_packages/design/" % self._package_path
self.package_mode = True
else:
self.log.info("No package path defined in config file")
self._package_path = None
self._src_prefix = conf['src_prefix']
self._design_dir = "%s/share/domogik/design/" % conf['src_prefix']
self.package_mode = False
# HTTP server ip and port
try:
cfg_rest = Loader('rest')
config_rest = cfg_rest.load()
conf_rest = dict(config_rest[1])
self.server_ip = conf_rest['rest_server_ip']
self.server_port = conf_rest['rest_server_port']
except KeyError:
# default parameters
self.server_ip = server_ip
self.server_port = server_port
self.log.info("Configuration : ip:port = %s:%s" % (self.server_ip, self.server_port))
# SSL configuration
try:
cfg_rest = Loader('rest')
config_rest = cfg_rest.load()
conf_rest = dict(config_rest[1])
self.use_ssl = conf_rest['rest_use_ssl']
if self.use_ssl == "True":
self.use_ssl = True
else:
self.use_ssl = False
self.ssl_certificate = conf_rest['rest_ssl_certificate']
except KeyError:
# default parameters
self.use_ssl = USE_SSL
self.ssl_certificate = SSL_CERTIFICATE
if self.use_ssl == True:
self.log.info("Configuration : SSL support activated (certificate : %s)" % self.ssl_certificate)
else:
self.log.info("Configuration : SSL support not activated")
# File repository
try:
cfg_rest = Loader('rest')
config_rest = cfg_rest.load()
conf_rest = dict(config_rest[1])
self.repo_dir = conf_rest['rest_repository']
except KeyError:
# default parameters
self.repo_dir = DEFAULT_REPO_DIR
# Gloal Queues config
self.log.debug("Get queues configuration")
self._config = Query(self.myxpl, self.log)
self._queue_timeout = self._config.query('rest', 'q-timeout')
if self._queue_timeout == None:
self._queue_timeout = QUEUE_TIMEOUT
self._queue_timeout = float(self._queue_timeout)
self._queue_package_size = self._config.query('rest', 'q-pkg-size')
if self._queue_package_size == None:
self._queue_package_size = QUEUE_PACKAGE_SIZE
self._queue_package_size = float(self._queue_package_size)
self._queue_size = self._config.query('rest', 'q-size')
if self._queue_size == None:
self._queue_size = QUEUE_SIZE
self._queue_size = float(self._queue_size)
self._queue_life_expectancy = self._config.query('rest', 'q-life-exp')
if self._queue_life_expectancy == None:
self._queue_life_expectancy = QUEUE_LIFE_EXPECTANCY
self._queue_life_expectancy = float(self._queue_life_expectancy)
self._queue_sleep = self._config.query('rest', 'q-sleep')
if self._queue_sleep == None:
self._queue_sleep = QUEUE_SLEEP
self._queue_sleep = float(self._queue_sleep)
# /command Queues config
self._queue_command_size = self._config.query('rest', 'q-cmd-size')
if self._queue_command_size == None:
self._queue_command_size = QUEUE_COMMAND_SIZE
self._queue_command_size = float(self._queue_command_size)
# /event Queues config
self._event_timeout = self._config.query('rest', 'evt-timeout')
if self._event_timeout == None:
self._event_timeout = EVENT_TIMEOUT
self._event_timeout = float(self._event_timeout)
self._queue_event_size = self._config.query('rest', 'q-evt-size')
if self._queue_event_size == None:
self._queue_event_size = QUEUE_EVENT_SIZE
self._queue_event_size = float(self._queue_event_size)
self._queue_event_timeout = self._config.query('rest', 'q-evt-timeout')
if self._queue_event_timeout == None:
self._queue_event_timeout = QUEUE_EVENT_TIMEOUT
self._queue_event_timeout = float(self._queue_event_timeout)
self._queue_event_life_expectancy = self._config.query('rest', 'q-evt-life-exp')
if self._queue_event_life_expectancy == None:
self._queue_event_life_expectancy = QUEUE_EVENT_LIFE_EXPECTANCY
self._queue_event_life_expectancy = float(self._queue_event_life_expectancy)
# Queues for xPL
# Queues for packages management
self._queue_package = Queue(self._queue_package_size)
# Queues for domogik system actions
self._queue_system_list = Queue(self._queue_size)
self._queue_system_detail = Queue(self._queue_size)
self._queue_system_start = Queue(self._queue_size)
self._queue_system_stop = Queue(self._queue_size)
# Queues for /command
self._queue_command = Queue(self._queue_command_size)
# Queues for /events/domogik
self._queue_event_dmg = Queue(self._queue_event_size)
# Queues for /events/request
# this queue will be fill by stat manager
self._event_requests = RequestEvents(self.get_stop,
self.log,
self._event_timeout,
self._queue_event_size,
self._queue_event_timeout,
self._queue_event_life_expectancy)
self.add_stop_cb(self._event_requests.set_stop_clean)
# Queues for /events/domogik
# this queue will be fill by stat manager
self._event_dmg = DmgEvents(self.get_stop,
self.log,
self._event_timeout,
self._queue_event_size,
self._queue_event_timeout,
self._queue_event_life_expectancy)
# notice : adding data in queue is made in _add_to_queue_system_list
self.add_stop_cb(self._event_dmg.set_stop_clean)
# define listeners for queues
self.log.debug("Create listeners")
if self.package_mode == True:
Listener(self._list_installed_packages, self.myxpl, \
{'schema': 'domogik.package',
'xpltype': 'xpl-trig',
'command' : 'installed-packages-list'})
Listener(self._add_to_queue_package, self.myxpl, \
{'schema': 'domogik.package',
'xpltype': 'xpl-trig'})
Listener(self._add_to_queue_system_list, self.myxpl, \
{'schema': 'domogik.system',
'xpltype': 'xpl-trig',
'command' : 'list'})
Listener(self._add_to_queue_system_list, self.myxpl, \
{'schema': 'domogik.system',
'xpltype': 'xpl-trig',
'command' : 'enable'})
Listener(self._add_to_queue_system_list, self.myxpl, \
{'schema': 'domogik.system',
'xpltype': 'xpl-trig',
'command' : 'disable'})
Listener(self._add_to_queue_system_detail, self.myxpl, \
{'schema': 'domogik.system',
'xpltype': 'xpl-trig',
'command' : 'detail'})
Listener(self._add_to_queue_system_start, self.myxpl, \
{'schema': 'domogik.system',
'xpltype': 'xpl-trig',
'command' : 'start'})
Listener(self._add_to_queue_system_stop, self.myxpl, \
{'schema': 'domogik.system',
'xpltype': 'xpl-trig',
'command' : 'stop'})
Listener(self._add_to_queue_command, self.myxpl, \
{'xpltype': 'xpl-trig'})
# Listener for hosts list
Listener(self._list_hosts, self.myxpl, \
{'schema': 'hbeat.app',
'xpltype': 'xpl-stat'})
# Background process to check if hosts has disappeared
thr_hbeat = XplTimer(10, \
self._refresh_status_for_list_hosts, \
self.myxpl)
thr_hbeat.start()
self._discover_hosts()
# Enable hbeat
self.enable_hbeat()
# Ask for installed packages on all hosts
# Semaphore init for installed package list update
self.sema_installed = Semaphore(value=1)
self._installed_packages = {}
if self.package_mode == True:
self._get_installed_packages_from_manager()
# Launch server, stats
self.log.info("REST Initialisation OK")
self.add_stop_cb(self.stop_http)
self.server = None
self.start_stats()
self.start_http()
except :
self.log.error("%s" % self.get_exception())
def _add_to_queue_package(self, message):
""" Add data in a queue
"""
self._put_in_queue(self._queue_package, message)
def _add_to_queue_system_list(self, message):
""" Add data in a queue
"""
self._put_in_queue(self._queue_system_list, message)
current_date = calendar.timegm(time.gmtime())
self._event_dmg.add_in_queues({"timestamp" : current_date,
"data" : "plugin-list-updated"})
def _add_to_queue_system_detail(self, message):
""" Add data in a queue
"""
self._put_in_queue(self._queue_system_detail, message)
def _add_to_queue_system_start(self, message):
""" Add data in a queue
"""
self._put_in_queue(self._queue_system_start, message)
def _add_to_queue_system_stop(self, message):
""" Add data in a queue
"""
self._put_in_queue(self._queue_system_stop, message)
def _add_to_queue_command(self, message):
""" Add data in a queue
"""
self._put_in_queue(self._queue_command, message)
def _get_from_queue(self, my_queue, filter_type = None, filter_schema = None, filter_data = None, nb_rec = 0, timeout = None):
""" Encapsulation for _get_from_queue_in
If timeout not elapsed and _get_from_queue didn't find a valid data
call again _get_from_queue until timeout
This encapsulation is used to process case where queue is not empty but there is
no valid data in it and we want to wait for timeout
"""
if timeout == None:
timeout = self._queue_timeout
start_time = time.time()
while time.time() - start_time < timeout:
try:
return self._get_from_queue_without_waiting(my_queue, filter_type, filter_schema, filter_data, nb_rec, timeout)
except Empty:
# no data in queue for us.... let's continue until time elapsed
# in order not rest not working so much, let it make a pause
time.sleep(self._queue_sleep)
# time elapsed... we can raise the Empty exception
raise Empty
def _get_from_queue_without_waiting(self, my_queue, filter_type = None, filter_schema = None, filter_data = None, nb_rec = 0, timeout = None):
""" Get an item from queue (recursive function)
Checks are made on :
- life expectancy of message
- filter given
- size of queue
If necessary, each item of queue is read.
@param my_queue : queue to get data from
@param filter_type : filter on a schema type
@param filter_schema : filter on a specific schema
@param filter_data : dictionnay of filters. Examples :
- {"command" : "start", ...}
- {"plugin" : "wol%", ...} : here "%" indicate that we search for something starting with "wol"
@param nb_rec : internal parameter (do not use it for first call). Used to check recursivity VS queue size
@param timeout : to use a different timeout from default one
"""
if timeout == None:
timeout = self._queue_timeout
self.log_queue.debug("Get from queue : %s (recursivity deepth : %s)" % (str(my_queue), nb_rec))
# check if recursivity doesn't exceed queue size
if nb_rec > my_queue.qsize():
self.log_queue.warning("Get from queue %s : number of call exceed queue size (%s) : return None" % (str(my_queue), my_queue.qsize()))
# we raise an "Empty" exception because we consider that if we don't find
# the good data, it is as if it was "empty"
raise Empty
msg_time, message = my_queue.get(True, timeout)
# if message not too old, we process it
if time.time() - msg_time < self._queue_life_expectancy:
# no filter defined
if filter_type == None and filter_schema == None and filter_data == None:
self.log_queue.debug("Get from queue %s : return %s" % (str(my_queue), str(message)))
return message
# we want to filter data
else:
keep_data = True
if filter_type != None and filter_type.lower() != message.type.lower():
keep_data = False
if filter_schema != None and filter_schema.lower() != message.schema.lower():
keep_data = False
if filter_data != None and keep_data == True:
# data
self.log_queue.debug("Filter on message %s WITH %s" % (message.data, filter_data))
for key in filter_data:
# take care of final "%" in order to search data starting by filter_data[key]
if filter_data[key][-1] == "%":
if message.data.has_key(key):
msg_data = str(message.data[key])
my_filter_data = str(filter_data[key])
len_data = len(my_filter_data) - 1
if msg_data[0:len_data] != my_filter_data[0:-1]:
keep_data = False
else:
keep_data = False
# normal search
else:
if message.data.has_key(key):
if message.data[key].lower() != filter_data[key].lower():
keep_data = False
else:
keep_data = False
# if message is ok for us, return it
if keep_data == True:
self.log_queue.debug("Get from queue %s : return %s" % (str(my_queue), str(message)))
return message
# else, message get back in queue and get another one
else:
self.log_queue.debug("Get from queue %s : bad data, check another one..." % (str(my_queue)))
self._put_in_queue(my_queue, message)
return self._get_from_queue_without_waiting(my_queue, filter_type, filter_schema, filter_data, nb_rec + 1, timeout)
# if message too old : get an other message
else:
self.log_queue.debug("Get from queue %s : data too old, check another one..." % (str(my_queue)))
return self._get_from_queue_without_waiting(my_queue, filter_type, filter_schema, filter_data, nb_rec + 1, timeout)
def _put_in_queue(self, my_queue, message):
""" put a message in a named queue
@param my_queue : queue
@param message : data to put in queue
"""
self.log_queue.debug("Put in queue %s : %s" % (str(my_queue), str(message)))
try:
my_queue.put((time.time(), message), True, self._queue_timeout)
# Clean queue to make space
except Full:
msg = "Queue '%s' is full : cleaning it to make some space..." % my_queue
self.log_queue.debug(msg)
print(msg)
# queue is full : start cleaning it
nb_ck = 0
while nb_ck < my_queue.qsize():
(q_time, q_data) = my_queue.get()
# data to keep
if time.time() - self._queue_life_expectancy < q_time:
my_queue.put((q_time, q_data), True, self._queue_timeout)
nb_ck += 1
my_queue.put((time.time(), message), True, self._queue_timeout)
self.log_queue.debug("Cleaning finished")
def _discover_hosts(self):
""" Send a hbeat.request to discover managers
"""
mess = XplMessage()
mess.set_type('xpl-cmnd')
mess.set_target("*")
mess.set_schema('hbeat.request')
mess.add_data({'command' : 'request'})
self.myxpl.send(mess)
def _list_hosts(self, message):
""" Maintain list of Domogik hosts
@param message : hbeat.app xpl message
"""
tmp1 = message.source.split(".")
tmp2 = tmp1[0].split("-")
vendor = tmp2[0]
device = tmp2[1]
instance = tmp1[1]
if vendor == "domogik" and device == "manager":
# host not in the list
if self._hosts_list.has_key(instance) == False:
self._hosts_list[instance] = {"primary" : False}
self._hosts_list[instance]["status"] = "on"
self._hosts_list[instance]["last_seen"] = time.time()
self._hosts_list[instance]["interval"] = 60 * int(message.data["interval"])
self._hosts_list[instance]["ip"] = message.data["remote-ip"]
self._hosts_list[instance]["id"] = instance
def _refresh_status_for_list_hosts(self):
""" Check if hosts has disappeared
"""
now = time.time()
for instance in self._hosts_list:
if (now - self._hosts_list[instance]["last_seen"] > self._hosts_list[instance]["interval"]):
self._hosts_list[instance]["status"] = "off"
def _get_installed_packages_from_manager(self):
""" Send a xpl message to all managers to get installed packages list
"""
### Send xpl message to get list
message = XplMessage()
message.set_type("xpl-cmnd")
message.set_schema("domogik.package")
message.add_data({"command" : "installed-packages-list"})
message.add_data({"host" : "*"})
self.myxpl.send(message)
def get_installed_packages(self):
""" return list of installed packages
There is a semaphore in order not to return the list when it is
updated (may be incomplete)
"""
# developper mode : all plugins are installed
# TODO : remove log lines after tests
self.log.debug("*** get_installed_packages")
self.sema_installed.acquire()
self.log.debug("*** sema acquired")
ret = self._installed_packages
self.sema_installed.release()
self.log.debug("*** sema released")
return ret
def _list_installed_packages(self, message):
""" Called when a list of installed packages is received
@param host : host
@param pkg_type : type of package
"""
print("Get new installed packages list")
self.log.debug("Get new installed packages list")
self.sema_installed.acquire()
self.log.debug("*** sema acquired")
self.log.debug("*** msg = %s" % message)
# process message
host = message.data["host"]
self._installed_packages[host] = {}
pkg_mgr = PackageManager()
idx = 0
loop_again = True
self.log.debug("*** before while")
while loop_again:
try:
self.log.debug("*** in while : idx=%s" % idx)
pkg_type = message.data["type"+str(idx)]
if message.data["enabled"+str(idx)].lower() == "yes":
enabled = True
else:
enabled = False
data = {"fullname" : message.data["fullname"+str(idx)],
"id" : message.data["id"+str(idx)],
"version" : message.data["version"+str(idx)],
"type" : message.data["type"+str(idx)],
#"source" : message.data["source"+str(idx)],
"enabled" : enabled}
self.log.debug("*** call get_available_updates(%s, %s, %s)" % (data["type"], data["id"], data["version"]))
updates = pkg_mgr.get_available_updates(data["type"], data["id"], data["version"])
self.log.debug("*** after get_available_updates")
data["updates"] = updates
if self._installed_packages[host].has_key(pkg_type) == False:
self._installed_packages[host][pkg_type] = []
self._installed_packages[host][pkg_type].append(data)
self.log.debug("*** before idx += 1")
idx += 1
except KeyError:
self.log.debug("*** except keyerror")
loop_again = False
except:
self.log.debug("*** except global")
self.log.error("Error while creating list of installed packages : %s" % traceback.format_exc())
loop_again = False
self.log.debug("*** before release")
self.sema_installed.release()
self.log.debug("*** sema released")
def start_http(self):
""" Start HTTP Server
"""
# Start HTTP server
self.log.info("Start HTTP Server on %s:%s..." % (self.server_ip, self.server_port))
if self.use_ssl:
self.server = HTTPSServerWithParam((self.server_ip, int(self.server_port)), RestHandler, \
handler_params = [self])
else:
self.server = HTTPServerWithParam((self.server_ip, int(self.server_port)), RestHandler, \
handler_params = [self])
self.server.serve_forever()
def stop_http(self):
""" Stop HTTP Server
"""
self.server.stop_handling()
def start_stats(self):
""" Start Statistics manager
"""
print("Start Stats")
self.log.info("Starting statistics manager. Its logs will be in a dedicated log file")
self.stat_mgr = StatsManager(handler_params = [self], xpl = self.myxpl)
self.stat_mgr.load()
self.log.info("Stat manager started")
def reload_stats(self):
""" Reload Statistics manager
"""
time.sleep(1)
print("Reload Stats")
self.log.info("Reloading statistics manager. Its logs will be in a dedicated log file")
self.stat_mgr.load()
def get_exception(self):
""" Get exception and display it on stdout
"""
my_exception = str(traceback.format_exc()).replace('"', "'")
print("==== Error in REST ====")
print(my_exception)
print("=======================")
return my_exception