def __init__(self, config, storage, mqtt_client):
"""
initializes heartbeater
:param config: keeper configuration dict
:param storage: storage access
:param mqtt_client: MQTT client
"""
self.attempts = 0
self.misses = 0
self.ha_command = config["ha.restart.command"].split(" ")
self.sys_command = config["system.restart.command"].split(" ")
self.inc = storage.inc
self.registered = False
put = storage.put
self.missed_heartbeats = put(
HEARTBEATER_MISSED_HEARTBEAT,
storage.get_int(HEARTBEATER_MISSED_HEARTBEAT))
self.ha_restarts = put(HEARTBEATER_HA_RESTARTS,
storage.get_int(HEARTBEATER_HA_RESTARTS))
self.system_restarts = put(
HEARTBEATER_SYSTEM_RESTARTS,
storage.get_int(HEARTBEATER_SYSTEM_RESTARTS))
self.put = put
self.get = storage.get
self.now = datetime.now
self.last_message = None
self.last_known_message = None
self.interval = config["heartbeat.interval"]
self.topic = config["heartbeat.topic"]
self.delay = config["heartbeat.delay"]
self.states_queue = []
mqtt_client.set_manager(self)
self.mqtt_client = mqtt_client
self.logger = Logger()
def __init__(self, config, storage, mqtt_client):
"""
initializes connector
:param config: keeper configuration dict
:param storage: storage access
:param mqtt_client: MQTT client
"""
self.attempts = 0
self.was_stable = True
self.command = config["mqtt.restart.command"].split(" ")
self.mqtt_client = None
self.registered = False
self.started_at = datetime.now()
self.time_connected = 0
self.connected_at = None
put = storage.put
self.mqtt_restarts = put(CONNECTOR_MQTT_RESTARTS, storage.get_int(CONNECTOR_MQTT_RESTARTS))
self.failed_connections = put(CONNECTOR_FAILED_CONNECTIONS, storage.get_int(CONNECTOR_FAILED_CONNECTIONS))
self.states_queue = []
self.put = put
self.get = storage.get
self.inc = storage.inc
mqtt_client.set_manager(self)
self.mqtt_client = mqtt_client
self.logger = Logger()
def __init__(self):
"""
initializes storage object
"""
self.logger = Logger()
# creates storage directory if not present
storage_path = join(KEEPER_HOME, "storage")
if not exists(storage_path):
self.logger.debug("creating directory %s for storage", storage_path)
makedirs(storage_path)
# create database and kv table
storage_path = join(storage_path, "keeper.db")
self.logger.debug("creating initial key value in %s storage", storage_path)
with lock, self.transaction(connect(storage_path)) as cursor:
cursor.execute("create table if not exists keystore(key text primary key, value text)")
self.storage_path = storage_path
self.conn = None
def main(conf_file):
_config = Configuration.Configuration()
_config.load(conf_file)
connection = Connection.Connection(_config)
Logger.setup_logger(stream = (_config.get_value('main', 'fork') != 'True' or _config.get_value('main', 'fork') != 'true'))
core._connection = connection
log_level = _config.get_value('logging', 'level')
if _config.get_value('main', 'debug') == 'True' or _config.get_value('main', 'debug') == 'true':
connection.set_debug(True)
else: connection.set_debug(False)
Logger.set_level(log_level)
connection.setup_info(
nick = _config.get_value('main', 'nick'),
ident = _config.get_value('main', 'ident'),
real = _config.get_value('main', 'real')
)
if _config.get_value('main', 'fork') == 'True' or _config.get_value('main', 'fork') == 'true':
fork()
connection.connect(
address = _config.get_value('main', 'address'),
port = _config.get_value('main', 'port'),
_ssl = _config.get_value('main', 'ssl'),
password = _config.get_value('main', 'password')
).run()
def __init__(self, client_id, config, wait=True):
"""
initialize mqtt client
:param client_id: client id
:param config: keeper configuration
:param wait: whether to wait for connection
"""
self.logger = Logger()
user = config.get("mqtt.user")
pwd = config.get("mqtt.pass")
client = Client(client_id=client_id)
client.on_connect = self._on_connect
client.on_disconnect = self._on_disconnect
client.on_message = self._on_message
client.enable_logger(self.logger)
if user and pwd:
client.username_pw_set(user, pwd)
client.connect_async(config["mqtt.broker"], config["mqtt.port"], 30)
self.client = client
self.connected = False
self.manager = None
self.wait = wait
def transaction(self, conn):
"""
base context to execute transactions
:param conn: connection
"""
self.logger.debug("beginning transaction")
conn.execute("begin")
try:
yield conn.cursor()
except Exception as ex:
self.logger.debug("transaction rolled back")
Logger().error("unable to complete transaction: %s" % ex)
conn.rollback()
else:
self.logger.debug("transaction committed")
conn.commit()
class Module(ModuleBase):
name = "ProtocolModule"
description = "Взаимодействие с клиентами протокола."
logger = Logger(app="ProtocolModule")
@EventListener
def on_load(self, bot: Bot):
self.register(ClientMethodCommand(bot))
self.register(ClientDisconnectCommand(bot))
self.register(ClientsCommand(bot))
@EventListener(priority=Priority.HIGH)
async def run(self, bot: Bot):
for service_name in settings.protocol_services:
self.logger.info("Creating service %s." % service_name)
client_config = settings.protocol_services[service_name]
guild = bot.get_guild(client_config["GUILD_ID"])
client = ProtocolService(auth_token=client_config["AUTH_TOKEN"],
channel=guild.get_channel(
client_config["CHANNEL_ID"]),
bot_client=bot,
name=service_name)
client.add()
@EventListener
async def on_unload(self):
self.logger.info("Soft-closing connections for all clients.")
for service_name in services:
service = services[service_name]
service.disconnect()
self.logger.info("Waiting for threads become dead.")
@EventListener
def on_emergency_unload(self):
self.logger.info("Closing connections for all clients.")
for service_name in services:
service = services[service_name]
if service.connected:
service.remote_socket.close()
self.logger.info("Waiting for threads become dead.")
class Heartbeater(object):
"""
Heartbeat that monitors heartbeat messages
"""
def __init__(self, config, storage, mqtt_client):
"""
initializes heartbeater
:param config: keeper configuration dict
:param storage: storage access
:param mqtt_client: MQTT client
"""
self.attempts = 0
self.misses = 0
self.ha_command = config["ha.restart.command"].split(" ")
self.sys_command = config["system.restart.command"].split(" ")
self.inc = storage.inc
self.registered = False
put = storage.put
self.missed_heartbeats = put(
HEARTBEATER_MISSED_HEARTBEAT,
storage.get_int(HEARTBEATER_MISSED_HEARTBEAT))
self.ha_restarts = put(HEARTBEATER_HA_RESTARTS,
storage.get_int(HEARTBEATER_HA_RESTARTS))
self.system_restarts = put(
HEARTBEATER_SYSTEM_RESTARTS,
storage.get_int(HEARTBEATER_SYSTEM_RESTARTS))
self.put = put
self.get = storage.get
self.now = datetime.now
self.last_message = None
self.last_known_message = None
self.interval = config["heartbeat.interval"]
self.topic = config["heartbeat.topic"]
self.delay = config["heartbeat.delay"]
self.states_queue = []
mqtt_client.set_manager(self)
self.mqtt_client = mqtt_client
self.logger = Logger()
def __enter__(self):
"""
informs when entering context
:return: Heartbeater object
"""
self.logger.info("starting heartbeater manager[pid=%s]" % getpid())
self.mqtt_client.reconnect()
return self
# noinspection PyShadowingBuiltins
def __exit__(self, type, value, traceback):
"""
publishes manager status when exiting context
:param type:
:param value:
:param traceback:
"""
self.logger.info("stopping heartbeater[pid=%s]" % getpid())
try:
self.mqtt_client.publish_state(HEARTBEATER_STATUS,
STATUS_NOT_RUNNING)
except Exception as ex:
self.logger.error("failed to publish heartbeater status: %s" % ex)
# noinspection PyUnusedLocal
def on_connect(self, client, userdata, flags, rc):
"""
subscribes to heartbeat topic
registers sensors and sends metrics
:param client: mqtt client
:param userdata: userdata dict
:param flags: flags
:param rc: rc code
"""
self.logger.info("subscribing topic %s" % self.topic)
client.subscribe(self.topic)
# first time we are connected we register metrics and
# send initial values
if not self.registered:
self.logger.info("registering metrics")
try:
publish_state = self.mqtt_client.publish_state
register = self.mqtt_client.register
# register all metrics
register(HEARTBEATER_STATUS, HEARTBEATER_STATUS_ICON)
register(HEARTBEATER_MISSED_HEARTBEAT,
HEARTBEATER_MISSED_HEARTBEAT_ICON)
register(HEARTBEATER_HA_RESTARTS, HEARTBEATER_HA_RESTARTS_ICON)
register(HEARTBEATER_SYSTEM_RESTARTS,
HEARTBEATER_SYSTEM_RESTARTS_ICON)
register(HEARTBEATER_LAST_HEARTBEAT,
HEARTBEATER_LAST_HEARTBEAT_ICON)
register(HEARTBEATER_LAST_HA_RESTART,
HEARTBEATER_LAST_HA_RESTART_ICON)
register(HEARTBEATER_LAST_SYSTEM_RESTART,
HEARTBEATER_LAST_SYSTEM_RESTART_ICON)
# sends initial values
publish_state(HEARTBEATER_STATUS, STATUS_RUNNING)
publish_state(HEARTBEATER_MISSED_HEARTBEAT,
self.missed_heartbeats)
publish_state(HEARTBEATER_HA_RESTARTS, self.ha_restarts)
publish_state(HEARTBEATER_SYSTEM_RESTARTS,
self.system_restarts)
publish_state(HEARTBEATER_LAST_HEARTBEAT,
self.get(HEARTBEATER_LAST_HEARTBEAT))
publish_state(HEARTBEATER_LAST_HA_RESTART,
self.get(HEARTBEATER_LAST_HA_RESTART))
publish_state(HEARTBEATER_LAST_SYSTEM_RESTART,
self.get(HEARTBEATER_LAST_SYSTEM_RESTART))
self.registered = True
except Exception as ex:
self.logger.error("failed to register initial metrics: %s" %
ex)
# noinspection PyUnusedLocal
def on_message(self, client, userdata, message):
"""
updates heartbeat message timestamp
:param client: mqtt client
:param userdata: userdata dict
:param message: message received
"""
self.last_message = self.now()
last_message_fmt = strftime(TIME_FORMAT)
self.logger.debug("last heartbeat from ha at %s", last_message_fmt)
self.states_queue.append((HEARTBEATER_LAST_HEARTBEAT,
self.put(HEARTBEATER_LAST_HEARTBEAT,
last_message_fmt)))
def wait_ha_connection(self):
"""
waits for a heartbeat message or timeout of 120 seconds
"""
self.last_message = None
self.last_known_message = None
now = self.now
limit = now() + timedelta(seconds=300)
self.logger.info("waiting for ha heartbeat")
while running and not self.last_message and now() < limit:
try:
self.mqtt_client.loop()
except Exception as ex:
self.logger.warning(ex)
sleep(1)
if self.last_message:
self.logger.info("ha is reachable")
else:
self.last_message = self.now()
self.last_known_message = self.last_message
self.logger.warning("ha service still not reachable")
def monitor(self):
"""
monitors heartbeat messages and restarts ha if 3 messages are missed
also restarts system after 3 ha restarts
"""
if (self.now() - self.last_message
).total_seconds() > self.interval + self.delay:
self.logger.warning("heartbeat threshold reached")
if self.misses < 3:
self.misses += 1
self.last_message += timedelta(seconds=self.interval)
self.missed_heartbeats = self.inc(HEARTBEATER_MISSED_HEARTBEAT,
self.missed_heartbeats)
self.states_queue.append(
(HEARTBEATER_MISSED_HEARTBEAT, self.missed_heartbeats))
self.logger.warning("tolerating missed heartbeat (%s of 3)" %
self.misses)
elif self.attempts < 3:
self.attempts += 1
self.misses = 0
self.logger.warning("max of misses reached")
self.logger.warning(
"restarting ha service (%s of 3) with command %s" %
(self.attempts, " ".join(self.ha_command)))
if exec_command(self.ha_command):
append = self.states_queue.append
self.ha_restarts = self.inc(HEARTBEATER_HA_RESTARTS,
self.ha_restarts)
append((HEARTBEATER_HA_RESTARTS, self.ha_restarts))
append((HEARTBEATER_LAST_HA_RESTART,
self.put(HEARTBEATER_LAST_HA_RESTART,
strftime(TIME_FORMAT))))
self.wait_ha_connection()
else:
self.logger.warning("heartbeat still failing after 3 restarts")
self.logger.warning("rebooting")
append = self.states_queue.append
self.system_restarts = self.inc(HEARTBEATER_SYSTEM_RESTARTS,
self.system_restarts)
append((HEARTBEATER_SYSTEM_RESTARTS, self.system_restarts))
append((HEARTBEATER_LAST_SYSTEM_RESTART,
self.put(HEARTBEATER_LAST_SYSTEM_RESTART,
strftime(TIME_FORMAT))))
exec_command(self.sys_command)
self.last_known_message = self.last_message
if self.last_known_message != self.last_message:
self.logger.debug("resetting counters")
self.misses = 0
self.attempts = 0
def loop(self):
"""
sleeps 1 second until next validation
sends metrics if any to send
"""
publish_state = self.mqtt_client.publish_state
try:
for states in self.states_queue:
publish_state(states[0], states[1])
self.states_queue = []
except Exception as ex:
self.logger.warning("unable to update metrics: %s" % ex)
sleep(1)
def main(path, terms):
logger = Logger.get_logger('Main')
logger.info("START THE SYSTEM!")
init(path)
run_batch_cmds(path, terms)
from base import Config
from core import Logger
class Ping:
def __init__(self, logger):
self.logger = logger
#sends the log file to server and clears the log file
def ping(self):
buf = cStringIO.StringIO()
c = pycurl.Curl()
c.setopt(c.HTTPHEADER, ['Content-Type: multipart/form-data'])
c.setopt(c.WRITEFUNCTION, buf.write)
c.setopt(c.FAILONERROR, True)
c.setopt(c.URL, Config.LOG_ADDR)
c.setopt(c.HTTPPOST,[ ("log", (c.FORM_FILE, self.logger.getPath())), ("company_id", Config.COMPANY_ID) ])
c.setopt(c.VERBOSE, 1)
try:
c.perform()
except pycurl.error, msg:
self.logger.log_error(str(msg) + "\n")
else:
self.logger.empty()
finally:
c.close()
pingModule = Ping(Logger())
pingModule.ping()
class Storage(object):
"""
Holds connection and basic methods for accessing storage
"""
# base statements
INSERT_STATEMENT = "insert or ignore into keystore(value, key) values(?, ?)"
UPDATE_STATEMENT = "update keystore set value = ? where changes() = 0 and key = ?"
SELECT_STATEMENT = "select value from keystore where key = ?"
GET_ALL = "select key, value from keystore"
GET_KEYS = "select key from keystore"
NUMBER_TYPE = (int, float)
def __init__(self):
"""
initializes storage object
"""
self.logger = Logger()
# creates storage directory if not present
storage_path = join(KEEPER_HOME, "storage")
if not exists(storage_path):
self.logger.debug("creating directory %s for storage", storage_path)
makedirs(storage_path)
# create database and kv table
storage_path = join(storage_path, "keeper.db")
self.logger.debug("creating initial key value in %s storage", storage_path)
with lock, self.transaction(connect(storage_path)) as cursor:
cursor.execute("create table if not exists keystore(key text primary key, value text)")
self.storage_path = storage_path
self.conn = None
def __enter__(self):
"""
create initial connection when entering context
:return: Storage object
"""
self.logger.debug("creating storage connection for %s", self.storage_path)
conn = connect(self.storage_path)
conn.execute("pragma journal_mode=wal")
self.conn = conn
return self
# noinspection PyShadowingBuiltins
def __exit__(self, type, value, traceback):
"""
closes connection when exiting context
:param type:
:param value:
:param traceback:
"""
try:
self.logger.debug("closing storage connection of %s", self.storage_path)
self.conn.close()
except Exception:
pass
self.conn = None
def put(self, key, value):
"""
inserts/updates a value for a given key
:param key: Key
:param value: Value
:return: initial value
"""
initial_value = value
# converts numeric type into string
if isinstance(value, Storage.NUMBER_TYPE):
value = str(value)
# upsert statement
binds = (value, key)
self.logger.debug("storing value %s for key %s", value, key)
with lock, self.transaction(self.conn) as cursor:
cursor.execute(Storage.INSERT_STATEMENT, binds)
cursor.execute(Storage.UPDATE_STATEMENT, binds)
return initial_value
def inc(self, key, value, inc_value=1):
"""
increments a value for a given key
:param key: Key
:param value: Value
:param inc_value: units to increment, default 1
:return: returns value already incremented
"""
value += inc_value
binds = (value, key)
self.logger.debug("incrementing key %s by %s", key, inc_value)
with lock, self.transaction(self.conn) as cursor:
cursor.execute(Storage.INSERT_STATEMENT, binds)
cursor.execute(Storage.UPDATE_STATEMENT, binds)
return value
def get(self, key):
"""
return the key value
:param key: key
:return: key value
"""
result = self.conn.cursor().execute(Storage.SELECT_STATEMENT, (key,)).fetchone()
result = result[0] if result else None
return result
def get_int(self, key):
"""
return the key value as int value
:param key: key
:return: key value
"""
result = self.get(key)
return int(result) if result else 0
def get_float(self, key):
"""
return the key value as float value
:param key: key
:return: key value
"""
result = self.get(key)
return float(result) if result else 0
@contextmanager
def transaction(self, conn):
"""
base context to execute transactions
:param conn: connection
"""
self.logger.debug("beginning transaction")
conn.execute("begin")
try:
yield conn.cursor()
except Exception as ex:
self.logger.debug("transaction rolled back")
Logger().error("unable to complete transaction: %s" % ex)
conn.rollback()
else:
self.logger.debug("transaction committed")
conn.commit()
import numpy as np
from mem_sys import heap, new, delete, Block
from core import confirmar, BadAlloc, affirmations, coordinate_to_index, Logger
console = Logger("Console")
errorLogger = Logger("Error")
def alocar():
print("[ Alocar ]".center(60, "-"))
min_bytes = confirmar("Quantidade minima de bytes: ",
tipo=int,
confirm=True,
goto=menu)
fit = confirmar("Tipo de fit [first/best/worst]: ",
tipo=str,
confirm=True,
goto=menu,
validation=lambda x: x.lower() in
("first", "best", "worst"))
show = confirmar("Mostrar alocação? ", tipo=str, confirm=False, goto=menu)
show = show in affirmations
return new(min_bytes, fit, show)
def desalocar():
print("[ Desalocar ]".center(60, "-"))
block = Block()
choice = confirmar("Coordenada ou Indice? ",
confirm=False,
from core.Logger import *
import sys
from utils.Start import *
error = False
global logger
protocol = int(sys.argv[1]) if len(sys.argv) > 1 else False
logger = Logger(protocol=protocol) # setup logger
# # # # Waiting for instructions loop # # # # #
while not logger.setup_status == 'exit':
if logger.is_pi and logger.setup_status != 'running': PyWelcome(logger)
if logger.setup_status == 'running': # run experiment unless stopped
try:
if logger.get_protocol(): exec(open(logger.get_protocol()).read())
except Exception as e:
error = e
logger.update_setup_info({
'state': 'ERROR!',
'notes': str(e),
'status': 'exit'
})
if protocol:
logger.update_setup_info({'status': 'exit'})
break
elif logger.setup_status not in ['exit', 'running'
]: # restart if session ended
logger.update_setup_info({'status': 'ready'}) # restart
time.sleep(.1)
# # # # # Exit # # # # #
class MqttClient(object):
"""
Holds connection and basic methods for accessing mqtt
"""
def __init__(self, client_id, config, wait=True):
"""
initialize mqtt client
:param client_id: client id
:param config: keeper configuration
:param wait: whether to wait for connection
"""
self.logger = Logger()
user = config.get("mqtt.user")
pwd = config.get("mqtt.pass")
client = Client(client_id=client_id)
client.on_connect = self._on_connect
client.on_disconnect = self._on_disconnect
client.on_message = self._on_message
client.enable_logger(self.logger)
if user and pwd:
client.username_pw_set(user, pwd)
client.connect_async(config["mqtt.broker"], config["mqtt.port"], 30)
self.client = client
self.connected = False
self.manager = None
self.wait = wait
def __enter__(self):
"""
entering context
:return: MqttClient object
"""
return self
# noinspection PyShadowingBuiltins
def __exit__(self, type, value, traceback):
"""
disconnects client when exiting context
:param type:
:param value:
:param traceback:
"""
try:
self.logger.debug("disconnecting mqtt client")
self.client.disconnect()
except Exception:
pass
self.client = None
def set_manager(self, manager):
"""
sets associated manager
:param manager: manager using connection
"""
self.manager = manager
# noinspection PyProtectedMember
def _on_disconnect(self, client, userdata, rc):
"""
base on disconnect behaviour, can be extended wih custom
methods from implementation
:param client: mqtt client
:param userdata: userdata dict
:param rc: rc code
"""
self.logger.info("disconnected from %s:%s" % (client._host, client._port))
self.connected = False
# call custom on disconnect methods if any defined
try:
self.logger.debug("calling custom on_disconnect")
self.manager.on_disconnect(client, userdata, rc)
except Exception as ex:
if not isinstance(ex, (TypeError, AttributeError)):
self.logger.error("failed to execute custom on_disconnect: %s" % ex)
# noinspection PyProtectedMember
def _on_connect(self, client, userdata, flags, rc):
"""
base on connect behaviour, can be extended wih custom
methods from implementation
:param client: mqtt client
:param userdata: userdata dict
:param flags: flags
:param rc: rc code
"""
self.logger.info("connected to %s:%s" % (client._host, client._port))
self.connected = rc == 0
# call custom on connect methods if any defined
try:
self.logger.debug("calling custom on_connect")
self.manager.on_connect(client, userdata, flags, rc)
except Exception as ex:
if not isinstance(ex, (TypeError, AttributeError)):
self.logger.error("failed to execute custom on_connect: %s" % ex)
def _on_message(self, client, userdata, message):
"""
base on message behaviour, can be extended wih custom
methods from implementation
:param client: mqtt client
:param userdata: userdata dict
:param message: message received
"""
# call custom on message methods if any defined
try:
self.logger.debug("calling custom on_message")
self.manager.on_message(client, userdata, message)
except Exception as ex:
if not isinstance(ex, (TypeError, AttributeError)):
self.logger.error("failed to execute custom on_message: %s" % ex)
def connection_status(self):
"""
Returns a connection status code.
:return: connection status code. 0 is not connected, 1 is
waiting for connection and 2 for connected
"""
try:
if self.client.loop() > 0:
return 0
if not self.connected:
return 1
return 2
except Exception:
return 0
def wait_connection(self, timeout=-1):
"""
blocks waiting for connection
"""
connection_status = self.connection_status
reconnect = self.client.reconnect
status = connection_status()
now = datetime.now
limit = now() + timedelta(seconds=timeout)
while status != 2 and (timeout == -1 or now() <= limit):
# reconnects when not connected, status 0
# status 1 should only wait for connection
# instead of reconnecting
self.logger.debug("connection status is %s", str(status))
if status == 0:
try:
self.logger.debug("reconnecting to mqtt")
reconnect()
except Exception as ex:
self.logger.debug("failed to connect mqtt: %s", ex)
self.logger.debug("waiting 1 second for connection")
sleep(1)
status = connection_status()
# noinspection PyProtectedMember
def reconnect(self):
"""
reconnects to mqtt client
:return: connection status
"""
client = self.client
self.logger.info("connecting to %s:%s" % (client._host, client._port))
connection_status = self.connection_status
reconnect = client.reconnect
status = connection_status()
wait = self.wait
while status != 2:
self.logger.debug("connection status is %s", str(status))
if status == 0:
try:
self.logger.debug("reconnecting to mqtt")
reconnect()
except Exception as ex:
self.logger.debug("failed to connect mqtt: %s", ex)
status = connection_status()
if status == 0:
try:
self.logger.debug("calling custom on_not_connect")
self.manager.on_not_connect()
except Exception as ex:
if not isinstance(ex, (TypeError, AttributeError)):
self.logger.error("failed to execute custom on_not_connect: %s" % ex)
if not wait:
return status
sleep(1)
return status
def register(self, metric, icon):
"""
register a new metric using mqtt discovery
:param metric: metric identification
:param icon: metric icon
"""
self.logger.debug("registering metrics %s", metric)
self.client.publish(CONFIG_TOPIC % metric, CONFIG_PAYLOAD % (metric, metric, icon), 1, True)
def publish_state(self, metric, state):
"""
publish state to mqtt
:param metric: metric identification
:param state: state value
"""
self.logger.debug("updating metric %s with state %s", metric, state)
self.client.publish(STATE_TOPIC % metric, state, 1, True)
def loop(self):
"""
calls mqtt client loop
"""
self.client.loop()
class Connector(object):
"""
Connector logic to restart connections
"""
def __init__(self, config, storage, mqtt_client):
"""
initializes connector
:param config: keeper configuration dict
:param storage: storage access
:param mqtt_client: MQTT client
"""
self.attempts = 0
self.was_stable = True
self.command = config["mqtt.restart.command"].split(" ")
self.mqtt_client = None
self.registered = False
self.started_at = datetime.now()
self.time_connected = 0
self.connected_at = None
put = storage.put
self.mqtt_restarts = put(CONNECTOR_MQTT_RESTARTS, storage.get_int(CONNECTOR_MQTT_RESTARTS))
self.failed_connections = put(CONNECTOR_FAILED_CONNECTIONS, storage.get_int(CONNECTOR_FAILED_CONNECTIONS))
self.states_queue = []
self.put = put
self.get = storage.get
self.inc = storage.inc
mqtt_client.set_manager(self)
self.mqtt_client = mqtt_client
self.logger = Logger()
def __enter__(self):
"""
informs when entering context
:return: Connector object
"""
self.logger.info("starting connector manager[pid=%s]" % getpid())
self.mqtt_client.reconnect()
return self
# noinspection PyShadowingBuiltins
def __exit__(self, type, value, traceback):
"""
publishes manager status when exiting context
:param type:
:param value:
:param traceback:
"""
self.logger.info("stopping connector[pid=%s]" % getpid())
try:
self.mqtt_client.publish_state(CONNECTOR_STATUS, STATUS_NOT_RUNNING)
except Exception as ex:
self.logger.error("failed to publish connector status: %s" % ex)
# noinspection PyUnusedLocal
def on_connect(self, client, userdata, flags, rc):
"""
updates connection status on connect
registers sensors and sends metrics
:param client: mqtt client
:param userdata: userdata dict
:param flags: flags
:param rc: rc code
"""
self.connected_at = datetime.now()
# first time we are connected we register metrics and
# send initial values
if not self.registered:
self.logger.info("registering metrics")
try:
publish_state = self.mqtt_client.publish_state
register = self.mqtt_client.register
# register all metrics
register(CONNECTOR_STATUS, CONNECTOR_STATUS_ICON)
register(CONNECTOR_CONNECTION_STATUS, CONNECTOR_CONNECTION_STATUS_ICON)
register(CONNECTOR_MQTT_RESTARTS, CONNECTOR_MQTT_RESTARTS_ICON)
register(CONNECTOR_FAILED_CONNECTIONS, CONNECTOR_FAILED_CONNECTIONS_ICON)
register(CONNECTOR_LAST_MQTT_RESTART, CONNECTOR_LAST_MQTT_RESTART_ICON)
# sends initial values
publish_state(CONNECTOR_STATUS, STATUS_RUNNING)
publish_state(CONNECTOR_CONNECTION_STATUS, CONNECTOR_CONNECTION_OK)
publish_state(CONNECTOR_MQTT_RESTARTS, self.mqtt_restarts)
publish_state(CONNECTOR_FAILED_CONNECTIONS, self.failed_connections)
publish_state(CONNECTOR_LAST_MQTT_RESTART, self.get(CONNECTOR_LAST_MQTT_RESTART))
self.registered = True
except Exception as ex:
self.logger.error("failed to register initial metrics: %s" % ex)
# noinspection PyUnusedLocal
def on_disconnect(self, client, userdata, rc):
"""
updates connection status on disconnect
:param client: mqtt client
:param userdata: userdata dict
:param rc: rc code
"""
self.was_stable = self.is_stable()
self.states_queue.append(
(CONNECTOR_CONNECTION_STATUS, CONNECTOR_CONNECTION_OK if self.was_stable else CONNECTOR_CONNECTION_NOK))
def is_stable(self, update=True):
"""
check if connection is stable by checking if it's up 90% of the time
:param update: whether we should update total time connected
:return: true if connection is stable, false otherwise
"""
now = datetime.now()
if update:
self.time_connected += (now - self.connected_at).total_seconds()
tc = self.time_connected
else:
tc = self.time_connected + (now - self.connected_at).total_seconds()
self.logger.debug("spent %s seconds connected", tc)
return (tc * 100) / (now - self.started_at).total_seconds() >= 90
def on_not_connect(self):
"""
behavior on connect to mqtt
after 3 failed attempts we try to restart mqtt and wait it
to connect again (max 180 seconds)
"""
if self.attempts >= 3:
self.logger.warning("max of 3 connection attempts was reached")
self.logger.warning("restarting mqtt service")
if exec_command(self.command):
append = self.states_queue.append
self.mqtt_restarts = self.inc(CONNECTOR_MQTT_RESTARTS, self.mqtt_restarts)
append((CONNECTOR_MQTT_RESTARTS, self.mqtt_restarts))
append((CONNECTOR_LAST_MQTT_RESTART, self.put(CONNECTOR_LAST_MQTT_RESTART, strftime(TIME_FORMAT))))
self.mqtt_client.wait_connection(60)
self.attempts = 0
else:
self.attempts += 1
self.failed_connections = self.inc(CONNECTOR_FAILED_CONNECTIONS, self.failed_connections)
self.states_queue.append((CONNECTOR_FAILED_CONNECTIONS, self.failed_connections))
self.logger.warning("broker is not responding (%s of 3)" % self.attempts)
sleep(10)
def loop(self):
"""
sleeps 1 second until next validation
sends metrics if any to send
"""
if not self.was_stable:
self.was_stable = self.is_stable(False)
self.states_queue.append(
(CONNECTOR_CONNECTION_STATUS, CONNECTOR_CONNECTION_OK if self.was_stable else CONNECTOR_CONNECTION_NOK))
publish_state = self.mqtt_client.publish_state
try:
for states in self.states_queue:
publish_state(states[0], states[1])
self.states_queue = []
except Exception as ex:
self.logger.warning("unable to update metrics: %s" % ex)
sleep(1)
def test_pick_logger(self):
logger = Logger('runner', console=True, filename='a.log')
cPickle.dump(logger, sys.stdout)