class JNTTServer(JNTTBase):
"""Server base test
"""
server_class = None
server_conf = ""
server_section = None
hadd_ctrl = None
hadds = None
def setUp(self):
JNTTBase.setUp(self)
self.mqttc = None
self.message = None
self.message_received = False
self.hearbeat_mqttc = None
self.heartbeat_message = None
self.heartbeat_waiting = None
self.heartbeat_waitings = None
self.heartbeat_received = False
self.server = None
self.running_server = None
if self.hadd_ctrl is None and self.hadds is not None and len(self.hadds) > 0:
self.hadd_ctrl = self.hadds[0]
def tearDown(self):
self.stopClient()
self.mqttc = None
self.mqtthearbeat = None
self.stopClient()
self.stopServer()
self.mqttc = None
self.message = None
self.message_received = False
self.hearbeat_mqttc = None
self.heartbeat_message = None
self.heartbeat_waiting = None
self.heartbeat_waitings = None
self.heartbeat_received = False
self.server = None
self.running_server = None
JNTTBase.tearDown(self)
def startClient(self, conf=None):
if conf is None:
conf = self.server.options.data
if self.mqttc is None:
self.mqttc = MQTTClient(options=conf)
self.mqttc.connect()
self.mqttc.start()
if self.hearbeat_mqttc is None:
self.hearbeat_mqttc = MQTTClient(options=conf)
self.hearbeat_mqttc.connect()
self.hearbeat_mqttc.start()
self.hearbeat_mqttc.subscribe(topic=TOPIC_HEARTBEAT, callback=self.mqtt_on_heartbeat_message)
def stopClient(self):
if self.mqttc != None:
self.mqttc.stop()
if self.hearbeat_mqttc != None:
self.hearbeat_mqttc.unsubscribe(topic=TOPIC_HEARTBEAT)
self.hearbeat_mqttc.stop()
if self.mqttc != None:
if self.mqttc.is_alive():
self.mqttc.join()
self.mqttc = None
if self.hearbeat_mqttc != None:
if self.hearbeat_mqttc.is_alive():
self.hearbeat_mqttc.join()
self.hearbeat_mqttc = None
def startServer(self):
if self.server is None:
with mock.patch(
"sys.argv", ["%s" % self.server_class, "start", "--conf_file=%s" % self.getDataFile(self.server_conf)]
):
options = vars(jnt_parse_args())
self.server = self.server_class(options)
self.server.start()
self.running_server = threading.Timer(0.01, self.server.run)
self.running_server.start()
time.sleep(1.5)
def stopServer(self):
if self.server is not None:
self.server.stop()
time.sleep(5)
self.server = None
if self.running_server is not None:
self.running_server.cancel()
time.sleep(5)
self.running_server = None
self.message = None
def start(self):
self.startServer()
self.startClient()
def stop(self):
self.stopClient()
self.stopServer()
def mqtt_on_heartbeat_message(self, client, userdata, message):
"""On generic message
"""
self.heartbeat_message = message
hb = HeartbeatMessage(self.heartbeat_message)
hbadd_ctrl, hbadd_node, state = hb.get_heartbeat()
if hbadd_ctrl is not None and hbadd_node is not None:
if self.heartbeat_waiting is None:
if self.heartbeat_waitings is None:
self.heartbeat_received = True
elif HADD % (hbadd_ctrl, hbadd_node) in self.heartbeat_waitings:
self.heartbeat_waitings.remove(HADD % (hbadd_ctrl, hbadd_node))
if len(self.heartbeat_waitings) == 0:
self.heartbeat_received = True
elif self.heartbeat_waiting == HADD % (hbadd_ctrl, hbadd_node):
self.heartbeat_received = True
print "HADD : %s/%s = %s" % (hbadd_ctrl, hbadd_node, state)
def assertInLogfile(self, expr="^ERROR "):
"""Assert an expression is in logifle
Must be called at the end of process, when the server has closed the logfile.
"""
self.assertTrue(self.server_conf is not None)
options = JNTOptions(options={"conf_file": self.getDataFile(self.server_conf)})
log_file_from_config = options.get_option("handler_file", "args", None)
self.assertTrue(log_file_from_config is not None)
# I know, it's bad
log_args = eval(log_file_from_config)
log_file_from_config = log_args[0]
self.assertFile(log_file_from_config)
found = False
with open(log_file_from_config, "r") as hand:
for line in hand:
print line
if re.search(expr, line):
found = True
self.assertTrue(found)
def assertNotInLogfile(self, expr="^ERROR "):
"""Assert an expression is not in logifle.
Must be called at the end of process, when the server has closed the logfile.
"""
self.assertTrue(self.server_conf is not None)
options = JNTOptions(options={"conf_file": self.getDataFile(self.server_conf)})
log_file_from_config = options.get_option("handler_file", "args", None)
self.assertTrue(log_file_from_config is not None)
# I know, it's bad
log_args = eval(log_file_from_config)
log_file_from_config = log_args[0]
self.assertFile(log_file_from_config)
found = False
with open(log_file_from_config, "r") as hand:
for line in hand:
print line
if re.search(expr, line):
found = True
self.assertFalse(found)
def assertHeartbeatNode(self, hadd=None, timeout=60):
print "Waiting for %s" % (hadd)
self.heartbeat_waiting = hadd
self.heartbeat_waitings = None
self.heartbeat_message = None
self.heartbeat_received = False
i = 0
while i < timeout * 10000 and not self.heartbeat_received:
time.sleep(0.0001)
i += 1
self.assertTrue(self.heartbeat_received)
time.sleep(0.5)
def assertHeartbeatNodes(self, hadds=None, timeout=60):
if hadds is None:
hadds = self.hadds
print "Waiting for %s" % (hadds)
self.heartbeat_waiting = None
self.heartbeat_waitings = list(hadds)
self.heartbeat_message = None
self.heartbeat_received = False
i = 0
while i < timeout * 10000 and not self.heartbeat_received:
time.sleep(0.0001)
i += 1
print "Unreceived heartbeats %s" % self.heartbeat_waitings
self.assertTrue(self.heartbeat_received)
time.sleep(0.5)
def waitHeartbeatNodes(self, hadds=None, timeout=60):
if hadds is None:
hadds = self.hadds
print "Waiting for %s" % (hadds)
self.heartbeat_waiting = None
self.heartbeat_waitings = list(hadds)
self.heartbeat_message = None
self.heartbeat_received = False
i = 0
while i < timeout * 10000 and not self.heartbeat_received:
time.sleep(0.0001)
i += 1
print "Unreceived heartbeats %s" % self.heartbeat_waitings
time.sleep(0.5)
def assertNodeRequest(
self,
cmd_class=0,
genre=0x04,
uuid="request_info_nodes",
node_hadd=None,
client_hadd=None,
data=None,
is_writeonly=False,
is_readonly=False,
timeout=5,
):
self.message_received = False
print "Waiting for %s : %s" % (node_hadd, uuid)
def mqtt_on_message(client, userdata, message):
"""On generic message
"""
self.message = message
self.message_received = True
self.mqttc.subscribe(topic=TOPIC_NODES_REPLY % client_hadd, callback=mqtt_on_message)
time.sleep(0.5)
msg = {
"cmd_class": cmd_class,
"genre": genre,
"uuid": uuid,
"reply_hadd": client_hadd,
"data": data,
"hadd": node_hadd,
"is_writeonly": is_writeonly,
"is_readonly": is_readonly,
}
self.mqttc.publish("/nodes/%s/request" % (node_hadd), json_dumps(msg))
i = 0
while i < timeout * 10000 and not self.message_received:
time.sleep(0.0001)
i += 1
self.assertTrue(self.message_received)
self.mqttc.unsubscribe(topic=TOPIC_NODES_REPLY % client_hadd)
time.sleep(0.5)
def assertBroadcastRequest(
self,
cmd_class=0,
genre=0x04,
uuid="request_info_nodes",
node_hadd=None,
client_hadd=None,
data=None,
is_writeonly=False,
is_readonly=False,
timeout=5,
):
self.message_received = False
def mqtt_on_message(client, userdata, message):
"""On generic message
"""
self.message = message
self.message_received = True
self.mqttc.subscribe(topic=TOPIC_BROADCAST_REPLY % client_hadd, callback=mqtt_on_message)
time.sleep(0.5)
msg = {
"cmd_class": cmd_class,
"genre": genre,
"uuid": uuid,
"reply_hadd": client_hadd,
"data": data,
"is_writeonly": is_writeonly,
"is_readonly": is_readonly,
}
self.mqttc.publish(TOPIC_BROADCAST_REQUEST, json_dumps(msg))
i = 0
while i < timeout * 10000 and not self.message_received:
time.sleep(0.0001)
i += 1
self.assertTrue(self.message_received)
self.mqttc.unsubscribe(topic=TOPIC_BROADCAST_REPLY % client_hadd)
time.sleep(0.5)
def assertUpdateValue(
self,
type="user",
data=None,
cmd_class=0,
genre=0x04,
uuid="request_info_nodes",
node_hadd=None,
client_hadd=None,
is_writeonly=False,
is_readonly=False,
timeout=5,
):
self.message_received = False
self.message = None
print "Waiting for %s : %s" % (node_hadd, uuid)
def mqtt_on_message(client, userdata, message):
"""On generic message
"""
msg = json_loads(message.payload)
print "Received message %s" % msg
if msg["uuid"] == uuid and msg["hadd"] == node_hadd:
self.message = message
self.message_received = True
self.mqttc.subscribe(topic="/values/%s/%s/#" % (type, node_hadd), callback=mqtt_on_message)
print "Subscribe to /values/%s/%s/#" % (type, node_hadd)
time.sleep(0.5)
msg = {
"cmd_class": cmd_class,
"genre": genre,
"uuid": uuid,
"reply_hadd": client_hadd,
"data": data,
"hadd": node_hadd,
"is_writeonly": is_writeonly,
"is_readonly": is_readonly,
}
self.mqttc.publish("/nodes/%s/request" % (node_hadd), json_dumps(msg))
i = 0
while i < timeout * 10000 and not self.message_received:
time.sleep(0.0001)
i += 1
self.assertTrue(self.message_received)
self.assertTrue(self.message is not None)
self.assertTrue(self.message.payload is not None)
if data is not None:
msg = json_loads(self.message.payload)
self.assertEqual(msg["data"], data)
self.mqttc.unsubscribe(topic="/values/%s/%s/#" % (type, node_hadd))
time.sleep(0.5)
def assertNotUpdateValue(
self,
type="user",
data=None,
cmd_class=0,
genre=0x04,
uuid="request_info_nodes",
node_hadd=None,
client_hadd=None,
is_writeonly=False,
is_readonly=False,
timeout=5,
):
self.message_received = False
self.message = None
print "Waiting for %s : %s" % (node_hadd, uuid)
def mqtt_on_message(client, userdata, message):
"""On generic message
"""
msg = json_loads(message.payload)
print "Received message %s" % msg
if msg["uuid"] == uuid and msg["hadd"] == node_hadd:
self.message = message
self.message_received = True
self.mqttc.subscribe(topic="/values/%s/%s/#" % (type, node_hadd), callback=mqtt_on_message)
print "Subscribe to /values/%s/%s/#" % (type, node_hadd)
time.sleep(0.5)
msg = {
"cmd_class": cmd_class,
"genre": genre,
"uuid": uuid,
"reply_hadd": client_hadd,
"data": data,
"hadd": node_hadd,
"is_writeonly": is_writeonly,
"is_readonly": is_readonly,
}
self.mqttc.publish("/nodes/%s/request" % (node_hadd), json_dumps(msg))
i = 0
while i < timeout * 10000 and not self.message_received:
time.sleep(0.0001)
i += 1
self.assertTrue(self.message is None)
self.assertFalse(self.message_received)
self.mqttc.unsubscribe(topic="/values/%s/%s/#" % (type, node_hadd))
time.sleep(0.5)
def assertWaitValue(
self,
type="user",
data=None,
cmd_class=0,
genre=0x04,
uuid="request_info_nodes",
node_hadd=None,
client_hadd=None,
is_writeonly=False,
is_readonly=False,
timeout=5,
):
self.message_received = False
self.message = None
print "Waiting for %s : %s" % (node_hadd, uuid)
def mqtt_on_message(client, userdata, message):
"""On generic message
"""
msg = json_loads(message.payload)
print "Received message %s" % msg
if msg["uuid"] == uuid and msg["hadd"] == node_hadd:
self.message = message
self.message_received = True
self.mqttc.subscribe(topic="/values/%s/%s/#" % (type, node_hadd), callback=mqtt_on_message)
print "Subscribe to /values/%s/%s/#" % (type, node_hadd)
time.sleep(0.5)
i = 0
while i < timeout * 10000 and not self.message_received:
time.sleep(0.0001)
i += 1
self.assertTrue(self.message_received)
self.assertTrue(self.message is not None)
self.assertTrue(self.message.payload is not None)
if data is not None:
msg = json_loads(self.message.payload)
self.assertEqual(msg["data"], data)
self.mqttc.unsubscribe(topic="/values/%s/%s/#" % (type, node_hadd))
time.sleep(0.5)
def assertNotWaitValue(
self,
type="user",
data=None,
cmd_class=0,
genre=0x04,
uuid="request_info_nodes",
node_hadd=None,
client_hadd=None,
is_writeonly=False,
is_readonly=False,
timeout=5,
):
self.message_received = False
self.message = None
print "Waiting for %s : %s" % (node_hadd, uuid)
def mqtt_on_message(client, userdata, message):
"""On generic message
"""
msg = json_loads(message.payload)
print "Received message %s" % msg
if msg["uuid"] == uuid and msg["hadd"] == node_hadd:
self.message = message
self.message_received = True
self.mqttc.subscribe(topic="/values/%s/%s/#" % (type, node_hadd), callback=mqtt_on_message)
print "Subscribe to /values/%s/%s/#" % (type, node_hadd)
time.sleep(0.5)
i = 0
while i < timeout * 10000 and not self.message_received:
time.sleep(0.0001)
i += 1
self.assertTrue(self.message is None)
self.assertFalse(self.message_received)
self.mqttc.unsubscribe(topic="/values/%s/%s/#" % (type, node_hadd))
time.sleep(0.5)
def assertFsmBoot(self, state="booting"):
"""Assert Finish State Machine can boot
"""
thread = self.server.find_thread(self.server_section)
self.assertNotEqual(thread, None)
JNTTBase.assertFsmBoot(self, bus=thread.bus, state="booting")
class RemoteNodeComponent(JNTComponent):
""" A resource ie /rrd """
def __init__(self, bus=None, addr=None, **kwargs):
"""
"""
oid = kwargs.pop('oid', 'remote.node')
name = kwargs.pop('name', "Remote node")
product_name = kwargs.pop('product_name', "Remote node")
product_type = kwargs.pop('product_type', "Software")
JNTComponent.__init__(self, oid=oid, bus=bus, addr=addr, name=name,
product_name=product_name, product_type=product_type, **kwargs)
logger.debug("[%s] - __init__ node uuid:%s", self.__class__.__name__, self.uuid)
self.mqttc_heartbeat = None
self.mqttc_values = None
self.state = 'OFFLINE'
self.remote_hadd = (None,None)
uuid="remote_hadd"
self.values[uuid] = self.value_factory['config_string'](options=self.options, uuid=uuid,
node_uuid=self.uuid,
help='HADD of the remote node that we will listen to',
label='rhadd',
default=None,
)
uuid="user_read"
self.values[uuid] = self.value_factory['rread_value'](options=self.options, uuid=uuid,
node_uuid=self.uuid,
help='The user values to listen to : value_uuid:index',
label='ruser',
default=None,
)
uuid="user_write"
self.values[uuid] = self.value_factory['rwrite_value'](options=self.options, uuid=uuid,
node_uuid=self.uuid,
help='The user values to listen to : value_uuid:index',
label='wuser',
default=None,
)
uuid="basic_read"
self.values[uuid] = self.value_factory['rread_value'](options=self.options, uuid=uuid,
node_uuid=self.uuid,
help='The basic values to listen to : value_uuid:index',
label='rbasic',
default=None,
)
uuid="basic_write"
self.values[uuid] = self.value_factory['rwrite_value'](options=self.options, uuid=uuid,
node_uuid=self.uuid,
help='The basic values to listen to : value_uuid:index',
label='wbasic',
default=None,
)
def start(self, mqttc):
"""Start the component.
"""
self.state = 'BOOT'
JNTComponent.start(self, mqttc)
hadd = self.values['remote_hadd'].data
logger.debug("[%s] - Found remote HADD %s for node %s", self.__class__.__name__, hadd, self.node.uuid)
if hadd is None:
logger.debug("[%s] - No remote HADD. Exit ...", self.__class__.__name__)
return False
self.remote_hadd = hadd_split(hadd)
if self.remote_hadd[0] is None or self.remote_hadd[1] is None:
logger.warning("[%s] - Bad remote HADD %s", self.__class__.__name__, hadd)
return False
try:
self.mqttc_heartbeat = MQTTClient(options=self.options.data)
self.mqttc_heartbeat.connect()
self.mqttc_heartbeat.subscribe(topic=TOPIC_HEARTBEAT_NODE%(hadd), callback=self.on_heartbeat)
self.mqttc_heartbeat.start()
except Exception:
logger.exception("[%s] - start", self.__class__.__name__)
values_read = self.get_read_values()
values_write = self.get_write_values()
logger.debug("[%s] - found %s values_read", self.__class__.__name__, len(values_read))
logger.debug("[%s] - found %s values_write", self.__class__.__name__, len(values_write))
topics = []
for value in values_read:
if value[0] == 'user':
topic=TOPIC_VALUES_USER
else:
topic=TOPIC_VALUES_BASIC
topic = topic%(hadd+'/'+value[1])
topics.append((topic, 0))
logger.debug("[%s] - subscribe to %s", self.__class__.__name__, topic)
if len(topics)>0:
try:
self.mqttc_values = MQTTClient(options=self.options.data)
self.mqttc_values.connect()
self.mqttc_values.subscribe(topics=topics, callback=self.on_remote_value)
self.mqttc_values.start()
except Exception:
logger.exception("[%s] - start", self.__class__.__name__)
#~ print max_index
#~ for index in range(max_index):
#~ print index
return True
def get_read_values(self):
"""Return all the read values.
"""
values = []
for which in ['user_read', 'basic_read']:
nb_instances = self.values[which].get_length()
for i in range(nb_instances):
vuuid, vindex = self.values[which].get_value_config(index=i)
values.append((which.replace('_read',''), vuuid, int(vindex)))
return values
def get_write_values(self):
"""Return all the read values.
"""
values = []
for which in ['user_write', 'basic_write']:
nb_instances = self.values[which].get_length()
for i in range(nb_instances):
vuuid, vindex, cmdcls, ston, stoff = self.values[which].get_value_config(index=i)
values.append((which.replace('_write',''), vuuid, int(vindex), cmdcls, ston, stoff))
return values
def stop(self):
"""Stop the component.
"""
if self.mqttc_values is not None:
try:
hadd = HADD%(self.remote_hadd[0], self.remote_hadd[1])
values_read = self.get_read_values()
topics = []
for value in values_read:
if value[0] == 'user':
topic=TOPIC_VALUES_USER
else:
topic=TOPIC_VALUES_BASIC
topic = topic%(HADD%(self.remote_hadd)+'/'+value[1])
topics.append(topic)
logger.debug("[%s] - Unsubscribe to %s", self.__class__.__name__, topics)
self.mqttc_values.unsubscribe(topics)
self.mqttc_values.stop()
if self.mqttc_values.is_alive():
self.mqttc_values.join()
self.mqttc_values = None
except Exception:
logger.exception("[%s] - stop", self.__class__.__name__)
if self.mqttc_heartbeat is not None:
try:
hadd = HADD%(self.remote_hadd[0], self.remote_hadd[1])
logger.debug("[%s] - Unsubscribe to %s", self.__class__.__name__, TOPIC_HEARTBEAT_NODE%(hadd))
self.mqttc_heartbeat.unsubscribe(topic=TOPIC_HEARTBEAT_NODE%(hadd))
self.mqttc_heartbeat.stop()
if self.mqttc_heartbeat.is_alive():
self.mqttc_heartbeat.join()
self.mqttc_heartbeat = None
except Exception:
logger.exception("[%s] - stop", self.__class__.__name__)
JNTComponent.stop(self)
return True
def check_heartbeat(self):
"""Check that the component is 'available'
"""
#~ print "it's me %s : %s" % (self.values['upsname'].data, self._ups_stats_last)
if self.mqttc_heartbeat is not None:
return self.state
return False
def on_heartbeat(self, client, userdata, message):
"""On request
:param client: the Client instance that is calling the callback.
:type client: paho.mqtt.client.Client
:param userdata: user data of any type and can be set when creating a new client instance or with user_data_set(userdata).
:type userdata: all
:param message: The message variable is a MQTTMessage that describes all of the message parameters.
:type message: paho.mqtt.client.MQTTMessage
"""
hb = HeartbeatMessage(message)
add_ctrl, add_node, state = hb.get_heartbeat()
if add_ctrl is None or add_node is None:
return
if (add_ctrl == self.remote_hadd[0]) and \
(add_node == self.remote_hadd[1] or add_node == -1) :
self.state = state
def on_remote_value(self, client, userdata, message):
"""On request
:param client: the Client instance that is calling the callback.
:type client: paho.mqtt.client.Client
:param userdata: user data of any type and can be set when creating a new client instance or with user_data_set(userdata).
:type userdata: all
:param message: The message variable is a MQTTMessage that describes all of the message parameters.
:type message: paho.mqtt.client.MQTTMessage
"""
pass
class RrdStoreThread(BaseThread):
"""The Rdd cache thread
Implement a cache.
"""
def __init__(self, section, options={}):
"""Initialise the cache thread
Manage a cache for the rrd.
A timer in a separated thread will pickle the cache to disk every 30 seconds.
An other thread will update the rrd every hours
:param options: The options used to start the worker.
:type clientid: str
"""
self.section = section
BaseThread.__init__(self, options=options)
self.config_timeout_delay = 1
self.loop_sleep = 0.01
self._lock = threading.Lock()
#~ self._cache_rrd_ttl = 60*60
self._cache_rrd_ttl = 60*60
self._cache_pickle_ttl = 240
self._cache_pickle_timer = None
self._cache_dead_ttl = 60*60*24
self._cache_dead_timer = None
self._thread_delay = 0.01
self._rrd_rotate_next_run = None
self._rrd_rotate_ttl = 1
self._dirmask = None
self._filemask = None
self._cache = {}
"""
whe have a datasource with a unique_name haddnode but at component start, we don't have it.
whe receive data from mqtt with params : haddvalue, value_uuid, value_index at etime
whe should update the rrd with : rtime:val1:val2:val3 ...
Whe should be abble to flush a rrd file for graphing
cache { 'rrd_0001' : { 'step':300, 'values' : {'epoch_time' : { 0 : 'U', ...}}, 'labels':{0:...}}
When updating a data, we can find the latest serie values filtering on epoch_time
We should look at the cache to find rrd_files that need a new serie of values (now - last_epoch) > step.
In this case, we add a new serie of with 'U' as values.
hadd
value_uuid
value_index
"""
self.epoch = datetime.datetime(1970,1,1)
self._mqttc = None
"""
"""
self.params = {}
def config_thread(self, cache_rrd_ttl=None, cache_pickle_ttl=None, cache_dead_ttl=None,
dirmask=RRD_DIR, filemask='%(rrd_file)s.rrd'):
"""
"""
if dirmask is not None:
self._dirmask = dirmask
if filemask is not None:
self._filemask = filemask
if cache_rrd_ttl is not None:
self._cache_rrd_ttl = cache_rrd_ttl
if cache_dead_ttl is not None:
self._cache_dead_ttl = cache_dead_ttl
if cache_pickle_ttl is not None:
self._cache_pickle_ttl = cache_pickle_ttl
def pre_loop(self):
"""Launch before entering the run loop. The node manager is available.
"""
self._mqttc = MQTTClient(options=self.options.data)
self._mqttc.connect()
self._mqttc.subscribe(topic=TOPIC_VALUES, callback=self.on_value)
self._mqttc.start()
self.restore()
self.start_timer_pickle()
self._rrd_rotate_next_run = datetime.datetime.now()
self.start_timer_dead()
def post_loop(self):
"""Launch after finishing the run loop. The node manager is still available.
"""
logger.debug("[%s] - stop timers in postloop", self.__class__.__name__)
self.stop_timer_pickle()
self.stop_timer_dead()
self._mqttc.unsubscribe(topic=TOPIC_VALUES)
self._mqttc.stop()
#~ self.flush_all()
self.dump()
if self._mqttc.is_alive():
self._mqttc.join()
self._mqttc = None
def loop(self):
"""Launch after finishing the run loop. The node manager is still available.
"""
if self._rrd_rotate_next_run < datetime.datetime.now():
now = datetime.datetime.now()
#Check for data that need a rotation
etime = (datetime.datetime.now() - self.epoch).total_seconds()
try:
for key in self._cache.keys():
try:
epochs = sorted(self._cache[key]['values'].keys())
#~ print "epochs : ", epochs
#~ print "condition : ", epochs[0] + self._cache_rrd_ttl, etime
if len(epochs) == 0 or etime > epochs[-1] + self._cache[key]['step']:
#We should rotate the values
self.timer_rrd_rotate(key, etime)
elif len(epochs) > 1 and epochs[0] + self._cache_rrd_ttl < etime :
#We should flush to the rrd
self.flush(key)
except Exception:
logger.exception("[%s] - Exception when rotating %s in cache", self.__class__.__name__, key)
except Exception:
logger.exception("[%s] - Exception when rotating in cache", self.__class__.__name__)
self._rrd_rotate_next_run = datetime.datetime.now() + datetime.timedelta(seconds=self._rrd_rotate_ttl)
self._reloadevent.wait(self.loop_sleep)
def on_value(self, client, userdata, message):
"""On value
Do not lock as it don't add new values to dict. Should be ok using keys instead of iterator.
:param client: the Client instance that is calling the callback.
:type client: paho.mqtt.client.Client
:param userdata: user data of any type and can be set when creating a new client instance or with user_data_set(userdata).
:type userdata: all
:param message: The message variable is a MQTTMessage that describes all of the message parameters.
:type message: paho.mqtt.client.MQTTMessage
"""
#logger.debug("[%s] - on_value %s", self.__class__.__name__, message.payload)
try:
data = json_loads(message.payload)
if 'genre' in data:
data = {0:{0:data}}
elif 'genre' in data[data.keys()[0]]:
data = {0:data}
store_index = self.create_store_index()
for nval in data:
for kval in data[nval]:
if data[nval][kval]['genre'] in [0x02, 0x01]:
hadd = data[nval][kval]['hadd']
uuid = data[nval][kval]['uuid']
index = 0
if 'index' in data[nval][kval]:
index = data[nval][kval]['index']
index = str(index)
#~ logger.debug("[%s] - update_last %s,%s,%s : %s", self.__class__.__name__, hadd, value_uuid, value_index)
if (hadd, uuid, index) in store_index:
self.update_last(hadd, uuid, index, data[nval][kval]['data'])
except Exception:
logger.exception("[%s] - Exception in on_value", self.__class__.__name__)
def create_store_index(self):
""" Create an in dex of keys
:ret: a list of tuple () of values in cache
"""
ret = []
rrds = self._cache.keys()
for rrd in rrds:
try:
indexes = self._cache[rrd]['indexes'].keys()
for index in indexes:
try:
ret.append( (self._cache[rrd]['hadds'][index], \
self._cache[rrd]['uuids'][index], \
self._cache[rrd]['indexes'][index]) )
except Exception:
logger.exception('[%s] - Exception in create_store_index : rrd= %s, index = %s', self.__class__.__name__, rrd, index)
except Exception:
logger.exception('[%s] - Exception in create_store_index : rrd = %s', self.__class__.__name__, rrd)
#~ logger.debug("[%s] - create_store_index %s", self.__class__.__name__, ret)
return ret
def update_last(self, hadd, value_uuid, value_index, data):
""" An helper to find
:ret:
"""
#~ logger.debug("[%s] - update_last %s,%s,%s : %s", self.__class__.__name__, hadd, value_uuid, value_index, data)
ret = []
rrds = self._cache.keys()
for rrd in rrds:
indexes = self._cache[rrd]['indexes'].keys()
for index in indexes:
if self._cache[rrd]['hadds'][index]==hadd and \
self._cache[rrd]['uuids'][index]==value_uuid and \
self._cache[rrd]['indexes'][index]==value_index:
epochs = sorted(self._cache[rrd]['values'].keys())
if len(epochs) == 0:
logger.warning("[%s] - Can't update value. No epoch found for %s", self.__class__.__name__, rrd)
else:
if data is None:
data = 'U'
self._cache[rrd]['values'][epochs[-1]][index] = data
self._cache[rrd]['last_update'] = datetime.datetime.now()
logger.debug("[%s] - Value updated in store %s,%s,%s : %s", self.__class__.__name__, hadd, value_uuid, value_index, data)
def timer_rrd_rotate(self, rrd_file, etime=None):
"""Rotate via a separate thread in a timer
"""
if rrd_file is None or rrd_file not in self._cache:
return False
if etime is None:
etime = (datetime.datetime.now() - self.epoch).total_seconds()
th = threading.Timer(self._thread_delay, self.rrd_rotate, args=(rrd_file, etime))
th.start()
def rrd_rotate(self, rrd_file, etime=None):
"""Rotate
"""
if etime is None:
etime = (datetime.datetime.now() - self.epoch).total_seconds()
logger.debug("[%s] - Rotate the rrd data in cache", self.__class__.__name__)
self._lock.acquire()
try:
if rrd_file is None or rrd_file not in self._cache:
return False
self._cache[rrd_file]['values'][etime] = {}
for key in self._cache[rrd_file]['hadds'].keys():
self._cache[rrd_file]['values'][etime][key]='U'
except Exception:
logger.exception("[%s] - Exception when rotating %s in cache", self.__class__.__name__, rrd_file)
finally:
self._lock.release()
def run(self):
"""Run the loop
"""
self._stopevent.clear()
#~ self.boot()
self.trigger_reload()
logger.debug("[%s] - Wait for the thread reload event for initial startup", self.__class__.__name__)
while not self._reloadevent.isSet() and not self._stopevent.isSet():
self._reloadevent.wait(0.50)
logger.debug("[%s] - Entering the thread loop", self.__class__.__name__)
while not self._stopevent.isSet():
self._reloadevent.clear()
try:
self.pre_loop()
except Exception:
logger.exception('[%s] - Exception in pre_loop', self.__class__.__name__)
self._stopevent.set()
while not self._reloadevent.isSet() and not self._stopevent.isSet():
self.loop()
try:
self.post_loop()
except Exception:
logger.exception('[%s] - Exception in post_loop', self.__class__.__name__)
def get_rrd_directory(self, params):
"""Get and create the direcotry if needed
"""
dirname='.'
if 'home_dir' in self.options.data and self.options.data['home_dir'] is not None:
dirname = self.options.data['home_dir']
directory = os.path.join(dirname, self._dirmask %(params))
if not os.path.exists(directory):
os.makedirs(directory)
return directory
def get_rrd_public_directory(self, params):
"""Get and create the direcotry if needed
"""
dirname='.'
if 'home_dir' in self.options.data and self.options.data['home_dir'] is not None:
dirname = self.options.data['home_dir']
dirname = os.path.join(dirname, "public")
directory = os.path.join(dirname, 'rrd', 'rrds')
if not os.path.exists(directory):
os.makedirs(directory)
return directory
def get_rrd_file(self, params):
"""
"""
return self._filemask %(params)
def get_pickle_filename(self):
"""Restore data from disk using pickle
"""
params={"rrd_file":''}
dirname = self.get_rrd_directory(params)
return os.path.join(dirname, 'rrd_cache.pickle')
def get_list_filename(self):
"""Restore data from disk using pickle
"""
params={"rrd_file":''}
directory = self.get_rrd_public_directory(params)
return os.path.join(directory, "index.txt")
def get_rrd_filename(self, rrd_file):
"""
"""
if rrd_file is None:
logger.debug("Can't retrieve rrd_file %s", rrd_file)
return None
params={"rrd_file":rrd_file}
directory = self.get_rrd_public_directory(params)
filename = self.get_rrd_file(params)
return os.path.join(directory, filename)
def get_rrd_httpname(self, rrd_file):
"""
"""
if rrd_file is None:
logger.debug("Can't retrieve rrd_file %s", rrd_file)
return None
params={"rrd_file":rrd_file}
directory = self._dirmask %(params)
filename = self.get_rrd_file(params)
return os.path.join(directory, 'rrds', filename)
def get_rrd_label(self, index, rrd_file, config):
"""
"""
if config is None:
if rrd_file in self._cache and index in self._cache[rrd_file]["labels"]:
return self._cache[rrd_file]["labels"][index]
try:
hadd = None
hadd, value_uuid, value_index, rrd_type, rrd_label = config.split('|')
self._cache[rrd_file]["labels"][index] = rrd_label
return rrd_label
except Exception:
logger.exception("[%s] - Can't retrieve add_ctrl, add_node from hadd %s", self.__class__.__name__, hadd)
return None
def dump(self):
"""Dump data to disk using pickle
"""
self._lock.acquire()
logger.debug("[%s] - Dump cache to disk")
try:
filename = self.get_pickle_filename()
pickle.dump( self._cache, open( filename, "wb" ) )
except Exception:
logger.exception("[%s] - Exception when dumping data to file", self.__class__.__name__)
finally:
self._lock.release()
def restore(self):
"""Restore data from disk using pickle
"""
self._lock.acquire()
logger.debug("[%s] - Restore cache from disk")
try:
filename = self.get_pickle_filename()
if os.path.exists(filename):
self._cache = pickle.load( open( filename, "rb" ) )
except Exception:
self._cache = {}
logger.exception("[%s] - Exception when restoring data from dump", self.__class__.__name__)
finally:
self._lock.release()
def start_timer_dead(self):
"""
"""
if self._cache_dead_timer is None:
self._cache_dead_timer = threading.Timer(self._cache_dead_ttl / 3, self.timer_dead)
self._cache_dead_timer.start()
def timer_dead(self):
"""Remove dead entries from cache
"""
self.stop_timer_dead()
logger.debug("[%s] - Remove dead entries in cache", self.__class__.__name__)
try:
now = datetime.datetime.now()
dead_time = now - datetime.timedelta(seconds=self._cache_dead_ttl)
for key in self._cache.keys():
self._lock.acquire()
if 'last_update' not in self._cache[key]:
self._cache[key]['last_update'] = now
try:
if key in self._cache and self._cache[key]['last_update'] < dead_time:
logger.debug("[%s] - Remove dead entries in cache : %s", self.__class__.__name__, key)
self.remove_rrd_from_list(key)
del self._cache[key]
except Exception:
logger.exception("[%s] - Exception when removing dead entry %s in cache", self.__class__.__name__, key)
finally:
self._lock.release()
except Exception:
logger.exception("[%s] - Exception when removing dead entries", self.__class__.__name__)
self.start_timer_dead()
def stop_timer_dead(self):
"""
"""
if self._cache_dead_timer is not None:
self._cache_dead_timer.cancel()
self._cache_dead_timer = None
def start_timer_pickle(self):
"""
"""
if self._cache_pickle_timer is None:
self._cache_pickle_timer = threading.Timer(self._cache_pickle_ttl, self.timer_pickle)
self._cache_pickle_timer.start()
def timer_pickle(self):
"""Dump cache to file using pickle
"""
self.stop_timer_pickle()
self.dump()
self.start_timer_pickle()
def stop_timer_pickle(self):
"""
"""
if self._cache_pickle_timer is not None:
self._cache_pickle_timer.cancel()
self._cache_pickle_timer = None
def timer_flush_all(self):
"""Flush all data via a separate thread in a timer
"""
if hadd is None or value_uuid is None or value_index is None:
return False
th = threading.Timer(self._thread_delay, self.flush_all, args=(hadd, value_uuid, value_index))
th.start()
def flush_all(self):
"""Flush all data to rrd files and remove them from cache
"""
rrds = self._cache.keys()
for rrd in rrds:
try:
self.flush(rrd)
except Exception:
logger.exception("[%s] - Exception in flush_all : rrd = %s", self.__class__.__name__, rrd)
def timer_flush(self, rrd_file):
"""Flush data from in cache from a value via a separate thread in a timer
"""
if hadd is None or value_uuid is None or value_index is None:
return False
th = threading.Timer(self._thread_delay, self.flush, args=(rrd_file))
th.start()
def flush(self, rrd_file):
"""Flush data from a value to rrd file and remove them from cache
"""
if rrd_file is None or rrd_file not in self._cache:
return False
self._lock.acquire()
logger.info("[%s] - Flush rrd_file %s", self.__class__.__name__, rrd_file)
try:
rrd_dict = self._cache[rrd_file]
epochs = sorted(rrd_dict['values'].keys())
rrd_data = [self.get_rrd_filename(rrd_file)]
last_epoch = epochs[-1]
for epoch in epochs:
try:
rrd_line = ""
if epoch != last_epoch:
#We must let the last serie in cache
#Otherwise we could raise :
# error: /tmp/janitoo/home/public/datarrd_store/rrd/open_files.rrd: illegal attempt to update using time 1443837167 when last
# update time is 1443837240 (minimum one second step)
rrd_line = '%s' %(epoch)
for key_idx in rrd_dict['values'][epoch]:
try:
val = 'U'
if rrd_dict['values'][epoch][key_idx] is not None:
val = rrd_dict['values'][epoch][key_idx]
rrd_line = '%s:%s' %(rrd_line, val)
except Exception:
rrd_line = '%s:%s' %(rrd_line, 'U')
logger.exception("[%s] - Exception when flushing cache for %s epoch %s:%s", self.__class__.__name__, rrd_file, epoch, key_idx)
del self._cache[rrd_file]['values'][epoch]
if rrd_line != "":
rrd_data.append(rrd_line)
except Exception:
logger.exception("[%s] - Exception when flushing cache for %s epoch %s", self.__class__.__name__, rrd_file, epoch)
if len (rrd_data) > 1:
rrdtool.update(rrd_data)
except Exception:
logger.exception("[%s] - Exception when flushing cache for %s", self.__class__.__name__, rrd_file)
finally:
self._lock.release()
def get_count_values(self):
"""Retrieve the number of values cached
"""
return len(self._cache)
def get_count_series(self):
"""Retrieve the number of series of values cached
"""
numb=0
for rrd_file in self._cache.keys():
numb += len(self._cache[rrd_file]['values'])
return numb
def get_values_to_dump(self):
"""Return a list of tuples (hadd, value_uuid, value_index) of values in timeout. They must be flush to disk
"""
return 0
def remove_config(self, rrd_file):
"""Remove an entry in cache and its rrd file
"""
if rrd_file not in self._cache:
logger.warning("[%s] - Remove a non existent entry [%s] from cache ", self.__class__.__name__, rrd_file)
if len(self._cache[rrd_file]['values']) > 0:
logger.warning("[%s] - Remove a non empty entry [%s] from cache : %s ", self.__class__.__name__, rrd_file, self._cache[rrd_file])
self._lock.acquire()
try:
filename = self.get_rrd_filename(rrd_file)
if os.path.exists(filename) == True:
os.remove(filename)
if rrd_file is not None and rrd_file in self._cache:
del self._cache[rrd_file]
except Exception:
logger.exception("[%s] - Exception when removing config", self.__class__.__name__)
finally:
self._lock.release()
def timer_add_config(self, rrd_file, step, config):
"""Add a config via a separate thread in a timer
"""
if rrd_file is None or step is None or config is None:
return False
th = threading.Timer(self._thread_delay*5, self.add_config, args=(rrd_file, step, config))
th.start()
def add_config(self, rrd_file, step, config):
"""
ret = rrdtool.create("example.rrd", "--step", "1800", "--start", '0',
"DS:metric1:GAUGE:2000:U:U",
"DS:metric2:GAUGE:2000:U:U",
"RRA:AVERAGE:0.5:1:600",
"RRA:AVERAGE:0.5:6:700",
"RRA:AVERAGE:0.5:24:775",
"RRA:AVERAGE:0.5:288:797",
"RRA:MAX:0.5:1:600",
"RRA:MAX:0.5:6:700",
"RRA:MAX:0.5:24:775",
"RRA:MAX:0.5:444:797")
Let’s consider all lines in details. First line include name of RRD database (“example.rrd”) and you can use here any path you want,
step of parameters checking (30 minutes in our case), and the start point (0 or N means ‘now’).
‘DS’ in line 4-5 means Data Source, these lines include two our metrics.
‘2000’ means that RRD can wait for 2000 seconds to get new values until it considers them as unknown
(that’s is why we use 2000, which 200 seconds more of our 30 minutes interval).
Last two parameters – ‘U:U’ – stand for min and max values of each metric (‘unknown’ in our case).
Lines 6-13 describe what types of gained values RRD should store in its database.
It’s pretty self-describing (average and max values). Mentioned values describe how many parameters RRD should keep.
Considering it can be confusing I will omit explanation but note that these values were choosen to be compatible with MRTG (actually, it’s not quite true
since we use 1800 seconds periods and not 5 minutes, so you might need to modify it (if you also don’t use 5 minutes period) or keep like I did).
GAUGE, COUNTER, DERIVE or ABSOLUTE
"""
#print "add_config", rrd_file, config
self._lock.acquire()
rrd_sources = []
try:
if rrd_file is None:
logger.warning("[%s] - Can't add %s in cache", self.__class__.__name__, rrd_file)
return False
if rrd_file not in self._cache:
self._cache[rrd_file] = {'step':step, 'last_update':datetime.datetime.now(), 'labels' : {}, 'types':{}, 'hadds':{}, 'uuids':{}, 'indexes':{}, 'values':{}}
for key in sorted(config.keys()):
hadd, value_uuid, value_index, rrd_type, rrd_label = config[key].split('|')
self._cache[rrd_file]["labels"][key] = rrd_label
self._cache[rrd_file]["types"][key] = rrd_type
self._cache[rrd_file]["hadds"][key] = hadd
self._cache[rrd_file]["uuids"][key] = value_uuid
self._cache[rrd_file]["indexes"][key] = value_index
rrd_sources.append("DS:%s:%s:%s:U:U" %(rrd_label, rrd_type, step*2))
except Exception:
logger.exception("[%s] - Exception when adding config in cache", self.__class__.__name__)
finally:
self._lock.release()
#print "rrd_sources :", rrd_sources
try:
filename = self.get_rrd_filename(rrd_file)
if os.path.exists(filename) == False:
rrdtool.create(filename, "--step", str(step), "--start", '0',
rrd_sources,
"RRA:AVERAGE:0.5:1:1440",
"RRA:AVERAGE:0.5:12:1440",
"RRA:AVERAGE:0.5:144:1440",
"RRA:AVERAGE:0.5:288:1440",
"RRA:MAX:0.5:1:1440",
"RRA:MAX:0.5:12:1440",
"RRA:MAX:0.5:144:1440",
"RRA:MAX:0.5:288:1440",
"RRA:MIN:0.5:1:1440",
"RRA:MIN:0.5:12:1440",
"RRA:MIN:0.5:144:1440",
"RRA:MIN:0.5:288:1440")
self.add_rrd_to_list(rrd_file)
except Exception:
logger.exception("[%s] - Exception when creating rrd file %s", self.__class__.__name__, rrd_file)
def add_rrd_to_list(self, rrd_file):
"""Add the rrd_file to index.txt
"""
filename = self.get_list_filename()
rrd_list = []
if os.path.exists(filename) == True:
with open(filename) as file: # Use file to refer to the file object
data = file.read()
rrd_list = data.split("|")
if rrd_file in rrd_list:
return
rrd_list.append(rrd_file)
line = '|'.join(rrd_list)
with open(filename, "w") as file:
file.write(line)
def remove_rrd_from_list(self, rrd_file):
"""Remove the rrd from index.txt
"""
filename = self.get_list_filename()
rrd_list = []
if os.path.exists(filename) == True:
with open(filename) as file: # Use file to refer to the file object
data = file.read()
rrd_list = data.split("|")
if rrd_file not in rrd_list:
return
rrd_list.remove(rrd_file)
line = '|'.join(rrd_list)
with open(filename, "w") as file:
file.write(line)
