def init_mqtt(self):
self.mqtt = None
try:
if "pub.mqtt.host" in dict(self.config.items("IO")):
self.host = self.config.get("IO", "pub.mqtt.host")
else:
self.host = self.config.get("IO", "mqtt.host")
self.port = self.config.get("IO", "mqtt.port")
if "mqtt.port" in self.topic_params.keys():
self.port = self.topic_params["mqtt.port"]
if "qos" in self.topic_params.keys():
self.qos = int(self.topic_params["qos"])
else:
self.qos = 1
self.client_id = "client_publish" + str(randrange(100000)) + str(time.time()).replace(".","")
self.mqtt = MQTTClient(str(self.host), self.port, self.client_id,
username=self.config.get("IO", "mqtt.username", fallback=None),
password=self.config.get("IO", "mqtt.password", fallback=None),
ca_cert_path=self.config.get("IO", "mqtt.ca.cert.path", fallback=None),
set_insecure=bool(self.config.get("IO", "mqtt.insecure.flag", fallback=False)),
id=self.id)
except Exception as e:
self.logger.error(e)
# error for mqtt will be caught at parent
raise e
def init_mqtt(self):
try:
client_id = "client_publish" + str(randrange(100000)) + str(
time.time()).replace(".", "")
self.mqtt = MQTTClient(str(self.host),
self.port,
client_id,
id=self.id,
connect_check_flag=True)
self.logger.info("successfully subscribed")
except Exception as e:
self.logger.error(e)
def init_mqtt(self):
try:
if "pub.mqtt.host" in dict(self.config.items("IO")):
host = self.config.get("IO", "pub.mqtt.host")
else:
host = self.config.get("IO", "mqtt.host")
port = self.config.get("IO", "mqtt.port")
client_id = "client_publish" + str(randrange(100000)) + str(
time.time()).replace(".", "")
mqtt = MQTTClient(str(host),
port,
client_id,
username=self.config.get("IO",
"mqtt.username",
fallback=None),
password=self.config.get("IO",
"mqtt.password",
fallback=None),
ca_cert_path=self.config.get("IO",
"mqtt.ca.cert.path",
fallback=None),
set_insecure=bool(
self.config.get("IO",
"mqtt.insecure.flag",
fallback=False)))
return mqtt
except Exception as e:
logger.error(e)
raise e
def init_mqtt(self):
###Connection to the mqtt broker
self.logger.debug("Starting init mqtt")
self.redisDB.set("Error mqtt" + self.id, False)
try:
for key, value in self.mqtt_params.items():
self.logger.debug("key " + str(key) + " value " + str(value))
# self.output_mqtt[key2] = {"host":host, "topic":topic, "qos":qos}
client_id = "client_publish" + str(randrange(100000)) + str(time.time()).replace(".", "")
host = str(value["host"])
port = value["mqtt.port"]
self.logger.debug("client " + client_id)
self.logger.debug("host " + host)
self.logger.debug("port " + str(port))
client_key = host+":"+str(port)
if client_key not in self.mqtt.keys():
self.mqtt[client_key] = MQTTClient(str(host), port, client_id,
username=value["username"], password=value["password"],
ca_cert_path=value["ca_cert_path"], set_insecure=value["insecure"], id=self.id)
self.logger.info("successfully subscribed")
except Exception as e:
self.logger.debug("Exception while starting mqtt")
self.redisDB.set("Error mqtt" + self.id, True)
self.logger.error(e)
class MonitorPub:
def __init__(self, config, id):
self.logger = MessageLogger.get_logger(__name__, id)
self.id = id
self.config = config
self.host = config.get("IO", "mqtt.host")
self.port = config.getint("IO", "mqtt.port", fallback=1883)
self.topic_params = json.loads(config.get("IO", "monitor.mqtt.topic"))
self.host, host_params, self.qos, self.topic, self.port = ConfigParserUtils.extract_host_params(
self.host, self.port, self.topic_params, self.config, None)
self.mqtt = None
self.init_mqtt()
def init_mqtt(self):
try:
client_id = "client_publish" + str(randrange(100000)) + str(
time.time()).replace(".", "")
self.mqtt = MQTTClient(str(self.host),
self.port,
client_id,
id=self.id,
connect_check_flag=True)
self.logger.info("successfully subscribed")
except Exception as e:
self.logger.error(e)
def send_monitor_ping(self, control_frequency):
try:
if self.mqtt:
msg = self.to_senml(control_frequency)
self.mqtt.publish(self.topic, msg, qos=self.qos)
self.logger.debug("published monitor ping")
else:
self.logger.warning("mqtt not initialized")
except Exception as e:
self.logger.error("Error sending monitor ping " + str(e))
def to_senml(self, value):
meas = senml.SenMLMeasurement()
meas.name = self.id
meas.value = value
meas.time = int(time.time())
doc = senml.SenMLDocument([meas])
return json.dumps(doc.to_json())
def init_mqtt(self, topic_qos):
self.logger.info("Initializing mqtt subscription client")
#if we set it to false here again then it may overwrite previous true value
#self.redisDB.set("Error mqtt"+self.id, False)
try:
if not self.port:
self.port = 1883
#read from config
self.client_id = "client_receive" + str(randrange(100000)) + str(time.time()).replace(".","")
self.mqtt = MQTTClient(str(self.host), self.port, self.client_id, username=self.host_params["username"],
password=self.host_params["password"], ca_cert_path=self.host_params["ca_cert_path"],
set_insecure=self.host_params["insecure_flag"], id=self.id)
self.mqtt.subscribe_to_topics(topic_qos, self.on_msg_received)
self.logger.info("successfully subscribed")
except Exception as e:
self.logger.error(e)
# error for mqtt will be caught by parent
raise e
def init_mqtt(self):
self.mqtt = None
try:
if "pub.mqtt.host" in dict(self.config.items("IO")):
self.host = self.config.get("IO", "pub.mqtt.host")
else:
self.host = self.config.get("IO", "mqtt.host")
self.port = self.config.get("IO", "mqtt.port")
self.host, host_params, self.qos, topic, self.port = ConfigParserUtils.extract_host_params(self.host, self.port,
self.topic_params,
self.config, None)
self.client_id = "client_publish" + str(randrange(100000)) + str(time.time()).replace(".","")
self.mqtt = MQTTClient(str(self.host), self.port, self.client_id,
username=host_params["username"],
password=host_params["password"],
ca_cert_path=host_params["ca_cert_path"],
set_insecure=host_params["insecure_flag"],
id=self.id)
except Exception as e:
self.logger.error(e)
# error for mqtt will be caught at parent
raise e
class DataPublisher(ABC,threading.Thread):
def __init__(self, internal, topic_params, config, publish_frequency, id=None):
super().__init__()
self.logger = MessageLogger.get_logger(__name__, id)
self.internal = internal
self.config = config
self.channel = "MQTT"
self.id = id
self.logger.debug("id = " + str(self.id))
if internal:
self.channel = config.get("IO", "channel")
if topic_params is None:
self.topic_params = {}
else:
self.topic_params = topic_params
self.publish_frequency = publish_frequency
self.stopRequest = threading.Event()
if self.channel == "MQTT":
self.init_mqtt()
elif self.channel == "ZMQ":
self.init_zmq()
self.logger.info("Initializing data publisher thread for topic " + str(self.topic_params))
def init_mqtt(self):
self.mqtt = None
try:
if "pub.mqtt.host" in dict(self.config.items("IO")):
self.host = self.config.get("IO", "pub.mqtt.host")
else:
self.host = self.config.get("IO", "mqtt.host")
self.port = self.config.get("IO", "mqtt.port")
if "mqtt.port" in self.topic_params.keys():
self.port = self.topic_params["mqtt.port"]
if "qos" in self.topic_params.keys():
self.qos = int(self.topic_params["qos"])
else:
self.qos = 1
self.client_id = "client_publish" + str(randrange(100000)) + str(time.time()).replace(".","")
self.mqtt = MQTTClient(str(self.host), self.port, self.client_id,
username=self.config.get("IO", "mqtt.username", fallback=None),
password=self.config.get("IO", "mqtt.password", fallback=None),
ca_cert_path=self.config.get("IO", "mqtt.ca.cert.path", fallback=None),
set_insecure=bool(self.config.get("IO", "mqtt.insecure.flag", fallback=False)),
id=self.id)
except Exception as e:
self.logger.error(e)
# error for mqtt will be caught at parent
raise e
def init_zmq(self):
self.host = self.config.get("IO", "zmq.host")
self.port = self.config.get("IO", "zmq.pub.port")
self.zmq = ZMQClient(self.host, self.port, None)
self.zmq.init_publisher(self.id)
def join(self, timeout=None):
super(DataPublisher, self).join(timeout)
def Stop(self):
self.logger.info("start data publisher thread exit")
self.stopRequest.set()
if self.channel == "MQTT" and self.mqtt is not None:
self.mqtt.MQTTExit()
elif self.channel == "ZMQ":
self.zmq.stop()
if self.isAlive():
self.join(4)
self.logger.info("data publisher thread exit")
def run(self):
"""Get data from internet or any other source"""
if "topic" not in self.topic_params.keys():
fetch_topic = True
else:
fetch_topic = False
topic = None
while not self.stopRequest.is_set():
if fetch_topic:
data, topic = self.get_data()
else:
data = self.get_data()
if data:
self.data_publish(data, topic)
time.sleep(self.publish_frequency)
def data_publish(self, data, topic=None):
if self.channel == "MQTT":
self.mqtt_publish(data, topic)
elif self.channel == "ZMQ":
self.zmq_publish(data, topic)
def mqtt_publish(self, data, topic=None):
try:
if topic is None:
topic = self.topic_params["topic"]
if self.internal:
topic = topic + "/" + self.id
self.logger.debug("Sending results to mqtt on this topic: " + topic)
self.mqtt.publish(topic, data, True, self.qos)
self.logger.debug("Results published")
except Exception as e:
self.logger.error(e)
def zmq_publish(self, data, topic=None):
if topic is None:
topic = self.topic_params["topic"]
if self.internal:
topic = topic + "/" + self.id
self.logger.debug("Sending results to zmq on this topic: " + topic)
self.zmq.publish_message(topic, data)
self.logger.debug("Results published")
@abstractmethod
def get_data(self):
pass
step = (end - start) / 60
setattr(wd, col, start + step * j)
weather.append(wd)
@staticmethod
def get_coordinate(city):
"""
Get geocoordinate from City name
:param city:
:return: Union[Type[JSONDecoder], Any]
"""
try:
config = configparser.RawConfigParser()
config.read("utils/ConfigFile.properties")
googlekey = config.get("SolverSection", "googleapikey")
request = requests.get(
"https://maps.googleapis.com/maps/api/geocode/json?address=" +
city + "&key=" + googlekey)
text = request.json()
return text["results"][0]["geometry"]["location"]
except KeyError:
return ""
we = Weather.get_weather("Bonn, Germany")
js = json.dumps([wea.__dict__ for wea in we], default=str)
m = MQTTClient("optiframework_mosquitto_1", 1883, "weatherClient")
m.publish("data/weather", str(js), True)
m.MQTTExit()
class DataReceiver(ABC):
def __init__(self, internal, topic_params, config, emptyValue={}, id=None, section=None, prepare_topic_qos=True, sub_pub=False):
super().__init__()
self.logger = MessageLogger.get_logger(__name__, id)
self.stop_request = False
self.internal = internal
self.topic_params = topic_params
self.prepare_topic_qos = prepare_topic_qos
self.emptyValue = emptyValue
self.data = self.emptyValue.copy()
self.data_update = False
self.config = config
self.channel = "MQTT"
self.topics = None
self.port = None
self.host_params = {}
self.first_time = 0
self.last_time = 0
self.id = id
self.section = section
self.sub_pub = sub_pub
if self.section is None:
self.section = "IO"
self.setup()
if self.channel == "MQTT":
self.init_mqtt(self.topics)
elif self.channel == "ZMQ":
self.init_zmq(self.topics)
def setup(self):
if self.internal:
self.channel = self.config.get("IO", "channel")
self.topics, self.host_params = self.get_internal_channel_params()
else:
self.topics, self.host, self.host_params = self.get_external_channel_params()
def get_external_channel_params(self):
topic_qos = []
host_params = {}
# read from config
sub_mqtt = "sub.mqtt.host"
if self.sub_pub:
sub_mqtt = "pub.mqtt.host"
if sub_mqtt in dict(self.config.items(self.section)):
host = self.config.get(self.section, sub_mqtt)
else:
host = self.config.get("IO", "mqtt.host")
host_params["username"] = self.config.get("IO", "mqtt.username", fallback=None)
host_params["password"] = self.config.get("IO", "mqtt.password", fallback=None)
host_params["ca_cert_path"] = self.config.get("IO", "mqtt.ca.cert.path", fallback=None)
host_params["insecure_flag"] = bool(self.config.get("IO", "mqtt.insecure.flag", fallback=False))
if "mqtt.username" in dict(self.config.items(self.section)):
host_params["username"] = self.config.get(self.section, "mqtt.username", fallback=None)
if "mqtt.password" in dict(self.config.items(self.section)):
host_params["password"] = self.config.get(self.section, "mqtt.password", fallback=None)
if "mqtt.ca.cert.path" in dict(self.config.items(self.section)):
host_params["ca_cert_path"] = self.config.get(self.section, "mqtt.ca.cert.path", fallback=None)
if "mqtt.insecure.flag" in dict(self.config.items(self.section)):
host_params["insecure_flag"] = bool(self.config.get(self.section, "mqtt.insecure.flag", fallback=False))
qos = 1
if self.topic_params:
topic = self.topic_params["topic"]
if "host" in self.topic_params.keys():
host = self.topic_params["host"]
if "qos" in self.topic_params.keys():
qos = self.topic_params["qos"]
if "mqtt.port" in self.topic_params.keys():
self.port = self.topic_params["mqtt.port"]
if "username" in self.topic_params.keys():
host_params["username"] = self.topic_params["username"]
if "password" in self.topic_params.keys():
host_params["password"] = self.topic_params["password"]
if "ca_cert_path" in self.topic_params.keys():
host_params["ca_cert_path"] = self.topic_params["ca_cert_path"]
if "insecure" in self.topic_params.keys():
host_params["insecure_flag"] = self.topic_params["insecure"]
topic_qos.append((topic, qos))
return topic_qos, host, host_params
def get_internal_channel_params(self):
if self.channel == "MQTT":
sub_mqtt = "sub.mqtt.host"
if self.sub_pub:
sub_mqtt = "pub.mqtt.host"
topic_qos = []
host_params = {}
if self.prepare_topic_qos:
for k, v in self.topic_params.items():
if k == "topic":
topic_qos.append((v + "/" + self.id,1))
elif k == "mqtt.port":
self.port = v
elif isinstance(self.topic_params, list):
topic_qos = self.topic_params
self.port = self.config.get("IO", "mqtt.port")
if sub_mqtt in dict(self.config.items("IO")):
self.host = self.config.get("IO", sub_mqtt)
if "mqtt.host" in dict(self.config.items("IO")):
self.host = self.config.get("IO", "mqtt.host")
host_params["username"] = self.config.get("IO", "mqtt.username", fallback=None)
host_params["password"] = self.config.get("IO", "mqtt.password", fallback=None)
host_params["ca_cert_path"] = self.config.get("IO", "mqtt.ca.cert.path", fallback=None)
host_params["insecure_flag"] = bool(self.config.get("IO", "mqtt.insecure.flag", fallback=False))
return topic_qos, host_params
elif self.channel == "ZMQ":
topics = []
for k, v in self.topic_params.items():
if k == "topic":
topics.append(v + "/" + self.id)
self.port = self.config.get("IO", "zmq.sub.port")
self.host = self.config.get("IO", "zmq.host")
return topics, None
def init_mqtt(self, topic_qos):
self.logger.info("Initializing mqtt subscription client")
#if we set it to false here again then it may overwrite previous true value
#self.redisDB.set("Error mqtt"+self.id, False)
try:
if not self.port:
self.port = 1883
#read from config
self.client_id = "client_receive" + str(randrange(100000)) + str(time.time()).replace(".","")
self.mqtt = MQTTClient(str(self.host), self.port, self.client_id, username=self.host_params["username"],
password=self.host_params["password"], ca_cert_path=self.host_params["ca_cert_path"],
set_insecure=self.host_params["insecure_flag"], id=self.id)
self.mqtt.subscribe_to_topics(topic_qos, self.on_msg_received)
self.logger.info("successfully subscribed")
except Exception as e:
self.logger.error(e)
# error for mqtt will be caught by parent
raise e
def init_zmq(self, topics):
self.logger.info("Initializing zmq subscription client")
self.zmq = ZMQClient(self.host, None, self.port)
self.zmq.init_subscriber(topics, self.id)
@abstractmethod
def on_msg_received(self, payload):
pass
def get_mqtt_data(self, require_updated, clearData):
if require_updated == 1 and not self.data:
require_updated = 0
while require_updated == 0 and not self.data_update and not self.stop_request:
self.logger.debug("wait for data "+str(self.topics))
time.sleep(0.5)
return self.get_and_update_data(clearData)
def exit(self):
self.stop_request = True
if self.channel == "MQTT":
self.mqtt.MQTTExit()
elif self.channel == "ZMQ":
self.zmq.stop()
self.logger.info("InputController safe exit")
def get_zmq_msg(self, clearData):
while True and not self.stop_request:
self.logger.debug("get zmq msg")
flag, topic, message = self.zmq.receive_message()
self.logger.debug("zmq subscription msg received for topic "+str(topic)+" for id "+str(self.id))
if flag:
self.on_msg_received(message)
break
time.sleep(1)
return self.get_and_update_data(clearData)
def get_and_update_data(self, clearData):
new_data = self.data.copy()
self.data_update = False
if clearData:
self.clear_data()
return new_data
def clear_data(self):
self.data = self.emptyValue.copy()
def get_data(self, require_updated=0, clearData=False):
"""
:param require_updated: 0 -> wait for new data
1 -> wait for new data if no prev data
2 -> return prev data, even if empty
:return:
"""
data = {}
if self.channel == "MQTT":
data = self.get_mqtt_data(require_updated, clearData)
elif self.channel == "ZMQ":
data = self.get_zmq_msg(clearData)
return data