def clear_redis(logger):
logger.info("reset redis training key locks")
training_lock_key = "training_lock"
from IO.redisDB import RedisDB
redisDB = RedisDB()
try:
redisDB.remove(training_lock_key)
except Exception as e:
logger.debug("training_lock key does not exist")
def delete_output(id): # noqa: E501
"""Deletes the output of the framework
# noqa: E501
:param id: Name of the registry to be deleted
:type id: str
:rtype: None
"""
redisDB = RedisDB()
output_keys = redisDB.get_keys_for_pattern("o:" + id + ":*")
if output_keys is not None:
for key in output_keys:
redisDB.remove(key)
return "success"
class ThreadFactory:
def __init__(self, model_name, control_frequency, horizon_in_steps,
dT_in_seconds, repetition, solver, id, optimization_type,
single_ev):
self.logger = MessageLogger.get_logger(__name__, id)
self.model_name = model_name
self.control_frequency = control_frequency
self.horizon_in_steps = horizon_in_steps
self.dT_in_seconds = dT_in_seconds
self.repetition = repetition
self.solver = solver
self.id = id
self.optimization_type = optimization_type
self.single_ev = single_ev
self.redisDB = RedisDB()
self.pyro_mip_server = None
def getFilePath(self, dir, file_name):
# print(os.path.sep)
# print(os.environ.get("HOME"))
project_dir = os.path.dirname(os.path.realpath(__file__))
data_file = os.path.join("/usr/src/app", dir, file_name)
return data_file
def startOptControllerThread(self):
self.logger.info("Creating optimization controller thread")
self.logger.info("Number of repetitions: " + str(self.repetition))
self.logger.info("Output with the following control_frequency: " +
str(self.control_frequency))
self.logger.info(
"Optimization calculated with the following horizon_in_steps: " +
str(self.horizon_in_steps))
self.logger.info(
"Optimization calculated with the following dT_in_seconds: " +
str(self.dT_in_seconds))
self.logger.info("Optimization calculated with the following model: " +
self.model_name)
self.logger.info(
"Optimization calculated with the following solver: " +
self.solver)
self.logger.info(
"Optimization calculated with the following optimization_type: " +
self.optimization_type)
self.redisDB.set("Error mqtt" + self.id, False)
self.logger.debug("Error mqtt " +
str(self.redisDB.get("Error mqtt" + self.id)))
# Creating an object of the configuration file (standard values)
try:
config = configparser.RawConfigParser()
config.read(
self.getFilePath("optimization/resources",
"ConfigFile.properties"))
except Exception as e:
self.logger.error(e)
# Loads the solver name if it was not given thorough the endpoint command/start/id
if not self.model_name:
self.model_name = config.get("SolverSection", "model.name")
self.logger.debug("This is the model name: " + self.model_name)
self.model_path = os.path.join(
config.get("SolverSection", "model.base.path"),
self.model_name) + ".py"
self.logger.debug("This is the path of the model: " +
str(self.model_path))
# Loads the solver name if not specified in command/start/id
if not self.solver:
self.solver_name = config.get("SolverSection", "solver.name")
else:
self.solver_name = self.solver
self.logger.debug(
"Optimization calculated with the following solver: " +
self.solver_name)
##############################################################################################
output_config = None
try:
# Reads the registry/output and stores it into an object
path = os.path.join(os.getcwd(), "optimization/resources",
str(self.id), "Output.registry.mqtt")
if not os.path.exists(path):
self.logger.debug(
"Output.registry.mqtt not set, only file output available")
else:
with open(path, "r") as file:
output_config = json.loads(file.read())
except Exception as e:
self.logger.error(
"Output.registry.mqtt not set, only file output available")
try:
# Reads the registry/input and stores it into an object
path = os.path.join(os.getcwd(), "optimization/resources",
str(self.id), "Input.registry.file")
if not os.path.exists(path):
input_config_file = {}
self.logger.debug("Not Input.registry.file present")
else:
with open(path, "r") as file:
input_config_file = json.loads(file.read())
self.logger.debug("Input.registry.file found")
except Exception as e:
self.logger.error("Input file not found")
input_config_file = {}
self.logger.error(e)
try:
# Reads the registry/input and stores it into an object
path = os.path.join(os.getcwd(), "optimization/resources",
str(self.id), "Input.registry.mqtt")
if not os.path.exists(path):
input_config_mqtt = {}
self.logger.debug("Not Input.registry.mqtt present")
else:
with open(path, "r") as file:
input_config_mqtt = json.loads(file.read())
self.logger.debug("Input.registry.mqtt found")
except Exception as e:
self.logger.error("Input file not found")
input_config_mqtt = {}
self.logger.error(e)
input_config_parser = InputConfigParser(input_config_file,
input_config_mqtt,
self.model_name, self.id,
self.optimization_type)
missing_keys = input_config_parser.check_keys_for_completeness()
if len(missing_keys) > 0:
raise MissingKeysException(
"Data source for following keys not declared: " +
str(missing_keys))
self.prediction_threads = {}
self.prediction_names = input_config_parser.get_prediction_names()
if self.prediction_names is not None and len(
self.prediction_names) > 0:
for prediction_name in self.prediction_names:
flag = input_config_parser.get_forecast_flag(prediction_name)
if flag:
self.logger.info(
"Creating prediction controller thread for topic " +
str(prediction_name))
topic_param = input_config_parser.get_params(
prediction_name)
parameters = json.dumps({
"control_frequency": self.control_frequency,
"horizon_in_steps": self.horizon_in_steps,
"topic_param": topic_param,
"dT_in_seconds": self.dT_in_seconds
})
self.redisDB.set(
"train:" + self.id + ":" + prediction_name, parameters)
self.prediction_threads[prediction_name] = LoadPrediction(
config, self.control_frequency, self.horizon_in_steps,
prediction_name, topic_param, self.dT_in_seconds,
self.id, True)
# self.prediction_threads[prediction_name].start()
self.non_prediction_threads = {}
self.non_prediction_names = input_config_parser.get_non_prediction_names(
)
if self.non_prediction_names is not None and len(
self.non_prediction_names) > 0:
for non_prediction_name in self.non_prediction_names:
flag = input_config_parser.get_forecast_flag(
non_prediction_name)
if flag:
if non_prediction_name == "P_PV":
self.non_prediction_threads[
non_prediction_name] = PVPrediction(
config, input_config_parser, self.id,
self.control_frequency, self.horizon_in_steps,
self.dT_in_seconds, non_prediction_name)
self.non_prediction_threads[non_prediction_name].start(
)
# Initializing constructor of the optimization controller thread
if self.optimization_type == "MPC":
self.opt = OptControllerMPC(
self.id, self.solver_name, self.model_path,
self.control_frequency, self.repetition, output_config,
input_config_parser, config, self.horizon_in_steps,
self.dT_in_seconds, self.optimization_type)
elif self.optimization_type == "discrete":
self.opt = OptControllerDiscrete(
self.id, self.solver_name, self.model_path,
self.control_frequency, self.repetition, output_config,
input_config_parser, config, self.horizon_in_steps,
self.dT_in_seconds, self.optimization_type)
elif self.optimization_type == "stochastic":
self.opt = OptControllerStochastic(
self.id, self.solver_name, self.model_path,
self.control_frequency, self.repetition, output_config,
input_config_parser, config, self.horizon_in_steps,
self.dT_in_seconds, self.optimization_type, self.single_ev)
try:
####starts the optimization controller thread
self.logger.debug("Mqtt issue " +
str(self.redisDB.get("Error mqtt" + self.id)))
if "False" in self.redisDB.get("Error mqtt" + self.id):
self.opt.start()
self.logger.debug("Optimization object started")
return 0
else:
self.redisDB.set("run:" + self.id, "stopping")
self.stopOptControllerThread()
self.redisDB.set("run:" + self.id, "stopped")
self.logger.error("Optimization object could not be started")
return 1
except Exception as e:
self.logger.error(e)
return 1
def stopOptControllerThread(self):
try:
# stop as per ID
for name, obj in self.prediction_threads.items():
self.redisDB.remove("train:" + self.id + ":" + name)
obj.Stop()
for name, obj in self.non_prediction_threads.items():
obj.Stop()
del obj
self.logger.info("Stopping optimization controller thread")
self.opt.Stop()
del self.opt
self.logger.info("Optimization controller thread stopped")
del self.logger
return "Optimization controller thread stopped"
except Exception as e:
self.logger.error(e)
return e
def is_running(self):
return not self.opt.get_finish_status()
def update_training_params(self, key, parameters):
while True:
self.redisDB.set(key, parameters)
time.sleep("60")
class OutputController:
def __init__(self, id=None, output_config=None):
self.logger = MessageLogger.get_logger(__name__, id)
self.logger.info("Output Class started")
self.output_config = output_config
self.mqtt = {}
self.redisDB = RedisDB()
self.mqtt_params = {}
self.output_mqtt = {}
self.id = id
self.config_parser_utils = ConfigParserUtils()
self.logger.debug("output_config: " + str(self.output_config) + " " + str(type(self.output_config)))
if self.output_config is not None:
self.extract_mqtt_params()
self.init_mqtt()
def extract_mqtt_params(self):
self.logger.debug("Output config = " + str(self.output_config))
for key, value in self.output_config.items():
self.logger.debug("key " + str(key) + " value " + str(value))
for key2, value2 in value.items():
self.logger.debug("key2 " + str(key2) + " value2 " + str(value2))
mqtt = self.config_parser_utils.get_mqtt(value2)
unit, horizon_values = self.read_extra_values(value2)
if mqtt is not None:
self.mqtt_params[key2] = mqtt.copy()
self.mqtt_params[key2]["unit"] = unit
self.mqtt_params[key2]["horizon_values"] = horizon_values
self.logger.debug("params = " + str(self.mqtt_params))
def read_extra_values(self, value2):
unit = None
horizon_values = False
if isinstance(value2, dict):
if "unit" in value2.keys():
unit = value2["unit"]
if "horizon_values" in value2.keys():
horizon_values = value2["horizon_values"]
return unit, horizon_values
def init_mqtt(self):
###Connection to the mqtt broker
self.logger.debug("Starting init mqtt")
self.redisDB.set("Error mqtt" + self.id, False)
try:
for key, value in self.mqtt_params.items():
self.logger.debug("key " + str(key) + " value " + str(value))
# self.output_mqtt[key2] = {"host":host, "topic":topic, "qos":qos}
client_id = "client_publish" + str(randrange(100000)) + str(time.time()).replace(".", "")
host = str(value["host"])
port = value["mqtt.port"]
self.logger.debug("client " + client_id)
self.logger.debug("host " + host)
self.logger.debug("port " + str(port))
client_key = host+":"+str(port)
if client_key not in self.mqtt.keys():
self.mqtt[client_key] = MQTTClient(str(host), port, client_id,
username=value["username"], password=value["password"],
ca_cert_path=value["ca_cert_path"], set_insecure=value["insecure"], id=self.id)
self.logger.info("successfully subscribed")
except Exception as e:
self.logger.debug("Exception while starting mqtt")
self.redisDB.set("Error mqtt" + self.id, True)
self.logger.error(e)
def publish_data(self, id, data, dT):
self.logger.debug("output data : "+ json.dumps(data, indent=4))
current_time = int(time.time())
try:
senml_data = self.senml_message_format(data, current_time, dT)
for mqtt_key, value in senml_data.items():
v = json.dumps(value)
# self.logger.debug("key: "+str(key))
# self.logger.debug("mqtt params: " + str(self.mqtt_params.keys()))
if mqtt_key in self.mqtt_params.keys():
value2 = self.mqtt_params[mqtt_key]
topic = value2["topic"]
host = value2["host"]
port = value2["mqtt.port"]
qos = value2["qos"]
client_key = host + ":" + str(port)
self.mqtt[client_key].sendResults(topic, v, qos)
except Exception as e:
self.logger.error("error in publish data ", e)
self.save_to_redis(id, data, current_time)
def Stop(self):
self.stop_request = True
try:
for key, value in self.mqtt_params.items():
self.logger.debug("key " + str(key) + " value " + str(value))
self.mqtt[key].MQTTExit()
self.logger.info("OutputController safe exit")
except Exception as e:
self.logger.error(e)
def senml_message_format(self, data, current_time, dT):
new_data = {}
# self.logger.debug("data for senml "+str(data))
for key, value in data.items():
flag = False
time = current_time
u = None
base = None
if isinstance(value, dict):
bn, n, val = self.get_names(value)
else:
bn, n, val = None, None, value
if bn:
base = senml.SenMLMeasurement()
base.name = bn
if key in self.mqtt_params.keys():
if self.mqtt_params[key]["unit"] is not None:
u = self.mqtt_params[key]["unit"]
"""
else:
u = "W"
"""
flag = self.mqtt_params[key]["horizon_values"]
meas_list = []
for v in val:
meas = senml.SenMLMeasurement()
meas.name = n
meas.time = time
meas.value = v
if u:
meas.unit = u
meas_list.append(meas)
time += dT
if not flag:
break # only want the first value
if len(meas_list) > 0:
doc = senml.SenMLDocument(meas_list, base=base)
new_data[key] = doc.to_json()
# self.logger.debug("Topic MQTT Senml message: "+str(new_data))
return new_data
def save_to_redis(self, id, data, time):
try:
part_key = "o:" + id + ":"
output_keys = self.redisDB.get_keys_for_pattern(part_key+"*")
if output_keys is not None:
for key in output_keys:
self.redisDB.remove(key)
for key, value in data.items():
key = key.replace("~","/")
if isinstance(value, dict):
bn, n, val = self.get_names(value)
else:
bn, n, val = None, key, value
if bn:
n = bn + "/" + n
index = 0
for v in val:
k = part_key + n + ":" + str(index)
self.redisDB.set(k, json.dumps({str(time): v}))
index += 1
except Exception as e:
self.logger.error("error adding to redis " + str(e))
def get_names(self, dict):
bn = None
n = None
v = None
if "bn" in dict.keys():
bn = dict["bn"]
if "n" in dict.keys():
n = dict["n"]
if "v" in dict.keys():
v = dict["v"]
return bn,n,v
