def clear_redis(logger): logger.info("reset redis training key locks") training_lock_key = "training_lock" from IO.redisDB import RedisDB redisDB = RedisDB() try: redisDB.remove(training_lock_key) except Exception as e: logger.debug("training_lock key does not exist")
def delete_output(id): # noqa: E501 """Deletes the output of the framework # noqa: E501 :param id: Name of the registry to be deleted :type id: str :rtype: None """ redisDB = RedisDB() output_keys = redisDB.get_keys_for_pattern("o:" + id + ":*") if output_keys is not None: for key in output_keys: redisDB.remove(key) return "success"
class ThreadFactory: def __init__(self, model_name, control_frequency, horizon_in_steps, dT_in_seconds, repetition, solver, id, optimization_type, single_ev): self.logger = MessageLogger.get_logger(__name__, id) self.model_name = model_name self.control_frequency = control_frequency self.horizon_in_steps = horizon_in_steps self.dT_in_seconds = dT_in_seconds self.repetition = repetition self.solver = solver self.id = id self.optimization_type = optimization_type self.single_ev = single_ev self.redisDB = RedisDB() self.pyro_mip_server = None def getFilePath(self, dir, file_name): # print(os.path.sep) # print(os.environ.get("HOME")) project_dir = os.path.dirname(os.path.realpath(__file__)) data_file = os.path.join("/usr/src/app", dir, file_name) return data_file def startOptControllerThread(self): self.logger.info("Creating optimization controller thread") self.logger.info("Number of repetitions: " + str(self.repetition)) self.logger.info("Output with the following control_frequency: " + str(self.control_frequency)) self.logger.info( "Optimization calculated with the following horizon_in_steps: " + str(self.horizon_in_steps)) self.logger.info( "Optimization calculated with the following dT_in_seconds: " + str(self.dT_in_seconds)) self.logger.info("Optimization calculated with the following model: " + self.model_name) self.logger.info( "Optimization calculated with the following solver: " + self.solver) self.logger.info( "Optimization calculated with the following optimization_type: " + self.optimization_type) self.redisDB.set("Error mqtt" + self.id, False) self.logger.debug("Error mqtt " + str(self.redisDB.get("Error mqtt" + self.id))) # Creating an object of the configuration file (standard values) try: config = configparser.RawConfigParser() config.read( self.getFilePath("optimization/resources", "ConfigFile.properties")) except Exception as e: self.logger.error(e) # Loads the solver name if it was not given thorough the endpoint command/start/id if not self.model_name: self.model_name = config.get("SolverSection", "model.name") self.logger.debug("This is the model name: " + self.model_name) self.model_path = os.path.join( config.get("SolverSection", "model.base.path"), self.model_name) + ".py" self.logger.debug("This is the path of the model: " + str(self.model_path)) # Loads the solver name if not specified in command/start/id if not self.solver: self.solver_name = config.get("SolverSection", "solver.name") else: self.solver_name = self.solver self.logger.debug( "Optimization calculated with the following solver: " + self.solver_name) ############################################################################################## output_config = None try: # Reads the registry/output and stores it into an object path = os.path.join(os.getcwd(), "optimization/resources", str(self.id), "Output.registry.mqtt") if not os.path.exists(path): self.logger.debug( "Output.registry.mqtt not set, only file output available") else: with open(path, "r") as file: output_config = json.loads(file.read()) except Exception as e: self.logger.error( "Output.registry.mqtt not set, only file output available") try: # Reads the registry/input and stores it into an object path = os.path.join(os.getcwd(), "optimization/resources", str(self.id), "Input.registry.file") if not os.path.exists(path): input_config_file = {} self.logger.debug("Not Input.registry.file present") else: with open(path, "r") as file: input_config_file = json.loads(file.read()) self.logger.debug("Input.registry.file found") except Exception as e: self.logger.error("Input file not found") input_config_file = {} self.logger.error(e) try: # Reads the registry/input and stores it into an object path = os.path.join(os.getcwd(), "optimization/resources", str(self.id), "Input.registry.mqtt") if not os.path.exists(path): input_config_mqtt = {} self.logger.debug("Not Input.registry.mqtt present") else: with open(path, "r") as file: input_config_mqtt = json.loads(file.read()) self.logger.debug("Input.registry.mqtt found") except Exception as e: self.logger.error("Input file not found") input_config_mqtt = {} self.logger.error(e) input_config_parser = InputConfigParser(input_config_file, input_config_mqtt, self.model_name, self.id, self.optimization_type) missing_keys = input_config_parser.check_keys_for_completeness() if len(missing_keys) > 0: raise MissingKeysException( "Data source for following keys not declared: " + str(missing_keys)) self.prediction_threads = {} self.prediction_names = input_config_parser.get_prediction_names() if self.prediction_names is not None and len( self.prediction_names) > 0: for prediction_name in self.prediction_names: flag = input_config_parser.get_forecast_flag(prediction_name) if flag: self.logger.info( "Creating prediction controller thread for topic " + str(prediction_name)) topic_param = input_config_parser.get_params( prediction_name) parameters = json.dumps({ "control_frequency": self.control_frequency, "horizon_in_steps": self.horizon_in_steps, "topic_param": topic_param, "dT_in_seconds": self.dT_in_seconds }) self.redisDB.set( "train:" + self.id + ":" + prediction_name, parameters) self.prediction_threads[prediction_name] = LoadPrediction( config, self.control_frequency, self.horizon_in_steps, prediction_name, topic_param, self.dT_in_seconds, self.id, True) # self.prediction_threads[prediction_name].start() self.non_prediction_threads = {} self.non_prediction_names = input_config_parser.get_non_prediction_names( ) if self.non_prediction_names is not None and len( self.non_prediction_names) > 0: for non_prediction_name in self.non_prediction_names: flag = input_config_parser.get_forecast_flag( non_prediction_name) if flag: if non_prediction_name == "P_PV": self.non_prediction_threads[ non_prediction_name] = PVPrediction( config, input_config_parser, self.id, self.control_frequency, self.horizon_in_steps, self.dT_in_seconds, non_prediction_name) self.non_prediction_threads[non_prediction_name].start( ) # Initializing constructor of the optimization controller thread if self.optimization_type == "MPC": self.opt = OptControllerMPC( self.id, self.solver_name, self.model_path, self.control_frequency, self.repetition, output_config, input_config_parser, config, self.horizon_in_steps, self.dT_in_seconds, self.optimization_type) elif self.optimization_type == "discrete": self.opt = OptControllerDiscrete( self.id, self.solver_name, self.model_path, self.control_frequency, self.repetition, output_config, input_config_parser, config, self.horizon_in_steps, self.dT_in_seconds, self.optimization_type) elif self.optimization_type == "stochastic": self.opt = OptControllerStochastic( self.id, self.solver_name, self.model_path, self.control_frequency, self.repetition, output_config, input_config_parser, config, self.horizon_in_steps, self.dT_in_seconds, self.optimization_type, self.single_ev) try: ####starts the optimization controller thread self.logger.debug("Mqtt issue " + str(self.redisDB.get("Error mqtt" + self.id))) if "False" in self.redisDB.get("Error mqtt" + self.id): self.opt.start() self.logger.debug("Optimization object started") return 0 else: self.redisDB.set("run:" + self.id, "stopping") self.stopOptControllerThread() self.redisDB.set("run:" + self.id, "stopped") self.logger.error("Optimization object could not be started") return 1 except Exception as e: self.logger.error(e) return 1 def stopOptControllerThread(self): try: # stop as per ID for name, obj in self.prediction_threads.items(): self.redisDB.remove("train:" + self.id + ":" + name) obj.Stop() for name, obj in self.non_prediction_threads.items(): obj.Stop() del obj self.logger.info("Stopping optimization controller thread") self.opt.Stop() del self.opt self.logger.info("Optimization controller thread stopped") del self.logger return "Optimization controller thread stopped" except Exception as e: self.logger.error(e) return e def is_running(self): return not self.opt.get_finish_status() def update_training_params(self, key, parameters): while True: self.redisDB.set(key, parameters) time.sleep("60")
class OutputController: def __init__(self, id=None, output_config=None): self.logger = MessageLogger.get_logger(__name__, id) self.logger.info("Output Class started") self.output_config = output_config self.mqtt = {} self.redisDB = RedisDB() self.mqtt_params = {} self.output_mqtt = {} self.id = id self.config_parser_utils = ConfigParserUtils() self.logger.debug("output_config: " + str(self.output_config) + " " + str(type(self.output_config))) if self.output_config is not None: self.extract_mqtt_params() self.init_mqtt() def extract_mqtt_params(self): self.logger.debug("Output config = " + str(self.output_config)) for key, value in self.output_config.items(): self.logger.debug("key " + str(key) + " value " + str(value)) for key2, value2 in value.items(): self.logger.debug("key2 " + str(key2) + " value2 " + str(value2)) mqtt = self.config_parser_utils.get_mqtt(value2) unit, horizon_values = self.read_extra_values(value2) if mqtt is not None: self.mqtt_params[key2] = mqtt.copy() self.mqtt_params[key2]["unit"] = unit self.mqtt_params[key2]["horizon_values"] = horizon_values self.logger.debug("params = " + str(self.mqtt_params)) def read_extra_values(self, value2): unit = None horizon_values = False if isinstance(value2, dict): if "unit" in value2.keys(): unit = value2["unit"] if "horizon_values" in value2.keys(): horizon_values = value2["horizon_values"] return unit, horizon_values def init_mqtt(self): ###Connection to the mqtt broker self.logger.debug("Starting init mqtt") self.redisDB.set("Error mqtt" + self.id, False) try: for key, value in self.mqtt_params.items(): self.logger.debug("key " + str(key) + " value " + str(value)) # self.output_mqtt[key2] = {"host":host, "topic":topic, "qos":qos} client_id = "client_publish" + str(randrange(100000)) + str(time.time()).replace(".", "") host = str(value["host"]) port = value["mqtt.port"] self.logger.debug("client " + client_id) self.logger.debug("host " + host) self.logger.debug("port " + str(port)) client_key = host+":"+str(port) if client_key not in self.mqtt.keys(): self.mqtt[client_key] = MQTTClient(str(host), port, client_id, username=value["username"], password=value["password"], ca_cert_path=value["ca_cert_path"], set_insecure=value["insecure"], id=self.id) self.logger.info("successfully subscribed") except Exception as e: self.logger.debug("Exception while starting mqtt") self.redisDB.set("Error mqtt" + self.id, True) self.logger.error(e) def publish_data(self, id, data, dT): self.logger.debug("output data : "+ json.dumps(data, indent=4)) current_time = int(time.time()) try: senml_data = self.senml_message_format(data, current_time, dT) for mqtt_key, value in senml_data.items(): v = json.dumps(value) # self.logger.debug("key: "+str(key)) # self.logger.debug("mqtt params: " + str(self.mqtt_params.keys())) if mqtt_key in self.mqtt_params.keys(): value2 = self.mqtt_params[mqtt_key] topic = value2["topic"] host = value2["host"] port = value2["mqtt.port"] qos = value2["qos"] client_key = host + ":" + str(port) self.mqtt[client_key].sendResults(topic, v, qos) except Exception as e: self.logger.error("error in publish data ", e) self.save_to_redis(id, data, current_time) def Stop(self): self.stop_request = True try: for key, value in self.mqtt_params.items(): self.logger.debug("key " + str(key) + " value " + str(value)) self.mqtt[key].MQTTExit() self.logger.info("OutputController safe exit") except Exception as e: self.logger.error(e) def senml_message_format(self, data, current_time, dT): new_data = {} # self.logger.debug("data for senml "+str(data)) for key, value in data.items(): flag = False time = current_time u = None base = None if isinstance(value, dict): bn, n, val = self.get_names(value) else: bn, n, val = None, None, value if bn: base = senml.SenMLMeasurement() base.name = bn if key in self.mqtt_params.keys(): if self.mqtt_params[key]["unit"] is not None: u = self.mqtt_params[key]["unit"] """ else: u = "W" """ flag = self.mqtt_params[key]["horizon_values"] meas_list = [] for v in val: meas = senml.SenMLMeasurement() meas.name = n meas.time = time meas.value = v if u: meas.unit = u meas_list.append(meas) time += dT if not flag: break # only want the first value if len(meas_list) > 0: doc = senml.SenMLDocument(meas_list, base=base) new_data[key] = doc.to_json() # self.logger.debug("Topic MQTT Senml message: "+str(new_data)) return new_data def save_to_redis(self, id, data, time): try: part_key = "o:" + id + ":" output_keys = self.redisDB.get_keys_for_pattern(part_key+"*") if output_keys is not None: for key in output_keys: self.redisDB.remove(key) for key, value in data.items(): key = key.replace("~","/") if isinstance(value, dict): bn, n, val = self.get_names(value) else: bn, n, val = None, key, value if bn: n = bn + "/" + n index = 0 for v in val: k = part_key + n + ":" + str(index) self.redisDB.set(k, json.dumps({str(time): v})) index += 1 except Exception as e: self.logger.error("error adding to redis " + str(e)) def get_names(self, dict): bn = None n = None v = None if "bn" in dict.keys(): bn = dict["bn"] if "n" in dict.keys(): n = dict["n"] if "v" in dict.keys(): v = dict["v"] return bn,n,v