def data_formater(self, data): json_data = json.loads(data) doc = None try: doc = senml.SenMLDocument.from_json(json_data) except Exception as e: pass if not doc: try: meas = senml.SenMLMeasurement.from_json(json_data) doc = senml.SenMLDocument([meas]) except Exception as e: pass new_data = {} if doc: base_data = doc.base bn = None if base_data: bn = base_data.name bn = bn.replace("/", "") if len(doc.measurements) > 0: for meas in doc.measurements: n = meas.name u = meas.unit v = meas.value t = meas.time new_data[bn + "." + n] = v return new_data
def add_formated_data(self, json_data): doc = None try: doc = senml.SenMLDocument.from_json(json_data) except Exception as e: pass if not doc: try: meas = senml.SenMLMeasurement.from_json(json_data) doc = senml.SenMLDocument([meas]) except Exception as e: pass if doc: base_data = doc.base bn, bu = None, None if base_data: bn = base_data.name bu = base_data.unit data = {} doc.measurements = sorted(doc.measurements, key=lambda x: x.time) if len(doc.measurements) > 0: for meas in doc.measurements: n = meas.name t = meas.time v = meas.value data[n] = {"last_time": t, "freq": v} return data return {}
def to_senml(self, value): meas = senml.SenMLMeasurement() meas.name = self.id meas.value = value meas.time = int(time.time()) doc = senml.SenMLDocument([meas]) return json.dumps(doc.to_json())
def convert_to_senml(self): meas = [] if len(self.pv_data) > 0: for row in self.pv_data: meas.append(self.get_senml_meas(float(row[1]), row[0])) doc = senml.SenMLDocument(meas) val = doc.to_json() return json.dumps(val)
def add_formated_data(self, json_data): doc = None try: doc = senml.SenMLDocument.from_json(json_data) except Exception as e: pass if not doc: try: meas = senml.SenMLMeasurement.from_json(json_data) doc = senml.SenMLDocument([meas]) except Exception as e: pass if doc: base_data = doc.base bn, bu = None, None if base_data: bn = base_data.name bu = base_data.unit data = {} raw_data = [] doc.measurements = sorted(doc.measurements, key=lambda x: x.time) if len(doc.measurements) > 0: self.length = len(doc.measurements) for meas in doc.measurements: n = meas.name u = meas.unit v = meas.value t = meas.time t = self.time_conversion(t) if not u: u = bu # dont check bn if not n: n = self.generic_name try: processed_value = self.preprocess_data(bn, n, v, u) if processed_value is not None and processed_value is not {}: raw_data.append([t, processed_value]) except Exception as e: self.logger.error("error " + str(e) + " n = " + str(n)) #self.logger.debug("data: " + str(data)) raw_data = self.expand_and_resample(raw_data, self.dT) if len(raw_data) > 0: bucket = self.time_to_bucket(raw_data[0][0]) for row in raw_data: bucket_key = str( self.current_day_index) + "_" + str(bucket) bucket += 1 if bucket >= self.total_steps_in_day: bucket = 0 self.current_day_index += 1 if self.current_day_index >= self.number_of_bucket_days: self.current_day_index = 0 data[bucket_key] = row[1] return data return {}
def extract_horizon_data(self): meas = [] list = self.load_data.items() list = sorted(list) list = list[-self.horizon_in_steps:] for i in range(self.horizon_in_steps): value = list[i][1] if value < 0: value = 0 meas.append(self.get_senml_meas(value, list[i][0])) doc = senml.SenMLDocument(meas) val = doc.to_json() return json.dumps(val)
def to_senml(self, name, value, timestamp): meas = senml.SenMLMeasurement() meas.name = name if isinstance(value, str): try: value = float(value) except Exception: pass meas.value = value meas.time = timestamp doc = senml.SenMLDocument([meas], base=self.base) val = doc.to_json() val = json.dumps(val) return val
def extract_horizon_data(self): meas = [] if len(self.pv_data) > 0: current_timestamp = datetime.datetime.now().timestamp() closest_index = self.find_closest_prev_timestamp( self.pv_data, current_timestamp) for i in range(self.horizon_in_steps): row = self.pv_data[closest_index] meas.append(self.get_senml_meas(float(row[1]), row[0])) closest_index += 1 if closest_index >= len(self.pv_data): closest_index = 0 doc = senml.SenMLDocument(meas) val = doc.to_json() return json.dumps(val)
def to_senml(self, results): meas = [] base = None if "base_name" in self.topic_params: base = senml.SenMLMeasurement() base.name = self.topic_params["base_name"] for time, errors in results.items(): rmse = errors["rmse"] mae = errors["mae"] meas.append( self.get_senml_meas(rmse, time, self.topic_name + "/rmse")) meas.append( self.get_senml_meas(mae, time, self.topic_name + "/mae")) doc = senml.SenMLDocument(meas, base=base) val = doc.to_json() return json.dumps(val)
def get_data(self): try: if self.source == "constant": val = self.constant_value logger.debug("Sent MQTT:" + str(val)) return val else: meas_list = [] current_time = int(math.floor(time.time())) if self.source == "file": vals = self.get_file_line(current_time) logger.debug("Length: " + str(self.length)) if len(vals) < self.length: logger.error( str(self.generic_name) + " mock file has invalid data. Less values than horizon_step" ) return None elif self.source == "random": vals = self.get_random_floats() logger.debug("Vals: " + str(vals)) logger.debug("Length: " + str(self.length)) prev_time = 0 for index in range(self.length): meas = senml.SenMLMeasurement() if self.is_timed: meas.value = vals[index][1] if prev_time > vals[index][0]: meas.time = prev_time + self.delta_time else: meas.time = int(vals[index][0]) prev_time = meas.time else: meas.value = vals[index] meas.time = int(current_time) meas.name = self.generic_name meas_list.append(meas) current_time += self.delta_time doc = senml.SenMLDocument(meas_list) val = doc.to_json() val = json.dumps(val) logger.debug("Sent MQTT:" + str(val)) return val except Exception as e: logger.error(e)
def add_formated_data(self, json_data): doc = None try: doc = senml.SenMLDocument.from_json(json_data) except Exception as e: pass if not doc: try: meas = senml.SenMLMeasurement.from_json(json_data) doc = senml.SenMLDocument([meas]) except Exception as e: pass if doc: base_data = doc.base bn, bu = None, None if base_data: bn = base_data.name bu = base_data.unit data = {} index = {} doc.measurements = sorted(doc.measurements, key=lambda x: x.time) for meas in doc.measurements: n = meas.name u = meas.unit v = meas.value t = meas.time if not u: u = bu # dont check bn if not n: n = self.generic_name try: v = self.unit_value_change(v, u) if n not in index.keys(): index[n] = 0 if n not in data.keys(): data[n] = {} data[n][index[n]] = v index[n] += 1 except Exception as e: logger.error("error " + str(e) + " n = " + str(n)) logger.debug("data: " + str(data)) return data return {}
def add_formated_data(self, json_data): doc = None try: doc = senml.SenMLDocument.from_json(json_data) except Exception as e: pass if not doc: try: meas = senml.SenMLMeasurement.from_json(json_data) doc = senml.SenMLDocument([meas]) except Exception as e: pass if doc: base_data = doc.base bn, bu = None, None if base_data: bn = base_data.name bu = base_data.unit data = {} raw_data = [] doc.measurements = sorted(doc.measurements, key=lambda x: x.time) if len(doc.measurements) > 0: self.length = len(doc.measurements) for meas in doc.measurements: n = meas.name u = meas.unit v = meas.value t = meas.time if not u: u = bu if not n: n = self.generic_name try: processed_value = self.preprocess_data(bn, n, v, u) if processed_value is not None and processed_value is not {}: raw_data.append([t, processed_value]) except Exception as e: self.logger.error("error " + str(e) + " n = " + str(n)) if len(raw_data) > 0: data[self.generic_name] = raw_data return data return {}
def senml_message_format(self, data, current_time, dT): new_data = {} # self.logger.debug("data for senml "+str(data)) for key, value in data.items(): flag = False time = current_time u = None base = None if isinstance(value, dict): bn, n, val = self.get_names(value) else: bn, n, val = None, None, value if bn: base = senml.SenMLMeasurement() base.name = bn if key in self.mqtt_params.keys(): if self.mqtt_params[key]["unit"] is not None: u = self.mqtt_params[key]["unit"] """ else: u = "W" """ flag = self.mqtt_params[key]["horizon_values"] meas_list = [] for v in val: meas = senml.SenMLMeasurement() meas.name = n meas.time = time meas.value = v if u: meas.unit = u meas_list.append(meas) time += dT if not flag: break # only want the first value if len(meas_list) > 0: doc = senml.SenMLDocument(meas_list, base=base) new_data[key] = doc.to_json() # self.logger.debug("Topic MQTT Senml message: "+str(new_data)) return new_data
def fetch_data(self, q, hr, min): """Data fetch thread. Runs at 00:30 every day""" while True: try: self.update_url() data = self.fetch_data_api() if data is not None: data_list = self.extract_data(data) meas_list = [] for row in data_list: meas = self.to_senml(row[0], row[1], row[2], self.name) meas_list.append(meas) logger.info("length of data = " + str(len(meas_list))) doc = senml.SenMLDocument(meas_list) json_data = doc.to_json() json_data = json.dumps(json_data) q.put(json_data) delay = self.get_delay_time(hr, min) time.sleep(delay) except Exception as e: logger.error(e) time.sleep(10)
def format_data(data=[]): new_data = [] doc = None try: doc = senml.SenMLDocument.from_json(data) except Exception as e: pass if not doc: try: meas = senml.SenMLMeasurement.from_json(data) doc = senml.SenMLDocument([meas]) except Exception as e: pass if doc: for meas in doc.measurements: try: n = meas.name v = meas.value t = meas.time t = RawDataReader.time_conversion(t) cols = [t, v] new_data.append(cols) except Exception as e: logger.error("Exception in formating meas " + str(e)) return new_data else: for row in data: try: cols = row.replace('\n', '').strip().split(",") dateTime = float(cols[0]) cols = cols[1:] cols = list(map(float, cols)) cols.insert(0, dateTime) new_data.append(cols) except Exception as e: logger.error("Exception in formating line " + str(row) + " " + str(e)) return new_data