def data_formater(self, data):
json_data = json.loads(data)
doc = None
try:
doc = senml.SenMLDocument.from_json(json_data)
except Exception as e:
pass
if not doc:
try:
meas = senml.SenMLMeasurement.from_json(json_data)
doc = senml.SenMLDocument([meas])
except Exception as e:
pass
new_data = {}
if doc:
base_data = doc.base
bn = None
if base_data:
bn = base_data.name
bn = bn.replace("/", "")
if len(doc.measurements) > 0:
for meas in doc.measurements:
n = meas.name
u = meas.unit
v = meas.value
t = meas.time
new_data[bn + "." + n] = v
return new_data
def add_formated_data(self, json_data):
doc = None
try:
doc = senml.SenMLDocument.from_json(json_data)
except Exception as e:
pass
if not doc:
try:
meas = senml.SenMLMeasurement.from_json(json_data)
doc = senml.SenMLDocument([meas])
except Exception as e:
pass
if doc:
base_data = doc.base
bn, bu = None, None
if base_data:
bn = base_data.name
bu = base_data.unit
data = {}
doc.measurements = sorted(doc.measurements, key=lambda x: x.time)
if len(doc.measurements) > 0:
for meas in doc.measurements:
n = meas.name
t = meas.time
v = meas.value
data[n] = {"last_time": t, "freq": v}
return data
return {}
def to_senml(self, value):
meas = senml.SenMLMeasurement()
meas.name = self.id
meas.value = value
meas.time = int(time.time())
doc = senml.SenMLDocument([meas])
return json.dumps(doc.to_json())
def convert_to_senml(self):
meas = []
if len(self.pv_data) > 0:
for row in self.pv_data:
meas.append(self.get_senml_meas(float(row[1]), row[0]))
doc = senml.SenMLDocument(meas)
val = doc.to_json()
return json.dumps(val)
def add_formated_data(self, json_data):
doc = None
try:
doc = senml.SenMLDocument.from_json(json_data)
except Exception as e:
pass
if not doc:
try:
meas = senml.SenMLMeasurement.from_json(json_data)
doc = senml.SenMLDocument([meas])
except Exception as e:
pass
if doc:
base_data = doc.base
bn, bu = None, None
if base_data:
bn = base_data.name
bu = base_data.unit
data = {}
raw_data = []
doc.measurements = sorted(doc.measurements, key=lambda x: x.time)
if len(doc.measurements) > 0:
self.length = len(doc.measurements)
for meas in doc.measurements:
n = meas.name
u = meas.unit
v = meas.value
t = meas.time
t = self.time_conversion(t)
if not u:
u = bu
# dont check bn
if not n:
n = self.generic_name
try:
processed_value = self.preprocess_data(bn, n, v, u)
if processed_value is not None and processed_value is not {}:
raw_data.append([t, processed_value])
except Exception as e:
self.logger.error("error " + str(e) + " n = " +
str(n))
#self.logger.debug("data: " + str(data))
raw_data = self.expand_and_resample(raw_data, self.dT)
if len(raw_data) > 0:
bucket = self.time_to_bucket(raw_data[0][0])
for row in raw_data:
bucket_key = str(
self.current_day_index) + "_" + str(bucket)
bucket += 1
if bucket >= self.total_steps_in_day:
bucket = 0
self.current_day_index += 1
if self.current_day_index >= self.number_of_bucket_days:
self.current_day_index = 0
data[bucket_key] = row[1]
return data
return {}
def extract_horizon_data(self):
meas = []
list = self.load_data.items()
list = sorted(list)
list = list[-self.horizon_in_steps:]
for i in range(self.horizon_in_steps):
value = list[i][1]
if value < 0:
value = 0
meas.append(self.get_senml_meas(value, list[i][0]))
doc = senml.SenMLDocument(meas)
val = doc.to_json()
return json.dumps(val)
def to_senml(self, name, value, timestamp):
meas = senml.SenMLMeasurement()
meas.name = name
if isinstance(value, str):
try:
value = float(value)
except Exception:
pass
meas.value = value
meas.time = timestamp
doc = senml.SenMLDocument([meas], base=self.base)
val = doc.to_json()
val = json.dumps(val)
return val
def extract_horizon_data(self):
meas = []
if len(self.pv_data) > 0:
current_timestamp = datetime.datetime.now().timestamp()
closest_index = self.find_closest_prev_timestamp(
self.pv_data, current_timestamp)
for i in range(self.horizon_in_steps):
row = self.pv_data[closest_index]
meas.append(self.get_senml_meas(float(row[1]), row[0]))
closest_index += 1
if closest_index >= len(self.pv_data):
closest_index = 0
doc = senml.SenMLDocument(meas)
val = doc.to_json()
return json.dumps(val)
def to_senml(self, results):
meas = []
base = None
if "base_name" in self.topic_params:
base = senml.SenMLMeasurement()
base.name = self.topic_params["base_name"]
for time, errors in results.items():
rmse = errors["rmse"]
mae = errors["mae"]
meas.append(
self.get_senml_meas(rmse, time, self.topic_name + "/rmse"))
meas.append(
self.get_senml_meas(mae, time, self.topic_name + "/mae"))
doc = senml.SenMLDocument(meas, base=base)
val = doc.to_json()
return json.dumps(val)
def get_data(self):
try:
if self.source == "constant":
val = self.constant_value
logger.debug("Sent MQTT:" + str(val))
return val
else:
meas_list = []
current_time = int(math.floor(time.time()))
if self.source == "file":
vals = self.get_file_line(current_time)
logger.debug("Length: " + str(self.length))
if len(vals) < self.length:
logger.error(
str(self.generic_name) +
" mock file has invalid data. Less values than horizon_step"
)
return None
elif self.source == "random":
vals = self.get_random_floats()
logger.debug("Vals: " + str(vals))
logger.debug("Length: " + str(self.length))
prev_time = 0
for index in range(self.length):
meas = senml.SenMLMeasurement()
if self.is_timed:
meas.value = vals[index][1]
if prev_time > vals[index][0]:
meas.time = prev_time + self.delta_time
else:
meas.time = int(vals[index][0])
prev_time = meas.time
else:
meas.value = vals[index]
meas.time = int(current_time)
meas.name = self.generic_name
meas_list.append(meas)
current_time += self.delta_time
doc = senml.SenMLDocument(meas_list)
val = doc.to_json()
val = json.dumps(val)
logger.debug("Sent MQTT:" + str(val))
return val
except Exception as e:
logger.error(e)
def add_formated_data(self, json_data):
doc = None
try:
doc = senml.SenMLDocument.from_json(json_data)
except Exception as e:
pass
if not doc:
try:
meas = senml.SenMLMeasurement.from_json(json_data)
doc = senml.SenMLDocument([meas])
except Exception as e:
pass
if doc:
base_data = doc.base
bn, bu = None, None
if base_data:
bn = base_data.name
bu = base_data.unit
data = {}
index = {}
doc.measurements = sorted(doc.measurements, key=lambda x: x.time)
for meas in doc.measurements:
n = meas.name
u = meas.unit
v = meas.value
t = meas.time
if not u:
u = bu
# dont check bn
if not n:
n = self.generic_name
try:
v = self.unit_value_change(v, u)
if n not in index.keys():
index[n] = 0
if n not in data.keys():
data[n] = {}
data[n][index[n]] = v
index[n] += 1
except Exception as e:
logger.error("error " + str(e) + " n = " + str(n))
logger.debug("data: " + str(data))
return data
return {}
def add_formated_data(self, json_data):
doc = None
try:
doc = senml.SenMLDocument.from_json(json_data)
except Exception as e:
pass
if not doc:
try:
meas = senml.SenMLMeasurement.from_json(json_data)
doc = senml.SenMLDocument([meas])
except Exception as e:
pass
if doc:
base_data = doc.base
bn, bu = None, None
if base_data:
bn = base_data.name
bu = base_data.unit
data = {}
raw_data = []
doc.measurements = sorted(doc.measurements, key=lambda x: x.time)
if len(doc.measurements) > 0:
self.length = len(doc.measurements)
for meas in doc.measurements:
n = meas.name
u = meas.unit
v = meas.value
t = meas.time
if not u:
u = bu
if not n:
n = self.generic_name
try:
processed_value = self.preprocess_data(bn, n, v, u)
if processed_value is not None and processed_value is not {}:
raw_data.append([t, processed_value])
except Exception as e:
self.logger.error("error " + str(e) + " n = " +
str(n))
if len(raw_data) > 0:
data[self.generic_name] = raw_data
return data
return {}
def senml_message_format(self, data, current_time, dT):
new_data = {}
# self.logger.debug("data for senml "+str(data))
for key, value in data.items():
flag = False
time = current_time
u = None
base = None
if isinstance(value, dict):
bn, n, val = self.get_names(value)
else:
bn, n, val = None, None, value
if bn:
base = senml.SenMLMeasurement()
base.name = bn
if key in self.mqtt_params.keys():
if self.mqtt_params[key]["unit"] is not None:
u = self.mqtt_params[key]["unit"]
"""
else:
u = "W"
"""
flag = self.mqtt_params[key]["horizon_values"]
meas_list = []
for v in val:
meas = senml.SenMLMeasurement()
meas.name = n
meas.time = time
meas.value = v
if u:
meas.unit = u
meas_list.append(meas)
time += dT
if not flag:
break # only want the first value
if len(meas_list) > 0:
doc = senml.SenMLDocument(meas_list, base=base)
new_data[key] = doc.to_json()
# self.logger.debug("Topic MQTT Senml message: "+str(new_data))
return new_data
def fetch_data(self, q, hr, min):
"""Data fetch thread. Runs at 00:30 every day"""
while True:
try:
self.update_url()
data = self.fetch_data_api()
if data is not None:
data_list = self.extract_data(data)
meas_list = []
for row in data_list:
meas = self.to_senml(row[0], row[1], row[2], self.name)
meas_list.append(meas)
logger.info("length of data = " + str(len(meas_list)))
doc = senml.SenMLDocument(meas_list)
json_data = doc.to_json()
json_data = json.dumps(json_data)
q.put(json_data)
delay = self.get_delay_time(hr, min)
time.sleep(delay)
except Exception as e:
logger.error(e)
time.sleep(10)
def format_data(data=[]):
new_data = []
doc = None
try:
doc = senml.SenMLDocument.from_json(data)
except Exception as e:
pass
if not doc:
try:
meas = senml.SenMLMeasurement.from_json(data)
doc = senml.SenMLDocument([meas])
except Exception as e:
pass
if doc:
for meas in doc.measurements:
try:
n = meas.name
v = meas.value
t = meas.time
t = RawDataReader.time_conversion(t)
cols = [t, v]
new_data.append(cols)
except Exception as e:
logger.error("Exception in formating meas " + str(e))
return new_data
else:
for row in data:
try:
cols = row.replace('\n', '').strip().split(",")
dateTime = float(cols[0])
cols = cols[1:]
cols = list(map(float, cols))
cols.insert(0, dateTime)
new_data.append(cols)
except Exception as e:
logger.error("Exception in formating line " + str(row) +
" " + str(e))
return new_data
