def read_file_data(self, file_path):
with open(file_path, "r") as f:
file_lines = f.readlines()
if any(";" in s for s in file_lines):
timed_vals = []
# format the file according to val;time
self.is_timed = True
for line in file_lines:
line = line.replace("\n", "")
vals = line.strip().split(";")
val = vals[0]
val = float(val.replace(",", "."))
time = vals[1]
time = float(time.replace(",", "."))
timed_vals.append([time, val])
timed_vals.sort(key=lambda x: x[0])
timed_vals = TimeSeries.expand_and_resample(
timed_vals, self.delta_time)
return timed_vals
else:
vals = []
for line in file_lines:
line = line.replace("\n", "")
val = float(line.replace(",", "."))
vals.append(val)
return vals
def get_data(self, timestamp):
if len(self.base_data) == 0:
self.fetch_base_data()
if len(self.base_data) > 0:
data, start_time = self.filter_next_horizon_hrs(timestamp)
data = self.append_timestamp(data, start_time)
data = TimeSeries.expand_and_resample(data, self.dT_in_seconds, False)
return data
def add_formated_data(self, json_data):
doc = None
try:
doc = senml.SenMLDocument.from_json(json_data)
except Exception as e:
pass
if not doc:
try:
meas = senml.SenMLMeasurement.from_json(json_data)
doc = senml.SenMLDocument([meas])
except Exception as e:
pass
if doc:
base_data = doc.base
bn, bu = None, None
if base_data:
bn = base_data.name
bu = base_data.unit
data = {}
raw_data = []
doc.measurements = sorted(doc.measurements, key=lambda x: x.time)
if len(doc.measurements) > 0:
for meas in doc.measurements:
n = meas.name
u = meas.unit
v = meas.value
t = meas.time
t = self.time_conversion(t)
if not u:
u = bu
# dont check bn
if not n:
n = self.generic_name
try:
processed_value = self.preprocess_data(bn, n, v, u)
if processed_value is not None and processed_value is not {}:
raw_data.append([t, processed_value])
except Exception as e:
self.logger.error("error " + str(e) + " n = " + str(n))
#self.logger.debug("raw data: " + str(raw_data))
raw_data = TimeSeries.expand_and_resample(raw_data, self.dT, True)
if len(raw_data) > 0:
self.length = len(raw_data)
bucket = self.time_to_bucket(raw_data[0][0])
for row in raw_data:
bucket_key = str(self.current_day_index) + "_" + str(bucket)
bucket += 1
if bucket >= self.total_steps_in_day:
bucket = 0
self.current_day_index += 1
if self.current_day_index >= self.number_of_bucket_days:
self.current_day_index = 0
data[bucket_key] = row[1]
return data
return {}
def expand_and_resample_into_blocks(self, raw_data, model_data_dT, input_size, output_size):
if len(raw_data) > 0:
min_length = input_size + output_size
blocks = self.break_series_into_countinous_blocks(raw_data, model_data_dT, min_length)
logger.info("num blocks = " + str(len(blocks)))
resampled_blocks = []
block_has_min_length = []
merged = False
for block in blocks:
resampled_block = TimeSeries.expand_and_resample(block, model_data_dT)
if len(resampled_block) > 0:
resampled_blocks.append(resampled_block)
logger.info("block size = " + str(len(resampled_block)))
if len(resampled_block) >= min_length:
block_has_min_length.append(True)
else:
block_has_min_length.append(False)
if len(block_has_min_length) > 0 and not any(block_has_min_length):
logger.info("merging block because insufficient data")
new_block = []
end_time = resampled_blocks[-1][-1][0]
# TODO : check logic
for i in reversed(range(len(resampled_blocks))):
rsb = resampled_blocks[i]
start_time = rsb[0][0]
if end_time - start_time < min_length * model_data_dT:
rsb.extend(new_block)
new_block = rsb
merged = True
else:
rsb.extend(new_block)
new_block = rsb
merged = True
break
if merged:
new_block = TimeSeries.expand_and_resample(new_block, model_data_dT)
logger.info("length of merged blocks after expand = " + str(len(new_block)))
resampled_blocks = [new_block]
return resampled_blocks, merged
else:
return [], False
def read_raw_data(self, start_time):
start_of_day = self.get_start_of_the_day(start_time)
end_time = start_time + self.dT_in_seconds * self.horizon_in_steps
data = self.raw_data.get_raw_data_by_time_influx(
self.influxDB, self.topic_name, start_time, end_time, self.id)
data = TimeSeries.expand_and_resample(data, self.dT_in_seconds)
#data = data[:-1]
data = data[:self.horizon_in_steps]
new_data = {}
for t, v in data:
bucket = self.time_to_bucket(t, start_of_day)
new_data[bucket] = v
return new_data
def postprocess_data(self, prediction, startTimestamp, delta, horizon_steps, Xmax, Xmin):
data = prediction.reshape(-1, 1)
# data = scaler.inverse_transform(data)
data = (data - self.min) / (self.max - self.min)
data = data * (Xmax - Xmin) + Xmin
data = data.reshape(-1)
startTime = datetime.datetime.fromtimestamp(startTimestamp)
result = []
for pred in data:
result.append([startTime.timestamp(), pred])
startTime += datetime.timedelta(seconds=60)
result = TimeSeries.expand_and_resample_reversed(result, delta, False)
result = result[:horizon_steps]
logger.debug("pred out start val = " + str(result[0]))
output = {}
for t, v in result:
output[datetime.datetime.fromtimestamp(t)] = v
return output
def run(self):
self.logger.debug("Running pv prediction")
while not self.stopRequest.is_set():
if not self.redisDB.get_bool(Constants.get_data_flow_key(self.id)):
time.sleep(30)
continue
self.logger.debug("pv prediction data flow true")
try:
start = time.time()
data = self.raw_data.get_raw_data()
self.logger.debug("pv data in run is " + str(data))
if len(data) > 0:
value = data[0][1]
current_timestamp = data[0][0]
self.logger.debug("pv received timestamp " +
str(current_timestamp) + " val " +
str(value))
base_data = self.radiation.get_data(current_timestamp)
shifted_base_data = TimeSeries.shift_by_timestamp(
base_data, current_timestamp, self.dT_in_seconds)
self.logger.debug("base_data = " + str(shifted_base_data))
adjusted_data = self.adjust_data(shifted_base_data, value,
current_timestamp)
predicted_data = self.extract_horizon_data(adjusted_data)
self.logger.debug("pv predicted timestamp " +
str(predicted_data[0][0]))
if predicted_data is not None and len(predicted_data) > 0:
self.q.put(predicted_data)
self.old_predictions[int(
predicted_data[0][0])] = predicted_data
start = self.control_frequency - (time.time() - start)
if start > 0:
time.sleep(start)
except Exception as e:
self.logger.error(
str(self.generic_name) + " prediction thread exception " +
str(e))
def run(self):
while not self.stopRequest.is_set():
if not self.redisDB.get_bool(Constants.get_data_flow_key(self.id)):
time.sleep(30)
continue
try:
data = self.raw_data.get_raw_data()
self.logger.debug("len data = " + str(len(data)))
data = TimeSeries.expand_and_resample(data, 60)
self.logger.debug("len resample data = " + str(len(data)))
true_data = data
if len(data) > 0:
data = self.processingData.append_mock_data(data, self.input_size, 60)
self.logger.debug("len appended data = " + str(len(data)))
if len(data) > self.input_size:
st = time.time()
test_predictions = []
model, graph = self.models.get_model(self.id + "_" + self.topic_name, True, self.redisDB)
predicted_flag = False
if model is not None and graph is not None:
if self.type == "load":
Xtest, Xmax, Xmin, latest_timestamp = self.processingData.preprocess_data_predict_load(data,
self.input_size)
else:
Xtest, Xmax, Xmin, latest_timestamp = self.processingData.preprocess_data_predict_pv(data,
self.input_size,
self.input_size_hist)
try:
self.logger.debug(
"model present, so predicting data for " + str(self.id) + " " + str(self.topic_name))
from prediction.predictModel import PredictModel
predictModel = PredictModel(self.stop_request_status)
prediction_time = time.time()
test_predictions = predictModel.predict_next_horizon(model, Xtest, self.batch_size, graph, self.type)
self.logger.debug("Prediction successful for " + str(self.id) + " " + str(self.topic_name) +
" which took "+str(time.time()-prediction_time) + " seconds")
predicted_flag = True
except Exception as e:
predicted_flag = False
self.models.remove_saved_model()
self.logger.error("exception when prediction using model : " + str(e))
if predicted_flag:
test_predictions = self.processingData.postprocess_data(test_predictions, latest_timestamp,
self.dT_in_seconds,
self.horizon_in_steps, Xmax, Xmin)
self.logger.debug("predictions values Xmax "+str(Xmax)+" Xmin "+str(Xmin))
self.q.put(test_predictions)
self.old_predictions.append(test_predictions)
if not predicted_flag:
self.logger.info("prediction model is none, extending the known values")
data = TimeSeries.expand_and_resample(true_data, self.dT_in_seconds)
test_predictions = self.processingData.get_regression_values(data, self.input_size,
self.output_size + 1,
self.dT_in_seconds)
self.q.put(test_predictions)
self.logger.debug(str(self.topic_name) + " predictions " + str(len(test_predictions)))
st = time.time() - st
ss = self.control_frequency - st
if ss < 0:
ss = 0
time.sleep(ss)
else:
time.sleep(1)
except Exception as e:
self.logger.error(str(self.topic_name) + " prediction thread exception " + str(e))