def extract_sensor_data_tocsv(self,sensorId):
# Takes the MySQL data and stores as a csv file for building ML models
# Specify the sensorId for which the model needs to be generated
logger.info("Extracting data from database to csv")
extract_query = "select * from" + " " + self.table_name + " " +"where sensorId=\"" + sensorId + "\"" + " " +\
"order by" + " " + "timeval;"
query_result = self.sql_object.query_table(extract_query)
df_dict = {} ## Dictionary to create the dataframe from the query results
df_dict["values"] = []
timestamps = []
start_timestamp = next(iter(query_result))
compare_timestamp = start_timestamp
df_dict["values"].append(query_result[start_timestamp]) # Insert the first value anyway
timestamps.append(start_timestamp)
for key in query_result.keys():
# Keep only 60 seconds different time values
compare_timestamp = key
if int((compare_timestamp - start_timestamp).total_seconds()/60) == 1:
timestamps.append(key)
df_dict["values"].append(query_result[key])
start_timestamp = compare_timestamp
data_frame = pd.DataFrame(df_dict)
data_frame.index = timestamps
data_frame.to_csv(self.data_path + self.mode_directory+"/" + sensorId + ".csv",index=True,index_label="timestamp")
return query_result
def __init__(self):
# Initiaizing the SQL Utils
logger.info("Initializing SQL Class")
parser = ConfigParser()
parser.read(CONFIGURATION_FILE)
self.host = parser.get(CONFIG_SECTION,"host")
self.user = parser.get(CONFIG_SECTION, 'user')
self.database = parser.get(CONFIG_SECTION,"database")
self.password = parser.get(CONFIG_SECTION,"password")
def insert(self,query):
# Insert data based on the query
logger.info("inserting query " + query)
mydb = mysql.connector.connect(
host=self.host,
user=self.user,
passwd=self.password,
database='sys',
auth_plugin='mysql_native_password'
)
mycursor = mydb.cursor()
print (query)
mycursor.execute(query)
mydb.commit()
mydb.close()
def read_data(self, filepath):
logger.info("reading the dataset")
# Load the dataset into the memory and start the learning
# Takes as input the path of the csv which contains the descriptions of energy consumed
# Datasets reside in the mode folder
aggregated_df = read_csv(filepath,
header=0,
parse_dates=[0],
index_col=0,
squeeze=True,
date_parser=self.parser,
nrows=600)
aggregated_series = aggregated_df.values # Convert the dataframe to a 2D array and pass back to the calling function
return aggregated_series, aggregated_df
def query_table(self,query):
# Query the table to generate the results
logger.info("Inside the query table call")
mydb = mysql.connector.connect(
host=self.host,
user=self.user,
passwd=self.password,
database='sys',
auth_plugin='mysql_native_password'
)
cursor = mydb.cursor()
cursor.execute(query)
data_dict = {}
for (sensorId, timeVal, dataval) in cursor:
#print (reviewId + " " + review_text)
data_dict[timeVal] = dataval
cursor.close()
mydb.close()
return data_dict
def post(self):
# Update the new pattern
logger.info("Inside Pattern Changer")
response_json = {}
response_json["status"] = "failed"
try:
json_request_string = self.request.body
json_object = json.loads(json_request_string)
pattern = json_object["pattern"]
print(pattern)
adaptation_file = open(init_object.adaptation_file, "w")
string_to_write = "ada " + pattern
adaptation_file.write(string_to_write)
adaptation_file.close()
response_json["status"] = "success"
self.write(json_encode(response_json))
#print (pattern)
except Exception as e:
self.write(json_encode(response_json))
logger.error(e)
def extract_sensor_data(self):
# Extracts the sensor data and stores it in MySQL Database
file = open(self.data_path + "log.txt", "r")
logger.info("inside the extract sensor data function")
prev_time = 0.0 # Keep a check on the time
line_count = 0
df_dict = {}
current_time = 0
try:
for line in file.readlines():
if "Time" in line:
if line_count > 0:
df_dict[prev_time] = 0
current_time = float(line.split(":")[1].split(" ")[1])
# print (current_time)
if ("is writing the message") in line:
if any(sensor in line for sensor in self.sensor_list):
time_val = self.start_time + timedelta(seconds=float(current_time))
sensor_id = line.split(" ")[0]
value_text = line.split(" ")[6]
print (value_text)
value = value_text.split("#")[-1].strip("\"")
print (value)
query = "insert into " + self.table_name +"values(\"" + sensor_id + "\"," + "\"" + str(time_val) + "\"," + str(
float(value)) + ");"
logger.info(query)
self.sql_object.insert(query)
logger.info("Data Extraction to Database completed successfully")
except Exception as e:
logger.error(e)
def stream_data(self):
# Stream data to Kafka
producer_instance = producer_object.connect_kafka_producer()
sensor_id_seconds = {}
sensor_id_value = {}
for id in self.sensor_list:
sensor_id_seconds[id] = 0
sensor_id_value[id] = 0
seconds = 0
with open(init_object.data_traffic_path+ init_object.data_traffic_file) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=';')
count = 0
total_seconds = 0
while (True):
#print (start_timestamp)
for row in csv.reader(iter(csv_file.readline, '')):
if len(row) > 0:
line = row[0].strip("\n")
#print (line)
# print (line_data.split(";"))
if "Time" in line:
if ":" in line and " " in line:
time_list = line.split(":")[1].split(" ")
if len(time_list) > 1:
if "." in time_list[1]:
try:
current_time = float(line.split(":")[1].split(" ")[1])
#start_timestamp = self.start_time + timedelta(seconds=float(current_time))
except:
pass
# print (current_time)
if ("is writing the message") in line:
if any(sensor in line for sensor in self.sensor_list):
#res = list(filter(lambda x: x in line, self.sensor_list))
if count == 0:
start_timestamp = self.start_time + timedelta(seconds=float(current_time))
#print (start_timestamp)
#print (line)
sensor_id = line.split(" ")[0]
print (sensor_id)
if sensor_id in self.sensor_list:
#print(line)
# It can happen that S2 is matched in the list by any() above instead of S20 since both have S2
value_text = line.split(" ")[6]
value = value_text.split("#")[-1].strip("\"")
#seconds = sensor_id_seconds[sensor_id]
sensor_id_seconds[sensor_id] += 0
sensor_id_value[sensor_id] = value
line_data = sensor_id + ";" +str(seconds) + ";" + str(value)
query = "insert into " + self.table_name + " values(\"" + sensor_id + "\"," + "\"" + str(
start_timestamp) + "\"," + str(
float(value)) + ");"
logger.info(query)
print(query)
# self.sql_object.insert(query)
producer_object.publish_message(producer_instance, "sensorData", "data", line_data)
logger.info("published message ", line_data)
#print (sensor_id_seconds[sensor_id])
if int(sensor_id_seconds[sensor_id]) >= 60:
#if sensor_id[seconds] >= 600:
# Reset the value for each 10 values
sensor_id_seconds[sensor_id] = 0
#print(line_data)
#sensor_id_seconds[sensor_id] = sensor_id_seconds[sensor_id] + 60
elif count>0:
#print (current_time)
#print (current_time)
time_val = self.start_time + timedelta(milliseconds=float(current_time))
#print (time_val)
#print (start_timestamp, time_val)
#print (int((time_val - start_timestamp).total_seconds()/60), count)
#if (int((time_val - start_timestamp).total_seconds()/60)) == count:
#if (time_val - start_timestamp).total_seconds() % 60 == 0:
#print (time_val, start_timestamp)
#print (line)
sensor_id = line.split(" ")[0]
if sensor_id in self.sensor_list:
#print (line)
# It can happen that S2 is matched in the list by any() above instead of S20 since both have S2
value_text = line.split(" ")[6]
value = value_text.split("#")[-1].strip("\"")
#seconds = sensor_id_seconds[sensor_id]
seconds = int((time_val - start_timestamp).total_seconds())
#sensor_id_seconds[sensor_id]+=seconds # Add the seconds\
#if sensor_id=="S1":
#print (seconds)
#print (sensor_id,sensor_id_seconds[sensor_id])
#print (line)
if value!="A" and value!="N":
sensor_id_value[sensor_id] = value
#print(value)
mod_value = int(current_time / 60)
if mod_value > sensor_id_seconds[sensor_id]:
#print(sensor_id, mod_value,sensor_id_seconds[sensor_id] )
sensor_id_seconds[sensor_id] = mod_value
sensor_id_value[sensor_id] = value
line_data = sensor_id + ";" + str(sensor_id_seconds[sensor_id]) + ";" + str(sensor_id_value[sensor_id]) ## To easily get the sensorId
#line_data = sensor_id + ";" + str(total_seconds) + ";" + str(
# sensor_id_value[sensor_id]) ## To easily get the sensorId
print (line_data)
producer_object.publish_message(producer_instance, "sensorData", "data",
line_data)
logger.info("published message ", line_data)
#seconds = seconds + 60
#sensor_id_seconds[sensor_id] = sensor_id_seconds[sensor_id] + 60
#if seconds>600:
# Reset the value for each 10 values
#seconds = 0
#print (sensor_id_seconds)
count+=1
def parser(self, x):
# date-time parsing function for loading the dataset
logger.info("Inside date parser")
return datetime.strptime(x, '%Y-%m-%d %H:%M:%S')
def proactive(self,
inverse_forecast_features,
energy_forecast_total,
horizon=10):
self.time_count += self.init_obj.lag
# First form the list with the accumulated sum of each forecast for the time horizon
#print (inverse_forecast_features)
in_energy_list = []
energy_value_list = []
for j in range(0, inverse_forecast_features.shape[1]):
energy_value_list.append(inverse_forecast_features[0, j])
if (len(energy_value_list) == 22):
in_energy_list.append(energy_value_list)
energy_value_list = []
energy_list = []
print(in_energy_list)
for index in range(22):
sum_val = 0
for i in range(horizon):
sum_val = sum_val + in_energy_list[i][index]
energy_list.append(sum_val)
#print (energy_list)
print(energy_forecast_total)
max_value = 0
max_index = 0
for index in range(0, len(energy_list)):
if (index != 20):
if energy_list[index] > max_value:
max_value = energy_list[index]
max_index = index
# Calculate the frequency reduction and write to the text file
frequency_map = self.dict_sensor_freq_keys.copy()
# Calculate the data transfer frequency reduction
total_energy_consumed = sum(energy_list)
print("proactive plan")
logger.info("Inside Adaptation Planner --proactive")
#total_energy_consumed = total_energy_consumed + random.randint(0,3)
total_energy_consumed = energy_forecast_total
if total_energy_consumed >= self.high_power:
#self.time_count += self.init_obj.lag
self.adapation_count += 1
for index in self.sensor_id_list:
reduction_freq_critical = 0
reduction_freq_normal = 0
if energy_list[index] == max_value:
#print ("here")
reduction_freq_normal = self.reduction_freq_normal_hp
reduction_freq_critical = self.reduction_freq_critical_hp
else:
reduction_percent = ((max_value - energy_list[index]) /
max_value)
reduction_freq_normal = int(self.reduction_freq_normal_hp *
reduction_percent)
reduction_freq_critical = int(
self.reduction_freq_critical_hp * reduction_percent)
sensor_key = "S" + str(
index
) # Form the sensor id to be used to get data from the reverse map
sensor_freq_key = self.sensor_id_key_map[
self.reverse_sensor_map[sensor_key]]
frequency_map[sensor_freq_key + "Norm"] = frequency_map[
sensor_freq_key + "Norm"] + reduction_freq_normal
frequency_map[sensor_freq_key + "Crit"] = frequency_map[
sensor_freq_key + "Crit"] + reduction_freq_critical
# Write the adaptation to the file
write_string = ""
for key in frequency_map:
write_string = write_string + key + " " + str(
frequency_map[key]) + "\n"
write_string = write_string[:-1]
text_file = open("config.txt", "w")
text_file.write(write_string)
text_file.close()
elif total_energy_consumed < self.high_power and total_energy_consumed >= self.base_power:
self.adapation_count += 1
for index in self.sensor_id_list:
reduction_freq_critical = 0
reduction_freq_normal = 0
if energy_list[index] == max_value:
reduction_freq_normal = self.reduction_freq_normal_bp
reduction_freq_critical = self.reduction_freq_critical_bp
else:
reduction_percent = ((max_value - energy_list[index]) /
max_value)
reduction_freq_normal = int(self.reduction_freq_normal_bp *
reduction_percent)
reduction_freq_critical = int(
self.reduction_freq_critical_bp * reduction_percent)
sensor_key = "S" + str(
index
) # Form the sensor id to be used to get data from the reverse map
sensor_freq_key = self.sensor_id_key_map[
self.reverse_sensor_map[sensor_key]]
frequency_map[sensor_freq_key + "Norm"] = frequency_map[
sensor_freq_key + "Norm"] + reduction_freq_normal
frequency_map[sensor_freq_key + "Crit"] = frequency_map[
sensor_freq_key + "Crit"] + reduction_freq_critical
# Write the adaptation to the file
write_string = ""
for key in frequency_map:
write_string = write_string + key + " " + str(
frequency_map[key]) + "\n"
write_string = write_string[:-1]
text_file = open("config.txt", "w")
text_file.write(write_string)
text_file.close()
elif total_energy_consumed < self.base_power:
# Means no need to perform any adaptation the original frequency remains as it is
self.bp_time += 1
if (self.bp_time >= self.bp_count):
# Restore back to original frequencies
# Write the adaptation to the file
write_string = ""
for key in frequency_map:
write_string = write_string + key + " " + str(
frequency_map[key]) + "\n"
write_string = write_string[:-1]
text_file = open("config.txt", "w")
text_file.write(write_string)
text_file.close()
logger.info("Adaptation reactive executor")
def reactive(self, in_energy_list):
# Get the list of energy consumption and decide on the adaptation
# Change the frequency of sensors in CupCarbon
# Ignore the index which has not to be accounted for computing the increase
# Energy list consists of 1 lists with energy of each component
'''
energy_list = []
for index in range(22):
sum_val = 0
for i in range (len(in_energy_list)):
sum_val = sum_val + in_energy_list[i][index]
energy_list.append(sum_val)
'''
max_value = 0
max_index = 0
energy_list = in_energy_list[0]
#energy_list = in_energy_list
for index in range(0, len(energy_list)):
if (index != 20):
if energy_list[index] > max_value:
max_value = energy_list[index]
max_index = index
# Calculate the frequency reduction and write to the text file
frequency_map = self.dict_sensor_freq_keys.copy()
# Calculate the data transfer frequency reduction
total_energy_consumed = sum(energy_list)
print("plan")
logger.info("Inside Adaptation Planner")
print(total_energy_consumed)
self.time_count += self.init_obj.lag
if total_energy_consumed >= self.high_power:
for index in self.sensor_id_list:
reduction_freq_critical = 0
reduction_freq_normal = 0
self.adapation_count += 1
if energy_list[index] == max_value:
print("here")
reduction_freq_normal = self.reduction_freq_normal_hp
reduction_freq_critical = self.reduction_freq_critical_hp
else:
reduction_percent = ((max_value - energy_list[index]) /
max_value)
reduction_freq_normal = int(self.reduction_freq_normal_hp *
reduction_percent)
reduction_freq_critical = int(
self.reduction_freq_critical_hp * reduction_percent)
sensor_key = "S" + str(
index
) # Form the sensor id to be used to get data from the reverse map
sensor_freq_key = self.sensor_id_key_map[
self.reverse_sensor_map[sensor_key]]
frequency_map[sensor_freq_key + "Norm"] = frequency_map[
sensor_freq_key + "Norm"] + reduction_freq_normal
frequency_map[sensor_freq_key + "Crit"] = frequency_map[
sensor_freq_key + "Crit"] + reduction_freq_critical
# Write the adaptation to the file
write_string = ""
for key in frequency_map:
write_string = write_string + key + " " + str(
frequency_map[key]) + "\n"
write_string = write_string[:-1]
text_file = open("config.txt", "w")
text_file.write(write_string)
text_file.close()
elif total_energy_consumed < self.high_power and total_energy_consumed >= self.base_power:
self.adapation_count += 1
for index in self.sensor_id_list:
reduction_freq_critical = 0
reduction_freq_normal = 0
if energy_list[index] == max_value:
reduction_freq_normal = self.reduction_freq_normal_bp
reduction_freq_critical = self.reduction_freq_critical_bp
else:
reduction_percent = ((max_value - energy_list[index]) /
max_value)
reduction_freq_normal = int(self.reduction_freq_normal_bp *
reduction_percent)
reduction_freq_critical = int(
self.reduction_freq_critical_bp * reduction_percent)
sensor_key = "S" + str(
index
) # Form the sensor id to be used to get data from the reverse map
sensor_freq_key = self.sensor_id_key_map[
self.reverse_sensor_map[sensor_key]]
frequency_map[sensor_freq_key + "Norm"] = frequency_map[
sensor_freq_key + "Norm"] + reduction_freq_normal
frequency_map[sensor_freq_key + "Crit"] = frequency_map[
sensor_freq_key + "Crit"] + reduction_freq_critical
# Write the adaptation to the file
write_string = ""
for key in frequency_map:
write_string = write_string + key + " " + str(
frequency_map[key]) + "\n"
write_string = write_string[:-1]
text_file = open("config.txt", "w")
text_file.write(write_string)
text_file.close()
elif total_energy_consumed < self.base_power:
# Means no need to perform any adaptation the original frequency remains as it is
self.bp_time += 1
if (self.bp_time >= self.init_obj.bp_count
): # Change this depending on the lag value
# Restore back to original frequencies
# Write the adaptation to the file
write_string = ""
for key in frequency_map:
write_string = write_string + key + " " + str(
frequency_map[key]) + "\n"
write_string = write_string[:-1]
text_file = open("config.txt", "w")
text_file.write(write_string)
text_file.close()
logger.info("Adaptation reactive executor")
def arima_forecastor(self):
# performs the basic naive forecast F_t = y_t - 1
# Reads the csv file into a dataframe and perform forecasts for each series to get the total sum
freq = 5 # Data every minute
horizon = 10 # Look ahead steps
aggregated_df = read_csv(self.data_path,
header=0,
parse_dates=[0],
index_col=0,
squeeze=True,
date_parser=self.parser)
# df_no_constants = aggregated_df.loc[:, (aggregated_df != aggregated_df.iloc[0]).any()]
actual_energy_list = []
data_list = []
energy_sum = 0
for i, row in aggregated_df.iterrows():
for j, column in row.iteritems():
if j not in ["S21"]:
energy_sum = energy_sum + column
actual_energy_list.append(
energy_sum) ## Directly append to main list
# print (energy_sum)
energy_sum = 0
#if (len(data_list) == 10):
# actual_energy_list.append(data_list)
# data_list = []
test_size = int(0.8 *
len(actual_energy_list)) # Divide into train and test
test_set = actual_energy_list[:-test_size]
forecast_list = []
history_list = []
history_list = actual_energy_list
history_data = Series(history_list).values
minutes_day = 1440
differenced = self.difference(history_data, minutes_day)
model = ARIMA(differenced, order=(10, 0, 1))
model_fit = model.fit(disp=1)
history_list.append(test_set[0]) ## Add the first element
logger.info("starting to forecast the test set")
#for elem in test_set:
# For each set of 10 forecasts perform the forecast for the next point and add it to list
#history_list.extend(elem)
# fit1 = ARIMA(history_list,order=(5,1,0)).fit(disp=1)
# Applying any the model from the package
# forecast_list_vals = fit1.forecast()
#print(forecast_list_vals)
#for i in forecast_list_vals:
# forecast_list.append(forecast_list_vals[0]) # Rolling forecast
# forecast_sum = sum(forecast_list_vals)
# forecast_list.append(forecast_sum) # Sum of the expected energy to make the comparison
# history_list.append(elem)
#logger.info("finished the test set forecast, last element forecast remaining")
#fit1 = ARIMA(history_list, order=(5, 1, 0)).fit(disp=0)
# Applying any the model from the package
#forecast_list_vals = fit1.forecast()
#forecast_list.append(forecast_list_vals[0]) # Rolling forecast
print(forecast_list)
logger.info(forecast_list)
# Compute the actual sum list
actual_list = []
for elem in test_set:
# print (elem_list)
#for i in elem_list:
actual_list.append(elem)
print(actual_list)
logger.info("*********************************************")
logger.info("length of forecast list", len(forecast_list))
logger.info("length of actual list", len(actual_list))
# print (len(actual_list))
# print (len(forecast_list))
try:
rmse_total = sqrt(mean_squared_error(forecast_list, actual_list))
print(rmse_total)
logger.info("RMSE Value ", rmse_total)
pyplot.plot(forecast_list[:100],
label="predicted",
linewidth=2.0,
color='#ED6A5A')
pyplot.plot(actual_list[:100],
label="actual",
linewidth=2.0,
color='#5386E4')
# pyplot.legend()
pyplot.ylabel("Energy Consumption (Joules)")
pyplot.xlabel("Time in Minutes")
pyplot.grid(True)
# plt.axhline(y=1.58, color='green', linestyle='--', linewidth=1.5)
pyplot.legend(loc="upper left")
pyplot.savefig("./plots/rmse_plot_arima.png", dpi=300)
except Exception as e:
logger.error(e)
def gather_data(self,adaptation_type,horizon=10,lag=10,decision_period=10):
global prev_vals
global main_energy_forecast
global main_traffic_forecast
consumer = KafkaConsumer(auto_offset_reset='latest',
bootstrap_servers=['localhost:9092'], api_version=(0, 10), consumer_timeout_ms=1000)
consumer.subscribe(pattern='^sensor.*') # Subscribe to a pattern
main_energy_list = []
while True:
for message in consumer:
if message.topic == "sensor":
# The QoS data comes here and the prediction needs to be done here
row = str(message.value).split(";")
if (len(row) > 3):
time_string = row[0]
second_level_data = []
row.pop() # remove the unwanted last element
vals = [x1 - float(x2) for (x1, x2) in zip(prev_vals, row[1:])]
# print (len (vals))
if (len(vals) == 22):
# Check if we have 22 elements always
# spark_predictor.main_energy_list.append(vals)
main_energy_list.append(vals)
#final_energy_list = [x + y for x, y in zip(final_energy_list, vals)] ## Keep addding them
prev_vals = [float(i) for i in row[1:]]
if adaptation_type == "reactive":
if (len(main_energy_list) == 1):
#print (main_energy_list)
# Compute the energy consumed by each sensor
ada_obj.reactive(main_energy_list)
logger.info("adaptation count " + str(ada_obj.adapation_count) + " " + str(ada_obj.time_count))
main_energy_list = [] # This will mean only every 10 minutes an adaptation will be performed
elif adaptation_type == "proactive":
#print (ada_obj.adapation_count)
if (len(main_energy_list) == lag):
print ("reached")
predict_array = np.array(main_energy_list)
predict_array = scalar_energy.fit_transform(predict_array)
# print (predict_array.shape)
predict_array = predict_array.reshape(1, lag, 22)
with graph.as_default():
energy_forecast = loaded_model_energy.predict(predict_array)
# K.clear_session()
inverse_forecast = energy_forecast.reshape(horizon, 22)
inverse_forecast = scalar_energy.inverse_transform(inverse_forecast)
inverse_forecast_features = inverse_forecast.reshape(energy_forecast.shape[0], 22*horizon)
energy_forecast_total = 0
for j in range(0, inverse_forecast_features.shape[1]):
#for j in range(0, 22*horizon): # Number of components * horizon equals inverse_forecast_Features.shape[1]
if j not in [20,42,64,86,108,130,152,174,196,218,240,262,284,306,328,350,372,394,416,438,460,482,504,526,548,570,592,614,636,658]:
# Ignore the database forecasts
energy_forecast_total = energy_forecast_total + inverse_forecast_features[0, j]
#print("Energy forecast")
#print(energy_forecast_total)
ada_obj.proactive(inverse_forecast_features,energy_forecast_total,horizon=horizon)
logger.info("adaptation count " + str(ada_obj.adapation_count) + " " + str(ada_obj.time_count))
#main_energy_list = [] # This will mean only every 10 minutes an adaptation will be performed
main_energy_list = main_energy_list[decision_period:] # This will mean only every 10 minutes an adaptation will be performed
def arima(self):
# performs the basic naive forecast F_t = y_t - 1
# Reads the csv file into a dataframe and perform forecasts for each series to get the total sum
freq = 1 # Data every minute
horizon = 10 # Look ahead steps
aggregated_df = read_csv(self.data_path, header=0, parse_dates=[0], index_col=0,
squeeze=True, date_parser=self.parser)
#df_no_constants = aggregated_df.loc[:, (aggregated_df != aggregated_df.iloc[0]).any()]
actual_energy_list = []
data_list = []
energy_sum = 0
for i, row in aggregated_df.iterrows():
energy_sum = 0
for j, column in row.iteritems():
if j not in ["S21"]:
energy_sum = energy_sum + column
data_list.append(energy_sum)
#print (energy_sum)
#if (len(data_list) == 10):
# actual_energy_list.append(data_list)
# data_list = []
actual_energy_list = data_list
#test_size = int(0.8 * len(actual_energy_list)) # Divide into train and test
with open(init_object.data_path + "list.json", "r") as json_file:
test_set = json.load(json_file)
test_set = test_set["data"]
value = test_set[0]
# Take the last 25 minutes data and make a fit
# Reason of not taking the full data for a rolling forecast, is that practically this is impossible as if
# the size of the data set becomes large, this process itself might consume a lot of time and memory for computation
forecast_list = []
actual_set = test_set
#print(len(test_set))
for index in range(0,100,1):
test_set.insert(index,actual_energy_list[(len(actual_energy_list)-1)-index])
search_index = test_set.index(value)
last_index = search_index # Get the index of the element to start the operation with
#print (len(test_set))
#time.sleep(3)
count = 0
logger.info("Total samples ",str(len(test_set[last_index:])))
print("Total samples ",str(len(test_set[last_index:])))
for elem in test_set[last_index:]:
# For each set of 10 forecasts perform the forecast for the next point and add it to list
# Applying any the model from the package
history_list = test_set[last_index-100:last_index]
#print (last_index-25, last_index)
#print (history_list)
model = ARIMA(history_list, order=(10, 0, 1)).fit(disp=0)
forecast_list_vals= model.forecast(steps=10)[0]
#fit1 = SimpleExpSmoothing(history_list).fit(smoothing_level=0.3, optimized=False)
#print (forecasts)
#print (forecast_list_vals)
#for i in forecast_list_vals:
forecast_list.append(sum(forecast_list_vals))
last_index = last_index + 1
#forecast_sum = sum(forecast_list_vals)
#forecast_list.append(forecast_sum) # Sum of the expected energy to make the comparison
count+=1
print ("sample ",count)
logger.info("Sample ", count)
#fit1 = SimpleExpSmoothing(history_list).fit(smoothing_level=0.2, optimized=False)
# Applying any the model from the package
#forecast_list_vals = fit1.forecast(10)
#forecast_list.append(sum(forecast_list_vals))
# Compute the actual sum list
print (last_index)
actual_list = []
for index in range(0, len(test_set[100:])):
# print (elem_list)
actual_list.append(sum(test_set[index:index + 10]))
print(len(actual_list))
#print (len(actual_list))
#print (len(forecast_list))
rmse_total = sqrt(mean_squared_error(actual_list, forecast_list))
naive_mae = 1.64 # from Naive Forecasts
print("RMSE ", rmse_total)
s_mape_total = self.s_mean_absolute_percentage_error(actual_list, forecast_list)
print("SMAPE ", s_mape_total)
mase_total = self.mean_absolute_scaled_error(actual_list, forecast_list, naive_mae)
print("MASE ", mase_total)
owa_score = (s_mape_total + mase_total) / 2
print("OWA ", owa_score)
pyplot.plot(forecast_list[:100], label="predicted", linewidth=2.0, color='#5386E4')
pyplot.plot(actual_list[:100], label="actual", linewidth=2.0, color='#ED6A5A')
#pyplot.legend()
pyplot.ylabel("Energy Consumption (Joules)")
pyplot.xlabel("Time in Minutes")
#pyplot.axis([0, 100, 4, 12])
pyplot.grid(True)
# plt.axhline(y=1.58, color='green', linestyle='--', linewidth=1.5)
pyplot.legend(loc="upper left")
pyplot.savefig("./plots/rmse_plot_arima.png", dpi=300)