def make_test(self):
result_json = {}
steps = 10
value = "Dew Point"
current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
result_json["Test started: "] = current_time
result_json["Value: "] = value
result_json["Steps in the future"] = steps
result_json["Use second data type: "] = self.use_second_data_type
sql_client = MySQLClient("xxx.xxx.xxx.xxx", "login", "pwd", "dbname")
station_locations_rec = sql_client.get_latitude_and_longitude()
datetime_unit_of_measure = "minute"
period_value = 11
optimize = True
time_handler = TimeHandler(datetime_unit_of_measure, period_value,
self.date_from, self.date_till)
for num in range(1, 55):
station_code = "LV"
if len(str(num)) == 1:
station_code += "0" + str(num)
else:
station_code += str(num)
print(station_code)
station_data = {}
records = sql_client.get_info_by_station(station_code)
if not records:
station_data["Data points: "] = 0
continue
list_of_measurements = DataHandler.get_filled_list_of_measurements(
records, value)
list_of_datetime = DataHandler.get_exact_value_from_many_my_sql_records(
[records], 2)[0]
data_df = time_handler.get_datetime_and_measurements_dataframe(
list_of_datetime, list_of_measurements)
data_df = data_df[~data_df.index.duplicated(keep='first')]
if data_df.empty:
station_data["Data points: "] = 0
continue
station_data["Data points: "] = len(data_df)
actual_data_df = time_handler. \
get_real_data_for_comparing_with_forecast(list_of_datetime, list_of_measurements,
date_from_for_actual_data=time_handler.date_till,
steps=steps)
if data_df.isnull().values.any() or actual_data_df.isnull(
).values.any():
station_data["None values: "] = int(
data_df.isna().sum()["Measurements"])
station_clusters_df = Cluster(
station_locations_rec).get_stations_clusters_df()
if data_df.isnull().values.any() or actual_data_df.isnull(
).values.any():
data_df, actual_data_df = DataFiller(
data_df, station_clusters_df, station_code,
time_handler, sql_client, station_locations_rec,
self.use_second_data_type, actual_data_df,
steps).fill_data_none_values(value)
station_data["None values after filling: "] = int(
data_df.isna().sum()["Measurements"])
if data_df.isnull().values.any():
continue
else:
station_data["None values: "] = 0
use_kf_values = [False, True]
result_json[station_code] = station_data
station_rmse_and_model_results = {}
# Kalman Filter
for use_fk in use_kf_values:
# using Kalman Filter
if use_fk:
filtered_values_list = KF_1D(
DataHandler.get_list_of_dataframe_values(
data_df.values)).get_filtered_values()
data_df["Measurements"] = filtered_values_list
data_df.sort_index(inplace=True)
arima_model = Forecasting_Model("ARIMA", data_df, steps,
station_code, optimize)
forecast_arima = arima_model.get_forecast()
if forecast_arima is not None:
arima_rmse = self.get_forecast_accuracy_with_real_data(
forecast_arima, actual_data_df.values)
arima_model_order = arima_model.best_model_order
else:
arima_rmse = None
arima_model_order = None
if use_fk:
station_rmse_and_model_results["With Kalman Filter: "] = {
"RMSE: ": arima_rmse,
"ARIMA model": arima_model_order
}
else:
station_rmse_and_model_results[
"Without Kalman Filter: "] = {
"RMSE: ": arima_rmse,
"ARIMA model": arima_model_order
}
station_data[
"Forecast results: "] = station_rmse_and_model_results
result_json[station_code] = station_data
current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
result_json["Test ended: "] = current_time
return result_json
from openpyxl import Workbook
from openpyxl import load_workbook
from Program.DataHandler import DataHandler
from Program.GraphEditor import GraphEditor
from Program.KalmanFilter import KalmanFilter
from Program.MySQLClient import MySQLClient
if __name__ == '__main__':
sql_client = MySQLClient("xxx.xxx.xxx.xxx", "login", "pwd", "dbname")
value = 'Dew Point °C'
period = '19 Jan - 19 Feb 2020'
records = sql_client.get_all_info_by_stations()
index = DataHandler.get_index_of_value('Dew Point')
lists_of_measurements = DataHandler.get_exact_value_from_many_my_sql_records(
records, index)
lists_of_measurements = DataHandler.get_lists_of_floats(
lists_of_measurements)
error_in_estimate = 1.0
error_in_measurement = 1.0
lists_of_measurements = DataHandler.replace_none_with_dash(
lists_of_measurements)
lists_of_measurements = DataHandler.fill_missing_data_in_lists(
lists_of_measurements)
station_codes = DataHandler.get_station_codes()
def get_forecast_all_models():
sql_client = MySQLClient("xxx.xxx.xxx.xxx", "login", "pwd", "dbname")
value = request.args["value"]
station_code = request.args["station"]
steps = int(request.args["steps"])
datetime_unit_of_measure = request.args["datetime_unit_of_measure"]
period_value = int(request.args["period_value"])
try:
use_fk = request.args["use_Kalman_Filter"] in ["True", "true"]
except:
use_fk = False
try:
use_second_data_type = request.args["use_second_data_type"] in [
"True", "true"
]
except:
use_second_data_type = False
try:
optimize = request.args["optimize"] in ["True", "true"]
except:
optimize = True
try:
# YYYY-MM-DD ---> YYYY-MM-DD HH:MM:SS
date_from = request.args["date_from"].replace("_", " ")
date_till = request.args["date_till"].replace("_", " ")
except:
date_from = None
date_till = None
records = sql_client.get_info_by_station(station_code)
list_of_measurements = DataHandler.get_filled_list_of_measurements(
records, value)
list_of_datetime = DataHandler.get_exact_value_from_many_my_sql_records(
[records], 2)[0]
# if optimize:
# if len(list_of_measurements) > 1000:
# list_of_measurements = list_of_measurements[-1000:]
# list_of_datetime = list_of_datetime[-1000:]
#
# if len(list_of_measurements) < 100:
# return data_is_too_small(station_code)
time_handler = TimeHandler(datetime_unit_of_measure,
period_value,
date_from=date_from,
date_till=date_till)
data_df = time_handler.get_datetime_and_measurements_dataframe(
list_of_datetime, list_of_measurements)
data_df = data_df[~data_df.index.duplicated(keep='first')]
if data_df.empty:
return data_problems(station_code)
if optimize and len(data_df) > 1100:
data_df = data_df.iloc[-1100:]
contains_none_values = False
if data_df.isnull().values.any():
contains_none_values = True
station_locations_rec = sql_client.get_latitude_and_longitude()
station_clusters_df = Cluster(
station_locations_rec).get_stations_clusters_df()
data_df = DataFiller(data_df, station_clusters_df, station_code,
time_handler, sql_client, station_locations_rec,
use_second_data_type).fill_data_none_values(value)
if data_df.isnull().values.any():
return data_still_has_none_values(station_code)
if use_fk:
filtered_values_list = KF_1D(
DataHandler.get_list_of_dataframe_values(
data_df.values)).get_filtered_values()
data_df["Measurements"] = filtered_values_list
data_df.sort_index(inplace=True)
arima_model = Forecasting_Model("ARIMA", data_df, steps, station_code,
optimize)
forecast_arima = arima_model.get_forecast()
arima_data_points = arima_model.data_points_to_use
arima_model_order = arima_model.best_model_order
arma_model = Forecasting_Model("ARMA", data_df, steps, station_code,
optimize)
forecast_arma = arma_model.get_forecast()
arma_data_points = arma_model.data_points_to_use
arma_model_order = arma_model.best_model_order
ar_model = Forecasting_Model("AR", data_df, steps, station_code, optimize)
forecast_ar = ar_model.get_forecast()
ar_data_points = ar_model.data_points_to_use
ar_model_order = ar_model.best_model_order
ma_model = Forecasting_Model("MA", data_df, steps, station_code, optimize)
forecast_ma = ma_model.get_forecast()
ma_data_points = ma_model.data_points_to_use
ma_model_order = ma_model.best_model_order
results_dict = dict()
response_json = dict()
if contains_none_values:
results_dict[
"Data overview: "] = "In the selected data were found and filled None values"
else:
results_dict["Data overview: "] = "Data has no None values"
# ARIMA results
if forecast_arima is not None:
p, d, q = arima_model_order
arima_model_order = {"p: ": p, "d: ": d, "q: ": q}
results_dict["Data points for ARIMA: "] = arima_data_points
results_dict["ARIMA model order: "] = arima_model_order
results_dict["Steps: "] = steps
results_dict["ARIMA forecast: "] = forecast_arima
else:
results_dict["ARIMA results: "] = "ARIMA model was not created"
# ARMA results
if forecast_arma is not None:
p, d, q = arma_model_order
arma_model_order = {"p: ": p, "q: ": q}
results_dict["Data points for ARMA: "] = arma_data_points
results_dict["ARMA model order: "] = arma_model_order
results_dict["Steps: "] = steps
results_dict["ARMA forecast: "] = forecast_arma
else:
results_dict["ARMA results: "] = "ARMA model was not created"
# AR results
if forecast_ar is not None:
p, d, q = ar_model_order
ar_model_order = {"p: ": p}
results_dict["Data points for AR: "] = ar_data_points
results_dict["AR model order: "] = ar_model_order
results_dict["Steps: "] = steps
results_dict["AR forecast: "] = forecast_ar
else:
results_dict["AR results: "] = "AR model was not created"
# MA results
if forecast_ma is not None:
p, d, q = ma_model_order
ma_model_order = {"q: ": q}
results_dict["Data points for MA: "] = ma_data_points
results_dict["MA model order: "] = ma_model_order
results_dict["Steps: "] = steps
results_dict["MA forecast: "] = forecast_ma
else:
results_dict["MA results: "] = "MA model was not created"
response_json["Kalman Filter: "] = use_fk
response_json[station_code] = results_dict
return response_json
def fill_data_none_values(self, value):
stations_codes_to_use_list = self.get_list_of_stations_codes_to_use()
df_s = []
act_df_s = []
for station_code in stations_codes_to_use_list:
records = self.sql_client.get_info_by_station(station_code)
if not records:
continue
list_of_measurements = DataHandler.get_filled_list_of_measurements(records, value)
list_of_datetime = DataHandler.get_exact_value_from_many_my_sql_records([records], 2)[0]
df = self.time_handler.get_datetime_and_measurements_dataframe(list_of_datetime, list_of_measurements)
if not df.empty:
df_s.append(df)
if self.actual_data_df is not None:
act_df = self.time_handler. \
get_real_data_for_comparing_with_forecast(list_of_datetime, list_of_measurements,
date_from_for_actual_data=self.time_handler.date_till,
steps=self.steps)
if not act_df.empty:
act_df_s.append(act_df)
for datetime, measurement in self.data_df[self.data_df["Measurements"].isnull()].iterrows():
temp_measurement = 0
i_temp = 0
for df in df_s:
temp_datetime = self.time_handler. \
get_datetime_from_dataframe_with_delta_limitation(df, datetime, self.time_handler.period_value)
if temp_datetime != -1:
temp_value = df.loc[temp_datetime]["Measurements"]
if type(temp_value) == pandas.Series:
continue
if temp_value is not None and not math.isnan(temp_value):
temp_measurement += temp_value
i_temp += 1
if i_temp != 0:
temp_measurement = temp_measurement / i_temp
self.data_df.loc[datetime]["Measurements"] = temp_measurement
if self.use_second_data_type:
self.lvgmc_station.get_lvgmc_station_data()
second_data_type_value = self.lvgmc_station.get_second_data_type_value(datetime)
if second_data_type_value is not None:
temp_measurement = (temp_measurement + second_data_type_value) / 2
self.data_df.loc[datetime]["Measurements"] = temp_measurement
else:
if self.use_second_data_type:
self.lvgmc_station.get_lvgmc_station_data()
second_data_type_value = self.lvgmc_station.get_second_data_type_value(datetime)
if second_data_type_value is not None:
temp_measurement = second_data_type_value
self.data_df.loc[datetime]["Measurements"] = temp_measurement
if self.actual_data_df is not None:
for datetime, measurement in self.actual_data_df[self.actual_data_df["Measurements"].isnull()].iterrows():
temp_measurement = 0
i_temp = 0
for act_df in act_df_s:
temp_datetime = self.time_handler. \
get_datetime_from_dataframe_with_delta_limitation(act_df, datetime,
self.time_handler.period_value)
if temp_datetime != -1:
temp_value = act_df.loc[temp_datetime]["Measurements"]
if type(temp_value) == pandas.Series:
continue
if temp_value is not None and not math.isnan(temp_value):
temp_measurement += temp_value
i_temp += 1
if i_temp != 0:
temp_measurement = temp_measurement / i_temp
self.data_df.loc[datetime]["Measurements"] = temp_measurement
if self.use_second_data_type:
self.lvgmc_station.get_lvgmc_station_data()
second_data_type_value = self.lvgmc_station.get_second_data_type_value(datetime)
if second_data_type_value is not None:
temp_measurement = (temp_measurement + second_data_type_value) / 2
self.actual_data_df.loc[datetime]["Measurements"] = temp_measurement
else:
if self.use_second_data_type:
self.lvgmc_station.get_lvgmc_station_data()
second_data_type_value = self.lvgmc_station.get_second_data_type_value(datetime)
if second_data_type_value is not None:
temp_measurement = second_data_type_value
self.actual_data_df.loc[datetime]["Measurements"] = temp_measurement
return self.data_df, self.actual_data_df
else:
return self.data_df
def get_filtered_values_using_1d_kalman_filter():
sql_client = MySQLClient("xxx.xxx.xxx.xxx", "login", "pwd", "dbname")
value = request.args["value"]
station_code = request.args["station"]
datetime_unit_of_measure = request.args["datetime_unit_of_measure"]
period_value = int(request.args["period_value"])
try:
use_second_data_type = request.args["use_second_data_type"] in [
"True", "true"
]
except:
use_second_data_type = False
try:
# YYYY-MM-DD ---> YYYY-MM-DD HH:MM:SS
date_from = request.args["date_from"].replace("_", " ")
date_till = request.args["date_till"].replace("_", " ")
except:
date_from = None
date_till = None
records = sql_client.get_info_by_station(station_code)
list_of_measurements = DataHandler.get_filled_list_of_measurements(
records, value)
list_of_datetime = DataHandler.get_exact_value_from_many_my_sql_records(
[records], 2)[0]
time_handler = TimeHandler(datetime_unit_of_measure,
period_value,
date_from=date_from,
date_till=date_till)
data_df = time_handler.get_datetime_and_measurements_dataframe(
list_of_datetime, list_of_measurements)
data_df = data_df[~data_df.index.duplicated(keep='first')]
if data_df.empty:
return data_problems(station_code)
contains_none_values = False
if data_df.isnull().values.any():
contains_none_values = True
station_locations_rec = sql_client.get_latitude_and_longitude()
station_clusters_df = Cluster(
station_locations_rec).get_stations_clusters_df()
data_df = DataFiller(data_df, station_clusters_df, station_code,
time_handler, sql_client, station_locations_rec,
use_second_data_type).fill_data_none_values(value)
if data_df.isnull().values.any():
return data_still_has_none_values(station_code)
filtered_values_list = KF_1D(
DataHandler.get_list_of_dataframe_values(
data_df.values)).get_filtered_values()
results_dict = dict()
response_json = dict()
if contains_none_values:
results_dict[
"Data overview: "] = "In the selected data were found and filled None values"
else:
results_dict["Data overview: "] = "Data has no None values"
results_dict[
"Filtered values using 1D Kalman Filter: "] = filtered_values_list
response_json[station_code] = results_dict
return response_json