def get_train_data(self, DAY, SEC):
"""
obs_reader = obs_reader()
obs_reader.find_max([1,2,3,4,99999])
obs_reader.find_obs( 20170731, 4, "wgl")
"""
location = self.location
loc_id = self.config.loc_id
element = self.element
train_day = self.train_day
read = reader(loc_id)
obs_reader = obs_read(location, element)
obs_ARRAY = self.act_array()
LC_date = self.LC_date_array(obs_ARRAY, DAY)
obs = []
train_ens = []
for i in range(0, train_day):
obs_value = obs_reader.find_obs(obs_ARRAY[i], SEC)
if not obs_value == Config.MISSING:
element_list = read.get_element(element, LC_date[i], DAY, SEC,
Config.TYPE)
train_ens.append(element_list)
obs.append(obs_value)
print("")
print("Actual Date : " + ''.join(str(x) + " " for x in obs_ARRAY))
print("LC_date : " + ''.join(str(x) + " " for x in LC_date))
print("Observation : " + ''.join(str(x) + " " for x in obs))
#coeff = out.get_coeff()
return train_ens, obs
def gen_bias(self, fc_day, section):
read = reader(self.config.loc_dict[self.location])
OBS = []
EPS_TRAIN = []
MEAN = []
BIAS = []
for day in range(1, fc_day + 1):
for sec in range(1, section + 1):
EPS_TRAIN, OBS = self.get_train_data(day, sec)
for k in range(len(OBS)):
MEAN.extend([np.mean(EPS_TRAIN[k])])
BIAS.extend([MEAN[k] - OBS[k]])
colors = (0, 0, 0)
area = np.pi * 1
x = MEAN
y = BIAS
plt.scatter(x, y, s=area, c=colors, alpha=0.5)
plt.title('Scatter plot ')
plt.xlabel('MEAN')
plt.ylabel('Bias')
plt.xlim(0, 35.0)
plt.ylim(-15.0, 20.0)
out_fname = 'C:\Users\wlsinaa\Desktop\scatter' + '\\' + self.location + '_' + str(
self.init_dtg) + '_' + str(self.train_day) + '_' + str(
day) + '_' + str(sec) + '.png'
plt.savefig(out_fname, dpi=300)
plt.gcf().clear()
MEAN = []
BIAS = []
def gen_coeff(self, fc_day, section):
read = reader(self.config.loc_dict[self.location])
EPS_data = read.get_element(self.element,
self.D.roll_dtg(self.init_dtg, 0), fc_day,
section, Config.TYPE)
eps_mean = np.mean(EPS_data)
delta = calMBM.delta(EPS_data)
NEW_EPS = np.copy(EPS_data)
ens_amount = len(EPS_data)
hkt = self.D.roll_dtg(self.init_dtg, fc_day)
obs_value = obs_read(self.location,
self.element).find_obs(hkt, section)
print("------------")
EPS_TRAIN, OBS = self.get_train_data(fc_day, section)
ens_minimizer = Minimizer_mbm(EPS_TRAIN, OBS, self.lamda,
"reliability")
ens_minimizer.process_data()
ens_minimizer.Minimizer()
coeff = ens_minimizer.get_coeff()
# debias
#bias_coeff = self.pre_coeff
#bias = bias_coeff[0] + bias_coeff[1] * eps_mean
#EPS_data = np.array(EPS_data) - bias
for ens_no in range(ens_amount):
NEW_EPS[ens_no] = coeff[0] + coeff[1] * eps_mean + (
coeff[2] * coeff[2] +
coeff[3] * coeff[3] / delta) * (EPS_data[ens_no] - eps_mean)
print(
"--------------------------------------------------------------------------------------------------------------------------------------------------------"
)
with open(self.csv_path, 'ab') as csvfile:
writer = csv.writer(csvfile)
line = [
self.init_dtg,
str(hkt),
str(fc_day),
str(section),
str(round(coeff[0], 4)),
str(round(coeff[1], 4)),
str(round(coeff[2] * coeff[2], 4)),
str(round(coeff[3] * coeff[3], 4)),
str(delta),
str(obs_value)
]
writer.writerow(line)
csvfile.close()
return NEW_EPS
def get_train_data(self, DAY, SEC):
"""
obs_reader = obs_reader()
obs_reader.find_max([1,2,3,4,99999])
obs_reader.find_obs( 20170731, 4, "wgl")
"""
loc_id = self.config.loc_dict[self.location]
read = reader(loc_id)
obs_reader = obs_read(self.location, self.element)
obs_ARRAY = self.act_array()
LC_date = self.LC_date_array(obs_ARRAY, DAY)
obs = []
train_ens = []
for i in range(0, self.train_day):
obs_value = obs_reader.find_obs(obs_ARRAY[i], SEC)
if not obs_value == Config.MISSING:
element_list = read.get_element(self.element, LC_date[i], DAY,
SEC, Config.TYPE)
train_ens.append(element_list)
obs.append(obs_value)
element = self.element
delete_list = []
threshold = 0.0
if element == 'windspd' or element == 'windmax':
for i in range(len(obs)):
if obs[i] < threshold or obs[i] == self.config.MISSING:
delete_list.append(i)
train_ens = np.array(train_ens)
obs = np.array(obs)
train_ens = np.delete(train_ens, delete_list, 0)
obs = np.delete(obs, delete_list, 0)
sample_no = len(train_ens)
# debias
#self.pre_coeff = self.bias_coeff(train_ens,obs, method = 'LinearRegression')
#train_ens = self.preproccessing(train_ens,self.pre_coeff)
self.obs_list = obs
print("")
print("Actual Date : " + ''.join(str(x) + " " for x in obs_ARRAY))
print("LC_date : " + ''.join(str(x) + " " for x in LC_date))
print("Observation : " + ''.join(str(x) + " " for x in obs))
#coeff = out.get_coeff()
return train_ens, obs
sys.path.append("../MBM_VSV")
import KernelDenity as kde
from LC_Reader import LC_Reader as reader
from config import Config
import numpy as np
import matplotlib.pyplot as plt
from obs_reader import obs_reader
import MBM_main as mbm
import numpy as np
"""MBM = mbm.MBM(201708211200,'windspd','wgl')
EPS = MBM.gen_ALL(2,4)
new_EPS = np.array(EPS)
new_EPS = new_EPS.T"""
for i in range(0, 52):
ensembles = reader(Config.loc_id).get_eps_timeseries(
i, 201708241200, "windspd")
x = np.arange(0, len(ensembles), 1)
if i == 0:
plt.plot(x, ensembles, linewidth=1.0, color="blue")
elif i == 1:
plt.plot(x, ensembles, linewidth=1.0, color="green")
else:
plt.plot(x, ensembles, linewidth=0.3)
obs = obs_reader("windspd",
Config().obs_format_dict["windspd"]).act_mix_lc_TS(
2017082512, 14, 4)
plt.plot(x, obs, linewidth=2.0, color="red")
#print(obs)
#print(len(obs))
plt.xlim(0, 60)
def process_section(self):
loc = self.location
element = self.element
distribution = self.distribution
init_dtg = self.init_dtg
self.LC.make_outdir(init_dtg, loc, element) ##check output directory
self.LC.make_csv(init_dtg, loc, element, self.csv_filename,
self.config.header[distribution]) ##create csv
line = []
read = reader(self.config.loc_dict[self.location])
for day in range(1, self.fc_day + 1):
for sec in range(1, self.section + 1):
data = read.get_element(element,
self.D.roll_dtg(self.init_dtg, 0), day,
sec, Config.TYPE)
initial = self.init_dtg
hkt = self.D.roll_dtg(self.init_dtg, day)
dmo_hres = self.LC.get_hres(data)
dmo_ctrl = self.LC.get_ctrl(data)
dmo_mean = self.LC.get_mean(data)
dmo_var = self.LC.get_var(data)
coeff_ngr = self.train_data_ngr(day, sec)
if self.distribution != 'TN_gust':
obs_value = obs_read(loc, element).find_obs(hkt, sec)
elif self.distribution == 'TN_gust':
obs_value = obs_read(loc, 'windmax').find_obs(hkt, sec)
if distribution == 'gev':
new_ngr_mean = Minimizer_ngr.ngr_mean(
coeff_ngr, dmo_hres, dmo_ctrl, dmo_mean, dmo_var)
new_ngr_sigma = Minimizer_ngr.ngr_var(
coeff_ngr, dmo_hres, dmo_ctrl, dmo_mean, dmo_var)
with open(self.csv_path, 'ab') as csvfile:
writer = csv.writer(csvfile)
line = [
initial, hkt,
str(day),
str(sec),
str(dmo_hres),
str(dmo_ctrl),
str(dmo_mean),
str(dmo_var), coeff_ngr[0], coeff_ngr[1],
coeff_ngr[2], coeff_ngr[3], coeff_ngr[4],
str(new_ngr_mean),
str(new_ngr_sigma),
str(obs_value)
]
writer.writerow(line)
elif distribution == 'LN':
new_ngr_mean = Minimizer_ngr.LN_mu(coeff_ngr, dmo_hres,
dmo_ctrl, dmo_mean,
dmo_var)
new_ngr_sigma = Minimizer_ngr.LN_sigma(
coeff_ngr, dmo_hres, dmo_ctrl, dmo_mean, dmo_var)
a_0 = round(coeff_ngr[0] * coeff_ngr[0], 5)
a_1 = round(coeff_ngr[1] * coeff_ngr[1], 5)
a_2 = round(coeff_ngr[2] * coeff_ngr[2], 5)
a_3 = round(coeff_ngr[3] * coeff_ngr[3], 5)
b_0 = round(coeff_ngr[4] * coeff_ngr[4], 5)
b_1 = round(coeff_ngr[5] * coeff_ngr[5], 5)
with open(self.csv_path, 'ab') as csvfile:
writer = csv.writer(csvfile)
line = [
initial, hkt,
str(day),
str(sec),
str(dmo_hres),
str(dmo_ctrl),
str(dmo_mean),
str(dmo_var), a_0, a_1, a_2, a_3, b_0, b_1,
str(new_ngr_mean),
str(new_ngr_sigma),
str(obs_value)
]
writer.writerow(line)
elif distribution == 'TN':
new_ngr_mean = Minimizer_ngr.TN_mu(coeff_ngr, dmo_hres,
dmo_ctrl, dmo_mean,
dmo_var)
new_ngr_var = Minimizer_ngr.TN_var(coeff_ngr, dmo_hres,
dmo_ctrl, dmo_mean,
dmo_var)
new_ngr_sigma = np.sqrt(new_ngr_var)
a_0 = round(coeff_ngr[0] * coeff_ngr[0], 5)
a_1 = round(coeff_ngr[1] * coeff_ngr[1], 5)
a_2 = round(coeff_ngr[2] * coeff_ngr[2], 5)
a_3 = round(coeff_ngr[3] * coeff_ngr[3], 5)
b_0 = round(coeff_ngr[4] * coeff_ngr[4], 5)
b_1 = round(coeff_ngr[5] * coeff_ngr[5], 5)
with open(self.csv_path, 'ab') as csvfile:
writer = csv.writer(csvfile)
line = [
initial, hkt,
str(day),
str(sec),
str(dmo_hres),
str(dmo_ctrl),
str(dmo_mean),
str(dmo_var), a_0, a_1, a_2, a_3, b_0, b_1,
str(new_ngr_mean),
str(new_ngr_sigma),
str(obs_value)
]
writer.writerow(line)
"""
***
Still in progress for gust factor method
***
elif distribution == 'TN_gust':
mu = Minimizer_ngr.TN_mu(coeff_ngr, dmo_hres,dmo_ctrl, dmo_mean,dmo_var)
var = Minimizer_ngr.TN_var(coeff_ngr, dmo_hres,dmo_ctrl, dmo_mean,dmo_var)
sigma = np.sqrt(new_ngr_var)
new_ngr_mean = mu
new_ngr_sigma = sigma
a_0 = round( coeff_ngr[0] * coeff_ngr[0],5)
a_1 = round( coeff_ngr[1] * coeff_ngr[1],5)
a_2 = round( coeff_ngr[2] * coeff_ngr[2],5)
a_3 = round( coeff_ngr[3] * coeff_ngr[3],5)
b_0 = round( coeff_ngr[4] * coeff_ngr[4],5)
b_1 = round( coeff_ngr[5] * coeff_ngr[5],5)
with open(self.csv_path,'ab') as csvfile:
writer = csv.writer(csvfile)
line = [initial,hkt,str(day),str(sec),str(dmo_hres),str(dmo_ctrl),str(dmo_mean),str(dmo_var),a_0,a_1,a_2,a_3,b_0,b_1, str(new_ngr_mean),str(new_ngr_sigma),str(obs_value)]
writer.writerow(line)
"""
csvfile.close()
print(
"--------------------------------------------------------------------------------------------------------------------------------------------------------"
)
def train_data_ngr(self, DAY, SEC):
"""
obs_reader = obs_reader()
obs_reader.find_max([1,2,3,4,99999])
obs_reader.find_obs( 20170731, 4, "wgl")
"""
location = self.location
element = self.element
loc_id = self.config.loc_dict[location]
read = reader(loc_id)
MISSING = self.config.MISSING
if self.distribution != 'TN_gust':
obs_reader = obs_read(location, element)
"""
elif self.distribution == 'TN_gust':
obs_reader = obs_read('windmax',Config().obs_format_dict[location])
"""
obs_ARRAY = self.act_array()
LC_date = self.LC_date_array(obs_ARRAY, DAY)
predictive_distribution = self.distribution
obs = []
train_ens = []
obs = []
dmo_hres = []
dmo_ctrl = []
dmo_mean = []
dmo_var = []
for i in range(0, self.train):
obs_value = obs_reader.find_obs(obs_ARRAY[i], SEC)
if not obs_value == MISSING:
element_list = read.get_element(element, LC_date[i], DAY, SEC,
Config.TYPE)
hres = self.LC.get_hres(element_list)
ctrl = self.LC.get_ctrl(element_list)
mean = self.LC.get_mean(element_list)
var = self.LC.get_var(element_list)
obs.append(obs_value)
dmo_hres.append(hres)
dmo_ctrl.append(ctrl)
dmo_mean.append(round(mean, 3))
dmo_var.append(var)
delete_list = []
threshold = 0.0
if element == 'windspd' or element == 'windmax':
for i in range(len(obs)):
if obs[i] < threshold or obs[i] == MISSING:
delete_list.append(i)
train_ens = np.array(train_ens)
obs = np.array(obs)
dmo_hres = np.array(dmo_hres)
dmo_ctrl = np.array(dmo_ctrl)
dmo_mean = np.array(dmo_mean)
dmo_var = np.array(dmo_var)
# delete the missing list
train_ens = np.delete(train_ens, delete_list, 0)
obs = np.delete(obs, delete_list, 0)
dmo_hres = np.delete(dmo_hres, delete_list, 0)
dmo_ctrl = np.delete(dmo_ctrl, delete_list, 0)
dmo_mean = np.delete(dmo_mean, delete_list, 0)
dmo_var = np.delete(dmo_var, delete_list, 0)
##output_list.append(obs,dmo_hres,dmo_ctrl,dmo_mean,dmo_var)
print("")
print("Actual Date : " + ''.join(str(x) + " " for x in obs_ARRAY))
print("LC_date : " + ''.join(str(x)[0:10] + " " for x in LC_date))
print("Observation : " + ''.join(str(x) + " " for x in obs))
print("Hres : " + ''.join(str(x) + " " for x in dmo_hres))
print("ctrl : " + ''.join(str(x) + " " for x in dmo_ctrl))
print("dmo_mean : " + ''.join(str(x) + " " for x in dmo_mean))
print("")
out = Minimizer_ngr.Minimizer(obs, dmo_hres, dmo_ctrl, dmo_mean,
dmo_var, predictive_distribution)
coeff = out.get_coeff()
return coeff