def get_train_data(self, DAY, SEC): """ obs_reader = obs_reader() obs_reader.find_max([1,2,3,4,99999]) obs_reader.find_obs( 20170731, 4, "wgl") """ location = self.location loc_id = self.config.loc_id element = self.element train_day = self.train_day read = reader(loc_id) obs_reader = obs_read(location, element) obs_ARRAY = self.act_array() LC_date = self.LC_date_array(obs_ARRAY, DAY) obs = [] train_ens = [] for i in range(0, train_day): obs_value = obs_reader.find_obs(obs_ARRAY[i], SEC) if not obs_value == Config.MISSING: element_list = read.get_element(element, LC_date[i], DAY, SEC, Config.TYPE) train_ens.append(element_list) obs.append(obs_value) print("") print("Actual Date : " + ''.join(str(x) + " " for x in obs_ARRAY)) print("LC_date : " + ''.join(str(x) + " " for x in LC_date)) print("Observation : " + ''.join(str(x) + " " for x in obs)) #coeff = out.get_coeff() return train_ens, obs
def gen_bias(self, fc_day, section): read = reader(self.config.loc_dict[self.location]) OBS = [] EPS_TRAIN = [] MEAN = [] BIAS = [] for day in range(1, fc_day + 1): for sec in range(1, section + 1): EPS_TRAIN, OBS = self.get_train_data(day, sec) for k in range(len(OBS)): MEAN.extend([np.mean(EPS_TRAIN[k])]) BIAS.extend([MEAN[k] - OBS[k]]) colors = (0, 0, 0) area = np.pi * 1 x = MEAN y = BIAS plt.scatter(x, y, s=area, c=colors, alpha=0.5) plt.title('Scatter plot ') plt.xlabel('MEAN') plt.ylabel('Bias') plt.xlim(0, 35.0) plt.ylim(-15.0, 20.0) out_fname = 'C:\Users\wlsinaa\Desktop\scatter' + '\\' + self.location + '_' + str( self.init_dtg) + '_' + str(self.train_day) + '_' + str( day) + '_' + str(sec) + '.png' plt.savefig(out_fname, dpi=300) plt.gcf().clear() MEAN = [] BIAS = []
def gen_coeff(self, fc_day, section): read = reader(self.config.loc_dict[self.location]) EPS_data = read.get_element(self.element, self.D.roll_dtg(self.init_dtg, 0), fc_day, section, Config.TYPE) eps_mean = np.mean(EPS_data) delta = calMBM.delta(EPS_data) NEW_EPS = np.copy(EPS_data) ens_amount = len(EPS_data) hkt = self.D.roll_dtg(self.init_dtg, fc_day) obs_value = obs_read(self.location, self.element).find_obs(hkt, section) print("------------") EPS_TRAIN, OBS = self.get_train_data(fc_day, section) ens_minimizer = Minimizer_mbm(EPS_TRAIN, OBS, self.lamda, "reliability") ens_minimizer.process_data() ens_minimizer.Minimizer() coeff = ens_minimizer.get_coeff() # debias #bias_coeff = self.pre_coeff #bias = bias_coeff[0] + bias_coeff[1] * eps_mean #EPS_data = np.array(EPS_data) - bias for ens_no in range(ens_amount): NEW_EPS[ens_no] = coeff[0] + coeff[1] * eps_mean + ( coeff[2] * coeff[2] + coeff[3] * coeff[3] / delta) * (EPS_data[ens_no] - eps_mean) print( "--------------------------------------------------------------------------------------------------------------------------------------------------------" ) with open(self.csv_path, 'ab') as csvfile: writer = csv.writer(csvfile) line = [ self.init_dtg, str(hkt), str(fc_day), str(section), str(round(coeff[0], 4)), str(round(coeff[1], 4)), str(round(coeff[2] * coeff[2], 4)), str(round(coeff[3] * coeff[3], 4)), str(delta), str(obs_value) ] writer.writerow(line) csvfile.close() return NEW_EPS
def get_train_data(self, DAY, SEC): """ obs_reader = obs_reader() obs_reader.find_max([1,2,3,4,99999]) obs_reader.find_obs( 20170731, 4, "wgl") """ loc_id = self.config.loc_dict[self.location] read = reader(loc_id) obs_reader = obs_read(self.location, self.element) obs_ARRAY = self.act_array() LC_date = self.LC_date_array(obs_ARRAY, DAY) obs = [] train_ens = [] for i in range(0, self.train_day): obs_value = obs_reader.find_obs(obs_ARRAY[i], SEC) if not obs_value == Config.MISSING: element_list = read.get_element(self.element, LC_date[i], DAY, SEC, Config.TYPE) train_ens.append(element_list) obs.append(obs_value) element = self.element delete_list = [] threshold = 0.0 if element == 'windspd' or element == 'windmax': for i in range(len(obs)): if obs[i] < threshold or obs[i] == self.config.MISSING: delete_list.append(i) train_ens = np.array(train_ens) obs = np.array(obs) train_ens = np.delete(train_ens, delete_list, 0) obs = np.delete(obs, delete_list, 0) sample_no = len(train_ens) # debias #self.pre_coeff = self.bias_coeff(train_ens,obs, method = 'LinearRegression') #train_ens = self.preproccessing(train_ens,self.pre_coeff) self.obs_list = obs print("") print("Actual Date : " + ''.join(str(x) + " " for x in obs_ARRAY)) print("LC_date : " + ''.join(str(x) + " " for x in LC_date)) print("Observation : " + ''.join(str(x) + " " for x in obs)) #coeff = out.get_coeff() return train_ens, obs
sys.path.append("../MBM_VSV") import KernelDenity as kde from LC_Reader import LC_Reader as reader from config import Config import numpy as np import matplotlib.pyplot as plt from obs_reader import obs_reader import MBM_main as mbm import numpy as np """MBM = mbm.MBM(201708211200,'windspd','wgl') EPS = MBM.gen_ALL(2,4) new_EPS = np.array(EPS) new_EPS = new_EPS.T""" for i in range(0, 52): ensembles = reader(Config.loc_id).get_eps_timeseries( i, 201708241200, "windspd") x = np.arange(0, len(ensembles), 1) if i == 0: plt.plot(x, ensembles, linewidth=1.0, color="blue") elif i == 1: plt.plot(x, ensembles, linewidth=1.0, color="green") else: plt.plot(x, ensembles, linewidth=0.3) obs = obs_reader("windspd", Config().obs_format_dict["windspd"]).act_mix_lc_TS( 2017082512, 14, 4) plt.plot(x, obs, linewidth=2.0, color="red") #print(obs) #print(len(obs)) plt.xlim(0, 60)
def process_section(self): loc = self.location element = self.element distribution = self.distribution init_dtg = self.init_dtg self.LC.make_outdir(init_dtg, loc, element) ##check output directory self.LC.make_csv(init_dtg, loc, element, self.csv_filename, self.config.header[distribution]) ##create csv line = [] read = reader(self.config.loc_dict[self.location]) for day in range(1, self.fc_day + 1): for sec in range(1, self.section + 1): data = read.get_element(element, self.D.roll_dtg(self.init_dtg, 0), day, sec, Config.TYPE) initial = self.init_dtg hkt = self.D.roll_dtg(self.init_dtg, day) dmo_hres = self.LC.get_hres(data) dmo_ctrl = self.LC.get_ctrl(data) dmo_mean = self.LC.get_mean(data) dmo_var = self.LC.get_var(data) coeff_ngr = self.train_data_ngr(day, sec) if self.distribution != 'TN_gust': obs_value = obs_read(loc, element).find_obs(hkt, sec) elif self.distribution == 'TN_gust': obs_value = obs_read(loc, 'windmax').find_obs(hkt, sec) if distribution == 'gev': new_ngr_mean = Minimizer_ngr.ngr_mean( coeff_ngr, dmo_hres, dmo_ctrl, dmo_mean, dmo_var) new_ngr_sigma = Minimizer_ngr.ngr_var( coeff_ngr, dmo_hres, dmo_ctrl, dmo_mean, dmo_var) with open(self.csv_path, 'ab') as csvfile: writer = csv.writer(csvfile) line = [ initial, hkt, str(day), str(sec), str(dmo_hres), str(dmo_ctrl), str(dmo_mean), str(dmo_var), coeff_ngr[0], coeff_ngr[1], coeff_ngr[2], coeff_ngr[3], coeff_ngr[4], str(new_ngr_mean), str(new_ngr_sigma), str(obs_value) ] writer.writerow(line) elif distribution == 'LN': new_ngr_mean = Minimizer_ngr.LN_mu(coeff_ngr, dmo_hres, dmo_ctrl, dmo_mean, dmo_var) new_ngr_sigma = Minimizer_ngr.LN_sigma( coeff_ngr, dmo_hres, dmo_ctrl, dmo_mean, dmo_var) a_0 = round(coeff_ngr[0] * coeff_ngr[0], 5) a_1 = round(coeff_ngr[1] * coeff_ngr[1], 5) a_2 = round(coeff_ngr[2] * coeff_ngr[2], 5) a_3 = round(coeff_ngr[3] * coeff_ngr[3], 5) b_0 = round(coeff_ngr[4] * coeff_ngr[4], 5) b_1 = round(coeff_ngr[5] * coeff_ngr[5], 5) with open(self.csv_path, 'ab') as csvfile: writer = csv.writer(csvfile) line = [ initial, hkt, str(day), str(sec), str(dmo_hres), str(dmo_ctrl), str(dmo_mean), str(dmo_var), a_0, a_1, a_2, a_3, b_0, b_1, str(new_ngr_mean), str(new_ngr_sigma), str(obs_value) ] writer.writerow(line) elif distribution == 'TN': new_ngr_mean = Minimizer_ngr.TN_mu(coeff_ngr, dmo_hres, dmo_ctrl, dmo_mean, dmo_var) new_ngr_var = Minimizer_ngr.TN_var(coeff_ngr, dmo_hres, dmo_ctrl, dmo_mean, dmo_var) new_ngr_sigma = np.sqrt(new_ngr_var) a_0 = round(coeff_ngr[0] * coeff_ngr[0], 5) a_1 = round(coeff_ngr[1] * coeff_ngr[1], 5) a_2 = round(coeff_ngr[2] * coeff_ngr[2], 5) a_3 = round(coeff_ngr[3] * coeff_ngr[3], 5) b_0 = round(coeff_ngr[4] * coeff_ngr[4], 5) b_1 = round(coeff_ngr[5] * coeff_ngr[5], 5) with open(self.csv_path, 'ab') as csvfile: writer = csv.writer(csvfile) line = [ initial, hkt, str(day), str(sec), str(dmo_hres), str(dmo_ctrl), str(dmo_mean), str(dmo_var), a_0, a_1, a_2, a_3, b_0, b_1, str(new_ngr_mean), str(new_ngr_sigma), str(obs_value) ] writer.writerow(line) """ *** Still in progress for gust factor method *** elif distribution == 'TN_gust': mu = Minimizer_ngr.TN_mu(coeff_ngr, dmo_hres,dmo_ctrl, dmo_mean,dmo_var) var = Minimizer_ngr.TN_var(coeff_ngr, dmo_hres,dmo_ctrl, dmo_mean,dmo_var) sigma = np.sqrt(new_ngr_var) new_ngr_mean = mu new_ngr_sigma = sigma a_0 = round( coeff_ngr[0] * coeff_ngr[0],5) a_1 = round( coeff_ngr[1] * coeff_ngr[1],5) a_2 = round( coeff_ngr[2] * coeff_ngr[2],5) a_3 = round( coeff_ngr[3] * coeff_ngr[3],5) b_0 = round( coeff_ngr[4] * coeff_ngr[4],5) b_1 = round( coeff_ngr[5] * coeff_ngr[5],5) with open(self.csv_path,'ab') as csvfile: writer = csv.writer(csvfile) line = [initial,hkt,str(day),str(sec),str(dmo_hres),str(dmo_ctrl),str(dmo_mean),str(dmo_var),a_0,a_1,a_2,a_3,b_0,b_1, str(new_ngr_mean),str(new_ngr_sigma),str(obs_value)] writer.writerow(line) """ csvfile.close() print( "--------------------------------------------------------------------------------------------------------------------------------------------------------" )
def train_data_ngr(self, DAY, SEC): """ obs_reader = obs_reader() obs_reader.find_max([1,2,3,4,99999]) obs_reader.find_obs( 20170731, 4, "wgl") """ location = self.location element = self.element loc_id = self.config.loc_dict[location] read = reader(loc_id) MISSING = self.config.MISSING if self.distribution != 'TN_gust': obs_reader = obs_read(location, element) """ elif self.distribution == 'TN_gust': obs_reader = obs_read('windmax',Config().obs_format_dict[location]) """ obs_ARRAY = self.act_array() LC_date = self.LC_date_array(obs_ARRAY, DAY) predictive_distribution = self.distribution obs = [] train_ens = [] obs = [] dmo_hres = [] dmo_ctrl = [] dmo_mean = [] dmo_var = [] for i in range(0, self.train): obs_value = obs_reader.find_obs(obs_ARRAY[i], SEC) if not obs_value == MISSING: element_list = read.get_element(element, LC_date[i], DAY, SEC, Config.TYPE) hres = self.LC.get_hres(element_list) ctrl = self.LC.get_ctrl(element_list) mean = self.LC.get_mean(element_list) var = self.LC.get_var(element_list) obs.append(obs_value) dmo_hres.append(hres) dmo_ctrl.append(ctrl) dmo_mean.append(round(mean, 3)) dmo_var.append(var) delete_list = [] threshold = 0.0 if element == 'windspd' or element == 'windmax': for i in range(len(obs)): if obs[i] < threshold or obs[i] == MISSING: delete_list.append(i) train_ens = np.array(train_ens) obs = np.array(obs) dmo_hres = np.array(dmo_hres) dmo_ctrl = np.array(dmo_ctrl) dmo_mean = np.array(dmo_mean) dmo_var = np.array(dmo_var) # delete the missing list train_ens = np.delete(train_ens, delete_list, 0) obs = np.delete(obs, delete_list, 0) dmo_hres = np.delete(dmo_hres, delete_list, 0) dmo_ctrl = np.delete(dmo_ctrl, delete_list, 0) dmo_mean = np.delete(dmo_mean, delete_list, 0) dmo_var = np.delete(dmo_var, delete_list, 0) ##output_list.append(obs,dmo_hres,dmo_ctrl,dmo_mean,dmo_var) print("") print("Actual Date : " + ''.join(str(x) + " " for x in obs_ARRAY)) print("LC_date : " + ''.join(str(x)[0:10] + " " for x in LC_date)) print("Observation : " + ''.join(str(x) + " " for x in obs)) print("Hres : " + ''.join(str(x) + " " for x in dmo_hres)) print("ctrl : " + ''.join(str(x) + " " for x in dmo_ctrl)) print("dmo_mean : " + ''.join(str(x) + " " for x in dmo_mean)) print("") out = Minimizer_ngr.Minimizer(obs, dmo_hres, dmo_ctrl, dmo_mean, dmo_var, predictive_distribution) coeff = out.get_coeff() return coeff