def calculate_change_factor(idir,
ifile,
prob=[0.10, 0.90],
aggregation=[24, 48, 72, 96]):
r'''
'''
ifiles = sorted(os.listdir(idir))
dic = {}
dic_dsf = {}
for i in ifiles:
fname = os.path.join(idir, i)
d = wg_io.load(fname)
mu = numpy.array(d[0])
nu = numpy.array(d[1])
mu_dsf = mu / (1 - mu)
nu_dsf = nu / (1 - nu)
# print 'CF: ', numpy.mean(nu/mu)
# print '\n'
prob_up = prob[1]
prob_down = prob[0]
axis = 0
alpha = 0.0
beta = 1.0
up = scipy.stats.mstats.mquantiles(nu / mu,
prob=prob_up,
alphap=alpha,
betap=beta,
axis=axis)
down = scipy.stats.mstats.mquantiles(nu / mu,
prob=prob_down,
alphap=alpha,
betap=beta,
axis=axis)
dic[i] = [numpy.mean(nu / mu), down, up]
up_dsf = scipy.stats.mstats.mquantiles(nu_dsf / mu_dsf,
prob=prob_up,
alphap=alpha,
betap=beta,
axis=axis)
down_dsf = scipy.stats.mstats.mquantiles(nu_dsf / mu_dsf,
prob=prob_down,
alphap=alpha,
betap=beta,
axis=axis)
dic_dsf[i] = [numpy.mean(nu_dsf / mu_dsf), down_dsf, up_dsf]
# aggregation = [24]
# Preprocess
#size_in = size_cm[0]/2.54, size_cm[1]/2.54
data_mean = {}
data_var = {}
data_skew = {}
data_dsf = {}
# TODO: include the other type of groupings!
for g in range(12):
temp_mean = []
temp_var = []
temp_skew = []
temp_dsf = []
for k in dic:
for agg in aggregation:
if ('agg_' + str(agg) in k) and ('stat_Eh'
in k) and ('g_' + str(g) + '_'
in k):
temp_mean.append([k, dic[k][0], dic[k][1], dic[k][2]])
if ('agg_' + str(agg) in k) and ('stat_VARh'
in k) and ('g_' + str(g) + '_'
in k):
temp_var.append([k, dic[k][0], dic[k][1], dic[k][2]])
if ('agg_' + str(agg) in k) and ('stat_SKh'
in k) and ('g_' + str(g) + '_'
in k):
temp_skew.append([k, dic[k][0], dic[k][1], dic[k][2]])
if ('agg_' + str(agg) in k) and ('stat_FFh'
in k) and ('g_' + str(g) + '_'
in k):
temp_dsf.append([k, dic[k][0], dic[k][1], dic[k][2]])
#temp_dsf.append([k, dic_dsf[k][0], dic_dsf[k][1], dic_dsf[k][2]])
data_mean[str(g)] = sorted(temp_mean)
data_var[str(g)] = sorted(temp_var)
data_skew[str(g)] = sorted(temp_skew)
data_dsf[str(g)] = sorted(temp_dsf)
dic = wg_io.load(ifile)
ts = 1
lag = 1
accumulate = True
group_type = 'm'
dic_stat = wg_tool.series_statistics(dic,
ts,
aggregation,
lag=lag,
group_type=group_type,
accumulate=accumulate)
temp_mean = []
temp_cv = []
temp_var = []
temp_dsf = []
temp_skew = []
temp_correl = []
# [Ey, VARy, CVy, SKy, Rly, DSFy]
dicdic = dic_stat['period']
for g in range(12):
mean_index = 0
cf_mean = data_mean[str(g)]
cf_mean = [i[1] for i in cf_mean]
mean = dicdic[mean_index][g]
new_mean = numpy.array(mean) * numpy.array(cf_mean)
temp_mean.append(new_mean)
var_index = 1
cf_var = data_var[str(g)]
cf_var = [i[1] for i in cf_var]
var = dicdic[var_index][g]
new_var = numpy.array(var) * numpy.array(cf_var)
temp_var.append(new_var)
new_cv = (new_var**0.5) / new_mean
temp_cv.append(new_cv)
skew_index = 3
cf_skew = data_skew[str(g)]
cf_skew = [i[1] for i in cf_skew]
skew = dicdic[skew_index][g]
new_skew = numpy.array(skew) * numpy.array(cf_skew)
temp_skew.append(new_skew)
dsf_index = 5
cf_dsf = data_dsf[str(g)]
cf_dsf = [i[1] for i in cf_dsf]
dsf = dicdic[dsf_index][g]
new_dsf = numpy.array(dsf) * numpy.array(cf_dsf)
temp_dsf.append(new_dsf)
ret_dic = {
'mean': temp_mean,
'var': temp_var,
'skew': temp_skew,
'cv': temp_cv,
'dsf': temp_dsf,
'aggregation': aggregation
}
return ret_dic
def change_factors(idir, ifile, extend_aggregation, aggregation, stats=None):
r'''
'''
dic = calculate_change_factor(idir, ifile)
fmin = scipy.optimize.fmin_l_bfgs_b
fmin = scipy.optimize.fmin
# Changed factors original data
dsf = dic['dsf']
mean = dic['mean']
var = dic['var']
skew = dic['skew']
cv_ori = []
correl_ori = []
skew_ori = []
mean_ori = []
var_ori = []
dsf_ori = []
# Find the original
# -----------------
dic = wg_io.load(ifile)
ts = 1
lag = 1
accumulate = True
group_type = 'm'
dic_stat = wg_tool.series_statistics(dic,
ts,
extend_aggregation,
lag=lag,
group_type=group_type,
accumulate=accumulate)
# [Ey, VARy, CVy, SKy, Rly, DSFy]
dicdic = dic_stat['period']
for g in range(12):
mean_index = 0
mean_ori.append(dicdic[mean_index][g])
var_index = 1
var_ori.append(dicdic[var_index][g])
cv_index = 2
cv_ori.append(dicdic[cv_index][g])
skew_index = 3
skew_ori.append(dicdic[skew_index][g])
correl_index = 4
correl_ori.append(dicdic[correl_index][g])
dsf_index = 5
dsf_ori.append(dicdic[dsf_index][g])
# Find indexes to use extension values for bellow 24 hours and calculates
# values for over 24 hours
n_group = len(dsf)
pre_index = []
post_index = []
for i in range(len(extend_aggregation)):
if not (extend_aggregation[i] in aggregation):
pre_index.append(i)
for i in range(len(aggregation)):
if (aggregation[i] in extend_aggregation):
post_index.append(i)
h = numpy.array(aggregation)
h_ext = numpy.array(extend_aggregation)
# Extend mean: E(h) = A*h. Is linear and cuts in 0. Either one value of future
# is used, or a regression is made...
# ----------------------------------------------------------------------
mean_new = []
for g in range(n_group):
mean_h = mean[g]
Xo = [1] # A
Xf = fmin(mean_min, Xo, args=(h, mean_h), disp=0)
A = Xf[0]
mean_adj = A * h_ext
mean_res = [mean_adj[i]
for i in pre_index] + [mean_h[j] for j in post_index]
mean_new.append(numpy.array(mean_res))
# matplotlib.pyplot.plot(h_ext, mean_adj, 'o--')
# matplotlib.pyplot.plot(h, mean_h)
# matplotlib.pyplot.show()
# Extend dry spell fraction: dsf(h) = A*(e^h*B)
# ---------------------------------------------
# TODO: include the different groupings
dsf_new = []
for g in range(n_group):
dsf_h = dsf[g]
Xo = [-1] # A
if False:
# Only use 24 72
hh = [h[0], h[1]]
dsf_hh = [dsf_h[0], dsf_h[1]]
else:
hh = h
dsf_hh = dsf_h
#print fmin(dsf_min, Xo, args=(h, dsf_h), approx_grad=True, bounds=bounds, maxfun=3000)
Xf = fmin(dsf_min, Xo, args=(hh, dsf_hh), disp=0)
A = Xf[0]
dsf_adj = numpy.exp(A * h_ext)
dsf_res = [dsf_adj[i]
for i in pre_index] + [dsf_h[j] for j in post_index]
dsf_new.append(numpy.array(dsf_res))
# matplotlib.pyplot.plot(h_ext, dsf_adj, 'o--')
# matplotlib.pyplot.plot(h, dsf_h)
# matplotlib.pyplot.show()
# Skewness.... hmmmm difficult so far so only observed is downscaled
# ---------------------------------------------
# TODO: include the different groupings
skew_new = []
for g in range(n_group):
skew_h = skew[g]
skew_adj = skew_ori[g]
skew_res = [skew_adj[i]
for i in pre_index] + [skew_h[j] for j in post_index]
skew_new.append(numpy.array(skew_res))
# Autocorrelation.... very difficult indeed not taken into account
# ---------------------------------------------
correl_new = correl_ori
# Variance
# -------------------------------------------------
var_new = []
for g in range(n_group):
var_h = var[g]
Xo = [0.5, 1, 0.8]
Xf = fmin(var_min, Xo, args=(aggregation, var_h), disp=0)
eps = Xf[0] ## [h]
Sig2i = Xf[1] ## [mm^2]
alp = Xf[2] ## []
var_adj = variance_t_downscaling(extend_aggregation, eps, Sig2i, alp)
var_res = [var_adj[i]
for i in pre_index] + [var_h[j] for j in post_index]
var_new.append(numpy.array(var_res))
# Coefficient of variance
#-----------------------
cv_new = [(var_new[i])**0.5 / mean_new[i] for i in range(len(mean_new))]
dic = {
'aggregation': extend_aggregation,
'dsf': dsf_new,
'mean': mean_new,
'var': var_new,
'skew': skew_new,
'correl': correl_new,
'cv': cv_new
}
mean_cf = numpy.array(mean_new) / numpy.array(mean_ori)
var_cf = numpy.array(var_new) / numpy.array(var_ori)
cv_cf = numpy.array(cv_new) / numpy.array(cv_ori)
skew_cf = numpy.array(skew_new) / numpy.array(skew_ori)
correl_cf = numpy.array(correl_new) / numpy.array(correl_ori)
dsf_cf = numpy.array(dsf_new) / numpy.array(dsf_ori)
## Define the statistics to use in the model, this way it can be more inter
## actively decided which ones to use
# 0 1 2 3 4 5
stats_names = ['Eh', 'VARh', 'CVh', 'Rlh', 'SKh', 'FFh']
if stats == None:
stats = [2, 3, 4, 5]
len_stats = len(stats)
stats_names_used = [stats_names[i] for i in stats]
print stats_names_used
all_stats = []
ep_stats = []
# Now arrange so it is easy to use in the parameter generator
for g in range(n_group):
temp_stats = []
temp_ep = []
for ag in range(len(extend_aggregation)):
stats_list = [
mean_cf[g][ag], var_cf[g][ag], cv_cf[g][ag], correl_cf[g][ag],
skew_cf[g][ag], dsf_cf[g][ag]
]
stats_used = [stats_list[i] for i in stats]
temp_stats = temp_stats + stats_used
temp_ep = temp_ep + [mean_cf[g][ag]]
all_stats.append(numpy.array(temp_stats))
ep_stats.append(numpy.array(temp_ep))
#stats_list.append(Eh, VARh, CVh, Rlh, SKh, FFh]
print all_stats
print ep_stats
return {'cf_stats': all_stats, 'cf_mean': ep_stats}
def extend_fine(idir, ifile, extend_aggregation, aggregation, ofile):
r'''
'''
dic = calculate_change_factor(idir, ifile)
fmin = scipy.optimize.fmin_l_bfgs_b
fmin = scipy.optimize.fmin
# Changed factors original data
dsf = dic['dsf']
mean = dic['mean']
var = dic['var']
skew = dic['skew']
correl_ori = []
skew_ori = []
# Find the original
# -----------------
dic = wg_io.load(ifile)
ts = 1
lag = 1
accumulate = True
group_type = 'm'
dic_stat = wg_tool.series_statistics(dic,
ts,
extend_aggregation,
lag=lag,
group_type=group_type,
accumulate=accumulate)
# [Ey, VARy, CVy, SKy, Rly, DSFy]
dicdic = dic_stat['period']
for g in range(12):
skew_index = 3
skew_ori.append(dicdic[skew_index][g])
correl_index = 4
correl_ori.append(dicdic[correl_index][g])
# Find indexes to use extension values for bellow 24 hours and calculates
# values for over 24 hours
n_group = len(dsf)
pre_index = []
post_index = []
for i in range(len(extend_aggregation)):
if not (extend_aggregation[i] in aggregation):
pre_index.append(i)
for i in range(len(aggregation)):
if (aggregation[i] in extend_aggregation):
post_index.append(i)
h = numpy.array(aggregation)
h_ext = numpy.array(extend_aggregation)
# Extend mean: E(h) = A*h. Is linear and cuts in 0. Either one value of future
# is used, or a regression is made...
# ----------------------------------------------------------------------
mean_new = []
for g in range(n_group):
mean_h = mean[g]
Xo = [1] # A
Xf = fmin(mean_min, Xo, args=(h, mean_h), disp=0)
A = Xf[0]
mean_adj = A * h_ext
mean_res = [mean_adj[i]
for i in pre_index] + [mean_h[j] for j in post_index]
mean_new.append(numpy.array(mean_res))
# matplotlib.pyplot.plot(h_ext, mean_adj, 'o--')
# matplotlib.pyplot.plot(h, mean_h)
# matplotlib.pyplot.show()
# Extend dry spell fraction: dsf(h) = A*(e^h*B)
# ---------------------------------------------
# TODO: include the different groupings
dsf_new = []
for g in range(n_group):
dsf_h = dsf[g]
Xo = [-1] # A
if False:
# Only use 24 72
hh = [h[0], h[1]]
dsf_hh = [dsf_h[0], dsf_h[1]]
else:
hh = h
dsf_hh = dsf_h
#print fmin(dsf_min, Xo, args=(h, dsf_h), approx_grad=True, bounds=bounds, maxfun=3000)
Xf = fmin(dsf_min, Xo, args=(hh, dsf_hh), disp=0)
A = Xf[0]
dsf_adj = numpy.exp(A * h_ext)
dsf_res = [dsf_adj[i]
for i in pre_index] + [dsf_h[j] for j in post_index]
dsf_new.append(numpy.array(dsf_res))
# matplotlib.pyplot.plot(h_ext, dsf_adj, 'o--')
# matplotlib.pyplot.plot(h, dsf_h)
# matplotlib.pyplot.show()
# Skewness.... hmmmm difficult so far so only observed is downscaled
# ---------------------------------------------
# TODO: include the different groupings
skew_new = []
for g in range(n_group):
skew_h = skew[g]
skew_adj = skew_ori[g]
skew_res = [skew_adj[i]
for i in pre_index] + [skew_h[j] for j in post_index]
skew_new.append(numpy.array(skew_res))
# Autocorrelation.... very difficult indeed not taken into account
# ---------------------------------------------
correl_new = correl_ori
# Variance
# -------------------------------------------------
var_new = []
for g in range(n_group):
var_h = var[g]
Xo = [0.5, 1, 0.8]
Xf = fmin(var_min, Xo, args=(aggregation, var_h), disp=0)
eps = Xf[0] ## [h]
Sig2i = Xf[1] ## [mm^2]
alp = Xf[2] ## []
var_adj = variance_t_downscaling(extend_aggregation, eps, Sig2i, alp)
var_res = [var_adj[i]
for i in pre_index] + [var_h[j] for j in post_index]
var_new.append(numpy.array(var_res))
# Coefficient of variance
#-----------------------
cv_new = [(var_new[i])**0.5 / mean_new[i] for i in range(len(mean_new))]
dic = {
'aggregation': extend_aggregation,
'dsf': dsf_new,
'mean': mean_new,
'var': var_new,
'skew': skew_new,
'correl': correl_new,
'cv': cv_new
}
# Plot test
# for g in range(n_group):
# for key in dic:
# x = dic['aggregation']
# if not(key == 'aggregation'):
# y = dic[key][g]
#
# fig = matplotlib.pyplot.figure()
## fig.subplots_adjust(left=0.10, bottom=0.10, wspace=0.40, hspace=0.20)
# fig.suptitle(key)
# ax = fig.add_subplot(111)
# ax.set_axisbelow(True)
# ax.plot(x, y, 'b-', label='period statistics')
# matplotlib.pyplot.show()
wg_io.save(dic, ofile)
return dic
def calculate_change_factor(idir, ifile, prob=[0.10, 0.90], aggregation=[24,48,72,96]):
r'''
'''
ifiles = sorted(os.listdir(idir))
dic = {}
dic_dsf= {}
for i in ifiles:
fname = os.path.join(idir, i)
d = wg_io.load(fname)
mu = numpy.array(d[0])
nu = numpy.array(d[1])
mu_dsf = mu/(1 - mu)
nu_dsf = nu/(1 - nu)
# print 'CF: ', numpy.mean(nu/mu)
# print '\n'
prob_up = prob[1]
prob_down = prob[0]
axis = 0
alpha = 0.0
beta = 1.0
up = scipy.stats.mstats.mquantiles(nu/mu, prob=prob_up, alphap=alpha, betap=beta, axis=axis)
down = scipy.stats.mstats.mquantiles(nu/mu, prob=prob_down, alphap=alpha, betap=beta, axis=axis)
dic[i] = [numpy.mean(nu/mu), down, up]
up_dsf = scipy.stats.mstats.mquantiles(nu_dsf/mu_dsf, prob=prob_up, alphap=alpha, betap=beta, axis=axis)
down_dsf = scipy.stats.mstats.mquantiles(nu_dsf/mu_dsf, prob=prob_down, alphap=alpha, betap=beta, axis=axis)
dic_dsf[i] = [numpy.mean(nu_dsf/mu_dsf), down_dsf, up_dsf]
# aggregation = [24]
# Preprocess
#size_in = size_cm[0]/2.54, size_cm[1]/2.54
data_mean = {}
data_var = {}
data_skew = {}
data_dsf = {}
# TODO: include the other type of groupings!
for g in range(12):
temp_mean = []
temp_var = []
temp_skew = []
temp_dsf = []
for k in dic:
for agg in aggregation:
if ('agg_' + str(agg) in k) and ('stat_Eh' in k) and ('g_' + str(g) + '_' in k):
temp_mean.append([k, dic[k][0], dic[k][1], dic[k][2]])
if ('agg_' + str(agg) in k) and ('stat_VARh' in k) and ('g_' + str(g) + '_' in k):
temp_var.append([k, dic[k][0], dic[k][1], dic[k][2]])
if ('agg_' + str(agg) in k) and ('stat_SKh' in k) and ('g_' + str(g) + '_' in k):
temp_skew.append([k, dic[k][0], dic[k][1], dic[k][2]])
if ('agg_' + str(agg) in k) and ('stat_FFh' in k) and ('g_' + str(g) + '_' in k):
temp_dsf.append([k, dic[k][0], dic[k][1], dic[k][2]])
#temp_dsf.append([k, dic_dsf[k][0], dic_dsf[k][1], dic_dsf[k][2]])
data_mean[str(g)] = sorted(temp_mean)
data_var[str(g)] = sorted(temp_var)
data_skew[str(g)] = sorted(temp_skew)
data_dsf[str(g)] = sorted(temp_dsf)
dic = wg_io.load(ifile)
ts = 1
lag = 1
accumulate = True
group_type = 'm'
dic_stat = wg_tool.series_statistics(dic, ts, aggregation, lag=lag,
group_type=group_type, accumulate=accumulate)
temp_mean = []
temp_cv = []
temp_var = []
temp_dsf = []
temp_skew = []
temp_correl = []
# [Ey, VARy, CVy, SKy, Rly, DSFy]
dicdic = dic_stat['period']
for g in range(12):
mean_index = 0
cf_mean = data_mean[str(g)]
cf_mean = [i[1] for i in cf_mean]
mean = dicdic[mean_index][g]
new_mean = numpy.array(mean) * numpy.array(cf_mean)
temp_mean.append(new_mean)
var_index = 1
cf_var = data_var[str(g)]
cf_var = [i[1] for i in cf_var]
var = dicdic[var_index][g]
new_var = numpy.array(var) * numpy.array(cf_var)
temp_var.append(new_var)
new_cv = (new_var**0.5)/new_mean
temp_cv.append(new_cv)
skew_index = 3
cf_skew = data_skew[str(g)]
cf_skew = [i[1] for i in cf_skew]
skew = dicdic[skew_index][g]
new_skew = numpy.array(skew) * numpy.array(cf_skew)
temp_skew.append(new_skew)
dsf_index = 5
cf_dsf = data_dsf[str(g)]
cf_dsf = [i[1] for i in cf_dsf]
dsf = dicdic[dsf_index][g]
new_dsf = numpy.array(dsf) * numpy.array(cf_dsf)
temp_dsf.append(new_dsf)
ret_dic = {'mean': temp_mean,
'var': temp_var,
'skew': temp_skew,
'cv': temp_cv,
'dsf': temp_dsf,
'aggregation': aggregation}
return ret_dic
def change_factors(idir, ifile, extend_aggregation, aggregation, stats=None):
r'''
'''
dic = calculate_change_factor(idir, ifile)
fmin = scipy.optimize.fmin_l_bfgs_b
fmin = scipy.optimize.fmin
# Changed factors original data
dsf = dic['dsf']
mean = dic['mean']
var = dic['var']
skew = dic['skew']
cv_ori = []
correl_ori = []
skew_ori = []
mean_ori = []
var_ori = []
dsf_ori = []
# Find the original
# -----------------
dic = wg_io.load(ifile)
ts = 1
lag = 1
accumulate = True
group_type = 'm'
dic_stat = wg_tool.series_statistics(dic, ts, extend_aggregation, lag=lag,
group_type=group_type, accumulate=accumulate)
# [Ey, VARy, CVy, SKy, Rly, DSFy]
dicdic = dic_stat['period']
for g in range(12):
mean_index = 0
mean_ori.append(dicdic[mean_index][g])
var_index = 1
var_ori.append(dicdic[var_index][g])
cv_index = 2
cv_ori.append(dicdic[cv_index][g])
skew_index = 3
skew_ori.append(dicdic[skew_index][g])
correl_index = 4
correl_ori.append(dicdic[correl_index][g])
dsf_index = 5
dsf_ori.append(dicdic[dsf_index][g])
# Find indexes to use extension values for bellow 24 hours and calculates
# values for over 24 hours
n_group = len(dsf)
pre_index = []
post_index = []
for i in range(len(extend_aggregation)):
if not(extend_aggregation[i] in aggregation):
pre_index.append(i)
for i in range(len(aggregation)):
if (aggregation[i] in extend_aggregation):
post_index.append(i)
h = numpy.array(aggregation)
h_ext = numpy.array(extend_aggregation)
# Extend mean: E(h) = A*h. Is linear and cuts in 0. Either one value of future
# is used, or a regression is made...
# ----------------------------------------------------------------------
mean_new = []
for g in range(n_group):
mean_h = mean[g]
Xo = [1] # A
Xf = fmin(mean_min, Xo, args=(h, mean_h), disp=0)
A = Xf[0]
mean_adj = A * h_ext
mean_res = [mean_adj[i] for i in pre_index] + [mean_h[j] for j in post_index]
mean_new.append(numpy.array(mean_res))
# matplotlib.pyplot.plot(h_ext, mean_adj, 'o--')
# matplotlib.pyplot.plot(h, mean_h)
# matplotlib.pyplot.show()
# Extend dry spell fraction: dsf(h) = A*(e^h*B)
# ---------------------------------------------
# TODO: include the different groupings
dsf_new = []
for g in range(n_group):
dsf_h = dsf[g]
Xo = [-1] # A
if False:
# Only use 24 72
hh = [h[0], h[1]]
dsf_hh = [dsf_h[0], dsf_h[1]]
else:
hh = h
dsf_hh = dsf_h
#print fmin(dsf_min, Xo, args=(h, dsf_h), approx_grad=True, bounds=bounds, maxfun=3000)
Xf = fmin(dsf_min, Xo, args=(hh, dsf_hh), disp=0)
A = Xf[0]
dsf_adj = numpy.exp(A * h_ext)
dsf_res = [dsf_adj[i] for i in pre_index] + [dsf_h[j] for j in post_index]
dsf_new.append(numpy.array(dsf_res))
# matplotlib.pyplot.plot(h_ext, dsf_adj, 'o--')
# matplotlib.pyplot.plot(h, dsf_h)
# matplotlib.pyplot.show()
# Skewness.... hmmmm difficult so far so only observed is downscaled
# ---------------------------------------------
# TODO: include the different groupings
skew_new = []
for g in range(n_group):
skew_h = skew[g]
skew_adj = skew_ori[g]
skew_res = [skew_adj[i] for i in pre_index] + [skew_h[j] for j in post_index]
skew_new.append(numpy.array(skew_res))
# Autocorrelation.... very difficult indeed not taken into account
# ---------------------------------------------
correl_new = correl_ori
# Variance
# -------------------------------------------------
var_new = []
for g in range(n_group):
var_h = var[g]
Xo = [0.5, 1, 0.8]
Xf = fmin(var_min, Xo, args=(aggregation, var_h), disp=0)
eps =Xf[0] ## [h]
Sig2i = Xf[1] ## [mm^2]
alp = Xf[2] ## []
var_adj = variance_t_downscaling(extend_aggregation, eps, Sig2i, alp)
var_res = [var_adj[i] for i in pre_index] + [var_h[j] for j in post_index]
var_new.append(numpy.array(var_res))
# Coefficient of variance
#-----------------------
cv_new = [(var_new[i])**0.5 / mean_new[i] for i in range(len(mean_new))]
dic = {'aggregation': extend_aggregation,
'dsf': dsf_new,
'mean': mean_new,
'var': var_new,
'skew': skew_new,
'correl': correl_new,
'cv': cv_new}
mean_cf = numpy.array(mean_new) / numpy.array(mean_ori)
var_cf = numpy.array(var_new) / numpy.array(var_ori)
cv_cf = numpy.array(cv_new) / numpy.array(cv_ori)
skew_cf = numpy.array(skew_new) / numpy.array(skew_ori)
correl_cf = numpy.array(correl_new) / numpy.array(correl_ori)
dsf_cf = numpy.array(dsf_new) / numpy.array(dsf_ori)
## Define the statistics to use in the model, this way it can be more inter
## actively decided which ones to use
# 0 1 2 3 4 5
stats_names = ['Eh', 'VARh', 'CVh', 'Rlh', 'SKh', 'FFh']
if stats == None:
stats = [2, 3, 4, 5]
len_stats = len(stats)
stats_names_used = [stats_names[i] for i in stats]
print stats_names_used
all_stats = []
ep_stats = []
# Now arrange so it is easy to use in the parameter generator
for g in range(n_group):
temp_stats = []
temp_ep = []
for ag in range(len(extend_aggregation)):
stats_list = [mean_cf[g][ag], var_cf[g][ag], cv_cf[g][ag],
correl_cf[g][ag], skew_cf[g][ag], dsf_cf[g][ag]]
stats_used = [stats_list[i] for i in stats]
temp_stats = temp_stats + stats_used
temp_ep = temp_ep + [mean_cf[g][ag]]
all_stats.append(numpy.array(temp_stats))
ep_stats.append(numpy.array(temp_ep))
#stats_list.append(Eh, VARh, CVh, Rlh, SKh, FFh]
print all_stats
print ep_stats
return {'cf_stats': all_stats,
'cf_mean': ep_stats}
def extend_fine(idir, ifile, extend_aggregation, aggregation, ofile):
r'''
'''
dic = calculate_change_factor(idir, ifile)
fmin = scipy.optimize.fmin_l_bfgs_b
fmin = scipy.optimize.fmin
# Changed factors original data
dsf = dic['dsf']
mean = dic['mean']
var = dic['var']
skew = dic['skew']
correl_ori = []
skew_ori = []
# Find the original
# -----------------
dic = wg_io.load(ifile)
ts = 1
lag = 1
accumulate = True
group_type = 'm'
dic_stat = wg_tool.series_statistics(dic, ts, extend_aggregation, lag=lag,
group_type=group_type, accumulate=accumulate)
# [Ey, VARy, CVy, SKy, Rly, DSFy]
dicdic = dic_stat['period']
for g in range(12):
skew_index = 3
skew_ori.append(dicdic[skew_index][g])
correl_index = 4
correl_ori.append(dicdic[correl_index][g])
# Find indexes to use extension values for bellow 24 hours and calculates
# values for over 24 hours
n_group = len(dsf)
pre_index = []
post_index = []
for i in range(len(extend_aggregation)):
if not(extend_aggregation[i] in aggregation):
pre_index.append(i)
for i in range(len(aggregation)):
if (aggregation[i] in extend_aggregation):
post_index.append(i)
h = numpy.array(aggregation)
h_ext = numpy.array(extend_aggregation)
# Extend mean: E(h) = A*h. Is linear and cuts in 0. Either one value of future
# is used, or a regression is made...
# ----------------------------------------------------------------------
mean_new = []
for g in range(n_group):
mean_h = mean[g]
Xo = [1] # A
Xf = fmin(mean_min, Xo, args=(h, mean_h), disp=0)
A = Xf[0]
mean_adj = A * h_ext
mean_res = [mean_adj[i] for i in pre_index] + [mean_h[j] for j in post_index]
mean_new.append(numpy.array(mean_res))
# matplotlib.pyplot.plot(h_ext, mean_adj, 'o--')
# matplotlib.pyplot.plot(h, mean_h)
# matplotlib.pyplot.show()
# Extend dry spell fraction: dsf(h) = A*(e^h*B)
# ---------------------------------------------
# TODO: include the different groupings
dsf_new = []
for g in range(n_group):
dsf_h = dsf[g]
Xo = [-1] # A
if False:
# Only use 24 72
hh = [h[0], h[1]]
dsf_hh = [dsf_h[0], dsf_h[1]]
else:
hh = h
dsf_hh = dsf_h
#print fmin(dsf_min, Xo, args=(h, dsf_h), approx_grad=True, bounds=bounds, maxfun=3000)
Xf = fmin(dsf_min, Xo, args=(hh, dsf_hh), disp=0)
A = Xf[0]
dsf_adj = numpy.exp(A * h_ext)
dsf_res = [dsf_adj[i] for i in pre_index] + [dsf_h[j] for j in post_index]
dsf_new.append(numpy.array(dsf_res))
# matplotlib.pyplot.plot(h_ext, dsf_adj, 'o--')
# matplotlib.pyplot.plot(h, dsf_h)
# matplotlib.pyplot.show()
# Skewness.... hmmmm difficult so far so only observed is downscaled
# ---------------------------------------------
# TODO: include the different groupings
skew_new = []
for g in range(n_group):
skew_h = skew[g]
skew_adj = skew_ori[g]
skew_res = [skew_adj[i] for i in pre_index] + [skew_h[j] for j in post_index]
skew_new.append(numpy.array(skew_res))
# Autocorrelation.... very difficult indeed not taken into account
# ---------------------------------------------
correl_new = correl_ori
# Variance
# -------------------------------------------------
var_new = []
for g in range(n_group):
var_h = var[g]
Xo = [0.5, 1, 0.8]
Xf = fmin(var_min, Xo, args=(aggregation, var_h), disp=0)
eps =Xf[0] ## [h]
Sig2i = Xf[1] ## [mm^2]
alp = Xf[2] ## []
var_adj = variance_t_downscaling(extend_aggregation, eps, Sig2i, alp)
var_res = [var_adj[i] for i in pre_index] + [var_h[j] for j in post_index]
var_new.append(numpy.array(var_res))
# Coefficient of variance
#-----------------------
cv_new = [(var_new[i])**0.5 / mean_new[i] for i in range(len(mean_new))]
dic = {'aggregation': extend_aggregation,
'dsf': dsf_new,
'mean': mean_new,
'var': var_new,
'skew': skew_new,
'correl': correl_new,
'cv': cv_new}
# Plot test
# for g in range(n_group):
# for key in dic:
# x = dic['aggregation']
# if not(key == 'aggregation'):
# y = dic[key][g]
#
# fig = matplotlib.pyplot.figure()
## fig.subplots_adjust(left=0.10, bottom=0.10, wspace=0.40, hspace=0.20)
# fig.suptitle(key)
# ax = fig.add_subplot(111)
# ax.set_axisbelow(True)
# ax.plot(x, y, 'b-', label='period statistics')
# matplotlib.pyplot.show()
wg_io.save(dic, ofile)
return dic
