def create_dqs_data():
"""Generates the set of input data files for this study. This method
supercedes the other create_* methods below, which are retained for
reference."""
# calculate seasonal and climatological means
obs_rainfall_vals = get_rain_values(model_loc + "/inputs_gujarat/weather/original/rainfall.dat")
obs_rainfall_climate_mean = obs_rainfall_vals.mean()
obs_rainfall_climate_std = obs_rainfall_vals.std()
obs_yield_climate_mean = obs_yield.mean()
obs_yield_climate_std = obs_yield.std()
obs_temp_season_mean = [] # line_num : seasonal_mean
obs_temp_season_std = [] # line_num : seasonal_std
obs_temp_avg_svals = array([]) # array of all temp values from the season we are interested in
for i in range(obs_min_temp_data.shape[0]): # tmin and tmax files have same structure
tmin_svals = obs_min_temp_data[i, 5:9] # June -> September (inclusive)
tmax_svals = obs_max_temp_data[i, 5:9]
tavg_svals = (tmin_svals + tmax_svals) / 2.0
obs_temp_season_mean.append(tavg_svals.mean())
obs_temp_season_std.append(tavg_svals.std())
obs_temp_avg_svals = append(obs_temp_avg_svals, tavg_svals)
obs_temp_climate_mean = obs_temp_avg_svals.mean()
obs_temp_climate_std = obs_temp_avg_svals.std()
# create directories
mkdir(model_loc + "/dqs_data")
for var_name, bias_type in dqs_data_config.iteritems():
mkdir(model_loc + "/dqs_data/" + var_name)
for b in bias_type:
mkdir(model_loc + "/dqs_data/" + var_name + "/" + b)
for s in dqs_seeds:
r = Random(s)
for p in percent_std_dev:
# -- daily uncorrelated precip --
out_file = open(model_loc + "/dqs_data/prec/day/p-" + str(p) + "_s-" + str(s) + ".dat", "w")
for row in obs_rain_data:
if len(row) in (51, 81): # there are 10 or 16 {3.1}f values
new_row = ""
index = 0
while index < (len(row) - 1): # there is a trailing '\n'
orig_value = float(row[index : (index + 5)])
out_std_dev = p / 100.0 * obs_rainfall_climate_std # was: * orig_value
new_value = r.normalvariate(orig_value, out_std_dev)
if new_value < 0:
new_value = 0.0
elif new_value >= 1000: # column format is {3.1}f
new_value = 999.9
new_row += "{0:5.1f}".format(new_value)
index += 5
new_row += "\n"
out_file.write(new_row)
else:
out_file.write(row)
out_file.close()
# -- seasonally biased precip --
out_file = open(model_loc + "/dqs_data/prec/season/p-" + str(p) + "_s-" + str(s) + ".dat", "w")
for rain_year in range(1966, 1990):
obs_rainfall_year_mean = get_rain_year_values(rain_year).mean()
obs_rainfall_year_std = get_rain_year_values(rain_year).std()
out_std_dev = p / 100.0 * obs_rainfall_year_std # was: * obs_rainfall_year_mean
diff_val = r.normalvariate(obs_rainfall_year_mean, out_std_dev) - obs_rainfall_year_mean
out_file.write(" BLOCK-NO 25 DAILY RF OF " + str(rain_year) + "\n")
for row in obs_rain_data[obs_rain_layout[rain_year][0] : (obs_rain_layout[rain_year][1] + 1)]:
new_row = ""
index = 0
while index < (len(row) - 1): # there is a trailing '\n'
orig_value = float(row[index : (index + 5)])
new_value = orig_value + diff_val
if new_value < 0:
new_value = 0.0
elif new_value >= 1000.0: # column format is {3.1}f
new_value = 999.9
new_row += "{0:5.1f}".format(new_value)
index += 5
new_row += "\n"
out_file.write(new_row)
out_file.write("\n\n")
out_file.close()
# -- climatologically biased precip --
out_file = open(model_loc + "/dqs_data/prec/climate/p-" + str(p) + "_s-" + str(s) + ".dat", "w")
out_std_dev = p / 100.0 * obs_rainfall_climate_std # was: obs_rainfall_climate_mean
diff_val = r.normalvariate(obs_rainfall_climate_mean, out_std_dev) - obs_rainfall_climate_mean
for row in obs_rain_data:
if len(row) in (51, 81): # there are 10 or 16 {3.1}f values
new_row = ""
index = 0
while index < (len(row) - 1): # there is a trailing '\n'
orig_value = float(row[index : (index + 5)])
new_value = orig_value + diff_val
if new_value < 0:
new_value = 0.0
elif new_value >= 1000.0: # column format is {3.1}f
new_value = 999.9
new_row += "{0:5.1f}".format(new_value)
index += 5
new_row += "\n"
out_file.write(new_row)
else:
out_file.write(row)
out_file.close()
# -- seasonally biased yield --
out_file = open(model_loc + "/dqs_data/yield/season/p-" + str(p) + "_s-" + str(s) + ".dat", "w")
for row in obs_yield_data:
out_mean = row[3]
out_std_dev = p / 100.0 * obs_yield_climate_std # was: out_mean
out_yield = r.normalvariate(out_mean, out_std_dev)
if out_yield < 0:
out_yield = 0.0
out_file.write(
str(int(row[0])).ljust(7)
+ str(int(row[1])).ljust(4)
+ str(int(row[2]))
+ "{0:.5f}".format(out_yield).rjust(13)
+ "\n"
)
out_file.close()
# -- climatologically biased yield --
out_file = open(model_loc + "/dqs_data/yield/climate/p-" + str(p) + "_s-" + str(s) + ".dat", "w")
out_std_dev = p / 100.0 * obs_yield_climate_std # was: obs_yield_climate_mean
diff_val = r.normalvariate(obs_yield_climate_mean, out_std_dev) - obs_yield_climate_mean
for row in obs_yield_data:
out_yield = row[3] + diff_val
if out_yield < 0:
out_yield = 0.0
out_file.write(
str(int(row[0])).ljust(7)
+ str(int(row[1])).ljust(4)
+ str(int(row[2]))
+ "{0:.5f}".format(out_yield).rjust(13)
+ "\n"
)
out_file.close()
# -- seasonally biased mean temp --
tmin_out_file = open(model_loc + "/dqs_data/mtemp/season/tmin_p-" + str(p) + "_s-" + str(s) + ".dat", "w")
tmax_out_file = open(model_loc + "/dqs_data/mtemp/season/tmax_p-" + str(p) + "_s-" + str(s) + ".dat", "w")
for i in range(obs_min_temp_data.shape[0]): # tmin and tmax files have same structure
out_std_dev = p / 100.0 * obs_temp_season_std[i] # was: obs_temp_season_mean[i]
diff_val = int(r.normalvariate(obs_temp_season_mean[i], out_std_dev) - obs_temp_season_mean[i])
new_tmin_row = obs_min_temp_first_column[i]
new_tmax_row = obs_max_temp_first_column[i]
for j in range(obs_min_temp_data.shape[1]):
new_tmin_val = obs_min_temp_data[i, j] + diff_val
if new_tmin_val < 0:
new_tmin_val = 0
elif new_tmin_val >= 1000: # data format is (2XI3) in input.f90
new_tmin_val = 999
new_tmax_val = obs_max_temp_data[i, j] + diff_val
if new_tmax_val < 0:
new_tmax_val = 0
elif new_tmax_val >= 1000:
new_tmax_val = 999
new_tmin_row += "{0:5d}".format(new_tmin_val)
new_tmax_row += "{0:5d}".format(new_tmax_val)
new_tmin_row += "\n"
new_tmax_row += "\n"
tmin_out_file.write(new_tmin_row)
tmax_out_file.write(new_tmax_row)
tmin_out_file.write("\n")
tmax_out_file.write("\n")
tmin_out_file.close()
tmax_out_file.close()
# -- climatologically biased mean temp --
tmin_out_file = open(model_loc + "/dqs_data/mtemp/climate/tmin_p-" + str(p) + "_s-" + str(s) + ".dat", "w")
tmax_out_file = open(model_loc + "/dqs_data/mtemp/climate/tmax_p-" + str(p) + "_s-" + str(s) + ".dat", "w")
out_std_dev = p / 100.0 * obs_temp_climate_std # was: obs_temp_climate_mean
diff_val = int(r.normalvariate(obs_temp_climate_mean, out_std_dev) - obs_temp_climate_mean)
for i in range(obs_min_temp_data.shape[0]): # tmin and tmax files have same structure
new_tmin_row = obs_min_temp_first_column[i]
new_tmax_row = obs_max_temp_first_column[i]
for j in range(obs_min_temp_data.shape[1]):
new_tmin_val = obs_min_temp_data[i, j] + diff_val
if new_tmin_val < 0:
new_tmin_val = 0
elif new_tmin_val >= 1000: # data format is (2XI3) in input.f90
new_tmin_val = 999
new_tmax_val = obs_max_temp_data[i, j] + diff_val
if new_tmax_val < 0:
new_tmax_val = 0
elif new_tmax_val >= 1000:
new_tmax_val = 999
new_tmin_row += "{0:5d}".format(new_tmin_val)
new_tmax_row += "{0:5d}".format(new_tmax_val)
new_tmin_row += "\n"
new_tmax_row += "\n"
tmin_out_file.write(new_tmin_row)
tmax_out_file.write(new_tmax_row)
tmin_out_file.write("\n")
tmax_out_file.write("\n")
tmin_out_file.close()
tmax_out_file.close()
def create_dqs_data():
'''Generates the set of input data files for this study. This method
supercedes the other create_* methods below, which are retained for
reference.'''
# calculate seasonal and climatological means
obs_rainfall_vals = get_rain_values(model_loc + '/inputs_gujarat/weather/original/rainfall.dat')
obs_rainfall_climate_mean = obs_rainfall_vals.mean()
obs_rainfall_climate_std = obs_rainfall_vals.std()
obs_yield_climate_mean = obs_yield.mean()
obs_yield_climate_std = obs_yield.std()
obs_temp_season_mean = [] # line_num : seasonal_mean
obs_temp_season_std = [] # line_num : seasonal_std
obs_temp_avg_svals = array([]) # array of all temp values from the season we are interested in
for i in range(obs_min_temp_data.shape[0]): # tmin and tmax files have same structure
tmin_svals = obs_min_temp_data[i,5:9] # June -> September (inclusive)
tmax_svals = obs_max_temp_data[i,5:9]
tavg_svals = (tmin_svals + tmax_svals) / 2.0
obs_temp_season_mean.append(tavg_svals.mean())
obs_temp_season_std.append(tavg_svals.std())
obs_temp_avg_svals = append(obs_temp_avg_svals, tavg_svals)
obs_temp_climate_mean = obs_temp_avg_svals.mean()
obs_temp_climate_std = obs_temp_avg_svals.std()
# create directories
mkdir(model_loc + '/dqs_data')
for var_name, bias_type in dqs_data_config.iteritems():
mkdir(model_loc + '/dqs_data/' + var_name)
for b in bias_type:
mkdir(model_loc + '/dqs_data/' + var_name + '/' + b)
for s in dqs_seeds:
r = Random(s)
for p in percent_std_dev:
# -- daily uncorrelated precip --
out_file = open(model_loc + '/dqs_data/prec/day/p-' + str(p) + '_s-' + str(s) + '.dat', 'w')
for row in obs_rain_data:
if len(row) in (51, 81): # there are 10 or 16 {3.1}f values
new_row = ''
index = 0
while index < (len(row)-1): # there is a trailing '\n'
orig_value = float(row[index:(index+5)])
out_std_dev = p / 100.0 * obs_rainfall_climate_std # was: * orig_value
new_value = r.normalvariate(orig_value, out_std_dev)
if new_value < 0:
new_value = 0.0
elif new_value >= 1000: # column format is {3.1}f
new_value = 999.9
new_row += '{0:5.1f}'.format(new_value)
index += 5
new_row += '\n'
out_file.write(new_row)
else:
out_file.write(row)
out_file.close()
# -- seasonally biased precip --
out_file = open(model_loc + '/dqs_data/prec/season/p-' + str(p) + '_s-' + str(s) + '.dat', 'w')
for rain_year in range(1966,1990):
obs_rainfall_year_mean = get_rain_year_values(rain_year).mean()
obs_rainfall_year_std = get_rain_year_values(rain_year).std()
out_std_dev = p / 100.0 * obs_rainfall_year_std # was: * obs_rainfall_year_mean
diff_val = r.normalvariate(obs_rainfall_year_mean, out_std_dev) - obs_rainfall_year_mean
out_file.write(' BLOCK-NO 25 DAILY RF OF ' + str(rain_year) + '\n')
for row in obs_rain_data[obs_rain_layout[rain_year][0]:(obs_rain_layout[rain_year][1]+1)]:
new_row = ''
index = 0
while index < (len(row)-1): # there is a trailing '\n'
orig_value = float(row[index:(index+5)])
new_value = orig_value + diff_val
if new_value < 0:
new_value = 0.0
elif new_value >= 1000.0: # column format is {3.1}f
new_value = 999.9
new_row += '{0:5.1f}'.format(new_value)
index += 5
new_row += '\n'
out_file.write(new_row)
out_file.write('\n\n')
out_file.close()
# -- climatologically biased precip --
out_file = open(model_loc + '/dqs_data/prec/climate/p-' + str(p) + '_s-' + str(s) + '.dat', 'w')
out_std_dev = p / 100.0 * obs_rainfall_climate_std # was: obs_rainfall_climate_mean
diff_val = r.normalvariate(obs_rainfall_climate_mean, out_std_dev) - obs_rainfall_climate_mean
for row in obs_rain_data:
if len(row) in (51, 81): # there are 10 or 16 {3.1}f values
new_row = ''
index = 0
while index < (len(row)-1): # there is a trailing '\n'
orig_value = float(row[index:(index+5)])
new_value = orig_value + diff_val
if new_value < 0:
new_value = 0.0
elif new_value >= 1000.0: # column format is {3.1}f
new_value = 999.9
new_row += '{0:5.1f}'.format(new_value)
index += 5
new_row += '\n'
out_file.write(new_row)
else:
out_file.write(row)
out_file.close()
# -- seasonally biased yield --
out_file = open(model_loc + '/dqs_data/yield/season/p-' + str(p) + '_s-' + str(s) + '.dat', 'w')
for row in obs_yield_data:
out_mean = row[3]
out_std_dev = p / 100.0 * obs_yield_climate_std # was: out_mean
out_yield = r.normalvariate(out_mean, out_std_dev)
if out_yield < 0:
out_yield = 0.0
out_file.write(str(int(row[0])).ljust(7) + str(int(row[1])).ljust(4) + str(int(row[2])) + '{0:.5f}'.format(out_yield).rjust(13) + '\n')
out_file.close()
# -- climatologically biased yield --
out_file = open(model_loc + '/dqs_data/yield/climate/p-' + str(p) + '_s-' + str(s) + '.dat', 'w')
out_std_dev = p / 100.0 * obs_yield_climate_std # was: obs_yield_climate_mean
diff_val = r.normalvariate(obs_yield_climate_mean, out_std_dev) - obs_yield_climate_mean
for row in obs_yield_data:
out_yield = row[3] + diff_val
if out_yield < 0:
out_yield = 0.0
out_file.write(str(int(row[0])).ljust(7) + str(int(row[1])).ljust(4) + str(int(row[2])) + '{0:.5f}'.format(out_yield).rjust(13) + '\n')
out_file.close()
# -- seasonally biased mean temp --
tmin_out_file = open(model_loc + '/dqs_data/mtemp/season/tmin_p-' + str(p) + '_s-' + str(s) + '.dat', 'w')
tmax_out_file = open(model_loc + '/dqs_data/mtemp/season/tmax_p-' + str(p) + '_s-' + str(s) + '.dat', 'w')
for i in range(obs_min_temp_data.shape[0]): # tmin and tmax files have same structure
out_std_dev = p / 100.0 * obs_temp_season_std[i] # was: obs_temp_season_mean[i]
diff_val = int(r.normalvariate(obs_temp_season_mean[i], out_std_dev) - obs_temp_season_mean[i])
new_tmin_row = obs_min_temp_first_column[i]
new_tmax_row = obs_max_temp_first_column[i]
for j in range(obs_min_temp_data.shape[1]):
new_tmin_val = obs_min_temp_data[i,j] + diff_val
if new_tmin_val < 0:
new_tmin_val = 0
elif new_tmin_val >= 1000: # data format is (2XI3) in input.f90
new_tmin_val = 999
new_tmax_val = obs_max_temp_data[i,j] + diff_val
if new_tmax_val < 0:
new_tmax_val = 0
elif new_tmax_val >= 1000:
new_tmax_val = 999
new_tmin_row += '{0:5d}'.format(new_tmin_val)
new_tmax_row += '{0:5d}'.format(new_tmax_val)
new_tmin_row += '\n'
new_tmax_row += '\n'
tmin_out_file.write(new_tmin_row)
tmax_out_file.write(new_tmax_row)
tmin_out_file.write('\n')
tmax_out_file.write('\n')
tmin_out_file.close()
tmax_out_file.close()
# -- climatologically biased mean temp --
tmin_out_file = open(model_loc + '/dqs_data/mtemp/climate/tmin_p-' + str(p) + '_s-' + str(s) + '.dat', 'w')
tmax_out_file = open(model_loc + '/dqs_data/mtemp/climate/tmax_p-' + str(p) + '_s-' + str(s) + '.dat', 'w')
out_std_dev = p / 100.0 * obs_temp_climate_std # was: obs_temp_climate_mean
diff_val = int(r.normalvariate(obs_temp_climate_mean, out_std_dev) - obs_temp_climate_mean)
for i in range(obs_min_temp_data.shape[0]): # tmin and tmax files have same structure
new_tmin_row = obs_min_temp_first_column[i]
new_tmax_row = obs_max_temp_first_column[i]
for j in range(obs_min_temp_data.shape[1]):
new_tmin_val = obs_min_temp_data[i,j] + diff_val
if new_tmin_val < 0:
new_tmin_val = 0
elif new_tmin_val >= 1000: # data format is (2XI3) in input.f90
new_tmin_val = 999
new_tmax_val = obs_max_temp_data[i,j] + diff_val
if new_tmax_val < 0:
new_tmax_val = 0
elif new_tmax_val >= 1000:
new_tmax_val = 999
new_tmin_row += '{0:5d}'.format(new_tmin_val)
new_tmax_row += '{0:5d}'.format(new_tmax_val)
new_tmin_row += '\n'
new_tmax_row += '\n'
tmin_out_file.write(new_tmin_row)
tmax_out_file.write(new_tmax_row)
tmin_out_file.write('\n')
tmax_out_file.write('\n')
tmin_out_file.close()
tmax_out_file.close()
# create list of unique random number seeds
seed(21)
shuffle_seeds = {}
for v in shuffled_data_config.keys():
shuffle_seeds[v] = []
while len(shuffle_seeds[v]) < num_seeds:
s = randint(1, 9999)
if s not in shuffle_seeds[v]:
shuffle_seeds[v].append(s)
# load original observations
prec_obs = {} # year : daily values
for year in range(1966, 1990):
prec_obs[year] = get_rain_year_values(year)
def save_prec_data(prec_data, prec_filename):
"""Saves the data in the dictionary prec_data to the file with the
given filename.
prec_data should be of the form {year : array_of_daily_values}."""
d_file = open(prec_filename, "w")
for year in sorted(prec_data.keys()):
assert prec_data[year].shape == (122,)
d_file.write(" BLOCK-NO 25 DAILY RF OF " + str(year) + "\n")
i = 0
for line in range(7):
for val in range(16):
# create list of unique random number seeds
seed(21)
shuffle_seeds = {}
for v in shuffled_data_config.keys():
shuffle_seeds[v] = []
while len(shuffle_seeds[v]) < num_seeds:
s = randint(1,9999)
if s not in shuffle_seeds[v]:
shuffle_seeds[v].append(s)
# load original observations
prec_obs = {} # year : daily values
for year in range(1966,1990):
prec_obs[year] = get_rain_year_values(year)
def save_prec_data(prec_data, prec_filename):
'''Saves the data in the dictionary prec_data to the file with the
given filename.
prec_data should be of the form {year : array_of_daily_values}.'''
d_file = open(prec_filename, 'w')
for year in sorted(prec_data.keys()):
assert (prec_data[year].shape == (122,))
d_file.write(' BLOCK-NO 25 DAILY RF OF ' + str(year) + '\n')
i = 0
for line in range(7):
for val in range(16):
