def create_dqs_data(): """Generates the set of input data files for this study. This method supercedes the other create_* methods below, which are retained for reference.""" # calculate seasonal and climatological means obs_rainfall_vals = get_rain_values(model_loc + "/inputs_gujarat/weather/original/rainfall.dat") obs_rainfall_climate_mean = obs_rainfall_vals.mean() obs_rainfall_climate_std = obs_rainfall_vals.std() obs_yield_climate_mean = obs_yield.mean() obs_yield_climate_std = obs_yield.std() obs_temp_season_mean = [] # line_num : seasonal_mean obs_temp_season_std = [] # line_num : seasonal_std obs_temp_avg_svals = array([]) # array of all temp values from the season we are interested in for i in range(obs_min_temp_data.shape[0]): # tmin and tmax files have same structure tmin_svals = obs_min_temp_data[i, 5:9] # June -> September (inclusive) tmax_svals = obs_max_temp_data[i, 5:9] tavg_svals = (tmin_svals + tmax_svals) / 2.0 obs_temp_season_mean.append(tavg_svals.mean()) obs_temp_season_std.append(tavg_svals.std()) obs_temp_avg_svals = append(obs_temp_avg_svals, tavg_svals) obs_temp_climate_mean = obs_temp_avg_svals.mean() obs_temp_climate_std = obs_temp_avg_svals.std() # create directories mkdir(model_loc + "/dqs_data") for var_name, bias_type in dqs_data_config.iteritems(): mkdir(model_loc + "/dqs_data/" + var_name) for b in bias_type: mkdir(model_loc + "/dqs_data/" + var_name + "/" + b) for s in dqs_seeds: r = Random(s) for p in percent_std_dev: # -- daily uncorrelated precip -- out_file = open(model_loc + "/dqs_data/prec/day/p-" + str(p) + "_s-" + str(s) + ".dat", "w") for row in obs_rain_data: if len(row) in (51, 81): # there are 10 or 16 {3.1}f values new_row = "" index = 0 while index < (len(row) - 1): # there is a trailing '\n' orig_value = float(row[index : (index + 5)]) out_std_dev = p / 100.0 * obs_rainfall_climate_std # was: * orig_value new_value = r.normalvariate(orig_value, out_std_dev) if new_value < 0: new_value = 0.0 elif new_value >= 1000: # column format is {3.1}f new_value = 999.9 new_row += "{0:5.1f}".format(new_value) index += 5 new_row += "\n" out_file.write(new_row) else: out_file.write(row) out_file.close() # -- seasonally biased precip -- out_file = open(model_loc + "/dqs_data/prec/season/p-" + str(p) + "_s-" + str(s) + ".dat", "w") for rain_year in range(1966, 1990): obs_rainfall_year_mean = get_rain_year_values(rain_year).mean() obs_rainfall_year_std = get_rain_year_values(rain_year).std() out_std_dev = p / 100.0 * obs_rainfall_year_std # was: * obs_rainfall_year_mean diff_val = r.normalvariate(obs_rainfall_year_mean, out_std_dev) - obs_rainfall_year_mean out_file.write(" BLOCK-NO 25 DAILY RF OF " + str(rain_year) + "\n") for row in obs_rain_data[obs_rain_layout[rain_year][0] : (obs_rain_layout[rain_year][1] + 1)]: new_row = "" index = 0 while index < (len(row) - 1): # there is a trailing '\n' orig_value = float(row[index : (index + 5)]) new_value = orig_value + diff_val if new_value < 0: new_value = 0.0 elif new_value >= 1000.0: # column format is {3.1}f new_value = 999.9 new_row += "{0:5.1f}".format(new_value) index += 5 new_row += "\n" out_file.write(new_row) out_file.write("\n\n") out_file.close() # -- climatologically biased precip -- out_file = open(model_loc + "/dqs_data/prec/climate/p-" + str(p) + "_s-" + str(s) + ".dat", "w") out_std_dev = p / 100.0 * obs_rainfall_climate_std # was: obs_rainfall_climate_mean diff_val = r.normalvariate(obs_rainfall_climate_mean, out_std_dev) - obs_rainfall_climate_mean for row in obs_rain_data: if len(row) in (51, 81): # there are 10 or 16 {3.1}f values new_row = "" index = 0 while index < (len(row) - 1): # there is a trailing '\n' orig_value = float(row[index : (index + 5)]) new_value = orig_value + diff_val if new_value < 0: new_value = 0.0 elif new_value >= 1000.0: # column format is {3.1}f new_value = 999.9 new_row += "{0:5.1f}".format(new_value) index += 5 new_row += "\n" out_file.write(new_row) else: out_file.write(row) out_file.close() # -- seasonally biased yield -- out_file = open(model_loc + "/dqs_data/yield/season/p-" + str(p) + "_s-" + str(s) + ".dat", "w") for row in obs_yield_data: out_mean = row[3] out_std_dev = p / 100.0 * obs_yield_climate_std # was: out_mean out_yield = r.normalvariate(out_mean, out_std_dev) if out_yield < 0: out_yield = 0.0 out_file.write( str(int(row[0])).ljust(7) + str(int(row[1])).ljust(4) + str(int(row[2])) + "{0:.5f}".format(out_yield).rjust(13) + "\n" ) out_file.close() # -- climatologically biased yield -- out_file = open(model_loc + "/dqs_data/yield/climate/p-" + str(p) + "_s-" + str(s) + ".dat", "w") out_std_dev = p / 100.0 * obs_yield_climate_std # was: obs_yield_climate_mean diff_val = r.normalvariate(obs_yield_climate_mean, out_std_dev) - obs_yield_climate_mean for row in obs_yield_data: out_yield = row[3] + diff_val if out_yield < 0: out_yield = 0.0 out_file.write( str(int(row[0])).ljust(7) + str(int(row[1])).ljust(4) + str(int(row[2])) + "{0:.5f}".format(out_yield).rjust(13) + "\n" ) out_file.close() # -- seasonally biased mean temp -- tmin_out_file = open(model_loc + "/dqs_data/mtemp/season/tmin_p-" + str(p) + "_s-" + str(s) + ".dat", "w") tmax_out_file = open(model_loc + "/dqs_data/mtemp/season/tmax_p-" + str(p) + "_s-" + str(s) + ".dat", "w") for i in range(obs_min_temp_data.shape[0]): # tmin and tmax files have same structure out_std_dev = p / 100.0 * obs_temp_season_std[i] # was: obs_temp_season_mean[i] diff_val = int(r.normalvariate(obs_temp_season_mean[i], out_std_dev) - obs_temp_season_mean[i]) new_tmin_row = obs_min_temp_first_column[i] new_tmax_row = obs_max_temp_first_column[i] for j in range(obs_min_temp_data.shape[1]): new_tmin_val = obs_min_temp_data[i, j] + diff_val if new_tmin_val < 0: new_tmin_val = 0 elif new_tmin_val >= 1000: # data format is (2XI3) in input.f90 new_tmin_val = 999 new_tmax_val = obs_max_temp_data[i, j] + diff_val if new_tmax_val < 0: new_tmax_val = 0 elif new_tmax_val >= 1000: new_tmax_val = 999 new_tmin_row += "{0:5d}".format(new_tmin_val) new_tmax_row += "{0:5d}".format(new_tmax_val) new_tmin_row += "\n" new_tmax_row += "\n" tmin_out_file.write(new_tmin_row) tmax_out_file.write(new_tmax_row) tmin_out_file.write("\n") tmax_out_file.write("\n") tmin_out_file.close() tmax_out_file.close() # -- climatologically biased mean temp -- tmin_out_file = open(model_loc + "/dqs_data/mtemp/climate/tmin_p-" + str(p) + "_s-" + str(s) + ".dat", "w") tmax_out_file = open(model_loc + "/dqs_data/mtemp/climate/tmax_p-" + str(p) + "_s-" + str(s) + ".dat", "w") out_std_dev = p / 100.0 * obs_temp_climate_std # was: obs_temp_climate_mean diff_val = int(r.normalvariate(obs_temp_climate_mean, out_std_dev) - obs_temp_climate_mean) for i in range(obs_min_temp_data.shape[0]): # tmin and tmax files have same structure new_tmin_row = obs_min_temp_first_column[i] new_tmax_row = obs_max_temp_first_column[i] for j in range(obs_min_temp_data.shape[1]): new_tmin_val = obs_min_temp_data[i, j] + diff_val if new_tmin_val < 0: new_tmin_val = 0 elif new_tmin_val >= 1000: # data format is (2XI3) in input.f90 new_tmin_val = 999 new_tmax_val = obs_max_temp_data[i, j] + diff_val if new_tmax_val < 0: new_tmax_val = 0 elif new_tmax_val >= 1000: new_tmax_val = 999 new_tmin_row += "{0:5d}".format(new_tmin_val) new_tmax_row += "{0:5d}".format(new_tmax_val) new_tmin_row += "\n" new_tmax_row += "\n" tmin_out_file.write(new_tmin_row) tmax_out_file.write(new_tmax_row) tmin_out_file.write("\n") tmax_out_file.write("\n") tmin_out_file.close() tmax_out_file.close()
def create_dqs_data(): '''Generates the set of input data files for this study. This method supercedes the other create_* methods below, which are retained for reference.''' # calculate seasonal and climatological means obs_rainfall_vals = get_rain_values(model_loc + '/inputs_gujarat/weather/original/rainfall.dat') obs_rainfall_climate_mean = obs_rainfall_vals.mean() obs_rainfall_climate_std = obs_rainfall_vals.std() obs_yield_climate_mean = obs_yield.mean() obs_yield_climate_std = obs_yield.std() obs_temp_season_mean = [] # line_num : seasonal_mean obs_temp_season_std = [] # line_num : seasonal_std obs_temp_avg_svals = array([]) # array of all temp values from the season we are interested in for i in range(obs_min_temp_data.shape[0]): # tmin and tmax files have same structure tmin_svals = obs_min_temp_data[i,5:9] # June -> September (inclusive) tmax_svals = obs_max_temp_data[i,5:9] tavg_svals = (tmin_svals + tmax_svals) / 2.0 obs_temp_season_mean.append(tavg_svals.mean()) obs_temp_season_std.append(tavg_svals.std()) obs_temp_avg_svals = append(obs_temp_avg_svals, tavg_svals) obs_temp_climate_mean = obs_temp_avg_svals.mean() obs_temp_climate_std = obs_temp_avg_svals.std() # create directories mkdir(model_loc + '/dqs_data') for var_name, bias_type in dqs_data_config.iteritems(): mkdir(model_loc + '/dqs_data/' + var_name) for b in bias_type: mkdir(model_loc + '/dqs_data/' + var_name + '/' + b) for s in dqs_seeds: r = Random(s) for p in percent_std_dev: # -- daily uncorrelated precip -- out_file = open(model_loc + '/dqs_data/prec/day/p-' + str(p) + '_s-' + str(s) + '.dat', 'w') for row in obs_rain_data: if len(row) in (51, 81): # there are 10 or 16 {3.1}f values new_row = '' index = 0 while index < (len(row)-1): # there is a trailing '\n' orig_value = float(row[index:(index+5)]) out_std_dev = p / 100.0 * obs_rainfall_climate_std # was: * orig_value new_value = r.normalvariate(orig_value, out_std_dev) if new_value < 0: new_value = 0.0 elif new_value >= 1000: # column format is {3.1}f new_value = 999.9 new_row += '{0:5.1f}'.format(new_value) index += 5 new_row += '\n' out_file.write(new_row) else: out_file.write(row) out_file.close() # -- seasonally biased precip -- out_file = open(model_loc + '/dqs_data/prec/season/p-' + str(p) + '_s-' + str(s) + '.dat', 'w') for rain_year in range(1966,1990): obs_rainfall_year_mean = get_rain_year_values(rain_year).mean() obs_rainfall_year_std = get_rain_year_values(rain_year).std() out_std_dev = p / 100.0 * obs_rainfall_year_std # was: * obs_rainfall_year_mean diff_val = r.normalvariate(obs_rainfall_year_mean, out_std_dev) - obs_rainfall_year_mean out_file.write(' BLOCK-NO 25 DAILY RF OF ' + str(rain_year) + '\n') for row in obs_rain_data[obs_rain_layout[rain_year][0]:(obs_rain_layout[rain_year][1]+1)]: new_row = '' index = 0 while index < (len(row)-1): # there is a trailing '\n' orig_value = float(row[index:(index+5)]) new_value = orig_value + diff_val if new_value < 0: new_value = 0.0 elif new_value >= 1000.0: # column format is {3.1}f new_value = 999.9 new_row += '{0:5.1f}'.format(new_value) index += 5 new_row += '\n' out_file.write(new_row) out_file.write('\n\n') out_file.close() # -- climatologically biased precip -- out_file = open(model_loc + '/dqs_data/prec/climate/p-' + str(p) + '_s-' + str(s) + '.dat', 'w') out_std_dev = p / 100.0 * obs_rainfall_climate_std # was: obs_rainfall_climate_mean diff_val = r.normalvariate(obs_rainfall_climate_mean, out_std_dev) - obs_rainfall_climate_mean for row in obs_rain_data: if len(row) in (51, 81): # there are 10 or 16 {3.1}f values new_row = '' index = 0 while index < (len(row)-1): # there is a trailing '\n' orig_value = float(row[index:(index+5)]) new_value = orig_value + diff_val if new_value < 0: new_value = 0.0 elif new_value >= 1000.0: # column format is {3.1}f new_value = 999.9 new_row += '{0:5.1f}'.format(new_value) index += 5 new_row += '\n' out_file.write(new_row) else: out_file.write(row) out_file.close() # -- seasonally biased yield -- out_file = open(model_loc + '/dqs_data/yield/season/p-' + str(p) + '_s-' + str(s) + '.dat', 'w') for row in obs_yield_data: out_mean = row[3] out_std_dev = p / 100.0 * obs_yield_climate_std # was: out_mean out_yield = r.normalvariate(out_mean, out_std_dev) if out_yield < 0: out_yield = 0.0 out_file.write(str(int(row[0])).ljust(7) + str(int(row[1])).ljust(4) + str(int(row[2])) + '{0:.5f}'.format(out_yield).rjust(13) + '\n') out_file.close() # -- climatologically biased yield -- out_file = open(model_loc + '/dqs_data/yield/climate/p-' + str(p) + '_s-' + str(s) + '.dat', 'w') out_std_dev = p / 100.0 * obs_yield_climate_std # was: obs_yield_climate_mean diff_val = r.normalvariate(obs_yield_climate_mean, out_std_dev) - obs_yield_climate_mean for row in obs_yield_data: out_yield = row[3] + diff_val if out_yield < 0: out_yield = 0.0 out_file.write(str(int(row[0])).ljust(7) + str(int(row[1])).ljust(4) + str(int(row[2])) + '{0:.5f}'.format(out_yield).rjust(13) + '\n') out_file.close() # -- seasonally biased mean temp -- tmin_out_file = open(model_loc + '/dqs_data/mtemp/season/tmin_p-' + str(p) + '_s-' + str(s) + '.dat', 'w') tmax_out_file = open(model_loc + '/dqs_data/mtemp/season/tmax_p-' + str(p) + '_s-' + str(s) + '.dat', 'w') for i in range(obs_min_temp_data.shape[0]): # tmin and tmax files have same structure out_std_dev = p / 100.0 * obs_temp_season_std[i] # was: obs_temp_season_mean[i] diff_val = int(r.normalvariate(obs_temp_season_mean[i], out_std_dev) - obs_temp_season_mean[i]) new_tmin_row = obs_min_temp_first_column[i] new_tmax_row = obs_max_temp_first_column[i] for j in range(obs_min_temp_data.shape[1]): new_tmin_val = obs_min_temp_data[i,j] + diff_val if new_tmin_val < 0: new_tmin_val = 0 elif new_tmin_val >= 1000: # data format is (2XI3) in input.f90 new_tmin_val = 999 new_tmax_val = obs_max_temp_data[i,j] + diff_val if new_tmax_val < 0: new_tmax_val = 0 elif new_tmax_val >= 1000: new_tmax_val = 999 new_tmin_row += '{0:5d}'.format(new_tmin_val) new_tmax_row += '{0:5d}'.format(new_tmax_val) new_tmin_row += '\n' new_tmax_row += '\n' tmin_out_file.write(new_tmin_row) tmax_out_file.write(new_tmax_row) tmin_out_file.write('\n') tmax_out_file.write('\n') tmin_out_file.close() tmax_out_file.close() # -- climatologically biased mean temp -- tmin_out_file = open(model_loc + '/dqs_data/mtemp/climate/tmin_p-' + str(p) + '_s-' + str(s) + '.dat', 'w') tmax_out_file = open(model_loc + '/dqs_data/mtemp/climate/tmax_p-' + str(p) + '_s-' + str(s) + '.dat', 'w') out_std_dev = p / 100.0 * obs_temp_climate_std # was: obs_temp_climate_mean diff_val = int(r.normalvariate(obs_temp_climate_mean, out_std_dev) - obs_temp_climate_mean) for i in range(obs_min_temp_data.shape[0]): # tmin and tmax files have same structure new_tmin_row = obs_min_temp_first_column[i] new_tmax_row = obs_max_temp_first_column[i] for j in range(obs_min_temp_data.shape[1]): new_tmin_val = obs_min_temp_data[i,j] + diff_val if new_tmin_val < 0: new_tmin_val = 0 elif new_tmin_val >= 1000: # data format is (2XI3) in input.f90 new_tmin_val = 999 new_tmax_val = obs_max_temp_data[i,j] + diff_val if new_tmax_val < 0: new_tmax_val = 0 elif new_tmax_val >= 1000: new_tmax_val = 999 new_tmin_row += '{0:5d}'.format(new_tmin_val) new_tmax_row += '{0:5d}'.format(new_tmax_val) new_tmin_row += '\n' new_tmax_row += '\n' tmin_out_file.write(new_tmin_row) tmax_out_file.write(new_tmax_row) tmin_out_file.write('\n') tmax_out_file.write('\n') tmin_out_file.close() tmax_out_file.close()
# create list of unique random number seeds seed(21) shuffle_seeds = {} for v in shuffled_data_config.keys(): shuffle_seeds[v] = [] while len(shuffle_seeds[v]) < num_seeds: s = randint(1, 9999) if s not in shuffle_seeds[v]: shuffle_seeds[v].append(s) # load original observations prec_obs = {} # year : daily values for year in range(1966, 1990): prec_obs[year] = get_rain_year_values(year) def save_prec_data(prec_data, prec_filename): """Saves the data in the dictionary prec_data to the file with the given filename. prec_data should be of the form {year : array_of_daily_values}.""" d_file = open(prec_filename, "w") for year in sorted(prec_data.keys()): assert prec_data[year].shape == (122,) d_file.write(" BLOCK-NO 25 DAILY RF OF " + str(year) + "\n") i = 0 for line in range(7): for val in range(16):
# create list of unique random number seeds seed(21) shuffle_seeds = {} for v in shuffled_data_config.keys(): shuffle_seeds[v] = [] while len(shuffle_seeds[v]) < num_seeds: s = randint(1,9999) if s not in shuffle_seeds[v]: shuffle_seeds[v].append(s) # load original observations prec_obs = {} # year : daily values for year in range(1966,1990): prec_obs[year] = get_rain_year_values(year) def save_prec_data(prec_data, prec_filename): '''Saves the data in the dictionary prec_data to the file with the given filename. prec_data should be of the form {year : array_of_daily_values}.''' d_file = open(prec_filename, 'w') for year in sorted(prec_data.keys()): assert (prec_data[year].shape == (122,)) d_file.write(' BLOCK-NO 25 DAILY RF OF ' + str(year) + '\n') i = 0 for line in range(7): for val in range(16):