def get_cmb_data(cmb_shapefile, extract_path, simulation_period, save_csv,
static_inputs, initial_path):
cmb_dict = cmb_sample_site_data(cmb_shapefile)
get_etrm_time_series(cmb_dict, extract_path)
for key, val in cmb_dict.iteritems():
# instantiate for each item to get a clean master dict
etrm = Processes(simulation_period,
save_csv,
static_inputs=static_inputs,
point_dict=cmb_dict,
initial_inputs=initial_path)
# print 'amf dict, pre-etrm run {}'.format(amf_dict)
print 'key : {}'.format(key)
# print 'find etrm dataframe as amf_dict[key][''etrm'']\n{}'.format(amf_dict[key]['etrm'])
tracker = etrm.run(simulation_period,
point_dict=cmb_dict[key],
point_dict_key=key)
# print 'tracker after etrm run: \n {}'.format(tracker)
csv_path_filename = os.path.join(save_csv,
'{}.csv'.format(val['Name']))
print 'this should be your csv: {}'.format(csv_path_filename)
tracker.to_csv(csv_path_filename, na_rep='nan', index_label='Date')
print 'tracker for {}: {}'.format(key, tracker)
return None
def get_cmb_data(cmb_shapefile, extract_path, simulation_period, save_csv, static_inputs, initial_path):
cmb_dict = cmb_sample_site_data(cmb_shapefile)
get_etrm_time_series(cmb_dict, extract_path)
for key, val in cmb_dict.iteritems():
# instantiate for each item to get a clean master dict
etrm = Processes(simulation_period, save_csv, static_inputs=static_inputs, point_dict=cmb_dict,
initial_inputs=initial_path)
# print 'amf dict, pre-etrm run {}'.format(amf_dict)
print 'key : {}'.format(key)
# print 'find etrm dataframe as amf_dict[key][''etrm'']\n{}'.format(amf_dict[key]['etrm'])
tracker = etrm.run(simulation_period, point_dict=cmb_dict[key], point_dict_key=key)
# print 'tracker after etrm run: \n {}'.format(tracker)
csv_path_filename = os.path.join(save_csv, '{}.csv'.format(val['Name']))
print 'this should be your csv: {}'.format(csv_path_filename)
tracker.to_csv(csv_path_filename, na_rep='nan', index_label='Date')
print 'tracker for {}: {}'.format(key, tracker)
return None
def get_ameriflux_data(amf_file_path,
simulation_period,
etrm_extract=None,
static_inputs=None,
initial_path=None,
save_csv=None,
save_cleaned_data=None,
save_combo=False):
amf_dict = amf_obs_time_series(BASE_AMF_DICT,
amf_file_path,
complete_days_only=True,
save_cleaned_data_path=save_cleaned_data,
return_low_err=True)
if save_cleaned_data:
return None
# print 'amf dict w/ AMF time series: \n{}'.format(amf_dict)
get_etrm_time_series(etrm_extract, dict_=amf_dict)
# print 'amf dict w/ etrm input time series: \n{}'.format(amf_dict) # fix this so it appends to all sites
# print 'ameriflux dict: {}'.format(amf_dict)
for key, val in amf_dict.iteritems():
# instantiate for each item to get a clean master dict
etrm = Processes(simulation_period,
save_csv,
static_inputs=static_inputs,
point_dict=amf_dict,
initial_inputs=initial_path)
# print 'amf dict, pre-etrm run {}'.format(amf_dict)
print '\n key : {}'.format(key)
# print 'find etrm dataframe as amf_dict[key][''etrm'']\n{}'.format(amf_dict[key]['etrm'])
tracker = etrm.run(simulation_period,
point_dict=amf_dict,
point_dict_key=key,
modify_soils=True,
apply_rofrac=0.7,
allen_ceff=0.8)
# print 'tracker after etrm run: \n {}'.format(tracker)
csv_path_filename = os.path.join(save_csv,
'{}.csv'.format(val['Name']))
print 'this should be your csv: {}'.format(csv_path_filename)
tracker.to_csv(csv_path_filename, na_rep='nan', index_label='Date')
amf_obs_etrm_combo = DataFrame(
concat((val['AMF_Data'], tracker), axis=1, join='outer'))
obs_etrm_comb_out = os.path.join(save_combo,
'{}_Ceff.csv'.format(val['Name']))
print 'this should be your combo csv: {}'.format(obs_etrm_comb_out)
amf_obs_etrm_combo.to_csv(obs_etrm_comb_out, index_label='Date')
# print 'tracker for {}: {}'.format(key, tracker)
return None
def get_extracted_data(point_dict, simulation_period, etrm_extract=None,
static_inputs=None, initial_path=None, save_csv=None, save_cleaned_data=None,
save_combo=False): # maybe won't need some of these arguments.
""" This function here takes the point dict and runs ETRM processes on it along with the inputs."""
get_etrm_time_series(etrm_extract, dict_=point_dict)
print 'amf dict w/ etrm input time series: \n{}'.format(point_dict) # fix this so it appends to all sites
# print 'ameriflux dict: {}'.format(amf_dict)
for key, val in point_dict.iteritems():
# instantiate for each item to get a clean master dict
etrm = Processes(simulation_period, save_csv, static_inputs=static_inputs, point_dict=point_dict,
initial_inputs=initial_path)
print 'point dict, pre-etrm run {}'.format(point_dict) # for testing
print '\n key : {}'.format(key)
# print 'find etrm dataframe as amf_dict[key][''etrm'']\n{}'.format(amf_dict[key]['etrm'])
tracker = etrm.run(simulation_period, point_dict=point_dict, point_dict_key=key, modify_soils=True,
ro_reinf_frac=0.7, allen_ceff=0.8)
# look out for that key! might be a problem
# print 'tracker after etrm run: \n {}'.format(tracker)
csv_path_filename = os.path.join(save_csv, '{}.csv'.format(val['Name']))
print 'this should be your csv: {}'.format(csv_path_filename)
# saves the model results to the tracker. Keep this part.
tracker.to_csv(csv_path_filename, na_rep='nan', index_label='Date')
# I think we don't need these. They are the obs AMF or combo files
# we only want trackers
# amf_obs_etrm_combo = DataFrame(concat((val['AMF_Data'], tracker), axis=1, join='outer'))
#
# obs_etrm_comb_out = os.path.join(save_combo, '{}_Ceff.csv'.format(val['Name']))
#
# print 'this should be your combo csv: {}'.format(obs_etrm_comb_out)
# amf_obs_etrm_combo.to_csv(obs_etrm_comb_out, index_label='Date')
# # print 'tracker for {}: {}'.format(key, tracker)
return None
def run(input_root, coords_path):
paths.build(input_root)
# a function that takes uses GDAL to get the coordinates of points.
point_dict = get_point_dict(coords_path)
simulation_period = datetime(2007, 1, 1), datetime(2013, 12, 29)
get_etrm_time_series(paths.amf_ex_sac_extract, dict_=point_dict)
print 'amf dict w/ etrm input time series: \n{}'.format(point_dict) # fix this so it appends to all sites
output = os.path.join(paths.amf_ex_sac_output_root, 'trackers')
for key, val in point_dict.iteritems():
# TODO: use of point dicts no longer supported by Processes. need to update accordingly to use masks.
# =================================================================
# instantiate for each item to get a clean master dict
etrm = Processes(simulation_period,
paths.amf_trackers,
point_dict=point_dict)
print 'point dict, pre-etrm run {}'.format(point_dict) # for testing
print 'key : {}'.format(key)
# =================================================================
# TODO: `run` does not return anything. need different way to access tracker.
# probably not even necessary as tracker results are saved at the end of `run`
tracker = etrm.run(simulation_period, point_dict=point_dict, point_dict_key=key, modify_soils=True,
ro_reinf_frac=0.7, allen_ceff=0.8)
# print 'tracker after etrm run: \n {}'.format(tracker)
csv_path_filename = os.path.join(output, '{}.csv'.format(val['Name']))
print 'this should be your csv: {}'.format(csv_path_filename)
# saves the model results to the tracker. Keep this part.
tracker.to_csv(csv_path_filename, na_rep='nan', index_label='Date')
def get_ameriflux_data(amf_file_path, simulation_period, etrm_extract=None,
static_inputs=None, initial_path=None, save_csv=None, save_cleaned_data=None,
save_combo=False):
amf_dict = amf_obs_time_series(BASE_AMF_DICT, amf_file_path, complete_days_only=True,
save_cleaned_data_path=save_cleaned_data, return_low_err=True)
if save_cleaned_data:
return None
# print 'amf dict w/ AMF time series: \n{}'.format(amf_dict)
get_etrm_time_series(etrm_extract, dict_=amf_dict)
# print 'amf dict w/ etrm input time series: \n{}'.format(amf_dict) # fix this so it appends to all sites
# print 'ameriflux dict: {}'.format(amf_dict)
for key, val in amf_dict.iteritems():
# instantiate for each item to get a clean master dict
etrm = Processes(simulation_period, save_csv, static_inputs=static_inputs, point_dict=amf_dict,
initial_inputs=initial_path)
# print 'amf dict, pre-etrm run {}'.format(amf_dict)
print '\n key : {}'.format(key)
# print 'find etrm dataframe as amf_dict[key][''etrm'']\n{}'.format(amf_dict[key]['etrm'])
tracker = etrm.run(simulation_period, point_dict=amf_dict, point_dict_key=key, modify_soils=True,
apply_rofrac=0.7, allen_ceff=0.8)
# print 'tracker after etrm run: \n {}'.format(tracker)
csv_path_filename = os.path.join(save_csv, '{}.csv'.format(val['Name']))
print 'this should be your csv: {}'.format(csv_path_filename)
tracker.to_csv(csv_path_filename, na_rep='nan', index_label='Date')
amf_obs_etrm_combo = DataFrame(concat((val['AMF_Data'], tracker), axis=1, join='outer'))
obs_etrm_comb_out = os.path.join(save_combo, '{}_Ceff.csv'.format(val['Name']))
print 'this should be your combo csv: {}'.format(obs_etrm_comb_out)
amf_obs_etrm_combo.to_csv(obs_etrm_comb_out, index_label='Date')
# print 'tracker for {}: {}'.format(key, tracker)
return None
def get_sensitivity_analysis(extracts, points, statics, initials, pickle=None):
temps = range(-5, 6)
all_pct = [x * 0.1 for x in range(5, 16)]
ndvi_range = linspace(0.9, 1.7, 11)
ndvi_range = array([round_to_value(x, 0.05) for x in ndvi_range])
var_arrs = []
y = 0
for x in range(0, 6):
ones_ = ones((5, 11), dtype=float)
zeros = [x * 0.0 for x in range(5, 16)]
norm_ndvi = array([1.25 for x in zeros])
if y == 0:
arr = insert(ones_, y, temps, axis=0)
arr = insert(arr, 4, norm_ndvi, axis=0)
arr = arr[0:6]
var_arrs.append(arr)
arr = []
elif y == 4:
arr = insert(ones_, 0, zeros, axis=0)
arr = insert(arr, y, ndvi_range, axis=0)
arr = arr[0:6]
var_arrs.append(arr)
print('shape arr: {}'.format(arr.shape))
arr = []
elif y == 5:
arr = insert(ones_, 0, zeros, axis=0)
arr = insert(arr, 4, norm_ndvi, axis=0)
arr = arr[0:5]
arr = insert(arr, y, all_pct, axis=0)
var_arrs.append(arr)
arr = []
else:
arr = insert(ones_, 0, zeros, axis=0)
arr = insert(arr, y, all_pct, axis=0)
arr = insert(arr, 4, norm_ndvi, axis=0)
arr = arr[0:6]
var_arrs.append(arr)
arr = []
y += 1
print('variable arrays: {}'.format(var_arrs))
normalize_list = [2, 0.20, 0.20, 2, 0.20, 0.50]
# site_list = ['Bateman', 'Navajo_Whiskey_Ck', 'Quemazon', 'Sierra_Blanca', 'SB_1', 'SB_2', 'SB_4', 'SB_5', 'VC_1',
# 'VC_2', 'VC_3', 'CH_1', 'CH_3', 'MG_1', 'MG_2', 'WHLR_PK', 'LP', 'South_Baldy',
# 'Water_Canyon', 'La_Jencia', 'Socorro']
site_list = ['Sierra_Blanca', 'Great_Western_Mine', 'Bonito', 'Nogal']
df = DataFrame(columns=FACTORS, index=site_list)
df_norm = DataFrame(columns=FACTORS, index=site_list)
site_dict = {
'Sierra_Blanca': {},
'Great_Western_Mine': {},
'Bonito': {},
'Nogal': {}
}
ds = Open(points)
lyr = ds.GetLayer()
# defs = lyr.GetLayerDefn()
for j, feat in enumerate(lyr):
name = feat.GetField("Name")
name = name.replace(' ', '_')
geom = feat.GetGeometryRef()
mx, my = int(geom.GetX()), int(geom.GetY())
site_dict[name]['Coords'] = '{} {}'.format(mx, my)
file_name = os.path.join(extracts, '{}.csv'.format(name))
print(file_name)
site_dict[name]['etrm'] = get_etrm_time_series(file_name,
single_file=True)
# print 'site dict before running etrm: {}'.format(site_dict)
for i, var_arr in enumerate(var_arrs):
factor = FACTORS[i]
print('running modified factor: {}'.format(factor))
print('')
for key, val in site_dict.iteritems():
print('\n site: {} \n '.format(key))
results = []
for col in var_arr.T:
etrm = Processes(SIMULATION_PERIOD,
static_inputs=statics,
initial_inputs=initials,
output_root=pickle,
point_dict=site_dict)
tracker = etrm.run(point_dict=site_dict,
point_dict_key=key,
sensitivity_matrix_column=col,
sensitivity=True)
# print 'tracker: {}'.format(tracker)
results.append(tracker['tot_infil'][-1])
print('total infil: {} \n results: {}'.format(
tracker['tot_infil'][-1], results))
df.iloc[site_list.index(key),
FACTORS.index(factor)] = divide(array(results), 14.0)
print('df after site {}: \n {}'.format(key, df))
print('df: {}'.format(df))
# tot_data : precip, et, tot_transp, tot_evap, infil, runoff, snow_fall, cum_mass, end_mass
# "SI = [Q(Po + delP] -Q(Po - delP] / (2 * delP)"
# where SI = Sensitivity Index, Q = etrm, Po = base value of input parameter,
# delP = change in value input
# find sensitivity index
xx = 0
for param in df.iteritems():
data_cube = param[1]
var_arr = var_arrs[xx]
yy = 0
for site in data_cube:
site_name = site_list[yy]
normal = normalize_list[xx]
site_obj = [x for x in site]
sens_list = []
zz = 0
for var in var_arr[xx]:
if var != var_arr[xx][5]:
base = var_arr[xx][5]
deltap = var - base
obj = site_obj[zz]
sen = ((obj * (base + deltap) - obj * (base - deltap)) /
(2 * deltap)) * normal
sens_list.append(sen)
zz += 1
sens_list = array(sens_list)
df_norm.iloc[site_list.index(site_name),
FACTORS.index(param[0])] = sens_list
if yy == 20:
print('done')
break
yy += 1
xx += 1
# why not save the data as pickle, so we don't have to do the analysis each time
# we debug the plotting
df.to_pickle(os.path.join(pickle, '_basic_sensitivity_2.pkl'))
df_norm.to_pickle(os.path.join(pickle, 'norm_sensitivity_2.pkl'))
def get_sensitivity_analysis(extracts, points, statics, initials, pickle=None):
temps = range(-5, 6)
all_pct = [x * 0.1 for x in range(5, 16)]
ndvi_range = linspace(0.9, 1.7, 11)
ndvi_range = array([round_to_value(x, 0.05) for x in ndvi_range])
var_arrs = []
y = 0
for x in range(0, 6):
ones_ = ones((5, 11), dtype=float)
zeros = [x * 0.0 for x in range(5, 16)]
norm_ndvi = array([1.25 for x in zeros])
if y == 0:
arr = insert(ones_, y, temps, axis=0)
arr = insert(arr, 4, norm_ndvi, axis=0)
arr = arr[0:6]
var_arrs.append(arr)
arr = []
elif y == 4:
arr = insert(ones_, 0, zeros, axis=0)
arr = insert(arr, y, ndvi_range, axis=0)
arr = arr[0:6]
var_arrs.append(arr)
print 'shape arr: {}'.format(arr.shape)
arr = []
elif y == 5:
arr = insert(ones_, 0, zeros, axis=0)
arr = insert(arr, 4, norm_ndvi, axis=0)
arr = arr[0:5]
arr = insert(arr, y, all_pct, axis=0)
var_arrs.append(arr)
arr = []
else:
arr = insert(ones_, 0, zeros, axis=0)
arr = insert(arr, y, all_pct, axis=0)
arr = insert(arr, 4, norm_ndvi, axis=0)
arr = arr[0:6]
var_arrs.append(arr)
arr = []
y += 1
print 'variable arrays: {}'.format(var_arrs)
normalize_list = [2, 0.20, 0.20, 2, 0.20, 0.50]
# site_list = ['Bateman', 'Navajo_Whiskey_Ck', 'Quemazon', 'Sierra_Blanca', 'SB_1', 'SB_2', 'SB_4', 'SB_5', 'VC_1',
# 'VC_2', 'VC_3', 'CH_1', 'CH_3', 'MG_1', 'MG_2', 'WHLR_PK', 'LP', 'South_Baldy',
# 'Water_Canyon', 'La_Jencia', 'Socorro']
site_list = ['Sierra_Blanca', 'Great_Western_Mine', 'Bonito', 'Nogal']
df = DataFrame(columns=FACTORS, index=site_list)
df_norm = DataFrame(columns=FACTORS, index=site_list)
site_dict = {'Sierra_Blanca': {}, 'Great_Western_Mine': {}, 'Bonito': {}, 'Nogal': {}}
ds = Open(points)
lyr = ds.GetLayer()
# defs = lyr.GetLayerDefn()
for j, feat in enumerate(lyr):
name = feat.GetField("Name")
name = name.replace(' ', '_')
geom = feat.GetGeometryRef()
mx, my = int(geom.GetX()), int(geom.GetY())
site_dict[name]['Coords'] = '{} {}'.format(mx, my)
file_name = os.path.join(extracts, '{}.csv'.format(name))
print file_name
site_dict[name]['etrm'] = get_etrm_time_series(file_name, single_file=True)
# print 'site dict before running etrm: {}'.format(site_dict)
for i, var_arr in enumerate(var_arrs):
factor = FACTORS[i]
print 'running modified factor: {}'.format(factor)
print ''
for key, val in site_dict.iteritems():
print '\n site: {} \n '.format(key)
results = []
for col in var_arr.T:
etrm = Processes(SIMULATION_PERIOD, static_inputs=statics, initial_inputs=initials,
output_root=pickle, point_dict=site_dict)
tracker = etrm.run(point_dict=site_dict, point_dict_key=key, sensitivity_matrix_column=col,
sensitivity=True)
# print 'tracker: {}'.format(tracker)
results.append(tracker['tot_infil'][-1])
print 'total infil: {} \n results: {}'.format(tracker['tot_infil'][-1], results)
df.iloc[site_list.index(key), FACTORS.index(factor)] = divide(array(results), 14.0)
print 'df after site {}: \n {}'.format(key, df)
print 'df: {}'.format(df)
# tot_data : precip, et, tot_transp, tot_evap, infil, runoff, snow_fall, cum_mass, end_mass
# "SI = [Q(Po + delP] -Q(Po - delP] / (2 * delP)"
# where SI = Sensitivity Index, Q = recharge, Po = base value of input parameter,
# delP = change in value input
# find sensitivity index
xx = 0
for param in df.iteritems():
data_cube = param[1]
var_arr = var_arrs[xx]
yy = 0
for site in data_cube:
site_name = site_list[yy]
normal = normalize_list[xx]
site_obj = [x for x in site]
sens_list = []
zz = 0
for var in var_arr[xx]:
if var != var_arr[xx][5]:
base = var_arr[xx][5]
deltap = var - base
obj = site_obj[zz]
sen = ((obj * (base + deltap) - obj * (base - deltap)) / (2 * deltap)) * normal
sens_list.append(sen)
zz += 1
sens_list = array(sens_list)
df_norm.iloc[site_list.index(site_name), FACTORS.index(param[0])] = sens_list
if yy == 20:
print 'done'
break
yy += 1
xx += 1
# why not save the data as pickle, so we don't have to do the analysis each time
# we debug the plotting
df.to_pickle(os.path.join(pickle, '_basic_sensitivity_2.pkl'))
df_norm.to_pickle(os.path.join(pickle, 'norm_sensitivity_2.pkl'))