def get_horarata_data():
out = pd.read_csv(
ksl_env.shared_drives(
r"Z2003_SLMACC\pasture_growth_modelling\dryland tuning\hororata_dryland.csv"
))
out.loc[:, 'date'] = pd.to_datetime(out.loc[:, 'date'])
out.loc[:, 'doy'] = out.loc[:, 'date'].dt.dayofyear
out.set_index('doy', inplace=True)
full_out = pd.DataFrame(index=range(1, (367) * 3), columns=['pg'])
idx = out.index.values
full_out.loc[idx, 'pg'] = out.loc[:, 'pg'].values
idx += 365
full_out.loc[idx, 'pg'] = out.loc[:, 'pg'].values
full_out.loc[idx + 365, 'pg'] = out.loc[:, 'pg'].values
full_out.loc[:, 'pg'] = pd.to_numeric(full_out.loc[:, 'pg'])
full_out.loc[:, 'pg'] = full_out.loc[:, 'pg'].interpolate(method='linear')
idx = np.arange(1, 367)
full_out = full_out.loc[idx + 365]
full_out.loc[:, 'doy'] = idx
full_out.set_index('doy', inplace=True)
return full_out
def get_horarata_data_old():
out = pd.read_csv(
ksl_env.shared_drives(
r"Z2003_SLMACC\pasture_growth_modelling\dryland tuning\hororata_dryland.csv"
))
out.loc[:, 'date'] = pd.to_datetime(out.loc[:, 'date'])
out.loc[:, 'month'] = out.loc[:, 'date'].dt.month
out.set_index('date', inplace=True)
out_sum = pd.DataFrame(index=pd.date_range('2011-01-01', '2011-12-31'),
columns=['pg'],
dtype=float)
out_sum.loc[out.index, 'pg'] = out.loc[:, 'pg'].values
temp = pd.concat(
[out_sum, out_sum,
out_sum]).reset_index().loc[:,
'pg'].interpolate(method='linear').values
out_sum.loc[:, 'pg'] = temp[366:366 + 365]
out_sum.loc[:, 'month'] = out_sum.index.month
out_sum = pd.DataFrame(out_sum.groupby('month').sum().loc[:, 'pg'])
out_sum.loc[:,
'pgr'] = out_sum.loc[:,
'pg'] / [ndays[e] for e in out_sum.index]
return out_sum
def build_example_storylines_for_greg(outdir):
if not os.path.exists(outdir):
os.makedirs(outdir)
data = pd.read_excel(ksl_env.shared_drives(
r"Z2003_SLMACC\event_definition\example_story_lines.xlsx"),
header=[0, 1],
skiprows=1)
ndays = {
1: 31,
2: 28,
3: 31,
4: 30,
5: 31,
6: 30,
7: 31,
8: 31,
9: 30,
10: 31,
11: 30,
12: 31,
}
rest_vals = {
1: 10,
2: 17,
3: 17,
4: 10,
5: 7,
6: 10,
7: 10,
8: 10,
9: 7,
10: 5,
11: 5,
12: 7,
}
rest_vals = {k: rest_vals[k] / ndays[k] for k in range(1, 13)}
for k in data.columns.levels[0]:
storyline = data.loc[:, k].dropna()
storyline.loc[:, 'month'] = storyline.loc[:, 'month'].astype(int)
storyline.loc[:, 'year'] = storyline.loc[:, 'year'].astype(int) + 2024
storyline.loc[storyline.rest > 0,
'rest'] = storyline.loc[:, 'month'].replace(rest_vals)
try:
ensure_no_impossible_events(storyline)
except Exception as val:
print('{} raised:\n {}'.format(k, val))
storyline.to_csv(os.path.join(outdir, '{}.csv'.format(k)), index=False)
def get_vcsn_record(version='trended', site='eyrewell', recalc=False):
if version == 'trended':
pass
elif version == 'detrended' and site == 'eyrewell':
raise ValueError('depreciated')
return _get_eyrewell_detrended(1)
elif version == 'detrended2' and site == 'eyrewell':
return _get_eyrewell_detrended(2)
else:
raise ValueError('incorrect {} for version'.format(version))
if site == 'eyrewell':
lat, lon = -43.372, 172.333
elif site == 'oxford':
lat, lon = -43.296, 172.192
else:
raise NotImplementedError('site: {} not implemented'.format(site))
key = 'weather_data'
data_path = ksl_env.shared_drives(
r"Z2003_SLMACC\weather_date\{}.hdf".format(site))
if not os.path.exists(os.path.dirname(data_path)):
os.makedirs(os.path.dirname(data_path))
if os.path.exists(data_path) and not recalc:
data = pd.read_hdf(data_path, key=key)
return data
data, use_cords = vcsn_pull_single_site(lat,
lon,
year_min=1972,
year_max=2019,
use_vars=('evspsblpot', 'rsds',
'tasmax', 'tasmin',
'pr'))
data.rename(columns={
'evspsblpot': 'pet',
'pr': 'rain',
'rsds': 'radn',
'tasmax': 'tmax',
'tasmin': 'tmin'
},
inplace=True)
print(use_cords)
data = data.set_index('date')
data.to_hdf(data_path, key=key, mode='w')
return data
def get_indicative_irrigated():
data = pd.read_csv(ksl_env.shared_drives(
r"Z2003_SLMACC\pasture_growth_modelling\SamSBPastureGrowth_irrigated.csv"
),
index_col=0).to_dict()
out_sum = pd.DataFrame(index=pd.date_range('2011-01-01',
'2011-12-31',
name='date'),
columns=['pg'],
dtype=int)
out_sum.loc[:, 'pg'] = pd.to_numeric(out_sum.index.month)
out_sum = out_sum.replace(data)
out_sum.loc[:, 'month'] = out_sum.index.month
out_sum = pd.DataFrame(out_sum.groupby('month').sum().loc[:, 'pg'])
out_sum.loc[:,
'pgr'] = out_sum.loc[:,
'pg'] / [ndays[e] for e in out_sum.index]
return out_sum
def get_restriction_record(version='trended', recalc=False):
if version == 'trended':
data_path = os.path.join(os.path.dirname(event_def_path),
'restriction_record.csv')
dt_format = '%Y-%m-%d'
raw_data_path = ksl_env.shared_drives(
r"Z2003_SLMACC\WIL data\OSHB_WaimakRiverData_withRestrictionInfo.xlsx"
)
elif version == 'detrended' or version == 'detrended2':
dt_format = '%Y-%m-%d'
data_path = os.path.join(os.path.dirname(event_def_path),
'restriction_record_detrend.csv')
else:
raise ValueError(
'unexpected argument for version {} expected either trended or detrended'
.format(version))
if not recalc and os.path.exists(data_path):
int_keys = {
'day': int,
'doy': int,
'month': int,
'year': int,
'f_rest': float,
'flow': float,
'take': float,
}
data = pd.read_csv(data_path, dtype=int_keys)
data.loc[:, 'date'] = pd.to_datetime(data.loc[:, 'date'],
format=dt_format)
data.set_index('date', inplace=True)
data.sort_index(inplace=True)
return data
if version == 'trended':
data = pd.read_excel(
raw_data_path
).loc[:, ['OHB flow m3/s', 'Take rate', 'Day', 'Month', 'Year']]
data = data.rename(
columns={
'OHB flow m3/s': 'flow',
'Take rate': 'take',
'Day': 'day',
'Month': 'month',
'Year': 'year'
})
data.loc[:,
'f_rest'] = 1 - data.loc[:, 'take'] / data.loc[:,
'take'].max()
data = data.loc[(data.year <= 2019) & (data.year >= 1972)]
data = data.groupby(['day', 'month', 'year']).mean().reset_index()
strs = [
'{}-{:02d}-{:02d}'.format(y, m, d)
for y, m, d in data[['year', 'month', 'day']].itertuples(
False, None)
]
data.loc[:, 'date'] = pd.Series(pd.to_datetime(strs))
data.loc[:, 'doy'] = data.date.dt.dayofyear
data = data.set_index('date')
elif version == 'detrended':
raise ValueError('detrended record is calculated via another process')
else:
raise ValueError(
'unexpected argument for version {} expected either trended or detrended'
.format(version))
data.loc[data.f_rest < 0.001, 'f_rest'] = 0
outdata = pd.DataFrame(
index=pd.date_range('1972-01-01', '2019-12-31', name='date'),
columns=['flow', 'take', 'day', 'month', 'year', 'f_rest'])
outdata.loc[:] = np.nan
outdata = outdata.combine_first(data)
outdata = outdata.fillna(method='ffill')
outdata.loc[:, 'day'] = outdata.index.day
outdata.loc[:, 'year'] = outdata.index.year
outdata.loc[:, 'month'] = outdata.index.month
outdata.loc[:, 'doy'] = outdata.index.dayofyear
outdata.to_csv(data_path)
return outdata
freq='{}D'.format(freq))
idx = np.in1d(pd.to_datetime(strs, format='%Y-%j'), harv_days)
days_harvest.loc[idx, 'harv_trig'] = trig
days_harvest.loc[idx, 'harv_targ'] = targ
out = run_basgra_nz(params,
matrix_weather,
days_harvest,
doy_irr,
verbose=False)
return out
if __name__ == '__main__':
outdir = ksl_env.shared_drives(
r"Z2003_SLMACC\pasture_growth_modelling\basgra_harvest_tuning\irr_harv_testing"
)
data = {
'Woodward_model': run_old_basgra(),
}
freq = [10, 10, 10]
trigs = [1501, 1600, 1700]
targs = [1500, 1500, 1500]
for f, tr, ta, in zip(freq, trigs, targs):
data['freq: {}, Trig:{}, Targ:{}'.format(f, tr,
ta)] = run_frequent_harvest(
f, tr, ta)
plot_multiple_results(
data,
"""
Author: Matt Hanson
Created: 21/12/2020 8:47 AM
"""
import pandas as pd
import numpy as np
import os
import ksl_env
from Climate_Shocks.get_past_record import get_restriction_record
from Climate_Shocks.climate_shocks_env import event_def_path
outdir = ksl_env.shared_drives(r"Z2003_SLMACC\event_definition\mutual_info")
if not os.path.exists(outdir):
os.makedirs(outdir)
event_months = {
'hot': [11, 12, 1, 2, 3],
'cold': list(range(5, 10)),
'dry': list(range(8, 13)) + list(range(1, 6)),
'wet': list(range(5, 10)),
'rest': list(range(9, 13)) + list(range(1, 5))
}
def make_data(org_data,
save=False,
restrict_cat=True,
save_paths=(event_def_path, )):
"""
make the final data for greg
:param org_data: from final_event_recurance import get_org_data
temp2.loc[:, :] = temp
temp2.loc[:, 'f_rest'] = restrict + 3
temp2.loc[:, 'pg'] = calc_pasture_growth(temp2, days_harvest, 'from_yield', '1D', resamp_fun='mean')
out['{}_paddock_rest'.format(base_name)] = temp2
for k in out:
out[k].loc[:, 'per_PAW'] = out[k].loc[:, 'PAW'] / out[k].loc[:, 'MXPAW']
return out, padock_values
if __name__ == '__main__':
save = False
params, doy = get_params_doy_irr(mode)
outdir = ksl_env.shared_drives(r"Z2003_SLMACC\pasture_growth_modelling\irrigation_tuning")
if not os.path.exists(outdir):
os.makedirs(outdir)
out, paddock_values = create_irrigation_abandomnet_data('b', params, reseed_trig=0.691, reseed_basal=0.722,
site='eyrewell')
# run some stats
outkeys = ['BASAL', 'f_rest', 'IRRIG', 'per_PAW', 'pg']
for k, v in out.items():
v.loc[:, 'month'] = v.index.month
rest = get_restriction_record()
rest = rest.groupby(['year', 'month']).sum()
idxs = pd.MultiIndex.from_product([pd.unique(out['b_paddock_rest'].loc[:, 'year']),
pd.unique(out['b_paddock_rest'].loc[:, 'month'])], names=['year', 'month'])
outdata = pd.DataFrame(index=idxs,
"""
import pandas as pd
import numpy as np
import os
import ksl_env
# add basgra nz functions
ksl_env.add_basgra_nz_path()
from supporting_functions.plotting import plot_multiple_results, plot_multiple_monthly_results
from Pasture_Growth_Modelling.initialisation_support.comparison_support import make_mean_comparison, \
get_horarata_data_old, get_indicative_irrigated
from Pasture_Growth_Modelling.historical_average_baseline import run_past_basgra_dryland, run_past_basgra_irrigated
if __name__ == '__main__':
outdir = ksl_env.shared_drives(
r"Z2003_SLMACC\pasture_growth_modelling\historical_runs_v2")
save = False
data = {
'irrigated_eyrewell': run_past_basgra_irrigated(),
#'irrigated_oxford': run_past_basgra_irrigated(site='oxford'),
'dryland_oxford': run_past_basgra_dryland(site='oxford'),
}
for i, k in enumerate(data.keys()):
data[
k].loc[:,
'RESEEDED'] += i # any more fo these to raise up to see with multiple runs
data2 = {e: make_mean_comparison(v, 'mean') for e, v in data.items()}
data2['Horoata'] = get_horarata_data_old()
data2['indicative_irr'] = get_indicative_irrigated()
"""
import ksl_env
import pandas as pd
import numpy as np
import os
from Climate_Shocks.vcsn_pull import vcsn_pull_single_site
# add basgra nz functions
ksl_env.add_basgra_nz_path()
from basgra_python import run_basgra_nz
from supporting_functions.plotting import plot_multiple_results
from supporting_functions.woodward_2020_params import get_woodward_mean_full_params
from input_output_keys import matrix_weather_keys_pet
backed_dir = ksl_env.shared_drives(
"Z2003_SLMACC\pasture_growth_modelling\dryland tuning")
unbacked_dir = ksl_env.mh_unbacked(
"Z2003_SLMACC\pasture_growth_modelling\dryland tuning")
if not os.path.exists(backed_dir):
os.makedirs(backed_dir)
if not os.path.exists(unbacked_dir):
os.makedirs(unbacked_dir)
def get_org_data():
raise NotImplementedError
def get_param_set(BASALI=0.25):
(en_u, _describe_names)] = temp.loc[:,
_describe_names].values
outdata = outdata.sort_index(axis=1, level=0, sort_remaining=False)
return outdata
if __name__ == '__main__':
# initial events recurrence must be run first. then this creates, the final events.
# visualied_events.csv come from Storylines.check_storyline
# event def data comes from Climate_Shocks\note_worthy_events\rough_stats.py
# below is historical... moved to Climate_Shocks\make_new_event_definition_data.py
run_old = False
run_detrend_test = True
if run_old:
event_def_dir = ksl_env.shared_drives(
"Z2003_SLMACC\event_definition/v5")
out = make_prob_impact_data()
t = pd.Series([' '.join(e) for e in out.columns])
idx = ~((t.str.contains('sum')) | (t.str.contains('count')))
out.loc[:, out.columns[idx]] *= 100
out.to_csv(os.path.join(event_def_dir, 'current_choice.csv'),
float_format='%.1f%%')
out.to_csv(os.path.join(os.path.dirname(event_def_path),
'event_historical_prob_impact.csv'),
float_format='%.1f%%')
make_data(get_org_data(), save=True)
if run_detrend_test:
temp = make_data(
get_org_data(
ksl_env.shared_drives(
7: 10,
8: 10,
9: 7,
10: 5,
11: 5,
12: 7,
}
rest_vals = {k: rest_vals[k] / ndays[k] for k in range(1, 13)}
for k in data.columns.levels[0]:
storyline = data.loc[:, k].dropna()
storyline.loc[:, 'month'] = storyline.loc[:, 'month'].astype(int)
storyline.loc[:, 'year'] = storyline.loc[:, 'year'].astype(int) + 2024
storyline.loc[storyline.rest > 0,
'rest'] = storyline.loc[:, 'month'].replace(rest_vals)
try:
ensure_no_impossible_events(storyline)
except Exception as val:
print('{} raised:\n {}'.format(k, val))
storyline.to_csv(os.path.join(outdir, '{}.csv'.format(k)), index=False)
if __name__ == '__main__':
# these will no longer run as they do not have the correct index
build_example_storylines_for_greg(
ksl_env.shared_drives(
r"Z2003_SLMACC\event_definition\example_storys_for_greg"))
from BS_work.SWG.check_1_month_runs import make_event_prob
from Climate_Shocks.note_worthy_events.rough_stats import make_data
from Climate_Shocks.climate_shocks_env import event_def_path, supporting_data_dir, storyline_dir
from Climate_Shocks.note_worthy_events.final_event_recurance import get_org_data
from Storylines.check_storyline import get_past_event_frequency, get_acceptable_events
from Storylines.storyline_evaluation.storyline_eval_support import get_pgr_prob_baseline_stiched
if __name__ == '__main__':
t = input('this will re-run most of the system and takes c. 15 HOURS are you sure you want to proceed '
'with make_new_event_definition_data.py \nY/N')
if t.lower() != 'y':
raise ValueError('stopped to prevent override')
re_run_SWG = True
re_run_pgr = True
prev_event_path = ksl_env.shared_drives(r"Z2003_SLMACC\event_definition\v5_detrend\detrend_event_data.csv")
event_def_dir = ksl_env.shared_drives(r"Z2003_SLMACC\event_definition/v6_detrend")
vcsn_version = 'detrended2'
root_dir = os.path.dirname(os.path.dirname(__file__))
# run inital event recurance
print('running inital for vcsn: {} and event_def_dir: {}'.format(vcsn_version, event_def_dir))
inital = os.path.join(root_dir, 'Climate_Shocks/note_worthy_events/inital_event_recurance.py')
result = subprocess.run([sys.executable, inital, event_def_dir, vcsn_version],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if result.returncode != 0:
raise ChildProcessError('{}\n{}'.format(result.stdout, result.stderr))
# make event data
print('making final event_data for vcsn: {} and event_def_dir: {}'.format(vcsn_version, event_def_dir))