def test_SplitFileWriterNode():
from awrams.utils import extents
from awrams.utils import datetools as dt
import awrams.models.awral.description
awrams.models.awral.description.CLIMATE_DATA = os.path.join(
os.path.dirname(__file__), '..', '..', 'test_data', 'simulation')
from awrams.utils.nodegraph import nodes
from awrams.simulation.ondemand import OnDemandSimulator
from awrams.models import awral
input_map = awral.get_default_mapping()
from awrams.utils.nodegraph import nodes
from awrams.utils.metatypes import ObjectDict
# output_path = './'
mapping = {}
mapping['qtot'] = nodes.write_to_annual_ncfile('./', 'qtot')
output_map = ObjectDict(
mapping=ObjectDict(mapping)) #,output_path=output_path)
runner = OnDemandSimulator(awral,
input_map.mapping,
omapping=output_map.mapping)
period = dt.dates('2010-2011')
extent = extents.from_cell_offset(200, 200)
r = runner.run(period, extent)
def test_multiple_catchment():
import os
import pandas as pd
import pickle
import sys
import awrams.calibration.calibration as c
from awrams.models import awral
from awrams.utils.metatypes import ObjectDict
path = os.path.join(os.path.dirname(__file__), '..', '..', 'test_data',
'calibration')
awral.CLIMATE_DATA = path
cal = c.CalibrationInstance(awral)
print(sys.argv)
if sys.argv[0].endswith('nosetests') or sys.argv[1].endswith('nosetests'):
cal.node_settings.num_workers = 1
cal.num_nodes = 1
cal.termp.max_iter = 40
cal.node_settings.catchment_ids = ['105001', '145003'
] #['4508', '105001'] #, '145003']
cal.node_settings.catchment_extents = pickle.load(
open(os.path.join(path, 'cal_catchment_extents.pkl'), 'rb'))
# cal.node_settings.run_period = pd.date_range("01/01/1950", "31/12/2011")
# cal.node_settings.eval_period = pd.date_range("01/01/1981", "31/12/2011")
cal.node_settings.run_period = pd.date_range("1 jan 2005", "31 dec 2010")
cal.node_settings.eval_period = pd.date_range("1 jan 2005", "31 dec 2010")
cal.node_settings.output_variables = ['qtot', 'etot', 'w0']
awral.set_outputs({
'OUTPUTS_CELL': ['qtot'],
'OUTPUTS_HRU': [],
'OUTPUTS_AVG': ['etot', 'w0']
})
cal.node_settings.observations.qtot = ObjectDict()
cal.node_settings.observations.etot = ObjectDict()
cal.node_settings.observations.w0 = ObjectDict()
cal.node_settings.observations.qtot.source_type = 'csv'
cal.node_settings.observations.etot.source_type = 'csv'
cal.node_settings.observations.w0.source_type = 'csv'
# HostPath is a portable paths API; it allows you specify common bases on multiple systems that are resovled at runtime
cal.node_settings.observations.qtot.filename = os.path.join(
path, 'q_obs.csv')
cal.node_settings.observations.etot.filename = os.path.join(
path, 'cmrset_obs.csv')
cal.node_settings.observations.w0.filename = os.path.join(
path, 'sm_amsre_obs.csv')
# View the localised hostpath...
# cal.node_settings.objective.localf.filename = os.path.join(os.path.dirname(__file__),'objectives','multivar_objectives.py')
cal.node_settings.objective.localf.classname = 'TestLocalMulti'
# cal.node_settings.objective.globalf.filename = os.path.join(os.path.dirname(__file__),'objectives','multivar_objectives.py')
cal.node_settings.objective.globalf.classname = 'GlobalMultiEval'
cal.setup_local()
cal.run_local()
def dimensions_from_georef(gr):
idim = NCDimension(dtype=np.dtype('int32'),
size=None,
data=None,
meta=ObjectDict(units="days since 1900-01-01",
calendar="gregorian",
name="time",
long_name="time",
standard_name='time'))
ydim = NCDimension(
size=gr['nlats'],
dtype=np.dtype('float64'),
meta=ObjectDict(standard_name='latitude',
long_name='latitude',
name='latitude',
units='degrees_north'),
data=gr['lat_origin'] -
gr['cellsize'] * np.arange(gr['nlats'], dtype=np.float64))
xdim = NCDimension(
size=gr['nlons'],
dtype=np.dtype('float64'),
meta=ObjectDict(standard_name='longitude',
long_name='longitude',
name='longitude',
units='degrees_east'),
data=gr['lon_origin'] +
gr['cellsize'] * np.arange(gr['nlons'], dtype=np.float64))
return (idim, ydim, xdim)
def _add_model(self,model_df,name,freq='d'):
self.selection._add(name)
m = ObjectDict(freq=SAMPLE_RATE[freq])
self.models[name] = m
m.name = name
m.data = ObjectDict()
m.obs = ObjectDict()
m.data.raw = model_df.loc[self.period]
if self.obs.freq != 'y':
if freq == 'd' and self.obs.freq == 'd':
m.data.daily,m.obs.daily = self._intersect(m.data.raw, self.obs.data)
m.data.monthly = resample_to_months_df(m.data.daily, self.aggr_how)
m.obs.monthly = resample_to_months_df(m.obs.daily, self.aggr_how)
elif freq == 'm' or self.obs.freq == 'm':
if freq == 'm':
_mod = m.data.raw.resample(rule='m', how=self.aggr_how)
else: # assume must be daily
_mod = resample_to_months_df(m.data.raw, self.aggr_how)
if self.obs.freq == 'm':
_obs = self.obs.data.resample(rule='m', how=self.aggr_how)
else:
_obs = resample_to_months_df(self.obs.data, self.aggr_how)
m.data.monthly,m.obs.monthly = self._intersect(_mod,_obs)
else:
raise Exception('model freq is %s' % repr(freq))
m.data.annual = resample_to_years_df(m.data.monthly, self.aggr_how, min_months=6)
m.obs.annual = resample_to_years_df(m.obs.monthly, self.aggr_how, min_months=6)
m.data.annual,m.obs.annual = self._intersect(m.data.annual,m.obs.annual)
else: #obs are annual (recharge)
m.data.annual = m.data.raw.resample(rule='a', how=self.aggr_how)
m.obs.annual = self.obs.data
m.stats = ObjectDict(freq=freq)
m.stats.daily = None
m.stats.monthly = None
if freq == 'd' and self.obs.freq == 'd':
m.stats.daily = build_stats_df(m.obs.daily, m.data.daily, m.obs.daily.keys())
if self.obs.freq != 'y':
m.stats.monthly = build_stats_df(m.obs.monthly, m.data.monthly, m.obs.monthly.keys())
if m.obs.annual is not None and m.data.annual is not None:
m.stats.annual = build_stats_df(m.obs.annual, m.data.annual, m.obs.annual.keys())
self.build_objfunc_stats(m.stats)
self._assign_colours()
def get_best(self):
index = np.where(self.fh['global_score'][...] ==
self.fh['global_score'][...].min())[0][0]
score = self.fh['global_score'][index]
parameters = dict(
zip(self.parameters, self.fh['parameter_values'][index]))
return ObjectDict(index=index, score=score, parameters=parameters)
def __init__(self,
obs_df,
ref_name,
var_name,
period,
how=np.mean,
annual_rule='A'):
self.period = period
self.var_name = var_name
self.ref_name = ref_name
self.aggr_how = how
self.annual_rule = annual_rule
obs_df = obs_df.loc[(obs_df.index >= period[0])
& (obs_df.index <= period[-1])]
self.obs = New()
self.obs.data = obs_df
self.obs.freq = infer_freq(obs_df)
if self.obs.freq != 'y':
self.obs.monthly = self.obs.data.resample(rule='m',
how=self.aggr_how)
self.obs.annual = self.obs.data.resample(rule='A', how=self.aggr_how)
# self.obs.freq = infer_freq(obs_df)
self.models = ObjectDict()
self._cmap = plt.get_cmap('gist_rainbow')
self.selection = Selector([])
def __init__(self, model):
ns = default_node_settings(model)
self.node_settings = ObjectDict(ns)
# self.hyperp = ObjectDict(complex_sz=43,n_complexes=14,sub_sz=22,n_offspring=1,n_evol=43,min_complexes=1)
self.hyperp = ObjectDict(complex_sz=5,
n_complexes=5,
sub_sz=2,
n_offspring=1,
n_evol=10,
min_complexes=2)
self.num_nodes = 4
self.termp = default_term_params()
self.params = ns['default_params'] #default_cal_params(model)
self.server = None
def load(self, csv_path, id_list=None, convert_units=1.0):
"""
load observed data from csv
:param csv_path: path to csvs containing observations
expects to find csv files: sm_top.csv,sm_shallow.csv,sm_middle.csv,sm_deep.csv,sm_profile.csv
:param id_list: None for comparison of all ids in csv or list of ids for subset
:param convert_units: factor to apply to observations
:return:
"""
self.benchmark = ObjectDict()
for layer in self.layers:
df = self._load_data(os.path.join(csv_path, layer + '.csv'),
id_list, convert_units)
self.benchmark[layer] = ComparisonSet(df,
self.obs_name,
self.var_name,
self.period,
how=self.how,
annual_rule=self.annual_rule)
self.sites = list(self._extents.keys())
self.cfg = {s: self.sites_meta[s] for s in self.sites}
self.add_model = self._add_model
def get_output_nodes(template):
from awrams.utils.nodegraph import nodes
from awrams.utils.metatypes import ObjectDict
from . import ffi_wrapper as fw
outputs = dict((k,template[k]) for k in ['OUTPUTS_HRU','OUTPUTS_AVG','OUTPUTS_CELL'])
mapping = {}
output_vars = []
for v in outputs['OUTPUTS_AVG'] + outputs['OUTPUTS_CELL']:
output_vars.append(v)
for v in outputs['OUTPUTS_HRU']:
output_vars.extend([v+'_sr',v+'_dr'])
for v in output_vars:
mapping[v] = nodes.model_output(v)
return ObjectDict(mapping=ObjectDict(mapping)) #,output_path=output_path)
def get_default_output_mapping(path='./'):
from awrams.utils.nodegraph import nodes
from awrams.utils.metatypes import ObjectDict
#+++ not dealing with sr and dr versions of HRUS
outputs = dict((k,_DT[k]) for k in ['OUTPUTS_HRU','OUTPUTS_AVG','OUTPUTS_CELL'])
mapping = {}
output_vars = []
for v in outputs['OUTPUTS_AVG'] + outputs['OUTPUTS_CELL']:
output_vars.append(v)
for v in outputs['OUTPUTS_HRU']:
output_vars.extend([v+'_sr',v+'_dr'])
for v in output_vars:
mapping[v] = nodes.write_to_ncfile(path,v)
return ObjectDict(mapping=ObjectDict(mapping)) #,output_path=output_path)
def default_term_params():
tp = ObjectDict()
tp.max_shuffle = 1000 # Max shuffling loops
tp.max_iter = 20000 # Max model evaluations
tp.target_score = 1e-8
tp.max_nsni = 5 # Max shuffle without improvement (as defined below)
tp.min_imp = 0.01 # Minimum change required for 'improvement' metric
return tp
def __init__(self, model_version, results_name=None):
self._variables = VariableGroup(self)
self.extent = None
self.period = None
self.name = results_name
self._path = None
self._model_version = model_version
self.parameters = ObjectDict()
self.parameters.spatial = None
self.parameters.landscape = None
def _infer_time_period(filename, results_folder):
filenames = sorted(
glob(os.path.join(results_folder,
_variable_name(filename) + '*.nc')))
start = start_date(managed_dataset(filenames[0], 'r'))
end = end_date(managed_dataset(filenames[-1], 'r'))
freq = dataset_frequency(managed_dataset(filenames[0], 'r'))
return ObjectDict(type=freq,
start=start.strftime('%Y-%m-%d'),
end=end.strftime('%Y-%m-%d'),
representation='YYYY-MM-DD')
def default_node_settings(model):
import imp
import awrams.calibration.objectives.multivar_objectives as w
from awrams.utils.nodegraph import nodes
ns = {}
ns['run_period'] = 'UNSPECIFIED' # the period for which the model is actually run
ns['eval_period'] = 'UNSPECIFIED' # the period over which it is evaluated against observations
from multiprocessing import cpu_count
ns['num_workers'] = 2 #cpu_count()
ns['inputs'] = model.get_default_mapping()
data_path = model.CLIMATE_DATA #'/data/cwd_awra_data/awra_inputs/climate_generated/'
FORCING = {
'tmin': ('temp_min', 'temp_min_day'),
'tmax': ('temp_max', 'temp_max_day'),
'precip': ('rain', 'rain_day'),
'solar': ('solar', 'solar_exposure_day')
}
for k, v in FORCING.items():
ns['inputs'].mapping[k + '_f'] = nodes.forcing_from_ncfiles(
data_path + '/', v[0] + '*', v[1], cache=False)
# Example with single catchment...
ns['catchment_ids'] = []
ns['catchment_extents'] = {}
ns['logfile'] = 'calibration.h5'
# All cal catchments
#ns['catchment_ids'] = [cid.strip() for cid in open('./Catchment_IDs.csv').readlines()[2:]]
from .calibrate import get_parameter_df
ns['default_params'] = get_parameter_df(ns['inputs'].mapping)
ns['observations'] = ObjectDict(qtot=ObjectDict())
ns['observations'].qtot.source_type = 'csv'
ns['observations'].qtot.filename = '/mnt/awramsi_test_data/Calibration/Catchment_Qobs.csv'
ns['objective'] = ObjectDict({
'localf': ObjectDict(),
'globalf': ObjectDict()
})
imp.load_source('lobjf_mod', w.__file__)
ns['objective']['localf']['filename'] = w.__file__
ns['objective']['localf']['classname'] = 'LocalEval'
# Any arguments required by the evaluator are stored in this dict
ns['objective']['localf']['arguments'] = ObjectDict()
# e.g
# ns['objective']['localf']['arguments']['min_valid'] = 15
ns['objective']['globalf']['filename'] = w.__file__
ns['objective']['globalf']['classname'] = 'GlobalMultiEval'
return ns
def _index_results(results_folder):
filenames = _identify_results_files(results_folder)
expected_variables = _identify_variables(filenames)
result = ObjectDict()
result.metadata_from = filenames[0]
result.name = None
result.extent = _infer_extent(result.metadata_from, results_folder)
result.period = _infer_time_period(result.metadata_from, results_folder)
result.variables = expected_variables
result.model_version = _infer_model_version(result.metadata_from,
results_folder)
return result
def _build(self, names):
from functools import partial
def select(n):
if not n in self._sel:
self._sel.append(n)
def unselect(n):
if n in self._sel:
self._sel.remove(n)
for name in names:
self._sel.append(name)
self.__dict__[name] = ObjectDict()
self.__dict__[name]['select'] = partial(select, name)
self.__dict__[name]['unselect'] = partial(unselect, name)
def get_all_settings():
from awrams.utils.metatypes import ObjectDict
import importlib.machinery
import types
HOME = os.path.expanduser('~')
sdict = ObjectDict()
import glob
local_setting_files = glob.glob(os.path.join(HOME, '.awrams/*.py'))
for f in local_setting_files:
submod = os.path.split(f)[-1].split('.')[0]
modname = 'awrams.{submod}.settings'.format(**locals())
try:
mod = importlib.import_module(modname)
sdict[submod] = mod
except:
loader = importlib.machinery.SourceFileLoader(submod, f)
mod = types.ModuleType(loader.name)
loader.exec_module(mod)
sdict[submod] = mod
return sdict
def dict_to_od(d):
'''
Simple JSON hook to make sure we get tab completeable dict objects
'''
return ObjectDict(d)
def __init__(self, **pars):
self.nc_par = ObjectDict(zlib=False) #fill_value=-999., zlib=False)
self.update(pars)
def __init__(self, source, name, units):
self.source = source
self.name = name
self.units = units
self.meta = ObjectDict()
def get_default_mapping():
import json
from awrams.utils.nodegraph import graph, nodes
from awrams.utils.metatypes import ObjectDict
from . import transforms
import numpy as np
dparams = json.load(open(DEFAULT_PARAMETER_FILE,'r'))
#dparams = dict([(k.lower(),v) for k,v in dparams.items()])
for entry in dparams:
entry['MemberName'] = entry['MemberName'].lower()
mapping = {}
# for k,v in dparams.items():
# mapping[k] = nodes.const(v)
for entry in dparams:
tmp = entry.copy()
tmp.pop('MemberName')
tmp.pop('Value')
mapping[entry['MemberName']] = nodes.const(entry['Value'],**tmp)
# Setup a new-style functional input map
import h5py
ds = h5py.File(SPATIAL_FILE,mode='r')
SPATIAL_GRIDS = list(ds['parameters'])
ds.close()
# FORCING = {
# 'tmin': ('tmin*','temp_min_day'),
# 'tmax': ('tmax*','temp_max_day'),
# 'precip': ('rr*','rain_day'),
# 'solar': ('solar*','solar_exposure_day')
# }
FORCING = {
'tmin': ('temp_min*','temp_min_day'),
'tmax': ('temp_max*','temp_max_day'),
'precip': ('rain*','rain_day'),
'solar': ('solar*','solar_exposure_day')
}
for k,v in FORCING.items():
mapping[k+'_f'] = nodes.forcing_from_ncfiles(CLIMATE_DATA,v[0],v[1])
for grid in SPATIAL_GRIDS:
if grid == 'height':
mapping['height'] = nodes.hypso_from_hdf5(SPATIAL_FILE,'parameters/height')
else:
mapping[grid.lower()+'_grid'] = nodes.spatial_from_hdf5(SPATIAL_FILE,'parameters/%s' % grid)
mapping.update({
'tmin': nodes.transform(np.minimum,['tmin_f','tmax_f']),
'tmax': nodes.transform(np.maximum,['tmin_f','tmax_f']),
'hypsperc_f': nodes.const_from_hdf5(SPATIAL_FILE,'dimensions/hypsometric_percentile',['hypsometric_percentile']),
'hypsperc': nodes.mul('hypsperc_f',0.01), # Model needs 0-1.0, file represents as 0-100
'fday': transforms.fday(),
'u2t': transforms.u2t('windspeed_grid','fday')
})
mapping['er_frac_ref_hrusr'] = nodes.mul('er_frac_ref_hrudr',0.5)
mapping['k_rout'] = nodes.transform(transforms.k_rout,('k_rout_scale','k_rout_int','meanpet_grid'))
mapping['k_gw'] = nodes.mul('k_gw_scale','k_gw_grid')
mapping['s0max'] = nodes.mul('s0max_scale','s0fracawc_grid',100.)
mapping['ssmax'] = nodes.mul('ssmax_scale','ssfracawc_grid',900.)
mapping['sdmax'] = nodes.mul('ssmax_scale','sdmax_scale','ssfracawc_grid',5000.)
mapping['k0sat'] = nodes.mul('k0sat_scale','k0sat_v5_grid')
mapping['kssat'] = nodes.mul('kssat_scale','kssat_v5_grid')
mapping['kdsat'] = nodes.mul('kdsat_scale','kdsat_v5_grid')
mapping['kr_0s'] = nodes.transform(transforms.interlayer_k,('k0sat','kssat'))
mapping['kr_sd'] = nodes.transform(transforms.interlayer_k,('kssat','kdsat'))
mapping['prefr'] = nodes.mul('pref_gridscale','pref_grid')
mapping['fhru_hrusr'] = nodes.sub(1.0,'f_tree_grid')
mapping['fhru_hrudr'] = nodes.assign('f_tree_grid')
mapping['ne'] = nodes.mul('ne_scale','ne_grid')
mapping['slope'] = nodes.assign('slope_grid')
mapping['hveg_hrudr'] = nodes.assign('hveg_dr_grid')
mapping['hveg_hrusr'] = nodes.const(0.5)
mapping['laimax_hrusr'] = nodes.assign('lai_max_grid')
mapping['laimax_hrudr'] = nodes.assign('lai_max_grid')
mapping['pair'] = nodes.const(97500.)
mapping['pt'] = nodes.assign('precip_f')
mapping['rgt'] = nodes.transform(np.maximum,['solar_f',0.1])
mapping['tat'] = nodes.mix('tmin','tmax',0.75)
mapping['avpt'] = nodes.transform(transforms.pe,'tmin')
mapping['radcskyt'] = transforms.radcskyt()
mapping['init_sr'] = nodes.const(0.0)
mapping['init_sg'] = nodes.const(100.0)
for hru in ('_hrusr','_hrudr'):
mapping['init_mleaf'+hru] = nodes.div(2.0,'sla'+hru)
for state in ["s0","ss","sd"]:
mapping['init_'+state+hru] = nodes.mul(state+'max',0.5)
# +++dims only required due to having to allocate shared-memory buffer before running...
dims = ObjectDict(hypsometric_percentile=20,latitude=None,longitude=None,time=None)
return ObjectDict(mapping=ObjectDict(mapping),dimensions=dims)