def change_path_to_forcing(imap):
from awrams.utils.nodegraph.nodes import forcing_from_ncfiles
from os.path import join, dirname
data_path = join(dirname(__file__), '..', '..', 'test_data', 'simulation')
imap['precip_f'] = forcing_from_ncfiles(data_path, "rain*", 'rain_day')
imap['tmax_f'] = forcing_from_ncfiles(data_path, "temp_max*",
'temp_max_day')
imap['tmin_f'] = forcing_from_ncfiles(data_path, "temp_min*",
'temp_min_day')
imap['solar_f'] = forcing_from_ncfiles(data_path, "solar*",
'solar_exposure_day')
def default_node_settings(model):
import imp
import awrams.calibration.objectives.multivar_objectives as w
from awrams.utils.nodegraph import nodes
ns = {}
ns['run_period'] = 'UNSPECIFIED' # the period for which the model is actually run
ns['eval_period'] = 'UNSPECIFIED' # the period over which it is evaluated against observations
from multiprocessing import cpu_count
ns['num_workers'] = 2 #cpu_count()
ns['inputs'] = model.get_default_mapping()
data_path = model.CLIMATE_DATA #'/data/cwd_awra_data/awra_inputs/climate_generated/'
FORCING = {
'tmin': ('temp_min', 'temp_min_day'),
'tmax': ('temp_max', 'temp_max_day'),
'precip': ('rain', 'rain_day'),
'solar': ('solar', 'solar_exposure_day')
}
for k, v in FORCING.items():
ns['inputs'].mapping[k + '_f'] = nodes.forcing_from_ncfiles(
data_path + '/', v[0] + '*', v[1], cache=False)
# Example with single catchment...
ns['catchment_ids'] = []
ns['catchment_extents'] = {}
ns['logfile'] = 'calibration.h5'
# All cal catchments
#ns['catchment_ids'] = [cid.strip() for cid in open('./Catchment_IDs.csv').readlines()[2:]]
from .calibrate import get_parameter_df
ns['default_params'] = get_parameter_df(ns['inputs'].mapping)
ns['observations'] = ObjectDict(qtot=ObjectDict())
ns['observations'].qtot.source_type = 'csv'
ns['observations'].qtot.filename = '/mnt/awramsi_test_data/Calibration/Catchment_Qobs.csv'
ns['objective'] = ObjectDict({
'localf': ObjectDict(),
'globalf': ObjectDict()
})
imp.load_source('lobjf_mod', w.__file__)
ns['objective']['localf']['filename'] = w.__file__
ns['objective']['localf']['classname'] = 'LocalEval'
# Any arguments required by the evaluator are stored in this dict
ns['objective']['localf']['arguments'] = ObjectDict()
# e.g
# ns['objective']['localf']['arguments']['min_valid'] = 15
ns['objective']['globalf']['filename'] = w.__file__
ns['objective']['globalf']['classname'] = 'GlobalMultiEval'
return ns
def get_default_mapping(self):
'''
Load the default (NodeGraph) mapping for this model
'''
from awrams.utils.nodegraph import nodes
from awrams.models.settings import CLIMATE_DATA
mapping = {}
mapping['c'] = nodes.parameter(0.5, 0.0, 1.0, False)
mapping['precip'] = nodes.forcing_from_ncfiles(CLIMATE_DATA, 'rr_*',
'rain_day')
return mapping
def change_path_to_forcing(imap):
from awrams.utils.nodegraph import nodes
# path = os.path.join(os.path.dirname(__file__),'../../test_data')
FORCING = {
'tmin': ('temp_min*', 'temp_min_day'),
'tmax': ('temp_max*', 'temp_max_day'),
'precip': ('rain_day*', 'rain_day'),
'solar': ('solar*', 'solar_exposure_day')
}
for k, v in FORCING.items():
imap.mapping[k + '_f'] = nodes.forcing_from_ncfiles(path,
v[0],
v[1],
cache=True)
def get_input_mapping(model_settings=None):
"""
Return the default input mapping for this model
This is a dict of key:GraphNode mappings
Return:
mapping (dict)
"""
if model_settings is None:
model_settings = get_settings()
mapping = {}
#for k,spec in config_options['PARAMETER_SPECS'].items():
# mapping[k] = nodes.parameter_from_json(
# model_settings['PARAMETER_FILE'], k, spec[0],spec[1],spec[2])
param_df = parameters.wirada_json_to_param_df(model_settings['PARAMETER_FILE'])
mapping = parameters.param_df_to_mapping(param_df,mapping)
SPATIAL_GRIDS = ['f_tree', 'height', 'hveg_dr', 'k0sat_v5', 'k_gw', 'kdsat_v5', 'kssat_v5', 'lai_max',
'meanPET', 'ne', 'pref', 's0fracAWC', 'slope', 'ssfracAWC', 'windspeed']
for grid in SPATIAL_GRIDS:
mapping[grid.lower()+'_grid'] = nodes.spatial_from_file(
model_settings['SPATIAL_FILE'], 'parameters/%s' % grid)
FORCING_DATA = model_settings['CLIMATE_DATASET']['FORCING']
CLIMATOLOGY = model_settings['CLIMATE_DATASET']['CLIMATOLOGY']
for k in ['tmin', 'tmax', 'precip']:
var_map = FORCING_DATA['MAPPING'][k]
mapping[k+'_f'] = nodes.forcing_from_ncfiles(FORCING_DATA['PATH'], var_map[0], var_map[1])
mapping['solar_f'] = nodes.forcing_gap_filler(FORCING_DATA['PATH'],FORCING_DATA['MAPPING']['solar'][0], \
FORCING_DATA['MAPPING']['solar'][1],CLIMATOLOGY['solar'][0])
mapping.update({
'tmin': nodes.transform(np.minimum, ['tmin_f', 'tmax_f']),
'tmax': nodes.transform(np.maximum, ['tmin_f', 'tmax_f']),
'hypsperc_f': nodes.const_from_hdf5(model_settings['SPATIAL_FILE'], 'dimensions/hypsometric_percentile', ['hypsometric_percentile']),
# Model needs 0-1.0, file represents as 0-100
'hypsperc': nodes.mul('hypsperc_f', 0.01),
'fday': transforms.fday(),
'u2t': transforms.u2t('windspeed_grid','fday')
})
mapping['height'] = nodes.assign('height_grid')
mapping['er_frac_ref_hrusr'] = nodes.mul('er_frac_ref_hrudr', 0.5)
mapping['k_rout'] = nodes.transform(
transforms.k_rout, ('k_rout_scale', 'k_rout_int', 'meanpet_grid'))
mapping['k_gw'] = nodes.mul('k_gw_scale', 'k_gw_grid')
mapping['s0max'] = nodes.mul('s0max_scale', 's0fracawc_grid', 100.)
mapping['ssmax'] = nodes.mul('ssmax_scale', 'ssfracawc_grid', 900.)
mapping['sdmax'] = nodes.mul('ssmax_scale','sdmax_scale','ssfracawc_grid',5000.)
mapping['k0sat'] = nodes.mul('k0sat_scale', 'k0sat_v5_grid')
mapping['kssat'] = nodes.mul('kssat_scale', 'kssat_v5_grid')
mapping['kdsat'] = nodes.mul('kdsat_scale', 'kdsat_v5_grid')
mapping['kr_0s'] = nodes.transform(
transforms.interlayer_k, ('k0sat', 'kssat'))
mapping['kr_sd'] = nodes.transform(
transforms.interlayer_k, ('kssat', 'kdsat'))
mapping['prefr'] = nodes.mul('pref_gridscale', 'pref_grid')
mapping['fhru_hrusr'] = nodes.sub(1.0, 'f_tree_grid')
mapping['fhru_hrudr'] = nodes.assign('f_tree_grid')
mapping['ne'] = nodes.mul('ne_scale', 'ne_grid')
mapping['slope'] = nodes.assign('slope_grid')
mapping['hveg_hrudr'] = nodes.assign('hveg_dr_grid')
mapping['laimax_hrusr'] = nodes.assign('lai_max_grid')
mapping['laimax_hrudr'] = nodes.assign('lai_max_grid')
mapping['pair'] = nodes.const(97500.)
mapping['pt'] = nodes.assign('precip_f')
mapping['rgt'] = nodes.transform(np.maximum, ['solar_f', 0.1])
mapping['tat'] = nodes.mix('tmin', 'tmax', 0.75)
mapping['avpt'] = nodes.transform(transforms.pe, 'tmin')
mapping['radcskyt'] = transforms.radcskyt()
mapping['init_sr'] = nodes.const(0.0)
mapping['init_sg'] = nodes.const(100.0)
for hru in ('_hrusr', '_hrudr'):
mapping['init_mleaf'+hru] = nodes.div(2.0, 'sla'+hru)
for state in ["s0", "ss", "sd"]:
mapping['init_'+state+hru] = nodes.mul(state+'max', 0.5)
return objectify(mapping)
def get_default_mapping():
import json
from awrams.utils.nodegraph import graph, nodes
from awrams.utils.metatypes import ObjectDict
from . import transforms
import numpy as np
dparams = json.load(open(DEFAULT_PARAMETER_FILE,'r'))
#dparams = dict([(k.lower(),v) for k,v in dparams.items()])
for entry in dparams:
entry['MemberName'] = entry['MemberName'].lower()
mapping = {}
# for k,v in dparams.items():
# mapping[k] = nodes.const(v)
for entry in dparams:
tmp = entry.copy()
tmp.pop('MemberName')
tmp.pop('Value')
mapping[entry['MemberName']] = nodes.const(entry['Value'],**tmp)
# Setup a new-style functional input map
import h5py
ds = h5py.File(SPATIAL_FILE,mode='r')
SPATIAL_GRIDS = list(ds['parameters'])
ds.close()
# FORCING = {
# 'tmin': ('tmin*','temp_min_day'),
# 'tmax': ('tmax*','temp_max_day'),
# 'precip': ('rr*','rain_day'),
# 'solar': ('solar*','solar_exposure_day')
# }
FORCING = {
'tmin': ('temp_min*','temp_min_day'),
'tmax': ('temp_max*','temp_max_day'),
'precip': ('rain*','rain_day'),
'solar': ('solar*','solar_exposure_day')
}
for k,v in FORCING.items():
mapping[k+'_f'] = nodes.forcing_from_ncfiles(CLIMATE_DATA,v[0],v[1])
for grid in SPATIAL_GRIDS:
if grid == 'height':
mapping['height'] = nodes.hypso_from_hdf5(SPATIAL_FILE,'parameters/height')
else:
mapping[grid.lower()+'_grid'] = nodes.spatial_from_hdf5(SPATIAL_FILE,'parameters/%s' % grid)
mapping.update({
'tmin': nodes.transform(np.minimum,['tmin_f','tmax_f']),
'tmax': nodes.transform(np.maximum,['tmin_f','tmax_f']),
'hypsperc_f': nodes.const_from_hdf5(SPATIAL_FILE,'dimensions/hypsometric_percentile',['hypsometric_percentile']),
'hypsperc': nodes.mul('hypsperc_f',0.01), # Model needs 0-1.0, file represents as 0-100
'fday': transforms.fday(),
'u2t': transforms.u2t('windspeed_grid','fday')
})
mapping['er_frac_ref_hrusr'] = nodes.mul('er_frac_ref_hrudr',0.5)
mapping['k_rout'] = nodes.transform(transforms.k_rout,('k_rout_scale','k_rout_int','meanpet_grid'))
mapping['k_gw'] = nodes.mul('k_gw_scale','k_gw_grid')
mapping['s0max'] = nodes.mul('s0max_scale','s0fracawc_grid',100.)
mapping['ssmax'] = nodes.mul('ssmax_scale','ssfracawc_grid',900.)
mapping['sdmax'] = nodes.mul('ssmax_scale','sdmax_scale','ssfracawc_grid',5000.)
mapping['k0sat'] = nodes.mul('k0sat_scale','k0sat_v5_grid')
mapping['kssat'] = nodes.mul('kssat_scale','kssat_v5_grid')
mapping['kdsat'] = nodes.mul('kdsat_scale','kdsat_v5_grid')
mapping['kr_0s'] = nodes.transform(transforms.interlayer_k,('k0sat','kssat'))
mapping['kr_sd'] = nodes.transform(transforms.interlayer_k,('kssat','kdsat'))
mapping['prefr'] = nodes.mul('pref_gridscale','pref_grid')
mapping['fhru_hrusr'] = nodes.sub(1.0,'f_tree_grid')
mapping['fhru_hrudr'] = nodes.assign('f_tree_grid')
mapping['ne'] = nodes.mul('ne_scale','ne_grid')
mapping['slope'] = nodes.assign('slope_grid')
mapping['hveg_hrudr'] = nodes.assign('hveg_dr_grid')
mapping['hveg_hrusr'] = nodes.const(0.5)
mapping['laimax_hrusr'] = nodes.assign('lai_max_grid')
mapping['laimax_hrudr'] = nodes.assign('lai_max_grid')
mapping['pair'] = nodes.const(97500.)
mapping['pt'] = nodes.assign('precip_f')
mapping['rgt'] = nodes.transform(np.maximum,['solar_f',0.1])
mapping['tat'] = nodes.mix('tmin','tmax',0.75)
mapping['avpt'] = nodes.transform(transforms.pe,'tmin')
mapping['radcskyt'] = transforms.radcskyt()
mapping['init_sr'] = nodes.const(0.0)
mapping['init_sg'] = nodes.const(100.0)
for hru in ('_hrusr','_hrudr'):
mapping['init_mleaf'+hru] = nodes.div(2.0,'sla'+hru)
for state in ["s0","ss","sd"]:
mapping['init_'+state+hru] = nodes.mul(state+'max',0.5)
# +++dims only required due to having to allocate shared-memory buffer before running...
dims = ObjectDict(hypsometric_percentile=20,latitude=None,longitude=None,time=None)
return ObjectDict(mapping=ObjectDict(mapping),dimensions=dims)
def get_default_mapping(self):
"""
Return the default input mapping for this model
This is a dict of key:GraphNode mappings
Return:
mapping (dict)
"""
import json
from awrams.utils.nodegraph import graph, nodes
from awrams.utils.metatypes import PrettyObjectDict
from . import transforms
import numpy as np
dparams = json.load(open(DEFAULT_PARAMETER_FILE, 'r'))
#dparams = dict([(k.lower(),v) for k,v in dparams.items()])
for entry in dparams:
entry['MemberName'] = entry['MemberName'].lower()
mapping = {}
# for k,v in dparams.items():
# mapping[k] = nodes.const(v)
for entry in dparams:
#tmp = entry.copy()
#tmp.pop('MemberName')
#tmp.pop('Value')
mapping[entry['MemberName']] = nodes.parameter(
entry['Value'],
entry['Min'],
entry['Max'],
entry['Fixed'],
description=entry['DisplayName'])
# Setup a new-style functional input map
import h5py
ds = h5py.File(SPATIAL_FILE, mode='r')
SPATIAL_GRIDS = list(ds['parameters'])
ds.close()
for k, v in FORCING.items():
mapping[k + '_f'] = nodes.forcing_from_ncfiles(
CLIMATE_DATA, v[0], v[1])
for grid in SPATIAL_GRIDS:
#if grid == 'height':
# mapping['height'] = nodes.hypso_from_hdf5(SPATIAL_FILE,'parameters/height')
#else:
mapping[grid.lower() + '_grid'] = nodes.spatial_from_file(
SPATIAL_FILE, 'parameters/%s' % grid)
mapping.update({
'tmin':
nodes.transform(np.minimum, ['tmin_f', 'tmax_f']),
'tmax':
nodes.transform(np.maximum, ['tmin_f', 'tmax_f']),
'hypsperc_f':
nodes.const_from_hdf5(SPATIAL_FILE,
'dimensions/hypsometric_percentile',
['hypsometric_percentile']),
'hypsperc':
nodes.mul('hypsperc_f',
0.01), # Model needs 0-1.0, file represents as 0-100
'fday':
transforms.fday(),
'u2t':
transforms.u2t('windspeed_grid', 'fday')
})
mapping['height'] = nodes.assign('height_grid')
mapping['er_frac_ref_hrusr'] = nodes.mul('er_frac_ref_hrudr', 0.5)
mapping['k_rout'] = nodes.transform(
transforms.k_rout, ('k_rout_scale', 'k_rout_int', 'meanpet_grid'))
mapping['k_gw'] = nodes.mul('k_gw_scale', 'k_gw_grid')
mapping['s0max'] = nodes.mul('s0max_scale', 's0fracawc_grid', 100.)
mapping['ssmax'] = nodes.mul('ssmax_scale', 'ssfracawc_grid', 900.)
mapping['sdmax'] = nodes.mul('ssmax_scale', 'sdmax_scale',
'ssfracawc_grid', 5000.)
mapping['k0sat'] = nodes.mul('k0sat_scale', 'k0sat_v5_grid')
mapping['kssat'] = nodes.mul('kssat_scale', 'kssat_v5_grid')
mapping['kdsat'] = nodes.mul('kdsat_scale', 'kdsat_v5_grid')
mapping['kr_0s'] = nodes.transform(transforms.interlayer_k,
('k0sat', 'kssat'))
mapping['kr_sd'] = nodes.transform(transforms.interlayer_k,
('kssat', 'kdsat'))
mapping['prefr'] = nodes.mul('pref_gridscale', 'pref_grid')
mapping['fhru_hrusr'] = nodes.sub(1.0, 'f_tree_grid')
mapping['fhru_hrudr'] = nodes.assign('f_tree_grid')
mapping['ne'] = nodes.mul('ne_scale', 'ne_grid')
mapping['slope'] = nodes.assign('slope_grid')
mapping['hveg_hrudr'] = nodes.assign('hveg_dr_grid')
mapping['laimax_hrusr'] = nodes.assign('lai_max_grid')
mapping['laimax_hrudr'] = nodes.assign('lai_max_grid')
mapping['pair'] = nodes.const(97500.)
mapping['pt'] = nodes.assign('precip_f')
mapping['rgt'] = nodes.transform(np.maximum, ['solar_f', 0.1])
mapping['tat'] = nodes.mix('tmin', 'tmax', 0.75)
mapping['avpt'] = nodes.transform(transforms.pe, 'tmin')
mapping['radcskyt'] = transforms.radcskyt()
mapping['init_sr'] = nodes.const(0.0)
mapping['init_sg'] = nodes.const(100.0)
for hru in ('_hrusr', '_hrudr'):
mapping['init_mleaf' + hru] = nodes.div(2.0, 'sla' + hru)
for state in ["s0", "ss", "sd"]:
mapping['init_' + state + hru] = nodes.mul(state + 'max', 0.5)
return PrettyObjectDict(mapping)