def build_pickle_from_spec(cal_spec,
max_nodes,
pickle_fn,
core_min=1,
max_over=1.02):
n_cores = settings.CORES_PER_NODE * core_min
node_alloc, catch_node_map = allocate_catchments_to_nodes(
cal_spec['extent_map'], max_nodes, n_cores, max_over=max_over)
n_workers = len(node_alloc)
cal_spec = cal_spec.copy()
for k in ['prerun_action', 'postrun_action']:
calldef = cal_spec.get(k)
if calldef is not None:
if isinstance(calldef, dict):
pass
else:
cal_spec[k] = callable_to_funcspec(calldef)
cal_spec['node_alloc'] = node_alloc
cal_spec['catch_node_map'] = catch_node_map
cal_spec['n_workers'] = n_workers
import pickle
with open(pickle_fn, 'wb') as pkl_out:
pickle.dump(cal_spec, pkl_out)
return cal_spec
def set_fast_forcing(node_mapping):
from awrams.utils.io.data_mapping import map_filename_annual
forcing_nodes = [
k for k, v in node_mapping.items()
if v.node_type == 'forcing_from_ncfiles'
]
for f in forcing_nodes:
node_mapping[f].args['map_func'] = nodes.callable_to_funcspec(
map_filename_annual)
def build_pickle_from_spec(cal_spec, cores_per_node, max_nodes, pickle_fn,
core_min=1, max_over=1.02):
"""Build a fully realised pickle file from the supplied cal_spec
Args:
cal_spec (dict): Calibration spec as built in example notebooks
cores_per_node (int): Number of cores to use on each (machine) node
max_nodes (int): Maximum number of (machine) nodes to allocate
pickle_fn (str): Output filename for generated pickle file
core_min (int, optional): Minimum number of cells per core
max_over (float, optional): Scaling factor for maximum cells/node
Returns:
dict: Calibration spec dict updated with node allocation information
"""
node_alloc, catch_node_map = allocate_catchments_to_nodes(
cal_spec['extent_map'], max_nodes, cores_per_node, max_over=max_over)
n_workers = len(node_alloc)
cal_spec = cal_spec.copy()
for k in ['prerun_action', 'postrun_action']:
calldef = cal_spec.get(k)
if calldef is not None:
if isinstance(calldef, dict):
pass
else:
cal_spec[k] = callable_to_funcspec(calldef)
cal_spec['node_alloc'] = node_alloc
cal_spec['catch_node_map'] = catch_node_map
cal_spec['n_workers'] = n_workers
cal_spec['n_sub_workers'] = cores_per_node
with open(pickle_fn, 'wb') as pkl_out:
pickle.dump(cal_spec, pkl_out)
return cal_spec
def get_task_specification():
from os.path import join
from awrams.utils import gis
from awrams.utils import datetools as dt
from awrams.utils import extents
from awrams.utils.settings_manager import get_settings
paths = get_settings('data_paths')
cal_extents = join(paths.CATCH_PATH, 'calibration_extents_5k.nc')
nces = gis.ExtentStoreNC(cal_extents, 'r')
catch_list_fn = join(paths.CATCH_PATH, 'Catchment_IDs.csv')
catch_df = open(catch_list_fn).readlines()
cids = [k.strip().zfill(6) for k in catch_df[1:]]
extent_map = {}
extent_map = dict([(e, nces[e]) for e in cids])
print([e.cell_count for e in extent_map.values()])
run_period = dt.dates('1950 - 2011')
eval_period = dt.dates('1970 - 2011')
from awrams.calibration.sensitivity import SobolOptimizer
#evolver_spec = EvolverSpec(sce.CCEvolver,evolver_run_args=dict(n_offspring=1,n_evolutions=5,elitism=2.0))
#optimizer_spec = OptimizerSpec(sce.ShuffledOptimizer,evolver_spec=evolver_spec,n_complexes=14,max_nsni=1e100) #n_complex 14
optimizer_spec = OptimizerSpec(SobolOptimizer,
threshold=0.005,
max_eval=25000)
from awrams.utils.nodegraph.nodes import callable_to_funcspec
from awrams.calibration.objectives import test_objectives as tobj
#local_objfspec = ObjectiveFunctionSpec(tobj.TestLocalSingle)
#global_objfspec = callable_to_funcspec(tobj.TestGlobalSingle)
observations = dict(qtot=join(paths.OBS_PATH, 'qobs_zfill.csv'))
local_objfspec = ObjectiveFunctionSpec(tobj.LocalQTotal)
global_objfspec = callable_to_funcspec(tobj.GlobalQTotal)
objective_spec = ObjectiveSpec(global_objfspec, local_objfspec,
observations, eval_period)
from awrams.models import awral
from awrams.utils.nodegraph import nodes, graph
node_mapping = awral.get_default_mapping()['mapping']
model = callable_to_funcspec(awral)
from awrams.models.settings import CLIMATOLOGY
node_mapping['solar_clim'] = nodes.monthly_climatology(
*CLIMATOLOGY['solar'])
node_mapping['solar_filled'] = nodes.gap_filler('solar_f', 'solar_clim')
node_mapping['rgt'].inputs[0] = 'solar_filled'
'''
User specifiable calibration description
'''
cal_spec = {}
cal_spec['optimizer_spec'] = optimizer_spec
cal_spec['objective_spec'] = objective_spec
cal_spec['extent_map'] = extent_map
cal_spec['run_period'] = run_period
cal_spec['model'] = model
cal_spec['node_mapping'] = node_mapping
cal_spec['logfile'] = '/short/er4/dss548/test_sens.h5'
cal_spec['prerun_action'] = callable_to_funcspec(prerun_raijin)
cal_spec['postrun_action'] = callable_to_funcspec(postrun_raijin)
return cal_spec
def __init__(self, global_objf, catchment_objf, observations, eval_period):
#+++ Possibly require separate obs for catchment, local...
self._global_objf = callable_to_funcspec(global_objf)
self.catchment_objf = catchment_objf
self.observations = observations #+++ Not all objectives require obs...
self.eval_period = eval_period
def __init__(self, objf_class, init_args=None):
self.funcspec = callable_to_funcspec(objf_class)
if init_args is None: init_args = {}
self.init_args = init_args
self.inputs_from_model = objf_class.input_schema
self.output_schema = objf_class.output_schema