def test_SplitFileWriterNode():
from awrams.utils import extents
from awrams.utils import datetools as dt
import awrams.models.awral.description
awrams.models.awral.description.CLIMATE_DATA = os.path.join(
os.path.dirname(__file__), '..', '..', 'test_data', 'simulation')
from awrams.utils.nodegraph import nodes
from awrams.simulation.ondemand import OnDemandSimulator
from awrams.models import awral
input_map = awral.get_default_mapping()
from awrams.utils.nodegraph import nodes
from awrams.utils.metatypes import ObjectDict
# output_path = './'
mapping = {}
mapping['qtot'] = nodes.write_to_annual_ncfile('./', 'qtot')
output_map = ObjectDict(
mapping=ObjectDict(mapping)) #,output_path=output_path)
runner = OnDemandSimulator(awral,
input_map.mapping,
omapping=output_map.mapping)
period = dt.dates('2010-2011')
extent = extents.from_cell_offset(200, 200)
r = runner.run(period, extent)
def test_output_graph_processing_splitfm_B():
from awrams.utils import extents
from awrams.utils import datetools as dt
import awrams.models.awral.description
awrams.models.awral.description.CLIMATE_DATA = os.path.join(
os.path.dirname(__file__), '..', '..', 'test_data', 'simulation')
from awrams.utils.nodegraph import nodes, graph
from awrams.simulation.ondemand import OnDemandSimulator
from awrams.models import awral
input_map = awral.get_default_mapping()
output_map = {
's0_save':
nodes.write_to_annual_ncfile(os.path.dirname(__file__), 's0', mode='w')
}
# outputs = graph.OutputGraph(output_map)
runner = OnDemandSimulator(awral, input_map.mapping, omapping=output_map)
print("RUNNER NEW (FILES EXISTING): multiple cells, multiple years")
period = dt.dates('2010-2011')
extent = extents.from_boundary_offset(200, 200, 201, 201)
r = runner.run(period, extent)
print("RUNNER OLD (FILES EXISTING): single cell, single year")
period = dt.dates('2015')
extent = extents.from_cell_offset(202, 202)
r = runner.run(period, extent)
def setup():
from os.path import join, dirname
from awrams.utils import datetools as dt
from awrams.utils.nodegraph import nodes, graph
from awrams.models import awral
from awrams.models.awral.template import DEFAULT_TEMPLATE
from awrams.utils.mapping_types import period_to_tc
global period
period = dt.dates('dec 2010 - jan 2011')
global input_map
input_map = awral.get_default_mapping()
change_path_to_forcing(input_map)
global output_map
output_map = awral.get_output_nodes(DEFAULT_TEMPLATE)
global outpath
outpath = join(dirname(__file__), '..', '..', 'test_data', 'simulation',
'outputs')
output_map.mapping['s0_ncsave'] = nodes.write_to_annual_ncfile(
outpath, 's0')
outgraph = graph.OutputGraph(output_map.mapping)
def test_simple_outputs(period,extent,output_mapping=None):
# from awrams.utils.nodegraph import nodes, graph
# from awrams.models.awral import transforms
# import numpy as np
# from awrams.utils import extents
from awrams.utils import mapping_types as mt
from awrams.utils import datetools as dt
# from awrams.utils.messaging import message
from awrams.models import awral
mapping = awral.get_default_mapping()
if output_mapping is None:
from awrams.models.awral.template import DEFAULT_TEMPLATE
output_mapping = awral.get_output_nodes(DEFAULT_TEMPLATE)
### initialise output ncfiles
# coords = [mt.period_to_tc(period)]
# coords.extend(mt.extent_to_spatial_coords(extent))
# o = graph.OutputGraph(output_mapping.mapping)
# o.initialise(coords)
max_dims = {'cell': 32*32,'time': 366}
for k,v in mapping.dimensions.items():
if v is not None:
max_dims[k] = v
# periods = dt.split_period(period,'a')
#chunks = [extents.from_boundary_offset(400,400,431,431),\
# extents.from_boundary_offset(200,200,201,201)]
# from awrams.utils.catchments import subdivide_extent
# #e = extents.default()
# e = extent #.from_boundary_offset(200,200,250,250)
# chunks = subdivide_extent(e,32)
import time
start = time.time()
test_sim(awral,max_dims,mapping,output_mapping,period,extent) #periods,chunks)
end = time.time()
print(end-start)
def test_OutputNode():
import awrams.models.awral.description
awrams.models.awral.description.CLIMATE_DATA = os.path.join(
os.path.dirname(__file__), '..', '..', 'test_data', 'simulation')
from awrams.simulation.ondemand import OnDemandSimulator
from awrams.models import awral
input_map = awral.get_default_mapping()
output_map = awral.get_default_output_mapping()
runner = OnDemandSimulator(awral,
input_map.mapping,
omapping=output_map.mapping)
print(runner.outputs)
def __init__(self, len_period, catchments, inputs, outputs):
self.catchments = catchments
self.inputs = inputs
self.outputs = outputs
self.params = None
self.len_period = len_period
from awrams.models import awral
imap = awral.get_default_mapping()
dspec = {}
for k in 'tmin_f', 'tmax_f', 'precip_f', 'solar_f', 'fday', 'radcskyt':
dspec[k] = ['time', 'latitude', 'longitude']
sp = [k for k in imap.mapping if k.endswith('_grid')]
for k in sp:
dspec[k] = ['latitude', 'longitude']
dspec['height'] = ['hypsometric_percentile', 'latitude', 'longitude']
dspec['hypsperc_f'] = ['hypsometric_percentile']
data_map = {k: {} for k in dspec}
for cid in self.catchments:
input = shm_to_nd_dict_inputs(**(self.inputs[cid]))
for k in dspec:
data_map[k][cid] = input[k]
for k in dspec:
imap.mapping[k] = forcing_from_dict(data_map[k], k, dims=dspec[k])
self.igraph = graph.ExecutionGraph(imap.mapping)
self.sim = awral.get_runner(self.igraph.get_dataspecs(True))
def get_task_specification():
from os.path import join
from awrams.utils import gis
from awrams.utils import datetools as dt
from awrams.utils import extents
from awrams.utils.settings_manager import get_settings
paths = get_settings('data_paths')
cal_extents = join(paths.CATCH_PATH, 'calibration_extents_5k.nc')
nces = gis.ExtentStoreNC(cal_extents, 'r')
catch_list_fn = join(paths.CATCH_PATH, 'Catchment_IDs.csv')
catch_df = open(catch_list_fn).readlines()
cids = [k.strip().zfill(6) for k in catch_df[1:]]
extent_map = {}
extent_map = dict([(e, nces[e]) for e in cids])
print([e.cell_count for e in extent_map.values()])
run_period = dt.dates('1950 - 2011')
eval_period = dt.dates('1970 - 2011')
from awrams.calibration.sensitivity import SobolOptimizer
#evolver_spec = EvolverSpec(sce.CCEvolver,evolver_run_args=dict(n_offspring=1,n_evolutions=5,elitism=2.0))
#optimizer_spec = OptimizerSpec(sce.ShuffledOptimizer,evolver_spec=evolver_spec,n_complexes=14,max_nsni=1e100) #n_complex 14
optimizer_spec = OptimizerSpec(SobolOptimizer,
threshold=0.005,
max_eval=25000)
from awrams.utils.nodegraph.nodes import callable_to_funcspec
from awrams.calibration.objectives import test_objectives as tobj
#local_objfspec = ObjectiveFunctionSpec(tobj.TestLocalSingle)
#global_objfspec = callable_to_funcspec(tobj.TestGlobalSingle)
observations = dict(qtot=join(paths.OBS_PATH, 'qobs_zfill.csv'))
local_objfspec = ObjectiveFunctionSpec(tobj.LocalQTotal)
global_objfspec = callable_to_funcspec(tobj.GlobalQTotal)
objective_spec = ObjectiveSpec(global_objfspec, local_objfspec,
observations, eval_period)
from awrams.models import awral
from awrams.utils.nodegraph import nodes, graph
node_mapping = awral.get_default_mapping()['mapping']
model = callable_to_funcspec(awral)
from awrams.models.settings import CLIMATOLOGY
node_mapping['solar_clim'] = nodes.monthly_climatology(
*CLIMATOLOGY['solar'])
node_mapping['solar_filled'] = nodes.gap_filler('solar_f', 'solar_clim')
node_mapping['rgt'].inputs[0] = 'solar_filled'
'''
User specifiable calibration description
'''
cal_spec = {}
cal_spec['optimizer_spec'] = optimizer_spec
cal_spec['objective_spec'] = objective_spec
cal_spec['extent_map'] = extent_map
cal_spec['run_period'] = run_period
cal_spec['model'] = model
cal_spec['node_mapping'] = node_mapping
cal_spec['logfile'] = '/short/er4/dss548/test_sens.h5'
cal_spec['prerun_action'] = callable_to_funcspec(prerun_raijin)
cal_spec['postrun_action'] = callable_to_funcspec(postrun_raijin)
return cal_spec