def test_output_graph_processing_splitfm_B():
from awrams.utils import extents
from awrams.utils import datetools as dt
import awrams.models.awral.description
awrams.models.awral.description.CLIMATE_DATA = os.path.join(
os.path.dirname(__file__), '..', '..', 'test_data', 'simulation')
from awrams.utils.nodegraph import nodes, graph
from awrams.simulation.ondemand import OnDemandSimulator
from awrams.models import awral
input_map = awral.get_default_mapping()
output_map = {
's0_save':
nodes.write_to_annual_ncfile(os.path.dirname(__file__), 's0', mode='w')
}
# outputs = graph.OutputGraph(output_map)
runner = OnDemandSimulator(awral, input_map.mapping, omapping=output_map)
print("RUNNER NEW (FILES EXISTING): multiple cells, multiple years")
period = dt.dates('2010-2011')
extent = extents.from_boundary_offset(200, 200, 201, 201)
r = runner.run(period, extent)
print("RUNNER OLD (FILES EXISTING): single cell, single year")
period = dt.dates('2015')
extent = extents.from_cell_offset(202, 202)
r = runner.run(period, extent)
def test_SplitFileWriterNode():
from awrams.utils import extents
from awrams.utils import datetools as dt
import awrams.models.awral.description
awrams.models.awral.description.CLIMATE_DATA = os.path.join(
os.path.dirname(__file__), '..', '..', 'test_data', 'simulation')
from awrams.utils.nodegraph import nodes
from awrams.simulation.ondemand import OnDemandSimulator
from awrams.models import awral
input_map = awral.get_default_mapping()
from awrams.utils.nodegraph import nodes
from awrams.utils.metatypes import ObjectDict
# output_path = './'
mapping = {}
mapping['qtot'] = nodes.write_to_annual_ncfile('./', 'qtot')
output_map = ObjectDict(
mapping=ObjectDict(mapping)) #,output_path=output_path)
runner = OnDemandSimulator(awral,
input_map.mapping,
omapping=output_map.mapping)
period = dt.dates('2010-2011')
extent = extents.from_cell_offset(200, 200)
r = runner.run(period, extent)
def test_ondemand_region_v5():
### test region
extent = extents.get_default_extent()
extent = extent.icoords[-32:-34,115:118]
from awrams.simulation.ondemand import OnDemandSimulator
sim = OnDemandSimulator(awral,input_map,omapping=output_map)
r = sim.run(period,extent)
def test_ondemand_point():
### test point
extent = extents.get_default_extent()
extent = extent.icoords[-32.1,115.1]
from awrams.simulation.ondemand import OnDemandSimulator
sim = OnDemandSimulator(awral,input_map,omapping=output_map)
r = sim.run(period,extent)
def test_output_graph_processing_splitfm_B():
from awrams.utils import extents
from awrams.utils import datetools as dt
e_all = extents.get_default_extent()
from awrams.utils.nodegraph import nodes, graph
from awrams.simulation.ondemand import OnDemandSimulator
input_map = awral.get_default_mapping()
climate_mod(input_map)
output_map = awral.get_output_mapping()
output_map['s0_save'] = nodes.write_to_annual_ncfile(
os.path.dirname(__file__), 's0', mode='w')
runner = OnDemandSimulator(awral, input_map, omapping=output_map)
print("RUNNER NEW (FILES EXISTING): multiple cells, multiple years")
period = dt.dates('2010-2011')
extent = e_all.ioffset[200:202, 200:202]
r = runner.run(period, extent)
clear_files()
print("RUNNER OLD (FILES EXISTING): single cell, single year")
period = dt.dates('2015')
extent = e_all.ioffset[202, 202]
r = runner.run(period, extent)
def test_ondemand_point():
from awrams.utils import extents
from awrams.simulation.ondemand import OnDemandSimulator
from awrams.models import awral
### test point
extent = extents.from_cell_coords(-32.1, 115.1)
sim = OnDemandSimulator(awral,
input_map.mapping,
omapping=output_map.mapping)
r = sim.run(period, extent)
def test_climatology_region():
### test region
period = dt.dates('dec 2010')
extent = extents.get_default_extent()
extent = extent.icoords[-32:-35,115:118]
from awrams.simulation.ondemand import OnDemandSimulator
sim = OnDemandSimulator(awral,input_map)
r,i = sim.run(period,extent,return_inputs=True)
assert not np.isnan(i['solar_f']).any()
def test_ondemand_region():
from awrams.utils import extents
from awrams.simulation.ondemand import OnDemandSimulator
from awrams.models import awral
### test region
extent = extents.from_boundary_coords(-32, 115, -35, 118)
sim = OnDemandSimulator(awral,
input_map.mapping,
omapping=output_map.mapping)
r = sim.run(period, extent)
def test_initial_states_point():
import numpy as np
from awrams.utils import extents
from awrams.utils import datetools as dt
from awrams.utils.nodegraph import nodes, graph
from awrams.simulation.ondemand import OnDemandSimulator
#from awrams.models import awral
period = dt.dates('dec 2010')
### test a single cell
extent = extents.get_default_extent()
extent = extent.icoords[-30, 120.5]
### simulation with default initial states
sim = OnDemandSimulator(awral, input_map)
r, i = sim.run(period, extent, return_inputs=True)
outputs_default = r['final_states']
### simulation with initial states read from nc files
get_initial_states(input_map)
sim = OnDemandSimulator(awral, input_map)
r, i = sim.run(period, extent, return_inputs=True)
outputs_init = r['final_states']
### compare final states with default states simulation
### should be different
for k, o in outputs_init.items():
assert not o == outputs_default[k]
### save initial states to compare
ini_states = {}
for k in i:
try:
if k.startswith('init'):
ini_states[k] = i[k]
except:
pass
### simulation with initial states read from dict
get_initial_states_dict(input_map, period, extent)
sim = OnDemandSimulator(awral, input_map)
r, i = sim.run(period, extent, return_inputs=True)
outputs_init_dict = r['final_states']
### compare final states with other ini states simulation
### should be same
for k, o in outputs_init_dict.items():
assert o == outputs_init[k]
### compare initial states from both methods
### should be same
for k in i:
try:
if k.startswith('init'):
assert ini_states[k] == i[k]
except:
pass
def __init__(self, period, extent, observations):
self.period = period
self.extent = extent
self.obs = observations
self.evaluations = 0
logger.info("initialising model simulator...")
self.ods = OnDemandSimulator(MODEL, input_map.mapping)
self.initial_results, self.inputs = self.ods.run(period,
extent,
return_inputs=True)
logger.info("done")
# print("XXXX",self.inputs)
self.obj_fn = objective(self.obs[period])
def test_ondemand_with_mask():
# Make output map with daily frequency
output_map['mleaf_hrudr_state'] = nodes.write_to_ncfile_snapshot(outpath,'mleaf_hrudr', freq='D')
period = dt.dates('25 dec 2010', '26 dec 2010')
from awrams.simulation.ondemand import OnDemandSimulator
sim = OnDemandSimulator(awral, input_map, omapping=output_map)
r = sim.run(period, extents.get_default_extent())
# Grab a new copy of the default extent in case the simulator mutated it
default_mask = extents.get_default_extent().mask
# Check that the results are masked arrays, using the first results and the final states
# as examples. Then check the masks are the default mask - masked arrays ensure that masked
# values are not used in computations.
assert all(type(r[key] == np.ma.core.MaskedArray) for key in r.keys())
assert all(type(r['final_states'][key] == np.ma.core.MaskedArray) for key in r['final_states'].keys())
assert all(np.array_equal(r[key].mask[0], default_mask) for key in r.keys() if key != 'final_states')
assert all(np.array_equal(r['final_states'][key].mask, default_mask) for key in r['final_states'].keys())
def test_output_graph_processing_splitfm_G():
from awrams.utils import extents
from awrams.utils import datetools as dt
e_all = extents.get_default_extent()
from awrams.utils.nodegraph import nodes,graph
from awrams.simulation.ondemand import OnDemandSimulator
print("RUNNER NEW: single cell ncf, multiple years")
period = dt.dates('2010-2011')
extent = e_all.ioffset[202,202]
#input_map = awral.get_default_mapping()
output_map = awral.get_output_mapping()
output_map['s0_save'] = nodes.write_to_annual_ncfile(os.path.dirname(__file__),'s0')
# outputs = graph.OutputGraph(output_map)
runner = OnDemandSimulator(awral,input_map,omapping=output_map)
r = runner.run(period,extent)
def test_output_graph_processing_snapshotfm_A():
from awrams.utils import extents
from awrams.utils import datetools as dt
e_all = extents.get_default_extent()
from awrams.utils.nodegraph import nodes,graph
from awrams.simulation.ondemand import OnDemandSimulator
#input_map = awral.get_default_mapping()
output_map = awral.get_output_mapping()
output_map['s0_save'] = nodes.write_to_ncfile_snapshot(
os.path.dirname(__file__), 's0')
runner = OnDemandSimulator(awral, input_map, omapping=output_map)
print("RUNNER NEW: multiple cells, multiple years")
period = dt.dates('2010-2011')
extent = e_all.ioffset[200, 200:202]
r = runner.run(period,extent)
def test_SplitFileWriterNode():
from awrams.utils import extents
from awrams.utils import datetools as dt
extent = extents.get_default_extent()
from awrams.utils.nodegraph import nodes
from awrams.simulation.ondemand import OnDemandSimulator
input_map = awral.get_default_mapping()
climate_mod(input_map)
from awrams.utils.nodegraph import nodes
from awrams.utils.metatypes import ObjectDict
# output_path = './'
output_map = awral.get_output_mapping()
output_map['qtot_save'] = nodes.write_to_annual_ncfile('./', 'qtot')
runner = OnDemandSimulator(awral, input_map, omapping=output_map)
period = dt.dates('2010-2011')
extent = extent.ioffset[200, 200:202]
r = runner.run(period, extent)
def test_OutputNode():
import awrams.models.awral.description
awrams.models.awral.description.CLIMATE_DATA = os.path.join(
os.path.dirname(__file__), '..', '..', 'test_data', 'simulation')
from awrams.simulation.ondemand import OnDemandSimulator
from awrams.models import awral
input_map = awral.get_default_mapping()
output_map = awral.get_default_output_mapping()
runner = OnDemandSimulator(awral,
input_map.mapping,
omapping=output_map.mapping)
print(runner.outputs)
def test_climatology_region():
import numpy as np
from awrams.utils import extents
from awrams.utils import datetools as dt
from awrams.simulation.ondemand import OnDemandSimulator
#from awrams.models import awral
### test region
period = dt.dates('dec 2010')
extent = extents.get_default_extent()
extent = extent.icoords[-32:-35, 115:118]
sim = OnDemandSimulator(awral, input_map) #,omapping=output_map.mapping)
r, i = sim.run(period, extent, return_inputs=True)
### this should be true
assert np.isnan(i['solar_f']).any()
insert_climatology(input_map)
sim = OnDemandSimulator(awral, input_map) #,omapping=output_map.mapping)
r, i = sim.run(period, extent, return_inputs=True)
assert not np.isnan(i['solar_f']).any()
def test_climatology_point():
import numpy as np
from awrams.utils import extents
from awrams.utils import datetools as dt
from awrams.simulation.ondemand import OnDemandSimulator
from awrams.models import awral
### test point
period = dt.dates('dec 2010')
extent = extents.from_cell_coords(-30, 120.5)
sim = OnDemandSimulator(awral,
input_map.mapping) #,omapping=output_map.mapping)
r, i = sim.run(period, extent, return_inputs=True)
### this should be true
assert np.isnan(i['solar_f']).any()
insert_climatology(input_map)
sim = OnDemandSimulator(awral,
input_map.mapping) #,omapping=output_map.mapping)
r, i = sim.run(period, extent, return_inputs=True)
assert not np.isnan(i['solar_f']).any()
class RunoffEvaluator(Evaluator):
'''
compare qtot against gauging station data observations
'''
def __init__(self, period, extent, observations):
self.period = period
self.extent = extent
self.obs = observations
self.evaluations = 0
logger.info("initialising model simulator...")
self.ods = OnDemandSimulator(MODEL, input_map.mapping)
self.initial_results, self.inputs = self.ods.run(period,
extent,
return_inputs=True)
logger.info("done")
# print("XXXX",self.inputs)
self.obj_fn = objective(self.obs[period])
def new(self):
return self
def run_sim(self, parameters):
for k, v in parameters.iteritems():
try:
# self.ods.input_runner.input_graph[k]['exe'].value = v
self.inputs[k] = v
except KeyError: ### ignore score column
pass
# return self.ods.run(self.period,self.extent)
return self.ods.run_prepack(self.inputs, self.period, self.extent)
def evaluate(self, parameters):
self.evaluations += 1
results = self.run_sim(parameters)
qtot_results = results['qtot']
### mm_to cumecs is approximate, could be improved by using actual cell areas
### this would require osgeo.ogr etc
catchment_qtot_mod = qtot_results.mean(axis=1) * mm_to_cumecs(
qtot_results.shape[1])
res = 1 - self.obj_fn(catchment_qtot_mod)
return res
def plot(self, results, period=None):
if period is None:
period = self.period
cal_catchment_qtot = results['qtot'].mean(axis=1) * mm_to_cumecs(
results['qtot'].shape[1])
ax = pd.DataFrame(cal_catchment_qtot,
columns=['modelled'],
index=self.period).loc[period].plot(figsize=(18, 6),
legend=True)
pd.DataFrame(self.obs[period].values,
columns=['observed'],
index=period).plot(legend=True, ax=ax)
def test_initial_states_region():
period = dt.dates('dec 2010')
### test a region
extent = extents.get_default_extent()
extent = extent.ioffset[400:408,170:178]
print(extent)
### simulation with default initial states
from awrams.simulation.ondemand import OnDemandSimulator
sim = OnDemandSimulator(awral,input_map)
r,i = sim.run(period,extent,return_inputs=True)
outputs_default = r['final_states']
### simulation with initial states read from nc files
get_initial_states(input_map)
sim = OnDemandSimulator(awral,input_map)
r,i = sim.run(period,extent,return_inputs=True)
outputs_init = r['final_states']
### compare final states with default states simulation
### should be different
for k,o in outputs_init.items():
assert not (o == outputs_default[k]).any()
### save initial states to compare
ini_states = {}
for k in i:
try:
if k.startswith('init'):
ini_states[k] = i[k]
except:
pass
### simulation with initial states read from dict
get_initial_states_dict(input_map,period,extent)
sim = OnDemandSimulator(awral,input_map)
r,i = sim.run(period,extent,return_inputs=True)
outputs_init_dict = r['final_states']
### compare final states with other ini states simulation
### should be same
for k,o in outputs_init_dict.items():
assert (o == outputs_init[k]).any()
### compare initial states from both methods
### should be same
for k in i:
try:
if k.startswith('init'):
assert ini_states[k] == i[k]
except:
pass
### simulation with initial states read from nc files
get_initial_states(input_map)
sim = OnDemandSimulator(awral,input_map)
r,i = sim.run(period,extent,return_inputs=True)
outputs_init = r['final_states']
### compare final states with other ini states simulation
### should be same
for k,o in outputs_init_dict.items():
assert (o == outputs_init[k]).any()
### compare initial states from both methods
### should be same
for k in i:
try:
if k.startswith('init'):
assert ini_states[k] == i[k]
except:
pass
def test_initial_states_region():
import numpy as np
from awrams.utils import extents
from awrams.utils import datetools as dt
from awrams.utils.nodegraph import nodes, graph
from awrams.simulation.ondemand import OnDemandSimulator
from awrams.models import awral
period = dt.dates('dec 2010')
### test a region
extent = extents.from_boundary_offset(400, 170, 407, 177)
print(extent)
### simulation with default initial states
sim = OnDemandSimulator(awral, input_map.mapping)
r, i = sim.run(period, extent, return_inputs=True)
outputs_default = r['final_states']
### simulation with initial states read from nc files
get_initial_states(input_map)
sim = OnDemandSimulator(awral, input_map.mapping)
r, i = sim.run(period, extent, return_inputs=True)
outputs_init = r['final_states']
### compare final states with default states simulation
### should be different
for k, o in outputs_init.items():
assert not (o == outputs_default[k]).any()
### save initial states to compare
ini_states = {}
for k in i:
try:
if k.startswith('init'):
ini_states[k] = i[k]
except:
pass
### simulation with initial states read from dict
get_initial_states_dict(input_map, period, extent)
sim = OnDemandSimulator(awral, input_map.mapping)
r, i = sim.run(period, extent, return_inputs=True)
outputs_init_dict = r['final_states']
### compare final states with other ini states simulation
### should be same
for k, o in outputs_init_dict.items():
assert (o == outputs_init[k]).any()
### compare initial states from both methods
### should be same
for k in i:
try:
if k.startswith('init'):
assert ini_states[k] == i[k]
except:
pass
extent = extents.from_boundary_coords(-30, 120.5, -30.35, 120.85)
print(extent)
### simulation with initial states read from nc files
get_initial_states(input_map)
sim = OnDemandSimulator(awral, input_map.mapping)
r, i = sim.run(period, extent, return_inputs=True)
outputs_init = r['final_states']
### compare final states with other ini states simulation
### should be same
for k, o in outputs_init_dict.items():
assert (o == outputs_init[k]).any()
### compare initial states from both methods
### should be same
for k in i:
try:
if k.startswith('init'):
assert ini_states[k] == i[k]
except:
pass
from awrams.utils import extents
import numpy as np
from awrams.utils.nodegraph import nodes, graph
from awrams.simulation.ondemand import OnDemandSimulator
full_extent = extents.get_default_extent()
period = dt.dates('dec 2010 - jan 2011')
extent = full_extent.ioffset[200:250, 200:250]
from awrams.models.awral import model
m = model.AWRALModel()
imap = m.get_default_mapping()
ods = OnDemandSimulator(m, imap)
print("running...")
res = ods.run(period, extent)
from awrams.utils.io.data_mapping import SplitFileManager
print("opening comparison results")
sfm = SplitFileManager.open_existing('./test_sim_outputs', 'qtot*', 'qtot')
qtot = sfm.get_data(period, extent)
max_diff = np.max(np.abs(res['qtot'] - qtot))
print(max_diff)
assert (max_diff < 1e-5)
