def test_shapefile_db():
try:
import osgeo
except ImportError:
return
from awrams.utils import config_manager
sys_profile = config_manager.get_system_profile().get_settings()
CATCHMENT_SHAPEFILE = join(sys_profile['DATA_PATHS']['SHAPEFILES'],
'Final_list_all_attributes.shp')
sdb = gis.ShapefileDB(CATCHMENT_SHAPEFILE)
e = extents.get_default_extent()
df = sdb.get_records_df()
df = df.set_index('StationID')
rec = df.loc['003303']
extent = sdb.get_extent_by_field('StationID', '003303', e)
area_diff = np.abs(extent.areas.sum() - (rec['AlbersArea'] * 1e6))
assert (area_diff / extent.areas.sum() < 1.0001)
br = extent.bounding_rect()
assert (between(rec['CentrLat'], br[0], br[1]))
assert (between(rec['CentrLon'], br[2], br[3]))
def get_sites_by_ids(site_ids, site_id_type, site_ref_type, site_set_name=None, extent=None):
"""
cross reference site ids with site meta contained is BENCHMARK_SITES csv
:param site_ids: list of site ids
:param site_id_type: id type for ids in site_ids (one of column names in BENCHMARK_SITES csv)
:param site_ref_type: column name to use for id reference name
:param site_set_name: name of set id belongs to, only required if id appears more than once in BENCHMARK_SITES csv
:param extent: reference extent from which to return site locations; will use default if none specified
:return: map of extents, map of site name
"""
if extent is None:
extent = extents.get_default_extent()
site_idx = get_obs_sites()
site_extents = {}
site_name_map = {}
for site_id in site_ids:
ixs = [k for k in list(site_idx.keys()) if site_idx[k][site_id_type] == site_id]
if site_set_name is not None:
ixs = [k for k in ixs if site_idx[k]['set'] in site_set_name]
if len(ixs) == 0: # site_id not found in SiteLocationsWithIndex.csv
continue
# use the first occurrence of pred_index
ref = site_idx[ixs[0]]
site_name = ref[site_ref_type]
site_name_map[site_name] = site_id
#site_extents[site_name] = extents.from_cell_coords(*sanitize_cell((float(ref['Y']), float(ref['X']))))
site_extents[site_name] = extent.icoords[float(ref['Y']), float(ref['X'])]
return site_extents, site_name_map
def test_multiextent():
gsd = extents.Extent.from_file(MASK_FILE)
full = extents.ExtentFactory(extents.get_default_extent())
m = dict(a=full.get_by_cell_offset(250, 250),
b=gsd.extent_from_cell_offset(275, 275),
c=gsd.extent_from_cell_offset(300, 300),
d=gsd.extent_from_boundary_offset(200, 290, 202, 292))
e = gsd.extent_from_multiple(m)
# print(e.mask.shape)
# print(e.mask[49:52,:2])
# print(e.mask[:4,40:44])
for k, cell in m.items():
# print(k,cell)
assert (e.contains(cell))
# for cell in e:
# print("LLL",cell)
#
assert e.x_min == 250
assert e.x_max == 300
assert e.y_min == 200
assert e.y_max == 300
assert e.cell_count == 12
def test_initial_states_point():
import numpy as np
from awrams.utils import extents
from awrams.utils import datetools as dt
from awrams.utils.nodegraph import nodes, graph
from awrams.simulation.ondemand import OnDemandSimulator
#from awrams.models import awral
period = dt.dates('dec 2010')
### test a single cell
extent = extents.get_default_extent()
extent = extent.icoords[-30, 120.5]
### simulation with default initial states
sim = OnDemandSimulator(awral, input_map)
r, i = sim.run(period, extent, return_inputs=True)
outputs_default = r['final_states']
### simulation with initial states read from nc files
get_initial_states(input_map)
sim = OnDemandSimulator(awral, input_map)
r, i = sim.run(period, extent, return_inputs=True)
outputs_init = r['final_states']
### compare final states with default states simulation
### should be different
for k, o in outputs_init.items():
assert not o == outputs_default[k]
### save initial states to compare
ini_states = {}
for k in i:
try:
if k.startswith('init'):
ini_states[k] = i[k]
except:
pass
### simulation with initial states read from dict
get_initial_states_dict(input_map, period, extent)
sim = OnDemandSimulator(awral, input_map)
r, i = sim.run(period, extent, return_inputs=True)
outputs_init_dict = r['final_states']
### compare final states with other ini states simulation
### should be same
for k, o in outputs_init_dict.items():
assert o == outputs_init[k]
### compare initial states from both methods
### should be same
for k in i:
try:
if k.startswith('init'):
assert ini_states[k] == i[k]
except:
pass
def test_ondemand_region_v5():
### test region
extent = extents.get_default_extent()
extent = extent.icoords[-32:-34,115:118]
from awrams.simulation.ondemand import OnDemandSimulator
sim = OnDemandSimulator(awral,input_map,omapping=output_map)
r = sim.run(period,extent)
def test_ondemand_point():
### test point
extent = extents.get_default_extent()
extent = extent.icoords[-32.1,115.1]
from awrams.simulation.ondemand import OnDemandSimulator
sim = OnDemandSimulator(awral,input_map,omapping=output_map)
r = sim.run(period,extent)
def test_extraction():
from awrams.utils import gis
from awrams.utils import config_manager
sys_settings = config_manager.get_system_profile().get_settings()
CATCHMENT_SHAPEFILE = os.path.join(sys_settings['DATA_PATHS']['SHAPEFILES'], \
'Final_list_all_attributes.shp')
catchments = gis.ShapefileDB(CATCHMENT_SHAPEFILE)
e_all = extents.get_default_extent()
e = catchments.get_extent_by_field('StationID', '421103', e_all)
period = dt.dates('jun 1990 - jan 1995')
var_name = 'rain_day'
path, _ = os.path.split(os.path.abspath(__file__))
data_path = os.path.join(sys_settings['DATA_PATHS']['TEST_DATA'],
'calibration/')
pattern = data_path + '/%s*' % var_name
df = extract(data_path, pattern, var_name, {
'241': e,
'512': e_all.ioffset[400, 400]
}, period)
assert ((df.index == period).all())
def test_output_graph_processing_splitfm_B():
from awrams.utils import extents
from awrams.utils import datetools as dt
e_all = extents.get_default_extent()
from awrams.utils.nodegraph import nodes, graph
from awrams.simulation.ondemand import OnDemandSimulator
input_map = awral.get_default_mapping()
climate_mod(input_map)
output_map = awral.get_output_mapping()
output_map['s0_save'] = nodes.write_to_annual_ncfile(
os.path.dirname(__file__), 's0', mode='w')
runner = OnDemandSimulator(awral, input_map, omapping=output_map)
print("RUNNER NEW (FILES EXISTING): multiple cells, multiple years")
period = dt.dates('2010-2011')
extent = e_all.ioffset[200:202, 200:202]
r = runner.run(period, extent)
clear_files()
print("RUNNER OLD (FILES EXISTING): single cell, single year")
period = dt.dates('2015')
extent = e_all.ioffset[202, 202]
r = runner.run(period, extent)
def test_server():
extent = extents.get_default_extent()
extent = extent.icoords[-32:-35,115:118]
from awrams.simulation.server import SimulationServer
sim = SimulationServer(awral)
sim.run(input_map,output_map,period,extent)
def test_climatology_region():
### test region
period = dt.dates('dec 2010')
extent = extents.get_default_extent()
extent = extent.icoords[-32:-35,115:118]
from awrams.simulation.ondemand import OnDemandSimulator
sim = OnDemandSimulator(awral,input_map)
r,i = sim.run(period,extent,return_inputs=True)
assert not np.isnan(i['solar_f']).any()
def test_ondemand_with_mask():
# Make output map with daily frequency
output_map['mleaf_hrudr_state'] = nodes.write_to_ncfile_snapshot(outpath,'mleaf_hrudr', freq='D')
period = dt.dates('25 dec 2010', '26 dec 2010')
from awrams.simulation.ondemand import OnDemandSimulator
sim = OnDemandSimulator(awral, input_map, omapping=output_map)
r = sim.run(period, extents.get_default_extent())
# Grab a new copy of the default extent in case the simulator mutated it
default_mask = extents.get_default_extent().mask
# Check that the results are masked arrays, using the first results and the final states
# as examples. Then check the masks are the default mask - masked arrays ensure that masked
# values are not used in computations.
assert all(type(r[key] == np.ma.core.MaskedArray) for key in r.keys())
assert all(type(r['final_states'][key] == np.ma.core.MaskedArray) for key in r['final_states'].keys())
assert all(np.array_equal(r[key].mask[0], default_mask) for key in r.keys() if key != 'final_states')
assert all(np.array_equal(r['final_states'][key].mask, default_mask) for key in r['final_states'].keys())
def test_extents_geoarray(): e = extents.get_default_extent() lats,lons = e.to_coords() lats_geo = geo.GeoArray.from_degrees(lats.index) lons_geo = geo.GeoArray.from_degrees(lons.index) print(lats.index) print(lats_geo.to_degrees()) print(lats.index - lats_geo.to_degrees()) assert((lats.index == lats_geo.to_degrees()).all()) assert((lons.index == lons_geo.to_degrees()).all())
def test_server():
from awrams.utils import extents
from awrams.simulation.server import SimulationServer
#from awrams.models import awral
extent = extents.get_default_extent()
extent = extent.icoords[-32:-35, 115:118]
get_initial_states_dict(input_map, period, extent)
insert_climatology(input_map)
sim = SimulationServer(awral)
sim.run(input_map, output_map, period, extent)
def __init__(self, model, imapping, omapping=None, extent=None):
if extent is None:
extent = extents.get_default_extent()
imapping = graph.get_input_tree(model.get_input_keys(), imapping)
#+++
#Document the use of this manually, don't just change the graph behind the scenes...
#imapping = graph.map_rescaling_nodes(imapping,extent)
self.input_runner = graph.ExecutionGraph(imapping)
self.model_runner = model.get_runner(
self.input_runner.get_dataspecs(True))
self.outputs = None
if omapping is not None:
self.outputs = graph.OutputGraph(omapping)
def test_output_graph_processing_snapshotfm_A():
from awrams.utils import extents
from awrams.utils import datetools as dt
e_all = extents.get_default_extent()
from awrams.utils.nodegraph import nodes,graph
from awrams.simulation.ondemand import OnDemandSimulator
#input_map = awral.get_default_mapping()
output_map = awral.get_output_mapping()
output_map['s0_save'] = nodes.write_to_ncfile_snapshot(
os.path.dirname(__file__), 's0')
runner = OnDemandSimulator(awral, input_map, omapping=output_map)
print("RUNNER NEW: multiple cells, multiple years")
period = dt.dates('2010-2011')
extent = e_all.ioffset[200, 200:202]
r = runner.run(period,extent)
def test_output_graph_processing_splitfm_G():
from awrams.utils import extents
from awrams.utils import datetools as dt
e_all = extents.get_default_extent()
from awrams.utils.nodegraph import nodes,graph
from awrams.simulation.ondemand import OnDemandSimulator
print("RUNNER NEW: single cell ncf, multiple years")
period = dt.dates('2010-2011')
extent = e_all.ioffset[202,202]
#input_map = awral.get_default_mapping()
output_map = awral.get_output_mapping()
output_map['s0_save'] = nodes.write_to_annual_ncfile(os.path.dirname(__file__),'s0')
# outputs = graph.OutputGraph(output_map)
runner = OnDemandSimulator(awral,input_map,omapping=output_map)
r = runner.run(period,extent)
def get_catchments_by_ids(ids,extent=None):
"""
get catchment extents for ids
:param ids: list of ids
:return: map of extents
"""
from awrams.utils.gis import ShapefileDB
from awrams.utils.settings import CATCHMENT_SHAPEFILE
extent_map = {}
if extent is None:
extent = extents.get_default_extent()
for idx in ids:
try:
extent_map[str(idx)] = ShapefileDB(CATCHMENT_SHAPEFILE).get_extent_by_field('StationID',idx,extent.geospatial_reference())
except Exception as e:
logger.warning(e)
continue
return extent_map
def test_climatology_region():
import numpy as np
from awrams.utils import extents
from awrams.utils import datetools as dt
from awrams.simulation.ondemand import OnDemandSimulator
#from awrams.models import awral
### test region
period = dt.dates('dec 2010')
extent = extents.get_default_extent()
extent = extent.icoords[-32:-35, 115:118]
sim = OnDemandSimulator(awral, input_map) #,omapping=output_map.mapping)
r, i = sim.run(period, extent, return_inputs=True)
### this should be true
assert np.isnan(i['solar_f']).any()
insert_climatology(input_map)
sim = OnDemandSimulator(awral, input_map) #,omapping=output_map.mapping)
r, i = sim.run(period, extent, return_inputs=True)
assert not np.isnan(i['solar_f']).any()
def test_shapefile_db():
try:
import osgeo
except ImportError:
return
sdb = gis.ShapefileDB(CATCHMENT_SHAPEFILE)
e = extents.get_default_extent()
df = sdb.get_records_df()
df = df.set_index('StationID')
rec = df.loc['003303']
extent = sdb.get_extent_by_field('StationID', '003303', e)
area_diff = np.abs(extent.areas.sum() - (rec['AlbersArea'] * 1e6))
assert (area_diff / extent.areas.sum() < 1.0001)
br = extent.bounding_rect()
assert (between(rec['CentrLat'], br[0], br[1]))
assert (between(rec['CentrLon'], br[2], br[3]))
def test_SplitFileWriterNode():
from awrams.utils import extents
from awrams.utils import datetools as dt
extent = extents.get_default_extent()
from awrams.utils.nodegraph import nodes
from awrams.simulation.ondemand import OnDemandSimulator
input_map = awral.get_default_mapping()
climate_mod(input_map)
from awrams.utils.nodegraph import nodes
from awrams.utils.metatypes import ObjectDict
# output_path = './'
output_map = awral.get_output_mapping()
output_map['qtot_save'] = nodes.write_to_annual_ncfile('./', 'qtot')
runner = OnDemandSimulator(awral, input_map, omapping=output_map)
period = dt.dates('2010-2011')
extent = extent.ioffset[200, 200:202]
r = runner.run(period, extent)
def test_sim():
'''
Required inputs
mortons: tmin,tmax,solar,vapour pressure ### apparently not albedo
fao56: tmin,tmax,solar,vapour pressure,wind
penpan: tmin2da,tmax,solar,vapour pressure,wind
penman: tmin,tmax,solar,vapour pressure,wind,veg,albedo
'''
from awrams.utils import extents
from awrams.simulation.server import SimulationServer
et.set_models_to_run([
"fao56", "asce", "morton_shallow_lake", "morton_areal", "penpan",
"penman"
])
extent = extents.get_default_extent()
extent = extent.icoords[-32:-35, 115:118]
period = pd.date_range("1 dec 2010", "31 jan 2011", freq='D')
sim = SimulationServer(et)
sim.run(imap, omap, period, extent)
def test_extraction():
from awrams.utils import gis
catchments = gis.ShapefileDB(gis.CATCHMENT_SHAPEFILE)
e_all = extents.get_default_extent()
e = catchments.get_extent_by_field('StationID', '421103', e_all)
period = dt.dates('jun 1990 - jan 1995')
var_name = 'rain_day'
path, _ = os.path.split(os.path.abspath(__file__))
data_path = os.path.join(path, '../../test_data/calibration/')
pattern = data_path + '/%s*' % var_name
df = extract(data_path, pattern, var_name, {
'241': e,
'512': e_all.ioffset[400, 400]
}, period)
assert ((df.index == period).all())
def test_initial_states_region():
period = dt.dates('dec 2010')
### test a region
extent = extents.get_default_extent()
extent = extent.ioffset[400:408,170:178]
print(extent)
### simulation with default initial states
from awrams.simulation.ondemand import OnDemandSimulator
sim = OnDemandSimulator(awral,input_map)
r,i = sim.run(period,extent,return_inputs=True)
outputs_default = r['final_states']
### simulation with initial states read from nc files
get_initial_states(input_map)
sim = OnDemandSimulator(awral,input_map)
r,i = sim.run(period,extent,return_inputs=True)
outputs_init = r['final_states']
### compare final states with default states simulation
### should be different
for k,o in outputs_init.items():
assert not (o == outputs_default[k]).any()
### save initial states to compare
ini_states = {}
for k in i:
try:
if k.startswith('init'):
ini_states[k] = i[k]
except:
pass
### simulation with initial states read from dict
get_initial_states_dict(input_map,period,extent)
sim = OnDemandSimulator(awral,input_map)
r,i = sim.run(period,extent,return_inputs=True)
outputs_init_dict = r['final_states']
### compare final states with other ini states simulation
### should be same
for k,o in outputs_init_dict.items():
assert (o == outputs_init[k]).any()
### compare initial states from both methods
### should be same
for k in i:
try:
if k.startswith('init'):
assert ini_states[k] == i[k]
except:
pass
### simulation with initial states read from nc files
get_initial_states(input_map)
sim = OnDemandSimulator(awral,input_map)
r,i = sim.run(period,extent,return_inputs=True)
outputs_init = r['final_states']
### compare final states with other ini states simulation
### should be same
for k,o in outputs_init_dict.items():
assert (o == outputs_init[k]).any()
### compare initial states from both methods
### should be same
for k in i:
try:
if k.startswith('init'):
assert ini_states[k] == i[k]
except:
pass
def test_asce():
import awrams.models.settings
import awrams.utils.nodegraph.nodes as nodes
# awrams.models.settings.CLIMATE_DATA = _join(TRAINING_DATA_PATH,'climate/BOM_climate/')
bom_data_path = '/data/cwd_awra_data/awra_inputs/climate_generated_sdcvd-awrap01/'
wind_data_path = '/data/cwd_awra_data/awra_inputs/mcvicar_wind/'
fveg_data_path = '/data/cwd_awra_data/awra_inputs/modis_fveg/8day/pv/'
albedo_data_path = '/data/cwd_awra_data/awra_inputs/randall_albedo/'
temp_min_2day_datapath = '/data/cwd_awra_data/awra_inputs/2dayAveTmin/'
FORCING = {
'tmin':
('temp_min*.nc', 'temp_min_day', bom_data_path + 'temp_min_day/'),
'tmin2da':
('temp_min*.nc', 'temp_min_2dayave', temp_min_2day_datapath),
'tmax':
('temp_max*.nc', 'temp_max_day', bom_data_path + 'temp_max_day/'),
'precip': ('rain_day*.nc', 'rain_day', bom_data_path + 'rain_day/'),
'solar': ('solar*.nc', 'solar_exposure_day',
bom_data_path + 'solar_exposure_day/'), #,
'vprp': ('vapour_pressure*.nc', 'vapour_pressure',
bom_data_path + 'vapour_pressure/'), #, #h09
'wind': ('wind*.nc', 'wind', wind_data_path),
'fveg': ('fveg*.nc', 'fveg', fveg_data_path),
'alb': ('albedo*.nc', 'albedo', albedo_data_path)
}
import awrams.models.et.settings
awrams.models.et.settings.FORCING = {
k + "_f": dict(zip(("pattern", "nc_var", "path"), v))
for k, v in FORCING.items()
}
'''
Required inputs
mortons: tmin,tmax,solar,vapour pressure ### apparently not albedo
fao56: tmin,tmax,solar,vapour pressure,wind
penpan: tmin2da,tmax,solar,vapour pressure,wind
penman: tmin,tmax,solar,vapour pressure,wind,veg,albedo
'''
from awrams.utils import extents
from awrams.simulation.ondemand import OnDemandSimulator
from awrams.simulation.server import SimulationServer
from awrams.models.et.model import ETModel
et = ETModel()
# et.set_models_to_run(["fao56","morton_shallow_lake","morton_areal","penpan","penman"])
# et.set_models_to_run(["fao56","morton_areal","penpan"])
# et.set_models_to_run(["morton_areal"])
# et.set_models_to_run(["fao56"])
# et.set_models_to_run(["penpan"])
# et.set_models_to_run(["penman"])
et.set_models_to_run(["asce"])
# et.set_models_to_run(["morton_shallow_lake"])
imap = et.get_default_mapping()
# print(imap['fveg'])
# print(imap['fveg_f'])
# imap['fveg_f'] = nodes.forcing_from_modis_ncfiles(fveg_data_path,'fveg_filled.nc','fveg')
# imap['fveg'] = nodes.mul('fveg_f',0.01)
# print(imap['fveg_f'])
# exit()
omap = et.get_output_mapping()
et.save_outputs(omap, path="./outputs/ss/")
# et.save_outputs(omap,path="/data/cwd_awra_data/awra_test_outputs/Scheduled_et_sdcvt-awrap01/")
# print(omap)
# omap['nc_msl_et'] = nodes.write_to_annual_ncfile("./",'msl_et')
extent = extents.get_default_extent()
# extent = extent.icoords[-32:-34,115:118]
# extent = extent.ioffset[440,95]
# extent = extent.ioffset[490,90]
# extent = extent.ioffset[277:304,50]
# extent = extent.ioffset[256:265,60]
# extent = extent.ioffset[455,770:791]
# extent = extent.ioffset[230:240,120]
# print("Extent",extent)
# period = pd.date_range("1 jan 2010","31 dec 2011",freq='D')
# period = pd.date_range("1 jan 2010",periods=1,freq='D')
period = pd.date_range("1 jan 2011", "31 dec 2011", freq='D')
# period = pd.date_range("1 jan 2016","31 jan 2016",freq='D')
# period = pd.date_range("1 jan 1990","30 apr 2018",freq='D')
# sim = OnDemandSimulator(et,imap,omapping=omap)
# print(sim.input_runner.input_graph['models']['exe'].get_dataspec())
# r = sim.run(period,extent)
# # print(r)
sim = SimulationServer(et)
sim.run(imap, omap, period, extent)
"""
Simple script to verify that running an OnDemandSimulator
will produce the same results as our cluster sim
"""
from awrams.utils import datetools as dt
from awrams.utils import extents
import numpy as np
from awrams.utils.nodegraph import nodes, graph
from awrams.simulation.ondemand import OnDemandSimulator
full_extent = extents.get_default_extent()
period = dt.dates('dec 2010 - jan 2011')
extent = full_extent.ioffset[200:250, 200:250]
from awrams.models.awral import model
m = model.AWRALModel()
imap = m.get_default_mapping()
ods = OnDemandSimulator(m, imap)
print("running...")
res = ods.run(period, extent)
from awrams.utils.io.data_mapping import SplitFileManager
print("opening comparison results")
sfm = SplitFileManager.open_existing('./test_sim_outputs', 'qtot*', 'qtot')
