def test_shapefile_db(): try: import osgeo except ImportError: return from awrams.utils import config_manager sys_profile = config_manager.get_system_profile().get_settings() CATCHMENT_SHAPEFILE = join(sys_profile['DATA_PATHS']['SHAPEFILES'], 'Final_list_all_attributes.shp') sdb = gis.ShapefileDB(CATCHMENT_SHAPEFILE) e = extents.get_default_extent() df = sdb.get_records_df() df = df.set_index('StationID') rec = df.loc['003303'] extent = sdb.get_extent_by_field('StationID', '003303', e) area_diff = np.abs(extent.areas.sum() - (rec['AlbersArea'] * 1e6)) assert (area_diff / extent.areas.sum() < 1.0001) br = extent.bounding_rect() assert (between(rec['CentrLat'], br[0], br[1])) assert (between(rec['CentrLon'], br[2], br[3]))
def get_sites_by_ids(site_ids, site_id_type, site_ref_type, site_set_name=None, extent=None): """ cross reference site ids with site meta contained is BENCHMARK_SITES csv :param site_ids: list of site ids :param site_id_type: id type for ids in site_ids (one of column names in BENCHMARK_SITES csv) :param site_ref_type: column name to use for id reference name :param site_set_name: name of set id belongs to, only required if id appears more than once in BENCHMARK_SITES csv :param extent: reference extent from which to return site locations; will use default if none specified :return: map of extents, map of site name """ if extent is None: extent = extents.get_default_extent() site_idx = get_obs_sites() site_extents = {} site_name_map = {} for site_id in site_ids: ixs = [k for k in list(site_idx.keys()) if site_idx[k][site_id_type] == site_id] if site_set_name is not None: ixs = [k for k in ixs if site_idx[k]['set'] in site_set_name] if len(ixs) == 0: # site_id not found in SiteLocationsWithIndex.csv continue # use the first occurrence of pred_index ref = site_idx[ixs[0]] site_name = ref[site_ref_type] site_name_map[site_name] = site_id #site_extents[site_name] = extents.from_cell_coords(*sanitize_cell((float(ref['Y']), float(ref['X'])))) site_extents[site_name] = extent.icoords[float(ref['Y']), float(ref['X'])] return site_extents, site_name_map
def test_multiextent(): gsd = extents.Extent.from_file(MASK_FILE) full = extents.ExtentFactory(extents.get_default_extent()) m = dict(a=full.get_by_cell_offset(250, 250), b=gsd.extent_from_cell_offset(275, 275), c=gsd.extent_from_cell_offset(300, 300), d=gsd.extent_from_boundary_offset(200, 290, 202, 292)) e = gsd.extent_from_multiple(m) # print(e.mask.shape) # print(e.mask[49:52,:2]) # print(e.mask[:4,40:44]) for k, cell in m.items(): # print(k,cell) assert (e.contains(cell)) # for cell in e: # print("LLL",cell) # assert e.x_min == 250 assert e.x_max == 300 assert e.y_min == 200 assert e.y_max == 300 assert e.cell_count == 12
def test_initial_states_point(): import numpy as np from awrams.utils import extents from awrams.utils import datetools as dt from awrams.utils.nodegraph import nodes, graph from awrams.simulation.ondemand import OnDemandSimulator #from awrams.models import awral period = dt.dates('dec 2010') ### test a single cell extent = extents.get_default_extent() extent = extent.icoords[-30, 120.5] ### simulation with default initial states sim = OnDemandSimulator(awral, input_map) r, i = sim.run(period, extent, return_inputs=True) outputs_default = r['final_states'] ### simulation with initial states read from nc files get_initial_states(input_map) sim = OnDemandSimulator(awral, input_map) r, i = sim.run(period, extent, return_inputs=True) outputs_init = r['final_states'] ### compare final states with default states simulation ### should be different for k, o in outputs_init.items(): assert not o == outputs_default[k] ### save initial states to compare ini_states = {} for k in i: try: if k.startswith('init'): ini_states[k] = i[k] except: pass ### simulation with initial states read from dict get_initial_states_dict(input_map, period, extent) sim = OnDemandSimulator(awral, input_map) r, i = sim.run(period, extent, return_inputs=True) outputs_init_dict = r['final_states'] ### compare final states with other ini states simulation ### should be same for k, o in outputs_init_dict.items(): assert o == outputs_init[k] ### compare initial states from both methods ### should be same for k in i: try: if k.startswith('init'): assert ini_states[k] == i[k] except: pass
def test_ondemand_region_v5(): ### test region extent = extents.get_default_extent() extent = extent.icoords[-32:-34,115:118] from awrams.simulation.ondemand import OnDemandSimulator sim = OnDemandSimulator(awral,input_map,omapping=output_map) r = sim.run(period,extent)
def test_ondemand_point(): ### test point extent = extents.get_default_extent() extent = extent.icoords[-32.1,115.1] from awrams.simulation.ondemand import OnDemandSimulator sim = OnDemandSimulator(awral,input_map,omapping=output_map) r = sim.run(period,extent)
def test_extraction(): from awrams.utils import gis from awrams.utils import config_manager sys_settings = config_manager.get_system_profile().get_settings() CATCHMENT_SHAPEFILE = os.path.join(sys_settings['DATA_PATHS']['SHAPEFILES'], \ 'Final_list_all_attributes.shp') catchments = gis.ShapefileDB(CATCHMENT_SHAPEFILE) e_all = extents.get_default_extent() e = catchments.get_extent_by_field('StationID', '421103', e_all) period = dt.dates('jun 1990 - jan 1995') var_name = 'rain_day' path, _ = os.path.split(os.path.abspath(__file__)) data_path = os.path.join(sys_settings['DATA_PATHS']['TEST_DATA'], 'calibration/') pattern = data_path + '/%s*' % var_name df = extract(data_path, pattern, var_name, { '241': e, '512': e_all.ioffset[400, 400] }, period) assert ((df.index == period).all())
def test_output_graph_processing_splitfm_B(): from awrams.utils import extents from awrams.utils import datetools as dt e_all = extents.get_default_extent() from awrams.utils.nodegraph import nodes, graph from awrams.simulation.ondemand import OnDemandSimulator input_map = awral.get_default_mapping() climate_mod(input_map) output_map = awral.get_output_mapping() output_map['s0_save'] = nodes.write_to_annual_ncfile( os.path.dirname(__file__), 's0', mode='w') runner = OnDemandSimulator(awral, input_map, omapping=output_map) print("RUNNER NEW (FILES EXISTING): multiple cells, multiple years") period = dt.dates('2010-2011') extent = e_all.ioffset[200:202, 200:202] r = runner.run(period, extent) clear_files() print("RUNNER OLD (FILES EXISTING): single cell, single year") period = dt.dates('2015') extent = e_all.ioffset[202, 202] r = runner.run(period, extent)
def test_server(): extent = extents.get_default_extent() extent = extent.icoords[-32:-35,115:118] from awrams.simulation.server import SimulationServer sim = SimulationServer(awral) sim.run(input_map,output_map,period,extent)
def test_climatology_region(): ### test region period = dt.dates('dec 2010') extent = extents.get_default_extent() extent = extent.icoords[-32:-35,115:118] from awrams.simulation.ondemand import OnDemandSimulator sim = OnDemandSimulator(awral,input_map) r,i = sim.run(period,extent,return_inputs=True) assert not np.isnan(i['solar_f']).any()
def test_ondemand_with_mask(): # Make output map with daily frequency output_map['mleaf_hrudr_state'] = nodes.write_to_ncfile_snapshot(outpath,'mleaf_hrudr', freq='D') period = dt.dates('25 dec 2010', '26 dec 2010') from awrams.simulation.ondemand import OnDemandSimulator sim = OnDemandSimulator(awral, input_map, omapping=output_map) r = sim.run(period, extents.get_default_extent()) # Grab a new copy of the default extent in case the simulator mutated it default_mask = extents.get_default_extent().mask # Check that the results are masked arrays, using the first results and the final states # as examples. Then check the masks are the default mask - masked arrays ensure that masked # values are not used in computations. assert all(type(r[key] == np.ma.core.MaskedArray) for key in r.keys()) assert all(type(r['final_states'][key] == np.ma.core.MaskedArray) for key in r['final_states'].keys()) assert all(np.array_equal(r[key].mask[0], default_mask) for key in r.keys() if key != 'final_states') assert all(np.array_equal(r['final_states'][key].mask, default_mask) for key in r['final_states'].keys())
def test_extents_geoarray(): e = extents.get_default_extent() lats,lons = e.to_coords() lats_geo = geo.GeoArray.from_degrees(lats.index) lons_geo = geo.GeoArray.from_degrees(lons.index) print(lats.index) print(lats_geo.to_degrees()) print(lats.index - lats_geo.to_degrees()) assert((lats.index == lats_geo.to_degrees()).all()) assert((lons.index == lons_geo.to_degrees()).all())
def test_server(): from awrams.utils import extents from awrams.simulation.server import SimulationServer #from awrams.models import awral extent = extents.get_default_extent() extent = extent.icoords[-32:-35, 115:118] get_initial_states_dict(input_map, period, extent) insert_climatology(input_map) sim = SimulationServer(awral) sim.run(input_map, output_map, period, extent)
def __init__(self, model, imapping, omapping=None, extent=None): if extent is None: extent = extents.get_default_extent() imapping = graph.get_input_tree(model.get_input_keys(), imapping) #+++ #Document the use of this manually, don't just change the graph behind the scenes... #imapping = graph.map_rescaling_nodes(imapping,extent) self.input_runner = graph.ExecutionGraph(imapping) self.model_runner = model.get_runner( self.input_runner.get_dataspecs(True)) self.outputs = None if omapping is not None: self.outputs = graph.OutputGraph(omapping)
def test_output_graph_processing_snapshotfm_A(): from awrams.utils import extents from awrams.utils import datetools as dt e_all = extents.get_default_extent() from awrams.utils.nodegraph import nodes,graph from awrams.simulation.ondemand import OnDemandSimulator #input_map = awral.get_default_mapping() output_map = awral.get_output_mapping() output_map['s0_save'] = nodes.write_to_ncfile_snapshot( os.path.dirname(__file__), 's0') runner = OnDemandSimulator(awral, input_map, omapping=output_map) print("RUNNER NEW: multiple cells, multiple years") period = dt.dates('2010-2011') extent = e_all.ioffset[200, 200:202] r = runner.run(period,extent)
def test_output_graph_processing_splitfm_G(): from awrams.utils import extents from awrams.utils import datetools as dt e_all = extents.get_default_extent() from awrams.utils.nodegraph import nodes,graph from awrams.simulation.ondemand import OnDemandSimulator print("RUNNER NEW: single cell ncf, multiple years") period = dt.dates('2010-2011') extent = e_all.ioffset[202,202] #input_map = awral.get_default_mapping() output_map = awral.get_output_mapping() output_map['s0_save'] = nodes.write_to_annual_ncfile(os.path.dirname(__file__),'s0') # outputs = graph.OutputGraph(output_map) runner = OnDemandSimulator(awral,input_map,omapping=output_map) r = runner.run(period,extent)
def get_catchments_by_ids(ids,extent=None): """ get catchment extents for ids :param ids: list of ids :return: map of extents """ from awrams.utils.gis import ShapefileDB from awrams.utils.settings import CATCHMENT_SHAPEFILE extent_map = {} if extent is None: extent = extents.get_default_extent() for idx in ids: try: extent_map[str(idx)] = ShapefileDB(CATCHMENT_SHAPEFILE).get_extent_by_field('StationID',idx,extent.geospatial_reference()) except Exception as e: logger.warning(e) continue return extent_map
def test_climatology_region(): import numpy as np from awrams.utils import extents from awrams.utils import datetools as dt from awrams.simulation.ondemand import OnDemandSimulator #from awrams.models import awral ### test region period = dt.dates('dec 2010') extent = extents.get_default_extent() extent = extent.icoords[-32:-35, 115:118] sim = OnDemandSimulator(awral, input_map) #,omapping=output_map.mapping) r, i = sim.run(period, extent, return_inputs=True) ### this should be true assert np.isnan(i['solar_f']).any() insert_climatology(input_map) sim = OnDemandSimulator(awral, input_map) #,omapping=output_map.mapping) r, i = sim.run(period, extent, return_inputs=True) assert not np.isnan(i['solar_f']).any()
def test_shapefile_db(): try: import osgeo except ImportError: return sdb = gis.ShapefileDB(CATCHMENT_SHAPEFILE) e = extents.get_default_extent() df = sdb.get_records_df() df = df.set_index('StationID') rec = df.loc['003303'] extent = sdb.get_extent_by_field('StationID', '003303', e) area_diff = np.abs(extent.areas.sum() - (rec['AlbersArea'] * 1e6)) assert (area_diff / extent.areas.sum() < 1.0001) br = extent.bounding_rect() assert (between(rec['CentrLat'], br[0], br[1])) assert (between(rec['CentrLon'], br[2], br[3]))
def test_SplitFileWriterNode(): from awrams.utils import extents from awrams.utils import datetools as dt extent = extents.get_default_extent() from awrams.utils.nodegraph import nodes from awrams.simulation.ondemand import OnDemandSimulator input_map = awral.get_default_mapping() climate_mod(input_map) from awrams.utils.nodegraph import nodes from awrams.utils.metatypes import ObjectDict # output_path = './' output_map = awral.get_output_mapping() output_map['qtot_save'] = nodes.write_to_annual_ncfile('./', 'qtot') runner = OnDemandSimulator(awral, input_map, omapping=output_map) period = dt.dates('2010-2011') extent = extent.ioffset[200, 200:202] r = runner.run(period, extent)
def test_sim(): ''' Required inputs mortons: tmin,tmax,solar,vapour pressure ### apparently not albedo fao56: tmin,tmax,solar,vapour pressure,wind penpan: tmin2da,tmax,solar,vapour pressure,wind penman: tmin,tmax,solar,vapour pressure,wind,veg,albedo ''' from awrams.utils import extents from awrams.simulation.server import SimulationServer et.set_models_to_run([ "fao56", "asce", "morton_shallow_lake", "morton_areal", "penpan", "penman" ]) extent = extents.get_default_extent() extent = extent.icoords[-32:-35, 115:118] period = pd.date_range("1 dec 2010", "31 jan 2011", freq='D') sim = SimulationServer(et) sim.run(imap, omap, period, extent)
def test_extraction(): from awrams.utils import gis catchments = gis.ShapefileDB(gis.CATCHMENT_SHAPEFILE) e_all = extents.get_default_extent() e = catchments.get_extent_by_field('StationID', '421103', e_all) period = dt.dates('jun 1990 - jan 1995') var_name = 'rain_day' path, _ = os.path.split(os.path.abspath(__file__)) data_path = os.path.join(path, '../../test_data/calibration/') pattern = data_path + '/%s*' % var_name df = extract(data_path, pattern, var_name, { '241': e, '512': e_all.ioffset[400, 400] }, period) assert ((df.index == period).all())
def test_initial_states_region(): period = dt.dates('dec 2010') ### test a region extent = extents.get_default_extent() extent = extent.ioffset[400:408,170:178] print(extent) ### simulation with default initial states from awrams.simulation.ondemand import OnDemandSimulator sim = OnDemandSimulator(awral,input_map) r,i = sim.run(period,extent,return_inputs=True) outputs_default = r['final_states'] ### simulation with initial states read from nc files get_initial_states(input_map) sim = OnDemandSimulator(awral,input_map) r,i = sim.run(period,extent,return_inputs=True) outputs_init = r['final_states'] ### compare final states with default states simulation ### should be different for k,o in outputs_init.items(): assert not (o == outputs_default[k]).any() ### save initial states to compare ini_states = {} for k in i: try: if k.startswith('init'): ini_states[k] = i[k] except: pass ### simulation with initial states read from dict get_initial_states_dict(input_map,period,extent) sim = OnDemandSimulator(awral,input_map) r,i = sim.run(period,extent,return_inputs=True) outputs_init_dict = r['final_states'] ### compare final states with other ini states simulation ### should be same for k,o in outputs_init_dict.items(): assert (o == outputs_init[k]).any() ### compare initial states from both methods ### should be same for k in i: try: if k.startswith('init'): assert ini_states[k] == i[k] except: pass ### simulation with initial states read from nc files get_initial_states(input_map) sim = OnDemandSimulator(awral,input_map) r,i = sim.run(period,extent,return_inputs=True) outputs_init = r['final_states'] ### compare final states with other ini states simulation ### should be same for k,o in outputs_init_dict.items(): assert (o == outputs_init[k]).any() ### compare initial states from both methods ### should be same for k in i: try: if k.startswith('init'): assert ini_states[k] == i[k] except: pass
def test_asce(): import awrams.models.settings import awrams.utils.nodegraph.nodes as nodes # awrams.models.settings.CLIMATE_DATA = _join(TRAINING_DATA_PATH,'climate/BOM_climate/') bom_data_path = '/data/cwd_awra_data/awra_inputs/climate_generated_sdcvd-awrap01/' wind_data_path = '/data/cwd_awra_data/awra_inputs/mcvicar_wind/' fveg_data_path = '/data/cwd_awra_data/awra_inputs/modis_fveg/8day/pv/' albedo_data_path = '/data/cwd_awra_data/awra_inputs/randall_albedo/' temp_min_2day_datapath = '/data/cwd_awra_data/awra_inputs/2dayAveTmin/' FORCING = { 'tmin': ('temp_min*.nc', 'temp_min_day', bom_data_path + 'temp_min_day/'), 'tmin2da': ('temp_min*.nc', 'temp_min_2dayave', temp_min_2day_datapath), 'tmax': ('temp_max*.nc', 'temp_max_day', bom_data_path + 'temp_max_day/'), 'precip': ('rain_day*.nc', 'rain_day', bom_data_path + 'rain_day/'), 'solar': ('solar*.nc', 'solar_exposure_day', bom_data_path + 'solar_exposure_day/'), #, 'vprp': ('vapour_pressure*.nc', 'vapour_pressure', bom_data_path + 'vapour_pressure/'), #, #h09 'wind': ('wind*.nc', 'wind', wind_data_path), 'fveg': ('fveg*.nc', 'fveg', fveg_data_path), 'alb': ('albedo*.nc', 'albedo', albedo_data_path) } import awrams.models.et.settings awrams.models.et.settings.FORCING = { k + "_f": dict(zip(("pattern", "nc_var", "path"), v)) for k, v in FORCING.items() } ''' Required inputs mortons: tmin,tmax,solar,vapour pressure ### apparently not albedo fao56: tmin,tmax,solar,vapour pressure,wind penpan: tmin2da,tmax,solar,vapour pressure,wind penman: tmin,tmax,solar,vapour pressure,wind,veg,albedo ''' from awrams.utils import extents from awrams.simulation.ondemand import OnDemandSimulator from awrams.simulation.server import SimulationServer from awrams.models.et.model import ETModel et = ETModel() # et.set_models_to_run(["fao56","morton_shallow_lake","morton_areal","penpan","penman"]) # et.set_models_to_run(["fao56","morton_areal","penpan"]) # et.set_models_to_run(["morton_areal"]) # et.set_models_to_run(["fao56"]) # et.set_models_to_run(["penpan"]) # et.set_models_to_run(["penman"]) et.set_models_to_run(["asce"]) # et.set_models_to_run(["morton_shallow_lake"]) imap = et.get_default_mapping() # print(imap['fveg']) # print(imap['fveg_f']) # imap['fveg_f'] = nodes.forcing_from_modis_ncfiles(fveg_data_path,'fveg_filled.nc','fveg') # imap['fveg'] = nodes.mul('fveg_f',0.01) # print(imap['fveg_f']) # exit() omap = et.get_output_mapping() et.save_outputs(omap, path="./outputs/ss/") # et.save_outputs(omap,path="/data/cwd_awra_data/awra_test_outputs/Scheduled_et_sdcvt-awrap01/") # print(omap) # omap['nc_msl_et'] = nodes.write_to_annual_ncfile("./",'msl_et') extent = extents.get_default_extent() # extent = extent.icoords[-32:-34,115:118] # extent = extent.ioffset[440,95] # extent = extent.ioffset[490,90] # extent = extent.ioffset[277:304,50] # extent = extent.ioffset[256:265,60] # extent = extent.ioffset[455,770:791] # extent = extent.ioffset[230:240,120] # print("Extent",extent) # period = pd.date_range("1 jan 2010","31 dec 2011",freq='D') # period = pd.date_range("1 jan 2010",periods=1,freq='D') period = pd.date_range("1 jan 2011", "31 dec 2011", freq='D') # period = pd.date_range("1 jan 2016","31 jan 2016",freq='D') # period = pd.date_range("1 jan 1990","30 apr 2018",freq='D') # sim = OnDemandSimulator(et,imap,omapping=omap) # print(sim.input_runner.input_graph['models']['exe'].get_dataspec()) # r = sim.run(period,extent) # # print(r) sim = SimulationServer(et) sim.run(imap, omap, period, extent)
""" Simple script to verify that running an OnDemandSimulator will produce the same results as our cluster sim """ from awrams.utils import datetools as dt from awrams.utils import extents import numpy as np from awrams.utils.nodegraph import nodes, graph from awrams.simulation.ondemand import OnDemandSimulator full_extent = extents.get_default_extent() period = dt.dates('dec 2010 - jan 2011') extent = full_extent.ioffset[200:250, 200:250] from awrams.models.awral import model m = model.AWRALModel() imap = m.get_default_mapping() ods = OnDemandSimulator(m, imap) print("running...") res = ods.run(period, extent) from awrams.utils.io.data_mapping import SplitFileManager print("opening comparison results") sfm = SplitFileManager.open_existing('./test_sim_outputs', 'qtot*', 'qtot')