def setup_v5():
from awrams.utils import config_manager
sys_settings = config_manager.get_system_profile('default').get_settings()
model_profile = config_manager.get_model_profile('awral','v5_default')
model_settings = model_profile.get_settings()
global awral
awral = model_profile.get_model(model_settings)
global period
period = dt.dates('dec 2010 - jan 2011')
global input_map
input_map = model_profile.get_input_mapping(model_settings)
model_settings['CLIMATE_DATASET'] = sys_settings['CLIMATE_DATASETS']['TESTING']
global output_map
output_map = awral.get_output_mapping()
global outpath
outpath = join(dirname(__file__),'..','..','test_data','simulation','outputs')
output_map['s0_ncsave'] = nodes.write_to_annual_ncfile(outpath,'s0')
output_map['mleaf_hrudr_state'] = nodes.write_to_ncfile_snapshot(outpath,'mleaf_hrudr')
def __init__(self, model, sys_settings=None):
if not isinstance(model, Model):
raise TypeError("model must be of type awrams.models.model.Model")
### defaults
if sys_settings is None:
sys_settings = config_manager.get_system_profile().get_settings()
self.sys_settings = sys_settings
sim_settings = sys_settings['SIMULATION']
self.spatial_chunk = sim_settings['SPATIAL_CHUNK']
self.time_chunk = sim_settings['TIME_CHUNK']
self.min_cells_per_worker = sim_settings['MIN_CELLS_PER_WORKER']
self.num_workers = os.cpu_count()
self.task_buffers = sim_settings['TASK_BUFFERS']
# This value is deprecated as of AWRACMS 1.2, but kept for old scripts that may attempt
# to set it. Setting read_ahead will have no effect, and will be removed in later versions
self.read_ahead = 1
self.model = model
self.logger = logger
def get_settings(sys_settings=None):
if sys_settings is None:
sys_settings = config_manager.get_system_profile().get_settings()
model_data_path = sys_settings['DATA_PATHS']['MODEL_DATA']
model_code_path = _join(sys_settings['DATA_PATHS']['CODE'],'models')
model_build_path = _join(sys_settings['DATA_PATHS']['BUILD_CACHE'],'models')
version = config_options['MODEL_VERSION']
model_config_path = config_manager.get_config_path('models/awral')
profile = {
'SPATIAL_FILE': _join(model_data_path, 'awral/spatial_parameters_v5.h5'),
'PARAMETER_FILE': _join(model_config_path, 'parameters/DefaultParameters_v5.json'),
'OUTPUTS': config_options['OUTPUTS'],
'CLIMATE_DATASET': sys_settings['CLIMATE_DATASETS']['TRAINING'],
'BUILD_SETTINGS': {
'SRC_FILENAME_BASE': 'awral',
'CORE_SRC_PATH': _join(model_code_path, 'awral/%s' % version),
'BUILD_STR': config_options['BUILD_STRINGS'][sys_settings['COMPILER']],
'BUILD_DEST_PATH': _join(model_build_path, 'awral/%s' % version)
},
'CONFIG_OPTIONS': config_options
}
return objectify(profile)
def build_output_mapping(model,outpath,save_vars=None,mode='a',dtype=np.float32,save_states_freq=None):
'''
Convenience function to save all models outputs to outpath as annual netCDF files
'''
### populate output map with all model outputs
io_settings = config_manager.get_system_profile().get_settings()['IO_SETTINGS']
output_mapping = model.get_output_mapping()
output_map_ncwrite = output_mapping.copy()
if save_vars is None:
save_vars = list(output_mapping)
for k in save_vars:
if k not in output_mapping:
raise Exception("Variable %s not activated in model output settings" %k)
output_map_ncwrite[k+'_ncsave'] = nodes.write_to_annual_ncfile(outpath,k,mode=mode,dtype=dtype)
if save_states_freq is not None:
state_keys = model.get_state_keys()
state_path = os.path.join(outpath,'states')
spatial_chunk = io_settings['CHUNKSIZES']['SPATIAL']
for k in state_keys:
if k not in output_mapping:
raise Exception("State %s not activated in model output settings" %k)
output_map_ncwrite[k+'_statesave'] = nodes.write_to_ncfile_snapshot(state_path,k,mode=mode, \
freq=save_states_freq,dtype=np.float64, \
chunksizes = (1,spatial_chunk,spatial_chunk))
return output_map_ncwrite
def test_shapefile_db():
try:
import osgeo
except ImportError:
return
from awrams.utils import config_manager
sys_profile = config_manager.get_system_profile().get_settings()
CATCHMENT_SHAPEFILE = join(sys_profile['DATA_PATHS']['SHAPEFILES'],
'Final_list_all_attributes.shp')
sdb = gis.ShapefileDB(CATCHMENT_SHAPEFILE)
e = extents.get_default_extent()
df = sdb.get_records_df()
df = df.set_index('StationID')
rec = df.loc['003303']
extent = sdb.get_extent_by_field('StationID', '003303', e)
area_diff = np.abs(extent.areas.sum() - (rec['AlbersArea'] * 1e6))
assert (area_diff / extent.areas.sum() < 1.0001)
br = extent.bounding_rect()
assert (between(rec['CentrLat'], br[0], br[1]))
assert (between(rec['CentrLon'], br[2], br[3]))
def test_extraction():
from awrams.utils import gis
from awrams.utils import config_manager
sys_settings = config_manager.get_system_profile().get_settings()
CATCHMENT_SHAPEFILE = os.path.join(sys_settings['DATA_PATHS']['SHAPEFILES'], \
'Final_list_all_attributes.shp')
catchments = gis.ShapefileDB(CATCHMENT_SHAPEFILE)
e_all = extents.get_default_extent()
e = catchments.get_extent_by_field('StationID', '421103', e_all)
period = dt.dates('jun 1990 - jan 1995')
var_name = 'rain_day'
path, _ = os.path.split(os.path.abspath(__file__))
data_path = os.path.join(sys_settings['DATA_PATHS']['TEST_DATA'],
'calibration/')
pattern = data_path + '/%s*' % var_name
df = extract(data_path, pattern, var_name, {
'241': e,
'512': e_all.ioffset[400, 400]
}, period)
assert ((df.index == period).all())
def get_default_extent():
warnings.warn("Pending deprecation: create extents from files instead",
PendingDeprecationWarning)
sys_settings = config_manager.get_system_profile().get_settings()
full_mask_filename = os.path.join(sys_settings['DATA_PATHS']['MASKS'],
sys_settings['DEFAULT_MASK_FILE'])
return Extent.from_file(full_mask_filename)
def get_initial_states(imap):
data_paths = config_manager.get_system_profile('default').get_settings()['DATA_PATHS']
data_path = join(data_paths['BASE_DATA'],'test_data','simulation')
mapping = imap
mapping['init_sr'] = nodes.init_state_from_ncfile(data_path,'sr_bal*','sr_bal')
mapping['init_sg'] = nodes.init_state_from_ncfile(data_path,'sg_bal*','sg_bal')
HRU = {'_hrusr':'******','_hrudr':'_dr'}
for hru in ('_hrusr','_hrudr'):
for state in ["s0","ss","sd",'mleaf']:
mapping['init_'+state+hru] = nodes.init_state_from_ncfile(data_path,state+HRU[hru]+'*',state+HRU[hru])
def setup():
clear_files()
from awrams.models.awral import model
from awrams.utils import config_manager
sys_settings = config_manager.get_system_profile().get_settings()
model_profile = config_manager.get_model_profile('awral','v6_default')
model_settings = model_profile.get_settings()
model_settings['CLIMATE_DATASET'] = sys_settings['CLIMATE_DATASETS']['TESTING']
global awral, input_map
awral = model_profile.get_model(model_settings)
input_map = model_profile.get_input_mapping(model_settings)
def get_initial_states_dict(imap,period,extent):
data_map = {}
data_paths = config_manager.get_system_profile('default').get_settings()['DATA_PATHS']
data_path = join(data_paths['BASE_DATA'],'test_data','simulation')
period = [period[0] - 1]
node_names = {'mleaf_dr': 'init_mleaf_hrudr',
'mleaf_sr': 'init_mleaf_hrusr',
's0_dr': 'init_s0_hrudr',
's0_sr': 'init_s0_hrusr',
'ss_dr': 'init_ss_hrudr',
'ss_sr': 'init_ss_hrusr',
'sd_dr': 'init_sd_hrudr',
'sd_sr': 'init_sd_hrusr',
'sg_bal': 'init_sg',
'sr_bal': 'init_sr'}
for k in 'mleaf_dr','s0_dr','sd_dr','sg_bal','ss_dr','mleaf_sr','s0_sr','sd_sr','sr_bal','ss_sr':
sfm = SplitFileManager.open_existing(data_path,k+'*nc',k)
data_map[node_names[k]] = sfm.get_data(period,extent)[0]
nodes.init_states_from_dict(imap,data_map,extent)
def test_get_padded_by_coords():
from awrams.utils.io.data_mapping import SplitFileManager
from awrams.utils.mapping_types import gen_coordset
import awrams.utils.datetools as dt
data_paths = config_manager.get_system_profile().get_settings(
)['DATA_PATHS']
path = os.path.join(data_paths['BASE_DATA'], 'test_data', 'simulation',
'climate', 'temp_min_day')
sfm = SplitFileManager.open_existing(path, 'temp_min_day_*.nc',
'temp_min_day')
# return sfm
extent = sfm.get_extent().ioffset[200:230, 200:230]
period = dt.dates('2011')
coords = gen_coordset(period, extent)
data = sfm.get_padded_by_coords(coords)
assert data.shape == coords.shape
from nose.tools import nottest
import matplotlib
import os
matplotlib.use('agg')
from awrams.utils import config_manager
system_profile = config_manager.get_system_profile().get_settings()
TEST_DATA_PATH = system_profile['DATA_PATHS']['TEST_DATA']
def test_imports():
import awrams.benchmarking
import awrams.benchmarking.meta
#import awrams.benchmarking.processing
import awrams.benchmarking.benchmark
import awrams.benchmarking.comparison
import awrams.benchmarking.stats
import awrams.benchmarking.utils
import awrams.benchmarking.meta.oznet
import awrams.benchmarking.meta.sasmas
# import awrams.benchmarking.processing.oznet
# import awrams.benchmarking.processing.sasmas
@nottest
def test_force_fail():
assert False
def _get_max_chunk():
from awrams.utils import config_manager
sys_settings = config_manager.get_system_profile().get_settings()
return sys_settings['IO_SETTINGS']['MAX_EXTRACT_CHUNK']
import h5py
import numpy as np
from collections import OrderedDict
import types
from awrams.utils.awrams_log import get_module_logger
logger = get_module_logger('awrams.utils.io')
#from awrams.utils.settings import VAR_CHUNK_CACHE_SIZE, VAR_CHUNK_CACHE_NELEMS, VAR_CHUNK_CACHE_PREEMPTION#pylint: disable=no-name-in-module
from awrams.utils.config_manager import get_system_profile
sys_settings = get_system_profile().get_settings()
VAR_CHUNK_CACHE_SIZE = sys_settings['IO_SETTINGS']['VAR_CHUNK_CACHE_SIZE']
VAR_CHUNK_CACHE_NELEMS = sys_settings['IO_SETTINGS']['VAR_CHUNK_CACHE_NELEMS']
VAR_CHUNK_CACHE_PREEMPTION = sys_settings['IO_SETTINGS'][
'VAR_CHUNK_CACHE_PREEMPTION']
propfaid = h5py.h5p.create(h5py.h5p.FILE_ACCESS)
settings = list(propfaid.get_cache())
#settings[1] # size of hash table
settings[2] = 0 #2**17 # =131072 size of chunk cache in bytes
# which is big enough for 5x(75, 1, 50 chunks;
# default is 2**20 =1048576
settings[3] = 1. # preemption 1 suited to whole chunk read/write
propfaid.set_cache(*settings)
propfaid.set_fapl_sec2()
propfaid.set_sieve_buf_size(0)
propfaid.set_fclose_degree(h5py.h5f.CLOSE_STRONG)
#propfaid.set_fapl_stdio()
import numpy as np
import os
from os.path import join
from awrams.utils import config_manager
import awrams.utils.extents as extents
from awrams.utils import geo
from numpy.testing import assert_allclose
from nose.tools import nottest, assert_almost_equal
SYS_PROFILE = config_manager.get_system_profile().get_settings()
MASK_FILE = join(SYS_PROFILE['DATA_PATHS']['MASKS'], 'web_mask_v5.h5')
@nottest
def build_mock_array(extent):
data = np.zeros(shape=(extent.shape))
for i in range(0, data.shape[0]):
for j in range(0, data.shape[1]):
data[i, j] = i * data.shape[0] + j
return data
# @nottest
def test_extent_all():
e = extents.Extent.from_file(MASK_FILE)
assert (e.shape == (681, 841))
assert (e.cell_count == 281655)
import numpy as np
from awrams.utils.config_manager import get_system_profile
CHUNKSIZE = get_system_profile().get_settings()['IO_SETTINGS']['DEFAULT_CHUNKSIZE']
class InputReader:
def __init__(self, variable, nlons=886):
self.ncd_time_slices = None
self.variable = variable
self.row = None
self.nlons=886
def __getitem__(self, spatial_slice):
return np.ma.concatenate(
[nct[0].variables[self.variable][nct[1],spatial_slice[0],spatial_slice[1]]
for nct in self.ncd_time_slices])
class InputChunkReader(InputReader):
# +++ Not referenced?
#from memory_tester import memory_usage
def __getitem__(self, cell):
if self.row is None:
self.row_chunk = int(cell[1] / CHUNKSIZE[2])
self.read_chunks((cell[0], self.row_chunk)) #int(cell[1] / CHUNKSIZE[2]) * CHUNKSIZE[2]))
self.row = cell[0]
if cell[0] != self.row or self.row_chunk != int(cell[1] / CHUNKSIZE[2]):
self.row_chunk = int(cell[1] / CHUNKSIZE[2])
self.read_chunks((cell[0], self.row_chunk)) #, int(cell[1] / CHUNKSIZE[2]) * CHUNKSIZE[2]))
self.row = cell[0]