def get_settings(sys_settings=None):
if sys_settings is None:
sys_settings = config_manager.get_system_profile().get_settings()
model_data_path = sys_settings['DATA_PATHS']['MODEL_DATA']
model_code_path = _join(sys_settings['DATA_PATHS']['CODE'],'models')
model_build_path = _join(sys_settings['DATA_PATHS']['BUILD_CACHE'],'models')
version = config_options['MODEL_VERSION']
model_config_path = config_manager.get_config_path('models/awral')
profile = {
'SPATIAL_FILE': _join(model_data_path, 'awral/spatial_parameters_v5.h5'),
'PARAMETER_FILE': _join(model_config_path, 'parameters/DefaultParameters_v5.json'),
'OUTPUTS': config_options['OUTPUTS'],
'CLIMATE_DATASET': sys_settings['CLIMATE_DATASETS']['TRAINING'],
'BUILD_SETTINGS': {
'SRC_FILENAME_BASE': 'awral',
'CORE_SRC_PATH': _join(model_code_path, 'awral/%s' % version),
'BUILD_STR': config_options['BUILD_STRINGS'][sys_settings['COMPILER']],
'BUILD_DEST_PATH': _join(model_build_path, 'awral/%s' % version)
},
'CONFIG_OPTIONS': config_options
}
return objectify(profile)
def get_input_mapping(model_settings=None):
"""
Return the default input mapping for this model
This is a dict of key:GraphNode mappings
Return:
mapping (dict)
"""
if model_settings is None:
model_settings = get_settings()
mapping = {}
#for k,spec in config_options['PARAMETER_SPECS'].items():
# mapping[k] = nodes.parameter_from_json(
# model_settings['PARAMETER_FILE'], k, spec[0],spec[1],spec[2])
param_df = parameters.wirada_json_to_param_df(model_settings['PARAMETER_FILE'])
mapping = parameters.param_df_to_mapping(param_df,mapping)
SPATIAL_GRIDS = ['f_tree', 'height', 'hveg_dr', 'k0sat_v5', 'k_gw', 'kdsat_v5', 'kssat_v5', 'lai_max',
'meanPET', 'ne', 'pref', 's0fracAWC', 'slope', 'ssfracAWC', 'windspeed']
for grid in SPATIAL_GRIDS:
mapping[grid.lower()+'_grid'] = nodes.spatial_from_file(
model_settings['SPATIAL_FILE'], 'parameters/%s' % grid)
FORCING_DATA = model_settings['CLIMATE_DATASET']['FORCING']
CLIMATOLOGY = model_settings['CLIMATE_DATASET']['CLIMATOLOGY']
for k in ['tmin', 'tmax', 'precip']:
var_map = FORCING_DATA['MAPPING'][k]
mapping[k+'_f'] = nodes.forcing_from_ncfiles(FORCING_DATA['PATH'], var_map[0], var_map[1])
mapping['solar_f'] = nodes.forcing_gap_filler(FORCING_DATA['PATH'],FORCING_DATA['MAPPING']['solar'][0], \
FORCING_DATA['MAPPING']['solar'][1],CLIMATOLOGY['solar'][0])
mapping.update({
'tmin': nodes.transform(np.minimum, ['tmin_f', 'tmax_f']),
'tmax': nodes.transform(np.maximum, ['tmin_f', 'tmax_f']),
'hypsperc_f': nodes.const_from_hdf5(model_settings['SPATIAL_FILE'], 'dimensions/hypsometric_percentile', ['hypsometric_percentile']),
# Model needs 0-1.0, file represents as 0-100
'hypsperc': nodes.mul('hypsperc_f', 0.01),
'fday': transforms.fday(),
'u2t': transforms.u2t('windspeed_grid','fday')
})
mapping['height'] = nodes.assign('height_grid')
mapping['er_frac_ref_hrusr'] = nodes.mul('er_frac_ref_hrudr', 0.5)
mapping['k_rout'] = nodes.transform(
transforms.k_rout, ('k_rout_scale', 'k_rout_int', 'meanpet_grid'))
mapping['k_gw'] = nodes.mul('k_gw_scale', 'k_gw_grid')
mapping['s0max'] = nodes.mul('s0max_scale', 's0fracawc_grid', 100.)
mapping['ssmax'] = nodes.mul('ssmax_scale', 'ssfracawc_grid', 900.)
mapping['sdmax'] = nodes.mul('ssmax_scale','sdmax_scale','ssfracawc_grid',5000.)
mapping['k0sat'] = nodes.mul('k0sat_scale', 'k0sat_v5_grid')
mapping['kssat'] = nodes.mul('kssat_scale', 'kssat_v5_grid')
mapping['kdsat'] = nodes.mul('kdsat_scale', 'kdsat_v5_grid')
mapping['kr_0s'] = nodes.transform(
transforms.interlayer_k, ('k0sat', 'kssat'))
mapping['kr_sd'] = nodes.transform(
transforms.interlayer_k, ('kssat', 'kdsat'))
mapping['prefr'] = nodes.mul('pref_gridscale', 'pref_grid')
mapping['fhru_hrusr'] = nodes.sub(1.0, 'f_tree_grid')
mapping['fhru_hrudr'] = nodes.assign('f_tree_grid')
mapping['ne'] = nodes.mul('ne_scale', 'ne_grid')
mapping['slope'] = nodes.assign('slope_grid')
mapping['hveg_hrudr'] = nodes.assign('hveg_dr_grid')
mapping['laimax_hrusr'] = nodes.assign('lai_max_grid')
mapping['laimax_hrudr'] = nodes.assign('lai_max_grid')
mapping['pair'] = nodes.const(97500.)
mapping['pt'] = nodes.assign('precip_f')
mapping['rgt'] = nodes.transform(np.maximum, ['solar_f', 0.1])
mapping['tat'] = nodes.mix('tmin', 'tmax', 0.75)
mapping['avpt'] = nodes.transform(transforms.pe, 'tmin')
mapping['radcskyt'] = transforms.radcskyt()
mapping['init_sr'] = nodes.const(0.0)
mapping['init_sg'] = nodes.const(100.0)
for hru in ('_hrusr', '_hrudr'):
mapping['init_mleaf'+hru] = nodes.div(2.0, 'sla'+hru)
for state in ["s0", "ss", "sd"]:
mapping['init_'+state+hru] = nodes.mul(state+'max', 0.5)
return objectify(mapping)
def get_settings():
TEST_DATA_PATH = join(BASE_DATA_PATH, 'test_data')
TRAINING_DATA_PATH = join(BASE_DATA_PATH, 'training')
CLIMATOLOGIES = {
'AWAP_DAILY': {
'solar':
(join(BASE_DATA_PATH,
'climatology/climatology_daily_solar_exposure_day.nc'),
'solar_exposure_day')
}
}
settings = {
'config_options':
config_options,
'DATA_PATHS': {
'AWRAMS_BASE': AWRAMS_BASE_PATH,
'BASE_DATA': BASE_DATA_PATH,
'MASKS': join(BASE_DATA_PATH, 'spatial/masks'),
'SHAPEFILES': join(BASE_DATA_PATH, 'spatial/shapefiles'),
'TEST_DATA': TEST_DATA_PATH,
'MODEL_DATA': join(BASE_DATA_PATH, 'model_data'),
'CODE': join(AWRAMS_BASE_PATH, 'code'),
'BUILD_CACHE': join(AWRAMS_BASE_PATH, 'build_cache')
},
'SIMULATION': {
'SPATIAL_CHUNK': 128,
'TIME_CHUNK': 32,
'MIN_CELLS_PER_WORKER': 32,
'TASK_BUFFERS': 3
},
# +++ Should move to external file so datasets can be shared between profiles
'CLIMATE_DATASETS': {
'TRAINING': {
'FORCING': {
'PATH': join(TRAINING_DATA_PATH, 'climate/bom_awap'),
'MAPPING': FORCING_MAP_AWAP
},
'CLIMATOLOGY': CLIMATOLOGIES['AWAP_DAILY']
},
'TESTING': {
'FORCING': {
'PATH': join(TEST_DATA_PATH, 'simulation/climate'),
'MAPPING': FORCING_MAP_AWAP
},
'CLIMATOLOGY': CLIMATOLOGIES['AWAP_DAILY']
}
},
'MPI_COMMON_OPTIONS': [
'--oversubscribe', '--allow-run-as-root',
'--mca plm_rsh_agent false', '-x TMPDIR=/dev/shm/'
],
'REMOTE_SETTINGS':
config_options['REMOTE_SETTINGS'].copy(),
'COMPILER':
'ICC_DEFAULT'
}
if config_options['DEV_MODE'] == True:
DEV_ACTIVATE = 'export PYTHONPATH=$PYTHONPATH:{REPO_PATH}\n'.format(
REPO_PATH=config_options['REPO_PATH'])
else:
DEV_ACTIVATE = ''
activation_str = settings['REMOTE_SETTINGS']['ACTIVATION'].format(
AWRAMS_BASE_PATH=AWRAMS_BASE_PATH, DEV_ACTIVATE=DEV_ACTIVATE)
settings['REMOTE_SETTINGS']['ACTIVATION'] = ''.join(
[s.lstrip(' ') + '\n' for s in activation_str.splitlines()])
return objectify(settings)
# Cell level outputs; ie those not contained within the HRU
OUTPUTS_CELL = ['qtot', 'sr', 'sg']
),
#Core Code Options
# - compiler settings
# - source/target file locations
'BUILD_STRINGS' : dict(
ICC_DEFAULT = "icc %s -march=native -std=c99 -static-intel --shared -fPIC -O3 -o %s",
GCC_DEFAULT = "gcc %s -std=c99 --shared -fPIC -O3 -o %s",
CLANG_WINDOWS = "clang %s --shared -std=c99 -O3 -o %s"
)
}
config_options = objectify(config_options)
def get_settings(sys_settings=None):
if sys_settings is None:
sys_settings = config_manager.get_system_profile().get_settings()
model_data_path = sys_settings['DATA_PATHS']['MODEL_DATA']
model_code_path = _join(sys_settings['DATA_PATHS']['CODE'],'models')
model_build_path = _join(sys_settings['DATA_PATHS']['BUILD_CACHE'],'models')
version = config_options['MODEL_VERSION']
model_config_path = config_manager.get_config_path('models/awral')
profile = {
def get_settings():
TEST_DATA_PATH = join(BASE_DATA_PATH, 'test_data')
TRAINING_DATA_PATH = join(BASE_DATA_PATH, 'training')
benchmark_sites_file = join(BASE_DATA_PATH,
'benchmarking/SiteLocationsWithUniqueID.csv')
SHAPEFILES = join(BASE_DATA_PATH, 'spatial/shapefiles')
CLIMATOLOGIES = {
'AWAP_DAILY': {
'solar':
(join(BASE_DATA_PATH,
'climatology/climatology_daily_solar_exposure_day.nc'),
'solar_exposure_day')
}
}
if os.name == 'nt':
COMPILER = 'CLANG_WINDOWS'
else:
COMPILER = 'GCC_DEFAULT'
settings = {
'DATA_PATHS': {
'AWRAMS_BASE': AWRAMS_BASE_PATH,
'BASE_DATA': BASE_DATA_PATH,
'MASKS': join(BASE_DATA_PATH, 'spatial/masks'),
'SHAPEFILES': SHAPEFILES,
'CATCHMENT_SHAPEFILE': join(SHAPEFILES,'Final_list_all_attributes.shp'),
'TEST_DATA': TEST_DATA_PATH,
'TRAINING_DATA': TRAINING_DATA_PATH,
'MODEL_DATA': join(BASE_DATA_PATH, 'model_data'),
'CODE': join(AWRAMS_BASE_PATH, 'code'),
'ASCAT': {
'TRAINING': join(TRAINING_DATA_PATH, 'benchmarking/ascat/'),
'TEST': join(TEST_DATA_PATH, 'benchmarking/ascat/')
},
'BUILD_CACHE': join(AWRAMS_BASE_PATH, 'build_cache')
},
'SIMULATION': {
'SPATIAL_CHUNK': 128,
'TIME_CHUNK': 32,
'MIN_CELLS_PER_WORKER': 32,
'TASK_BUFFERS': 3
},
# +++ Should move to external file so datasets can be shared between profiles
'CLIMATE_DATASETS': {
'TRAINING': {
'FORCING': {
'PATH': join(TRAINING_DATA_PATH, 'climate/bom_awap'),
'MAPPING': FORCING_MAP_AWAP
},
'CLIMATOLOGY': CLIMATOLOGIES['AWAP_DAILY']
},
'TESTING': {
'FORCING': {
'PATH': join(TEST_DATA_PATH, 'simulation/climate'),
'MAPPING': FORCING_MAP_AWAP
},
'CLIMATOLOGY': CLIMATOLOGIES['AWAP_DAILY']
}
},
'BENCHMARKING': {
'BENCHMARK_SITES': benchmark_sites_file,
'MONTHLY_REJECTION_THRESHOLD': 15,
'ANNUAL_REJECTION_THRESHOLD': 6,
'SM_MODEL_VARNAMES': ['s0_avg', 'ss_avg', 'sd_avg'],
'SM_MODEL_LAYERS': {'s0_avg': 100., 'ss_avg': 900., 'sd_avg':
5000.},
'SM_OBSERVED_LAYERS': ('profile','top','shallow','middle','deep'),
'FIG_SIZE': (14,6),
'CELLSIZE': 0.05,
'LANDSCAPE_VERSION_EQUIVALENCE': {"5":"45","5R":"45","5Q":"45"}
},
# Preferred compiler; referenced in model settings
'COMPILER': COMPILER,
'IO_SETTINGS' : {
'CHUNKSIZES': config_options['CHUNKSIZES'],
'DEFAULT_CHUNKSIZE': (config_options['CHUNKSIZES']['TIME'], \
config_options['CHUNKSIZES']['SPATIAL'], \
config_options['CHUNKSIZES']['SPATIAL']),
'VAR_CHUNK_CACHE_SIZE': 2**20, # =1048576 ie 1Mb
'VAR_CHUNK_CACHE_NELEMS': 1009, # prime number
'VAR_CHUNK_CACHE_PREEMPTION': 0.75, # 1 for read or write only
# '_fallthrough' will attempt to use _h5py, then netCDF4 if that fails
'DB_OPEN_WITH': '_fallthrough', #"_h5py" OR "_nc"
'MAX_FILES_PER_SFM': 32, # Maximum files allowed open in each SplitFileManager.
# Maximum chunksize to read during extraction (in bytes)
'MAX_EXTRACT_CHUNK': 2**24
},
'LOGGER_SETTINGS': config_options['LOGGER_SETTINGS'],
# Used in extents.get_default_extent
# Consider creating extents objects explicitly from files rather than using this method.
# It exists for backwards compatibility, and will be deprecated
'DEFAULT_MASK_FILE': 'web_mask_v5.h5'
}
return objectify(settings)