def setup_config(self, autocloseFileLimitFraction=0.5, maxChunkSize=10000):
config = MpasAnalysisConfigParser()
config.add_section('input')
config.set('input', 'autocloseFileLimitFraction',
str(autocloseFileLimitFraction))
config.set('input', 'maxChunkSize', str(maxChunkSize))
return config
def get_latlon_array_descriptor(self):
configPath = str(self.datadir.join('config.analysis'))
config = MpasAnalysisConfigParser()
config.read(configPath)
lat = numpy.array(config.getExpression('interpolate', 'lat',
usenumpyfunc=True))
lon = numpy.array(config.getExpression('interpolate', 'lon',
usenumpyfunc=True))
descriptor = LatLonGridDescriptor.create(lat, lon, units='degrees')
return descriptor
def doTest(generate, expectedResults):
config = MpasAnalysisConfigParser()
config.add_section('output')
config.set('output', 'generate', generate)
for analysisName in expectedResults:
expectedResult = expectedResults[analysisName]
result = checkGenerate(
config,
analysisName=analysisName,
mpasCore=cores[analysisName],
analysisCategory=categories[analysisName])
self.assertEqual(result, expectedResult)
def doTest(generate, expectedResults):
config = MpasAnalysisConfigParser()
config.add_section('output')
config.set('output', 'generate', generate)
for taskName in expectedResults:
genericTask = AnalysisTask(config=config,
taskName=taskName,
componentName=cores[taskName],
tags=tags[taskName])
expectedResult = expectedResults[taskName]
result = genericTask.check_generate()
self.assertEqual(result, expectedResult)
def test_destination_from_numpy_lat_lon(self):
"""
test horizontal interpolation from an MPAS mesh to a destination
lat/lon grid determined from config options 'lat' and 'lon'.
Xylar Asay-Davis
02/25/2017
"""
configPath = str(self.datadir.join('config.analysis'))
config = MpasAnalysisConfigParser()
config.read(configPath)
lat = numpy.array(
config.getExpression('interpolate', 'lat', usenumpyfunc=True))
lon = numpy.array(
config.getExpression('interpolate', 'lon', usenumpyfunc=True))
mpasMeshFileName = str(self.datadir.join('mpasMesh.nc'))
timeSeriesFileName = str(self.datadir.join('timeSeries.0002-01-01.nc'))
suffix = 'destination_from_config_options'
weightFileName = '{}/weights_{}.nc'.format(self.test_dir, suffix)
outFileName = '{}/remapped_{}.nc'.format(self.test_dir, suffix)
interpolate.build_remap_weights(sourceFileName=mpasMeshFileName,
outWeightFileName=weightFileName,
sourceFileType='mpas',
method='bilinear',
destinationLat=lat,
destinationLon=lon)
assert os.path.exists(weightFileName)
interpolate.remap(inFileName=timeSeriesFileName,
outFileName=outFileName,
inWeightFileName=weightFileName,
sourceFileType='mpas')
assert os.path.exists(outFileName)
print ""
print "Plotting 2-d maps of sea-ice concentration and thickness " \
"climatologies..."
from mpas_analysis.sea_ice.modelvsobs import seaice_modelvsobs
seaice_modelvsobs(config,
streamMap=seaIceStreamMap,
variableMap=seaIceVariableMap)
# GENERATE LAND-ICE DIAGNOSTICS
if config.getboolean('plot', 'displayToScreen'):
plt.show()
return # }}}
if __name__ == "__main__":
# process command line arguments and run analysis from configuration
if len(sys.argv) <= 1:
print "usage: %s <in_config_file> [<in_config_file2>]" % sys.argv[0]
exit(1)
configFileNames = sys.argv[1:]
config = MpasAnalysisConfigParser()
config.read(configFileNames)
analysis(config)
# vim: foldmethod=marker ai ts=4 sts=4 et sw=4 ft=python
def setup_config(self, autocloseFileLimitFraction=0.5):
config = MpasAnalysisConfigParser()
config.add_section('input')
config.set('input', 'autocloseFileLimitFraction',
str(autocloseFileLimitFraction))
return config
import matplotlib.colors as colors
import pyproj
import sys
from mpas_analysis.shared.interpolation import Remapper
from mpas_analysis.shared.grid import ProjectionGridDescriptor
from mpas_analysis.shared.mpas_xarray.mpas_xarray import subset_variables
from mpas_analysis.shared.climatology \
import get_Antarctic_stereographic_comparison_descriptor
from mpas_analysis.configuration.MpasAnalysisConfigParser \
import MpasAnalysisConfigParser
inFileName = '/media/xylar/extra_data/data_overflow/observations/Antarctica/' \
'Rignot_et_al._2013/Ant_MeltingRate.nc'
config = MpasAnalysisConfigParser()
config.read('config.default')
ds = xarray.open_dataset(inFileName)
ds = subset_variables(ds, ['melt_actual', 'xaxis', 'yaxis'])
lx = numpy.abs(1e-3 * (ds.xaxis.values[-1] - ds.xaxis.values[0]))
ly = numpy.abs(1e-3 * (ds.yaxis.values[-1] - ds.yaxis.values[0]))
maskedMeltRate = numpy.ma.masked_array(ds.melt_actual,
mask=(ds.melt_actual.values == 0.))
ds['meltRate'] = xarray.DataArray(maskedMeltRate,
dims=ds.melt_actual.dims,
coords=ds.melt_actual.coords,
attrs=ds.melt_actual.attrs)
def setup_config(self, autocloseFileLimitFraction=0.5,
maxChunkSize=10000):
config = MpasAnalysisConfigParser()
config.add_section('input')
config.set('input', 'autocloseFileLimitFraction',
str(autocloseFileLimitFraction))
config.set('input', 'maxChunkSize', str(maxChunkSize))
config.set('input', 'mpasMeshName', 'QU240')
config.add_section('output')
config.set('output', 'baseDirectory', self.test_dir)
config.set('output', 'mappingSubdirectory', '.')
config.set('output', 'mpasClimatologySubdirectory', 'clim/mpas')
config.set('output', 'mpasRemappedClimSubdirectory',
'clim/mpas/remap')
config.add_section('climatology')
config.set('climatology', 'startYear', '2')
config.set('climatology', 'endYear', '2')
config.set('climatology', 'comparisonLatResolution', '0.5')
config.set('climatology', 'comparisonLonResolution', '0.5')
config.set('climatology', 'mpasInterpolationMethod', 'bilinear')
config.add_section('oceanObservations')
config.set('oceanObservations', 'interpolationMethod', 'bilinear')
config.set('oceanObservations', 'climatologySubdirectory', 'clim/obs')
config.set('oceanObservations', 'remappedClimSubdirectory',
'clim/obs/remap')
return config
def setup_config(self):
configPath = self.datadir.join('config.analysis')
self.config = MpasAnalysisConfigParser()
self.config.read(str(configPath))
class TestMPASAnalysisConfigParser(TestCase):
def setup_config(self):
configPath = self.datadir.join('config.analysis')
self.config = MpasAnalysisConfigParser()
self.config.read(str(configPath))
def check_container(self, container, container_type, item_type):
assert isinstance(container, container_type)
for item in container:
assert isinstance(item, item_type)
def test_read_config(self):
self.setup_config()
colorMapName = self.config.get('sst_modelvsobs', 'cmapDiff')
self.assertEqual(colorMapName, 'coolwarm')
self.assertEqual(self.config.getint('Test', 'testInt'), 15)
self.assertEqual(self.config.getExpression('Test', 'testInt'), 15)
self.assertEqual(self.config.getfloat('Test', 'testFloat'), 18.0)
self.assertEqual(self.config.getExpression('Test', 'testFloat'), 18.0)
self.assertEqual(self.config.getfloat('Test', 'testFloat2'), 3.)
self.assertEqual(self.config.getExpression('Test', 'testFloat2'), 3.)
self.assertEqual(self.config.getboolean('Test', 'testBool'), True)
self.assertEqual(self.config.getExpression('Test', 'testBool'), True)
testList = self.config.getExpression('sst_modelvsobs',
'cmapIndicesModelObs')
self.check_container(testList, list, int)
self.assertEqual(testList, [0, 40, 80, 110, 140, 170, 200, 230, 255])
testList = self.config.getExpression('sst_modelvsobs',
'cmapIndicesModelObs',
elementType=float)
self.check_container(testList, list, float)
self.assertEqual(testList, [0., 40., 80., 110., 140., 170., 200.,
230., 255.])
testList = self.config.getExpression('sst_modelvsobs',
'comparisonTimes')
self.check_container(testList, list, str)
self.assertEqual(testList, ['JFM', 'JAS', 'ANN'])
testList = self.config.getExpression('Test', 'testList')
self.check_container(testList, list, float)
self.assertEqual(testList, [0.5, 0.1, 0.5])
testTuple = self.config.getExpression('Test', 'testTuple')
assert isinstance(testTuple, tuple)
self.assertEqual(testTuple, (5, 0.1, 'string'))
testDict = self.config.getExpression('Test', 'testDict')
assert isinstance(testDict, dict)
self.assertEqual(testDict, {'key1': 'string',
'key2': -12,
'key3': False})
with self.assertRaisesRegexp(
configparser.NoOptionError,
"No option 'doesntexist' in section: 'Test'"):
self.config.getExpression(str('Test'), str('doesntexist'))
@requires_numpy
def test_read_config_numpy(self):
self.setup_config()
# tests numpy evaluation capability
import numpy as np
for testname in ['testNumpyarange' + str(ii) for ii in np.arange(3)]:
self.assertArrayEqual(self.config.getExpression('TestNumpy',
testname,
usenumpyfunc=True),
np.arange(0, 1, 10))
for testname in ['testNumpylinspace' + str(ii) for ii in np.arange(3)]:
self.assertArrayEqual(self.config.getExpression('TestNumpy',
testname,
usenumpyfunc=True),
np.linspace(0, 1, 10))
for testNumpy in ['testNumpypi' + str(ii) for ii in np.arange(3)] + \
['testNumpyPi']:
self.assertEqual(self.config.getExpression('TestNumpy', testNumpy,
usenumpyfunc=True),
np.pi)
with self.assertRaisesRegexp(
AssertionError,
"'__' is not allowed in .* for `usenumpyfunc=True`"):
self.config.getExpression('TestNumpy', 'testBadStr',
usenumpyfunc=True),
def test_get_with_default(self):
self.setup_config()
def check_get_with_default(name, value, dtype):
# test an options that doesn't exist using getWithDefault
var = self.config.getWithDefault('sst_modelvsobs', name, value)
assert isinstance(var, dtype)
self.assertEqual(var, value)
# test several types with getWithDefault
check_get_with_default(name='aBool', value=True, dtype=bool)
check_get_with_default(name='anInt', value=1, dtype=six.integer_types)
check_get_with_default(name='aFloat', value=1.0, dtype=float)
check_get_with_default(name='aList', value=[1, 2, 3], dtype=list)
check_get_with_default(name='aTuple', value=(1, 2, 3), dtype=tuple)
check_get_with_default(name='aDict', value={'blah': 1}, dtype=dict)
check_get_with_default(name='aStr', value='blah', dtype=six.string_types)
if config.getboolean('seaice_modelvsobs', 'generate'):
print ""
print "Plotting 2-d maps of sea-ice concentration and thickness " \
"climatologies..."
from mpas_analysis.sea_ice.modelvsobs import seaice_modelvsobs
seaice_modelvsobs(config, streamMap=seaIceStreamMap,
variableMap=seaIceVariableMap)
# GENERATE LAND-ICE DIAGNOSTICS
if config.getboolean('plot', 'displayToScreen'):
plt.show()
return # }}}
if __name__ == "__main__":
# process command line arguments and run analysis from configuration
if len(sys.argv) <= 1:
print "usage: %s <in_config_file> [<in_config_file2>]" % sys.argv[0]
exit(1)
configFileNames = sys.argv[1:]
config = MpasAnalysisConfigParser()
config.read(configFileNames)
analysis(config)
# vim: foldmethod=marker ai ts=4 sts=4 et sw=4 ft=python
def setup_config(self):
configPath = self.datadir.join('config.analysis')
self.config = MpasAnalysisConfigParser()
self.config.read(str(configPath))
class TestMPASAnalysisConfigParser(TestCase):
def setup_config(self):
configPath = self.datadir.join('config.analysis')
self.config = MpasAnalysisConfigParser()
self.config.read(str(configPath))
def check_container(self, container, container_type, item_type):
assert isinstance(container, container_type)
for item in container:
assert isinstance(item, item_type)
def test_read_config(self):
self.setup_config()
colorMapName = self.config.get('sst_modelvsobs', 'cmapDiff')
self.assertEqual(colorMapName, 'coolwarm')
self.assertEqual(self.config.getint('Test', 'testInt'), 15)
self.assertEqual(self.config.getfloat('Test', 'testFloat'), 18.)
self.assertEqual(self.config.getboolean('Test', 'testBool'), True)
testList = self.config.getExpression('sst_modelvsobs',
'cmapIndicesModelObs')
self.check_container(testList, list, int)
self.assertEqual(testList, [0, 40, 80, 110, 140, 170, 200, 230, 255])
testList = self.config.getExpression('sst_modelvsobs',
'cmapIndicesModelObs',
elementType=float)
self.check_container(testList, list, float)
self.assertEqual(testList, [0., 40., 80., 110., 140., 170., 200.,
230., 255.])
testList = self.config.getExpression('sst_modelvsobs',
'comparisonTimes')
self.check_container(testList, list, str)
self.assertEqual(testList, ['JFM', 'JAS', 'ANN'])
testList = self.config.getExpression('Test', 'testList')
self.check_container(testList, list, float)
self.assertEqual(testList, [0.5, 0.1, 0.5])
testTuple = self.config.getExpression('Test', 'testTuple')
assert isinstance(testTuple, tuple)
self.assertEqual(testTuple, (5, 0.1, 'string'))
testDict = self.config.getExpression('Test', 'testDict')
assert isinstance(testDict, dict)
self.assertEqual(testDict, {'key1': 'string',
'key2': -12,
'key3': False})
class TestMPASAnalysisConfigParser(TestCase):
def setup_config(self):
configPath = self.datadir.join('config.analysis')
self.config = MpasAnalysisConfigParser()
self.config.read(str(configPath))
def check_container(self, container, container_type, item_type):
assert isinstance(container, container_type)
for item in container:
assert isinstance(item, item_type)
def test_read_config(self):
self.setup_config()
colorMapName = self.config.get('sst_modelvsobs', 'cmapDiff')
self.assertEqual(colorMapName, 'coolwarm')
self.assertEqual(self.config.getint('Test', 'testInt'), 15)
self.assertEqual(self.config.getfloat('Test', 'testFloat'), 18.)
self.assertEqual(self.config.getboolean('Test', 'testBool'), True)
testList = self.config.getExpression('sst_modelvsobs',
'cmapIndicesModelObs')
self.check_container(testList, list, int)
self.assertEqual(testList, [0, 40, 80, 110, 140, 170, 200, 230, 255])
testList = self.config.getExpression('sst_modelvsobs',
'cmapIndicesModelObs',
elementType=float)
self.check_container(testList, list, float)
self.assertEqual(testList,
[0., 40., 80., 110., 140., 170., 200., 230., 255.])
testList = self.config.getExpression('sst_modelvsobs',
'comparisonTimes')
self.check_container(testList, list, str)
self.assertEqual(testList, ['JFM', 'JAS', 'ANN'])
testList = self.config.getExpression('Test', 'testList')
self.check_container(testList, list, float)
self.assertEqual(testList, [0.5, 0.1, 0.5])
testTuple = self.config.getExpression('Test', 'testTuple')
assert isinstance(testTuple, tuple)
self.assertEqual(testTuple, (5, 0.1, 'string'))
testDict = self.config.getExpression('Test', 'testDict')
assert isinstance(testDict, dict)
self.assertEqual(testDict, {
'key1': 'string',
'key2': -12,
'key3': False
})
help='config file')
args = parser.parse_args()
# add config.default to cover default not included in the config files
# provided on the command line
defaultConfig = '{}/config.default'.format(
os.path.dirname(os.path.realpath(__file__)))
if os.path.exists(defaultConfig):
configFiles = [defaultConfig] + args.configFiles
else:
warnings.warn('WARNING: Did not find config.default. Assuming other '
'config file(s) contain a\n'
'full set of configuration options.')
configFiles = args.configFiles
config = MpasAnalysisConfigParser()
config.read(configFiles)
if args.generate:
update_generate(config, args.generate)
logsDirectory = build_config_full_path(config, 'output',
'logsSubdirectory')
make_directories(logsDirectory)
make_directories('{}/configs/'.format(logsDirectory))
analyses = build_analysis_list(config)
parallelTaskCount = config.getWithDefault('execute',
'parallelTaskCount',
default=1)