def main(): """ Entry point for the main script ``mpas_analysis`` """ parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('-v', '--version', action='version', version='mpas_analysis {}'.format( mpas_analysis.__version__), help="Show version number and exit") parser.add_argument("--setup_only", dest="setup_only", action='store_true', help="If only the setup phase, not the run or HTML " "generation phases, should be executed.") parser.add_argument("--html_only", dest="html_only", action='store_true', help="If only the setup and HTML generation phases, " "not the run phase, should be executed.") parser.add_argument("-g", "--generate", dest="generate", help="A list of analysis modules to generate " "(nearly identical generate option in config file).", metavar="ANALYSIS1[,ANALYSIS2,ANALYSIS3,...]") parser.add_argument("-l", "--list", dest="list", action='store_true', help="List the available analysis tasks") parser.add_argument("-p", "--purge", dest="purge", action='store_true', help="Purge the analysis by deleting the output" "directory before running") parser.add_argument('configFiles', metavar='CONFIG', type=str, nargs='*', help='config file') parser.add_argument("--plot_colormaps", dest="plot_colormaps", action='store_true', help="Make a plot displaying all available colormaps") parser.add_argument("--verbose", dest="verbose", action='store_true', help="Verbose error reporting during setup-and-check " "phase") args = parser.parse_args() if len(sys.argv) == 1: parser.print_help() sys.exit(0) for configFile in args.configFiles: if not os.path.exists(configFile): raise OSError('Config file {} not found.'.format(configFile)) # add config.default to cover default not included in the config files # provided on the command line if pkg_resources.resource_exists('mpas_analysis', 'config.default'): defaultConfig = pkg_resources.resource_filename( 'mpas_analysis', 'config.default') configFiles = [defaultConfig] + args.configFiles else: print('WARNING: Did not find config.default. Assuming other config ' 'file(s) contain a\n' 'full set of configuration options.') defaultConfig = None configFiles = args.configFiles config = MpasAnalysisConfigParser() config.read(configFiles) if args.list: analyses = build_analysis_list(config, controlConfig=None) for analysisTask in analyses: print('task: {}'.format(analysisTask.taskName)) print(' component: {}'.format(analysisTask.componentName)), print(' tags: {}'.format(', '.join(analysisTask.tags))) sys.exit(0) if args.plot_colormaps: _register_custom_colormaps() _plot_color_gradients() sys.exit(0) if config.has_option('runs', 'controlRunConfigFile'): controlConfigFile = config.get('runs', 'controlRunConfigFile') if not os.path.exists(controlConfigFile): raise OSError('A control config file {} was specified but the ' 'file does not exist'.format(controlConfigFile)) controlConfigFiles = [controlConfigFile] if defaultConfig is not None: controlConfigFiles = [defaultConfig] + controlConfigFiles controlConfig = MpasAnalysisConfigParser() controlConfig.read(controlConfigFiles) # replace the log directory so log files get written to this run's # log directory, not the control run's logsDirectory = build_config_full_path(config, 'output', 'logsSubdirectory') controlConfig.set('output', 'logsSubdirectory', logsDirectory) print('Comparing to control run {} rather than observations. \n' 'Make sure that MPAS-Analysis has been run previously with the ' 'control config file.'.format( controlConfig.get('runs', 'mainRunName'))) else: controlConfig = None if args.purge: purge_output(config) if config.has_option('runs', 'mainRunConfigFile'): symlink_main_run(config, defaultConfig) if args.generate: update_generate(config, args.generate) if controlConfig is not None: # we want to use the "generate" option from the current run, not # the control config file controlConfig.set('output', 'generate', config.get('output', 'generate')) logsDirectory = build_config_full_path(config, 'output', 'logsSubdirectory') make_directories(logsDirectory) update_time_bounds_in_config(config) file_cache_maxsize = config.getint('input', 'file_cache_maxsize') try: xarray.set_options(file_cache_maxsize=file_cache_maxsize) except ValueError: # xarray version doesn't support file_cache_maxsize yet... pass startTime = time.time() analyses = build_analysis_list(config, controlConfig) analyses = determine_analyses_to_generate(analyses, args.verbose) setupDuration = time.time() - startTime if not args.setup_only and not args.html_only: run_analysis(config, analyses) runDuration = time.time() - startTime m, s = divmod(setupDuration, 60) h, m = divmod(int(m), 60) print('Total setup time: {}:{:02d}:{:05.2f}'.format(h, m, s)) m, s = divmod(runDuration, 60) h, m = divmod(int(m), 60) print('Total run time: {}:{:02d}:{:05.2f}'.format(h, m, s)) if not args.setup_only: generate_html(config, analyses, controlConfig)
class TestMPASAnalysisConfigParser(TestCase): def setup_config(self): configPath = self.datadir.join('config.analysis') self.config = MpasAnalysisConfigParser() self.config.read(str(configPath)) def check_container(self, container, container_type, item_type): assert isinstance(container, container_type) for item in container: assert isinstance(item, item_type) def test_read_config(self): self.setup_config() colorMapName = self.config.get('sst_modelvsobs', 'cmapDiff') self.assertEqual(colorMapName, 'coolwarm') self.assertEqual(self.config.getint('Test', 'testInt'), 15) self.assertEqual(self.config.getExpression('Test', 'testInt'), 15) self.assertEqual(self.config.getfloat('Test', 'testFloat'), 18.0) self.assertEqual(self.config.getExpression('Test', 'testFloat'), 18.0) self.assertEqual(self.config.getfloat('Test', 'testFloat2'), 3.) self.assertEqual(self.config.getExpression('Test', 'testFloat2'), 3.) self.assertEqual(self.config.getboolean('Test', 'testBool'), True) self.assertEqual(self.config.getExpression('Test', 'testBool'), True) testList = self.config.getExpression('sst_modelvsobs', 'cmapIndicesModelObs') self.check_container(testList, list, int) self.assertEqual(testList, [0, 40, 80, 110, 140, 170, 200, 230, 255]) testList = self.config.getExpression('sst_modelvsobs', 'cmapIndicesModelObs', elementType=float) self.check_container(testList, list, float) self.assertEqual(testList, [0., 40., 80., 110., 140., 170., 200., 230., 255.]) testList = self.config.getExpression('sst_modelvsobs', 'comparisonTimes') self.check_container(testList, list, str) self.assertEqual(testList, ['JFM', 'JAS', 'ANN']) testList = self.config.getExpression('Test', 'testList') self.check_container(testList, list, float) self.assertEqual(testList, [0.5, 0.1, 0.5]) testTuple = self.config.getExpression('Test', 'testTuple') assert isinstance(testTuple, tuple) self.assertEqual(testTuple, (5, 0.1, 'string')) testDict = self.config.getExpression('Test', 'testDict') assert isinstance(testDict, dict) self.assertEqual(testDict, { 'key1': 'string', 'key2': -12, 'key3': False }) with six.assertRaisesRegex( self, configparser.NoOptionError, "No option 'doesntexist' in section: 'Test'"): self.config.getExpression(str('Test'), str('doesntexist')) @requires_numpy def test_read_config_numpy(self): self.setup_config() # tests numpy evaluation capability import numpy as np for testname in ['testNumpyarange' + str(ii) for ii in np.arange(3)]: self.assertArrayEqual( self.config.getExpression('TestNumpy', testname, usenumpyfunc=True), np.arange(0, 1, 10)) for testname in ['testNumpylinspace' + str(ii) for ii in np.arange(3)]: self.assertArrayEqual( self.config.getExpression('TestNumpy', testname, usenumpyfunc=True), np.linspace(0, 1, 10)) for testNumpy in ['testNumpypi' + str(ii) for ii in np.arange(3)] + \ ['testNumpyPi']: self.assertEqual( self.config.getExpression('TestNumpy', testNumpy, usenumpyfunc=True), np.pi) with six.assertRaisesRegex( self, AssertionError, "'__' is not allowed in .* for `usenumpyfunc=True`"): self.config.getExpression('TestNumpy', 'testBadStr', usenumpyfunc=True), def test_get_with_default(self): self.setup_config() def check_get_with_default(name, value, dtype): # test an options that doesn't exist using getWithDefault var = self.config.getWithDefault('sst_modelvsobs', name, value) assert isinstance(var, dtype) self.assertEqual(var, value) # test several types with getWithDefault check_get_with_default(name='aBool', value=True, dtype=bool) check_get_with_default(name='anInt', value=1, dtype=six.integer_types) check_get_with_default(name='aFloat', value=1.0, dtype=float) check_get_with_default(name='aList', value=[1, 2, 3], dtype=list) check_get_with_default(name='aTuple', value=(1, 2, 3), dtype=tuple) check_get_with_default(name='aDict', value={'blah': 1}, dtype=dict) check_get_with_default(name='aStr', value='blah', dtype=six.string_types)