def setup_config(self, autocloseFileLimitFraction=0.5, maxChunkSize=10000): config = MpasAnalysisConfigParser() config.add_section('input') config.set('input', 'autocloseFileLimitFraction', str(autocloseFileLimitFraction)) config.set('input', 'maxChunkSize', str(maxChunkSize)) return config
def setup_config(self): configPath = self.datadir.join('config.QU240') config = MpasAnalysisConfigParser() config.read(str(configPath)) config.set('input', 'baseDirectory', str(self.datadir)) config.set('output', 'baseDirectory', str(self.test_dir)) return config
def doTest(generate, expectedResults): config = MpasAnalysisConfigParser() config.add_section('output') config.set('output', 'generate', generate) for taskName in expectedResults: genericTask = AnalysisTask(config=config, taskName=taskName, componentName=cores[taskName], tags=tags[taskName]) expectedResult = expectedResults[taskName] result = genericTask.check_generate() self.assertEqual(result, expectedResult)
def setup_config(self): configPath = self.datadir.join('config.remap_obs') config = MpasAnalysisConfigParser() config.read(str(configPath)) config.set('input', 'baseDirectory', str(self.datadir)) config.set('diagnostics', 'baseDirectory', str(self.datadir)) config.set('oceanObservations', 'obsSubdirectory', '.') config.set('output', 'baseDirectory', str(self.test_dir)) return config
def main(): """ Entry point for the main script ``mpas_analysis`` """ parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('-v', '--version', action='version', version='mpas_analysis {}'.format( mpas_analysis.__version__), help="Show version number and exit") parser.add_argument("--setup_only", dest="setup_only", action='store_true', help="If only the setup phase, not the run or HTML " "generation phases, should be executed.") parser.add_argument("--html_only", dest="html_only", action='store_true', help="If only the setup and HTML generation phases, " "not the run phase, should be executed.") parser.add_argument("-g", "--generate", dest="generate", help="A list of analysis modules to generate " "(nearly identical generate option in config file).", metavar="ANALYSIS1[,ANALYSIS2,ANALYSIS3,...]") parser.add_argument("-l", "--list", dest="list", action='store_true', help="List the available analysis tasks") parser.add_argument("-p", "--purge", dest="purge", action='store_true', help="Purge the analysis by deleting the output" "directory before running") parser.add_argument('configFiles', metavar='CONFIG', type=str, nargs='*', help='config file') parser.add_argument("--plot_colormaps", dest="plot_colormaps", action='store_true', help="Make a plot displaying all available colormaps") parser.add_argument("--verbose", dest="verbose", action='store_true', help="Verbose error reporting during setup-and-check " "phase") args = parser.parse_args() if len(sys.argv) == 1: parser.print_help() sys.exit(0) for configFile in args.configFiles: if not os.path.exists(configFile): raise OSError('Config file {} not found.'.format(configFile)) # add config.default to cover default not included in the config files # provided on the command line if pkg_resources.resource_exists('mpas_analysis', 'config.default'): defaultConfig = pkg_resources.resource_filename( 'mpas_analysis', 'config.default') configFiles = [defaultConfig] + args.configFiles else: print('WARNING: Did not find config.default. Assuming other config ' 'file(s) contain a\n' 'full set of configuration options.') defaultConfig = None configFiles = args.configFiles config = MpasAnalysisConfigParser() config.read(configFiles) if args.list: analyses = build_analysis_list(config, controlConfig=None) for analysisTask in analyses: print('task: {}'.format(analysisTask.taskName)) print(' component: {}'.format(analysisTask.componentName)), print(' tags: {}'.format(', '.join(analysisTask.tags))) sys.exit(0) if args.plot_colormaps: _register_custom_colormaps() _plot_color_gradients() sys.exit(0) if config.has_option('runs', 'controlRunConfigFile'): controlConfigFile = config.get('runs', 'controlRunConfigFile') if not os.path.exists(controlConfigFile): raise OSError('A control config file {} was specified but the ' 'file does not exist'.format(controlConfigFile)) controlConfigFiles = [controlConfigFile] if defaultConfig is not None: controlConfigFiles = [defaultConfig] + controlConfigFiles controlConfig = MpasAnalysisConfigParser() controlConfig.read(controlConfigFiles) # replace the log directory so log files get written to this run's # log directory, not the control run's logsDirectory = build_config_full_path(config, 'output', 'logsSubdirectory') controlConfig.set('output', 'logsSubdirectory', logsDirectory) print('Comparing to control run {} rather than observations. \n' 'Make sure that MPAS-Analysis has been run previously with the ' 'control config file.'.format( controlConfig.get('runs', 'mainRunName'))) else: controlConfig = None if args.purge: purge_output(config) if config.has_option('runs', 'mainRunConfigFile'): symlink_main_run(config, defaultConfig) if args.generate: update_generate(config, args.generate) if controlConfig is not None: # we want to use the "generate" option from the current run, not # the control config file controlConfig.set('output', 'generate', config.get('output', 'generate')) logsDirectory = build_config_full_path(config, 'output', 'logsSubdirectory') make_directories(logsDirectory) update_time_bounds_in_config(config) file_cache_maxsize = config.getint('input', 'file_cache_maxsize') try: xarray.set_options(file_cache_maxsize=file_cache_maxsize) except ValueError: # xarray version doesn't support file_cache_maxsize yet... pass startTime = time.time() analyses = build_analysis_list(config, controlConfig) analyses = determine_analyses_to_generate(analyses, args.verbose) setupDuration = time.time() - startTime if not args.setup_only and not args.html_only: run_analysis(config, analyses) runDuration = time.time() - startTime m, s = divmod(setupDuration, 60) h, m = divmod(int(m), 60) print('Total setup time: {}:{:02d}:{:05.2f}'.format(h, m, s)) m, s = divmod(runDuration, 60) h, m = divmod(int(m), 60) print('Total run time: {}:{:02d}:{:05.2f}'.format(h, m, s)) if not args.setup_only: generate_html(config, analyses, controlConfig)
def setup_config(self, maxChunkSize=10000): config = MpasAnalysisConfigParser() config.add_section('input') config.set('input', 'maxChunkSize', str(maxChunkSize)) return config
def main(): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("-i", "--inDir", dest="inDir", required=True, help="Directory where intermediate files used in " "processing should be downloaded") parser.add_argument("-o", "--outDir", dest="outDir", required=True, help="Directory where final preprocessed observation " "are stored") args = parser.parse_args() date = '20190603' inGridName = 'SouthernOcean_0.167x0.167degree_{}'.format(date) inTPrefix = 'THETA_mnthlyBar.0000000100' inSPrefix = 'SALT_mnthlyBar.0000000100' inMLDPrefix = 'MLD_mnthlyBar.0000000100' inUPrefix = 'UVEL_mnthlyBar.0000000100' inVPrefix = 'VVEL_mnthlyBar.0000000100' inGammaNPrefix = 'GAMMA_mnthlyBar.0000000100' # size in km of the polar stereographic grid antarcticStereoWidth = 10000 config = MpasAnalysisConfigParser() config.read('mpas_analysis/config.default') config.set('climatology', 'comparisonAntarcticStereoWidth', '{}'.format(antarcticStereoWidth)) outDescriptor = get_comparison_descriptor(config, 'antarctic') outGridName = '{}_{}'.format(outDescriptor.meshName, date) inPrefixes = [ inTPrefix, inSPrefix, inMLDPrefix, inUPrefix, inVPrefix, inGammaNPrefix ] inGridFileName = '{}/grid.mat'.format(args.inDir) try: os.makedirs(args.inDir) except OSError: pass try: os.makedirs(args.outDir) except OSError: pass # dowload the desired file download_files(['GRID_README.txt'], urlBase='http://sose.ucsd.edu/DATA', outDir=args.inDir) urlBase = 'http://sose.ucsd.edu/DATA/SO6_V2' fileList = ['grid.mat'] for prefix in inPrefixes: fileList.append('{}.data.gz'.format(prefix)) fileList.append('{}.meta'.format(prefix)) download_files(fileList, urlBase, args.inDir) unzip_sose_data(inPrefixes, args.inDir) prefix = '{}/SOSE'.format(args.outDir) sose_volume_to_nc(prefix, inGridName, inGridFileName, args.inDir) prefix = '{}/SOSE_2005-2010_monthly'.format(args.outDir) remap_pt_s(prefix, inGridName, inGridFileName, args.inDir, inTPrefix, inSPrefix, inGammaNPrefix, outDescriptor, outGridName) remap_mld(prefix, inGridName, inGridFileName, args.inDir, inMLDPrefix, outDescriptor, outGridName) remap_u(prefix, inGridName, inGridFileName, args.inDir, inUPrefix, outDescriptor, outGridName) remap_v(prefix, inGridName, inGridFileName, args.inDir, inVPrefix, outDescriptor, outGridName) compute_vel_mag(prefix, inGridName, args.inDir, outGridName) compute_pot_density(prefix, inGridName, args.inDir, outGridName)
resolution. ''' from mpas_analysis.shared.interpolation import Remapper from mpas_analysis.shared.grid import MpasMeshDescriptor from mpas_analysis.shared.climatology import get_comparison_descriptor from mpas_analysis.configuration import MpasAnalysisConfigParser # replace with the MPAS mesh name inGridName = 'oQU240' # replace with the path to the desired mesh or restart file inGridFileName = '/media/xylar/extra_data/analysis/edison/G-QU240-master-intel/run/mpaso.rst.0001-01-06_00000.nc' config = MpasAnalysisConfigParser() config.read('mpas_analysis/config.default') # replace 1.0 with the desired resolution of the output mesh config.set('climatology', 'comparisonLatResolution', '1.0') config.set('climatology', 'comparisonLonResolution', '1.0') inDescriptor = MpasMeshDescriptor(inGridFileName, inGridName) outDescriptor = get_comparison_descriptor(config, 'latlon') outGridName = outDescriptor.meshName mappingFileName = 'map_{}_to_{}.nc'.format(inGridName, outGridName) remapper = Remapper(inDescriptor, outDescriptor, mappingFileName) remapper.build_mapping_file(method='bilinear')
def setup_config(self, autocloseFileLimitFraction=0.5, maxChunkSize=10000): config = MpasAnalysisConfigParser() config.add_section('input') config.set('input', 'autocloseFileLimitFraction', str(autocloseFileLimitFraction)) config.set('input', 'maxChunkSize', str(maxChunkSize)) config.set('input', 'mpasMeshName', 'QU240') config.add_section('output') config.set('output', 'baseDirectory', self.test_dir) config.set('output', 'mappingSubdirectory', '.') config.set('output', 'mpasClimatologySubdirectory', 'clim/mpas') config.add_section('climatology') config.set('climatology', 'startYear', '2') config.set('climatology', 'endYear', '2') config.set('climatology', 'comparisonLatResolution', '0.5') config.set('climatology', 'comparisonLonResolution', '0.5') config.set('climatology', 'mpasInterpolationMethod', 'bilinear') config.add_section('oceanObservations') config.set('oceanObservations', 'interpolationMethod', 'bilinear') config.set('oceanObservations', 'climatologySubdirectory', 'clim/obs') config.set('oceanObservations', 'remappedClimSubdirectory', 'clim/obs/remap') return config
''' from mpas_analysis.shared.interpolation import Remapper from mpas_analysis.shared.grid import MpasMeshDescriptor from mpas_analysis.shared.climatology import get_comparison_descriptor from mpas_analysis.configuration import MpasAnalysisConfigParser # replace with the MPAS mesh name inGridName = 'oQU240' # replace with the path to the desired mesh or restart file inGridFileName = '/media/xylar/extra_data/analysis/edison/G-QU240-master-intel/run/mpaso.rst.0001-01-06_00000.nc' config = MpasAnalysisConfigParser() config.read('mpas_analysis/config.default') # replace these numbers with the desired size and resolution of the output mesh config.set('climatology', 'comparisonAntarcticStereoWidth', '6000.0') config.set('climatology', 'comparisonAntarcticStereoResolution', '10.0') inDescriptor = MpasMeshDescriptor(inGridFileName, inGridName) outDescriptor = get_comparison_descriptor(config, 'antarctic') outGridName = outDescriptor.meshName mappingFileName = 'map_{}_to_{}.nc'.format(inGridName, outGridName) remapper = Remapper(inDescriptor, outDescriptor, mappingFileName) remapper.build_mapping_file(method='bilinear')