def initialize_envDict(envDict, caseroot, debugMsg, standalone): """initialize_main - initialize settings on rank 0 Arguments: envDict (dictionary) - environment dictionary caseroot (string) - case root debugMsg (object) - vprinter object for printing debugging messages Return: envDict (dictionary) - environment dictionary """ # setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files env_file_list = ['./env_postprocess.xml', './env_diags_ice.xml'] envDict = cesmEnvLib.readXML(caseroot, env_file_list) # debug print out the envDict debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2) # refer to the caseroot that was specified on the command line instead of what # is read in the environment as the caseroot may have changed from what is listed # in the env xml envDict['CASEROOT'] = caseroot # add the os.environ['PATH'] to the envDict['PATH'] envDict['ICEDIAG_PATH'] = os.pathsep + os.environ['PATH'] # strip the ICEDIAG_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, 'ICEDIAG_') envDict['seas'] = ['jfm', 'fm', 'amj', 'jas', 'ond', 'on', 'ann'] return envDict
def initialize_main(envDict, caseroot, debugMsg, standalone): """initialize_main - initialize settings on rank 0 Arguments: envDict (dictionary) - environment dictionary caseroot (string) - case root debugMsg (object) - vprinter object for printing debugging messages standalone (boolean) - specify if postprocess caseroot is standalone (true) of not (false) Return: envDict (dictionary) - environment dictionary """ env_file_list = ['./env_postprocess.xml', './env_diags_ilamb.xml'] envDict = cesmEnvLib.readXML(caseroot, env_file_list) # debug print out the envDict debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2) # refer to the caseroot that was specified on the command line instead of what # is read in the environment as the caseroot may have changed from what is listed # in the env xml envDict['CASEROOT'] = caseroot # add the os.environ['PATH'] to the envDict['PATH'] envDict['ILAMBDIAG_PATH'] = os.pathsep + os.environ['PATH'] # strip the ILAMBDIAG_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, 'ILAMBDIAG_') # setup the working directories sys.path.append(envDict['PATH']) return envDict
def initialize_main(envDict, caseroot, debugMsg, standalone, imb_name): """initialize_main - initialize settings on rank 0 Arguments: envDict (dictionary) - environment dictionary caseroot (string) - case root debugMsg (object) - vprinter object for printing debugging messages standalone (boolean) - specify if postprocess caseroot is standalone (true) of not (false) Return: envDict (dictionary) - environment dictionary """ env_file_list = [ './env_postprocess.xml', './env_diags_{0}.xml'.format(imb_name) ] debugMsg('in initialize_main before readXML', header=True, verbosity=1) envDict = cesmEnvLib.readXML(caseroot, env_file_list) # debug print out the envDict debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2) # refer to the caseroot that was specified on the command line instead of what # is read in the environment as the caseroot may have changed from what is listed # in the env xml envDict['CASEROOT'] = caseroot diag_name = "{0}DIAG".format(imb_name.upper()) # add the os.environ['PATH'] to the envDict['PATH'] diag_path = "{0}_PATH".format(diag_name) envDict[diag_path] = os.pathsep + os.environ['PATH'] # strip the ILAMBDIAG_ or IOMBDIAG_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, '{0}_'.format(diag_name)) debugMsg('envDict OUTPUTROOT = {0}'.format(envDict['OUTPUTROOT']), header=True, verbosity=1) # ILAMB requires output directory to already exist if not os.path.exists(envDict['OUTPUTROOT']): os.makedirs(envDict['OUTPUTROOT']) debugMsg('in initialize_main before path.append', header=True, verbosity=1) # setup the working directories sys.path.append(envDict['PATH']) debugMsg('in initialize_main get machine name', header=True, verbosity=1) hostname = cesmEnvLib.get_hostname() envDict['MACH'] = cesmEnvLib.get_machine_name( hostname, '{0}/Machines/machine_postprocess.xml'.format( envDict['POSTPROCESS_PATH'])) return envDict
def initialize_main(envDict, caseroot, debugMsg, standalone): """initialize_main - initialize settings on rank 0 Arguments: envDict (dictionary) - environment dictionary caseroot (string) - case root debugMsg (object) - vprinter object for printing debugging messages Return: envDict (dictionary) - environment dictionary """ # setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files ## env_file_list = ['../env_case.xml', '../env_run.xml', '../env_build.xml', '../env_mach_pes.xml', './env_postprocess.xml', './env_diags_atm.xml'] ## if standalone: env_file_list = ['./env_postprocess.xml', './env_diags_atm.xml'] envDict = cesmEnvLib.readXML(caseroot, env_file_list) # debug print out the envDict debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2) # refer to the caseroot that was specified on the command line instead of what # is read in the environment as the caseroot may have changed from what is listed # in the env xml envDict['CASEROOT'] = caseroot # add the os.environ['PATH'] to the envDict['PATH'] envDict['ATMDIAG_PATH'] = os.pathsep + os.environ['PATH'] # strip the ATMDIAG_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, 'ATMDIAG_') # TODO - create the list of necessary climatology files for model filelist = list() # check average files debugMsg('calling checkAvgFiles', header=True) rc = diagUtilsLib.checkAvgFiles(filelist) if not rc: print( '---------------------------------------------------------------------------' ) print('ERROR: atmosphere climatology files do not exist in:') print(' {0}'.format(envDict['TAVGDIR'])) print('Please run the {0}.atm_avg_generator script first.'.format( envDict['CASE'])) print( '---------------------------------------------------------------------------' ) sys.exit(1) # TODO - create the list of necessary climatology files for control return envDict
def initialize_main(envDict, caseroot, debugMsg, standalone): """initialize_main - initialize settings on rank 0 Arguments: envDict (dictionary) - environment dictionary caseroot (string) - case root debugMsg (object) - vprinter object for printing debugging messages standalong (boolean) - indicate stand-alone post processing caseroot Return: envDict (dictionary) - environment dictionary """ # setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files env_file_list = ['./env_postprocess.xml', './env_diags_ocn.xml'] envDict = cesmEnvLib.readXML(caseroot, env_file_list) # debug print out the envDict debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2) # refer to the caseroot that was specified on the command line instead of what # is read in the environment as the caseroot may have changed from what is listed # in the env xml envDict['CASEROOT'] = caseroot # add the os.environ['PATH'] to the envDict['PATH'] envDict['OCNDIAG_PATH'] += os.pathsep + os.environ['PATH'] # strip the OCNDIAG_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, 'OCNDIAG_') # special variable mapped from the CESM env to the OCN diags env envDict['RESOLUTION'] = envDict['OCN_GRID'] # setup the TOBSFILE and SOBSFILE variables based on the vertical levels # 60 (default) or 42 if envDict['VERTICAL'] == '42': envDict['TOBSFILE'] = envDict['TOBSFILE_V42'] envDict['SOBSFILE'] = envDict['SOBSFILE_V42'] debugMsg('TOBSFILE = {0}, SOBSFILE= {1}'.format(envDict['TOBSFILE'], envDict['SOBSFILE']), header=True, verbosity=2) # TODO - create the list of necessary climatology files for model filelist = list() # check average files debugMsg('calling checkAvgFiles', header=True, verbosity=2) rc = diagUtilsLib.checkAvgFiles(filelist) if not rc: print('---------------------------------------------------------------------------') print('ERROR: ocean climatology files do not exist in:') print(' {0}'.format(envDict['TAVGDIR'])) print('Please run the {0}.ocn_avg_generator script first.'.format(envDict['CASE'])) print('---------------------------------------------------------------------------') sys.exit(1) return envDict
def initialize_envDict(envDict, caseroot, debugMsg, standalone): """initialize_main - initialize settings on rank 0 Arguments: envDict (dictionary) - environment dictionary caseroot (string) - case root debugMsg (object) - vprinter object for printing debugging messages Return: envDict (dictionary) - environment dictionary """ # setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files env_file_list = ['./env_postprocess.xml', './env_diags_atm.xml'] envDict = cesmEnvLib.readXML(caseroot, env_file_list) # debug print out the envDict debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2) # refer to the caseroot that was specified on the command line instead of what # is read in the environment as the caseroot may have changed from what is listed # in the env xml envDict['CASEROOT'] = caseroot # add the os.environ['PATH'] to the envDict['PATH'] envDict['ATMDIAG_PATH'] = os.pathsep + os.environ['PATH'] # strip the ATMDIAG_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, 'ATMDIAG_') envDict['seas'] = [] if envDict['plot_ANN_climo']: envDict['seas'].append('ANN') if envDict['plot_DJF_climo']: envDict['seas'].append('DJF') if envDict['plot_MAM_climo']: envDict['seas'].append('MAM') if envDict['plot_JJA_climo']: envDict['seas'].append('JJA') if envDict['plot_SON_climo']: envDict['seas'].append('SON') if envDict['test_path_history_subdir'].endswith('/'): envDict['test_path_history_subdir'] = envDict[ 'test_path_history_subdir'][:-1] if envDict['cntl_path_history_subdir'].endswith('/'): envDict['cntl_path_history_subdir'] = envDict[ 'cntl_path_history_subdir'][:-1] return envDict
def initialize_envDict(envDict, caseroot, debugMsg, standalone): """initialize_main - initialize settings on rank 0 Arguments: envDict (dictionary) - environment dictionary caseroot (string) - case root debugMsg (object) - vprinter object for printing debugging messages Return: envDict (dictionary) - environment dictionary """ # setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files env_file_list = ['./env_postprocess.xml', './env_diags_atm.xml'] envDict = cesmEnvLib.readXML(caseroot, env_file_list) # debug print out the envDict debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2) # refer to the caseroot that was specified on the command line instead of what # is read in the environment as the caseroot may have changed from what is listed # in the env xml envDict['CASEROOT'] = caseroot # add the os.environ['PATH'] to the envDict['PATH'] envDict['ATMDIAG_PATH'] = os.pathsep + os.environ['PATH'] # strip the ATMDIAG_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, 'ATMDIAG_') envDict['seas'] = [] if envDict['plot_ANN_climo']: envDict['seas'].append('ANN') if envDict['plot_DJF_climo']: envDict['seas'].append('DJF') if envDict['plot_MAM_climo']: envDict['seas'].append('MAM') if envDict['plot_JJA_climo']: envDict['seas'].append('JJA') if envDict['plot_SON_climo']: envDict['seas'].append('SON') if envDict['test_path_history_subdir'].endswith('/'): envDict['test_path_history_subdir'] = envDict['test_path_history_subdir'][:-1] if envDict['cntl_path_history_subdir'].endswith('/'): envDict['cntl_path_history_subdir'] = envDict['cntl_path_history_subdir'][:-1] return envDict
def initialize_main(envDict, caseroot, debugMsg, standalone): """initialize_main - initialize settings on rank 0 Arguments: envDict (dictionary) - environment dictionary caseroot (string) - case root debugMsg (object) - vprinter object for printing debugging messages Return: envDict (dictionary) - environment dictionary """ # setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files env_file_list = ['./env_postprocess.xml', './env_diags_atm.xml'] envDict = cesmEnvLib.readXML(caseroot, env_file_list) # debug print out the envDict debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2) # refer to the caseroot that was specified on the command line instead of what # is read in the environment as the caseroot may have changed from what is listed # in the env xml envDict['CASEROOT'] = caseroot # add the os.environ['PATH'] to the envDict['PATH'] envDict['ATMDIAG_PATH'] = os.pathsep + os.environ['PATH'] # strip the ATMDIAG_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, 'ATMDIAG_') # TODO - create the list of necessary climatology files for model filelist = list() # check average files debugMsg('calling checkAvgFiles', header=True) rc = diagUtilsLib.checkAvgFiles(filelist) if not rc: print('---------------------------------------------------------------------------') print('ERROR: atmosphere climatology files do not exist in:') print(' {0}'.format(envDict['TAVGDIR'])) print('Please run the {0}.atm_avg_generator script first.'.format(envDict['CASE'])) print('---------------------------------------------------------------------------') sys.exit(1) # TODO - create the list of necessary climatology files for control return envDict
def initialize_main(envDict, caseroot, debugMsg, standalone): """initialize_main - initialize settings on rank 0 Arguments: envDict (dictionary) - environment dictionary caseroot (string) - case root debugMsg (object) - vprinter object for printing debugging messages Return: envDict (dictionary) - environment dictionary """ # setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files ## env_file_list = ['../env_case.xml', '../env_run.xml', '../env_build.xml', '../env_mach_pes.xml', './env_postprocess.xml', './env_diags_ice.xml'] ## envDict['STANDALONE'] = False ## if standalone: env_file_list = ['./env_postprocess.xml', './env_diags_ice.xml'] ## envDict['STANDALONE'] = True envDict = cesmEnvLib.readXML(caseroot, env_file_list) # debug print out the envDict debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2) # add the os.environ['PATH'] to the envDict['PATH'] envDict['ICEDIAG_PATH'] = str(os.pathsep + os.environ['PATH']) # strip the ICEDIAG_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, 'ICEDIAG_') # TODO - create the list of necessary climatology files for model filelist = list() # check average files debugMsg('calling checkAvgFiles', header=True) rc = diagUtilsLib.checkAvgFiles(filelist) if not rc: print('---------------------------------------------------------------------------') print('ERROR: ice climatology files do not exist in:') print(' {0}'.format(envDict['TAVGDIR'])) print('Please run the {0}.ice_avg_generator script first.'.format(envDict['CASE'])) print('---------------------------------------------------------------------------') sys.exit(1) # TODO - create the list of necessary climatology files for control return envDict
def initialize_envDict(envDict, caseroot, debugMsg, standalone): """initialize_main - initialize settings on rank 0 Arguments: envDict (dictionary) - environment dictionary caseroot (string) - case root debugMsg (object) - vprinter object for printing debugging messages standalone (boolean) - indicate stand-alone post processing caseroot Return: envDict (dictionary) - environment dictionary """ # setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files env_file_list = ['./env_postprocess.xml', './env_diags_ocn.xml'] envDict = cesmEnvLib.readXML(caseroot, env_file_list) # debug print out the envDict debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2) # refer to the caseroot that was specified on the command line instead of what # is read in the environment as the caseroot may have changed from what is listed # in the env xml envDict['CASEROOT'] = caseroot # add the os.environ['PATH'] to the envDict['PATH'] envDict['OCNDIAG_PATH'] += os.pathsep + os.environ['PATH'] # initialize varLists envDict['MODEL_VARLIST'] = [] if len(envDict['OCNDIAG_PYAVG_MODELCASE_VARLIST']) > 0: envDict['MODEL_VARLIST'] = envDict[ 'OCNDIAG_PYAVG_MODELCASE_VARLIST'].split(',') envDict['CNTRL_VARLIST'] = [] if len(envDict['OCNDIAG_PYAVG_CNTRLCASE_VARLIST']) > 0: envDict['CNTRL_VARLIST'] = envDict[ 'OCNDIAG_PYAVG_CNTRLCASE_VARLIST'].split(',') # strip the OCNDIAG_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, 'OCNDIAG_') return envDict
def initialize_envDict(envDict, caseroot, debugMsg, standalone, main_comm): """initialize_main - initialize settings on rank 0 Arguments: envDict (dictionary) - environment dictionary caseroot (string) - case root debugMsg (object) - vprinter object for printing debugging messages Return: envDict (dictionary) - environment dictionary """ # setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files # TODO put this file list into the config_postprocess definition ## env_file_list = ['../env_case.xml', '../env_run.xml', '../env_build.xml', '../env_mach_pes.xml', './env_postprocess.xml', './env_diags_lnd.xml'] ## if standalone: env_file_list = ['./env_postprocess.xml', './env_diags_lnd.xml'] envDict = cesmEnvLib.readXML(caseroot, env_file_list) # debug print out the envDict if main_comm.is_manager(): debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=1) # refer to the caseroot that was specified on the command line instead of what # is read in the environment as the caseroot may have changed from what is listed # in the env xml # add the os.environ['PATH'] to the envDict['PATH'] envDict['LANDDIAG_PATH'] = os.pathsep + os.environ['PATH'] # strip the LNDDIAG_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, 'LNDDIAG_') envDict['seas'] = ['djf', 'mam', 'jja', 'son', 'ann'] return envDict
def initialize_envDict(envDict, caseroot, debugMsg, standalone, main_comm): """initialize_main - initialize settings on rank 0 Arguments: envDict (dictionary) - environment dictionary caseroot (string) - case root debugMsg (object) - vprinter object for printing debugging messages Return: envDict (dictionary) - environment dictionary """ # setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files # TODO put this file list into the config_postprocess definition ## env_file_list = ['../env_case.xml', '../env_run.xml', '../env_build.xml', '../env_mach_pes.xml', './env_postprocess.xml', './env_diags_lnd.xml'] ## if standalone: env_file_list = ['./env_postprocess.xml', './env_diags_lnd.xml'] envDict = cesmEnvLib.readXML(caseroot, env_file_list) # debug print out the envDict if main_comm.is_manager(): debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=1) # refer to the caseroot that was specified on the command line instead of what # is read in the environment as the caseroot may have changed from what is listed # in the env xml # add the os.environ['PATH'] to the envDict['PATH'] envDict['LANDDIAG_PATH'] = os.pathsep + os.environ['PATH'] # strip the LNDDIAG_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, 'LNDDIAG_') envDict['seas'] = ['djf','mam','jja','son','ann'] return envDict
def initialize_main(envDict, caseroot, debugMsg, standalone): """initialize_main - initialize settings on rank 0 Arguments: envDict (dictionary) - environment dictionary caseroot (string) - case root debugMsg (object) - vprinter object for printing debugging messages Return: envDict (dictionary) - environment dictionary """ # setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files ## env_file_list = ['../env_case.xml', '../env_run.xml', '../env_build.xml', '../env_mach_pes.xml', './env_postprocess.xml', './env_diags_lnd.xml'] ## envDict['STANDALONE'] = False ## if standalone: env_file_list = ['./env_postprocess.xml', './env_diags_lnd.xml'] ## envDict['STANDALONE'] = True envDict = cesmEnvLib.readXML(caseroot, env_file_list) # debug print out the envDict debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2) # refer to the caseroot that was specified on the command line instead of what # is read in the environment as the caseroot may have changed from what is listed # in the env xml envDict['CASEROOT'] = caseroot # add the os.environ['PATH'] to the envDict['PATH'] envDict['LNDDIAG_PATH'] = os.pathsep + os.environ['PATH'] # strip the LNDDIAG_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, 'LNDDIAG_') # TODO - create the list of necessary climatology files for model filelist = list() # check average files debugMsg('calling checkAvgFiles', header=True, verbosity=1) rc = diagUtilsLib.checkAvgFiles(filelist) if not rc: print('---------------------------------------------------------------------------') print('ERROR: land climatology files do not exist') print('Please run the {0}.lnd_avg_generator script first.'.format(envDict['CASE'])) print('---------------------------------------------------------------------------') sys.exit(1) # TODO - create the list of necessary climatology files for control # setup the working directories sys.path.append(envDict['PATH']) # the WKDIR variable is very confusing... it gets reset in setup_workdir if (envDict['MODEL_VS_MODEL'] == 'True'): envDict['WKDIR'] = envDict['PTMPDIR_1']+'/diag/'+envDict['caseid_1']+'-'+envDict['caseid_2']+'/' else: envDict['WKDIR'] = envDict['PTMPDIR_1']+'/diag/'+envDict['caseid_1']+'-obs/' if envDict['CASA'] == '1': envDict['VAR_MASTER'] = envDict['var_master_casa'] else: envDict['VAR_MASTER'] = envDict['var_master_cn'] return envDict
def initialize_main(envDict, caseroot, debugMsg, standalone): """initialize_main - initialize settings on rank 0 Arguments: envDict (dictionary) - environment dictionary caseroot (string) - case root debugMsg (object) - vprinter object for printing debugging messages Return: envDict (dictionary) - environment dictionary """ # setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files ## env_file_list = ['../env_case.xml', '../env_run.xml', '../env_build.xml', '../env_mach_pes.xml', './env_postprocess.xml', './env_diags_lnd.xml'] ## envDict['STANDALONE'] = False ## if standalone: env_file_list = ['./env_postprocess.xml', './env_diags_lnd.xml'] ## envDict['STANDALONE'] = True envDict = cesmEnvLib.readXML(caseroot, env_file_list) # debug print out the envDict debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2) # refer to the caseroot that was specified on the command line instead of what # is read in the environment as the caseroot may have changed from what is listed # in the env xml envDict['CASEROOT'] = caseroot # add the os.environ['PATH'] to the envDict['PATH'] envDict['LNDDIAG_PATH'] = os.pathsep + os.environ['PATH'] # strip the LNDDIAG_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, 'LNDDIAG_') # TODO - create the list of necessary climatology files for model filelist = list() # check average files debugMsg('calling checkAvgFiles', header=True, verbosity=1) rc = diagUtilsLib.checkAvgFiles(filelist) if not rc: print( '---------------------------------------------------------------------------' ) print('ERROR: land climatology files do not exist') print('Please run the {0}.lnd_avg_generator script first.'.format( envDict['CASE'])) print( '---------------------------------------------------------------------------' ) sys.exit(1) # TODO - create the list of necessary climatology files for control # setup the working directories sys.path.append(envDict['PATH']) # the WKDIR variable is very confusing... it gets reset in setup_workdir if (envDict['MODEL_VS_MODEL'] == 'True'): envDict['WKDIR'] = envDict['PTMPDIR_1'] + '/diag/' + envDict[ 'caseid_1'] + '-' + envDict['caseid_2'] + '/' else: envDict['WKDIR'] = envDict['PTMPDIR_1'] + '/diag/' + envDict[ 'caseid_1'] + '-obs/' if envDict['CASA'] == '1': envDict['VAR_MASTER'] = envDict['var_master_casa'] else: envDict['VAR_MASTER'] = envDict['var_master_cn'] return envDict
def main(options, main_comm, debugMsg): """ read env_ocn_remap.xml settings to call the ocean_remap class """ # initialize the environment dictionary envDict = dict() # Get rank and size rank = main_comm.get_rank() size = main_comm.get_size() # CASEROOT is given on the command line as required option --caseroot if rank == 0: caseroot = options.caseroot[0] envDict['CASEROOT'] = options.caseroot[0] debugMsg('caseroot = {0}'.format(envDict['CASEROOT']), header=True, verbosity=2) env_file_list = ['./env_postprocess.xml', './env_ocn_remap.xml'] envDict = cesmEnvLib.readXML(caseroot, env_file_list) # strip the OCNREMAP_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, 'OCNREMAP_') print("cmip6: {0}".format(envDict['cmip6'])) print("filelist: {0}".format(envDict['filelist'])) print("matrix_2d_fname: {0}".format(envDict['matrix_2d_fname'])) print("matrix_3d_fname: {0}".format(envDict['matrix_3d_fname'])) print("indir: {0}".format(envDict['indir'])) print("outdir: {0}".format(envDict['outdir'])) print("chunk size: {0}".format(envDict['chunk'])) # broadcast envDict to all tasks envDict = main_comm.partition(data=envDict, func=partition.Duplicate(), involved=True) main_comm.sync() files = [] if rank == 0: # Find files to regrid #Do we have a cmip6 variable list? if envDict['cmip6'] is not None: if envDict['indir'] is not None: with open(envDict['cmip6']) as f: for l in f: t = l.strip().split(':')[0] v = l.strip().split(':')[1] print("Trying to find: {0}_{1}*.nc".format(v, t)) for root, dirs, fns in os.walk(envDict['indir']): for fn in fnmatch.filter(fns, v + '_' + t + "*.nc"): if 'tmp.nc' not in fn and 'gr' not in fn.split( '_'): print("Found: {0}".format(fn.split('/'))) files.append(os.path.join(root, fn)) else: print( "You need to specify an indir argument with the cmip6 argument" ) file = None elif envDict['filelist'] is not None: with open(envDict['filelist']) as f: for l in f: files.append(l.strip()) elif envDict['indir'] is not None: for root, dirs, fns in os.walk(envDict['indir']): for fn in fnmatch.filter(fns, "*.nc"): files.append(os.path.join(root, fn)) else: print('Exiting because no input path or files where given') files = None # All call this main_comm.sync() files = main_comm.partition(files, func=partition.Duplicate(), involved=True) if files is None: sys.exit() #matrix_2d_fname = 'POP_gx1v7_to_latlon_1x1_0E_mask_conserve_20181015.nc' matrix_2d = remap.ocean_remap(envDict['matrix_2d_fname']) #matrix_3d_fname = 'POP_gx1v7_to_latlon_1x1_0E_fulldepth_conserve_20181015.nc' matrix_3d = remap.ocean_remap(envDict['matrix_3d_fname']) # names of coordinate dimensions in output files dim_names = {'depth': 'olevel', 'lat': 'latitude', 'lon': 'longitude'} dim_names = {'depth': 'lev', 'lat': 'lat', 'lon': 'lon'} main_comm.sync() # Have only root create these files if rank == 0: if len(files) > 0 and envDict['cmip6'] is not None: temp = files[0] # create CMIP Ofx files for var_name in ('areacello', 'deptho', 'thkcello', 'volcello'): new_outdir = temp.replace( temp.split('/')[-4], var_name).replace( temp.split('/')[-5], 'Ofx').replace( temp.split('/')[-3], 'gr').replace( '_' + temp.split('_')[-1], '') + '.nc' d = os.path.dirname(new_outdir) if not os.path.exists(d): os.makedirs(d) fptr_out = nc.Dataset(new_outdir, 'w') # pylint: disable=E1101 matrix_3d.dst_grid.def_dims_common(fptr_out, dim_names) matrix_3d.dst_grid.write_vars_common(fptr_out, dim_names) matrix_3d.dst_grid.write_var_CMIP_Ofx(fptr_out, dim_names, var_name) # Create a master slave parallel protocol GWORK_TAG = 10 # global comm mpi tag if (rank == 0): for i in files: main_comm.ration(data=i, tag=GWORK_TAG) for i in range(1, size): main_comm.ration(data=-99, tag=GWORK_TAG) else: f = -999 while f != -99: f = main_comm.ration(tag=GWORK_TAG) if f != -99: print("working on: {0}".format(f)) testfile_in_fname = f testfile_out_fname = f.replace(f.split('/')[-3], 'gr') if not os.path.exists(testfile_out_fname): d = os.path.dirname(testfile_out_fname) if not os.path.exists(d): os.makedirs(d) fptr_in = nc.Dataset(testfile_in_fname, 'r') # pylint: disable=E1101 if (len(fptr_in[f.split('/')[-4]].dimensions) == 4 or len(fptr_in[f.split('/')[-4]].dimensions) == 3): fptr_out = nc.Dataset(testfile_out_fname + '.tmp', 'w') # pylint: disable=E1101 remap.copy_time(fptr_in, fptr_out) remap.copy_gAttr(fptr_in, fptr_out) if dim_names['depth'] in fptr_in.dimensions: matrix_3d.dst_grid.def_dims_common( fptr_out, dim_names) matrix_3d.dst_grid.write_vars_common( fptr_out, dim_names) else: matrix_2d.dst_grid.def_dims_common( fptr_out, dim_names) matrix_2d.dst_grid.write_vars_common( fptr_out, dim_names) field_names = [] for v in fptr_in.variables: if v not in [ 'lat', 'lat_bnds', 'lon', 'lon_bnds', 'lev', 'lev_bnds', 'time', 'time_bnds', 'nlat', 'nlon' ]: field_names.append(v) for field_name in field_names: varid_out = remap.def_var(field_name, fptr_in, fptr_out, dim_names) # use appropriate matrix for regridding c = envDict['chunk'] if c is None: c = 1 else: c = int(c) try: if dim_names['depth'] in varid_out.dimensions: #print ("Running a 3D variable") b = 0 for i in range( 0, fptr_in.dimensions['time'].size, c): if b + c >= fptr_in.dimensions[ 'time'].size: c = fptr_in.dimensions[ 'time'].size - b varid_out[b:( b + c), :, :, :] = matrix_3d.remap_var( fptr_in.variables[field_name][ b:(b + c), :, :, :]) #, #fill_value=getattr(varid_out, 'missing_value')) b = b + c else: #print ("Running a 2D variable") b = 0 for i in range( 0, fptr_in.dimensions['time'].size, c): if b + c >= fptr_in.dimensions[ 'time'].size: c = fptr_in.dimensions[ 'time'].size - b varid_out[b:( b + c), :, :] = matrix_2d.remap_var( fptr_in.variables[field_name][ b:(b + c), :, :]) #, #fill_value=getattr(varid_out, 'missing_value')) b = b + c except TypeError as e: print('Type Error for variable {0} '.format( field_name)) fptr_in.close() fptr_out.close() try: os.rename(testfile_out_fname + '.tmp', testfile_out_fname) except OSError as e: print('Could not create {0}'.format( testfile_out_fname)) else: print("Not creating {0}".format(testfile_out_fname)) main_comm.sync()