def initialize_envDict(envDict, caseroot, debugMsg, standalone):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
Return:
envDict (dictionary) - environment dictionary
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
env_file_list = ['./env_postprocess.xml', './env_diags_ice.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
debugMsg('envDict after readXML = {0}'.format(envDict),
header=True,
verbosity=2)
# refer to the caseroot that was specified on the command line instead of what
# is read in the environment as the caseroot may have changed from what is listed
# in the env xml
envDict['CASEROOT'] = caseroot
# add the os.environ['PATH'] to the envDict['PATH']
envDict['ICEDIAG_PATH'] = os.pathsep + os.environ['PATH']
# strip the ICEDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'ICEDIAG_')
envDict['seas'] = ['jfm', 'fm', 'amj', 'jas', 'ond', 'on', 'ann']
return envDict
def initialize_main(envDict, caseroot, debugMsg, standalone):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
standalone (boolean) - specify if postprocess caseroot is standalone (true) of not (false)
Return:
envDict (dictionary) - environment dictionary
"""
env_file_list = ['./env_postprocess.xml', './env_diags_ilamb.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2)
# refer to the caseroot that was specified on the command line instead of what
# is read in the environment as the caseroot may have changed from what is listed
# in the env xml
envDict['CASEROOT'] = caseroot
# add the os.environ['PATH'] to the envDict['PATH']
envDict['ILAMBDIAG_PATH'] = os.pathsep + os.environ['PATH']
# strip the ILAMBDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'ILAMBDIAG_')
# setup the working directories
sys.path.append(envDict['PATH'])
return envDict
def main(options, scomm, rank, size):
"""
"""
# initialize the CASEROOT environment dictionary
cesmEnv = dict()
# CASEROOT is given on the command line as required option --caseroot
caseroot = options.caseroot[0]
# set the caseroot based on standalone or not
pp_caseroot = caseroot
if not options.standalone:
caseroot, pp_subdir = os.path.split(caseroot)
if rank == 0:
print('cesm_tseries_generator: caseroot = {0}'.format(caseroot))
# set the debug level
debug = options.debug[0]
# cesmEnv["id"] = "value" parsed from the CASEROOT/env_*.xml files
env_file_list = [
'env_case.xml', 'env_run.xml', 'env_build.xml', 'env_mach_pes.xml'
]
# check if the standalone option is set
if options.standalone:
env_file_list = ['env_postprocess.xml']
cesmEnv = cesmEnvLib.readXML(caseroot, env_file_list)
# initialize the specifiers list to contain the list of specifier classes
specifiers = list()
# loading the specifiers from the env_timeseries.xml only needs to run on the master task (rank=0)
if rank == 0:
specifiers = readArchiveXML(caseroot, cesmEnv['DOUT_S_ROOT'],
cesmEnv['CASE'], options.standalone, debug)
scomm.sync()
# specifiers is a list of pyreshaper specification objects ready to pass to the reshaper
specifiers = scomm.partition(specifiers,
func=partition.Duplicate(),
involved=True)
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers,
serial=False,
verbosity=debug)
# Run the conversion (slice-to-series) process
reshpr.convert()
# Print timing diagnostics
reshpr.print_diagnostics()
# TO-DO check if DOUT_S_SAVE_HISTORY_FILES is true or false and
# delete history files accordingly
return 0
def initialize_main(envDict, caseroot, debugMsg, standalone, imb_name):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
standalone (boolean) - specify if postprocess caseroot is standalone (true) of not (false)
Return:
envDict (dictionary) - environment dictionary
"""
env_file_list = [
'./env_postprocess.xml', './env_diags_{0}.xml'.format(imb_name)
]
debugMsg('in initialize_main before readXML', header=True, verbosity=1)
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
debugMsg('envDict after readXML = {0}'.format(envDict),
header=True,
verbosity=2)
# refer to the caseroot that was specified on the command line instead of what
# is read in the environment as the caseroot may have changed from what is listed
# in the env xml
envDict['CASEROOT'] = caseroot
diag_name = "{0}DIAG".format(imb_name.upper())
# add the os.environ['PATH'] to the envDict['PATH']
diag_path = "{0}_PATH".format(diag_name)
envDict[diag_path] = os.pathsep + os.environ['PATH']
# strip the ILAMBDIAG_ or IOMBDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, '{0}_'.format(diag_name))
debugMsg('envDict OUTPUTROOT = {0}'.format(envDict['OUTPUTROOT']),
header=True,
verbosity=1)
# ILAMB requires output directory to already exist
if not os.path.exists(envDict['OUTPUTROOT']):
os.makedirs(envDict['OUTPUTROOT'])
debugMsg('in initialize_main before path.append', header=True, verbosity=1)
# setup the working directories
sys.path.append(envDict['PATH'])
debugMsg('in initialize_main get machine name', header=True, verbosity=1)
hostname = cesmEnvLib.get_hostname()
envDict['MACH'] = cesmEnvLib.get_machine_name(
hostname, '{0}/Machines/machine_postprocess.xml'.format(
envDict['POSTPROCESS_PATH']))
return envDict
def initialize_main(envDict, caseroot, debugMsg, standalone):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
Return:
envDict (dictionary) - environment dictionary
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
## env_file_list = ['../env_case.xml', '../env_run.xml', '../env_build.xml', '../env_mach_pes.xml', './env_postprocess.xml', './env_diags_atm.xml']
## if standalone:
env_file_list = ['./env_postprocess.xml', './env_diags_atm.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
debugMsg('envDict after readXML = {0}'.format(envDict),
header=True,
verbosity=2)
# refer to the caseroot that was specified on the command line instead of what
# is read in the environment as the caseroot may have changed from what is listed
# in the env xml
envDict['CASEROOT'] = caseroot
# add the os.environ['PATH'] to the envDict['PATH']
envDict['ATMDIAG_PATH'] = os.pathsep + os.environ['PATH']
# strip the ATMDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'ATMDIAG_')
# TODO - create the list of necessary climatology files for model
filelist = list()
# check average files
debugMsg('calling checkAvgFiles', header=True)
rc = diagUtilsLib.checkAvgFiles(filelist)
if not rc:
print(
'---------------------------------------------------------------------------'
)
print('ERROR: atmosphere climatology files do not exist in:')
print(' {0}'.format(envDict['TAVGDIR']))
print('Please run the {0}.atm_avg_generator script first.'.format(
envDict['CASE']))
print(
'---------------------------------------------------------------------------'
)
sys.exit(1)
# TODO - create the list of necessary climatology files for control
return envDict
def initialize_main(envDict, caseroot, debugMsg, standalone):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
standalong (boolean) - indicate stand-alone post processing caseroot
Return:
envDict (dictionary) - environment dictionary
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
env_file_list = ['./env_postprocess.xml', './env_diags_ocn.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2)
# refer to the caseroot that was specified on the command line instead of what
# is read in the environment as the caseroot may have changed from what is listed
# in the env xml
envDict['CASEROOT'] = caseroot
# add the os.environ['PATH'] to the envDict['PATH']
envDict['OCNDIAG_PATH'] += os.pathsep + os.environ['PATH']
# strip the OCNDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'OCNDIAG_')
# special variable mapped from the CESM env to the OCN diags env
envDict['RESOLUTION'] = envDict['OCN_GRID']
# setup the TOBSFILE and SOBSFILE variables based on the vertical levels
# 60 (default) or 42
if envDict['VERTICAL'] == '42':
envDict['TOBSFILE'] = envDict['TOBSFILE_V42']
envDict['SOBSFILE'] = envDict['SOBSFILE_V42']
debugMsg('TOBSFILE = {0}, SOBSFILE= {1}'.format(envDict['TOBSFILE'], envDict['SOBSFILE']), header=True, verbosity=2)
# TODO - create the list of necessary climatology files for model
filelist = list()
# check average files
debugMsg('calling checkAvgFiles', header=True, verbosity=2)
rc = diagUtilsLib.checkAvgFiles(filelist)
if not rc:
print('---------------------------------------------------------------------------')
print('ERROR: ocean climatology files do not exist in:')
print(' {0}'.format(envDict['TAVGDIR']))
print('Please run the {0}.ocn_avg_generator script first.'.format(envDict['CASE']))
print('---------------------------------------------------------------------------')
sys.exit(1)
return envDict
def initialize_main(envDict, caseroot, debugMsg, standalone):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
standalong (boolean) - indicate stand-alone post processing caseroot
Return:
envDict (dictionary) - environment dictionary
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
env_file_list = ['./env_postprocess.xml', './env_diags_ocn.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2)
# refer to the caseroot that was specified on the command line instead of what
# is read in the environment as the caseroot may have changed from what is listed
# in the env xml
envDict['CASEROOT'] = caseroot
# add the os.environ['PATH'] to the envDict['PATH']
envDict['OCNDIAG_PATH'] += os.pathsep + os.environ['PATH']
# strip the OCNDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'OCNDIAG_')
# special variable mapped from the CESM env to the OCN diags env
envDict['RESOLUTION'] = envDict['OCN_GRID']
# setup the TOBSFILE and SOBSFILE variables based on the vertical levels
# 60 (default) or 42
if envDict['VERTICAL'] == '42':
envDict['TOBSFILE'] = envDict['TOBSFILE_V42']
envDict['SOBSFILE'] = envDict['SOBSFILE_V42']
debugMsg('TOBSFILE = {0}, SOBSFILE= {1}'.format(envDict['TOBSFILE'], envDict['SOBSFILE']), header=True, verbosity=2)
# TODO - create the list of necessary climatology files for model
filelist = list()
# check average files
debugMsg('calling checkAvgFiles', header=True, verbosity=2)
rc = diagUtilsLib.checkAvgFiles(filelist)
if not rc:
print('---------------------------------------------------------------------------')
print('ERROR: ocean climatology files do not exist in:')
print(' {0}'.format(envDict['TAVGDIR']))
print('Please run the {0}.ocn_avg_generator script first.'.format(envDict['CASE']))
print('---------------------------------------------------------------------------')
sys.exit(1)
return envDict
def initialize_envDict(envDict, caseroot, debugMsg, standalone):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
Return:
envDict (dictionary) - environment dictionary
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
env_file_list = ['./env_postprocess.xml', './env_diags_atm.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
debugMsg('envDict after readXML = {0}'.format(envDict),
header=True,
verbosity=2)
# refer to the caseroot that was specified on the command line instead of what
# is read in the environment as the caseroot may have changed from what is listed
# in the env xml
envDict['CASEROOT'] = caseroot
# add the os.environ['PATH'] to the envDict['PATH']
envDict['ATMDIAG_PATH'] = os.pathsep + os.environ['PATH']
# strip the ATMDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'ATMDIAG_')
envDict['seas'] = []
if envDict['plot_ANN_climo']:
envDict['seas'].append('ANN')
if envDict['plot_DJF_climo']:
envDict['seas'].append('DJF')
if envDict['plot_MAM_climo']:
envDict['seas'].append('MAM')
if envDict['plot_JJA_climo']:
envDict['seas'].append('JJA')
if envDict['plot_SON_climo']:
envDict['seas'].append('SON')
if envDict['test_path_history_subdir'].endswith('/'):
envDict['test_path_history_subdir'] = envDict[
'test_path_history_subdir'][:-1]
if envDict['cntl_path_history_subdir'].endswith('/'):
envDict['cntl_path_history_subdir'] = envDict[
'cntl_path_history_subdir'][:-1]
return envDict
def run_initialization(caseroot, debug):
"""initialize_main - initialize settings on rank 0
Arguments:
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
env_file_list = ['env_conform.xml', 'env_postprocess.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
#debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2)
# get the variables needed to send to iconform
global_attributes = envDict[
'CONFORM_GLOBAL_ATTRIBUTES'] + "," + caseroot + "/../archive_files/db.json"
definitions = envDict['CONFORM_CESM_DEFINITIONS']
exp_name = envDict['CONFORM_EXP_NAME']
json_dir = envDict['CONFORM_JSON_DIRECTORY'] + "/PyConform_input/"
output_dir = envDict['CONFORM_OUTPUT_DIR']
extra_field_json = envDict['CONFORM_EXTRA_FIELD_JSON']
try:
pipe = subprocess.Popen([
'iconform -g {0} -d {1} -tt xml -e {2} -p {3} -o {4} -to True'.
format(global_attributes, definitions, exp_name, output_dir,
json_dir)
],
env=envDict,
shell=True,
stdout=subprocess.PIPE)
output = pipe.communicate()[0]
print('iconform: {0}'.format(output))
while pipe.poll() is None:
time.sleep(0.5)
except OSError as e:
print('WARNING', e.errno, e.strerror)
try:
pipe = subprocess.Popen(
['cesm_extras -e {0} {1}/*'.format(extra_field_json, json_dir)],
env=envDict,
shell=True,
stdout=subprocess.PIPE)
output = pipe.communicate()[0]
print('cesm_extras: {0}'.format(output))
while pipe.poll() is None:
time.sleep(0.5)
except OSError as e:
print('WARNING', e.errno, e.strerror)
def initialize_envDict(envDict, caseroot, debugMsg, standalone):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
Return:
envDict (dictionary) - environment dictionary
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
env_file_list = ['./env_postprocess.xml', './env_diags_atm.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2)
# refer to the caseroot that was specified on the command line instead of what
# is read in the environment as the caseroot may have changed from what is listed
# in the env xml
envDict['CASEROOT'] = caseroot
# add the os.environ['PATH'] to the envDict['PATH']
envDict['ATMDIAG_PATH'] = os.pathsep + os.environ['PATH']
# strip the ATMDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'ATMDIAG_')
envDict['seas'] = []
if envDict['plot_ANN_climo']:
envDict['seas'].append('ANN')
if envDict['plot_DJF_climo']:
envDict['seas'].append('DJF')
if envDict['plot_MAM_climo']:
envDict['seas'].append('MAM')
if envDict['plot_JJA_climo']:
envDict['seas'].append('JJA')
if envDict['plot_SON_climo']:
envDict['seas'].append('SON')
if envDict['test_path_history_subdir'].endswith('/'):
envDict['test_path_history_subdir'] = envDict['test_path_history_subdir'][:-1]
if envDict['cntl_path_history_subdir'].endswith('/'):
envDict['cntl_path_history_subdir'] = envDict['cntl_path_history_subdir'][:-1]
return envDict
def initialize_main(envDict, caseroot, debugMsg, standalone):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
Return:
envDict (dictionary) - environment dictionary
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
env_file_list = ['./env_postprocess.xml', './env_diags_atm.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2)
# refer to the caseroot that was specified on the command line instead of what
# is read in the environment as the caseroot may have changed from what is listed
# in the env xml
envDict['CASEROOT'] = caseroot
# add the os.environ['PATH'] to the envDict['PATH']
envDict['ATMDIAG_PATH'] = os.pathsep + os.environ['PATH']
# strip the ATMDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'ATMDIAG_')
# TODO - create the list of necessary climatology files for model
filelist = list()
# check average files
debugMsg('calling checkAvgFiles', header=True)
rc = diagUtilsLib.checkAvgFiles(filelist)
if not rc:
print('---------------------------------------------------------------------------')
print('ERROR: atmosphere climatology files do not exist in:')
print(' {0}'.format(envDict['TAVGDIR']))
print('Please run the {0}.atm_avg_generator script first.'.format(envDict['CASE']))
print('---------------------------------------------------------------------------')
sys.exit(1)
# TODO - create the list of necessary climatology files for control
return envDict
def initialize_main(envDict, caseroot, debugMsg, standalone):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
Return:
envDict (dictionary) - environment dictionary
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
## env_file_list = ['../env_case.xml', '../env_run.xml', '../env_build.xml', '../env_mach_pes.xml', './env_postprocess.xml', './env_diags_ice.xml']
## envDict['STANDALONE'] = False
## if standalone:
env_file_list = ['./env_postprocess.xml', './env_diags_ice.xml']
## envDict['STANDALONE'] = True
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2)
# add the os.environ['PATH'] to the envDict['PATH']
envDict['ICEDIAG_PATH'] = str(os.pathsep + os.environ['PATH'])
# strip the ICEDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'ICEDIAG_')
# TODO - create the list of necessary climatology files for model
filelist = list()
# check average files
debugMsg('calling checkAvgFiles', header=True)
rc = diagUtilsLib.checkAvgFiles(filelist)
if not rc:
print('---------------------------------------------------------------------------')
print('ERROR: ice climatology files do not exist in:')
print(' {0}'.format(envDict['TAVGDIR']))
print('Please run the {0}.ice_avg_generator script first.'.format(envDict['CASE']))
print('---------------------------------------------------------------------------')
sys.exit(1)
# TODO - create the list of necessary climatology files for control
return envDict
def initialize_envDict(envDict, caseroot, debugMsg, standalone):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
standalone (boolean) - indicate stand-alone post processing caseroot
Return:
envDict (dictionary) - environment dictionary
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
env_file_list = ['./env_postprocess.xml', './env_diags_ocn.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
debugMsg('envDict after readXML = {0}'.format(envDict),
header=True,
verbosity=2)
# refer to the caseroot that was specified on the command line instead of what
# is read in the environment as the caseroot may have changed from what is listed
# in the env xml
envDict['CASEROOT'] = caseroot
# add the os.environ['PATH'] to the envDict['PATH']
envDict['OCNDIAG_PATH'] += os.pathsep + os.environ['PATH']
# initialize varLists
envDict['MODEL_VARLIST'] = []
if len(envDict['OCNDIAG_PYAVG_MODELCASE_VARLIST']) > 0:
envDict['MODEL_VARLIST'] = envDict[
'OCNDIAG_PYAVG_MODELCASE_VARLIST'].split(',')
envDict['CNTRL_VARLIST'] = []
if len(envDict['OCNDIAG_PYAVG_CNTRLCASE_VARLIST']) > 0:
envDict['CNTRL_VARLIST'] = envDict[
'OCNDIAG_PYAVG_CNTRLCASE_VARLIST'].split(',')
# strip the OCNDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'OCNDIAG_')
return envDict
def run_initialization(caseroot, debug):
"""initialize_main - initialize settings on rank 0
Arguments:
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
env_file_list = ['env_conform.xml','env_postprocess.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
#debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2)
# get the variables needed to send to iconform
global_attributes = envDict['CONFORM_GLOBAL_ATTRIBUTES']+","+caseroot+"/../archive_files/db.json"
definitions = envDict['CONFORM_CESM_DEFINITIONS']
exp_name = envDict['CONFORM_EXP_NAME']
json_dir = envDict['CONFORM_JSON_DIRECTORY']+"/PyConform_input/"
output_dir = envDict['CONFORM_OUTPUT_DIR']
extra_field_json = envDict['CONFORM_EXTRA_FIELD_JSON']
try:
pipe = subprocess.Popen(['iconform -g {0} -d {1} -tt xml -e {2} -p {3} -o {4} -to True'.format(
global_attributes, definitions, exp_name, output_dir, json_dir)], env=envDict, shell=True, stdout=subprocess.PIPE)
output = pipe.communicate()[0]
print('iconform: {0}'.format(output))
while pipe.poll() is None:
time.sleep(0.5)
except OSError as e:
print('WARNING',e.errno,e.strerror)
try:
pipe = subprocess.Popen(['cesm_extras -e {0} {1}/*'.format(
extra_field_json, json_dir)], env=envDict, shell=True, stdout=subprocess.PIPE)
output = pipe.communicate()[0]
print('cesm_extras: {0}'.format(output))
while pipe.poll() is None:
time.sleep(0.5)
except OSError as e:
print('WARNING',e.errno,e.strerror)
def initialize_envDict(envDict, caseroot, debugMsg, standalone, main_comm):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
Return:
envDict (dictionary) - environment dictionary
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
# TODO put this file list into the config_postprocess definition
## env_file_list = ['../env_case.xml', '../env_run.xml', '../env_build.xml', '../env_mach_pes.xml', './env_postprocess.xml', './env_diags_lnd.xml']
## if standalone:
env_file_list = ['./env_postprocess.xml', './env_diags_lnd.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
if main_comm.is_manager():
debugMsg('envDict after readXML = {0}'.format(envDict),
header=True,
verbosity=1)
# refer to the caseroot that was specified on the command line instead of what
# is read in the environment as the caseroot may have changed from what is listed
# in the env xml
# add the os.environ['PATH'] to the envDict['PATH']
envDict['LANDDIAG_PATH'] = os.pathsep + os.environ['PATH']
# strip the LNDDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'LNDDIAG_')
envDict['seas'] = ['djf', 'mam', 'jja', 'son', 'ann']
return envDict
def initialize_main(envDict, caseroot, debugMsg, standalone):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
standalone (boolean) - specify if postprocess caseroot is standalone (true) of not (false)
Return:
envDict (dictionary) - environment dictionary
"""
env_file_list = ['./env_postprocess.xml', './env_diags_ilamb.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
debugMsg('envDict after readXML = {0}'.format(envDict),
header=True,
verbosity=2)
# refer to the caseroot that was specified on the command line instead of what
# is read in the environment as the caseroot may have changed from what is listed
# in the env xml
envDict['CASEROOT'] = caseroot
# add the os.environ['PATH'] to the envDict['PATH']
envDict['ILAMBDIAG_PATH'] = os.pathsep + os.environ['PATH']
# strip the ILAMBDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'ILAMBDIAG_')
# setup the working directories
sys.path.append(envDict['PATH'])
return envDict
def initialize_envDict(envDict, caseroot, debugMsg, standalone, main_comm):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
Return:
envDict (dictionary) - environment dictionary
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
# TODO put this file list into the config_postprocess definition
## env_file_list = ['../env_case.xml', '../env_run.xml', '../env_build.xml', '../env_mach_pes.xml', './env_postprocess.xml', './env_diags_lnd.xml']
## if standalone:
env_file_list = ['./env_postprocess.xml', './env_diags_lnd.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
if main_comm.is_manager():
debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=1)
# refer to the caseroot that was specified on the command line instead of what
# is read in the environment as the caseroot may have changed from what is listed
# in the env xml
# add the os.environ['PATH'] to the envDict['PATH']
envDict['LANDDIAG_PATH'] = os.pathsep + os.environ['PATH']
# strip the LNDDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'LNDDIAG_')
envDict['seas'] = ['djf','mam','jja','son','ann']
return envDict
def main(options, scomm, rank, size, debug, debugMsg):
"""
"""
# initialize the CASEROOT environment dictionary
cesmEnv = dict()
# CASEROOT is given on the command line as required option --caseroot
caseroot = options.caseroot[0]
# get the XML variables loaded into a hash
env_file_list = ['env_postprocess.xml']
cesmEnv = cesmEnvLib.readXML(caseroot, env_file_list)
# initialize the specifiers list to contain the list of specifier classes
specifiers = list()
# loading the specifiers from the env_timeseries.xml only needs to run on the master task (rank=0)
if rank == 0:
tseries_input_rootdir = cesmEnv['TIMESERIES_INPUT_ROOTDIR']
tseries_output_rootdir = cesmEnv['TIMESERIES_OUTPUT_ROOTDIR']
case = cesmEnv['CASE']
completechunk = cesmEnv['TIMESERIES_COMPLETECHUNK']
if completechunk.upper() in ['T', 'TRUE']:
completechunk = 1
else:
completechunk = 0
specifiers, log = readArchiveXML(caseroot, tseries_input_rootdir,
tseries_output_rootdir, case,
options.standalone, completechunk,
debug, debugMsg)
scomm.sync()
# specifiers is a list of pyreshaper specification objects ready to pass to the reshaper
specifiers = scomm.partition(specifiers,
func=partition.Duplicate(),
involved=True)
if rank == 0:
debugMsg("# of Specifiers: " + str(len(specifiers)),
header=True,
verbosity=1)
if len(specifiers) > 0:
# setup subcommunicators to do streams and chunks in parallel
# everyone participates except for root
inter_comm, lsubcomms = divide_comm(scomm, len(specifiers))
color = inter_comm.get_color()
lsize = inter_comm.get_size()
lrank = inter_comm.get_rank()
GWORK_TAG = 10 # global comm mpi tag
LWORK_TAG = 20 # local comm mpi tag
# global root - hands out specifiers to work on. When complete, it must tell each subcomm all work is done.
if (rank == 0):
for i in range(0, len(specifiers)): # hand out all specifiers
scomm.ration(data=i, tag=GWORK_TAG)
for i in range(0,
lsubcomms): # complete, signal this to all subcomms
scomm.ration(data=-99, tag=GWORK_TAG)
# subcomm root - performs the same tasks as other subcomm ranks, but also gets the specifier to work on and sends
# this information to all ranks within subcomm
elif (lrank == 0):
i = -999
while i != -99:
i = scomm.ration(tag=GWORK_TAG) # recv from global
for x in range(1, lsize):
inter_comm.ration(i, LWORK_TAG) # send to local ranks
if i != -99:
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers[i],
serial=False,
verbosity=debug,
simplecomm=inter_comm)
# Run the conversion (slice-to-series) process
reshpr.convert()
inter_comm.sync()
# all subcomm ranks - recv the specifier to work on and call the reshaper
else:
i = -999
while i != -99:
i = inter_comm.ration(tag=LWORK_TAG) # recv from local root
if i != -99:
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers[i],
serial=False,
verbosity=debug,
simplecomm=inter_comm)
# Run the conversion (slice-to-series) process
reshpr.convert()
inter_comm.sync()
if rank == 0:
# Update system log with the dates that were just converted
debugMsg('before chunking.write_log', header=True, verbosity=1)
chunking.write_log('{0}/logs/ts_status.log'.format(caseroot), log)
debugMsg('after chunking.write_log', header=True, verbosity=1)
scomm.sync()
return 0
def initialize_main(envDict, caseroot, debugMsg, standalone):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
Return:
envDict (dictionary) - environment dictionary
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
## env_file_list = ['../env_case.xml', '../env_run.xml', '../env_build.xml', '../env_mach_pes.xml', './env_postprocess.xml', './env_diags_lnd.xml']
## envDict['STANDALONE'] = False
## if standalone:
env_file_list = ['./env_postprocess.xml', './env_diags_lnd.xml']
## envDict['STANDALONE'] = True
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
debugMsg('envDict after readXML = {0}'.format(envDict), header=True, verbosity=2)
# refer to the caseroot that was specified on the command line instead of what
# is read in the environment as the caseroot may have changed from what is listed
# in the env xml
envDict['CASEROOT'] = caseroot
# add the os.environ['PATH'] to the envDict['PATH']
envDict['LNDDIAG_PATH'] = os.pathsep + os.environ['PATH']
# strip the LNDDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'LNDDIAG_')
# TODO - create the list of necessary climatology files for model
filelist = list()
# check average files
debugMsg('calling checkAvgFiles', header=True, verbosity=1)
rc = diagUtilsLib.checkAvgFiles(filelist)
if not rc:
print('---------------------------------------------------------------------------')
print('ERROR: land climatology files do not exist')
print('Please run the {0}.lnd_avg_generator script first.'.format(envDict['CASE']))
print('---------------------------------------------------------------------------')
sys.exit(1)
# TODO - create the list of necessary climatology files for control
# setup the working directories
sys.path.append(envDict['PATH'])
# the WKDIR variable is very confusing... it gets reset in setup_workdir
if (envDict['MODEL_VS_MODEL'] == 'True'):
envDict['WKDIR'] = envDict['PTMPDIR_1']+'/diag/'+envDict['caseid_1']+'-'+envDict['caseid_2']+'/'
else:
envDict['WKDIR'] = envDict['PTMPDIR_1']+'/diag/'+envDict['caseid_1']+'-obs/'
if envDict['CASA'] == '1':
envDict['VAR_MASTER'] = envDict['var_master_casa']
else:
envDict['VAR_MASTER'] = envDict['var_master_cn']
return envDict
def main(options, scomm, rank, size, debug, debugMsg):
"""
"""
# initialize the CASEROOT environment dictionary
cesmEnv = dict()
# CASEROOT is given on the command line as required option --caseroot
caseroot = options.caseroot[0]
# get the XML variables loaded into a hash
env_file_list = ['env_postprocess.xml']
cesmEnv = cesmEnvLib.readXML(caseroot, env_file_list);
# initialize the specifiers list to contain the list of specifier classes
specifiers = list()
tseries_input_rootdir = cesmEnv['TIMESERIES_INPUT_ROOTDIR']
tseries_output_rootdir = cesmEnv['TIMESERIES_OUTPUT_ROOTDIR']
case = cesmEnv['CASE']
completechunk = cesmEnv['TIMESERIES_COMPLETECHUNK']
generate_all = cesmEnv['TIMESERIES_GENERATE_ALL']
if completechunk.upper() in ['T','TRUE']:
completechunk = 1
else:
completechunk = 0
specifiers,log = readArchiveXML(caseroot, tseries_input_rootdir, tseries_output_rootdir,
case, options.standalone, completechunk, generate_all,
debug, debugMsg, scomm, rank, size)
scomm.sync()
# specifiers is a list of pyreshaper specification objects ready to pass to the reshaper
if rank == 0:
debugMsg("# of Specifiers: "+str(len(specifiers)), header=True, verbosity=1)
if len(specifiers) > 0:
# setup subcommunicators to do streams and chunks in parallel
# everyone participates except for root
inter_comm, lsubcomms = divide_comm(scomm, len(specifiers))
color = inter_comm.get_color()
lsize = inter_comm.get_size()
lrank = inter_comm.get_rank()
GWORK_TAG = 10 # global comm mpi tag
LWORK_TAG = 20 # local comm mpi tag
# global root - hands out specifiers to work on. When complete, it must tell each subcomm all work is done.
if (rank == 0):
for i in range(0,len(specifiers)): # hand out all specifiers
scomm.ration(data=i, tag=GWORK_TAG)
for i in range(0,lsubcomms): # complete, signal this to all subcomms
scomm.ration(data=-99, tag=GWORK_TAG)
# subcomm root - performs the same tasks as other subcomm ranks, but also gets the specifier to work on and sends
# this information to all ranks within subcomm
elif (lrank == 0):
i = -999
while i != -99:
i = scomm.ration(tag=GWORK_TAG) # recv from global
for x in range(1,lsize):
inter_comm.ration(i, LWORK_TAG) # send to local ranks
if i != -99:
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers[i], serial=False, verbosity=debug, simplecomm=inter_comm)
# Run the conversion (slice-to-series) process
reshpr.convert()
inter_comm.sync()
# all subcomm ranks - recv the specifier to work on and call the reshaper
else:
i = -999
while i != -99:
i = inter_comm.ration(tag=LWORK_TAG) # recv from local root
if i != -99:
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers[i], serial=False, verbosity=debug, simplecomm=inter_comm)
# Run the conversion (slice-to-series) process
reshpr.convert()
inter_comm.sync()
if rank == 0:
# Update system log with the dates that were just converted
debugMsg('before chunking.write_log', header=True, verbosity=1)
chunking.write_log('{0}/logs/ts_status.log'.format(caseroot), log)
debugMsg('after chunking.write_log', header=True, verbosity=1)
scomm.sync()
return 0
def main(options, scomm, rank, size):
"""
"""
# initialize the CASEROOT environment dictionary
cesmEnv = dict()
# CASEROOT is given on the command line as required option --caseroot
caseroot = options.caseroot[0]
# set the debug level
debug = options.debug[0]
# is there only one mip definition in each file?
ind = "True"
# get the XML variables loaded into a hash
env_file_list = ['env_postprocess.xml','env_conform.xml']
cesmEnv = cesmEnvLib.readXML(caseroot, env_file_list);
# We want to have warnings and not errors (at least for the first sets of cmip simulations)
simplefilter("default", ValidationWarning)
# Get the extra modules pyconform needs
pp_path = cesmEnv["POSTPROCESS_PATH"]
conform_module_path = pp_path+'/conformer/conformer/source/pyconform/modules/'
for i, m in enumerate(external_mods):
print("Loading: "+conform_module_path+"/"+m)
load_source('user{}'.format(i), conform_module_path+"/"+m)
# create the cesm stream to table mapping
# if rank == 0:
dout_s_root = cesmEnv['DOUT_S_ROOT']
case = cesmEnv['CASE']
pc_inpur_dir = cesmEnv['CONFORM_JSON_DIRECTORY']+'/PyConform_input/'
#readArchiveXML(caseroot, dout_s_root, case, debug)
nc_files = find_nc_files(dout_s_root)
variable_list = fill_list(nc_files, pc_inpur_dir, cesmEnv["CONFORM_EXTRA_FIELD_NETCDF_DIR"], scomm, rank, size)
mappings = {}
if rank == 0:
mappings = match_tableSpec_to_stream(pc_inpur_dir, variable_list)
for k,v in sorted(mappings.iteritems()):
print k
for f in sorted(v):
print f
print len(v),'\n\n'
scomm.sync()
# Pass the stream and mapping information to the other procs
mappings = scomm.partition(mappings, func=partition.Duplicate(), involved=True)
print("I CAN RUN ",len(mappings.keys())," json files")
failures = 0
if len(mappings.keys()) > 0:
# setup subcommunicators to do streams and chunks in parallel
# everyone participates except for root
inter_comm, lsubcomms = divide_comm(scomm, len(mappings.keys()), ind)
color = inter_comm.get_color()
lsize = inter_comm.get_size()
lrank = inter_comm.get_rank()
print "MPI INFO: ",color," ",lrank,"/",lsize," ",rank,"/",size
GWORK_TAG = 10 # global comm mpi tag
LWORK_TAG = 20 # local comm mpi tag
# global root - hands out mappings to work on. When complete, it must tell each subcomm all work is done.
if (rank == 0):
#for i in range(0,len(mappings.keys())): # hand out all mappings
for i in mappings.keys():
scomm.ration(data=i, tag=GWORK_TAG)
for i in range(1,lsubcomms): # complete, signal this to all subcomms
scomm.ration(data=-99, tag=GWORK_TAG)
# subcomm root - performs the same tasks as other subcomm ranks, but also gets the specifier to work on and sends
# this information to all ranks within subcomm
elif (lrank == 0):
i = -999
while i != -99:
i = scomm.ration(tag=GWORK_TAG) # recv from global
for x in range(1,lsize):
inter_comm.ration(i, LWORK_TAG) # send to local ranks
if i != -99:
print "(",rank,"/",lrank,")"," start running ",i
failures += run_PyConform(i, mappings[i], inter_comm)
print "(",rank,"/",lrank,")"," finished running ",i
print "(",rank,"/",lrank,")","FAILURES: ",failures
inter_comm.sync()
# all subcomm ranks - recv the specifier to work on and call the reshaper
else:
i = -999
while i != -99:
i = inter_comm.ration(tag=LWORK_TAG) # recv from local root
if i != -99:
print "(",rank,"/",lrank,")"," start running ",i
failures += run_PyConform(i, mappings[i], inter_comm)
print "(",rank,"/",lrank,")"," finished running ",i
print "(",rank,"/",lrank,")","FAILURES: ",failures
inter_comm.sync()
print "(",rank,"/",lrank,")"," FINISHED"
scomm.sync()
def main(options, scomm, rank, size):
"""
"""
# setup an overall timer
timer = timekeeper.TimeKeeper()
timer.start("Total Time")
# initialize the CASEROOT environment dictionary
cesmEnv = dict()
# CASEROOT is given on the command line as required option --caseroot
caseroot = options.caseroot[0]
# set the debug level
debug = options.debug[0]
# is there only one mip definition in each file?
ind = "True"
# get the XML variables loaded into a hash
env_file_list = ['env_postprocess.xml','env_conform.xml']
cesmEnv = cesmEnvLib.readXML(caseroot, env_file_list);
# We want to have warnings and not errors (at least for the first sets of cmip simulations)
simplefilter("default", ValidationWarning)
# Get the extra modules pyconform needs
pp_path = cesmEnv["POSTPROCESS_PATH"]
conform_module_path = pp_path+'/conformer/conformer/source/pyconform/modules/'
for i, m in enumerate(external_mods):
print("Loading: "+conform_module_path+"/"+m)
load_source('user{}'.format(i), conform_module_path+"/"+m)
# create the cesm stream to table mapping
# if rank == 0:
dout_s_root = cesmEnv['TIMESERIES_OUTPUT_ROOTDIR']
case = cesmEnv['CASE']
pc_inpur_dir = cesmEnv['CONFORM_JSON_DIRECTORY']+'/PyConform_input/'
#readArchiveXML(caseroot, dout_s_root, case, debug)
nc_files = find_nc_files(dout_s_root)
variable_list = fill_list(nc_files, pc_inpur_dir, cesmEnv["CONFORM_EXTRA_FIELD_NETCDF_DIR"], scomm, rank, size)
mappings = {}
if rank == 0:
mappings = match_tableSpec_to_stream(pc_inpur_dir, variable_list, dout_s_root, case)
for k,v in sorted(mappings.iteritems()):
print k
for f in sorted(v):
print f
print len(v),'\n\n'
scomm.sync()
# Pass the stream and mapping information to the other procs
mappings = scomm.partition(mappings, func=partition.Duplicate(), involved=True)
print("I CAN RUN ",len(mappings.keys())," json files")
failures = 0
if len(mappings.keys()) > 0:
# setup subcommunicators to do streams and chunks in parallel
# everyone participates except for root
inter_comm, lsubcomms = divide_comm(scomm, len(mappings.keys()), ind)
color = inter_comm.get_color()
lsize = inter_comm.get_size()
lrank = inter_comm.get_rank()
print "MPI INFO: ",color," ",lrank,"/",lsize," ",rank,"/",size
GWORK_TAG = 10 # global comm mpi tag
LWORK_TAG = 20 # local comm mpi tag
# global root - hands out mappings to work on. When complete, it must tell each subcomm all work is done.
if (rank == 0):
#for i in range(0,len(mappings.keys())): # hand out all mappings
for i in mappings.keys():
scomm.ration(data=i, tag=GWORK_TAG)
for i in range(1,lsubcomms): # complete, signal this to all subcomms
scomm.ration(data=-99, tag=GWORK_TAG)
# subcomm root - performs the same tasks as other subcomm ranks, but also gets the specifier to work on and sends
# this information to all ranks within subcomm
elif (lrank == 0):
i = -999
while i != -99:
i = scomm.ration(tag=GWORK_TAG) # recv from global
for x in range(1,lsize):
inter_comm.ration(i, LWORK_TAG) # send to local ranks
if i != -99:
print "(",rank,"/",lrank,")"," start running ",i
failures += run_PyConform(i, mappings[i], inter_comm)
print "(",rank,"/",lrank,")"," finished running ",i
print "(",rank,"/",lrank,")","FAILURES: ",failures
inter_comm.sync()
# all subcomm ranks - recv the specifier to work on and call the reshaper
else:
i = -999
while i != -99:
i = inter_comm.ration(tag=LWORK_TAG) # recv from local root
if i != -99:
print "(",rank,"/",lrank,")"," start running ",i
failures += run_PyConform(i, mappings[i], inter_comm)
print "(",rank,"/",lrank,")"," finished running ",i
print "(",rank,"/",lrank,")","FAILURES: ",failures
inter_comm.sync()
print "(",rank,"/",lrank,")"," FINISHED"
scomm.sync()
timer.stop("Total Time")
if rank == 0:
print('************************************************************')
print('Total Time: {0} seconds'.format(timer.get_time("Total Time")))
print('************************************************************')
def initialize_main(envDict, caseroot, debugMsg, standalone):
"""initialize_main - initialize settings on rank 0
Arguments:
envDict (dictionary) - environment dictionary
caseroot (string) - case root
debugMsg (object) - vprinter object for printing debugging messages
Return:
envDict (dictionary) - environment dictionary
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
## env_file_list = ['../env_case.xml', '../env_run.xml', '../env_build.xml', '../env_mach_pes.xml', './env_postprocess.xml', './env_diags_lnd.xml']
## envDict['STANDALONE'] = False
## if standalone:
env_file_list = ['./env_postprocess.xml', './env_diags_lnd.xml']
## envDict['STANDALONE'] = True
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# debug print out the envDict
debugMsg('envDict after readXML = {0}'.format(envDict),
header=True,
verbosity=2)
# refer to the caseroot that was specified on the command line instead of what
# is read in the environment as the caseroot may have changed from what is listed
# in the env xml
envDict['CASEROOT'] = caseroot
# add the os.environ['PATH'] to the envDict['PATH']
envDict['LNDDIAG_PATH'] = os.pathsep + os.environ['PATH']
# strip the LNDDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'LNDDIAG_')
# TODO - create the list of necessary climatology files for model
filelist = list()
# check average files
debugMsg('calling checkAvgFiles', header=True, verbosity=1)
rc = diagUtilsLib.checkAvgFiles(filelist)
if not rc:
print(
'---------------------------------------------------------------------------'
)
print('ERROR: land climatology files do not exist')
print('Please run the {0}.lnd_avg_generator script first.'.format(
envDict['CASE']))
print(
'---------------------------------------------------------------------------'
)
sys.exit(1)
# TODO - create the list of necessary climatology files for control
# setup the working directories
sys.path.append(envDict['PATH'])
# the WKDIR variable is very confusing... it gets reset in setup_workdir
if (envDict['MODEL_VS_MODEL'] == 'True'):
envDict['WKDIR'] = envDict['PTMPDIR_1'] + '/diag/' + envDict[
'caseid_1'] + '-' + envDict['caseid_2'] + '/'
else:
envDict['WKDIR'] = envDict['PTMPDIR_1'] + '/diag/' + envDict[
'caseid_1'] + '-obs/'
if envDict['CASA'] == '1':
envDict['VAR_MASTER'] = envDict['var_master_casa']
else:
envDict['VAR_MASTER'] = envDict['var_master_cn']
return envDict
def main(options, main_comm, debugMsg):
"""
read env_ocn_remap.xml settings to call the ocean_remap class
"""
# initialize the environment dictionary
envDict = dict()
# Get rank and size
rank = main_comm.get_rank()
size = main_comm.get_size()
# CASEROOT is given on the command line as required option --caseroot
if rank == 0:
caseroot = options.caseroot[0]
envDict['CASEROOT'] = options.caseroot[0]
debugMsg('caseroot = {0}'.format(envDict['CASEROOT']),
header=True,
verbosity=2)
env_file_list = ['./env_postprocess.xml', './env_ocn_remap.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
# strip the OCNREMAP_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'OCNREMAP_')
print("cmip6: {0}".format(envDict['cmip6']))
print("filelist: {0}".format(envDict['filelist']))
print("matrix_2d_fname: {0}".format(envDict['matrix_2d_fname']))
print("matrix_3d_fname: {0}".format(envDict['matrix_3d_fname']))
print("indir: {0}".format(envDict['indir']))
print("outdir: {0}".format(envDict['outdir']))
print("chunk size: {0}".format(envDict['chunk']))
# broadcast envDict to all tasks
envDict = main_comm.partition(data=envDict,
func=partition.Duplicate(),
involved=True)
main_comm.sync()
files = []
if rank == 0:
# Find files to regrid
#Do we have a cmip6 variable list?
if envDict['cmip6'] is not None:
if envDict['indir'] is not None:
with open(envDict['cmip6']) as f:
for l in f:
t = l.strip().split(':')[0]
v = l.strip().split(':')[1]
print("Trying to find: {0}_{1}*.nc".format(v, t))
for root, dirs, fns in os.walk(envDict['indir']):
for fn in fnmatch.filter(fns,
v + '_' + t + "*.nc"):
if 'tmp.nc' not in fn and 'gr' not in fn.split(
'_'):
print("Found: {0}".format(fn.split('/')))
files.append(os.path.join(root, fn))
else:
print(
"You need to specify an indir argument with the cmip6 argument"
)
file = None
elif envDict['filelist'] is not None:
with open(envDict['filelist']) as f:
for l in f:
files.append(l.strip())
elif envDict['indir'] is not None:
for root, dirs, fns in os.walk(envDict['indir']):
for fn in fnmatch.filter(fns, "*.nc"):
files.append(os.path.join(root, fn))
else:
print('Exiting because no input path or files where given')
files = None
# All call this
main_comm.sync()
files = main_comm.partition(files,
func=partition.Duplicate(),
involved=True)
if files is None:
sys.exit()
#matrix_2d_fname = 'POP_gx1v7_to_latlon_1x1_0E_mask_conserve_20181015.nc'
matrix_2d = remap.ocean_remap(envDict['matrix_2d_fname'])
#matrix_3d_fname = 'POP_gx1v7_to_latlon_1x1_0E_fulldepth_conserve_20181015.nc'
matrix_3d = remap.ocean_remap(envDict['matrix_3d_fname'])
# names of coordinate dimensions in output files
dim_names = {'depth': 'olevel', 'lat': 'latitude', 'lon': 'longitude'}
dim_names = {'depth': 'lev', 'lat': 'lat', 'lon': 'lon'}
main_comm.sync()
# Have only root create these files
if rank == 0:
if len(files) > 0 and envDict['cmip6'] is not None:
temp = files[0]
# create CMIP Ofx files
for var_name in ('areacello', 'deptho', 'thkcello', 'volcello'):
new_outdir = temp.replace(
temp.split('/')[-4], var_name).replace(
temp.split('/')[-5], 'Ofx').replace(
temp.split('/')[-3], 'gr').replace(
'_' + temp.split('_')[-1], '') + '.nc'
d = os.path.dirname(new_outdir)
if not os.path.exists(d):
os.makedirs(d)
fptr_out = nc.Dataset(new_outdir, 'w') # pylint: disable=E1101
matrix_3d.dst_grid.def_dims_common(fptr_out, dim_names)
matrix_3d.dst_grid.write_vars_common(fptr_out, dim_names)
matrix_3d.dst_grid.write_var_CMIP_Ofx(fptr_out, dim_names,
var_name)
# Create a master slave parallel protocol
GWORK_TAG = 10 # global comm mpi tag
if (rank == 0):
for i in files:
main_comm.ration(data=i, tag=GWORK_TAG)
for i in range(1, size):
main_comm.ration(data=-99, tag=GWORK_TAG)
else:
f = -999
while f != -99:
f = main_comm.ration(tag=GWORK_TAG)
if f != -99:
print("working on: {0}".format(f))
testfile_in_fname = f
testfile_out_fname = f.replace(f.split('/')[-3], 'gr')
if not os.path.exists(testfile_out_fname):
d = os.path.dirname(testfile_out_fname)
if not os.path.exists(d):
os.makedirs(d)
fptr_in = nc.Dataset(testfile_in_fname, 'r') # pylint: disable=E1101
if (len(fptr_in[f.split('/')[-4]].dimensions) == 4
or len(fptr_in[f.split('/')[-4]].dimensions) == 3):
fptr_out = nc.Dataset(testfile_out_fname + '.tmp', 'w') # pylint: disable=E1101
remap.copy_time(fptr_in, fptr_out)
remap.copy_gAttr(fptr_in, fptr_out)
if dim_names['depth'] in fptr_in.dimensions:
matrix_3d.dst_grid.def_dims_common(
fptr_out, dim_names)
matrix_3d.dst_grid.write_vars_common(
fptr_out, dim_names)
else:
matrix_2d.dst_grid.def_dims_common(
fptr_out, dim_names)
matrix_2d.dst_grid.write_vars_common(
fptr_out, dim_names)
field_names = []
for v in fptr_in.variables:
if v not in [
'lat', 'lat_bnds', 'lon', 'lon_bnds',
'lev', 'lev_bnds', 'time', 'time_bnds',
'nlat', 'nlon'
]:
field_names.append(v)
for field_name in field_names:
varid_out = remap.def_var(field_name, fptr_in,
fptr_out, dim_names)
# use appropriate matrix for regridding
c = envDict['chunk']
if c is None:
c = 1
else:
c = int(c)
try:
if dim_names['depth'] in varid_out.dimensions:
#print ("Running a 3D variable")
b = 0
for i in range(
0, fptr_in.dimensions['time'].size,
c):
if b + c >= fptr_in.dimensions[
'time'].size:
c = fptr_in.dimensions[
'time'].size - b
varid_out[b:(
b +
c), :, :, :] = matrix_3d.remap_var(
fptr_in.variables[field_name][
b:(b + c), :, :, :]) #,
#fill_value=getattr(varid_out, 'missing_value'))
b = b + c
else:
#print ("Running a 2D variable")
b = 0
for i in range(
0, fptr_in.dimensions['time'].size,
c):
if b + c >= fptr_in.dimensions[
'time'].size:
c = fptr_in.dimensions[
'time'].size - b
varid_out[b:(
b +
c), :, :] = matrix_2d.remap_var(
fptr_in.variables[field_name][
b:(b + c), :, :]) #,
#fill_value=getattr(varid_out, 'missing_value'))
b = b + c
except TypeError as e:
print('Type Error for variable {0} '.format(
field_name))
fptr_in.close()
fptr_out.close()
try:
os.rename(testfile_out_fname + '.tmp',
testfile_out_fname)
except OSError as e:
print('Could not create {0}'.format(
testfile_out_fname))
else:
print("Not creating {0}".format(testfile_out_fname))
main_comm.sync()
