def check_prerequisites(self, env):
"""This method does some generic checks for the plots prerequisites
that are common to all plots
"""
print(' Checking generic prerequisites for land diagnostics plot.')
cesmEnvLib.setXmlEnv(env)
def check_prerequisites(self, env):
"""This method does some generic checks for the plots prerequisites
that are common to all plots
"""
print(' Checking generic prerequisites for atmosphere diagnostics plot.')
cesmEnvLib.setXmlEnv(env)
def check_prerequisites(self, env):
"""This method does some generic checks for the plots prerequisites
that are common to all plots
"""
print(' Checking generic prerequisites for ocean diagnostics plot.')
# set SEASAVGFILE env var to the envDict['MAVGFILE'] file
env['SEASAVGFILE'] = env['MAVGFILE']
# set SEASAVGTEMP env var to the envDict['MAVGFILE'] file
env['SEASAVGTEMP'] = env['MAVGFILE']
# set SEASAVGSALT env var to the envDict['MAVGFILE'] file
env['SEASAVGSALT'] = env['MAVGFILE']
cesmEnvLib.setXmlEnv(env)
def main(options, main_comm, debugMsg):
"""setup the environment for running the diagnostics in parallel.
Calls 6 different diagnostics generation types:
model vs. observation (optional BGC - ecosystem)
model vs. control (optional BGC - ecosystem)
model time-series (optional BGC - ecosystem)
Arguments:
options (object) - command line options
main_comm (object) - MPI simple communicator object
debugMsg (object) - vprinter object for printing debugging messages
The env_diags_ocn.xml configuration file defines the way the diagnostics are generated.
See (website URL here...) for a complete desciption of the env_diags_ocn XML options.
"""
# initialize the environment dictionary
envDict = dict()
# CASEROOT is given on the command line as required option --caseroot
if main_comm.is_manager():
caseroot = options.caseroot[0]
debugMsg('caseroot = {0}'.format(caseroot), header=True, verbosity=2)
debugMsg('calling initialize_main', header=True, verbosity=2)
envDict = initialize_main(envDict, caseroot, debugMsg, options.standalone)
debugMsg('calling check_ncl_nco', header=True, verbosity=2)
diagUtilsLib.check_ncl_nco(envDict)
# broadcast envDict to all tasks
envDict = main_comm.partition(data=envDict, func=partition.Duplicate(), involved=True)
sys.path.append(envDict['PATH'])
main_comm.sync()
# get list of diagnostics types to be created
diag_list = list()
num_of_diags = 0
if main_comm.is_manager():
diag_list, diag_dict = setup_diags(envDict)
num_of_diags = len(diag_list)
if num_of_diags == 0:
print('No ocean diagnostics specified. Please check the {0}/env_diags_ocn.xml settings.'.format(envDict['PP_CASE_PATH']))
sys.exit(1)
print('User requested diagnostics:')
for diag in diag_list:
print(' {0}'.format(diag))
try:
os.makedirs(envDict['WORKDIR'])
except OSError as exception:
if exception.errno != errno.EEXIST:
err_msg = 'ERROR: ocn_diags_generator.py problem accessing the working directory {0}'.format(envDict['WORKDIR'])
raise OSError(err_msg)
debugMsg('Ocean diagnostics - Creating main index.html page', header=True, verbosity=2)
# define the templatePath
templatePath = '{0}/diagnostics/diagnostics/ocn/Templates'.format(envDict['POSTPROCESS_PATH'])
templateLoader = jinja2.FileSystemLoader( searchpath=templatePath )
templateEnv = jinja2.Environment( loader=templateLoader )
template_file = 'ocean_diagnostics.tmpl'
template = templateEnv.get_template( template_file )
# get the current datatime string for the template
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
# set the template variables
templateVars = { 'casename' : envDict['CASE'],
'tagname' : envDict['CESM_TAG'],
'diag_dict' : diag_dict,
'control_casename' : envDict['CNTRLCASE'],
'start_year' : envDict['YEAR0'],
'stop_year' : envDict['YEAR1'],
'control_start_year' : envDict['CNTRLYEAR0'],
'control_stop_year' : envDict['CNTRLYEAR1'],
'today': now,
'tseries_start_year' : envDict['TSERIES_YEAR0'],
'tseries_stop_year' : envDict['TSERIES_YEAR1']
}
# write the main index.html page to the top working directory
main_html = template.render( templateVars )
with open( '{0}/index.html'.format(envDict['WORKDIR']), 'w') as index:
index.write(main_html)
debugMsg('Ocean diagnostics - Copying stylesheet', header=True, verbosity=2)
shutil.copy2('{0}/Templates/diag_style.css'.format(envDict['POSTPROCESS_PATH']), '{0}/diag_style.css'.format(envDict['WORKDIR']))
debugMsg('Ocean diagnostics - Copying logo files', header=True, verbosity=2)
if not os.path.exists('{0}/logos'.format(envDict['WORKDIR'])):
os.mkdir('{0}/logos'.format(envDict['WORKDIR']))
for filename in glob.glob(os.path.join('{0}/Templates/logos'.format(envDict['POSTPROCESS_PATH']), '*.*')):
shutil.copy(filename, '{0}/logos'.format(envDict['WORKDIR']))
# setup the unique OCNDIAG_WEBDIR output file
env_file = '{0}/env_diags_ocn.xml'.format(envDict['PP_CASE_PATH'])
key = 'OCNDIAG_WEBDIR'
value = envDict['WORKDIR']
##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(envDict['PP_CASE_PATH'], key, main_comm.get_size(), main_comm.get_rank() )
web_file = '{0}/web_dirs/{1}.{2}'.format(envDict['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S'))
try:
diagUtilsLib.write_web_file(web_file, 'ocn', key, value)
except:
print('WARNING ocn_diags_generator unable to write {0}={1} to {2}'.format(key, value, web_file))
main_comm.sync()
# broadcast the diag_list to all tasks
num_of_diags = main_comm.partition(num_of_diags, func=partition.Duplicate(), involved=True)
diag_list = main_comm.partition(data=diag_list, func=partition.Duplicate(), involved=True)
main_comm.sync()
# initialize some variables for distributing diagnostics across the communicators
diags_send = diag_list
gmaster = main_comm.is_manager()
gsize = main_comm.get_size()
grank = main_comm.get_rank()
local_diag_list = list()
# divide the main communicator into sub_communicators to be passed to each diag class
# split mpi comm world if the size of the communicator > 1 and the num_of_diags > 1
if gsize > 1 and num_of_diags > 1:
temp_color = (grank % num_of_diags)
if (temp_color == num_of_diags):
temp_color = temp_color - 1
groups = list()
for g in range(0,num_of_diags):
groups.append(g)
debugMsg('global_rank {0}, temp_color {1}, #of groups(diag types) {2}, groups {3}, diag_list {4}'.format(grank, temp_color, num_of_diags, groups, diag_list), header=True, verbosity=2)
group = groups[temp_color]
inter_comm, multi_comm = main_comm.divide(group)
color = inter_comm.get_color()
lsize = inter_comm.get_size()
lrank = inter_comm.get_rank()
lmaster = inter_comm.is_manager()
debugMsg('color {0}, lsize {1}, lrank {2}, lmaster {3}'.format(color, lsize, lrank, lmaster), header=True, verbosity=2)
# partition the diag_list between communicators
DIAG_LIST_TAG = 10
if lmaster:
local_diag_list = multi_comm.partition(diag_list,func=partition.EqualStride(),involved=True)
debugMsg('lrank = {0} local_diag_list = {1}'.format(lrank, local_diag_list), header=True, verbosity=2)
for b in range(1, lsize):
diags_send = inter_comm.ration(data=local_diag_list, tag=DIAG_LIST_TAG)
debugMsg('b = {0} diags_send = {1} lsize = {2}'.format(b, diags_send, lsize), header=True, verbosity=2)
else:
local_diag_list = inter_comm.ration(tag=DIAG_LIST_TAG)
debugMsg('local_diag_list {0}',format(local_diag_list), header=True, verbosity=2)
else:
inter_comm = main_comm
lmaster = main_comm.is_manager()
lsize = main_comm.get_size()
lrank = main_comm.get_rank()
local_diag_list = diag_list
inter_comm.sync()
main_comm.sync()
# loop through the local_diag_list
for requested_diag in local_diag_list:
try:
debugMsg('requested_diag {0}, lrank {1}, lsize {2}, lmaster {3}'.format(requested_diag, lrank, lsize, lmaster), header=True, verbosity=2)
diag = ocn_diags_factory.oceanDiagnosticsFactory(requested_diag)
# check the prerequisites for the diagnostics types
debugMsg('Checking prerequisites for {0}'.format(diag.__class__.__name__), header=True, verbosity=2)
skip_key = '{0}_SKIP'.format(requested_diag)
if lmaster:
try:
envDict = diag.check_prerequisites(envDict)
except ocn_diags_bc.PrerequisitesError:
print("Problem with check_prerequisites for '{0}' skipping!".format(requested_diag))
envDict[skip_key] = True
except RuntimeError as e:
# unrecoverable error, bail!
print(e)
envDict['unrecoverableErrorOnMaster'] = True
inter_comm.sync()
# broadcast the envDict
envDict = inter_comm.partition(data=envDict, func=partition.Duplicate(), involved=True)
if envDict.has_key('unrecoverableErrorOnMaster'):
raise RuntimeError
# run the diagnostics type on each inter_comm
if not envDict.has_key(skip_key):
# set the shell env using the values set in the XML and read into the envDict across all tasks
cesmEnvLib.setXmlEnv(envDict)
# run the diagnostics
envDict = diag.run_diagnostics(envDict, inter_comm)
inter_comm.sync()
except ocn_diags_bc.RecoverableError as e:
# catch all recoverable errors, print a message and continue.
print(e)
print("Skipped '{0}' and continuing!".format(requested_diag))
except RuntimeError as e:
# unrecoverable error, bail!
print(e)
return 1
main_comm.sync()
def run_diagnostics(self, env, scomm):
""" call the necessary plotting routines to generate diagnostics plots
"""
super(modelVsModel, self).run_diagnostics(env, scomm)
scomm.sync()
# setup some global variables
requested_plot_sets = list()
local_requested_plots = list()
local_html_list = list()
# all the plot module XML vars start with 'set_'
for key, value in env.iteritems():
if ("set_" in key and value == 'True'):
requested_plot_sets.append(key)
scomm.sync()
if scomm.is_manager():
print('DEBUG model_vs_model requested_plot_sets = {0}'.format(
requested_plot_sets))
# partition requested plots to all tasks
# first, create plotting classes and get the number of plots each will created
requested_plots = {}
set_sizes = {}
#plots_weights = []
for plot_set in requested_plot_sets:
requested_plots.update(
lnd_diags_plot_factory.LandDiagnosticPlotFactory(
plot_set, env))
#for plot_id,plot_class in requested_plots.iteritems():
# if hasattr(plot_class, 'weight'):
# factor = plot_class.weight
# else:
# factor = 1
# plots_weights.append((plot_id,len(plot_class.expectedPlots)*factor))
# partition based on the number of plots each set will create
#local_plot_list = scomm.partition(plots_weights, func=partition.WeightBalanced(), involved=True)
if scomm.is_manager():
print('DEBUG model_vs_model requested_plots.keys = {0}'.format(
requested_plots.keys()))
local_plot_list = scomm.partition(requested_plots.keys(),
func=partition.EqualStride(),
involved=True)
scomm.sync()
timer = timekeeper.TimeKeeper()
# loop over local plot lists - set env and then run plotting script
timer.start(str(scomm.get_rank()) + "ncl total time on task")
for plot_set in local_plot_list:
timer.start(str(scomm.get_rank()) + plot_set)
plot_class = requested_plots[plot_set]
# set all env variables (global and particular to this plot call
print(
'model vs. model - Checking prerequisite for {0} on rank {1}'.
format(plot_class.__class__.__name__, scomm.get_rank()))
plot_class.check_prerequisites(env)
# Stringify the env dictionary
for name, value in plot_class.plot_env.iteritems():
plot_class.plot_env[name] = str(value)
# call script to create plots
for script in plot_class.ncl_scripts:
print(
'model vs. model - Generating plots for {0} on rank {1} with script {2}'
.format(plot_class.__class__.__name__, scomm.get_rank(),
script))
diagUtilsLib.generate_ncl_plots(plot_class.plot_env, script)
timer.stop(str(scomm.get_rank()) + plot_set)
timer.stop(str(scomm.get_rank()) + "ncl total time on task")
scomm.sync()
print(timer.get_all_times())
#w = 0
#for p in plots_weights:
# if p[0] in local_plot_list:
# w = w + p[1]
#print(str(scomm.get_rank())+' weight:'+str(w))
# set html files
if scomm.is_manager():
# Create web dirs and move images/tables to that web dir
for n in ('1', '2', '3', '4', '5', '6', '7', '8', '9'):
web_dir = env['WKDIR']
set_dir = web_dir + '/set' + n
# Create the plot set web directory
if not os.path.exists(set_dir):
os.makedirs(set_dir)
# Copy plots into the correct web dir
glob_string = web_dir + '/set' + n + '_*'
imgs = glob.glob(glob_string)
if len(imgs) > 0:
for img in imgs:
new_fn = set_dir + '/' + os.path.basename(img)
os.rename(img, new_fn)
# Copy the set1Diff and set1Anom plots to set_1 web dir
if n == '1':
glob_string = web_dir + '/set1Diff' + '_*'
imgs = glob.glob(glob_string)
if len(imgs) > 0:
for img in imgs:
new_fn = set_dir + '/' + os.path.basename(img)
os.rename(img, new_fn)
glob_string = web_dir + '/set1Anom' + '_*'
imgs = glob.glob(glob_string)
if len(imgs) > 0:
for img in imgs:
new_fn = set_dir + '/' + os.path.basename(img)
os.rename(img, new_fn)
env['WEB_DIR'] = web_dir
shutil.copy2(
env['POSTPROCESS_PATH'] + '/lnd_diag/inputFiles/' +
env['VAR_MASTER'], web_dir + '/variable_master.ncl')
if n == '9':
web_script = env[
'POSTPROCESS_PATH'] + '/lnd_diag/shared/lnd_statTable.pl'
rc2, err_msg = cesmEnvLib.checkFile(web_script, 'read')
if rc2:
try:
subprocess.check_call(web_script)
except subprocess.CalledProcessError as e:
print('WARNING: {0} error executing command:'.format(
web_script))
print(' {0}'.format(e.cmd))
print(' rc = {0}'.format(e.returncode))
else:
print('{0}... {1} file not found'.format(
err_msg, web_script))
# copy the set9_statTable.html to the set9 subdir
set9_statTable = web_dir + '/set9_statTable.html'
rc2, err_msg = cesmEnvLib.checkFile(set9_statTable, 'read')
if rc2:
new_fn = web_dir + '/set9/' + os.path.basename(
set9_statTable)
try:
os.rename(set9_statTable, new_fn)
except os.OSError as e:
print('WARNING: rename error for file {0}'.format(
set9_statTable))
print(' rc = {0}'.format(e.returncode))
else:
print('{0}... {1} file not found'.format(
err_msg, set9_statTable))
web_script_1 = env[
'POSTPROCESS_PATH'] + '/lnd_diag/shared/lnd_create_webpage.pl'
web_script_2 = env[
'POSTPROCESS_PATH'] + '/lnd_diag/shared/lnd_lookupTable.pl'
print('Creating Web Pages')
# set the shell environment
cesmEnvLib.setXmlEnv(env)
# lnd_create_webpage.pl call
rc1, err_msg = cesmEnvLib.checkFile(web_script_1, 'read')
if rc1:
try:
subprocess.check_call(web_script_1)
except subprocess.CalledProcessError as e:
print('WARNING: {0} error executing command:'.format(
web_script_1))
print(' {0}'.format(e.cmd))
print(' rc = {0}'.format(e.returncode))
else:
print('{0}... {1} file not found'.format(
err_msg, web_script_1))
# lnd_lookupTable.pl call
rc2, err_msg = cesmEnvLib.checkFile(web_script_2, 'read')
if rc2:
try:
subprocess.check_call(web_script_2)
except subprocess.CalledProcessError as e:
print('WARNING: {0} error executing command:'.format(
web_script_2))
print(' {0}'.format(e.cmd))
print(' rc = {0}'.format(e.returncode))
else:
print('{0}... {1} file not found'.format(
err_msg, web_script_2))
# move all the plots to the diag_path with the years appended to the path
endYr1 = (int(env['clim_first_yr_1']) +
int(env['clim_num_yrs_1'])) - 1
endYr2 = (int(env['clim_first_yr_2']) +
int(env['clim_num_yrs_2'])) - 1
diag_path = '{0}/diag/{1}.{2}_{3}-{4}.{5}_{6}'.format(
env['OUTPUT_ROOT_PATH'], env['caseid_1'],
env['clim_first_yr_1'], str(endYr1), env['caseid_2'],
env['clim_first_yr_2'], str(endYr2))
move_files = True
try:
os.makedirs(diag_path)
except OSError as exception:
if exception.errno != errno.EEXIST:
err_msg = 'ERROR: {0} problem accessing directory {1}'.format(
self.__class__.__name__, diag_path)
raise OSError(err_msg)
move_files = False
elif env['CLEANUP_FILES'].lower() in ['t', 'true']:
# delete all the files in the diag_path directory
for root, dirs, files in os.walk('diag_path'):
for f in files:
os.unlink(os.path.join(root, f))
for d in dirs:
shutil.rmtree(os.path.join(root, d))
elif env['CLEANUP_FILES'].lower() in ['f', 'false']:
print(
'WARNING: {0} exists and is not empty and LNDDIAG_CLEANUP_FILES = False. Leaving new diagnostics files in {1}'
.format(diag_path, web_dir))
diag_path = web_dir
move_files = False
# move the files to the new diag_path
if move_files:
try:
print('DEBUG: model_vs_model renaming web files')
os.rename(web_dir, diag_path)
except OSError as e:
print('WARNING: Error renaming %s to %s: %s' %
(web_dir, diag_path, e))
diag_path = web_dir
print('DEBUG: model vs. model web_dir = {0}'.format(web_dir))
print('DEBUG: model vs. model diag_path = {0}'.format(diag_path))
# setup the unique LNDDIAG_WEBDIR_MODEL_VS_MODEL output file
env_file = '{0}/env_diags_lnd.xml'.format(env['PP_CASE_PATH'])
key = 'LNDDIAG_WEBDIR_{0}'.format(self._name)
value = diag_path
##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(env['PP_CASE_PATH'], key, scomm.get_size(), scomm.get_rank() )
web_file = '{0}/web_dirs/{1}.{2}'.format(
env['PP_CASE_PATH'], key,
datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S'))
try:
diagUtilsLib.write_web_file(web_file, 'lnd', key, value)
except:
print(
'WARNING lnd model_vs_model unable to write {0}={1} to {2}'
.format(key, value, web_file))
print(
'*******************************************************************************'
)
print(
'Successfully completed generating land diagnostics model vs. model plots'
)
print(
'*******************************************************************************'
)
def main(options, main_comm, debugMsg, timer):
"""setup the environment for running the diagnostics in parallel.
Calls 2 different diagnostics generation types:
model vs. observation
model vs. model
Arguments:
options (object) - command line options
main_comm (object) - MPI simple communicator object
debugMsg (object) - vprinter object for printing debugging messages
timer (object) - timer object for keeping times
The env_diags_lnd.xml configuration file defines the way the diagnostics are generated.
See (website URL here...) for a complete desciption of the env_diags_lnd XML options.
"""
# initialize the environment dictionary
envDict = dict()
# CASEROOT is given on the command line as required option --caseroot
if main_comm.is_manager():
caseroot = options.caseroot[0]
debugMsg('caseroot = {0}'.format(caseroot), header=True, verbosity=1)
debugMsg('calling initialize_main', header=True, verbosity=1)
envDict = initialize_main(envDict, caseroot, debugMsg, options.standalone)
debugMsg('calling check_ncl_nco', header=True, verbosity=1)
diagUtilsLib.check_ncl_nco(envDict)
# broadcast envDict to all tasks
envDict = main_comm.partition(data=envDict, func=partition.Duplicate(), involved=True)
main_comm.sync()
# get list of diagnostics types to be created
diag_list = list()
diag_list = setup_diags(envDict)
if main_comm.is_manager():
print('User requested diagnostics:')
for diag in diag_list:
print(' {0}'.format(diag))
main_comm.sync()
# broadcast the diag_list to all tasks
num_of_diags = len(diag_list)
num_of_diags = main_comm.partition(num_of_diags, func=partition.Duplicate(), involved=True)
diag_list = main_comm.partition(data=diag_list, func=partition.Duplicate(), involved=True)
main_comm.sync()
# initialize some variables for distributing diagnostics across the communicators
diags_send = diag_list
gmaster = main_comm.is_manager()
gsize = main_comm.get_size()
grank = main_comm.get_rank()
local_diag_list = list()
# divide the main communicator into sub_communicators to be passed to each diag class
# split mpi comm world if the size of the communicator > 1 and the num_of_diags > 1
if gsize > 1 and num_of_diags > 1:
temp_color = (grank % num_of_diags)
if (temp_color == num_of_diags):
temp_color = temp_color - 1
groups = list()
for g in range(0,num_of_diags):
groups.append(g)
debugMsg('global_rank {0}, temp_color {1}, #of groups(diag types) {2}, groups {3}, diag_list {4}'.format(grank, temp_color, num_of_diags, groups, diag_list), header=True, verbosity=1)
group = groups[temp_color]
inter_comm, multi_comm = main_comm.divide(group)
color = inter_comm.get_color()
lsize = inter_comm.get_size()
lrank = inter_comm.get_rank()
lmaster = inter_comm.is_manager()
if lmaster:
debugMsg('color {0}, lsize {1}, lrank {2}, lmaster {3}'.format(color, lsize, lrank, lmaster), header=True, verbosity=1)
# partition the diag_list between communicators
DIAG_LIST_TAG = 10
if lmaster:
local_diag_list = multi_comm.partition(diag_list,func=partition.EqualStride(),involved=True)
for b in range(1, lsize):
diags_send = inter_comm.ration(data=local_diag_list, tag=DIAG_LIST_TAG)
else:
local_diag_list = inter_comm.ration(tag=DIAG_LIST_TAG)
if lmaster:
debugMsg('local_diag_list {0}',format(local_diag_list), header=True, verbosity=1)
else:
inter_comm = main_comm
lmaster = main_comm.is_manager()
lsize = main_comm.get_size()
lrank = main_comm.get_rank()
local_diag_list = diag_list
inter_comm.sync()
main_comm.sync()
if lmaster:
debugMsg('lsize = {0}, lrank = {1}'.format(lsize, lrank), header=True, verbosity=1)
inter_comm.sync()
# loop through the local_diag_list list
for requested_diag in local_diag_list:
try:
diag = lnd_diags_factory.landDiagnosticsFactory(requested_diag,envDict)
# check the prerequisites for the diagnostics types
if lmaster:
debugMsg('Checking prerequisites for {0}'.format(diag.__class__.__name__), header=True, verbosity=1)
envDict = diag.check_prerequisites(envDict, inter_comm)
inter_comm.sync()
# broadcast the envDict
envDict = inter_comm.partition(data=envDict, func=partition.Duplicate(), involved=True)
# set the shell env using the values set in the XML and read into the envDict across all tasks
cesmEnvLib.setXmlEnv(envDict)
# debug check if the envDict contains a non-string entry
if lmaster:
for k,v in envDict.iteritems():
if not isinstance(v, basestring):
debugMsg('lnd_diags_generator - envDict: key = {0}, value = {1}'.format(k,v), header=True, verbosity=1)
debugMsg('inter_comm size = {0}'.format(inter_comm.get_size()), header=True, verbosity=1)
diag.run_diagnostics(envDict, inter_comm)
if lmaster:
debugMsg('lnd_diags_generator: return from run_diagnostics', header=True, verbosity=1)
except lnd_diags_bc.RecoverableError as e:
# catch all recoverable errors, print a message and continue.
print(e)
print("Skipped '{0}' and continuing!".format(request_diag))
except RuntimeError as e:
# unrecoverable error, bail!
print(e)
return 1
def run_diagnostics(self, env, scomm):
""" call the necessary plotting routines to generate diagnostics plots
"""
super(modelVsObs, self).run_diagnostics(env, scomm)
scomm.sync()
# setup some global variables
requested_plot_sets = list()
local_requested_plots = list()
local_html_list = list()
# all the plot module XML vars start with 'set_'
for key, value in env.iteritems():
if ("set_" in key and value == 'True'):
requested_plot_sets.append(key)
scomm.sync()
if scomm.is_manager():
print('DEBUG model_vs_obs requested_plot_sets = {0}'.format(requested_plot_sets))
# partition requested plots to all tasks
# first, create plotting classes and get the number of plots each will created
requested_plots = {}
set_sizes = {}
#plots_weights = []
for plot_set in requested_plot_sets:
requested_plots.update(lnd_diags_plot_factory.LandDiagnosticPlotFactory(plot_set,env))
#for plot_id,plot_class in requested_plots.iteritems():
# if hasattr(plot_class, 'weight'):
# factor = plot_class.weight
# else:
# factor = 1
# plots_weights.append((plot_id,len(plot_class.expectedPlots)*factor))
# partition based on the number of plots each set will create
#local_plot_list = scomm.partition(plots_weights, func=partition.WeightBalanced(), involved=True)
local_plot_list = scomm.partition(requested_plots.keys(), func=partition.EqualStride(), involved=True)
scomm.sync()
timer = timekeeper.TimeKeeper()
# loop over local plot lists - set env and then run plotting script
timer.start(str(scomm.get_rank())+"ncl total time on task")
for plot_set in local_plot_list:
timer.start(str(scomm.get_rank())+plot_set)
plot_class = requested_plots[plot_set]
print('DEBUG model vs. obs - Checking prerequisite for {0} on rank {1}'.format(plot_class.__class__.__name__, scomm.get_rank()))
plot_class.check_prerequisites(env)
# Stringify the env dictionary
for name, value in plot_class.plot_env.iteritems():
plot_class.plot_env[name] = str(value)
# call script to create plots
for script in plot_class.ncl_scripts:
print('DEBUG model vs. obs - Generating plots for {0} on rank {1} with script {2}'.format(plot_class.__class__.__name__, scomm.get_rank(),script))
diagUtilsLib.generate_ncl_plots(plot_class.plot_env, script)
timer.stop(str(scomm.get_rank())+plot_set)
timer.stop(str(scomm.get_rank())+"ncl total time on task")
scomm.sync()
print(timer.get_all_times())
#w = 0
#for p in plots_weights:
# if p[0] in local_plot_list:
# w = w + p[1]
#print(str(scomm.get_rank())+' weight:'+str(w))
# set html files
if scomm.is_manager():
# Create web dirs and move images/tables to that web dir
for n in ('1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12'):
web_dir = env['WKDIR']
set_dir = web_dir + '/set' + n
# Create the plot set web directory
if not os.path.exists(set_dir):
os.makedirs(set_dir)
# Copy plots into the correct web dir
glob_string = web_dir+'/set'+n+'_*'
imgs = glob.glob(glob_string)
if len(imgs) > 0:
for img in imgs:
new_fn = set_dir + '/' + os.path.basename(img)
os.rename(img,new_fn)
env['WEB_DIR'] = web_dir
shutil.copy2(env['POSTPROCESS_PATH']+'/lnd_diag/inputFiles/'+env['VAR_MASTER'],web_dir+'/variable_master.ncl')
web_script_1 = env['POSTPROCESS_PATH']+'/lnd_diag/shared/lnd_create_webpage.pl'
web_script_2 = env['POSTPROCESS_PATH']+'/lnd_diag/shared/lnd_lookupTable.pl'
print('Creating Web Pages')
# set the shell environment
cesmEnvLib.setXmlEnv(env)
# lnd_create_webpage.pl call
rc1, err_msg = cesmEnvLib.checkFile(web_script_1,'read')
if rc1:
try:
subprocess.check_call(web_script_1)
except subprocess.CalledProcessError as e:
print('WARNING: {0} error executing command:'.format(web_script_1))
print(' {0}'.format(e.cmd))
print(' rc = {0}'.format(e.returncode))
else:
print('{0}... {1} file not found'.format(err_msg,web_script_1))
# lnd_lookupTable.pl call
rc2, err_msg = cesmEnvLib.checkFile(web_script_2,'read')
if rc2:
try:
subprocess.check_call(web_script_2)
except subprocess.CalledProcessError as e:
print('WARNING: {0} error executing command:'.format(web_script_2))
print(' {0}'.format(e.cmd))
print(' rc = {0}'.format(e.returncode))
else:
print('{0}... {1} file not found'.format(err_msg,web_script_2))
# move all the plots to the diag_path with the years appended to the path
endYr = (int(env['clim_first_yr_1']) + int(env['clim_num_yrs_1'])) - 1
diag_path = '{0}/diag/{1}-obs.{2}_{3}'.format(env['OUTPUT_ROOT_PATH'], env['caseid_1'],
env['clim_first_yr_1'], str(endYr))
move_files = True
try:
os.makedirs(diag_path)
except OSError as exception:
if exception.errno != errno.EEXIST:
err_msg = 'ERROR: {0} problem accessing directory {1}'.format(self.__class__.__name__, diag_path)
raise OSError(err_msg)
move_files = False
elif env['CLEANUP_FILES'].lower() in ['t','true']:
# delete all the files in the diag_path directory
for root, dirs, files in os.walk(diag_path):
for f in files:
os.unlink(os.path.join(root, f))
for d in dirs:
shutil.rmtree(os.path.join(root, d))
elif env['CLEANUP_FILES'].lower() in ['f','false']:
print('WARNING: {0} exists and is not empty and LNDDIAG_CLEANUP_FILES = False. Leaving new diagnostics files in {1}'.format(diag_path, web_dir))
diag_path = web_dir
move_files = False
print('DEBUG: model vs. obs web_dir = {0}'.format(web_dir))
print('DEBUG: model vs. obs diag_path = {0}'.format(diag_path))
# move the files to the new diag_path
if move_files:
try:
print('DEBUG: model_vs_obs renaming web files')
os.rename(web_dir, diag_path)
except OSError as e:
print ('WARNING: Error renaming %s to %s: %s' % (web_dir, diag_path, e))
diag_path = web_dir
# setup the LNDDIAG_WEBDIR_MODEL_VS_OBS output file
env_file = '{0}/env_diags_lnd.xml'.format(env['PP_CASE_PATH'])
key = 'LNDDIAG_WEBDIR_{0}'.format(self._name)
value = diag_path
web_file = '{0}/web_dirs/{1}.{2}'.format(env['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S'))
try:
diagUtilsLib.write_web_file(web_file, 'lnd', key, value)
except:
print('WARNING lnd model_vs_obs unable to write {0}={1} to {2}'.format(key, value, web_file))
print('*******************************************************************************')
print('Successfully completed generating land diagnostics model vs. observation plots')
print('*******************************************************************************')
def main(options, main_comm, debugMsg):
"""setup the environment for running the diagnostics in parallel.
Calls 6 different diagnostics generation types:
model vs. observation (optional BGC - ecosystem)
model vs. control (optional BGC - ecosystem)
model time-series (optional BGC - ecosystem)
Arguments:
options (object) - command line options
main_comm (object) - MPI simple communicator object
debugMsg (object) - vprinter object for printing debugging messages
The env_diags_ocn.xml configuration file defines the way the diagnostics are generated.
See (website URL here...) for a complete desciption of the env_diags_ocn XML options.
"""
# initialize the environment dictionary
envDict = dict()
# CASEROOT is given on the command line as required option --caseroot
if main_comm.is_manager():
caseroot = options.caseroot[0]
debugMsg('caseroot = {0}'.format(caseroot), header=True, verbosity=2)
debugMsg('calling initialize_main', header=True, verbosity=2)
envDict = initialize_main(envDict, caseroot, debugMsg, options.standalone)
debugMsg('calling check_ncl_nco', header=True, verbosity=2)
diagUtilsLib.check_ncl_nco(envDict)
# broadcast envDict to all tasks
envDict = main_comm.partition(data=envDict, func=partition.Duplicate(), involved=True)
sys.path.append(envDict['PATH'])
main_comm.sync()
# get list of diagnostics types to be created
diag_list = list()
num_of_diags = 0
if main_comm.is_manager():
diag_list, diag_dict = setup_diags(envDict)
num_of_diags = len(diag_list)
if num_of_diags == 0:
print('No ocean diagnostics specified. Please check the {0}/env_diags_ocn.xml settings.'.format(envDict['PP_CASE_PATH']))
sys.exit(1)
print('User requested diagnostics:')
for diag in diag_list:
print(' {0}'.format(diag))
try:
os.makedirs(envDict['WORKDIR'])
except OSError as exception:
if exception.errno != errno.EEXIST:
err_msg = 'ERROR: ocn_diags_generator.py problem accessing the working directory {0}'.format(envDict['WORKDIR'])
raise OSError(err_msg)
debugMsg('Ocean diagnostics - Creating main index.html page', header=True, verbosity=2)
# define the templatePath
templatePath = '{0}/diagnostics/diagnostics/ocn/Templates'.format(envDict['POSTPROCESS_PATH'])
templateLoader = jinja2.FileSystemLoader( searchpath=templatePath )
templateEnv = jinja2.Environment( loader=templateLoader )
template_file = 'ocean_diagnostics.tmpl'
template = templateEnv.get_template( template_file )
# get the current datatime string for the template
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
# set the template variables
templateVars = { 'casename' : envDict['CASE'],
'tagname' : envDict['CESM_TAG'],
'username' : envDict['USER_NAME'],
'diag_dict' : diag_dict,
'control_casename' : envDict['CNTRLCASE'],
'start_year' : envDict['YEAR0'],
'stop_year' : envDict['YEAR1'],
'control_start_year' : envDict['CNTRLYEAR0'],
'control_stop_year' : envDict['CNTRLYEAR1'],
'today': now,
'tseries_start_year' : envDict['TSERIES_YEAR0'],
'tseries_stop_year' : envDict['TSERIES_YEAR1']
}
# write the main index.html page to the top working directory
main_html = template.render( templateVars )
with open( '{0}/index.html'.format(envDict['WORKDIR']), 'w') as index:
index.write(main_html)
debugMsg('Ocean diagnostics - Copying stylesheet', header=True, verbosity=2)
shutil.copy2('{0}/Templates/diag_style.css'.format(envDict['POSTPROCESS_PATH']), '{0}/diag_style.css'.format(envDict['WORKDIR']))
debugMsg('Ocean diagnostics - Copying logo files', header=True, verbosity=2)
if not os.path.exists('{0}/logos'.format(envDict['WORKDIR'])):
os.mkdir('{0}/logos'.format(envDict['WORKDIR']))
for filename in glob.glob(os.path.join('{0}/Templates/logos'.format(envDict['POSTPROCESS_PATH']), '*.*')):
shutil.copy(filename, '{0}/logos'.format(envDict['WORKDIR']))
# setup the unique OCNDIAG_WEBDIR output file
env_file = '{0}/env_diags_ocn.xml'.format(envDict['PP_CASE_PATH'])
key = 'OCNDIAG_WEBDIR'
value = envDict['WORKDIR']
##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(envDict['PP_CASE_PATH'], key, main_comm.get_size(), main_comm.get_rank() )
web_file = '{0}/web_dirs/{1}.{2}'.format(envDict['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S'))
try:
diagUtilsLib.write_web_file(web_file, 'ocn', key, value)
except:
print('WARNING ocn_diags_generator unable to write {0}={1} to {2}'.format(key, value, web_file))
main_comm.sync()
# broadcast the diag_list to all tasks
num_of_diags = main_comm.partition(num_of_diags, func=partition.Duplicate(), involved=True)
diag_list = main_comm.partition(data=diag_list, func=partition.Duplicate(), involved=True)
main_comm.sync()
# initialize some variables for distributing diagnostics across the communicators
diags_send = diag_list
gmaster = main_comm.is_manager()
gsize = main_comm.get_size()
grank = main_comm.get_rank()
local_diag_list = list()
# divide the main communicator into sub_communicators to be passed to each diag class
# split mpi comm world if the size of the communicator > 1 and the num_of_diags > 1
if gsize > 1 and num_of_diags > 1:
temp_color = (grank % num_of_diags)
if (temp_color == num_of_diags):
temp_color = temp_color - 1
groups = list()
for g in range(0,num_of_diags):
groups.append(g)
debugMsg('global_rank {0}, temp_color {1}, #of groups(diag types) {2}, groups {3}, diag_list {4}'.format(grank, temp_color, num_of_diags, groups, diag_list), header=True, verbosity=2)
group = groups[temp_color]
inter_comm, multi_comm = main_comm.divide(group)
color = inter_comm.get_color()
lsize = inter_comm.get_size()
lrank = inter_comm.get_rank()
lmaster = inter_comm.is_manager()
debugMsg('color {0}, lsize {1}, lrank {2}, lmaster {3}'.format(color, lsize, lrank, lmaster), header=True, verbosity=2)
# partition the diag_list between communicators
DIAG_LIST_TAG = 10
if lmaster:
local_diag_list = multi_comm.partition(diag_list,func=partition.EqualStride(),involved=True)
debugMsg('lrank = {0} local_diag_list = {1}'.format(lrank, local_diag_list), header=True, verbosity=2)
for b in range(1, lsize):
diags_send = inter_comm.ration(data=local_diag_list, tag=DIAG_LIST_TAG)
debugMsg('b = {0} diags_send = {1} lsize = {2}'.format(b, diags_send, lsize), header=True, verbosity=2)
else:
local_diag_list = inter_comm.ration(tag=DIAG_LIST_TAG)
debugMsg('local_diag_list {0}',format(local_diag_list), header=True, verbosity=2)
else:
inter_comm = main_comm
lmaster = main_comm.is_manager()
lsize = main_comm.get_size()
lrank = main_comm.get_rank()
local_diag_list = diag_list
inter_comm.sync()
main_comm.sync()
# loop through the local_diag_list
for requested_diag in local_diag_list:
try:
debugMsg('requested_diag {0}, lrank {1}, lsize {2}, lmaster {3}'.format(requested_diag, lrank, lsize, lmaster), header=True, verbosity=2)
diag = ocn_diags_factory.oceanDiagnosticsFactory(requested_diag)
# check the prerequisites for the diagnostics types
debugMsg('Checking prerequisites for {0}'.format(diag.__class__.__name__), header=True, verbosity=2)
skip_key = '{0}_SKIP'.format(requested_diag)
if lmaster:
try:
envDict = diag.check_prerequisites(envDict)
except ocn_diags_bc.PrerequisitesError:
print("Problem with check_prerequisites for '{0}' skipping!".format(requested_diag))
envDict[skip_key] = True
except RuntimeError as e:
# unrecoverable error, bail!
print(e)
envDict['unrecoverableErrorOnMaster'] = True
inter_comm.sync()
# broadcast the envDict
envDict = inter_comm.partition(data=envDict, func=partition.Duplicate(), involved=True)
if envDict.has_key('unrecoverableErrorOnMaster'):
raise RuntimeError
# run the diagnostics type on each inter_comm
if not envDict.has_key(skip_key):
# set the shell env using the values set in the XML and read into the envDict across all tasks
cesmEnvLib.setXmlEnv(envDict)
# run the diagnostics
envDict = diag.run_diagnostics(envDict, inter_comm)
inter_comm.sync()
except ocn_diags_bc.RecoverableError as e:
# catch all recoverable errors, print a message and continue.
print(e)
print("Skipped '{0}' and continuing!".format(requested_diag))
except RuntimeError as e:
# unrecoverable error, bail!
print(e)
return 1
main_comm.sync()
def main(options, main_comm, debugMsg, timer):
"""setup the environment for running the diagnostics in parallel.
Calls 2 different diagnostics generation types:
model vs. observation
model vs. model
Arguments:
options (object) - command line options
main_comm (object) - MPI simple communicator object
debugMsg (object) - vprinter object for printing debugging messages
timer (object) - timer object for keeping times
The env_diags_lnd.xml configuration file defines the way the diagnostics are generated.
See (website URL here...) for a complete desciption of the env_diags_lnd XML options.
"""
# initialize the environment dictionary
envDict = dict()
# CASEROOT is given on the command line as required option --caseroot
if main_comm.is_manager():
caseroot = options.caseroot[0]
debugMsg('caseroot = {0}'.format(caseroot), header=True, verbosity=1)
debugMsg('calling initialize_main', header=True, verbosity=1)
envDict = initialize_main(envDict, caseroot, debugMsg,
options.standalone)
debugMsg('calling check_ncl_nco', header=True, verbosity=1)
diagUtilsLib.check_ncl_nco(envDict)
# broadcast envDict to all tasks
envDict = main_comm.partition(data=envDict,
func=partition.Duplicate(),
involved=True)
main_comm.sync()
# get list of diagnostics types to be created
diag_list = list()
diag_list = setup_diags(envDict)
if main_comm.is_manager():
print('User requested diagnostics:')
for diag in diag_list:
print(' {0}'.format(diag))
main_comm.sync()
# broadcast the diag_list to all tasks
num_of_diags = len(diag_list)
num_of_diags = main_comm.partition(num_of_diags,
func=partition.Duplicate(),
involved=True)
diag_list = main_comm.partition(data=diag_list,
func=partition.Duplicate(),
involved=True)
main_comm.sync()
# initialize some variables for distributing diagnostics across the communicators
diags_send = diag_list
gmaster = main_comm.is_manager()
gsize = main_comm.get_size()
grank = main_comm.get_rank()
local_diag_list = list()
# divide the main communicator into sub_communicators to be passed to each diag class
# split mpi comm world if the size of the communicator > 1 and the num_of_diags > 1
if gsize > 1 and num_of_diags > 1:
temp_color = (grank % num_of_diags)
if (temp_color == num_of_diags):
temp_color = temp_color - 1
groups = list()
for g in range(0, num_of_diags):
groups.append(g)
debugMsg(
'global_rank {0}, temp_color {1}, #of groups(diag types) {2}, groups {3}, diag_list {4}'
.format(grank, temp_color, num_of_diags, groups, diag_list),
header=True,
verbosity=1)
group = groups[temp_color]
inter_comm, multi_comm = main_comm.divide(group)
color = inter_comm.get_color()
lsize = inter_comm.get_size()
lrank = inter_comm.get_rank()
lmaster = inter_comm.is_manager()
if lmaster:
debugMsg('color {0}, lsize {1}, lrank {2}, lmaster {3}'.format(
color, lsize, lrank, lmaster),
header=True,
verbosity=1)
# partition the diag_list between communicators
DIAG_LIST_TAG = 10
if lmaster:
local_diag_list = multi_comm.partition(
diag_list, func=partition.EqualStride(), involved=True)
for b in range(1, lsize):
diags_send = inter_comm.ration(data=local_diag_list,
tag=DIAG_LIST_TAG)
else:
local_diag_list = inter_comm.ration(tag=DIAG_LIST_TAG)
if lmaster:
debugMsg('local_diag_list {0}',
format(local_diag_list),
header=True,
verbosity=1)
else:
inter_comm = main_comm
lmaster = main_comm.is_manager()
lsize = main_comm.get_size()
lrank = main_comm.get_rank()
local_diag_list = diag_list
inter_comm.sync()
main_comm.sync()
if lmaster:
debugMsg('lsize = {0}, lrank = {1}'.format(lsize, lrank),
header=True,
verbosity=1)
inter_comm.sync()
# loop through the local_diag_list list
for requested_diag in local_diag_list:
try:
diag = lnd_diags_factory.landDiagnosticsFactory(
requested_diag, envDict)
# check the prerequisites for the diagnostics types
if lmaster:
debugMsg('Checking prerequisites for {0}'.format(
diag.__class__.__name__),
header=True,
verbosity=1)
envDict = diag.check_prerequisites(envDict, inter_comm)
inter_comm.sync()
# broadcast the envDict
envDict = inter_comm.partition(data=envDict,
func=partition.Duplicate(),
involved=True)
# set the shell env using the values set in the XML and read into the envDict across all tasks
cesmEnvLib.setXmlEnv(envDict)
# debug check if the envDict contains a non-string entry
if lmaster:
for k, v in envDict.iteritems():
if not isinstance(v, basestring):
debugMsg(
'lnd_diags_generator - envDict: key = {0}, value = {1}'
.format(k, v),
header=True,
verbosity=1)
debugMsg('inter_comm size = {0}'.format(inter_comm.get_size()),
header=True,
verbosity=1)
diag.run_diagnostics(envDict, inter_comm)
if lmaster:
debugMsg('lnd_diags_generator: return from run_diagnostics',
header=True,
verbosity=1)
except lnd_diags_bc.RecoverableError as e:
# catch all recoverable errors, print a message and continue.
print(e)
print("Skipped '{0}' and continuing!".format(request_diag))
except RuntimeError as e:
# unrecoverable error, bail!
print(e)
return 1