def check_prerequisites(self, env): """This method does some generic checks for the plots prerequisites that are common to all plots """ print(' Checking generic prerequisites for land diagnostics plot.') cesmEnvLib.setXmlEnv(env)
def check_prerequisites(self, env): """This method does some generic checks for the plots prerequisites that are common to all plots """ print(' Checking generic prerequisites for atmosphere diagnostics plot.') cesmEnvLib.setXmlEnv(env)
def check_prerequisites(self, env): """This method does some generic checks for the plots prerequisites that are common to all plots """ print(' Checking generic prerequisites for ocean diagnostics plot.') # set SEASAVGFILE env var to the envDict['MAVGFILE'] file env['SEASAVGFILE'] = env['MAVGFILE'] # set SEASAVGTEMP env var to the envDict['MAVGFILE'] file env['SEASAVGTEMP'] = env['MAVGFILE'] # set SEASAVGSALT env var to the envDict['MAVGFILE'] file env['SEASAVGSALT'] = env['MAVGFILE'] cesmEnvLib.setXmlEnv(env)
def main(options, main_comm, debugMsg): """setup the environment for running the diagnostics in parallel. Calls 6 different diagnostics generation types: model vs. observation (optional BGC - ecosystem) model vs. control (optional BGC - ecosystem) model time-series (optional BGC - ecosystem) Arguments: options (object) - command line options main_comm (object) - MPI simple communicator object debugMsg (object) - vprinter object for printing debugging messages The env_diags_ocn.xml configuration file defines the way the diagnostics are generated. See (website URL here...) for a complete desciption of the env_diags_ocn XML options. """ # initialize the environment dictionary envDict = dict() # CASEROOT is given on the command line as required option --caseroot if main_comm.is_manager(): caseroot = options.caseroot[0] debugMsg('caseroot = {0}'.format(caseroot), header=True, verbosity=2) debugMsg('calling initialize_main', header=True, verbosity=2) envDict = initialize_main(envDict, caseroot, debugMsg, options.standalone) debugMsg('calling check_ncl_nco', header=True, verbosity=2) diagUtilsLib.check_ncl_nco(envDict) # broadcast envDict to all tasks envDict = main_comm.partition(data=envDict, func=partition.Duplicate(), involved=True) sys.path.append(envDict['PATH']) main_comm.sync() # get list of diagnostics types to be created diag_list = list() num_of_diags = 0 if main_comm.is_manager(): diag_list, diag_dict = setup_diags(envDict) num_of_diags = len(diag_list) if num_of_diags == 0: print('No ocean diagnostics specified. Please check the {0}/env_diags_ocn.xml settings.'.format(envDict['PP_CASE_PATH'])) sys.exit(1) print('User requested diagnostics:') for diag in diag_list: print(' {0}'.format(diag)) try: os.makedirs(envDict['WORKDIR']) except OSError as exception: if exception.errno != errno.EEXIST: err_msg = 'ERROR: ocn_diags_generator.py problem accessing the working directory {0}'.format(envDict['WORKDIR']) raise OSError(err_msg) debugMsg('Ocean diagnostics - Creating main index.html page', header=True, verbosity=2) # define the templatePath templatePath = '{0}/diagnostics/diagnostics/ocn/Templates'.format(envDict['POSTPROCESS_PATH']) templateLoader = jinja2.FileSystemLoader( searchpath=templatePath ) templateEnv = jinja2.Environment( loader=templateLoader ) template_file = 'ocean_diagnostics.tmpl' template = templateEnv.get_template( template_file ) # get the current datatime string for the template now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # set the template variables templateVars = { 'casename' : envDict['CASE'], 'tagname' : envDict['CESM_TAG'], 'diag_dict' : diag_dict, 'control_casename' : envDict['CNTRLCASE'], 'start_year' : envDict['YEAR0'], 'stop_year' : envDict['YEAR1'], 'control_start_year' : envDict['CNTRLYEAR0'], 'control_stop_year' : envDict['CNTRLYEAR1'], 'today': now, 'tseries_start_year' : envDict['TSERIES_YEAR0'], 'tseries_stop_year' : envDict['TSERIES_YEAR1'] } # write the main index.html page to the top working directory main_html = template.render( templateVars ) with open( '{0}/index.html'.format(envDict['WORKDIR']), 'w') as index: index.write(main_html) debugMsg('Ocean diagnostics - Copying stylesheet', header=True, verbosity=2) shutil.copy2('{0}/Templates/diag_style.css'.format(envDict['POSTPROCESS_PATH']), '{0}/diag_style.css'.format(envDict['WORKDIR'])) debugMsg('Ocean diagnostics - Copying logo files', header=True, verbosity=2) if not os.path.exists('{0}/logos'.format(envDict['WORKDIR'])): os.mkdir('{0}/logos'.format(envDict['WORKDIR'])) for filename in glob.glob(os.path.join('{0}/Templates/logos'.format(envDict['POSTPROCESS_PATH']), '*.*')): shutil.copy(filename, '{0}/logos'.format(envDict['WORKDIR'])) # setup the unique OCNDIAG_WEBDIR output file env_file = '{0}/env_diags_ocn.xml'.format(envDict['PP_CASE_PATH']) key = 'OCNDIAG_WEBDIR' value = envDict['WORKDIR'] ##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(envDict['PP_CASE_PATH'], key, main_comm.get_size(), main_comm.get_rank() ) web_file = '{0}/web_dirs/{1}.{2}'.format(envDict['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S')) try: diagUtilsLib.write_web_file(web_file, 'ocn', key, value) except: print('WARNING ocn_diags_generator unable to write {0}={1} to {2}'.format(key, value, web_file)) main_comm.sync() # broadcast the diag_list to all tasks num_of_diags = main_comm.partition(num_of_diags, func=partition.Duplicate(), involved=True) diag_list = main_comm.partition(data=diag_list, func=partition.Duplicate(), involved=True) main_comm.sync() # initialize some variables for distributing diagnostics across the communicators diags_send = diag_list gmaster = main_comm.is_manager() gsize = main_comm.get_size() grank = main_comm.get_rank() local_diag_list = list() # divide the main communicator into sub_communicators to be passed to each diag class # split mpi comm world if the size of the communicator > 1 and the num_of_diags > 1 if gsize > 1 and num_of_diags > 1: temp_color = (grank % num_of_diags) if (temp_color == num_of_diags): temp_color = temp_color - 1 groups = list() for g in range(0,num_of_diags): groups.append(g) debugMsg('global_rank {0}, temp_color {1}, #of groups(diag types) {2}, groups {3}, diag_list {4}'.format(grank, temp_color, num_of_diags, groups, diag_list), header=True, verbosity=2) group = groups[temp_color] inter_comm, multi_comm = main_comm.divide(group) color = inter_comm.get_color() lsize = inter_comm.get_size() lrank = inter_comm.get_rank() lmaster = inter_comm.is_manager() debugMsg('color {0}, lsize {1}, lrank {2}, lmaster {3}'.format(color, lsize, lrank, lmaster), header=True, verbosity=2) # partition the diag_list between communicators DIAG_LIST_TAG = 10 if lmaster: local_diag_list = multi_comm.partition(diag_list,func=partition.EqualStride(),involved=True) debugMsg('lrank = {0} local_diag_list = {1}'.format(lrank, local_diag_list), header=True, verbosity=2) for b in range(1, lsize): diags_send = inter_comm.ration(data=local_diag_list, tag=DIAG_LIST_TAG) debugMsg('b = {0} diags_send = {1} lsize = {2}'.format(b, diags_send, lsize), header=True, verbosity=2) else: local_diag_list = inter_comm.ration(tag=DIAG_LIST_TAG) debugMsg('local_diag_list {0}',format(local_diag_list), header=True, verbosity=2) else: inter_comm = main_comm lmaster = main_comm.is_manager() lsize = main_comm.get_size() lrank = main_comm.get_rank() local_diag_list = diag_list inter_comm.sync() main_comm.sync() # loop through the local_diag_list for requested_diag in local_diag_list: try: debugMsg('requested_diag {0}, lrank {1}, lsize {2}, lmaster {3}'.format(requested_diag, lrank, lsize, lmaster), header=True, verbosity=2) diag = ocn_diags_factory.oceanDiagnosticsFactory(requested_diag) # check the prerequisites for the diagnostics types debugMsg('Checking prerequisites for {0}'.format(diag.__class__.__name__), header=True, verbosity=2) skip_key = '{0}_SKIP'.format(requested_diag) if lmaster: try: envDict = diag.check_prerequisites(envDict) except ocn_diags_bc.PrerequisitesError: print("Problem with check_prerequisites for '{0}' skipping!".format(requested_diag)) envDict[skip_key] = True except RuntimeError as e: # unrecoverable error, bail! print(e) envDict['unrecoverableErrorOnMaster'] = True inter_comm.sync() # broadcast the envDict envDict = inter_comm.partition(data=envDict, func=partition.Duplicate(), involved=True) if envDict.has_key('unrecoverableErrorOnMaster'): raise RuntimeError # run the diagnostics type on each inter_comm if not envDict.has_key(skip_key): # set the shell env using the values set in the XML and read into the envDict across all tasks cesmEnvLib.setXmlEnv(envDict) # run the diagnostics envDict = diag.run_diagnostics(envDict, inter_comm) inter_comm.sync() except ocn_diags_bc.RecoverableError as e: # catch all recoverable errors, print a message and continue. print(e) print("Skipped '{0}' and continuing!".format(requested_diag)) except RuntimeError as e: # unrecoverable error, bail! print(e) return 1 main_comm.sync()
def run_diagnostics(self, env, scomm): """ call the necessary plotting routines to generate diagnostics plots """ super(modelVsModel, self).run_diagnostics(env, scomm) scomm.sync() # setup some global variables requested_plot_sets = list() local_requested_plots = list() local_html_list = list() # all the plot module XML vars start with 'set_' for key, value in env.iteritems(): if ("set_" in key and value == 'True'): requested_plot_sets.append(key) scomm.sync() if scomm.is_manager(): print('DEBUG model_vs_model requested_plot_sets = {0}'.format( requested_plot_sets)) # partition requested plots to all tasks # first, create plotting classes and get the number of plots each will created requested_plots = {} set_sizes = {} #plots_weights = [] for plot_set in requested_plot_sets: requested_plots.update( lnd_diags_plot_factory.LandDiagnosticPlotFactory( plot_set, env)) #for plot_id,plot_class in requested_plots.iteritems(): # if hasattr(plot_class, 'weight'): # factor = plot_class.weight # else: # factor = 1 # plots_weights.append((plot_id,len(plot_class.expectedPlots)*factor)) # partition based on the number of plots each set will create #local_plot_list = scomm.partition(plots_weights, func=partition.WeightBalanced(), involved=True) if scomm.is_manager(): print('DEBUG model_vs_model requested_plots.keys = {0}'.format( requested_plots.keys())) local_plot_list = scomm.partition(requested_plots.keys(), func=partition.EqualStride(), involved=True) scomm.sync() timer = timekeeper.TimeKeeper() # loop over local plot lists - set env and then run plotting script timer.start(str(scomm.get_rank()) + "ncl total time on task") for plot_set in local_plot_list: timer.start(str(scomm.get_rank()) + plot_set) plot_class = requested_plots[plot_set] # set all env variables (global and particular to this plot call print( 'model vs. model - Checking prerequisite for {0} on rank {1}'. format(plot_class.__class__.__name__, scomm.get_rank())) plot_class.check_prerequisites(env) # Stringify the env dictionary for name, value in plot_class.plot_env.iteritems(): plot_class.plot_env[name] = str(value) # call script to create plots for script in plot_class.ncl_scripts: print( 'model vs. model - Generating plots for {0} on rank {1} with script {2}' .format(plot_class.__class__.__name__, scomm.get_rank(), script)) diagUtilsLib.generate_ncl_plots(plot_class.plot_env, script) timer.stop(str(scomm.get_rank()) + plot_set) timer.stop(str(scomm.get_rank()) + "ncl total time on task") scomm.sync() print(timer.get_all_times()) #w = 0 #for p in plots_weights: # if p[0] in local_plot_list: # w = w + p[1] #print(str(scomm.get_rank())+' weight:'+str(w)) # set html files if scomm.is_manager(): # Create web dirs and move images/tables to that web dir for n in ('1', '2', '3', '4', '5', '6', '7', '8', '9'): web_dir = env['WKDIR'] set_dir = web_dir + '/set' + n # Create the plot set web directory if not os.path.exists(set_dir): os.makedirs(set_dir) # Copy plots into the correct web dir glob_string = web_dir + '/set' + n + '_*' imgs = glob.glob(glob_string) if len(imgs) > 0: for img in imgs: new_fn = set_dir + '/' + os.path.basename(img) os.rename(img, new_fn) # Copy the set1Diff and set1Anom plots to set_1 web dir if n == '1': glob_string = web_dir + '/set1Diff' + '_*' imgs = glob.glob(glob_string) if len(imgs) > 0: for img in imgs: new_fn = set_dir + '/' + os.path.basename(img) os.rename(img, new_fn) glob_string = web_dir + '/set1Anom' + '_*' imgs = glob.glob(glob_string) if len(imgs) > 0: for img in imgs: new_fn = set_dir + '/' + os.path.basename(img) os.rename(img, new_fn) env['WEB_DIR'] = web_dir shutil.copy2( env['POSTPROCESS_PATH'] + '/lnd_diag/inputFiles/' + env['VAR_MASTER'], web_dir + '/variable_master.ncl') if n == '9': web_script = env[ 'POSTPROCESS_PATH'] + '/lnd_diag/shared/lnd_statTable.pl' rc2, err_msg = cesmEnvLib.checkFile(web_script, 'read') if rc2: try: subprocess.check_call(web_script) except subprocess.CalledProcessError as e: print('WARNING: {0} error executing command:'.format( web_script)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) else: print('{0}... {1} file not found'.format( err_msg, web_script)) # copy the set9_statTable.html to the set9 subdir set9_statTable = web_dir + '/set9_statTable.html' rc2, err_msg = cesmEnvLib.checkFile(set9_statTable, 'read') if rc2: new_fn = web_dir + '/set9/' + os.path.basename( set9_statTable) try: os.rename(set9_statTable, new_fn) except os.OSError as e: print('WARNING: rename error for file {0}'.format( set9_statTable)) print(' rc = {0}'.format(e.returncode)) else: print('{0}... {1} file not found'.format( err_msg, set9_statTable)) web_script_1 = env[ 'POSTPROCESS_PATH'] + '/lnd_diag/shared/lnd_create_webpage.pl' web_script_2 = env[ 'POSTPROCESS_PATH'] + '/lnd_diag/shared/lnd_lookupTable.pl' print('Creating Web Pages') # set the shell environment cesmEnvLib.setXmlEnv(env) # lnd_create_webpage.pl call rc1, err_msg = cesmEnvLib.checkFile(web_script_1, 'read') if rc1: try: subprocess.check_call(web_script_1) except subprocess.CalledProcessError as e: print('WARNING: {0} error executing command:'.format( web_script_1)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) else: print('{0}... {1} file not found'.format( err_msg, web_script_1)) # lnd_lookupTable.pl call rc2, err_msg = cesmEnvLib.checkFile(web_script_2, 'read') if rc2: try: subprocess.check_call(web_script_2) except subprocess.CalledProcessError as e: print('WARNING: {0} error executing command:'.format( web_script_2)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) else: print('{0}... {1} file not found'.format( err_msg, web_script_2)) # move all the plots to the diag_path with the years appended to the path endYr1 = (int(env['clim_first_yr_1']) + int(env['clim_num_yrs_1'])) - 1 endYr2 = (int(env['clim_first_yr_2']) + int(env['clim_num_yrs_2'])) - 1 diag_path = '{0}/diag/{1}.{2}_{3}-{4}.{5}_{6}'.format( env['OUTPUT_ROOT_PATH'], env['caseid_1'], env['clim_first_yr_1'], str(endYr1), env['caseid_2'], env['clim_first_yr_2'], str(endYr2)) move_files = True try: os.makedirs(diag_path) except OSError as exception: if exception.errno != errno.EEXIST: err_msg = 'ERROR: {0} problem accessing directory {1}'.format( self.__class__.__name__, diag_path) raise OSError(err_msg) move_files = False elif env['CLEANUP_FILES'].lower() in ['t', 'true']: # delete all the files in the diag_path directory for root, dirs, files in os.walk('diag_path'): for f in files: os.unlink(os.path.join(root, f)) for d in dirs: shutil.rmtree(os.path.join(root, d)) elif env['CLEANUP_FILES'].lower() in ['f', 'false']: print( 'WARNING: {0} exists and is not empty and LNDDIAG_CLEANUP_FILES = False. Leaving new diagnostics files in {1}' .format(diag_path, web_dir)) diag_path = web_dir move_files = False # move the files to the new diag_path if move_files: try: print('DEBUG: model_vs_model renaming web files') os.rename(web_dir, diag_path) except OSError as e: print('WARNING: Error renaming %s to %s: %s' % (web_dir, diag_path, e)) diag_path = web_dir print('DEBUG: model vs. model web_dir = {0}'.format(web_dir)) print('DEBUG: model vs. model diag_path = {0}'.format(diag_path)) # setup the unique LNDDIAG_WEBDIR_MODEL_VS_MODEL output file env_file = '{0}/env_diags_lnd.xml'.format(env['PP_CASE_PATH']) key = 'LNDDIAG_WEBDIR_{0}'.format(self._name) value = diag_path ##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(env['PP_CASE_PATH'], key, scomm.get_size(), scomm.get_rank() ) web_file = '{0}/web_dirs/{1}.{2}'.format( env['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S')) try: diagUtilsLib.write_web_file(web_file, 'lnd', key, value) except: print( 'WARNING lnd model_vs_model unable to write {0}={1} to {2}' .format(key, value, web_file)) print( '*******************************************************************************' ) print( 'Successfully completed generating land diagnostics model vs. model plots' ) print( '*******************************************************************************' )
def main(options, main_comm, debugMsg, timer): """setup the environment for running the diagnostics in parallel. Calls 2 different diagnostics generation types: model vs. observation model vs. model Arguments: options (object) - command line options main_comm (object) - MPI simple communicator object debugMsg (object) - vprinter object for printing debugging messages timer (object) - timer object for keeping times The env_diags_lnd.xml configuration file defines the way the diagnostics are generated. See (website URL here...) for a complete desciption of the env_diags_lnd XML options. """ # initialize the environment dictionary envDict = dict() # CASEROOT is given on the command line as required option --caseroot if main_comm.is_manager(): caseroot = options.caseroot[0] debugMsg('caseroot = {0}'.format(caseroot), header=True, verbosity=1) debugMsg('calling initialize_main', header=True, verbosity=1) envDict = initialize_main(envDict, caseroot, debugMsg, options.standalone) debugMsg('calling check_ncl_nco', header=True, verbosity=1) diagUtilsLib.check_ncl_nco(envDict) # broadcast envDict to all tasks envDict = main_comm.partition(data=envDict, func=partition.Duplicate(), involved=True) main_comm.sync() # get list of diagnostics types to be created diag_list = list() diag_list = setup_diags(envDict) if main_comm.is_manager(): print('User requested diagnostics:') for diag in diag_list: print(' {0}'.format(diag)) main_comm.sync() # broadcast the diag_list to all tasks num_of_diags = len(diag_list) num_of_diags = main_comm.partition(num_of_diags, func=partition.Duplicate(), involved=True) diag_list = main_comm.partition(data=diag_list, func=partition.Duplicate(), involved=True) main_comm.sync() # initialize some variables for distributing diagnostics across the communicators diags_send = diag_list gmaster = main_comm.is_manager() gsize = main_comm.get_size() grank = main_comm.get_rank() local_diag_list = list() # divide the main communicator into sub_communicators to be passed to each diag class # split mpi comm world if the size of the communicator > 1 and the num_of_diags > 1 if gsize > 1 and num_of_diags > 1: temp_color = (grank % num_of_diags) if (temp_color == num_of_diags): temp_color = temp_color - 1 groups = list() for g in range(0,num_of_diags): groups.append(g) debugMsg('global_rank {0}, temp_color {1}, #of groups(diag types) {2}, groups {3}, diag_list {4}'.format(grank, temp_color, num_of_diags, groups, diag_list), header=True, verbosity=1) group = groups[temp_color] inter_comm, multi_comm = main_comm.divide(group) color = inter_comm.get_color() lsize = inter_comm.get_size() lrank = inter_comm.get_rank() lmaster = inter_comm.is_manager() if lmaster: debugMsg('color {0}, lsize {1}, lrank {2}, lmaster {3}'.format(color, lsize, lrank, lmaster), header=True, verbosity=1) # partition the diag_list between communicators DIAG_LIST_TAG = 10 if lmaster: local_diag_list = multi_comm.partition(diag_list,func=partition.EqualStride(),involved=True) for b in range(1, lsize): diags_send = inter_comm.ration(data=local_diag_list, tag=DIAG_LIST_TAG) else: local_diag_list = inter_comm.ration(tag=DIAG_LIST_TAG) if lmaster: debugMsg('local_diag_list {0}',format(local_diag_list), header=True, verbosity=1) else: inter_comm = main_comm lmaster = main_comm.is_manager() lsize = main_comm.get_size() lrank = main_comm.get_rank() local_diag_list = diag_list inter_comm.sync() main_comm.sync() if lmaster: debugMsg('lsize = {0}, lrank = {1}'.format(lsize, lrank), header=True, verbosity=1) inter_comm.sync() # loop through the local_diag_list list for requested_diag in local_diag_list: try: diag = lnd_diags_factory.landDiagnosticsFactory(requested_diag,envDict) # check the prerequisites for the diagnostics types if lmaster: debugMsg('Checking prerequisites for {0}'.format(diag.__class__.__name__), header=True, verbosity=1) envDict = diag.check_prerequisites(envDict, inter_comm) inter_comm.sync() # broadcast the envDict envDict = inter_comm.partition(data=envDict, func=partition.Duplicate(), involved=True) # set the shell env using the values set in the XML and read into the envDict across all tasks cesmEnvLib.setXmlEnv(envDict) # debug check if the envDict contains a non-string entry if lmaster: for k,v in envDict.iteritems(): if not isinstance(v, basestring): debugMsg('lnd_diags_generator - envDict: key = {0}, value = {1}'.format(k,v), header=True, verbosity=1) debugMsg('inter_comm size = {0}'.format(inter_comm.get_size()), header=True, verbosity=1) diag.run_diagnostics(envDict, inter_comm) if lmaster: debugMsg('lnd_diags_generator: return from run_diagnostics', header=True, verbosity=1) except lnd_diags_bc.RecoverableError as e: # catch all recoverable errors, print a message and continue. print(e) print("Skipped '{0}' and continuing!".format(request_diag)) except RuntimeError as e: # unrecoverable error, bail! print(e) return 1
def run_diagnostics(self, env, scomm): """ call the necessary plotting routines to generate diagnostics plots """ super(modelVsObs, self).run_diagnostics(env, scomm) scomm.sync() # setup some global variables requested_plot_sets = list() local_requested_plots = list() local_html_list = list() # all the plot module XML vars start with 'set_' for key, value in env.iteritems(): if ("set_" in key and value == 'True'): requested_plot_sets.append(key) scomm.sync() if scomm.is_manager(): print('DEBUG model_vs_obs requested_plot_sets = {0}'.format(requested_plot_sets)) # partition requested plots to all tasks # first, create plotting classes and get the number of plots each will created requested_plots = {} set_sizes = {} #plots_weights = [] for plot_set in requested_plot_sets: requested_plots.update(lnd_diags_plot_factory.LandDiagnosticPlotFactory(plot_set,env)) #for plot_id,plot_class in requested_plots.iteritems(): # if hasattr(plot_class, 'weight'): # factor = plot_class.weight # else: # factor = 1 # plots_weights.append((plot_id,len(plot_class.expectedPlots)*factor)) # partition based on the number of plots each set will create #local_plot_list = scomm.partition(plots_weights, func=partition.WeightBalanced(), involved=True) local_plot_list = scomm.partition(requested_plots.keys(), func=partition.EqualStride(), involved=True) scomm.sync() timer = timekeeper.TimeKeeper() # loop over local plot lists - set env and then run plotting script timer.start(str(scomm.get_rank())+"ncl total time on task") for plot_set in local_plot_list: timer.start(str(scomm.get_rank())+plot_set) plot_class = requested_plots[plot_set] print('DEBUG model vs. obs - Checking prerequisite for {0} on rank {1}'.format(plot_class.__class__.__name__, scomm.get_rank())) plot_class.check_prerequisites(env) # Stringify the env dictionary for name, value in plot_class.plot_env.iteritems(): plot_class.plot_env[name] = str(value) # call script to create plots for script in plot_class.ncl_scripts: print('DEBUG model vs. obs - Generating plots for {0} on rank {1} with script {2}'.format(plot_class.__class__.__name__, scomm.get_rank(),script)) diagUtilsLib.generate_ncl_plots(plot_class.plot_env, script) timer.stop(str(scomm.get_rank())+plot_set) timer.stop(str(scomm.get_rank())+"ncl total time on task") scomm.sync() print(timer.get_all_times()) #w = 0 #for p in plots_weights: # if p[0] in local_plot_list: # w = w + p[1] #print(str(scomm.get_rank())+' weight:'+str(w)) # set html files if scomm.is_manager(): # Create web dirs and move images/tables to that web dir for n in ('1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12'): web_dir = env['WKDIR'] set_dir = web_dir + '/set' + n # Create the plot set web directory if not os.path.exists(set_dir): os.makedirs(set_dir) # Copy plots into the correct web dir glob_string = web_dir+'/set'+n+'_*' imgs = glob.glob(glob_string) if len(imgs) > 0: for img in imgs: new_fn = set_dir + '/' + os.path.basename(img) os.rename(img,new_fn) env['WEB_DIR'] = web_dir shutil.copy2(env['POSTPROCESS_PATH']+'/lnd_diag/inputFiles/'+env['VAR_MASTER'],web_dir+'/variable_master.ncl') web_script_1 = env['POSTPROCESS_PATH']+'/lnd_diag/shared/lnd_create_webpage.pl' web_script_2 = env['POSTPROCESS_PATH']+'/lnd_diag/shared/lnd_lookupTable.pl' print('Creating Web Pages') # set the shell environment cesmEnvLib.setXmlEnv(env) # lnd_create_webpage.pl call rc1, err_msg = cesmEnvLib.checkFile(web_script_1,'read') if rc1: try: subprocess.check_call(web_script_1) except subprocess.CalledProcessError as e: print('WARNING: {0} error executing command:'.format(web_script_1)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) else: print('{0}... {1} file not found'.format(err_msg,web_script_1)) # lnd_lookupTable.pl call rc2, err_msg = cesmEnvLib.checkFile(web_script_2,'read') if rc2: try: subprocess.check_call(web_script_2) except subprocess.CalledProcessError as e: print('WARNING: {0} error executing command:'.format(web_script_2)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) else: print('{0}... {1} file not found'.format(err_msg,web_script_2)) # move all the plots to the diag_path with the years appended to the path endYr = (int(env['clim_first_yr_1']) + int(env['clim_num_yrs_1'])) - 1 diag_path = '{0}/diag/{1}-obs.{2}_{3}'.format(env['OUTPUT_ROOT_PATH'], env['caseid_1'], env['clim_first_yr_1'], str(endYr)) move_files = True try: os.makedirs(diag_path) except OSError as exception: if exception.errno != errno.EEXIST: err_msg = 'ERROR: {0} problem accessing directory {1}'.format(self.__class__.__name__, diag_path) raise OSError(err_msg) move_files = False elif env['CLEANUP_FILES'].lower() in ['t','true']: # delete all the files in the diag_path directory for root, dirs, files in os.walk(diag_path): for f in files: os.unlink(os.path.join(root, f)) for d in dirs: shutil.rmtree(os.path.join(root, d)) elif env['CLEANUP_FILES'].lower() in ['f','false']: print('WARNING: {0} exists and is not empty and LNDDIAG_CLEANUP_FILES = False. Leaving new diagnostics files in {1}'.format(diag_path, web_dir)) diag_path = web_dir move_files = False print('DEBUG: model vs. obs web_dir = {0}'.format(web_dir)) print('DEBUG: model vs. obs diag_path = {0}'.format(diag_path)) # move the files to the new diag_path if move_files: try: print('DEBUG: model_vs_obs renaming web files') os.rename(web_dir, diag_path) except OSError as e: print ('WARNING: Error renaming %s to %s: %s' % (web_dir, diag_path, e)) diag_path = web_dir # setup the LNDDIAG_WEBDIR_MODEL_VS_OBS output file env_file = '{0}/env_diags_lnd.xml'.format(env['PP_CASE_PATH']) key = 'LNDDIAG_WEBDIR_{0}'.format(self._name) value = diag_path web_file = '{0}/web_dirs/{1}.{2}'.format(env['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S')) try: diagUtilsLib.write_web_file(web_file, 'lnd', key, value) except: print('WARNING lnd model_vs_obs unable to write {0}={1} to {2}'.format(key, value, web_file)) print('*******************************************************************************') print('Successfully completed generating land diagnostics model vs. observation plots') print('*******************************************************************************')
def main(options, main_comm, debugMsg): """setup the environment for running the diagnostics in parallel. Calls 6 different diagnostics generation types: model vs. observation (optional BGC - ecosystem) model vs. control (optional BGC - ecosystem) model time-series (optional BGC - ecosystem) Arguments: options (object) - command line options main_comm (object) - MPI simple communicator object debugMsg (object) - vprinter object for printing debugging messages The env_diags_ocn.xml configuration file defines the way the diagnostics are generated. See (website URL here...) for a complete desciption of the env_diags_ocn XML options. """ # initialize the environment dictionary envDict = dict() # CASEROOT is given on the command line as required option --caseroot if main_comm.is_manager(): caseroot = options.caseroot[0] debugMsg('caseroot = {0}'.format(caseroot), header=True, verbosity=2) debugMsg('calling initialize_main', header=True, verbosity=2) envDict = initialize_main(envDict, caseroot, debugMsg, options.standalone) debugMsg('calling check_ncl_nco', header=True, verbosity=2) diagUtilsLib.check_ncl_nco(envDict) # broadcast envDict to all tasks envDict = main_comm.partition(data=envDict, func=partition.Duplicate(), involved=True) sys.path.append(envDict['PATH']) main_comm.sync() # get list of diagnostics types to be created diag_list = list() num_of_diags = 0 if main_comm.is_manager(): diag_list, diag_dict = setup_diags(envDict) num_of_diags = len(diag_list) if num_of_diags == 0: print('No ocean diagnostics specified. Please check the {0}/env_diags_ocn.xml settings.'.format(envDict['PP_CASE_PATH'])) sys.exit(1) print('User requested diagnostics:') for diag in diag_list: print(' {0}'.format(diag)) try: os.makedirs(envDict['WORKDIR']) except OSError as exception: if exception.errno != errno.EEXIST: err_msg = 'ERROR: ocn_diags_generator.py problem accessing the working directory {0}'.format(envDict['WORKDIR']) raise OSError(err_msg) debugMsg('Ocean diagnostics - Creating main index.html page', header=True, verbosity=2) # define the templatePath templatePath = '{0}/diagnostics/diagnostics/ocn/Templates'.format(envDict['POSTPROCESS_PATH']) templateLoader = jinja2.FileSystemLoader( searchpath=templatePath ) templateEnv = jinja2.Environment( loader=templateLoader ) template_file = 'ocean_diagnostics.tmpl' template = templateEnv.get_template( template_file ) # get the current datatime string for the template now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # set the template variables templateVars = { 'casename' : envDict['CASE'], 'tagname' : envDict['CESM_TAG'], 'username' : envDict['USER_NAME'], 'diag_dict' : diag_dict, 'control_casename' : envDict['CNTRLCASE'], 'start_year' : envDict['YEAR0'], 'stop_year' : envDict['YEAR1'], 'control_start_year' : envDict['CNTRLYEAR0'], 'control_stop_year' : envDict['CNTRLYEAR1'], 'today': now, 'tseries_start_year' : envDict['TSERIES_YEAR0'], 'tseries_stop_year' : envDict['TSERIES_YEAR1'] } # write the main index.html page to the top working directory main_html = template.render( templateVars ) with open( '{0}/index.html'.format(envDict['WORKDIR']), 'w') as index: index.write(main_html) debugMsg('Ocean diagnostics - Copying stylesheet', header=True, verbosity=2) shutil.copy2('{0}/Templates/diag_style.css'.format(envDict['POSTPROCESS_PATH']), '{0}/diag_style.css'.format(envDict['WORKDIR'])) debugMsg('Ocean diagnostics - Copying logo files', header=True, verbosity=2) if not os.path.exists('{0}/logos'.format(envDict['WORKDIR'])): os.mkdir('{0}/logos'.format(envDict['WORKDIR'])) for filename in glob.glob(os.path.join('{0}/Templates/logos'.format(envDict['POSTPROCESS_PATH']), '*.*')): shutil.copy(filename, '{0}/logos'.format(envDict['WORKDIR'])) # setup the unique OCNDIAG_WEBDIR output file env_file = '{0}/env_diags_ocn.xml'.format(envDict['PP_CASE_PATH']) key = 'OCNDIAG_WEBDIR' value = envDict['WORKDIR'] ##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(envDict['PP_CASE_PATH'], key, main_comm.get_size(), main_comm.get_rank() ) web_file = '{0}/web_dirs/{1}.{2}'.format(envDict['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S')) try: diagUtilsLib.write_web_file(web_file, 'ocn', key, value) except: print('WARNING ocn_diags_generator unable to write {0}={1} to {2}'.format(key, value, web_file)) main_comm.sync() # broadcast the diag_list to all tasks num_of_diags = main_comm.partition(num_of_diags, func=partition.Duplicate(), involved=True) diag_list = main_comm.partition(data=diag_list, func=partition.Duplicate(), involved=True) main_comm.sync() # initialize some variables for distributing diagnostics across the communicators diags_send = diag_list gmaster = main_comm.is_manager() gsize = main_comm.get_size() grank = main_comm.get_rank() local_diag_list = list() # divide the main communicator into sub_communicators to be passed to each diag class # split mpi comm world if the size of the communicator > 1 and the num_of_diags > 1 if gsize > 1 and num_of_diags > 1: temp_color = (grank % num_of_diags) if (temp_color == num_of_diags): temp_color = temp_color - 1 groups = list() for g in range(0,num_of_diags): groups.append(g) debugMsg('global_rank {0}, temp_color {1}, #of groups(diag types) {2}, groups {3}, diag_list {4}'.format(grank, temp_color, num_of_diags, groups, diag_list), header=True, verbosity=2) group = groups[temp_color] inter_comm, multi_comm = main_comm.divide(group) color = inter_comm.get_color() lsize = inter_comm.get_size() lrank = inter_comm.get_rank() lmaster = inter_comm.is_manager() debugMsg('color {0}, lsize {1}, lrank {2}, lmaster {3}'.format(color, lsize, lrank, lmaster), header=True, verbosity=2) # partition the diag_list between communicators DIAG_LIST_TAG = 10 if lmaster: local_diag_list = multi_comm.partition(diag_list,func=partition.EqualStride(),involved=True) debugMsg('lrank = {0} local_diag_list = {1}'.format(lrank, local_diag_list), header=True, verbosity=2) for b in range(1, lsize): diags_send = inter_comm.ration(data=local_diag_list, tag=DIAG_LIST_TAG) debugMsg('b = {0} diags_send = {1} lsize = {2}'.format(b, diags_send, lsize), header=True, verbosity=2) else: local_diag_list = inter_comm.ration(tag=DIAG_LIST_TAG) debugMsg('local_diag_list {0}',format(local_diag_list), header=True, verbosity=2) else: inter_comm = main_comm lmaster = main_comm.is_manager() lsize = main_comm.get_size() lrank = main_comm.get_rank() local_diag_list = diag_list inter_comm.sync() main_comm.sync() # loop through the local_diag_list for requested_diag in local_diag_list: try: debugMsg('requested_diag {0}, lrank {1}, lsize {2}, lmaster {3}'.format(requested_diag, lrank, lsize, lmaster), header=True, verbosity=2) diag = ocn_diags_factory.oceanDiagnosticsFactory(requested_diag) # check the prerequisites for the diagnostics types debugMsg('Checking prerequisites for {0}'.format(diag.__class__.__name__), header=True, verbosity=2) skip_key = '{0}_SKIP'.format(requested_diag) if lmaster: try: envDict = diag.check_prerequisites(envDict) except ocn_diags_bc.PrerequisitesError: print("Problem with check_prerequisites for '{0}' skipping!".format(requested_diag)) envDict[skip_key] = True except RuntimeError as e: # unrecoverable error, bail! print(e) envDict['unrecoverableErrorOnMaster'] = True inter_comm.sync() # broadcast the envDict envDict = inter_comm.partition(data=envDict, func=partition.Duplicate(), involved=True) if envDict.has_key('unrecoverableErrorOnMaster'): raise RuntimeError # run the diagnostics type on each inter_comm if not envDict.has_key(skip_key): # set the shell env using the values set in the XML and read into the envDict across all tasks cesmEnvLib.setXmlEnv(envDict) # run the diagnostics envDict = diag.run_diagnostics(envDict, inter_comm) inter_comm.sync() except ocn_diags_bc.RecoverableError as e: # catch all recoverable errors, print a message and continue. print(e) print("Skipped '{0}' and continuing!".format(requested_diag)) except RuntimeError as e: # unrecoverable error, bail! print(e) return 1 main_comm.sync()
def main(options, main_comm, debugMsg, timer): """setup the environment for running the diagnostics in parallel. Calls 2 different diagnostics generation types: model vs. observation model vs. model Arguments: options (object) - command line options main_comm (object) - MPI simple communicator object debugMsg (object) - vprinter object for printing debugging messages timer (object) - timer object for keeping times The env_diags_lnd.xml configuration file defines the way the diagnostics are generated. See (website URL here...) for a complete desciption of the env_diags_lnd XML options. """ # initialize the environment dictionary envDict = dict() # CASEROOT is given on the command line as required option --caseroot if main_comm.is_manager(): caseroot = options.caseroot[0] debugMsg('caseroot = {0}'.format(caseroot), header=True, verbosity=1) debugMsg('calling initialize_main', header=True, verbosity=1) envDict = initialize_main(envDict, caseroot, debugMsg, options.standalone) debugMsg('calling check_ncl_nco', header=True, verbosity=1) diagUtilsLib.check_ncl_nco(envDict) # broadcast envDict to all tasks envDict = main_comm.partition(data=envDict, func=partition.Duplicate(), involved=True) main_comm.sync() # get list of diagnostics types to be created diag_list = list() diag_list = setup_diags(envDict) if main_comm.is_manager(): print('User requested diagnostics:') for diag in diag_list: print(' {0}'.format(diag)) main_comm.sync() # broadcast the diag_list to all tasks num_of_diags = len(diag_list) num_of_diags = main_comm.partition(num_of_diags, func=partition.Duplicate(), involved=True) diag_list = main_comm.partition(data=diag_list, func=partition.Duplicate(), involved=True) main_comm.sync() # initialize some variables for distributing diagnostics across the communicators diags_send = diag_list gmaster = main_comm.is_manager() gsize = main_comm.get_size() grank = main_comm.get_rank() local_diag_list = list() # divide the main communicator into sub_communicators to be passed to each diag class # split mpi comm world if the size of the communicator > 1 and the num_of_diags > 1 if gsize > 1 and num_of_diags > 1: temp_color = (grank % num_of_diags) if (temp_color == num_of_diags): temp_color = temp_color - 1 groups = list() for g in range(0, num_of_diags): groups.append(g) debugMsg( 'global_rank {0}, temp_color {1}, #of groups(diag types) {2}, groups {3}, diag_list {4}' .format(grank, temp_color, num_of_diags, groups, diag_list), header=True, verbosity=1) group = groups[temp_color] inter_comm, multi_comm = main_comm.divide(group) color = inter_comm.get_color() lsize = inter_comm.get_size() lrank = inter_comm.get_rank() lmaster = inter_comm.is_manager() if lmaster: debugMsg('color {0}, lsize {1}, lrank {2}, lmaster {3}'.format( color, lsize, lrank, lmaster), header=True, verbosity=1) # partition the diag_list between communicators DIAG_LIST_TAG = 10 if lmaster: local_diag_list = multi_comm.partition( diag_list, func=partition.EqualStride(), involved=True) for b in range(1, lsize): diags_send = inter_comm.ration(data=local_diag_list, tag=DIAG_LIST_TAG) else: local_diag_list = inter_comm.ration(tag=DIAG_LIST_TAG) if lmaster: debugMsg('local_diag_list {0}', format(local_diag_list), header=True, verbosity=1) else: inter_comm = main_comm lmaster = main_comm.is_manager() lsize = main_comm.get_size() lrank = main_comm.get_rank() local_diag_list = diag_list inter_comm.sync() main_comm.sync() if lmaster: debugMsg('lsize = {0}, lrank = {1}'.format(lsize, lrank), header=True, verbosity=1) inter_comm.sync() # loop through the local_diag_list list for requested_diag in local_diag_list: try: diag = lnd_diags_factory.landDiagnosticsFactory( requested_diag, envDict) # check the prerequisites for the diagnostics types if lmaster: debugMsg('Checking prerequisites for {0}'.format( diag.__class__.__name__), header=True, verbosity=1) envDict = diag.check_prerequisites(envDict, inter_comm) inter_comm.sync() # broadcast the envDict envDict = inter_comm.partition(data=envDict, func=partition.Duplicate(), involved=True) # set the shell env using the values set in the XML and read into the envDict across all tasks cesmEnvLib.setXmlEnv(envDict) # debug check if the envDict contains a non-string entry if lmaster: for k, v in envDict.iteritems(): if not isinstance(v, basestring): debugMsg( 'lnd_diags_generator - envDict: key = {0}, value = {1}' .format(k, v), header=True, verbosity=1) debugMsg('inter_comm size = {0}'.format(inter_comm.get_size()), header=True, verbosity=1) diag.run_diagnostics(envDict, inter_comm) if lmaster: debugMsg('lnd_diags_generator: return from run_diagnostics', header=True, verbosity=1) except lnd_diags_bc.RecoverableError as e: # catch all recoverable errors, print a message and continue. print(e) print("Skipped '{0}' and continuing!".format(request_diag)) except RuntimeError as e: # unrecoverable error, bail! print(e) return 1