def run_diagnostics(self, env, scomm):
        """ call the necessary plotting routines to generate diagnostics plots
        """
        super(modelVsModel, self).run_diagnostics(env, scomm)
        scomm.sync()

        # setup some global variables
        requested_plot_sets = list()
        local_requested_plots = list()
        local_html_list = list()

        # all the plot module XML vars start with 'set_'  need to strip that off
        for key, value in env.iteritems():
            if ("wset_" in key
                    and (value == 'True' or env['all_waccm_sets'] == 'True')):
                requested_plot_sets.append(key)
            elif ("cset_" in key
                  and (value == 'True' or env['all_chem_sets'] == 'True')):
                requested_plot_sets.append(key)
            elif ("set_" in key
                  and (value == 'True' or env['all_sets'] == 'True')):
                if ("wset_" not in key and "cset_" not in key):
                    requested_plot_sets.append(key)

        scomm.sync()

        # partition requested plots to all tasks
        # first, create plotting classes and get the number of plots each will created
        requested_plots = {}
        set_sizes = {}
        plots_weights = []
        for plot_set in requested_plot_sets:
            requested_plots.update(
                atm_diags_plot_factory.atmosphereDiagnosticPlotFactory(
                    plot_set, env))
        for plot_id, plot_class in requested_plots.iteritems():
            if hasattr(plot_class, 'weight'):
                factor = plot_class.weight
            else:
                factor = 1
            plots_weights.append(
                (plot_id, len(plot_class.expectedPlots) * factor))
        # partition based on the number of plots each set will create
        local_plot_list = scomm.partition(plots_weights,
                                          func=partition.WeightBalanced(),
                                          involved=True)

        timer = timekeeper.TimeKeeper()
        # loop over local plot lists - set env and then run plotting script
        #for plot_id,plot_class in local_plot_list.interitems():
        timer.start(str(scomm.get_rank()) + "ncl total time on task")
        for plot_set in local_plot_list:
            timer.start(str(scomm.get_rank()) + plot_set)
            plot_class = requested_plots[plot_set]
            # set all env variables (global and particular to this plot call
            plot_class.check_prerequisites(env)
            # Stringify the env dictionary
            for name, value in plot_class.plot_env.iteritems():
                plot_class.plot_env[name] = str(value)
            # call script to create plots
            for script in plot_class.ncl_scripts:
                diagUtilsLib.generate_ncl_plots(plot_class.plot_env, script)
                plot_class.plot_env['NCDF_MODE'] = 'write'
                plot_class.plot_env['VAR_MODE'] = 'write'
            timer.stop(str(scomm.get_rank()) + plot_set)
        timer.stop(str(scomm.get_rank()) + "ncl total time on task")
        scomm.sync()
        print(timer.get_all_times())
        w = 0
        for p in plots_weights:
            if p[0] in local_plot_list:
                w = w + p[1]
        print(str(scomm.get_rank()) + ' weight:' + str(w))

        # set html files
        if scomm.is_manager():
            env['HTML_HOME'] = env['DIAG_HOME'] + '/html/model1-model2/'
            # Get web dir name and create it if it does not exist
            #web_dir = '{0}/{1}-{2}'.format(env['test_path_diag'], env['test_casename'], env['cntl_casename'])
            web_dir = env['test_path_diag']
            #if not os.path.exists(web_dir):
            #    os.makedirs(web_dir)
            # Copy over some files needed by web pages
            if not os.path.exists(web_dir + '/images'):
                os.mkdir(web_dir + '/images')
            diag_imgs = glob.glob(env['DIAG_HOME'] + '/html/images/*')
            for img in diag_imgs:
                shutil.copy(img, web_dir + '/images/')

            # Create set dirs, copy plots to set dir, and create html file for set
            requested_plot_sets.append(
                'sets')  # Add 'sets' to create top level html files
            for plot_set in requested_plot_sets:
                if 'set_5' == plot_set or 'set_6' == plot_set:
                    glob_set = plot_set.replace('_', '')
                    plot_set = 'set5_6'
                elif 'set_1' == plot_set:
                    glob_set = 'table_'
                    plot_set = plot_set.replace('_', '')
                elif 'sets' == plot_set:
                    set_dir = web_dir + '/'
                else:
                    plot_set = plot_set.replace('_', '')
                    glob_set = plot_set
                if 'sets' not in plot_set:  #'sets' is top level, don't create directory or copy images files
                    set_dir = web_dir + '/' + plot_set
                    # Create the plot set web directory
                    if not os.path.exists(set_dir):
                        os.makedirs(set_dir)
                    # Copy plots into the correct web dir
                    glob_string = env['test_path_diag'] + '/' + glob_set + '*.*'
                    imgs = glob.glob(glob_string)
                    if imgs > 0:
                        for img in imgs:
                            new_fn = set_dir + '/' + os.path.basename(img)
                            os.rename(img, new_fn)
                # Copy/Process html files
                if 'sets' in plot_set:
                    orig_html = env['HTML_HOME'] + '/' + plot_set
                else:
                    orig_html = env[
                        'HTML_HOME'] + '/' + plot_set + '/' + plot_set
                create_atm_html.create_plotset_html(orig_html, set_dir,
                                                    plot_set, env,
                                                    'model_vs_model')

            # Remove any plotvar netcdf files that exists in the diag directory
            if env['save_ncdfs'] == 'False':
                cesmEnvLib.purge(env['test_path_diag'], '.*\.nc')
                cesmEnvLib.purge(env['test_path_diag'], '/station_ids')

            # move all the plots to the diag_path with the years appended to the path
            endYr1 = (int(env['test_first_yr']) + int(env['test_nyrs'])) - 1
            endYr2 = (int(env['cntl_first_yr']) + int(env['cntl_nyrs'])) - 1
            diag_path = '{0}/diag/{1}.{2}_{3}-{4}.{5}_{6}'.format(
                env['OUTPUT_ROOT_PATH'], env['test_casename'],
                env['test_first_yr'], str(endYr1), env['cntl_casename'],
                env['cntl_first_yr'], str(endYr2))
            move_files = True

            try:
                os.makedirs(diag_path)
            except OSError as exception:
                if exception.errno != errno.EEXIST:
                    err_msg = 'ERROR: {0} problem accessing directory {1}'.format(
                        self.__class__.__name__, diag_path)
                    raise OSError(err_msg)
                    move_files = False

                elif env['CLEANUP_FILES'].lower() in ['t', 'true']:
                    # delete all the files in the diag_path directory
                    for root, dirs, files in os.walk('diag_path'):
                        for f in files:
                            os.unlink(os.path.join(root, f))
                        for d in dirs:
                            shutil.rmtree(os.path.join(root, d))

                elif env['CLEANUP_FILES'].lower() in ['f', 'false']:
                    print(
                        'WARNING: {0} exists and is not empty and ATMDIAG_CLEANUP_FILES = False. Leaving new diagnostics files in {1}'
                        .format(diag_path, web_dir))
                    diag_path = web_dir
                    move_files = False

            # move the files to the new diag_path
            if move_files:
                try:
                    print('DEBUG: model_vs_model renaming web files')
                    os.rename(web_dir, diag_path)
                except OSError as e:
                    print('WARNING: Error renaming %s to %s: %s' %
                          (web_dir, diag_path, e))
                    diag_path = web_dir

            # setup the unique ATMDIAG_WEBDIR_MODEL_VS_OBS output file
            env_file = '{0}/env_diags_atm.xml'.format(env['PP_CASE_PATH'])
            key = 'ATMDIAG_WEBDIR_{0}'.format(self._name)
            value = diag_path
            ##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(env['PP_CASE_PATH'], key, scomm.get_size(), scomm.get_rank() )
            web_file = '{0}/web_dirs/{1}.{2}'.format(
                env['PP_CASE_PATH'], key,
                datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S'))
            try:
                diagUtilsLib.write_web_file(web_file, 'atm', key, value)
            except:
                print(
                    'WARNING atm model_vs_model unable to write {0}={1} to {2}'
                    .format(key, value, web_file))

            print(
                '*******************************************************************************'
            )
            print(
                'Successfully completed generating atmosphere diagnostics model vs. model plots'
            )
            print(
                '*******************************************************************************'
            )
def main(options, main_comm, debugMsg):
    """setup the environment for running the diagnostics in parallel. 

    Calls 6 different diagnostics generation types:
    model vs. observation (optional BGC - ecosystem)
    model vs. control (optional BGC - ecosystem)
    model time-series (optional BGC - ecosystem)

    Arguments:
    options (object) - command line options
    main_comm (object) - MPI simple communicator object
    debugMsg (object) - vprinter object for printing debugging messages

    The env_diags_ocn.xml configuration file defines the way the diagnostics are generated. 
    See (website URL here...) for a complete desciption of the env_diags_ocn XML options.
    """

    # initialize the environment dictionary
    envDict = dict()

    # CASEROOT is given on the command line as required option --caseroot
    if main_comm.is_manager():
        caseroot = options.caseroot[0]
        debugMsg('caseroot = {0}'.format(caseroot), header=True, verbosity=2)

        debugMsg('calling initialize_main', header=True, verbosity=2)
        envDict = initialize_main(envDict, caseroot, debugMsg, options.standalone)

        debugMsg('calling check_ncl_nco', header=True, verbosity=2)
        diagUtilsLib.check_ncl_nco(envDict)

    # broadcast envDict to all tasks
    envDict = main_comm.partition(data=envDict, func=partition.Duplicate(), involved=True)
    sys.path.append(envDict['PATH'])
    main_comm.sync()

    # get list of diagnostics types to be created
    diag_list = list()
    num_of_diags = 0

    if main_comm.is_manager():
        diag_list, diag_dict = setup_diags(envDict)

        num_of_diags = len(diag_list)
        if num_of_diags == 0:
            print('No ocean diagnostics specified. Please check the {0}/env_diags_ocn.xml settings.'.format(envDict['PP_CASE_PATH']))
            sys.exit(1)

        print('User requested diagnostics:')
        for diag in diag_list:
            print('  {0}'.format(diag))

        try:
            os.makedirs(envDict['WORKDIR'])
        except OSError as exception:
            if exception.errno != errno.EEXIST:
                err_msg = 'ERROR: ocn_diags_generator.py problem accessing the working directory {0}'.format(envDict['WORKDIR'])
                raise OSError(err_msg)

        debugMsg('Ocean diagnostics - Creating main index.html page', header=True, verbosity=2)

        # define the templatePath
        templatePath = '{0}/diagnostics/diagnostics/ocn/Templates'.format(envDict['POSTPROCESS_PATH']) 

        templateLoader = jinja2.FileSystemLoader( searchpath=templatePath )
        templateEnv = jinja2.Environment( loader=templateLoader )
            
        template_file = 'ocean_diagnostics.tmpl'
        template = templateEnv.get_template( template_file )
            
        # get the current datatime string for the template
        now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')

        # set the template variables
        templateVars = { 'casename' : envDict['CASE'],
                         'tagname' : envDict['CESM_TAG'],
                         'diag_dict' : diag_dict,
                         'control_casename' : envDict['CNTRLCASE'],
                         'start_year' : envDict['YEAR0'],
                         'stop_year' : envDict['YEAR1'],
                         'control_start_year' : envDict['CNTRLYEAR0'],
                         'control_stop_year' : envDict['CNTRLYEAR1'],
                         'today': now,
                         'tseries_start_year' : envDict['TSERIES_YEAR0'],
                         'tseries_stop_year' : envDict['TSERIES_YEAR1']
                         }

        # write the main index.html page to the top working directory
        main_html = template.render( templateVars )
        with open( '{0}/index.html'.format(envDict['WORKDIR']), 'w') as index:
            index.write(main_html)

        debugMsg('Ocean diagnostics - Copying stylesheet', header=True, verbosity=2)
        shutil.copy2('{0}/Templates/diag_style.css'.format(envDict['POSTPROCESS_PATH']), '{0}/diag_style.css'.format(envDict['WORKDIR']))

        debugMsg('Ocean diagnostics - Copying logo files', header=True, verbosity=2)
        if not os.path.exists('{0}/logos'.format(envDict['WORKDIR'])):
            os.mkdir('{0}/logos'.format(envDict['WORKDIR']))

        for filename in glob.glob(os.path.join('{0}/Templates/logos'.format(envDict['POSTPROCESS_PATH']), '*.*')):
            shutil.copy(filename, '{0}/logos'.format(envDict['WORKDIR']))
 
        # setup the unique OCNDIAG_WEBDIR output file
        env_file = '{0}/env_diags_ocn.xml'.format(envDict['PP_CASE_PATH'])
        key = 'OCNDIAG_WEBDIR'
        value = envDict['WORKDIR']
        ##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(envDict['PP_CASE_PATH'], key, main_comm.get_size(), main_comm.get_rank() )
        web_file = '{0}/web_dirs/{1}.{2}'.format(envDict['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S'))
        try:
            diagUtilsLib.write_web_file(web_file, 'ocn', key, value)
        except:
            print('WARNING ocn_diags_generator unable to write {0}={1} to {2}'.format(key, value, web_file))

    main_comm.sync()

    # broadcast the diag_list to all tasks
    num_of_diags = main_comm.partition(num_of_diags, func=partition.Duplicate(), involved=True)
    diag_list = main_comm.partition(data=diag_list, func=partition.Duplicate(), involved=True)
    main_comm.sync()

    # initialize some variables for distributing diagnostics across the communicators
    diags_send = diag_list
    gmaster = main_comm.is_manager()
    gsize = main_comm.get_size()
    grank = main_comm.get_rank()
    local_diag_list = list()

    # divide the main communicator into sub_communicators to be passed to each diag class
    # split mpi comm world if the size of the communicator > 1 and the num_of_diags > 1
    if gsize > 1 and num_of_diags > 1:
        temp_color = (grank % num_of_diags)
        if (temp_color == num_of_diags):
            temp_color = temp_color - 1
        groups = list()
        for g in range(0,num_of_diags):
            groups.append(g)
        debugMsg('global_rank {0}, temp_color {1}, #of groups(diag types) {2}, groups {3}, diag_list {4}'.format(grank, temp_color, num_of_diags, groups, diag_list), header=True, verbosity=2)
        group = groups[temp_color]
        inter_comm, multi_comm = main_comm.divide(group)
        color = inter_comm.get_color()
        lsize = inter_comm.get_size()
        lrank = inter_comm.get_rank()
        lmaster = inter_comm.is_manager()
        debugMsg('color {0}, lsize {1}, lrank {2}, lmaster {3}'.format(color, lsize, lrank, lmaster), header=True, verbosity=2)

        # partition the diag_list between communicators
        DIAG_LIST_TAG = 10
        if lmaster:
            local_diag_list = multi_comm.partition(diag_list,func=partition.EqualStride(),involved=True)
            debugMsg('lrank = {0} local_diag_list = {1}'.format(lrank, local_diag_list), header=True, verbosity=2)
            for b in range(1, lsize):
                diags_send = inter_comm.ration(data=local_diag_list, tag=DIAG_LIST_TAG) 
                debugMsg('b = {0} diags_send = {1} lsize = {2}'.format(b, diags_send, lsize), header=True, verbosity=2)
        else:
            local_diag_list = inter_comm.ration(tag=DIAG_LIST_TAG)
        debugMsg('local_diag_list {0}',format(local_diag_list), header=True, verbosity=2)
    else:
        inter_comm = main_comm
        lmaster = main_comm.is_manager()
        lsize = main_comm.get_size()
        lrank = main_comm.get_rank()
        local_diag_list = diag_list

    inter_comm.sync()
    main_comm.sync()

    # loop through the local_diag_list 
    for requested_diag in local_diag_list:
        try:
            debugMsg('requested_diag {0}, lrank {1}, lsize {2}, lmaster {3}'.format(requested_diag, lrank, lsize, lmaster), header=True, verbosity=2)
            diag = ocn_diags_factory.oceanDiagnosticsFactory(requested_diag)

            # check the prerequisites for the diagnostics types
            debugMsg('Checking prerequisites for {0}'.format(diag.__class__.__name__), header=True, verbosity=2)
            
            skip_key = '{0}_SKIP'.format(requested_diag)
            if lmaster:
                try:
                    envDict = diag.check_prerequisites(envDict)
                except ocn_diags_bc.PrerequisitesError:
                    print("Problem with check_prerequisites for '{0}' skipping!".format(requested_diag))
                    envDict[skip_key] = True
                except RuntimeError as e:
                    # unrecoverable error, bail!
                    print(e)
                    envDict['unrecoverableErrorOnMaster'] = True

            inter_comm.sync()

            # broadcast the envDict
            envDict = inter_comm.partition(data=envDict, func=partition.Duplicate(), involved=True)

            if envDict.has_key('unrecoverableErrorOnMaster'):
                raise RuntimeError

            # run the diagnostics type on each inter_comm
            if not envDict.has_key(skip_key):
                # set the shell env using the values set in the XML and read into the envDict across all tasks
                cesmEnvLib.setXmlEnv(envDict)
                # run the diagnostics
                envDict = diag.run_diagnostics(envDict, inter_comm)

            inter_comm.sync()
            
        except ocn_diags_bc.RecoverableError as e:
            # catch all recoverable errors, print a message and continue.
            print(e)
            print("Skipped '{0}' and continuing!".format(requested_diag))
        except RuntimeError as e:
            # unrecoverable error, bail!
            print(e)
            return 1

    main_comm.sync()
Esempio n. 3
0
    def run_diagnostics(self, env, scomm):
        """ call the necessary plotting routines to generate diagnostics plots
        """
        super(modelVsModel, self).run_diagnostics(env, scomm)
        scomm.sync()

        # setup some global variables
        requested_plot_sets = list()
        local_requested_plots = list()
        local_html_list = list()

        # all the plot module XML vars start with 'set_'  need to strip that off
        for key, value in env.iteritems():
            if ("PLOT_" in key and value == 'True'):
                ##                if ("DIFF" in key or 'PLOT_REGIONS' in key):
                if ("DIFF" in key):
                    requested_plot_sets.append(key)
        scomm.sync()

        # partition requested plots to all tasks
        # first, create plotting classes and get the number of plots each will created
        requested_plots = {}
        set_sizes = {}
        for plot_set in requested_plot_sets:
            requested_plots.update(
                ice_diags_plot_factory.iceDiagnosticPlotFactory(plot_set, env))
        # partition based on the number of plots each set will create
        local_plot_list = scomm.partition(requested_plots.keys(),
                                          func=partition.EqualStride(),
                                          involved=True)

        timer = timekeeper.TimeKeeper()
        # loop over local plot lists - set env and then run plotting script
        #for plot_id,plot_class in local_plot_list.interitems():
        timer.start(str(scomm.get_rank()) + "ncl total time on task")
        for plot_set in local_plot_list:
            timer.start(str(scomm.get_rank()) + plot_set)
            plot_class = requested_plots[plot_set]
            # set all env variables (global and particular to this plot call
            plot_class.check_prerequisites(env)
            # Stringify the env dictionary
            for name, value in plot_class.plot_env.iteritems():
                plot_class.plot_env[name] = str(value)
            # call script to create plots
            for script in plot_class.ncl_scripts:
                diagUtilsLib.generate_ncl_plots(plot_class.plot_env, script)
            timer.stop(str(scomm.get_rank()) + plot_set)
        timer.stop(str(scomm.get_rank()) + "ncl total time on task")
        scomm.sync()
        print(timer.get_all_times())

        # set html files
        if scomm.is_manager():
            # Setup (local) web directories
            env['HTML_HOME'] = env['POSTPROCESS_PATH'] + '/ice_diag/web/'
            web_dir = env['WKDIR'] + '/yrs' + env['BEGYR_CONT'] + '-' + env[
                'ENDYR_CONT']
            if env['CLEANUP_FILES'].lower() in ['t', 'true'
                                                ] and os.path.exists(web_dir):
                shutil.rmtree(web_dir)
            if not os.path.exists(web_dir):
                os.mkdir(web_dir)
            if not os.path.exists(web_dir + '/contour'):
                os.mkdir(web_dir + '/contour')
            if not os.path.exists(web_dir + '/vector'):
                os.mkdir(web_dir + '/vector')
            if not os.path.exists(web_dir + '/line'):
                os.mkdir(web_dir + '/line')
            if not os.path.exists(web_dir + '/obs'):
                os.mkdir(web_dir + '/obs')

            # Move images tot the oppropriate directories
            plot_dir_map = {
                'icesat': 'obs',
                'ASPeCt': 'obs',
                'diff_con_': 'contour',
                'diff_vec_': 'vector',
                'line': 'line',
                'clim': 'line'
            }
            for key, dir in plot_dir_map.iteritems():
                glob_string = env['WKDIR'] + '/*' + key + '*.png'
                imgs = glob.glob(glob_string)
                if imgs > 0:
                    for img in imgs:
                        new_fn = web_dir + '/' + dir + '/' + os.path.basename(
                            img)
                        os.rename(img, new_fn)

            # Create/format the html files and copy txt and map files
            shutil.copy(env['HTML_HOME'] + '/ICESat.txt',
                        web_dir + '/obs/ICESat.txt')
            shutil.copy(env['HTML_HOME'] + '/ASPeCt.txt',
                        web_dir + '/obs/ASPeCt.txt')
            glob_string = env['HTML_HOME'] + '/maps/*'
            maps = glob.glob(glob_string)
            for map in maps:
                mn = os.path.basename(map)
                shutil.copy(map, web_dir + '/' + mn)

            print('DEBUG: model_vs_model web_dir = {0}'.format(web_dir))

            html_dir = env['HTML_HOME']
            create_ice_html.create_plotset_html(
                html_dir + '/index_diff_temp.html', web_dir + '/index.html',
                env)
            create_ice_html.create_plotset_html(
                html_dir + '/contour_diff.html', web_dir + '/contour.html',
                env)
            create_ice_html.create_plotset_html(
                html_dir + '/timeseries_diff.html',
                web_dir + '/timeseries.html', env)
            create_ice_html.create_plotset_html(
                html_dir + '/regional_diff.html', web_dir + '/regional.html',
                env)
            create_ice_html.create_plotset_html(html_dir + '/vector_diff.html',
                                                web_dir + '/vector.html', env)

            # setup the unique ICEDIAG_WEBDIR_MODEL_VS_MODEL output file
            env_file = '{0}/env_diags_ice.xml'.format(env['PP_CASE_PATH'])
            key = 'ICEDIAG_WEBDIR_{0}'.format(self._name)
            value = env['WKDIR']
            ##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(env['PP_CASE_PATH'], key, scomm.get_size(), scomm.get_rank() )
            web_file = '{0}/web_dirs/{1}.{2}'.format(
                env['PP_CASE_PATH'], key,
                datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S'))
            try:
                diagUtilsLib.write_web_file(web_file, 'ice', key, value)
            except:
                print(
                    'WARNING ice model_vs_model unable to write {0}={1} to {2}'
                    .format(key, value, web_file))

            print(
                '*******************************************************************************'
            )
            print(
                'Successfully completed generating ice diagnostics model vs. model plots'
            )
            print(
                '*******************************************************************************'
            )
Esempio n. 4
0
    def run_diagnostics(self, env, scomm):
        """ call the necessary plotting routines to generate diagnostics plots
        """
        super(modelVsModel, self).run_diagnostics(env, scomm)
        scomm.sync()

        # setup some global variables
        requested_plot_sets = list()
        local_requested_plots = list()
        local_html_list = list()

        # all the plot module XML vars start with 'set_'  need to strip that off
        for key, value in env.iteritems():
            if   ("wset_"in key and (value == 'True' or env['all_waccm_sets'] == 'True')):
                requested_plot_sets.append(key)
            elif ("cset_"in key and (value == 'True' or env['all_chem_sets'] == 'True')):
                requested_plot_sets.append(key)
            elif ("set_" in key and (value == 'True' or env['all_sets'] == 'True')):
                if ("wset_" not in key and "cset_" not in key):
                    requested_plot_sets.append(key)
        
        scomm.sync()

        # partition requested plots to all tasks
        # first, create plotting classes and get the number of plots each will created 
        requested_plots = {}
        set_sizes = {}
        plots_weights = []
        for plot_set in requested_plot_sets:
            requested_plots.update(atm_diags_plot_factory.atmosphereDiagnosticPlotFactory(plot_set,env))
        for plot_id,plot_class in requested_plots.iteritems(): 
            if hasattr(plot_class, 'weight'):
                factor = plot_class.weight
            else:
                factor = 1
            plots_weights.append((plot_id,len(plot_class.expectedPlots)*factor))
        # partition based on the number of plots each set will create
        local_plot_list = scomm.partition(plots_weights, func=partition.WeightBalanced(), involved=True)  

        timer = timekeeper.TimeKeeper()
        # loop over local plot lists - set env and then run plotting script         
        #for plot_id,plot_class in local_plot_list.interitems():
        timer.start(str(scomm.get_rank())+"ncl total time on task")
        for plot_set in local_plot_list:
            timer.start(str(scomm.get_rank())+plot_set)
            plot_class = requested_plots[plot_set]
            # set all env variables (global and particular to this plot call
            plot_class.check_prerequisites(env)
            # Stringify the env dictionary
            for name,value in plot_class.plot_env.iteritems():
                plot_class.plot_env[name] = str(value)
            # call script to create plots
            for script in plot_class.ncl_scripts:
                diagUtilsLib.generate_ncl_plots(plot_class.plot_env,script)
                plot_class.plot_env['NCDF_MODE'] = 'write'
                plot_class.plot_env['VAR_MODE'] = 'write'
            timer.stop(str(scomm.get_rank())+plot_set) 
        timer.stop(str(scomm.get_rank())+"ncl total time on task")
        scomm.sync() 
        print(timer.get_all_times())
        w = 0
        for p in plots_weights:
            if p[0] in local_plot_list:
                w = w + p[1]
        print(str(scomm.get_rank())+' weight:'+str(w))

        # set html files
        if scomm.is_manager():
            env['HTML_HOME'] = env['DIAG_HOME']+'/html/model1-model2/'
            # Get web dir name and create it if it does not exist
            #web_dir = '{0}/{1}-{2}'.format(env['test_path_diag'], env['test_casename'], env['cntl_casename'])
            web_dir = env['test_path_diag']
            #if not os.path.exists(web_dir):
            #    os.makedirs(web_dir)
            # Copy over some files needed by web pages
            if not os.path.exists(web_dir+'/images'):
                os.mkdir(web_dir+'/images')
            diag_imgs = glob.glob(env['DIAG_HOME']+'/html/images/*')
            for img in diag_imgs:
                shutil.copy(img,web_dir+'/images/')
          
            # Create set dirs, copy plots to set dir, and create html file for set 
            requested_plot_sets.append('sets') # Add 'sets' to create top level html files
            glob_set = list()
            for plot_set in requested_plot_sets:
                 if 'set_5' == plot_set or 'set_6' == plot_set:
                     glob_set.append(plot_set.replace('_',''))
                     plot_set = 'set5_6'
                 elif 'cset_1' == plot_set:
                     glob_set.append('table_soa')
                     glob_set.append('table_chem')
                     plot_set = plot_set.replace('_','')     
                 elif 'set_1' == plot_set:
                     glob_set.append('table_GLBL')
                     glob_set.append('table_NEXT')
                     glob_set.append('table_SEXT')
                     glob_set.append('table_TROP')
                     plot_set = plot_set.replace('_','')
                 elif 'sets' == plot_set:
                     set_dir = web_dir + '/' 
                 else:
                     glob_set.append(plot_set.replace('_',''))
                     plot_set = plot_set.replace('_','')

                 if 'sets' not in plot_set: #'sets' is top level, don't create directory or copy images files
                     set_dir = web_dir + '/' + plot_set
                     # Create the plot set web directory
                     if not os.path.exists(set_dir):
                         os.makedirs(set_dir) 
                     # Copy plots into the correct web dir
                     for gs in glob_set:
                         glob_string = env['test_path_diag']+'/'+gs+'*.*'
                         imgs = glob.glob(glob_string) 
                         if imgs > 0:
                             for img in imgs:
                                 new_fn = set_dir + '/' + os.path.basename(img)
                                 os.rename(img,new_fn)

                 # Copy/Process html files
                 if 'sets' in plot_set:
                     orig_html = env['HTML_HOME']+'/'+plot_set 
                 else:
                     orig_html = env['HTML_HOME']+'/'+plot_set+'/'+plot_set 
                 create_atm_html.create_plotset_html(orig_html,set_dir,plot_set,env,'model_vs_model')

            # Remove any plotvar netcdf files that exists in the diag directory
            if env['save_ncdfs'] == 'False':
                cesmEnvLib.purge(env['test_path_diag'], '.*\.nc')
                cesmEnvLib.purge(env['test_path_diag'], '/station_ids')
            
            # move all the plots to the diag_path with the years appended to the path
            endYr1 = (int(env['test_first_yr']) + int(env['test_nyrs'])) - 1 
            endYr2 = (int(env['cntl_first_yr']) + int(env['cntl_nyrs'])) - 1 
            diag_path = '{0}/diag/{1}.{2}_{3}-{4}.{5}_{6}'.format(env['OUTPUT_ROOT_PATH'], 
                          env['test_casename'], env['test_first_yr'], str(endYr1),
                          env['cntl_casename'], env['cntl_first_yr'], str(endYr2))
            move_files = True

            try:
                os.makedirs(diag_path)
            except OSError as exception:
                if exception.errno != errno.EEXIST:
                    err_msg = 'ERROR: {0} problem accessing directory {1}'.format(self.__class__.__name__, diag_path)
                    raise OSError(err_msg)
                    move_files = False

                elif env['CLEANUP_FILES'].lower() in ['t','true']:
                    # delete all the files in the diag_path directory
                    for root, dirs, files in os.walk(diag_path):
                        for f in files:
                            os.unlink(os.path.join(root, f))
                        for d in dirs:
                            shutil.rmtree(os.path.join(root, d))

                elif env['CLEANUP_FILES'].lower() in ['f','false']:
                    print('WARNING: {0} exists and is not empty and ATMDIAG_CLEANUP_FILES = False. Leaving new diagnostics files in {1}'.format(diag_path, web_dir))
                    diag_path = web_dir
                    move_files = False

            print('DEBUG: model vs. model web_dir = {0}'.format(web_dir))
            print('DEBUG: model vs. model diag_path = {0}'.format(diag_path))

            # move the files to the new diag_path 
            if move_files:
                try:
                    print('DEBUG: model_vs_model renaming web files')
                    os.rename(web_dir, diag_path)
                except OSError as e:
                    print ('WARNING: Error renaming %s to %s: %s' % (web_dir, diag_path, e))
                    diag_path = web_dir

            # setup the unique ATMDIAG_WEBDIR_MODEL_VS_OBS output file
            env_file = '{0}/env_diags_atm.xml'.format(env['PP_CASE_PATH'])
            key = 'ATMDIAG_WEBDIR_{0}'.format(self._name)
            value = diag_path
            web_file = '{0}/web_dirs/{1}.{2}'.format(env['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S'))
            try:
                diagUtilsLib.write_web_file(web_file, 'atm', key, value)
            except:
                print('WARNING atm model_vs_model unable to write {0}={1} to {2}'.format(key, value, web_file))

            print('*******************************************************************************')
            print('Successfully completed generating atmosphere diagnostics model vs. model plots')
            print('*******************************************************************************')
Esempio n. 5
0
    def run_diagnostics(self, env, scomm):
        """ call the necessary plotting routines to generate diagnostics plots
        """
        super(modelVsModel, self).run_diagnostics(env, scomm)
        scomm.sync()

        # setup some global variables
        requested_plot_sets = list()
        local_requested_plots = list()
        local_html_list = list()

        # all the plot module XML vars start with 'set_'
        for key, value in env.iteritems():
            if ("set_" in key and value == 'True'):
                requested_plot_sets.append(key)
        scomm.sync()

        if scomm.is_manager():
            print('DEBUG model_vs_model requested_plot_sets = {0}'.format(
                requested_plot_sets))

        # partition requested plots to all tasks
        # first, create plotting classes and get the number of plots each will created
        requested_plots = {}
        set_sizes = {}
        #plots_weights = []
        for plot_set in requested_plot_sets:
            requested_plots.update(
                lnd_diags_plot_factory.LandDiagnosticPlotFactory(
                    plot_set, env))

        #for plot_id,plot_class in requested_plots.iteritems():
        #    if hasattr(plot_class, 'weight'):
        #        factor = plot_class.weight
        #    else:
        #        factor = 1
        #    plots_weights.append((plot_id,len(plot_class.expectedPlots)*factor))
        # partition based on the number of plots each set will create
        #local_plot_list = scomm.partition(plots_weights, func=partition.WeightBalanced(), involved=True)

        if scomm.is_manager():
            print('DEBUG model_vs_model requested_plots.keys = {0}'.format(
                requested_plots.keys()))

        local_plot_list = scomm.partition(requested_plots.keys(),
                                          func=partition.EqualStride(),
                                          involved=True)
        scomm.sync()

        timer = timekeeper.TimeKeeper()

        # loop over local plot lists - set env and then run plotting script
        timer.start(str(scomm.get_rank()) + "ncl total time on task")
        for plot_set in local_plot_list:

            timer.start(str(scomm.get_rank()) + plot_set)
            plot_class = requested_plots[plot_set]

            # set all env variables (global and particular to this plot call
            print(
                'model vs. model - Checking prerequisite for {0} on rank {1}'.
                format(plot_class.__class__.__name__, scomm.get_rank()))
            plot_class.check_prerequisites(env)

            # Stringify the env dictionary
            for name, value in plot_class.plot_env.iteritems():
                plot_class.plot_env[name] = str(value)

            # call script to create plots
            for script in plot_class.ncl_scripts:
                print(
                    'model vs. model - Generating plots for {0} on rank {1} with script {2}'
                    .format(plot_class.__class__.__name__, scomm.get_rank(),
                            script))
                diagUtilsLib.generate_ncl_plots(plot_class.plot_env, script)

            timer.stop(str(scomm.get_rank()) + plot_set)
        timer.stop(str(scomm.get_rank()) + "ncl total time on task")

        scomm.sync()
        print(timer.get_all_times())
        #w = 0
        #for p in plots_weights:
        #    if p[0] in local_plot_list:
        #        w = w + p[1]
        #print(str(scomm.get_rank())+' weight:'+str(w))

        # set html files
        if scomm.is_manager():

            # Create web dirs and move images/tables to that web dir
            for n in ('1', '2', '3', '4', '5', '6', '7', '8', '9'):
                web_dir = env['WKDIR']
                set_dir = web_dir + '/set' + n
                # Create the plot set web directory
                if not os.path.exists(set_dir):
                    os.makedirs(set_dir)
                # Copy plots into the correct web dir
                glob_string = web_dir + '/set' + n + '_*'
                imgs = glob.glob(glob_string)
                if len(imgs) > 0:
                    for img in imgs:
                        new_fn = set_dir + '/' + os.path.basename(img)
                        os.rename(img, new_fn)
                # Copy the set1Diff and set1Anom plots to set_1 web dir
                if n == '1':
                    glob_string = web_dir + '/set1Diff' + '_*'
                    imgs = glob.glob(glob_string)
                    if len(imgs) > 0:
                        for img in imgs:
                            new_fn = set_dir + '/' + os.path.basename(img)
                            os.rename(img, new_fn)

                    glob_string = web_dir + '/set1Anom' + '_*'
                    imgs = glob.glob(glob_string)
                    if len(imgs) > 0:
                        for img in imgs:
                            new_fn = set_dir + '/' + os.path.basename(img)
                            os.rename(img, new_fn)

            env['WEB_DIR'] = web_dir
            shutil.copy2(
                env['POSTPROCESS_PATH'] + '/lnd_diag/inputFiles/' +
                env['VAR_MASTER'], web_dir + '/variable_master.ncl')

            if n == '9':
                web_script = env[
                    'POSTPROCESS_PATH'] + '/lnd_diag/shared/lnd_statTable.pl'
                rc2, err_msg = cesmEnvLib.checkFile(web_script, 'read')
                if rc2:
                    try:
                        subprocess.check_call(web_script)
                    except subprocess.CalledProcessError as e:
                        print('WARNING: {0} error executing command:'.format(
                            web_script))
                        print('    {0}'.format(e.cmd))
                        print('    rc = {0}'.format(e.returncode))
                else:
                    print('{0}... {1} file not found'.format(
                        err_msg, web_script))

                # copy the set9_statTable.html to the set9 subdir
                set9_statTable = web_dir + '/set9_statTable.html'
                rc2, err_msg = cesmEnvLib.checkFile(set9_statTable, 'read')
                if rc2:
                    new_fn = web_dir + '/set9/' + os.path.basename(
                        set9_statTable)
                    try:
                        os.rename(set9_statTable, new_fn)
                    except os.OSError as e:
                        print('WARNING: rename error for file {0}'.format(
                            set9_statTable))
                        print('    rc = {0}'.format(e.returncode))
                else:
                    print('{0}... {1} file not found'.format(
                        err_msg, set9_statTable))

            web_script_1 = env[
                'POSTPROCESS_PATH'] + '/lnd_diag/shared/lnd_create_webpage.pl'
            web_script_2 = env[
                'POSTPROCESS_PATH'] + '/lnd_diag/shared/lnd_lookupTable.pl'

            print('Creating Web Pages')

            # set the shell environment
            cesmEnvLib.setXmlEnv(env)

            # lnd_create_webpage.pl call
            rc1, err_msg = cesmEnvLib.checkFile(web_script_1, 'read')
            if rc1:
                try:
                    subprocess.check_call(web_script_1)
                except subprocess.CalledProcessError as e:
                    print('WARNING: {0} error executing command:'.format(
                        web_script_1))
                    print('    {0}'.format(e.cmd))
                    print('    rc = {0}'.format(e.returncode))
            else:
                print('{0}... {1} file not found'.format(
                    err_msg, web_script_1))

            # lnd_lookupTable.pl call
            rc2, err_msg = cesmEnvLib.checkFile(web_script_2, 'read')
            if rc2:
                try:
                    subprocess.check_call(web_script_2)
                except subprocess.CalledProcessError as e:
                    print('WARNING: {0} error executing command:'.format(
                        web_script_2))
                    print('    {0}'.format(e.cmd))
                    print('    rc = {0}'.format(e.returncode))
            else:
                print('{0}... {1} file not found'.format(
                    err_msg, web_script_2))

            # move all the plots to the diag_path with the years appended to the path
            endYr1 = (int(env['clim_first_yr_1']) +
                      int(env['clim_num_yrs_1'])) - 1
            endYr2 = (int(env['clim_first_yr_2']) +
                      int(env['clim_num_yrs_2'])) - 1
            diag_path = '{0}/diag/{1}.{2}_{3}-{4}.{5}_{6}'.format(
                env['OUTPUT_ROOT_PATH'], env['caseid_1'],
                env['clim_first_yr_1'], str(endYr1), env['caseid_2'],
                env['clim_first_yr_2'], str(endYr2))
            move_files = True

            try:
                os.makedirs(diag_path)
            except OSError as exception:
                if exception.errno != errno.EEXIST:
                    err_msg = 'ERROR: {0} problem accessing directory {1}'.format(
                        self.__class__.__name__, diag_path)
                    raise OSError(err_msg)
                    move_files = False

                elif env['CLEANUP_FILES'].lower() in ['t', 'true']:
                    # delete all the files in the diag_path directory
                    for root, dirs, files in os.walk('diag_path'):
                        for f in files:
                            os.unlink(os.path.join(root, f))
                        for d in dirs:
                            shutil.rmtree(os.path.join(root, d))

                elif env['CLEANUP_FILES'].lower() in ['f', 'false']:
                    print(
                        'WARNING: {0} exists and is not empty and LNDDIAG_CLEANUP_FILES = False. Leaving new diagnostics files in {1}'
                        .format(diag_path, web_dir))
                    diag_path = web_dir
                    move_files = False

            # move the files to the new diag_path
            if move_files:
                try:
                    print('DEBUG: model_vs_model renaming web files')
                    os.rename(web_dir, diag_path)
                except OSError as e:
                    print('WARNING: Error renaming %s to %s: %s' %
                          (web_dir, diag_path, e))
                    diag_path = web_dir

            print('DEBUG: model vs. model web_dir = {0}'.format(web_dir))
            print('DEBUG: model vs. model diag_path = {0}'.format(diag_path))

            # setup the unique LNDDIAG_WEBDIR_MODEL_VS_MODEL output file
            env_file = '{0}/env_diags_lnd.xml'.format(env['PP_CASE_PATH'])
            key = 'LNDDIAG_WEBDIR_{0}'.format(self._name)
            value = diag_path
            ##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(env['PP_CASE_PATH'], key, scomm.get_size(), scomm.get_rank() )
            web_file = '{0}/web_dirs/{1}.{2}'.format(
                env['PP_CASE_PATH'], key,
                datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S'))
            try:
                diagUtilsLib.write_web_file(web_file, 'lnd', key, value)
            except:
                print(
                    'WARNING lnd model_vs_model unable to write {0}={1} to {2}'
                    .format(key, value, web_file))

            print(
                '*******************************************************************************'
            )
            print(
                'Successfully completed generating land diagnostics model vs. model plots'
            )
            print(
                '*******************************************************************************'
            )
Esempio n. 6
0
    def run_diagnostics(self, env, scomm):
        """ call the necessary plotting routines to generate diagnostics plots
        """
        super(modelVsObs, self).run_diagnostics(env, scomm)
        scomm.sync()

        # setup some global variables
        requested_plot_sets = list()
        local_requested_plots = list()
        local_html_list = list()

        # all the plot module XML vars start with 'set_' 
        for key, value in env.iteritems():
            if ("set_" in key and value == 'True'):
                requested_plot_sets.append(key)
        scomm.sync()

        if scomm.is_manager():
            print('DEBUG model_vs_obs requested_plot_sets = {0}'.format(requested_plot_sets))

        # partition requested plots to all tasks
        # first, create plotting classes and get the number of plots each will created 
        requested_plots = {}
        set_sizes = {}
        #plots_weights = []
        for plot_set in requested_plot_sets:
            requested_plots.update(lnd_diags_plot_factory.LandDiagnosticPlotFactory(plot_set,env))

        #for plot_id,plot_class in requested_plots.iteritems():
        #    if hasattr(plot_class, 'weight'):
        #        factor = plot_class.weight
        #    else:
        #        factor = 1
        #    plots_weights.append((plot_id,len(plot_class.expectedPlots)*factor))
        # partition based on the number of plots each set will create
        #local_plot_list = scomm.partition(plots_weights, func=partition.WeightBalanced(), involved=True)  

        local_plot_list = scomm.partition(requested_plots.keys(), func=partition.EqualStride(), involved=True)
        scomm.sync()

        timer = timekeeper.TimeKeeper()
        # loop over local plot lists - set env and then run plotting script         
        timer.start(str(scomm.get_rank())+"ncl total time on task")

        for plot_set in local_plot_list:
            timer.start(str(scomm.get_rank())+plot_set)
            plot_class = requested_plots[plot_set]

            print('DEBUG model vs. obs - Checking prerequisite for {0} on rank {1}'.format(plot_class.__class__.__name__, scomm.get_rank()))
            plot_class.check_prerequisites(env)

            # Stringify the env dictionary
            for name, value in plot_class.plot_env.iteritems():
                plot_class.plot_env[name] = str(value)

            # call script to create plots
            for script in plot_class.ncl_scripts:
                print('DEBUG model vs. obs - Generating plots for {0} on rank {1} with script {2}'.format(plot_class.__class__.__name__, scomm.get_rank(),script))
                diagUtilsLib.generate_ncl_plots(plot_class.plot_env, script)

            timer.stop(str(scomm.get_rank())+plot_set)
        timer.stop(str(scomm.get_rank())+"ncl total time on task") 

        scomm.sync()
        print(timer.get_all_times())
        #w = 0
        #for p in plots_weights:
        #    if p[0] in local_plot_list:
        #        w = w + p[1]
        #print(str(scomm.get_rank())+' weight:'+str(w))

        # set html files
        if scomm.is_manager():

            # Create web dirs and move images/tables to that web dir
            for n in ('1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12'):
                web_dir = env['WKDIR'] 
                set_dir = web_dir + '/set' + n
                # Create the plot set web directory
                if not os.path.exists(set_dir):
                    os.makedirs(set_dir)
                # Copy plots into the correct web dir
                glob_string = web_dir+'/set'+n+'_*'
                imgs = glob.glob(glob_string)
                if len(imgs) > 0:
                    for img in imgs:
                        new_fn = set_dir + '/' + os.path.basename(img)
                        os.rename(img,new_fn)
            env['WEB_DIR'] = web_dir
            shutil.copy2(env['POSTPROCESS_PATH']+'/lnd_diag/inputFiles/'+env['VAR_MASTER'],web_dir+'/variable_master.ncl')
            web_script_1 = env['POSTPROCESS_PATH']+'/lnd_diag/shared/lnd_create_webpage.pl'
            web_script_2 = env['POSTPROCESS_PATH']+'/lnd_diag/shared/lnd_lookupTable.pl'

            print('Creating Web Pages')

            # set the shell environment
            cesmEnvLib.setXmlEnv(env)

            # lnd_create_webpage.pl call
            rc1, err_msg = cesmEnvLib.checkFile(web_script_1,'read')
            if rc1:
                try:
                    subprocess.check_call(web_script_1)
                except subprocess.CalledProcessError as e:
                    print('WARNING: {0} error executing command:'.format(web_script_1))
                    print('    {0}'.format(e.cmd))
                    print('    rc = {0}'.format(e.returncode))
            else:
                print('{0}... {1} file not found'.format(err_msg,web_script_1))

            # lnd_lookupTable.pl call          
            rc2, err_msg = cesmEnvLib.checkFile(web_script_2,'read')
            if rc2:
                try:
                    subprocess.check_call(web_script_2)
                except subprocess.CalledProcessError as e:
                    print('WARNING: {0} error executing command:'.format(web_script_2))
                    print('    {0}'.format(e.cmd))
                    print('    rc = {0}'.format(e.returncode))
            else:
                print('{0}... {1} file not found'.format(err_msg,web_script_2))

            # move all the plots to the diag_path with the years appended to the path
            endYr = (int(env['clim_first_yr_1']) + int(env['clim_num_yrs_1'])) - 1  
            diag_path = '{0}/diag/{1}-obs.{2}_{3}'.format(env['OUTPUT_ROOT_PATH'], env['caseid_1'],
                                                      env['clim_first_yr_1'], str(endYr))
            move_files = True
            try:
                os.makedirs(diag_path)
            except OSError as exception:
                if exception.errno != errno.EEXIST:
                    err_msg = 'ERROR: {0} problem accessing directory {1}'.format(self.__class__.__name__, diag_path)
                    raise OSError(err_msg)
                    move_files = False

                elif env['CLEANUP_FILES'].lower() in ['t','true']:
                    # delete all the files in the diag_path directory
                    for root, dirs, files in os.walk(diag_path):
                        for f in files:
                            os.unlink(os.path.join(root, f))
                        for d in dirs:
                            shutil.rmtree(os.path.join(root, d))

                elif env['CLEANUP_FILES'].lower() in ['f','false']:
                    print('WARNING: {0} exists and is not empty and LNDDIAG_CLEANUP_FILES = False. Leaving new diagnostics files in {1}'.format(diag_path, web_dir))
                    diag_path = web_dir
                    move_files = False

            print('DEBUG: model vs. obs web_dir = {0}'.format(web_dir))
            print('DEBUG: model vs. obs diag_path = {0}'.format(diag_path))

            # move the files to the new diag_path 
            if move_files:
                try:
                    print('DEBUG: model_vs_obs renaming web files')
                    os.rename(web_dir, diag_path)
                except OSError as e:
                    print ('WARNING: Error renaming %s to %s: %s' % (web_dir, diag_path, e))
                    diag_path = web_dir

            # setup the LNDDIAG_WEBDIR_MODEL_VS_OBS output file
            env_file = '{0}/env_diags_lnd.xml'.format(env['PP_CASE_PATH'])
            key = 'LNDDIAG_WEBDIR_{0}'.format(self._name)
            value = diag_path
            web_file = '{0}/web_dirs/{1}.{2}'.format(env['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S'))
            try:
                diagUtilsLib.write_web_file(web_file, 'lnd', key, value)
            except:
                print('WARNING lnd model_vs_obs unable to write {0}={1} to {2}'.format(key, value, web_file))

            print('*******************************************************************************')
            print('Successfully completed generating land diagnostics model vs. observation plots')
            print('*******************************************************************************')
Esempio n. 7
0
    def run_diagnostics(self, env, scomm):
        """ call the necessary plotting routines to generate diagnostics plots
        """
        super(modelVsModel, self).run_diagnostics(env, scomm)
        scomm.sync()

        # setup some global variables
        requested_plot_sets = list()
        local_requested_plots = list()
        local_html_list = list()

        # all the plot module XML vars start with 'set_'  need to strip that off
        for key, value in env.iteritems():
            if   ("PLOT_"in key and value == 'True'):
##                if ("DIFF" in key or 'PLOT_REGIONS' in key):
                if ("DIFF" in key):
                    requested_plot_sets.append(key)        
        scomm.sync()

        # partition requested plots to all tasks
        # first, create plotting classes and get the number of plots each will created 
        requested_plots = {}
        set_sizes = {}
        for plot_set in requested_plot_sets:
            requested_plots.update(ice_diags_plot_factory.iceDiagnosticPlotFactory(plot_set,env))
        # partition based on the number of plots each set will create
        local_plot_list = scomm.partition(requested_plots.keys(), func=partition.EqualStride(), involved=True)  

        timer = timekeeper.TimeKeeper()
        # loop over local plot lists - set env and then run plotting script         
        #for plot_id,plot_class in local_plot_list.interitems():
        timer.start(str(scomm.get_rank())+"ncl total time on task")
        for plot_set in local_plot_list:
            timer.start(str(scomm.get_rank())+plot_set)
            plot_class = requested_plots[plot_set]
            # set all env variables (global and particular to this plot call
            plot_class.check_prerequisites(env)
            # Stringify the env dictionary
            for name,value in plot_class.plot_env.iteritems():
                plot_class.plot_env[name] = str(value)
            # call script to create plots
            for script in plot_class.ncl_scripts:
                diagUtilsLib.generate_ncl_plots(plot_class.plot_env,script)
            timer.stop(str(scomm.get_rank())+plot_set) 
        timer.stop(str(scomm.get_rank())+"ncl total time on task")
        scomm.sync() 
        print(timer.get_all_times())

        # set html files
        if scomm.is_manager():
            # Setup (local) web directories
            env['HTML_HOME'] = env['POSTPROCESS_PATH']+'/ice_diag/web/'
            web_dir = env['WKDIR']+'/yrs'+env['BEGYR_CONT']+'-'+env['ENDYR_CONT']
            if env['CLEANUP_FILES'].lower() in ['t','true'] and os.path.exists(web_dir):
                shutil.rmtree(web_dir)
            if not os.path.exists(web_dir):
                os.mkdir(web_dir)
            if not os.path.exists(web_dir+'/contour'):
                os.mkdir(web_dir+'/contour')
            if not os.path.exists(web_dir+'/vector'):
                os.mkdir(web_dir+'/vector')
            if not os.path.exists(web_dir+'/line'):
                os.mkdir(web_dir+'/line')
            if not os.path.exists(web_dir+'/obs'):
                os.mkdir(web_dir+'/obs')

            # Move images tot the oppropriate directories
            plot_dir_map = {'icesat':'obs', 'ASPeCt':'obs', 'diff_con_':'contour', 'diff_vec_':'vector', 'line':'line', 'clim':'line'}
            for key,dir in plot_dir_map.iteritems():
                glob_string = env['WKDIR']+'/*'+key+'*.png'
                imgs = glob.glob(glob_string)
                if imgs > 0:
                    for img in imgs:
                        new_fn = web_dir + '/' + dir + '/' + os.path.basename(img)
                        os.rename(img,new_fn)

            # Create/format the html files and copy txt and map files
            shutil.copy(env['HTML_HOME']+'/ICESat.txt', web_dir+'/obs/ICESat.txt')
            shutil.copy(env['HTML_HOME']+'/ASPeCt.txt', web_dir+'/obs/ASPeCt.txt')
            glob_string = env['HTML_HOME']+'/maps/*'
            maps = glob.glob(glob_string)
            for map in maps:
                mn = os.path.basename(map)
                shutil.copy(map,web_dir+'/'+mn)

            print('DEBUG: model_vs_model web_dir = {0}'.format(web_dir))

            html_dir = env['HTML_HOME']
            var_name_type_cont = env['VAR_NAME_TYPE_CONT']
            var_name_type_diff = env['VAR_NAME_TYPE_DIFF']
            create_ice_html.create_plotset_html(html_dir+'/index_diff_temp.html',web_dir+'/index.html',env)
            if 'SIMIP' in var_name_type_cont or 'SIMIP' in var_name_type_diff:
                create_ice_html.create_plotset_html(html_dir+'/contour_diff_simip.html',web_dir+'/contour.html',env)
            else:
                create_ice_html.create_plotset_html(html_dir+'/contour_diff.html',web_dir+'/contour.html',env)

            create_ice_html.create_plotset_html(html_dir+'/timeseries_diff.html',web_dir+'/timeseries.html',env)
            create_ice_html.create_plotset_html(html_dir+'/regional_diff.html',web_dir+'/regional.html',env)
            create_ice_html.create_plotset_html(html_dir+'/vector_diff.html',web_dir+'/vector.html',env)

            # setup the unique ICEDIAG_WEBDIR_MODEL_VS_MODEL output file
            env_file = '{0}/env_diags_ice.xml'.format(env['PP_CASE_PATH'])
            key = 'ICEDIAG_WEBDIR_{0}'.format(self._name)
            value = env['WKDIR']
            ##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(env['PP_CASE_PATH'], key, scomm.get_size(), scomm.get_rank() )
            web_file = '{0}/web_dirs/{1}.{2}'.format(env['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S'))
            try:
                diagUtilsLib.write_web_file(web_file, 'ice', key, value)
            except:
                print('WARNING ice model_vs_model unable to write {0}={1} to {2}'.format(key, value, web_file))

            print('*******************************************************************************')
            print('Successfully completed generating ice diagnostics model vs. model plots')
            print('*******************************************************************************')
def main(options, main_comm, debugMsg):
    """setup the environment for running the diagnostics in parallel. 

    Calls 6 different diagnostics generation types:
    model vs. observation (optional BGC - ecosystem)
    model vs. control (optional BGC - ecosystem)
    model time-series (optional BGC - ecosystem)

    Arguments:
    options (object) - command line options
    main_comm (object) - MPI simple communicator object
    debugMsg (object) - vprinter object for printing debugging messages

    The env_diags_ocn.xml configuration file defines the way the diagnostics are generated. 
    See (website URL here...) for a complete desciption of the env_diags_ocn XML options.
    """

    # initialize the environment dictionary
    envDict = dict()

    # CASEROOT is given on the command line as required option --caseroot
    if main_comm.is_manager():
        caseroot = options.caseroot[0]
        debugMsg('caseroot = {0}'.format(caseroot), header=True, verbosity=2)

        debugMsg('calling initialize_main', header=True, verbosity=2)
        envDict = initialize_main(envDict, caseroot, debugMsg, options.standalone)

        debugMsg('calling check_ncl_nco', header=True, verbosity=2)
        diagUtilsLib.check_ncl_nco(envDict)

    # broadcast envDict to all tasks
    envDict = main_comm.partition(data=envDict, func=partition.Duplicate(), involved=True)
    sys.path.append(envDict['PATH'])
    main_comm.sync()

    # get list of diagnostics types to be created
    diag_list = list()
    num_of_diags = 0

    if main_comm.is_manager():
        diag_list, diag_dict = setup_diags(envDict)

        num_of_diags = len(diag_list)
        if num_of_diags == 0:
            print('No ocean diagnostics specified. Please check the {0}/env_diags_ocn.xml settings.'.format(envDict['PP_CASE_PATH']))
            sys.exit(1)

        print('User requested diagnostics:')
        for diag in diag_list:
            print('  {0}'.format(diag))

        try:
            os.makedirs(envDict['WORKDIR'])
        except OSError as exception:
            if exception.errno != errno.EEXIST:
                err_msg = 'ERROR: ocn_diags_generator.py problem accessing the working directory {0}'.format(envDict['WORKDIR'])
                raise OSError(err_msg)

        debugMsg('Ocean diagnostics - Creating main index.html page', header=True, verbosity=2)

        # define the templatePath
        templatePath = '{0}/diagnostics/diagnostics/ocn/Templates'.format(envDict['POSTPROCESS_PATH']) 

        templateLoader = jinja2.FileSystemLoader( searchpath=templatePath )
        templateEnv = jinja2.Environment( loader=templateLoader )
            
        template_file = 'ocean_diagnostics.tmpl'
        template = templateEnv.get_template( template_file )
            
        # get the current datatime string for the template
        now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')

        # set the template variables
        templateVars = { 'casename' : envDict['CASE'],
                         'tagname' : envDict['CESM_TAG'],
                         'username' : envDict['USER_NAME'],
                         'diag_dict' : diag_dict,
                         'control_casename' : envDict['CNTRLCASE'],
                         'start_year' : envDict['YEAR0'],
                         'stop_year' : envDict['YEAR1'],
                         'control_start_year' : envDict['CNTRLYEAR0'],
                         'control_stop_year' : envDict['CNTRLYEAR1'],
                         'today': now,
                         'tseries_start_year' : envDict['TSERIES_YEAR0'],
                         'tseries_stop_year' : envDict['TSERIES_YEAR1']
                         }

        # write the main index.html page to the top working directory
        main_html = template.render( templateVars )
        with open( '{0}/index.html'.format(envDict['WORKDIR']), 'w') as index:
            index.write(main_html)

        debugMsg('Ocean diagnostics - Copying stylesheet', header=True, verbosity=2)
        shutil.copy2('{0}/Templates/diag_style.css'.format(envDict['POSTPROCESS_PATH']), '{0}/diag_style.css'.format(envDict['WORKDIR']))

        debugMsg('Ocean diagnostics - Copying logo files', header=True, verbosity=2)
        if not os.path.exists('{0}/logos'.format(envDict['WORKDIR'])):
            os.mkdir('{0}/logos'.format(envDict['WORKDIR']))

        for filename in glob.glob(os.path.join('{0}/Templates/logos'.format(envDict['POSTPROCESS_PATH']), '*.*')):
            shutil.copy(filename, '{0}/logos'.format(envDict['WORKDIR']))
 
        # setup the unique OCNDIAG_WEBDIR output file
        env_file = '{0}/env_diags_ocn.xml'.format(envDict['PP_CASE_PATH'])
        key = 'OCNDIAG_WEBDIR'
        value = envDict['WORKDIR']
        ##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(envDict['PP_CASE_PATH'], key, main_comm.get_size(), main_comm.get_rank() )
        web_file = '{0}/web_dirs/{1}.{2}'.format(envDict['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S'))
        try:
            diagUtilsLib.write_web_file(web_file, 'ocn', key, value)
        except:
            print('WARNING ocn_diags_generator unable to write {0}={1} to {2}'.format(key, value, web_file))

    main_comm.sync()

    # broadcast the diag_list to all tasks
    num_of_diags = main_comm.partition(num_of_diags, func=partition.Duplicate(), involved=True)
    diag_list = main_comm.partition(data=diag_list, func=partition.Duplicate(), involved=True)
    main_comm.sync()

    # initialize some variables for distributing diagnostics across the communicators
    diags_send = diag_list
    gmaster = main_comm.is_manager()
    gsize = main_comm.get_size()
    grank = main_comm.get_rank()
    local_diag_list = list()

    # divide the main communicator into sub_communicators to be passed to each diag class
    # split mpi comm world if the size of the communicator > 1 and the num_of_diags > 1
    if gsize > 1 and num_of_diags > 1:
        temp_color = (grank % num_of_diags)
        if (temp_color == num_of_diags):
            temp_color = temp_color - 1
        groups = list()
        for g in range(0,num_of_diags):
            groups.append(g)
        debugMsg('global_rank {0}, temp_color {1}, #of groups(diag types) {2}, groups {3}, diag_list {4}'.format(grank, temp_color, num_of_diags, groups, diag_list), header=True, verbosity=2)
        group = groups[temp_color]
        inter_comm, multi_comm = main_comm.divide(group)
        color = inter_comm.get_color()
        lsize = inter_comm.get_size()
        lrank = inter_comm.get_rank()
        lmaster = inter_comm.is_manager()
        debugMsg('color {0}, lsize {1}, lrank {2}, lmaster {3}'.format(color, lsize, lrank, lmaster), header=True, verbosity=2)

        # partition the diag_list between communicators
        DIAG_LIST_TAG = 10
        if lmaster:
            local_diag_list = multi_comm.partition(diag_list,func=partition.EqualStride(),involved=True)
            debugMsg('lrank = {0} local_diag_list = {1}'.format(lrank, local_diag_list), header=True, verbosity=2)
            for b in range(1, lsize):
                diags_send = inter_comm.ration(data=local_diag_list, tag=DIAG_LIST_TAG) 
                debugMsg('b = {0} diags_send = {1} lsize = {2}'.format(b, diags_send, lsize), header=True, verbosity=2)
        else:
            local_diag_list = inter_comm.ration(tag=DIAG_LIST_TAG)
        debugMsg('local_diag_list {0}',format(local_diag_list), header=True, verbosity=2)
    else:
        inter_comm = main_comm
        lmaster = main_comm.is_manager()
        lsize = main_comm.get_size()
        lrank = main_comm.get_rank()
        local_diag_list = diag_list

    inter_comm.sync()
    main_comm.sync()

    # loop through the local_diag_list 
    for requested_diag in local_diag_list:
        try:
            debugMsg('requested_diag {0}, lrank {1}, lsize {2}, lmaster {3}'.format(requested_diag, lrank, lsize, lmaster), header=True, verbosity=2)
            diag = ocn_diags_factory.oceanDiagnosticsFactory(requested_diag)

            # check the prerequisites for the diagnostics types
            debugMsg('Checking prerequisites for {0}'.format(diag.__class__.__name__), header=True, verbosity=2)
            
            skip_key = '{0}_SKIP'.format(requested_diag)
            if lmaster:
                try:
                    envDict = diag.check_prerequisites(envDict)
                except ocn_diags_bc.PrerequisitesError:
                    print("Problem with check_prerequisites for '{0}' skipping!".format(requested_diag))
                    envDict[skip_key] = True
                except RuntimeError as e:
                    # unrecoverable error, bail!
                    print(e)
                    envDict['unrecoverableErrorOnMaster'] = True

            inter_comm.sync()

            # broadcast the envDict
            envDict = inter_comm.partition(data=envDict, func=partition.Duplicate(), involved=True)

            if envDict.has_key('unrecoverableErrorOnMaster'):
                raise RuntimeError

            # run the diagnostics type on each inter_comm
            if not envDict.has_key(skip_key):
                # set the shell env using the values set in the XML and read into the envDict across all tasks
                cesmEnvLib.setXmlEnv(envDict)
                # run the diagnostics
                envDict = diag.run_diagnostics(envDict, inter_comm)

            inter_comm.sync()
            
        except ocn_diags_bc.RecoverableError as e:
            # catch all recoverable errors, print a message and continue.
            print(e)
            print("Skipped '{0}' and continuing!".format(requested_diag))
        except RuntimeError as e:
            # unrecoverable error, bail!
            print(e)
            return 1

    main_comm.sync()