def run_diagnostics(self, env, scomm): """ call the necessary plotting routines to generate diagnostics plots """ super(modelTimeseries, self).run_diagnostics(env, scomm) scomm.sync() # setup some global variables requested_plots = list() local_requested_plots = list() # define the templatePath for all tasks templatePath = '{0}/diagnostics/diagnostics/ocn/Templates'.format(env['POSTPROCESS_PATH']) # all the plot module XML vars start with MVO_PM_ need to strip that off for key, value in env.iteritems(): if (re.search("\AMTS_PM_", key) and value.upper() in ['T','TRUE']): k = key[4:] requested_plots.append(k) scomm.sync() print('model timeseries - after scomm.sync requested_plots = {0}'.format(requested_plots)) if scomm.is_manager(): print('model timeseries - User requested plot modules:') for plot in requested_plots: print(' {0}'.format(plot)) print('model timeseries - Creating plot html header') templateLoader = jinja2.FileSystemLoader( searchpath=templatePath ) templateEnv = jinja2.Environment( loader=templateLoader ) template_file = 'model_timeseries.tmpl' template = templateEnv.get_template( template_file ) # get the current datatime string for the template now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # test the template variables templateVars = { 'casename' : env['CASE'], 'tagname' : env['CESM_TAG'], 'username' : env['USER_NAME'], 'start_year' : env['TSERIES_YEAR0'], 'stop_year' : env['TSERIES_YEAR1'], 'today': now } print('model timeseries - Rendering plot html header') plot_html = template.render( templateVars ) scomm.sync() print('model timeseries - Partition requested plots') # partition requested plots to all tasks local_requested_plots = scomm.partition(requested_plots, func=partition.EqualStride(), involved=True) scomm.sync() for requested_plot in local_requested_plots: try: plot = ocn_diags_plot_factory.oceanDiagnosticPlotFactory('timeseries', requested_plot) print('model timeseries - Checking prerequisite for {0} on rank {1}'.format(plot.__class__.__name__, scomm.get_rank())) plot.check_prerequisites(env) print('model timeseries - Generating plots for {0} on rank {1}'.format(plot.__class__.__name__, scomm.get_rank())) plot.generate_plots(env) print('model timeseries - Converting plots for {0} on rank {1}'.format(plot.__class__.__name__, scomm.get_rank())) plot.convert_plots(env['WORKDIR'], env['IMAGEFORMAT']) except RuntimeError as e: # unrecoverable error, bail! print("model timeseries - Skipped '{0}' and continuing!".format(requested_plot)) print(e) scomm.sync() # initialize OrderedDict with plot_order list entries as key html_order = collections.OrderedDict() for plot in env['MTS_PLOT_ORDER'].split(): html_order[plot] = ''; if scomm.is_manager(): for plot in env['MTS_PLOT_ORDER'].split(): if plot in requested_plots: print('calling get_html for plot = {0}'.format(plot)) plot_obj = ocn_diags_plot_factory.oceanDiagnosticPlotFactory('timeseries', plot) shortname, html = plot_obj.get_html(env['WORKDIR'], templatePath, env['IMAGEFORMAT']) html_order[shortname] = html for k, v in html_order.iteritems(): print('Adding html for plot = {0}'.format(k)) plot_html += v print('model timeseries - Adding footer html') with open('{0}/footer.tmpl'.format(templatePath), 'r') as tmpl: plot_html += tmpl.read() print('model timeseries - Writing plot index.html') with open( '{0}/index.html'.format(env['WORKDIR']), 'w') as index: index.write(plot_html) print('**************************************************************************') print('Successfully completed generating ocean diagnostics model timeseries plots') print('**************************************************************************') scomm.sync() # append the web_dir location to the env key = 'OCNDIAG_WEBDIR_{0}'.format(self._name) env[key] = env['WORKDIR'] return env
def run_diagnostics(self, env, scomm): """ call the necessary plotting routines to generate diagnostics plots """ super(modelVsObsEcosys, self).run_diagnostics(env, scomm) scomm.sync() # setup some global variables requested_plots = list() local_requested_plots = list() local_html_list = list() # define the templatePath for all tasks templatePath = '{0}/diagnostics/diagnostics/ocn/Templates'.format( env['POSTPROCESS_PATH']) # all the plot module XML vars start with MVOECOSYS_PM_ need to strip that off for key, value in env.iteritems(): if (re.search("\AMVOECOSYS_PM_", key) and value.upper() in ['T', 'TRUE']): k = key[10:] requested_plots.append(k) scomm.sync() print('model vs. obs ecosys - after scomm.sync requested_plots = {0}'. format(requested_plots)) if scomm.is_manager(): print('model vs. obs ecosys - User requested plot modules:') for plot in requested_plots: print(' {0}'.format(plot)) print('model vs. obs ecosys - Creating plot html header') templateLoader = jinja2.FileSystemLoader(searchpath=templatePath) templateEnv = jinja2.Environment(loader=templateLoader) template_file = 'model_vs_obs_ecosys.tmpl' template = templateEnv.get_template(template_file) # get the current datatime string for the template now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # test the template variables templateVars = { 'casename': env['CASE'], 'username': env['USER_NAME'], 'tagname': env['CESM_TAG'], 'start_year': env['YEAR0'], 'stop_year': env['YEAR1'], 'today': now } print('model vs. obs ecosys - Rendering plot html header') plot_html = template.render(templateVars) scomm.sync() print('model vs. obs ecosys - Partition requested plots') # partition requested plots to all tasks local_requested_plots = scomm.partition(requested_plots, func=partition.EqualStride(), involved=True) scomm.sync() for requested_plot in local_requested_plots: try: plot = ocn_diags_plot_factory.oceanDiagnosticPlotFactory( 'obs', requested_plot) print( 'model vs. obs ecosys - Checking prerequisite for {0} on rank {1}' .format(plot.__class__.__name__, scomm.get_rank())) plot.check_prerequisites(env) print( 'model vs. obs ecosys - Generating plots for {0} on rank {1}' .format(plot.__class__.__name__, scomm.get_rank())) plot.generate_plots(env) print( 'model vs. obs ecosys - Converting plots for {0} on rank {1}' .format(plot.__class__.__name__, scomm.get_rank())) plot.convert_plots(env['WORKDIR'], env['IMAGEFORMAT']) print( 'model vs. obs ecosys - Creating HTML for {0} on rank {1}'. format(plot.__class__.__name__, scomm.get_rank())) html = plot.get_html(env['WORKDIR'], templatePath, env['IMAGEFORMAT']) local_html_list.append(str(html)) #print('local_html_list = {0}'.format(local_html_list)) except RecoverableError as e: # catch all recoverable errors, print a message and continue. print(e) print("model vs. obs ecosys - Skipped '{0}' and continuing!". format(requested_plot)) except RuntimeError as e: # unrecoverable error, bail! print(e) return 1 scomm.sync() # define a tag for the MPI collection of all local_html_list variables html_msg_tag = 1 all_html = list() all_html = [local_html_list] if scomm.get_size() > 1: if scomm.is_manager(): all_html = [local_html_list] for n in range(1, scomm.get_size()): rank, temp_html = scomm.collect(tag=html_msg_tag) all_html.append(temp_html) #print('all_html = {0}'.format(all_html)) else: return_code = scomm.collect(data=local_html_list, tag=html_msg_tag) scomm.sync() if scomm.is_manager(): # merge the all_html list of lists into a single list all_html = list(itertools.chain.from_iterable(all_html)) for each_html in all_html: #print('each_html = {0}'.format(each_html)) plot_html += each_html print('model vs. obs ecosys - Adding footer html') with open('{0}/footer.tmpl'.format(templatePath), 'r') as tmpl: plot_html += tmpl.read() print('model vs. obs ecosys - Writing plot index.html') with open('{0}/index.html'.format(env['WORKDIR']), 'w') as index: index.write(plot_html) print( '*****************************************************************************************' ) print( 'Successfully completed generating ocean diagnostics model vs. observation ecosystem plots' ) print( '*****************************************************************************************' ) scomm.sync() # append the web_dir location to the env key = 'OCNDIAG_WEBDIR_{0}'.format(self._name) env[key] = env['WORKDIR'] return env
def run_diagnostics(self, env, scomm): """ call the necessary plotting routines to generate diagnostics plots """ super(modelVsControl, self).run_diagnostics(env, scomm) scomm.sync() # setup some global variables requested_plots = list() local_requested_plots = list() # define the templatePath for all tasks templatePath = '{0}/diagnostics/diagnostics/ocn/Templates'.format( env['POSTPROCESS_PATH']) # all the plot module XML vars start with MVC_PM_ need to strip off MVC_ for key, value in env.iteritems(): if (re.search("\AMVC_PM_", key) and value.upper() in ['T', 'TRUE']): k = key[4:] requested_plots.append(k) scomm.sync() print('model vs. control - after scomm.sync requested_plots = {0}'. format(requested_plots)) if scomm.is_manager(): print('model vs. control - User requested plot modules:') for plot in requested_plots: print(' {0}'.format(plot)) print('model vs. control - Creating plot html header') templateLoader = jinja2.FileSystemLoader(searchpath=templatePath) templateEnv = jinja2.Environment(loader=templateLoader) template_file = 'model_vs_control.tmpl' template = templateEnv.get_template(template_file) # get the current datatime string for the template now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # set the template variables templateVars = { 'casename': env['CASE'], 'control_casename': env['CNTRLCASE'], 'tagname': env['CESM_TAG'], 'start_year': env['YEAR0'], 'stop_year': env['YEAR1'], 'control_start_year': env['CNTRLYEAR0'], 'control_stop_year': env['CNTRLYEAR1'], 'today': now } print('model vs. control - Rendering plot html header') plot_html = template.render(templateVars) scomm.sync() print('model vs. control - Partition requested plots') # partition requested plots to all tasks local_requested_plots = scomm.partition(requested_plots, func=partition.EqualStride(), involved=True) scomm.sync() for requested_plot in local_requested_plots: try: plot = ocn_diags_plot_factory.oceanDiagnosticPlotFactory( 'control', requested_plot) print( 'model vs. control - Checking prerequisite for {0} on rank {1}' .format(plot.__class__.__name__, scomm.get_rank())) plot.check_prerequisites(env) print( 'model vs. control - Generating plots for {0} on rank {1}'. format(plot.__class__.__name__, scomm.get_rank())) plot.generate_plots(env) print( 'model vs. control - Converting plots for {0} on rank {1}'. format(plot.__class__.__name__, scomm.get_rank())) plot.convert_plots(env['WORKDIR'], env['IMAGEFORMAT']) except RuntimeError as e: # unrecoverable error, bail! print("Skipped '{0}' and continuing!".format(requested_plot)) print(e) scomm.sync() # initialize OrderedDict with plot_order list entries as key html_order = collections.OrderedDict() for plot in env['MVC_PLOT_ORDER'].split(): html_order[plot] = '' if scomm.is_manager(): for plot in env['MVC_PLOT_ORDER'].split(): if plot in requested_plots: print('calling get_html for plot = {0}'.format(plot)) plot_obj = ocn_diags_plot_factory.oceanDiagnosticPlotFactory( 'control', plot) shortname, html = plot_obj.get_html( env['WORKDIR'], templatePath, env['IMAGEFORMAT']) html_order[shortname] = html for k, v in html_order.iteritems(): print('Adding html for plot = {0}'.format(k)) plot_html += v print('model vs. control - Adding footer html') with open('{0}/footer.tmpl'.format(templatePath), 'r') as tmpl: plot_html += tmpl.read() print('model vs. control - Writing plot index.html') with open('{0}/index.html'.format(env['WORKDIR']), 'w') as index: index.write(plot_html) print( '***************************************************************************' ) print( 'Successfully completed generating ocean diagnostics model vs. control plots' ) print( '***************************************************************************' ) scomm.sync() # append the web_dir location to the env key = 'OCNDIAG_WEBDIR_{0}'.format(self._name) env[key] = env['WORKDIR'] return env
def run_diagnostics(self, env, scomm): """ call the necessary plotting routines to generate diagnostics plots """ super(modelVsObsEcosys, self).run_diagnostics(env, scomm) scomm.sync() # setup some global variables requested_plots = list() local_requested_plots = list() local_html_list = list() # define the templatePath for all tasks templatePath = '{0}/diagnostics/diagnostics/ocn/Templates'.format(env['POSTPROCESS_PATH']) # all the plot module XML vars start with MVOECOSYS_PM_ need to strip that off for key, value in env.iteritems(): if (re.search("\AMVOECOSYS_PM_", key) and value.upper() in ['T','TRUE']): k = key[10:] requested_plots.append(k) scomm.sync() print('model vs. obs ecosys - after scomm.sync requested_plots = {0}'.format(requested_plots)) if scomm.is_manager(): print('model vs. obs ecosys - User requested plot modules:') for plot in requested_plots: print(' {0}'.format(plot)) print('model vs. obs ecosys - Creating plot html header') templateLoader = jinja2.FileSystemLoader( searchpath=templatePath ) templateEnv = jinja2.Environment( loader=templateLoader ) template_file = 'model_vs_obs_ecosys.tmpl' template = templateEnv.get_template( template_file ) # get the current datatime string for the template now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # test the template variables templateVars = { 'casename' : env['CASE'], 'username' : env['USER_NAME'], 'tagname' : env['CESM_TAG'], 'start_year' : env['YEAR0'], 'stop_year' : env['YEAR1'], 'today': now } print('model vs. obs ecosys - Rendering plot html header') plot_html = template.render( templateVars ) scomm.sync() print('model vs. obs ecosys - Partition requested plots') # partition requested plots to all tasks local_requested_plots = scomm.partition(requested_plots, func=partition.EqualStride(), involved=True) scomm.sync() for requested_plot in local_requested_plots: try: plot = ocn_diags_plot_factory.oceanDiagnosticPlotFactory('obs', requested_plot) print('model vs. obs ecosys - Checking prerequisite for {0} on rank {1}'.format(plot.__class__.__name__, scomm.get_rank())) plot.check_prerequisites(env) print('model vs. obs ecosys - Generating plots for {0} on rank {1}'.format(plot.__class__.__name__, scomm.get_rank())) plot.generate_plots(env) print('model vs. obs ecosys - Converting plots for {0} on rank {1}'.format(plot.__class__.__name__, scomm.get_rank())) plot.convert_plots(env['WORKDIR'], env['IMAGEFORMAT']) print('model vs. obs ecosys - Creating HTML for {0} on rank {1}'.format(plot.__class__.__name__, scomm.get_rank())) html = plot.get_html(env['WORKDIR'], templatePath, env['IMAGEFORMAT']) local_html_list.append(str(html)) #print('local_html_list = {0}'.format(local_html_list)) except RecoverableError as e: # catch all recoverable errors, print a message and continue. print(e) print("model vs. obs ecosys - Skipped '{0}' and continuing!".format(requested_plot)) except RuntimeError as e: # unrecoverable error, bail! print(e) return 1 scomm.sync() # define a tag for the MPI collection of all local_html_list variables html_msg_tag = 1 all_html = list() all_html = [local_html_list] if scomm.get_size() > 1: if scomm.is_manager(): all_html = [local_html_list] for n in range(1,scomm.get_size()): rank, temp_html = scomm.collect(tag=html_msg_tag) all_html.append(temp_html) #print('all_html = {0}'.format(all_html)) else: return_code = scomm.collect(data=local_html_list, tag=html_msg_tag) scomm.sync() if scomm.is_manager(): # merge the all_html list of lists into a single list all_html = list(itertools.chain.from_iterable(all_html)) for each_html in all_html: #print('each_html = {0}'.format(each_html)) plot_html += each_html print('model vs. obs ecosys - Adding footer html') with open('{0}/footer.tmpl'.format(templatePath), 'r') as tmpl: plot_html += tmpl.read() print('model vs. obs ecosys - Writing plot index.html') with open( '{0}/index.html'.format(env['WORKDIR']), 'w') as index: index.write(plot_html) print('*****************************************************************************************') print('Successfully completed generating ocean diagnostics model vs. observation ecosystem plots') print('*****************************************************************************************') scomm.sync() # append the web_dir location to the env key = 'OCNDIAG_WEBDIR_{0}'.format(self._name) env[key] = env['WORKDIR'] return env