Exemplo n.º 1
0
def summarize_set_dir(set_dir, results_dir, summ_obj=None, tstplot_obj=None, vis_opts=None, hist_obj=None, overwrite=False, fail_on_error=False,  verbose=False):
    logger = logging.getLogger(LOGGER_NAME)

    # Remove trailing slashes
    set_dir = set_dir.rstrip('/')
    results_dir = results_dir.rstrip('/')

    if not os.path.exists(results_dir):
        os.makedirs(results_dir)

    # Open log file for set dir
    log_handler = L2_Log_Util.open_log_file( os.path.join(results_dir, LOG_FILE_BASE) )

    # Create necessary objects if not passed in
    if summ_obj == None:
        summ_obj = testset_summary()

    # Create status files
    run_lists = stats.gather_status(set_dir, results_dir)

    for (run_type, run_directories) in run_lists.items():
        if run_type not in SUMMARY_TYPES:
            continue

        input_files_only = run_type == 'forward_model' or run_type == 'jacobian_only'
        
        summ_file = SUMM_FILE_FMT % (results_dir, run_type)
        stat_file = STAT_FILE_FMT % (results_dir, run_type)
        plot_file = PLOT_FILE_FMT % (results_dir, run_type)
        time_file = TIME_FILE_FMT % (results_dir, run_type)

        plots_dir = PLOTS_DIR_FMT % (results_dir, run_type)

        processing_msg = 'generating summary files for run type: %s for set dir: %s' % (run_type, set_dir)
        with error_trapping(fail_on_error, processing_msg):
            summ_obj.make_summary_output(run_directories, stat_filename=stat_file, summ_filename=summ_file, overwrite=overwrite)

        ####

        if tstplot_obj != None:
            processing_msg = 'generating summary plots for run type: %s for set dir: %s' % (run_type, set_dir)
            with error_trapping(fail_on_error, processing_msg):
                tstplot_obj.make_summary_output(stat_filename=stat_file, summ_filename=summ_file, plot_filename=plot_file, overwrite=overwrite)

        ####

        if vis_opts != None and not input_files_only:
            import l2_vis
            if not os.path.exists(plots_dir):
                logger.info('Creating plots dir:', plots_dir)
                os.makedirs(plots_dir)

            processing_msg = 'creating per run directory plots for run type: %s for set dir: %s' % (run_type, set_dir)
            with error_trapping(fail_on_error, processing_msg):
                l2_vis.make_run_plots(run_directories, plots_dir, overwrite=overwrite, **vis_opts)

        ####

# Commented out as the new L2 code does not have the output that this routine reads anymore
#         processing_msg = 'generating summary timings info file for run type: %s for set dir: %s' % (run_type, set_dir)
#         with error_trapping(fail_on_error, processing_msg):
#             make_time_file = True
#             if not overwrite and os.path.exists(time_file):
#                 time_obj = OCO_Matrix()
#                 time_obj.read(time_file, read_data=False)

#                 if time_obj.dims[0] == len(run_directories):
#                     if verbose:
#                         logger.info('Skipping recreating timings file: %s' % time_file)
#                     make_time_file = False

#             if make_time_file:
#                 compute_runtimes(run_directories, time_file)
    ####

    if hist_obj != None:
        processing_msg = 'creating solar zenith angle histogram plot for set dir: %s' % set_dir
        with error_trapping(fail_on_error, processing_msg):
            hist_obj.plot(set_dir, results_dir, verbose)

    L2_Log_Util.close_log_file(log_handler)
def summarize_set_dir(set_dir,
                      results_dir,
                      summ_obj=None,
                      tstplot_obj=None,
                      vis_opts=None,
                      hist_obj=None,
                      overwrite=False,
                      fail_on_error=False,
                      verbose=False):
    logger = logging.getLogger(LOGGER_NAME)

    # Remove trailing slashes
    set_dir = set_dir.rstrip('/')
    results_dir = results_dir.rstrip('/')

    if not os.path.exists(results_dir):
        os.makedirs(results_dir)

    # Open log file for set dir
    log_handler = L2_Log_Util.open_log_file(
        os.path.join(results_dir, LOG_FILE_BASE))

    # Create necessary objects if not passed in
    if summ_obj == None:
        summ_obj = testset_summary()

    # Create status files
    run_lists = stats.gather_status(set_dir, results_dir)

    for (run_type, run_directories) in run_lists.items():
        if run_type not in SUMMARY_TYPES:
            continue

        input_files_only = run_type == 'forward_model' or run_type == 'jacobian_only'

        summ_file = SUMM_FILE_FMT % (results_dir, run_type)
        stat_file = STAT_FILE_FMT % (results_dir, run_type)
        plot_file = PLOT_FILE_FMT % (results_dir, run_type)
        time_file = TIME_FILE_FMT % (results_dir, run_type)

        plots_dir = PLOTS_DIR_FMT % (results_dir, run_type)

        processing_msg = 'generating summary files for run type: %s for set dir: %s' % (
            run_type, set_dir)
        with error_trapping(fail_on_error, processing_msg):
            summ_obj.make_summary_output(run_directories,
                                         stat_filename=stat_file,
                                         summ_filename=summ_file,
                                         overwrite=overwrite)

        ####

        if tstplot_obj != None:
            processing_msg = 'generating summary plots for run type: %s for set dir: %s' % (
                run_type, set_dir)
            with error_trapping(fail_on_error, processing_msg):
                tstplot_obj.make_summary_output(stat_filename=stat_file,
                                                summ_filename=summ_file,
                                                plot_filename=plot_file,
                                                overwrite=overwrite)

        ####

        if vis_opts != None and not input_files_only:
            import l2_vis
            if not os.path.exists(plots_dir):
                logger.info('Creating plots dir:', plots_dir)
                os.makedirs(plots_dir)

            processing_msg = 'creating per run directory plots for run type: %s for set dir: %s' % (
                run_type, set_dir)
            with error_trapping(fail_on_error, processing_msg):
                l2_vis.make_run_plots(run_directories,
                                      plots_dir,
                                      overwrite=overwrite,
                                      **vis_opts)

        ####


# Commented out as the new L2 code does not have the output that this routine reads anymore
#         processing_msg = 'generating summary timings info file for run type: %s for set dir: %s' % (run_type, set_dir)
#         with error_trapping(fail_on_error, processing_msg):
#             make_time_file = True
#             if not overwrite and os.path.exists(time_file):
#                 time_obj = OCO_Matrix()
#                 time_obj.read(time_file, read_data=False)

#                 if time_obj.dims[0] == len(run_directories):
#                     if verbose:
#                         logger.info('Skipping recreating timings file: %s' % time_file)
#                     make_time_file = False

#             if make_time_file:
#                 compute_runtimes(run_directories, time_file)
####

    if hist_obj != None:
        processing_msg = 'creating solar zenith angle histogram plot for set dir: %s' % set_dir
        with error_trapping(fail_on_error, processing_msg):
            hist_obj.plot(set_dir, results_dir, verbose)

    L2_Log_Util.close_log_file(log_handler)
Exemplo n.º 3
0
def plot_group(run_directories, output_directory=None, filename_filters=None, overwrite=False, debugging_logs=False, **kwargs):
    """Plot files for a group of supposedly similar run directories"""

    logger = logging.getLogger()

    # Find common name among basenames of run directories
    dir_group_name = os.path.commonprefix( [os.path.basename(curr_dir.strip('/')) for curr_dir in run_directories] )

    logger.info('Ploting run group: ' + dir_group_name)

    for curr_glob, curr_routine in PLOT_ROUTINE_GLOBS.items():

        # For each type of plotting glob find matching files
        plot_files = []
        for curr_dir in run_directories:
            # glob may return multiple
            glob_results = glob.glob( os.path.join(curr_dir, curr_glob) )

            if filename_filters == None:
                plot_files += glob_results
            else:
                for curr_filter in filename_filters:
                    for curr_result in glob_results:
                        if re.search(curr_filter, curr_result):
                            plot_files.append(curr_result)

        # Group together globbed files according to basename
        # Safe way to only plot files that appear in all grouped dirs
        file_groups = get_path_groups(plot_files)

        for curr_group_files in file_groups:
            group_basename = os.path.splitext(os.path.basename(curr_group_files[0]))[0]
            group_title = '%s %s' % (dir_group_name, group_basename)

            output_filename = '%s_%s.%s' % (dir_group_name, group_basename, OUTPUT_FORMAT)
            if output_directory != None:
                if not os.path.exists(output_directory):
                    os.makedirs(output_directory)
                
                output_filename = os.path.join(output_directory, output_filename)

            if debugging_logs:
                log_filename = '%s.%s' % (os.path.splitext(output_filename)[0], 'log')
                logger.info('Writing log file: %s' % log_filename)
                log_handler = L2_Log_Util.open_log_file(log_filename)

            logger.info('Plotting file group: ' + group_basename)
            logger.debug('Group files:')
            for curr_file in curr_group_files:
                logger.debug(os.path.realpath(curr_file))
            
            # Remove common parts from end of filenames
            group_file_labels = [ os.path.dirname(re.sub(dir_group_name + '.*', '', curr_file)) for curr_file in curr_group_files ]

            # Remove common directory part from beginning of labels
            common_dir_prefix = os.path.commonprefix([ os.path.dirname(curr_lbl) for curr_lbl in group_file_labels])
            if len(common_dir_prefix) > 0:
                group_file_labels = [ curr_lbl.replace(common_dir_prefix, '') for curr_lbl in group_file_labels ]

            if os.path.exists(output_filename):
                logger.info('Skipping existing plot file: %s' % output_filename)
            else:
                logger.info('Creating plot file: %s' % output_filename)

                # Plot files using plotting routine associated with file glob
                try:
                    plotted_filenames = curr_routine(curr_group_files, group_title, group_file_labels, **kwargs)
                    combine_pdfs(plotted_filenames, output_filename, remove_source_files=True)

                    logger.debug('Combined %d plot files' % len(plotted_filenames))
                except:
                    logger.error('Failed to create plot file: %s' % output_filename)
                    logger.error('Error running routine %s for files %s:' % (curr_routine, curr_group_files))
                    logger.error(''.join(traceback.format_exception(*sys.exc_info(), limit=2)))

            if debugging_logs:
                L2_Log_Util.close_log_file(log_handler)