def summarize_set_dir(set_dir, results_dir, summ_obj=None, tstplot_obj=None, vis_opts=None, hist_obj=None, overwrite=False, fail_on_error=False, verbose=False):
logger = logging.getLogger(LOGGER_NAME)
# Remove trailing slashes
set_dir = set_dir.rstrip('/')
results_dir = results_dir.rstrip('/')
if not os.path.exists(results_dir):
os.makedirs(results_dir)
# Open log file for set dir
log_handler = L2_Log_Util.open_log_file( os.path.join(results_dir, LOG_FILE_BASE) )
# Create necessary objects if not passed in
if summ_obj == None:
summ_obj = testset_summary()
# Create status files
run_lists = stats.gather_status(set_dir, results_dir)
for (run_type, run_directories) in run_lists.items():
if run_type not in SUMMARY_TYPES:
continue
input_files_only = run_type == 'forward_model' or run_type == 'jacobian_only'
summ_file = SUMM_FILE_FMT % (results_dir, run_type)
stat_file = STAT_FILE_FMT % (results_dir, run_type)
plot_file = PLOT_FILE_FMT % (results_dir, run_type)
time_file = TIME_FILE_FMT % (results_dir, run_type)
plots_dir = PLOTS_DIR_FMT % (results_dir, run_type)
processing_msg = 'generating summary files for run type: %s for set dir: %s' % (run_type, set_dir)
with error_trapping(fail_on_error, processing_msg):
summ_obj.make_summary_output(run_directories, stat_filename=stat_file, summ_filename=summ_file, overwrite=overwrite)
####
if tstplot_obj != None:
processing_msg = 'generating summary plots for run type: %s for set dir: %s' % (run_type, set_dir)
with error_trapping(fail_on_error, processing_msg):
tstplot_obj.make_summary_output(stat_filename=stat_file, summ_filename=summ_file, plot_filename=plot_file, overwrite=overwrite)
####
if vis_opts != None and not input_files_only:
import l2_vis
if not os.path.exists(plots_dir):
logger.info('Creating plots dir:', plots_dir)
os.makedirs(plots_dir)
processing_msg = 'creating per run directory plots for run type: %s for set dir: %s' % (run_type, set_dir)
with error_trapping(fail_on_error, processing_msg):
l2_vis.make_run_plots(run_directories, plots_dir, overwrite=overwrite, **vis_opts)
####
# Commented out as the new L2 code does not have the output that this routine reads anymore
# processing_msg = 'generating summary timings info file for run type: %s for set dir: %s' % (run_type, set_dir)
# with error_trapping(fail_on_error, processing_msg):
# make_time_file = True
# if not overwrite and os.path.exists(time_file):
# time_obj = OCO_Matrix()
# time_obj.read(time_file, read_data=False)
# if time_obj.dims[0] == len(run_directories):
# if verbose:
# logger.info('Skipping recreating timings file: %s' % time_file)
# make_time_file = False
# if make_time_file:
# compute_runtimes(run_directories, time_file)
####
if hist_obj != None:
processing_msg = 'creating solar zenith angle histogram plot for set dir: %s' % set_dir
with error_trapping(fail_on_error, processing_msg):
hist_obj.plot(set_dir, results_dir, verbose)
L2_Log_Util.close_log_file(log_handler)
def summarize_set_dir(set_dir,
results_dir,
summ_obj=None,
tstplot_obj=None,
vis_opts=None,
hist_obj=None,
overwrite=False,
fail_on_error=False,
verbose=False):
logger = logging.getLogger(LOGGER_NAME)
# Remove trailing slashes
set_dir = set_dir.rstrip('/')
results_dir = results_dir.rstrip('/')
if not os.path.exists(results_dir):
os.makedirs(results_dir)
# Open log file for set dir
log_handler = L2_Log_Util.open_log_file(
os.path.join(results_dir, LOG_FILE_BASE))
# Create necessary objects if not passed in
if summ_obj == None:
summ_obj = testset_summary()
# Create status files
run_lists = stats.gather_status(set_dir, results_dir)
for (run_type, run_directories) in run_lists.items():
if run_type not in SUMMARY_TYPES:
continue
input_files_only = run_type == 'forward_model' or run_type == 'jacobian_only'
summ_file = SUMM_FILE_FMT % (results_dir, run_type)
stat_file = STAT_FILE_FMT % (results_dir, run_type)
plot_file = PLOT_FILE_FMT % (results_dir, run_type)
time_file = TIME_FILE_FMT % (results_dir, run_type)
plots_dir = PLOTS_DIR_FMT % (results_dir, run_type)
processing_msg = 'generating summary files for run type: %s for set dir: %s' % (
run_type, set_dir)
with error_trapping(fail_on_error, processing_msg):
summ_obj.make_summary_output(run_directories,
stat_filename=stat_file,
summ_filename=summ_file,
overwrite=overwrite)
####
if tstplot_obj != None:
processing_msg = 'generating summary plots for run type: %s for set dir: %s' % (
run_type, set_dir)
with error_trapping(fail_on_error, processing_msg):
tstplot_obj.make_summary_output(stat_filename=stat_file,
summ_filename=summ_file,
plot_filename=plot_file,
overwrite=overwrite)
####
if vis_opts != None and not input_files_only:
import l2_vis
if not os.path.exists(plots_dir):
logger.info('Creating plots dir:', plots_dir)
os.makedirs(plots_dir)
processing_msg = 'creating per run directory plots for run type: %s for set dir: %s' % (
run_type, set_dir)
with error_trapping(fail_on_error, processing_msg):
l2_vis.make_run_plots(run_directories,
plots_dir,
overwrite=overwrite,
**vis_opts)
####
# Commented out as the new L2 code does not have the output that this routine reads anymore
# processing_msg = 'generating summary timings info file for run type: %s for set dir: %s' % (run_type, set_dir)
# with error_trapping(fail_on_error, processing_msg):
# make_time_file = True
# if not overwrite and os.path.exists(time_file):
# time_obj = OCO_Matrix()
# time_obj.read(time_file, read_data=False)
# if time_obj.dims[0] == len(run_directories):
# if verbose:
# logger.info('Skipping recreating timings file: %s' % time_file)
# make_time_file = False
# if make_time_file:
# compute_runtimes(run_directories, time_file)
####
if hist_obj != None:
processing_msg = 'creating solar zenith angle histogram plot for set dir: %s' % set_dir
with error_trapping(fail_on_error, processing_msg):
hist_obj.plot(set_dir, results_dir, verbose)
L2_Log_Util.close_log_file(log_handler)
def plot_group(run_directories, output_directory=None, filename_filters=None, overwrite=False, debugging_logs=False, **kwargs):
"""Plot files for a group of supposedly similar run directories"""
logger = logging.getLogger()
# Find common name among basenames of run directories
dir_group_name = os.path.commonprefix( [os.path.basename(curr_dir.strip('/')) for curr_dir in run_directories] )
logger.info('Ploting run group: ' + dir_group_name)
for curr_glob, curr_routine in PLOT_ROUTINE_GLOBS.items():
# For each type of plotting glob find matching files
plot_files = []
for curr_dir in run_directories:
# glob may return multiple
glob_results = glob.glob( os.path.join(curr_dir, curr_glob) )
if filename_filters == None:
plot_files += glob_results
else:
for curr_filter in filename_filters:
for curr_result in glob_results:
if re.search(curr_filter, curr_result):
plot_files.append(curr_result)
# Group together globbed files according to basename
# Safe way to only plot files that appear in all grouped dirs
file_groups = get_path_groups(plot_files)
for curr_group_files in file_groups:
group_basename = os.path.splitext(os.path.basename(curr_group_files[0]))[0]
group_title = '%s %s' % (dir_group_name, group_basename)
output_filename = '%s_%s.%s' % (dir_group_name, group_basename, OUTPUT_FORMAT)
if output_directory != None:
if not os.path.exists(output_directory):
os.makedirs(output_directory)
output_filename = os.path.join(output_directory, output_filename)
if debugging_logs:
log_filename = '%s.%s' % (os.path.splitext(output_filename)[0], 'log')
logger.info('Writing log file: %s' % log_filename)
log_handler = L2_Log_Util.open_log_file(log_filename)
logger.info('Plotting file group: ' + group_basename)
logger.debug('Group files:')
for curr_file in curr_group_files:
logger.debug(os.path.realpath(curr_file))
# Remove common parts from end of filenames
group_file_labels = [ os.path.dirname(re.sub(dir_group_name + '.*', '', curr_file)) for curr_file in curr_group_files ]
# Remove common directory part from beginning of labels
common_dir_prefix = os.path.commonprefix([ os.path.dirname(curr_lbl) for curr_lbl in group_file_labels])
if len(common_dir_prefix) > 0:
group_file_labels = [ curr_lbl.replace(common_dir_prefix, '') for curr_lbl in group_file_labels ]
if os.path.exists(output_filename):
logger.info('Skipping existing plot file: %s' % output_filename)
else:
logger.info('Creating plot file: %s' % output_filename)
# Plot files using plotting routine associated with file glob
try:
plotted_filenames = curr_routine(curr_group_files, group_title, group_file_labels, **kwargs)
combine_pdfs(plotted_filenames, output_filename, remove_source_files=True)
logger.debug('Combined %d plot files' % len(plotted_filenames))
except:
logger.error('Failed to create plot file: %s' % output_filename)
logger.error('Error running routine %s for files %s:' % (curr_routine, curr_group_files))
logger.error(''.join(traceback.format_exception(*sys.exc_info(), limit=2)))
if debugging_logs:
L2_Log_Util.close_log_file(log_handler)