def makeEotestReport(self, butler): """After running eotest, generate pdf(s) of the results. Generate a sensor test report from the output data in config.eotestOutputPath, one for each CCD. The pdf file(s), along with the .tex file(s) and the individual plots are written to the eotestOutputPath. .pdf generation requires a TeX distro including pdflatex to be installed. """ ccds = butler.queryMetadata('raw', self.config.ccdKey) for ccd in ccds: self.log.info("Starting test report generation for %s" % ccd) try: plotPath = os.path.join(self.config.eotestOutputPath, 'plots') if not os.path.exists(plotPath): os.makedirs(plotPath) plots = sensorTest.EOTestPlots(ccd, self.config.eotestOutputPath, plotPath) eoTestReport = sensorTest.EOTestReport(plots, wl_dir='') eoTestReport.make_figures() eoTestReport.make_pdf() except Exception as e: self.log.warn("Failed to make eotest report for %s: %s" % (ccd, e)) self.log.info("Finished test report generation.")
def qe_task(run, det_name, lambda_files, pd_ratio_file, gains, mask_files=(), correction_image=None, temp_set_point=-100, bias_frame=None, mondiode_func=None): """Single sensor execution of the QE task.""" file_prefix = make_file_prefix(run, det_name) task = sensorTest.QeTask() task.config.temp_set_point = temp_set_point task.run(file_prefix, lambda_files, pd_ratio_file, mask_files, gains, correction_image=correction_image, bias_frame=bias_frame, mondiode_func=mondiode_func) results_file = '%s_eotest_results.fits' % file_prefix plots = sensorTest.EOTestPlots(file_prefix, results_file=results_file) siteUtils.make_png_file(plots.qe, '%s_qe.png' % file_prefix, qe_file='%s_QE.fits' % file_prefix) try: plots.flat_fields(os.path.dirname(lambda_files[0]), annotation='e-/pixel, gain-corrected, bias-subtracted') except Exception as eobj: print("Exception raised while creating flat fields:") print(str(eobj))
def plot_cte_results(run, det_name, superflat_file, eotest_results_file, mask_files=()): """ Create single CCD superflat mosaic plots and plots of the serial and parallel CTE profiles. """ file_prefix = make_file_prefix(run, det_name) flux_level = 'low' if 'low' in os.path.basename(superflat_file) else 'high' plots \ = sensorTest.EOTestPlots(file_prefix, results_file=eotest_results_file) png_files = [] png_files.append(superflat_file.replace('.fits', '.png')) siteUtils.make_png_file(sensorTest.plot_flat, png_files[-1], superflat_file, title=('%s, %s, CTE superflat, %s flux ' % (run, det_name, flux_level)), annotation='ADU/pixel', flatten=True, binsize=4) png_files.append('%s_serial_oscan_%s.png' % (file_prefix, flux_level)) siteUtils.make_png_file(plots.cte_profiles, png_files[-1], flux_level, superflat_file, mask_files, serial=True) png_files.append('%s_parallel_oscan_%s.png' % (file_prefix, flux_level)) siteUtils.make_png_file(plots.cte_profiles, png_files[-1], flux_level, superflat_file, mask_files, serial=False) return png_files
def flat_pairs_task(run, det_name, flat_files, gains, mask_files=(), flat2_finder=find_flat2_bot, linearity_spec_range=(1e4, 9e4), use_exptime=False, bias_frame=None, mondiode_func=None, dark_frame=None): """Single sensor execution of the flat pairs task.""" file_prefix = make_file_prefix(run, det_name) task = sensorTest.FlatPairTask() task.run(file_prefix, flat_files, mask_files, gains, linearity_spec_range=linearity_spec_range, use_exptime=use_exptime, flat2_finder=flat2_finder, bias_frame=bias_frame, mondiode_func=mondiode_func, linearity_correction=get_nlc_func(det_name), dark_frame=dark_frame) results_file = '%s_eotest_results.fits' % file_prefix plots = sensorTest.EOTestPlots(file_prefix, results_file=results_file) detresp_file = '%s_det_response.fits' % file_prefix siteUtils.make_png_file(plots.linearity, '%s_linearity.png' % file_prefix, detresp_file=detresp_file, max_dev=0.03, use_exptime=use_exptime, Ne_bounds=linearity_spec_range) siteUtils.make_png_file(plots.linearity_resids, '%s_linearity_resids.png' % file_prefix, detresp_file=detresp_file, max_dev=0.03, Ne_bounds=linearity_spec_range, use_exptime=use_exptime) siteUtils.make_png_file(row_means_var_plot, f'{file_prefix}_row_means_variance.png', detresp_file, file_prefix)
def run_ptc_task(sensor_id): import lsst.eotest.sensor as sensorTest import siteUtils import eotestUtils file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) flat_files = siteUtils.dependency_glob( 'S*/%s_flat*flat?_*.fits' % sensor_id, jobname=siteUtils.getProcessName('flat_pair_raft_acq'), description='Flat files:') bias_frame = siteUtils.dependency_glob('%s_sflat*median_bias.fits' % sensor_id, description='Super bias frame:')[0] mask_files = \ eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id) gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis', sensor_id=sensor_id) task = sensorTest.PtcTask() task.run(sensor_id, flat_files, mask_files, gains, bias_frame=bias_frame) results_file = '%s_eotest_results.fits' % sensor_id plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file) siteUtils.make_png_file(plots.ptcs, '%s_ptcs.png' % file_prefix, ptc_file='%s_ptc.fits' % sensor_id)
def run_dark_current_task(sensor_id): "Single sensor execution of dark current analysis." import lsst.eotest.sensor as sensorTest import siteUtils import eotestUtils file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) dark_files = siteUtils.dependency_glob( 'S*/%s_dark_dark_*.fits' % sensor_id, jobname=siteUtils.getProcessName('dark_raft_acq'), description='Dark files:') bias_frame = siteUtils.dependency_glob('%s_sflat*median_bias.fits' % sensor_id, description='Super bias frame:')[0] mask_files = \ eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id) gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis', sensor_id=sensor_id) task = sensorTest.DarkCurrentTask() task.config.temp_set_point = -100. dark_curr_pixels, dark95s \ = task.run(sensor_id, dark_files, mask_files, gains, bias_frame=bias_frame) results_file \ = siteUtils.dependency_glob('%s_eotest_results.fits' % sensor_id, jobname='read_noise_raft')[0] plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file) siteUtils.make_png_file(plots.total_noise, '%s_noise.png' % file_prefix, dark95s=dark95s)
def run_bf_task(sensor_id): import lsst.eotest.sensor as sensorTest import siteUtils import eotestUtils file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) flat_files = siteUtils.dependency_glob( 'S*/%s_flat*flat1*.fits' % sensor_id, jobname=siteUtils.getProcessName('flat_pair_raft_acq'), description='Flat files:') bias_frame = siteUtils.dependency_glob('%s_sflat*median_bias.fits' % sensor_id, description='Superbias files:')[0] mask_files = \ eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id) task = sensorTest.BFTask() task.run(sensor_id, flat_files, mask_files=mask_files, bias_frame=bias_frame) results_file = '%s_eotest_results.fits' % sensor_id plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file) siteUtils.make_png_file(plots.bf_curves, '%s_brighter-fatter.png' % file_prefix, bf_file='%s_bf.fits' % sensor_id)
def plot_ccd_total_noise(run, det_name, dark_curr_pixels, dark95s, eotest_results_file): """ Make CCD-level total noise summary plots using the dark current measurements and an existing eotest results file containing the read noise measurements. """ file_prefix = make_file_prefix(run, det_name) plots = sensorTest.EOTestPlots(det_name, results_file=eotest_results_file) siteUtils.make_png_file(plots.total_noise, '%s_noise.png' % file_prefix, dark95s=dark95s)
def bf_task(run, det_name, flat_files, gains, mask_files=(), flat2_finder=None, bias_frame=None): """Single sensor execution of the brighter-fatter task.""" file_prefix = make_file_prefix(run, det_name) task = sensorTest.BFTask() task.run(file_prefix, flat_files, mask_files=mask_files, flat2_finder=flat2_finder, bias_frame=bias_frame, linearity_correction=get_nlc_func(det_name), gains=gains) results_file = '%s_eotest_results.fits' % file_prefix plots = sensorTest.EOTestPlots(file_prefix, results_file=results_file) siteUtils.make_png_file(plots.bf_curves, '%s_brighter-fatter.png' % file_prefix, bf_file='%s_bf.fits' % file_prefix)
def ptc_task(run, det_name, flat_files, gains, mask_files=(), flat2_finder=find_flat2_bot, bias_frame=None): """Single sensor execution of the PTC task.""" file_prefix = make_file_prefix(run, det_name) task = sensorTest.PtcTask() task.run(file_prefix, flat_files, mask_files, gains, flat2_finder=flat2_finder, bias_frame=bias_frame, linearity_correction=get_nlc_func(det_name)) results_file = '%s_eotest_results.fits' % file_prefix plots = sensorTest.EOTestPlots(file_prefix, results_file=results_file) siteUtils.make_png_file(plots.ptcs, '%s_ptcs.png' % file_prefix, ptc_file='%s_ptc.fits' % file_prefix)
def run_flat_pair_task(sensor_id): import lsst.eotest.sensor as sensorTest import siteUtils import eotestUtils file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) flat_files = siteUtils.dependency_glob('S*/%s_flat*flat?_*.fits' % sensor_id, jobname=siteUtils.getProcessName('flat_pair_raft_acq'), description='Flat files:') bias_frame = siteUtils.dependency_glob('%s_sflat*median_bias.fits' % sensor_id, description='Super bias frame:')[0] mask_files = \ eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id) gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis', sensor_id=sensor_id) use_exptime = True if siteUtils.getSiteName() == 'SLAC': # Since slit-width can be set individually for each exposure # on TS-8 at IR-2 (LSSTTD-1231), we need to use the MONDIODE # keyword for computing the integrated incident flux. use_exptime = False task = sensorTest.FlatPairTask() task.run(sensor_id, flat_files, mask_files, gains, linearity_spec_range=(1e4, 9e4), use_exptime=use_exptime, bias_frame=bias_frame) results_file = '%s_eotest_results.fits' % sensor_id plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file) Ne_bounds = (1e4, 9e4) detresp_file = '%s_det_response.fits' % sensor_id siteUtils.make_png_file(plots.linearity, '%s_linearity.png' % file_prefix, detresp_file=detresp_file, max_dev=0.03, use_exptime=use_exptime, Ne_bounds=Ne_bounds) siteUtils.make_png_file(plots.linearity_resids, '%s_linearity_resids.png' % file_prefix, detresp_file=detresp_file, max_dev=0.03, Ne_bounds=Ne_bounds, use_exptime=use_exptime)
def run_fe55_task(sensor_id): "Single sensor execution of the Fe55 analysis task." import os import glob import lsst.eotest.image_utils as imutils import lsst.eotest.sensor as sensorTest import siteUtils import eotestUtils file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) acq_jobname = siteUtils.getProcessName('fe55_raft_acq') fe55_files = siteUtils.dependency_glob('S*/%s_fe55_fe55_*.fits' % sensor_id, jobname=acq_jobname, description='Fe55 files:') # Reverse sort the fe55 files to avoid transient effects arising # from using frames taken right after cool down that could bias # the gain measurement. fe55_files = sorted(fe55_files, reverse=True) bias_files = siteUtils.dependency_glob('S*/%s_fe55_bias_*.fits' % sensor_id, jobname=acq_jobname, description='Bias files:') bias_frame = eotestUtils.make_median_bias_frame(bias_files, sensor_id, 'fe55_raft_acq', skip=0) # # Create a png zoom of the upper right corner of segment 1 for an Fe55 # exposure for inclusion in the test report. # print("processing fe55_zoom:", fe55_files[0]) siteUtils.make_png_file(sensorTest.fe55_zoom, '%(file_prefix)s_fe55_zoom.png' % locals(), fe55_files[0], size=250, amp=1, annotation='ADU/pixel') # # Perform analysis of 9-pixel statistics for Fe55 charge clusters. # try: pixel_stats = sensorTest.Fe55PixelStats(fe55_files, sensor_id=sensor_id) siteUtils.make_png_file(pixel_stats.pixel_hists, '%s_fe55_p3_p5_hists.png' % file_prefix, pix0='p3', pix1='p5') siteUtils.make_png_file(pixel_stats.pixel_diff_profile, '%s_fe55_p3_p5_profiles.png' % file_prefix, pixel_coord='x', pix0='p3', pix1='p5') siteUtils.make_png_file(pixel_stats.apflux_profile, '%s_fe55_apflux_serial.png' % file_prefix) siteUtils.make_png_file(pixel_stats.apflux_profile, '%s_fe55_apflux_parallel.png' % file_prefix, pixel_coord='y') except Exception as eobj: print("Exception raised while creating pixel statistics plots:") print(str(eobj)) print("Skipping these plots.") # Roll-off defects mask needs an input file to get the vendor # geometry, and will be used for all analyses. rolloff_mask_file = '%s_rolloff_defects_mask.fits' % sensor_id sensorTest.rolloff_mask(fe55_files[0], rolloff_mask_file) task = sensorTest.Fe55Task() task.config.temp_set_point = -100. hist_nsig = 10 dn_range = 1590. / 2., 1590. / 0.5 task.run(sensor_id, fe55_files, (rolloff_mask_file, ), bias_frame=bias_frame, accuracy_req=0.01, hist_nsig=hist_nsig, dn_range=dn_range) # Fe55 gain and psf analysis results plots for the test report. results_file = '%s_eotest_results.fits' % sensor_id plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file) siteUtils.make_png_file(plots.gains, '%s_gains.png' % file_prefix) siteUtils.make_png_file(sensorTest.plot_flat, '%s_median_bias.png' % file_prefix, bias_frame, title='%s, median bias frame' % sensor_id, annotation='ADU/pixel, overscan-subtracted') fe55_file = glob.glob('%s_psf_results*.fits' % sensor_id)[0] siteUtils.make_png_file(plots.fe55_dists, '%s_fe55_dists.png' % file_prefix, fe55_file=fe55_file, xrange_scale=3, dn_range=dn_range) siteUtils.make_png_file(plots.psf_dists, '%s_psf_dists.png' % file_prefix, fe55_file=fe55_file)
append_prnu(results_file, processName_dependencyGlob(results_file, jobname='prnu')[0]) qe_file = processName_dependencyGlob('*%s_QE.fits' % sensor_id, jobname='qe_analysis')[0] shutil.copy(qe_file, ".") try: xtalk_file = processName_dependencyGlob('*%s_xtalk_matrix.fits' % sensor_id, jobname='crosstalk')[0] except IndexError: xtalk_file = None plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file, xtalk_file=xtalk_file) plots.specs.add_job_ids(summary_files) # Fe55 flux distribution fits fe55_file = processName_dependencyGlob('%s_psf_results*.fits' % sensor_id, jobname='fe55_analysis')[0] plots.fe55_dists(fe55_file=fe55_file) plt.savefig('%s_fe55_dists.png' % sensor_id) plt.close('all') # PSF distributions from Fe55 fits plots.psf_dists(fe55_file=fe55_file) plt.savefig('%s_psf_dists.png' % sensor_id) plt.close('all')
def run_cte_task(sensor_id): "Single sensor execution of the cte task." import os import shutil import glob import lsst.eotest.sensor as sensorTest import siteUtils import eotestUtils file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) bias_frame = siteUtils.dependency_glob('%s_sflat*median_bias.fits' % sensor_id, description='Super bias frame:')[0] mask_files = \ eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id) gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis', sensor_id=sensor_id) # Omit rolloff defects mask since it would mask some of the edges used # in the eper method. mask_files = [ item for item in mask_files if item.find('rolloff_defects') == -1 ] print("Using mask files:") for mask_file in mask_files: print(" " + mask_file) results_file \ = siteUtils.dependency_glob('%s_eotest_results.fits' % sensor_id, jobname='fe55_raft_analysis', description='Fe55 results file')[0] shutil.copy(results_file, os.path.basename(results_file)) results_file = os.path.basename(results_file) sflat_high_files = \ siteUtils.dependency_glob('S*/%s_sflat_500_flat_H*.fits' % sensor_id, jobname=siteUtils.getProcessName('sflat_raft_acq'), description='Superflat high flux files:') task = sensorTest.CteTask() task.run(sensor_id, sflat_high_files, flux_level='high', gains=gains, mask_files=mask_files, bias_frame=bias_frame) sflat_low_files = \ siteUtils.dependency_glob('S*/%s_sflat_500_flat_L*.fits' % sensor_id, jobname=siteUtils.getProcessName('sflat_raft_acq'), description='Superflat low flux files:') task.run(sensor_id, sflat_low_files, flux_level='low', gains=gains, mask_files=mask_files, bias_frame=bias_frame) plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file) superflat_files = sorted(glob.glob('%s_superflat_*.fits' % sensor_id)) mask_files = [ x for x in glob.glob('%s*mask.fits' % sensor_id) if x.find('rolloff') == -1 ] for sflat_file in superflat_files: flux_level = 'low' if sflat_file.find('high') != -1: flux_level = 'high' siteUtils.make_png_file( sensorTest.plot_flat, sflat_file.replace('.fits', '.png').replace(sensor_id, file_prefix), sflat_file, title=('%s, CTE superflat, %s flux ' % (sensor_id, flux_level)), annotation='ADU/pixel', flatten=True, binsize=4) siteUtils.make_png_file(plots.cte_profiles, ('%s_serial_oscan_%s.png' % (file_prefix, flux_level)), flux_level, sflat_file, mask_files, serial=True) siteUtils.make_png_file(plots.cte_profiles, ('%s_parallel_oscan_%s.png' % (file_prefix, flux_level)), flux_level, sflat_file, mask_files, serial=False)
def fe55_task(run, det_name, fe55_files, bias_frame=None): "Single sensor execution of the Fe55 analysis task." file_prefix = make_file_prefix(run, det_name) title = '{}, {}'.format(run, det_name) if bias_frame is None: bias_frame = bias_filename(run, det_name) png_files = [] try: pixel_stats = sensorTest.Fe55PixelStats(fe55_files, sensor_id=file_prefix) png_files.append('%s_fe55_p3_p5_hists.png' % file_prefix) siteUtils.make_png_file(pixel_stats.pixel_hists, png_files[-1], pix0='p3', pix1='p5') png_files.append('%s_fe55_p3_p5_profiles.png' % file_prefix) siteUtils.make_png_file(pixel_stats.pixel_diff_profile, png_files[-1], pixel_coord='x', pix0='p3', pix1='p5') except Exception: # Encountered error processing data or generating pngs so skip # these plots. pass rolloff_mask_file = '%s_edge_rolloff_mask.fits' % file_prefix sensorTest.rolloff_mask(fe55_files[0], rolloff_mask_file) hist_nsig = 20 task = sensorTest.Fe55Task() task.config.temp_set_point = -100. task.run(file_prefix, fe55_files, (rolloff_mask_file,), bias_frame=bias_frame, accuracy_req=0.01, hist_nsig=hist_nsig, linearity_correction=get_nlc_func(det_name)) # Fe55 gain and psf analysis results plots for the test report. results_file = '%s_eotest_results.fits' % file_prefix plots = sensorTest.EOTestPlots(file_prefix, results_file=results_file) png_files.append('%s_gains.png' % file_prefix) siteUtils.make_png_file(plots.gains, png_files[-1]) png_files.append('%s_fe55_median_bias.png' % file_prefix) siteUtils.make_png_file(sensorTest.plot_flat, png_files[-1], bias_frame, title='%s, median bias frame' % title, annotation='ADU/pixel, overscan-subtracted') fe55_file = glob.glob('%s_psf_results*.fits' % file_prefix)[0] png_files.append('%s_fe55_dists.png' % file_prefix) siteUtils.make_png_file(plots.fe55_dists, png_files[-1], fe55_file=fe55_file, xrange_scale=5, hist_nsig=hist_nsig) png_files.append('%s_psf_dists.png' % file_prefix) siteUtils.make_png_file(plots.psf_dists, png_files[-1], fe55_file=fe55_file) png_file_list = '{}_fe55_task_png_files.txt'.format(det_name) with open(png_file_list, 'w') as output: for item in png_files: if os.path.isfile(item): output.write('{}\n'.format(item))
def run_qe_task(sensor_id): "Single sensor execution of the QE task." import os import sys import lsst.eotest.sensor as sensorTest import siteUtils import eotestUtils file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) lambda_files = siteUtils.dependency_glob( 'S*/%s_lambda_flat_*.fits' % sensor_id, jobname=siteUtils.getProcessName('qe_raft_acq'), description='Lambda files:') pd_ratio_file = eotestUtils.getPhotodiodeRatioFile() if pd_ratio_file is None: message = ("The test-stand specific photodiode ratio file is " + "not given in config/%s/eotest_calibrations.cfg." % siteUtils.getSiteName()) raise RuntimeError(message) correction_image = eotestUtils.getIlluminationNonUniformityImage() if correction_image is None: print() print("WARNING: The correction image file is not given in") print("config/%s/eotest_calibrations.cfg." % siteUtils.getSiteName()) print("No correction for non-uniform illumination will be applied.") print() sys.stdout.flush() bias_frame = siteUtils.dependency_glob('%s_sflat*median_bias.fits' % sensor_id, description='Super bias frame:')[0] mask_files = \ eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id) gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis', sensor_id=sensor_id) task = sensorTest.QeTask() task.config.temp_set_point = -100. task.run(sensor_id, lambda_files, pd_ratio_file, mask_files, gains, correction_image=correction_image, bias_frame=bias_frame) results_file \ = siteUtils.dependency_glob('%s_eotest_results.fits' % sensor_id, jobname='fe55_raft_analysis', description='Fe55 results file')[0] plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file) siteUtils.make_png_file(plots.qe, '%s_qe.png' % file_prefix, qe_file='%s_QE.fits' % sensor_id) try: plots.flat_fields( os.path.dirname(lambda_files[0]), annotation='e-/pixel, gain-corrected, bias-subtracted') except Exception as eobj: print("Exception raised while creating flat fields:") print(str(eobj))