def test_amplifier_info(): """Test that the correct number of amplifiers are found for a given file """ data_dir = os.path.join(get_config()['test_dir'], 'dark_monitor') fullframe = instrument_properties.amplifier_info(os.path.join(data_dir, 'test_image_ff.fits')) fullframe_truth = (4, {'1': [(4, 4), (512, 2044)], '2': [(512, 4), (1024, 2044)], '3': [(1024, 4), (1536, 2044)], '4': [(1536, 4), (2044, 2044)]}) assert fullframe == fullframe_truth fullframe = instrument_properties.amplifier_info(os.path.join(data_dir, 'test_image_ff.fits'), omit_reference_pixels=False) fullframe_truth = (4, {'1': [(0, 0), (512, 2048)], '2': [(512, 0), (1024, 2048)], '3': [(1024, 0), (1536, 2048)], '4': [(1536, 0), (2048, 2048)]}) assert fullframe == fullframe_truth subarray = instrument_properties.amplifier_info(os.path.join(data_dir, 'test_image_1.fits')) subarray_truth = (1, {'1': [(0, 0), (10, 10)]}) assert subarray == subarray_truth subarray_one = instrument_properties.amplifier_info(os.path.join(data_dir, 'test_image_grismstripe_one_amp.fits')) subarray_one_truth = (1, {'1': [(4, 4), (2044, 64)]}) assert subarray_one == subarray_one_truth subarray_four = instrument_properties.amplifier_info(os.path.join(data_dir, 'test_image_grismstripe_four_amp.fits')) subarray_four_truth = (4, {'1': [(4, 4), (512, 64)], '2': [(512, 4), (1024, 64)], '3': [(1024, 4), (1536, 64)], '4': [(1536, 4), (2044, 64)]}) assert subarray_four == subarray_four_truth
def process(self, file_list): """The main method for processing darks. See module docstrings for further details. Parameters ---------- file_list : list List of filenames (including full paths) to the dark current files. """ for filename in file_list: logging.info('\tWorking on file: {}'.format(filename)) # Get relevant header information for this file self.get_metadata(filename) # Run the file through the necessary pipeline steps pipeline_steps = self.determine_pipeline_steps() logging.info('\tRunning pipeline on {}'.format(filename)) try: processed_file = pipeline_tools.run_calwebb_detector1_steps( filename, pipeline_steps) logging.info( '\tPipeline complete. Output: {}'.format(processed_file)) set_permissions(processed_file) except: logging.info( '\tPipeline processing failed for {}'.format(filename)) continue # Find amplifier boundaries so per-amp statistics can be calculated _, amp_bounds = instrument_properties.amplifier_info( processed_file, omit_reference_pixels=True) logging.info('\tAmplifier boundaries: {}'.format(amp_bounds)) # Get the ramp data; remove first 5 groups and last group for MIRI to avoid reset/rscd effects cal_data = fits.getdata(processed_file, 'SCI', uint=False) if self.instrument == 'MIRI': cal_data = cal_data[:, 5:-1, :, :] # Make the readnoise image readnoise_outfile = os.path.join( self.data_dir, os.path.basename( processed_file.replace('.fits', '_readnoise.fits'))) readnoise = self.make_readnoise_image(cal_data) fits.writeto(readnoise_outfile, readnoise, overwrite=True) logging.info( '\tReadnoise image saved to {}'.format(readnoise_outfile)) # Calculate the full image readnoise stats clipped = sigma_clip(readnoise, sigma=3.0, maxiters=5) full_image_mean, full_image_stddev = np.nanmean( clipped), np.nanstd(clipped) full_image_n, full_image_bin_centers = self.make_histogram( readnoise) logging.info('\tReadnoise image stats: {:.5f} +/- {:.5f}'.format( full_image_mean, full_image_stddev)) # Calculate readnoise stats in each amp separately amp_stats = self.get_amp_stats(readnoise, amp_bounds) logging.info( '\tReadnoise image stats by amp: {}'.format(amp_stats)) # Get the current JWST Readnoise Reference File data parameters = self.make_crds_parameter_dict() reffile_mapping = crds.getreferences(parameters, reftypes=['readnoise']) readnoise_file = reffile_mapping['readnoise'] if 'NOT FOUND' in readnoise_file: logging.warning( '\tNo pipeline readnoise reffile match for this file - assuming all zeros.' ) pipeline_readnoise = np.zeros(readnoise.shape) else: logging.info('\tPipeline readnoise reffile is {}'.format( readnoise_file)) pipeline_readnoise = fits.getdata(readnoise_file) # Find the difference between the current readnoise image and the pipeline readnoise reffile, and record image stats. # Sometimes, the pipeline readnoise reffile needs to be cutout to match the subarray. pipeline_readnoise = pipeline_readnoise[self.substrt2 - 1:self.substrt2 + self.subsize2 - 1, self.substrt1 - 1:self.substrt1 + self.subsize1 - 1] readnoise_diff = readnoise - pipeline_readnoise clipped = sigma_clip(readnoise_diff, sigma=3.0, maxiters=5) diff_image_mean, diff_image_stddev = np.nanmean( clipped), np.nanstd(clipped) diff_image_n, diff_image_bin_centers = self.make_histogram( readnoise_diff) logging.info( '\tReadnoise difference image stats: {:.5f} +/- {:.5f}'.format( diff_image_mean, diff_image_stddev)) # Save a png of the readnoise difference image for visual inspection logging.info('\tCreating png of readnoise difference image') readnoise_diff_png = self.image_to_png( readnoise_diff, outname=os.path.basename(readnoise_outfile).replace( '.fits', '_diff')) # Construct new entry for this file for the readnoise database table. # Can't insert values with numpy.float32 datatypes into database # so need to change the datatypes of these values. readnoise_db_entry = { 'uncal_filename': filename, 'aperture': self.aperture, 'detector': self.detector, 'subarray': self.subarray, 'read_pattern': self.read_pattern, 'nints': self.nints, 'ngroups': self.ngroups, 'expstart': self.expstart, 'readnoise_filename': readnoise_outfile, 'full_image_mean': float(full_image_mean), 'full_image_stddev': float(full_image_stddev), 'full_image_n': full_image_n.astype(float), 'full_image_bin_centers': full_image_bin_centers.astype(float), 'readnoise_diff_image': readnoise_diff_png, 'diff_image_mean': float(diff_image_mean), 'diff_image_stddev': float(diff_image_stddev), 'diff_image_n': diff_image_n.astype(float), 'diff_image_bin_centers': diff_image_bin_centers.astype(float), 'entry_date': datetime.datetime.now() } for key in amp_stats.keys(): if isinstance(amp_stats[key], (int, float)): readnoise_db_entry[key] = float(amp_stats[key]) else: readnoise_db_entry[key] = amp_stats[key].astype(float) # Add this new entry to the readnoise database table self.stats_table.__table__.insert().execute(readnoise_db_entry) logging.info('\tNew entry added to readnoise database table') # Remove the raw and calibrated files to save memory space os.remove(filename) os.remove(processed_file)
def process(self, file_list): """The main method for processing darks. See module docstrings for further details. Parameters ---------- file_list : list List of filenames (including full paths) to the dark current files """ for filename in file_list: logging.info('\tWorking on file: {}'.format(filename)) # Skip processing if an entry for this file already exists in # the bias stats database. file_exists = self.file_exists_in_database(filename) if file_exists: logging.info( '\t{} already exists in the bias database table.'.format( filename)) continue # Get the exposure start time of this file expstart = '{}T{}'.format( fits.getheader(filename, 0)['DATE-OBS'], fits.getheader(filename, 0)['TIME-OBS']) # Determine if the file needs group_scale in pipeline run read_pattern = fits.getheader(filename, 0)['READPATT'] if read_pattern not in pipeline_tools.GROUPSCALE_READOUT_PATTERNS: group_scale = False else: group_scale = True # Run the file through the pipeline up through the refpix step logging.info('\tRunning pipeline on {}'.format(filename)) processed_file = self.run_early_pipeline(filename, odd_even_rows=False, odd_even_columns=True, use_side_ref_pixels=True, group_scale=group_scale) logging.info( '\tPipeline complete. Output: {}'.format(processed_file)) # Find amplifier boundaries so per-amp statistics can be calculated _, amp_bounds = instrument_properties.amplifier_info( processed_file, omit_reference_pixels=True) logging.info('\tAmplifier boundaries: {}'.format(amp_bounds)) # Get the uncalibrated 0th group data for this file uncal_data = fits.getdata(filename, 'SCI')[0, 0, :, :].astype(float) # Calculate the uncal median values of each amplifier for odd/even columns amp_medians = self.get_amp_medians(uncal_data, amp_bounds) logging.info('\tCalculated uncalibrated image stats: {}'.format( amp_medians)) # Calculate image statistics and the collapsed row/column values # in the calibrated image cal_data = fits.getdata(processed_file, 'SCI')[0, 0, :, :] dq = fits.getdata(processed_file, 'PIXELDQ') mean, median, stddev = sigma_clipped_stats(cal_data[dq == 0], sigma=3.0, maxiters=5) logging.info( '\tCalculated calibrated image stats: {:.3f} +/- {:.3f}'. format(mean, stddev)) collapsed_rows, collapsed_columns = self.collapse_image(cal_data) logging.info( '\tCalculated collapsed row/column values of calibrated image.' ) # Save a png of the calibrated image for visual inspection logging.info('\tCreating png of calibrated image') output_png = self.image_to_png( cal_data, outname=os.path.basename(processed_file).replace('.fits', '')) # Construct new entry for this file for the bias database table. # Can't insert values with numpy.float32 datatypes into database # so need to change the datatypes of these values. bias_db_entry = { 'aperture': self.aperture, 'uncal_filename': filename, 'cal_filename': processed_file, 'cal_image': output_png, 'expstart': expstart, 'mean': float(mean), 'median': float(median), 'stddev': float(stddev), 'collapsed_rows': collapsed_rows.astype(float), 'collapsed_columns': collapsed_columns.astype(float), 'entry_date': datetime.datetime.now() } for key in amp_medians.keys(): bias_db_entry[key] = float(amp_medians[key]) # Add this new entry to the bias database table self.stats_table.__table__.insert().execute(bias_db_entry) logging.info('\tNew entry added to bias database table: {}'.format( bias_db_entry))
def process(self, file_list): """The main method for processing darks. See module docstrings for further details. Parameters ---------- file_list : list List of filenames (including full paths) to the dark current files. """ for filename in file_list: logging.info('\tWorking on file: {}'.format(filename)) # Get relevant header info for this file self.read_pattern = fits.getheader(filename, 0)['READPATT'] self.expstart = '{}T{}'.format( fits.getheader(filename, 0)['DATE-OBS'], fits.getheader(filename, 0)['TIME-OBS']) # Run the file through the necessary pipeline steps pipeline_steps = self.determine_pipeline_steps() logging.info('\tRunning pipeline on {}'.format(filename)) try: processed_file = pipeline_tools.run_calwebb_detector1_steps( filename, pipeline_steps) logging.info( '\tPipeline complete. Output: {}'.format(processed_file)) set_permissions(processed_file) except: logging.info( '\tPipeline processing failed for {}'.format(filename)) os.remove(filename) continue # Find amplifier boundaries so per-amp statistics can be calculated _, amp_bounds = instrument_properties.amplifier_info( processed_file, omit_reference_pixels=True) logging.info('\tAmplifier boundaries: {}'.format(amp_bounds)) # Get the uncalibrated 0th group data for this file uncal_data = fits.getdata(filename, 'SCI')[0, 0, :, :].astype(float) # Calculate the uncal median values of each amplifier for odd/even columns amp_medians = self.get_amp_medians(uncal_data, amp_bounds) logging.info('\tCalculated uncalibrated image stats: {}'.format( amp_medians)) # Calculate image statistics on the calibrated image cal_data = fits.getdata(processed_file, 'SCI')[0, 0, :, :] mean, median, stddev = sigma_clipped_stats(cal_data, sigma=3.0, maxiters=5) collapsed_rows, collapsed_columns = self.collapse_image(cal_data) counts, bin_centers = self.make_histogram(cal_data) logging.info( '\tCalculated calibrated image stats: {:.3f} +/- {:.3f}'. format(mean, stddev)) # Save a png of the calibrated image for visual inspection logging.info('\tCreating png of calibrated image') output_png = self.image_to_png( cal_data, outname=os.path.basename(processed_file).replace('.fits', '')) # Construct new entry for this file for the bias database table. # Can't insert values with numpy.float32 datatypes into database # so need to change the datatypes of these values. bias_db_entry = { 'aperture': self.aperture, 'uncal_filename': filename, 'cal_filename': processed_file, 'cal_image': output_png, 'expstart': self.expstart, 'mean': float(mean), 'median': float(median), 'stddev': float(stddev), 'collapsed_rows': collapsed_rows.astype(float), 'collapsed_columns': collapsed_columns.astype(float), 'counts': counts.astype(float), 'bin_centers': bin_centers.astype(float), 'entry_date': datetime.datetime.now() } for key in amp_medians.keys(): bias_db_entry[key] = float(amp_medians[key]) # Add this new entry to the bias database table self.stats_table.__table__.insert().execute(bias_db_entry) logging.info('\tNew entry added to bias database table: {}'.format( bias_db_entry)) # Remove the raw and calibrated files to save memory space os.remove(filename) os.remove(processed_file)
def process(self, file_list): """The main method for processing darks. See module docstrings for further details. Parameters ---------- file_list : list List of filenames (including full paths) to the dark current files """ # Basic metadata that will be needed later self.get_metadata(file_list[0]) # Determine which pipeline steps need to be executed required_steps = pipeline_tools.get_pipeline_steps(self.instrument) logging.info( '\tRequired calwebb1_detector pipeline steps to have the data in the ' 'correct format:') for item in required_steps: logging.info('\t\t{}: {}'.format(item, required_steps[item])) # Modify the list of pipeline steps to skip those not needed for the # preparation of dark current data required_steps['dark_current'] = False required_steps['persistence'] = False # NIRSpec IR^2 readout pattern NRSIRS2 is the only one with # nframes not a power of 2 if self.read_pattern not in pipeline_tools.GROUPSCALE_READOUT_PATTERNS: required_steps['group_scale'] = False # Run pipeline steps on files, generating slope files slope_files = [] for filename in file_list: completed_steps = pipeline_tools.completed_pipeline_steps(filename) steps_to_run = pipeline_tools.steps_to_run(required_steps, completed_steps) logging.info('\tWorking on file: {}'.format(filename)) logging.info('\tPipeline steps that remain to be run:') for item in steps_to_run: logging.info('\t\t{}: {}'.format(item, steps_to_run[item])) # Run any remaining required pipeline steps if any(steps_to_run.values()) is False: slope_files.append(filename) else: processed_file = filename.replace('.fits', '_{}.fits'.format('rate')) # If the slope file already exists, skip the pipeline call if not os.path.isfile(processed_file): logging.info('\tRunning pipeline on {}'.format(filename)) processed_file = pipeline_tools.run_calwebb_detector1_steps( os.path.abspath(filename), steps_to_run) logging.info('\tPipeline complete. Output: {}'.format( processed_file)) else: logging.info( '\tSlope file {} already exists. Skipping call to pipeline.' .format(processed_file)) pass slope_files.append(processed_file) # Delete the original dark ramp file to save disk space os.remove(filename) obs_times = [] logging.info( '\tSlope images to use in the dark monitor for {}, {}:'.format( self.instrument, self.aperture)) for item in slope_files: logging.info('\t\t{}'.format(item)) # Get the observation time for each file obstime = instrument_properties.get_obstime(item) obs_times.append(obstime) # Find the earliest and latest observation time, and calculate # the mid-time. min_time = np.min(obs_times) max_time = np.max(obs_times) mid_time = instrument_properties.mean_time(obs_times) # Read in all slope images and place into a list slope_image_stack, slope_exptimes = pipeline_tools.image_stack( slope_files) # Calculate a mean slope image from the inputs slope_image, stdev_image = calculations.mean_image(slope_image_stack, sigma_threshold=3) mean_slope_file = self.save_mean_slope_image(slope_image, stdev_image, slope_files) logging.info( '\tSigma-clipped mean of the slope images saved to: {}'.format( mean_slope_file)) # ----- Search for new hot/dead/noisy pixels ----- # Read in baseline mean slope image and stdev image # The baseline image is used to look for hot/dead/noisy pixels, # but not for comparing mean dark rates. Therefore, updates to # the baseline can be minimal. # Limit checks for hot/dead/noisy pixels to full frame data since # subarray data have much shorter exposure times and therefore lower # signal-to-noise aperture_type = Siaf(self.instrument)[self.aperture].AperType if aperture_type == 'FULLSCA': baseline_file = self.get_baseline_filename() if baseline_file is None: logging.warning(( '\tNo baseline dark current countrate image for {} {}. Setting the ' 'current mean slope image to be the new baseline.'.format( self.instrument, self.aperture))) baseline_file = mean_slope_file baseline_mean = deepcopy(slope_image) baseline_stdev = deepcopy(stdev_image) else: logging.info('\tBaseline file is {}'.format(baseline_file)) baseline_mean, baseline_stdev = self.read_baseline_slope_image( baseline_file) # Check the hot/dead pixel population for changes new_hot_pix, new_dead_pix = self.find_hot_dead_pixels( slope_image, baseline_mean) # Shift the coordinates to be in full frame coordinate system new_hot_pix = self.shift_to_full_frame(new_hot_pix) new_dead_pix = self.shift_to_full_frame(new_dead_pix) # Exclude hot and dead pixels found previously new_hot_pix = self.exclude_existing_badpix(new_hot_pix, 'hot') new_dead_pix = self.exclude_existing_badpix(new_dead_pix, 'dead') # Add new hot and dead pixels to the database logging.info('\tFound {} new hot pixels'.format(len( new_hot_pix[0]))) logging.info('\tFound {} new dead pixels'.format( len(new_dead_pix[0]))) self.add_bad_pix(new_hot_pix, 'hot', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time) self.add_bad_pix(new_dead_pix, 'dead', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time) # Check for any pixels that are significantly more noisy than # in the baseline stdev image new_noisy_pixels = self.noise_check(stdev_image, baseline_stdev) # Shift coordinates to be in full_frame coordinate system new_noisy_pixels = self.shift_to_full_frame(new_noisy_pixels) # Exclude previously found noisy pixels new_noisy_pixels = self.exclude_existing_badpix( new_noisy_pixels, 'noisy') # Add new noisy pixels to the database logging.info('\tFound {} new noisy pixels'.format( len(new_noisy_pixels[0]))) self.add_bad_pix(new_noisy_pixels, 'noisy', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time) # ----- Calculate image statistics ----- # Find amplifier boundaries so per-amp statistics can be calculated number_of_amps, amp_bounds = instrument_properties.amplifier_info( slope_files[0]) logging.info('\tAmplifier boundaries: {}'.format(amp_bounds)) # Calculate mean and stdev values, and fit a Gaussian to the # histogram of the pixels in each amp (amp_mean, amp_stdev, gauss_param, gauss_chisquared, double_gauss_params, double_gauss_chisquared, histogram, bins) = self.stats_by_amp(slope_image, amp_bounds) # Construct new entry for dark database table source_files = [os.path.basename(item) for item in file_list] for key in amp_mean.keys(): dark_db_entry = { 'aperture': self.aperture, 'amplifier': key, 'mean': amp_mean[key], 'stdev': amp_stdev[key], 'source_files': source_files, 'obs_start_time': min_time, 'obs_mid_time': mid_time, 'obs_end_time': max_time, 'gauss_amplitude': list(gauss_param[key][0]), 'gauss_peak': list(gauss_param[key][1]), 'gauss_width': list(gauss_param[key][2]), 'gauss_chisq': gauss_chisquared[key], 'double_gauss_amplitude1': double_gauss_params[key][0], 'double_gauss_peak1': double_gauss_params[key][1], 'double_gauss_width1': double_gauss_params[key][2], 'double_gauss_amplitude2': double_gauss_params[key][3], 'double_gauss_peak2': double_gauss_params[key][4], 'double_gauss_width2': double_gauss_params[key][5], 'double_gauss_chisq': double_gauss_chisquared[key], 'mean_dark_image_file': os.path.basename(mean_slope_file), 'hist_dark_values': bins, 'hist_amplitudes': histogram, 'entry_date': datetime.datetime.now() } self.stats_table.__table__.insert().execute(dark_db_entry)