def test_completed_pipeline_steps(): """Test that the list of completed pipeline steps for a file is correct Parameters ---------- filename : str File to be checked """ filename = os.path.join(get_config()['filesystem'], 'jw00312', 'jw00312002001_02102_00001_nrcb4_rateints.fits') completed_steps = pipeline_tools.completed_pipeline_steps(filename) true_completed = OrderedDict([('group_scale', False), ('dq_init', True), ('saturation', True), ('ipc', False), ('refpix', True), ('superbias', True), ('persistence', True), ('dark_current', True), ('linearity', True), ('firstframe', False), ('lastframe', False), ('rscd', False), ('jump', True), ('rate', True)]) # Only test steps that have a value of True completed_steps = OrderedDict( (k, v) for k, v in completed_steps.items() if v is True) true_completed = OrderedDict( (k, v) for k, v in true_completed.items() if v is True) assert completed_steps == true_completed
def process(self, file_list): """The main method for processing darks. See module docstrings for further details. Parameters ---------- file_list : list List of filenames (including full paths) to the dark current files """ # Basic metadata that will be needed later self.get_metadata(file_list[0]) # Determine which pipeline steps need to be executed required_steps = pipeline_tools.get_pipeline_steps(self.instrument) logging.info( '\tRequired calwebb1_detector pipeline steps to have the data in the ' 'correct format:') for item in required_steps: logging.info('\t\t{}: {}'.format(item, required_steps[item])) # Modify the list of pipeline steps to skip those not needed for the # preparation of dark current data required_steps['dark_current'] = False required_steps['persistence'] = False # NIRSpec IR^2 readout pattern NRSIRS2 is the only one with # nframes not a power of 2 if self.read_pattern not in pipeline_tools.GROUPSCALE_READOUT_PATTERNS: required_steps['group_scale'] = False # Run pipeline steps on files, generating slope files slope_files = [] for filename in file_list: completed_steps = pipeline_tools.completed_pipeline_steps(filename) steps_to_run = pipeline_tools.steps_to_run(required_steps, completed_steps) logging.info('\tWorking on file: {}'.format(filename)) logging.info('\tPipeline steps that remain to be run:') for item in steps_to_run: logging.info('\t\t{}: {}'.format(item, steps_to_run[item])) # Run any remaining required pipeline steps if any(steps_to_run.values()) is False: slope_files.append(filename) else: processed_file = filename.replace('.fits', '_{}.fits'.format('rate')) # If the slope file already exists, skip the pipeline call if not os.path.isfile(processed_file): logging.info('\tRunning pipeline on {}'.format(filename)) processed_file = pipeline_tools.run_calwebb_detector1_steps( os.path.abspath(filename), steps_to_run) logging.info('\tPipeline complete. Output: {}'.format( processed_file)) else: logging.info( '\tSlope file {} already exists. Skipping call to pipeline.' .format(processed_file)) pass slope_files.append(processed_file) # Delete the original dark ramp file to save disk space os.remove(filename) obs_times = [] logging.info( '\tSlope images to use in the dark monitor for {}, {}:'.format( self.instrument, self.aperture)) for item in slope_files: logging.info('\t\t{}'.format(item)) # Get the observation time for each file obstime = instrument_properties.get_obstime(item) obs_times.append(obstime) # Find the earliest and latest observation time, and calculate # the mid-time. min_time = np.min(obs_times) max_time = np.max(obs_times) mid_time = instrument_properties.mean_time(obs_times) # Read in all slope images and place into a list slope_image_stack, slope_exptimes = pipeline_tools.image_stack( slope_files) # Calculate a mean slope image from the inputs slope_image, stdev_image = calculations.mean_image(slope_image_stack, sigma_threshold=3) mean_slope_file = self.save_mean_slope_image(slope_image, stdev_image, slope_files) logging.info( '\tSigma-clipped mean of the slope images saved to: {}'.format( mean_slope_file)) # ----- Search for new hot/dead/noisy pixels ----- # Read in baseline mean slope image and stdev image # The baseline image is used to look for hot/dead/noisy pixels, # but not for comparing mean dark rates. Therefore, updates to # the baseline can be minimal. # Limit checks for hot/dead/noisy pixels to full frame data since # subarray data have much shorter exposure times and therefore lower # signal-to-noise aperture_type = Siaf(self.instrument)[self.aperture].AperType if aperture_type == 'FULLSCA': baseline_file = self.get_baseline_filename() if baseline_file is None: logging.warning(( '\tNo baseline dark current countrate image for {} {}. Setting the ' 'current mean slope image to be the new baseline.'.format( self.instrument, self.aperture))) baseline_file = mean_slope_file baseline_mean = deepcopy(slope_image) baseline_stdev = deepcopy(stdev_image) else: logging.info('\tBaseline file is {}'.format(baseline_file)) baseline_mean, baseline_stdev = self.read_baseline_slope_image( baseline_file) # Check the hot/dead pixel population for changes new_hot_pix, new_dead_pix = self.find_hot_dead_pixels( slope_image, baseline_mean) # Shift the coordinates to be in full frame coordinate system new_hot_pix = self.shift_to_full_frame(new_hot_pix) new_dead_pix = self.shift_to_full_frame(new_dead_pix) # Exclude hot and dead pixels found previously new_hot_pix = self.exclude_existing_badpix(new_hot_pix, 'hot') new_dead_pix = self.exclude_existing_badpix(new_dead_pix, 'dead') # Add new hot and dead pixels to the database logging.info('\tFound {} new hot pixels'.format(len( new_hot_pix[0]))) logging.info('\tFound {} new dead pixels'.format( len(new_dead_pix[0]))) self.add_bad_pix(new_hot_pix, 'hot', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time) self.add_bad_pix(new_dead_pix, 'dead', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time) # Check for any pixels that are significantly more noisy than # in the baseline stdev image new_noisy_pixels = self.noise_check(stdev_image, baseline_stdev) # Shift coordinates to be in full_frame coordinate system new_noisy_pixels = self.shift_to_full_frame(new_noisy_pixels) # Exclude previously found noisy pixels new_noisy_pixels = self.exclude_existing_badpix( new_noisy_pixels, 'noisy') # Add new noisy pixels to the database logging.info('\tFound {} new noisy pixels'.format( len(new_noisy_pixels[0]))) self.add_bad_pix(new_noisy_pixels, 'noisy', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time) # ----- Calculate image statistics ----- # Find amplifier boundaries so per-amp statistics can be calculated number_of_amps, amp_bounds = instrument_properties.amplifier_info( slope_files[0]) logging.info('\tAmplifier boundaries: {}'.format(amp_bounds)) # Calculate mean and stdev values, and fit a Gaussian to the # histogram of the pixels in each amp (amp_mean, amp_stdev, gauss_param, gauss_chisquared, double_gauss_params, double_gauss_chisquared, histogram, bins) = self.stats_by_amp(slope_image, amp_bounds) # Construct new entry for dark database table source_files = [os.path.basename(item) for item in file_list] for key in amp_mean.keys(): dark_db_entry = { 'aperture': self.aperture, 'amplifier': key, 'mean': amp_mean[key], 'stdev': amp_stdev[key], 'source_files': source_files, 'obs_start_time': min_time, 'obs_mid_time': mid_time, 'obs_end_time': max_time, 'gauss_amplitude': list(gauss_param[key][0]), 'gauss_peak': list(gauss_param[key][1]), 'gauss_width': list(gauss_param[key][2]), 'gauss_chisq': gauss_chisquared[key], 'double_gauss_amplitude1': double_gauss_params[key][0], 'double_gauss_peak1': double_gauss_params[key][1], 'double_gauss_width1': double_gauss_params[key][2], 'double_gauss_amplitude2': double_gauss_params[key][3], 'double_gauss_peak2': double_gauss_params[key][4], 'double_gauss_width2': double_gauss_params[key][5], 'double_gauss_chisq': double_gauss_chisquared[key], 'mean_dark_image_file': os.path.basename(mean_slope_file), 'hist_dark_values': bins, 'hist_amplitudes': histogram, 'entry_date': datetime.datetime.now() } self.stats_table.__table__.insert().execute(dark_db_entry)