def callalgorithm(self): """Call the bpm algorithm. The only requirement is that the output reference file is saved as self.args.outputreffilename mkrefs.py will supply the input files in self.inputimages['output_name']. This will be a list containing the filenames to use as input. The file types (e.g. dark, flat) associated with each filename are contained in self.inputimages['imtype']. From this, you can specify the appropriate file names in the call to your module. """ # Organize the input files into a group of darks and a group of # flats flatfiles = [] darkfiles = [] for row in self.inputimagestable: if row['imlabel'] == 'D': darkfiles.append(row['fitsfile']) elif row['imlabel'] == 'F': flatfiles.append(row['fitsfile']) # Since this module is called after calib_prep, all of the requested # outputs from the various pipeline steps should be present in the # output directory. Create lists of these files. The files listed in # self.inputimagestable['fitsfile'] should all be the same in terms # of their calibration state. So we should only have to check one in # order to know what state all of them are. dark_slope_files = [] dark_uncal_files = [] dark_jump_files = [] dark_fitopt_files = [] directory, filename = os.path.split(darkfiles[0]) # Get the suffix of the input file so we know the calibration state suffix = None for ramp_suffix in RATE_FILE_SUFFIXES: if ramp_suffix in filename: dark_slope_files = copy.deepcopy(darkfiles) suffix = ramp_suffix if suffix is None: suffix = filename.split('_')[-1] suffix = suffix.replace('.fits', '') if suffix == 'uncal': dark_uncal_files = copy.deepcopy(darkfiles) elif suffix == 'jump': dark_jump_files = copy.deepcopy(darkfiles) else: raise ValueError('Unexpected suffixes for input dark files.') # Create lists of the needed calibration state files if len(dark_slope_files) > 0: dark_uncal_files = [ elem.replace(suffix, '_uncal') for elem in dark_slope_files ] dark_jump_files = [ elem.replace(suffix, '_jump') for elem in dark_slope_files ] dark_fitopt_files = [ elem.replace(suffix, '_fitopt') for elem in dark_slope_files ] elif len(dark_uncal_files) > 0: dark_slope_files = [ elem.replace(suffix, '_1_ramp_fit') for elem in dark_uncal_files ] dark_jump_files = [ elem.replace(suffix, '_jump') for elem in dark_uncal_files ] dark_fitopt_files = [ elem.replace(suffix, '_fitopt') for elem in dark_uncal_files ] elif len(dark_jump_files) > 0: dark_uncal_files = [ elem.replace(suffix, '_uncal') for elem in dark_jump_files ] dark_slope_files = [ elem.replace(suffix, '_1_ramp_fit') for elem in dark_jump_files ] dark_fitopt_files = [ elem.replace(suffix, '_fitopt') for elem in dark_jump_files ] # Repeat for flat field files flat_slope_files = [] flat_uncal_files = [] directory, filename = os.path.split(flatfiles[0]) # Get the suffix of the input file so we know the calibration state suffix = None for ramp_suffix in RATE_FILE_SUFFIXES: if ramp_suffix in filename: flat_slope_files = copy.deepcopy(flatfiles) suffix = ramp_suffix if suffix is None: suffix = filename.split('_')[-1] suffix = suffix.replace('.fits', '') if suffix == 'uncal': flat_uncal_files = copy.deepcopy(flatfiles) else: raise ValueError( 'Unexpected suffixes for input flat field files.') # Create lists of the needed calibration state files if len(flat_slope_files) > 0: flat_uncal_files = [ elem.replace(suffix, '_uncal') for elem in flat_slope_files ] elif len(flat_uncal_files) > 0: flat_slope_files = [ elem.replace(suffix, '_1_ramp_fit') for elem in flat_uncal_files ] # The bad pixel mask module needs to use the file with the individual # slopes (_1_ramp_fit.fits), rather than the mean slope (_0_ramp_fit.fits), # for exposures with more than one integration per exposure. But for # exposures with only one integration, only the _0_ramp_fit file will be # produced. So go through the lists of slope files and check to see # which versions are present, and adjust the lists accordingly. # Call the wrapped module and provide the proper arguments from the # self.parameters dictionary. bpm.bad_pixels( flat_slope_files=flat_slope_files, dead_search=self.parameters['dead_search'], low_qe_and_open_search=self.parameters['low_qe_and_open_search'], dead_search_type=self.parameters['dead_search_type'], flat_mean_sigma_threshold=self. parameters['flat_mean_sigma_threshold'], flat_mean_normalization_method=self. parameters['flat_mean_normalization_method'], smoothing_box_width=self.parameters['smoothing_box_width'], smoothing_type=self.parameters['smoothing_type'], dead_sigma_threshold=self.parameters['dead_sigma_threshold'], max_dead_norm_signal=self.parameters['max_dead_norm_signal'], run_dead_flux_check=self.parameters['run_dead_flux_check'], dead_flux_check_files=flat_uncal_files, flux_check=self.parameters['flux_check'], max_low_qe_norm_signal=self.parameters['max_low_qe_norm_signal'], max_open_adj_norm_signal=self. parameters['max_open_adj_norm_signal'], manual_flag_file=self.parameters['manual_flag_file'], flat_do_not_use=self.parameters['flat_do_not_use'], dark_slope_files=dark_slope_files, dark_uncal_files=dark_uncal_files, dark_jump_files=dark_jump_files, dark_fitopt_files=dark_fitopt_files, dark_stdev_clipping_sigma=self. parameters['dark_stdev_clipping_sigma'], dark_max_clipping_iters=self.parameters['dark_max_clipping_iters'], dark_noisy_threshold=self.parameters['dark_noisy_threshold'], max_saturated_fraction=self.parameters['max_saturated_fraction'], max_jump_limit=self.parameters['max_jump_limit'], jump_ratio_threshold=self.parameters['jump_ratio_threshold'], early_cutoff_fraction=self.parameters['early_cutoff_fraction'], pedestal_sigma_threshold=self. parameters['pedestal_sigma_threshold'], rc_fraction_threshold=self.parameters['rc_fraction_threshold'], low_pedestal_fraction=self.parameters['low_pedestal_fraction'], high_cr_fraction=self.parameters['high_cr_fraction'], flag_values=self.parameters['flag_values'], dark_do_not_use=self.parameters['dark_do_not_use'], plot=self.parameters['plot'], output_file=self.args.outputreffilename, author=self.parameters['author'], description=self.parameters['description'], pedigree=self.parameters['pedigree'], useafter=self.parameters['useafter'], history=self.parameters['history'], quality_check=self.parameters['quality_check']) return (0)
def process(self, illuminated_raw_files, illuminated_slope_files, dark_raw_files, dark_slope_files): """The main method for processing darks. See module docstrings for further details. Parameters ---------- illuminated_raw_files : list List of filenames (including full paths) of raw (uncal) flat field files. These should all be for the same detector and aperture. illuminated_slope_files : list List of filenames (including full paths) of flat field slope files. These should all be for the same detector and aperture and correspond one-to-one with ``illuminated_raw_files``. For cases where a raw file exists but no slope file, the slope file should be None dark_raw_files : list List of filenames (including full paths) of raw (uncal) dark files. These should all be for the same detector and aperture. dark_slope_files : list List of filenames (including full paths) of dark current slope files. These should all be for the same detector and aperture and correspond one-to-one with ``dark_raw_files``. For cases where a raw file exists but no slope file, the slope file should be ``None`` """ # Illuminated files - run entirety of calwebb_detector1 for uncal # files where corresponding rate file is 'None' all_files = [] badpix_types = [] badpix_types_from_flats = ['DEAD', 'LOW_QE', 'OPEN', 'ADJ_OPEN'] badpix_types_from_darks = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH'] illuminated_obstimes = [] if illuminated_raw_files: index = 0 badpix_types.extend(badpix_types_from_flats) for uncal_file, rate_file in zip(illuminated_raw_files, illuminated_slope_files): self.get_metadata(uncal_file) if rate_file == 'None': jump_output, rate_output, _ = pipeline_tools.calwebb_detector1_save_jump(uncal_file, self.data_dir, ramp_fit=True, save_fitopt=False) if self.nints > 1: illuminated_slope_files[index] = rate_output.replace('0_ramp_fit', '1_ramp_fit') else: illuminated_slope_files[index] = deepcopy(rate_output) index += 1 # Get observation time for all files illuminated_obstimes.append(instrument_properties.get_obstime(uncal_file)) all_files = deepcopy(illuminated_slope_files) min_illum_time = min(illuminated_obstimes) max_illum_time = max(illuminated_obstimes) mid_illum_time = instrument_properties.mean_time(illuminated_obstimes) # Dark files - Run calwebb_detector1 on all uncal files, saving the # Jump step output. If corresponding rate file is 'None', then also # run the ramp-fit step and save the output dark_jump_files = [] dark_fitopt_files = [] dark_obstimes = [] if dark_raw_files: index = 0 badpix_types.extend(badpix_types_from_darks) # In this case we need to run the pipeline on all input files, # even if the rate file is present, because we also need the jump # and fitops files, which are not saved by default for uncal_file, rate_file in zip(dark_raw_files, dark_slope_files): jump_output, rate_output, fitopt_output = pipeline_tools.calwebb_detector1_save_jump(uncal_file, self.data_dir, ramp_fit=True, save_fitopt=True) self.get_metadata(uncal_file) dark_jump_files.append(jump_output) dark_fitopt_files.append(fitopt_output) if self.nints > 1: #dark_slope_files[index] = rate_output.replace('rate', 'rateints') dark_slope_files[index] = rate_output.replace('0_ramp_fit', '1_ramp_fit') else: dark_slope_files[index] = deepcopy(rate_output) dark_obstimes.append(instrument_properties.get_obstime(uncal_file)) index += 1 if len(all_files) == 0: all_files = deepcopy(dark_slope_files) else: all_files = all_files + dark_slope_files min_dark_time = min(dark_obstimes) max_dark_time = max(dark_obstimes) mid_dark_time = instrument_properties.mean_time(dark_obstimes) # For the dead flux check, filter out any files that have less than # 4 groups dead_flux_files = [] if illuminated_raw_files: for illum_file in illuminated_raw_files: ngroup = fits.getheader(illum_file)['NGROUPS'] if ngroup >= 4: dead_flux_files.append(illum_file) if len(dead_flux_files) == 0: dead_flux_files = None # Instrument-specific preferences from jwst_reffiles meetings if self.instrument in ['nircam', 'niriss', 'fgs']: dead_search_type = 'sigma_rate' elif self.instrument in ['miri', 'nirspec']: dead_search_type = 'absolute_rate' flat_mean_normalization_method = 'smoothed' # Call the bad pixel search module from jwst_reffiles. Lots of # other possible parameters. Only specify the non-default params # in order to make things easier to read. query_string = 'darks_{}_flats_{}_to_{}'.format(self.dark_query_start, self.flat_query_start, self.query_end) output_file = '{}_{}_{}_bpm.fits'.format(self.instrument, self.aperture, query_string) output_file = os.path.join(self.output_dir, output_file) bad_pixel_mask.bad_pixels(flat_slope_files=illuminated_slope_files, dead_search_type=dead_search_type, flat_mean_normalization_method=flat_mean_normalization_method, run_dead_flux_check=True, dead_flux_check_files=dead_flux_files, flux_check=35000, dark_slope_files=dark_slope_files, dark_uncal_files=dark_raw_files, dark_jump_files=dark_jump_files, dark_fitopt_files=dark_fitopt_files, plot=False, output_file=output_file, author='jwst_reffiles', description='A bad pix mask', pedigree='GROUND', useafter='2222-04-01 00:00:00', history='This file was created by JWQL', quality_check=False) # Read in the newly-created bad pixel file set_permissions(output_file) badpix_map = fits.getdata(output_file) # Locate and read in the current bad pixel mask parameters = self.make_crds_parameter_dict() mask_dictionary = crds_tools.get_reffiles(parameters, ['mask'], download=True) baseline_file = mask_dictionary['mask'] if 'NOT FOUND' in baseline_file: logging.warning(('\tNo baseline bad pixel file for {} {}. Any bad ' 'pixels found as part of this search will be considered new'.format(self.instrument, self.aperture))) baseline_file = new_badpix_file yd, xd = badpix_mask.shape baseline_badpix_mask = np.zeros((yd, xd), type=np.int) else: logging.info('\tBaseline bad pixel file is {}'.format(baseline_file)) baseline_badpix_mask = fits.getdata(baseline_file) # Exclude hot and dead pixels in the current bad pixel mask #new_hot_pix = self.exclude_existing_badpix(new_hot_pix, 'hot') new_since_reffile = exclude_crds_mask_pix(badpix_map, baseline_badpix_mask) # Create a list of the new instances of each type of bad pixel for bad_type in badpix_types: bad_location_list = bad_map_to_list(new_since_reffile, bad_type) # Add new hot and dead pixels to the database logging.info('\tFound {} new {} pixels'.format(len(bad_location_list[0]), bad_type)) if bad_type in badpix_types_from_flats: self.add_bad_pix(bad_location_list, bad_type, illuminated_slope_files, min_illum_time, mid_illum_time, max_illum_time, baseline_file) elif bad_type in badpix_types_from_darks: self.add_bad_pix(bad_location_list, bad_type, dark_slope_files, min_dark_time, mid_dark_time, max_dark_time, baseline_file) else: raise ValueError("Unrecognized type of bad pixel: {}. Cannot update database table.".format(bad_type))