def test_get_pipeline_steps():
"""Test that the proper pipeline steps are returned for an
instrument
"""
# FGS, NIRCam, and NIRISS have the same required steps
instruments = ['fgs', 'nircam', 'niriss']
for instrument in instruments:
req_steps = pipeline_tools.get_pipeline_steps(instrument)
steps = [
'dq_init', 'saturation', 'superbias', 'refpix', 'linearity',
'persistence', 'dark_current', 'jump', 'rate'
]
not_required = [
'group_scale', 'ipc', 'firstframe', 'lastframe', 'rscd'
]
steps_dict = OrderedDict({})
for step in steps:
steps_dict[step] = True
for step in not_required:
steps_dict[step] = False
assert req_steps == steps_dict
# NIRSpec and MIRI have different required steps
nrs_req_steps = pipeline_tools.get_pipeline_steps('nirspec')
nrs_steps = [
'group_scale', 'dq_init', 'saturation', 'superbias', 'refpix',
'linearity', 'dark_current', 'jump', 'rate'
]
not_required = ['ipc', 'persistence', 'firstframe', 'lastframe', 'rscd']
nrs_dict = OrderedDict({})
for step in nrs_steps:
nrs_dict[step] = True
for step in not_required:
nrs_dict[step] = False
assert nrs_req_steps == nrs_dict
miri_req_steps = pipeline_tools.get_pipeline_steps('miri')
miri_steps = [
'dq_init', 'saturation', 'firstframe', 'lastframe', 'linearity',
'rscd', 'dark_current', 'refpix', 'jump', 'rate'
]
not_required = ['group_scale', 'ipc', 'superbias', 'persistence']
miri_dict = OrderedDict({})
for step in miri_steps:
miri_dict[step] = True
for step in not_required:
miri_dict[step] = False
assert miri_req_steps == miri_dict
def test_get_pipeline_steps():
"""Test that the proper pipeline steps are returned for an
instrument
"""
# FGS, NIRCam, and NIRISS
instruments = ['fgs', 'nircam', 'niriss']
for instrument in instruments:
req_steps = pipeline_tools.get_pipeline_steps(instrument)
steps = [
'dq_init', 'saturation', 'superbias', 'refpix', 'linearity',
'persistence', 'dark_current', 'jump', 'rate'
]
not_required = [
'group_scale', 'ipc', 'firstframe', 'lastframe', 'rscd'
]
steps_dict = OrderedDict({})
for step in steps:
steps_dict[step] = True
for step in not_required:
steps_dict[step] = False
# Only test steps that have a value of True
req_steps = OrderedDict(
(k, v) for k, v in req_steps.items() if v is True)
steps_dict = OrderedDict(
(k, v) for k, v in steps_dict.items() if v is True)
assert req_steps == steps_dict
# NIRSpec
nrs_req_steps = pipeline_tools.get_pipeline_steps('nirspec')
nrs_steps = [
'group_scale', 'dq_init', 'saturation', 'superbias', 'refpix',
'linearity', 'dark_current', 'jump', 'rate'
]
not_required = ['ipc', 'persistence', 'firstframe', 'lastframe', 'rscd']
nrs_dict = OrderedDict({})
for step in nrs_steps:
nrs_dict[step] = True
for step in not_required:
nrs_dict[step] = False
# Only test steps that have a value of True
nrs_req_steps = OrderedDict(
(k, v) for k, v in nrs_req_steps.items() if v is True)
nrs_dict = OrderedDict((k, v) for k, v in nrs_dict.items() if v is True)
assert nrs_req_steps == nrs_dict
# MIRI
miri_req_steps = pipeline_tools.get_pipeline_steps('miri')
miri_steps = [
'dq_init', 'saturation', 'firstframe', 'lastframe', 'linearity',
'rscd', 'dark_current', 'refpix', 'jump', 'rate'
]
not_required = ['group_scale', 'ipc', 'superbias', 'persistence']
miri_dict = OrderedDict({})
for step in miri_steps:
miri_dict[step] = True
for step in not_required:
miri_dict[step] = False
# Only test steps that have a value of True
miri_req_steps = OrderedDict(
(k, v) for k, v in miri_req_steps.items() if v is True)
miri_dict = OrderedDict((k, v) for k, v in miri_dict.items() if v is True)
assert miri_req_steps == miri_dict
def process(self, file_list):
"""The main method for processing darks. See module docstrings
for further details.
Parameters
----------
file_list : list
List of filenames (including full paths) to the dark current
files
"""
# Basic metadata that will be needed later
self.get_metadata(file_list[0])
# Determine which pipeline steps need to be executed
required_steps = pipeline_tools.get_pipeline_steps(self.instrument)
logging.info(
'\tRequired calwebb1_detector pipeline steps to have the data in the '
'correct format:')
for item in required_steps:
logging.info('\t\t{}: {}'.format(item, required_steps[item]))
# Modify the list of pipeline steps to skip those not needed for the
# preparation of dark current data
required_steps['dark_current'] = False
required_steps['persistence'] = False
# NIRSpec IR^2 readout pattern NRSIRS2 is the only one with
# nframes not a power of 2
if self.read_pattern not in pipeline_tools.GROUPSCALE_READOUT_PATTERNS:
required_steps['group_scale'] = False
# Run pipeline steps on files, generating slope files
slope_files = []
for filename in file_list:
completed_steps = pipeline_tools.completed_pipeline_steps(filename)
steps_to_run = pipeline_tools.steps_to_run(required_steps,
completed_steps)
logging.info('\tWorking on file: {}'.format(filename))
logging.info('\tPipeline steps that remain to be run:')
for item in steps_to_run:
logging.info('\t\t{}: {}'.format(item, steps_to_run[item]))
# Run any remaining required pipeline steps
if any(steps_to_run.values()) is False:
slope_files.append(filename)
else:
processed_file = filename.replace('.fits',
'_{}.fits'.format('rate'))
# If the slope file already exists, skip the pipeline call
if not os.path.isfile(processed_file):
logging.info('\tRunning pipeline on {}'.format(filename))
processed_file = pipeline_tools.run_calwebb_detector1_steps(
os.path.abspath(filename), steps_to_run)
logging.info('\tPipeline complete. Output: {}'.format(
processed_file))
else:
logging.info(
'\tSlope file {} already exists. Skipping call to pipeline.'
.format(processed_file))
pass
slope_files.append(processed_file)
# Delete the original dark ramp file to save disk space
os.remove(filename)
obs_times = []
logging.info(
'\tSlope images to use in the dark monitor for {}, {}:'.format(
self.instrument, self.aperture))
for item in slope_files:
logging.info('\t\t{}'.format(item))
# Get the observation time for each file
obstime = instrument_properties.get_obstime(item)
obs_times.append(obstime)
# Find the earliest and latest observation time, and calculate
# the mid-time.
min_time = np.min(obs_times)
max_time = np.max(obs_times)
mid_time = instrument_properties.mean_time(obs_times)
# Read in all slope images and place into a list
slope_image_stack, slope_exptimes = pipeline_tools.image_stack(
slope_files)
# Calculate a mean slope image from the inputs
slope_image, stdev_image = calculations.mean_image(slope_image_stack,
sigma_threshold=3)
mean_slope_file = self.save_mean_slope_image(slope_image, stdev_image,
slope_files)
logging.info(
'\tSigma-clipped mean of the slope images saved to: {}'.format(
mean_slope_file))
# ----- Search for new hot/dead/noisy pixels -----
# Read in baseline mean slope image and stdev image
# The baseline image is used to look for hot/dead/noisy pixels,
# but not for comparing mean dark rates. Therefore, updates to
# the baseline can be minimal.
# Limit checks for hot/dead/noisy pixels to full frame data since
# subarray data have much shorter exposure times and therefore lower
# signal-to-noise
aperture_type = Siaf(self.instrument)[self.aperture].AperType
if aperture_type == 'FULLSCA':
baseline_file = self.get_baseline_filename()
if baseline_file is None:
logging.warning((
'\tNo baseline dark current countrate image for {} {}. Setting the '
'current mean slope image to be the new baseline.'.format(
self.instrument, self.aperture)))
baseline_file = mean_slope_file
baseline_mean = deepcopy(slope_image)
baseline_stdev = deepcopy(stdev_image)
else:
logging.info('\tBaseline file is {}'.format(baseline_file))
baseline_mean, baseline_stdev = self.read_baseline_slope_image(
baseline_file)
# Check the hot/dead pixel population for changes
new_hot_pix, new_dead_pix = self.find_hot_dead_pixels(
slope_image, baseline_mean)
# Shift the coordinates to be in full frame coordinate system
new_hot_pix = self.shift_to_full_frame(new_hot_pix)
new_dead_pix = self.shift_to_full_frame(new_dead_pix)
# Exclude hot and dead pixels found previously
new_hot_pix = self.exclude_existing_badpix(new_hot_pix, 'hot')
new_dead_pix = self.exclude_existing_badpix(new_dead_pix, 'dead')
# Add new hot and dead pixels to the database
logging.info('\tFound {} new hot pixels'.format(len(
new_hot_pix[0])))
logging.info('\tFound {} new dead pixels'.format(
len(new_dead_pix[0])))
self.add_bad_pix(new_hot_pix, 'hot', file_list, mean_slope_file,
baseline_file, min_time, mid_time, max_time)
self.add_bad_pix(new_dead_pix, 'dead', file_list, mean_slope_file,
baseline_file, min_time, mid_time, max_time)
# Check for any pixels that are significantly more noisy than
# in the baseline stdev image
new_noisy_pixels = self.noise_check(stdev_image, baseline_stdev)
# Shift coordinates to be in full_frame coordinate system
new_noisy_pixels = self.shift_to_full_frame(new_noisy_pixels)
# Exclude previously found noisy pixels
new_noisy_pixels = self.exclude_existing_badpix(
new_noisy_pixels, 'noisy')
# Add new noisy pixels to the database
logging.info('\tFound {} new noisy pixels'.format(
len(new_noisy_pixels[0])))
self.add_bad_pix(new_noisy_pixels, 'noisy', file_list,
mean_slope_file, baseline_file, min_time,
mid_time, max_time)
# ----- Calculate image statistics -----
# Find amplifier boundaries so per-amp statistics can be calculated
number_of_amps, amp_bounds = instrument_properties.amplifier_info(
slope_files[0])
logging.info('\tAmplifier boundaries: {}'.format(amp_bounds))
# Calculate mean and stdev values, and fit a Gaussian to the
# histogram of the pixels in each amp
(amp_mean, amp_stdev, gauss_param, gauss_chisquared,
double_gauss_params, double_gauss_chisquared, histogram,
bins) = self.stats_by_amp(slope_image, amp_bounds)
# Construct new entry for dark database table
source_files = [os.path.basename(item) for item in file_list]
for key in amp_mean.keys():
dark_db_entry = {
'aperture': self.aperture,
'amplifier': key,
'mean': amp_mean[key],
'stdev': amp_stdev[key],
'source_files': source_files,
'obs_start_time': min_time,
'obs_mid_time': mid_time,
'obs_end_time': max_time,
'gauss_amplitude': list(gauss_param[key][0]),
'gauss_peak': list(gauss_param[key][1]),
'gauss_width': list(gauss_param[key][2]),
'gauss_chisq': gauss_chisquared[key],
'double_gauss_amplitude1': double_gauss_params[key][0],
'double_gauss_peak1': double_gauss_params[key][1],
'double_gauss_width1': double_gauss_params[key][2],
'double_gauss_amplitude2': double_gauss_params[key][3],
'double_gauss_peak2': double_gauss_params[key][4],
'double_gauss_width2': double_gauss_params[key][5],
'double_gauss_chisq': double_gauss_chisquared[key],
'mean_dark_image_file': os.path.basename(mean_slope_file),
'hist_dark_values': bins,
'hist_amplitudes': histogram,
'entry_date': datetime.datetime.now()
}
self.stats_table.__table__.insert().execute(dark_db_entry)