def test_amplifier_info():
"""Test that the correct number of amplifiers are found for a given
file
"""
data_dir = os.path.join(get_config()['test_dir'], 'dark_monitor')
fullframe = instrument_properties.amplifier_info(os.path.join(data_dir, 'test_image_ff.fits'))
fullframe_truth = (4, {'1': [(4, 4), (512, 2044)],
'2': [(512, 4), (1024, 2044)],
'3': [(1024, 4), (1536, 2044)],
'4': [(1536, 4), (2044, 2044)]})
assert fullframe == fullframe_truth
fullframe = instrument_properties.amplifier_info(os.path.join(data_dir, 'test_image_ff.fits'), omit_reference_pixels=False)
fullframe_truth = (4, {'1': [(0, 0), (512, 2048)],
'2': [(512, 0), (1024, 2048)],
'3': [(1024, 0), (1536, 2048)],
'4': [(1536, 0), (2048, 2048)]})
assert fullframe == fullframe_truth
subarray = instrument_properties.amplifier_info(os.path.join(data_dir, 'test_image_1.fits'))
subarray_truth = (1, {'1': [(0, 0), (10, 10)]})
assert subarray == subarray_truth
subarray_one = instrument_properties.amplifier_info(os.path.join(data_dir, 'test_image_grismstripe_one_amp.fits'))
subarray_one_truth = (1, {'1': [(4, 4), (2044, 64)]})
assert subarray_one == subarray_one_truth
subarray_four = instrument_properties.amplifier_info(os.path.join(data_dir, 'test_image_grismstripe_four_amp.fits'))
subarray_four_truth = (4, {'1': [(4, 4), (512, 64)],
'2': [(512, 4), (1024, 64)],
'3': [(1024, 4), (1536, 64)],
'4': [(1536, 4), (2044, 64)]})
assert subarray_four == subarray_four_truth
def process(self, file_list):
"""The main method for processing darks. See module docstrings
for further details.
Parameters
----------
file_list : list
List of filenames (including full paths) to the dark current
files.
"""
for filename in file_list:
logging.info('\tWorking on file: {}'.format(filename))
# Get relevant header information for this file
self.get_metadata(filename)
# Run the file through the necessary pipeline steps
pipeline_steps = self.determine_pipeline_steps()
logging.info('\tRunning pipeline on {}'.format(filename))
try:
processed_file = pipeline_tools.run_calwebb_detector1_steps(
filename, pipeline_steps)
logging.info(
'\tPipeline complete. Output: {}'.format(processed_file))
set_permissions(processed_file)
except:
logging.info(
'\tPipeline processing failed for {}'.format(filename))
continue
# Find amplifier boundaries so per-amp statistics can be calculated
_, amp_bounds = instrument_properties.amplifier_info(
processed_file, omit_reference_pixels=True)
logging.info('\tAmplifier boundaries: {}'.format(amp_bounds))
# Get the ramp data; remove first 5 groups and last group for MIRI to avoid reset/rscd effects
cal_data = fits.getdata(processed_file, 'SCI', uint=False)
if self.instrument == 'MIRI':
cal_data = cal_data[:, 5:-1, :, :]
# Make the readnoise image
readnoise_outfile = os.path.join(
self.data_dir,
os.path.basename(
processed_file.replace('.fits', '_readnoise.fits')))
readnoise = self.make_readnoise_image(cal_data)
fits.writeto(readnoise_outfile, readnoise, overwrite=True)
logging.info(
'\tReadnoise image saved to {}'.format(readnoise_outfile))
# Calculate the full image readnoise stats
clipped = sigma_clip(readnoise, sigma=3.0, maxiters=5)
full_image_mean, full_image_stddev = np.nanmean(
clipped), np.nanstd(clipped)
full_image_n, full_image_bin_centers = self.make_histogram(
readnoise)
logging.info('\tReadnoise image stats: {:.5f} +/- {:.5f}'.format(
full_image_mean, full_image_stddev))
# Calculate readnoise stats in each amp separately
amp_stats = self.get_amp_stats(readnoise, amp_bounds)
logging.info(
'\tReadnoise image stats by amp: {}'.format(amp_stats))
# Get the current JWST Readnoise Reference File data
parameters = self.make_crds_parameter_dict()
reffile_mapping = crds.getreferences(parameters,
reftypes=['readnoise'])
readnoise_file = reffile_mapping['readnoise']
if 'NOT FOUND' in readnoise_file:
logging.warning(
'\tNo pipeline readnoise reffile match for this file - assuming all zeros.'
)
pipeline_readnoise = np.zeros(readnoise.shape)
else:
logging.info('\tPipeline readnoise reffile is {}'.format(
readnoise_file))
pipeline_readnoise = fits.getdata(readnoise_file)
# Find the difference between the current readnoise image and the pipeline readnoise reffile, and record image stats.
# Sometimes, the pipeline readnoise reffile needs to be cutout to match the subarray.
pipeline_readnoise = pipeline_readnoise[self.substrt2 -
1:self.substrt2 +
self.subsize2 - 1,
self.substrt1 -
1:self.substrt1 +
self.subsize1 - 1]
readnoise_diff = readnoise - pipeline_readnoise
clipped = sigma_clip(readnoise_diff, sigma=3.0, maxiters=5)
diff_image_mean, diff_image_stddev = np.nanmean(
clipped), np.nanstd(clipped)
diff_image_n, diff_image_bin_centers = self.make_histogram(
readnoise_diff)
logging.info(
'\tReadnoise difference image stats: {:.5f} +/- {:.5f}'.format(
diff_image_mean, diff_image_stddev))
# Save a png of the readnoise difference image for visual inspection
logging.info('\tCreating png of readnoise difference image')
readnoise_diff_png = self.image_to_png(
readnoise_diff,
outname=os.path.basename(readnoise_outfile).replace(
'.fits', '_diff'))
# Construct new entry for this file for the readnoise database table.
# Can't insert values with numpy.float32 datatypes into database
# so need to change the datatypes of these values.
readnoise_db_entry = {
'uncal_filename': filename,
'aperture': self.aperture,
'detector': self.detector,
'subarray': self.subarray,
'read_pattern': self.read_pattern,
'nints': self.nints,
'ngroups': self.ngroups,
'expstart': self.expstart,
'readnoise_filename': readnoise_outfile,
'full_image_mean': float(full_image_mean),
'full_image_stddev': float(full_image_stddev),
'full_image_n': full_image_n.astype(float),
'full_image_bin_centers': full_image_bin_centers.astype(float),
'readnoise_diff_image': readnoise_diff_png,
'diff_image_mean': float(diff_image_mean),
'diff_image_stddev': float(diff_image_stddev),
'diff_image_n': diff_image_n.astype(float),
'diff_image_bin_centers': diff_image_bin_centers.astype(float),
'entry_date': datetime.datetime.now()
}
for key in amp_stats.keys():
if isinstance(amp_stats[key], (int, float)):
readnoise_db_entry[key] = float(amp_stats[key])
else:
readnoise_db_entry[key] = amp_stats[key].astype(float)
# Add this new entry to the readnoise database table
self.stats_table.__table__.insert().execute(readnoise_db_entry)
logging.info('\tNew entry added to readnoise database table')
# Remove the raw and calibrated files to save memory space
os.remove(filename)
os.remove(processed_file)
def process(self, file_list):
"""The main method for processing darks. See module docstrings
for further details.
Parameters
----------
file_list : list
List of filenames (including full paths) to the dark current
files
"""
for filename in file_list:
logging.info('\tWorking on file: {}'.format(filename))
# Skip processing if an entry for this file already exists in
# the bias stats database.
file_exists = self.file_exists_in_database(filename)
if file_exists:
logging.info(
'\t{} already exists in the bias database table.'.format(
filename))
continue
# Get the exposure start time of this file
expstart = '{}T{}'.format(
fits.getheader(filename, 0)['DATE-OBS'],
fits.getheader(filename, 0)['TIME-OBS'])
# Determine if the file needs group_scale in pipeline run
read_pattern = fits.getheader(filename, 0)['READPATT']
if read_pattern not in pipeline_tools.GROUPSCALE_READOUT_PATTERNS:
group_scale = False
else:
group_scale = True
# Run the file through the pipeline up through the refpix step
logging.info('\tRunning pipeline on {}'.format(filename))
processed_file = self.run_early_pipeline(filename,
odd_even_rows=False,
odd_even_columns=True,
use_side_ref_pixels=True,
group_scale=group_scale)
logging.info(
'\tPipeline complete. Output: {}'.format(processed_file))
# Find amplifier boundaries so per-amp statistics can be calculated
_, amp_bounds = instrument_properties.amplifier_info(
processed_file, omit_reference_pixels=True)
logging.info('\tAmplifier boundaries: {}'.format(amp_bounds))
# Get the uncalibrated 0th group data for this file
uncal_data = fits.getdata(filename, 'SCI')[0,
0, :, :].astype(float)
# Calculate the uncal median values of each amplifier for odd/even columns
amp_medians = self.get_amp_medians(uncal_data, amp_bounds)
logging.info('\tCalculated uncalibrated image stats: {}'.format(
amp_medians))
# Calculate image statistics and the collapsed row/column values
# in the calibrated image
cal_data = fits.getdata(processed_file, 'SCI')[0, 0, :, :]
dq = fits.getdata(processed_file, 'PIXELDQ')
mean, median, stddev = sigma_clipped_stats(cal_data[dq == 0],
sigma=3.0,
maxiters=5)
logging.info(
'\tCalculated calibrated image stats: {:.3f} +/- {:.3f}'.
format(mean, stddev))
collapsed_rows, collapsed_columns = self.collapse_image(cal_data)
logging.info(
'\tCalculated collapsed row/column values of calibrated image.'
)
# Save a png of the calibrated image for visual inspection
logging.info('\tCreating png of calibrated image')
output_png = self.image_to_png(
cal_data,
outname=os.path.basename(processed_file).replace('.fits', ''))
# Construct new entry for this file for the bias database table.
# Can't insert values with numpy.float32 datatypes into database
# so need to change the datatypes of these values.
bias_db_entry = {
'aperture': self.aperture,
'uncal_filename': filename,
'cal_filename': processed_file,
'cal_image': output_png,
'expstart': expstart,
'mean': float(mean),
'median': float(median),
'stddev': float(stddev),
'collapsed_rows': collapsed_rows.astype(float),
'collapsed_columns': collapsed_columns.astype(float),
'entry_date': datetime.datetime.now()
}
for key in amp_medians.keys():
bias_db_entry[key] = float(amp_medians[key])
# Add this new entry to the bias database table
self.stats_table.__table__.insert().execute(bias_db_entry)
logging.info('\tNew entry added to bias database table: {}'.format(
bias_db_entry))
def process(self, file_list):
"""The main method for processing darks. See module docstrings
for further details.
Parameters
----------
file_list : list
List of filenames (including full paths) to the dark current
files.
"""
for filename in file_list:
logging.info('\tWorking on file: {}'.format(filename))
# Get relevant header info for this file
self.read_pattern = fits.getheader(filename, 0)['READPATT']
self.expstart = '{}T{}'.format(
fits.getheader(filename, 0)['DATE-OBS'],
fits.getheader(filename, 0)['TIME-OBS'])
# Run the file through the necessary pipeline steps
pipeline_steps = self.determine_pipeline_steps()
logging.info('\tRunning pipeline on {}'.format(filename))
try:
processed_file = pipeline_tools.run_calwebb_detector1_steps(
filename, pipeline_steps)
logging.info(
'\tPipeline complete. Output: {}'.format(processed_file))
set_permissions(processed_file)
except:
logging.info(
'\tPipeline processing failed for {}'.format(filename))
os.remove(filename)
continue
# Find amplifier boundaries so per-amp statistics can be calculated
_, amp_bounds = instrument_properties.amplifier_info(
processed_file, omit_reference_pixels=True)
logging.info('\tAmplifier boundaries: {}'.format(amp_bounds))
# Get the uncalibrated 0th group data for this file
uncal_data = fits.getdata(filename, 'SCI')[0,
0, :, :].astype(float)
# Calculate the uncal median values of each amplifier for odd/even columns
amp_medians = self.get_amp_medians(uncal_data, amp_bounds)
logging.info('\tCalculated uncalibrated image stats: {}'.format(
amp_medians))
# Calculate image statistics on the calibrated image
cal_data = fits.getdata(processed_file, 'SCI')[0, 0, :, :]
mean, median, stddev = sigma_clipped_stats(cal_data,
sigma=3.0,
maxiters=5)
collapsed_rows, collapsed_columns = self.collapse_image(cal_data)
counts, bin_centers = self.make_histogram(cal_data)
logging.info(
'\tCalculated calibrated image stats: {:.3f} +/- {:.3f}'.
format(mean, stddev))
# Save a png of the calibrated image for visual inspection
logging.info('\tCreating png of calibrated image')
output_png = self.image_to_png(
cal_data,
outname=os.path.basename(processed_file).replace('.fits', ''))
# Construct new entry for this file for the bias database table.
# Can't insert values with numpy.float32 datatypes into database
# so need to change the datatypes of these values.
bias_db_entry = {
'aperture': self.aperture,
'uncal_filename': filename,
'cal_filename': processed_file,
'cal_image': output_png,
'expstart': self.expstart,
'mean': float(mean),
'median': float(median),
'stddev': float(stddev),
'collapsed_rows': collapsed_rows.astype(float),
'collapsed_columns': collapsed_columns.astype(float),
'counts': counts.astype(float),
'bin_centers': bin_centers.astype(float),
'entry_date': datetime.datetime.now()
}
for key in amp_medians.keys():
bias_db_entry[key] = float(amp_medians[key])
# Add this new entry to the bias database table
self.stats_table.__table__.insert().execute(bias_db_entry)
logging.info('\tNew entry added to bias database table: {}'.format(
bias_db_entry))
# Remove the raw and calibrated files to save memory space
os.remove(filename)
os.remove(processed_file)
def process(self, file_list):
"""The main method for processing darks. See module docstrings
for further details.
Parameters
----------
file_list : list
List of filenames (including full paths) to the dark current
files
"""
# Basic metadata that will be needed later
self.get_metadata(file_list[0])
# Determine which pipeline steps need to be executed
required_steps = pipeline_tools.get_pipeline_steps(self.instrument)
logging.info(
'\tRequired calwebb1_detector pipeline steps to have the data in the '
'correct format:')
for item in required_steps:
logging.info('\t\t{}: {}'.format(item, required_steps[item]))
# Modify the list of pipeline steps to skip those not needed for the
# preparation of dark current data
required_steps['dark_current'] = False
required_steps['persistence'] = False
# NIRSpec IR^2 readout pattern NRSIRS2 is the only one with
# nframes not a power of 2
if self.read_pattern not in pipeline_tools.GROUPSCALE_READOUT_PATTERNS:
required_steps['group_scale'] = False
# Run pipeline steps on files, generating slope files
slope_files = []
for filename in file_list:
completed_steps = pipeline_tools.completed_pipeline_steps(filename)
steps_to_run = pipeline_tools.steps_to_run(required_steps,
completed_steps)
logging.info('\tWorking on file: {}'.format(filename))
logging.info('\tPipeline steps that remain to be run:')
for item in steps_to_run:
logging.info('\t\t{}: {}'.format(item, steps_to_run[item]))
# Run any remaining required pipeline steps
if any(steps_to_run.values()) is False:
slope_files.append(filename)
else:
processed_file = filename.replace('.fits',
'_{}.fits'.format('rate'))
# If the slope file already exists, skip the pipeline call
if not os.path.isfile(processed_file):
logging.info('\tRunning pipeline on {}'.format(filename))
processed_file = pipeline_tools.run_calwebb_detector1_steps(
os.path.abspath(filename), steps_to_run)
logging.info('\tPipeline complete. Output: {}'.format(
processed_file))
else:
logging.info(
'\tSlope file {} already exists. Skipping call to pipeline.'
.format(processed_file))
pass
slope_files.append(processed_file)
# Delete the original dark ramp file to save disk space
os.remove(filename)
obs_times = []
logging.info(
'\tSlope images to use in the dark monitor for {}, {}:'.format(
self.instrument, self.aperture))
for item in slope_files:
logging.info('\t\t{}'.format(item))
# Get the observation time for each file
obstime = instrument_properties.get_obstime(item)
obs_times.append(obstime)
# Find the earliest and latest observation time, and calculate
# the mid-time.
min_time = np.min(obs_times)
max_time = np.max(obs_times)
mid_time = instrument_properties.mean_time(obs_times)
# Read in all slope images and place into a list
slope_image_stack, slope_exptimes = pipeline_tools.image_stack(
slope_files)
# Calculate a mean slope image from the inputs
slope_image, stdev_image = calculations.mean_image(slope_image_stack,
sigma_threshold=3)
mean_slope_file = self.save_mean_slope_image(slope_image, stdev_image,
slope_files)
logging.info(
'\tSigma-clipped mean of the slope images saved to: {}'.format(
mean_slope_file))
# ----- Search for new hot/dead/noisy pixels -----
# Read in baseline mean slope image and stdev image
# The baseline image is used to look for hot/dead/noisy pixels,
# but not for comparing mean dark rates. Therefore, updates to
# the baseline can be minimal.
# Limit checks for hot/dead/noisy pixels to full frame data since
# subarray data have much shorter exposure times and therefore lower
# signal-to-noise
aperture_type = Siaf(self.instrument)[self.aperture].AperType
if aperture_type == 'FULLSCA':
baseline_file = self.get_baseline_filename()
if baseline_file is None:
logging.warning((
'\tNo baseline dark current countrate image for {} {}. Setting the '
'current mean slope image to be the new baseline.'.format(
self.instrument, self.aperture)))
baseline_file = mean_slope_file
baseline_mean = deepcopy(slope_image)
baseline_stdev = deepcopy(stdev_image)
else:
logging.info('\tBaseline file is {}'.format(baseline_file))
baseline_mean, baseline_stdev = self.read_baseline_slope_image(
baseline_file)
# Check the hot/dead pixel population for changes
new_hot_pix, new_dead_pix = self.find_hot_dead_pixels(
slope_image, baseline_mean)
# Shift the coordinates to be in full frame coordinate system
new_hot_pix = self.shift_to_full_frame(new_hot_pix)
new_dead_pix = self.shift_to_full_frame(new_dead_pix)
# Exclude hot and dead pixels found previously
new_hot_pix = self.exclude_existing_badpix(new_hot_pix, 'hot')
new_dead_pix = self.exclude_existing_badpix(new_dead_pix, 'dead')
# Add new hot and dead pixels to the database
logging.info('\tFound {} new hot pixels'.format(len(
new_hot_pix[0])))
logging.info('\tFound {} new dead pixels'.format(
len(new_dead_pix[0])))
self.add_bad_pix(new_hot_pix, 'hot', file_list, mean_slope_file,
baseline_file, min_time, mid_time, max_time)
self.add_bad_pix(new_dead_pix, 'dead', file_list, mean_slope_file,
baseline_file, min_time, mid_time, max_time)
# Check for any pixels that are significantly more noisy than
# in the baseline stdev image
new_noisy_pixels = self.noise_check(stdev_image, baseline_stdev)
# Shift coordinates to be in full_frame coordinate system
new_noisy_pixels = self.shift_to_full_frame(new_noisy_pixels)
# Exclude previously found noisy pixels
new_noisy_pixels = self.exclude_existing_badpix(
new_noisy_pixels, 'noisy')
# Add new noisy pixels to the database
logging.info('\tFound {} new noisy pixels'.format(
len(new_noisy_pixels[0])))
self.add_bad_pix(new_noisy_pixels, 'noisy', file_list,
mean_slope_file, baseline_file, min_time,
mid_time, max_time)
# ----- Calculate image statistics -----
# Find amplifier boundaries so per-amp statistics can be calculated
number_of_amps, amp_bounds = instrument_properties.amplifier_info(
slope_files[0])
logging.info('\tAmplifier boundaries: {}'.format(amp_bounds))
# Calculate mean and stdev values, and fit a Gaussian to the
# histogram of the pixels in each amp
(amp_mean, amp_stdev, gauss_param, gauss_chisquared,
double_gauss_params, double_gauss_chisquared, histogram,
bins) = self.stats_by_amp(slope_image, amp_bounds)
# Construct new entry for dark database table
source_files = [os.path.basename(item) for item in file_list]
for key in amp_mean.keys():
dark_db_entry = {
'aperture': self.aperture,
'amplifier': key,
'mean': amp_mean[key],
'stdev': amp_stdev[key],
'source_files': source_files,
'obs_start_time': min_time,
'obs_mid_time': mid_time,
'obs_end_time': max_time,
'gauss_amplitude': list(gauss_param[key][0]),
'gauss_peak': list(gauss_param[key][1]),
'gauss_width': list(gauss_param[key][2]),
'gauss_chisq': gauss_chisquared[key],
'double_gauss_amplitude1': double_gauss_params[key][0],
'double_gauss_peak1': double_gauss_params[key][1],
'double_gauss_width1': double_gauss_params[key][2],
'double_gauss_amplitude2': double_gauss_params[key][3],
'double_gauss_peak2': double_gauss_params[key][4],
'double_gauss_width2': double_gauss_params[key][5],
'double_gauss_chisq': double_gauss_chisquared[key],
'mean_dark_image_file': os.path.basename(mean_slope_file),
'hist_dark_values': bins,
'hist_amplitudes': histogram,
'entry_date': datetime.datetime.now()
}
self.stats_table.__table__.insert().execute(dark_db_entry)
