def test_save_qc_results(mock_es):
context = FakeContext()
context.post_to_elasticsearch = True
context.elasticsearch_url = '/'
stage = FakeStage(context)
qc.save_qc_results(stage.runtime_context, {}, FakeImage())
assert mock_es.called
def do_stage(self, image):
npixels = np.product(image.data.shape)
fraction_1000s = float(np.sum(image.data == 1000)) / npixels
logging_tags = {
'FRAC1000': fraction_1000s,
'threshold': self.THOUSANDS_THRESHOLD
}
has_1000s_error = fraction_1000s > self.THOUSANDS_THRESHOLD
qc_results = {
'sinistro_thousands.failed': has_1000s_error,
'sinistro_thousands.fraction': fraction_1000s,
'sinistro_thousands.threshold': self.THOUSANDS_THRESHOLD
}
if has_1000s_error:
logger.error('Image is mostly 1000s. Rejecting image',
image=image,
extra_tags=logging_tags)
qc_results['rejected'] = True
return None
else:
logger.info('Measuring fraction of 1000s.',
image=image,
extra_tags=logging_tags)
qc.save_qc_results(self.runtime_context, qc_results, image)
return image
def check_ra_range(self, image, bad_keywords=None):
"""
Logs an error if the keyword right_ascension is not inside
the expected range (0<ra<360 degrees) in the image header.
Parameters
----------
image : object
a banzais.image.Image object.
bad_keywords: list
a list of any keywords that are missing or NA
"""
if bad_keywords is None:
bad_keywords = []
if 'CRVAL1' not in bad_keywords:
ra_value = image.header['CRVAL1']
is_bad_ra_value = (ra_value > self.RA_MAX) | (ra_value <
self.RA_MIN)
if is_bad_ra_value:
sentence = 'The header CRVAL1 key got the unexpected value : {0}'.format(
ra_value)
logger.error(sentence, image=image)
qc_results = {
"header.ra.failed": is_bad_ra_value,
"header.ra.value": ra_value
}
qc.save_qc_results(self.runtime_context, qc_results, image)
def do_stage(self, image):
lab_lines = find_nearest(image.features['wavelength'],
np.sort(image.line_list))
delta_lambda = image.features['wavelength'] - lab_lines
sigma_delta_lambda = robust_standard_deviation(delta_lambda)
low_scatter_lines = delta_lambda < 3. * sigma_delta_lambda
matched_sigma_delta_lambda = robust_standard_deviation(
delta_lambda[low_scatter_lines])
num_detected_lines = len(image.features['wavelength'])
num_matched_lines = np.count_nonzero(low_scatter_lines)
feature_centroid_uncertainty = image.features['centroid_err']
reduced_chi2 = get_reduced_chi_squared(
delta_lambda[low_scatter_lines],
feature_centroid_uncertainty[low_scatter_lines])
velocity_precision = get_velocity_precision(
image.features['wavelength'][low_scatter_lines],
lab_lines[low_scatter_lines], num_matched_lines)
if num_matched_lines == 0: # get rid of nans in the matched statistics if we have zero matched lines.
matched_sigma_delta_lambda, reduced_chi2, velocity_precision = 0, 0, 0 * units.meter / units.second
# opensearch keys don't have to be the same as the fits headers
qc_results = {
'SIGLAM':
np.round(matched_sigma_delta_lambda, 4),
'RVPRECSN':
np.round(
velocity_precision.to(units.meter / units.second).value, 4),
'WAVRCHI2':
np.round(reduced_chi2, 4),
'NLINEDET':
num_detected_lines,
'NLINES':
num_matched_lines
}
qc_description = {
'SIGLAM': 'wavecal residuals [Angstroms]',
'RVPRECSN': 'wavecal precision [m/s]',
'WAVRCHI2': 'reduced chisquared goodness of wavecal fit',
'NLINEDET': 'Number of lines found on detector',
'NLINES': 'Number of matched lines'
}
qc.save_qc_results(self.runtime_context, qc_results, image)
# saving the results to the image header
for key in qc_results.keys():
image.meta[key] = (qc_results[key], qc_description[key])
logger.info(f'wavecal precision (m/s) = {qc_results["RVPRECSN"]}',
image=image)
if qc_results['RVPRECSN'] > 10 or qc_results['RVPRECSN'] < 3:
logger.warning(
f' Final calibration precision is outside the expected range '
f'wavecal precision (m/s) = '
f'{qc_results["RVPRECSN"]}',
image=image)
return image
def check_dec_range(self, image, bad_keywords=None):
"""
Logs an error if the keyword declination is not inside
the expected range (-90<dec<90 degrees) in the image header.
Parameters
----------
image : object
a banzais.image.Image object.
bad_keywords: list
a list of any keywords that are missing or NA
"""
if bad_keywords is None:
bad_keywords = []
if 'CRVAL2' not in bad_keywords:
dec_value = image.header['CRVAL2']
is_bad_dec_value = (dec_value > self.DEC_MAX) | (dec_value <
self.DEC_MIN)
if is_bad_dec_value:
sentence = 'The header CRVAL2 key got the unexpected value : {0}'.format(
dec_value)
logger.error(sentence, image=image)
qc_results = {
"header.dec.failed": is_bad_dec_value,
"header.dec.value": dec_value
}
qc.save_qc_results(self.runtime_context, qc_results, image)
def test_save_qc_results(mock_es):
context = FakeContext()
context.post_to_elasticsearch = True
context.elasticsearch_url = '/'
stage = FakeStage(context)
qc.save_qc_results(stage.runtime_context, {}, FakeImage())
assert mock_es.called
def do_stage(self, image):
saturation_level = float(image.header['SATURATE'])
saturated_pixels = image.data >= saturation_level
total_pixels = image.data.size
saturation_fraction = float(saturated_pixels.sum()) / total_pixels
logging_tags = {
'SATFRAC': saturation_fraction,
'threshold': self.SATURATION_THRESHOLD
}
logger.info('Measured saturation fraction.',
image=image,
extra_tags=logging_tags)
is_saturated = saturation_fraction >= self.SATURATION_THRESHOLD
qc_results = {
'saturated.failed': is_saturated,
'saturated.fraction': saturation_fraction,
'saturated.threshold': self.SATURATION_THRESHOLD
}
if is_saturated:
logger.error('SATFRAC exceeds threshold.',
image=image,
extra_tags=logging_tags)
qc_results['rejected'] = True
else:
image.header['SATFRAC'] = (saturation_fraction,
"Fraction of Pixels that are Saturated")
qc.save_qc_results(self.runtime_context, qc_results, image)
return None if is_saturated else image
def do_stage(self, image):
pattern_noise_is_bad, fraction_pixels_above_threshold = self.check_for_pattern_noise(
image.data)
logging_tags = {
'snr_threshold': self.SNR_THRESHOLD,
'min_fraction_pixels_above_threshold':
self.MIN_FRACTION_PIXELS_ABOVE_THRESHOLD,
'min_adjacent_pixels': self.MIN_ADJACENT_PIXELS,
'fraction_pixels_above_threshold': fraction_pixels_above_threshold
}
if pattern_noise_is_bad:
logger.error('Image found to have pattern noise.',
image=image,
extra_tags=logging_tags)
else:
logger.info('No pattern noise found.',
image=image,
extra_tags=logging_tags)
qc_results = {
'pattern_noise.failed':
pattern_noise_is_bad,
'pattern_noise.snr_threshold':
self.SNR_THRESHOLD,
'pattern_noise.min_fraction_pixels_above_threshold':
self.MIN_FRACTION_PIXELS_ABOVE_THRESHOLD,
'pattern_noise.min_adjacent_pixels':
self.MIN_ADJACENT_PIXELS,
'patter_noise.fraction_pixels_above_threshold':
fraction_pixels_above_threshold
}
qc.save_qc_results(self.runtime_context, qc_results, image)
return image
def do_stage(self, image):
if image.exptime <= 0.0:
logger.error('EXPTIME is <= 0.0. Rejecting frame', image=image)
qc_results = {'exptime': image.exptime, 'rejected': True}
qc.save_qc_results(self.runtime_context, qc_results, image)
return None
image.data /= image.exptime
logger.info('Normalizing dark by exposure time', image=image)
return image
def apply_master_calibration(self, image, master_calibration_image):
# Short circuit
if master_calibration_image.data is None:
return image
# We assume the image has already been normalized before this stage is run.
bad_pixel_fraction = np.abs(image.data - master_calibration_image.data)
# Estimate the noise of the image
noise = self.noise_model(image)
bad_pixel_fraction /= noise
bad_pixel_fraction = bad_pixel_fraction >= self.SIGNAL_TO_NOISE_THRESHOLD
bad_pixel_fraction = bad_pixel_fraction.sum() / float(
bad_pixel_fraction.size)
frame_is_bad = bad_pixel_fraction > self.ACCEPTABLE_PIXEL_FRACTION
qc_results = {
"master_comparison.fraction":
bad_pixel_fraction,
"master_comparison.snr_threshold":
self.SIGNAL_TO_NOISE_THRESHOLD,
"master_comparison.pixel_threshold":
self.ACCEPTABLE_PIXEL_FRACTION,
"master_comparison.comparison_master_filename":
master_calibration_image.filename
}
logging_tags = {}
for qc_check, qc_result in qc_results.items():
logging_tags[qc_check] = qc_result
logging_tags[
'master_comparison_filename'] = master_calibration_image.filename
msg = "Performing comparison to last good master {caltype} frame"
logger.info(msg.format(caltype=self.calibration_type),
image=image,
extra_tags=logging_tags)
# This needs to be added after the qc_results dictionary is used for the logging tags because
# they can't handle booleans
qc_results["master_comparison.failed"] = frame_is_bad
if frame_is_bad:
# Flag the image as bad and log an error
image.is_bad = True
qc_results['rejected'] = True
msg = 'Flagging {caltype} as bad because it deviates too much from the previous master'
logger.error(msg.format(caltype=self.calibration_type),
image=image,
extra_tags=logging_tags)
qc.save_qc_results(self.runtime_context, qc_results, image)
return image
def do_stage(self, image):
pattern_noise_is_bad, fraction_pixels_above_threshold = self.check_for_pattern_noise(image.data)
logging_tags = {'snr_threshold': self.SNR_THRESHOLD,
'min_fraction_pixels_above_threshold': self.MIN_FRACTION_PIXELS_ABOVE_THRESHOLD,
'min_adjacent_pixels': self.MIN_ADJACENT_PIXELS,
'fraction_pixels_above_threshold': fraction_pixels_above_threshold}
if pattern_noise_is_bad:
logger.error('Image found to have pattern noise.', image=image, extra_tags=logging_tags)
else:
logger.info('No pattern noise found.', image=image, extra_tags=logging_tags)
qc_results = {'pattern_noise.failed': pattern_noise_is_bad,
'pattern_noise.snr_threshold': self.SNR_THRESHOLD,
'pattern_noise.min_fraction_pixels_above_threshold': self.MIN_FRACTION_PIXELS_ABOVE_THRESHOLD,
'pattern_noise.min_adjacent_pixels': self.MIN_ADJACENT_PIXELS,
'patter_noise.fraction_pixels_above_threshold': fraction_pixels_above_threshold}
qc.save_qc_results(self.runtime_context, qc_results, image)
return image
def do_stage(self, image):
npixels = np.product(image.data.shape)
fraction_1000s = float(np.sum(image.data == 1000)) / npixels
logging_tags = {'FRAC1000': fraction_1000s,
'threshold': self.THOUSANDS_THRESHOLD}
has_1000s_error = fraction_1000s > self.THOUSANDS_THRESHOLD
qc_results = {'sinistro_thousands.failed': has_1000s_error,
'sinistro_thousands.fraction': fraction_1000s,
'sinistro_thousands.threshold': self.THOUSANDS_THRESHOLD}
if has_1000s_error:
logger.error('Image is mostly 1000s. Rejecting image', image=image, extra_tags=logging_tags)
qc_results['rejected'] = True
return None
else:
logger.info('Measuring fraction of 1000s.', image=image, extra_tags=logging_tags)
qc.save_qc_results(self.runtime_context, qc_results, image)
return image
def check_keywords_missing_or_na(self, image):
"""
Logs an error if the keyword is missing or 'N/A' (the default placeholder value).
Parameters
----------
image : object
a banzais.image.Image object.
Returns
-------
bad_keywords: list
a list of any keywords that are missing or NA
Notes
-----
Some header keywords for bias and dark frames (e.g., 'OFST-RA') are excpted to be non-valued,
but the 'N/A' placeholder values should be overwritten by 'NaN'.
"""
qc_results = {}
missing_keywords = []
na_keywords = []
for keyword in self.expected_header_keywords:
if keyword not in image.header:
sentence = 'The header key {0} is not in image header!'.format(
keyword)
logger.error(sentence, image=image)
missing_keywords.append(keyword)
elif image.header[keyword] == 'N/A':
sentence = 'The header key {0} got the unexpected value : N/A'.format(
keyword)
logger.error(sentence, image=image)
na_keywords.append(keyword)
are_keywords_missing = len(missing_keywords) > 0
are_keywords_na = len(na_keywords) > 0
qc_results["header.keywords.missing.failed"] = are_keywords_missing
qc_results["header.keywords.na.failed"] = are_keywords_na
if are_keywords_missing:
qc_results["header.keywords.missing.names"] = missing_keywords
if are_keywords_na:
qc_results["header.keywords.na.names"] = na_keywords
qc.save_qc_results(self.runtime_context, qc_results, image)
return missing_keywords + na_keywords
def check_keywords_missing_or_na(self, image):
"""
Logs an error if the keyword is missing or 'N/A' (the default placeholder value).
Parameters
----------
image : object
a banzais.image.Image object.
Returns
-------
bad_keywords: list
a list of any keywords that are missing or NA
Notes
-----
Some header keywords for bias and dark frames (e.g., 'OFST-RA') are excpted to be non-valued,
but the 'N/A' placeholder values should be overwritten by 'NaN'.
"""
qc_results = {}
missing_keywords = []
na_keywords = []
for keyword in self.expected_header_keywords:
if keyword not in image.header:
sentence = 'The header key {0} is not in image header!'.format(keyword)
logger.error(sentence, image=image)
missing_keywords.append(keyword)
elif image.header[keyword] == 'N/A':
sentence = 'The header key {0} got the unexpected value : N/A'.format(keyword)
logger.error(sentence, image=image)
na_keywords.append(keyword)
are_keywords_missing = len(missing_keywords) > 0
are_keywords_na = len(na_keywords) > 0
qc_results["header.keywords.missing.failed"] = are_keywords_missing
qc_results["header.keywords.na.failed"] = are_keywords_na
if are_keywords_missing:
qc_results["header.keywords.missing.names"] = missing_keywords
if are_keywords_na:
qc_results["header.keywords.na.names"] = na_keywords
qc.save_qc_results(self.runtime_context, qc_results, image)
return missing_keywords + na_keywords
def do_stage(self, image):
try:
# OFST-RA/DEC is the same as CAT-RA/DEC but includes user requested offset
requested_coords = SkyCoord(image.header['OFST-RA'], image.header['OFST-DEC'],
unit=(u.hour, u.deg), frame='icrs')
except ValueError as e:
try:
# Fallback to CAT-RA and CAT-DEC
requested_coords = SkyCoord(image.header['CAT-RA'], image.header['CAT-DEC'],
unit=(u.hour, u.deg), frame='icrs')
except:
logger.error(e, image=image)
return image
# This only works assuming CRPIX is at the center of the image
solved_coords = SkyCoord(image.header['CRVAL1'], image.header['CRVAL2'],
unit=(u.deg, u.deg), frame='icrs')
angular_separation = solved_coords.separation(requested_coords).arcsec
logging_tags = {'PNTOFST': angular_separation}
pointing_severe = abs(angular_separation) > self.SEVERE_THRESHOLD
pointing_warning = abs(angular_separation) > self.WARNING_THRESHOLD
if pointing_severe:
logger.error('Pointing offset exceeds threshold', image=image, extra_tags=logging_tags)
elif pointing_warning:
logger.warning('Pointing offset exceeds threshhold', image=image, extra_tags=logging_tags)
qc_results = {'pointing.failed': pointing_severe,
'pointing.failed_threshold': self.SEVERE_THRESHOLD,
'pointing.warning': pointing_warning,
'pointing.warning_threshold': self.WARNING_THRESHOLD,
'pointing.offset': angular_separation}
qc.save_qc_results(self.runtime_context, qc_results, image)
image.header['PNTOFST'] = (
angular_separation, '[arcsec] offset of requested and solved center'
)
return image
def apply_master_calibration(self, image, master_calibration_image):
# Short circuit
if master_calibration_image.data is None:
return image
# We assume the image has already been normalized before this stage is run.
bad_pixel_fraction = np.abs(image.data - master_calibration_image.data)
# Estimate the noise of the image
noise = self.noise_model(image)
bad_pixel_fraction /= noise
bad_pixel_fraction = bad_pixel_fraction >= self.SIGNAL_TO_NOISE_THRESHOLD
bad_pixel_fraction = bad_pixel_fraction.sum() / float(bad_pixel_fraction.size)
frame_is_bad = bad_pixel_fraction > self.ACCEPTABLE_PIXEL_FRACTION
qc_results = {"master_comparison.fraction": bad_pixel_fraction,
"master_comparison.snr_threshold": self.SIGNAL_TO_NOISE_THRESHOLD,
"master_comparison.pixel_threshold": self.ACCEPTABLE_PIXEL_FRACTION,
"master_comparison.comparison_master_filename": master_calibration_image.filename}
logging_tags = {}
for qc_check, qc_result in qc_results.items():
logging_tags[qc_check] = qc_result
logging_tags['master_comparison_filename'] = master_calibration_image.filename
msg = "Performing comparison to last good master {caltype} frame"
logger.info(msg.format(caltype=self.calibration_type), image=image, extra_tags=logging_tags)
# This needs to be added after the qc_results dictionary is used for the logging tags because
# they can't handle booleans
qc_results["master_comparison.failed"] = frame_is_bad
if frame_is_bad:
# Flag the image as bad and log an error
image.is_bad = True
qc_results['rejected'] = True
msg = 'Flagging {caltype} as bad because it deviates too much from the previous master'
logger.error(msg.format(caltype=self.calibration_type), image=image, extra_tags=logging_tags)
qc.save_qc_results(self.runtime_context, qc_results, image)
return image
def do_stage(self, image):
saturation_level = float(image.header['SATURATE'])
saturated_pixels = image.data >= saturation_level
total_pixels = image.data.size
saturation_fraction = float(saturated_pixels.sum()) / total_pixels
logging_tags = {'SATFRAC': saturation_fraction,
'threshold': self.SATURATION_THRESHOLD}
logger.info('Measured saturation fraction.', image=image, extra_tags=logging_tags)
is_saturated = saturation_fraction >= self.SATURATION_THRESHOLD
qc_results = {'saturated.failed': is_saturated,
'saturated.fraction': saturation_fraction,
'saturated.threshold': self.SATURATION_THRESHOLD}
if is_saturated:
logger.error('SATFRAC exceeds threshold.', image=image, extra_tags=logging_tags)
qc_results['rejected'] = True
return None
else:
image.header['SATFRAC'] = (saturation_fraction,
"Fraction of Pixels that are Saturated")
qc.save_qc_results(self.runtime_context, qc_results, image)
return image
def check_exptime_value(self, image, bad_keywords=None):
"""
Logs an error if OBSTYPE is not BIAS and EXPTIME <= 0
Parameters
----------
image : object
a banzais.image.Image object.
bad_keywords: list
a list of any keywords that are missing or NA
"""
if bad_keywords is None:
bad_keywords = []
if 'EXPTIME' not in bad_keywords and 'OBSTYPE' not in bad_keywords:
exptime_value = image.header['EXPTIME']
qc_results = {"header.exptime.value": exptime_value}
if image.header['OBSTYPE'] != 'BIAS':
is_exptime_null = exptime_value <= 0.0
if is_exptime_null:
sentence = 'The header EXPTIME key got the unexpected value {0}:' \
'null or negative value'.format(exptime_value)
logger.error(sentence, image=image)
qc_results["header.exptime.failed"] = is_exptime_null
qc.save_qc_results(self.runtime_context, qc_results, image)
def check_ra_range(self, image, bad_keywords=None):
"""
Logs an error if the keyword right_ascension is not inside
the expected range (0<ra<360 degrees) in the image header.
Parameters
----------
image : object
a banzais.image.Image object.
bad_keywords: list
a list of any keywords that are missing or NA
"""
if bad_keywords is None:
bad_keywords = []
if 'CRVAL1' not in bad_keywords:
ra_value = image.header['CRVAL1']
is_bad_ra_value = (ra_value > self.RA_MAX) | (ra_value < self.RA_MIN)
if is_bad_ra_value:
sentence = 'The header CRVAL1 key got the unexpected value : {0}'.format(ra_value)
logger.error(sentence, image=image)
qc_results = {"header.ra.failed": is_bad_ra_value,
"header.ra.value": ra_value}
qc.save_qc_results(self.runtime_context, qc_results, image)
def check_exptime_value(self, image, bad_keywords=None):
"""
Logs an error if OBSTYPE is not BIAS and EXPTIME <= 0
Parameters
----------
image : object
a banzais.image.Image object.
bad_keywords: list
a list of any keywords that are missing or NA
"""
if bad_keywords is None:
bad_keywords = []
if 'EXPTIME' not in bad_keywords and 'OBSTYPE' not in bad_keywords:
exptime_value = image.header['EXPTIME']
qc_results = {"header.exptime.value": exptime_value}
if image.header['OBSTYPE'] != 'BIAS':
is_exptime_null = exptime_value <= 0.0
if is_exptime_null:
sentence = 'The header EXPTIME key got the unexpected value {0}:' \
'null or negative value'.format(exptime_value)
logger.error(sentence, image=image)
qc_results["header.exptime.failed"] = is_exptime_null
qc.save_qc_results(self.runtime_context, qc_results, image)
def check_dec_range(self, image, bad_keywords=None):
"""
Logs an error if the keyword declination is not inside
the expected range (-90<dec<90 degrees) in the image header.
Parameters
----------
image : object
a banzais.image.Image object.
bad_keywords: list
a list of any keywords that are missing or NA
"""
if bad_keywords is None:
bad_keywords = []
if 'CRVAL2' not in bad_keywords:
dec_value = image.header['CRVAL2']
is_bad_dec_value = (dec_value > self.DEC_MAX) | (dec_value < self.DEC_MIN)
if is_bad_dec_value:
sentence = 'The header CRVAL2 key got the unexpected value : {0}'.format(dec_value)
logger.error(sentence, image=image)
qc_results = {"header.dec.failed": is_bad_dec_value,
"header.dec.value": dec_value}
qc.save_qc_results(self.runtime_context, qc_results, image)
def test_save_qc_results_no_post_to_elasticsearch_attribute():
stage = FakeStage(FakeContext())
assert qc.save_qc_results(stage.runtime_context, {}, FakeImage()) == {}
def test_save_qc_results_no_post_to_elasticsearch_attribute():
stage = FakeStage(FakeContext())
assert qc.save_qc_results(stage.runtime_context, {}, FakeImage()) == {}