def do_stage(self, images):
for image in images:
image.header['RLEVEL'] = (self.pipeline_context.rlevel,
'Reduction level')
image.header['PIPEVER'] = (banzai.__version__, 'Pipeline version')
if instantly_public(image.header['PROPID']):
image.header['L1PUBDAT'] = (
image.header['DATE-OBS'],
'[UTC] Date the frame becomes public')
else:
# Wait a year
date_observed = date_utils.parse_date_obs(
image.header['DATE-OBS'])
next_year = date_observed + timedelta(days=365)
image.header['L1PUBDAT'] = (
date_utils.date_obs_to_string(next_year),
'[UTC] Date the frame becomes public')
logging_tags = logs.image_config_to_tags(image,
self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
logs.add_tag(logging_tags, 'rlevel', int(image.header['RLEVEL']))
logs.add_tag(logging_tags, 'pipeline_version',
image.header['PIPEVER'])
logs.add_tag(logging_tags, 'l1pubdat', image.header['L1PUBDAT'])
self.logger.info('Updating header', extra=logging_tags)
return images
def do_stage(self, images):
for image in images:
if len(image.data.shape) > 2:
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
n_amps = image.data.shape[0]
crosstalk_matrix = np.identity(n_amps)
for j in range(n_amps):
for i in range(n_amps):
if i != j:
crosstalk_keyword = 'CRSTLK{0}{1}'.format(i + 1, j + 1)
crosstalk_matrix[i, j] = -float(image.header[crosstalk_keyword])
logs.add_tag(logging_tags, crosstalk_keyword,
image.header[crosstalk_keyword])
self.logger.info('Removing crosstalk', extra=logging_tags)
# Techinally, we should iterate this process because crosstalk doesn't
# produce more crosstalk
"""This dot product is effectivly the following:
coeffs = [[Q11, Q12, Q13, Q14],
[Q21, Q22, Q23, Q24],
[Q31, Q32, Q33, Q34],
[Q41, Q42, Q43, Q44]]
The corrected data, D, from quadrant i is
D1 = D1 - Q21 D2 - Q31 D3 - Q41 D4
D2 = D2 - Q12 D1 - Q32 D3 - Q42 D4
D3 = D3 - Q13 D1 - Q23 D2 - Q43 D4
D4 = D4 - Q14 D1 - Q24 D2 - Q34 D3
"""
image.data = np.dot(crosstalk_matrix.T, np.swapaxes(image.data, 0, 1))
return images
def do_stage(self, images):
for image in images:
logging_tags = logs.image_config_to_tags(image,
self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
logs.add_tag(logging_tags, 'gain', image.gain)
self.logger.info('Multiplying by gain', extra=logging_tags)
if len(image.data.shape) > 2:
n_amps = image.data.shape[0]
gain = np.array(eval(image.gain))
for i in range(n_amps):
image.data[i] *= gain[i]
image.header['SATURATE'] *= min(gain)
image.header['MAXLIN'] *= min(gain)
else:
image.data *= image.gain
image.header['SATURATE'] *= image.gain
image.header['MAXLIN'] *= 1.0
image.gain = 1.0
image.header['GAIN'] = 1.0
return images
def sinistro_mode_is_supported(image):
"""
Check to make sure the Sinistro image was taken in a supported mode.
Parameters
----------
image: banzai.images.Image
Sinistro image to check
Returns
-------
supported: bool
True if reduction is supported
Notes
-----
Currently we only support 1x1 binning images.
"""
# TODO Add support for other binnings
supported = True
tags = logs.image_config_to_tags(image, None)
logs.add_tag(tags, 'filename', image.filename)
if image.header['CCDSUM'] != '1 1':
supported = False
logger.error('Non-supported Sinistro mode', logging_tags=tags)
if image.instrument not in crosstalk_coefficients.keys():
supported = False
logger.error('Crosstalk Coefficients missing!', extra=tags)
return supported
def image_has_valid_saturate_value(image):
"""
Check if the image has a valid saturate value.
Parameters
----------
image: banzai.images.Image
Returns
-------
valid: bool
True if the image has a non-zero saturate value. False otherwise.
Notes
-----
The saturate keyword being zero causes a lot of headaches so we should just dump
the image if the saturate value is zero after we have fixed the typical incorrect values.
"""
valid = True
if float(image.header['SATURATE']) == 0.0:
tags = logs.image_config_to_tags(image, None)
logs.add_tag(tags, 'filename', image.filename)
logger.error('SATURATE keyword cannot be zero', extra=tags)
valid = False
return valid
def do_stage(self, images):
images_to_remove = []
for image in images:
logging_tags = logs.image_config_to_tags(image,
self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
logs.add_tag(logging_tags, 'gain', image.gain)
self.logger.info('Multiplying by gain', extra=logging_tags)
gain = image.gain
if validate_gain(gain):
self.logger.error('Gain missing. Rejecting image.',
extra=logging_tags)
images_to_remove.append(image)
else:
if len(image.data.shape) > 2:
n_amps = image.data.shape[0]
for i in range(n_amps):
image.data[i] *= gain[i]
image.header['SATURATE'] *= min(gain)
image.header['MAXLIN'] *= min(gain)
else:
image.data *= image.gain
image.header['SATURATE'] *= image.gain
image.header['MAXLIN'] *= image.gain
image.gain = 1.0
image.header['GAIN'] = 1.0
for image in images_to_remove:
images.remove(image)
return images
def check_exptime_value(self, image):
"""Logs an error if :
-1) the keyword exptime is not higher than 0.0
-2) the keyword exptime is equal to 0.0 and 'OBSTYPE' keyword is 'EXPOSE'
Parameters
----------
image : object
a banzais.image.Image object.
"""
exptime_value = image.header['EXPTIME']
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
if exptime_value < 0.0:
sentence = 'The header EXPTIME key got the unexpected value : negative value'
self.logger.error(sentence, extra=logging_tags)
return
obstype = image.header['OBSTYPE']
if (exptime_value == 0.0) & (obstype == 'EXPOSE'):
sentence = 'The header EXPTIME key got the unexpected value : 0.0'
self.logger.error(sentence, extra=logging_tags)
return
def do_stage(self, images):
images_to_remove = []
for image in images:
saturation_level = float(image.header['SATURATE'])
saturated_pixels = image.data >= saturation_level
total_pixels = image.data.size
saturation_fraction = float(saturated_pixels.sum()) / total_pixels
logging_tags = logs.image_config_to_tags(image,
self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
logs.add_tag(logging_tags, 'SATFRAC', saturation_fraction)
self.logger.info('Measured saturation fraction.',
extra=logging_tags)
if saturation_fraction >= self.threshold:
self.logger.error('SATFRAC exceeds threshold.',
extra=logging_tags)
images_to_remove.append(image)
else:
image.header['SATFRAC'] = (
saturation_fraction,
"Fraction of Pixels that are Saturated")
for image in images_to_remove:
images.remove(image)
return images
def do_stage(self, images):
for image in images:
if len(image.data.shape) > 2:
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
n_amps = image.data.shape[0]
nx, ny = get_mosaic_size(image, n_amps)
mosaiced_data = np.zeros((ny, nx), dtype=np.float32)
mosaiced_bpm = np.zeros((ny, nx), dtype=np.uint8)
for i in range(n_amps):
datasec = image.header['DATASEC{0}'.format(i + 1)]
amp_slice = fits_utils.parse_region_keyword(datasec)
logs.add_tag(logging_tags, 'DATASEC{0}'.format(i + 1), datasec)
detsec = image.header['DETSEC{0}'.format(i + 1)]
mosaic_slice = fits_utils.parse_region_keyword(detsec)
logs.add_tag(logging_tags, 'DETSEC{0}'.format(i + 1), datasec)
mosaiced_data[mosaic_slice] = image.data[i][amp_slice]
mosaiced_bpm[mosaic_slice] = image.bpm[i][amp_slice]
image.data = mosaiced_data
image.bpm = mosaiced_bpm
image.update_shape(nx, ny)
image.header['NAXIS'] = 2
image.header.pop('NAXIS3')
self.logger.info('Mosaiced image', extra=logging_tags)
return images
def create_master_calibration_header(images):
header = fits.Header()
for h in images[0].header.keys():
try:
# Dump empty header keywords
if len(h) > 0:
header[h] = images[0].header[h]
except ValueError as e:
logging_tags = logs.image_config_to_tags(images[0], None)
logs.add_tag(logging_tags, 'filename', images[0].filename)
logger.error('Could not add keyword {0}'.format(h),
extra=logging_tags)
continue
header = sanitizeheader(header)
observation_dates = [image.dateobs for image in images]
mean_dateobs = date_utils.mean_date(observation_dates)
header['DATE-OBS'] = date_utils.date_obs_to_string(mean_dateobs)
header.add_history("Images combined to create master calibration image:")
for image in images:
header.add_history(image.filename)
return header
def check_exptime_value(self, image):
"""Logs an error if :
-1) the keyword exptime is not higher than 0.0
-2) the keyword exptime is equal to 0.0 and 'OBSTYPE' keyword is 'EXPOSE'
Parameters
----------
image : object
a banzais.image.Image object.
"""
exptime_value = image.header['EXPTIME']
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
if exptime_value < 0.0:
sentence = 'The header EXPTIME key got the unexpected value : negative value'
self.logger.error(sentence, extra=logging_tags)
return
obstype = image.header['OBSTYPE']
if (exptime_value == 0.0) & (obstype == 'EXPOSE'):
sentence = 'The header EXPTIME key got the unexpected value : 0.0'
self.logger.error(sentence, extra=logging_tags)
return
def do_stage(self, images):
for i, image in enumerate(images):
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
# Save the catalog to a temporary file
filename = os.path.basename(image.filename)
logs.add_tag(logging_tags, 'filename', filename)
# Skip the image if we don't have some kind of initial RA and Dec guess
if np.isnan(image.ra) or np.isnan(image.dec):
self.logger.error('Skipping WCS solution. No initial pointing guess from header.',
extra=logging_tags)
continue
with tempfile.TemporaryDirectory() as tmpdirname:
catalog_name = os.path.join(tmpdirname, filename.replace('.fits', '.cat.fits'))
try:
image.write_catalog(catalog_name, nsources=40)
except image_utils.MissingCatalogException:
image.header['WCSERR'] = (4, 'Error status of WCS fit. 0 for no error')
self.logger.error('No source catalog. Not attempting WCS solution',
extra=logging_tags)
continue
# Run astrometry.net
wcs_name = os.path.join(tmpdirname, filename.replace('.fits', '.wcs.fits'))
command = self.cmd.format(ra=image.ra, dec=image.dec, scale_low=0.9*image.pixel_scale,
scale_high=1.1*image.pixel_scale, wcs_name=wcs_name,
catalog_name=catalog_name, nx=image.nx, ny=image.ny)
console_output = subprocess.check_output(shlex.split(command))
self.logger.debug(console_output, extra=logging_tags)
if os.path.exists(wcs_name):
# Copy the WCS keywords into original image
new_header = fits.getheader(wcs_name)
header_keywords_to_update = ['CTYPE1', 'CTYPE2', 'CRPIX1', 'CRPIX2', 'CRVAL1',
'CRVAL2', 'CD1_1', 'CD1_2', 'CD2_1', 'CD2_2']
for keyword in header_keywords_to_update:
image.header[keyword] = new_header[keyword]
image.header['WCSERR'] = (0, 'Error status of WCS fit. 0 for no error')
# Update the RA and Dec header keywords
image.header['RA'], image.header['DEC'] = get_ra_dec_in_sexagesimal(image.header['CRVAL1'],
image.header['CRVAL2'])
# Clean up wcs file
os.remove(wcs_name)
# Add the RA and Dec values to the catalog
add_ra_dec_to_catalog(image)
else:
image.header['WCSERR'] = (4, 'Error status of WCS fit. 0 for no error')
logs.add_tag(logging_tags, 'WCSERR', image.header['WCSERR'])
self.logger.info('Attempted WCS Solve', extra=logging_tags)
return images
def do_stage(self, images):
for image in images:
if len(image.data.shape) > 2:
logging_tags = logs.image_config_to_tags(
image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
n_amps = image.data.shape[0]
nx, ny = get_mosaic_size(image, n_amps)
mosaiced_data = np.zeros((ny, nx), dtype=np.float32)
mosaiced_bpm = np.zeros((ny, nx), dtype=np.uint8)
for i in range(n_amps):
datasec = image.header['DATASEC{0}'.format(i + 1)]
amp_slice = fits_utils.parse_region_keyword(datasec)
logs.add_tag(logging_tags, 'DATASEC{0}'.format(i + 1),
datasec)
detsec = image.header['DETSEC{0}'.format(i + 1)]
mosaic_slice = fits_utils.parse_region_keyword(detsec)
logs.add_tag(logging_tags, 'DETSEC{0}'.format(i + 1),
datasec)
mosaiced_data[mosaic_slice] = image.data[i][amp_slice]
mosaiced_bpm[mosaic_slice] = image.bpm[i][amp_slice]
image.data = mosaiced_data
image.bpm = mosaiced_bpm
image.update_shape(nx, ny)
image.header['NAXIS'] = 2
image.header.pop('NAXIS3')
self.logger.info('Mosaiced image', extra=logging_tags)
return images
def get_bpm(image, pipeline_context):
bpm_filename = dbs.get_bpm(image.telescope_id,
image.ccdsum,
db_address=pipeline_context.db_address)
if bpm_filename is None:
bpm = None
image.header['L1IDMASK'] = ('', 'Id. of mask file used')
else:
bpm_hdu = fits_utils.open_fits_file(bpm_filename)
bpm_extensions = fits_utils.get_extensions_by_name(bpm_hdu, 'BPM')
if len(bpm_extensions) > 1:
extension_shape = bpm_extensions[0].data.shape
bpm_shape = (len(bpm_extensions), extension_shape[0],
extension_shape[1])
bpm = np.zeros(bpm_shape, dtype=np.uint8)
for i, extension in enumerate(bpm_extensions):
bpm[i, :, :] = extension.data[:, :]
elif len(bpm_extensions) == 1:
bpm = np.array(bpm_extensions[0].data, dtype=np.uint8)
else:
bpm = np.array(bpm_hdu[0].data, dtype=np.uint8)
if not bpm_has_valid_size(bpm, image):
tags = logs.image_config_to_tags(image, None)
logs.add_tag(tags, 'filename', image.filename)
logger.error('BPM shape mismatch', extra=tags)
err_msg = 'BPM shape mismatch for {filename} ' \
'{site}/{instrument}'.format(filename=image.filename, site=image.site,
instrument=image.instrument)
raise ValueError(err_msg)
image.header['L1IDMASK'] = (os.path.basename(bpm_filename),
'Id. of mask file used')
return bpm
def get_bpm(image, pipeline_context):
bpm_filename = dbs.get_bpm(image.telescope_id, image.ccdsum,
db_address=pipeline_context.db_address)
if bpm_filename is None:
bpm = None
image.header['L1IDMASK'] = ('', 'Id. of mask file used')
else:
bpm_hdu = fits_utils.open_fits_file(bpm_filename)
bpm_extensions = fits_utils.get_extensions_by_name(bpm_hdu, 'BPM')
if len(bpm_extensions) > 1:
extension_shape = bpm_extensions[0].data.shape
bpm_shape = (len(bpm_extensions), extension_shape[0], extension_shape[1])
bpm = np.zeros(bpm_shape, dtype=np.uint8)
for i, extension in enumerate(bpm_extensions):
bpm[i, :, :] = extension.data[:, :]
elif len(bpm_extensions) == 1:
bpm = np.array(bpm_extensions[0].data, dtype=np.uint8)
else:
bpm = np.array(bpm_hdu[0].data, dtype=np.uint8)
if not bpm_has_valid_size(bpm, image):
tags = logs.image_config_to_tags(image, None)
logs.add_tag(tags, 'filename', image.filename)
logger.error('BPM shape mismatch', extra=tags)
err_msg = 'BPM shape mismatch for {filename} ' \
'{site}/{instrument}'.format(filename=image.filename, site=image.site,
instrument=image.instrument)
raise ValueError(err_msg)
image.header['L1IDMASK'] = (os.path.basename(bpm_filename), 'Id. of mask file used')
return bpm
def do_stage(self, images):
if len(images) < self.min_images:
# Do nothing
self.logger.warning('Not enough images to combine.')
return []
else:
image_config = image_utils.check_image_homogeneity(images)
logging_tags = logs.image_config_to_tags(image_config, self.group_by_keywords)
return self.make_master_calibration_frame(images, image_config, logging_tags)
def do_stage(self, images):
if len(images) < self.min_images:
# Do nothing
self.logger.warning('Not enough images to combine.')
return []
else:
image_config = image_utils.check_image_homogeneity(images)
logging_tags = logs.image_config_to_tags(image_config, self.group_by_keywords)
return self.make_master_calibration_frame(images, image_config, logging_tags)
def do_stage(self, images):
for image in images:
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
logs.add_tag(logging_tags, 'trimsec', image.header['TRIMSEC'])
self.logger.info('Trimming image', extra=logging_tags)
nx, ny = _trim_image(image)
image.update_shape(nx, ny)
return images
def writeto(self, filename, fpack=False):
image_hdu = fits.PrimaryHDU(self.data.astype(np.float32),
header=self.header)
image_hdu.header['BITPIX'] = -32
image_hdu.header['BSCALE'] = 1.0
image_hdu.header['BZERO'] = 0.0
image_hdu.header['SIMPLE'] = True
image_hdu.header['EXTEND'] = True
image_hdu.name = 'SCI'
hdu_list = [image_hdu]
if self.catalog is not None:
table_hdu = fits_utils.table_to_fits(self.catalog)
table_hdu.name = 'CAT'
hdu_list.append(table_hdu)
if self.bpm is not None:
bpm_hdu = fits.ImageHDU(self.bpm.astype(np.uint8))
bpm_hdu.name = 'BPM'
hdu_list.append(bpm_hdu)
hdu_list = fits.HDUList(hdu_list)
try:
hdu_list.verify(option='exception')
except fits.VerifyError as fits_error:
logging_tags = logs.image_config_to_tags(self, None)
logs.add_tag(logging_tags, 'filename',
os.path.basename(self.filename))
logger.warn(
'Error in FITS Verification. {0}. Attempting fix.'.format(
fits_error),
extra=logging_tags)
try:
hdu_list.verify(option='silentfix+exception')
except fits.VerifyError as fix_attempt_error:
logger.error('Could not repair FITS header. {0}'.format(
fix_attempt_error),
extra=logging_tags)
with tempfile.TemporaryDirectory() as temp_directory:
base_filename = os.path.basename(filename)
hdu_list.writeto(os.path.join(temp_directory, base_filename),
overwrite=True,
output_verify='fix+warn')
if fpack:
filename += '.fz'
if os.path.exists(filename):
os.remove(filename)
os.system('fpack -q 64 {temp_directory}/{basename}'.format(
temp_directory=temp_directory, basename=base_filename))
base_filename += '.fz'
self.filename += '.fz'
shutil.move(os.path.join(temp_directory, base_filename), filename)
def do_stage(self, images):
if len(images) == 0:
# Abort!
return []
else:
image_config = image_utils.check_image_homogeneity(images)
logging_tags = logs.image_config_to_tags(image_config, self.group_by_keywords)
master_calibration_filename = self.get_calibration_filename(images[0])
if master_calibration_filename is None:
self.logger.error('Master Calibration file does not exist for {stage}'.format(stage=self.stage_name),
extra=logging_tags)
raise MasterCalibrationDoesNotExist
master_calibration_image = Image(self.pipeline_context,
filename=master_calibration_filename)
return self.apply_master_calibration(images, master_calibration_image, logging_tags)
def do_stage(self, images):
for image in images:
if len(image.data.shape) > 2:
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
n_amps = image.data.shape[0]
crosstalk_matrix = np.identity(n_amps)
for j in range(n_amps):
for i in range(n_amps):
if i != j:
crosstalk_keyword = 'CRSTLK{0}{1}'.format(i + 1, j + 1)
crosstalk_matrix[j, i] = -float(image.header[crosstalk_keyword])
logs.add_tag(logging_tags, crosstalk_keyword,
image.header[crosstalk_keyword])
self.logger.info('Removing crosstalk', extra=logging_tags)
image.data = np.dot(crosstalk_matrix, np.swapaxes(image.data, 0, 1))
return images
def do_stage(self, images):
if len(images) == 0:
# Abort!
return []
else:
image_config = image_utils.check_image_homogeneity(images)
logging_tags = logs.image_config_to_tags(image_config, self.group_by_keywords)
master_calibration_filename = self.get_calibration_filename(images[0])
if master_calibration_filename is None:
self.logger.error('Master Calibration file does not exist for {stage}'.format(stage=self.stage_name),
extra=logging_tags)
raise MasterCalibrationDoesNotExist
master_calibration_image = Image(self.pipeline_context,
filename=master_calibration_filename)
return self.apply_master_calibration(images, master_calibration_image, logging_tags)
def writeto(self, filename, fpack=False):
image_hdu = fits.PrimaryHDU(self.data.astype(np.float32), header=self.header)
image_hdu.header['BITPIX'] = -32
image_hdu.header['BSCALE'] = 1.0
image_hdu.header['BZERO'] = 0.0
image_hdu.header['SIMPLE'] = True
image_hdu.header['EXTEND'] = True
image_hdu.name = 'SCI'
hdu_list = [image_hdu]
if self.catalog is not None:
table_hdu = fits_utils.table_to_fits(self.catalog)
table_hdu.name = 'CAT'
hdu_list.append(table_hdu)
if self.bpm is not None:
bpm_hdu = fits.ImageHDU(self.bpm.astype(np.uint8))
bpm_hdu.name = 'BPM'
hdu_list.append(bpm_hdu)
hdu_list = fits.HDUList(hdu_list)
try:
hdu_list.verify(option='exception')
except fits.VerifyError as fits_error:
logging_tags = logs.image_config_to_tags(self, None)
logs.add_tag(logging_tags, 'filename', os.path.basename(self.filename))
logger.warn('Error in FITS Verification. {0}. Attempting fix.'.format(fits_error),
extra=logging_tags)
try:
hdu_list.verify(option='silentfix+exception')
except fits.VerifyError as fix_attempt_error:
logger.error('Could not repair FITS header. {0}'.format(fix_attempt_error),
extra=logging_tags)
with tempfile.TemporaryDirectory() as temp_directory:
base_filename = os.path.basename(filename)
hdu_list.writeto(os.path.join(temp_directory, base_filename), overwrite=True,
output_verify='fix+warn')
if fpack:
filename += '.fz'
if os.path.exists(filename):
os.remove(filename)
os.system('fpack -q 64 {temp_directory}/{basename}'.format(temp_directory=temp_directory,
basename=base_filename))
base_filename += '.fz'
self.filename += '.fz'
shutil.move(os.path.join(temp_directory, base_filename), filename)
def do_stage(self, images):
images_to_remove = []
for image in images:
npixels = np.product(image.data.shape)
fraction_1000s = float(np.sum(image.data == 1000)) / npixels
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
logs.add_tag(logging_tags, 'FRAC1000', fraction_1000s)
if fraction_1000s > self.threshold:
self.logger.error('Image is mostly 1000s. Rejecting image', extra=logging_tags)
images_to_remove.append(image)
else:
self.logger.info('Measuring fraction of 1000s.', extra=logging_tags)
for image in images_to_remove:
images.remove(image)
return images
def check_dec_range(self, image):
"""Logs an error if the keyword declination is not inside
the expected range (-90<dec<90 degrees) in the image header.
Parameters
----------
image : object
a banzais.image.Image object.
"""
dec_value = image.header['CRVAL2']
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
if (dec_value > 90) | (dec_value < -90):
sentence = 'The header CRVAL2 key got the unexpected value : ' + str(dec_value)
self.logger.error(sentence, extra=logging_tags)
def do_stage(self, images):
for image in images:
saturation_level = float(image.header['SATURATE'])
saturated_pixels = image.data >= saturation_level
total_pixels = image.data.size
saturation_fraction = float(saturated_pixels.sum()) / total_pixels
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
logs.add_tag(logging_tags, 'SATFRAC', saturation_fraction)
self.logger.info('Measured saturation fraction.', extra=logging_tags)
if saturation_fraction >= self.threshold:
self.logger.error('SATFRAC exceeds threshold.', extra=logging_tags)
images.remove(image)
else:
image.header['SATFRAC'] = (saturation_fraction, "Fraction of Pixels that are Saturated")
return images
def do_stage(self, images):
for image in images:
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
# Subtract the overscan if it exists
if len(image.data.shape) > 2:
for i in range(image.data.shape[0]):
overscan_level = _subtract_overscan_3d(image, i)
logs.add_tag(logging_tags, 'OVERSCN{0}'.format(i + 1), overscan_level)
self.logger.info('Subtracting overscan', extra=logging_tags)
else:
overscan_level = _subtract_overscan_2d(image)
logs.add_tag(logging_tags, 'OVERSCAN', overscan_level)
self.logger.info('Subtracting overscan', extra=logging_tags)
return images
def writeto(self, filename, fpack=False):
image_hdu = fits.PrimaryHDU(self.data.astype(np.float32),
header=self.header)
image_hdu.header['BITPIX'] = -32
image_hdu.header['BSCALE'] = 1.0
image_hdu.header['BZERO'] = 0.0
image_hdu.header['SIMPLE'] = True
image_hdu.header['EXTEND'] = True
image_hdu.update_ext_name('SCI')
hdu_list = [image_hdu]
if self.catalog is not None:
table_hdu = fits_utils.table_to_fits(self.catalog)
table_hdu.update_ext_name('CAT')
hdu_list.append(table_hdu)
if self.bpm is not None:
bpm_hdu = fits.ImageHDU(self.bpm.astype(np.uint8))
bpm_hdu.update_ext_name('BPM')
hdu_list.append(bpm_hdu)
hdu_list = fits.HDUList(hdu_list)
try:
hdu_list.verify(option='exception')
except fits.VerifyError as fits_error:
logging_tags = logs.image_config_to_tags(self, None)
logs.add_tag(logging_tags, 'filename',
os.path.basename(self.filename))
logger.warn(
'Error in FITS Verification. {0}. Attempting fix.'.format(
fits_error),
extra=logging_tags)
try:
hdu_list.verify(option='silentfix+exception')
except fits.VerifyError as fix_attempt_error:
logger.error('Could not repair FITS header. {0}'.format(
fix_attempt_error),
extra=logging_tags)
hdu_list.writeto(filename, clobber=True, output_verify='fix+warn')
if fpack:
if os.path.exists(filename + '.fz'):
os.remove(filename + '.fz')
os.system('fpack -q 64 {0}'.format(filename))
os.remove(filename)
self.filename += '.fz'
def check_header_keyword_present(self, keyword, image):
""" Logs an error if the keyword is not in the image header.
Parameters
----------
keyword : str
the keyword of interest
image : object
a banzais.image.Image object.
"""
header = image.header
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
if keyword not in header:
sentence = 'The header key ' + keyword + ' is not in image header!'
self.logger.error(sentence, extra=logging_tags)
def save_pipeline_metadata(image, pipeline_context):
image.header['RLEVEL'] = (pipeline_context.rlevel, 'Reduction level')
image.header['PIPEVER'] = (banzai.__version__, 'Pipeline version')
if file_utils.instantly_public(image.header['PROPID']):
image.header['L1PUBDAT'] = (image.header['DATE-OBS'],
'[UTC] Date the frame becomes public')
else:
# Wait a year
date_observed = date_utils.parse_date_obs(image.header['DATE-OBS'])
next_year = date_observed + timedelta(days=365)
image.header['L1PUBDAT'] = (date_utils.date_obs_to_string(next_year),
'[UTC] Date the frame becomes public')
logging_tags = logs.image_config_to_tags(image, None)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
logs.add_tag(logging_tags, 'rlevel', int(image.header['RLEVEL']))
logs.add_tag(logging_tags, 'pipeline_version', image.header['PIPEVER'])
logs.add_tag(logging_tags, 'l1pubdat', image.header['L1PUBDAT'])
logger.info('Updating header', extra=logging_tags)
def check_header_keyword_present(self, keyword, image):
""" Logs an error if the keyword is not in the image header.
Parameters
----------
keyword : str
the keyword of interest
image : object
a banzais.image.Image object.
"""
header = image.header
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
if keyword not in header:
sentence = 'The header key ' + keyword + ' is not in image header!'
self.logger.error(sentence, extra=logging_tags)
def check_ra_range(self, image):
""" Logs an error if the keyword right_ascension is not inside
the expected range (0<ra<360 degrees) in the image header.
Parameters
----------
image : object
a banzais.image.Image object.
"""
ra_value = image.header['CRVAL1']
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
if isinstance(ra_value, float):
if (ra_value > 360) | (ra_value < 0):
sentence = 'The header CRVAL1 key got the unexpected value : ' + str(ra_value)
self.logger.error(sentence, extra=logging_tags)
def check_dec_range(self, image):
"""Logs an error if the keyword declination is not inside
the expected range (-90<dec<90 degrees) in the image header.
Parameters
----------
image : object
a banzais.image.Image object.
"""
dec_value = image.header['CRVAL2']
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
if (dec_value > 90) | (dec_value < -90):
sentence = 'The header CRVAL2 key got the unexpected value : ' + str(
dec_value)
self.logger.error(sentence, extra=logging_tags)
def check_ra_range(self, image):
""" Logs an error if the keyword right_ascension is not inside
the expected range (0<ra<360 degrees) in the image header.
Parameters
----------
image : object
a banzais.image.Image object.
"""
ra_value = image.header['CRVAL1']
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
if isinstance(ra_value, float):
if (ra_value > 360) | (ra_value < 0):
sentence = 'The header CRVAL1 key got the unexpected value : ' + str(
ra_value)
self.logger.error(sentence, extra=logging_tags)
def _trim_image(image):
trimsec = fits_utils.parse_region_keyword(image.header['TRIMSEC'])
if trimsec is not None:
image.data = image.data[trimsec]
image.bpm = image.bpm[trimsec]
# Update the NAXIS and CRPIX keywords
image.header['NAXIS1'] = trimsec[1].stop - trimsec[1].start
image.header['NAXIS2'] = trimsec[0].stop - trimsec[0].start
image.header['CRPIX1'] -= trimsec[1].start
image.header['CRPIX2'] -= trimsec[0].start
image.header['L1STATTR'] = (1, 'Status flag for overscan trimming')
else:
logging_tags = logs.image_config_to_tags(image, None)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
logs.add_tag(logging_tags, 'trimsec', image.header['TRIMSEC'])
logger.warning('TRIMSEC was not defined.', extra=logging_tags)
image.header['L1STATTR'] = (0, 'Status flag for overscan trimming')
return image.header['NAXIS1'], image.header['NAXIS2']
def do_stage(self, images):
images_to_remove = []
for image in images:
npixels = np.product(image.data.shape)
fraction_1000s = float(np.sum(image.data == 1000)) / npixels
logging_tags = logs.image_config_to_tags(image,
self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
logs.add_tag(logging_tags, 'FRAC1000', fraction_1000s)
if fraction_1000s > self.threshold:
self.logger.error('Image is mostly 1000s. Rejecting image',
extra=logging_tags)
images_to_remove.append(image)
else:
self.logger.info('Measuring fraction of 1000s.',
extra=logging_tags)
for image in images_to_remove:
images.remove(image)
return images
def writeto(self, filename, fpack=False):
image_hdu = fits.PrimaryHDU(self.data.astype(np.float32), header=self.header)
image_hdu.header['BITPIX'] = -32
image_hdu.header['BSCALE'] = 1.0
image_hdu.header['BZERO'] = 0.0
image_hdu.header['SIMPLE'] = True
image_hdu.header['EXTEND'] = True
image_hdu.update_ext_name('SCI')
hdu_list = [image_hdu]
if self.catalog is not None:
table_hdu = fits_utils.table_to_fits(self.catalog)
table_hdu.update_ext_name('CAT')
hdu_list.append(table_hdu)
if self.bpm is not None:
bpm_hdu = fits.ImageHDU(self.bpm.astype(np.uint8))
bpm_hdu.update_ext_name('BPM')
hdu_list.append(bpm_hdu)
hdu_list = fits.HDUList(hdu_list)
try:
hdu_list.verify(option='exception')
except fits.VerifyError as fits_error:
logging_tags = logs.image_config_to_tags(self, None)
logs.add_tag(logging_tags, 'filename', os.path.basename(self.filename))
logger.warn('Error in FITS Verification. {0}. Attempting fix.'.format(fits_error),
extra=logging_tags)
try:
hdu_list.verify(option='silentfix+exception')
except fits.VerifyError as fix_attempt_error:
logger.error('Could not repair FITS header. {0}'.format(fix_attempt_error),
extra=logging_tags)
hdu_list.writeto(filename, clobber=True, output_verify='fix+warn')
if fpack:
if os.path.exists(filename + '.fz'):
os.remove(filename + '.fz')
os.system('fpack -q 64 {0}'.format(filename))
os.remove(filename)
self.filename += '.fz'
def check_header_na(self, keyword, image):
""" Logs an error if the keyword is 'N/A' instead of the
expected type in the image header.
Parameters
----------
keyword : str
the keyword of interest
image : object
a banzais.image.Image object.
"""
header_value = image.header[keyword]
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
if isinstance(header_value, str):
if 'N/A' in header_value:
sentence = 'The header key ' + keyword + ' got the unexpected value : N/A'
self.logger.error(sentence, extra=logging_tags)
def check_header_na(self, keyword, image):
""" Logs an error if the keyword is 'N/A' instead of the
expected type in the image header.
Parameters
----------
keyword : str
the keyword of interest
image : object
a banzais.image.Image object.
"""
header_value = image.header[keyword]
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
if isinstance(header_value, str):
if 'N/A' in header_value:
sentence = 'The header key ' + keyword + ' got the unexpected value : N/A'
self.logger.error(sentence, extra=logging_tags)
def do_stage(self, images):
for image in images:
if len(image.data.shape) > 2:
logging_tags = logs.image_config_to_tags(
image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
n_amps = image.data.shape[0]
crosstalk_matrix = np.identity(n_amps)
for j in range(n_amps):
for i in range(n_amps):
if i != j:
crosstalk_keyword = 'CRSTLK{0}{1}'.format(
i + 1, j + 1)
crosstalk_matrix[j, i] = -float(
image.header[crosstalk_keyword])
logs.add_tag(logging_tags, crosstalk_keyword,
image.header[crosstalk_keyword])
self.logger.info('Removing crosstalk', extra=logging_tags)
image.data = np.dot(crosstalk_matrix,
np.swapaxes(image.data, 0, 1))
return images
def do_stage(self, images):
for image in images:
logging_tags = logs.image_config_to_tags(image,
self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
# Subtract the overscan if it exists
if len(image.data.shape) > 2:
for i in range(image.data.shape[0]):
overscan_level = _subtract_overscan_3d(image, i)
logs.add_tag(logging_tags, 'OVERSCN{0}'.format(i + 1),
overscan_level)
self.logger.info('Subtracting overscan',
extra=logging_tags)
else:
overscan_level = _subtract_overscan_2d(image)
logs.add_tag(logging_tags, 'OVERSCAN', overscan_level)
self.logger.info('Subtracting overscan', extra=logging_tags)
return images
def do_stage(self, images):
for image in images:
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
logs.add_tag(logging_tags, 'gain', image.gain)
self.logger.info('Multiplying by gain', extra=logging_tags)
if len(image.data.shape) > 2:
n_amps = image.data.shape[0]
gain = np.array(eval(image.gain))
for i in range(n_amps):
image.data[i] *= gain[i]
image.header['SATURATE'] *= min(gain)
image.header['MAXLIN'] *= min(gain)
else:
image.data *= image.gain
image.header['SATURATE'] *= image.gain
image.header['MAXLIN'] *= image.gain
image.gain = 1.0
image.header['GAIN'] = 1.0
return images
def check_header_format(self, keyword, image):
""" Logs an error if the keyword is not the expected type
in the image header.
Parameters
----------
keyword : str
the keyword of interest
image : object
a banzais.image.Image object.
"""
header_value = image.header[keyword]
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
if not isinstance(header_value, self.header_expected_format[keyword]):
sentence = ('The header key ' + keyword + ' got an unexpected format : ' + type(
header_value).__name__ + ' in place of ' + self.header_expected_format[
keyword].__name__)
self.logger.error(sentence, extra=logging_tags)
def create_master_calibration_header(images):
header = fits.Header()
for h in images[0].header.keys():
try:
# Dump empty header keywords
if len(h) > 0:
header[h] = images[0].header[h]
except ValueError as e:
logging_tags = logs.image_config_to_tags(images[0], None)
logs.add_tag(logging_tags, 'filename', images[0].filename)
logger.error('Could not add keyword {0}'.format(h), extra=logging_tags)
continue
header = sanitizeheader(header)
observation_dates = [image.dateobs for image in images]
mean_dateobs = date_utils.mean_date(observation_dates)
header['DATE-OBS'] = date_utils.date_obs_to_string(mean_dateobs)
header.add_history("Images combined to create master calibration image:")
for image in images:
header.add_history(image.filename)
return header
def do_stage(self, images):
for image in images:
if len(image.data.shape) > 2:
logging_tags = logs.image_config_to_tags(
image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
n_amps = image.data.shape[0]
crosstalk_matrix = np.identity(n_amps)
for j in range(n_amps):
for i in range(n_amps):
if i != j:
crosstalk_keyword = 'CRSTLK{0}{1}'.format(
i + 1, j + 1)
crosstalk_matrix[i, j] = -float(
image.header[crosstalk_keyword])
logs.add_tag(logging_tags, crosstalk_keyword,
image.header[crosstalk_keyword])
self.logger.info('Removing crosstalk', extra=logging_tags)
# Techinally, we should iterate this process because crosstalk doesn't
# produce more crosstalk
"""This dot product is effectivly the following:
coeffs = [[Q11, Q12, Q13, Q14],
[Q21, Q22, Q23, Q24],
[Q31, Q32, Q33, Q34],
[Q41, Q42, Q43, Q44]]
The corrected data, D, from quadrant i is
D1 = D1 - Q21 D2 - Q31 D3 - Q41 D4
D2 = D2 - Q12 D1 - Q32 D3 - Q42 D4
D3 = D3 - Q13 D1 - Q23 D2 - Q43 D4
D4 = D4 - Q14 D1 - Q24 D2 - Q34 D3
"""
image.data = np.dot(crosstalk_matrix.T,
np.swapaxes(image.data, 0, 1))
return images
def check_header_format(self, keyword, image):
""" Logs an error if the keyword is not the expected type
in the image header.
Parameters
----------
keyword : str
the keyword of interest
image : object
a banzais.image.Image object.
"""
header_value = image.header[keyword]
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
if not isinstance(header_value, self.header_expected_format[keyword]):
sentence = ('The header key ' + keyword +
' got an unexpected format : ' +
type(header_value).__name__ + ' in place of ' +
self.header_expected_format[keyword].__name__)
self.logger.error(sentence, extra=logging_tags)
def run_stage(self, image_set, image_config):
image_set = list(image_set)
tags = logs.image_config_to_tags(image_set[0], self.group_by_keywords)
self.logger.info('Running {0}'.format(self.stage_name), extra=tags)
return self.do_stage(image_set)
def do_stage(self, images):
for i, image in enumerate(images):
try:
# Set the number of source pixels to be 5% of the total. This keeps us safe from
# satellites and airplanes.
sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))
data = image.data.copy()
error = (np.abs(data) + image.readnoise ** 2.0) ** 0.5
mask = image.bpm > 0
# Fits can be backwards byte order, so fix that if need be and subtract
# the background
try:
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
except ValueError:
data = data.byteswap(True).newbyteorder()
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
bkg.subfrom(data)
# Do an initial source detection
# TODO: Add back in masking after we are sure SEP works
sources = sep.extract(data, self.threshold, minarea=self.min_area,
err=error, deblend_cont=0.005)
# Convert the detections into a table
sources = Table(sources)
# Calculate the ellipticity
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# Fix any value of theta that are invalid due to floating point rounding
# -pi / 2 < theta < pi / 2
sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi
# Calculate the kron radius
kronrad, krflag = sep.kron_radius(data, sources['x'], sources['y'],
sources['a'], sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
sources['kronrad'] = kronrad
# Calcuate the equivilent of flux_auto
flux, fluxerr, flag = sep.sum_ellipse(data, sources['x'], sources['y'],
sources['a'], sources['b'],
np.pi / 2.0, 2.5 * kronrad,
subpix=1, err=error)
sources['flux'] = flux
sources['fluxerr'] = fluxerr
sources['flag'] |= flag
# Calculate the FWHMs of the stars:
fwhm = 2.0 * (np.log(2) * (sources['a'] ** 2.0 + sources['b'] ** 2.0)) ** 0.5
sources['fwhm'] = fwhm
# Cut individual bright pixels. Often cosmic rays
sources = sources[fwhm > 1.0]
# Measure the flux profile
flux_radii, flag = sep.flux_radius(data, sources['x'], sources['y'],
6.0 * sources['a'], [0.25, 0.5, 0.75],
normflux=sources['flux'], subpix=5)
sources['flag'] |= flag
sources['fluxrad25'] = flux_radii[:, 0]
sources['fluxrad50'] = flux_radii[:, 1]
sources['fluxrad75'] = flux_radii[:, 2]
# Calculate the windowed positions
sig = 2.0 / 2.35 * sources['fluxrad50']
xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'], sig)
sources['flag'] |= flag
sources['xwin'] = xwin
sources['ywin'] = ywin
# Calculate the average background at each source
bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(), sources['x'], sources['y'],
sources['a'], sources['b'], np.pi / 2.0,
2.5 * sources['kronrad'], subpix=1)
#masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
# sources['a'], sources['b'], np.pi / 2.0,
# 2.5 * kronrad, subpix=1)
background_area = (2.5 * sources['kronrad']) ** 2.0 * sources['a'] * sources['b'] * np.pi # - masksum
sources['background'] = bkgflux
sources['background'][background_area > 0] /= background_area[background_area > 0]
# Update the catalog to match fits convention instead of python array convention
sources['x'] += 1.0
sources['y'] += 1.0
sources['xpeak'] += 1
sources['ypeak'] += 1
sources['xwin'] += 1.0
sources['ywin'] += 1.0
sources['theta'] = np.degrees(sources['theta'])
image.catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak', 'ypeak',
'flux', 'fluxerr', 'background', 'fwhm',
'a', 'b', 'theta', 'kronrad', 'ellipticity',
'fluxrad25', 'fluxrad50', 'fluxrad75',
'x2', 'y2', 'xy', 'flag']
# Add the units and description to the catalogs
image.catalog['x'].unit = 'pixel'
image.catalog['x'].description = 'X coordinate of the object'
image.catalog['y'].unit = 'pixel'
image.catalog['y'].description = 'Y coordinate of the object'
image.catalog['xwin'].unit = 'pixel'
image.catalog['xwin'].description = 'Windowed X coordinate of the object'
image.catalog['ywin'].unit = 'pixel'
image.catalog['ywin'].description = 'Windowed Y coordinate of the object'
image.catalog['xpeak'].unit = 'pixel'
image.catalog['xpeak'].description = 'X coordinate of the peak'
image.catalog['ypeak'].unit = 'pixel'
image.catalog['ypeak'].description = 'Windowed Y coordinate of the peak'
image.catalog['flux'].unit = 'counts'
image.catalog['flux'].description = 'Flux within a Kron-like elliptical aperture'
image.catalog['fluxerr'].unit = 'counts'
image.catalog['fluxerr'].description = 'Erronr on the flux within a Kron-like elliptical aperture'
image.catalog['background'].unit = 'counts'
image.catalog['background'].description = 'Average background value in the aperture'
image.catalog['fwhm'].unit = 'pixel'
image.catalog['fwhm'].description = 'FWHM of the object'
image.catalog['a'].unit = 'pixel'
image.catalog['a'].description = 'Semi-major axis of the object'
image.catalog['b'].unit = 'pixel'
image.catalog['b'].description = 'Semi-minor axis of the object'
image.catalog['theta'].unit = 'degrees'
image.catalog['theta'].description = 'Position angle of the object'
image.catalog['kronrad'].unit = 'pixel'
image.catalog['kronrad'].description = 'Kron radius used for extraction'
image.catalog['ellipticity'].description = 'Ellipticity'
image.catalog['fluxrad25'].unit = 'pixel'
image.catalog['fluxrad25'].description = 'Radius containing 25% of the flux'
image.catalog['fluxrad50'].unit = 'pixel'
image.catalog['fluxrad50'].description = 'Radius containing 50% of the flux'
image.catalog['fluxrad75'].unit = 'pixel'
image.catalog['fluxrad75'].description = 'Radius containing 75% of the flux'
image.catalog['x2'].unit = 'pixel^2'
image.catalog['x2'].description = 'Variance on X coordinate of the object'
image.catalog['y2'].unit = 'pixel^2'
image.catalog['y2'].description = 'Variance on Y coordinate of the object'
image.catalog['xy'].unit = 'pixel^2'
image.catalog['xy'].description = 'XY covariance of the object'
image.catalog['flag'].description = 'Bit mask combination of extraction and photometry flags'
image.catalog.sort('flux')
image.catalog.reverse()
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
# Save some background statistics in the header
mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
image.header['L1MEAN'] = (mean_background,
'[counts] Sigma clipped mean of frame background')
logs.add_tag(logging_tags, 'L1MEAN', float(mean_background))
median_background = np.median(bkg.back())
image.header['L1MEDIAN'] = (median_background,
'[counts] Median of frame background')
logs.add_tag(logging_tags, 'L1MEDIAN', float(median_background))
std_background = stats.robust_standard_deviation(bkg.back())
image.header['L1SIGMA'] = (std_background,
'[counts] Robust std dev of frame background')
logs.add_tag(logging_tags, 'L1SIGMA', float(std_background))
# Save some image statistics to the header
good_objects = image.catalog['flag'] == 0
seeing = np.median(image.catalog['fwhm'][good_objects]) * image.pixel_scale
image.header['L1FWHM'] = (seeing, '[arcsec] Frame FWHM in arcsec')
logs.add_tag(logging_tags, 'L1FWHM', float(seeing))
mean_ellipticity = stats.sigma_clipped_mean(sources['ellipticity'][good_objects],
3.0)
image.header['L1ELLIP'] = (mean_ellipticity, 'Mean image ellipticity (1-B/A)')
logs.add_tag(logging_tags, 'L1ELLIP', float(mean_ellipticity))
mean_position_angle = stats.sigma_clipped_mean(sources['theta'][good_objects], 3.0)
image.header['L1ELLIPA'] = (mean_position_angle,
'[deg] PA of mean image ellipticity')
logs.add_tag(logging_tags, 'L1ELLIPA', float(mean_position_angle))
self.logger.info('Extracted sources', extra=logging_tags)
except Exception as e:
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
self.logger.error(e, extra=logging_tags)
return images
def setup_logging(self, image):
self.logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(self.logging_tags, 'filename', os.path.basename(image.filename))
def do_stage(self, images):
for i, image in enumerate(images):
logging_tags = logs.image_config_to_tags(image,
self.group_by_keywords)
# Save the catalog to a temporary file
filename = os.path.basename(image.filename)
logs.add_tag(logging_tags, 'filename', filename)
with tempfile.TemporaryDirectory() as tmpdirname:
catalog_name = os.path.join(
tmpdirname, filename.replace('.fits', '.cat.fits'))
try:
image.write_catalog(catalog_name, nsources=40)
except image_utils.MissingCatalogException:
image.header['WCSERR'] = (
4, 'Error status of WCS fit. 0 for no error')
self.logger.error(
'No source catalog. Not attempting WCS solution',
extra=logging_tags)
continue
# Run astrometry.net
wcs_name = os.path.join(tmpdirname,
filename.replace('.fits', '.wcs.fits'))
command = self.cmd.format(ra=image.ra,
dec=image.dec,
scale_low=0.9 * image.pixel_scale,
scale_high=1.1 * image.pixel_scale,
wcs_name=wcs_name,
catalog_name=catalog_name,
nx=image.nx,
ny=image.ny)
console_output = subprocess.check_output(shlex.split(command))
self.logger.debug(console_output, extra=logging_tags)
if os.path.exists(wcs_name):
# Copy the WCS keywords into original image
new_header = fits.getheader(wcs_name)
header_keywords_to_update = [
'CTYPE1', 'CTYPE2', 'CRPIX1', 'CRPIX2', 'CRVAL1',
'CRVAL2', 'CD1_1', 'CD1_2', 'CD2_1', 'CD2_2'
]
for keyword in header_keywords_to_update:
image.header[keyword] = new_header[keyword]
image.header['WCSERR'] = (
0, 'Error status of WCS fit. 0 for no error')
# Update the RA and Dec header keywords
image.header['RA'], image.header[
'DEC'] = get_ra_dec_in_sexagesimal(
image.header['CRVAL1'], image.header['CRVAL2'])
# Clean up wcs file
os.remove(wcs_name)
# Add the RA and Dec values to the catalog
add_ra_dec_to_catalog(image)
else:
image.header['WCSERR'] = (
4, 'Error status of WCS fit. 0 for no error')
logs.add_tag(logging_tags, 'WCSERR', image.header['WCSERR'])
self.logger.info('Attempted WCS Solve', extra=logging_tags)
return images
def run_stage(self, image_set, image_config):
image_set = list(image_set)
tags = logs.image_config_to_tags(image_set[0], self.group_by_keywords)
self.logger.info('Running {0}'.format(self.stage_name), extra=tags)
return self.do_stage(image_set)
def setup_logging(self, image):
self.logging_tags = logs.image_config_to_tags(image,
self.group_by_keywords)
logs.add_tag(self.logging_tags, 'filename',
os.path.basename(image.filename))
def do_stage(self, images):
for i, image in enumerate(images):
try:
# Set the number of source pixels to be 5% of the total. This keeps us safe from
# satellites and airplanes.
sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))
data = image.data.copy()
error = (np.abs(data) + image.readnoise**2.0)**0.5
mask = image.bpm > 0
# Fits can be backwards byte order, so fix that if need be and subtract
# the background
try:
bkg = sep.Background(data,
mask=mask,
bw=32,
bh=32,
fw=3,
fh=3)
except ValueError:
data = data.byteswap(True).newbyteorder()
bkg = sep.Background(data,
mask=mask,
bw=32,
bh=32,
fw=3,
fh=3)
bkg.subfrom(data)
# Do an initial source detection
# TODO: Add back in masking after we are sure SEP works
sources = sep.extract(data,
self.threshold,
minarea=self.min_area,
err=error,
deblend_cont=0.005)
# Convert the detections into a table
sources = Table(sources)
# Calculate the ellipticity
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# Fix any value of theta that are invalid due to floating point rounding
# -pi / 2 < theta < pi / 2
sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi
# Calculate the kron radius
kronrad, krflag = sep.kron_radius(data, sources['x'],
sources['y'], sources['a'],
sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
sources['kronrad'] = kronrad
# Calcuate the equivilent of flux_auto
flux, fluxerr, flag = sep.sum_ellipse(data,
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 * kronrad,
subpix=1,
err=error)
sources['flux'] = flux
sources['fluxerr'] = fluxerr
sources['flag'] |= flag
# Calculate the FWHMs of the stars:
fwhm = 2.0 * (np.log(2) *
(sources['a']**2.0 + sources['b']**2.0))**0.5
sources['fwhm'] = fwhm
# Cut individual bright pixels. Often cosmic rays
sources = sources[fwhm > 1.0]
# Measure the flux profile
flux_radii, flag = sep.flux_radius(data,
sources['x'],
sources['y'],
6.0 * sources['a'],
[0.25, 0.5, 0.75],
normflux=sources['flux'],
subpix=5)
sources['flag'] |= flag
sources['fluxrad25'] = flux_radii[:, 0]
sources['fluxrad50'] = flux_radii[:, 1]
sources['fluxrad75'] = flux_radii[:, 2]
# Calculate the windowed positions
sig = 2.0 / 2.35 * sources['fluxrad50']
xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'],
sig)
sources['flag'] |= flag
sources['xwin'] = xwin
sources['ywin'] = ywin
# Calculate the average background at each source
bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(),
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 *
sources['kronrad'],
subpix=1)
#masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
# sources['a'], sources['b'], np.pi / 2.0,
# 2.5 * kronrad, subpix=1)
background_area = (
2.5 * sources['kronrad']
)**2.0 * sources['a'] * sources['b'] * np.pi # - masksum
sources['background'] = bkgflux
sources['background'][background_area > 0] /= background_area[
background_area > 0]
# Update the catalog to match fits convention instead of python array convention
sources['x'] += 1.0
sources['y'] += 1.0
sources['xpeak'] += 1
sources['ypeak'] += 1
sources['xwin'] += 1.0
sources['ywin'] += 1.0
sources['theta'] = np.degrees(sources['theta'])
image.catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak',
'ypeak', 'flux', 'fluxerr',
'background', 'fwhm', 'a', 'b',
'theta', 'kronrad', 'ellipticity',
'fluxrad25', 'fluxrad50', 'fluxrad75',
'x2', 'y2', 'xy', 'flag']
# Add the units and description to the catalogs
image.catalog['x'].unit = 'pixel'
image.catalog['x'].description = 'X coordinate of the object'
image.catalog['y'].unit = 'pixel'
image.catalog['y'].description = 'Y coordinate of the object'
image.catalog['xwin'].unit = 'pixel'
image.catalog[
'xwin'].description = 'Windowed X coordinate of the object'
image.catalog['ywin'].unit = 'pixel'
image.catalog[
'ywin'].description = 'Windowed Y coordinate of the object'
image.catalog['xpeak'].unit = 'pixel'
image.catalog['xpeak'].description = 'X coordinate of the peak'
image.catalog['ypeak'].unit = 'pixel'
image.catalog[
'ypeak'].description = 'Windowed Y coordinate of the peak'
image.catalog['flux'].unit = 'counts'
image.catalog[
'flux'].description = 'Flux within a Kron-like elliptical aperture'
image.catalog['fluxerr'].unit = 'counts'
image.catalog[
'fluxerr'].description = 'Erronr on the flux within a Kron-like elliptical aperture'
image.catalog['background'].unit = 'counts'
image.catalog[
'background'].description = 'Average background value in the aperture'
image.catalog['fwhm'].unit = 'pixel'
image.catalog['fwhm'].description = 'FWHM of the object'
image.catalog['a'].unit = 'pixel'
image.catalog[
'a'].description = 'Semi-major axis of the object'
image.catalog['b'].unit = 'pixel'
image.catalog[
'b'].description = 'Semi-minor axis of the object'
image.catalog['theta'].unit = 'degrees'
image.catalog[
'theta'].description = 'Position angle of the object'
image.catalog['kronrad'].unit = 'pixel'
image.catalog[
'kronrad'].description = 'Kron radius used for extraction'
image.catalog['ellipticity'].description = 'Ellipticity'
image.catalog['fluxrad25'].unit = 'pixel'
image.catalog[
'fluxrad25'].description = 'Radius containing 25% of the flux'
image.catalog['fluxrad50'].unit = 'pixel'
image.catalog[
'fluxrad50'].description = 'Radius containing 50% of the flux'
image.catalog['fluxrad75'].unit = 'pixel'
image.catalog[
'fluxrad75'].description = 'Radius containing 75% of the flux'
image.catalog['x2'].unit = 'pixel^2'
image.catalog[
'x2'].description = 'Variance on X coordinate of the object'
image.catalog['y2'].unit = 'pixel^2'
image.catalog[
'y2'].description = 'Variance on Y coordinate of the object'
image.catalog['xy'].unit = 'pixel^2'
image.catalog['xy'].description = 'XY covariance of the object'
image.catalog[
'flag'].description = 'Bit mask combination of extraction and photometry flags'
image.catalog.sort('flux')
image.catalog.reverse()
logging_tags = logs.image_config_to_tags(
image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
# Save some background statistics in the header
mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
image.header['L1MEAN'] = (
mean_background,
'[counts] Sigma clipped mean of frame background')
logs.add_tag(logging_tags, 'L1MEAN', float(mean_background))
median_background = np.median(bkg.back())
image.header['L1MEDIAN'] = (
median_background, '[counts] Median of frame background')
logs.add_tag(logging_tags, 'L1MEDIAN',
float(median_background))
std_background = stats.robust_standard_deviation(bkg.back())
image.header['L1SIGMA'] = (
std_background,
'[counts] Robust std dev of frame background')
logs.add_tag(logging_tags, 'L1SIGMA', float(std_background))
# Save some image statistics to the header
good_objects = image.catalog['flag'] == 0
seeing = np.median(
image.catalog['fwhm'][good_objects]) * image.pixel_scale
image.header['L1FWHM'] = (seeing,
'[arcsec] Frame FWHM in arcsec')
logs.add_tag(logging_tags, 'L1FWHM', float(seeing))
mean_ellipticity = stats.sigma_clipped_mean(
sources['ellipticity'][good_objects], 3.0)
image.header['L1ELLIP'] = (mean_ellipticity,
'Mean image ellipticity (1-B/A)')
logs.add_tag(logging_tags, 'L1ELLIP', float(mean_ellipticity))
mean_position_angle = stats.sigma_clipped_mean(
sources['theta'][good_objects], 3.0)
image.header['L1ELLIPA'] = (
mean_position_angle, '[deg] PA of mean image ellipticity')
logs.add_tag(logging_tags, 'L1ELLIPA',
float(mean_position_angle))
self.logger.info('Extracted sources', extra=logging_tags)
except Exception as e:
logging_tags = logs.image_config_to_tags(
image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
self.logger.error(e, extra=logging_tags)
return images
