def make_master_calibration_frame(self, images, image_config,
logging_tags):
bias_data = np.zeros((image_config.ny, image_config.nx, len(images)),
dtype=np.float32)
bias_mask = np.zeros((image_config.ny, image_config.nx, len(images)),
dtype=np.uint8)
bias_level_array = np.zeros(len(images), dtype=np.float32)
master_bias_filename = self.get_calibration_filename(image_config)
logs.add_tag(logging_tags, 'master_bias',
os.path.basename(master_bias_filename))
for i, image in enumerate(images):
bias_level_array[i] = stats.sigma_clipped_mean(image.data,
3.5,
mask=image.bpm)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
logs.add_tag(logging_tags, 'BIASLVL', float(bias_level_array[i]))
self.logger.debug('Calculating bias level', extra=logging_tags)
# Subtract the bias level for each image
bias_data[:, :, i] = image.data[:, :] - bias_level_array[i]
bias_mask[:, :, i] = image.bpm[:, :]
mean_bias_level = stats.sigma_clipped_mean(bias_level_array, 3.0)
master_bias = stats.sigma_clipped_mean(bias_data,
3.0,
axis=2,
mask=bias_mask,
inplace=True)
del bias_data
del bias_mask
master_bpm = np.array(master_bias == 0.0, dtype=np.uint8)
header = fits_utils.create_master_calibration_header(images)
header['BIASLVL'] = (mean_bias_level, 'Mean bias level of master bias')
master_bias_image = Image(self.pipeline_context,
data=master_bias,
header=header)
master_bias_image.filename = master_bias_filename
master_bias_image.bpm = master_bpm
logs.pop_tag(logging_tags, 'master_bias')
logs.add_tag(logging_tags, 'filename',
os.path.basename(master_bias_image.filename))
logs.add_tag(logging_tags, 'BIASLVL', mean_bias_level)
self.logger.debug('Average bias level in ADU', extra=logging_tags)
return [master_bias_image]
def make_master_calibration_frame(self, images, image_config, logging_tags):
dark_data = np.zeros((images[0].ny, images[0].nx, len(images)), dtype=np.float32)
dark_mask = np.zeros((images[0].ny, images[0].nx, len(images)), dtype=np.uint8)
master_dark_filename = self.get_calibration_filename(images[0])
logs.add_tag(logging_tags, 'master_dark', os.path.basename(master_dark_filename))
for i, image in enumerate(images):
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
self.logger.debug('Combining dark', extra=logging_tags)
dark_data[:, :, i] = image.data[:, :]
dark_data[:, :, i] /= image.exptime
dark_mask[:, :, i] = image.bpm[:, :]
master_dark = stats.sigma_clipped_mean(dark_data, 3.0, axis=2, mask=dark_mask, inplace=True)
# Memory cleanup
del dark_data
del dark_mask
master_bpm = np.array(master_dark == 0.0, dtype=np.uint8)
master_dark[master_bpm] = 0.0
# Save the master dark image with all of the combined images in the header
master_dark_header = fits_utils.create_master_calibration_header(images)
master_dark_image = Image(self.pipeline_context, data=master_dark,
header=master_dark_header)
master_dark_image.filename = master_dark_filename
master_dark_image.bpm = master_bpm
logs.pop_tag(logging_tags, 'master_dark')
logs.add_tag(logging_tags, 'filename', os.path.basename(master_dark_image.filename))
self.logger.info('Created master dark', extra=logging_tags)
return [master_dark_image]
def make_master_calibration_frame(self, images):
# Sort the images by reverse observation date, so that the most recent one
# is used to create the filename and select the day directory
images.sort(key=lambda image: image.dateobs, reverse=True)
data_stack = np.zeros((images[0].ny, images[0].nx, len(images)), dtype=np.float32)
stack_mask = np.zeros((images[0].ny, images[0].nx, len(images)), dtype=np.uint8)
make_calibration_name = settings.CALIBRATION_FILENAME_FUNCTIONS[self.calibration_type]
master_calibration_filename = make_calibration_name(images[0])
for i, image in enumerate(images):
logger.debug('Stacking Frames', image=image,
extra_tags={'master_calibration': os.path.basename(master_calibration_filename)})
data_stack[:, :, i] = image.data[:, :]
stack_mask[:, :, i] = image.bpm[:, :]
stacked_data = stats.sigma_clipped_mean(data_stack, 3.0, axis=2, mask=stack_mask, inplace=True)
# Memory cleanup
del data_stack
del stack_mask
master_bpm = np.array(stacked_data == 0.0, dtype=np.uint8)
# Save the master dark image with all of the combined images in the header
master_header = create_master_calibration_header(images[0].header, images)
master_image = FRAME_CLASS(self.runtime_context, data=stacked_data, header=master_header)
master_image.filename = master_calibration_filename
master_image.bpm = master_bpm
logger.info('Created master calibration stack', image=master_image,
extra_tags={'calibration_type': self.calibration_type})
return master_image
def calculate_rv(image, orders_to_use, template):
# for steps in 1 km/s from -2000 to +2000 km/s
coarse_velocities = np.arange(-2000, 2001, 1) * units.km / units.s
coarse_ccfs = cross_correlate_over_traces(image, orders_to_use,
coarse_velocities, template)
# take the peak
velocity_peaks = np.array([
coarse_velocities[np.argmax(ccf['xcor'])].to('km / s').value
for ccf in coarse_ccfs
])
best_v = stats.sigma_clipped_mean(velocity_peaks, 3.0)
velocities = np.arange(best_v - 2, best_v + 2 + 1e-4,
1e-3) * units.km / units.s
ccfs = cross_correlate_over_traces(image, orders_to_use, velocities,
template)
# Calculate the peak velocity
rvs_per_order = np.array(
[ccf['v'][np.argmax(ccf['xcor'])].to('km / s').value for ccf in ccfs])
# iterative sigma clipping using robust_standard_deviation to reject outliers and centering on the median.
rvs_per_order = sigma_clip(rvs_per_order,
sigma=4,
cenfunc='median',
maxiters=5,
stdfunc=stats.robust_standard_deviation,
axis=None,
masked=True,
return_bounds=False,
copy=True)
rv = np.ma.mean(rvs_per_order) * units.km / units.s
rv_err = np.ma.std(rvs_per_order) / np.sqrt(
np.ma.count(rvs_per_order)) * units.km / units.s
return rv, rv_err, coarse_ccfs, ccfs
def do_stage(self, image):
# Get the sigma clipped mean of the central 25% of the image
flat_normalization = stats.sigma_clipped_mean(image.get_inner_image_section(), 3.5)
image.data /= flat_normalization
image.header['FLATLVL'] = flat_normalization
logger.info('Calculate flat normalization', image=image,
extra_tags={'flat_normalization': flat_normalization})
return image
def make_master_calibration_frame(self, images, image_config, logging_tags):
flat_data = np.zeros((images[0].ny, images[0].nx, len(images)), dtype=np.float32)
flat_mask = np.zeros((images[0].ny, images[0].nx, len(images)), dtype=np.uint8)
quarter_nx = images[0].nx // 4
quarter_ny = images[0].ny // 4
master_flat_filename = self.get_calibration_filename(images[0])
logs.add_tag(logging_tags, 'master_flat', os.path.basename(master_flat_filename))
for i, image in enumerate(images):
# Get the sigma clipped mean of the central 25% of the image
flat_normalization = stats.sigma_clipped_mean(image.data[quarter_ny: -quarter_ny,
quarter_nx:-quarter_nx], 3.5)
flat_data[:, :, i] = image.data[:, :]
flat_data[:, :, i] /= flat_normalization
flat_mask[:, :, i] = image.bpm[:, :]
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
logs.add_tag(logging_tags, 'flat_normalization', flat_normalization)
self.logger.debug('Calculating flat normalization', extra=logging_tags)
logs.pop_tag(logging_tags, 'flat_normalization')
master_flat = stats.sigma_clipped_mean(flat_data, 3.0, axis=2, mask=flat_mask,
fill_value=1.0, inplace=True)
master_bpm = np.array(master_flat == 1.0, dtype=np.uint8)
master_bpm = np.logical_and(master_bpm, master_flat < 0.2)
master_flat[master_flat < 0.2] = 1.0
master_flat_header = fits_utils.create_master_calibration_header(images)
master_flat_image = Image(self.pipeline_context, data=master_flat,
header=master_flat_header)
master_flat_image.filename = master_flat_filename
master_flat_image.bpm = master_bpm
logs.pop_tag(logging_tags, 'master_flat')
logs.add_tag(logging_tags, 'filename', os.path.basename(master_flat_image.filename))
self.logger.info('Created master flat', extra=logging_tags)
return [master_flat_image]
def do_stage(self, image):
# Get the sigma clipped mean of the central 25% of the image
flat_normalization = stats.sigma_clipped_mean(
image.get_inner_image_section(), 3.5)
image.data /= flat_normalization
image.header['FLATLVL'] = flat_normalization
logger.info('Calculate flat normalization',
image=image,
extra_tags={'flat_normalization': flat_normalization})
return image
def test_makes_a_sensible_master_dark(mock_frame):
nimages = 20
images = [FakeDarkImage() for x in range(nimages)]
for i, image in enumerate(images):
image.data = np.ones((image.ny, image.nx)) * i
expected_master_dark = stats.sigma_clipped_mean(np.arange(nimages), 3.0)
maker = DarkMaker(FakeContext(frame_class=FakeDarkImage))
stacked_images = maker.do_stage(images)
assert (stacked_images[0].data == expected_master_dark).all()
def _subtract_overscan_2d(image):
overscan_region = fits_utils.parse_region_keyword(image.header.get('BIASSEC'))
if overscan_region is not None:
overscan_level = stats.sigma_clipped_mean(image.data[overscan_region], 3)
image.header['L1STATOV'] = (1, 'Status flag for overscan correction')
else:
overscan_level = 0.0
image.header['L1STATOV'] = (0, 'Status flag for overscan correction')
image.header['OVERSCAN'] = (overscan_level, 'Overscan value that was subtracted')
image.data -= overscan_level
return overscan_level
def _subtract_overscan_2d(image):
overscan_region = fits_utils.parse_region_keyword(image.header.get('BIASSEC'))
if overscan_region is not None:
overscan_level = stats.sigma_clipped_mean(image.data[overscan_region], 3)
image.header['L1STATOV'] = (1, 'Status flag for overscan correction')
else:
overscan_level = 0.0
image.header['L1STATOV'] = (0, 'Status flag for overscan correction')
image.header['OVERSCAN'] = (overscan_level, 'Overscan value that was subtracted')
image.data -= overscan_level
return overscan_level
def test_makes_a_sensible_master_dark(mock_frame, mock_namer):
mock_namer.return_value = lambda *x: 'foo.fits'
nimages = 20
images = [FakeDarkImage() for x in range(nimages)]
for i, image in enumerate(images):
image.data = np.ones((image.ny, image.nx)) * i
expected_master_dark = stats.sigma_clipped_mean(np.arange(nimages), 3.0)
maker = DarkMaker(FakeContext(frame_class=FakeDarkImage))
stacked_images = maker.do_stage(images)
assert (stacked_images[0].data == expected_master_dark).all()
def make_master_calibration_frame(self, images, image_config, logging_tags):
bias_data = np.zeros((image_config.ny, image_config.nx, len(images)), dtype=np.float32)
bias_mask = np.zeros((image_config.ny, image_config.nx, len(images)), dtype=np.uint8)
bias_level_array = np.zeros(len(images), dtype=np.float32)
master_bias_filename = self.get_calibration_filename(image_config)
logs.add_tag(logging_tags, 'master_bias', os.path.basename(master_bias_filename))
for i, image in enumerate(images):
bias_level_array[i] = stats.sigma_clipped_mean(image.data, 3.5, mask=image.bpm)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
logs.add_tag(logging_tags, 'BIASLVL', float(bias_level_array[i]))
self.logger.debug('Calculating bias level', extra=logging_tags)
# Subtract the bias level for each image
bias_data[:, :, i] = image.data[:, :] - bias_level_array[i]
bias_mask[:, :, i] = image.bpm[:, :]
mean_bias_level = stats.sigma_clipped_mean(bias_level_array, 3.0)
master_bias = stats.sigma_clipped_mean(bias_data, 3.0, axis=2, mask=bias_mask, inplace=True)
del bias_data
del bias_mask
master_bpm = np.array(master_bias == 0.0, dtype=np.uint8)
header = fits_utils.create_master_calibration_header(images)
header['BIASLVL'] = (mean_bias_level, 'Mean bias level of master bias')
master_bias_image = Image(self.pipeline_context, data=master_bias, header=header)
master_bias_image.filename = master_bias_filename
master_bias_image.bpm = master_bpm
logs.pop_tag(logging_tags, 'master_bias')
logs.add_tag(logging_tags, 'filename', os.path.basename(master_bias_image.filename))
logs.add_tag(logging_tags, 'BIASLVL', mean_bias_level)
self.logger.debug('Average bias level in ADU', extra=logging_tags)
return [master_bias_image]
def make_master_calibration_frame(self, images):
# Sort the images by reverse observation date, so that the most recent one
# is used to create the filename and select the day directory
images.sort(key=lambda image: image.dateobs, reverse=True)
data_stack = np.zeros((images[0].ny, images[0].nx, len(images)),
dtype=np.float32)
stack_mask = np.zeros((images[0].ny, images[0].nx, len(images)),
dtype=np.uint8)
make_calibration_name = file_utils.make_calibration_filename_function(
self.calibration_type, self.runtime_context)
master_calibration_filename = make_calibration_name(images[0])
for i, image in enumerate(images):
logger.debug('Stacking Frames',
image=image,
extra_tags={
'master_calibration':
os.path.basename(master_calibration_filename)
})
data_stack[:, :, i] = image.data[:, :]
stack_mask[:, :, i] = image.bpm[:, :]
stacked_data = stats.sigma_clipped_mean(data_stack,
3.0,
axis=2,
mask=stack_mask,
inplace=True)
# Memory cleanup
del data_stack
del stack_mask
master_bpm = np.array(stacked_data == 0.0, dtype=np.uint8)
# Save the master dark image with all of the combined images in the header
master_header = create_master_calibration_header(
images[0].header, images)
master_image = FRAME_CLASS(self.runtime_context,
data=stacked_data,
header=master_header)
master_image.filename = master_calibration_filename
master_image.bpm = master_bpm
logger.info('Created master calibration stack',
image=master_image,
extra_tags={'calibration_type': self.calibration_type})
return master_image
def _subtract_overscan_3d(image, i):
overscan_region = fits_utils.parse_region_keyword(image.extension_headers[i].get('BIASSEC'))
if overscan_region is not None:
overscan_level = stats.sigma_clipped_mean(image.data[i][overscan_region], 3)
image.header['L1STATOV'] = (1, 'Status flag for overscan correction')
else:
overscan_level = 0.0
image.header['L1STATOV'] = (0, 'Status flag for overscan correction')
overscan_comment = 'Overscan value that was subtracted from Q{0}'.format(i + 1)
image.header['OVERSCN{0}'.format(i + 1)] = (overscan_level, overscan_comment)
image.data[i] -= overscan_level
return overscan_level
def _subtract_overscan_3d(image, i):
overscan_region = fits_utils.parse_region_keyword(image.extension_headers[i].get('BIASSEC'))
if overscan_region is not None:
overscan_level = stats.sigma_clipped_mean(image.data[i][overscan_region], 3)
image.header['L1STATOV'] = (1, 'Status flag for overscan correction')
else:
overscan_level = 0.0
image.header['L1STATOV'] = (0, 'Status flag for overscan correction')
overscan_comment = 'Overscan value that was subtracted from Q{0}'.format(i + 1)
image.header['OVERSCN{0}'.format(i + 1)] = (overscan_level, overscan_comment)
image.data[i] -= overscan_level
return overscan_level
def make_master_calibration_frame(self, images, image_config,
logging_tags):
dark_data = np.zeros((images[0].ny, images[0].nx, len(images)),
dtype=np.float32)
dark_mask = np.zeros((images[0].ny, images[0].nx, len(images)),
dtype=np.uint8)
master_dark_filename = self.get_calibration_filename(images[0])
logs.add_tag(logging_tags, 'master_dark',
os.path.basename(master_dark_filename))
for i, image in enumerate(images):
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
self.logger.debug('Combining dark', extra=logging_tags)
dark_data[:, :, i] = image.data[:, :]
dark_data[:, :, i] /= image.exptime
dark_mask[:, :, i] = image.bpm[:, :]
master_dark = stats.sigma_clipped_mean(dark_data,
3.0,
axis=2,
mask=dark_mask,
inplace=True)
# Memory cleanup
del dark_data
del dark_mask
master_bpm = np.array(master_dark == 0.0, dtype=np.uint8)
master_dark[master_bpm] = 0.0
# Save the master dark image with all of the combined images in the header
master_dark_header = fits_utils.create_master_calibration_header(
images)
master_dark_image = Image(self.pipeline_context,
data=master_dark,
header=master_dark_header)
master_dark_image.filename = master_dark_filename
master_dark_image.bpm = master_bpm
logs.pop_tag(logging_tags, 'master_dark')
logs.add_tag(logging_tags, 'filename',
os.path.basename(master_dark_image.filename))
self.logger.info('Created master dark', extra=logging_tags)
return [master_dark_image]
def do_stage(self, images):
for i, image in enumerate(images):
try:
# Set the number of source pixels to be 5% of the total. This keeps us safe from
# satellites and airplanes.
sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))
data = image.data.copy()
error = (np.abs(data) + image.readnoise ** 2.0) ** 0.5
mask = image.bpm > 0
# Fits can be backwards byte order, so fix that if need be and subtract
# the background
try:
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
except ValueError:
data = data.byteswap(True).newbyteorder()
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
bkg.subfrom(data)
# Do an initial source detection
# TODO: Add back in masking after we are sure SEP works
sources = sep.extract(data, self.threshold, minarea=self.min_area,
err=error, deblend_cont=0.005)
# Convert the detections into a table
sources = Table(sources)
# Calculate the ellipticity
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# Fix any value of theta that are invalid due to floating point rounding
# -pi / 2 < theta < pi / 2
sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi
# Calculate the kron radius
kronrad, krflag = sep.kron_radius(data, sources['x'], sources['y'],
sources['a'], sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
sources['kronrad'] = kronrad
# Calcuate the equivilent of flux_auto
flux, fluxerr, flag = sep.sum_ellipse(data, sources['x'], sources['y'],
sources['a'], sources['b'],
np.pi / 2.0, 2.5 * kronrad,
subpix=1, err=error)
sources['flux'] = flux
sources['fluxerr'] = fluxerr
sources['flag'] |= flag
# Calculate the FWHMs of the stars:
fwhm = 2.0 * (np.log(2) * (sources['a'] ** 2.0 + sources['b'] ** 2.0)) ** 0.5
sources['fwhm'] = fwhm
# Cut individual bright pixels. Often cosmic rays
sources = sources[fwhm > 1.0]
# Measure the flux profile
flux_radii, flag = sep.flux_radius(data, sources['x'], sources['y'],
6.0 * sources['a'], [0.25, 0.5, 0.75],
normflux=sources['flux'], subpix=5)
sources['flag'] |= flag
sources['fluxrad25'] = flux_radii[:, 0]
sources['fluxrad50'] = flux_radii[:, 1]
sources['fluxrad75'] = flux_radii[:, 2]
# Calculate the windowed positions
sig = 2.0 / 2.35 * sources['fluxrad50']
xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'], sig)
sources['flag'] |= flag
sources['xwin'] = xwin
sources['ywin'] = ywin
# Calculate the average background at each source
bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(), sources['x'], sources['y'],
sources['a'], sources['b'], np.pi / 2.0,
2.5 * sources['kronrad'], subpix=1)
#masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
# sources['a'], sources['b'], np.pi / 2.0,
# 2.5 * kronrad, subpix=1)
background_area = (2.5 * sources['kronrad']) ** 2.0 * sources['a'] * sources['b'] * np.pi # - masksum
sources['background'] = bkgflux
sources['background'][background_area > 0] /= background_area[background_area > 0]
# Update the catalog to match fits convention instead of python array convention
sources['x'] += 1.0
sources['y'] += 1.0
sources['xpeak'] += 1
sources['ypeak'] += 1
sources['xwin'] += 1.0
sources['ywin'] += 1.0
sources['theta'] = np.degrees(sources['theta'])
image.catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak', 'ypeak',
'flux', 'fluxerr', 'background', 'fwhm',
'a', 'b', 'theta', 'kronrad', 'ellipticity',
'fluxrad25', 'fluxrad50', 'fluxrad75',
'x2', 'y2', 'xy', 'flag']
# Add the units and description to the catalogs
image.catalog['x'].unit = 'pixel'
image.catalog['x'].description = 'X coordinate of the object'
image.catalog['y'].unit = 'pixel'
image.catalog['y'].description = 'Y coordinate of the object'
image.catalog['xwin'].unit = 'pixel'
image.catalog['xwin'].description = 'Windowed X coordinate of the object'
image.catalog['ywin'].unit = 'pixel'
image.catalog['ywin'].description = 'Windowed Y coordinate of the object'
image.catalog['xpeak'].unit = 'pixel'
image.catalog['xpeak'].description = 'X coordinate of the peak'
image.catalog['ypeak'].unit = 'pixel'
image.catalog['ypeak'].description = 'Windowed Y coordinate of the peak'
image.catalog['flux'].unit = 'counts'
image.catalog['flux'].description = 'Flux within a Kron-like elliptical aperture'
image.catalog['fluxerr'].unit = 'counts'
image.catalog['fluxerr'].description = 'Erronr on the flux within a Kron-like elliptical aperture'
image.catalog['background'].unit = 'counts'
image.catalog['background'].description = 'Average background value in the aperture'
image.catalog['fwhm'].unit = 'pixel'
image.catalog['fwhm'].description = 'FWHM of the object'
image.catalog['a'].unit = 'pixel'
image.catalog['a'].description = 'Semi-major axis of the object'
image.catalog['b'].unit = 'pixel'
image.catalog['b'].description = 'Semi-minor axis of the object'
image.catalog['theta'].unit = 'degrees'
image.catalog['theta'].description = 'Position angle of the object'
image.catalog['kronrad'].unit = 'pixel'
image.catalog['kronrad'].description = 'Kron radius used for extraction'
image.catalog['ellipticity'].description = 'Ellipticity'
image.catalog['fluxrad25'].unit = 'pixel'
image.catalog['fluxrad25'].description = 'Radius containing 25% of the flux'
image.catalog['fluxrad50'].unit = 'pixel'
image.catalog['fluxrad50'].description = 'Radius containing 50% of the flux'
image.catalog['fluxrad75'].unit = 'pixel'
image.catalog['fluxrad75'].description = 'Radius containing 75% of the flux'
image.catalog['x2'].unit = 'pixel^2'
image.catalog['x2'].description = 'Variance on X coordinate of the object'
image.catalog['y2'].unit = 'pixel^2'
image.catalog['y2'].description = 'Variance on Y coordinate of the object'
image.catalog['xy'].unit = 'pixel^2'
image.catalog['xy'].description = 'XY covariance of the object'
image.catalog['flag'].description = 'Bit mask combination of extraction and photometry flags'
image.catalog.sort('flux')
image.catalog.reverse()
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
# Save some background statistics in the header
mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
image.header['L1MEAN'] = (mean_background,
'[counts] Sigma clipped mean of frame background')
logs.add_tag(logging_tags, 'L1MEAN', float(mean_background))
median_background = np.median(bkg.back())
image.header['L1MEDIAN'] = (median_background,
'[counts] Median of frame background')
logs.add_tag(logging_tags, 'L1MEDIAN', float(median_background))
std_background = stats.robust_standard_deviation(bkg.back())
image.header['L1SIGMA'] = (std_background,
'[counts] Robust std dev of frame background')
logs.add_tag(logging_tags, 'L1SIGMA', float(std_background))
# Save some image statistics to the header
good_objects = image.catalog['flag'] == 0
seeing = np.median(image.catalog['fwhm'][good_objects]) * image.pixel_scale
image.header['L1FWHM'] = (seeing, '[arcsec] Frame FWHM in arcsec')
logs.add_tag(logging_tags, 'L1FWHM', float(seeing))
mean_ellipticity = stats.sigma_clipped_mean(sources['ellipticity'][good_objects],
3.0)
image.header['L1ELLIP'] = (mean_ellipticity, 'Mean image ellipticity (1-B/A)')
logs.add_tag(logging_tags, 'L1ELLIP', float(mean_ellipticity))
mean_position_angle = stats.sigma_clipped_mean(sources['theta'][good_objects], 3.0)
image.header['L1ELLIPA'] = (mean_position_angle,
'[deg] PA of mean image ellipticity')
logs.add_tag(logging_tags, 'L1ELLIPA', float(mean_position_angle))
self.logger.info('Extracted sources', extra=logging_tags)
except Exception as e:
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
self.logger.error(e, extra=logging_tags)
return images
def do_stage(self, image):
bias_level = stats.sigma_clipped_mean(image.data, 3.5, mask=image.bpm)
logger.debug('Subtracting bias level', image=image, extra_tags={'BIASLVL': float(bias_level)})
image.data -= bias_level
image.header['BIASLVL'] = bias_level, 'Bias Level that was removed'
return image
def do_stage(self, image):
try:
# Set the number of source pixels to be 5% of the total. This keeps us safe from
# satellites and airplanes.
sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))
data = image.data.copy()
error = (np.abs(data) + image.readnoise**2.0)**0.5
mask = image.bpm > 0
# Fits can be backwards byte order, so fix that if need be and subtract
# the background
try:
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
except ValueError:
data = data.byteswap(True).newbyteorder()
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
bkg.subfrom(data)
# Do an initial source detection
# TODO: Add back in masking after we are sure SEP works
sources = sep.extract(data,
self.threshold,
minarea=self.min_area,
err=error,
deblend_cont=0.005)
# Convert the detections into a table
sources = Table(sources)
# We remove anything with a detection flag >= 8
# This includes memory overflows and objects that are too close the edge
sources = sources[sources['flag'] < 8]
sources = array_utils.prune_nans_from_table(sources)
# Calculate the ellipticity
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# Fix any value of theta that are invalid due to floating point rounding
# -pi / 2 < theta < pi / 2
sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi
# Calculate the kron radius
kronrad, krflag = sep.kron_radius(data, sources['x'], sources['y'],
sources['a'], sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
sources['kronrad'] = kronrad
# Calcuate the equivilent of flux_auto
flux, fluxerr, flag = sep.sum_ellipse(data,
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 * kronrad,
subpix=1,
err=error)
sources['flux'] = flux
sources['fluxerr'] = fluxerr
sources['flag'] |= flag
# Do circular aperture photometry for diameters of 1" to 6"
for diameter in [1, 2, 3, 4, 5, 6]:
flux, fluxerr, flag = sep.sum_circle(data,
sources['x'],
sources['y'],
diameter / 2.0 /
image.pixel_scale,
gain=1.0,
err=error)
sources['fluxaper{0}'.format(diameter)] = flux
sources['fluxerr{0}'.format(diameter)] = fluxerr
sources['flag'] |= flag
# Calculate the FWHMs of the stars:
fwhm = 2.0 * (np.log(2) *
(sources['a']**2.0 + sources['b']**2.0))**0.5
sources['fwhm'] = fwhm
# Cut individual bright pixels. Often cosmic rays
sources = sources[fwhm > 1.0]
# Measure the flux profile
flux_radii, flag = sep.flux_radius(data,
sources['x'],
sources['y'],
6.0 * sources['a'],
[0.25, 0.5, 0.75],
normflux=sources['flux'],
subpix=5)
sources['flag'] |= flag
sources['fluxrad25'] = flux_radii[:, 0]
sources['fluxrad50'] = flux_radii[:, 1]
sources['fluxrad75'] = flux_radii[:, 2]
# Calculate the windowed positions
sig = 2.0 / 2.35 * sources['fluxrad50']
xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'],
sig)
sources['flag'] |= flag
sources['xwin'] = xwin
sources['ywin'] = ywin
# Calculate the average background at each source
bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(),
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 * sources['kronrad'],
subpix=1)
# masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
# sources['a'], sources['b'], np.pi / 2.0,
# 2.5 * kronrad, subpix=1)
background_area = (
2.5 * sources['kronrad']
)**2.0 * sources['a'] * sources['b'] * np.pi # - masksum
sources['background'] = bkgflux
sources['background'][background_area > 0] /= background_area[
background_area > 0]
# Update the catalog to match fits convention instead of python array convention
sources['x'] += 1.0
sources['y'] += 1.0
sources['xpeak'] += 1
sources['ypeak'] += 1
sources['xwin'] += 1.0
sources['ywin'] += 1.0
sources['theta'] = np.degrees(sources['theta'])
catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak', 'ypeak',
'flux', 'fluxerr', 'peak', 'fluxaper1',
'fluxerr1', 'fluxaper2', 'fluxerr2', 'fluxaper3',
'fluxerr3', 'fluxaper4', 'fluxerr4', 'fluxaper5',
'fluxerr5', 'fluxaper6', 'fluxerr6',
'background', 'fwhm', 'a', 'b', 'theta',
'kronrad', 'ellipticity', 'fluxrad25',
'fluxrad50', 'fluxrad75', 'x2', 'y2', 'xy',
'flag']
# Add the units and description to the catalogs
catalog['x'].unit = 'pixel'
catalog['x'].description = 'X coordinate of the object'
catalog['y'].unit = 'pixel'
catalog['y'].description = 'Y coordinate of the object'
catalog['xwin'].unit = 'pixel'
catalog['xwin'].description = 'Windowed X coordinate of the object'
catalog['ywin'].unit = 'pixel'
catalog['ywin'].description = 'Windowed Y coordinate of the object'
catalog['xpeak'].unit = 'pixel'
catalog['xpeak'].description = 'X coordinate of the peak'
catalog['ypeak'].unit = 'pixel'
catalog['ypeak'].description = 'Windowed Y coordinate of the peak'
catalog['flux'].unit = 'count'
catalog[
'flux'].description = 'Flux within a Kron-like elliptical aperture'
catalog['fluxerr'].unit = 'count'
catalog[
'fluxerr'].description = 'Error on the flux within Kron aperture'
catalog['peak'].unit = 'count'
catalog['peak'].description = 'Peak flux (flux at xpeak, ypeak)'
for diameter in [1, 2, 3, 4, 5, 6]:
catalog['fluxaper{0}'.format(diameter)].unit = 'count'
catalog['fluxaper{0}'.format(
diameter
)].description = 'Flux from fixed circular aperture: {0}" diameter'.format(
diameter)
catalog['fluxerr{0}'.format(diameter)].unit = 'count'
catalog['fluxerr{0}'.format(
diameter
)].description = 'Error on Flux from circular aperture: {0}"'.format(
diameter)
catalog['background'].unit = 'count'
catalog[
'background'].description = 'Average background value in the aperture'
catalog['fwhm'].unit = 'pixel'
catalog['fwhm'].description = 'FWHM of the object'
catalog['a'].unit = 'pixel'
catalog['a'].description = 'Semi-major axis of the object'
catalog['b'].unit = 'pixel'
catalog['b'].description = 'Semi-minor axis of the object'
catalog['theta'].unit = 'degree'
catalog['theta'].description = 'Position angle of the object'
catalog['kronrad'].unit = 'pixel'
catalog['kronrad'].description = 'Kron radius used for extraction'
catalog['ellipticity'].description = 'Ellipticity'
catalog['fluxrad25'].unit = 'pixel'
catalog[
'fluxrad25'].description = 'Radius containing 25% of the flux'
catalog['fluxrad50'].unit = 'pixel'
catalog[
'fluxrad50'].description = 'Radius containing 50% of the flux'
catalog['fluxrad75'].unit = 'pixel'
catalog[
'fluxrad75'].description = 'Radius containing 75% of the flux'
catalog['x2'].unit = 'pixel^2'
catalog[
'x2'].description = 'Variance on X coordinate of the object'
catalog['y2'].unit = 'pixel^2'
catalog[
'y2'].description = 'Variance on Y coordinate of the object'
catalog['xy'].unit = 'pixel^2'
catalog['xy'].description = 'XY covariance of the object'
catalog[
'flag'].description = 'Bit mask of extraction/photometry flags'
catalog.sort('flux')
catalog.reverse()
# Save some background statistics in the header
mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
image.header['L1MEAN'] = (
mean_background,
'[counts] Sigma clipped mean of frame background')
median_background = np.median(bkg.back())
image.header['L1MEDIAN'] = (median_background,
'[counts] Median of frame background')
std_background = stats.robust_standard_deviation(bkg.back())
image.header['L1SIGMA'] = (
std_background, '[counts] Robust std dev of frame background')
# Save some image statistics to the header
good_objects = catalog['flag'] == 0
for quantity in ['fwhm', 'ellipticity', 'theta']:
good_objects = np.logical_and(
good_objects, np.logical_not(np.isnan(catalog[quantity])))
if good_objects.sum() == 0:
image.header['L1FWHM'] = ('NaN',
'[arcsec] Frame FWHM in arcsec')
image.header['L1ELLIP'] = ('NaN',
'Mean image ellipticity (1-B/A)')
image.header['L1ELLIPA'] = (
'NaN', '[deg] PA of mean image ellipticity')
else:
seeing = np.median(
catalog['fwhm'][good_objects]) * image.pixel_scale
image.header['L1FWHM'] = (seeing,
'[arcsec] Frame FWHM in arcsec')
mean_ellipticity = stats.sigma_clipped_mean(
catalog['ellipticity'][good_objects], 3.0)
image.header['L1ELLIP'] = (mean_ellipticity,
'Mean image ellipticity (1-B/A)')
mean_position_angle = stats.sigma_clipped_mean(
catalog['theta'][good_objects], 3.0)
image.header['L1ELLIPA'] = (
mean_position_angle, '[deg] PA of mean image ellipticity')
logging_tags = {
key: float(image.header[key])
for key in [
'L1MEAN', 'L1MEDIAN', 'L1SIGMA', 'L1FWHM', 'L1ELLIP',
'L1ELLIPA'
]
}
logger.info('Extracted sources',
image=image,
extra_tags=logging_tags)
# adding catalog (a data table) to the appropriate images attribute.
image.data_tables['catalog'] = DataTable(data_table=catalog,
name='CAT')
except Exception:
logger.error(logs.format_exception(), image=image)
return image
def do_stage(self, images):
for i, image in enumerate(images):
try:
# Set the number of source pixels to be 5% of the total. This keeps us safe from
# satellites and airplanes.
sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))
data = image.data.copy()
error = (np.abs(data) + image.readnoise**2.0)**0.5
mask = image.bpm > 0
# Fits can be backwards byte order, so fix that if need be and subtract
# the background
try:
bkg = sep.Background(data,
mask=mask,
bw=32,
bh=32,
fw=3,
fh=3)
except ValueError:
data = data.byteswap(True).newbyteorder()
bkg = sep.Background(data,
mask=mask,
bw=32,
bh=32,
fw=3,
fh=3)
bkg.subfrom(data)
# Do an initial source detection
# TODO: Add back in masking after we are sure SEP works
sources = sep.extract(data,
self.threshold,
minarea=self.min_area,
err=error,
deblend_cont=0.005)
# Convert the detections into a table
sources = Table(sources)
# Calculate the ellipticity
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# Fix any value of theta that are invalid due to floating point rounding
# -pi / 2 < theta < pi / 2
sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi
# Calculate the kron radius
kronrad, krflag = sep.kron_radius(data, sources['x'],
sources['y'], sources['a'],
sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
sources['kronrad'] = kronrad
# Calcuate the equivilent of flux_auto
flux, fluxerr, flag = sep.sum_ellipse(data,
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 * kronrad,
subpix=1,
err=error)
sources['flux'] = flux
sources['fluxerr'] = fluxerr
sources['flag'] |= flag
# Calculate the FWHMs of the stars:
fwhm = 2.0 * (np.log(2) *
(sources['a']**2.0 + sources['b']**2.0))**0.5
sources['fwhm'] = fwhm
# Cut individual bright pixels. Often cosmic rays
sources = sources[fwhm > 1.0]
# Measure the flux profile
flux_radii, flag = sep.flux_radius(data,
sources['x'],
sources['y'],
6.0 * sources['a'],
[0.25, 0.5, 0.75],
normflux=sources['flux'],
subpix=5)
sources['flag'] |= flag
sources['fluxrad25'] = flux_radii[:, 0]
sources['fluxrad50'] = flux_radii[:, 1]
sources['fluxrad75'] = flux_radii[:, 2]
# Calculate the windowed positions
sig = 2.0 / 2.35 * sources['fluxrad50']
xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'],
sig)
sources['flag'] |= flag
sources['xwin'] = xwin
sources['ywin'] = ywin
# Calculate the average background at each source
bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(),
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 *
sources['kronrad'],
subpix=1)
#masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
# sources['a'], sources['b'], np.pi / 2.0,
# 2.5 * kronrad, subpix=1)
background_area = (
2.5 * sources['kronrad']
)**2.0 * sources['a'] * sources['b'] * np.pi # - masksum
sources['background'] = bkgflux
sources['background'][background_area > 0] /= background_area[
background_area > 0]
# Update the catalog to match fits convention instead of python array convention
sources['x'] += 1.0
sources['y'] += 1.0
sources['xpeak'] += 1
sources['ypeak'] += 1
sources['xwin'] += 1.0
sources['ywin'] += 1.0
sources['theta'] = np.degrees(sources['theta'])
image.catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak',
'ypeak', 'flux', 'fluxerr',
'background', 'fwhm', 'a', 'b',
'theta', 'kronrad', 'ellipticity',
'fluxrad25', 'fluxrad50', 'fluxrad75',
'x2', 'y2', 'xy', 'flag']
# Add the units and description to the catalogs
image.catalog['x'].unit = 'pixel'
image.catalog['x'].description = 'X coordinate of the object'
image.catalog['y'].unit = 'pixel'
image.catalog['y'].description = 'Y coordinate of the object'
image.catalog['xwin'].unit = 'pixel'
image.catalog[
'xwin'].description = 'Windowed X coordinate of the object'
image.catalog['ywin'].unit = 'pixel'
image.catalog[
'ywin'].description = 'Windowed Y coordinate of the object'
image.catalog['xpeak'].unit = 'pixel'
image.catalog['xpeak'].description = 'X coordinate of the peak'
image.catalog['ypeak'].unit = 'pixel'
image.catalog[
'ypeak'].description = 'Windowed Y coordinate of the peak'
image.catalog['flux'].unit = 'counts'
image.catalog[
'flux'].description = 'Flux within a Kron-like elliptical aperture'
image.catalog['fluxerr'].unit = 'counts'
image.catalog[
'fluxerr'].description = 'Erronr on the flux within a Kron-like elliptical aperture'
image.catalog['background'].unit = 'counts'
image.catalog[
'background'].description = 'Average background value in the aperture'
image.catalog['fwhm'].unit = 'pixel'
image.catalog['fwhm'].description = 'FWHM of the object'
image.catalog['a'].unit = 'pixel'
image.catalog[
'a'].description = 'Semi-major axis of the object'
image.catalog['b'].unit = 'pixel'
image.catalog[
'b'].description = 'Semi-minor axis of the object'
image.catalog['theta'].unit = 'degrees'
image.catalog[
'theta'].description = 'Position angle of the object'
image.catalog['kronrad'].unit = 'pixel'
image.catalog[
'kronrad'].description = 'Kron radius used for extraction'
image.catalog['ellipticity'].description = 'Ellipticity'
image.catalog['fluxrad25'].unit = 'pixel'
image.catalog[
'fluxrad25'].description = 'Radius containing 25% of the flux'
image.catalog['fluxrad50'].unit = 'pixel'
image.catalog[
'fluxrad50'].description = 'Radius containing 50% of the flux'
image.catalog['fluxrad75'].unit = 'pixel'
image.catalog[
'fluxrad75'].description = 'Radius containing 75% of the flux'
image.catalog['x2'].unit = 'pixel^2'
image.catalog[
'x2'].description = 'Variance on X coordinate of the object'
image.catalog['y2'].unit = 'pixel^2'
image.catalog[
'y2'].description = 'Variance on Y coordinate of the object'
image.catalog['xy'].unit = 'pixel^2'
image.catalog['xy'].description = 'XY covariance of the object'
image.catalog[
'flag'].description = 'Bit mask combination of extraction and photometry flags'
image.catalog.sort('flux')
image.catalog.reverse()
logging_tags = logs.image_config_to_tags(
image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
# Save some background statistics in the header
mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
image.header['L1MEAN'] = (
mean_background,
'[counts] Sigma clipped mean of frame background')
logs.add_tag(logging_tags, 'L1MEAN', float(mean_background))
median_background = np.median(bkg.back())
image.header['L1MEDIAN'] = (
median_background, '[counts] Median of frame background')
logs.add_tag(logging_tags, 'L1MEDIAN',
float(median_background))
std_background = stats.robust_standard_deviation(bkg.back())
image.header['L1SIGMA'] = (
std_background,
'[counts] Robust std dev of frame background')
logs.add_tag(logging_tags, 'L1SIGMA', float(std_background))
# Save some image statistics to the header
good_objects = image.catalog['flag'] == 0
seeing = np.median(
image.catalog['fwhm'][good_objects]) * image.pixel_scale
image.header['L1FWHM'] = (seeing,
'[arcsec] Frame FWHM in arcsec')
logs.add_tag(logging_tags, 'L1FWHM', float(seeing))
mean_ellipticity = stats.sigma_clipped_mean(
sources['ellipticity'][good_objects], 3.0)
image.header['L1ELLIP'] = (mean_ellipticity,
'Mean image ellipticity (1-B/A)')
logs.add_tag(logging_tags, 'L1ELLIP', float(mean_ellipticity))
mean_position_angle = stats.sigma_clipped_mean(
sources['theta'][good_objects], 3.0)
image.header['L1ELLIPA'] = (
mean_position_angle, '[deg] PA of mean image ellipticity')
logs.add_tag(logging_tags, 'L1ELLIPA',
float(mean_position_angle))
self.logger.info('Extracted sources', extra=logging_tags)
except Exception as e:
logging_tags = logs.image_config_to_tags(
image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
self.logger.error(e, extra=logging_tags)
return images