def build_dqmask(self, chip=None):
# apply any DQ array, if available
dqmask = None
if chip:
dqarr = self.imghdu[('DQ', chip)].data
else:
dqarr = np.concatenate(
[self.imghdu[('DQ', i + 1)].data for i in range(self.num_sci)])
# "grow out" regions in DQ mask flagged as saturated by several
# pixels in every direction to prevent the
# source match algorithm from trying to match multiple sources
# from one image to a single source in the
# other or vice-versa.
# Create temp DQ mask containing all pixels flagged with any value EXCEPT 256
non_sat_mask = bitfield_to_boolean_mask(dqarr, ignore_flags=256 + 2048)
# Create temp DQ mask containing saturated pixels ONLY
sat_mask = bitfield_to_boolean_mask(dqarr, ignore_flags=~(256 + 2048))
# Ignore sources where only a couple of pixels are flagged as saturated
sat_mask = ndimage.binary_erosion(sat_mask, iterations=1)
# Grow out saturated pixels by a few pixels in every direction
grown_sat_mask = ndimage.binary_dilation(sat_mask, iterations=5)
# combine the two temporary DQ masks into a single composite DQ mask.
dqmask = np.bitwise_or(non_sat_mask, grown_sat_mask)
return dqmask
def _mergeUserMaskAndDQ(dq, mask, dqbits):
from astropy.nddata.bitmask import (interpret_bit_flags,
bitfield_to_boolean_mask)
dqbits = interpret_bit_flags(dqbits)
if dqbits is None:
if mask is None:
return None
else:
return mask.copy().astype(dtype=np.uint8)
if dq is None:
raise ValueError("DQ array is None while 'dqbits' is not None.")
dqmask = bitfield_to_boolean_mask(dq,
dqbits,
good_mask_value=1,
dtype=np.uint8)
if mask is None:
return dqmask
# merge user mask with DQ mask:
dqmask[mask == 0] = 0
return dqmask
def _imodel2skyim(self, image_model):
# create
if self._dqbits is None:
dqmask = np.isfinite(image_model.data).astype(dtype=np.uint8)
else:
dqmask = bitfield_to_boolean_mask(
image_model.dq,
self._dqbits,
good_mask_value=1,
dtype=np.uint8
) * np.isfinite(image_model.data)
# see if 'skymatch' was previously run and raise an exception
# if 'subtract' mode has changed compared to the previous pass:
if image_model.meta.background.subtracted is None:
if image_model.meta.background.level is not None:
# report inconsistency:
raise ValueError("Background level was set but the "
"'subtracted' property is undefined (None).")
level = 0.0
else:
level = image_model.meta.background.level
if level is None:
# NOTE: In principle we could assume that level is 0 and
# possibly add a log entry documenting this, however,
# at this moment I think it is saver to quit and...
#
# report inconsistency:
raise ValueError("Background level was subtracted but the "
"'level' property is undefined (None).")
if image_model.meta.background.subtracted != self.subtract:
# cannot run 'skymatch' step on already "skymatched" images
# when 'subtract' spec is inconsistent with
# meta.background.subtracted:
raise ValueError("'subtract' step's specification is "
"inconsistent with background info already "
"present in image '{:s}' meta."
.format(image_model.meta.filename))
sky_im = SkyImage(
image=image_model.data,
wcs_fwd=image_model.meta.wcs.__call__,
wcs_inv=image_model.meta.wcs.invert,
pix_area=1.0, #TODO: pixel area
convf=1.0, #TODO: conv. factor to brightness
mask=dqmask,
id=image_model.meta.filename, # file name?
skystat=self._skystat,
stepsize=self.stepsize,
meta={'image_model': image_model}
)
if self.subtract:
sky_im.sky = level
return sky_im
def _imodel2skyim(self, image_model):
# create
if self._dqbits is None:
dqmask = np.isfinite(image_model.data).astype(dtype=np.uint8)
else:
dqmask = bitfield_to_boolean_mask(
image_model.dq,
self._dqbits,
good_mask_value=1,
dtype=np.uint8
) * np.isfinite(image_model.data)
# see if 'skymatch' was previously run and raise an exception
# if 'subtract' mode has changed compared to the previous pass:
if image_model.meta.background.subtracted is None:
if image_model.meta.background.level is not None:
# report inconsistency:
raise ValueError("Background level was set but the "
"'subtracted' property is undefined (None).")
level = 0.0
else:
level = image_model.meta.background.level
if level is None:
# NOTE: In principle we could assume that level is 0 and
# possibly add a log entry documenting this, however,
# at this moment I think it is saver to quit and...
#
# report inconsistency:
raise ValueError("Background level was subtracted but the "
"'level' property is undefined (None).")
if image_model.meta.background.subtracted != self.subtract:
# cannot run 'skymatch' step on already "skymatched" images
# when 'subtract' spec is inconsistent with
# meta.background.subtracted:
raise ValueError("'subtract' step's specification is "
"inconsistent with background info already "
"present in image '{:s}' meta."
.format(image_model.meta.filename))
sky_im = SkyImage(
image=image_model.data,
wcs_fwd=image_model.meta.wcs.__call__,
wcs_inv=image_model.meta.wcs.invert,
pix_area=1.0, #TODO: pixel area
convf=1.0, #TODO: conv. factor to brightness
mask=dqmask,
id=image_model.meta.filename, # file name?
skystat=self._skystat,
stepsize=self.stepsize,
meta={'image_model': image_model}
)
if self.subtract:
sky_im.sky = level
return sky_im
def test_bitfield_to_boolean_mask(data, flags, flip, goodval, dtype, ref):
mask = bitmask.bitfield_to_boolean_mask(bitfield=data,
ignore_flags=flags,
flip_bits=flip,
good_mask_value=goodval,
dtype=dtype)
assert (mask.dtype == dtype)
assert np.all(mask == ref)
def ix_masked(self,maskcol,maskval=None,indices=None):
# get the indices based on input.
indices=self.getindices(indices)
if maskval is None:
(keep,) = np.where(self.t.loc[indices,maskcol].ne(0))
else:
(keep,) = np.where(bitmask.bitfield_to_boolean_mask(self.t.loc[indices,maskcol].astype('int'),ignore_flags=~maskval))
indices = indices[keep]
return(indices)
def test_bitfield_to_boolean_mask(data, flags, flip, goodval, dtype, ref):
mask = bitmask.bitfield_to_boolean_mask(
bitfield=data,
ignore_flags=flags,
flip_bits=flip,
good_mask_value=goodval,
dtype=dtype
)
assert(mask.dtype == dtype)
assert np.all(mask == ref)
def build_dqmask(self, chip=None):
# apply any DQ array, if available
dqmask = None
if chip:
dqarr = self.imghdu[('DQ', chip)].data
else:
dqarr = np.concatenate(
[self.imghdu[('DQ', i + 1)].data for i in range(self.num_sci)])
# Create temp DQ mask containing all pixels flagged with any value EXCEPT 512 (bad ref pixel)
dqmask = bitfield_to_boolean_mask(dqarr, ignore_flags=512)
return dqmask
def test_bitfield_must_be_integer_check():
with pytest.raises(TypeError):
bitmask.bitfield_to_boolean_mask(1.0, 1)
def process(self, input1, input2):
cube_models = datamodels.ModelContainer(input1)
models2d = datamodels.ModelContainer(input2)
dqbits = interpret_bit_flags(self.dqbits, flag_name_map=pixel)
# set sky statistics:
self._skystat = SkyStats(
skystat=self.skystat,
lower=self.lower,
upper=self.upper,
nclip=self.nclip,
lsig=self.lsigma,
usig=self.usigma,
binwidth=self.binwidth
)
# At this moment running 'cube_skymatch' on images whose
# background has been previously subtracted is not supported.
# Raise an exception if background was subtracted:
self._check_background(cube_models)
self._check_background(models2d)
self._reset_background(cube_models)
self._reset_background(models2d)
# create a list of SkyCubes:
skycubes = []
for cm in cube_models:
# process weights and combine with DQ:
if not hasattr(cm, 'weightmap') or cm.weightmap is None:
weights = np.ones_like(cm.data, dtype=np.float64)
else:
weights = cm.weightmap.copy()
if dqbits is not None:
dq = bitfield_to_boolean_mask(
cm.dq,
self._dqbits,
good_mask_value=0,
dtype=bool
)
weights[dq] = 0.0
wcs = cm.meta.wcs if hasattr(cm.meta, 'wcs') else None
wcsinfo = cm.meta.wcsinfo if hasattr(cm.meta, 'wcsinfo') else None
exptime = cm.meta.exposure.exposure_time
if exptime is None:
exptime = 1.0
sc = SkyCube(
data=cm.data,
wcs=wcs,
wcsinfo=wcsinfo,
weights=weights,
cube_weight=exptime,
bkg_deg=self.bkg_degree,
bkg_center=None,
id=None,
meta={'original_cube_model': cm}
)
skycubes.append(sc)
skymatch_cubes, nsubspace = match(skycubes, subtract=self.subtract)
if nsubspace > 1:
self.log.warning("Not all cubes have been sky matched as "
"some of them do not overlap.")
# save background info in 'meta' and subtract sky from 2D images
# if requested:
# model.meta.instrument.channel
for c in skymatch_cubes:
self._set_cube_bkg_meta(c.meta['original_cube_model'], c)
model2d, channel = _find_associated_2d_image(
c.meta['original_cube_model'], models2d
)
if model2d is None:
continue
self._set_model2d_bkg_meta(
c.meta['original_cube_model'],
model2d,
channel
)
if self.subtract2d:
self._apply_sky_2d(model2d, channel)
return cube_models, models2d
def run_extract1d(input_model, spectrace_ref_name, wavemap_ref_name,
specprofile_ref_name, speckernel_ref_name, subarray,
soss_filter, soss_kwargs):
"""Run the spectral extraction on NIRISS SOSS data.
Parameters
----------
input_model : DataModel
The input DataModel.
spectrace_ref_name : str
Name of the spectrace reference file.
wavemap_ref_name : str
Name of the wavemap reference file.
specprofile_ref_name : str
Name of the specprofile reference file.
speckernel_ref_name : str
Name of the speckernel reference file.
subarray : str
Subarray on which the data were recorded; one of 'SUBSTRIPT96',
'SUBSTRIP256' or 'FULL'.
soss_filter : str
Filter in place during observations; one of 'CLEAR' or 'F277W'.
soss_kwargs : dict
Dictionary of keyword arguments passed from extract_1d_step.
Returns
-------
output_model : DataModel
DataModel containing the extracted spectra.
"""
# Map the order integer names to the string names
order_str_2_int = {f'Order {order}': order for order in [1, 2, 3]}
# Read the reference files.
spectrace_ref = datamodels.SpecTraceModel(spectrace_ref_name)
wavemap_ref = datamodels.WaveMapModel(wavemap_ref_name)
specprofile_ref = datamodels.SpecProfileModel(specprofile_ref_name)
speckernel_ref = datamodels.SpecKernelModel(speckernel_ref_name)
ref_files = dict()
ref_files['spectrace'] = spectrace_ref
ref_files['wavemap'] = wavemap_ref
ref_files['specprofile'] = specprofile_ref
ref_files['speckernel'] = speckernel_ref
# Initialize the output model and output references (model of the detector and box aperture weights).
output_model = datamodels.MultiSpecModel()
output_model.update(input_model) # Copy meta data from input to output.
output_references = datamodels.SossExtractModel()
output_references.update(input_model)
all_tracemodels = dict()
all_box_weights = dict()
# Extract depending on the type of datamodels (Image or Cube)
if isinstance(input_model, datamodels.ImageModel):
log.info('Input is an ImageModel, processing a single integration.')
# Initialize the theta, dx, dy transform parameters
transform = soss_kwargs.pop('transform')
# Received a single 2D image; set dtype to float64 and convert DQ to boolean mask.
scidata = input_model.data.astype('float64')
scierr = input_model.err.astype('float64')
scimask = input_model.dq > 0 # Mask bad pixels with True.
refmask = bitfield_to_boolean_mask(
input_model.dq,
ignore_flags=dqflags.pixel['REFERENCE_PIXEL'],
flip_bits=True)
# Perform background correction.
bkg_mask = make_background_mask(scidata, width=40)
scidata_bkg, col_bkg, npix_bkg = soss_background(scidata,
scimask,
bkg_mask=bkg_mask)
# Determine the theta, dx, dy transform needed to match scidata trace position to ref file position.
if transform is None:
log.info('Solving for the transformation parameters.')
# Unpack the expected order 1 & 2 positions.
spectrace_ref = ref_files['spectrace']
xref_o1 = spectrace_ref.trace[0].data['X']
yref_o1 = spectrace_ref.trace[0].data['Y']
xref_o2 = spectrace_ref.trace[1].data['X']
yref_o2 = spectrace_ref.trace[1].data['Y']
# Use the solver on the background subtracted image.
if subarray == 'SUBSTRIP96' or soss_filter == 'F277W':
# Use only order 1 to solve theta, dx, dy
transform = solve_transform(scidata_bkg,
scimask,
xref_o1,
yref_o1,
soss_filter=soss_filter)
else:
transform = solve_transform(scidata_bkg,
scimask,
xref_o1,
yref_o1,
xref_o2,
yref_o2,
soss_filter=soss_filter)
log.info(
'Measured to Reference trace position transform: theta={:.4f}, dx={:.4f}, dy={:.4f}'
.format(transform[0], transform[1], transform[2]))
# Prepare the reference file arguments.
ref_file_args = get_ref_file_args(ref_files, transform)
# Make sure wavelength maps cover only parts where the centroid is inside the detector image
_mask_wv_map_centroid_outside(ref_file_args[0], ref_files, transform,
scidata_bkg.shape[0])
# Model the traces based on optics filter configuration (CLEAR or F277W)
if soss_filter == 'CLEAR':
# Model the image.
kwargs = dict()
kwargs['transform'] = transform
kwargs['tikfac'] = soss_kwargs['tikfac']
kwargs['n_os'] = soss_kwargs['n_os']
kwargs['threshold'] = soss_kwargs['threshold']
result = model_image(scidata_bkg, scierr, scimask, refmask,
ref_file_args, **kwargs)
tracemodels, soss_kwargs['tikfac'], logl = result
else:
# No model can be fit for F277W yet, missing throughput reference files.
msg = f"No extraction possible for filter {soss_filter}."
log.critical(msg)
raise ValueError(msg)
# Save trace models for output reference
for order in tracemodels:
# Save as a list (convert to array at the end)
all_tracemodels[order] = [tracemodels[order]]
# Use the trace models to perform a decontaminated extraction.
kwargs = dict()
kwargs['width'] = soss_kwargs['width']
kwargs['bad_pix'] = soss_kwargs['bad_pix']
result = extract_image(scidata_bkg, scierr, scimask, tracemodels,
ref_files, transform, subarray, **kwargs)
wavelengths, fluxes, fluxerrs, npixels, box_weights = result
# Save box weights for output reference
for order in box_weights:
# Save as a list (convert to array at the end)
all_box_weights[order] = [box_weights[order]]
# Copy spectral data for each order into the output model.
for order in wavelengths.keys():
table_size = len(wavelengths[order])
out_table = np.zeros(table_size,
dtype=datamodels.SpecModel().spec_table.dtype)
out_table['WAVELENGTH'] = wavelengths[order]
out_table['FLUX'] = fluxes[order]
out_table['FLUX_ERROR'] = fluxerrs[order]
out_table['DQ'] = np.zeros(table_size)
out_table['BACKGROUND'] = col_bkg
out_table['NPIXELS'] = npixels[order]
spec = datamodels.SpecModel(spec_table=out_table)
# Add integration number and spectral order
spec.spectral_order = order_str_2_int[order]
output_model.spec.append(spec)
output_model.meta.soss_extract1d.width = kwargs['width']
output_model.meta.soss_extract1d.tikhonov_factor = soss_kwargs[
'tikfac']
output_model.meta.soss_extract1d.delta_x = transform[1]
output_model.meta.soss_extract1d.delta_y = transform[2]
output_model.meta.soss_extract1d.theta = transform[0]
output_model.meta.soss_extract1d.oversampling = soss_kwargs['n_os']
output_model.meta.soss_extract1d.threshold = soss_kwargs['threshold']
elif isinstance(input_model, datamodels.CubeModel):
nimages = len(input_model.data)
log.info(
'Input is a CubeModel containing {} integrations.'.format(nimages))
# Initialize the theta, dx, dy transform parameters
transform = soss_kwargs.pop('transform')
# Loop over images.
for i in range(nimages):
log.info('Processing integration {} of {}.'.format(i + 1, nimages))
# Unpack the i-th image, set dtype to float64 and convert DQ to boolean mask.
scidata = input_model.data[i].astype('float64')
scierr = input_model.err[i].astype('float64')
scimask = np.bitwise_and(input_model.dq[i],
dqflags.pixel['DO_NOT_USE']).astype(bool)
refmask = bitfield_to_boolean_mask(
input_model.dq[i],
ignore_flags=dqflags.pixel['REFERENCE_PIXEL'],
flip_bits=True)
# Perform background correction.
bkg_mask = make_background_mask(scidata, width=40)
scidata_bkg, col_bkg, npix_bkg = soss_background(scidata,
scimask,
bkg_mask=bkg_mask)
# Determine the theta, dx, dy transform needed to match scidata trace position to ref file position.
if transform is None:
log.info('Solving for the transformation parameters.')
# Unpack the expected order 1 & 2 positions.
spectrace_ref = ref_files['spectrace']
xref_o1 = spectrace_ref.trace[0].data['X']
yref_o1 = spectrace_ref.trace[0].data['Y']
xref_o2 = spectrace_ref.trace[1].data['X']
yref_o2 = spectrace_ref.trace[1].data['Y']
# Use the solver on the background subtracted image.
if subarray == 'SUBSTRIP96' or soss_filter == 'F277W':
# Use only order 1 to solve theta, dx, dy
transform = solve_transform(scidata_bkg,
scimask,
xref_o1,
yref_o1,
soss_filter=soss_filter)
else:
transform = solve_transform(scidata_bkg,
scimask,
xref_o1,
yref_o1,
xref_o2,
yref_o2,
soss_filter=soss_filter)
log.info(
'Measured to Reference trace position transform: theta={:.4f}, dx={:.4f}, dy={:.4f}'
.format(transform[0], transform[1], transform[2]))
# Prepare the reference file arguments.
ref_file_args = get_ref_file_args(ref_files, transform)
# Make sure wavelength maps cover only parts where the centroid is inside the detector image
_mask_wv_map_centroid_outside(ref_file_args[0], ref_files,
transform, scidata_bkg.shape[0])
# Model the traces based on optics filter configuration (CLEAR or F277W)
if soss_filter == 'CLEAR':
# Model the image.
kwargs = dict()
kwargs['transform'] = transform
kwargs['tikfac'] = soss_kwargs['tikfac']
kwargs['n_os'] = soss_kwargs['n_os']
kwargs['threshold'] = soss_kwargs['threshold']
result = model_image(scidata_bkg, scierr, scimask, refmask,
ref_file_args, **kwargs)
tracemodels, soss_kwargs['tikfac'], logl = result
else:
# No model can be fit for F277W yet, missing throughput reference files.
msg = f"No extraction possible for filter {soss_filter}."
log.critical(msg)
raise ValueError(msg)
# Save trace models for output reference
for order in tracemodels:
# Initialize a list for first integration
if i == 0:
all_tracemodels[order] = []
all_tracemodels[order].append(tracemodels[order])
# Use the trace models to perform a de-contaminated extraction.
kwargs = dict()
kwargs['width'] = soss_kwargs['width']
kwargs['bad_pix'] = soss_kwargs['bad_pix']
result = extract_image(scidata_bkg, scierr, scimask, tracemodels,
ref_files, transform, subarray, **kwargs)
wavelengths, fluxes, fluxerrs, npixels, box_weights = result
# Save box weights for output reference
for order in box_weights:
# Initialize a list for first integration
if i == 0:
all_box_weights[order] = []
all_box_weights[order].append(box_weights[order])
# Copy spectral data for each order into the output model.
for order in wavelengths.keys():
table_size = len(wavelengths[order])
out_table = np.zeros(
table_size, dtype=datamodels.SpecModel().spec_table.dtype)
out_table['WAVELENGTH'] = wavelengths[order]
out_table['FLUX'] = fluxes[order]
out_table['FLUX_ERROR'] = fluxerrs[order]
out_table['DQ'] = np.zeros(table_size)
out_table['BACKGROUND'] = col_bkg
out_table['NPIXELS'] = npixels[order]
spec = datamodels.SpecModel(spec_table=out_table)
# Add integration number and spectral order
spec.spectral_order = order_str_2_int[order]
spec.int_num = i + 1 # integration number starts at 1, not 0 like python
output_model.spec.append(spec)
output_model.meta.soss_extract1d.width = kwargs['width']
output_model.meta.soss_extract1d.tikhonov_factor = soss_kwargs[
'tikfac']
output_model.meta.soss_extract1d.delta_x = transform[1]
output_model.meta.soss_extract1d.delta_y = transform[2]
output_model.meta.soss_extract1d.theta = transform[0]
output_model.meta.soss_extract1d.oversampling = soss_kwargs['n_os']
output_model.meta.soss_extract1d.threshold = soss_kwargs[
'threshold']
else:
msg = "Only ImageModel and CubeModel are implemented for the NIRISS SOSS extraction."
log.critical(msg)
raise ValueError(msg)
# Save output references
for order in all_tracemodels:
# Convert from list to array
tracemod_ord = np.array(all_tracemodels[order])
# Save
order_int = order_str_2_int[order]
setattr(output_references, f'order{order_int}', tracemod_ord)
for order in all_box_weights:
# Convert from list to array
box_w_ord = np.array(all_box_weights[order])
# Save
order_int = order_str_2_int[order]
setattr(output_references, f'aperture{order_int}', box_w_ord)
return output_model, output_references
def get_mask(self, from_sources=None,
tracks=True, ghosts=True, spillage=True, spikes=True,
dead=True, nan=True, saturated=True, brightstarhalo=True,
lowresponsivity=True, highresponsivity=True, noisy=True,
sexsources=False, psfsources=False,
alltrue=False, flip_bits=True,
verbose=False, getflags=False,
**kwargs):
""" get a boolean mask (or associated flags). You have the chooce of
what you want to mask out.
Image pixels to be mask are set to True.
A pixel is masked if it corresponds to any of the requested entry.
For instance if a bitmask is '3', it corresponds to condition 1(2^0) et 2(2^1).
Since 0 -> tracks and 1 -> sexsources, if tracks or sexsources (or both) is (are) true,
then the pixel will be set to True.
Uses: astropy.nddata.bitmask.bitfield_to_boolean_mask
Parameters
----------
from_source: [None/bool/DataFrame] -optional-
A mask will be extracted from the given source.
(This uses, sep.mask_ellipse)
Accepted format:
- None or False: ignored.
- True: this uses the self.sources
- DataFrame: this will using this dataframe as source.
this dataframe must contain: x,y,a,b,theta
=> accepted kwargs: 'r' the scale (diameter) of the ellipse (5 default)
// If from_source is used, rest is ignored.
// These corresponds to the bitmasking definition for the IPAC pipeline //
Special parameters
alltrue: [bool] -optional-
Short cut to set everything to true. Supposedly only background left
flip_bits: [bool] -optional-
This should be True to have the aforementioned masking proceedure.
See astropy.nddata.bitmask.bitfield_to_boolean_mask
verbose: [bool] -optional-
Shall this print what you requested ?
getflags: [bool]
Get the bitmask power of 2 you requested instead of the actual masking.
Returns
-------
boolean mask (or list of int, see getflags)
"""
if from_sources is not None and from_sources is not False:
return super().get_mask(from_sources=from_sources, **kwargs)
# // BitMasking
if alltrue and not getflags:
return np.asarray(self.mask>0, dtype="bool")
locals_ = locals()
if verbose:
print({k:locals_[k] for k in self.BITMASK_KEY})
flags = [2**i for i,k in enumerate(self.BITMASK_KEY) if locals_[k] or alltrue]
if getflags:
return flags
return bitmask.bitfield_to_boolean_mask(self.mask, ignore_flags=flags, flip_bits=flip_bits)
def generate_source_catalog(image, **kwargs):
""" Build source catalogs for each chip using photutils.
The catalog returned by this function includes sources found in all chips
of the input image with the positions translated to the coordinate frame
defined by the reference WCS `refwcs`. The sources will be
- identified using photutils segmentation-based source finding code
- ignore any input pixel which has been flagged as 'bad' in the DQ
array, should a DQ array be found in the input HDUList.
- classified as probable cosmic-rays (if enabled) using central_moments
properties of each source, with these sources being removed from the
catalog.
Parameters
-----------
image : HDUList object
Input image as an astropy.io.fits HDUList object
dqname : string
EXTNAME for the DQ array, if present, in the input image HDUList.
output : boolean
Specify whether or not to write out a separate catalog file for all the
sources found in each chip. Default: None (False)
Optional Parameters
--------------------
threshold : float, optional
This parameter controls the threshold used for identifying sources in
the image relative to the background RMS.
If None, compute a default value of (background+3*rms(background)).
If threshold < 0.0, use absolute value as scaling factor for default value.
Default: None
fwhm : float, optional
FWHM (in pixels) of the expected sources from the image, comparable to the
'conv_width' parameter from 'tweakreg'. Objects with FWHM closest to
this value will be identified as sources in the catalog. Default: 3.0.
Returns
--------
source_cats : dict
Dict of astropy Tables identified by chip number with
each table containing sources from image extension `('sci',chip)`.
"""
if not isinstance(image, pf.HDUList):
raise ValueError("Input {} not fits.HDUList object".format(image))
dqname = kwargs.get('dqname','DQ')
output = kwargs.get('output',None)
# Build source catalog for entire image
source_cats = {}
numSci = countExtn(image, extname='SCI')
for chip in range(numSci):
chip += 1
# find sources in image
if output:
rootname = image[0].header['rootname']
outroot = '{}_sci{}_src'.format(rootname, chip)
kwargs['output'] = outroot
imgarr = image['sci',chip].data
if 'photmode' in image[0].header:
photmode = image[0].header['photmode']
else:
photmode = image['sci',chip].header['photmode']
# apply any DQ array, if available
dqmask = None
if image.index_of(dqname):
dqarr = image[dqname,chip].data
# "grow out" regions in DQ mask flagged as saturated by several pixels in every direction to prevent the
# source match algorithm from trying to match multiple sources from one image to a single source in the
# other or vice-versa.
non_sat_mask = bitfield_to_boolean_mask(dqarr,ignore_flags =256) # Create temp DQ mask containing all pixels flagged with any value EXCEPT 256
sat_mask = bitfield_to_boolean_mask(dqarr,ignore_flags =~256) # Create temp DQ mask containing saturated pixels ONLY
grown_sat_mask = ndimage.binary_dilation(sat_mask,iterations = 5) # Grow out saturated pixels by a few pixels in every direction
dqmask = np.bitwise_or(non_sat_mask,grown_sat_mask) # combine the two temporary DQ masks into a single composite DQ mask.
# dqmask = bitfield_to_boolean_mask(dqarr, good_mask_value=False) #TODO: <---Remove this old no-sat bit grow line once this thing works
seg_tab, segmap = extract_sources(imgarr, dqmask=dqmask, **kwargs)
seg_tab_phot = seg_tab #compute_photometry(seg_tab,photmode)
source_cats[chip] = seg_tab_phot
return source_cats
def test_bitfield_must_be_integer_check():
with pytest.raises(TypeError):
bitmask.bitfield_to_boolean_mask(1.0, 1)
def generate_source_catalog(image,
dqname="DQ",
output=False,
fwhm=3.0,
**detector_pars):
""" Build source catalogs for each chip using photutils.
The catalog returned by this function includes sources found in all chips
of the input image with the positions translated to the coordinate frame
defined by the reference WCS `refwcs`. The sources will be
- identified using photutils segmentation-based source finding code
- ignore any input pixel which has been flagged as 'bad' in the DQ
array, should a DQ array be found in the input HDUList.
- classified as probable cosmic-rays (if enabled) using central_moments
properties of each source, with these sources being removed from the
catalog.
Parameters
----------
image : `~astropy.io.fits.HDUList`
Input image as an astropy.io.fits HDUList.
dqname : str
EXTNAME for the DQ array, if present, in
the input image HDUList.
output : bool
Specify whether or not to write out a separate catalog file for all the
sources found in each chip.
fwhm : float
Full-width half-maximum (fwhm) of the PSF in pixels.
Returns
-------
source_cats : dict
Dict of astropy Tables identified by chip number with
each table containing sources from image extension ``('sci', chip)``.
"""
if not isinstance(image, fits.HDUList):
raise ValueError("Input {} not fits.HDUList object".format(image))
# remove parameters that are not needed by subsequent functions
del detector_pars['fwhmpsf']
# Build source catalog for entire image
source_cats = {}
numSci = countExtn(image, extname='SCI')
outroot = None
for chip in range(numSci):
chip += 1
# find sources in image
if output:
rootname = image[0].header['rootname']
outroot = '{}_sci{}_src'.format(rootname, chip)
imgarr = image['sci', chip].data
# apply any DQ array, if available
dqmask = None
if image.index_of(dqname):
dqarr = image[dqname, chip].data
# "grow out" regions in DQ mask flagged as saturated by several
# pixels in every direction to prevent the
# source match algorithm from trying to match multiple sources
# from one image to a single source in the
# other or vice-versa.
# Create temp DQ mask containing all pixels flagged with any value EXCEPT 256
non_sat_mask = bitfield_to_boolean_mask(dqarr, ignore_flags=256)
# Create temp DQ mask containing saturated pixels ONLY
sat_mask = bitfield_to_boolean_mask(dqarr, ignore_flags=~256)
# Grow out saturated pixels by a few pixels in every direction
grown_sat_mask = ndimage.binary_dilation(sat_mask, iterations=5)
# combine the two temporary DQ masks into a single composite DQ mask.
dqmask = np.bitwise_or(non_sat_mask, grown_sat_mask)
# dqmask = bitfield_to_boolean_mask(dqarr, good_mask_value=False)
# TODO: <---Remove this old no-sat bit grow line once this
# thing works
seg_tab, segmap = extract_sources(imgarr,
dqmask=dqmask,
outroot=outroot,
fwhm=fwhm,
**detector_pars)
seg_tab_phot = seg_tab
source_cats[chip] = seg_tab_phot
return source_cats
def process(self, input1, input2):
cube_models = datamodels.ModelContainer(input1, persist=False)
models2d = datamodels.ModelContainer(input2, persist=False)
dqbits = interpret_bit_flags(self.dqbits)
# set sky stattistics:
self._skystat = SkyStats(
skystat=self.skystat,
lower=self.lower,
upper=self.upper,
nclip=self.nclip,
lsig=self.lsigma,
usig=self.usigma,
binwidth=self.binwidth
)
# At this moment running 'cube_skymatch' on images whose
# background has been previously subtracted is not supported.
# Raise an exception if background was subtracted:
self._check_background(cube_models)
self._check_background(models2d)
self._reset_background(cube_models)
self._reset_background(models2d)
# create a list of SkyCubes:
skycubes = []
for cm in cube_models:
# process weights and combine with DQ:
if not hasattr(cm, 'weightmap') or cm.weightmap is None:
weights = np.ones_like(cm.data, dtype=np.float64)
else:
weights = cm.weightmap.copy()
if dqbits is not None:
dq = bitfield_to_boolean_mask(
cm.dq,
self._dqbits,
good_mask_value=0,
dtype=np.bool
)
weights[dq] = 0.0
wcs = cm.meta.wcs if hasattr(cm.meta, 'wcs') else None
wcsinfo = cm.meta.wcsinfo if hasattr(cm.meta, 'wcsinfo') else None
exptime = cm.meta.exposure.exposure_time
if exptime is None:
exptime = 1.0
sc = SkyCube(
data=cm.data,
wcs=wcs,
wcsinfo=wcsinfo,
weights=weights,
cube_weight=exptime,
bkg_deg=self.bkg_degree,
bkg_center=None,
id=None,
meta={'original_cube_model': cm}
)
skycubes.append(sc)
skymatch_cubes, nsubspace = match(skycubes, subtract=self.subtract)
if nsubspace > 1:
self.log.warning("Not all cubes have been sky matched as "
"some of them do not overlap.")
# save background info in 'meta' and subtract sky from 2D images
# if requested:
##### model.meta.instrument.channel
for c in skymatch_cubes:
self._set_cube_bkg_meta(c.meta['original_cube_model'], c)
model2d, channel = _find_associated_2d_image(
c.meta['original_cube_model'], models2d
)
if model2d is None:
continue
self._set_model2d_bkg_meta(
c.meta['original_cube_model'],
model2d,
channel
)
if self.subtract2d:
self._apply_sky_2d(model2d, channel)
return cube_models, models2d
