def build_dqmask(self, chip=None): # apply any DQ array, if available dqmask = None if chip: dqarr = self.imghdu[('DQ', chip)].data else: dqarr = np.concatenate( [self.imghdu[('DQ', i + 1)].data for i in range(self.num_sci)]) # "grow out" regions in DQ mask flagged as saturated by several # pixels in every direction to prevent the # source match algorithm from trying to match multiple sources # from one image to a single source in the # other or vice-versa. # Create temp DQ mask containing all pixels flagged with any value EXCEPT 256 non_sat_mask = bitfield_to_boolean_mask(dqarr, ignore_flags=256 + 2048) # Create temp DQ mask containing saturated pixels ONLY sat_mask = bitfield_to_boolean_mask(dqarr, ignore_flags=~(256 + 2048)) # Ignore sources where only a couple of pixels are flagged as saturated sat_mask = ndimage.binary_erosion(sat_mask, iterations=1) # Grow out saturated pixels by a few pixels in every direction grown_sat_mask = ndimage.binary_dilation(sat_mask, iterations=5) # combine the two temporary DQ masks into a single composite DQ mask. dqmask = np.bitwise_or(non_sat_mask, grown_sat_mask) return dqmask
def _mergeUserMaskAndDQ(dq, mask, dqbits): from astropy.nddata.bitmask import (interpret_bit_flags, bitfield_to_boolean_mask) dqbits = interpret_bit_flags(dqbits) if dqbits is None: if mask is None: return None else: return mask.copy().astype(dtype=np.uint8) if dq is None: raise ValueError("DQ array is None while 'dqbits' is not None.") dqmask = bitfield_to_boolean_mask(dq, dqbits, good_mask_value=1, dtype=np.uint8) if mask is None: return dqmask # merge user mask with DQ mask: dqmask[mask == 0] = 0 return dqmask
def _imodel2skyim(self, image_model): # create if self._dqbits is None: dqmask = np.isfinite(image_model.data).astype(dtype=np.uint8) else: dqmask = bitfield_to_boolean_mask( image_model.dq, self._dqbits, good_mask_value=1, dtype=np.uint8 ) * np.isfinite(image_model.data) # see if 'skymatch' was previously run and raise an exception # if 'subtract' mode has changed compared to the previous pass: if image_model.meta.background.subtracted is None: if image_model.meta.background.level is not None: # report inconsistency: raise ValueError("Background level was set but the " "'subtracted' property is undefined (None).") level = 0.0 else: level = image_model.meta.background.level if level is None: # NOTE: In principle we could assume that level is 0 and # possibly add a log entry documenting this, however, # at this moment I think it is saver to quit and... # # report inconsistency: raise ValueError("Background level was subtracted but the " "'level' property is undefined (None).") if image_model.meta.background.subtracted != self.subtract: # cannot run 'skymatch' step on already "skymatched" images # when 'subtract' spec is inconsistent with # meta.background.subtracted: raise ValueError("'subtract' step's specification is " "inconsistent with background info already " "present in image '{:s}' meta." .format(image_model.meta.filename)) sky_im = SkyImage( image=image_model.data, wcs_fwd=image_model.meta.wcs.__call__, wcs_inv=image_model.meta.wcs.invert, pix_area=1.0, #TODO: pixel area convf=1.0, #TODO: conv. factor to brightness mask=dqmask, id=image_model.meta.filename, # file name? skystat=self._skystat, stepsize=self.stepsize, meta={'image_model': image_model} ) if self.subtract: sky_im.sky = level return sky_im
def _imodel2skyim(self, image_model): # create if self._dqbits is None: dqmask = np.isfinite(image_model.data).astype(dtype=np.uint8) else: dqmask = bitfield_to_boolean_mask( image_model.dq, self._dqbits, good_mask_value=1, dtype=np.uint8 ) * np.isfinite(image_model.data) # see if 'skymatch' was previously run and raise an exception # if 'subtract' mode has changed compared to the previous pass: if image_model.meta.background.subtracted is None: if image_model.meta.background.level is not None: # report inconsistency: raise ValueError("Background level was set but the " "'subtracted' property is undefined (None).") level = 0.0 else: level = image_model.meta.background.level if level is None: # NOTE: In principle we could assume that level is 0 and # possibly add a log entry documenting this, however, # at this moment I think it is saver to quit and... # # report inconsistency: raise ValueError("Background level was subtracted but the " "'level' property is undefined (None).") if image_model.meta.background.subtracted != self.subtract: # cannot run 'skymatch' step on already "skymatched" images # when 'subtract' spec is inconsistent with # meta.background.subtracted: raise ValueError("'subtract' step's specification is " "inconsistent with background info already " "present in image '{:s}' meta." .format(image_model.meta.filename)) sky_im = SkyImage( image=image_model.data, wcs_fwd=image_model.meta.wcs.__call__, wcs_inv=image_model.meta.wcs.invert, pix_area=1.0, #TODO: pixel area convf=1.0, #TODO: conv. factor to brightness mask=dqmask, id=image_model.meta.filename, # file name? skystat=self._skystat, stepsize=self.stepsize, meta={'image_model': image_model} ) if self.subtract: sky_im.sky = level return sky_im
def test_bitfield_to_boolean_mask(data, flags, flip, goodval, dtype, ref): mask = bitmask.bitfield_to_boolean_mask(bitfield=data, ignore_flags=flags, flip_bits=flip, good_mask_value=goodval, dtype=dtype) assert (mask.dtype == dtype) assert np.all(mask == ref)
def ix_masked(self,maskcol,maskval=None,indices=None): # get the indices based on input. indices=self.getindices(indices) if maskval is None: (keep,) = np.where(self.t.loc[indices,maskcol].ne(0)) else: (keep,) = np.where(bitmask.bitfield_to_boolean_mask(self.t.loc[indices,maskcol].astype('int'),ignore_flags=~maskval)) indices = indices[keep] return(indices)
def test_bitfield_to_boolean_mask(data, flags, flip, goodval, dtype, ref): mask = bitmask.bitfield_to_boolean_mask( bitfield=data, ignore_flags=flags, flip_bits=flip, good_mask_value=goodval, dtype=dtype ) assert(mask.dtype == dtype) assert np.all(mask == ref)
def build_dqmask(self, chip=None): # apply any DQ array, if available dqmask = None if chip: dqarr = self.imghdu[('DQ', chip)].data else: dqarr = np.concatenate( [self.imghdu[('DQ', i + 1)].data for i in range(self.num_sci)]) # Create temp DQ mask containing all pixels flagged with any value EXCEPT 512 (bad ref pixel) dqmask = bitfield_to_boolean_mask(dqarr, ignore_flags=512) return dqmask
def test_bitfield_must_be_integer_check(): with pytest.raises(TypeError): bitmask.bitfield_to_boolean_mask(1.0, 1)
def process(self, input1, input2): cube_models = datamodels.ModelContainer(input1) models2d = datamodels.ModelContainer(input2) dqbits = interpret_bit_flags(self.dqbits, flag_name_map=pixel) # set sky statistics: self._skystat = SkyStats( skystat=self.skystat, lower=self.lower, upper=self.upper, nclip=self.nclip, lsig=self.lsigma, usig=self.usigma, binwidth=self.binwidth ) # At this moment running 'cube_skymatch' on images whose # background has been previously subtracted is not supported. # Raise an exception if background was subtracted: self._check_background(cube_models) self._check_background(models2d) self._reset_background(cube_models) self._reset_background(models2d) # create a list of SkyCubes: skycubes = [] for cm in cube_models: # process weights and combine with DQ: if not hasattr(cm, 'weightmap') or cm.weightmap is None: weights = np.ones_like(cm.data, dtype=np.float64) else: weights = cm.weightmap.copy() if dqbits is not None: dq = bitfield_to_boolean_mask( cm.dq, self._dqbits, good_mask_value=0, dtype=bool ) weights[dq] = 0.0 wcs = cm.meta.wcs if hasattr(cm.meta, 'wcs') else None wcsinfo = cm.meta.wcsinfo if hasattr(cm.meta, 'wcsinfo') else None exptime = cm.meta.exposure.exposure_time if exptime is None: exptime = 1.0 sc = SkyCube( data=cm.data, wcs=wcs, wcsinfo=wcsinfo, weights=weights, cube_weight=exptime, bkg_deg=self.bkg_degree, bkg_center=None, id=None, meta={'original_cube_model': cm} ) skycubes.append(sc) skymatch_cubes, nsubspace = match(skycubes, subtract=self.subtract) if nsubspace > 1: self.log.warning("Not all cubes have been sky matched as " "some of them do not overlap.") # save background info in 'meta' and subtract sky from 2D images # if requested: # model.meta.instrument.channel for c in skymatch_cubes: self._set_cube_bkg_meta(c.meta['original_cube_model'], c) model2d, channel = _find_associated_2d_image( c.meta['original_cube_model'], models2d ) if model2d is None: continue self._set_model2d_bkg_meta( c.meta['original_cube_model'], model2d, channel ) if self.subtract2d: self._apply_sky_2d(model2d, channel) return cube_models, models2d
def run_extract1d(input_model, spectrace_ref_name, wavemap_ref_name, specprofile_ref_name, speckernel_ref_name, subarray, soss_filter, soss_kwargs): """Run the spectral extraction on NIRISS SOSS data. Parameters ---------- input_model : DataModel The input DataModel. spectrace_ref_name : str Name of the spectrace reference file. wavemap_ref_name : str Name of the wavemap reference file. specprofile_ref_name : str Name of the specprofile reference file. speckernel_ref_name : str Name of the speckernel reference file. subarray : str Subarray on which the data were recorded; one of 'SUBSTRIPT96', 'SUBSTRIP256' or 'FULL'. soss_filter : str Filter in place during observations; one of 'CLEAR' or 'F277W'. soss_kwargs : dict Dictionary of keyword arguments passed from extract_1d_step. Returns ------- output_model : DataModel DataModel containing the extracted spectra. """ # Map the order integer names to the string names order_str_2_int = {f'Order {order}': order for order in [1, 2, 3]} # Read the reference files. spectrace_ref = datamodels.SpecTraceModel(spectrace_ref_name) wavemap_ref = datamodels.WaveMapModel(wavemap_ref_name) specprofile_ref = datamodels.SpecProfileModel(specprofile_ref_name) speckernel_ref = datamodels.SpecKernelModel(speckernel_ref_name) ref_files = dict() ref_files['spectrace'] = spectrace_ref ref_files['wavemap'] = wavemap_ref ref_files['specprofile'] = specprofile_ref ref_files['speckernel'] = speckernel_ref # Initialize the output model and output references (model of the detector and box aperture weights). output_model = datamodels.MultiSpecModel() output_model.update(input_model) # Copy meta data from input to output. output_references = datamodels.SossExtractModel() output_references.update(input_model) all_tracemodels = dict() all_box_weights = dict() # Extract depending on the type of datamodels (Image or Cube) if isinstance(input_model, datamodels.ImageModel): log.info('Input is an ImageModel, processing a single integration.') # Initialize the theta, dx, dy transform parameters transform = soss_kwargs.pop('transform') # Received a single 2D image; set dtype to float64 and convert DQ to boolean mask. scidata = input_model.data.astype('float64') scierr = input_model.err.astype('float64') scimask = input_model.dq > 0 # Mask bad pixels with True. refmask = bitfield_to_boolean_mask( input_model.dq, ignore_flags=dqflags.pixel['REFERENCE_PIXEL'], flip_bits=True) # Perform background correction. bkg_mask = make_background_mask(scidata, width=40) scidata_bkg, col_bkg, npix_bkg = soss_background(scidata, scimask, bkg_mask=bkg_mask) # Determine the theta, dx, dy transform needed to match scidata trace position to ref file position. if transform is None: log.info('Solving for the transformation parameters.') # Unpack the expected order 1 & 2 positions. spectrace_ref = ref_files['spectrace'] xref_o1 = spectrace_ref.trace[0].data['X'] yref_o1 = spectrace_ref.trace[0].data['Y'] xref_o2 = spectrace_ref.trace[1].data['X'] yref_o2 = spectrace_ref.trace[1].data['Y'] # Use the solver on the background subtracted image. if subarray == 'SUBSTRIP96' or soss_filter == 'F277W': # Use only order 1 to solve theta, dx, dy transform = solve_transform(scidata_bkg, scimask, xref_o1, yref_o1, soss_filter=soss_filter) else: transform = solve_transform(scidata_bkg, scimask, xref_o1, yref_o1, xref_o2, yref_o2, soss_filter=soss_filter) log.info( 'Measured to Reference trace position transform: theta={:.4f}, dx={:.4f}, dy={:.4f}' .format(transform[0], transform[1], transform[2])) # Prepare the reference file arguments. ref_file_args = get_ref_file_args(ref_files, transform) # Make sure wavelength maps cover only parts where the centroid is inside the detector image _mask_wv_map_centroid_outside(ref_file_args[0], ref_files, transform, scidata_bkg.shape[0]) # Model the traces based on optics filter configuration (CLEAR or F277W) if soss_filter == 'CLEAR': # Model the image. kwargs = dict() kwargs['transform'] = transform kwargs['tikfac'] = soss_kwargs['tikfac'] kwargs['n_os'] = soss_kwargs['n_os'] kwargs['threshold'] = soss_kwargs['threshold'] result = model_image(scidata_bkg, scierr, scimask, refmask, ref_file_args, **kwargs) tracemodels, soss_kwargs['tikfac'], logl = result else: # No model can be fit for F277W yet, missing throughput reference files. msg = f"No extraction possible for filter {soss_filter}." log.critical(msg) raise ValueError(msg) # Save trace models for output reference for order in tracemodels: # Save as a list (convert to array at the end) all_tracemodels[order] = [tracemodels[order]] # Use the trace models to perform a decontaminated extraction. kwargs = dict() kwargs['width'] = soss_kwargs['width'] kwargs['bad_pix'] = soss_kwargs['bad_pix'] result = extract_image(scidata_bkg, scierr, scimask, tracemodels, ref_files, transform, subarray, **kwargs) wavelengths, fluxes, fluxerrs, npixels, box_weights = result # Save box weights for output reference for order in box_weights: # Save as a list (convert to array at the end) all_box_weights[order] = [box_weights[order]] # Copy spectral data for each order into the output model. for order in wavelengths.keys(): table_size = len(wavelengths[order]) out_table = np.zeros(table_size, dtype=datamodels.SpecModel().spec_table.dtype) out_table['WAVELENGTH'] = wavelengths[order] out_table['FLUX'] = fluxes[order] out_table['FLUX_ERROR'] = fluxerrs[order] out_table['DQ'] = np.zeros(table_size) out_table['BACKGROUND'] = col_bkg out_table['NPIXELS'] = npixels[order] spec = datamodels.SpecModel(spec_table=out_table) # Add integration number and spectral order spec.spectral_order = order_str_2_int[order] output_model.spec.append(spec) output_model.meta.soss_extract1d.width = kwargs['width'] output_model.meta.soss_extract1d.tikhonov_factor = soss_kwargs[ 'tikfac'] output_model.meta.soss_extract1d.delta_x = transform[1] output_model.meta.soss_extract1d.delta_y = transform[2] output_model.meta.soss_extract1d.theta = transform[0] output_model.meta.soss_extract1d.oversampling = soss_kwargs['n_os'] output_model.meta.soss_extract1d.threshold = soss_kwargs['threshold'] elif isinstance(input_model, datamodels.CubeModel): nimages = len(input_model.data) log.info( 'Input is a CubeModel containing {} integrations.'.format(nimages)) # Initialize the theta, dx, dy transform parameters transform = soss_kwargs.pop('transform') # Loop over images. for i in range(nimages): log.info('Processing integration {} of {}.'.format(i + 1, nimages)) # Unpack the i-th image, set dtype to float64 and convert DQ to boolean mask. scidata = input_model.data[i].astype('float64') scierr = input_model.err[i].astype('float64') scimask = np.bitwise_and(input_model.dq[i], dqflags.pixel['DO_NOT_USE']).astype(bool) refmask = bitfield_to_boolean_mask( input_model.dq[i], ignore_flags=dqflags.pixel['REFERENCE_PIXEL'], flip_bits=True) # Perform background correction. bkg_mask = make_background_mask(scidata, width=40) scidata_bkg, col_bkg, npix_bkg = soss_background(scidata, scimask, bkg_mask=bkg_mask) # Determine the theta, dx, dy transform needed to match scidata trace position to ref file position. if transform is None: log.info('Solving for the transformation parameters.') # Unpack the expected order 1 & 2 positions. spectrace_ref = ref_files['spectrace'] xref_o1 = spectrace_ref.trace[0].data['X'] yref_o1 = spectrace_ref.trace[0].data['Y'] xref_o2 = spectrace_ref.trace[1].data['X'] yref_o2 = spectrace_ref.trace[1].data['Y'] # Use the solver on the background subtracted image. if subarray == 'SUBSTRIP96' or soss_filter == 'F277W': # Use only order 1 to solve theta, dx, dy transform = solve_transform(scidata_bkg, scimask, xref_o1, yref_o1, soss_filter=soss_filter) else: transform = solve_transform(scidata_bkg, scimask, xref_o1, yref_o1, xref_o2, yref_o2, soss_filter=soss_filter) log.info( 'Measured to Reference trace position transform: theta={:.4f}, dx={:.4f}, dy={:.4f}' .format(transform[0], transform[1], transform[2])) # Prepare the reference file arguments. ref_file_args = get_ref_file_args(ref_files, transform) # Make sure wavelength maps cover only parts where the centroid is inside the detector image _mask_wv_map_centroid_outside(ref_file_args[0], ref_files, transform, scidata_bkg.shape[0]) # Model the traces based on optics filter configuration (CLEAR or F277W) if soss_filter == 'CLEAR': # Model the image. kwargs = dict() kwargs['transform'] = transform kwargs['tikfac'] = soss_kwargs['tikfac'] kwargs['n_os'] = soss_kwargs['n_os'] kwargs['threshold'] = soss_kwargs['threshold'] result = model_image(scidata_bkg, scierr, scimask, refmask, ref_file_args, **kwargs) tracemodels, soss_kwargs['tikfac'], logl = result else: # No model can be fit for F277W yet, missing throughput reference files. msg = f"No extraction possible for filter {soss_filter}." log.critical(msg) raise ValueError(msg) # Save trace models for output reference for order in tracemodels: # Initialize a list for first integration if i == 0: all_tracemodels[order] = [] all_tracemodels[order].append(tracemodels[order]) # Use the trace models to perform a de-contaminated extraction. kwargs = dict() kwargs['width'] = soss_kwargs['width'] kwargs['bad_pix'] = soss_kwargs['bad_pix'] result = extract_image(scidata_bkg, scierr, scimask, tracemodels, ref_files, transform, subarray, **kwargs) wavelengths, fluxes, fluxerrs, npixels, box_weights = result # Save box weights for output reference for order in box_weights: # Initialize a list for first integration if i == 0: all_box_weights[order] = [] all_box_weights[order].append(box_weights[order]) # Copy spectral data for each order into the output model. for order in wavelengths.keys(): table_size = len(wavelengths[order]) out_table = np.zeros( table_size, dtype=datamodels.SpecModel().spec_table.dtype) out_table['WAVELENGTH'] = wavelengths[order] out_table['FLUX'] = fluxes[order] out_table['FLUX_ERROR'] = fluxerrs[order] out_table['DQ'] = np.zeros(table_size) out_table['BACKGROUND'] = col_bkg out_table['NPIXELS'] = npixels[order] spec = datamodels.SpecModel(spec_table=out_table) # Add integration number and spectral order spec.spectral_order = order_str_2_int[order] spec.int_num = i + 1 # integration number starts at 1, not 0 like python output_model.spec.append(spec) output_model.meta.soss_extract1d.width = kwargs['width'] output_model.meta.soss_extract1d.tikhonov_factor = soss_kwargs[ 'tikfac'] output_model.meta.soss_extract1d.delta_x = transform[1] output_model.meta.soss_extract1d.delta_y = transform[2] output_model.meta.soss_extract1d.theta = transform[0] output_model.meta.soss_extract1d.oversampling = soss_kwargs['n_os'] output_model.meta.soss_extract1d.threshold = soss_kwargs[ 'threshold'] else: msg = "Only ImageModel and CubeModel are implemented for the NIRISS SOSS extraction." log.critical(msg) raise ValueError(msg) # Save output references for order in all_tracemodels: # Convert from list to array tracemod_ord = np.array(all_tracemodels[order]) # Save order_int = order_str_2_int[order] setattr(output_references, f'order{order_int}', tracemod_ord) for order in all_box_weights: # Convert from list to array box_w_ord = np.array(all_box_weights[order]) # Save order_int = order_str_2_int[order] setattr(output_references, f'aperture{order_int}', box_w_ord) return output_model, output_references
def get_mask(self, from_sources=None, tracks=True, ghosts=True, spillage=True, spikes=True, dead=True, nan=True, saturated=True, brightstarhalo=True, lowresponsivity=True, highresponsivity=True, noisy=True, sexsources=False, psfsources=False, alltrue=False, flip_bits=True, verbose=False, getflags=False, **kwargs): """ get a boolean mask (or associated flags). You have the chooce of what you want to mask out. Image pixels to be mask are set to True. A pixel is masked if it corresponds to any of the requested entry. For instance if a bitmask is '3', it corresponds to condition 1(2^0) et 2(2^1). Since 0 -> tracks and 1 -> sexsources, if tracks or sexsources (or both) is (are) true, then the pixel will be set to True. Uses: astropy.nddata.bitmask.bitfield_to_boolean_mask Parameters ---------- from_source: [None/bool/DataFrame] -optional- A mask will be extracted from the given source. (This uses, sep.mask_ellipse) Accepted format: - None or False: ignored. - True: this uses the self.sources - DataFrame: this will using this dataframe as source. this dataframe must contain: x,y,a,b,theta => accepted kwargs: 'r' the scale (diameter) of the ellipse (5 default) // If from_source is used, rest is ignored. // These corresponds to the bitmasking definition for the IPAC pipeline // Special parameters alltrue: [bool] -optional- Short cut to set everything to true. Supposedly only background left flip_bits: [bool] -optional- This should be True to have the aforementioned masking proceedure. See astropy.nddata.bitmask.bitfield_to_boolean_mask verbose: [bool] -optional- Shall this print what you requested ? getflags: [bool] Get the bitmask power of 2 you requested instead of the actual masking. Returns ------- boolean mask (or list of int, see getflags) """ if from_sources is not None and from_sources is not False: return super().get_mask(from_sources=from_sources, **kwargs) # // BitMasking if alltrue and not getflags: return np.asarray(self.mask>0, dtype="bool") locals_ = locals() if verbose: print({k:locals_[k] for k in self.BITMASK_KEY}) flags = [2**i for i,k in enumerate(self.BITMASK_KEY) if locals_[k] or alltrue] if getflags: return flags return bitmask.bitfield_to_boolean_mask(self.mask, ignore_flags=flags, flip_bits=flip_bits)
def generate_source_catalog(image, **kwargs): """ Build source catalogs for each chip using photutils. The catalog returned by this function includes sources found in all chips of the input image with the positions translated to the coordinate frame defined by the reference WCS `refwcs`. The sources will be - identified using photutils segmentation-based source finding code - ignore any input pixel which has been flagged as 'bad' in the DQ array, should a DQ array be found in the input HDUList. - classified as probable cosmic-rays (if enabled) using central_moments properties of each source, with these sources being removed from the catalog. Parameters ----------- image : HDUList object Input image as an astropy.io.fits HDUList object dqname : string EXTNAME for the DQ array, if present, in the input image HDUList. output : boolean Specify whether or not to write out a separate catalog file for all the sources found in each chip. Default: None (False) Optional Parameters -------------------- threshold : float, optional This parameter controls the threshold used for identifying sources in the image relative to the background RMS. If None, compute a default value of (background+3*rms(background)). If threshold < 0.0, use absolute value as scaling factor for default value. Default: None fwhm : float, optional FWHM (in pixels) of the expected sources from the image, comparable to the 'conv_width' parameter from 'tweakreg'. Objects with FWHM closest to this value will be identified as sources in the catalog. Default: 3.0. Returns -------- source_cats : dict Dict of astropy Tables identified by chip number with each table containing sources from image extension `('sci',chip)`. """ if not isinstance(image, pf.HDUList): raise ValueError("Input {} not fits.HDUList object".format(image)) dqname = kwargs.get('dqname','DQ') output = kwargs.get('output',None) # Build source catalog for entire image source_cats = {} numSci = countExtn(image, extname='SCI') for chip in range(numSci): chip += 1 # find sources in image if output: rootname = image[0].header['rootname'] outroot = '{}_sci{}_src'.format(rootname, chip) kwargs['output'] = outroot imgarr = image['sci',chip].data if 'photmode' in image[0].header: photmode = image[0].header['photmode'] else: photmode = image['sci',chip].header['photmode'] # apply any DQ array, if available dqmask = None if image.index_of(dqname): dqarr = image[dqname,chip].data # "grow out" regions in DQ mask flagged as saturated by several pixels in every direction to prevent the # source match algorithm from trying to match multiple sources from one image to a single source in the # other or vice-versa. non_sat_mask = bitfield_to_boolean_mask(dqarr,ignore_flags =256) # Create temp DQ mask containing all pixels flagged with any value EXCEPT 256 sat_mask = bitfield_to_boolean_mask(dqarr,ignore_flags =~256) # Create temp DQ mask containing saturated pixels ONLY grown_sat_mask = ndimage.binary_dilation(sat_mask,iterations = 5) # Grow out saturated pixels by a few pixels in every direction dqmask = np.bitwise_or(non_sat_mask,grown_sat_mask) # combine the two temporary DQ masks into a single composite DQ mask. # dqmask = bitfield_to_boolean_mask(dqarr, good_mask_value=False) #TODO: <---Remove this old no-sat bit grow line once this thing works seg_tab, segmap = extract_sources(imgarr, dqmask=dqmask, **kwargs) seg_tab_phot = seg_tab #compute_photometry(seg_tab,photmode) source_cats[chip] = seg_tab_phot return source_cats
def test_bitfield_must_be_integer_check(): with pytest.raises(TypeError): bitmask.bitfield_to_boolean_mask(1.0, 1)
def generate_source_catalog(image, dqname="DQ", output=False, fwhm=3.0, **detector_pars): """ Build source catalogs for each chip using photutils. The catalog returned by this function includes sources found in all chips of the input image with the positions translated to the coordinate frame defined by the reference WCS `refwcs`. The sources will be - identified using photutils segmentation-based source finding code - ignore any input pixel which has been flagged as 'bad' in the DQ array, should a DQ array be found in the input HDUList. - classified as probable cosmic-rays (if enabled) using central_moments properties of each source, with these sources being removed from the catalog. Parameters ---------- image : `~astropy.io.fits.HDUList` Input image as an astropy.io.fits HDUList. dqname : str EXTNAME for the DQ array, if present, in the input image HDUList. output : bool Specify whether or not to write out a separate catalog file for all the sources found in each chip. fwhm : float Full-width half-maximum (fwhm) of the PSF in pixels. Returns ------- source_cats : dict Dict of astropy Tables identified by chip number with each table containing sources from image extension ``('sci', chip)``. """ if not isinstance(image, fits.HDUList): raise ValueError("Input {} not fits.HDUList object".format(image)) # remove parameters that are not needed by subsequent functions del detector_pars['fwhmpsf'] # Build source catalog for entire image source_cats = {} numSci = countExtn(image, extname='SCI') outroot = None for chip in range(numSci): chip += 1 # find sources in image if output: rootname = image[0].header['rootname'] outroot = '{}_sci{}_src'.format(rootname, chip) imgarr = image['sci', chip].data # apply any DQ array, if available dqmask = None if image.index_of(dqname): dqarr = image[dqname, chip].data # "grow out" regions in DQ mask flagged as saturated by several # pixels in every direction to prevent the # source match algorithm from trying to match multiple sources # from one image to a single source in the # other or vice-versa. # Create temp DQ mask containing all pixels flagged with any value EXCEPT 256 non_sat_mask = bitfield_to_boolean_mask(dqarr, ignore_flags=256) # Create temp DQ mask containing saturated pixels ONLY sat_mask = bitfield_to_boolean_mask(dqarr, ignore_flags=~256) # Grow out saturated pixels by a few pixels in every direction grown_sat_mask = ndimage.binary_dilation(sat_mask, iterations=5) # combine the two temporary DQ masks into a single composite DQ mask. dqmask = np.bitwise_or(non_sat_mask, grown_sat_mask) # dqmask = bitfield_to_boolean_mask(dqarr, good_mask_value=False) # TODO: <---Remove this old no-sat bit grow line once this # thing works seg_tab, segmap = extract_sources(imgarr, dqmask=dqmask, outroot=outroot, fwhm=fwhm, **detector_pars) seg_tab_phot = seg_tab source_cats[chip] = seg_tab_phot return source_cats
def process(self, input1, input2): cube_models = datamodels.ModelContainer(input1, persist=False) models2d = datamodels.ModelContainer(input2, persist=False) dqbits = interpret_bit_flags(self.dqbits) # set sky stattistics: self._skystat = SkyStats( skystat=self.skystat, lower=self.lower, upper=self.upper, nclip=self.nclip, lsig=self.lsigma, usig=self.usigma, binwidth=self.binwidth ) # At this moment running 'cube_skymatch' on images whose # background has been previously subtracted is not supported. # Raise an exception if background was subtracted: self._check_background(cube_models) self._check_background(models2d) self._reset_background(cube_models) self._reset_background(models2d) # create a list of SkyCubes: skycubes = [] for cm in cube_models: # process weights and combine with DQ: if not hasattr(cm, 'weightmap') or cm.weightmap is None: weights = np.ones_like(cm.data, dtype=np.float64) else: weights = cm.weightmap.copy() if dqbits is not None: dq = bitfield_to_boolean_mask( cm.dq, self._dqbits, good_mask_value=0, dtype=np.bool ) weights[dq] = 0.0 wcs = cm.meta.wcs if hasattr(cm.meta, 'wcs') else None wcsinfo = cm.meta.wcsinfo if hasattr(cm.meta, 'wcsinfo') else None exptime = cm.meta.exposure.exposure_time if exptime is None: exptime = 1.0 sc = SkyCube( data=cm.data, wcs=wcs, wcsinfo=wcsinfo, weights=weights, cube_weight=exptime, bkg_deg=self.bkg_degree, bkg_center=None, id=None, meta={'original_cube_model': cm} ) skycubes.append(sc) skymatch_cubes, nsubspace = match(skycubes, subtract=self.subtract) if nsubspace > 1: self.log.warning("Not all cubes have been sky matched as " "some of them do not overlap.") # save background info in 'meta' and subtract sky from 2D images # if requested: ##### model.meta.instrument.channel for c in skymatch_cubes: self._set_cube_bkg_meta(c.meta['original_cube_model'], c) model2d, channel = _find_associated_2d_image( c.meta['original_cube_model'], models2d ) if model2d is None: continue self._set_model2d_bkg_meta( c.meta['original_cube_model'], model2d, channel ) if self.subtract2d: self._apply_sky_2d(model2d, channel) return cube_models, models2d