def estimate_sky_background(data, method='scalar', mask_sources=True, path=None): """Estimate a scalar or image sky background with uncertainties. Args: data (np.array or str): Image or data cube with sky observations used to extract the sky background mean and uncertainty. Str type input is interpreted as a file name to read the data from. method (str, optional): Can be `scalar` or `image` for setting the shape of the output. mask_sources (bool, optional): Creates a source mask to exclude sources from the measurement. This should not be set if working in `image` mode. path (str, optional): Path to the data files. This is used only if data is provided as a file name. Returns: mean (float or np.array): Mean sky background as a scalar or image, depending on the method parameter. std (float or np.array): Uncertainty on the sky background estimate as a scalar or image, depending on the method parameter. """ # Handle str type data if isinstance(data, str): file = data if path is not None: file = os.path.join(path, file) data = fits.getdata(filename=file) # Create source mask if mask_sources: if data.ndim == 3: mask = make_source_mask(np.sum(data, axis=0), nsigma=2, npixels=5, dilate_size=11) mask = np.repeat(np.expand_dims(mask, 0), data.shape[0], axis=0) else: mask = make_source_mask(data, nsigma=2, npixels=5, dilate_size=11) else: mask = None # Derive statistics if method == 'scalar': mean, _, std = sigma_clipped_stats(data, sigma=3.0, mask=mask) else: raise NotImplementedError( f"Method {method} is not implemented yet for sky background estimation!" ) return mean, std
def create_flat(flat_list,fil,red_path,mbias=None,log=None): log.info('Processing files for filter: '+fil) log.info(str(len(flat_list))+' files found.') flats = [] masks = [] flat_scale = [] for flat in flat_list: log.info('Loading file: '+flat) raw = CCDData.read(flat,unit=u.adu) red = ccdproc.ccd_process(raw, gain=raw.header['SYSGAIN']*u.electron/u.adu, readnoise=rdnoise(raw.header)*u.electron) log.info('Exposure time of image is '+str(red.header['TRUITIME']*red.header['COADDONE'])) mask = make_source_mask(red, nsigma=3, npixels=5) bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80) masked = np.array(red) masked[mask] = bkg.background[mask] log.info('Median flat level: '+str(np.median(masked))) norm = 1/np.median(masked[224:1824,224:1824]) log.info('Flat normalization: '+str(norm)) flat_scale.append(norm) flats.append(CCDData(masked,meta=red.header,unit=u.electron)) mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True) log.info('Created master flat for filter: '+fil) mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True) log.info('Master flat written to mflat_'+fil+'.fits') return
def findStars(image): # In order to use the MAD as a consistent estimator for the estimation # of the standard deviation σ, one takes σ = k * MAD where k is a constant # scale factor, which depends on the distribution. For normally distributed # data k is taken to be k = 1.48[26] bkg_sigma = 1.48 * mad(image) t = 5 * bkg_sigma daofind = DAOStarFinder(fwhm=3.0, threshold=t) stars = daofind(image) #stars['signal'] = stars['flux'] * t # data = image mask = make_source_mask(data, snr=2, npixels=5, dilate_size=11) mean, median, std = sigma_clipped_stats(data, sigma=3.0, mask=mask) madstd = mad_std(data) snrs = [] for peak in stars['peak']: snrs.append(peak / madstd / 7.4) stars['snr'] = snrs print((mean, median, std, bkg_sigma, t, madstd)) # #print stars return stars
def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None): masks = [] processed = [] for sci in sci_list: log.info('Loading file: '+sci) log.info('Applying gain correction, dark subtraction, overscan correction, flat correction, and trimming image.') raw = CCDData.read(sci,hdu=1,unit=u.adu) red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron) log.info('Exposure time of science image is '+str(red.header['EXPTIME'])) log.info('Loading correct master dark') mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron) red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second)#dark_exposure=1*u.second, data_exposure=red.header['EXPTIME']*u.second, scale=True) red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3)) red = ccdproc.flat_correct(red, mflat) processed_data = ccdproc.ccd_process(red, trim=raw.header['DATASEC']) log.info('File proccessed and trimmed.') log.info('Cleaning cosmic rays and creating mask.') mask = make_source_mask(processed_data, nsigma=3, npixels=5) masks.append(mask) clean, com_mask = create_mask.create_mask(sci,processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(red.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log) processed_data.data = clean log.info('Calculating 2D background.') bkg = Background2D(processed_data, (510, 510), filter_size=(9, 9),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80) log.info('Median background: '+str(np.median(bkg.background))) fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits'),bkg.background,overwrite=True) final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['EXPTIME']*u.second,propagate_uncertainties=True,handle_meta='first_found') log.info('Background subtracted and image divided by exposure time.') final.write(sci.replace('/raw/','/red/'),overwrite=True) processed.append(final) return processed, masks
def create_flat(flat_list,fil,red_path,mbias=None,log=None): log.info('Processing files for filter: '+fil) log.info(str(len(flat_list))+' files found.') flats = [] flat_scale = [] for flat in flat_list: log.info('Loading file: '+flat) raw = CCDData.read(flat,hdu=1,unit=u.adu) red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron) log.info('Exposure time of image is '+str(red.header['EXPTIME'])) log.info('Loading correct master dark') mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron) red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second) red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3)) mask = make_source_mask(red, nsigma=3, npixels=5) bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80) masked = np.array(red) masked[mask] = bkg.background[mask] log.info('Median flat level: '+str(np.median(masked))) norm = 1/np.median(masked[500:1500,500:1500]) log.info('Flat normalization: '+str(norm)) flat_scale.append(norm) flats.append(CCDData(masked,meta=red.header,unit=u.electron)) mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True) log.info('Created master flat for filter: '+fil) mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True) log.info('Master flat written to mflat_'+fil+'.fits') return
def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None): masks = [] processed = [] for sci in sci_list: log.info('Loading file: '+sci) log.info('Applying gain correction and flat correction.') with fits.open(sci) as hdr: header = hdr[0].header data = hdr[0].data data[np.isnan(data)] = np.nanmedian(data) raw = CCDData(data,meta=header,unit=u.adu) red = ccdproc.ccd_process(raw, gain=raw.header['SYSGAIN']*u.electron/u.adu, readnoise=rdnoise(raw.header)*u.electron) log.info('Exposure time of science image is '+str(red.header['TRUITIME']*red.header['COADDONE'])) flat = np.array(ccdproc.flat_correct(red, mflat)) flat[np.isnan(flat)] = np.nanmedian(flat) processed_data = CCDData(flat,unit=u.electron,header=red.header,wcs=red.wcs) log.info('File proccessed.') log.info('Cleaning cosmic rays and creating mask.') mask = make_source_mask(processed_data, nsigma=3, npixels=5) masks.append(mask) clean, com_mask = create_mask.create_mask(sci.replace('.gz',''),processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(raw.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log) processed_data.data = clean log.info('Calculating 2D background.') bkg = Background2D(processed_data, (128, 128), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80) log.info('Median background: '+str(np.median(bkg.background))) fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits').replace('.gz',''),bkg.background,overwrite=True) final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['TRUITIME']*red.header['COADDONE']*u.second,propagate_uncertainties=True,handle_meta='first_found') log.info('Background subtracted and image divided by exposure time.') final.write(sci.replace('/raw/','/red/').replace('.gz',''),overwrite=True) processed.append(final) return processed, masks
def make_side_msk(img, snr=2.5, npixels=5, dilate_size=5, ct_QSO_mask=False): from photutils import make_source_mask mask_o = make_source_mask(img, snr=snr, npixels=npixels, dilate_size=dilate_size) exp_center = np.zeros_like(mask_o) cent = len(exp_center) / 2 exp_center[cent, cent] = 1 count = [-1, 0, 1] i = 0 exp_center_comp = np.zeros_like(mask_o) while exp_center_comp.sum() != exp_center.sum(): exp_center_comp = copy.deepcopy(exp_center) for i in range(exp_center.sum()): x, y = np.where(exp_center == 1)[0][i], np.where( exp_center == 1)[1][i] for j in count: for k in count: if exp_center[x + j, y + k] == 0 and mask_o[x + j, y + k] == 1: exp_center[x + j, y + k] = 1 if ct_QSO_mask == False: mask_ = mask_o * ~exp_center return ~mask_ elif ct_QSO_mask == True: return exp_center
def __init__(self, gid, field, flter, nearby_fitting_region, nearby_gids = None): if not (field == 'S' or field == 'N'): raise Exception('Field must be S or N.') if not type(flter) == str: raise Exception('Filter must be string.') self.gid = gid self.field = field self.flter = flter self.nearby_fitting_region = nearby_fitting_region if self.flter == '850': if self.field == 'N': self.fitsfile = 'GOODS-N_acsz_sci_sub.fits' self.fullfield = 'North' if self.field == 'S': self.fitsfile = 'goodss_3dhst.v4.0.F850LP_orig_sci.fits' self.fullfield = 'South' else: if self.field =='S': self.fitsfile = 'goodss_3dhst.v4.0.F'+flter+'W_orig_sci.fits' self.fullfield = 'South' if self.field =='N': self.fitsfile = 'goodsn_3dhst.v4.0.F'+flter+'W_orig_sci.fits' self.fullfield = 'North' ### CATALOG ### if (field == 'S') or (field == 'South'): catalog = '/Users/rosaliaobrien/research/website/catalogs/goodss_3dhst.v4.4.cat' if (field == 'N') or (field == 'North'): catalog = '/Users/rosaliaobrien/research/website/catalogs/goodsn_3dhst.v4.4.cat' self.cat = ascii.read(catalog) ### GET SKY ESTIMATE ### if not nearby_gids == None: xmin, xmax, ymin, ymax = self.get_boundaries(nearby_gids) path = '/Users/rosaliaobrien/research/GALFIT_CLEAR/running_GALFIT/'+self.fullfield+'/'+self.flter+'/' skydata = fits.open(path+self.fitsfile)[0].data xmid = (xmin+xmax)/2 ymid = (ymin+ymax)/2 cutout = Cutout2D(skydata, (xmid, ymid), (400, 400), copy=True, mode='partial') masked_cutout = make_source_mask(cutout.data, snr = 1.5, npixels=10, dilate_size=11) cutout.data[masked_cutout] = np.nan self.scidata = cutout.data sigma_clip = SigmaClip(sigma=3) bkg = SExtractorBackground(sigma_clip) rms = MADStdBackgroundRMS(sigma_clip) self.sky_value = bkg.calc_background(cutout.data) self.rms_value = rms.calc_background_rms(cutout.data)
def estimate_background(im): # im must be BW if len(im.shape) == 3: im=im.squeeze() mask = make_source_mask(im, snr=2, npixels=5, dilate_size=11) mean, median, std = sigma_clipped_stats(im, sigma=3.0, mask=mask) flat_background = np.ones_like(im) * mean fake_background = np.random.normal(mean, std, size=im.shape) return flat_background, fake_background, std
def reduce_object(self, object_frame, flatfield, master_dark, bkg_method="mesh"): print("Opening object frame", object_frame) obj_frame = fits.open(object_frame) obj_frame_data = obj_frame[0].data obj_frame_header = obj_frame[0].header # Subtract master dark print("Opening master dark frame") master_dark = fits.open(master_dark) master_dark_data = master_dark[0].data master_dark_header = master_dark[0].header print("Subtracting master dark from object") reduced_obj_frame_data = obj_frame_data - master_dark_data # Flatfield correct print("Opening flatfield frame") flatfield = fits.open(flatfield) flatfield_data = flatfield[0].data flatfield_header = flatfield[0].header print("Dividing object by flatfield") reduced_obj_frame_data /= flatfield_data # Remove cosmic rays #sepmed = False #cleantype = "medmask" #print("Detecting cosmic rays") #artifact_mask, reduced_obj_frame_data = astroscrappy.detect_cosmics(reduced_obj_frame_data, sepmed=sepmed, cleantype=cleantype) # Subtract background if bkg_method == "mesh": nsigma = 2.0 npixels = 5 dilate_size = 31 print("Creating mask") mask = make_source_mask(reduced_obj_frame_data, nsigma=nsigma, npixels=npixels, dilate_size=dilate_size) print("Subtracting background mesh") background = sep.Background(reduced_obj_frame_data, mask=mask) reduced_obj_frame_data -= background elif bkg_method == "sigma": print("Subtracting sigma-clipped background") mean, median, std = sigma_clipped_stats(reduced_obj_frame_data, sigma=bkg_sigma) reduced_obj_frame_data -= median print("Converting reduced array to FITS") reduced_hdu = fits.PrimaryHDU(reduced_obj_frame_data, header=obj_frame_header) return reduced_hdu
def set_mask( self, *args, method: str = "source", inner: float = None, outer: float = None, position: SkyCoord = None, wcs: WCS = None, **kwargs, ): """ Set a mask to the data image :param method: the method to set mask, "source", or "annulus"defaults to "source" :type method: str, optional :param inner: the inner radius of the annulus, defaults to None :type inner: float, optional :param outer: the outer radius of the annulus, defaults to None :type outer: float, optional :param position: the center of the annulus, defaults to None :type position: SkyCoord, optional :param wcs: the wcs used to convert mask to pixel, defaults to None :type wcs: WCS, optional :raises ValueError: "annulus" must have position and inner/outer radius """ if method == "source": self._mask = make_source_mask(self.data, *args, **kwargs) elif method == "annulus": if not position or not inner or not outer: raise ValueError( "Use annulus mask but missing position or inner/outer radius" ) else: if not wcs: logging.warning( "Not assign the wcs, try the wcs from the header") wcs = self.header mask = (SkyCircularAnnulus( position, inner * u.arcsec, outer * u.arcsec).to_pixel(wcs).to_mask("center")) self._mask = self.ImageMask(shape="annulus", position=position, rin=inner, rout=outer, mask=mask) # self.mask = self.aper.to_mask("center") else: raise ValueError(f"No supported method: {method}")
def Background_DilatedSources(IMG, results, options): """ Compute a global background value for an image. Performed by identifying pixels which are beyond 3 sigma above the average signal and masking them, also further masking a boarder of 20 pixels around the initial masked pixels. Returns a dictionary of parameters describing the background level. """ # Mask main body of image so only outer 1/5th is used # for background calculation. if 'mask' in results and not results['mask'] is None: mask = results['mask'] else: mask = np.zeros(IMG.shape) mask[int(IMG.shape[0] / 5.):int(4. * IMG.shape[0] / 5.), int(IMG.shape[1] / 5.):int(4. * IMG.shape[1] / 5.)] = 1 # Run photutils source mask to remove pixels with sources # such as stars and galaxies, including a boarder # around each source. if not ('ap_set_background' in options and 'ap_set_background_noise' in options): source_mask = make_source_mask( IMG, nsigma=3, npixels=int(1. / options['ap_pixscale']), dilate_size=40, filter_fwhm=1. / options['ap_pixscale'], filter_size=int(3. / options['ap_pixscale']), sigclip_iters=5) mask = np.logical_or(mask, source_mask) # Return statistics from background sky bkgrnd = options[ 'ap_set_background'] if 'ap_set_background' in options else np.median( IMG[np.logical_not(mask)]) noise = options[ 'ap_set_background_noise'] if 'ap_set_background_noise' in options else iqr( IMG[np.logical_not(mask)], rng=[16, 84]) / 2 uncertainty = noise / np.sqrt(np.sum(np.logical_not(mask))) return IMG, { 'background': bkgrnd, 'background noise': noise, 'background uncertainty': uncertainty, 'auxfile background': 'background: %.5e +- %.2e flux/pix, noise: %.5e flux/pix' % (bkgrnd, uncertainty, noise) }
def sub_bkg(self, img): sigma_clip = SigmaClip(sigma=3.) bkg_estimator = SExtractorBackground() mask_0 = make_source_mask(img, nsigma=3, npixels=5, dilate_size=11) mask_1 = (np.isnan(img)) mask = mask_0 + mask_1 bkg = Background2D(img, (5, 5), filter_size=(3, 3), sigma_clip=sigma_clip, bkg_estimator=bkg_estimator, mask=mask) back = bkg.background * ~mask_1 return img - back
def _perform(self): """ Returns an Argument() with the parameters that depends on this operation. """ self.log.info(f"Running {self.__class__.__name__} action") snr = 5 npixels = 5 self.action.args.source_mask = [None] * len( self.action.args.kd.pixeldata) for i, pd in enumerate(self.action.args.kd.pixeldata): source_mask = photutils.make_source_mask(pd, snr, npixels) self.action.args.source_mask[i] = source_mask return self.action.args
def est_bkg(image, pltshow=1): print("Estimating the background light......") img = image # check the back grounp sigma_clip = SigmaClip(sigma=3., iters=10) bkg_estimator = SExtractorBackground() mask_0 = make_source_mask(img, snr=2, npixels=5, dilate_size=11) mask_1 = (np.isnan(img)) mask = mask_0 + mask_1 bkg = Background2D(img, (50, 50), filter_size=(3, 3), sigma_clip=sigma_clip, bkg_estimator=bkg_estimator, mask=mask) from matplotlib.colors import LogNorm fig = plt.figure(figsize=(15, 15)) ax = fig.add_subplot(1, 1, 1) ax.imshow(img, norm=LogNorm(), origin='lower') #bkg.plot_meshes(outlines=True, color='#1f77b4') ax.xaxis.set_visible(False) ax.yaxis.set_visible(False) if pltshow == 0: plt.close() else: plt.show() fig = plt.figure(figsize=(15, 15)) ax = fig.add_subplot(1, 1, 1) ax.imshow(mask, origin='lower') #bkg.plot_meshes(outlines=True, color='#1f77b4') ax.xaxis.set_visible(False) ax.yaxis.set_visible(False) if pltshow == 0: plt.close() else: plt.show() back = bkg.background * ~mask_1 fig = plt.figure(figsize=(15, 15)) ax = fig.add_subplot(1, 1, 1) ax.imshow(back, origin='lower', cmap='Greys_r') ax.xaxis.set_visible(False) ax.yaxis.set_visible(False) if pltshow == 0: plt.close() else: plt.show() # img -= back # pyfits.PrimaryHDU(img).writeto('sub_coadd.fits',overwrite=True) return back
def bkg_subs(image, box, snr=2, npixels=5, dilate_size=15): """Substrackt the background of an image. Sources are idenfified first and masked, then the background is estimated as the (sigma-clipped) median. """ # get data from file data = fits.getdata(image) # create mask nan_mask = np.isnan(data) source_mask = make_source_mask(data, snr=snr, npixels=npixels, dilate_size=dilate_size) mask = np.logical_or(nan_mask, source_mask) bkg_estimator = MedianBackground() bkg = Background2D(data, box, filter_size=1, bkg_estimator=bkg_estimator, mask=mask) return data - bkg.background
def mask_sources(self): """ This function ... :return: """ # Inform the user log.info("Masking sources ...") # Create sources mask mask = make_source_mask(self.sources_with_noise.data, snr=2, npixels=5, dilate_size=11, mask=self.rotation_mask) self.sources_mask = Mask(mask) # Plot if self.config.plot: plotting.plot_mask(self.sources_mask, title="sources mask")
def _perform(self): """ Returns an Argument() with the parameters that depend on this operation. """ self.log.info(f"Running {self.__class__.__name__} action") snr = self.cfg['Extract'].getfloat('source_mask_snr', 5) self.log.debug(f" Using snr = {snr}") npixels = self.cfg['Extract'].getfloat('source_mask_npixels', 5) self.log.debug(f" Using npixels = {npixels}") source_mask = photutils.make_source_mask(self.action.args.ccddata, snr, npixels) self.action.args.source_mask = source_mask return self.action.args
def sub_bkg(img, plot=True): from astropy.stats import SigmaClip from photutils import Background2D, SExtractorBackground sigma_clip = SigmaClip(sigma=3., iters=10) bkg_estimator = SExtractorBackground() from photutils import make_source_mask mask_0 = make_source_mask(img, snr=2, npixels=5, dilate_size=11) mask_1 = (np.isnan(img)) mask = mask_0 + mask_1 bkg = Background2D(img, (50, 50), filter_size=(3, 3), sigma_clip=sigma_clip, bkg_estimator=bkg_estimator, mask=mask) from matplotlib.colors import LogNorm fig = plt.figure(figsize=(15, 15)) ax = fig.add_subplot(1, 1, 1) ax.imshow(img, norm=LogNorm(), origin='lower') #bkg.plot_meshes(outlines=True, color='#1f77b4') ax.xaxis.set_visible(False) ax.yaxis.set_visible(False) if plot: plt.show() else: plt.close() fig = plt.figure(figsize=(15, 15)) ax = fig.add_subplot(1, 1, 1) ax.imshow(mask, origin='lower') #bkg.plot_meshes(outlines=True, color='#1f77b4') ax.xaxis.set_visible(False) ax.yaxis.set_visible(False) if plot: plt.show() else: plt.close() back = bkg.background * ~mask_1 fig = plt.figure(figsize=(15, 15)) ax = fig.add_subplot(1, 1, 1) ax.imshow(back, origin='lower', cmap='Greys_r') ax.xaxis.set_visible(False) ax.yaxis.set_visible(False) if plot: plt.show() else: plt.close() return img - back, back
def starphot(imdata, position, radius, r_in, r_out): """ sources: http://photutils.readthedocs.io/en/stable/ photutils/aperture.html ARGS: imdata: Numpy array containing the star. position: [x,y] coordinates where x corresponds to the second index of imdata radius: Radius of the circular aperture used to compute the flux of the star. r_in: Inner radius of the annulus used to compute the background mean. r_out: Outer radius of the annulus used to compute the background mean. Returns [flux, background variance, background mean] """ try: statmask = photutils.make_source_mask(imdata, snr=5, npixels=5, dilate_size=10) except TypeError: return None bkg_annulus = photutils.CircularAnnulus(position, r_in, r_out) bkg_phot_table = photutils.aperture_photometry(imdata, bkg_annulus, method='subpixel', mask=statmask) bkg_mean_per_pixel = bkg_phot_table['aperture_sum'] / bkg_annulus.area() src_aperture = photutils.CircularAperture(position, radius) src_phot_table = photutils.aperture_photometry(imdata, src_aperture, method='subpixel') signal = src_phot_table['aperture_sum'] - bkg_mean_per_pixel*\ src_aperture.area() #noise_squared = signal + bkg_mean_per_pixel*src_aperture.area() mean, median, std = sigma_clipped_stats(imdata, sigma=3.0, iters=5, mask=statmask) noise_squared = std**2 return float(str(signal.data[0])), noise_squared,\ float(str(bkg_mean_per_pixel.data[0]))
def subtract_sky_ccd_mask(ccd, skysub='SKYSUB', skyval='SKYVAL', snr=3, npixels=5, dilate_size=11, sigma=3, check_skysub=True, max_val=10000): if check_skysub and skysub is not None and skysub in ccd.header and ccd.header[skysub]: return ccd if not check_skysub and skysub is not None and skysub in ccd.header and ccd.header[skysub]: print ('WARNING: Image already sky subtracted! Subtracting AGAIN the sky! [set check_skysub = True for checking]') mask = make_source_mask(ccd.data, snr=sigma, npixels=npixels, dilate_size=dilate_size) mean, median, std = sigma_clipped_stats(ccd.data, sigma=sigma, mask=mask) sky = median if max_val is not None and sky >= max_val: if 'FILENAME' in ccd.header: print ('WARNING: File "%s" has high sky level (%s >= %s). Correction NOT applied' % (ccd.header['FILENAME'], sky, max_val)) else: print ('WARNING: File has high sky level (%s >= %s). Correction NOT applied' % (sky, max_val)) return ccd ccd.data -= sky ccd.header[skysub] = True ccd.header[skyval] = sky ccd.header['SKYTYPE'] = '1D' return ccd
def bkgstd(im_data, mask): """Estimate the RMS standard deviation of the background in the `im_data` image. Arguments --------- im_data : array_like Image data mask : array_like Mask to apply to image Returns ------- np.ndarray RMS standard deviation of the background image """ # use crude image segmentation to find sources above SNR=3, build a # source mask, and estimate the background RMS source_mask = make_source_mask(im_data, snr=3, npixels=5, dilate_size=15, mask=mask) # combine the bad pixel mask and source mask rough_mask = np.logical_or(mask, source_mask) # estimate the background standard deviation try: sigma_clip = SigmaClip(sigma=3, maxiters=5) # sigma clipping except TypeError: # in old astropy, "maxiters" was "iters" sigma_clip = SigmaClip(sigma=3, iters=5) try: bkg = Background2D(im_data, (50,50), filter_size=(5,5), sigma_clip=sigma_clip, bkg_estimator=MedianBackground(), mask=rough_mask) except ValueError: e = sys.exc_info() print("\nWhile attempting background estimation on the "+ "science image, the following error was raised: "+ f"\n{str(e[0])}\n{str(e[1])}\n--> exiting.") return return bkg.background_rms
def detect_sources(image): ''' By Anna Marini Extract the light sources from the image ''' # threshold = detect_threshold(image, nsigma=2.) # sigma = 3.0 * gaussian_fwhm_to_sigma # FWHM = 3. # kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3) # kernel.normalize() if isinstance(image, str): image = get_fits_data(image) mask = make_source_mask(image, nsigma=2, npixels=5, dilate_size=11) mean, median, std = sigma_clipped_stats(image, sigma=3, mask=mask) daofind = DAOStarFinder(fwhm=3.0, threshold=5. * std) sources = daofind(image - median) # Pixel coordinates of the sources x = np.array(sources['xcentroid']) y = np.array(sources['ycentroid']) return x, y
def background_median(data, sigma=3, npixels=5, maxiters=20, **kwargs): """Calculate background median for data. Calculates the background median for an image by iteratively sigma clipping. It first creates a source mask using segmentation and binary dilation (see https://photutils.readthedocs.io/en/stable/api/photutils.make_source_mask.html) for more details, before iteratively calculating the sigma-clipped median of the data. Args: data (str or numpy.ndarray or astropy.io.fits.PrimaryHDU): Data to calculate the background median for. If provided as a string, this is interpreted as a filename for a .fits HDU. sigma (float, optional): Level to perform sigma-clipping to. Defaults to 3. npixels (int, optional): Number of connected pixels greater than sigma to consider for a pixel to be part of a source when masking. Defaults to 5. maxiters (int,optional): The maximum number of sigma-clipping iterations to perform. Defaults to 20. Returns: float: Background median of the data. """ if isinstance(data, str): hdu = fits.open(data)[0] data = hdu.data elif isinstance(data, fits.PrimaryHDU): data = data.data mask = make_source_mask(data, snr=sigma, npixels=npixels, **kwargs) _, median, _ = sigma_clipped_stats(data, mask=mask, sigma=sigma, maxiters=maxiters, **kwargs) return median
def getstats(img, ff): gd = np.where(img != 0) print('there are ', len(gd), ' elements') arr = img[gd] arr = sorted(arr) n = len(arr) print('array is ', n, ' elements') i = round(ff * n) vmax = arr[i] print(ff, ' signal range value is ', vmax) print('making mask') mask = make_source_mask(img, snr=2, npixels=5, dilate_size=11) print('calculating stats') vmean, vmedian, vstd = sigma_clipped_stats(img, sigma=3.0, mask=mask, mask_value=0.) print('mean: ', vmean) print('median: ', vmedian) print('sigma: ', vstd) return vmean, vmedian, vstd, vmax
def background_subtract(im_data): """calculates background using a mask routine from photutils. Parameters ---------- im_data : numpy array requires an image data array returns ------ output_im: numpy array The background subtracted data. mask: numpy array bool The mask used to shield the object std: float The standard deviation on the background """ # import numpy as np # from astropy.io import fits # store the data from the HDU argument # im_data = HDU.data # Generate mask from photutils import make_source_mask from astropy.stats import sigma_clipped_stats mask = make_source_mask(im_data, snr=2, npixels=5, dilate_size=11) # calculate bias using mean # clipped stats are used, just in case mean, median, std = sigma_clipped_stats(im_data, sigma=3.0, mask=mask) print('Background mean: ' + str(mean)) print('Background median: ' + str(median)) print('Background standerd deviation: ' + str(std)) output_im = im_data - mean return output_im, mask, std
def cc_spec(image,width,p_instr,theta_instr): # global make_source_mask #crop the images with separate o and e ray sizex is the cropped half xcoordinate sizey1,sizey2 the boundary in y direction o_ray=[] ; e_ray=[]; noise=[]; ny=240 for img in image: img=rebin(img[:,20:280],[240,130]) #print 'size of the image', img.shape cen_o=int(round(101.0/(240/ny))) cen_e=int(round(135/(240/ny))) o_ray.append(img[(cen_o-width):(cen_o+width),:]) e_ray.append(img[(cen_e-width):(cen_e+width),:]) mask = make_source_mask(img, snr=1.5, npixels=10, dilate_size=10) maskr= np.invert(mask) mean, median, std = sigma_clipped_stats(img, sigma=2.3, iters=8, mask=mask) noise.append(std) xc=[];yc=[];crop_image_o=[];crop_image_e=[] I_1=np.mean(o_ray[0]+e_ray[0],axis=0); Q_1=np.mean(o_ray[0]-e_ray[0],axis=0) I_2=np.mean(o_ray[1]+e_ray[1],axis=0); Q_2=np.mean(o_ray[1]-e_ray[1],axis=0) I_3=np.mean(o_ray[2]+e_ray[2],axis=0); U_1=np.mean(o_ray[2]-e_ray[2],axis=0) I_4=np.mean(o_ray[3]+e_ray[3],axis=0); U_2=np.mean(o_ray[3]-e_ray[3],axis=0) Q=(Q_1-Q_2)/2.0 U=(U_1-U_2)/2.0 I=(I_1+I_2+I_3+I_4)/4.0 n=I_1.shape ###Instrument correction U_c=U-p_instr*np.cos(2.0*theta_instr*np.pi/180.0)*I Q_c=Q-p_instr*np.sin(2.0*theta_instr*np.pi/180.0)*I factor=((2.0*width+1.0)**0.5) err_Q=(noise[0]+noise[1])/(2.0*factor)+np.zeros((n[0])) #Q_1 noise =sqrt(2)*noise[0] Q noise=Q_1 noise/sqrt(2) err_U=(noise[2]+noise[3])/(2.0*factor)+np.zeros((n[0])) #same err_I=np.mean(noise)/((2.0)**0.5*factor)+np.zeros((n[0])) return Q, U, I, err_Q, err_U, err_I
def extract_source(images): # Calculate image noise stack = np.sum(images, axis = 0) mask = make_source_mask(stack, snr=2, npixels=5, dilate_size=11) mean, median, std = sigma_clipped_stats(stack, sigma=3.0, mask=mask) # Pull out all sources segm = detect_sources(stack, 2*std, npixels = 5) # Screen for our source cx = int(stack.shape[0]/2) cy = int(stack.shape[1]/2) center_pixel = segm.data[cx,cy] segm.data[np.where(segm.data != center_pixel)] = 0 region = np.zeros(stack.shape, dtype = bool) region[segm.data == center_pixel] = True cutout = np.where(region) pad = 3 xmin,xmax = min(cutout[0]) - pad, max(cutout[0]) + pad ymin,ymax = min(cutout[1]) - pad, max(cutout[1]) + pad rect_region = [xmin,xmax,ymin,ymax] dof2 = (xmax - xmin)*(ymax - ymin) dof = len(stack[region]) fig = plt.figure() ax1 = plt.subplot(121) ax1.imshow(stack[xmin:xmax,ymin:ymax], interpolation = 'none', cmap = 'gray') ax2 = plt.subplot(122) ax2.imshow(segm.data[xmin:xmax,ymin:ymax], interpolation = 'none') plt.show() return region, rect_region, std, dof
def bkgsub(im_file, mask_file=None, bkg_box=(5,5), bkg_filt=(5,5), plot_bkg=False, plot_bkgsubbed=False, scale_bkg="linear", scale_bkgsubbed="linear", write=True, output=None): """Subtract the background from an image. Arguments --------- im_file : str Filename for image of interest mask_file : str, optional Filename for bad pixels mask (default None) bkg_box : tuple, optional Sizes of box along each axis in which background is estimated (default (5,5)) bkg_filt : tuple, optional Size of the median filter pre-applied to the background before background estimation (default (5,5) where (1,1) --> no filtering) plot_bkg : bool, optional Whether to plot BACKGROUND image (default False) plot_bkgsubbed : bool, optional Whether to plot background-SUBTRACTED image (default False) scale_bkg : {"linear", "log", "asinh"}, optional Scale to apply to BACKGROUND image plot (default "linear") scale_bkgsubbed : {"linear", "log", "asinh"}, optional Scale to apply to background-SUBTRACTED image plot (default "linear") write : bool, optional Whether to write the background-SUBTRACTED image to a fits file (default True) output : str, optional Name for output fits file (default `im_file.replace(".fits", "_bkgsub.fits")`) Returns ------- astropy.io.fits.PrimaryHDU New HDU (image + header) with the image background-subtracted Notes ----- **TO-DO:** - Allow naming of output plots """ ## load in the image data image_data = fits.getdata(im_file) ## source detection and building masks if mask_file: # load a bad pixel mask if one is provided bp_mask = fits.getdata(mask_file).astype(bool) zeromask = image_data == 0 # mask out pixels equal to 0 nansmask = np.isnan(image_data) # mask out nans bp_mask = np.logical_or.reduce((bp_mask, zeromask, nansmask)) #bp_mask = np.logical_or(bp_mask, zeromask) # old way: chained #bp_mask = np.logical_or(bp_mask, nansmask) else: zeromask = image_data == 0 # mask out pixels equal to 0 nansmask = np.isnan(image_data) # mask out nans bp_mask = np.logical_or(nansmask, zeromask) # make a crude source mask source_mask = make_source_mask(image_data, snr=3, npixels=5, dilate_size=15, mask=bp_mask) # combine the bad pixel mask and source mask for background subtraction # make the final mask mask = np.logical_or(bp_mask, source_mask) ## estimate the background try: sigma_clip = SigmaClip(sigma=3, maxiters=5) # sigma clipping except TypeError: # in old astropy, "maxiters" was "iters" sigma_clip = SigmaClip(sigma=3, iters=5) bkg = Background2D(image_data, box_size=bkg_box, filter_size=bkg_filt, sigma_clip=sigma_clip, bkg_estimator=MedianBackground(), mask=mask) bkg_img = bkg.background bkgstd = bkg.background_rms_median ## subtract the background bkgsub_img = image_data - bkg_img bkgstd = bkg.background_rms_median # save this quantity to write to header ## finally, mask bad pixels # all bad pix are then set to 0 for consistency bkg_img_masked = np.ma.masked_where(bp_mask, bkg_img) bkg_img = np.ma.filled(bkg_img_masked, 0) bkgsub_img_masked = np.ma.masked_where(bp_mask, bkgsub_img) bkgsub_img = np.ma.filled(bkgsub_img_masked, 0) ## plotting (optional) if plot_bkg: # plot the background, if desired output_bkg = f"background_{scale_bkg}.png" __plot_bkg(im_header=fits.getheader(im_file), bkg_img_masked=bkg_img_masked, scale=scale_bkg, output=output_bkg) if plot_bkgsubbed: # plot the background-subtracted image, if desired output_bkgsubbed = f"background_subbed_{scale_bkgsubbed}.png" __plot_bkgsubbed(im_header=fits.getheader(im_file), bkgsub_img_masked=bkgsub_img_masked, scale=scale_bkgsubbed, output=output_bkgsubbed) ## building the final HDU hdr = fits.getheader(im_file) hdr["BKGSTD"] = bkgstd # useful header for later bkgsub_hdu = fits.PrimaryHDU(data=bkgsub_img, header=hdr) if write: # if we want to write the background-subtracted fits file if not(output): # if no output name given, set default output = im_file.replace(".fits", "_bkgsub.fits") bkgsub_hdu.writeto(output, overwrite=True, output_verify="ignore") return bkgsub_hdu
elif cut < 0: PSF = PSF[:, -cut:cut] PSF /= PSF.sum() if PSF.shape[0] != PSF.shape[1]: raise ValueError("PSF shape is not a square.") PSF_list.append(PSF) zp_list.append(zp) #============================================================================== # quickly evaluate the background rms #============================================================================== background_rms_list = [] for i in range(len(band_seq)): if i in run_list: mask = make_source_mask(QSO_im_list[i], snr=3, npixels=5, dilate_size=11) background_rms_list.append(np.std(QSO_im_list[i] * (1 - mask * 1))) else: background_rms_list.append([]) fit_frame_size = 81 #============================================================================== # Start set up for fitting: #============================================================================== psf_l, QSO_img_l, QSO_std_l = [], [], [] for k in range(len(band_seq)): if k in run_list: ct = int((len(QSO_im_list[k]) - fit_frame_size) / 2) # If want to cut to 61, QSO_im[ct:-ct,ct:-ct]
bt = np.int(np.round(0.5*cy)) tp = bt + 1000 tmpImg = tmpImg[bt:tp, lf:rt] # Grab the on-off target value for this image thisAB = subGroup['AB'][iFile] # Place the image in a list and store required background values if thisAB == 'B': # Place B images in the BimgList BimgList.append(tmpImg) # Place the median value of this off-target image in list mask = make_source_mask( tmpImg.data, snr=2, npixels=5, dilate_size=11 ) mean, median, std = sigma_clipped_stats( tmpImg.data, sigma=3.0, mask=mask ) BbkgList.append(median) # Place the time of this image in a list of time values BdatetimeList.append(tmpImg.julianDate) if thisAB == 'A': # Read in any associated masks and store them. maskFile = os.path.join(maskDir, os.path.basename(filename)) # If there is a mask for this file, then apply it! if os.path.isfile(maskFile):