def cutouts(image, stars, size=15): """Custom version to extract stars cutouts Parameters ---------- Parameters ---------- image: np.ndarray or path stars: np.ndarray stars positions with shape (n,2) size: int size of the cuts around stars (in pixels), by default 15 Returns ------- np.ndarray of shape (size, size) """ if isinstance(image, str): image = fits.getdata(image) stars_in = np.logical_and( np.all(stars < np.array(image.shape) - size - 2, axis=1), np.all(stars > np.ones(2) * size + 2, axis=1)) stars = stars[stars_in] # with warnings.catch_warnings(): warnings.simplefilter("ignore") stars_tbl = Table([stars[:, 0], stars[:, 1]], names=["x", "y"]) stars = extract_stars(NDData(data=image), stars_tbl, size=size) return np.argwhere(stars_in).flatten(), stars
def make_stars_guess( image: np.ndarray, star_finder: photutils.StarFinderBase, cutout_size: int = config.cutout_size) -> photutils.psf.EPSFStars: """ Given an image, extract stars as EPSFStars for psf fitting :param image: yes :param cutout_size: how big should the regions around each star used for fitting be? :param star_finder: which starfinder to use? :return: instance of exctracted EPSFStars """ # The idea here is to run a "greedy" starfinder that finds a lot more candidates than we need and then # to filter out the bright and isolated stars peaks_tbl = star_finder(image) peaks_tbl.rename_columns(['xcentroid', 'ycentroid'], ['x', 'y']) peaks_tbl = cut_edges(peaks_tbl, cutout_size, image.shape[0]) # TODO this gets medianed away with the image combine approach, so more star more good? # stars_tbl = cut_close_stars(peaks_tbl, cutoff_dist=3) stars_tbl = peaks_tbl image_no_background = image - np.median(image) stars = extract_stars(NDData(image_no_background), stars_tbl, size=cutout_size) return stars
def extract(self): epsf_builder = EPSFBuilder(oversampling=4, maxiters=3, progress_bar=True) for file in self.params.inFiles: logging.info("Extracting PSFs from file {}".format(file)) psf_file = PSFfile(file, self.params.tmpDir, frame_shape=(self.box_size, self.box_size)) frame_number = fits.getheader(file)['NAXIS3'] for frame_index in range(frame_number): print("\rExtracting PSF from frame {}/{}".format( frame_index + 1, frame_number), end='') with fits.open(file) as hdulist: frame = hdulist[0].data[frame_index] stars = extract_stars(NDData(data=frame), self.star_table, size=self.box_size) # Compute instantaneous PSF # epsf, fitted_stars = epsf_builder(stars) epsf = np.zeros(stars[0].data.shape) for star in stars: epsf += star.data psf_file.update_frame(frame_index, epsf) print('\r')
def _extractStars(self): self._findStars() self._starsTab = Table() self._starsTab['x'] = self._selectedStars['xcentroid'] self._starsTab['y'] = self._selectedStars['ycentroid'] self._starsCut = extract_stars(NDData(data=self._image), self._starsTab, self._size)
def extract_stars(self): # just in case sky was not properly subtracted mean_val, median_val, std_val = sigma_clipped_stats(self.data, sigma=2.) self.data -= median_val nddata = NDData(data=self.data) self.stars = extract_stars(nddata, self.stars_tbl, size=self.size) # check to make sure stars don't have zeros # Virgo 2017 pointing-4_R.coadd.fits is giving me trouble b/c a lot of stars have zeros keepflag = np.ones(len(self.stars), 'bool') for i, s in enumerate(self.stars): if len(np.where(s.data == 0)[0]) > 1: keepflag[i] = False self.keepflag2 = keepflag self.stars = extract_stars(nddata, self.stars_tbl[keepflag], size=self.size) #self.stars = self.stars[keepflag] if len(self.stars) < self.nstars: self.nstars = len(self.stars)
def cutouts(image, stars, size=15): """Custom version to extract stars cutouts Parameters ---------- Parameters ---------- image: np.ndarray or path stars: np.ndarray stars positions with shape (n,2) size: int size of the cuts around stars (in pixels), by default 15 Returns ------- np.ndarray of shape (size, size) """ if isinstance(image, str): image = fits.getdata(image) warnings.simplefilter("ignore") if np.shape(stars) > (1,2): stars_tbl = Table( [stars[:, 0], stars[:, 1], np.arange(len(stars))], names=["x", "y", "id"]) _stars = [None]*len(stars) stars = extract_stars(NDData(data=image), stars_tbl, size=size) idxs = np.array([s.id_label for s in stars]) for i, s in enumerate(stars): _stars[idxs[i]] = s return idxs, _stars else: stars_tbl = Table( data=np.array([stars[0][0], stars[0][1]]), names=["x", "y"]) stars = ([0], [extract_stars(NDData(data=image), stars_tbl, size=size)]) return stars
def construct_epsf(imdata, mean): peaks_tbl = find_peaks(imdata, threshold=20.0) peaks_tbl['peak_value'].info.format = '%.8g' for i in range(len(peaks_tbl)): print(i, peaks_tbl['x_peak'][i], peaks_tbl['y_peak'][i]) #rem_indA = [0,1,3,4,5,6,8,9,11,12,13,16,84,85,86,87,91,92,99,102] #rem_indB = [0,1,3,4,5,6,8,9,11,12,13,15,16,17,24,25,61,62,73,78,84,85,86,87,91,92,99,102] #rem_indC = [0,1,2,3,4,5,6,8,9,10,11,12,13,15,16,17,22,23,24,25,61,62,71,73,77,78,84,85,86,87,91,92,94,96,99,102] #rem_indVIS = [2,3,4,5,6,7,8,9,10,12,13,14] #rem_indD = [3,5] #rem_indVIS2 = [0,1,2,7] #rem_indE = [22,23,24,25,27,28,29] #rem_indF = [11,12,13,15,16,18,19,20,21,22,23,24,25,26] #rem_indG = [9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28] #rem_indH=[4,5,6,7,8,9,10,11,12,13] #rem_indI = [4,5,6,7,8,9,10,11,12,13,14,15,16,17,18] #rem_indJ = [0,1,2,3,4,5,7,8,9,10,11,12,13,14,15,16,22,25,27,28,29,30,33,34,36,37,38,39] #rem_indK = [0,1,2,3,4,5,6,7,8,9,10,11,13,14,15,23,24,25,26,29,30,31,33,34,35] #rem_indL = [4,5,6,7,8,10,11,12,13] #rem_indM = [5,6,7,8,9,10,11,13,14,15,16,18,27,29] #rem_indN = [4,5,6,7,8,9,10,11,13,14,15,16,17,19,20,21] #peaks_tbl.remove_rows([rem_indM]) plt.figure() plt.scatter(peaks_tbl['x_peak'], peaks_tbl['y_peak'], c='k') plt.imshow(imdata, vmin=-0.2, vmax=2., origin='lowerleft') plt.show() stars_tbl = Table() stars_tbl['x'] = peaks_tbl['x_peak'] stars_tbl['y'] = peaks_tbl['y_peak'] mean_val, median_val, std_val = sigma_clipped_stats(imdata, sigma=2.0) imdata -= median_val nddata = NDData(data=imdata) stars = extract_stars(nddata, stars_tbl, size=20) epsf_builder = EPSFBuilder(oversampling=4, maxiters=3, progress_bar=False) epsf, fitted_stars = epsf_builder(stars) hdu = fits.PrimaryHDU(epsf.data) hdul = fits.HDUList([hdu]) hdul.writeto('Serpens3/epsf.fits') return stars, epsf
def make_psf(data, catalog, show=False, boxsize=25.): catalog = catalog.copy() catalog['x'], catalog['y'] = data.wcs.all_world2pix(catalog['raMean'], catalog['decMean'], 0) bkg = np.nanmedian(data) nddata = NDData(data - bkg) stars = psf.extract_stars(nddata, catalog, size=boxsize) epsf_builder = EPSFBuilder(oversampling=1.) epsf, fitted_stars = epsf_builder(stars) if show: plt.figure() plt.imshow(epsf.data) plot_stars(fitted_stars) return epsf, fitted_stars
def make_psf(filepath, filename, filt_name, show, datamax, nstars, ncores, catalog_filepath): # TODO: add stage checking generally to stop errors t_start = time.time() if not nstars: nstars = 12 size = 25 print(f'Working on {filename}') full_filepath = filepath + filename data = getdata(full_filepath, 0) banzai = getdata(full_filepath, 1) metadata = load_pickle(filename) # TODO: be smarter about saving useful stars, add field option banzai_coords = _filter_banzai(banzai, datamax) stars_without_cosmics = _filter_cosmics(metadata, size, banzai_coords) all_stars = _filter_to_catalog(stars_without_cosmics, catalog_filepath, filt_name, filepath, filename) psf_stars = all_stars[:nstars] extracted_stars = extract_stars(NDData(data=data), psf_stars, size=size) print('\tBuilding psf . . .') epsf_builder = EPSFBuilder(oversampling=2, maxiters=3, progress_bar=False) epsf, fitted_stars = epsf_builder(extracted_stars) # can only pickle python built-ins? create_or_update_pickle(filename=filename, key='epsf', val=epsf.data.tolist()) create_or_update_pickle(filename=filename, key='psf_fitted_stars', val=np.array(all_stars).tolist()) #create_or_update_pickle(filename=filename, key='psf_fitted_stars', val=fitted_stars) t_end = time.time() print(f'Time to generate psf (s): {t_end-t_start:.2f}') if show and ncores == 1: check_psf(data, filename, epsf.data, all_stars, psf_stars, size) elif show and ncores != 1: print("Can't display psfs while using multiple cores") print()
def generate_epsf(img_file, x, y, size=51, oversampling=2, maxiters=5): # Construct stars table from bright stars_tbl = Table() stars_tbl['x'] = x stars_tbl['y'] = y print(stars_tbl) img_hdu = fits.open(img_file) ndimage = NDData(data=img_hdu[0].data) stars = extract_stars(ndimage, stars_tbl, size=size) print('Extracted {0} stars. Building EPSF...'.format(len(stars))) epsf_builder = EPSFBuilder(oversampling=oversampling, maxiters=maxiters, progress_bar=True) epsf, fitted_stars = epsf_builder(stars) return(epsf)
def calculate_psf_template(image, xx, yy, size, oversampling_rate, maxiters=11, mask=None, plot_candidates=False) : hsize = (size+2)/2 # remove stars close to the sample edges pos_mask = ((xx > hsize) & (xx < (image.shape[1] - 1 - hsize)) & (yy > hsize) & (yy < (image.shape[0] - 1 - hsize))) # Table of good star positions stars_tbl = Table() stars_tbl['x'] = xx[pos_mask] stars_tbl['y'] = yy[pos_mask] # Subtract background # print("image.shape ", image.shape) mean_val, median_val, std_val = sigma_clipped_stats(image, mask=mask, sigma=2.) image -= median_val nddata = NDData(data=image) stars = extract_stars(nddata, stars_tbl, size=size) print("number of stars ", len(stars)) # # Plot candidates (use with care!) # if(plot_candidates == True) : nrows = 1+int(np.sqrt(float(len(stars)))) ncols = 1+ int(float(len(stars))/float(nrows)) # print("nrows, ncols, len(stars) ",nrows, ncols, len(stars)) # nrows = 7 # ncols = 8 fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20),squeeze=True) ax = ax.ravel() for i in range(nrows*ncols): norm = simple_norm(stars[i], 'log', percent=99.) ax[i].imshow(stars[i], norm=norm, origin='lower', cmap='viridis') plt.show() # # build average PSF # epsf_builder = EPSFBuilder(oversampling=oversampling_rate, maxiters=3,progress_bar=False) epsf, fitted_stars = epsf_builder(stars) return epsf, fitted_stars
def make_psf(data, catalog, show=False): catalog = catalog.copy() catalog['x'], catalog['y'] = data.wcs.all_world2pix(catalog['raMean'], catalog['decMean'], 0) bkg = np.nanmedian(data) nddata = NDData(data - bkg) stars = psf.extract_stars(nddata, catalog, size=25.) epsf_builder = EPSFBuilder(oversampling=1.) epsf, fitted_stars = epsf_builder(stars) if show: plt.figure() plt.imshow(epsf.data) nrows = int(np.ceil(len(stars)**0.5)) fig, axarr = plt.subplots(nrows, nrows, figsize=(20, 20), squeeze=True) for ax, star in zip(axarr.ravel(), fitted_stars): ax.imshow(star) ax.plot(star.cutout_center[0], star.cutout_center[1], 'r+') return epsf, fitted_stars
def run_photometry(img_file, epsf, fwhm, x, y, subtract_back=False, forced=False): img_hdu = fits.open(img_file) if subtract_back: bkg = Background2D(img_hdu[0].data, (21,21), filter_size=(3,3)) image = img_hdu[0].data - bkg.background ndimage = NDData(data=backsub) else: image = img_hdu[0].data ndimage = NDData(data=img_hdu[0].data) psf = copy.copy(epsf) stars_tbl = Table() stars_tbl['x'] = x stars_tbl['y'] = y stars = extract_stars(ndimage, stars_tbl, size=51) stars_tbl['flux'] = np.array([stars[0].estimate_flux()]) targets = Table() targets['x_0'] = stars_tbl['x'] targets['y_0'] = stars_tbl['y'] targets['flux_0'] = stars_tbl['flux'] if forced: psf.x_0.fixed = True psf.y_0.fixed = True daogroup = DAOGroup(fwhm) photometry = BasicPSFPhotometry(group_maker=daogroup, bkg_estimator=mmm_bkg, psf_model=psf, fitter=fitter, fitshape=(51,51)) result_tab = photometry(image=image, init_guesses=targets) return(result_tab)
def build_epsf(image_num): size = 7 hsize = (size - 1) / 2 peaks_tbl = find_peaks(Reduced_Image_Data[image_num], threshold=750.) x = peaks_tbl['x_peak'] y = peaks_tbl['y_peak'] mask = ((x > hsize) & (x < (Reduced_Image_Data[image_num].shape[1] - 1 - hsize)) & (y > hsize) & (y < (Reduced_Image_Data[image_num].shape[0] - 1 - hsize))) stars_tbl = Table() stars_tbl['x'] = x[mask] stars_tbl['y'] = y[mask] nddata = NDData(data=Reduced_Image_Data[image_num]) stars = extract_stars(nddata, stars_tbl, size=10) #nrows = 5 #ncols = 5 #fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize = (15, 15), squeeze = True) #ax = ax.ravel() #for i in range(nrows*ncols): #norm = simple_norm(stars[i], 'log', percent = 99.) #ax[i].imshow(stars[i], norm = norm, origin = 'lower', cmap = 'gray') epsf_builder = EPSFBuilder(oversampling=8, maxiters=20, progress_bar=False) epsf, fitted_stars = epsf_builder(stars) #norm = simple_norm(epsf.data, 'log', percent = 99.) #plt.imshow(epsf.data, norm = norm, origin = 'lower', cmap = 'viridis') #plt.colorbar() return epsf
def cutouts(image, stars, size): stars_tbl = Table(stars.T, names=["x", "y"]) stars = extract_stars(NDData(data=image), stars_tbl, size=size) return stars
def removeStarsPSF(img, starPixX, starPixY, starR, mags, gX, gY, gR, p1, p2, radius, showProgress, inverted = False): remove = [] for i in range(0,len(starPixX)): if i%100 == 0: print(i) for j in range(i+1,len(starPixX)): if np.abs(starPixX[i]-starPixX[j])<2.5*radius and np.abs(starPixY[i]-starPixY[j])<2.5*radius: remove.append(i) remove.append(j) mask = np.ones(len(starPixX), dtype = bool) mask[np.array(remove)] = 0 starPixX = starPixX[mask] starPixY = starPixY[mask] starR = starR[mask] mags = mags[mask] print(radius) mask = np.ones(starPixX.shape[0],dtype=bool) for i in range(gX.shape[0]): mask = np.logical_and(mask, np.sqrt((starPixX-gX[i])**2+(starPixY-gY[i])**2)>gR[i]+2*radius) starPixX = starPixX[mask] starPixY = starPixY[mask] starR = starR[mask] mags = mags[mask] mask = (2*radius<starPixX) & (starPixX<img.shape[1]-2*radius) & (2*radius<starPixY) & (starPixY<img.shape[0]-2*radius) starPixX = starPixX[mask] starPixY = starPixY[mask] starR = starR[mask] mags = mags[mask] if p1[0]==p2[0]: psfMask = np.abs(starPixX-p1[0])>2.5*radius trailMask = np.abs(starPixX-p1[0])<10 psfX = starPixX[psfMask] psfY = starPixY[psfMask] psfR = starR[psfMask] psfMags = mags[psfMask] trailX = starPixX[trailMask] trailY = starPixY[trailMask] trailMags = mags[trailMask] else: m,b = TF.getLineParams(p1,p2) par,perp = GT.findDistances(starPixX,starPixY,m,b) psfMask = np.abs(perp)>2.5*radius trailMask = np.abs(perp)<10 psfX = starPixX[psfMask] psfY = starPixY[psfMask] psfR = starR[psfMask] psfMags = mags[psfMask] trailX = starPixX[trailMask] trailY = starPixY[trailMask] trailMags = mags[trailMask] mask = psfR>10 psfX = psfX[mask] psfY = psfY[mask] psfMags = psfMags[mask] print(psfX.shape[0]) showImageWithDetectedHotPixels(img, "Image with Star Picks", psfX, psfY, "b", inverted) psfX,psfY = centroid_sources(img, psfX, psfY, box_size=21, centroid_func=centroid_com) showImageWithDetectedHotPixels(img, "Image with Star Picks Centroid", psfX, psfY, "b", inverted) stars = Table() stars["x"] = psfX stars["y"] = psfY #stars2 = Table() #stars2["x_0"] = psfX #stars2["y_0"] = psfY stars = extract_stars(NDData(data=img), stars, size = 2*int(2*radius)+1) nrows = 5 ncols = 5 fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20), squeeze=True) ax = ax.ravel() for i in range(nrows*ncols): norm = simple_norm(stars[i], 'log', percent=99.) ax[i].imshow(stars[i], norm=norm, origin='lower', cmap='viridis') plt.show() #sigma_psf = 2.0 #daogroup = DAOGroup(2.0) #mmm_bkg = MMMBackground() #fitter = LevMarLSQFitter() #psf_model = PRF(sigma=sigma_psf) #photometry = BasicPSFPhotometry(group_maker=daogroup,bkg_estimator=mmm_bkg,psf_model=psf_model,fitter=LevMarLSQFitter(),fitshape=(11,11)) #result_tab = photometry(image=img, init_guesses=stars2) #residual_image = photometry.get_residual_image() epsfBuilder = EPSFBuilder(oversampling=2, maxiters=20, smoothing_kernel = "quadratic") epsf, starFits = epsfBuilder(stars) #showImage(residual_image,"Image No Stars",True) norm = simple_norm(epsf.data, 'log', percent=99.) plt.imshow(epsf.data, norm=norm, origin='lower', cmap='viridis') plt.colorbar() plt.show() nrows = 5 ncols = 5 fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20), squeeze=True) ax = ax.ravel() for i in range(nrows*ncols): norm = simple_norm(starFits[i], 'log', percent=99.) ax[i].imshow(starFits[i], norm=norm, origin='lower', cmap='viridis') plt.show() nrows = 5 ncols = 5 fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20), squeeze=True) ax = ax.ravel() for i in range(nrows*ncols): norm = simple_norm(stars[i].data-starFits[i].data, 'log', percent=99.) ax[i].imshow(stars[i].data-starFits[i].data, norm=norm, origin='lower', cmap='viridis') plt.show() showImage(finalImg, "Image With No Stars", inverted)
def photometry(fileid): """ Run photometry. .. codeauthor:: Rasmus Handberg <*****@*****.**> """ # Settings: ref_mag_limit = 17 # Lower limit on reference target brightness ref_target_dist_limit = 30 # Reference star must be further than this away to be included logger = logging.getLogger(__name__) tic = default_timer() # Use local copy of archive if configured to do so: config = load_config() # Get datafile dict from API: datafile = api.get_datafile(fileid) logger.debug("Datafile: %s", datafile) targetid = datafile['targetid'] photfilter = datafile['photfilter'] archive_local = config.get('photometry', 'archive_local', fallback=None) if archive_local is not None: datafile['archive_path'] = archive_local if not os.path.isdir(datafile['archive_path']): raise FileNotFoundError("ARCHIVE is not available") # Get the catalog containing the target and reference stars: # TODO: Include proper-motion to the time of observation catalog = api.get_catalog(targetid, output='table') target = catalog['target'][0] # Extract information about target: target_name = str(target['target_name']) target_coord = coords.SkyCoord(ra=target['ra'], dec=target['decl'], unit='deg', frame='icrs') # Folder to save output: # TODO: Change this! output_folder_root = config.get('photometry', 'output', fallback='.') output_folder = os.path.join(output_folder_root, target_name, '%04d' % fileid) os.makedirs(output_folder, exist_ok=True) # Also write any logging output to the formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') _filehandler = logging.FileHandler(os.path.join(output_folder, 'photometry.log'), mode='w') _filehandler.setFormatter(formatter) _filehandler.setLevel(logging.INFO) logger.addHandler(_filehandler) # The paths to the science image: filepath = os.path.join(datafile['archive_path'], datafile['path']) # TODO: Download datafile using API to local drive: # TODO: Is this a security concern? #if archive_local: # api.download_datafile(datafile, archive_local) # Translate photometric filter into table column: if photfilter == 'gp': ref_filter = 'g_mag' elif photfilter == 'rp': ref_filter = 'r_mag' elif photfilter == 'ip': ref_filter = 'i_mag' elif photfilter == 'zp': ref_filter = 'z_mag' elif photfilter == 'B': ref_filter = 'B_mag' elif photfilter == 'V': ref_filter = 'V_mag' else: logger.warning( "Could not find filter '%s' in catalogs. Using default gp filter.", photfilter) ref_filter = 'g_mag' references = catalog['references'] references.sort(ref_filter) # Load the image from the FITS file: image = load_image(filepath) # Calculate pixel-coordinates of references: row_col_coords = image.wcs.all_world2pix( np.array([[ref['ra'], ref['decl']] for ref in references]), 0) references['pixel_column'] = row_col_coords[:, 0] references['pixel_row'] = row_col_coords[:, 1] # Calculate the targets position in the image: target_pixel_pos = image.wcs.all_world2pix( [[target['ra'], target['decl']]], 0)[0] # Clean out the references: hsize = 10 x = references['pixel_column'] y = references['pixel_row'] references = references[ (np.sqrt((x - target_pixel_pos[0])**2 + (y - target_pixel_pos[1])**2) > ref_target_dist_limit) & (references[ref_filter] < ref_mag_limit) & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] #============================================================================================== # BARYCENTRIC CORRECTION OF TIME #============================================================================================== ltt_bary = image.obstime.light_travel_time(target_coord, ephemeris='jpl') image.obstime = image.obstime.tdb + ltt_bary #============================================================================================== # BACKGROUND ESITMATION #============================================================================================== fig, ax = plt.subplots(1, 2, figsize=(20, 18)) plot_image(image.clean, ax=ax[0], scale='log', cbar='right', title='Image') plot_image(image.mask, ax=ax[1], scale='linear', cbar='right', title='Mask') fig.savefig(os.path.join(output_folder, 'original.png'), bbox_inches='tight') plt.close(fig) # Estimate image background: # Not using image.clean here, since we are redefining the mask anyway bkg = Background2D( image.clean, (128, 128), filter_size=(5, 5), #mask=image.mask | (image.clean > background_cutoff), sigma_clip=SigmaClip(sigma=3.0), bkg_estimator=SExtractorBackground(), exclude_percentile=50.0) image.background = bkg.background # Create background-subtracted image: image.subclean = image.clean - image.background # Plot background estimation: fig, ax = plt.subplots(1, 3, figsize=(20, 6)) plot_image(image.clean, ax=ax[0], scale='log', title='Original') plot_image(image.background, ax=ax[1], scale='log', title='Background') plot_image(image.subclean, ax=ax[2], scale='log', title='Background subtracted') fig.savefig(os.path.join(output_folder, 'background.png'), bbox_inches='tight') plt.close(fig) # TODO: Is this correct?! image.error = calc_total_error(image.clean, bkg.background_rms, 1.0) #============================================================================================== # DETECTION OF STARS AND MATCHING WITH CATALOG #============================================================================================== logger.info("References:\n%s", references) radius = 10 fwhm_guess = 6.0 fwhm_min = 3.5 fwhm_max = 13.5 # Extract stars sub-images: #stars = extract_stars( # NDData(data=image.subclean, mask=image.mask), # stars_for_epsf, # size=size #) # Set up 2D Gaussian model for fitting to reference stars: g2d = models.Gaussian2D(amplitude=1.0, x_mean=radius, y_mean=radius, x_stddev=fwhm_guess * gaussian_fwhm_to_sigma) g2d.amplitude.bounds = (0.1, 2.0) g2d.x_mean.bounds = (0.5 * radius, 1.5 * radius) g2d.y_mean.bounds = (0.5 * radius, 1.5 * radius) g2d.x_stddev.bounds = (fwhm_min * gaussian_fwhm_to_sigma, fwhm_max * gaussian_fwhm_to_sigma) g2d.y_stddev.tied = lambda model: model.x_stddev g2d.theta.fixed = True gfitter = fitting.LevMarLSQFitter() fwhms = np.full(len(references), np.NaN) for i, (x, y) in enumerate( zip(references['pixel_column'], references['pixel_row'])): x = int(np.round(x)) y = int(np.round(y)) x0, y0, width, height = x - radius, y - radius, 2 * radius, 2 * radius cutout = slice(y0 - 1, y0 + height), slice(x0 - 1, x0 + width) curr_star = image.subclean[cutout] / np.max(image.subclean[cutout]) npix = len(curr_star) ypos, xpos = np.mgrid[:npix, :npix] gfit = gfitter(g2d, x=xpos, y=ypos, z=curr_star) fwhms[i] = gfit.x_fwhm mask = ~np.isfinite(fwhms) | (fwhms <= fwhm_min) | (fwhms >= fwhm_max) masked_fwhms = np.ma.masked_array(fwhms, mask) fwhm = np.mean(sigma_clip(masked_fwhms, maxiters=20, sigma=2.0)) logger.info("FWHM: %f", fwhm) # Use DAOStarFinder to search the image for stars, and only use reference-stars where a # star was actually detected close to the references-star coordinate: cleanout_references = (len(references) > 50) if cleanout_references: daofind_tbl = DAOStarFinder(100, fwhm=fwhm, roundlo=-0.5, roundhi=0.5).find_stars(image.subclean, mask=image.mask) indx_good = np.zeros(len(references), dtype='bool') for k, ref in enumerate(references): dist = np.sqrt((daofind_tbl['xcentroid'] - ref['pixel_column'])**2 + (daofind_tbl['ycentroid'] - ref['pixel_row'])**2) if np.any(dist <= fwhm / 4): # Cutoff set somewhat arbitrary indx_good[k] = True references = references[indx_good] fig, ax = plt.subplots(1, 1, figsize=(20, 18)) plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) ax.scatter(references['pixel_column'], references['pixel_row'], c='r', alpha=0.3) if cleanout_references: ax.scatter(daofind_tbl['xcentroid'], daofind_tbl['ycentroid'], c='g', alpha=0.3) ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', c='r') fig.savefig(os.path.join(output_folder, 'positions.png'), bbox_inches='tight') plt.close(fig) #============================================================================================== # CREATE EFFECTIVE PSF MODEL #============================================================================================== # Make cutouts of stars using extract_stars: # Scales with FWHM size = int(np.round(29 * fwhm / 6)) if size % 2 == 0: size += 1 # Make sure it's a uneven number size = max(size, 15) # Never go below 15 pixels hsize = (size - 1) / 2 x = references['pixel_column'] y = references['pixel_row'] mask_near_edge = ((x > hsize) & (x < (image.shape[1] - 1 - hsize)) & (y > hsize) & (y < (image.shape[0] - 1 - hsize))) stars_for_epsf = Table() stars_for_epsf['x'] = x[mask_near_edge] stars_for_epsf['y'] = y[mask_near_edge] # Store which stars were used in ePSF in the table: logger.info("Number of stars used for ePSF: %d", len(stars_for_epsf)) references['used_for_epsf'] = mask_near_edge # Extract stars sub-images: stars = extract_stars(NDData(data=image.subclean, mask=image.mask), stars_for_epsf, size=size) # Plot the stars being used for ePSF: nrows = 5 ncols = 5 imgnr = 0 for k in range(int(np.ceil(len(stars_for_epsf) / (nrows * ncols)))): fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20), squeeze=True) ax = ax.ravel() for i in range(nrows * ncols): if imgnr > len(stars_for_epsf) - 1: ax[i].axis('off') else: plot_image(stars[imgnr], ax=ax[i], scale='log', cmap='viridis') imgnr += 1 fig.savefig(os.path.join(output_folder, 'epsf_stars%02d.png' % (k + 1)), bbox_inches='tight') plt.close(fig) # Build the ePSF: epsf = EPSFBuilder(oversampling=1.0, maxiters=500, fitter=EPSFFitter(fit_boxsize=2 * fwhm), progress_bar=True)(stars)[0] logger.info('Successfully built PSF model') fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 15)) plot_image(epsf.data, ax=ax1, cmap='viridis') fwhms = [] for a, ax in ((0, ax3), (1, ax2)): # Collapse the PDF along this axis: profile = epsf.data.sum(axis=a) itop = profile.argmax() poffset = profile[itop] / 2 # Run a spline through the points, but subtract half of the peak value, and find the roots: # We have to use a cubic spline, since roots() is not supported for other splines # for some reason profile_intp = UnivariateSpline(np.arange(0, len(profile)), profile - poffset, k=3, s=0, ext=3) lr = profile_intp.roots() axis_fwhm = lr[1] - lr[0] fwhms.append(axis_fwhm) x_fine = np.linspace(-0.5, len(profile) - 0.5, 500) ax.plot(profile, 'k.-') ax.plot(x_fine, profile_intp(x_fine) + poffset, 'g-') ax.axvline(itop) ax.axvspan(lr[0], lr[1], facecolor='g', alpha=0.2) ax.set_xlim(-0.5, len(profile) - 0.5) # Let's make the final FWHM the largest one we found: fwhm = np.max(fwhms) logger.info("Final FWHM based on ePSF: %f", fwhm) #ax2.axvspan(itop - fwhm/2, itop + fwhm/2, facecolor='b', alpha=0.2) #ax3.axvspan(itop - fwhm/2, itop + fwhm/2, facecolor='b', alpha=0.2) ax4.axis('off') fig.savefig(os.path.join(output_folder, 'epsf.png'), bbox_inches='tight') plt.close(fig) #============================================================================================== # COORDINATES TO DO PHOTOMETRY AT #============================================================================================== coordinates = np.array([[ref['pixel_column'], ref['pixel_row']] for ref in references]) # Add the main target position as the first entry: if datafile.get('template') is None: coordinates = np.concatenate(([target_pixel_pos], coordinates), axis=0) #============================================================================================== # APERTURE PHOTOMETRY #============================================================================================== # Define apertures for aperture photometry: apertures = CircularAperture(coordinates, r=fwhm) annuli = CircularAnnulus(coordinates, r_in=1.5 * fwhm, r_out=2.5 * fwhm) apphot_tbl = aperture_photometry(image.subclean, [apertures, annuli], mask=image.mask, error=image.error) logger.debug("Aperture Photometry Table:\n%s", apphot_tbl) logger.info('Apperature Photometry Success') #============================================================================================== # PSF PHOTOMETRY #============================================================================================== # Are we fixing the postions? epsf.fixed.update({'x_0': False, 'y_0': False}) # Create photometry object: photometry = BasicPSFPhotometry(group_maker=DAOGroup(fwhm), bkg_estimator=SExtractorBackground(), psf_model=epsf, fitter=fitting.LevMarLSQFitter(), fitshape=size, aperture_radius=fwhm) psfphot_tbl = photometry(image=image.subclean, init_guesses=Table(coordinates, names=['x_0', 'y_0'])) logger.debug("PSF Photometry Table:\n%s", psfphot_tbl) logger.info('PSF Photometry Success') #============================================================================================== # TEMPLATE SUBTRACTION AND TARGET PHOTOMETRY #============================================================================================== if datafile.get('template') is not None: # Find the pixel-scale of the science image: pixel_area = proj_plane_pixel_area(image.wcs.celestial) pixel_scale = np.sqrt(pixel_area) * 3600 # arcsec/pixel #print(image.wcs.celestial.cunit) % Doesn't work? logger.info("Science image pixel scale: %f", pixel_scale) # Run the template subtraction, and get back # the science image where the template has been subtracted: diffimage = run_imagematch(datafile, target, star_coord=coordinates, fwhm=fwhm, pixel_scale=pixel_scale) # Include mask from original image: diffimage = np.ma.masked_array(diffimage, image.mask) # Create apertures around the target: apertures = CircularAperture(target_pixel_pos, r=fwhm) annuli = CircularAnnulus(target_pixel_pos, r_in=1.5 * fwhm, r_out=2.5 * fwhm) # Create two plots of the difference image: fig, ax = plt.subplots(1, 1, squeeze=True, figsize=(20, 20)) plot_image(diffimage, ax=ax, cbar='right', title=target_name) ax.plot(target_pixel_pos[0], target_pixel_pos[1], marker='+', color='r') fig.savefig(os.path.join(output_folder, 'diffimg.png'), bbox_inches='tight') apertures.plot(color='r') annuli.plot(color='k') ax.set_xlim(target_pixel_pos[0] - 50, target_pixel_pos[0] + 50) ax.set_ylim(target_pixel_pos[1] - 50, target_pixel_pos[1] + 50) fig.savefig(os.path.join(output_folder, 'diffimg_zoom.png'), bbox_inches='tight') plt.close(fig) # Run aperture photometry on subtracted image: target_apphot_tbl = aperture_photometry(diffimage, [apertures, annuli], mask=image.mask, error=image.error) # Run PSF photometry on template subtracted image: target_psfphot_tbl = photometry(diffimage, init_guesses=Table( target_pixel_pos, names=['x_0', 'y_0'])) # Combine the output tables from the target and the reference stars into one: apphot_tbl = vstack([target_apphot_tbl, apphot_tbl], join_type='exact') psfphot_tbl = vstack([target_psfphot_tbl, psfphot_tbl], join_type='exact') # Build results table: tab = references.copy() tab.insert_row( 0, { 'starid': 0, 'ra': target['ra'], 'decl': target['decl'], 'pixel_column': target_pixel_pos[0], 'pixel_row': target_pixel_pos[1] }) for key in ('pm_ra', 'pm_dec', 'gaia_mag', 'gaia_bp_mag', 'gaia_rp_mag', 'H_mag', 'J_mag', 'K_mag', 'g_mag', 'r_mag', 'i_mag', 'z_mag'): tab[0][key] = np.NaN # Subtract background estimated from annuli: flux_aperture = apphot_tbl['aperture_sum_0'] - ( apphot_tbl['aperture_sum_1'] / annuli.area()) * apertures.area() flux_aperture_error = np.sqrt(apphot_tbl['aperture_sum_err_0']**2 + (apphot_tbl['aperture_sum_err_1'] / annuli.area() * apertures.area())**2) # Add table columns with results: tab['flux_aperture'] = flux_aperture / image.exptime tab['flux_aperture_error'] = flux_aperture_error / image.exptime tab['flux_psf'] = psfphot_tbl['flux_fit'] / image.exptime tab['flux_psf_error'] = psfphot_tbl['flux_unc'] / image.exptime tab['pixel_column_psf_fit'] = psfphot_tbl['x_fit'] tab['pixel_row_psf_fit'] = psfphot_tbl['y_fit'] tab['pixel_column_psf_fit_error'] = psfphot_tbl['x_0_unc'] tab['pixel_row_psf_fit_error'] = psfphot_tbl['y_0_unc'] # Theck that we got valid photometry: if not np.isfinite(tab[0]['flux_psf']) or not np.isfinite( tab[0]['flux_psf_error']): raise Exception("Target magnitude is undefined.") #============================================================================================== # CALIBRATE #============================================================================================== # Convert PSF fluxes to magnitudes: mag_inst = -2.5 * np.log10(tab['flux_psf']) mag_inst_err = (2.5 / np.log(10)) * (tab['flux_psf_error'] / tab['flux_psf']) # Corresponding magnitudes in catalog: mag_catalog = tab[ref_filter] # Mask out things that should not be used in calibration: use_for_calibration = np.ones_like(mag_catalog, dtype='bool') use_for_calibration[0] = False # Do not use target for calibration use_for_calibration[~np.isfinite(mag_inst) | ~np.isfinite(mag_catalog)] = False # Just creating some short-hands: x = mag_catalog[use_for_calibration] y = mag_inst[use_for_calibration] yerr = mag_inst_err[use_for_calibration] # Fit linear function with fixed slope, using sigma-clipping: model = models.Linear1D(slope=1, fixed={'slope': True}) fitter = fitting.FittingWithOutlierRemoval(fitting.LinearLSQFitter(), sigma_clip, sigma=3.0) best_fit, sigma_clipped = fitter(model, x, y, weights=1.0 / yerr**2) # Extract zero-point and estimate its error: # I don't know why there is not an error-estimate attached directly to the Parameter? zp = -1 * best_fit.intercept.value # Negative, because that is the way zeropoints are usually defined zp_error = nanstd(y[~sigma_clipped] - best_fit(x[~sigma_clipped])) # Add calibrated magnitudes to the photometry table: tab['mag'] = mag_inst + zp tab['mag_error'] = np.sqrt(mag_inst_err**2 + zp_error**2) fig, ax = plt.subplots(1, 1) ax.errorbar(x, y, yerr=yerr, fmt='k.') ax.scatter(x[sigma_clipped], y[sigma_clipped], marker='x', c='r') ax.plot(x, best_fit(x), color='g', linewidth=3) ax.set_xlabel('Catalog magnitude') ax.set_ylabel('Instrumental magnitude') fig.savefig(os.path.join(output_folder, 'calibration.png'), bbox_inches='tight') plt.close(fig) #============================================================================================== # SAVE PHOTOMETRY #============================================================================================== # Descriptions of columns: tab['flux_aperture'].unit = u.count / u.second tab['flux_aperture_error'].unit = u.count / u.second tab['flux_psf'].unit = u.count / u.second tab['flux_psf_error'].unit = u.count / u.second tab['pixel_column'].unit = u.pixel tab['pixel_row'].unit = u.pixel tab['pixel_column_psf_fit'].unit = u.pixel tab['pixel_row_psf_fit'].unit = u.pixel tab['pixel_column_psf_fit_error'].unit = u.pixel tab['pixel_row_psf_fit_error'].unit = u.pixel # Meta-data: tab.meta['version'] = __version__ tab.meta['fileid'] = fileid tab.meta['template'] = None if datafile.get( 'template') is None else datafile['template']['fileid'] tab.meta['photfilter'] = photfilter tab.meta['fwhm'] = fwhm tab.meta['obstime-bmjd'] = float(image.obstime.mjd) tab.meta['zp'] = zp tab.meta['zp_error'] = zp_error # Filepath where to save photometry: photometry_output = os.path.join(output_folder, 'photometry.ecsv') # Write the final table to file: tab.write(photometry_output, format='ascii.ecsv', delimiter=',', overwrite=True) toc = default_timer() logger.info("Photometry took: %f seconds", toc - tic) return photometry_output
def get_psf(data, starlist_x, starlist_y, do_plot='no', n_resample=4): # create table with x and y coordinates for the epsf stars stars_tbl = Table() stars_tbl['x'], stars_tbl['y'] = starlist_x, starlist_y # create bkg subtracted cutouts around stars mean_val, median_val, std_val = sigma_clipped_stats(data, sigma=2.) data -= median_val # extraction function requires data as NDData object nddata = NDData(data=data) # extract 30 x 30 px cutouts around the stars stars = extract_stars(nddata, stars_tbl, size=30) # build the EPSF from the selected stars epsf_builder = EPSFBuilder(oversampling=n_resample, maxiters=10) epsf, fitted_stars = epsf_builder(stars) # fit gaussian through PSF to estimate FWHM params = lmfit.Parameters() params.add('h', 0.006, min=0.004, max=0.01, vary=True) params.add('std', 5, min=1, max=10, vary=True) params.add('cen', 50, min=45, max=55, vary=True) len_x = len(epsf.data[0, :]) x = np.linspace(0, 100, len_x) cutthrough = epsf.data[int(len(x) / 2), :] minimizer = lmfit.Minimizer(misc.single_egauss, params, fcn_args=(x, cutthrough)) result = minimizer.minimize() gauss_std = result.params['std'] h, cen = result.params['h'], result.params['cen'] if do_plot == 'yes': fig_psf, ax_psf = plt.subplots(figsize=(8, 8)) ax_psf.imshow(epsf.data, origin='lower', cmap='inferno') ax_psf.set_xlabel('x coordinate [px]') ax_psf.set_ylabel('y coordinate [px]') fig_psf.suptitle('3D PSF') fig_psf.savefig('PSF_2D.pdf', bbox_inches='tight') # plot cuts through the ePSF at different x positions len_x = len(epsf.data[0, :]) x = np.linspace(0, 100, len_x) fig1, ax1 = plt.subplots(figsize=(8, 8)) xvals = [int(len(x) / 4), int(len(x) / 2.1), int(len(x) / 2)] for xval in xvals: cutthrough = epsf.data[xval, :] lab = 'x = ' + str(xval) ax1.plot(x, cutthrough, label=lab) g = misc.emission_gaussian(x, h, cen, gauss_std) ax1.plot(x, g, label='Gaussian fit') ax1.set_xlabel('x coordinate [px]') ax1.set_ylabel('EPSF normalized flux') ax1.legend() fig1.suptitle('2D cut-through of the PSF') fig1.savefig('PSF_cutthrough.pdf', bbox_inches='tight') # return epsf, std devition of gaussian fit and resampling parameter return epsf, gauss_std, n_resample
def jwst_camera_fpa_data(data_dir, pattern, standardized_data_dir, parameters, overwrite_source_extraction=False): """Generate standardized focal plane alignment (fpa) data based on JWST camera image. """ save_plot = parameters['save_plot'] file_list = glob.glob(os.path.join(data_dir, '*{}'.format(pattern))) if len(file_list) == 0: raise RuntimeError('No data found') file_list.sort() for f in file_list: plt.close('all') print() print('Data directory: {}'.format(data_dir)) print('Image being processed: {}'.format(f)) im = datamodels.open(f) if hasattr(im, 'data') is False: im.data = fits.getdata(f) #im.dq = np.zeros(im.data.shape) header_info = OrderedDict() for attribute in 'telescope'.split(): header_info[attribute] = getattr(im.meta, attribute) # observations for attribute in 'date time visit_number visit_id visit_group activity_id program_number'.split( ): header_info['observation_{}'.format(attribute)] = getattr( im.meta.observation, attribute) header_info['epoch_isot'] = '{}T{}'.format( header_info['observation_date'], header_info['observation_time']) # instrument for attribute in 'name filter pupil detector'.split(): header_info['instrument_{}'.format(attribute)] = getattr( im.meta.instrument, attribute) # subarray for attribute in 'name'.split(): header_info['subarray_{}'.format(attribute)] = getattr( im.meta.subarray, attribute) # aperture for attribute in 'name position_angle pps_name'.split(): try: value = getattr(im.meta.aperture, attribute) except AttributeError: value = None header_info['aperture_{}'.format(attribute)] = value header_info['INSTRUME'] = header_info['instrument_name'] header_info['SIAFAPER'] = header_info['aperture_name'] instrument_name = getattr(im.meta.instrument, 'name') instrument_detector = getattr(im.meta.instrument, 'detector') instrument_filter = getattr(im.meta.instrument, 'filter') # temporary solution, this should come from populated aperture attributes #if header_info['subarray_name'] == 'FULL': # master_apertures = pysiaf.read.read_siaf_detector_layout() # if header_info['instrument_name'].lower() in ['niriss', 'miri']: # header_info['SIAFAPER'] = master_apertures['AperName'][np.where(master_apertures['InstrName']==header_info['instrument_name'])[0][0]] # elif header_info['instrument_name'].lower() in ['fgs']: # header_info['SIAFAPER'] = 'FGS{}_FULL'.format(header_info['instrument_detector'][-1]) # elif header_info['instrument_name'].lower() in ['nircam']: # header_info['SIAFAPER'] = header_info['aperture_name'] #else: # sys.exit('Only FULL arrays are currently supported.') # target for attribute in 'ra dec catalog_name proposer_name'.split(): header_info['target_{}'.format(attribute)] = getattr( im.meta.target, attribute) # pointing for attribute in 'ra_v1 dec_v1 pa_v3'.split(): try: value = getattr(im.meta.pointing, attribute) except AttributeError: value = None header_info['pointing_{}'.format(attribute)] = value # add HST style keywords header_info['PROGRAM_VISIT'] = '{}_{}'.format( header_info['observation_program_number'], header_info['observation_visit_id']) header_info['PROPOSID'] = header_info['observation_program_number'] header_info['DATE-OBS'] = header_info['observation_date'] header_info['TELESCOP'] = header_info['telescope'] header_info['INSTRUME'] = header_info['instrument_name'] try: header_info['APERTURE'] = header_info['SIAFAPER'] except KeyError: header_info['APERTURE'] = None header_info['CHIP'] = 0 # TBD: Need to remove making yet another directory #extracted_sources_dir = os.path.join(standardized_data_dir, 'extraction') #if os.path.isdir(extracted_sources_dir) is False: # os.makedirs(extracted_sources_dir) extracted_sources_file = os.path.join( standardized_data_dir, #extracted_sources_dir, '{}_extracted_sources.fits'.format( os.path.basename(f).split('.')[0])) mask_extreme_slope_values = False parameters['maximum_slope_value'] = 1000. # Check if extracted_sources_file exists, or overwrite_source_extraction is set to True if (not os.path.isfile(extracted_sources_file)) or ( overwrite_source_extraction): data = copy.deepcopy(im.data) #dq = copy.deepcopy(im.dq) # Convert image data to counts per second photmjsr = getattr(im.meta.photometry, 'conversion_megajanskys') data_cps = data / photmjsr if mask_extreme_slope_values: # clean up extreme slope values bad_index = np.where( np.abs(data) > parameters['maximum_slope_value']) data[bad_index] = 0. dq[bad_index] = -1 bkgrms = MADStdBackgroundRMS() mmm_bkg = MMMBackground() bgrms = bkgrms(data_cps) bgavg = mmm_bkg(data_cps) # Default parameters that generally works for NIRCam/NIRISS images sigma_factor = 10 round_lo, round_hi = 0.0, 0.6 sharp_lo, sharp_hi = 0.3, 1.4 fwhm_lo, fwhm_hi = 1.0, 20.0 fwhm = 2.0 minsep_fwhm = 7 # NOTE: minsep_fwhm>5 to reject artifacts around saturated stars flux_percent_lo, flux_percent_hi = 10, 99 # if 'sharp_lo' in parameters: # sharp_lo = parameters['sharp_lo'] ### ### TBD1: Relocate params below to config parts/files ### # Use different criteria for selecting good stars if parameters['nominalpsf']: # If using Nominal PSF models if instrument_name == 'NIRISS': #fwhm_lo, fwhm_hi = 1.0, 2.0 sharp_lo, sharp_hi = 0.6, 1.4 elif instrument_name == 'FGS': #fwhm_lo, fwhm_hi = 1.0, 1.4 sharp_lo, sharp_hi = 0.6, 1.4 elif instrument_name == 'NIRCAM': sharp_lo, sharp_hi = 0.6, 1.4 elif instrument_name == 'MIRI': sharp_lo, sharp_hi = 0.8, 1.0 fwhm_lo, fwhm_hi = 1.5, 2.2 sigma_factor = 3 elif instrument_name == 'NIRSPEC': sharp_lo, sharp_hi = 0.6, 0.8 round_lo, round_hi = 0.0, 0.3 fwhm_lo, fwhm_hi = 1.0, 1.75 else: ### ### For OTE commissioning, tweak the params below after finding ### the correct ranges by runnin the photometry notebook. ### # If using Commissioning (non-phased) PSF models if instrument_name == 'NIRISS': sharp_lo, sharp_hi = 0.6, 1.4 fwhm_lo, fwhm_hi = 1.4, 2.4 ################################################################################ ################################################################################ ################################################################################ elif instrument_name == 'FGS': sigma_factor = 10 minsep_fwhm = 2.5 sharp_lo, sharp_hi = 0.45, 0.7 round_lo, round_hi = 0.0, 0.3 flux_percent_lo, flux_percent_hi = 2, 99 fwhm = 4 ################################################################################ ################################################################################ ################################################################################ # Below works well for F200W and F356W images elif instrument_name == 'NIRCAM': sigma_factor = 3 minsep_fwhm = 2.5 sharp_lo, sharp_hi = 0.5, 0.7 round_lo, round_hi = 0.0, 0.2 flux_percent_lo, flux_percent_hi = 2, 99 if 'F200W' in instrument_filter: fwhm = 10 elif 'F356W' in instrument_filter: fwhm = 8 elif 'F090W' in instrument_filter: fwhm = 5.5 elif 'F277W' in instrument_filter: fwhm = 6.5 else: fwhm = 3 ################################################################################ ################################################################################ ################################################################################ elif instrument_name == 'MIRI': sharl_lo, sharp_hi = 0.5, 1.0 fwhm_lo, fwhm_hi = 1.5, 2.2 sigma_factor = 3 elif instrument_name == 'NIRSPEC': sharp_lo, sharp_hi = 0.5, 0.8 round_lo, round_hi = 0.0, 0.3 fwhm_lo, fwhm_hi = 1.0, 1.75 # Use IRAFStarFinder for source detection iraffind = IRAFStarFinder(threshold=sigma_factor * bgrms + bgavg, fwhm=fwhm, minsep_fwhm=minsep_fwhm, roundlo=round_lo, roundhi=round_hi, sharplo=sharp_lo, sharphi=sharp_hi) # Create default mask with all False values datamask = np.zeros( data_cps.shape, dtype=bool) # This creates an array with all False # Mask the left (for NRS1) and right regions (for NRS2) for NIRSpec if instrument_detector == 'NRS1': datamask[:, :1023] = True # Mask everything on the left side elif instrument_detector == 'NRS2': datamask[:, 1024:] = True # Mask everything on the right side iraf_extracted_sources = iraffind(data_cps, mask=datamask) # Perform some basic filtering # Remove sources based on flux percentile # 10-99% works well for filtering out too faint or saturated sources flux_min = np.percentile(iraf_extracted_sources['flux'], flux_percent_lo) flux_max = np.percentile(iraf_extracted_sources['flux'], flux_percent_hi) iraf_extracted_sources.remove_rows( np.where(iraf_extracted_sources['flux'] < flux_min)) iraf_extracted_sources.remove_rows( np.where(iraf_extracted_sources['flux'] > flux_max)) # Also remove sources based on fwhm ### ### Don't use below for now - 2/23/2022 (Don't use it unless we get lots of bad sources) ### #iraf_extracted_sources.remove_rows(np.where(iraf_extracted_sources['fwhm']<fwhm_lo)) #iraf_extracted_sources.remove_rows(np.where(iraf_extracted_sources['fwhm']>fwhm_hi)) # Now improve the positions by re-running centroiding algorithm if necessary. # NOTE: For now, re-centroiding will be turned off ### ### TBD2: Add re-centroiding algorithm adopted from Paul here ### #xarr = sources_masked['xcentroid'] #yarr = sources_masked['ycentroid'] #newx, newy = centroid_sources(data_cps, xarr, yarr, box_size=5, centroid_func=centroid_2dg) #coords = np.column_stack((newx, newy)) #srcaper = CircularAnnulus(coords, r_in=1, r_out=3) #srcaper_masks = srcaper.to_mask(method='center') #satflag = np.zeros((len(newx),),dtype=int) #i = 0 #for mask in srcaper_masks: # srcaper_dq = mask.multiply(dqarr) # srcaper_dq_1d = srcaper_dq[mask.data>0] # badpix = np.logical_and(srcaper_dq_1d>2, srcaper_dq_1d<7) # reallybad = np.where(srcaper_dq_1d==1) # if ((len(srcaper_dq_1d[badpix]) > 1) or (len(srcaper_dq_1d[reallybad]) > 0)): # satflag[i] = 1 # i =+1 #goodx = newx[np.where(satflag==0)] #goody = newy[np.where(satflag==0)] #print('Number of sources before removing saturated or bad pixels: ', len(xarr)) #print('Number of sources without saturated or bad pixels: ', len(goodx)) #print(' ') #coords = np.column_stack((goodx,goody)) print('Number of extracted sources after filtering: {} sources'. format(len(iraf_extracted_sources))) if parameters['use_epsf'] is True: size = 25 hsize = (size - 1) / 2 x = iraf_extracted_sources['xcentroid'] y = iraf_extracted_sources['ycentroid'] mask = ((x > hsize) & (x < (data_cps.shape[1] - 1 - hsize)) & (y > hsize) & (y < (data_cps.shape[0] - 1 - hsize))) stars_tbl = Table() stars_tbl['x'] = x[mask] stars_tbl['y'] = y[mask] print('Using {} stars to build epsf'.format(len(stars_tbl))) data_cps_bkgsub = data_cps.copy() data_cps_bkgsub -= bgavg nddata = NDData(data=data_cps_bkgsub) stars = extract_stars(nddata, stars_tbl, size=size) # # Figure - PSF stars # nrows = 10 ncols = 10 fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20), squeeze=True) ax = ax.ravel() for i in range(nrows * ncols): if i <= len(stars) - 1: norm = simple_norm(stars[i], 'log', percent=99.) ax[i].imshow(stars[i], norm=norm, origin='lower', cmap='viridis') plt.title('{} sample stars for epsf'.format( header_info['APERTURE'])) if save_plot: figname = os.path.join( extracted_sources_dir, '{}_sample_psfs.pdf'.format( os.path.basename(f).split('.')[0])) plt.savefig(figname) if parameters['show_extracted_sources']: plt.show() # # Timer for ePSF construction # tic = time.perf_counter() epsf_builder = EPSFBuilder(oversampling=4, maxiters=3, progress_bar=False) print("Building ePSF ...") epsf, fitted_stars = epsf_builder(stars) toc = time.perf_counter() print("Time elapsed for building ePSF:", toc - tic) # # Figure - ePSF plot # norm_epsf = simple_norm(epsf.data, 'log', percent=99.) plt.figure() plt.imshow(epsf.data, norm=norm_epsf, origin='lower', cmap='viridis') plt.colorbar() plt.title('{} epsf using {} stars'.format( header_info['APERTURE'], len(stars_tbl))) if save_plot: figname = os.path.join( extracted_sources_dir, '{}_epsf.pdf'.format( os.path.basename(f).split('.')[0])) plt.savefig(figname) if parameters['show_extracted_sources']: plt.show() daogroup = DAOGroup(5.0 * 2.0) psf_model = epsf.copy() tic = time.perf_counter() photometry = IterativelySubtractedPSFPhotometry( finder=iraffind, group_maker=daogroup, bkg_estimator=mmm_bkg, psf_model=psf_model, fitter=LevMarLSQFitter(), niters=1, fitshape=(11, 11), aperture_radius=5) print('Performing source extraction and photometry ...') epsf_extracted_sources = photometry(data_cps) toc = time.perf_counter() print("Time elapsed for PSF photometry:", toc - tic) print('Final source extraction with epsf: {} sources'.format( len(epsf_extracted_sources))) epsf_extracted_sources['xcentroid'] = epsf_extracted_sources[ 'x_fit'] epsf_extracted_sources['ycentroid'] = epsf_extracted_sources[ 'y_fit'] extracted_sources = epsf_extracted_sources extracted_sources.write(extracted_sources_file, overwrite=True) norm = simple_norm(data_cps, 'sqrt', percent=99.) diff = photometry.get_residual_image() plt.figure() ax1 = plt.subplot(1, 2, 1) plt.xlabel("X [pix]") plt.ylabel("Y [pix]") ax1.imshow(data_cps, norm=norm, cmap='Greys') ax2 = plt.subplot(1, 2, 2) plt.xlabel("X [pix]") plt.ylabel("Y [pix]") ax2.imshow(diff, norm=norm, cmap='Greys') plt.title('PSF subtracted image for {}'.format( os.path.basename(f))) if save_plot: figname = os.path.join( extracted_sources_dir, '{}_psfsubtracted_image.pdf'.format( os.path.basename(f).split('.')[0])) plt.savefig(figname) if parameters['show_psfsubtracted_image']: plt.show() else: extracted_sources = iraf_extracted_sources extracted_sources.write(extracted_sources_file, overwrite=True) positions = np.transpose((extracted_sources['xcentroid'], extracted_sources['ycentroid'])) apertures = CircularAperture(positions, r=10) norm = simple_norm(data_cps, 'sqrt', percent=99.) plt.figure(figsize=(12, 12)) plt.xlabel("X [pix]") plt.ylabel("Y [pix]") plt.imshow(data_cps, norm=norm, cmap='Greys', origin='lower') apertures.plot(color='blue', lw=1.5, alpha=0.5) title_string = '{}: {} selected sources'.format( os.path.basename(f), len(extracted_sources)) plt.title(title_string) plt.tight_layout() if save_plot: figname = os.path.join( standardized_data_dir, '{}_extracted_sources.pdf'.format( os.path.basename(f).split('.')[0])) plt.savefig(figname) if parameters['show_extracted_sources']: plt.show() plt.close() else: extracted_sources = Table.read(extracted_sources_file) print('Extracted {} sources from {}'.format(len(extracted_sources), f)) impose_positive_flux = True if impose_positive_flux and parameters['use_epsf']: extracted_sources.remove_rows( np.where(extracted_sources['flux_fit'] < 0)[0]) print('Only {} sources have positve flux'.format( len(extracted_sources))) astrometry_uncertainty_mas = 5 if len(extracted_sources) > 0: # Cal images are in DMS coordinates which correspond to the SIAF Science (SCI) frame extracted_sources['x_SCI'], extracted_sources[ 'y_SCI'] = extracted_sources['xcentroid'], extracted_sources[ 'ycentroid'] # For now, astrometric uncertainty defaults to 5 mas for each source. extracted_sources['sigma_x_mas'] = np.ones( len(extracted_sources)) * astrometry_uncertainty_mas extracted_sources['sigma_y_mas'] = np.ones( len(extracted_sources)) * astrometry_uncertainty_mas # transfer info to astropy table header for key, value in header_info.items(): extracted_sources.meta[key] = value extracted_sources.meta['DATAFILE'] = os.path.basename(f) extracted_sources.meta['DATAPATH'] = os.path.dirname(f) extracted_sources.meta['EPOCH'] = header_info['epoch_isot'] out_file = os.path.join( standardized_data_dir, '{}_FPA_data.fits'.format( extracted_sources.meta['DATAFILE'].split('.')[0])) print('Writing {}'.format(out_file)) with warnings.catch_warnings(): warnings.simplefilter('ignore', AstropyWarning, append=True) extracted_sources.write(out_file, overwrite=True) return im
def basic_psf_phot(target, centroided_sources, plots=False): def hmsm_to_days(hour=0,min=0,sec=0,micro=0): """ Convert hours, minutes, seconds, and microseconds to fractional days. """ days = sec + (micro / 1.e6) days = min + (days / 60.) days = hour + (days / 60.) return days / 24. def date_to_jd(year,month,day): """ Convert a date to Julian Day. Algorithm from 'Practical Astronomy with your Calculator or Spreadsheet', 4th ed., Duffet-Smith and Zwart, 2011. """ if month == 1 or month == 2: yearp = year - 1 monthp = month + 12 else: yearp = year monthp = month # this checks where we are in relation to October 15, 1582, the beginning # of the Gregorian calendar. if ((year < 1582) or (year == 1582 and month < 10) or (year == 1582 and month == 10 and day < 15)): # before start of Gregorian calendar B = 0 else: # after start of Gregorian calendar A = math.trunc(yearp / 100.) B = 2 - A + math.trunc(A / 4.) if yearp < 0: C = math.trunc((365.25 * yearp) - 0.75) else: C = math.trunc(365.25 * yearp) D = math.trunc(30.6001 * (monthp + 1)) jd = B + C + D + day + 1720994.5 return jd def gaussian(p, x, y): height, center_x, center_y, width_x, width_y, rotation = p rotation = np.deg2rad(rotation) x_0 = center_x * np.cos(rotation) - center_y * np.sin(rotation) y_0 = center_x * np.sin(rotation) + center_y * np.cos(rotation) def rotgauss(x,y): xp = x * np.cos(rotation) - y * np.sin(rotation) yp = x * np.sin(rotation) + y * np.cos(rotation) g = height*np.exp( -(((x_0-xp)/width_x)**2+ ((y_0-yp)/width_y)**2)/2.) return g g = rotgauss(x,y) return g def moments(data): total = np.nansum(data) X, Y = np.indices(data.shape) center_x = int(np.shape(data)[1]/2) center_y = int(np.shape(data)[0]/2) row = data[int(center_x), :] col = data[:, int(center_y)] width_x = np.nansum(np.sqrt(abs((np.arange(col.size)-center_y)**2*col)) /np.nansum(col)) width_y = np.nansum(np.sqrt(abs((np.arange(row.size)-center_x)**2*row)) /np.nansum(row)) height = np.nanmax(data) rotation = 0.0 return height, center_x, center_y, width_x, width_y, rotation def errorfunction(p, x, y, data): return gaussian(p, x, y) - data def fitgaussian(data): params = moments(data) X, Y = np.indices(data.shape) mask = ~np.isnan(data) x = X[mask] y = Y[mask] data = data[mask] p, success = optimize.leastsq(errorfunction, params, args=(x, y, data)) return p pines_path = pines_dir_check() short_name = short_name_creator(target) reduced_path = pines_path/('Objects/'+short_name+'/reduced/') reduced_files = np.array(natsort.natsorted([x for x in reduced_path.glob('*.fits')])) centroided_sources.columns = centroided_sources.columns.str.strip() source_names = natsort.natsorted(list(set([i[0:-2].replace('X','').replace('Y','').rstrip().lstrip() for i in centroided_sources.keys()]))) #Declare a new dataframe to hold the information for all targets for this . columns = ['Filename', 'Time UT', 'Time JD', 'Airmass', 'Seeing'] for i in range(0, len(source_names)): columns.append(source_names[i]+' Flux') columns.append(source_names[i]+' Flux Error') psf_df = pd.DataFrame(index=range(len(reduced_files)), columns=columns) output_filename = pines_path/('Objects/'+short_name+'/psf_phot/'+short_name+'_psf_phot.csv') for i in range(len(reduced_files)): #Read in image data/header. file = reduced_files[i] data = fits.open(file)[0].data header = fits.open(file)[0].header print('{}, image {} of {}.'.format(file.name, i+1, len(reduced_files))) #Read in some supporting information. log_path = pines_path/('Logs/'+file.name.split('.')[0]+'_log.txt') log = pines_log_reader(log_path) date_obs = header['DATE-OBS'] #Catch a case that can cause datetime strptime to crash; Mimir headers sometimes have DATE-OBS with seconds specified as 010.xx seconds, when it should be 10.xx seconds. if len(date_obs.split(':')[-1].split('.')[0]) == 3: date_obs = date_obs.split(':')[0] + ':' + date_obs.split(':')[1] + ':' + date_obs.split(':')[-1][1:] #Keep a try/except clause here in case other unknown DATE-OBS formats pop up. try: date = datetime.datetime.strptime(date_obs, '%Y-%m-%dT%H:%M:%S.%f') except: print('Header DATE-OBS format does not match the format code in strptime! Inspect/correct the DATE-OBS value.') pdb.set_trace() days = date.day + hmsm_to_days(date.hour,date.minute,date.second,date.microsecond) jd = date_to_jd(date.year,date.month,days) psf_df['Filename'][i] = file.name psf_df['Time UT'][i] = header['DATE-OBS'] psf_df['Time JD'][i] = jd psf_df['Airmass'][i] = header['AIRMASS'] psf_df['Seeing'][i] = log['X seeing'][np.where(log['Filename'] == file.name.split('_')[0]+'.fits')[0][0]] #Read in source centroids for this image x = np.zeros(len(source_names)) y = np.zeros(len(source_names)) seeing = psf_df['Seeing'][i] for j in range(len(source_names)): source = source_names[j] x[j] = centroided_sources[source+' X'][i] y[j] = centroided_sources[source+' Y'][i] #The extract_stars() function requires the input data as an NDData object. nddata = NDData(data=data) #Create table of good star positions stars_tbl = Table() stars_tbl['x'] = x stars_tbl['y'] = y size = 25 x, y = np.meshgrid(np.arange(0,size), np.arange(0,size)) #Extract star cutouts. stars = extract_stars(nddata, stars_tbl, size=size) fitter = fitting.LevMarLSQFitter() fig, ax = plt.subplots(nrows=len(stars), ncols=3, sharex=True, sharey=True, figsize=(12,40)) #Fit a 2D Gaussian to each star. for j in range(len(stars)): star = stars[j] source = source_names[j] mmm_bkg = MMMBackground() cutout = star.data - mmm_bkg(star.data) #Get the star's centroid position in the cutout. dtype = [('x_0', 'f8'), ('y_0', 'f8')] pos = Table(data=np.zeros(1, dtype=dtype)) source_x = stars_tbl['x'][j] source_y = stars_tbl['y'][j] pos['x_0'] = source_x - int(source_x - size/2 + 1) pos['y_0'] = source_y - int(source_y - size/2 + 1) parameters = fitgaussian(cutout) g2d_fit = gaussian(parameters, x, y) avg, med, std = sigma_clipped_stats(cutout) im = ax[j,0].imshow(cutout, origin='lower', vmin=med-std, vmax=med+8*std) divider = make_axes_locatable(ax[j,0]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') ax[j,0].plot(pos['x_0'], pos['y_0'], 'rx') ax[j,0].set_ylabel(source) ax[j,0].text(pos['x_0'], pos['y_0']+1, '('+str(np.round(source_x,1))+', '+str(np.round(source_y,1))+')', color='r', ha='center') ax[j,0].axis('off') axins = ax[j,0].inset_axes([0.75, 0.75, 0.25, 0.25]) axins.set_yticklabels([]) axins.set_yticks([]) axins.set_xticklabels([]) axins.set_xticks([]) axins.imshow(data, origin='lower', vmin=med-std, vmax=med+8*std) axins.plot(source_x, source_y, 'rx') im = ax[j,1].imshow(g2d_fit, origin='lower', vmin=med-std, vmax=med+8*std) divider = make_axes_locatable(ax[j,1]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') ax[j,1].axis('off') avg, med, std = sigma_clipped_stats(cutout - g2d_fit) im = ax[j,2].imshow(cutout - g2d_fit, origin='lower', vmin=med-std, vmax=med+8*std) divider = make_axes_locatable(ax[j,2]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') ax[j,2].axis('off') if j == 0: ax[j,0].set_title('Data') ax[j,1].set_title('2D Gaussian Model') ax[j,2].set_title('Data - Model') plt.tight_layout() output_filename = pines_path/('Objects/'+short_name+'/basic_psf_phot/'+reduced_files[i].name.split('_')[0]+'_'+'source_modeling.pdf') plt.savefig(output_filename) plt.close() return
def find_pinholes_regular(fname, sname, fdarkff, fdark, fff, files, ref_shape, size, threshold, fwhm, fitshape, range_psf, sigma=2., oversampling=4, maxiters=3): """Finds and fits regullary spread pinhole positions with a ePSF in a FITS image. Parameters ---------- fname : str Folder name of the input fits files. sname : str Folder name of the returned found and matched pinhole positions (txt files) fdarkff : string Location of the dark images for the flat field images. fdark : string Location of the dark images for the raw images. fff : string Location of the flat field images. files : (1, 2)-shaped int array File range to create a median image ref_shape : (1,2)-shaped array Number of reference stars in x and y direction [x, y]. size : int Rectangular size of the ePSF. Size must be an odd number. threshold : float The absolute image value above which to select sources. fwhm : float The full-width half-maximum (FWHM) of the major axis of the Gaussian kernel in units of pixels. fitshape : int or length-2 array-like Rectangular shape around the center of a star which will be used to collect the data to do the fitting. Can be an integer to be the same along both axes. E.g., 5 is the same as (5, 5), which means to fit only at the following relative pixel positions: [-2, -1, 0, 1, 2]. Each element of fitshape must be an odd number. range_psf : (1, 4)-shaped int array Position range to compute epsf [xmin,xmax,ymin,ymax] sigma : float Number of standard deviations used to perform sigma clip with a astropy.stats.SigmaClip object. oversampling : int or tuple of two int The oversampling factor(s) of the ePSF relative to the input stars along the x and y axes. The oversampling can either be a single float or a tuple of two floats of the form (x_oversamp, y_oversamp). If oversampling is a scalar then the oversampling will be the same for both the x and y axes. maxiters : int The maximum number of iterations to perform. Returns ------- positions_sort : (N,2)-shaped array Found and matched positions of the pinholes. ref_positions : (N,2)-shaped array Matched reference grid positions. """ #Load the sample of fits images entries = os.listdir(fname) data_col = np.array([fits.getdata(fname + '/' + entries[files[0]], ext=0)]) for k in range(files[0], files[1]): data_col = np.append(data_col, [fits.getdata(fname + '/' + entries[k], ext=0)], axis=0) #Data reduction: Darc current + Flatfield + bias data_col = data_correction(data_col, fdarkff, fdark, fff) #Claculate median image data_full = np.median(data_col, axis=0) data = data_full[range_psf[2]:range_psf[3], range_psf[0]:range_psf[1]] #Find peaks in data peaks_tbl = find_peaks(data, threshold=threshold) peaks_tbl['peak_value'].info.format = '%.8g' #Load data around found peaks hsize = (size - 1) / 2 x = peaks_tbl['x_peak'] y = peaks_tbl['y_peak'] mask = ((x > hsize) & (x < (data.shape[1] - 1 - hsize)) & (y > hsize) & (y < (data.shape[0] - 1 - hsize))) stars_tbl = Table() stars_tbl['x'] = x[mask] stars_tbl['y'] = y[mask] #Calculate mean, median, std mean_val, median_val, std_val = sigma_clipped_stats(data, sigma=sigma) data = data - median_val #Find pinholes and create ePSF nddata = NDData(data=data) stars = extract_stars(nddata, stars_tbl, size=size) epsf_builder = EPSFBuilder(oversampling=oversampling, maxiters=maxiters, progress_bar=False) epsf, fitted_stars = epsf_builder(stars) #Use ePSF to find precise locations of pinholes daofind = DAOPhotPSFPhotometry(crit_separation=30, threshold=threshold, fwhm=fwhm, psf_model=epsf, fitshape=fitshape, aperture_radius=12, niters=1) #Get positions sources = daofind(data_full) for col in sources.colnames: sources[col].info.format = '%.8g' pos_full = np.transpose((sources['x_fit'], sources['y_fit'])) #Plot found pinholes apertures = CircularAperture(pos_full, r=10) norm = ImageNormalize(stretch=SqrtStretch()) fig, ax = plt.subplots() ax.set_title('Pinhole Positions') ax.set(xlabel='x [pixel]', ylabel='y [pixel]') ax.imshow(data_full, cmap='Greys', origin='lower', norm=norm) apertures.plot(color='blue', lw=1.5, alpha=0.5) ax.legend(['#pinholes = ' + str(len(pos_full[:, 0]))], loc='lower left', prop={'size': 12}) plt.show() #Find central position xcent = (np.max(pos_full[:, 0]) + np.min(pos_full[:, 0])) / 2 ycent = (np.max(pos_full[:, 1]) + np.min(pos_full[:, 1])) / 2 #Find positions at the edges to set base positions for linear transformatio to match pinholes with reference grid distance = (pos_full[:, 0] - xcent)**2 + (pos_full[:, 1] - ycent)**2 pins = len(distance) sort_distance = np.partition(distance, (pins - 4, pins - 3, pins - 2, pins - 1)) maxpos = pos_full[distance == sort_distance[pins - 1]] maxpos = np.append(maxpos, pos_full[distance == sort_distance[pins - 2]], axis=0) maxpos = np.append(maxpos, pos_full[distance == sort_distance[pins - 3]], axis=0) maxpos = np.append(maxpos, pos_full[distance == sort_distance[pins - 4]], axis=0) b01 = maxpos[maxpos[:, 1] < ycent] b23 = maxpos[maxpos[:, 1] > ycent] posbase = np.array(b01[b01[:, 0] < xcent]) posbase = np.append(posbase, b01[b01[:, 0] > xcent], axis=0) posbase = np.append(posbase, b23[b23[:, 0] < xcent], axis=0) print(posbase) #Sort positions by matching with reference grid positions_sort, ref_positions = sort_positions_regular( pos_full, posbase, ref_shape) text = np.array([ positions_sort[:, 0], positions_sort[:, 1], ref_positions[:, 0], ref_positions[:, 1] ]) text_trans = np.zeros((len(positions_sort[:, 0]), 4)) #Transpose text matrix for k in range(0, 4): for l in range(0, len(positions_sort[:, 0])): text_trans[l][k] = text[k][l] #Save data as txt file np.savetxt( sname + '.txt', text_trans, fmt='%1.9E', delimiter='\t', header= ' x-measured y-measured x-reference y-reference', comments='') return positions_sort, ref_positions
# ['ibg403010_drz.fits', 597, 385], # ['ibg405010_drz.fits', 570, 701], # GAIA bad # ['ibg455010_drz.fits', 263, 444], # ['ibg456010_drz.fits', 530, 696], # ['ibg456010_drz.fits', 549, 462], # GAIA bad # ['ibg456010_drz.fits', 860, 408], # ['ibg456010_drz.fits', 911, 115], # ['ibg465010_drz.fits', 588, 723], # ['ibg471010_drz.fits', 600, 685], # ['ibg471010_drz.fits', 892, 511], #] # -1 because the above positions are measured in ds9, which counts from (1,1) # while the python code counts from (0,0) stars621 = extract_stars([NDData(fits.open(datadir + row[0])[1].data) for row in prflist], [Table({'x': [row[1] - 1], 'y': [row[2] - 1]}) for row in prflist], size=25) stars845 = extract_stars([NDData(fits.open(datadir + row[0].replace('10_', '20_'))[1].data) for row in prflist], [Table({'x': [row[1] - 1], 'y': [row[2] - 1]}) for row in prflist], size=25) def check_matching_source_exists(l1, l2, d, xname='xcentroid', yname='ycentroid'): '''Check for each source in l1, if one or more sources in l2 are close This is not the most efficient way to do things, but very quick to code and runtime is not a concern for this. Parameters ----------
def build_ePSF_astrometry(image_file, mask_file=None, nstars=40, image_source_file=None, astrom_sigma=5.0, psf_sigma=5.0, alim=10000, lowper=0.6, highper=0.9, keep=False, cutout=35, write=True, output=None, plot=False, output_plot=None, verbose=False): """Build the effective Point-Spread Function using a sample of stars from some image acquired via the `image2xy` tool of `astrometry.net`. Arguments --------- image_file : str Filename for a **background-subtracted** image mask_file : str, optional Filename for a mask file (default None) nstars : int, optional *Maximum* number of stars to use in building the ePSF (default 40; set to None to impose no limit) image_source_file : str, optional Filename for a `.xy.fits` file containing detected sources with their pixel coordinates and **background-subtracted** flux (default None, in which case a new such file is produced) astrom_sigma : float, optional Detection significance when using `image2xy` in `astrometry.net` to find sources (default 5.0) psf_sigma : float, optional Sigma of the approximate Gaussian PSF of the images (default 5.0) alim : int, optional *Maximum* allowed source area in square pixels for `astrometry.net`, above which sources will be deblended (default 10000) lowper, highper : float, optional Lower and upper flux percentiles (as a fraction between 0 and 1) such that sources outside the corresponding flux range will be excluded from ePSF building (default 0.6 and 0.9, respectively) keep : bool, optional Whether to keep the source list file (`.xy.fits` files; default False) cutout : int, optional Cutout size around each star in pixels (default 35; must be **odd**; rounded **down** if even) write : bool, optional Whether to write the ePSF to a new fits file (default True) output : str, optional Name for the output ePSF data fits file (default `image_file.replace(".fits", "_ePSF.fits")`) plot : bool, optional Whether to plot the newly-built ePSF (default False) output_plot : str, optional Name for the output figure (default `image_file.replace(".fits", "_ePSF.png")`) verbose : bool, optional Whether to be verbose (default False) Returns ------- np.ndarray The ePSF data in a 2D array Notes ----- Uses `astrometry.net` to obtain a list of sources in the image with their x, y coordinates, flux, and background at their location. (If a list of sources has already been obtained `solve-field` or `image2xy`, this can be input). Finally, selects stars between the `lowper` th and `highper` percentile fluxes. Finally, uses `EPSFBuilder` to empirically obtain the ePSF of these stars. Optionally writes and/or plots the obtained ePSF. **The ePSF obtained here should not be used in convolutions.** Instead, it can serve as a tool for estimating the seeing of an image. """ # ignore annoying warnings from photutils from astropy.utils.exceptions import AstropyWarning warnings.simplefilter('ignore', category=AstropyWarning) from astropy.nddata import NDData from photutils.psf import extract_stars from photutils import EPSFBuilder # load in data image_data = fits.getdata(image_file) image_header = fits.getheader(image_file) try: instrument = image_header["INSTRUME"] except KeyError: instrument = "Unknown" ### source detection ## use pre-existing file obtained by astrometry.net, if supplied if image_source_file: image_sources = np.logical_not(fits.getdata(image_source_file)) ## use astrometry.net to find the sources # -b --> no background-subtraction # -O --> overwrite # -p <astrom_sigma> --> signficance # -w <psf_sigma> --> estimated PSF sigma # -m <alim> --> max object size for deblending is <alim> else: options = f" -b -O -p {astrom_sigma} -w {psf_sigma} -m {alim}" run(f"image2xy {options} {image_file}", shell=True) image_sources_file = image_file.replace(".fits", ".xy.fits") image_sources = fits.getdata(image_sources_file) if not (keep): run(f"rm {image_sources_file}", shell=True) # file is not needed print(f'\n{len(image_sources)} stars at >{astrom_sigma} sigma found ' + f'in image {re.sub(".*/", "", image_file)} with astrometry.net') sources = Table() # build a table sources['x'] = image_sources['X'] # for EPSFBuilder sources['y'] = image_sources['Y'] sources['flux'] = image_sources['FLUX'] if nstars: sources = sources[:min(nstars, len(sources))] ## get WCS coords for all sources w = wcs.WCS(image_header) sources["ra"], sources["dec"] = w.all_pix2world(sources["x"], sources["y"], 1) ## mask out edge sources: # a bounding circle for WIRCam, rectangle for MegaPrime xsize = image_data.shape[1] ysize = image_data.shape[0] if "WIRCam" in instrument: rad_limit = xsize / 2.0 dist_to_center = np.sqrt((sources['x'] - xsize / 2.0)**2 + (sources['y'] - ysize / 2.0)**2) mask = dist_to_center <= rad_limit sources = sources[mask] else: x_lims = [int(0.05 * xsize), int(0.95 * xsize)] y_lims = [int(0.05 * ysize), int(0.95 * ysize)] mask = (sources['x'] > x_lims[0]) & (sources['x'] < x_lims[1]) & ( sources['y'] > y_lims[0]) & (sources['y'] < y_lims[1]) sources = sources[mask] ## empirically obtain the effective Point Spread Function (ePSF) nddata = NDData(image_data) # NDData object if mask_file: # supply a mask if needed nddata.mask = fits.getdata(mask_file) if cutout % 2 == 0: # if cutout even, subtract 1 cutout -= 1 stars = extract_stars(nddata, sources, size=cutout) # extract stars ## use only the stars with fluxes between two percentiles if using ## astrometry.net if image_source_file: stars_tab = Table() # temporary table stars_col = Column(data=range(len(stars.all_stars)), name="stars") stars_tab["stars"] = stars_col # column of indices of each star fluxes = [s.flux for s in stars] fluxes_col = Column(data=fluxes, name="flux") stars_tab["flux"] = fluxes_col # column of fluxes # get percentiles per_low = np.percentile(fluxes, lowper * 100) # lower percentile flux per_high = np.percentile(fluxes, highper * 100) # upper percentile flux mask = (stars_tab["flux"] >= per_low) & (stars_tab["flux"] <= per_high) stars_tab = stars_tab[mask] # include only stars between these fluxes idx_stars = (stars_tab["stars"]).data # indices of these stars # update stars object # have to manually update all_stars AND _data attributes stars.all_stars = [stars[i] for i in idx_stars] stars._data = stars.all_stars ## build the ePSF nstars_epsf = len(stars.all_stars) # no. of stars used in ePSF building if nstars_epsf == 0: print( "\nNo valid sources were found to build the ePSF with the given" + " conditions. Exiting.") return if verbose: print(f"{nstars_epsf} stars used in building the ePSF") epsf_builder = EPSFBuilder( oversampling=1, maxiters=7, # build it progress_bar=False) epsf, fitted_stars = epsf_builder(stars) epsf_data = epsf.data if write: # write, if desired epsf_hdu = fits.PrimaryHDU(data=epsf_data) if not (output): output = image_file.replace(".fits", "_ePSF.fits") epsf_hdu.writeto(output, overwrite=True, output_verify="ignore") if plot: # plot, if desired if not (output_plot): # set output name if not given output_plot = image_file.replace(".fits", "_ePSF.png") __plot_ePSF(epsf_data=epsf_data, output=output_plot) return epsf_data
def epsf_phot(target, centroided_sources, plots=False): def hmsm_to_days(hour=0,min=0,sec=0,micro=0): """ Convert hours, minutes, seconds, and microseconds to fractional days. """ days = sec + (micro / 1.e6) days = min + (days / 60.) days = hour + (days / 60.) return days / 24. def date_to_jd(year,month,day): """ Convert a date to Julian Day. Algorithm from 'Practical Astronomy with your Calculator or Spreadsheet', 4th ed., Duffet-Smith and Zwart, 2011. """ if month == 1 or month == 2: yearp = year - 1 monthp = month + 12 else: yearp = year monthp = month # this checks where we are in relation to October 15, 1582, the beginning # of the Gregorian calendar. if ((year < 1582) or (year == 1582 and month < 10) or (year == 1582 and month == 10 and day < 15)): # before start of Gregorian calendar B = 0 else: # after start of Gregorian calendar A = math.trunc(yearp / 100.) B = 2 - A + math.trunc(A / 4.) if yearp < 0: C = math.trunc((365.25 * yearp) - 0.75) else: C = math.trunc(365.25 * yearp) D = math.trunc(30.6001 * (monthp + 1)) jd = B + C + D + day + 1720994.5 return jd pines_path = pines_dir_check() short_name = short_name_creator(target) reduced_path = pines_path/('Objects/'+short_name+'/reduced/') reduced_filenames = natsort.natsorted([x.name for x in reduced_path.glob('*.fits')]) reduced_files = np.array([reduced_path/i for i in reduced_filenames]) centroided_sources.columns = centroided_sources.columns.str.strip() source_names = natsort.natsorted(list(set([i.split(' ')[0]+' '+i.split(' ')[1] for i in centroided_sources.keys() if (i[0] == '2') or (i[0] == 'R')]))) #Create output plot directories for each source. if plots: for name in source_names: #If the folders are already there, delete them. source_path = (pines_path/('Objects/'+short_name+'/psf_phot/'+name+'/')) if source_path.exists(): shutil.rmtree(source_path) #Create folders. os.mkdir(source_path) #Declare a new dataframe to hold the information for all targets for this . columns = ['Filename', 'Time UT', 'Time JD', 'Airmass', 'Seeing'] for i in range(0, len(source_names)): columns.append(source_names[i]+' Flux') columns.append(source_names[i]+' Flux Error') psf_df = pd.DataFrame(index=range(len(reduced_files)), columns=columns) output_filename = pines_path/('Objects/'+short_name+'/psf_phot/'+short_name+'_psf_phot.csv') for i in range(0, len(reduced_files)): #Read in image data/header. file = reduced_files[i] data = fits.open(file)[0].data header = fits.open(file)[0].header print('{}, image {} of {}.'.format(file.name, i+1, len(reduced_files))) #Read in some supporting information. log_path = pines_path/('Logs/'+file.name.split('.')[0]+'_log.txt') log = pines_log_reader(log_path) date_obs = header['DATE-OBS'] #Catch a case that can cause datetime strptime to crash; Mimir headers sometimes have DATE-OBS with seconds specified as 010.xx seconds, when it should be 10.xx seconds. if len(date_obs.split(':')[-1].split('.')[0]) == 3: date_obs = date_obs.split(':')[0] + ':' + date_obs.split(':')[1] + ':' + date_obs.split(':')[-1][1:] #Keep a try/except clause here in case other unknown DATE-OBS formats pop up. try: date = datetime.datetime.strptime(date_obs, '%Y-%m-%dT%H:%M:%S.%f') except: print('Header DATE-OBS format does not match the format code in strptime! Inspect/correct the DATE-OBS value.') pdb.set_trace() days = date.day + hmsm_to_days(date.hour,date.minute,date.second,date.microsecond) jd = date_to_jd(date.year,date.month,days) psf_df['Filename'][i] = file.name psf_df['Time UT'][i] = header['DATE-OBS'] psf_df['Time JD'][i] = jd psf_df['Airmass'][i] = header['AIRMASS'] psf_df['Seeing'][i] = log['X seeing'][np.where(log['Filename'] == file.name.split('_')[0]+'.fits')[0][0]] #Read in source centroids for this image x = np.zeros(len(source_names)) y = np.zeros(len(source_names)) for j in range(len(source_names)): source = source_names[j] x[j] = centroided_sources[source+' X'][i] y[j] = centroided_sources[source+' Y'][i] #Extract pixel cutouts of our stars, so let’s explicitly exclude stars that are too close to the image boundaries (because they cannot be extracted). size = 13 hsize = (size - 1) / 2 #mask = ((x > hsize) & (x < (data.shape[1] -1 - hsize)) & (y > hsize) & (y < (data.shape[0] -1 - hsize)) & (y > 100) & (y < 923)) #Create table of good star positions stars_tbl = Table() stars_tbl['x'] = x stars_tbl['y'] = y #Subtract background (star cutouts from which we build the ePSF must have background subtracted). mean_val, median_val, std_val = sigma_clipped_stats(data, sigma=2.) data -= median_val #Replace nans in data using Gaussian. # kernel = Gaussian2DKernel(x_stddev=0.5) # data = interpolate_replace_nans(data, kernel) #The extract_stars() function requires the input data as an NDData object. nddata = NDData(data=data) #Extract star cutouts. stars = extract_stars(nddata, stars_tbl, size=size) #Plot. nrows = 5 ncols = 5 fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(10, 10), squeeze=True) ax = ax.ravel() for j in range(len(stars)): norm = simple_norm(stars[j], 'log', percent=99.) ax[j].imshow(stars[j].data, norm=norm, origin='lower', cmap='viridis') pdb.set_trace() #Construct the ePSF using the star cutouts. epsf_fitter = EPSFFitter() epsf_builder = EPSFBuilder(maxiters=4, progress_bar=False, fitter=epsf_fitter) try: epsf, fitted_stars = epsf_builder(stars) output_filename = pines_path/('Objects/'+short_name+'/psf_phot/'+short_name+'_psf_phot.csv') for j in range(len(stars)): star = stars[j] source_name = source_names[j] sigma_psf = 1.85 dtype = [('x_0', 'f8'), ('y_0', 'f8')] pos = Table(data=np.zeros(1, dtype=dtype)) source_x = stars_tbl['x'][j] source_y = stars_tbl['y'][j] pos['x_0'] = source_x - int(source_x - size/2 + 1) pos['y_0'] = source_y - int(source_y - size/2 + 1) daogroup = DAOGroup(4.0*sigma_psf*gaussian_sigma_to_fwhm) mmm_bkg = MMMBackground() photometry = BasicPSFPhotometry(group_maker=daogroup, bkg_estimator=mmm_bkg, psf_model=epsf, fitter=LevMarLSQFitter(), fitshape=(13,13), aperture_radius=4.) result_tab = photometry(image=star, init_guesses=pos) residual_image = photometry.get_residual_image() psf_df[source_name+' Flux'][i] = result_tab['flux_fit'][0] psf_df[source_name+' Flux Error'][i] = result_tab['flux_unc'][0] if plots: fig, ax = plt.subplots(nrows=1, ncols=3, figsize=(12,4)) im = ax[0].imshow(star, origin='lower') divider = make_axes_locatable(ax[0]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') ax[0].plot(result_tab['x_fit'][0], result_tab['y_fit'][0], 'rx') ax[0].set_title('Data') im2 = ax[1].imshow(epsf.data, origin='lower') ax[1].set_title('EPSF Model') divider = make_axes_locatable(ax[1]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im2, cax=cax, orientation='vertical') im3 = ax[2].imshow(residual_image, origin='lower') ax[2].set_title('Residual Image') divider = make_axes_locatable(ax[2]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im3, cax=cax, orientation='vertical') plt.suptitle(source_name+'\n'+reduced_files[i].name+', image '+str(i+1)+' of '+str(len(reduced_files))) plt.subplots_adjust(wspace=0.5, top=0.95, bottom = 0.05) plot_output_name = pines_path/('Objects/'+short_name+'/psf_phot/'+source_name+'/'+str(i).zfill(4)+'.jpg') plt.savefig(plot_output_name) plt.close() except: print('') print('EPSF BUILDER FAILED, SKIPPING IMAGE.') print('') #Plot the ePSF. # plt.figure() # norm = simple_norm(epsf.data, 'log', percent=99.) # plt.imshow(epsf.data, norm=norm, origin='lower', cmap='viridis') # cb = plt.colorbar() # plt.tight_layout() print('Saving psf photometry output to {}.'.format(output_filename)) with open(output_filename, 'w') as f: for j in range(len(psf_df)): if j == 0: f.write('{:>21s}, {:>22s}, {:>17s}, {:>7s}, {:>7s}, '.format('Filename', 'Time UT', 'Time JD', 'Airmass', 'Seeing')) for i in range(len(source_names)): if i != len(source_names) - 1: f.write('{:>20s}, {:>26s}, '.format(source_names[i]+' Flux', source_names[i]+' Flux Error')) else: f.write('{:>20s}, {:>26s}\n'.format(source_names[i]+' Flux', source_names[i]+' Flux Error')) format_string = '{:21s}, {:22s}, {:17.9f}, {:7.2f}, {:7.1f}, ' #If the seeing value for this image is 'nan' (a string), convert it to a float. #TODO: Not sure why it's being read in as a string, fix that. if type(psf_df['Seeing'][j]) == str: psf_df['Seeing'][j] = float(psf_df['Seeing'][j]) #Do a try/except clause for writeout, in case it breaks in the future. try: f.write(format_string.format(psf_df['Filename'][j], psf_df['Time UT'][j], psf_df['Time JD'][j], psf_df['Airmass'][j], psf_df['Seeing'][j])) except: print('Writeout failed! Inspect quantities you are trying to write out.') pdb.set_trace() for i in range(len(source_names)): if i != len(source_names) - 1: format_string = '{:20.11f}, {:26.11f}, ' else: format_string = '{:20.11f}, {:26.11f}\n' f.write(format_string.format(psf_df[source_names[i]+' Flux'][j], psf_df[source_names[i]+' Flux Error'][j])) print('') return
def build_ePSF_imsegm(image_file, mask_file=None, nstars=40, thresh_sigma=5.0, pixelmin=20, etamax=1.4, areamax=500, cutout=35, write=True, output=None, plot=False, output_plot=None, verbose=False): """Build the effective Point-Spread Function using a sample of stars from some image acquired via image segmentation. Arguments --------- image_file : str Filename for a **background-subtracted** image mask_file : str, optional Filename for a mask file (default None) nstars : int, optional *Maximum* number of stars to use in building the ePSF (default 40; set to None to impose no limit) thresh_sigma : float, optional Sigma threshold for source detection with image segmentation (default 5.0) pixelmin : float, optional *Minimum* pixel area of an isophote to be considered a good source for building the ePSF (default 20) etamax : float, optional *Maximum* allowed elongation for an isophote to be considered a good source for building the ePSF (default 1.4) areamax : float, optional *Maximum* allowed area (in square pixels) for an isophote to be considered a good source for building the ePSF (default 500) cutout : int, optional Cutout size around each star in pixels (default 35; must be **odd**; rounded **down** if even) write : bool, optional Whether to write the ePSF to a new fits file (default True) output : str, optional Name for the output ePSF data fits file (default `image_file.replace(".fits", "_ePSF.fits")`) plot : bool, optional Whether to plot the newly-built ePSF (default False) output_plot : str, optional Name for the output figure (default `image_file.replace(".fits", "_ePSF.png")`) verbose : bool, optional Whether to be verbose (default False) Returns ------- np.ndarray The ePSF data in a 2D array Notes ----- Uses image segmentation via `photutils` to obtain a list of sources in the image with their x, y coordinates, flux, and background at their location. Then uses `EPSFBuilder` to empirically obtain the ePSF of these stars. Optionally writes and/or plots the obtained ePSF. **The ePSF obtained here should not be used in convolutions.** Instead, it can serve as a tool for estimating the seeing of an image. """ # ignore annoying warnings from photutils from astropy.utils.exceptions import AstropyWarning warnings.simplefilter('ignore', category=AstropyWarning) # imports from astropy.nddata import NDData from photutils.psf import extract_stars from photutils import EPSFBuilder # load in data image_data = fits.getdata(image_file) image_header = fits.getheader(image_file) try: instrument = image_header["INSTRUME"] except KeyError: instrument = "Unknown" ## source detection # add mask to image_data image_data = np.ma.masked_where(image_data == 0.0, image_data) # build an actual mask mask = (image_data == 0) if mask_file: mask = np.logical_or(mask, fits.getdata(mask_file)) # set detection standard deviation try: std = image_header["BKGSTD"] # header written by bkgsub function except KeyError: # make crude source mask, get standard deviation of background source_mask = make_source_mask(image_data, snr=3, npixels=5, dilate_size=15, mask=mask) final_mask = np.logical_or(mask, source_mask) std = np.std(np.ma.masked_where(final_mask, image_data)) # use the segmentation image to get the source properties segm = detect_sources(image_data, thresh_sigma * std, npixels=pixelmin, mask=mask) cat = source_properties(image_data, segm, mask=mask) ## get the catalog and coordinate/fluxes for sources, do some filtering try: tbl = cat.to_table() except ValueError: print("SourceCatalog contains no sources. Exiting.") return # restrict elongation and area to obtain only unsaturated stars tbl = tbl[(tbl["elongation"] <= etamax)] tbl = tbl[(tbl["area"].value <= areamax)] # build a table sources = Table() # build a table sources['x'] = tbl['xcentroid'] # for EPSFBuilder sources['y'] = tbl['ycentroid'] sources['flux'] = tbl['source_sum'].data / tbl["area"].data sources.sort("flux") sources.reverse() # restrict number of stars (if requested) if nstars: sources = sources[:min(nstars, len(sources))] ## get WCS coords for all sources w = wcs.WCS(image_header) sources["ra"], sources["dec"] = w.all_pix2world(sources["x"], sources["y"], 1) ## mask out edge sources: # a bounding circle for WIRCam, rectangle for MegaPrime xsize = image_data.shape[1] ysize = image_data.shape[0] if "WIRCam" in instrument: # bounding circle rad_limit = xsize / 2.0 dist_to_center = np.sqrt((sources['x'] - xsize / 2.0)**2 + (sources['y'] - ysize / 2.0)**2) mask = dist_to_center <= rad_limit sources = sources[mask] else: # rectangle x_lims = [int(0.05 * xsize), int(0.95 * xsize)] y_lims = [int(0.05 * ysize), int(0.95 * ysize)] mask = (sources['x'] > x_lims[0]) & (sources['x'] < x_lims[1]) & ( sources['y'] > y_lims[0]) & (sources['y'] < y_lims[1]) sources = sources[mask] ## empirically obtain the effective Point Spread Function (ePSF) nddata = NDData(image_data) # NDData object if mask_file: # supply a mask if needed nddata.mask = fits.getdata(mask_file) if cutout % 2 == 0: # if cutout even, subtract 1 cutout -= 1 stars = extract_stars(nddata, sources, size=cutout) # extract stars ## build the ePSF nstars_epsf = len(stars.all_stars) # no. of stars used in ePSF building if nstars_epsf == 0: print( "\nNo valid sources were found to build the ePSF with the given" + " conditions. Exiting.") return if verbose: print(f"{nstars_epsf} stars used in building the ePSF") epsf_builder = EPSFBuilder( oversampling=1, maxiters=7, # build it progress_bar=False) epsf, fitted_stars = epsf_builder(stars) epsf_data = epsf.data if write: # write, if desired epsf_hdu = fits.PrimaryHDU(data=epsf_data) if not (output): output = image_file.replace(".fits", "_ePSF.fits") epsf_hdu.writeto(output, overwrite=True, output_verify="ignore") if plot: # plot, if desired if not (output_plot): # set output name if not given output_plot = image_file.replace(".fits", "_ePSF.png") __plot_ePSF(epsf_data=epsf_data, output=output_plot) return epsf_data
def __fit_PSF(image_file, mask_file=None, nstars=40, thresh_sigma=5.0, pixelmin=20, elongation_lim=1.4, area_max=500, cutout=35, astrom_sigma=5.0, psf_sigma=5.0, alim=10000, clean=True, source_lim=None, write_ePSF=False, ePSF_output=None, plot_ePSF=True, ePSF_plotname=None, plot_residuals=False, resid_plotname=None, verbose=False): """ Input: general: - filename for a **BACKGROUND-SUBTRACTED** image - filename for a mask image (optional; default None) - maximum number of stars to use (optional; default 40; set to None to impose no limit) source detection: - sigma threshold for source detection with image segmentation (optional; default 5.0) - *minimum* number of isophotal pixels (optional; default 20) - *maximum* allowed elongation for sources found by image segmentation (optional; default 1.4) - *maximum* allowed area for sources found by image segmentation (optional; default 500 pix**2) - cutout size around each star in pix (optional; default 35 pix; must be ODD, rounded down if even) astrometry.net: - sigma threshold for astrometry.net source detection image (optional; default 5.0) - sigma of the Gaussian PSF of the image (optional; default 5.0) - maximum allowed source area in pix**2 for astrometry.net for deblending (optional; default 10000; only relevant if no source list file is provided) - whether to remove files output by image2xy once finished with them (optional; default True) misc: - limit on number of sources to fit with ePSF (optional; default None which imposes no limit) writing, plotting, verbosity: - whether to write the derived ePSF to a fits file (optional; default False) - name for output ePSF fits file (optional; default set below) - whether to plot the derived ePSF (optional; default True) - name for output ePSF plot (optional; default set below) - whether to plot the residuals of the iterative PSF fitting (optional; default False) - name for output residuals plot (optional; default set below) - be verbose (optional; default False) Uses image segmentation to obtain a list of sources in the image with their x, y coordinates. Uses EPSFBuilder to empirically obtain the ePSF of these stars. Optionally writes and/or plots the obtaind ePSF. Finally, uses astrometry.net to find all sources in the image, and fits them with the empirically obtained ePSF. The ePSF obtained here should NOT be used in convolutions. Instead, it can serve as a tool for estimating the seeing of an image. Output: table containing the coordinates and instrumental magnitudes of the detected, ePSF-fit sources """ # load in data image_data = fits.getdata(image_file) image_header = fits.getheader(image_file) try: instrument = image_header["INSTRUME"] except KeyError: instrument = "Unknown" pixscale = image_header["PIXSCAL1"] ### SOURCE DETECTION ### use image segmentation to find sources with an area > pixelmin pix**2 ### which are above the threshold sigma*std image_data = fits.getdata(image_file) # subfile data image_data = np.ma.masked_where(image_data==0.0, image_data) # mask bad pixels ## build an actual mask mask = (image_data==0) if mask_file: mask = np.logical_or(mask, fits.getdata(mask_file)) ## set detection standard deviation try: std = image_header["BKGSTD"] # header written by amakihi.bkgsub fn except KeyError: # make crude source mask, get standard deviation of background source_mask = make_source_mask(image_data, snr=3, npixels=5, dilate_size=15, mask=mask) final_mask = np.logical_or(mask, source_mask) std = np.std(np.ma.masked_where(final_mask, image_data)) ## use the segmentation image to get the source properties # use <mask>, which does not mask sources segm = detect_sources(image_data, thresh_sigma*std, npixels=pixelmin, mask=mask) cat = source_properties(image_data, segm, mask=mask) ## get the catalog and coordinates for sources try: tbl = cat.to_table() except ValueError: print("SourceCatalog contains no sources. Exiting.") return # restrict elongation and area to obtain only unsaturated stars tbl = tbl[(tbl["elongation"] <= elongation_lim)] tbl = tbl[(tbl["area"].value <= area_max)] sources = Table() # build a table sources['x'] = tbl['xcentroid'] # for EPSFBuilder sources['y'] = tbl['ycentroid'] sources['flux'] = tbl['source_sum'].data/tbl["area"].data sources.sort("flux") sources.reverse() if nstars: sources = sources[:min(nstars, len(sources))] ## setup: get WCS coords for all sources w = wcs.WCS(image_header) sources["ra"], sources["dec"] = w.all_pix2world(sources["x"], sources["y"], 1) ## mask out edge sources: # a bounding circle for WIRCam, rectangle for MegaPrime xsize = image_data.shape[1] ysize = image_data.shape[0] if "WIRCam" in instrument: rad_limit = xsize/2.0 dist_to_center = np.sqrt((sources['x']-xsize/2.0)**2 + (sources['y']-ysize/2.0)**2) dmask = dist_to_center <= rad_limit sources = sources[dmask] else: x_lims = [int(0.05*xsize), int(0.95*xsize)] y_lims = [int(0.05*ysize), int(0.95*ysize)] dmask = (sources['x']>x_lims[0]) & (sources['x']<x_lims[1]) & ( sources['y']>y_lims[0]) & (sources['y']<y_lims[1]) sources = sources[dmask] ## empirically obtain the effective Point Spread Function (ePSF) nddata = NDData(image_data) # NDData object if mask_file: # supply a mask if needed nddata.mask = fits.getdata(mask_file) if cutout%2 == 0: # if cutout even, subtract 1 cutout -= 1 stars = extract_stars(nddata, sources, size=cutout) # extract stars ## build the ePSF nstars_epsf = len(stars.all_stars) # no. of stars used in ePSF building if nstars_epsf == 0: print("\nNo valid sources were found to build the ePSF with the given"+ " conditions. Exiting.") return if verbose: print(f"\n{nstars_epsf} stars used in building the ePSF") start = timer() epsf_builder = EPSFBuilder(oversampling=1, maxiters=7, # build it progress_bar=False) epsf, fitted_stars = epsf_builder(stars) epsf_data = epsf.data end = timer() # timing time_elaps = end-start # print ePSF FWHM, if desired print(f"Time required for ePSF building {time_elaps:.2f} s\n") if verbose: ePSF_FWHM(epsf_data, True) epsf_hdu = fits.PrimaryHDU(data=epsf_data) if write_ePSF: # write, if desired if not(ePSF_output): ePSF_output = image_file.replace(".fits", "_ePSF.fits") epsf_hdu.writeto(ePSF_output, overwrite=True, output_verify="ignore") psf_model = epsf # set the model psf_model.x_0.fixed = True # fix centroids (known beforehand) psf_model.y_0.fixed = True ### USE ASTROMETRY.NET TO FIND SOURCES TO FIT # -b --> no background-subtraction # -O --> overwrite # -p <astrom_sigma> --> signficance # -w <psf_sigma> --> estimated PSF sigma # -m <alim> --> max object size for deblending is <alim> options = f"-O -b -p {astrom_sigma} -w {psf_sigma}" options += f" -m {alim}" run(f"image2xy {options} {image_file}", shell=True) image_sources_file = image_file.replace(".fits", ".xy.fits") image_sources = fits.getdata(image_sources_file) if clean: run(f"rm {image_sources_file}", shell=True) # this file is not needed print(f'\n{len(image_sources)} stars at >{astrom_sigma}'+ f' sigma found in image {re.sub(".*/", "", image_file)}'+ ' with astrometry.net') astrom_sources = Table() # build a table astrom_sources['x_mean'] = image_sources['X'] # for BasicPSFPhotometry astrom_sources['y_mean'] = image_sources['Y'] astrom_sources['flux'] = image_sources['FLUX'] # initial guesses for centroids, fluxes pos = Table(names=['x_0', 'y_0','flux_0'], data=[astrom_sources['x_mean'], astrom_sources['y_mean'], astrom_sources['flux']]) ### FIT THE ePSF TO ALL DETECTED SOURCES start = timer() # timing the fit # sources separated by less than this critical separation are grouped # together when fitting the PSF via the DAOGROUP algorithm sigma_psf = 2.0 # 2 pix crit_sep = 2.0*sigma_psf*gaussian_sigma_to_fwhm # twice the PSF FWHM daogroup = DAOGroup(crit_sep) # an astropy fitter, does Levenberg-Marquardt least-squares fitting fitter_tool = LevMarLSQFitter() # if we have a limit on the number of sources to fit if source_lim: try: import random # pick a given no. of random sources source_rows = random.choices(astrom_sources, k=source_lim) astrom_sources = Table(names=['x_mean', 'y_mean', 'flux'], rows=source_rows) pos = Table(names=['x_0', 'y_0','flux_0'], data=[astrom_sources['x_mean'], astrom_sources['y_mean'], astrom_sources['flux']]) except IndexError: print("The input source limit exceeds the number of sources"+ " detected by astrometry, so no limit is imposed.\n") photometry = BasicPSFPhotometry(group_maker=daogroup, bkg_estimator=None, # bg subtract already done psf_model=psf_model, fitter=fitter_tool, fitshape=(11,11)) result_tab = photometry(image=image_data, init_guesses=pos) # results residual_image = photometry.get_residual_image() # residuals of PSF fit residual_image = np.ma.masked_where(mask, residual_image) residual_image.fill_value = 0 # set to zero residual_image = residual_image.filled() end = timer() # timing time_elaps = end - start print(f"Time required fit ePSF to all sources {time_elaps:.2f} s\n") # include WCS coordinates pos["ra"], pos["dec"] = w.all_pix2world(pos["x_0"], pos["y_0"], 1) result_tab.add_column(pos['ra']) result_tab.add_column(pos['dec']) # mask out negative flux_fit values in the results mask_flux = (result_tab['flux_fit'] >= 0.0) psf_sources = result_tab[mask_flux] # PSF-fit sources # compute magnitudes and their errors and add to the table # error = (2.5/(ln(10)*flux_fit))*flux_unc mag_fit = -2.5*np.log10(psf_sources['flux_fit']) # instrumental mags mag_fit.name = 'mag_fit' mag_unc = 2.5/(psf_sources['flux_fit']*np.log(10)) mag_unc *= psf_sources['flux_unc'] mag_unc.name = 'mag_unc' psf_sources['mag_fit'] = mag_fit psf_sources['mag_unc'] = mag_unc # mask entries with large magnitude uncertainties mask_unc = psf_sources['mag_unc'] < 0.4 psf_sources = psf_sources[mask_unc] if plot_ePSF: # if we wish to see the ePSF plt.figure(figsize=(10,9)) plt.imshow(epsf_data, origin='lower', aspect=1, cmap='magma', interpolation="nearest") plt.xlabel("Pixels", fontsize=16) plt.ylabel("Pixels", fontsize=16) plt.title("Effective Point-Spread Function (1 pixel = " +str(pixscale)+ '")', fontsize=16) plt.colorbar(orientation="vertical", fraction=0.046, pad=0.08) plt.rc("xtick",labelsize=16) # not working? plt.rc("ytick",labelsize=16) if not(ePSF_plotname): ePSF_plotname = image_file.replace(".fits", "_ePSF.png") plt.savefig(ePSF_plotname, bbox_inches="tight") plt.close() if plot_residuals: # if we wish to see a plot of the residuals if "WIRCam" in instrument: plt.figure(figsize=(10,9)) else: plt.figure(figsize=(12,14)) ax = plt.subplot(projection=w) plt.imshow(residual_image, cmap='magma', aspect=1, interpolation='nearest', origin='lower') plt.xlabel("RA (J2000)", fontsize=16) plt.ylabel("Dec (J2000)", fontsize=16) plt.title("PSF residuals", fontsize=16) cb = plt.colorbar(orientation='vertical', fraction=0.046, pad=0.08) cb.set_label(label="ADU", fontsize=16) ax.coords["ra"].set_ticklabel(size=15) ax.coords["dec"].set_ticklabel(size=15) if not(resid_plotname): resid_plotname = image_file.replace(".fits", "_ePSFresiduals.png") plt.savefig(resid_plotname, bbox_inches="tight") plt.close() return psf_sources
def generate_PSFs(self, equivalent_radius=2., size=20., oversampling=1, plot=None, filepaths=None): ''' Generate effective point spread fuctions (ePSFs) for each image Parameters ---------- equivalent_radius : float, unit arcsec radius criteria to indentify star size : float, unit pixel use what size box to extract stars oversampling : int oversample the ePSF plot : None for not plot stars & ePSF list like [1,2,3] to plot rgb image filepaths : filepath to store the ePSFs ''' stars = self.common_catalog.copy() remolist = [] for loop in range(len(stars)): for loop2 in range(self.__length): a = ( self.image_list[loop2].sources_catalog['equivalent_radius'] [stars['sloop_{0}'.format(loop2 + 1)][loop]] ) * self.image_list[loop2].pixel_scales[0].value if (a > equivalent_radius): remolist.append(loop) break stars.remove_rows(remolist) star_images = [] PSFs = [] for loop2 in range(self.__length): newsc = self.image_list[loop2].sources_catalog.copy() indexes = np.delete( np.arange(len(self.image_list[loop2].sources_catalog)), stars['sloop_{0}'.format(loop2 + 1)]) newsc.remove_rows(indexes) stars_tbl = Table() stars_tbl['x'] = np.array(newsc['maxval_xpos']) stars_tbl['y'] = np.array(newsc['maxval_ypos']) nddata = NDData(data=np.array(self.image_list[loop2].ss_data)) Tstar = extract_stars(nddata, stars_tbl, size=size) epsf_builder = EPSFBuilder(oversampling=oversampling, maxiters=15, progress_bar=False) epsf, fitted_stars = epsf_builder(Tstar) self.image_list[loop2].PSF = epsf.data if filepaths is not None: hdu = fits.PrimaryHDU(epsf.data.astype('float32')) After = fits.HDUList([hdu]) After.writeto(filepaths[loop2], overwrite=True) if plot is not None: star_images.append(Tstar) PSFs.append(epsf.data) if plot is not None: tlens = len(stars) if (((tlens // 5) + 1) * 5 - tlens) < (( (tlens // 4) + 1) * 4 - tlens): ncols = 5 nrows = (tlens // 5) + 1 else: ncols = 4 nrows = (tlens // 4) + 1 fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(3 * ncols, 3 * nrows), squeeze=True) ax = ax.ravel() for i in range(tlens): if len(plot) > 2: star_b = star_images[plot[0]][i].data * 100. / np.sum( star_images[plot[0]][i].data) star_g = star_images[plot[1]][i].data * 100. / np.sum( star_images[plot[1]][i].data) star_r = star_images[plot[2]][i].data * 100. / np.sum( star_images[plot[2]][i].data) norm = simple_norm(star_b, 'log', percent=99.) image = make_lupton_rgb(star_r, star_g, star_b, Q=10) else: image = star_images[plot[0]][i].data norm = simple_norm(image, 'log', percent=99.) ax[i].imshow(image, norm=norm, origin='lower') plt.show() fig = plt.figure(figsize=(10, 10)) if len(plot) > 2: star_b = PSFs[plot[0]] * 100. / np.sum(PSFs[plot[0]]) star_g = PSFs[plot[1]] * 100. / np.sum(PSFs[plot[1]]) star_r = PSFs[plot[2]] * 100. / np.sum(PSFs[plot[2]]) norm = simple_norm(star_b, 'log', percent=99.) image = make_lupton_rgb(star_r, star_g, star_b, Q=10) else: image = PSFs[plot[0]] norm = simple_norm(image, 'log', percent=99.) plt.imshow(image, norm=norm, origin='lower') plt.show()
data += make_noise_image(data.shape, type='gaussian', mean=10., stddev=5., random_state=12345) from photutils import find_peaks peaks_tbl = find_peaks(data, threshold=500.) from astropy.table import Table stars_tbl = Table() stars_tbl['x'] = peaks_tbl['x_peak'] stars_tbl['y'] = peaks_tbl['y_peak'] from astropy.stats import sigma_clipped_stats mean_val, median_val, std_val = sigma_clipped_stats(data, sigma=2., iters=None) data -= median_val from astropy.nddata import NDData nddata = NDData(data=data) from photutils.psf import extract_stars stars = extract_stars(nddata, stars_tbl, size=25) import matplotlib.pyplot as plt nrows = 5 ncols = 5 fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20), squeeze=True) ax = ax.ravel() for i in range(nrows*ncols): norm = simple_norm(stars[i], 'log', percent=99.) ax[i].imshow(stars[i], norm=norm, origin='lower', cmap='viridis')
def find_pinholes_irregular(fname, freference, sname, fdarkff, fdark, fff, files, size, threshold, fwhm, fitshape, MAX_CONTROL_POINTS, PIXEL_TOL, range_psf, sigma=2., oversampling=4, maxiters=3, MIN_MATCHES_FRACTION=0.8, NUM_NEAREST_NEIGHBORS=5): """Finds and fits irregularly spread pinhole positions with a ePSF in a FITS image. Then matches them to the reference positions. Parameters ---------- fname : str Folder name of the input fits files. freference : str File name of the reference positions (txt file). sname : str Folder name of the returned found and matched pinhole positions (txt files). fdarkff : string Location of the dark images for the flat field images. fdark : string Location of the dark images for the raw images. fff : string Location of the flat field images. files : (1, 2)-shaped int array File range to create a median image size : int Rectangular size of the ePSF. Size must be an odd number. threshold : float The absolute image value above which to select sources. fwhm : float The full-width half-maximum (FWHM) of the major axis of the Gaussian kernel in units of pixels. fitshape : int or length-2 array-like Rectangular shape around the center of a star which will be used to collect the data to do the fitting. Can be an integer to be the same along both axes. E.g., 5 is the same as (5, 5), which means to fit only at the following relative pixel positions: [-2, -1, 0, 1, 2]. Each element of fitshape must be an odd number. MAX_CONTROL_POINTS : int The maximum control points (stars) to use to build the invariants. PIXEL_TOL : int The pixel distance tolerance to assume two invariant points are the same. range_psf : (1, 4)-shaped int array Position range to compute epsf [xmin,xmax,ymin,ymax] sigma : float Number of standard deviations used to perform sigma clip with a astropy.stats.SigmaClip object. oversampling : int or tuple of two int The oversampling factor(s) of the ePSF relative to the input stars along the x and y axes. The oversampling can either be a single float or a tuple of two floats of the form (x_oversamp, y_oversamp). If oversampling is a scalar then the oversampling will be the same for both the x and y axes. maxiters : int The maximum number of iterations to perform. MIN_MATCHES_FRACTION : float (0,1] The minimum fraction of triangle matches to accept a transformation. If the minimum fraction yields more than 10 triangles, 10 is used instead. NUM_NEAREST_NEIGHBORS : int The number of nearest neighbors of a given star (including itself) to construct the triangle invariants. Returns ------- s_list : (N,2)-shaped array Found and matched positions of the pinholes. t_list : (N,2)-shaped array Matched reference grid positions. """ #Load the sample of fits images entries = os.listdir(fname) data_col = np.array([fits.getdata(fname + '/' + entries[files[0]], ext=0)]) for k in range(files[0] + 1, files[1] + 1): data_col = np.append(data_col, [fits.getdata(fname + '/' + entries[k], ext=0)], axis=0) #Data reduction: Darc current + Flatfield data_col = data_correction(data_col, fdarkff, fdark, fff) #Claculate median image data_full = np.median(data_col, axis=0) pos_full = np.array([[0, 0]]) data = data_full[range_psf[2]:range_psf[3], range_psf[0]:range_psf[1]] #Find peaks in data peaks_tbl = find_peaks(data, threshold=threshold) peaks_tbl['peak_value'].info.format = '%.8g' #Load data around found peaks hsize = (size - 1) / 2 x = peaks_tbl['x_peak'] y = peaks_tbl['y_peak'] mask = ((x > hsize) & (x < (data.shape[1] - 1 - hsize)) & (y > hsize) & (y < (data.shape[0] - 1 - hsize))) stars_tbl = Table() stars_tbl['x'] = x[mask] stars_tbl['y'] = y[mask] #Calculate mean, median, std mean_val, median_val, std_val = sigma_clipped_stats(data, sigma=sigma) data = data - median_val #Find pinholes and create ePSF nddata = NDData(data=data) stars = extract_stars(nddata, stars_tbl, size=size) epsf_builder = EPSFBuilder(oversampling=oversampling, maxiters=maxiters, progress_bar=False) epsf, fitted_stars = epsf_builder(stars) #Use ePSF to find precise locations of pinholes daofind = DAOPhotPSFPhotometry(crit_separation=30, threshold=threshold, fwhm=fwhm, psf_model=epsf, fitshape=fitshape, aperture_radius=12, niters=1) #Get positions sources = daofind(data_full) for col in sources.colnames: sources[col].info.format = '%.8g' pos = np.transpose((sources['x_fit'], sources['y_fit'])) pos_full = np.append(pos_full, pos, axis=0) pos_full = pos_full[1:] #Plot found pinholes apertures = CircularAperture(pos_full, r=10) norm = ImageNormalize(stretch=SqrtStretch()) #Plot found pinholes fig, ax = plt.subplots() ax.set_title('Pinhole Positions') ax.set(xlabel='x [pixel]', ylabel='y [pixel]') ax.imshow(data_full, cmap='Greys', origin='lower', norm=norm) apertures.plot(color='blue', lw=1.5, alpha=0.5) ax.legend(['#pinholes = ' + str(len(pos_full[:, 0]))], loc='lower left', prop={'size': 12}) plt.show() #Sort positions by matching with reference grid positions_sort = pos_full ref_positions = np.genfromtxt(freference, skip_header=0) transf, (s_list, t_list) = find_transform(positions_sort, ref_positions, MAX_CONTROL_POINTS, PIXEL_TOL, MIN_MATCHES_FRACTION, NUM_NEAREST_NEIGHBORS) text = np.array([s_list[:, 0], s_list[:, 1], t_list[:, 0], t_list[:, 1]]) text_trans = np.zeros((len(s_list[:, 0]), 4)) #Transpose text matrix for k in range(0, 4): for l in range(0, len(s_list[:, 0])): text_trans[l][k] = text[k][l] #Save data as txt file np.savetxt(sname + '.txt', text_trans, fmt='%1.9E', delimiter='\t', comments='', header='x-measured y-measured x-reference y-reference') return s_list, t_list
if not os.path.exists(epsf_file): mean, median, std = sigma_clipped_stats(image) threshold = median + 10 * std finder = IRAFStarFinder(threshold=threshold, fwhm=4, minsep_fwhm=5, peakmax=image.max() / 0.8) star_table = finder(image) star_table.rename_columns(('xcentroid', 'ycentroid'),('x','y')) sigma_clip = SigmaClip(sigma=5.0) bkg_estimator = MMMBackground() bkg = Background2D(image, 5, filter_size=(3, 3), sigma_clip=sigma_clip, bkg_estimator=bkg_estimator) nddata = NDData(image-bkg.background) stars = extract_stars(nddata, star_table, size=51) epsf, fitted_stars = EPSFBuilder(oversampling=4, maxiters=3, progress_bar=True, smoothing_kernel='quadratic')(stars) epsf_model = prepare_psf_model(epsf, renormalize_psf=False) with open(epsf_file,'wb') as f: pickle.dump([epsf_model, finder], f) else: with open(epsf_file, 'rb') as f: epsf_model, finder = pickle.load(f) phot = IterativelySubtractedPSFPhotometry(group_maker=DAOGroup(5), bkg_estimator=MMMBackground(), psf_model=epsf_model, fitshape=[31,31], finder=finder,