def interped(self): if not hasattr(self, '_interped'): kernel = Box2DKernel(5) # Gaussian2DKernel(stddev=2.5) # crmask, _ = detect_cosmics(indat=np.ascontiguousarray( self.bkg_sub_img.filled(-9999)), inmask=self.bkg_sub_img.mask, sigclip=6., cleantype='medmask') self.bkg_sub_img.mask = np.ma.mask_or(self.bkg_sub_img.mask, crmask) self.bkg_sub_img.mask = np.ma.mask_or(self.bkg_sub_img.mask, np.isnan(self.bkg_sub_img)) print(('Masked pixels: ', np.sum(self.bkg_sub_img.mask))) img = self.bkg_sub_img.filled(np.nan) img_interp = interpolate_replace_nans(img, kernel, convolve=conv) while np.any(np.isnan(img_interp)): img_interp = interpolate_replace_nans(img_interp, kernel, convolve=conv) # clipped = sigma_clip(self.bkg_sub_img, # iters=5, sigma_upper=40).filled(np.nan) # img_interp = interpolate_replace_nans(img_interp, kernel) self._interped = img_interp return self._interped
def smoothing(blue_path, red_path, tell_wave, tell_flux, hi, lo, mid, mid2, pre): #Import co-added target spectra blue_coadd1d = fits.open(blue_path) blue_dat = blue_coadd1d[1].data blue_coadd1d.close() red_coadd1d = fits.open(red_path) red_dat = red_coadd1d[1].data red_coadd1d.close() #Get wavelengths and fluxes blue_wave = blue_dat['wave'] red_wave = red_dat['wave'] blue_ivar = blue_dat['ivar'] red_ivar = red_dat['ivar'] flux = np.copy(tell_flux) total_wave = np.concatenate((blue_wave, red_wave)) noise = 1/np.sqrt(np.concatenate((blue_ivar, red_ivar))) #Get resolution resb, resr = tell.lris_res(blue_wave, red_wave) total_res = np.concatenate((resb, resr)) #Get pixel positions of noisy telluric spikes spikes_hi, spikes_lo, spikes_mid, spikes_mid2, spikes_early = spikes(total_wave, flux, hi, lo, mid, mid2, pre) bad_pixels = [] for i in range(len(spikes_hi)): for j in range(len(spikes_hi[i])): bad_pixels.append(spikes_hi[i][j]) for i in range(len(spikes_lo)): for j in range(len(spikes_lo[i])): bad_pixels.append(spikes_lo[i][j]) for i in range(len(spikes_mid)): bad_pixels.append(spikes_mid[i]) for i in range(len(spikes_mid2)): bad_pixels.append(spikes_mid2[i]) for i in range(len(spikes_early)): bad_pixels.append(spikes_early[i]) bad_pixels = np.sort(list(set(bad_pixels))) #Remove duplicates #Interpolate over noisy spikes flux[bad_pixels] = np.nan noise[bad_pixels] = np.nan kernel = Gaussian1DKernel(stddev = 1) intp_flux = interpolate_replace_nans(flux, kernel, convolve = convolve_fft) intp_noise = interpolate_replace_nans(noise, kernel, convolve = convolve_fft) #Smooth by 200km/s c = 299792.458 #speed of light in_sigma_kms = 200 sigma_aa_desired = in_sigma_kms/c*total_wave smoothed_flux = utils.smoothing.smoothspec(total_wave, intp_flux, outwave=total_wave, smoothtype='lsf', resolution=sigma_aa_desired) smoothed_noise = utils.smoothing.smoothspec(total_wave, intp_noise, outwave=total_wave, smoothtype='lsf', resolution=sigma_aa_desired) return total_wave, smoothed_flux, smoothed_noise, bad_pixels
def __call__(self, abs_sci_name): ''' Bad pix fixing, for a single frame so as to parallelize job INPUTS: sci_name: science array filename ''' # read in the science frame from raw data directory sci, header_sci = fits.getdata(abs_sci_name, 0, header=True) # fix bad pixels (note conversion to 32-bit signed ints) sci_badnan = np.multiply(sci, self.badpix) image_fixpixed = interpolate_replace_nans(array=sci_badnan, kernel=self.kernel).astype( np.int32) # add a line to the header indicating last reduction step header_sci["RED_STEP"] = "bad-pixel-fixed" # write file out abs_image_fixpixed_name = str(self.config_data["data_dirs"]["DIR_PIXL_CORRTD"] + \ os.path.basename(abs_sci_name)) fits.writeto(filename=abs_image_fixpixed_name, data=image_fixpixed, header=header_sci, overwrite=True) print("Writing out bad-pixel-fixed frame " + os.path.basename(abs_image_fixpixed_name))
def fix_outlier(self, arr, interpolate=True, sigma=5, iters=2): kernel = Gaussian2DKernel(x_stddev=self.psf / 2.355) new_array = sigma_clip(arr, sigma=sigma, iters=iters) # remove 5 sigma outlier if interpolate: new_array = interpolate_replace_nans(new_array.filled(np.nan), kernel) return new_array
def loadmap_astroclean(image, x_stddev=1, x_size=8, y_size=8): #9x9 kernel for smoothing kernel = Gaussian2DKernel(x_stddev, x_size, y_size) image = imread(image) imageclean = interpolate_replace_nans(image, kernel) return imageclean
def quick_plot(array, interp=False): if interp: array = interpolate_replace_nans(array, kernel=Gaussian2DKernel(x_stddev=0.25)) plt.ion() plt.figure() norm = ImageNormalize(array, interval=ZScaleInterval()) im = plt.imshow(array, origin='lower', norm=norm) cb = plt.colorbar(im) plt.tight_layout()
def fix_bad_pixels(image, bad_map, add_bad=[], x_stddev=1): """ Replace bad pixels with values interpolated from their neighbors (interpolation is made with a gaussian kernel convolution).""" if len(add_bad) != 0: for j in range(len(add_bad)): bad_map[add_bad[j][1], add_bad[j][0]] = 1 img_nan = image.copy() img_nan[bad_map == 1] = np.nan kernel = Gaussian2DKernel(x_stddev=x_stddev) fixed_image = interpolate_replace_nans(img_nan, kernel) return fixed_image
def extend_field(grid, mask=None, gauss_widths=(1, 11), taper=None): """Extrapolate w/Gaussian average of increasing width and taper weights. Extend field (internal and external borders) by applying a Gaussian average of increasing width at each iteration, and scaling the result by a cosine/linear taper (from 1 to 0 with iterations). Args: grid: 2D field to extrapolate. mask: region to exclude from the Gaussian average. gauss_widths: width (n_pixels) of Gaussian average at each iteration. taper: cosine|linear|None, shape of the weight window. Notes: Cosine taper goes from 1 at gauss_widths[0] to 0 at gauss_widths[-1]. gauss_widths define the number of iterations (one per width). Widths can be passed as a range (1, N) or explicitly (1, 3, 5, ...). """ if len(gauss_widths) == 2: gauss_widths = range(*gauss_widths) if taper == "cosine": half_window = len(gauss_widths) taper_weights = tukey(half_window * 2)[half_window:] elif taper == "linear": taper_weights = np.linspace(1, 0, len(gauss_widths)) else: taper_weights = np.full_like(gauss_widths, 1) print("taper weights:\n", taper_weights) if mask is not None: grid[mask == 1] = np.nan # <= this is key !!! for i, k in enumerate(gauss_widths): print("gauss kernel size:", k) mask_before = np.isnan(grid) kernel = Gaussian2DKernel(k) grid = interpolate_replace_nans(grid, kernel, boundary="extend") if mask is not None: grid[mask == 1] = np.nan mask_after = np.isnan(grid) mask_extended = mask_before & ~mask_after grid[mask_extended] *= taper_weights[i] return grid
def maybe_interpolate_dead_pixels(self): if self.interpolate_dead_pixels: print('Interpolating dead pixels...') kernel = Gaussian2DKernel(1) not_valid_and_inside_brightfield = np.logical_and( np.logical_not(self.bvm), self.bmb) self.data[not_valid_and_inside_brightfield] = np.NaN # p = Pool(multiprocessing.cpu_count()) # p.map(replace_nans, data) for i, da in enumerate(self.data): self.data[i] = interpolate_replace_nans(da.copy(), kernel) self.data = np.nan_to_num(self.data, copy=False)
def identify_sky_pixels(sky, kernel=10.0): G = Gaussian1DKernel(kernel) cont = convolve(sky, G, boundary='extend') mask = safe_sigma_clip(sky - cont) for i in np.arange(5): nsky = sky * 1. mask.mask[1:] += mask.mask[:-1] mask.mask[:-1] += mask.mask[1:] nsky[mask.mask] = np.nan cont = convolve(nsky, G, boundary='extend') while np.isnan(cont).sum(): cont = interpolate_replace_nans(cont, G) mask = safe_sigma_clip(sky - cont) return mask.mask, cont
def interped(self): if not hasattr(self, "_interped"): kernel = Box2DKernel(5) # Gaussian2DKernel(stddev=2.5) # crmask, _ = detect_cosmics( indat=np.ascontiguousarray(self.bkg_sub_img.filled(-9999)), inmask=self.bkg_sub_img.mask, sigclip=6.0, cleantype="medmask", ) self.bkg_sub_img.mask = np.ma.mask_or(self.bkg_sub_img.mask, crmask) self.bkg_sub_img.mask = np.ma.mask_or(self.bkg_sub_img.mask, np.isnan(self.bkg_sub_img)) img = self.bkg_sub_img.filled(np.nan) img_interp = interpolate_replace_nans(img, kernel, convolve=conv) while np.any(np.isnan(img_interp)): img_interp = interpolate_replace_nans(img_interp, kernel, convolve=conv) self._interped = img_interp return self._interped
def fill_gaps( self, size=[3, 3] ): #Function fills any gaps, experimental version at the moment, comment out this one and uncomment the above one to go back to the old versoin n_lines, ny, n_velocity, nx = shape(self.cube) #Calculate pixel sizes mask = zeros([ny + 2, nx + 2], dtype=int) #Create an array to store pixle kernel = Gaussian2DKernel(stddev=0.25, x_size=size[0], y_size=size[1]) structure = ones(size, dtype=int) for i in xrange(n_lines): for j in xrange(n_velocity): cube_slice = self.cube[i, :, j, :] var_slice = self.var[i, :, j, :] smoothed_cube_slice = interpolate_replace_nans(cube_slice, kernel=kernel) smoothed_var_slice = interpolate_replace_nans(var_slice, kernel=kernel) mask[1:-1, 1:-1][isfinite(cube_slice)] = 1 pixels_to_replace = (mask - binary_closing( mask, structure=structure))[1:-1, 1:-1] < 0 mask[:] = 0 cube_slice[pixels_to_replace] = smoothed_cube_slice[ pixels_to_replace] var_sclice = smoothed_var_slice[pixels_to_replace]
def replacenans_2d_interpolate(data): """Interpolate over nans within array using astropy interpolate_replace_nans function Input: data = np.array of data values Output: data_out = np.array of data value with no nan values""" # We smooth with a Gaussian kernel with x_stddev=1 (and y_stddev=1) # It is a 9x9 array kernel = Gaussian2DKernel(x_stddev=1) # create a "fixed" image with NaNs replaced by interpolated values data_out = interpolate_replace_nans(data, kernel) return data_out
def interpolate_replace(ima, pos, size = 20): kernel = Box2DKernel(3, mode='center') ima_ = ima for x,y in pos: x= int(round(x)) y= int(round(y)) ima_[y-size:y+size,x-r_out:x+size] = interpolate_replace_nans(ima[y-size:y+size,x-r_out:x+size].filled(), kernel) if False: print x,y plt.figure(35); plt.clf(); plt.ioff() fig, (ax0,ax1) = plt.subplots(ncols=2, nrows=1, num=33) ax0.imshow(ima[y-size:y+size,x-r_out:x+size]) ax1.imshow(ima_[y-size:y+size,x-r_out:x+size]) plt.show() return ima_
def fix_bad_pixels(image, bad_map, add_bad=None, x_stddev=1): """Replace bad pixels with values interpolated from their neighbors (interpolation is made with a gaussian kernel convolution).""" from astropy.convolution import Gaussian2DKernel, interpolate_replace_nans if add_bad is None: add_bad = [] if len(add_bad) != 0: bad_map = bad_map.copy() # Don't modify input bad pixel map, use a copy for j in range(len(add_bad)): bad_map[add_bad[j][1], add_bad[j][0]] = 1 img_nan = image.copy() img_nan[bad_map == 1] = np.nan kernel = Gaussian2DKernel(x_stddev=x_stddev) fixed_image = interpolate_replace_nans(img_nan, kernel) return fixed_image
def interpolate(self): """ interpolate zeros and negative values in data using FFT convolve function """ for hdu in self: if hdu.data is None: continue with np.errstate(invalid='ignore'): non_finite = ~np.isfinite(hdu.data) less_zero = hdu.data <= 0 if np.any(less_zero) or np.any(non_finite): data = hdu.data.astype(float) mask_data = np.ma.masked_less_equal(data, 0) # mask_data = np.ma.masked_inside(data, -1e5, 0) mask_data.fill_value = np.nan data = mask_data.filled() data = interpolate_replace_nans(data, self.kernel, convolve=convolve_fft, allow_huge=True) hdu.data = data
def interp_nans(data, x_stddev=1): """Interpolate over any NaNs present in a final mosaic. Uses the Astropy.convolution interpolate_replace_nans to smooth over any gaps left in an image. This may be particularly useful for WFPC2 images, where there are small gaps between chips. Args: data (numpy.ndarray): Input data to interpolate NaNs over. x_stddev (int, optional): Standard deviation of the Gaussian kernel. Defaults to 1 (pixel). Returns: numpy.ndarray: The data with NaNs interpolated over """ kernel = Gaussian2DKernel(x_stddev=x_stddev) image_interp = interpolate_replace_nans(data, kernel) return image_interp
def interpolate(img: CCDData): """ Takes a image with a mask for bad pixels and interpolates over the bad pixels :param img: the image you want to interpolate bad pixels in :return: interpolated image """ # TODO combiner does not care about this and marks it invalid still from astropy.convolution import CustomKernel from astropy.convolution import interpolate_replace_nans # TODO this here doesn't really work all that well -> extended regions cause artifacts at border kernel_array = np.array([[1, 1, 1], [1, 1, 1], [1, 1, 1] ]) / 9 # average of all surrounding pixels # noinspection PyTypeChecker kernel = CustomKernel( kernel_array ) # TODO the original pipeline used fixpix, which says it uses linear interpolation img.data[np.logical_not(img.mask)] = np.NaN img.data = interpolate_replace_nans(img.data, kernel) return img
def MaxFilter(img, wlen): img_conv_max = [ maximum_filter(interpolate_replace_nans(img[i], np.ones((29, 29))), size=wlen) for i in range(0, len(img)) ] return img_conv_max
def interpolate_nans_kernel(data, kernel, max_iterations=10, not_converge="keep", ignore_nans_in=None, plot=False): """ This function ... :param data: :param kernel: :param max_iterations: :param not_converge: :param ignore_nans_in: :param plot: :return: """ from astropy.convolution import interpolate_replace_nans # Debugging nans = np.isnan(data) nnans = np.sum(nans) log.debug("The number of NaNs at the start is " + str(nnans)) # Plot? if plot: plotting.plot_mask(nans, title="NaNs") # Interpolate result = interpolate_replace_nans(data, kernel) niterations = 1 # Plot? if plot: plotting.plot_mask(result, title="Result after iteration 1") # Plot? if plot and ignore_nans_in is not None: plotting.plot_mask(ignore_nans_in, title="ignoring NaNs") # Get the current number of nans # previous_nnans = None # don't get it for performance if ignore_nans_in is not None: nnans = np.sum(np.isnan(result[np.logical_not(ignore_nans_in)])) else: nnans = np.sum(np.isnan(result)) # Plot? if plot: plotting.plot_mask(np.isnan(result), title="NaNs after iteration 1") # Debugging log.debug("The number of NaN values after iteration 1 is " + str(nnans)) # Are there still NaNs? while nnans > 0: # Check number of iterations if max_iterations is not None and niterations == max_iterations: raise RuntimeError("The maximum number of iterations has been reached without success") # Debugging log.debug("Interpolation iteration " + str(niterations + 1) + " ...") # Perform next interpolation result = interpolate_replace_nans(result, kernel) # Increment the niterations counter niterations += 1 # Get the current number of nans previous_nnans = nnans if ignore_nans_in is not None: nnans = np.sum(np.isnan(result[np.logical_not(ignore_nans_in)])) else: nnans = np.sum(np.isnan(result)) # Plot? if plot: plotting.plot_mask(np.isnan(result), title="NaNs after iteration " + str(niterations)) condition = nnans > previous_nnans if ignore_nans_in else nnans >= previous_nnans # Not converging if condition: if not_converge == "keep": log.warning("The number of NaNs could not converge to zero: " + str(nnans) + " NaN values will remain") break # break the loop elif not_converge == "error": raise RuntimeError("The number of NaNs is not converging to zero (nnans = " + str(nnans) + ", previous nnans = " + str(previous_nnans) + ")") else: raise ValueError("Invalid option for 'not_converge'") # Debugging log.debug("The number of NaN values after iteration " + str(niterations) + " is " + str(nnans)) # Return the resulting data return result
def MedianFilter(img, wlen): img_conv_med = [ median_filter(interpolate_replace_nans(img[i], np.ones((29, 29))), size=wlen) for i in range(0, len(img)) ] return img_conv_med
def PINES_quicklook(image_name='test.fits', interp=True): calibration_path = '/Users/obs72/Desktop/PINES_scripts/Calibrations/' if image_name == 'test.fits': file_path = '/Users/obs72/Desktop/PINES_scripts/test_image/test.fits' else: date_string = image_name.split('.')[0] file_path = '/mimir/data/obs72/' + date_string + '/' + image_name if os.path.exists(file_path): header = fits.open(file_path)[0].header band = header['FILTNME2'] exptime = str(header['EXPTIME']) flat_path = calibration_path + 'Flats/master_flat_' + band + '.fits' if os.path.exists(flat_path): flat = fits.open(flat_path)[0].data else: print('ERROR: No ', band, '-band flat exists in ', calibration_path, 'Flats/...make one.') return #Select the master dark on-disk with the closest exposure time to exptime. dark_top_level_path = calibration_path + 'Darks/' dark_files = np.array(glob(dark_top_level_path + '*.fits')) dark_exptimes = np.array([ float(i.split('master_dark_')[1].split('.fits')[0]) for i in dark_files ]) dark_files = dark_files[np.argsort(dark_exptimes)] dark_exptimes = dark_exptimes[np.argsort(dark_exptimes)] dark_ind = np.where( abs(dark_exptimes - float(exptime)) == np.min(abs(dark_exptimes - float(exptime))))[0][0] dark_path = dark_files[dark_ind] dark = fits.open(dark_path)[0].data ut_str = header['DATE-OBS'].split( 'T')[0] + ' ' + header['DATE-OBS'].split('T')[1].split('.')[0] from_zone = tz.gettz('UTC') to_zone = tz.gettz('America/Phoenix') utc = datetime.datetime.strptime(ut_str, '%Y-%m-%d %H:%M:%S') utc = utc.replace(tzinfo=from_zone) local = utc.astimezone(to_zone) local_str = local.strftime('%Y-%m-%d %H:%M:%S') raw_image = fits.open(file_path)[0].data[0:1024, :] reduced_image = (raw_image - dark) / flat avg, med, std = sigma_clipped_stats(reduced_image) if interp: bpm = fits.open( '/Users/obs72/Desktop/PINES_scripts/Calibrations/Bad_pixel_masks/bpm.fits' )[0].data reduced_image[bpm == 1] = np.nan reduced_image = interpolate_replace_nans( reduced_image, kernel=Gaussian2DKernel(0.5)) fig, ax = plt.subplots(figsize=(9, 8)) divider = make_axes_locatable(ax) cax = divider.append_axes('right', size='5%', pad=0.05) ax.set_aspect('equal') norm = ImageNormalize(reduced_image, interval=ZScaleInterval()) im = ax.imshow(reduced_image, origin='lower', norm=norm) fig.colorbar(im, cax=cax, orientation='vertical', label='ADU') ax.set_title(file_path.split('/')[-1], fontsize=16) plt.tight_layout() breakpoint() plt.close() else: print('ERROR: file ', file_path, ' does not exist.') return
def make_data_dict(regions=['DR21C'],datadirs=['DR21C'],alignment_iteration=0,DIST=7,length=200,kernel_sigma=6,wavelength = '850'): """ :param regions: a list of regions to run :param datadirs: a list of directories that hold the data for each region listed in parameter regions. :param alignment_iteration: there will be multiple iterations of the alignment run - this 0-based integer designates which alignment iteration the output file describes :param DIST: the distance used for linear fitting and gaussian fitting (use width = RADIUS*2 + 1) :param length: the distance used for linear fitting and gaussian fitting (use width = RADIUS*2 + 1) :param kernel_sigma: the smoothing kernel (in pixels) to subtract largescale structure (high pass filter) """ # + ===================== + # | Global parameters | # + ===================== + tol = 0.05 REGIONS = {} for i in regions: REGIONS[i] = {wavelength:1} align_smooth_kernel = Gaussian2DKernel(x_stddev=kernel_sigma, y_stddev=kernel_sigma) data = defaultdict(dict) for region,datadir in zip(regions,datadirs): data[region] = defaultdict(dict) Dates850 = [] Dates450 = [] DataRoot = datadir + "/" # where all the data is stored files = [] for eachfile in os.listdir(DataRoot): if os.path.isfile(os.path.join(DataRoot, eachfile)): if eachfile.split('.')[-1] == 'sdf': if wavelength in eachfile: files.append(eachfile) #files = [f for f in os.listdir(DataRoot) if (os.path.isfile(os.path.join(DataRoot, f)) and # os.path.join(DataRoot, f)[-4:] ==".sdf")] # all the files in dir for this wavelength files = sorted(files) # sorting to ensure we select the correct first region if wavelength == '450': scale = 2 elif wavelength == '850': scale = 3 else: scale = 0 data[region][wavelength] = defaultdict(dict) data[region][wavelength]['epoch'] = defaultdict(list) data[region][wavelength]['dates'] = list() # to collect all of the dates in the data[region] set data[region][wavelength]['JCMT_offset'] = defaultdict(str) # to use the date as the index data[region][wavelength]['header'] = defaultdict(dict) data[region][wavelength]['XC'] = defaultdict(dict) data[region][wavelength]['XC']['offset'] = defaultdict(list) data[region][wavelength]['XC']['offset_err'] = defaultdict(list) data[region][wavelength]['XC']['alignment'] = defaultdict(list) data[region][wavelength]['linear'] = defaultdict(dict) data[region][wavelength]['linear']['m'] = defaultdict(dict) data[region][wavelength]['linear']['m_err'] = defaultdict(dict) data[region][wavelength]['linear']['b'] = defaultdict(dict) data[region][wavelength]['linear']['b_err'] = defaultdict(dict) data[region][wavelength]['AC'] = defaultdict(dict) data[region][wavelength]['AC']['beam'] = defaultdict(list) data[region][wavelength]['AC']['amp'] = defaultdict(list) data[region][wavelength]['AC']['amp_err'] = defaultdict(list) data[region][wavelength]['AC']['sig_x'] = defaultdict(list) data[region][wavelength]['AC']['sig_x_err'] = defaultdict(list) data[region][wavelength]['AC']['sig_y'] = defaultdict(list) data[region][wavelength]['AC']['sig_y_err'] = defaultdict(list) data[region][wavelength]['AC']['theta'] = defaultdict(list) data[region][wavelength]['AC']['theta_err'] = defaultdict(list) FEN = files[0] FilePath = datadir + "/" + FEN OutPath = datadir + "/" + FEN.split('.sdf')[0] + ".fit" if os.path.isfile(OutPath): pass else: convert.ndf2fits(FilePath, OutPath) FilePath = OutPath print('\n\nFIRST EPOCH: '+FilePath+'\n\n') FirstEpoch = fits.open(FilePath) # opening the file in astropy FirstEpochData = FirstEpoch[0].data[0] # Numpy data array for the first epoch FirstEpochCentre = np.array([FirstEpoch[0].header['CRPIX1'], FirstEpoch[0].header['CRPIX2']]) # middle of the map of the first epoch FED_MidMapX = FirstEpochData.shape[1] // 2 FED_MidMapY = FirstEpochData.shape[0] // 2 FirstEpochVec = np.array([FirstEpochCentre[0] - FED_MidMapX, FirstEpochCentre[1] - FED_MidMapY]) FirstEpochData = FirstEpochData[ FED_MidMapY - length:FED_MidMapY + length + 1, FED_MidMapX - length:FED_MidMapX + length + 1] FirstEpochData_smooth = convolve(FirstEpochData, align_smooth_kernel, normalize_kernel=False) FirstEpochData -= FirstEpochData_smooth for fn in files: if wavelength in fn: FilePath = datadir + "/" + fn tau225_start = float(kappa.fitsval(FilePath, 'WVMTAUST').value) tau225_end = float(kappa.fitsval(FilePath, 'WVMTAUEN').value) tau225 = sum([tau225_start, tau225_end]) / 2 AirMass_start = float(kappa.fitsval(FilePath, 'AMSTART').value) AirMass_end = float(kappa.fitsval(FilePath, 'AMEND').value) AirMass = sum([AirMass_start, AirMass_end]) / 2 elev_start = float(kappa.fitsval(FilePath, 'ELSTART').value) elev_end = float(kappa.fitsval(FilePath, 'ELEND').value) elev = int(round(sum([elev_start, elev_end]) / 2, 0)) OutPath = datadir + "/" + fn[:-4] + ".fit" if os.path.isfile(OutPath): pass else: convert.ndf2fits(FilePath, OutPath) FilePath = OutPath hdul = fits.open(FilePath) # opening the file in astropy date = ''.join(str(hdul[0].header['DATE-OBS']).split('T')[0].split('-')) # extract date from the header date += '-' + str(hdul[0].header['OBSNUM']) JulianDate = str(float(hdul[0].header['MJD-OBS']) + 2400000.5) print('Epoch: {:14}'.format(date)) data[region][wavelength]['header']['airmass'][date] = AirMass data[region][wavelength]['header']['t225'][date] = tau225 data[region][wavelength]['header']['julian_date'][date] = JulianDate data[region][wavelength]['header']['elevation'][date] = elev data[region][wavelength]['dates'].append(date) centre = (hdul[0].header['CRPIX1'], hdul[0].header['CRPIX2']) # JCMT's alleged centre is hdu = hdul[0] # a nice compact way to store the data for later. # data[region][wavelength]['epoch'][date].append(hdu)o Epoch = hdu.data[0] # map of the region Map_of_Region = interpolate_replace_nans(correlate(Epoch, clip_only=True), Gaussian2DKernel(5)) Map_of_Region_smooth = convolve(Map_of_Region, align_smooth_kernel, normalize_kernel=False) Map_of_RegionXC = Map_of_Region - Map_of_Region_smooth XC = correlate(epoch_1=Map_of_RegionXC, epoch_2=FirstEpochData).real PS = correlate(Map_of_Region, psd=True) AC = correlate(Map_of_Region).real # auto correlation of the map centre = (hdul[0].header['CRPIX1'], hdul[0].header['CRPIX2']) # JCMT's alleged centre is Vec = np.array([centre[0] - (hdul[0].shape[2] // 2), centre[1] - (hdul[0].shape[1] // 2)]) JCMT_offset = FirstEpochVec - Vec # JCMT offset from headers data[region][wavelength]['JCMT_offset'][date] = JCMT_offset # used for accessing data later. [[AMP, SIGX, SIGY, THETA], [AMP_ERR, SIGX_ERR, SIGY_ERR, THETA_ERR]], _ = gaussian_fit_ac(AC) offset, offset_err = gaussian_fit_xc(XC) alignment = JCMT_offset - offset Length_Scale = np.sqrt(SIGX * SIGY) data[region][wavelength]['XC']['offset'][date] = offset * scale data[region][wavelength]['XC']['offset_err'][date] = offset_err data[region][wavelength]['XC']['alignment'][date] = alignment data[region][wavelength]['AC']['beam'][date] = Length_Scale data[region][wavelength]['AC']['amp'][date] = AMP data[region][wavelength]['AC']['amp_err'][date] = AMP_ERR data[region][wavelength]['AC']['sig_x'][date] = SIGX data[region][wavelength]['AC']['sig_x_err'][date] = SIGX_ERR data[region][wavelength]['AC']['sig_y'][date] = SIGY data[region][wavelength]['AC']['sig_y_err'][date] = SIGY_ERR data[region][wavelength]['AC']['theta'][date] = THETA data[region][wavelength]['AC']['theta_err'][date] = THETA_ERR Clipped_Map_of_Region_LENGTH = np.arange(0, Map_of_Region.shape[0]) loc = list(product(Clipped_Map_of_Region_LENGTH, Clipped_Map_of_Region_LENGTH)) MidMapX = AC.shape[1] // 2 # middle of the map x MidMapY = AC.shape[0] // 2 # and y radius, AC_pows = [], [] for idx in loc: # Determining the power at a certain radius r = ((idx[0] - MidMapX) ** 2 + (idx[1] - MidMapY) ** 2) ** (1 / 2) AC_pow = AC[idx[0], idx[1]].real radius.append(r) AC_pows.append(AC_pow) radius, AC_pows = zip(*sorted(list(zip(radius, AC_pows)), key=op.itemgetter(0))) radius = np.array(radius) AC_pows = np.array(AC_pows) num = len(radius[np.where(radius <= DIST)]) opt_fit_AC, cov_mat_AC = curve_fit(f, radius[1:num], AC_pows[1:num]) err = np.sqrt(np.diag(cov_mat_AC)) M = opt_fit_AC[0] M_err = err[0] B = opt_fit_AC[1] B_err = err[1] data[region][wavelength]['linear']['m'][date] = M data[region][wavelength]['linear']['m_err'][date] = M_err data[region][wavelength]['linear']['b'][date] = B data[region][wavelength]['linear']['b_err'][date] = B_err data = default_to_regular(data) if not os.path.exists('data'): os.system('mkdir data') with open('data/data_Transient_run_'+str(alignment_iteration)+'_'+wavelength+'.pickle', 'wb') as OUT: pickle.dump(data, OUT)
) da_gemb = xr.concat(grids, dim="t") print("averaging in time ...") da_gemb = da_gemb.rolling(t=window, center=True).mean() print("regridding in time ...") da_gemb = da_gemb.interp(t=t_cube) print(da_gemb) print("extending spatial boundaries before ...") for k in range(t_cube.size): da_gemb.values[:, :, k] = interpolate_replace_nans( da_gemb.values[:, :, k], Gaussian2DKernel(2), boundary="extend" ) print("regridding in space ...") da_gemb = da_gemb.interp(x=x_cube, y=y_cube) print("extending spatial boundaries after ...") for k in range(t_cube.size): da_gemb.values[:, :, k] = interpolate_replace_nans( da_gemb.values[:, :, k], Gaussian2DKernel(5), boundary="extend" ) da_gemb.values[:, :, k] = interpolate_replace_nans( da_gemb.values[:, :, k], Gaussian2DKernel(3), boundary="extend" ) if 0:
plt.figure() plt.pcolormesh(X_new, Y_new, mask_new) plt.show() sys.exit() """ # Filter artifacts and extend boundaries for k in range(melt.shape[2]): print(k) melt_k = melt[:, :, k] melt_k = filt_velocity_boundary(x, y, melt_k) melt_k = filt_positives(x, y, melt_k, [(-1574000, -1543250, -500000, -440000)]) melt[:, :, k] = interpolate_replace_nans(melt_k, Gaussian2DKernel(1), boundary="extend") # Mask with zeros if 0: mask3d_zeros = np.repeat(mask_zeros[:, :, np.newaxis], melt.shape[2], axis=2) melt[mask3d_zeros == 1] = 0.0 """ plt.matshow(np.nanmean(melt, axis=2), vmin=-5, vmax=5, cmap='RdBu') plt.show() sys.exit() """ # Regrid (comes out upside down)
plt.show() if count_plot == n_plots: sys.exit() # --- Spatial filtering --- # if 1: kernel = Gaussian2DKernel(1) for k in range(H.shape[2]): print("interpolating slice:", k) dHdt[:, :, k] = interpolate_replace_nans(dHdt[:, :, k], kernel, boundary="extend") H[:, :, k] = interpolate_replace_nans(H[:, :, k], kernel, boundary="extend") adv[:, :, k] = interpolate_replace_nans(adv[:, :, k], kernel, boundary="extend") str[:, :, k] = interpolate_replace_nans(str[:, :, k], kernel, boundary="extend") div[:, :, k] = interpolate_replace_nans(div[:, :, k], kernel, boundary="extend") smb[:, :, k] = interpolate_replace_nans(smb[:, :, k], kernel,
def master_synthetic_image_creator(target, seeing=''): def mimir_source_finder(image_path, sigma_above_bg, fwhm): """Find sources in Mimir images.""" np.seterr(all='ignore') #Ignore invalids (i.e. divide by zeros) #Find stars in the master image. avg, med, stddev = sigma_clipped_stats( image, sigma=3.0, maxiters=3) #Previously maxiters = 5! daofind = DAOStarFinder(fwhm=fwhm, threshold=sigma_above_bg * stddev, sky=med, ratio=0.8) new_sources = daofind(image) x_centroids = new_sources['xcentroid'] y_centroids = new_sources['ycentroid'] sharpness = new_sources['sharpness'] fluxes = new_sources['flux'] peaks = new_sources['peak'] #Cut sources that are found within 20 pix of the edges. use_x = np.where((x_centroids > 20) & (x_centroids < 1004))[0] x_centroids = x_centroids[use_x] y_centroids = y_centroids[use_x] sharpness = sharpness[use_x] fluxes = fluxes[use_x] peaks = peaks[use_x] use_y = np.where((y_centroids > 20) & (y_centroids < 1004))[0] x_centroids = x_centroids[use_y] y_centroids = y_centroids[use_y] sharpness = sharpness[use_y] fluxes = fluxes[use_y] peaks = peaks[use_y] #Also cut using sharpness, this seems to eliminate a lot of false detections. use_sharp = np.where(sharpness > 0.5)[0] x_centroids = x_centroids[use_sharp] y_centroids = y_centroids[use_sharp] sharpness = sharpness[use_sharp] fluxes = fluxes[use_sharp] peaks = peaks[use_sharp] #Cut sources in the lower left, if bars are present. #use_ll = np.where((x_centroids > 512) | (y_centroids > 512)) #x_centroids = x_centroids [use_ll] #y_centroids = y_centroids [use_ll] #sharpness = sharpness[use_ll] #fluxes = fluxes[use_ll] #peaks = peaks[use_ll] #Cut targets whose y centroids are near y = 512. These are usually bad. use_512 = np.where( np.logical_or((y_centroids < 509), (y_centroids > 515)))[0] x_centroids = x_centroids[use_512] y_centroids = y_centroids[use_512] sharpness = sharpness[use_512] fluxes = fluxes[use_512] peaks = peaks[use_512] #Cut sources with negative/saturated peaks use_peaks = np.where((peaks > 30) & (peaks < 7000))[0] x_centroids = x_centroids[use_peaks] y_centroids = y_centroids[use_peaks] sharpness = sharpness[use_peaks] fluxes = fluxes[use_peaks] peaks = peaks[use_peaks] #Do quick photometry on the remaining sources. positions = [(x_centroids[i], y_centroids[i]) for i in range(len(x_centroids))] apertures = CircularAperture(positions, r=4) phot_table = aperture_photometry(image - med, apertures) #Cut based on brightness. phot_table.sort('aperture_sum') cutoff = 1 * std * np.pi * 4**2 bad_source_locs = np.where(phot_table['aperture_sum'] < cutoff) phot_table.remove_rows(bad_source_locs) if len(phot_table) > 15: x_centroids = phot_table['xcenter'].value[-16:-1] y_centroids = phot_table['ycenter'].value[-16:-1] else: x_centroids = phot_table['xcenter'].value y_centroids = phot_table['ycenter'].value return (x_centroids, y_centroids) def synthetic_image_maker(x_centroids, y_centroids, fwhm): #Construct synthetic images from centroid/flux data. synthetic_image = np.zeros((1024, 1024)) sigma = fwhm / 2.355 for i in range(len(x_centroids)): #Cut out little boxes around each source and add in Gaussian representations. This saves time. int_centroid_x = int(np.round(x_centroids[i])) int_centroid_y = int(np.round(y_centroids[i])) y_cut, x_cut = np.mgrid[int_centroid_y - 10:int_centroid_y + 10, int_centroid_x - 10:int_centroid_x + 10] dist = np.sqrt((x_cut - x_centroids[i])**2 + (y_cut - y_centroids[i])**2) synthetic_image[y_cut, x_cut] += np.exp(-((dist)**2 / (2 * sigma**2) + ((dist)**2 / (2 * sigma**2)))) return (synthetic_image) def auto_correlation_seeing(im, cutout_w=15): plate_scale = 0.579 #arcsec/pix sigma_to_fwhm = 2.355 #Set a row to NaNs, which will dominate the autocorrelation of Mimir images. im[513] = np.nan #Interpolate nans in the image, repeating until no nans remain. while sum(sum(np.isnan(im))) > 0: im = interpolate_replace_nans(im, kernel=Gaussian2DKernel(0.5)) #Cut 80 pixels near top/bottom edges, which can dominate the fft if they have a "ski jump" feature. im = im[80:944, :] y_dim, x_dim = im.shape #Subtract off a simple estimate of the image background. im -= sigma_clipped_stats(im)[1] #Do auto correlation fft = signal.fftconvolve(im, im[::-1, ::-1], mode='same') #Do a cutout around the center of the fft. cutout = fft[int(y_dim / 2) - cutout_w:int(y_dim / 2) + cutout_w, int(x_dim / 2) - cutout_w:int(x_dim / 2) + cutout_w] #Set the midplane of the cutout to nans and interpolate. cutout[cutout_w] = np.nan while sum(sum(np.isnan(cutout))) > 0: cutout = interpolate_replace_nans(cutout, Gaussian2DKernel(0.25)) #Subtract off "background" cutout -= np.nanmedian(cutout) #Fit a 2D Gaussian to the cutout #Assume a seeing of 2".7, the average value measured for PINES. g_init = models.Gaussian2D( amplitude=np.nanmax(cutout), x_mean=cutout_w, y_mean=cutout_w, x_stddev=2.7 / plate_scale / sigma_to_fwhm * np.sqrt(2), y_stddev=2.7 / plate_scale / sigma_to_fwhm * np.sqrt(2)) g_init.x_mean.fixed = True g_init.y_mean.fixed = True #Set limits on the fitted gaussians between 1".6 and 7".0 #Factor of sqrt(2) corrects for autocorrelation of 2 gaussians. g_init.x_stddev.min = 1.6 / plate_scale / sigma_to_fwhm * np.sqrt(2) g_init.y_stddev.min = 1.6 / plate_scale / sigma_to_fwhm * np.sqrt(2) g_init.x_stddev.max = 7 / plate_scale / sigma_to_fwhm * np.sqrt(2) g_init.y_stddev.max = 7 / plate_scale / sigma_to_fwhm * np.sqrt(2) fit_g = fitting.LevMarLSQFitter() y, x = np.mgrid[:int(2 * cutout_w), :int(2 * cutout_w)] g = fit_g(g_init, x, y, cutout) #Convert to fwhm in arcsec. seeing_fwhm_as = g.y_stddev.value / np.sqrt( 2) * sigma_to_fwhm * plate_scale return seeing_fwhm_as plt.ion() target = target.replace(' ', '') #By default, point to today's date directory. ut_date = datetime.datetime.utcnow() if ut_date.month < 10: month_string = '0' + str(ut_date.month) else: month_string = str(ut_date.month) if ut_date.day < 10: day_string = '0' + str(ut_date.day) else: day_string = str(ut_date.day) date_string = str(ut_date.year) + month_string + day_string #Copy the test.fits file to the master_images directory in PINES scripts. #test_path = '/mimir/data/obs72/'+date_string+'/test.fits' test_path = '/Users/obs72/Desktop/PINES_scripts/test_image/test.fits' target_path = '/Users/obs72/Desktop/PINES_scripts/master_images/' + target + '_master.fits' shutil.copyfile(test_path, target_path) file_path = '/Users/obs72/Desktop/PINES_scripts/master_images/' + target + '_master.fits' calibration_path = '/Users/obs72/Desktop/PINES_scripts/Calibrations/' #Open the image and calibration files. header = fits.open(file_path)[0].header exptime = header['EXPTIME'] filter = header['FILTNME2'] image = fits.open(file_path)[0].data[0:1024, :].astype('float') dark = fits.open(calibration_path + 'Darks/master_dark_' + str(exptime) + '.fits')[0].data flat = fits.open(calibration_path + 'Flats/master_flat_' + filter + '.fits')[0].data bpm_path = calibration_path + 'Bad_pixel_masks/bpm.fits' bpm = fits.open(bpm_path)[0].data #Reduce image. image = (image - dark) / flat #Interpolate over bad pixels image[np.where(bpm)] = np.nan kernel = Gaussian2DKernel(x_stddev=1) image = interpolate_replace_nans(image, kernel) if seeing == '': seeing = auto_correlation_seeing(image) else: seeing = float(seeing) #old code had a 2.355 factor. PSM thinks this is a bug #daostarfinder_fwhm = seeing*2.355/0.579 #new code removes 2.355 factor print('Using seeing FWHM = {:1.1f}" to detect sources'.format(seeing)) daostarfinder_fwhm = seeing / 0.579 #Do a simple 2d background model. box_size = 32 sigma_clip = SigmaClip(sigma=3.) bkg_estimator = MedianBackground() bkg = Background2D(image, (box_size, box_size), filter_size=(3, 3), sigma_clip=sigma_clip, bkg_estimator=bkg_estimator) image = image - bkg.background avg, med, std = sigma_clipped_stats(image) #Save reduced image to test_image for inspection # hdu_reduced = fits.PrimaryHDU(image) # hdu_reduced.writeto('/Users/obs72/Desktop/PINES_scripts/test_image/master_reduced.fits',overwrite=True) #Find sources in the image. (x_centroids, y_centroids) = mimir_source_finder(image, sigma_above_bg=5, fwhm=daostarfinder_fwhm) #Plot the field with detected sources. plt.figure(figsize=(9, 9)) norm = ImageNormalize(image, interval=ZScaleInterval()) plt.imshow(image, origin='lower', norm=norm) plt.plot(x_centroids, y_centroids, 'rx') for i in range(len(x_centroids)): plt.text(x_centroids[i] + 8, y_centroids[i] + 8, str(i), color='r', fontsize=14) plt.title( 'Inspect to make sure stars were found!\nO for magnification tool, R to reset view' ) plt.tight_layout() plt.show() print('') print('') print('') #Prompt the user to remove any false detections. ids = input( 'Enter ids of sources to be removed separated by commas (i.e., 4,18,22). If none to remove, hit enter. To break, ctrl + D. ' ) if ids != '': ids_to_eliminate = [int(i) for i in ids.split(',')] ids = [ int(i) for i in np.linspace(0, len(x_centroids) - 1, len(x_centroids)) ] ids_to_keep = [] for i in range(len(ids)): if ids[i] not in ids_to_eliminate: ids_to_keep.append(ids[i]) else: ids_to_keep = [ int(i) for i in np.linspace(0, len(x_centroids) - 1, len(x_centroids)) ] plt.clf() plt.imshow(image, origin='lower', vmin=med, vmax=med + 5 * std) plt.plot(x_centroids[ids_to_keep], y_centroids[ids_to_keep], 'rx') for i in range(len(x_centroids[ids_to_keep])): plt.text(x_centroids[ids_to_keep][i] + 8, y_centroids[ids_to_keep][i] + 8, str(i), color='r') #Create the synthetic image using the accepted sources. synthetic_image = synthetic_image_maker(x_centroids[ids_to_keep], y_centroids[ids_to_keep], 8) plt.figure(figsize=(9, 7)) plt.imshow(synthetic_image, origin='lower') plt.title('Synthetic image') plt.show() print('') print('') print('') #Now write to a master synthetic image.fits file. hdu = fits.PrimaryHDU(synthetic_image, header=header) if os.path.exists('master_images/' + target + '_master_synthetic.fits'): ans = input( 'Master image already exists for target, type y to overwrite. ') if ans == 'y': os.remove('master_images/' + target + '_master_synthetic.fits') hdu.writeto('master_images/' + target + '_master_synthetic.fits') print('Writing master synthetic image to master_images/' + target + '_master_synthetic.fits') else: print('New master synthetic image not saved.') else: hdu.writeto('master_images/' + target + '_master_synthetic.fits') print('Writing master synthetic image to master_images/' + target + '_master_synthetic.fits') #Open list of master images and append new one. master_image_list = '/Users/obs72/Desktop/PINES_scripts/input_file.txt' file_object = open(master_image_list, 'a') append_str = '/Users/obs72/Desktop/PINES_scripts/master_images/' + target + '_master.fits, 2MASS ' + target.split( '2MASS')[1] file_object.write('\n') file_object.write(append_str) file_object.close()
JulianDate = str(float(hdul[0].header['MJD-OBS']) + 2400000.5) print('Epoch: {:14}'.format(date)) data[region][wavelength]['header']['airmass'][date] = AirMass data[region][wavelength]['header']['t225'][date] = tau225 data[region][wavelength]['header']['julian_date'][ date] = JulianDate data[region][wavelength]['header']['elevation'][date] = elev data[region][wavelength]['dates'].append(date) centre = (hdul[0].header['CRPIX1'], hdul[0].header['CRPIX2'] ) # JCMT's alleged centre is hdu = hdul[ 0] # a nice compact way to store the data for later. # data[region][wavelength]['epoch'][date].append(hdu)o Epoch = hdu.data[0] # map of the region Map_of_Region = interpolate_replace_nans( correlate(Epoch, clip_only=True), Gaussian2DKernel(5)) Map_of_Region_smooth = convolve(Map_of_Region, align_smooth_kernel, normalize_kernel=False) Map_of_RegionXC = Map_of_Region - Map_of_Region_smooth XC = correlate(epoch_1=Map_of_RegionXC, epoch_2=FirstEpochData).real PS = correlate(Map_of_Region, psd=True) AC = correlate( Map_of_Region).real # auto correlation of the map centre = (hdul[0].header['CRPIX1'], hdul[0].header['CRPIX2'] ) # JCMT's alleged centre is Vec = np.array([ centre[0] - (hdul[0].shape[2] // 2), centre[1] - (hdul[0].shape[1] // 2)
Data['sigma_y'] = [] Data['alpha'] = [] Count = [] for index in range(4, 60): atoms = Data["Raw"][index] - dark_mean # Generic Image preperation atoms_nans = (atoms <= 0) atoms[atoms_nans] = 1 OD = -np.log((atoms) / (probe - dark_mean)) OD[atoms_nans] = np.nan # Now use astropy to clean this up OD = interpolate_replace_nans(OD, kernel) BGCounts = np.sum(OD * BG_Mask['mask']) / BG_Mask['pts'] OD -= BGCounts Data["OD"].append(OD) # Construct smooth background images # Next we setup a fit: I'll do a deformed gaussian peak. pguess = lmfit.Parameters() pguess.add('amp', value=1, min=-0.5, max=6) pguess.add('xc', value=0, min=-10, max=10) pguess.add('yc', value=0, min=-10, max=10) pguess.add('sigma_x', value=10, min=2, max=30) pguess.add('sigma_y', value=10, min=2, max=30) pguess.add('alpha', value=1, min=0.1, max=4)
def merge_fits_files(infolder=None, infiles=None, suffix='', fout=None, method='mean', prefix='', sigma=3, target=None, filt=None, rotangle=None, northup=False, refbox=20, fitmeth='fastgauss', refsign=1, imgoffsetangle=None, verbose=False, tolrot=0.01, pfov=None, replace_nan_with='median', instrument=None): """ Take either a folder full of single (chop/nod) frames or a given list of files and simply average them into one merged file """ if infolder is not None and infiles is None: infiles = [ ff for ff in os.listdir(infolder) if (ff.endswith(suffix + '.fits')) & (ff.startswith(prefix)) ] if infolder is None: infolder = '' nfiles = len(infiles) ima = [] shapes = [] rots = [] ffull = infolder + "/" + infiles[0] hdu = fits.open(ffull) head0 = hdu[0].header hdu.close() if instrument is None: instrument = head0['INSTRUME'] print('Found instrument: ', instrument) if instrument == 'VISIR': if imgoffsetangle is None: imgoffsetangle = _vp.imgoffsetangle rotsense = _vp.rotsense rotoffset = _vp.rotoffset elif (instrument == 'NAOS+CONICA') | (instrument == 'NACO'): imgoffsetangle = 0 # apparently correct? rotsense = -1 # for NACO the ADA.POSANG really gives the PA on sky rotoffset = 0 elif instrument == "ISAAC": imgoffsetangle = _ip.imgoffsetangle rotsense = _ip.rotsense rotoffset = _ip.rotoffset print('Found n potential files: ', nfiles) if verbose: print("MERGE_FITS_FILES: fitmeth: ", fitmeth) print("MERGE_FITS_FILES: refbox: ", refbox) print("MERGE_FITS_FILES: refsign: ", refsign) for i in range(nfiles): ffull = infolder + "/" + infiles[i] hdu = fits.open(ffull) if verbose: print("MERGE_FITS_FILES: i, infiles[i]: ", i, infiles[i]) head = hdu[0].header rot = head['HIERARCH ESO ADA POSANG'] if "HIERARCH ESO TEL ROT ALTAZTRACK" in head: pupiltrack = (head["HIERARCH ESO TEL ROT ALTAZTRACK"]) else: pupiltrack = False # --- if pupil tracking is on then the parang in the VISIR fits-header is # not normalised by the offset angle of the VISIR imager with respect # to the adapter/rotator if pupiltrack: rot = rot - imgoffsetangle rot += rotoffset if verbose: print("MERGE_FITS_FILES: rot: ", rot) if target is not None: targ = head['HIERARCH ESO OBS TARG NAME'] if str(targ) != target: continue if filt is not None: filtf = head['HIERARCH ESO INS FILT1 NAME'] if str(filtf) != filt: continue if rotangle is not None: if rot != rotangle: continue im = hdu[0].data hdu.close() # --- rotate images if Northup is requested to be up if np.abs(rot) > tolrot and northup: # --- ndimage is not compaticble with nans, so we need to replace them print( "MERGE_FITS_FILES: Encountered NaNs not compatible with " + "rotation. Replacing with ", replace_nan_with) if ((type(replace_nan_with) == float) | (type(replace_nan_with) == int)): im[np.isnan(im)] = replace_nan_with elif replace_nan_with == 'median': im[np.isnan(im)] = np.nanmedian(im) # --- WARNING: does not work with large NaN areas at the borders elif replace_nan_with == 'interpol': kernel = Gaussian2DKernel(stddev=1) im = interpolate_replace_nans(im, kernel) # print(np.isnan(im).any()) # # plt.imshow(im, origin='bottom', interpolation='nearest', # norm=LogNorm()) # plt.title(str(i)+' - after replacing NaNs, before rotation') # plt.show() im = ndimage.interpolation.rotate(im, rotsense * rot, order=3) # plt.imshow(im, origin='bottom', interpolation='nearest', # norm=LogNorm()) # plt.show() ima.append(im) shapes.append(np.shape(im)) rots.append(rot) shapes = np.array(shapes) n = len(ima) if verbose: print("MERGE_FITS_FILES: n: ", n) # --- after rotating the images probably have different sizes if northup: minsize = [np.min(shapes[:, 0]), np.min(shapes[:, 1])] for i in range(n): if verbose: print("MERGE_FITS_FILES: i: ", i) print("MERGE_FITS_FILES: shapes: ", shapes[i]) print("MERGE_FITS_FILES: argmax: ", np.unravel_index(np.nanargmax(ima[i]), shapes[i])) plt.imshow(ima[i], origin='bottom', interpolation='nearest', norm=LogNorm()) plt.title( str(i) + ' - rot ' + str(rots[i]) + ' - before centered crop') plt.show() fit, _, _ = _find_source(ima[i], method=fitmeth, searchbox=refbox, fitbox=refbox, sign=refsign, verbose=verbose, plot=verbose) ima[i] = _crop_image(ima[i], box=minsize, cenpos=fit[2:4]) if verbose: plt.imshow(ima[i], origin='bottom', interpolation='nearest', norm=LogNorm()) plt.title(str(i) + ' - rot ' + str(rots[i])) plt.show() print(i, fit) # --- if WCS is present in the header, it needs to be updated to if "CTYPE1" in head0: if head0["CTYPE1"] == "RA---TAN": if pfov is None: pfov = _ if "HIERARCH ESO INS PFOV" in head0: pfov = float(head["HIERARCH ESO INS PFOV"]) # VISIR else: pfov = float(head["HIERARCH ESO INS PIXSCALE"]) # NACO ra = head0["HIERARCH ESO TEL TARG ALPHA"] sec = ra % 100 min = (ra % 10000 - sec) / 100 hour = (int(ra) / 10000) ra_deg = 15 * (hour + min / 60.0 + sec / 3600.0) # print(ra_deg) dec = head0["HIERARCH ESO TEL TARG DELTA"] sec = dec % 100 min = (dec % 10000 - sec) / 100 deg = (int(dec) / 10000) if deg < 0: dec_deg = deg - min / 60.0 - sec / 3600.0 else: dec_deg = deg + min / 60.0 + sec / 3600.0 # print(ra_deg) head0["CRPIX1"] = minsize[1] * 0.5 head0["CRPIX2"] = minsize[0] * 0.5 head0["CRVAL1"] = ra_deg head0["CRVAL2"] = dec_deg if "CD1_1" in head0: del head0["CD1_1"] del head0["CD1_2"] del head0["CD2_1"] del head0["CD2_2"] head0["CDELT1"] = -pfov / 3600.0 head0["CDELT2"] = pfov / 3600.0 ima = np.array(ima) #print(np.shape(ima)) if method == 'mean': outim = np.nanmean(ima, axis=0) if method == 'median': outim = np.nanmedian(ima, axis=0) if method == 'sigmaclip': outim = np.array( np.mean(sigma_clip(ima, sigma=sigma, axis=0, maxiters=2), axis=0)) print(" - " + str(n) + " images combined.") if fout is not None: fits.writeto(fout, outim, head0, overwrite=True) return (outim)
def auto_correlation_seeing(im, cutout_w=15): plate_scale = 0.579 #arcsec/pix sigma_to_fwhm = 2.355 #Set a row to NaNs, which will dominate the autocorrelation of Mimir images. im[513] = np.nan #Interpolate nans in the image, repeating until no nans remain. while sum(sum(np.isnan(im))) > 0: im = interpolate_replace_nans(im, kernel=Gaussian2DKernel(0.5)) #Cut 80 pixels near top/bottom edges, which can dominate the fft if they have a "ski jump" feature. im = im[80:944, :] y_dim, x_dim = im.shape #Subtract off a simple estimate of the image background. im -= sigma_clipped_stats(im)[1] #Do auto correlation fft = signal.fftconvolve(im, im[::-1, ::-1], mode='same') #Do a cutout around the center of the fft. cutout = fft[int(y_dim / 2) - cutout_w:int(y_dim / 2) + cutout_w, int(x_dim / 2) - cutout_w:int(x_dim / 2) + cutout_w] #Set the midplane of the cutout to nans and interpolate. cutout[cutout_w] = np.nan while sum(sum(np.isnan(cutout))) > 0: cutout = interpolate_replace_nans(cutout, Gaussian2DKernel(0.25)) #Subtract off "background" cutout -= np.nanmedian(cutout) #Fit a 2D Gaussian to the cutout #Assume a seeing of 2".7, the average value measured for PINES. g_init = models.Gaussian2D( amplitude=np.nanmax(cutout), x_mean=cutout_w, y_mean=cutout_w, x_stddev=2.7 / plate_scale / sigma_to_fwhm * np.sqrt(2), y_stddev=2.7 / plate_scale / sigma_to_fwhm * np.sqrt(2)) g_init.x_mean.fixed = True g_init.y_mean.fixed = True #Set limits on the fitted gaussians between 1".6 and 7".0 #Factor of sqrt(2) corrects for autocorrelation of 2 gaussians. g_init.x_stddev.min = 1.6 / plate_scale / sigma_to_fwhm * np.sqrt(2) g_init.y_stddev.min = 1.6 / plate_scale / sigma_to_fwhm * np.sqrt(2) g_init.x_stddev.max = 7 / plate_scale / sigma_to_fwhm * np.sqrt(2) g_init.y_stddev.max = 7 / plate_scale / sigma_to_fwhm * np.sqrt(2) fit_g = fitting.LevMarLSQFitter() y, x = np.mgrid[:int(2 * cutout_w), :int(2 * cutout_w)] g = fit_g(g_init, x, y, cutout) #Convert to fwhm in arcsec. seeing_fwhm_as = g.y_stddev.value / np.sqrt( 2) * sigma_to_fwhm * plate_scale return seeing_fwhm_as
def source_detect_astrometry(target, api_key, filename, download_data=False): ''' Authors: Pat Tamburo, Boston University, May 2021 Purpose: Uploads a single reduced image for a target to astrometry.net, downloads solution image, and updates the image header with the astrometry.net wcs. Inputs: target (str): The target's long 2MASS name (e.g., '2MASS J09161504+2139512') api_key (str): The api key of your account on astrometry.net. filename (pathlib.Path): Path to the source_detect_image. download_data (bool, optional): Whether or not to first download reduced data for the target from the PINES server. ''' kernel = Gaussian2DKernel(x_stddev=0.25) #If the header does not have a HISTORY keyword (which is added by astrometry.net), process it. header = fits.open(filename)[0].header if 'HISTORY' not in header: print('Uploading {} to astrometry.net.'.format(filename.name)) #Read in the image data, interpolate NaNs, and save to a temporary fits file. #Astrometry.net does not work with NaNs in images. original_image = fits.open(filename)[0].data image = interpolate_replace_nans(original_image, kernel=kernel) temp_filename = filename.parent / (filename.name.split('.fits')[0] + '_temp.fits') hdu = fits.PrimaryHDU(image, header=header) hdu.writeto(temp_filename, overwrite=True) #Upload the image with interpolated NaNs to astrometry.net. It will solve and download to a new image. pat.astrometry.upload_to_astrometry.upload_file( api_key, temp_filename, header) #Try to donwload the solved image and open it. try: #Grab the header of the astrometry.net solution image, and the original image data. astrometry_image_path = filename.parent / ( temp_filename.name.split('.fits')[0] + '_new_image.fits') wcs_header = fits.open(astrometry_image_path)[0].header wcs_hdu = fits.PrimaryHDU(original_image, header=wcs_header) #Save the original image data with the new wcs header. output_filename = filename wcs_hdu.writeto(output_filename, overwrite=True) #If the try clause didn't work, that's because the processing on astrometry.net failed. except: raise RuntimeError( 'Astrometry solution failed! Use a different source detect image for choosing reference stars.' ) #Delete temporary files. os.remove(temp_filename) os.remove(astrometry_image_path) time.sleep(1) print('') #If the header DOES have a HISTORY keyword, skip it, it has already been processed. else: print( 'Astrometric processing already complete for {}, skipping.'.format( filename.name)) print('')