def test_cube_derotate(imlib, interpolation, border_mode, edge_blend): """ Note: this calls most other routines defined in rotation.py """ # even res = CUBE_even.copy() angles = np.array([120, 90, 60, 45]) for i in range(24): # yields 360deg multiples for all derotation angles res = cube_derotate(res, angles, imlib=imlib, border_mode=border_mode, edge_blend=edge_blend, nproc=1 + (i % 2)) # just consider cropped image for evaluation # to avoid differences due to edge effects res = cube_crop_frames(res, 50) CUBE_test = cube_crop_frames(CUBE_even, 50) aarc(res, CUBE_test, rtol=1e-1, atol=1e-1) # odd res = CUBE_odd.copy() angles = np.array([120, 90, 60, 45]) for i in range(24): # yields 360deg multiples for all derotation angles res = cube_derotate(res, angles, imlib=imlib, border_mode=border_mode, edge_blend=edge_blend, nproc=1 + (i % 2)) # just consider cropped image for evaluation # to avoid differences due to edge effects res = cube_crop_frames(res, 51) CUBE_test = cube_crop_frames(CUBE_odd, 51) aarc(res, CUBE_test, rtol=1e-1, atol=1e-1)
def _inject_FC(cube, psf, angle_list, plsc, inrad, outrad, flux_dist_theta, k_list, sca, collapse_func, patch_size, lr_mode, interp, mode): """ One patch residuals per injection """ cubefc, cox, coy = create_synt_cube(cube, psf, angle_list, plsc, flux=flux_dist_theta[0], dist=flux_dist_theta[1], theta=flux_dist_theta[2], verbose=False) cox = int(np.round(cox)) coy = int(np.round(coy)) cube_residuals = svd_decomp(cubefc, angle_list, patch_size, inrad, outrad, sca, k_list, collapse_func, neg_ang=False, lr_mode=lr_mode, nproc=1, interp=interp, mode=mode) patch = cube_crop_frames(np.array(cube_residuals), patch_size, xy=(cox, coy), verbose=False, force=True) return patch
def crop_cube(self, arcsecond_diameter=3.5, verbose=True, debug=False): """ Crops frames in the master cube after recentering and bad frame removal. Recommended for post-processing ie. PCA in concentric annuli. If the provided arcsecond diameter happens to be larger than the cropping provided in recentering, no cropping will occur. Parameters ---------- arcsecond_diameter : float or int Size of the frames diameter in arcseconds. Default of 3" for NaCO corresponds to 111x111 (x,y) pixel frames. Note this is a diameter, not a radius. verbose : bool optional If True extra messages of completion are showed. Writes to fits file ------- cropped cube : numpy ndarray Cube with cropped frames """ if not os.path.isfile( self.outpath + '{}_master_cube.fits'.format(self.dataset_dict['source'])): raise NameError( 'Missing master cube from recentering and bad frame removal!') master_cube = open_fits( self.outpath + '{}_master_cube.fits'.format(self.dataset_dict['source']), verbose=debug) nz, ny, nx = master_cube.shape crop_size = int( (arcsecond_diameter) / (self.dataset_dict['pixel_scale'])) if not crop_size % 2: crop_size += 1 print('Crop size not odd, increased to {}'.format(crop_size)) if debug: print('Input crop size is {} pixels'.format(crop_size)) if ny <= crop_size: print( 'Crop size is larger than the frame size. Skipping cropping...' ) else: if verbose: print('######### Running frame cropping #########') master_cube = cube_crop_frames(master_cube, crop_size, force=False, verbose=debug, full_output=False) write_fits( self.outpath + '{}_master_cube.fits'.format(self.dataset_dict['source']), master_cube)
def test_cube_derotate_mask(imlib, interpolation, border_mode, edge_blend, interp_zeros): """ Note: this calls most other routines defined in rotation.py """ # mask with nans CUBE_odd_mask = mask_circle(CUBE_odd, 4, np.nan) res = CUBE_odd_mask.copy() angles = np.array([180, 120, 90, 60]) for i in range(6): # yields 360deg multiples for all derotation angles res = cube_derotate(res, angles, border_mode=border_mode, imlib=imlib, interpolation=interpolation, nproc=1 + (i % 2), mask_val=np.nan, edge_blend=edge_blend, interp_zeros=False) # just consider cropped image for evaluation # to avoid differences due to edge effects res = cube_crop_frames(res, 51) CUBE_test = cube_crop_frames(CUBE_odd_mask, 51) aarc(res, CUBE_test, rtol=1e-1, atol=1e-1) # mask with zeros CUBE_odd_mask = mask_circle(CUBE_odd, 4, 0) res = CUBE_odd_mask.copy() angles = np.array([180, 120, 90, 60]) for i in range(6): # yields 360deg multiples for all derotation angles res = cube_derotate(res, angles, border_mode=border_mode, imlib=imlib, interpolation=interpolation, nproc=1 + (i % 2), mask_val=0, edge_blend=edge_blend, interp_zeros=interp_zeros) # just consider cropped image for evaluation # to avoid differences due to edge effects res = cube_crop_frames(res, 51) CUBE_test = cube_crop_frames(CUBE_odd_mask, 51) aarc(res, CUBE_test, rtol=1e-1, atol=1e-1)
def example_dataset_rdi(): """ Download example FITS cube from github + prepare HCIDataset object. Returns ------- dataset : HCIDataset Notes ----- Astropy's ``download_file`` uses caching, so the file is downloaded at most once per test run. """ print("downloading data...") url_prefix = "https://github.com/carlgogo/VIP_extras/raw/master/datasets" f1 = download_file("{}/naco_betapic_cube.fits".format(url_prefix), cache=True) f2 = download_file("{}/naco_betapic_psf.fits".format(url_prefix), cache=True) f3 = download_file("{}/naco_betapic_pa.fits".format(url_prefix), cache=True) # load fits cube = vip.fits.open_fits(f1) angles = vip.fits.open_fits(f3).flatten() # shape (61,1) -> (61,) psf = vip.fits.open_fits(f2) # creating a flux screen scr = vip.var.create_synth_psf('moff', (101, 101), fwhm=50) scrcu = np.array([scr * i for i in np.linspace(1e3, 2e3, num=31)]) # upscaling (1.2) and taking half of the frames, reversing order cube_upsc = cube_px_resampling(cube[::-1], 1.2, verbose=False)[::2] # cropping and adding the flux screen cube_ref = cube_crop_frames(cube_upsc, 101, verbose=False) + scrcu # create dataset object dataset = vip.Dataset(cube, angles=angles, psf=psf, cuberef=cube_ref, px_scale=vip.conf.VLT_NACO['plsc']) dataset.normalize_psf(size=20, force_odd=False) # overwrite PSF for easy access dataset.psf = dataset.psfn return dataset
def prepare_patches(cube, angle_list, xy, fwhm, patch_size_px, delta_rot=0.5, normalization='slice', imlib='opencv', interpolation='bilinear', debug=False): """ Prepare patches for SODINN-PW. """ centy_fr, centx_fr = frame_center(cube[0]) angle_list = check_pa_vector(angle_list) xy_dist = dist(centy_fr, centx_fr, xy[1], xy[0]) res = _pairwise_diff_residuals(cube, angle_list, ann_center=xy_dist, fwhm=fwhm, delta_rot=delta_rot, debug=False) res_der = cube_derotate(res, angle_list, imlib=imlib, interpolation=interpolation) res_der_crop = cube_crop_frames(res_der, patch_size_px, xy=xy, verbose=False, force=True) patches = normalize_01_pw(res_der_crop, normalization) if debug: print('dist : {}'.format(xy_dist)) plot_frames( tuple(patches), axis=False, colorbar=False, ) return patches
def make_mlar_samples_ann_signal(input_array, angle_list, psf, n_samples, cevr_thresh, n_ks, force_klen, inrad, outrad, patch_size, flux_low, flux_high, plsc=0.01, normalize='slice', nproc=1, nproc2=1, interp='bilinear', lr_mode='eigen', mode='mlar', kss_window=None, tss_window=None, random_seed=42, verbose=False): """ n_samples : For ONEs, half_n_samples SVDs mask is a list of tuples X,Y inputarr is a 3d array or list of 3d arrays orig_zero_patches : percentage of original zero patches """ dist_flux_p1 = flux_low dist_flux_p2 = flux_high collapse_func = np.mean scaling = None # 'temp-mean' random_state = np.random.RandomState(random_seed) # making ones, injecting companions. The other half of n_samples if verbose: print("Creating the ONEs samples") frsize = int(input_array.shape[1]) cube = input_array # if frsize > outrad + outrad + patch_size + 2: # frsize = int(outrad + outrad + patch_size + 2) # cube = cube_crop_frames(input_array, frsize, force=True, # verbose=False) width = outrad - inrad yy, xx = get_annulus_segments((frsize, frsize), inrad, width, 1)[0] num_patches = yy.shape[0] k_list = get_cumexpvar(cube, 'annular', inrad, outrad, patch_size, k_list=None, cevr_thresh=cevr_thresh, n_ks=n_ks, match_len=force_klen, verbose=False) n_req_inject = n_samples if mode == 'mlar': # 4D: n_samples/2, n_k_list, patch_size, patch_size X_ones_array = np.empty((n_req_inject, len(k_list), patch_size, patch_size)) elif mode == 'tmlar': nfr = cube.shape[0] X_ones_array = np.empty((n_req_inject, nfr, patch_size, patch_size)) elif mode == 'tmlar4d': nfr = cube.shape[0] X_ones_array = np.empty((n_req_inject, nfr, len(k_list), patch_size, patch_size)) if verbose: print("{} injections".format(n_req_inject)) if not dist_flux_p2 > dist_flux_p1: err_msg = 'dist_flux_p2 must be larger than dist_flux_p1' raise ValueError(err_msg) fluxes = random_state.uniform(dist_flux_p1, dist_flux_p2, size=n_req_inject) fluxes = np.sort(fluxes) inds_inj = random_state.randint(0, num_patches, size=n_req_inject) dists = [] thetas = [] for m in range(n_req_inject): injx = xx[inds_inj[m]] injy = yy[inds_inj[m]] injx -= frame_center(cube[0])[1] injy -= frame_center(cube[0])[0] dist = np.sqrt(injx**2+injy**2) theta = np.mod(np.arctan2(injy, injx) / np.pi * 180, 360) dists.append(dist) thetas.append(theta) if not nproc: nproc = int((cpu_count()/4)) if nproc == 1: for m in range(n_req_inject): cufc, cox, coy = create_synt_cube(cube, psf, angle_list, plsc, theta=thetas[m], flux=fluxes[m], dist=dists[m], verbose=False) cox = int(np.round(cox)) coy = int(np.round(coy)) cube_residuals = svd_decomp(cufc, angle_list, patch_size, inrad, outrad, scaling, k_list, collapse_func, neg_ang=False, lr_mode=lr_mode, nproc=nproc2, interp=interp, mode=mode) # one patch residuals per injection X_ones_array[m] = cube_crop_frames(np.asarray(cube_residuals), patch_size, xy=(cox, coy), verbose=False, force=True) elif nproc > 1: if lr_mode in ['cupy', 'randcupy', 'eigencupy']: raise RuntimeError('CUPY does not play well with multiproc') if get_start_method() == 'fork' and lr_mode in ['pytorch', 'eigenpytorch', 'randpytorch']: raise RuntimeError("Cannot use pytorch and multiprocessing " "outside main (i.e. from a jupyter cell). " "See: http://pytorch.org/docs/0.3.1/notes/" "multiprocessing.html.") flux_dist_theta = zip(fluxes, dists, thetas) res = pool_map(nproc, _inject_FC, cube, psf, angle_list, plsc, inrad, outrad, iterable(flux_dist_theta), k_list, scaling, collapse_func, patch_size, lr_mode, interp, mode) for m in range(n_req_inject): X_ones_array[m] = res[m] # Moving-subsampling move_subsampling = 'median' if mode == 'mlar' and kss_window is not None: X_ones_array = cube_move_subsample(X_ones_array, kss_window, axis=1, mode=move_subsampling) elif mode == 'tmlar' and tss_window is not None: X_ones_array = cube_move_subsample(X_ones_array, kss_window, axis=1, mode=move_subsampling) elif mode == 'tmlar4d': if tss_window is not None: X_ones_array = cube_move_subsample(X_ones_array, tss_window, axis=1, mode=move_subsampling) if kss_window is not None: X_ones_array = cube_move_subsample(X_ones_array, kss_window, axis=2, mode=move_subsampling) if normalize is not None: if mode == 'tmlar4d': for i in range(X_ones_array.shape[0]): X_ones_array[i] = normalize_01(X_ones_array[i], normalize) else: X_ones_array = normalize_01(X_ones_array, normalize) return X_ones_array.astype('float32')
def make_mlar_samples_ann_noise(input_array, angle_list, cevr_thresh, n_ks, force_klen, inrad, outrad, patch_size, fwhm=4, normalize='slice', nproc2=1, interp='bilinear', lr_mode='eigen', mode='mlar', kss_window=None, tss_window=None, random_seed=42, nsamp_sep=None, verbose=False): """ patch_size in pxs mask is a list of tuples X,Y inputarr is a 3d array or list of 3d arrays orig_zero_patches : percentage of original zero patches """ collapse_func = np.mean scaling = None # 'temp-mean' random_state = np.random.RandomState(random_seed) patches_list = [] frsize = int(input_array.shape[1]) if frsize > outrad + outrad + patch_size + 2: frsize = int(outrad + outrad + patch_size + 2) cube = cube_crop_frames(input_array, frsize, force=True, verbose=False) # making zeros if verbose: print("Creating the ZEROs samples") if not inrad >= int(patch_size/2.) + fwhm: msg = "Warning: The patches are overlapping with the inner 1xFWHM " msg += "annulus" if not inrad > int(np.round(patch_size/2.)): raise RuntimeError("Inner radius must be > half patch_size") k_list = get_cumexpvar(cube, 'annular', inrad, outrad, patch_size, k_list=None, cevr_thresh=cevr_thresh, n_ks=n_ks, match_len=force_klen, verbose=False) resdec = svd_decomp(cube, angle_list, patch_size, inrad, outrad, scaling, k_list, collapse_func, interp=interp, nproc=nproc2, lr_mode=lr_mode, neg_ang=True, mode=mode) cube_residuals_negang = resdec width = outrad - inrad yy, xx = get_annulus_segments((frsize, frsize), inrad, width, 1)[0] if nsamp_sep is None: num_patches = yy.shape[0] else: num_patches = nsamp_sep if num_patches < yy.shape[0]: ind = random_state.choice(yy.shape[0], nsamp_sep, replace=False) else: ind = random_state.choice(yy.shape[0], nsamp_sep, replace=True) yy = yy[ind] xx = xx[ind] cube_residuals_negang = np.asarray(cube_residuals_negang) if verbose: print("Total patches in annulus = {:}".format(num_patches)) for i in range(num_patches): xy = (int(xx[i]), int(yy[i])) patches_list.append(cube_crop_frames(cube_residuals_negang, patch_size, xy=xy, verbose=False, force=True)) # For MLAR and TMLAR X_zeros_array is 4d: # [n_patches_annulus, n_k_list || n_time_steps, patch_size, patch_size] # For TMLAR4D X_zeros_array is 5d: # [n_patches_annulus, n_time_steps, n_k_list, patch_size, patch_size] X_zeros_array = np.array(patches_list) n_ks = len(k_list) # Moving-subsampling move_subsampling = 'median' if mode == 'mlar' and kss_window is not None: X_zeros_array = cube_move_subsample(X_zeros_array, kss_window, axis=1, mode=move_subsampling) elif mode == 'tmlar' and tss_window is not None: X_zeros_array = cube_move_subsample(X_zeros_array, kss_window, axis=1, mode=move_subsampling) elif mode == 'tmlar4d': if tss_window is not None: X_zeros_array = cube_move_subsample(X_zeros_array, tss_window, axis=1, mode=move_subsampling) if kss_window is not None: X_zeros_array = cube_move_subsample(X_zeros_array, kss_window, axis=2, mode=move_subsampling) # Normalization if normalize is not None: if mode == 'tmlar4d': for i in range(X_zeros_array.shape[0]): X_zeros_array[i] = normalize_01(X_zeros_array[i], normalize) else: X_zeros_array = normalize_01(X_zeros_array, normalize) return X_zeros_array.astype('float32'), k_list, n_ks
def do_negfc(self, do_firstguess=True, guess_xy=None, mcmc_negfc=True, inject_neg=True, ncomp=1, algo='pca_annular', nwalkers_ini=120, niteration_min=25, niteration_limit=10000, delta_rot=(0.5, 3), weights=False, coronagraph=False, overwrite=True, save_plot=True, verbose=True): """ Module for estimating the location and flux of a planet. A sub-folder 'negfc' is created for storing all output files. Using a first guess from the (x,y) coordinates in pixels for the planet, we can estimate a preliminary guess for the position and flux for each planet using a Nelder-Mead minimization. Saves the r, theta and flux If using MCMC, runs an affine invariant MCMC sampling algorithm in order to determine the position and the flux of the planet using the 'Negative Fake Companion' technique. The result of this procedure is a chain with the samples from the posterior distributions of each of the 3 parameters. Finally, we can inject a negative flux of the planet found in MCMC into the original dataset and apply post processing to determine the residuals in the data. Parameters: *********** do_firstguess : bool whether to determine a first guess for the position and the flux of a planet (not NEGFC) guess_xy : tuple if do_firstguess=True, this estimate of the source (x,y) location to be provided to firstguess(). Note Python's zero-based indexing mcmc_negfc : bool whether to run MCMC NEGFC sampling (computationally intensive) inject_neg : bool whether to inject negative flux of the planet and post process the data without signal from the planet ncomp : int, default 1 number of prinicple components to subtract algo : 'pca_annulus', 'pca_annular', 'pca'. default 'pca_annular' select which routine to be used to model and subtract the stellar PSF nwalkers_ini : int, default 120 for MCMC, the number of Goodman & Weare 'walkers' niteration_min : int, default 25 for MCMC, the simulation will run at least this number of steps per walker niteration_limit : int, default 10000 for MCMC, stops simulation if this many steps run without having reached the convergence criterion delta_rot : tuple same as for postprocessing module. Threshold rotation angle for PCA annular weights : bool should only be used on unsaturated datasets, where the flux of the star can be measured in each image of the cube. Applies a correction to each frame to account for variability of the adaptive optics, and hence flux from the star (reference Christiaens et al. 2021 2021MNRAS.502.6117C) coronagraph : bool for MCMC and injecting a negative companion, True if the observation utilised a coronagraph (AGPM). The known radial transmission of the NACO+AGPM coronagraph will be used overwrite : bool, default True whether to run a module and overwrite results if they already exist save_plot : bool, default True for firstguess the chi2 vs. flux plot is saved. For MCMC results are pickled and saved to the outpath along with corner plots verbose : bool prints more output and interediate files when True """ print("======= Starting NEGFC....=======") if guess_xy is None and do_firstguess is True: raise ValueError("Enter an approximate location into guess_xy!") if weights is True and coronagraph is True: raise ValueError("Dataset cannot be both non-coronagraphic and coronagraphic!!") outpath_sub = self.outpath + "negfc/" if not isdir(outpath_sub): os.system("mkdir " + outpath_sub) if verbose: print('Input path is {}'.format(self.inpath)) print('Output path is {}'.format(outpath_sub)) source = self.dataset_dict['source'] tn_shift = 0.568 # Launhardt et al. 2020, true North offset for NACO ADI_cube_name = '{}_master_cube.fits' # template name for input master cube derot_ang_name = 'derot_angles.fits' # template name for corresponding input derotation angles psfn_name = "master_unsat_psf_norm.fits" # normalised PSF ADI_cube = open_fits(self.inpath + ADI_cube_name.format(source), verbose=verbose) derot_angles = open_fits(self.inpath + derot_ang_name, verbose=verbose) + tn_shift psfn = open_fits(self.inpath + psfn_name, verbose=verbose) ref_cube = None if algo == 'pca_annular': label_pca = 'pca_annular' algo = pca_annular elif algo == 'pca_annulus': label_pca = 'pca_annulus' algo = pca_annulus elif algo == 'pca': label_pca = 'pca' algo = pca else: raise ValueError("Invalid algorithm. Select either pca_annular, pca_annulus or pca!") opt_npc = ncomp ap_rad = 1 * self.fwhm f_range = np.geomspace(0.1, 201, 40) asize = 3 * self.fwhm if weights: nfr = ADI_cube.shape[0] # number of frames star_flux = np.zeros([nfr]) # for storing the star flux found in each frame crop_sz_tmp = min(int(10 * self.fwhm), ADI_cube.shape[1] - 2) # crop around star, either 10*FWHM or size - 2 if crop_sz_tmp % 2 == 0: # if it's not even, crop crop_sz_tmp -= 1 for ii in range(nfr): _, star_flux[ii], _ = normalize_psf(ADI_cube[ii], fwhm=self.fwhm, size=crop_sz_tmp, full_output=True) # get star flux in 1*FWHM weights = star_flux / np.median(star_flux) star_flux = np.median(star_flux) # for use after MCMC when turning the chain into contrast else: weights = None flux_psf_name = "master_unsat-stellarpsf_fluxes.fits" # flux in a FWHM aperture found in calibration star_flux = open_fits(self.inpath + flux_psf_name, verbose=verbose)[1] # scaled fwhm flux if coronagraph: # radial transmission of the coronagraph, 2 columns (pixels from centre, off-axis transmission) # data provided by Valentin Christiaens. First entry in both columns was not zero, but VIP adds it in anyway transmission = np.array([[0, 3.5894626e-10, 5.0611424e-01, 1.0122285e+00, 1.5183427e+00, 2.0244570e+00, 2.5305712e+00, 3.0366855e+00, 3.5427995e+00, 4.0489140e+00, 4.5550284e+00, 5.0611424e+00, 5.5672565e+00, 6.0733705e+00, 6.5794849e+00, 7.0855989e+00, 7.5917134e+00, 8.6039419e+00, 9.1100569e+00, 9.6161709e+00, 1.0628398e+01, 1.1134513e+01, 1.2146742e+01, 1.2652856e+01, 1.3665085e+01, 1.4677314e+01, 1.6195656e+01, 1.7207884e+01, 1.8220114e+01, 1.9738455e+01, 2.1256796e+01, 2.2775141e+01, 2.4293484e+01, 2.6317940e+01, 2.8342396e+01, 3.0366854e+01, 3.2897423e+01, 3.4921883e+01, 3.7452454e+01, 4.0489140e+01, 4.3525822e+01, 4.6562508e+01, 5.0105309e+01, 5.4154221e+01, 5.7697018e+01, 6.2252052e+01, 6.6807076e+01, 7.1868225e+01], [0, 6.7836474e-05, 3.3822558e-03, 1.7766271e-02, 5.2646037e-02, 1.1413762e-01, 1.9890217e-01, 2.9460809e-01, 3.8605216e-01, 4.6217495e-01, 5.1963091e-01, 5.6185508e-01, 5.9548348e-01, 6.2670821e-01, 6.5912777e-01, 6.9335037e-01, 7.2783405e-01, 7.8866738e-01, 8.1227022e-01, 8.3128709e-01, 8.5912752e-01, 8.6968899e-01, 8.8677746e-01, 8.9409947e-01, 9.0848678e-01, 9.2426234e-01, 9.4704604e-01, 9.5787460e-01, 9.6538281e-01, 9.7379774e-01, 9.8088801e-01, 9.8751044e-01, 9.9255627e-01, 9.9640906e-01, 9.9917024e-01, 1.0009050e+00, 1.0021056e+00, 1.0026742e+00, 1.0027454e+00, 1.0027291e+00, 1.0023015e+00, 1.0016677e+00, 1.0009446e+00, 1.0000550e+00, 9.9953103e-01, 9.9917012e-01, 9.9915260e-01, 9.9922234e-01]]) else: transmission = None if (not isfile(outpath_sub + label_pca + "_npc{}_simplex_results.fits".format(opt_npc)) or overwrite) and do_firstguess: # find r, theta based on the provided estimate location cy, cx = frame_center(ADI_cube[0]) dy_pl = guess_xy[0][1] - cy dx_pl = guess_xy[0][0] - cx r_pl = np.sqrt(np.power(dx_pl, 2) + np.power(dy_pl, 2)) # pixel distance to the guess location theta_pl = (np.rad2deg(np.arctan2(dy_pl, dx_pl))) % 360 # theta (angle) to the guess location print("Estimated (r, theta) before first guess = ({:.1f},{:.1f})".format(r_pl, theta_pl)) ini_state = firstguess(ADI_cube, derot_angles, psfn, ncomp=opt_npc, plsc=self.pixel_scale, planets_xy_coord=guess_xy, fwhm=self.fwhm, annulus_width=12, aperture_radius=ap_rad, cube_ref=ref_cube, svd_mode='lapack', scaling=None, fmerit='stddev', imlib='opencv', interpolation='lanczos4', collapse='median', p_ini=None, transmission=transmission, weights=weights, algo=algo, f_range=f_range, simplex=True, simplex_options=None, plot=save_plot, verbose=verbose, save=save_plot) # when p_ini is set to None, it gets the value of planets_xy_coord ini_state = np.array([ini_state[0][0], ini_state[1][0], ini_state[2][0]]) write_fits(outpath_sub + label_pca + "_npc{}_simplex_results.fits".format(opt_npc), ini_state, verbose=verbose) # saves r, theta and flux. No print statement as firstguess() does that for us if (not isfile(outpath_sub + "MCMC_results") or overwrite) and mcmc_negfc: ini_state = open_fits(outpath_sub + label_pca + "_npc{}_simplex_results.fits".format(opt_npc), verbose=verbose) delta_theta_min = np.rad2deg(np.arctan(4./ r_pl)) # at least the angle corresponding to 2 azimuthal pixels delta_theta = max(delta_theta_min, 5.) bounds = [(max(r_pl - asize / 2., 1), r_pl + asize / 2.), # radius (theta_pl - delta_theta, theta_pl + delta_theta), # angle (0, 5 * abs(ini_state[2]))] if ini_state[0] < bounds[0][0] or ini_state[0] > bounds[0][1] or ini_state[1] < bounds[1][0] or \ ini_state[1] > bounds[1][1] or ini_state[2] < bounds[2][0] or ini_state[2] > bounds[2][1]: print("!!! WARNING: simplex results not in original bounds - NEGFC simplex MIGHT HAVE FAILED !!!") ini_state = np.array([r_pl, theta_pl, abs(ini_state[2])]) if verbose is True: verbosity = 2 print('MCMC NEGFC sampling is about to begin...') else: verbosity = 0 final_chain = mcmc_negfc_sampling(ADI_cube, derot_angles, psfn, ncomp=opt_npc, plsc=self.pixel_scale, initial_state=ini_state, fwhm=self.fwhm, weights=weights, annulus_width=12, aperture_radius=ap_rad, cube_ref=ref_cube, svd_mode='lapack', scaling=None, fmerit='stddev', imlib='opencv', interpolation='lanczos4', transmission=transmission, collapse='median', nwalkers=nwalkers_ini, bounds=bounds, a=2.0, ac_c=50, mu_sigma=(0, 1), burnin=0.3, rhat_threshold=1.01, rhat_count_threshold=1, conv_test='ac', # use autocorrelation 'ac' to ensure sufficient sampling. sample around # the area of best likelihood to make distribution niteration_min=niteration_min, niteration_limit=niteration_limit, niteration_supp=0, check_maxgap=50, nproc=self.nproc, algo=algo, output_dir=outpath_sub, output_file="MCMC_results", display=False, verbosity=verbosity, save=save_plot) final_chain[:, :, 2] = final_chain[:, :, 2] / star_flux # dividing by the star flux converts to a contrast show_walk_plot(final_chain, save=save_plot, output_dir=outpath_sub) show_corner_plot(final_chain, burnin=0.5, save=save_plot, output_dir=outpath_sub) # determine the highly probable value for each model parameter and the 1-sigma confidence interval isamples_flat = final_chain[:,int(final_chain.shape[1]//(1/0.3)):,:].reshape((-1,3)) # 0.3 is the burnin vals, err = confidence(isamples_flat, cfd=68.27, bins=100, gaussian_fit=False, weights=weights, verbose=verbose, save=True, output_dir=outpath_sub, filename='confidence.txt', plsc=self.pixel_scale) labels = ['r', 'theta', 'f'] mcmc_res = np.zeros([3,3]) # pull the values and confidence interval out for saving for i in range(3): mcmc_res[i,0] = vals[labels[i]] mcmc_res[i,1] = err[labels[i]][0] mcmc_res[i,2] = err[labels[i]][1] write_fits(outpath_sub + 'mcmc_results.fits', mcmc_res) # now gaussian fit gvals, gerr = confidence(isamples_flat, cfd=68.27, bins=100, gaussian_fit=True, weights=weights, verbose=verbose, save=True, output_dir=outpath_sub,filename='confidence_gauss.txt', plsc=self.pixel_scale) mcmc_res = np.zeros([3,2]) for i in range(3): mcmc_res[i,0] = gvals[i] mcmc_res[i,1] = gerr[i] write_fits(outpath_sub + 'mcmc_results_gauss.fits', mcmc_res) if inject_neg: pca_res = np.zeros([ADI_cube.shape[1], ADI_cube.shape[2]]) pca_res_emp = pca_res.copy() planet_params = open_fits(outpath_sub+'mcmc_results.fits') flux_psf_name = "master_unsat-stellarpsf_fluxes.fits" star_flux = open_fits(self.inpath + flux_psf_name, verbose=verbose)[1] ADI_cube_emp = cube_inject_companions(ADI_cube, psfn, derot_angles, flevel=-planet_params[2, 0] * star_flux, plsc=self.pixel_scale, rad_dists=[planet_params[0, 0]], n_branches=1, theta=planet_params[1, 0], imlib='opencv', interpolation='lanczos4', verbose=verbose, transmission=transmission) write_fits(outpath_sub+'ADI_cube_empty.fits', ADI_cube_emp) # the cube with the negative flux injected if algo == pca_annular: radius_int = int(np.floor(r_pl-asize/2)) # asize is 3 * FWHM, rounds down. To skip the inner region # crop the cube to just larger than the annulus to improve the speed of PCA crop_sz = int(2*np.ceil(r_pl+asize+1)) # rounds up if not crop_sz % 2: # make sure the crop is odd crop_sz += 1 if crop_sz < ADI_cube.shape[1] and crop_sz < ADI_cube.shape[2]: # crop if crop_sz is smaller than cube pad = int((ADI_cube.shape[1]-crop_sz)/2) crop_cube = cube_crop_frames(ADI_cube, crop_sz, verbose=verbose) else: crop_cube = ADI_cube # dont crop if the cube is already smaller pca_res_tmp = pca_annular(crop_cube, derot_angles, cube_ref=ref_cube, radius_int=radius_int, fwhm=self.fwhm, asize=asize, delta_rot=delta_rot, ncomp=opt_npc, svd_mode='lapack', scaling=None, imlib='opencv', interpolation='lanczos4', nproc=self.nproc, min_frames_lib=max(opt_npc, 10), verbose=verbose, full_output=False) pca_res = np.pad(pca_res_tmp, pad, mode='constant', constant_values=0) write_fits(outpath_sub + 'pca_annular_res_npc{}.fits'.format(opt_npc), pca_res) # emp if crop_sz < ADI_cube_emp.shape[1] and crop_sz < ADI_cube_emp.shape[2]: pad = int((ADI_cube_emp.shape[1]-crop_sz)/2) crop_cube = cube_crop_frames(ADI_cube_emp, crop_sz, verbose=verbose) else: crop_cube = ADI_cube_emp del ADI_cube_emp del ADI_cube pca_res_tmp = pca_annular(crop_cube, derot_angles, cube_ref=ref_cube, radius_int=radius_int, fwhm=self.fwhm, asize=asize, delta_rot=delta_rot, ncomp=opt_npc, svd_mode='lapack', scaling=None, imlib='opencv', interpolation='lanczos4', nproc=self.nproc, min_frames_lib=max(opt_npc, 10), verbose=verbose, full_output=False) # pad again now pca_res_emp = np.pad(pca_res_tmp, pad, mode='constant', constant_values=0) write_fits(outpath_sub+'pca_annular_res_empty_npc{}.fits'.format(opt_npc), pca_res_emp)
def recenter(self, nproc=1, sigfactor=4, subi_size=21, crop_sz=None, verbose=True, debug=False, plot=False, coro=True): """ Recenters cropped science images by fitting a double Gaussian (negative+positive) to each median combined SCI cube, or by fitting a single negative Gaussian to the coronagraph using the speckle pattern of each median combined SCI cube. Parameters: *********** sigfactor: float, default = 4 If thresholding is performed during 2gauss fitting, set the threshold in terms of gaussian sigma in the subimage (will depend on your cropping size) subi_size: int, default = 21 Size of the square subimage sides in pixels. crop_sz: int, optional, in units of pixels. None by default Crops to this size after recentering for memory management purposes. Useful for very large datasets verbose: True for False To provide extra information about the progress and results of the pipeline plot: True or False Set to False when running on M3 coro: True for coronagraph data. False otherwise. Recentering requires coronagraphic data Writes fits to file: *********** x_shifts.fits # writes the x shifts to the file y_shifts.fits # writes the y shifts to the file {source}_master_cube.fits # makes the recentered master cube derot_angles.fits # makes a vector of derotation angles """ if not coro: if self.recenter_method != '2dfit': raise ValueError('Recentering method invalid') if self.recenter_model == '2gauss': raise ValueError('2Gauss requires coronagraphic data') if verbose: print(len(self.sci_list), 'science cubes') if debug: print(self.sci_list) ncubes = len(self.sci_list) fwhm_all = open_fits( self.inpath + 'fwhm.fits', verbose=debug ) # changed this to open the file as sometimes we wont run get_stellar_psf() or it may have already run fwhm = fwhm_all[0] # fwhm is the first entry in the file fwhm = fwhm.item( ) # changes from numpy.float32 to regular float so it will work in VIP if verbose: print('fwhm:', fwhm, 'of type', type(fwhm)) mem = np.zeros(len(self.sci_list)) # Creates a master science cube with just the median of each cube bar = pyprind.ProgBar( len(self.sci_list), stream=1, title= 'Creating master science cube (median of each science cube)....') for sc, fits_name in enumerate( self.sci_list): # enumerate over the list of all science cubes tmp = open_fits(self.inpath + '4_sky_subtr_imlib_' + fits_name, verbose=debug) #open cube as tmp if sc == 0: self.ndit, ny, nx = tmp.shape #dimensions of cube tmp_tmp = np.zeros( [ncubes, ny, nx] ) # template cube with the median of each SCI cube. np.zeros is array filled with zeros #mem_msg = 'Set check_memory=False to override this memory check' tmp_tmp[sc] = np.median(tmp, axis=0) # median frame of cube tmp input_bytes = tmp.nbytes memory = check_enough_memory(input_bytes, verbose=True) tmp = None mem[sc] = memory bar.update() write_fits(self.outpath + 'memory.fits', mem, verbose=debug) if self.recenter_method == 'speckle': # FOR GAUSSIAN print('##### Recentering via speckle pattern #####') #registered science sube, low+high pass filtered cube,cube with stretched values, x shifts, y shifts tmp_tmp, cube_sci_lpf, cube_stret, sx, sy = cube_recenter_via_speckles( tmp_tmp, cube_ref=None, alignment_iter=5, gammaval=1, min_spat_freq=0.5, max_spat_freq=3, fwhm=fwhm, debug=debug, recenter_median=True, negative=coro, fit_type='gaus', crop=False, subframesize=subi_size, imlib='opencv', interpolation='lanczos4', plot=plot, full_output=True) del cube_sci_lpf del cube_stret elif self.recenter_method == '2dfit': # DOUBLE GAUSSIAN print('##### Recentering via 2dfit #####') params_2g = { 'fwhm_neg': 0.8 * fwhm, 'fwhm_pos': 2 * fwhm, 'theta_neg': 48., 'theta_pos': 135., 'neg_amp': 0.8 } res = cube_recenter_2dfit(tmp_tmp, xy=None, fwhm=fwhm, subi_size=subi_size, model=self.recenter_model, nproc=nproc, imlib='opencv', interpolation='lanczos4', offset=None, negative=False, threshold=True, sigfactor=sigfactor, fix_neg=False, params_2g=params_2g, save_shifts=False, full_output=True, verbose=verbose, debug=debug, plot=plot) sy = res[1] sx = res[2] # true_agpm_cen = (res[4][0],res[3][0]) # true_fwhm_pos = (res[5][0],res[6][0]) # true_fwhm_neg = (res[7][0],res[8][0]) # true_theta_pos = res[9][0] # true_theta_neg = res[10][0] # amp_pos = res[11][0] # amp_neg = res[12][0] # true_neg_amp = amp_neg/amp_pos # params_2g = {'fwhm_neg': true_fwhm_neg, 'fwhm_pos': true_fwhm_pos, # 'theta_neg': true_theta_neg, 'theta_pos':true_theta_pos, # 'neg_amp': true_neg_amp} # # second: fixing params for neg gaussian - applied on individual frames. returns recentered array, and x-y shifts # tmp_tmp, sy, sx = cube_recenter_2dfit(tmp_tmp, xy=true_agpm_cen, # fwhm=self.fwhm, subi_size=subi_size, # model=model, nproc=nproc, imlib='opencv', # interpolation='lanczos4', # offset=None, negative=False, # threshold=True, sigfactor=sigfactor, # fix_neg=True, params_2g=params_2g, # save_shifts=False, full_output=True, # verbose=verbose, debug=debug, plot=plot) # LOAD IN REAL_NDIT_SCI # Load original cubes, shift them, and create master cube tmp_tmp = np.zeros( [int(np.sum(self.real_ndit_sci)), ny, nx] ) #makes an array full of zeros, length of the sum of each entry in the sci dimensions file. we dont need our old tmp_tmp anymore angles_1dvector = np.zeros([ int(np.sum(self.real_ndit_sci)) ]) # makes empty array for derot angles, length of number of frames for sc, fits_name in enumerate(self.sci_list): tmp = open_fits(self.inpath + '4_sky_subtr_imlib_' + fits_name, verbose=debug) #opens science cube dim = int(self.real_ndit_sci[sc] ) #gets the integer dimensions of this science cube for dd in range(dim): #dd goes from 0 to the largest dimension tmp_tmp[int( np.sum(self.real_ndit_sci[:sc]) ) + dd] = frame_shift( tmp[dd], shift_y=sy[sc], shift_x=sx[sc], imlib='opencv' ) #this line applies the shifts to all the science images in the cube the loop is currently on. it also converts all cubes to a single long cube by adding the first dd frames, then the next dd frames from the next cube and so on angles_1dvector[int( np.sum(self.real_ndit_sci[:sc]) ) + dd] = self.derot_angles_cropped[sc][ dd] # turn 2d rotation file into a vector here same as for the mastercube above # sc*ndit+dd i don't think this line works for variable sized cubes tmp = None # memory management pathlib.Path(self.outpath).mkdir(parents=True, exist_ok=True) if crop_sz is not None: if not crop_sz % 2: crop_sz += 1 print('Crop size not odd, increased to {}'.format(crop_sz)) print('Cropping to {} pixels'.format(crop_sz)) tmp_tmp = cube_crop_frames(tmp_tmp, crop_sz, force=False, verbose=debug, full_output=False) # write all the shifts write_fits(self.outpath + 'x_shifts.fits', sx) # writes the x shifts to the file write_fits(self.outpath + 'y_shifts.fits', sy) # writes the y shifts to the file write_fits(self.outpath + '{}_master_cube.fits'.format(self.dataset_dict['source']), tmp_tmp) #makes the master cube write_fits(self.outpath + 'derot_angles.fits', angles_1dvector) # writes the 1D array of derotation angles if verbose: print('Shifts applied, master cube saved') tmp_tmp = None
def bad_frame_removal(self, pxl_shift_thres=0.5, sub_frame_sz=31, verbose=True, debug=False, plot='save'): """ For removing outlier frames often caused by AO errors. To be run after recentering is complete. Takes the recentered mastercube and removes frames with a shift greater than a user defined pixel threshold in x or y above the median shift. It then takes the median of those cubes and correlates them to the median combined mastercube. Removes all those frames below the threshold from the mastercube and rotation file, then saves both as new files for use in post processing pxl_shift_thres: decimal, in units of pixels. Default is 0.5 pixels. Any shifts in the x or y direction greater than this threshold will cause the frame/s to be labelled as bad and thus removed sub_frame_sz: integer, must be odd. Default is 31. This sets the cropping during frame correlation to the median plot: 'save' to write to file the correlation plot, None will not """ if verbose: print('\n') print('Beginning bad frame removal...') print('\n') angle_file = open_fits(self.outpath + 'derot_angles.fits', verbose=debug) #opens the rotation file recentered_cube = open_fits( self.outpath + '{}_master_cube.fits'.format(self.dataset_dict['source']), verbose=debug) # loads the master cube #open x shifts file for the respective method x_shifts = open_fits(self.outpath + "x_shifts.fits", verbose=debug) median_sx = np.median(x_shifts) #median of x shifts #opens y shifts file for the respective method y_shifts = open_fits(self.outpath + "y_shifts.fits", verbose=debug) median_sy = np.median(y_shifts) #median of y shifts #x_shifts_long = np.zeros([len(self.sci_list)*self.ndit]) # list with number of cubes times number of frames in each cube as the length #y_shifts_long = np.zeros([len(self.sci_list)*self.ndit]) if not self.fast_reduction: # long are shifts to be apply to each frame in each cube. fast reduction only has one cube x_shifts_long = np.zeros([int(np.sum(self.real_ndit_sci))]) y_shifts_long = np.zeros([int(np.sum(self.real_ndit_sci))]) for i in range(len( self.sci_list)): # from 0 to the length of sci_list ndit = self.real_ndit_sci[i] # gets the dimensions of the cube x_shifts_long[i * ndit:(i + 1) * ndit] = x_shifts[ i] # sets the average shifts of all frames in that cube y_shifts_long[i * ndit:(i + 1) * ndit] = y_shifts[i] write_fits(self.outpath + 'x_shifts_long.fits', x_shifts_long, verbose=debug) # saves shifts to file write_fits(self.outpath + 'y_shifts_long.fits', y_shifts_long, verbose=debug) x_shifts = x_shifts_long y_shifts = y_shifts_long if verbose: print("x shift median:", median_sx) print("y shift median:", median_sy) bad = [] good = [] # this system works, however it can let a couple of wild frames slip through at a 0.5 pixel threshold. may need a stricter threshold depending on the data and crop size i = 0 shifts = list(zip(x_shifts, y_shifts)) bar = pyprind.ProgBar(len(x_shifts), stream=1, title='Running pixel shift check...') for sx, sy in shifts: #iterate over the shifts to find any greater or less than pxl_shift_thres pixels from median if abs(sx) < ((abs(median_sx)) + pxl_shift_thres) and abs(sx) > ( (abs(median_sx)) - pxl_shift_thres) and abs(sy) < ( (abs(median_sy)) + pxl_shift_thres) and abs(sy) > ( (abs(median_sy)) - pxl_shift_thres): good.append(i) else: bad.append(i) i += 1 bar.update() # only keeps the files that weren't shifted above the threshold frames_pxl_threshold = recentered_cube[good] recentered_cube = None if verbose: print('Frames within pixel shift threshold:', len(frames_pxl_threshold)) #recentered_cube = None # only keeps the corresponding derotation entry for the frames that were kept angle_pxl_threshold = angle_file[good] if verbose: print( '########### Median combining {} frames for correlation check... ###########' .format(len(frames_pxl_threshold))) #makes array of good frames from the recentered mastercube subarray = cube_crop_frames( frames_pxl_threshold, size=sub_frame_sz, verbose=verbose) # crops all the frames to a common size frame_ref = np.median( subarray, axis=0 ) #median frame of remaining cropped frames, can be sped up with multi-processing if verbose: print('Running frame correlation check...') # calculates correlation threshold using the median of the Pearson correlation of all frames, minus 1 standard deviation #frame_ref = frame_crop(tmp_median, size = sub_frame_sz, verbose=verbose) # crops the median of all frames to a common size distances = cube_distance( subarray, frame_ref, mode='full', dist='pearson', plot=True ) # calculates the correlation of each frame to the median and saves as a list if plot == 'save': # save a plot of distances compared to the median for each frame if set to 'save' plt.savefig(self.outpath + 'distances.pdf') correlation_thres = np.median(distances) - np.std( distances ) # threshold is the median of the distances minus one stddev good_frames, bad_frames = cube_detect_badfr_correlation( subarray, frame_ref=frame_ref, dist='pearson', threshold=correlation_thres, plot=True, verbose=False) if plot == 'save': plt.savefig(self.outpath + 'frame_correlation.pdf') #only keeps the files that were above the correlation threshold frames_threshold = frames_pxl_threshold[good_frames] frames_pxl_threshold = None if verbose: print('Frames within correlation threshold:', len(frames_threshold)) #frames_pxl_threshold = None # only keeps the derotation entries for the good frames above the correlation threshold angle_threshold = angle_pxl_threshold[good_frames] # saves the good frames to a new file, and saves the derotation angles to a new file write_fits( self.outpath + '{}_master_cube.fits'.format(self.dataset_dict['source']), frames_threshold) write_fits(self.outpath + 'derot_angles.fits', angle_threshold) if verbose: print('Saved good frames and their respective rotations to file') frames_threshold = None
def correct_bad_pixels(self, verbose=True, debug=False): sci_list = [] with open(self.outpath + "sci_list.txt", "r") as f: tmp = f.readlines() for line in tmp: sci_list.append(line.split('\n')[0]) sky_list = [] with open(self.outpath + "sky_list.txt", "r") as f: tmp = f.readlines() for line in tmp: sky_list.append(line.split('\n')[0]) unsat_list = [] with open(self.outpath + "unsat_list.txt", "r") as f: tmp = f.readlines() for line in tmp: unsat_list.append(line.split('\n')[0]) n_sci = len(sci_list) ndit_sci = fits_info.ndit_sci n_sky = len(sky_list) ndit_sky = fits_info.ndit_sky tmp = open_fits(self.outpath + '1_crop_unsat_' + unsat_list[-1], header=False) nx_unsat_crop = tmp.shape[2] master_flat_frame = open_fits(self.outpath + 'master_flat_field.fits') # Create bpix map bpix = np.where(np.abs(master_flat_frame - 1.09) > 0.41) # i.e. for QE < 0.68 and QE > 1.5 bpix_map = np.zeros([self.com_sz, self.com_sz]) bpix_map[bpix] = 1 if nx_unsat_crop < bpix_map.shape[1]: bpix_map_unsat = frame_crop(bpix_map, nx_unsat_crop) else: bpix_map_unsat = bpix_map #number of bad pixels nbpix = int(np.sum(bpix_map)) ntotpix = self.com_sz**2 print("total number of bpix: ", nbpix) print("total number of pixels: ", ntotpix) print("=> {}% of bad pixels.".format(100 * nbpix / ntotpix)) write_fits(self.outpath + 'master_bpix_map.fits', bpix_map) write_fits(self.outpath + 'master_bpix_map_unsat.fits', bpix_map_unsat) plot_frames(bpix_map, bpix_map_unsat) #update final crop size final_sz = self.get_final_sz() #crop frames to that size for sc, fits_name in enumerate(sci_list): tmp = open_fits(self.outpath + '2_nan_corr_' + fits_name, verbose=False) tmp_tmp = cube_crop_frames(tmp, final_sz, xy=self.agpm_pos) write_fits(self.outpath + '2_crop_' + fits_name, tmp_tmp) if not debug: os.system("rm " + self.outpath + '2_nan_corr_' + fits_name) for sk, fits_name in enumerate(sky_list): tmp = open_fits(self.outpath + '2_nan_corr_' + fits_name, verbose=False) tmp_tmp = cube_crop_frames(tmp, final_sz, xy=self.agpm_pos) write_fits(self.outpath + '2_crop_' + fits_name, tmp_tmp) if not debug: os.system("rm " + self.outpath + '2_nan_corr_' + fits_name) if not debug: tmp = open_fits(self.outpath + '2_crop_' + sci_list[0])[-1] tmp_tmp = open_fits(self.outpath + '2_crop_' + sci_list[-1])[-1] plot_frames(tmp, tmp_tmp) else: # COMPARE BEFORE AND AFTER NAN_CORR + CROP old_tmp = open_fits(self.outpath + '2_ff_' + sci_list[0])[-1] old_tmp_tmp = open_fits(self.outpath + '2_ff_' + sci_list[-1])[-1] tmp = open_fits(self.outpath + '2_crop_' + sci_list[0])[-1] tmp_tmp = open_fits(self.outpath + '2_crop_' + sci_list[-1])[-1] plot_frames(old_tmp, tmp, old_tmp_tmp, tmp_tmp) # Crop the bpix map in a same way bpix_map = open_fits(self.outpath + 'master_bpix_map.fits') bpix_map_2ndcrop = frame_crop(bpix_map, final_sz, cenxy=self.agpm_pos) write_fits(self.outpath + 'master_bpix_map_2ndcrop.fits', bpix_map_2ndcrop) bpix_map = open_fits(self.outpath + 'master_bpix_map_2ndcrop.fits') t0 = time_ini() for sc, fits_name in enumerate(sci_list): tmp = open_fits(self.outpath + '2_crop_' + fits_name, verbose=False) # first with the bp max defined from the flat field (without protecting radius) tmp_tmp = cube_fix_badpix_isolated(tmp, bpm_mask=bpix_map, sigma_clip=7, num_neig=5, size=5, protect_mask=True, radius=9, verbose=False, debug=False) write_fits(self.outpath + '2_bpix_corr_' + fits_name, tmp_tmp, verbose=False) timing(t0) # second, residual hot pixels tmp_tmp, bpm = cube_fix_badpix_isolated(tmp_tmp, bpm_mask=None, sigma_clip=8, num_neig=5, size=5, protect_mask=True, radius=10, verbose=False, debug=False, full_output=True) write_fits(self.outpath + '2_bpix_corr2_' + fits_name, tmp_tmp) write_fits(self.outpath + '2_bpix_corr2_map_' + fits_name, bpm) timing(t0) if not debug: os.system("rm " + self.outpath + '2_crop_' + fits_name) bpix_map = open_fits(self.outpath + 'master_bpix_map_2ndcrop.fits') t0 = time_ini() for sk, fits_name in enumerate(sky_list): tmp = open_fits(self.outpath + '2_crop_' + fits_name, verbose=False) # first with the bp max defined from the flat field (without protecting radius) tmp_tmp = cube_fix_badpix_isolated(tmp, bpm_mask=bpix_map, sigma_clip=7, num_neig=5, size=5, protect_mask=True, radius=9, verbose=False, debug=False) write_fits(self.outpath + '2_bpix_corr_' + fits_name, tmp_tmp, verbose=False) timing(t0) # second, residual hot pixels tmp_tmp, bpm = cube_fix_badpix_isolated(tmp_tmp, bpm_mask=None, sigma_clip=8, num_neig=5, size=5, protect_mask=True, radius=10, verbose=False, debug=False, full_output=True) write_fits(self.outpath + '2_bpix_corr2_' + fits_name, tmp_tmp) write_fits(self.outpath + '2_bpix_corr2_map_' + fits_name, bpm) timing(t0) if not debug: os.system("rm " + self.outpath + '2_crop_' + fits_name) bpix_map_unsat = open_fits(self.outpath + 'master_bpix_map_unsat.fits') t0 = time_ini() for un, fits_name in enumerate(unsat_list): tmp = open_fits(self.outpath + '2_nan_corr_unsat_' + fits_name, verbose=False) # first with the bp max defined from the flat field (without protecting radius) tmp_tmp = cube_fix_badpix_isolated(tmp, bpm_mask=bpix_map_unsat, sigma_clip=7, num_neig=5, size=5, protect_mask=True, radius=9, verbose=False, debug=False) write_fits(self.outpath + '2_bpix_corr_unsat_' + fits_name, tmp_tmp) timing(t0) # second, residual hot pixels tmp_tmp, bpm = cube_fix_badpix_isolated(tmp_tmp, bpm_mask=None, sigma_clip=8, num_neig=5, size=5, protect_mask=True, radius=10, verbose=False, debug=False, full_output=True) write_fits(self.outpath + '2_bpix_corr2_unsat_' + fits_name, tmp_tmp) write_fits(self.outpath + '2_bpix_corr2_map_unsat_' + fits_name, bpm) timing(t0) if not debug: os.system("rm " + self.outpath + '2_nan_corr_unsat_' + fits_name) # FIRST CREATE MASTER CUBE FOR SCI tmp_tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' + sci_list[0], verbose=False) n_y = tmp_tmp_tmp.shape[1] n_x = tmp_tmp_tmp.shape[2] tmp_tmp_tmp = np.zeros([n_sci, n_y, n_x]) for sc, fits_name in enumerate(sci_list): tmp_tmp_tmp[sc] = open_fits( self.outpath + '2_bpix_corr2_' + fits_name, verbose=False)[int(random.randrange(min(ndit_sci)))] tmp_tmp_tmp = np.median(tmp_tmp_tmp, axis=0) write_fits(self.outpath + 'TMP_2_master_median_SCI.fits', tmp_tmp_tmp) # THEN CREATE MASTER CUBE FOR SKY tmp_tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' + sky_list[0], verbose=False) n_y = tmp_tmp_tmp.shape[1] n_x = tmp_tmp_tmp.shape[2] tmp_tmp_tmp = np.zeros([n_sky, n_y, n_x]) for sk, fits_name in enumerate(sky_list): tmp_tmp_tmp[sk] = open_fits( self.outpath + '2_bpix_corr2_' + fits_name, verbose=False)[int(random.randrange(min(ndit_sky)))] tmp_tmp_tmp = np.median(tmp_tmp_tmp, axis=0) write_fits(self.outpath + 'TMP_2_master_median_SKY.fits', tmp_tmp_tmp) bpix_map_ori = open_fits(self.outpath + 'master_bpix_map_2ndcrop.fits') bpix_map_sci_0 = open_fits(self.outpath + '2_bpix_corr2_map_' + sci_list[0]) bpix_map_sci_1 = open_fits(self.outpath + '2_bpix_corr2_map_' + sci_list[-1]) bpix_map_sky_0 = open_fits(self.outpath + '2_bpix_corr2_map_' + sky_list[0]) bpix_map_sky_1 = open_fits(self.outpath + '2_bpix_corr2_map_' + sky_list[-1]) bpix_map_unsat_0 = open_fits(self.outpath + '2_bpix_corr2_map_unsat_' + unsat_list[0]) bpix_map_unsat_1 = open_fits(self.outpath + '2_bpix_corr2_map_unsat_' + unsat_list[-1]) plot_frames( bpix_map_ori, bpix_map_sci_0, bpix_map_sci_1, #tmp_tmp_tmp, #bpix_tmp, bpix_map_sky_0, bpix_map_sky_1, #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY bpix_map_unsat_0, bpix_map_unsat_1 #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY ) tmpSCI = open_fits(self.outpath + 'TMP_2_master_median_SCI.fits') tmpSKY = open_fits(self.outpath + 'TMP_2_master_median_SKY.fits') if not debug: tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' + sci_list[1])[-1] tmp_tmp2 = open_fits(self.outpath + '2_bpix_corr2_' + sci_list[-1])[-1] plot_frames( #tmp, tmp-tmpSKY, #tmp_tmp_tmp, #bpix_tmp, tmp_tmp, tmp_tmp - tmpSKY, #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY #tmp2, tmp2-tmpSKY, #tmp_tmp_tmp, #bpix_tmp, tmp_tmp2, tmp_tmp2 - tmpSKY #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY ) else: # COMPARE BEFORE AND AFTER BPIX CORR (without sky subtr) tmp = open_fits(self.outpath + '2_crop_' + sci_list[-1])[-1] tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' + sci_list[-1])[-1] tmp2 = open_fits(self.outpath + '2_crop_' + sky_list[-1])[-1] tmp_tmp2 = open_fits(self.outpath + '2_bpix_corr2_' + sky_list[-1])[-1] ( tmp, tmp - tmpSKY, #tmp_tmp_tmp, #bpix_tmp, tmp_tmp, tmp_tmp - tmpSKY, #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY tmp2, tmp2 - tmpSKY, #tmp_tmp_tmp, #bpix_tmp, tmp_tmp2, tmp_tmp2 - tmpSKY #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY )
def dark_subtract(self, verbose=True, debug=True): sci_list = [] with open(self.outpath + "sci_list.txt", "r") as f: tmp = f.readlines() for line in tmp: sci_list.append(line.split('\n')[0]) sky_list = [] with open(self.outpath + "sky_list.txt", "r") as f: tmp = f.readlines() for line in tmp: sky_list.append(line.split('\n')[0]) unsat_list = [] with open(self.outpath + "unsat_list.txt", "r") as f: tmp = f.readlines() for line in tmp: unsat_list.append(line.split('\n')[0]) unsat_dark_list = [] with open(self.outpath + "unsat_dark_list.txt", "r") as f: tmp = f.readlines() for line in tmp: unsat_dark_list.append(line.split('\n')[0]) flat_list = [] with open(self.outpath + "flat_list.txt", "r") as f: tmp = f.readlines() for line in tmp: flat_list.append(line.split('\n')[0]) flat_dark_list = [] with open(self.outpath + "flat_dark_list.txt", "r") as f: tmp = f.readlines() for line in tmp: flat_dark_list.append(line.split('\n')[0]) sci_dark_list = [] with open(self.outpath + "sci_dark_list.txt", "r") as f: tmp = f.readlines() for line in tmp: sci_dark_list.append(line.split('\n')[0]) pixel_scale = fits_info.pixel_scale tmp = np.zeros([3, self.com_sz, self.com_sz]) #creating master dark cubes for fd, fd_name in enumerate(flat_dark_list): tmp_tmp = open_fits(self.inpath + fd_name, header=False, verbose=debug) tmp[fd] = frame_crop(tmp_tmp, self.com_sz, force=True, verbose=debug) write_fits(self.outpath + 'flat_dark_cube.fits', tmp) if verbose: print('Flat dark cubes have been cropped and saved') for sd, sd_name in enumerate(sci_dark_list): tmp_tmp = open_fits(self.inpath + sd_name, header=False, verbose=debug) n_dim = tmp_tmp.ndim if sd == 0: if n_dim == 2: tmp = np.array([ frame_crop(tmp_tmp, self.com_sz, force=True, verbose=debug) ]) else: tmp = cube_crop_frames(tmp_tmp, self.com_sz, force=True, verbose=debug) else: if n_dim == 2: tmp = np.append(tmp, [ frame_crop( tmp_tmp, self.com_sz, force=True, verbose=debug) ], axis=0) else: tmp = np.append(tmp, cube_crop_frames(tmp_tmp, self.com_sz, force=True, verbose=debug), axis=0) write_fits(self.outpath + 'sci_dark_cube.fits', tmp) if verbose: print('Sci dark cubes have been cropped and saved') #create an if stament for if the size is larger than sz and less than if less than crop by nx-1 for sd, sd_name in enumerate(unsat_dark_list): tmp_tmp = open_fits(self.inpath + sd_name, header=False, verbose=debug) tmp = np.zeros([ len(sci_dark_list) * tmp_tmp.shape[0], tmp_tmp.shape[1], tmp_tmp.shape[2] ]) n_dim = tmp_tmp.ndim if sd == 0: if n_dim == 2: ny, nx = tmp_tmp.shape if nx <= self.com_sz: tmp = np.array([ frame_crop(tmp_tmp, nx - 1, force=True, verbose=debug) ]) else: tmp = np.array( [frame_crop(tmp_tmp, self.com_sz, verbose=debug)]) else: nz, ny, nx = tmp_tmp.shape if nx <= self.com_sz: tmp = cube_crop_frames(tmp_tmp, nx - 1, force=True, verbose=debug) else: tmp = cube_crop_frames(tmp_tmp, self.com_sz, force=True, verbose=debug) else: if n_dim == 2: ny, nx = tmp_tmp.shape if nx <= self.com_sz: tmp = np.append(tmp, [ frame_crop( tmp_tmp, nx - 1, force=True, verbose=debug) ], axis=0) else: tmp = np.append(tmp, [ frame_crop(tmp_tmp, self.com_sz, force=True, verbose=debug) ], axis=0) else: nz, ny, nx = tmp_tmp.shape if nx <= self.com_sz: tmp = cube_crop_frames(tmp, nx - 1, force=True, verbose=debug) tmp = np.append(tmp, cube_crop_frames(tmp_tmp, nx - 1, force=True, verbose=debug), axis=0) else: tmp = np.append(tmp, cube_crop_frames(tmp_tmp, self.com_sz, force=True, verbose=debug), axis=0) tmp = np.zeros([ len(sci_dark_list) * tmp_tmp.shape[0], tmp_tmp.shape[1], tmp_tmp.shape[2] ]) write_fits(self.outpath + 'unsat_dark_cube.fits', tmp) if verbose: print('Unsat dark cubes have been cropped and saved') #defining the anulus (this is where we avoid correcting around the star) cy, cx = find_agpm_list(self, sci_list) self.agpm_pos = (cx, cy) if verbose: print(' The location of the AGPM has been calculated', 'cy = ', cy, 'cx = ', cx) agpm_dedge = min(self.agpm_pos[0], self.agpm_pos[1], self.com_sz - self.agpm_pos[0], self.com_sz - self.agpm_pos[1]) mask_arr = np.ones([self.com_sz, self.com_sz]) cy, cx = frame_center(mask_arr) mask_inner_rad = int(3.0 / pixel_scale) # 3arcsec min to avoid star emission mask_width = agpm_dedge - mask_inner_rad - 1 mask_AGPM_com = get_annulus_segments(mask_arr, mask_inner_rad, mask_width, mode='mask')[0] mask_AGPM_com = frame_shift( mask_AGPM_com, self.agpm_pos[1] - cy, self.agpm_pos[0] - cx) # agpm is not centered in the frame so shift the mask if verbose: print('AGPM mask has been defined') if debug: tmp = open_fits(self.outpath + sci_list[0]) #plot_frames(tmp[-1], circle = self.agpm_pos) #now begin the dark subtraction useing PCA npc_dark = 1 #val found this value gives the best result. tmp_tmp = np.zeros([len(flat_list), self.com_sz, self.com_sz]) tmp_tmp_tmp = open_fits(self.outpath + 'flat_dark_cube.fits') for fl, flat_name in enumerate(flat_list): tmp = open_fits(self.inpath + flat_name, header=False, verbose=debug) tmp_tmp[fl] = frame_crop(tmp, self.com_sz, force=True, verbose=debug) tmp_tmp = cube_subtract_sky_pca(tmp_tmp, tmp_tmp_tmp, mask_AGPM_com, ref_cube=None, ncomp=npc_dark) write_fits(self.outpath + '1_crop_flat_cube.fits', tmp_tmp) if verbose: print('Dark has been subtracted from Flats') if debug: #plot the median of dark cube median of cube before subtraction median after subtraction tmp_tmp_tmp = np.median(tmp_tmp_tmp, axis=0) #flat_dark cube median tmp = tmp #flat before subtraction tmp_tmp = np.median(tmp_tmp, axis=0) #median flat after dark subtract plot_frames((tmp_tmp_tmp, tmp, tmp_tmp)) tmp_tmp_tmp = open_fits(self.outpath + 'sci_dark_cube.fits') for sc, fits_name in enumerate(sci_list): tmp = open_fits(self.inpath + fits_name, header=False, verbose=debug) tmp = cube_crop_frames(tmp, self.com_sz, force=True, verbose=debug) tmp_tmp = cube_subtract_sky_pca(tmp, tmp_tmp_tmp, mask_AGPM_com, ref_cube=None, ncomp=npc_dark) write_fits(self.outpath + '1_crop_' + fits_name, tmp_tmp) if verbose: print('Dark has been subtracted from Sci') if debug: #plot the median of dark cube median of cube before subtraction median after subtraction tmp_tmp_tmp = np.median(tmp_tmp_tmp, axis=0) tmp = np.median(tmp, axis=0) tmp_tmp = np.median(tmp_tmp, axis=0) plot_frames((tmp_tmp_tmp, tmp, tmp_tmp)) tmp_tmp_tmp = open_fits(self.outpath + 'sci_dark_cube.fits') for sc, fits_name in enumerate(sky_list): tmp = open_fits(self.inpath + fits_name, header=False, verbose=debug) tmp = cube_crop_frames(tmp, self.com_sz, force=True, verbose=debug) tmp_tmp = cube_subtract_sky_pca(tmp, tmp_tmp_tmp, mask_AGPM_com, ref_cube=None, ncomp=npc_dark) write_fits(self.outpath + '1_crop_' + fits_name, tmp_tmp) if verbose: print('Dark has been subtracted from Sky') if debug: #plot the median of dark cube median of cube before subtraction median after subtraction tmp_tmp_tmp = np.median(tmp_tmp_tmp, axis=0) tmp = np.median(tmp, axis=0) tmp_tmp = np.median(tmp_tmp, axis=0) plot_frames((tmp_tmp_tmp, tmp, tmp_tmp)) tmp_tmp_tmp = open_fits(self.outpath + 'master_unsat_dark.fits') # no need to crop the unsat frame at the same size as the sci images if they are smaller for un, fits_name in enumerate(unsat_list): tmp = open_fits(self.inpath + fits_name, header=False) #tmp = cube_crop_frames(tmp,nx_unsat_crop) if tmp.shape[2] > self.com_sz: nx_unsat_crop = self.com_sz tmp_tmp = cube_crop_frames(tmp - tmp_tmp_tmp, nx_unsat_crop, force=True, verbose=debug) elif tmp.shape[2] % 2 == 0: nx_unsat_crop = tmp.shape[2] - 1 tmp_tmp = cube_crop_frames(tmp - tmp_tmp_tmp, nx_unsat_crop, force=True, verbose=debug) else: nx_unsat_crop = tmp.shape[2] tmp_tmp = tmp - tmp_tmp_tmp write_fits(self.outpath + '1_crop_unsat_' + fits_name, tmp_tmp) if verbose: print('unsat frames have been cropped') if debug: #plot the median of dark cube median of cube before subtraction median after subtraction tmp_tmp_tmp = np.median(tmp_tmp_tmp, axis=0) tmp = np.median(tmp, axis=0) tmp_tmp = np.median(tmp_tmp, axis=0) plot_frames((tmp_tmp_tmp, tmp, tmp_tmp))