def crop_cube(self, arcsecond_diameter=3.5, verbose=True, debug=False):
"""
Crops frames in the master cube after recentering and bad frame removal. Recommended for post-processing ie.
PCA in concentric annuli. If the provided arcsecond diameter happens to be larger than the cropping provided in
recentering, no cropping will occur.
Parameters
----------
arcsecond_diameter : float or int
Size of the frames diameter in arcseconds. Default of 3" for NaCO corresponds to 111x111 (x,y) pixel frames.
Note this is a diameter, not a radius.
verbose : bool optional
If True extra messages of completion are showed.
Writes to fits file
-------
cropped cube : numpy ndarray
Cube with cropped frames
"""
if not os.path.isfile(
self.outpath +
'{}_master_cube.fits'.format(self.dataset_dict['source'])):
raise NameError(
'Missing master cube from recentering and bad frame removal!')
master_cube = open_fits(
self.outpath +
'{}_master_cube.fits'.format(self.dataset_dict['source']),
verbose=debug)
nz, ny, nx = master_cube.shape
crop_size = int(
(arcsecond_diameter) / (self.dataset_dict['pixel_scale']))
if not crop_size % 2:
crop_size += 1
print('Crop size not odd, increased to {}'.format(crop_size))
if debug:
print('Input crop size is {} pixels'.format(crop_size))
if ny <= crop_size:
print(
'Crop size is larger than the frame size. Skipping cropping...'
)
else:
if verbose:
print('######### Running frame cropping #########')
master_cube = cube_crop_frames(master_cube,
crop_size,
force=False,
verbose=debug,
full_output=False)
write_fits(
self.outpath +
'{}_master_cube.fits'.format(self.dataset_dict['source']),
master_cube)
def find_agpm_list(self, file_list, coro = True, threshold = 0):
cube = open_fits(self.outpath + file_list[0])
nz, ny, nx = cube.shape
median_frame = np.median(cube, axis = 0)
median_frame = frame_filter_lowpass(median_frame, median_size = 7, mode = 'median')
write_fits(self.outpath +'median_frame',median_frame)
median_frame = frame_filter_lowpass(median_frame, mode = 'gauss',fwhm_size = 5)
write_fits(self.outpath +'median_frame_1',median_frame)
cy,cx = np.unravel_index(np.argmax(median_frame), median_frame.shape)
return cy, cx
def __init__(self,
inpath,
outpath,
dataset_dict,
recenter_method,
recenter_model,
coro=True):
self.inpath = inpath
self.outpath = outpath
self.derot_angles_cropped = open_fits(self.inpath +
'derot_angles_cropped.fits',
verbose=False)
self.recenter_method = recenter_method
self.recenter_model = recenter_model
self.sci_list = []
# get all the science cubes into a list
with open(self.inpath + "sci_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
self.sci_list.append(line.split('\n')[0])
self.sci_list.sort(
) # make sure they are in order so derotation doesn't make a mess of the frames
self.real_ndit_sci = []
for sc, fits_name in enumerate(
self.sci_list): # enumerate over the list of all science cubes
tmp = open_fits(self.inpath + '4_sky_subtr_imlib_' + fits_name,
verbose=False)
self.real_ndit_sci.append(
tmp.shape[0]) # gets length of each cube for later use
del tmp
self.dataset_dict = dataset_dict
self.fast_reduction = dataset_dict['fast_reduction']
os.system("cp " + self.inpath +
'master_unsat-stellarpsf_fluxes.fits ' +
self.outpath) # for use later
os.system("cp " + self.inpath + 'fwhm.fits ' +
self.outpath) # for use later
os.system("cp " + self.inpath + 'master_unsat_psf_norm.fits ' +
self.outpath) # for use later
def __init__(self, inpath, outpath, dataset_dict, nproc, npc):
self.inpath = inpath
self.outpath = outpath
self.nproc = nproc
self.npc = npc
try:
self.fwhm = open_fits(self.inpath + 'fwhm.fits', verbose=False)[0] # fwhm is first entry
except:
print("Alert: No FWHM file found. Setting to median value of 4.2")
self.fwhm = 4.2
self.dataset_dict = dataset_dict
self.pixel_scale = dataset_dict['pixel_scale']
if not isdir(self.outpath):
os.system("mkdir " + self.outpath)
def find_star_unsat(self, unsat_list, verbose=True, debug=False):
self.unsat_star_pos = {}
y_star = []
x_star = []
for un, fits_name in enumerate(unsat_list):
tmp = np.median(open_fits(self.outpath + '1_crop_unsat_' +
fits_name,
header=False),
axis=0)
table_res = detection(tmp,
fwhm=1.2 * self.resel,
bkg_sigma=1,
mode='lpeaks',
matched_filter=False,
mask=True,
snr_thresh=10,
plot=debug,
debug=debug,
full_output=debug,
verbose=verbose)
y_star = np.append(x_star, table_res['y'][0])
x_star = np.append(y_star, table_res['x'][0])
self.unsat_star_pos['y'] = y_star
self.unsat_star_pos['x'] = x_star
self.unsat_star_pos['fname'] = unsat_list
if verbose:
print('The star has been located in the unsat cubes')
if debug:
snr_star = table_res['px_snr']
for un, fits_name in enumerate(unsat_list):
tmp = np.median(open_fits(self.outpath + '1_crop_unsat_' +
fits_name,
header=False),
axis=0)
plot_frames(tmp, circle=(y_star[un], x_star[un]))
def __init__(self, inpath, outpath, final_sz=0, coro=True):
self.inpath = inpath
self.outpath = outpath
self.final_sz = final_sz
sci_list = []
with open(self.outpath + "sci_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sci_list.append(line.split('\n')[0])
nx = open_fits(self.inpath + sci_list[0]).shape[2]
self.com_sz = nx - 1
#self.resel = 3.5154884047173294
self.resel = float(
open(self.outpath + "resel.txt", "r").readlines()[0])
def bad_columns(self, verbose = True, debug = True):
bcm = np.zeros((1026, 1024) ,dtype=np.float64)
#creating bad pixel map
for i in range(3, 509, 8):
for j in range(512):
bcm[j,i] = 1
if verbose:
print(self.file_list)
for fname in self.file_list:
if verbose:
print('about to fix', fname)
cube_fname, header_fname = open_fits(self.inpath + fname,
header = True, verbose = debug)
test_fname = cube_fname.copy()
#crop the bad pixcel map to the same dimentions of the frames
if len(cube_fname.shape) == 3:
nz, ny, nx = cube_fname.shape
cy, cx = ny/2 , nx/2
ini_y, fin_y = int(512-cy), int(512+cy)
ini_x, fin_x = int(512-cx), int(512+cx)
bcm_crop = bcm[ini_y:fin_y,ini_x:fin_x]
for j in range(nz):
#replace bad columns in each frame of the cubes
test_fname[j] = frame_fix_badpix_isolated(test_fname[j],
bpm_mask= bcm_crop, sigma_clip=3,
num_neig=5, size=5, protect_mask=False,
radius=30, verbose=debug, debug=False)
write_fits(self.outpath + fname, test_fname,
header_fname, output_verify = 'fix')
else:
ny, nx = cube_fname.shape
cy, cx = ny/2 , nx/2
ini_y, fin_y = int(512-cy), int(512+cy)
ini_x, fin_x = int(512-cx), int(512+cx)
bcm_crop = bcm[ini_y:fin_y,ini_x:fin_x]
test_fname = frame_fix_badpix_isolated(test_fname,
bpm_mask= bcm_crop, sigma_clip=3, num_neig=5,
size=5, protect_mask=False, radius=30,
verbose=debug, debug=False)
write_fits(self.outpath + fname, test_fname,
header_fname, output_verify = 'fix')
if verbose:
print('done fixing',fname)
def find_AGPM_list(self, file_list, verbose=True, debug=False):
"""
This method will find the location of the AGPM
(roughly the location of the star)
"""
cube = open_fits(self.outpath + file_list[0])
nz, ny, nx = cube.shape
median_frame = np.median(cube, axis=0)
median_frame = frame_filter_lowpass(median_frame,
median_size=7,
mode='median')
median_frame = frame_filter_lowpass(median_frame,
mode='gauss',
fwhm_size=5)
ycom, xcom = np.unravel_index(np.argmax(median_frame), median_frame.shape)
if verbose:
print('The location of the AGPM is', 'ycom =', ycom, 'xcom =', xcom)
if debug:
pdb.set_trace()
return [ycom, xcom]
def postprocessing(self, do_adi=True, do_adi_contrast=True, do_pca_full=True, do_pca_ann=True, cropped=True,
do_snr_map=True, do_snr_map_opt=True, delta_rot=(0.5, 3), mask_IWA=1, overwrite=True, plot=True,
verbose=True, debug=False):
"""
For post processing the master cube via median ADI, full frame PCA-ADI, or annular PCA-ADI. Includes constrast
curves and SNR maps.
Parameters:
***********
do_adi : bool
Whether to do a median-ADI processing
do_adi_contrast : bool
Whether to compute contrast curve associated to median-ADI
do_pca_full : bool
Whether to apply PCA-ADI on full frame
do_pca_ann : bool, default is False
Whether to apply annular PCA-ADI (more computer intensive). Only runs if cropped=True
cropped : bool
whether the master cube was cropped in pre-processing
do_snr_map : bool
whether to compute an SNR map (warning: computer intensive); useful only when point-like features are seen
in the image
do_snr_map_opt : bool
Whether to compute a non-conventional (more realistic) SNR map
delta_rot : tuple
Threshold in rotation angle used in pca_annular to include frames in the PCA library (provided in terms of
FWHM). See description of pca_annular() for more details
mask_IWA : int, default 1
Size of the numerical mask that hides the inner part of post-processed images. Provided in terms of fwhm
overwrite : bool, default True
whether to overwrite pre-exisiting output files from previous reductions
plot : bool
Whether to save plots to the output path (PDF file, print quality)
verbose : bool
prints more output when True
debug : bool, default is False
Saves extra output files
"""
# make directories if they don't exist
print("======= Starting post-processing....=======")
outpath_sub = self.outpath + "sub_npc{}/".format(self.npc)
if not isdir(outpath_sub):
os.system("mkdir " + outpath_sub)
if verbose:
print('Input path is {}'.format(self.inpath))
print('Output path is {}'.format(outpath_sub))
source = self.dataset_dict['source']
tn_shift = 0.568 # Launhardt et al. 2020, true North offset for NACO
ADI_cube_name = '{}_master_cube.fits' # template name for input master cube
derot_ang_name = 'derot_angles.fits' # template name for corresponding input derotation angles
ADI_cube = open_fits(self.inpath + ADI_cube_name.format(source), verbose=verbose)
derot_angles = open_fits(self.inpath + derot_ang_name, verbose=verbose) + tn_shift
if do_adi_contrast:
psfn_name = "master_unsat_psf_norm.fits" # normalised PSF
flux_psf_name = "master_unsat-stellarpsf_fluxes.fits" # flux in a FWHM aperture found in calibration
psfn = open_fits(self.inpath + psfn_name, verbose=verbose)
starphot = open_fits(self.inpath + flux_psf_name, verbose=verbose)[1] # scaled fwhm flux is the second entry
mask_IWA_px = mask_IWA * self.fwhm
if verbose:
print("adopted mask size: {:.0f}".format(mask_IWA_px))
ann_sz = 3 # if PCA-ADI in a single annulus or in concentric annuli, this is the size of the annulus/i in FWHM
svd_mode = 'lapack' # python package used for Singular Value Decomposition for PCA reductions
n_randsvd = 3 # if svd package is set to 'randsvd' number of times we do PCA rand-svd, before taking the
# median of all results (there is a risk of significant self-subtraction when just doing it once)
ref_cube = None # if any, load here a centered calibrated cube of reference star observations - would then be
# used for PCA instead of the SCI cube itself
# TEST number of principal components
# PCA-FULL
if do_pca_full:
test_pcs_full = list(range(1, self.npc + 1))
# PCA-ANN
if do_pca_ann:
test_pcs_ann = list(range(1, self.npc + 1)) # needs a cropped cube
######################### Simple ADI ###########################
if do_adi:
if not isfile(outpath_sub + 'final_ADI_simple.fits') or overwrite:
if debug: # saves the residuals
tmp, _, tmp_tmp = median_sub(ADI_cube, derot_angles, fwhm=self.fwhm,
radius_int=0, asize=ann_sz, delta_rot=delta_rot,
full_output=debug, verbose=verbose)
tmp = mask_circle(tmp, mask_IWA_px)
write_fits(outpath_sub + 'TMP_ADI_simple_cube_der.fits', tmp, verbose=verbose)
# make median combination of the de-rotated cube.
tmp_tmp = median_sub(ADI_cube, derot_angles, fwhm=self.fwhm,
radius_int=0, asize=ann_sz, delta_rot=delta_rot,
full_output=False, verbose=verbose)
tmp_tmp = mask_circle(tmp_tmp, mask_IWA_px) # we mask the IWA
write_fits(outpath_sub + 'final_ADI_simple.fits', tmp_tmp, verbose=verbose)
## SNR map
if (not isfile(outpath_sub + 'final_ADI_simple_snrmap.fits') or overwrite) and do_snr_map:
tmp = open_fits(outpath_sub + 'final_ADI_simple.fits', verbose=verbose)
tmp = mask_circle(tmp, mask_IWA_px)
tmp_tmp = snrmap(tmp, self.fwhm, nproc=self.nproc, verbose=debug)
write_fits(outpath_sub + 'final_ADI_simple_snrmap.fits', tmp_tmp, verbose=verbose)
## Contrast curve
if (not isfile(outpath_sub + 'contrast_adi.pdf') or overwrite) and do_adi_contrast:
_ = contrast_curve(ADI_cube, derot_angles, psfn, self.fwhm, pxscale=self.pixel_scale,
starphot=starphot, algo=median_sub, sigma=5., nbranch=1, theta=0,
inner_rad=1, wedge=(0, 360), student=True, transmission=None,
smooth=True, plot=plot, dpi=300, debug=debug,
save_plot=outpath_sub + 'contrast_adi.pdf', verbose=verbose)
if verbose:
print("======= Completed Median-ADI =======")
####################### PCA-ADI full ###########################
if do_pca_full:
test_pcs_str_list = [str(x) for x in test_pcs_full]
ntest_pcs = len(test_pcs_full)
test_pcs_str = "npc" + "-".join(test_pcs_str_list)
PCA_ADI_cube = ADI_cube.copy()
tmp_tmp = np.zeros([ntest_pcs, PCA_ADI_cube.shape[1], PCA_ADI_cube.shape[2]])
if do_snr_map_opt:
tmp_tmp_tmp_tmp = np.zeros([ntest_pcs, PCA_ADI_cube.shape[1], PCA_ADI_cube.shape[2]])
for pp, npc in enumerate(test_pcs_full):
if svd_mode == 'randsvd':
tmp_tmp_tmp = np.zeros([n_randsvd, PCA_ADI_cube.shape[1], PCA_ADI_cube.shape[2]])
for nr in range(n_randsvd):
tmp_tmp_tmp[nr] = pca(PCA_ADI_cube, angle_list=derot_angles, cube_ref=ref_cube,
scale_list=None, ncomp=int(npc),
svd_mode='randsvd', scaling=None, mask_center_px=mask_IWA_px,
delta_rot=delta_rot, fwhm=self.fwhm, collapse='median', check_memory=True,
full_output=False, verbose=verbose)
tmp_tmp[pp] = np.median(tmp_tmp_tmp, axis=0)
else:
if not isfile(outpath_sub + 'final_PCA-ADI_full_' + test_pcs_str + '.fits') or overwrite:
tmp_tmp[pp] = pca(PCA_ADI_cube, angle_list=derot_angles, cube_ref=ref_cube,
scale_list=None, ncomp=int(npc),
svd_mode=svd_mode, scaling=None, mask_center_px=mask_IWA_px,
delta_rot=delta_rot, fwhm=self.fwhm, collapse='median', check_memory=True,
full_output=False, verbose=verbose)
if (not isfile(outpath_sub + 'final_PCA-ADI_full_' +test_pcs_str+'_snrmap_opt.fits') or overwrite) \
and do_snr_map_opt:
tmp_tmp_tmp_tmp[pp] = pca(PCA_ADI_cube, angle_list=-derot_angles, cube_ref=ref_cube,
scale_list=None, ncomp=int(npc),
svd_mode=svd_mode, scaling=None, mask_center_px=mask_IWA_px,
delta_rot=delta_rot, fwhm=self.fwhm, collapse='median',
check_memory=True,
full_output=False, verbose=verbose)
if not isfile(outpath_sub + 'final_PCA-ADI_full_' + test_pcs_str + '.fits') or overwrite:
write_fits(outpath_sub + 'final_PCA-ADI_full_' + test_pcs_str + '.fits', tmp_tmp, verbose=verbose)
if (not isfile(outpath_sub + 'neg_PCA-ADI_full_' + test_pcs_str + '.fits') or overwrite) and do_snr_map_opt:
write_fits(outpath_sub + 'neg_PCA-ADI_full_' + test_pcs_str + '.fits',
tmp_tmp_tmp_tmp, verbose=verbose)
### Convolution
if not isfile(outpath_sub + 'final_PCA-ADI_full_' + test_pcs_str + '_conv.fits') or overwrite:
tmp = open_fits(outpath_sub + 'final_PCA-ADI_full_' + test_pcs_str + '.fits', verbose=verbose)
for nn in range(tmp.shape[0]):
tmp[nn] = frame_filter_lowpass(tmp[nn], fwhm_size=self.fwhm, gauss_mode='conv')
write_fits(outpath_sub + 'final_PCA-ADI_full_' + test_pcs_str + '_conv.fits', tmp, verbose=verbose)
### SNR map
if (not isfile(outpath_sub + 'final_PCA-ADI_full_' + test_pcs_str + '_snrmap.fits') or overwrite) \
and do_snr_map:
tmp = open_fits(outpath_sub + 'final_PCA-ADI_full_' + test_pcs_str + '.fits', verbose=verbose)
for pp in range(ntest_pcs):
tmp[pp] = snrmap(tmp[pp], self.fwhm, nproc=self.nproc, verbose=debug)
tmp[pp] = mask_circle(tmp[pp], mask_IWA_px)
write_fits(outpath_sub + 'final_PCA-ADI_full_' + test_pcs_str + '_snrmap.fits', tmp, verbose=verbose)
### SNR map optimized
if (not isfile(outpath_sub + 'final_PCA-ADI_full_' + test_pcs_str + '_snrmap_opt.fits') or overwrite) \
and do_snr_map_opt:
tmp = open_fits(outpath_sub + 'final_PCA-ADI_full_' + test_pcs_str + '.fits', verbose=verbose)
for pp in range(ntest_pcs):
tmp[pp] = snrmap(tmp[pp], self.fwhm, array2=tmp_tmp_tmp_tmp[pp], incl_neg_lobes=False,
nproc=self.nproc, verbose=debug)
tmp[pp] = mask_circle(tmp[pp], mask_IWA_px)
write_fits(outpath_sub + 'final_PCA-ADI_full_' + test_pcs_str + '_snrmap_opt.fits', tmp,
verbose=verbose)
if verbose:
print("======= Completed PCA Full Frame =======")
######################## PCA-ADI annular #######################
if do_pca_ann:
if cropped == False:
raise ValueError('PCA-ADI annular requires a cropped cube!')
PCA_ADI_cube = ADI_cube.copy()
del ADI_cube
test_pcs_str_list = [str(x) for x in test_pcs_ann]
ntest_pcs = len(test_pcs_ann)
test_pcs_str = "npc" + "-".join(test_pcs_str_list)
tmp_tmp = np.zeros([ntest_pcs, PCA_ADI_cube.shape[1], PCA_ADI_cube.shape[2]])
if debug:
array_der = np.zeros([ntest_pcs, PCA_ADI_cube.shape[0], PCA_ADI_cube.shape[1], PCA_ADI_cube.shape[2]])
array_out = np.zeros([ntest_pcs, PCA_ADI_cube.shape[0], PCA_ADI_cube.shape[1], PCA_ADI_cube.shape[2]])
if do_snr_map_opt:
tmp_tmp_tmp_tmp = np.zeros([ntest_pcs, PCA_ADI_cube.shape[1], PCA_ADI_cube.shape[2]])
for pp, npc in enumerate(test_pcs_ann):
if debug and ((not isfile(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '_residuals.fits') and
not isfile(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '_residuals-derot.fits'))
or overwrite):
# saves residuals and median if debug is true and they either dont exist or are to be overwritten
array_out[pp], array_der[pp], tmp_tmp[pp] = pca_annular(PCA_ADI_cube, derot_angles,
cube_ref=ref_cube, scale_list=None,
radius_int=mask_IWA_px, fwhm=self.fwhm,
asize=ann_sz * self.fwhm,
n_segments=1, delta_rot=delta_rot, ncomp=int(npc),
svd_mode=svd_mode, nproc=self.nproc,
min_frames_lib=max(npc, 10),
max_frames_lib=200, tol=1e-1, scaling=None,
imlib='opencv',
interpolation='lanczos4', collapse='median',
ifs_collapse_range='all',
full_output=debug, verbose=verbose)
else:
if not isfile(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '.fits') or overwrite:
tmp_tmp[pp] = pca_annular(PCA_ADI_cube, derot_angles, cube_ref=ref_cube, scale_list=None,
radius_int=mask_IWA_px, fwhm=self.fwhm, asize=ann_sz * self.fwhm,
n_segments=1, delta_rot=delta_rot, ncomp=int(npc),
svd_mode=svd_mode, nproc=self.nproc, min_frames_lib=max(npc, 10),
max_frames_lib=200, tol=1e-1, scaling=None, imlib='opencv',
interpolation='lanczos4', collapse='median', ifs_collapse_range='all',
full_output=False, verbose=verbose)
if (not isfile(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '_snrmap_opt.fits') or overwrite) and do_snr_map_opt:
tmp_tmp_tmp_tmp[pp] = pca_annular(PCA_ADI_cube, -derot_angles, cube_ref=ref_cube,
scale_list=None, radius_int=mask_IWA_px, fwhm=self.fwhm,
asize=ann_sz * self.fwhm, n_segments=1, delta_rot=delta_rot,
ncomp=int(npc), svd_mode=svd_mode,
nproc=self.nproc, min_frames_lib=max(npc, 10),
max_frames_lib=200, tol=1e-1, scaling=None, imlib='opencv',
interpolation='lanczos4', collapse='median',
ifs_collapse_range='all', full_output=False, verbose=verbose)
if not isfile(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '.fits') or overwrite:
write_fits(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '.fits', tmp_tmp, verbose=verbose)
if debug and ((not isfile(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '_residuals.fits') and
not isfile(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '_residuals-derot.fits')) or overwrite):
write_fits(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '_residuals.fits', array_out, verbose=verbose)
write_fits(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '_residuals-derot.fits', array_der, verbose=verbose)
if (not isfile(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '_snrmap_opt.fits') or overwrite) and do_snr_map_opt:
write_fits(outpath_sub + 'neg_PCA-ADI_ann_' + test_pcs_str + '.fits', tmp_tmp_tmp_tmp, verbose=verbose)
### Convolution
if not isfile(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '_conv.fits') or overwrite:
tmp = open_fits(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '.fits', verbose=verbose)
for nn in range(tmp.shape[0]):
tmp[nn] = frame_filter_lowpass(tmp[nn], fwhm_size=self.fwhm, gauss_mode='conv')
write_fits(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '_conv.fits', tmp, verbose=verbose)
### SNR map
if (not isfile(
outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '_snrmap.fits') or overwrite) and do_snr_map:
tmp = open_fits(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '.fits', verbose=verbose)
for pp in range(ntest_pcs):
tmp[pp] = snrmap(tmp[pp], self.fwhm, nproc=self.nproc, verbose=debug)
tmp[pp] = mask_circle(tmp[pp], mask_IWA_px)
write_fits(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '_snrmap.fits', tmp, verbose=verbose)
### SNR map optimized
if (not isfile(
outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '_snrmap_opt.fits') or overwrite) and do_snr_map_opt:
tmp = open_fits(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '.fits', verbose=verbose)
for pp in range(ntest_pcs):
tmp[pp] = snrmap(tmp[pp], self.fwhm, plot=plot, array2=tmp_tmp_tmp_tmp[pp], nproc=self.nproc,
verbose=debug)
tmp[pp] = mask_circle(tmp[pp], mask_IWA_px)
write_fits(outpath_sub + 'final_PCA-ADI_ann_' + test_pcs_str + '_snrmap_opt.fits', tmp, verbose=verbose)
if verbose:
print("======= Completed PCA Annular =======")
vels = np.arange(vmin, vmax, dv)
CC = CrossCorr(vels)
from glob import glob
from scipy.interpolate import interp1d
files = glob("/Users/rakesh/Data/Templates/BT-Settl/lte14*")
files.sort()
fl = files[2]
Teff = np.float(fl.split('/')[-1].split('-')[0][-4:])
logg = np.float(fl.split('/')[-1].split('-')[1])
temp_flux, temp_wavs = CC.processTemplate(fl)
window_size = 101
order = 1
npc = 5
filename = "/Users/rakesh/Data/Final_derot_residual_cubes/final_derot_res_cube_npc{}_H_good0-28.fits".format(
npc)
fits_h = open_fits(filename)
filename = "/Users/rakesh/Data/Final_derot_residual_cubes/final_derot_res_cube_npc{}_K_good0-28.fits".format(
npc)
fits_k = open_fits(filename)
filename = "/Users/rakesh/Data/Final_derot_residual_cubes/lbda_vec_avg_K.fits"
waves_k = open_fits(filename)
filename = "/Users/rakesh/Data/Final_derot_residual_cubes/lbda_vec_avg_H.fits"
waves_h = open_fits(filename)
fwhm_h = open_fits(
"/Users/rakesh/Data/Final_derot_residual_cubes/fwhm_vec_avg_H.fits")
fwhm_k = open_fits(
"/Users/rakesh/Data/Final_derot_residual_cubes/fwhm_vec_avg_K.fits")
cropped_k = vip_hci.hci_dataset.cube_crop_frames(fits_k, 71)
cropped_h = vip_hci.hci_dataset.cube_crop_frames(fits_h, 71)
wavs = np.array(list(waves_h) + list(waves_k))
wmin = np.min(wavs)
def do_negfc(self, do_firstguess=True, guess_xy=None, mcmc_negfc=True, inject_neg=True, ncomp=1, algo='pca_annular',
nwalkers_ini=120, niteration_min=25, niteration_limit=10000, delta_rot=(0.5, 3), weights=False,
coronagraph=False, overwrite=True, save_plot=True, verbose=True):
"""
Module for estimating the location and flux of a planet. A sub-folder 'negfc' is created for storing all
output files.
Using a first guess from the (x,y) coordinates in pixels for the planet, we can estimate a preliminary guess for
the position and flux for each planet using a Nelder-Mead minimization. Saves the r, theta and flux
If using MCMC, runs an affine invariant MCMC sampling algorithm in order to determine the position and the flux
of the planet using the 'Negative Fake Companion' technique. The result of this procedure is a chain with the
samples from the posterior distributions of each of the 3 parameters.
Finally, we can inject a negative flux of the planet found in MCMC into the original dataset and apply
post processing to determine the residuals in the data.
Parameters:
***********
do_firstguess : bool
whether to determine a first guess for the position and the flux of a planet (not NEGFC)
guess_xy : tuple
if do_firstguess=True, this estimate of the source (x,y) location to be provided to firstguess(). Note
Python's zero-based indexing
mcmc_negfc : bool
whether to run MCMC NEGFC sampling (computationally intensive)
inject_neg : bool
whether to inject negative flux of the planet and post process the data without signal from the planet
ncomp : int, default 1
number of prinicple components to subtract
algo : 'pca_annulus', 'pca_annular', 'pca'. default 'pca_annular'
select which routine to be used to model and subtract the stellar PSF
nwalkers_ini : int, default 120
for MCMC, the number of Goodman & Weare 'walkers'
niteration_min : int, default 25
for MCMC, the simulation will run at least this number of steps per walker
niteration_limit : int, default 10000
for MCMC, stops simulation if this many steps run without having reached the convergence criterion
delta_rot : tuple
same as for postprocessing module. Threshold rotation angle for PCA annular
weights : bool
should only be used on unsaturated datasets, where the flux of the star can be measured in each image of
the cube. Applies a correction to each frame to account for variability of the adaptive optics, and hence
flux from the star (reference Christiaens et al. 2021 2021MNRAS.502.6117C)
coronagraph : bool
for MCMC and injecting a negative companion, True if the observation utilised a coronagraph (AGPM).
The known radial transmission of the NACO+AGPM coronagraph will be used
overwrite : bool, default True
whether to run a module and overwrite results if they already exist
save_plot : bool, default True
for firstguess the chi2 vs. flux plot is saved. For MCMC results are pickled and saved to the outpath
along with corner plots
verbose : bool
prints more output and interediate files when True
"""
print("======= Starting NEGFC....=======")
if guess_xy is None and do_firstguess is True:
raise ValueError("Enter an approximate location into guess_xy!")
if weights is True and coronagraph is True:
raise ValueError("Dataset cannot be both non-coronagraphic and coronagraphic!!")
outpath_sub = self.outpath + "negfc/"
if not isdir(outpath_sub):
os.system("mkdir " + outpath_sub)
if verbose:
print('Input path is {}'.format(self.inpath))
print('Output path is {}'.format(outpath_sub))
source = self.dataset_dict['source']
tn_shift = 0.568 # Launhardt et al. 2020, true North offset for NACO
ADI_cube_name = '{}_master_cube.fits' # template name for input master cube
derot_ang_name = 'derot_angles.fits' # template name for corresponding input derotation angles
psfn_name = "master_unsat_psf_norm.fits" # normalised PSF
ADI_cube = open_fits(self.inpath + ADI_cube_name.format(source), verbose=verbose)
derot_angles = open_fits(self.inpath + derot_ang_name, verbose=verbose) + tn_shift
psfn = open_fits(self.inpath + psfn_name, verbose=verbose)
ref_cube = None
if algo == 'pca_annular':
label_pca = 'pca_annular'
algo = pca_annular
elif algo == 'pca_annulus':
label_pca = 'pca_annulus'
algo = pca_annulus
elif algo == 'pca':
label_pca = 'pca'
algo = pca
else:
raise ValueError("Invalid algorithm. Select either pca_annular, pca_annulus or pca!")
opt_npc = ncomp
ap_rad = 1 * self.fwhm
f_range = np.geomspace(0.1, 201, 40)
asize = 3 * self.fwhm
if weights:
nfr = ADI_cube.shape[0] # number of frames
star_flux = np.zeros([nfr]) # for storing the star flux found in each frame
crop_sz_tmp = min(int(10 * self.fwhm),
ADI_cube.shape[1] - 2) # crop around star, either 10*FWHM or size - 2
if crop_sz_tmp % 2 == 0: # if it's not even, crop
crop_sz_tmp -= 1
for ii in range(nfr):
_, star_flux[ii], _ = normalize_psf(ADI_cube[ii], fwhm=self.fwhm, size=crop_sz_tmp,
full_output=True) # get star flux in 1*FWHM
weights = star_flux / np.median(star_flux)
star_flux = np.median(star_flux) # for use after MCMC when turning the chain into contrast
else:
weights = None
flux_psf_name = "master_unsat-stellarpsf_fluxes.fits" # flux in a FWHM aperture found in calibration
star_flux = open_fits(self.inpath + flux_psf_name, verbose=verbose)[1] # scaled fwhm flux
if coronagraph: # radial transmission of the coronagraph, 2 columns (pixels from centre, off-axis transmission)
# data provided by Valentin Christiaens. First entry in both columns was not zero, but VIP adds it in anyway
transmission = np.array([[0, 3.5894626e-10, 5.0611424e-01, 1.0122285e+00, 1.5183427e+00,
2.0244570e+00, 2.5305712e+00, 3.0366855e+00, 3.5427995e+00,
4.0489140e+00, 4.5550284e+00, 5.0611424e+00, 5.5672565e+00,
6.0733705e+00, 6.5794849e+00, 7.0855989e+00, 7.5917134e+00,
8.6039419e+00, 9.1100569e+00, 9.6161709e+00, 1.0628398e+01,
1.1134513e+01, 1.2146742e+01, 1.2652856e+01, 1.3665085e+01,
1.4677314e+01, 1.6195656e+01, 1.7207884e+01, 1.8220114e+01,
1.9738455e+01, 2.1256796e+01, 2.2775141e+01, 2.4293484e+01,
2.6317940e+01, 2.8342396e+01, 3.0366854e+01, 3.2897423e+01,
3.4921883e+01, 3.7452454e+01, 4.0489140e+01, 4.3525822e+01,
4.6562508e+01, 5.0105309e+01, 5.4154221e+01, 5.7697018e+01,
6.2252052e+01, 6.6807076e+01, 7.1868225e+01],
[0, 6.7836474e-05, 3.3822558e-03, 1.7766271e-02, 5.2646037e-02,
1.1413762e-01, 1.9890217e-01, 2.9460809e-01, 3.8605216e-01,
4.6217495e-01, 5.1963091e-01, 5.6185508e-01, 5.9548348e-01,
6.2670821e-01, 6.5912777e-01, 6.9335037e-01, 7.2783405e-01,
7.8866738e-01, 8.1227022e-01, 8.3128709e-01, 8.5912752e-01,
8.6968899e-01, 8.8677746e-01, 8.9409947e-01, 9.0848678e-01,
9.2426234e-01, 9.4704604e-01, 9.5787460e-01, 9.6538281e-01,
9.7379774e-01, 9.8088801e-01, 9.8751044e-01, 9.9255627e-01,
9.9640906e-01, 9.9917024e-01, 1.0009050e+00, 1.0021056e+00,
1.0026742e+00, 1.0027454e+00, 1.0027291e+00, 1.0023015e+00,
1.0016677e+00, 1.0009446e+00, 1.0000550e+00, 9.9953103e-01,
9.9917012e-01, 9.9915260e-01, 9.9922234e-01]])
else:
transmission = None
if (not isfile(outpath_sub + label_pca + "_npc{}_simplex_results.fits".format(opt_npc)) or overwrite) and do_firstguess:
# find r, theta based on the provided estimate location
cy, cx = frame_center(ADI_cube[0])
dy_pl = guess_xy[0][1] - cy
dx_pl = guess_xy[0][0] - cx
r_pl = np.sqrt(np.power(dx_pl, 2) + np.power(dy_pl, 2)) # pixel distance to the guess location
theta_pl = (np.rad2deg(np.arctan2(dy_pl, dx_pl))) % 360 # theta (angle) to the guess location
print("Estimated (r, theta) before first guess = ({:.1f},{:.1f})".format(r_pl, theta_pl))
ini_state = firstguess(ADI_cube, derot_angles, psfn, ncomp=opt_npc, plsc=self.pixel_scale,
planets_xy_coord=guess_xy, fwhm=self.fwhm,
annulus_width=12, aperture_radius=ap_rad, cube_ref=ref_cube,
svd_mode='lapack', scaling=None, fmerit='stddev', imlib='opencv',
interpolation='lanczos4', collapse='median', p_ini=None,
transmission=transmission, weights=weights, algo=algo,
f_range=f_range, simplex=True, simplex_options=None, plot=save_plot,
verbose=verbose, save=save_plot)
# when p_ini is set to None, it gets the value of planets_xy_coord
ini_state = np.array([ini_state[0][0], ini_state[1][0], ini_state[2][0]])
write_fits(outpath_sub + label_pca + "_npc{}_simplex_results.fits".format(opt_npc), ini_state,
verbose=verbose) # saves r, theta and flux. No print statement as firstguess() does that for us
if (not isfile(outpath_sub + "MCMC_results") or overwrite) and mcmc_negfc:
ini_state = open_fits(outpath_sub + label_pca + "_npc{}_simplex_results.fits".format(opt_npc),
verbose=verbose)
delta_theta_min = np.rad2deg(np.arctan(4./ r_pl)) # at least the angle corresponding to 2 azimuthal pixels
delta_theta = max(delta_theta_min, 5.)
bounds = [(max(r_pl - asize / 2., 1), r_pl + asize / 2.), # radius
(theta_pl - delta_theta, theta_pl + delta_theta), # angle
(0, 5 * abs(ini_state[2]))]
if ini_state[0] < bounds[0][0] or ini_state[0] > bounds[0][1] or ini_state[1] < bounds[1][0] or \
ini_state[1] > bounds[1][1] or ini_state[2] < bounds[2][0] or ini_state[2] > bounds[2][1]:
print("!!! WARNING: simplex results not in original bounds - NEGFC simplex MIGHT HAVE FAILED !!!")
ini_state = np.array([r_pl, theta_pl, abs(ini_state[2])])
if verbose is True:
verbosity = 2
print('MCMC NEGFC sampling is about to begin...')
else:
verbosity = 0
final_chain = mcmc_negfc_sampling(ADI_cube, derot_angles, psfn, ncomp=opt_npc, plsc=self.pixel_scale,
initial_state=ini_state, fwhm=self.fwhm, weights=weights,
annulus_width=12, aperture_radius=ap_rad, cube_ref=ref_cube,
svd_mode='lapack', scaling=None, fmerit='stddev',
imlib='opencv', interpolation='lanczos4', transmission=transmission,
collapse='median', nwalkers=nwalkers_ini, bounds=bounds, a=2.0,
ac_c=50, mu_sigma=(0, 1),
burnin=0.3, rhat_threshold=1.01, rhat_count_threshold=1, conv_test='ac',
# use autocorrelation 'ac' to ensure sufficient sampling. sample around
# the area of best likelihood to make distribution
niteration_min=niteration_min, niteration_limit=niteration_limit,
niteration_supp=0, check_maxgap=50, nproc=self.nproc, algo=algo,
output_dir=outpath_sub,
output_file="MCMC_results", display=False, verbosity=verbosity,
save=save_plot)
final_chain[:, :, 2] = final_chain[:, :, 2] / star_flux # dividing by the star flux converts to a contrast
show_walk_plot(final_chain, save=save_plot, output_dir=outpath_sub)
show_corner_plot(final_chain, burnin=0.5, save=save_plot, output_dir=outpath_sub)
# determine the highly probable value for each model parameter and the 1-sigma confidence interval
isamples_flat = final_chain[:,int(final_chain.shape[1]//(1/0.3)):,:].reshape((-1,3)) # 0.3 is the burnin
vals, err = confidence(isamples_flat, cfd=68.27, bins=100, gaussian_fit=False, weights=weights,
verbose=verbose, save=True, output_dir=outpath_sub, filename='confidence.txt',
plsc=self.pixel_scale)
labels = ['r', 'theta', 'f']
mcmc_res = np.zeros([3,3])
# pull the values and confidence interval out for saving
for i in range(3):
mcmc_res[i,0] = vals[labels[i]]
mcmc_res[i,1] = err[labels[i]][0]
mcmc_res[i,2] = err[labels[i]][1]
write_fits(outpath_sub + 'mcmc_results.fits', mcmc_res)
# now gaussian fit
gvals, gerr = confidence(isamples_flat, cfd=68.27, bins=100, gaussian_fit=True, weights=weights,
verbose=verbose, save=True, output_dir=outpath_sub,filename='confidence_gauss.txt',
plsc=self.pixel_scale)
mcmc_res = np.zeros([3,2])
for i in range(3):
mcmc_res[i,0] = gvals[i]
mcmc_res[i,1] = gerr[i]
write_fits(outpath_sub + 'mcmc_results_gauss.fits', mcmc_res)
if inject_neg:
pca_res = np.zeros([ADI_cube.shape[1], ADI_cube.shape[2]])
pca_res_emp = pca_res.copy()
planet_params = open_fits(outpath_sub+'mcmc_results.fits')
flux_psf_name = "master_unsat-stellarpsf_fluxes.fits"
star_flux = open_fits(self.inpath + flux_psf_name, verbose=verbose)[1]
ADI_cube_emp = cube_inject_companions(ADI_cube, psfn, derot_angles,
flevel=-planet_params[2, 0] * star_flux, plsc=self.pixel_scale,
rad_dists=[planet_params[0, 0]],
n_branches=1, theta=planet_params[1, 0],
imlib='opencv', interpolation='lanczos4',
verbose=verbose, transmission=transmission)
write_fits(outpath_sub+'ADI_cube_empty.fits', ADI_cube_emp) # the cube with the negative flux injected
if algo == pca_annular:
radius_int = int(np.floor(r_pl-asize/2)) # asize is 3 * FWHM, rounds down. To skip the inner region
# crop the cube to just larger than the annulus to improve the speed of PCA
crop_sz = int(2*np.ceil(r_pl+asize+1)) # rounds up
if not crop_sz % 2: # make sure the crop is odd
crop_sz += 1
if crop_sz < ADI_cube.shape[1] and crop_sz < ADI_cube.shape[2]: # crop if crop_sz is smaller than cube
pad = int((ADI_cube.shape[1]-crop_sz)/2)
crop_cube = cube_crop_frames(ADI_cube, crop_sz, verbose=verbose)
else:
crop_cube = ADI_cube # dont crop if the cube is already smaller
pca_res_tmp = pca_annular(crop_cube, derot_angles, cube_ref=ref_cube, radius_int=radius_int,
fwhm=self.fwhm, asize=asize, delta_rot=delta_rot, ncomp=opt_npc,
svd_mode='lapack', scaling=None, imlib='opencv', interpolation='lanczos4',
nproc=self.nproc, min_frames_lib=max(opt_npc, 10), verbose=verbose,
full_output=False)
pca_res = np.pad(pca_res_tmp, pad, mode='constant', constant_values=0)
write_fits(outpath_sub + 'pca_annular_res_npc{}.fits'.format(opt_npc), pca_res)
# emp
if crop_sz < ADI_cube_emp.shape[1] and crop_sz < ADI_cube_emp.shape[2]:
pad = int((ADI_cube_emp.shape[1]-crop_sz)/2)
crop_cube = cube_crop_frames(ADI_cube_emp, crop_sz, verbose=verbose)
else:
crop_cube = ADI_cube_emp
del ADI_cube_emp
del ADI_cube
pca_res_tmp = pca_annular(crop_cube, derot_angles, cube_ref=ref_cube, radius_int=radius_int,
fwhm=self.fwhm, asize=asize, delta_rot=delta_rot, ncomp=opt_npc,
svd_mode='lapack', scaling=None, imlib='opencv', interpolation='lanczos4',
nproc=self.nproc, min_frames_lib=max(opt_npc, 10), verbose=verbose,
full_output=False)
# pad again now
pca_res_emp = np.pad(pca_res_tmp, pad, mode='constant', constant_values=0)
write_fits(outpath_sub+'pca_annular_res_empty_npc{}.fits'.format(opt_npc), pca_res_emp)
# detection maps will penalize your result). Also, you must submit a single
# detection threshold which depends on the statistics of the algorithm/detection
# map, and finally the value of the full width at half maximum used.
# We assume a naming convention: instrument_{cube/pa/psf}_id.fits
list_instru = ["sphere_irdis", "nirc2"]
data_dir = "../../data/public_data/"
sub_dir = "../submission"
if not os.path.exists(sub_dir):
os.makedirs(sub_dir)
for instru in list_instru:
for i in range(N):
ds_id = str(i + 1) + '.fits'
try:
cube = open_fits(os.path.join(data_dir, instru + '_cube_' + ds_id))
print(instru, "_", i)
except:
continue
pa = open_fits(os.path.join(data_dir, instru + '_pa_' + ds_id))
psf = open_fits(os.path.join(data_dir, instru + '_psf_' + ds_id))
plsc = open_fits(os.path.join(data_dir, instru + '_plsc_' + ds_id))
if cube.ndim == 4:
wavelength = open_fits(
os.path.join(data_dir, instru + '_wl_' + ds_id))
# Let's assume we are using a single FWHM (fwhm = 4.8)
if len(psf.shape) == 3:
fit = fit_2dgaussian(np.mean(psf, axis=0), full_output=True)
fwhm = np.mean([fit['fwhm_x'][0], fit['fwhm_y'][0]])
else:
def recenter(self,
nproc=1,
sigfactor=4,
subi_size=21,
crop_sz=None,
verbose=True,
debug=False,
plot=False,
coro=True):
"""
Recenters cropped science images by fitting a double Gaussian (negative+positive) to each median combined SCI cube,
or by fitting a single negative Gaussian to the coronagraph using the speckle pattern of each median combined SCI cube.
Parameters:
***********
sigfactor: float, default = 4
If thresholding is performed during 2gauss fitting, set the threshold in terms of gaussian sigma in the subimage (will depend on your cropping size)
subi_size: int, default = 21
Size of the square subimage sides in pixels.
crop_sz: int, optional, in units of pixels. None by default
Crops to this size after recentering for memory management purposes. Useful for very large datasets
verbose: True for False
To provide extra information about the progress and results of the pipeline
plot: True or False
Set to False when running on M3
coro: True for coronagraph data. False otherwise. Recentering requires coronagraphic data
Writes fits to file:
***********
x_shifts.fits # writes the x shifts to the file
y_shifts.fits # writes the y shifts to the file
{source}_master_cube.fits # makes the recentered master cube
derot_angles.fits # makes a vector of derotation angles
"""
if not coro:
if self.recenter_method != '2dfit':
raise ValueError('Recentering method invalid')
if self.recenter_model == '2gauss':
raise ValueError('2Gauss requires coronagraphic data')
if verbose:
print(len(self.sci_list), 'science cubes')
if debug:
print(self.sci_list)
ncubes = len(self.sci_list)
fwhm_all = open_fits(
self.inpath + 'fwhm.fits', verbose=debug
) # changed this to open the file as sometimes we wont run get_stellar_psf() or it may have already run
fwhm = fwhm_all[0] # fwhm is the first entry in the file
fwhm = fwhm.item(
) # changes from numpy.float32 to regular float so it will work in VIP
if verbose:
print('fwhm:', fwhm, 'of type', type(fwhm))
mem = np.zeros(len(self.sci_list))
# Creates a master science cube with just the median of each cube
bar = pyprind.ProgBar(
len(self.sci_list),
stream=1,
title=
'Creating master science cube (median of each science cube)....')
for sc, fits_name in enumerate(
self.sci_list): # enumerate over the list of all science cubes
tmp = open_fits(self.inpath + '4_sky_subtr_imlib_' + fits_name,
verbose=debug) #open cube as tmp
if sc == 0:
self.ndit, ny, nx = tmp.shape #dimensions of cube
tmp_tmp = np.zeros(
[ncubes, ny, nx]
) # template cube with the median of each SCI cube. np.zeros is array filled with zeros
#mem_msg = 'Set check_memory=False to override this memory check'
tmp_tmp[sc] = np.median(tmp, axis=0) # median frame of cube tmp
input_bytes = tmp.nbytes
memory = check_enough_memory(input_bytes, verbose=True)
tmp = None
mem[sc] = memory
bar.update()
write_fits(self.outpath + 'memory.fits', mem, verbose=debug)
if self.recenter_method == 'speckle':
# FOR GAUSSIAN
print('##### Recentering via speckle pattern #####')
#registered science sube, low+high pass filtered cube,cube with stretched values, x shifts, y shifts
tmp_tmp, cube_sci_lpf, cube_stret, sx, sy = cube_recenter_via_speckles(
tmp_tmp,
cube_ref=None,
alignment_iter=5,
gammaval=1,
min_spat_freq=0.5,
max_spat_freq=3,
fwhm=fwhm,
debug=debug,
recenter_median=True,
negative=coro,
fit_type='gaus',
crop=False,
subframesize=subi_size,
imlib='opencv',
interpolation='lanczos4',
plot=plot,
full_output=True)
del cube_sci_lpf
del cube_stret
elif self.recenter_method == '2dfit':
# DOUBLE GAUSSIAN
print('##### Recentering via 2dfit #####')
params_2g = {
'fwhm_neg': 0.8 * fwhm,
'fwhm_pos': 2 * fwhm,
'theta_neg': 48.,
'theta_pos': 135.,
'neg_amp': 0.8
}
res = cube_recenter_2dfit(tmp_tmp,
xy=None,
fwhm=fwhm,
subi_size=subi_size,
model=self.recenter_model,
nproc=nproc,
imlib='opencv',
interpolation='lanczos4',
offset=None,
negative=False,
threshold=True,
sigfactor=sigfactor,
fix_neg=False,
params_2g=params_2g,
save_shifts=False,
full_output=True,
verbose=verbose,
debug=debug,
plot=plot)
sy = res[1]
sx = res[2]
# true_agpm_cen = (res[4][0],res[3][0])
# true_fwhm_pos = (res[5][0],res[6][0])
# true_fwhm_neg = (res[7][0],res[8][0])
# true_theta_pos = res[9][0]
# true_theta_neg = res[10][0]
# amp_pos = res[11][0]
# amp_neg = res[12][0]
# true_neg_amp = amp_neg/amp_pos
# params_2g = {'fwhm_neg': true_fwhm_neg, 'fwhm_pos': true_fwhm_pos,
# 'theta_neg': true_theta_neg, 'theta_pos':true_theta_pos,
# 'neg_amp': true_neg_amp}
# # second: fixing params for neg gaussian - applied on individual frames. returns recentered array, and x-y shifts
# tmp_tmp, sy, sx = cube_recenter_2dfit(tmp_tmp, xy=true_agpm_cen,
# fwhm=self.fwhm, subi_size=subi_size,
# model=model, nproc=nproc, imlib='opencv',
# interpolation='lanczos4',
# offset=None, negative=False,
# threshold=True, sigfactor=sigfactor,
# fix_neg=True, params_2g=params_2g,
# save_shifts=False, full_output=True,
# verbose=verbose, debug=debug, plot=plot)
# LOAD IN REAL_NDIT_SCI
# Load original cubes, shift them, and create master cube
tmp_tmp = np.zeros(
[int(np.sum(self.real_ndit_sci)), ny, nx]
) #makes an array full of zeros, length of the sum of each entry in the sci dimensions file. we dont need our old tmp_tmp anymore
angles_1dvector = np.zeros([
int(np.sum(self.real_ndit_sci))
]) # makes empty array for derot angles, length of number of frames
for sc, fits_name in enumerate(self.sci_list):
tmp = open_fits(self.inpath + '4_sky_subtr_imlib_' + fits_name,
verbose=debug) #opens science cube
dim = int(self.real_ndit_sci[sc]
) #gets the integer dimensions of this science cube
for dd in range(dim): #dd goes from 0 to the largest dimension
tmp_tmp[int(
np.sum(self.real_ndit_sci[:sc])
) + dd] = frame_shift(
tmp[dd], shift_y=sy[sc], shift_x=sx[sc], imlib='opencv'
) #this line applies the shifts to all the science images in the cube the loop is currently on. it also converts all cubes to a single long cube by adding the first dd frames, then the next dd frames from the next cube and so on
angles_1dvector[int(
np.sum(self.real_ndit_sci[:sc])
) + dd] = self.derot_angles_cropped[sc][
dd] # turn 2d rotation file into a vector here same as for the mastercube above
# sc*ndit+dd i don't think this line works for variable sized cubes
tmp = None # memory management
pathlib.Path(self.outpath).mkdir(parents=True, exist_ok=True)
if crop_sz is not None:
if not crop_sz % 2:
crop_sz += 1
print('Crop size not odd, increased to {}'.format(crop_sz))
print('Cropping to {} pixels'.format(crop_sz))
tmp_tmp = cube_crop_frames(tmp_tmp,
crop_sz,
force=False,
verbose=debug,
full_output=False)
# write all the shifts
write_fits(self.outpath + 'x_shifts.fits',
sx) # writes the x shifts to the file
write_fits(self.outpath + 'y_shifts.fits',
sy) # writes the y shifts to the file
write_fits(self.outpath +
'{}_master_cube.fits'.format(self.dataset_dict['source']),
tmp_tmp) #makes the master cube
write_fits(self.outpath + 'derot_angles.fits',
angles_1dvector) # writes the 1D array of derotation angles
if verbose:
print('Shifts applied, master cube saved')
tmp_tmp = None
def correct_bad_pixels(self, verbose=True, debug=False):
sci_list = []
with open(self.outpath + "sci_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sci_list.append(line.split('\n')[0])
sky_list = []
with open(self.outpath + "sky_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sky_list.append(line.split('\n')[0])
unsat_list = []
with open(self.outpath + "unsat_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
unsat_list.append(line.split('\n')[0])
n_sci = len(sci_list)
ndit_sci = fits_info.ndit_sci
n_sky = len(sky_list)
ndit_sky = fits_info.ndit_sky
tmp = open_fits(self.outpath + '1_crop_unsat_' + unsat_list[-1],
header=False)
nx_unsat_crop = tmp.shape[2]
master_flat_frame = open_fits(self.outpath + 'master_flat_field.fits')
# Create bpix map
bpix = np.where(np.abs(master_flat_frame - 1.09) >
0.41) # i.e. for QE < 0.68 and QE > 1.5
bpix_map = np.zeros([self.com_sz, self.com_sz])
bpix_map[bpix] = 1
if nx_unsat_crop < bpix_map.shape[1]:
bpix_map_unsat = frame_crop(bpix_map, nx_unsat_crop)
else:
bpix_map_unsat = bpix_map
#number of bad pixels
nbpix = int(np.sum(bpix_map))
ntotpix = self.com_sz**2
print("total number of bpix: ", nbpix)
print("total number of pixels: ", ntotpix)
print("=> {}% of bad pixels.".format(100 * nbpix / ntotpix))
write_fits(self.outpath + 'master_bpix_map.fits', bpix_map)
write_fits(self.outpath + 'master_bpix_map_unsat.fits', bpix_map_unsat)
plot_frames(bpix_map, bpix_map_unsat)
#update final crop size
final_sz = self.get_final_sz()
#crop frames to that size
for sc, fits_name in enumerate(sci_list):
tmp = open_fits(self.outpath + '2_nan_corr_' + fits_name,
verbose=False)
tmp_tmp = cube_crop_frames(tmp, final_sz, xy=self.agpm_pos)
write_fits(self.outpath + '2_crop_' + fits_name, tmp_tmp)
if not debug:
os.system("rm " + self.outpath + '2_nan_corr_' + fits_name)
for sk, fits_name in enumerate(sky_list):
tmp = open_fits(self.outpath + '2_nan_corr_' + fits_name,
verbose=False)
tmp_tmp = cube_crop_frames(tmp, final_sz, xy=self.agpm_pos)
write_fits(self.outpath + '2_crop_' + fits_name, tmp_tmp)
if not debug:
os.system("rm " + self.outpath + '2_nan_corr_' + fits_name)
if not debug:
tmp = open_fits(self.outpath + '2_crop_' + sci_list[0])[-1]
tmp_tmp = open_fits(self.outpath + '2_crop_' + sci_list[-1])[-1]
plot_frames(tmp, tmp_tmp)
else:
# COMPARE BEFORE AND AFTER NAN_CORR + CROP
old_tmp = open_fits(self.outpath + '2_ff_' + sci_list[0])[-1]
old_tmp_tmp = open_fits(self.outpath + '2_ff_' + sci_list[-1])[-1]
tmp = open_fits(self.outpath + '2_crop_' + sci_list[0])[-1]
tmp_tmp = open_fits(self.outpath + '2_crop_' + sci_list[-1])[-1]
plot_frames(old_tmp, tmp, old_tmp_tmp, tmp_tmp)
# Crop the bpix map in a same way
bpix_map = open_fits(self.outpath + 'master_bpix_map.fits')
bpix_map_2ndcrop = frame_crop(bpix_map, final_sz, cenxy=self.agpm_pos)
write_fits(self.outpath + 'master_bpix_map_2ndcrop.fits',
bpix_map_2ndcrop)
bpix_map = open_fits(self.outpath + 'master_bpix_map_2ndcrop.fits')
t0 = time_ini()
for sc, fits_name in enumerate(sci_list):
tmp = open_fits(self.outpath + '2_crop_' + fits_name,
verbose=False)
# first with the bp max defined from the flat field (without protecting radius)
tmp_tmp = cube_fix_badpix_isolated(tmp,
bpm_mask=bpix_map,
sigma_clip=7,
num_neig=5,
size=5,
protect_mask=True,
radius=9,
verbose=False,
debug=False)
write_fits(self.outpath + '2_bpix_corr_' + fits_name,
tmp_tmp,
verbose=False)
timing(t0)
# second, residual hot pixels
tmp_tmp, bpm = cube_fix_badpix_isolated(tmp_tmp,
bpm_mask=None,
sigma_clip=8,
num_neig=5,
size=5,
protect_mask=True,
radius=10,
verbose=False,
debug=False,
full_output=True)
write_fits(self.outpath + '2_bpix_corr2_' + fits_name, tmp_tmp)
write_fits(self.outpath + '2_bpix_corr2_map_' + fits_name, bpm)
timing(t0)
if not debug:
os.system("rm " + self.outpath + '2_crop_' + fits_name)
bpix_map = open_fits(self.outpath + 'master_bpix_map_2ndcrop.fits')
t0 = time_ini()
for sk, fits_name in enumerate(sky_list):
tmp = open_fits(self.outpath + '2_crop_' + fits_name,
verbose=False)
# first with the bp max defined from the flat field (without protecting radius)
tmp_tmp = cube_fix_badpix_isolated(tmp,
bpm_mask=bpix_map,
sigma_clip=7,
num_neig=5,
size=5,
protect_mask=True,
radius=9,
verbose=False,
debug=False)
write_fits(self.outpath + '2_bpix_corr_' + fits_name,
tmp_tmp,
verbose=False)
timing(t0)
# second, residual hot pixels
tmp_tmp, bpm = cube_fix_badpix_isolated(tmp_tmp,
bpm_mask=None,
sigma_clip=8,
num_neig=5,
size=5,
protect_mask=True,
radius=10,
verbose=False,
debug=False,
full_output=True)
write_fits(self.outpath + '2_bpix_corr2_' + fits_name, tmp_tmp)
write_fits(self.outpath + '2_bpix_corr2_map_' + fits_name, bpm)
timing(t0)
if not debug:
os.system("rm " + self.outpath + '2_crop_' + fits_name)
bpix_map_unsat = open_fits(self.outpath + 'master_bpix_map_unsat.fits')
t0 = time_ini()
for un, fits_name in enumerate(unsat_list):
tmp = open_fits(self.outpath + '2_nan_corr_unsat_' + fits_name,
verbose=False)
# first with the bp max defined from the flat field (without protecting radius)
tmp_tmp = cube_fix_badpix_isolated(tmp,
bpm_mask=bpix_map_unsat,
sigma_clip=7,
num_neig=5,
size=5,
protect_mask=True,
radius=9,
verbose=False,
debug=False)
write_fits(self.outpath + '2_bpix_corr_unsat_' + fits_name,
tmp_tmp)
timing(t0)
# second, residual hot pixels
tmp_tmp, bpm = cube_fix_badpix_isolated(tmp_tmp,
bpm_mask=None,
sigma_clip=8,
num_neig=5,
size=5,
protect_mask=True,
radius=10,
verbose=False,
debug=False,
full_output=True)
write_fits(self.outpath + '2_bpix_corr2_unsat_' + fits_name,
tmp_tmp)
write_fits(self.outpath + '2_bpix_corr2_map_unsat_' + fits_name,
bpm)
timing(t0)
if not debug:
os.system("rm " + self.outpath + '2_nan_corr_unsat_' +
fits_name)
# FIRST CREATE MASTER CUBE FOR SCI
tmp_tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' + sci_list[0],
verbose=False)
n_y = tmp_tmp_tmp.shape[1]
n_x = tmp_tmp_tmp.shape[2]
tmp_tmp_tmp = np.zeros([n_sci, n_y, n_x])
for sc, fits_name in enumerate(sci_list):
tmp_tmp_tmp[sc] = open_fits(
self.outpath + '2_bpix_corr2_' + fits_name,
verbose=False)[int(random.randrange(min(ndit_sci)))]
tmp_tmp_tmp = np.median(tmp_tmp_tmp, axis=0)
write_fits(self.outpath + 'TMP_2_master_median_SCI.fits', tmp_tmp_tmp)
# THEN CREATE MASTER CUBE FOR SKY
tmp_tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' + sky_list[0],
verbose=False)
n_y = tmp_tmp_tmp.shape[1]
n_x = tmp_tmp_tmp.shape[2]
tmp_tmp_tmp = np.zeros([n_sky, n_y, n_x])
for sk, fits_name in enumerate(sky_list):
tmp_tmp_tmp[sk] = open_fits(
self.outpath + '2_bpix_corr2_' + fits_name,
verbose=False)[int(random.randrange(min(ndit_sky)))]
tmp_tmp_tmp = np.median(tmp_tmp_tmp, axis=0)
write_fits(self.outpath + 'TMP_2_master_median_SKY.fits', tmp_tmp_tmp)
bpix_map_ori = open_fits(self.outpath + 'master_bpix_map_2ndcrop.fits')
bpix_map_sci_0 = open_fits(self.outpath + '2_bpix_corr2_map_' +
sci_list[0])
bpix_map_sci_1 = open_fits(self.outpath + '2_bpix_corr2_map_' +
sci_list[-1])
bpix_map_sky_0 = open_fits(self.outpath + '2_bpix_corr2_map_' +
sky_list[0])
bpix_map_sky_1 = open_fits(self.outpath + '2_bpix_corr2_map_' +
sky_list[-1])
bpix_map_unsat_0 = open_fits(self.outpath + '2_bpix_corr2_map_unsat_' +
unsat_list[0])
bpix_map_unsat_1 = open_fits(self.outpath + '2_bpix_corr2_map_unsat_' +
unsat_list[-1])
plot_frames(
bpix_map_ori,
bpix_map_sci_0,
bpix_map_sci_1, #tmp_tmp_tmp, #bpix_tmp,
bpix_map_sky_0,
bpix_map_sky_1, #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY
bpix_map_unsat_0,
bpix_map_unsat_1 #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY
)
tmpSCI = open_fits(self.outpath + 'TMP_2_master_median_SCI.fits')
tmpSKY = open_fits(self.outpath + 'TMP_2_master_median_SKY.fits')
if not debug:
tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' +
sci_list[1])[-1]
tmp_tmp2 = open_fits(self.outpath + '2_bpix_corr2_' +
sci_list[-1])[-1]
plot_frames( #tmp, tmp-tmpSKY, #tmp_tmp_tmp, #bpix_tmp,
tmp_tmp,
tmp_tmp -
tmpSKY, #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY
#tmp2, tmp2-tmpSKY, #tmp_tmp_tmp, #bpix_tmp,
tmp_tmp2,
tmp_tmp2 -
tmpSKY #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY
)
else:
# COMPARE BEFORE AND AFTER BPIX CORR (without sky subtr)
tmp = open_fits(self.outpath + '2_crop_' + sci_list[-1])[-1]
tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' +
sci_list[-1])[-1]
tmp2 = open_fits(self.outpath + '2_crop_' + sky_list[-1])[-1]
tmp_tmp2 = open_fits(self.outpath + '2_bpix_corr2_' +
sky_list[-1])[-1]
(
tmp,
tmp - tmpSKY, #tmp_tmp_tmp, #bpix_tmp,
tmp_tmp,
tmp_tmp -
tmpSKY, #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY
tmp2,
tmp2 - tmpSKY, #tmp_tmp_tmp, #bpix_tmp,
tmp_tmp2,
tmp_tmp2 -
tmpSKY #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY
)
def _derot_ang_ipag(self, sci_list=sci_list, loc='st'):
nsci = len(sci_list)
parang = np.zeros(nsci)
posang = np.zeros(nsci)
rot_pt_off = np.zeros(nsci)
n_frames_vec = np.ones(nsci, dtype=int)
if loc == 'st':
kw_par = 'HIERARCH ESO TEL PARANG START' # Parallactic angle at start
kw_pos = 'HIERARCH ESO ADA POSANG' # Position angle at start
elif loc == 'nd':
kw_par = 'HIERARCH ESO TEL PARANG END' # Parallactic angle at end
kw_pos = 'HIERARCH ESO ADA POSANG END' # Position angle at exposure end
# FIRST COMPILE PARANG, POSANG and PUPILPOS
for ff in range(len(sci_list)):
cube, header = open_fits(self.inpath + sci_list[ff],
header=True,
verbose=False)
n_frames_vec[ff] = cube.shape[
0] - 1 # "-1" is because the last frame is the median of all others
parang[ff] = header[kw_par]
posang[ff] = header[kw_pos]
pupilpos = 180.0 - parang[ff] + posang[ff]
rot_pt_off[ff] = 90 + 89.44 - pupilpos
if verbose:
print("parang: {}, posang: {}, rot_pt_off: {}".format(
parang[ff], posang[ff], rot_pt_off[ff]))
# NEXT CHECK IF THE OBSERVATION WENT THROUGH TRANSIT (change of sign in parang OR stddev of rot_pt_off > 1.)
rot_pt_off_med = np.median(rot_pt_off)
rot_pt_off_std = np.std(rot_pt_off)
if np.min(parang) * np.max(parang) < 0. or rot_pt_off_std > 1.:
if verbose:
print(
"The observation goes through transit and/or the pupil position was reset in the middle of the observation: "
)
if np.min(parang) * np.max(parang) < 0.:
print("min/max parang: ", np.min(parang),
np.max(parang))
if rot_pt_off_std > 1.:
print(
"the standard deviation of pupil positions is greater than 1: ",
rot_pt_off_std)
# find index where the transit occurs (change of sign of parang OR big difference in pupil pos)
n_changes = 0
for ff in range(len(sci_list) - 1):
if parang[ff] * parang[ff + 1] < 0. or np.abs(
rot_pt_off[ff] - rot_pt_off[ff + 1]) > 1.:
idx_transit = ff + 1
n_changes += 1
# check that these conditions only detected one passage through transit
if n_changes != 1:
print(
" {} passages of transit were detected (instead of 1!). Check that the input fits list is given in chronological order."
.format(n_changes))
pdb.set_trace()
rot_pt_off_med1 = np.median(rot_pt_off[:idx_transit])
rot_pt_off_med2 = np.median(rot_pt_off[idx_transit:])
final_derot_angs = rot_pt_off_med1 - parang
final_derot_angs[
idx_transit:] = rot_pt_off_med2 - parang[idx_transit:]
else:
final_derot_angs = rot_pt_off_med - parang
# MAKE SURE ANGLES ARE IN THE RANGE (-180,180)deg
min_derot_angs = np.amin(final_derot_angs)
nrot_min = min_derot_angs / 360.
if nrot_min < -0.5:
final_derot_angs[np.where(
final_derot_angs < -180)] = final_derot_angs[np.where(
final_derot_angs < -180)] + np.ceil(nrot_min) * 360.
max_derot_angs = np.amax(final_derot_angs)
nrot_max = max_derot_angs / 360.
if nrot_max > 0.5:
final_derot_angs[np.where(
final_derot_angs > 180)] = final_derot_angs[np.where(
final_derot_angs > 180)] - np.ceil(nrot_max) * 360.
return -1. * final_derot_angs, n_frames_vec
def bad_columns(self, verbose=True, debug=False):
"""
In NACO data there are systematic bad columns in the lower left quadrant
This method will correct those bad colums with the median of the neighbouring pixels
"""
#creating bad pixel map
bcm = np.zeros((1026, 1024), dtype=np.float64)
for i in range(3, 509, 8):
for j in range(512):
bcm[j, i] = 1
for fname in self.file_list:
if verbose:
print('Fixing', fname)
tmp, header_fname = open_fits(self.inpath + fname,
header=True,
verbose=debug)
if verbose:
print(tmp.shape)
#crop the bad pixel map to the same dimentions of the frames
if len(tmp.shape) == 3:
nz, ny, nx = tmp.shape
cy, cx = ny / 2, nx / 2
ini_y, fin_y = int(512 - cy), int(512 + cy)
ini_x, fin_x = int(512 - cx), int(512 + cx)
bcm_crop = bcm[ini_y:fin_y, ini_x:fin_x]
for j in range(nz):
#replace bad columns in each frame of the cubes
tmp[j] = frame_fix_badpix_isolated(tmp[j],
bpm_mask=bcm_crop,
sigma_clip=3,
num_neig=5,
size=5,
protect_mask=False,
radius=30,
verbose=debug,
debug=False)
write_fits(self.outpath + fname,
tmp,
header_fname,
output_verify='fix')
else:
ny, nx = tmp.shape
cy, cx = ny / 2, nx / 2
ini_y, fin_y = int(512 - cy), int(512 + cy)
ini_x, fin_x = int(512 - cx), int(512 + cx)
bcm_crop = bcm[ini_y:fin_y, ini_x:fin_x]
tmp = frame_fix_badpix_isolated(tmp,
bpm_mask=bcm_crop,
sigma_clip=3,
num_neig=5,
size=5,
protect_mask=False,
radius=30,
verbose=debug,
debug=False)
write_fits(self.outpath + fname,
tmp,
header_fname,
output_verify='fix')
if verbose:
print('done fixing', fname)
def find_sky_in_sci_cube(self,
nres=3,
coro=True,
verbose=True,
plot=None,
debug=False):
"""
Empty SKY list could be caused by a misclasification of the header in NACO data
This method will check the flux of the SCI cubes around the location of the AGPM
A SKY cube should be less bright at that location allowing the seperation of cubes
"""
flux_list = []
fname_list = []
sci_list = []
with open(self.outpath + "sci_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sci_list.append(line.split('\n')[0])
sky_list = []
with open(self.outpath + "sky_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sky_list.append(line.split('\n')[0])
self.resel = (fits_info.wavelength * 180 * 3600) / (
fits_info.size_telescope * np.pi * fits_info.pixel_scale)
agpm_pos = find_AGPM_list(self, sci_list)
if verbose:
print('The rougth location of the star is', 'y = ', agpm_pos[0],
'x =', agpm_pos[1])
#create the aperture
circ_aper = CircularAperture((agpm_pos[1], agpm_pos[0]),
round(nres * self.resel))
#total flux through the aperture
for fname in sci_list:
cube_fname = open_fits(self.outpath + fname, verbose=debug)
median_frame = np.median(cube_fname, axis=0)
circ_aper_phot = aperture_photometry(median_frame,
circ_aper,
method='exact')
#append it to the flux list.
circ_flux = np.array(circ_aper_phot['aperture_sum'])
flux_list.append(circ_flux[0])
fname_list.append(fname)
if verbose:
print('centre flux has been measured for', fname)
median_flux = np.median(flux_list)
sd_flux = np.std(flux_list)
if verbose:
print('Sorting Sky from Sci')
for i in range(len(flux_list)):
if flux_list[i] < median_flux - 2 * sd_flux:
sky_list.append(fname_list[i])
sci_list.remove(fname_list[i])
symbol = 'bo'
if plot:
if flux_list[i] > median_flux - 2 * sd_flux:
symbol = 'go'
else:
symbol = 'ro'
plt.plot(i, flux_list[i] / median_flux, symbol)
if plot:
plt.title('Normalised flux around star')
plt.ylabel('normalised flux')
if plot == 'save':
plt.savefig(self.outpath + 'flux_plot')
if plot == 'show':
plt.show()
with open(self.outpath + "sci_list.txt", "w") as f:
for sci in sci_list:
f.write(sci + '\n')
with open(self.outpath + "sky_list.txt", "w") as f:
for sci in sky_list:
f.write(sci + '\n')
if verbose:
print('done :)')
def flat_field_correction(self,
verbose=True,
debug_=False,
overwrite_basic=False):
sci_list = []
with open(self.outpath + "sci_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sci_list.append(line.split('\n')[0])
sky_list = []
with open(self.outpath + "sky_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sky_list.append(line.split('\n')[0])
flat_list = []
with open(self.outpath + "flat_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
flat_list.append(line.split('\n')[0])
unsat_list = []
with open(self.outpath + "unsat_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
unsat_list.append(line.split('\n')[0])
flat_X = []
flat_X_values = []
tmp = open_fits(self.outpath + '1_crop_unsat_' + unsat_list[-1],
header=False)
nx_unsat_crop = tmp.shape[2]
for fl, flat_name in enumerate(flat_list):
tmp, header = open_fits(self.inpath + flat_list[fl],
header=True,
verbose=debug_)
flat_X.append(header['AIRMASS'])
if fl == 0:
flat_X_values.append(header['AIRMASS'])
else:
#creates a list rejecting airmass values that differ more than the tolerance.
list_occ = [
isclose(header['AIRMASS'], x, atol=0.1)
for x in flat_X_values
]
if True not in list_occ:
flat_X_values.append(header['AIRMASS'])
print(flat_X)
flat_X_values = np.sort(
flat_X_values
) # !!! VERY IMPORTANT, DO NOT COMMENT, OR IT WILL SCREW EVERYTHING UP!!!
print(flat_X_values)
if verbose:
print('The airmass values have been sorted into a list')
# There should be 15 twilight flats in total with NACO; 5 at each airmass. BUG SOMETIMES!
flat_tmp_cube_1 = np.zeros([5, self.com_sz, self.com_sz])
flat_tmp_cube_2 = np.zeros([5, self.com_sz, self.com_sz])
flat_tmp_cube_3 = np.zeros([5, self.com_sz, self.com_sz])
counter_1 = 0
counter_2 = 0
counter_3 = 0
flat_cube_3X = np.zeros([3, self.com_sz, self.com_sz])
# TAKE MEDIAN OF each group of 5 frames with SAME AIRMASS
flat_cube = open_fits(self.outpath + '1_crop_flat_cube.fits',
header=False,
verbose=debug_)
for fl, self.flat_name in enumerate(flat_list):
if find_nearest(
flat_X_values,
flat_X[fl]) == 0: #could create the function find_nearest
flat_tmp_cube_1[counter_1] = flat_cube[fl]
counter_1 += 1
elif find_nearest(flat_X_values, flat_X[fl]) == 1:
flat_tmp_cube_2[counter_2] = flat_cube[fl]
counter_2 += 1
elif find_nearest(flat_X_values, flat_X[fl]) == 2:
flat_tmp_cube_3[counter_3] = flat_cube[fl]
counter_3 += 1
flat_cube_3X[0] = np.median(flat_tmp_cube_1, axis=0)
flat_cube_3X[1] = np.median(flat_tmp_cube_2, axis=0)
flat_cube_3X[2] = np.median(flat_tmp_cube_3, axis=0)
if verbose:
print('The median flat cubes with same airmass have been created')
med_fl = np.zeros(3)
gains_all = np.zeros([3, self.com_sz, self.com_sz])
#the method for avoiding the bad quadrant is removed since it is fixed in the preproc
for ii in range(3):
med_fl[ii] = np.median(flat_cube_3X[ii])
gains_all[ii] = flat_cube_3X[ii] / med_fl[ii]
master_flat_frame = np.median(gains_all, axis=0)
if nx_unsat_crop < master_flat_frame.shape[1]:
master_flat_unsat = frame_crop(master_flat_frame, nx_unsat_crop)
else:
master_flat_unsat = master_flat_frame
write_fits(self.outpath + 'master_flat_field.fits', master_flat_frame)
write_fits(self.outpath + 'master_flat_field_unsat.fits',
master_flat_unsat)
#plots(master_flat_frame, master_flat_unsat)
if verbose:
print('master flat frames has been saved')
master_flat_frame = open_fits(self.outpath + 'master_flat_field.fits')
if overwrite_basic or not isfile(self.outpath + '2_ff_' +
sci_list[-1]):
for sc, fits_name in enumerate(sci_list):
tmp = open_fits(self.outpath + '1_crop_' + fits_name,
verbose=debug_)
tmp_tmp = np.zeros_like(tmp)
for jj in range(tmp.shape[0]):
tmp_tmp[jj] = tmp[jj] / master_flat_frame
write_fits(self.outpath + '2_ff_' + fits_name,
tmp_tmp,
verbose=debug_)
if not debug_:
os.system("rm " + self.outpath + '1_crop_' + fits_name)
if verbose:
print('Done scaling SCI frames with respects to ff')
if overwrite_basic or not isfile(self.outpath + '2_ff_' +
sky_list[-1]):
for sk, fits_name in enumerate(sky_list):
tmp = open_fits(self.outpath + '1_crop_' + fits_name,
verbose=debug_)
tmp_tmp = np.zeros_like(tmp)
for jj in range(tmp.shape[0]):
tmp_tmp[jj] = tmp[jj] / master_flat_frame
write_fits(self.outpath + '2_ff_' + fits_name,
tmp_tmp,
verbose=debug_)
if not debug_:
os.system("rm " + self.outpath + '1_crop_' + fits_name)
if verbose:
print('Done scaling SKY frames with respects to ff ')
# COMPARE BEFORE AND AFTER FLAT-FIELD
tmp = np.median(open_fits(self.outpath + '2_ff_' + sci_list[0]),
axis=0)
tmp_tmp = np.median(open_fits(self.outpath + '2_ff_' + sci_list[-1]),
axis=0)
if debug_:
old_tmp = np.median(open_fits(self.outpath + '1_crop_' +
sci_list[0]),
axis=0)
old_tmp_tmp = np.median(open_fits(self.outpath + '1_crop_' +
sci_list[-1]),
axis=0)
plot_frames(old_tmp, tmp, old_tmp_tmp, tmp_tmp)
else:
plot_frames(tmp, tmp_tmp)
master_flat_unsat = open_fits(self.outpath +
'master_flat_field_unsat.fits')
for un, fits_name in enumerate(unsat_list):
tmp = open_fits(self.outpath + '1_crop_unsat_' + fits_name,
verbose=debug_)
tmp_tmp = np.zeros_like(tmp)
for jj in range(tmp.shape[0]):
tmp_tmp[jj] = tmp[jj] / master_flat_unsat
write_fits(self.outpath + '2_ff_unsat_' + fits_name,
tmp_tmp,
verbose=debug_)
if not debug_:
os.system("rm " + self.outpath + '1_crop_unsat_' + fits_name)
if verbose:
print('Done scaling UNSAT frames with respects to ff')
# COMPARE BEFORE AND AFTER FLAT-FIELD
tmp = open_fits(self.outpath + '2_ff_unsat_' + unsat_list[0])[-1]
tmp_tmp = open_fits(self.outpath + '2_ff_unsat_' + unsat_list[-1])[-1]
if debug_:
old_tmp = open_fits(self.outpath + '1_crop_unsat_' +
unsat_list[0])[-1]
old_tmp_tmp = open_fits(self.outpath + '1_crop_unsat_' +
unsat_list[-1])[-1]
plot_frames(old_tmp, tmp, old_tmp_tmp, tmp_tmp)
else:
plot_frames(tmp, tmp_tmp)
def find_sky_in_sci_cube(self, nres = 3, coro = True, verbose = True, debug = True):
flux_list = []
fname_list = []
sci_list = []
with open(self.outpath+"sci_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sci_list.append(line.split('\n')[0])
sky_list = []
with open(self.outpath+"sky_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sky_list.append(line.split('\n')[0])
#draw sci list from dico
#find corono centre of the cube.
cy,cx = find_agpm_list(self, sci_list)
if debug:
print(cy,cx)
#plot_frames(open_fits(self.outpath + sci_list[0])[-1], circle = (cy,cx))
#create the aperture
circ_aper = CircularAperture((cx,cy), round(nres*self.resel))
#total flux through the aperture
for fname in sci_list:
cube_fname = open_fits(self.outpath + fname)
median_frame = np.median(cube_fname, axis = 0)
circ_aper_phot = aperture_photometry(median_frame,
circ_aper, method='exact')
#append it to the flux list.
circ_flux = np.array(circ_aper_phot['aperture_sum'])
flux_list.append(circ_flux[0])
fname_list.append(fname)
median_flux = np.median(flux_list)
sd_flux = np.std(flux_list)
for i in range(len(flux_list)):
if flux_list[i] < median_flux - 2*sd_flux:
sky_list.append(fname_list[i])
sci_list.remove(fname_list[i])
with open(self.outpath+"sci_list.txt", "w") as f:
for sci in sci_list:
f.write(sci+'\n')
with open(self.outpath+"sky_list.txt", "w") as f:
for sci in sky_list:
f.write(sci+'\n')
#plot of centre flux
if debug:
plt.figure(1)
plt.title('Flux of median sci frames at the centre \
of the coronagraph')
plt.plot(flux_list, 'bo', label = 'centre flux')
plt.ylabel('Flux')
plt.legend()
plt.savefig(self.outpath + 'flux_plot')
plt.show()
def mk_dico(self, coro = True, verbose = True, debug = False):
if coro:
#creacting a dictionary
#add a list of the legnth of frames in the cubes add that to the dico
file_list = [f for f in listdir(self.outpath) if
isfile(join(self.outpath, f))]
fits_list = []
sci_list = []
sci_list_mjd = []
sky_list = []
unsat_list = []
unsat_list_mjd = []
flat_list = []
X_sci_list = []
X_unsat_list = []
flat_dark_list = []
sci_dark_list = []
unsat_dark_list = []
sci_frames = []
sky_frames = []
unsat_frames = []
flat_frames = []
for fname in file_list:
if fname.endswith('.fits') and fname.startswith('NACO'):
fits_list.append(fname)
cube, header = open_fits(self.outpath +fname, header=True,
verbose=debug)
if header['HIERARCH ESO DPR CATG'] == 'SCIENCE'and \
header['HIERARCH ESO DPR TYPE'] == 'OBJECT' and \
header['HIERARCH ESO DET DIT'] == self.dit_sci and \
header['HIERARCH ESO DET NDIT'] in self.ndit_sci and\
cube.shape[0] > 2/3*self.ndit_sci:
sci_list.append(fname)
sci_list_mjd.append(header['MJD-OBS'])
X_sci_list.append(header['AIRMASS'])
sci_frames.append(cube.shape[0])
elif (header['HIERARCH ESO DPR CATG'] == 'SCIENCE' and \
header['HIERARCH ESO DPR TYPE'] == 'SKY' and \
header['HIERARCH ESO DET DIT'] == self.dit_sci and\
header['HIERARCH ESO DET NDIT'] in self.ndit_sky) and\
cube.shape[0] > 2/3*self.ndit_sky:
sky_list.append(fname)
sky_frames.append(cube.shape[0])
elif header['HIERARCH ESO DPR CATG'] == 'SCIENCE' and \
header['HIERARCH ESO DET DIT'] == self.dit_unsat and \
header['HIERARCH ESO DET NDIT'] in self.ndit_unsat:
unsat_list.append(fname)
unsat_list_mjd.append(header['MJD-OBS'])
X_unsat_list.append(header['AIRMASS'])
unsat_frames.append(cube.shape[0])
elif 'FLAT,SKY' in header['HIERARCH ESO DPR TYPE']:
flat_list.append(fname)
flat_frames.append(cube.shape[0])
elif 'DARK' in header['HIERARCH ESO DPR TYPE']:
if header['HIERARCH ESO DET DIT'] == self.dit_flat:
flat_dark_list.append(fname)
if header['HIERARCH ESO DET DIT'] == self.dit_sci:
sci_dark_list.append(fname)
if header['HIERARCH ESO DET DIT'] == self.dit_unsat:
unsat_dark_list.append(fname)
with open(self.outpath+"sci_list.txt", "w") as f:
for sci in sci_list:
f.write(sci+'\n')
with open(self.outpath+"sky_list.txt", "w") as f:
for sci in sky_list:
f.write(sci+'\n')
with open(self.outpath+"unsat_list.txt", "w") as f:
for sci in unsat_list:
f.write(sci+'\n')
with open(self.outpath+"unsat_dark_list.txt", "w") as f:
for sci in unsat_dark_list:
f.write(sci+'\n')
with open(self.outpath+"flat_dark_list.txt", "w") as f:
for sci in flat_dark_list:
f.write(sci+'\n')
with open(self.outpath+"sci_dark_list.txt", "w") as f:
for sci in sci_dark_list:
f.write(sci+'\n')
with open(self.outpath+"flat_list.txt", "w") as f:
for sci in flat_list:
f.write(sci+'\n')
open(self.outpath+"resel.txt", "w").write(str(self.resel))
def bad_columns(self, sat_val=32768, verbose=True, debug=False):
"""
In NACO data there are systematic bad columns in the lower left quadrant
This method will correct those bad columns with the median of the neighbouring pixels.
May require manual inspection of one frame to confirm the saturated value.
sat_val : int, optional
value of the saturated column. Default 32768
"""
# creating bad pixel map
# bcm = np.zeros((1026, 1024) ,dtype=np.float64)
# for i in range(3, 509, 8):
# for j in range(512):
# bcm[j,i] = 1
for fname in self.file_list:
tmp, header_fname = open_fits(self.inpath + fname,
header=True,
verbose=debug)
print('Opened {} of type {}'.format(
fname, header_fname['HIERARCH ESO DPR TYPE']))
# ADD code here that checks for bad column and updates the mask
if verbose:
print('Fixing {} of shape {}'.format(fname, tmp.shape))
# crop the bad pixel map to the same dimensions of the frames
if len(tmp.shape) == 3:
nz, ny, nx = tmp.shape
bcm = np.zeros(
(ny, nx),
dtype=np.int8) # make mask the same dimensions as cube
tmp_median = np.median(tmp, axis=0) # median frame of cube
# loop through the median cube pixels and if any are 32768, add the location to the mask
bcm[np.where(tmp_median == sat_val)] = 1
# for i in range(0,nx): # all x pixels
# for j in range(0,ny): # all y pixels
# if tmp_median[j,i] == sat_val: # if saturated
# bcm[j,i] = 1 # mark as bad in mask
# cy, cx = ny/2 , nx/2
# ini_y, fin_y = int(512-cy), int(512+cy)
# ini_x, fin_x = int(512-cx), int(512+cx)
# bcm_crop = bcm[ini_y:fin_y,ini_x:fin_x]
for j in range(nz):
# replace bad columns in each frame of the cubes
tmp[j] = frame_fix_badpix_isolated(tmp[j],
bpm_mask=bcm,
sigma_clip=3,
num_neig=5,
size=5,
protect_mask=False,
radius=30,
verbose=debug,
debug=False)
write_fits(self.outpath + fname,
tmp,
header_fname,
output_verify='fix')
else:
print('File {} is not a cube ({})'.format(
fname, header_fname['HIERARCH ESO DPR TYPE']))
ny, nx = tmp.shape
bcm = np.zeros(
(ny, nx),
dtype=np.int8) # make mask the same dimensions as frame
bcm[np.where(tmp == sat_val)] = 1
# for i in range(0,nx):
# for j in range(0,ny):
# if tmp[j,i] == sat_val:
# bcm[j,i] = 1
# cy, cx = ny/2 , nx/2
# ini_y, fin_y = int(512-cy), int(512+cy)
# ini_x, fin_x = int(512-cx), int(512+cx)
# bcm_crop = bcm[ini_y:fin_y,ini_x:fin_x]
tmp = frame_fix_badpix_isolated(tmp,
bpm_mask=bcm,
sigma_clip=3,
num_neig=5,
size=5,
protect_mask=False,
radius=30,
verbose=debug,
debug=False)
write_fits(self.outpath + fname,
tmp,
header_fname,
output_verify='fix')
if verbose:
print('Fixed', fname)
def find_AGPM_or_star(self,
file_list,
rel_AGPM_pos_xy=(50.5, 6.5),
size=101,
verbose=True,
debug=False):
"""
added by Iain to prevent dust grains being picked up as the AGPM
This method will find the location of the AGPM or star (even when sky frames are mixed with science frames), by
using the known relative distance of the AGPM from the frame center in all VLT/NaCO datasets. It then creates a
subset square image around the expected location and applies a low pass filter + max search method and returns
the (y,x) location of the AGPM/star
Parameters
----------
file_list : list of str
List containing all science cube names
rel_AGPM_pos_xy : tuple, float
relative location of the AGPM from the frame center in pixels, should be left unchanged. This is used to
calculate how many pixels in x and y the AGPM is from the center and can be applied to almost all datasets
with VLT/NaCO as the AGPM is always in the same approximate position
size : int
pixel dimensions of the square to sample for the AGPM/star (ie size = 100 is 100 x 100 pixels)
verbose : bool
If True extra messages are shown
debug : bool, False by default
Enters pdb once the location has been found
Returns
----------
[ycom, xcom] : location of AGPM or star
"""
sci_cube = open_fits(self.inpath +
file_list[0]) # opens first sci/sky cube
nz, ny, nx = sci_cube.shape # gets size of it. science and sky cubes have same shape. assumes all cubes are the same ny and nx (they should be!)
cy, cx = frame_center(sci_cube,
verbose=verbose) # find central pixel coordinates
# then the position will be that plus the relative shift in y and x
rel_shift_x = rel_AGPM_pos_xy[
0] # 50.5 is pixels from frame center to AGPM in x in an example data set, thus providing the relative shift
rel_shift_y = rel_AGPM_pos_xy[
1] # 6.5 is pixels from frame center to AGPM in y in an example data set, thus providing the relative shift
# the center of the square to apply the low pass filter to - is the approximate position of the AGPM/star based on previous observations
y_tmp = cy + rel_shift_y
x_tmp = cx + rel_shift_x
median_all_cubes = np.zeros([len(file_list), ny, nx]) # makes empty array
for sc, fits_name in enumerate(file_list): # loops over all images
tmp = open_fits(self.inpath + fits_name,
verbose=debug) # opens the cube
median_all_cubes[sc] = tmp[
-1] # takes the last entry (the median) and adds it to the empty array
median_frame = np.median(median_all_cubes,
axis=0) # median of all median frames
# define a square of 100 x 100 with the center being the approximate AGPM/star position
median_frame, cornery, cornerx = get_square(median_frame,
size=size,
y=y_tmp,
x=x_tmp,
position=True,
verbose=True)
# apply low pass filter
median_frame = frame_filter_lowpass(median_frame,
median_size=7,
mode='median')
median_frame = frame_filter_lowpass(median_frame,
mode='gauss',
fwhm_size=5)
# find coordiates of max flux in the square
ycom_tmp, xcom_tmp = np.unravel_index(np.argmax(median_frame),
median_frame.shape)
# AGPM/star is the bottom-left corner coordinates plus the location of the max in the square
ycom = cornery + ycom_tmp
xcom = cornerx + xcom_tmp
if verbose:
print('The location of the AGPM/star is', 'ycom =', ycom, 'xcom =',
xcom)
if debug:
pdb.set_trace()
return [ycom, xcom]
print("Taking datconfig.ini")
config = configparser.ConfigParser()
config.read(configpath)
vmin=np.float(config.get("wavelengthparams","vmin"))
vmax=np.float(config.get("wavelengthparams","vmax"))
dv=np.float(config.get("wavelengthparams","dv"))
vels=np.arange(vmin,vmax,dv)
CC=CrossCorr(vels)
from glob import glob
from scipy.interpolate import interp1d
fl=config.get("paths","template_path")
Teff=fl.split('/')[-1].split('-')[0][-4:]
logg=fl.split('/')[-1].split('-')[1]
temp_flux,temp_wavs=CC.processTemplate(fl)
fwhm=open_fits("/Users/rakesh/Data/Final_derot_residual_cubes_HD142527/Rakesh_derot/fwhm_vec_WLcorr")
n_comps=int(config.get("wavelengthparams",'n_comps'))
wmin_max=[np.float(config.get("wavelengthparams","wmin")),
np.float(config.get("wavelengthparams","wmax"))]#[2.2,2.5]
datapath=config.get("paths","datapath")#"/Users/rakesh/Data/PDS70v2020/"
pp=(config.get("imageparams",'preprocess',fallback=True))
window_size=int(config.get("wavelengthparams","window_size",fallback=101))
order=int(config.get("wavelengthparams","order",fallback=1))
s=SINFONI(datpath=datapath,filename=config.get("paths","cubename"),
wavelen=config.get("paths","wavelen_file"),#"good_lambdas_WLcorr.fits",
fwhm=config.get("paths","fwhm_file"),#"good_fwhm_WLcorr.fits",
sz=int(config.get("imageparams","crop_size")),
wmin_max=wmin_max,
wmin_wmax_tellurics=[np.float(config.get("wavelengthparams","tell_wmin",fallback=1.75)),
np.float(config.get("wavelengthparams","tell_wmax",fallback=2.15))])
def correct_nan(self, verbose=True, debug=False):
sci_list = []
with open(self.outpath + "sci_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sci_list.append(line.split('\n')[0])
sky_list = []
with open(self.outpath + "sky_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sky_list.append(line.split('\n')[0])
unsat_list = []
with open(self.outpath + "unsat_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
unsat_list.append(line.split('\n')[0])
n_sci = len(sci_list)
n_sky = len(sky_list)
bar = pyprind.ProgBar(n_sci,
stream=1,
title='Correcting nan pixels in SCI frames')
for sc, fits_name in enumerate(sci_list):
tmp = open_fits(self.outpath + '2_ff_' + fits_name, verbose=debug)
tmp_tmp = cube_correct_nan(tmp,
neighbor_box=3,
min_neighbors=3,
verbose=debug)
write_fits(self.outpath + '2_nan_corr_' + fits_name,
tmp_tmp,
verbose=debug)
bar.update()
if not debug:
os.system("rm " + self.outpath + '2_ff_' + fits_name)
if verbose:
print('Done corecting NAN pixels in SCI frames')
bar = pyprind.ProgBar(n_sky,
stream=1,
title='Correcting nan pixels in SKY frames')
for sk, fits_name in enumerate(sky_list):
tmp = open_fits(self.outpath + '2_ff_' + fits_name, verbose=debug)
tmp_tmp = cube_correct_nan(tmp,
neighbor_box=3,
min_neighbors=3,
verbose=debug)
write_fits(self.outpath + '2_nan_corr_' + fits_name,
tmp_tmp,
verbose=debug)
bar.update()
if not debug:
os.system("rm " + self.outpath + '2_ff_' + fits_name)
if verbose:
print('Done corecting NAN pixels in SKY frames')
for un, fits_name in enumerate(unsat_list):
tmp = open_fits(self.outpath + '2_ff_unsat_' + fits_name,
verbose=debug)
tmp_tmp = cube_correct_nan(tmp,
neighbor_box=3,
min_neighbors=3,
verbose=debug)
write_fits(self.outpath + '2_nan_corr_unsat_' + fits_name,
tmp_tmp,
verbose=debug)
if not debug:
os.system("rm " + self.outpath + '2_ff_unsat_' + fits_name)
if verbose:
print('Done correcting NAN pixels in UNSAT frames')
def mk_dico(self, coro=True, verbose=True, debug=False):
if coro:
# creating a dictionary
file_list = [
f for f in listdir(self.outpath)
if isfile(join(self.outpath, f))
]
fits_list = []
sci_list = []
sci_list_mjd = []
sky_list = []
sky_list_mjd = []
unsat_list = []
flat_list = []
flat_dark_list = []
sci_dark_list = []
unsat_dark_list = []
master_mjd = []
master_airmass = []
if verbose:
print('Creating dictionary')
for fname in file_list:
if fname.endswith('.fits') and fname.startswith('NACO'):
fits_list.append(fname)
cube, header = open_fits(self.outpath + fname,
header=True,
verbose=debug)
if header['HIERARCH ESO DPR CATG'] == 'SCIENCE' and \
header['HIERARCH ESO DPR TYPE'] == 'OBJECT' and \
header['HIERARCH ESO DET DIT'] == self.dit_sci and \
header['HIERARCH ESO DET NDIT'] in self.ndit_sci and \
cube.shape[0] > 0.8 * min(self.ndit_sci): # avoid bad cubes
sci_list.append(fname)
sci_list_mjd.append(header['MJD-OBS'])
# sci_list_airmass.append(header['AIRMASS'])
elif (header['HIERARCH ESO DPR CATG'] == 'SCIENCE' and \
header['HIERARCH ESO DPR TYPE'] == 'SKY' and \
header['HIERARCH ESO DET DIT'] == self.dit_sci and \
header['HIERARCH ESO DET NDIT'] in self.ndit_sky) and \
cube.shape[0] > 0.8 * min(self.ndit_sky): # avoid bad cubes
sky_list.append(fname)
sky_list_mjd.append(header['MJD-OBS'])
# sky_list_airmass.append(header['AIRMASS'])
elif header['HIERARCH ESO DPR CATG'] == 'SCIENCE' and \
header['HIERARCH ESO DET DIT'] == self.dit_unsat and \
header['HIERARCH ESO DET NDIT'] in self.ndit_unsat:
unsat_list.append(fname)
# unsat_list_mjd.append(header['MJD-OBS'])
# unsat_list_airmass.append(header['AIRMASS'])
elif 'FLAT,SKY' in header['HIERARCH ESO DPR TYPE']:
flat_list.append(fname)
# flat_list_mjd.append(header['MJD-OBS'])
# flat_list_airmass.append(header['AIRMASS'])
elif 'DARK' in header['HIERARCH ESO DPR TYPE']:
if header['HIERARCH ESO DET DIT'] == self.dit_flat:
flat_dark_list.append(fname)
if header['HIERARCH ESO DET DIT'] == self.dit_sci:
sci_dark_list.append(fname)
if header['HIERARCH ESO DET DIT'] == self.dit_unsat:
unsat_dark_list.append(fname)
with open(self.outpath + "sci_list_mjd.txt", "w") as f:
for time in sci_list_mjd:
f.write(str(time) + '\n')
with open(self.outpath + "sky_list_mjd.txt", "w") as f:
for time in sky_list_mjd:
f.write(str(time) + '\n')
with open(self.outpath + "sci_list.txt", "w") as f:
for sci in sci_list:
f.write(sci + '\n')
with open(self.outpath + "sky_list.txt", "w") as f:
for sci in sky_list:
f.write(sci + '\n')
with open(self.outpath + "unsat_list.txt", "w") as f:
for sci in unsat_list:
f.write(sci + '\n')
with open(self.outpath + "unsat_dark_list.txt", "w") as f:
for sci in unsat_dark_list:
f.write(sci + '\n')
with open(self.outpath + "flat_dark_list.txt", "w") as f:
for sci in flat_dark_list:
f.write(sci + '\n')
with open(self.outpath + "sci_dark_list.txt", "w") as f:
for sci in sci_dark_list:
f.write(sci + '\n')
with open(self.outpath + "flat_list.txt", "w") as f:
for sci in flat_list:
f.write(sci + '\n')
if verbose:
print('Done :)')
def bad_frame_removal(self,
pxl_shift_thres=0.5,
sub_frame_sz=31,
verbose=True,
debug=False,
plot='save'):
"""
For removing outlier frames often caused by AO errors. To be run after recentering is complete. Takes the
recentered mastercube and removes frames with a shift greater than a user defined pixel threshold in x or y above
the median shift. It then takes the median of those cubes and correlates them to the median combined mastercube.
Removes all those frames below the threshold from the mastercube and rotation file, then saves both as new files
for use in post processing
pxl_shift_thres: decimal, in units of pixels. Default is 0.5 pixels.
Any shifts in the x or y direction greater than this threshold will cause the frame/s
to be labelled as bad and thus removed
sub_frame_sz: integer, must be odd. Default is 31.
This sets the cropping during frame correlation to the median
plot: 'save' to write to file the correlation plot, None will not
"""
if verbose:
print('\n')
print('Beginning bad frame removal...')
print('\n')
angle_file = open_fits(self.outpath + 'derot_angles.fits',
verbose=debug) #opens the rotation file
recentered_cube = open_fits(
self.outpath +
'{}_master_cube.fits'.format(self.dataset_dict['source']),
verbose=debug) # loads the master cube
#open x shifts file for the respective method
x_shifts = open_fits(self.outpath + "x_shifts.fits", verbose=debug)
median_sx = np.median(x_shifts) #median of x shifts
#opens y shifts file for the respective method
y_shifts = open_fits(self.outpath + "y_shifts.fits", verbose=debug)
median_sy = np.median(y_shifts) #median of y shifts
#x_shifts_long = np.zeros([len(self.sci_list)*self.ndit]) # list with number of cubes times number of frames in each cube as the length
#y_shifts_long = np.zeros([len(self.sci_list)*self.ndit])
if not self.fast_reduction: # long are shifts to be apply to each frame in each cube. fast reduction only has one cube
x_shifts_long = np.zeros([int(np.sum(self.real_ndit_sci))])
y_shifts_long = np.zeros([int(np.sum(self.real_ndit_sci))])
for i in range(len(
self.sci_list)): # from 0 to the length of sci_list
ndit = self.real_ndit_sci[i] # gets the dimensions of the cube
x_shifts_long[i * ndit:(i + 1) * ndit] = x_shifts[
i] # sets the average shifts of all frames in that cube
y_shifts_long[i * ndit:(i + 1) * ndit] = y_shifts[i]
write_fits(self.outpath + 'x_shifts_long.fits',
x_shifts_long,
verbose=debug) # saves shifts to file
write_fits(self.outpath + 'y_shifts_long.fits',
y_shifts_long,
verbose=debug)
x_shifts = x_shifts_long
y_shifts = y_shifts_long
if verbose:
print("x shift median:", median_sx)
print("y shift median:", median_sy)
bad = []
good = []
# this system works, however it can let a couple of wild frames slip through at a 0.5 pixel threshold. may need a stricter threshold depending on the data and crop size
i = 0
shifts = list(zip(x_shifts, y_shifts))
bar = pyprind.ProgBar(len(x_shifts),
stream=1,
title='Running pixel shift check...')
for sx, sy in shifts: #iterate over the shifts to find any greater or less than pxl_shift_thres pixels from median
if abs(sx) < ((abs(median_sx)) + pxl_shift_thres) and abs(sx) > (
(abs(median_sx)) - pxl_shift_thres) and abs(sy) < (
(abs(median_sy)) + pxl_shift_thres) and abs(sy) > (
(abs(median_sy)) - pxl_shift_thres):
good.append(i)
else:
bad.append(i)
i += 1
bar.update()
# only keeps the files that weren't shifted above the threshold
frames_pxl_threshold = recentered_cube[good]
recentered_cube = None
if verbose:
print('Frames within pixel shift threshold:',
len(frames_pxl_threshold))
#recentered_cube = None
# only keeps the corresponding derotation entry for the frames that were kept
angle_pxl_threshold = angle_file[good]
if verbose:
print(
'########### Median combining {} frames for correlation check... ###########'
.format(len(frames_pxl_threshold)))
#makes array of good frames from the recentered mastercube
subarray = cube_crop_frames(
frames_pxl_threshold, size=sub_frame_sz,
verbose=verbose) # crops all the frames to a common size
frame_ref = np.median(
subarray, axis=0
) #median frame of remaining cropped frames, can be sped up with multi-processing
if verbose:
print('Running frame correlation check...')
# calculates correlation threshold using the median of the Pearson correlation of all frames, minus 1 standard deviation
#frame_ref = frame_crop(tmp_median, size = sub_frame_sz, verbose=verbose) # crops the median of all frames to a common size
distances = cube_distance(
subarray, frame_ref, mode='full', dist='pearson', plot=True
) # calculates the correlation of each frame to the median and saves as a list
if plot == 'save': # save a plot of distances compared to the median for each frame if set to 'save'
plt.savefig(self.outpath + 'distances.pdf')
correlation_thres = np.median(distances) - np.std(
distances
) # threshold is the median of the distances minus one stddev
good_frames, bad_frames = cube_detect_badfr_correlation(
subarray,
frame_ref=frame_ref,
dist='pearson',
threshold=correlation_thres,
plot=True,
verbose=False)
if plot == 'save':
plt.savefig(self.outpath + 'frame_correlation.pdf')
#only keeps the files that were above the correlation threshold
frames_threshold = frames_pxl_threshold[good_frames]
frames_pxl_threshold = None
if verbose:
print('Frames within correlation threshold:',
len(frames_threshold))
#frames_pxl_threshold = None
# only keeps the derotation entries for the good frames above the correlation threshold
angle_threshold = angle_pxl_threshold[good_frames]
# saves the good frames to a new file, and saves the derotation angles to a new file
write_fits(
self.outpath +
'{}_master_cube.fits'.format(self.dataset_dict['source']),
frames_threshold)
write_fits(self.outpath + 'derot_angles.fits', angle_threshold)
if verbose:
print('Saved good frames and their respective rotations to file')
frames_threshold = None
def find_sky_in_sci_cube(self,
nres=3,
coro=True,
verbose=True,
plot=None,
debug=False):
"""
Empty SKY list could be caused by a misclassification of the header in NACO data
This method will check the flux of the SCI cubes around the location of the AGPM
A SKY cube should be less bright at that location allowing the separation of cubes
"""
flux_list = []
fname_list = []
sci_list = []
sky_list = []
sci_list_mjd = [] # observation time of each sci cube
sky_list_mjd = [] # observation time of each sky cube
with open(self.outpath + "sci_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sci_list.append(line.split('\n')[0])
with open(self.outpath + "sky_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sky_list.append(line.split('\n')[0])
with open(self.outpath + "sci_list_mjd.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sci_list_mjd.append(float(line.split('\n')[0]))
with open(self.outpath + "sky_list_mjd.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sky_list_mjd.append(float(line.split('\n')[0]))
self.resel = (self.dataset_dict['wavelength'] * 180 *
3600) / (self.dataset_dict['size_telescope'] * np.pi *
self.dataset_dict['pixel_scale'])
agpm_pos = find_AGPM_or_star(self, sci_list, verbose=debug)
if verbose:
print('The rough location of the star/AGPM is', 'y = ',
agpm_pos[0], 'x =', agpm_pos[1])
print('Measuring flux in SCI cubes...')
# create the aperture
circ_aper = CircularAperture((agpm_pos[1], agpm_pos[0]),
round(nres * self.resel))
# total flux through the aperture
for fname in sci_list:
cube_fname = open_fits(self.outpath + fname, verbose=debug)
median_frame = np.median(cube_fname, axis=0)
circ_aper_phot = aperture_photometry(median_frame,
circ_aper,
method='exact')
# append it to the flux list.
circ_flux = np.array(circ_aper_phot['aperture_sum'])
flux_list.append(circ_flux[0])
fname_list.append(fname)
if debug:
print('centre flux has been measured for', fname)
median_flux = np.median(flux_list)
sd_flux = np.std(flux_list)
if verbose:
print('Sorting Sky from Sci')
for i in range(len(flux_list)):
if flux_list[i] < median_flux - 2 * sd_flux:
sky_list.append(
fname_list[i]) # add the sky cube to the sky cube list
sky_list_mjd.append(
sci_list_mjd[i]
) # add the observation to the sky obs list from the sci obs list
sci_list.remove(
fname_list[i]) # remove the sky cube from the sci list
sci_list_mjd.remove(
sci_list_mjd[i]
) # remove the sky obs time from the sci obs list
symbol = 'bo'
if plot:
if flux_list[i] > median_flux - 2 * sd_flux:
symbol = 'go'
else:
symbol = 'ro'
plt.plot(i, flux_list[i] / median_flux, symbol)
if plot:
plt.title('Normalised flux around star')
plt.ylabel('normalised flux')
plt.xlabel('cube')
if plot == 'save':
plt.savefig(self.outpath + 'flux_plot.pdf')
if plot == 'show':
plt.show()
# sci_list.sort()
# with open(self.outpath + "sci_list.txt", "w") as f:
# for sci in sci_list:
# f.write(sci + '\n')
sci_list.sort()
if self.fast_reduction:
with open(self.outpath + "sci_list_ori.txt", "w") as f:
for ss, sci in enumerate(sci_list):
tmp = open_fits(self.inpath + sci, verbose=debug)
if ss == 0:
master_sci = np.zeros(
[len(sci_list), tmp.shape[1], tmp.shape[2]])
master_sci[ss] = tmp[-1]
f.write(sci + '\n')
with open(self.outpath + "sci_list.txt", "w") as f:
f.write('master_sci_fast_reduction.fits')
write_fits(self.outpath + 'master_sci_fast_reduction.fits',
master_sci,
verbose=debug)
print('Saved fast reduction master science cube')
else:
with open(self.outpath + "sci_list.txt", "w") as f:
for sci in sci_list:
f.write(sci + '\n')
sky_list.sort()
with open(self.outpath + "sky_list.txt", "w") as f:
for sky in sky_list:
f.write(sky + '\n')
sci_list_mjd.sort()
# save the sci observation time to text file
with open(self.outpath + "sci_list_mjd.txt", "w") as f:
for time in sci_list_mjd:
f.write(str(time) + '\n')
sky_list_mjd.sort()
# save the sky observation time to text file
with open(self.outpath + "sky_list_mjd.txt", "w") as f:
for time in sky_list_mjd:
f.write(str(time) + '\n')
if len(sci_list_mjd) != len(sci_list):
print(
'======== WARNING: SCI observation time list is a different length to SCI cube list!! ========'
)
if len(sky_list_mjd) != len(sky_list):
print(
'======== WARNING: SKY observation time list is a different length to SKY cube list!! ========'
)
if verbose:
print('done :)')
def median_binning(self, binning_factor=10, verbose=True, debug=False):
"""
Median combines the frames within the master science cube as per the binning factor, and makes the necessary
changes to the derotation file
Parameters:
***********
binning_factor: int,list or tuple
Defines how many frames to median combine. Use a list or tuple to run binning multiple times with different
factors. Default = 10
Writes to fits file:
********
the binned master cube
the binned derotation angles
"""
if isinstance(binning_factor, int) == False and isinstance(binning_factor,list) == False and \
isinstance(binning_factor,tuple) == False: # if it isnt int, tuple or list then raise an error
raise TypeError(
'Invalid binning_factor! Use either int, list or tuple')
if not os.path.isfile(
self.outpath +
'{}_master_cube.fits'.format(self.dataset_dict['source'])):
raise NameError(
'Missing master cube from recentering and bad frame removal!')
if not os.path.isfile(self.outpath + 'derot_angles.fits'):
raise NameError(
'Missing derotation angles files from recentering and bad frame removal!'
)
master_cube = open_fits(
self.outpath +
'{}_master_cube.fits'.format(self.dataset_dict['source']),
verbose=debug)
derot_angles = open_fits(self.outpath + 'derot_angles.fits',
verbose=debug)
def _binning(self, binning_factor, master_cube, derot_angles):
if binning_factor == 1 or binning_factor == 0: # doesn't bin with 1 but will loop over the other factors in the list or tuple
print(
'Binning factor is 1 or 0 (cant bin any frames). Skipping binning...'
)
else:
if verbose:
print(
'##### Median binning frames with binning_factor = {} #####'
.format(binning_factor))
nframes, ny, nx = master_cube.shape
derot_angles_binned = np.zeros([int(nframes / binning_factor)])
master_cube_binned = np.zeros(
[int(nframes / binning_factor), ny, nx])
for idx, frame in enumerate(
range(binning_factor, nframes, binning_factor)):
if idx == (int(nframes / binning_factor) - 1):
master_cube_binned[idx] = np.median(
master_cube[frame - binning_factor:], axis=0)
derot_angles_binned[idx] = np.median(
derot_angles[frame - binning_factor:])
master_cube_binned[idx] = np.median(
master_cube[frame - binning_factor:frame], axis=0)
derot_angles_binned[idx] = np.median(
derot_angles[frame - binning_factor:frame])
write_fits(
self.outpath + '{}_master_cube.fits'.format(
self.dataset_dict['source'], binning_factor),
master_cube_binned)
write_fits(
self.outpath + 'derot_angles.fits'.format(binning_factor),
derot_angles_binned)
if isinstance(binning_factor, int):
_binning(self, binning_factor, master_cube, derot_angles)
if isinstance(binning_factor, list) or isinstance(
binning_factor, tuple):
for binning_factor in binning_factor:
_binning(self, binning_factor, master_cube, derot_angles)
def find_derot_angles(inpath, fits_list, loc='st', ipag=True, verbose=False):
"""
Find the derotation angle vector to apply to a set of NACO cubes to align it with North up.
IMPORTANT: the list of fits has to be in chronological order of acquisition.
Parameters:
***********
inpath: str
Where the fits files are located
fits_list: list of str
List containing the name of every fits file to be considered
loc: {'st','nd'}, str, opt
Whether to consider the derotation angle at the beginning ('st') or end
('nd') of the cube exposure.
ipag: {False, True}, bool, opt
Whether to use the method recommended by ipag. If not, just return the value
of keyword 'POSANG'
"""
n_fits = len(fits_list)
rot_pt_off = np.zeros(n_fits)
parang = np.zeros(n_fits)
final_derot_angs = np.zeros(n_fits)
posang = np.zeros(n_fits)
if loc == 'st':
kw_par = 'HIERARCH ESO TEL PARANG START'
kw_pos = 'HIERARCH ESO ADA POSANG'
elif loc == 'nd':
kw_par = 'HIERARCH ESO TEL PARANG END'
kw_pos = 'HIERARCH ESO ADA POSANG END'
# FIRST COMPILE PARANG, POSANG and PUPILPOS
for ff in range(n_fits):
_, header = open_fits(inpath + fits_list[ff],
header=True,
verbose=False)
parang[ff] = header[kw_par]
posang[ff] = header[kw_pos]
pupilpos = 180.0 - parang[ff] + posang[ff]
rot_pt_off[ff] = 90 + 89.44 - pupilpos
if verbose:
print("parang: {}, posang: {}, rot_pt_off: {}".format(
parang[ff], posang[ff], rot_pt_off[ff]))
# NEXT CHECK IF THE OBSERVATION WENT THROUGH TRANSIT (change of sign in parang OR stddev of rot_pt_off > 1.)
rot_pt_off_med = np.median(rot_pt_off)
rot_pt_off_std = np.std(rot_pt_off)
if np.min(parang) * np.max(parang) < 0. or rot_pt_off_std > 1.:
if verbose:
print(
"The observation goes through transit and/or the pupil position was reset in the middle of the observation: "
)
if np.min(parang) * np.max(parang) < 0.:
print("min/max parang: ", np.min(parang), np.max(parang))
if rot_pt_off_std > 1.:
print(
"the standard deviation of pupil positions is greater than 1: ",
rot_pt_off_std)
# find index where the transit occurs (change of sign of parang OR big difference in pupil pos)
n_changes = 0
for ff in range(n_fits - 1):
if parang[ff] * parang[ff + 1] < 0. or np.abs(
rot_pt_off[ff] - rot_pt_off[ff + 1]) > 1.:
idx_transit = ff + 1
n_changes += 1
# check that these conditions only detected one passage through transit
if n_changes != 1:
print(
" {} passages of transit were detected (instead of 1!). Check that the input fits list is given in chronological order."
.format(n_changes))
pdb.set_trace()
rot_pt_off_med1 = np.median(rot_pt_off[:idx_transit])
rot_pt_off_med2 = np.median(rot_pt_off[idx_transit:])
final_derot_angs = rot_pt_off_med1 - parang
final_derot_angs[idx_transit:] = rot_pt_off_med2 - parang[idx_transit:]
else:
final_derot_angs = rot_pt_off_med - parang
# MAKE SURE ANGLES ARE IN THE RANGE (-180,180)deg
min_derot_angs = np.amin(final_derot_angs)
nrot_min = min_derot_angs / 360.
if nrot_min < -0.5:
final_derot_angs[np.where(
final_derot_angs < -180)] = final_derot_angs[np.where(
final_derot_angs < -180)] + np.ceil(nrot_min) * 360.
max_derot_angs = np.amax(final_derot_angs)
nrot_max = max_derot_angs / 360.
if nrot_max > 0.5:
final_derot_angs[np.where(
final_derot_angs > 180)] = final_derot_angs[np.where(
final_derot_angs > 180)] - np.ceil(nrot_max) * 360.
if ipag:
return -1. * final_derot_angs
else:
return posang
def dark_subtract(self, verbose=True, debug=True):
sci_list = []
with open(self.outpath + "sci_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sci_list.append(line.split('\n')[0])
sky_list = []
with open(self.outpath + "sky_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sky_list.append(line.split('\n')[0])
unsat_list = []
with open(self.outpath + "unsat_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
unsat_list.append(line.split('\n')[0])
unsat_dark_list = []
with open(self.outpath + "unsat_dark_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
unsat_dark_list.append(line.split('\n')[0])
flat_list = []
with open(self.outpath + "flat_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
flat_list.append(line.split('\n')[0])
flat_dark_list = []
with open(self.outpath + "flat_dark_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
flat_dark_list.append(line.split('\n')[0])
sci_dark_list = []
with open(self.outpath + "sci_dark_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sci_dark_list.append(line.split('\n')[0])
pixel_scale = fits_info.pixel_scale
tmp = np.zeros([3, self.com_sz, self.com_sz])
#creating master dark cubes
for fd, fd_name in enumerate(flat_dark_list):
tmp_tmp = open_fits(self.inpath + fd_name,
header=False,
verbose=debug)
tmp[fd] = frame_crop(tmp_tmp,
self.com_sz,
force=True,
verbose=debug)
write_fits(self.outpath + 'flat_dark_cube.fits', tmp)
if verbose:
print('Flat dark cubes have been cropped and saved')
for sd, sd_name in enumerate(sci_dark_list):
tmp_tmp = open_fits(self.inpath + sd_name,
header=False,
verbose=debug)
n_dim = tmp_tmp.ndim
if sd == 0:
if n_dim == 2:
tmp = np.array([
frame_crop(tmp_tmp,
self.com_sz,
force=True,
verbose=debug)
])
else:
tmp = cube_crop_frames(tmp_tmp,
self.com_sz,
force=True,
verbose=debug)
else:
if n_dim == 2:
tmp = np.append(tmp, [
frame_crop(
tmp_tmp, self.com_sz, force=True, verbose=debug)
],
axis=0)
else:
tmp = np.append(tmp,
cube_crop_frames(tmp_tmp,
self.com_sz,
force=True,
verbose=debug),
axis=0)
write_fits(self.outpath + 'sci_dark_cube.fits', tmp)
if verbose:
print('Sci dark cubes have been cropped and saved')
#create an if stament for if the size is larger than sz and less than if less than crop by nx-1
for sd, sd_name in enumerate(unsat_dark_list):
tmp_tmp = open_fits(self.inpath + sd_name,
header=False,
verbose=debug)
tmp = np.zeros([
len(sci_dark_list) * tmp_tmp.shape[0], tmp_tmp.shape[1],
tmp_tmp.shape[2]
])
n_dim = tmp_tmp.ndim
if sd == 0:
if n_dim == 2:
ny, nx = tmp_tmp.shape
if nx <= self.com_sz:
tmp = np.array([
frame_crop(tmp_tmp,
nx - 1,
force=True,
verbose=debug)
])
else:
tmp = np.array(
[frame_crop(tmp_tmp, self.com_sz, verbose=debug)])
else:
nz, ny, nx = tmp_tmp.shape
if nx <= self.com_sz:
tmp = cube_crop_frames(tmp_tmp,
nx - 1,
force=True,
verbose=debug)
else:
tmp = cube_crop_frames(tmp_tmp,
self.com_sz,
force=True,
verbose=debug)
else:
if n_dim == 2:
ny, nx = tmp_tmp.shape
if nx <= self.com_sz:
tmp = np.append(tmp, [
frame_crop(
tmp_tmp, nx - 1, force=True, verbose=debug)
],
axis=0)
else:
tmp = np.append(tmp, [
frame_crop(tmp_tmp,
self.com_sz,
force=True,
verbose=debug)
],
axis=0)
else:
nz, ny, nx = tmp_tmp.shape
if nx <= self.com_sz:
tmp = cube_crop_frames(tmp,
nx - 1,
force=True,
verbose=debug)
tmp = np.append(tmp,
cube_crop_frames(tmp_tmp,
nx - 1,
force=True,
verbose=debug),
axis=0)
else:
tmp = np.append(tmp,
cube_crop_frames(tmp_tmp,
self.com_sz,
force=True,
verbose=debug),
axis=0)
tmp = np.zeros([
len(sci_dark_list) * tmp_tmp.shape[0], tmp_tmp.shape[1],
tmp_tmp.shape[2]
])
write_fits(self.outpath + 'unsat_dark_cube.fits', tmp)
if verbose:
print('Unsat dark cubes have been cropped and saved')
#defining the anulus (this is where we avoid correcting around the star)
cy, cx = find_agpm_list(self, sci_list)
self.agpm_pos = (cx, cy)
if verbose:
print(' The location of the AGPM has been calculated', 'cy = ', cy,
'cx = ', cx)
agpm_dedge = min(self.agpm_pos[0], self.agpm_pos[1],
self.com_sz - self.agpm_pos[0],
self.com_sz - self.agpm_pos[1])
mask_arr = np.ones([self.com_sz, self.com_sz])
cy, cx = frame_center(mask_arr)
mask_inner_rad = int(3.0 /
pixel_scale) # 3arcsec min to avoid star emission
mask_width = agpm_dedge - mask_inner_rad - 1
mask_AGPM_com = get_annulus_segments(mask_arr,
mask_inner_rad,
mask_width,
mode='mask')[0]
mask_AGPM_com = frame_shift(
mask_AGPM_com, self.agpm_pos[1] - cy, self.agpm_pos[0] -
cx) # agpm is not centered in the frame so shift the mask
if verbose:
print('AGPM mask has been defined')
if debug:
tmp = open_fits(self.outpath + sci_list[0])
#plot_frames(tmp[-1], circle = self.agpm_pos)
#now begin the dark subtraction useing PCA
npc_dark = 1 #val found this value gives the best result.
tmp_tmp = np.zeros([len(flat_list), self.com_sz, self.com_sz])
tmp_tmp_tmp = open_fits(self.outpath + 'flat_dark_cube.fits')
for fl, flat_name in enumerate(flat_list):
tmp = open_fits(self.inpath + flat_name,
header=False,
verbose=debug)
tmp_tmp[fl] = frame_crop(tmp,
self.com_sz,
force=True,
verbose=debug)
tmp_tmp = cube_subtract_sky_pca(tmp_tmp,
tmp_tmp_tmp,
mask_AGPM_com,
ref_cube=None,
ncomp=npc_dark)
write_fits(self.outpath + '1_crop_flat_cube.fits', tmp_tmp)
if verbose:
print('Dark has been subtracted from Flats')
if debug:
#plot the median of dark cube median of cube before subtraction median after subtraction
tmp_tmp_tmp = np.median(tmp_tmp_tmp,
axis=0) #flat_dark cube median
tmp = tmp #flat before subtraction
tmp_tmp = np.median(tmp_tmp,
axis=0) #median flat after dark subtract
plot_frames((tmp_tmp_tmp, tmp, tmp_tmp))
tmp_tmp_tmp = open_fits(self.outpath + 'sci_dark_cube.fits')
for sc, fits_name in enumerate(sci_list):
tmp = open_fits(self.inpath + fits_name,
header=False,
verbose=debug)
tmp = cube_crop_frames(tmp, self.com_sz, force=True, verbose=debug)
tmp_tmp = cube_subtract_sky_pca(tmp,
tmp_tmp_tmp,
mask_AGPM_com,
ref_cube=None,
ncomp=npc_dark)
write_fits(self.outpath + '1_crop_' + fits_name, tmp_tmp)
if verbose:
print('Dark has been subtracted from Sci')
if debug:
#plot the median of dark cube median of cube before subtraction median after subtraction
tmp_tmp_tmp = np.median(tmp_tmp_tmp, axis=0)
tmp = np.median(tmp, axis=0)
tmp_tmp = np.median(tmp_tmp, axis=0)
plot_frames((tmp_tmp_tmp, tmp, tmp_tmp))
tmp_tmp_tmp = open_fits(self.outpath + 'sci_dark_cube.fits')
for sc, fits_name in enumerate(sky_list):
tmp = open_fits(self.inpath + fits_name,
header=False,
verbose=debug)
tmp = cube_crop_frames(tmp, self.com_sz, force=True, verbose=debug)
tmp_tmp = cube_subtract_sky_pca(tmp,
tmp_tmp_tmp,
mask_AGPM_com,
ref_cube=None,
ncomp=npc_dark)
write_fits(self.outpath + '1_crop_' + fits_name, tmp_tmp)
if verbose:
print('Dark has been subtracted from Sky')
if debug:
#plot the median of dark cube median of cube before subtraction median after subtraction
tmp_tmp_tmp = np.median(tmp_tmp_tmp, axis=0)
tmp = np.median(tmp, axis=0)
tmp_tmp = np.median(tmp_tmp, axis=0)
plot_frames((tmp_tmp_tmp, tmp, tmp_tmp))
tmp_tmp_tmp = open_fits(self.outpath + 'master_unsat_dark.fits')
# no need to crop the unsat frame at the same size as the sci images if they are smaller
for un, fits_name in enumerate(unsat_list):
tmp = open_fits(self.inpath + fits_name, header=False)
#tmp = cube_crop_frames(tmp,nx_unsat_crop)
if tmp.shape[2] > self.com_sz:
nx_unsat_crop = self.com_sz
tmp_tmp = cube_crop_frames(tmp - tmp_tmp_tmp,
nx_unsat_crop,
force=True,
verbose=debug)
elif tmp.shape[2] % 2 == 0:
nx_unsat_crop = tmp.shape[2] - 1
tmp_tmp = cube_crop_frames(tmp - tmp_tmp_tmp,
nx_unsat_crop,
force=True,
verbose=debug)
else:
nx_unsat_crop = tmp.shape[2]
tmp_tmp = tmp - tmp_tmp_tmp
write_fits(self.outpath + '1_crop_unsat_' + fits_name, tmp_tmp)
if verbose:
print('unsat frames have been cropped')
if debug:
#plot the median of dark cube median of cube before subtraction median after subtraction
tmp_tmp_tmp = np.median(tmp_tmp_tmp, axis=0)
tmp = np.median(tmp, axis=0)
tmp_tmp = np.median(tmp_tmp, axis=0)
plot_frames((tmp_tmp_tmp, tmp, tmp_tmp))
