timit=False)
#####################################################################################################################################################
# (3) ORDER TRACING #################################################################################################################################
# find orders roughly
P, tempmask = find_stripes(MW,
deg_polynomial=2,
min_peak=0.05,
gauss_filter_sigma=3.,
simu=False)
# assign physical diffraction order numbers (this is only a dummy function for now) to order-fit polynomials and bad-region masks
P_id = make_P_id(P)
mask = make_mask_dict(tempmask)
# extract stripes of user-defined width from the science image, centred on the polynomial fits defined in step (1)
MW_stripes, MW_indices = extract_stripes(MW,
P_id,
return_indices=True,
slit_height=5)
#####################################################################################################################################################
###
#if we want to determine spatial profiles, then we should remove cosmics and background from MW like so:
# cosmic_cleaned_MW = remove_cosmics(MW, ronmask, obsname, path, Flim=3.0, siglim=5.0, maxiter=1, savemask=False, savefile=False, save_err=False, verbose=True, timit=True)
# bg_corrected_MW = remove_background(cosmic_cleaned_MW, P_id, obsname, path, degpol=5, slit_height=5, save_bg=False, savefile=False, save_err=False, exclude_top_and_bottom=True, verbose=True, timit=True)
#before doing the following:
MW_stripes, MW_stripe_indices = extract_stripes(MW,
P_id,
return_indices=True,
slit_height=30)
err_MW_stripes = extract_stripes(err_MW,
P_id,
for o in P_id.keys():
P_id[o][0] -= 2.
np.save(path + 'P_id.npy', P_id)
np.save(path + 'mask.npy', mask)
# now redo the master white properly, incl background removal, and save to file
if choice_mw.lower() == 'r':
MW,err_MW = process_whites(flat_list, MB=medbias, ronmask=ronmask, MD=MDS, gain=gain, scalable=True, fancy=False, P_id=P_id,
clip=5., savefile=True, saveall=False, diffimg=False, remove_bg=True, path=path, debug_level=1, timit=False)
#####################################################################################################################################################
### (4) CREATE FIBRE PROFILES #######################################################################################################################
slit_height = 30
MW_stripes,MW_indices = extract_stripes(MW, P_id, return_indices=True, slit_height=slit_height)
del MW_stripes # save memory
# err_MW_stripes = extract_stripes(err_MW, P_id, return_indices=False, slit_height=slit_height)
choice_fp = 'r'
if os.path.isfile(fibparms_path + 'combined_fibre_profile_fits_' + date + '.npy'):
choice_fp = raw_input("FIBRE PROFILES for " + date + " already exists! Do you want to skip this step or recreate it? ['s' / 'r']")
if choice_fp.lower() == 's':
print('Loading fibre profiles for ' + date + '...')
fibparms = np.load(fibparms_path + 'combined_fibre_profile_fits_' + date + '.npy').item()
else:
sure = 'x'
while sure.lower() not in ['y','n']:
sure = raw_input("Are you sure you want to create fibre profiles for " + date + "??? This may take several hours!!! ['y' / 'n']")
if sure == 'y':
fp_in = fit_multiple_profiles_from_indices(P_id, MW, err_MW, MW_indices, slit_height=slit_height, timit=True, debug_level=1)
ny=4096
medbias,coeffs,offsets,rons = get_bias_and_readnoise_from_bias_frames(corrected_bias_list, degpol=5, clip=5., gain=gain, debug_level=0, timit=True)
# create MASTER BIAS frame and read-out noise mask (units = electrons)
offmask,ronmask = make_offmask_and_ronmask(offsets, rons, nx, ny, gain=gain, savefiles=True, path=path, timit=True)
MB = make_master_bias_from_coeffs(coeffs, nx, ny, savefile=True, path=path, timit=True)
#we did not have darks, so I did this
MD = np.zeros(MB.shape)
#create (bias- & dark-subtracted) MASTER WHITE frame and corresponding error array (units = ADUs)
MW,err_MW = process_whites(white_list, corrected_white_list, MB=MB, ronmask=ronmask, MD=MD, gain=gain, scalable=False, fancy=False, clip=5., savefile=True, saveall=True, diffimg=False, path=path, timit=False)
# find orders
P,tempmask = find_stripes(MW, deg_polynomial=2, min_peak=0.05, gauss_filter_sigma=3., simu=False)
# assign physical diffraction order numbers (this is only a dummy function for now) to order-fit polynomials and bad-region masks
P_id = make_P_id(P)
mask = make_mask_dict(tempmask)
#loop over all files you want to extract a 1-dim spectrum for
for filename in corrected_stellar_list:
#do some housekeeping with filenames
dum = filename.split('/')
dum2 = dum[-1].split('.')
obsname = dum2[0]
# (1) call routine that does all the bias and dark correction stuff and proper error treatment
img = correct_for_bias_and_dark_from_filename(filename, MB, MD, gain=gain, scalable=False, savefile=True, path=path, timit=True) #[e-]
err_img = np.sqrt(np.clip(img,0,None) + ronmask*ronmask) # [e-]
# (5) extract stripes
stripes,stripe_indices = extract_stripes(img, P_id, return_indices=True, slit_height=25, savefiles=True, obsname=obsname, path=path, timit=True)
pix,flux,err = extract_spectrum_from_indices(img, err_img, stripe_indices, method="quick", slit_height=25, RON=ronmask, savefile=True,
filetype='fits', obsname=obsname, path=path, timit=True)
def make_fibparms_by_fib(savefile=True):
path = '/Users/christoph/OneDrive - UNSW/fibre_profiles/'
fp_files = glob.glob(path + "sim/" + "fibre_profiles*.npy")
#mask_files = glob.glob(path+"masks/"+"mask*.npy")
fibparms = {}
for file in fp_files:
fibnum = file[-6:-4]
fib = 'fibre_' + fibnum
fp = np.load(file).item()
mask = np.load(path + "masks/" + "mask_" + fibnum + ".npy").item()
flatname = '/Volumes/BERGRAID/data/simu/veloce_flat_t70000_single_fib' + fibnum + '.fit'
flat = pyfits.getdata(flatname)
img = flat + 1.
#we need the flux later only to weight the fits
P, tempmask = find_stripes(flat, deg_polynomial=2)
P_id = make_P_id(P)
#mask = make_mask_dict(tempmask)
stripes = extract_stripes(img, P_id, slit_height=10)
fibparms[fib] = {}
for ord in sorted(fp.keys()):
fibparms[fib][ord] = {}
mu = np.array(fp[ord]['mu'])
#amp = np.array(fp[ord]['amp'])
sigma = np.array(fp[ord]['sigma'])
beta = np.array(fp[ord]['beta'])
#offset = np.array(fp[ord]['offset'])
#slope = np.array(fp[ord]['slope'])
onchip = mu >= 0
good = np.logical_and(mu >= 0, mask[ord])
stripe = stripes[ord]
sc, sr = flatten_single_stripe(stripe, slit_height=10, timit=False)
fluxsum = np.sum(sc, axis=0)
w = np.sqrt(
fluxsum
) #use relative error so that the order centres receive the largest weight-contribution
xx = np.arange(len(mu))
mu_fit = np.poly1d(np.polyfit(xx[good], mu[good], 5, w=w[good]))
# xarr = xx*4*np.pi/np.max(xx) - 2*np.pi
# amp_popt,amp_pcov = curve_fit(blaze,xarr,amp,sigma=w,p0=(0.2,np.max(amp),0.))
sigma_fit = np.poly1d(
np.polyfit(xx[good], sigma[good], 2, w=w[good]))
beta_fit = np.poly1d(np.polyfit(xx[good], beta[good], 2,
w=w[good]))
#offset_fit = np.poly1d(np.polyfit(xx[good], offset[good], 0, w=w[good]))
#offset_fit = np.average(offset, weights=w)
fibparms[fib][ord]['mu_fit'] = mu_fit
fibparms[fib][ord]['sigma_fit'] = sigma_fit
fibparms[fib][ord]['beta_fit'] = beta_fit
#fibparms[fib][ord]['offset_fit'] = offset_fit
fibparms[fib][ord]['onchip'] = onchip
if savefile:
np.save(
'/Users/christoph/OneDrive - UNSW/fibre_profiles/sim/fibparms_by_fib.npy',
fibparms)
return fibparms
norm=True)
#####################################################################################################################################################
# (3) order tracing #################################################################################################################################
# find orders roughly
P, tempmask = find_stripes(MW,
deg_polynomial=2,
min_peak=0.05,
gauss_filter_sigma=3.,
simu=True)
# assign physical diffraction order numbers (this is only a dummy function for now) to order-fit polynomials and bad-region masks
P_id = make_P_id(P)
mask = make_mask_dict(tempmask)
# extract stripes of user-defined width from the science image, centred on the polynomial fits defined in step (1)
flat_stripes, fs_indices = extract_stripes(MW,
P_id,
return_indices=True,
slit_height=5)
#####################################################################################################################################################
# (4) cosmic ray removal ############################################################################################################################
#####################################################################################################################################################
# (5) fit and remove background #####################################################################################################################
# extract and fit background
bg, bg_mask = extract_background(MW, P_id, return_mask=True, slit_height=10)
bg_coeffs, bg_image = fit_background(bg, deg=3, timit=True, return_full=True)
#####################################################################################################################################################
# (6) fibre-profile modelling #######################################################################################################################
# fit fibre profiles
def process_science_images(imglist, P_id, chipmask, mask=None, stripe_indices=None, quick_indices=None, sampling_size=25, slit_height=32, qsh=23, gain=[1.,1.,1.,1.], MB=None, ronmask=None, MD=None, scalable=False, saveall=False, path=None, ext_method='optimal',
from_indices=True, slope=True, offset=True, fibs='all', date=None, timit=False):
"""
Process all science / calibration lamp images. This includes:
(1) bias and dark subtraction
(2) cosmic ray removal
(3) background extraction and estimation
(4) flat-fielding (ie removal of pixel-to-pixel sensitivity variations)
=============================
(5) extraction of stripes
(6) extraction of 1-dim spectra
(7) get relative intensities of different fibres
(8) wavelength solution
(9) barycentric correction (for stellar observations only)
"""
print('WARNING: I commented out BARCYRORR')
# cont = raw_input('Do you still want to continue?')
cont='y'
assert cont.lower() == 'y', 'You chose to quit!'
if timit:
start_time = time.time()
# sort image list, just in case
imglist.sort()
# get a list with the object names
object_list = [pyfits.getval(file, 'OBJECT').split('+')[0] for file in imglist]
if object_list[0] == 'ARC - ThAr':
obstype = 'ARC'
elif object_list[0].lower() in ["lc", "lc-only", "lfc", "lfc-only", "simlc", "thxe", "thxe-only", "simth", "thxe+lfc", "lfc+thxe", "lc+simthxe", "lc+thxe"]:
obstype = 'simcalib'
else:
obstype = 'stellar'
if obstype in ['stellar', 'ARC']:
# and the indices where the object changes (to figure out which observations belong to one epoch)
changes = np.where(np.array(object_list)[:-1] != np.array(object_list)[1:])[0] + 1 # need the plus one to get the indices of the first occasion of a new object
# list of indices for individual epochs - there's gotta be a smarter way to do this...
all_epoch_list = []
if len(changes) > 0:
all_epoch_list.append(np.arange(0,changes[0]))
for j in range(len(changes) - 1):
all_epoch_list.append(np.arange(changes[j], changes[j+1]))
all_epoch_list.append(np.arange(changes[-1], len(object_list)))
else:
all_epoch_list.append(np.arange(0, len(object_list)))
#####################################
### (1) bias and dark subtraction ###
#####################################
# if INPUT arrays are not given, read them from default files
if path is None:
print('WARNING: output file directory not provided!!!')
print('Using same directory as input file...')
dum = imglist[0].split('/')
path = imglist[0][0: -len(dum[-1])]
if MB is None:
# no need to fix orientation, this is already a processed file [ADU]
# MB = pyfits.getdata(path + 'master_bias.fits')
MB = pyfits.getdata(path + 'median_bias.fits')
if ronmask is None:
# no need to fix orientation, this is already a processed file [e-]
ronmask = pyfits.getdata(path + 'read_noise_mask.fits')
if MD is None:
if scalable:
# no need to fix orientation, this is already a processed file [e-]
MD = pyfits.getdata(path + 'master_dark_scalable.fits', 0)
# err_MD = pyfits.getdata(path + 'master_dark_scalable.fits', 1)
else:
# no need to fix orientation, this is already a processed file [e-]
print('WARNING: scalable KW not properly implemented (stellar_list can have different exposure times...)')
texp = 600.
MD = pyfits.getdata(path + 'master_dark_t' + str(int(np.round(texp, 0))) + '.fits', 0)
# err_MD = pyfits.getdata(path + 'master_dark_t' + str(int(np.round(texp, 0))) + '.fits', 1)
if not from_indices:
ron_stripes = extract_stripes(ronmask, P_id, return_indices=False, slit_height=slit_height, savefiles=False, timit=True)
# loop over all files
for i,filename in enumerate(imglist):
# (0) do some housekeeping with filenames, and check if there are multiple exposures for a given epoch of a star
dum = filename.split('/')
dum2 = dum[-1].split('.')
obsname = dum2[0]
obsnum = int(obsname[-5:])
object = pyfits.getval(filename, 'OBJECT').split('+')[0]
object_indices = np.where(object == np.array(object_list))[0]
texp = pyfits.getval(filename, 'ELAPSED')
# check if this exposure belongs to the same epoch as the previous one
if obstype in ['stellar', 'ARC']:
if i > 0:
if filename in epoch_list:
new_epoch = False
else:
new_epoch = True
# delete existing temp bg files so we don't accidentally load them for a wrong epoch
if os.path.isfile(path + 'temp_bg_lfc.fits'):
os.remove(path + 'temp_bg_lfc.fits')
if os.path.isfile(path + 'temp_bg_thxe.fits'):
os.remove(path + 'temp_bgthxe.fits')
if os.path.isfile(path + 'temp_bg_both.fits'):
os.remove(path + 'temp_bg_both.fits')
if os.path.isfile(path + 'temp_bg_neither.fits'):
os.remove(path + 'temp_bg_neither.fits')
else:
new_epoch = True
# delete existing temp bg files so we don't accidentally load them for a wrong epoch
if os.path.isfile(path + 'temp_bg_lfc.fits'):
os.remove(path + 'temp_bg_lfc.fits')
if os.path.isfile(path + 'temp_bg_thxe.fits'):
os.remove(path + 'temp_bgthxe.fits')
if os.path.isfile(path + 'temp_bg_both.fits'):
os.remove(path + 'temp_bg_both.fits')
if os.path.isfile(path + 'temp_bg_neither.fits'):
os.remove(path + 'temp_bg_neither.fits')
else:
if i == 0:
new_epoch = True
else:
new_epoch = False
print('Extracting ' + obstype + ' spectrum ' + str(i + 1) + '/' + str(len(imglist)) + ': ' + obsname)
if obstype in ['stellar', 'ARC']:
# list of all the observations belonging to this epoch
epoch_ix = [sublist for sublist in all_epoch_list if i in sublist] # different from object_indices, as epoch_ix contains only indices for this particular epoch if there are multiple epochs of a target in a given night
epoch_list = list(np.array(imglist)[epoch_ix])
# make sublists according to the four possible calibration lamp configurations
epoch_sublists = {'lfc':[], 'thxe':[], 'both':[], 'neither':[]}
if int(date) < 20190503:
# look at the actual 2D image (using chipmasks for LFC and simThXe) to determine which calibration lamps fired
for file in epoch_list:
img = correct_for_bias_and_dark_from_filename(file, MB, MD, gain=gain, scalable=scalable, savefile=saveall, path=path)
lc = laser_on(img, chipmask)
thxe = thxe_on(img, chipmask)
if (not lc) and (not thxe):
epoch_sublists['neither'].append(file)
elif (lc) and (thxe):
epoch_sublists['both'].append(file)
else:
if lc:
epoch_sublists['lfc'].append(file)
elif thxe:
epoch_sublists['thxe'].append(file)
# now check the calibration lamp configuration for the main observation in question
img = correct_for_bias_and_dark_from_filename(filename, MB, MD, gain=gain, scalable=scalable,
savefile=saveall, path=path)
lc = laser_on(img, chipmask)
thxe = thxe_on(img, chipmask)
if (not lc) and (not thxe):
lamp_config = 'neither'
elif (lc) and (thxe):
lamp_config = 'both'
else:
if lc:
lamp_config = 'lfc'
elif thxe:
lamp_config = 'thxe'
else:
# since May 2019 the header keywords are correct, so check for LFC / ThXe in header, as that is MUCH faster
for file in epoch_list:
lc = 0
thxe = 0
h = pyfits.getheader(file)
if 'LCNEXP' in h.keys(): # this indicates the latest version of the FITS headers (from May 2019 onwards)
if ('LCEXP' in h.keys()) or ('LCMNEXP' in h.keys()): # this indicates the LFC actually was actually exposed (either automatically or manually)
lc = 1
else: # if not, just go with the OBJECT field
if ('LC' in pyfits.getval(filename, 'OBJECT').split('+')) or ('LFC' in pyfits.getval(filename, 'OBJECT').split('+')):
lc = 1
if h['SIMCALTT'] > 0:
thxe = 1
assert lc+thxe in [0,1,2], 'ERROR: could not establish status of LFC and simultaneous ThXe for ' + obsname + '.fits !!!'
if lc+thxe == 0:
epoch_sublists['neither'].append(file)
elif lc+thxe == 1:
if lc == 1:
epoch_sublists['lfc'].append(file)
else:
epoch_sublists['thxe'].append(file)
elif lc+thxe == 2:
epoch_sublists['both'].append(file)
# now check the calibration lamp configuration for the main observation in question
lc = 0
thxe = 0
h = pyfits.getheader(filename)
if 'LCNEXP' in h.keys(): # this indicates the latest version of the FITS headers (from May 2019 onwards)
if ('LCEXP' in h.keys()) or ('LCMNEXP' in h.keys()): # this indicates the LFC actually was actually exposed (either automatically or manually)
lc = 1
else: # if not latest header version, just go with the OBJECT field
if ('LC' in pyfits.getval(filename, 'OBJECT').split('+')) or ('LFC' in pyfits.getval(filename, 'OBJECT').split('+')):
lc = 1
if h['SIMCALTT'] > 0:
thxe = 1
if lc + thxe == 0:
lamp_config = 'neither'
elif lc + thxe == 1:
if lc == 1:
lamp_config = 'lfc'
else:
lamp_config = 'thxe'
elif lc + thxe == 2:
lamp_config = 'both'
else:
# for sim. calibration images we don't need to check for the calibration lamp configuration for all exposures (done external to this function)!
# just for the file in question and then create a dummy copy of the image list so that it is in the same format that ix expected for stellar
# observations
if int(date) < 20190503:
# now check the calibration lamp configuration for the main observation in question
img = correct_for_bias_and_dark_from_filename(filename, MB, MD, gain=gain, scalable=scalable, savefile=saveall, path=path)
lc = laser_on(img, chipmask)
thxe = thxe_on(img, chipmask)
if (not lc) and (not thxe):
lamp_config = 'neither'
elif (lc) and (thxe):
lamp_config = 'both'
else:
if lc:
lamp_config = 'lfc'
elif thxe:
lamp_config = 'thxe'
else:
# now check the calibration lamp configuration for the main observation in question
lc = 0
thxe = 0
h = pyfits.getheader(filename)
if 'LCNEXP' in h.keys(): # this indicates the latest version of the FITS headers (from May 2019 onwards)
if ('LCEXP' in h.keys()) or (
'LCMNEXP' in h.keys()): # this indicates the LFC actually was actually exposed (either automatically or manually)
lc = 1
else: # if not latest header version, just go with the OBJECT field
if ('LC' in pyfits.getval(filename, 'OBJECT').split('+')) or (
'LFC' in pyfits.getval(filename, 'OBJECT').split('+')):
lc = 1
if h['SIMCALTT'] > 0:
thxe = 1
if lc + thxe == 0:
lamp_config = 'neither'
elif lc + thxe == 1:
if lc == 1:
lamp_config = 'lfc'
else:
lamp_config = 'thxe'
elif lc + thxe == 2:
lamp_config = 'both'
epoch_sublists = {}
epoch_sublists[lamp_config] = imglist[:]
# (1) call routine that does all the overscan-, bias- & dark-correction stuff and proper error treatment
img = correct_for_bias_and_dark_from_filename(filename, MB, MD, gain=gain, scalable=scalable, savefile=saveall, path=path) # [e-]
#err = np.sqrt(img + ronmask*ronmask) # [e-]
#TEMPFIX: (how should I be doing this properly???)
err_img = np.sqrt(np.clip(img,0,None) + ronmask*ronmask) # [e-]
## (2) remove cosmic rays (ERRORS MUST REMAIN UNCHANGED)
## check if there are multiple exposures for this epoch (if yes, we can do the much simpler "median_remove_cosmics")
if len(epoch_sublists[lamp_config]) == 1:
# do it the hard way using LACosmic
# identify and extract background
bg_raw = extract_background(img, chipmask['bg'], timit=timit)
# remove cosmics, but only from background
cosmic_cleaned_img = remove_cosmics(bg_raw.todense(), ronmask, obsname, path, Flim=3.0, siglim=5.0, maxiter=1, savemask=False, savefile=False, save_err=False, verbose=True, timit=True) # [e-]
# identify and extract background from cosmic-cleaned image
bg = extract_background(cosmic_cleaned_img, chipmask['bg'], timit=timit)
# bg = extract_background_pid(cosmic_cleaned_img, P_id, slit_height=30, exclude_top_and_bottom=True, timit=timit)
# fit background
bg_coeffs, bg_img = fit_background(bg, clip=10, return_full=True, timit=timit)
elif len(epoch_sublists[lamp_config]) == 2:
if new_epoch or not os.path.isfile(path + 'temp_bg_' + lamp_config + '.fits'):
# list of individual exposure times for this epoch
subepoch_texp_list = [pyfits.getval(file, 'ELAPSED') for file in epoch_sublists[lamp_config]]
tscale = np.array(subepoch_texp_list) / texp
# get background from the element-wise minimum-image of the two images
img1 = correct_for_bias_and_dark_from_filename(epoch_sublists[lamp_config][0], MB, MD, gain=gain, scalable=scalable, savefile=False)
img2 = correct_for_bias_and_dark_from_filename(epoch_sublists[lamp_config][1], MB, MD, gain=gain, scalable=scalable, savefile=False)
min_img = np.minimum(img1/tscale[0], img2/tscale[1])
# identify and extract background from the minimum-image
bg = extract_background(min_img, chipmask['bg'], timit=timit)
# bg = extract_background_pid(min_img, P_id, slit_height=30, exclude_top_and_bottom=True, timit=timit)
del min_img
# fit background
bg_coeffs, bg_img = fit_background(bg, clip=10, return_full=True, timit=timit)
# save background image to temporary file for re-use later (when reducing the next file of this sublist)
pyfits.writeto(path + 'temp_bg_' + lamp_config + '.fits', bg_img, clobber=True)
else:
# no need to re-compute background, just load it from file
print('Loading background image for this epoch and lamp configuration...')
bg_img = pyfits.getdata(path + 'temp_bg_' + lamp_config + '.fits')
else:
if new_epoch or not os.path.isfile(path + 'temp_bg_' + lamp_config + '.fits'):
# make sure this sublist is not too long (otherwise we might run out of memory in this step)
if len(epoch_sublists[lamp_config]) > 10:
mainix = epoch_sublists[lamp_config].index(filename)
if mainix < 5:
epoch_sublists[lamp_config] = epoch_sublists[lamp_config][:11]
elif mainix > len(epoch_sublists[lamp_config]) - 6:
epoch_sublists[lamp_config] = epoch_sublists[lamp_config][-11:]
else:
epoch_sublists[lamp_config] = epoch_sublists[lamp_config][mainix-5:mainix+6]
# list of individual exposure times for this epoch
subepoch_texp_list = [pyfits.getval(file, 'ELAPSED') for file in epoch_sublists[lamp_config]]
tscale = np.array(subepoch_texp_list) / texp
# make list of actual images
img_list = []
for file in epoch_sublists[lamp_config]:
img_list.append(correct_for_bias_and_dark_from_filename(file, MB, MD, gain=gain, scalable=scalable, savefile=False))
# # index indicating which one of the files in the epoch list is the "main" one
# main_index = np.where(np.array(epoch_ix) == i)[0][0]
# take median after scaling to same exposure time as main exposure
med_img = np.median(np.array(img_list) / tscale.reshape(len(img_list), 1, 1), axis=0)
del img_list
# identify and extract background from the median image
bg = extract_background(med_img, chipmask['bg'], timit=timit)
# bg = extract_background_pid(med_img, P_id, slit_height=30, exclude_top_and_bottom=True, timit=timit)
del med_img
# fit background
bg_coeffs, bg_img = fit_background(bg, clip=10, return_full=True, timit=timit)
# save background image to temporary file for re-use later (when reducing the next file of this sublist)
pyfits.writeto(path + 'temp_bg_' + lamp_config + '.fits', bg_img, clobber=True)
else:
# no need to re-compute background, just load it from file
print('Loading background image for this epoch and lamp configuration...')
bg_img = pyfits.getdata(path + 'temp_bg_' + lamp_config + '.fits')
# now actually subtract the background model
bg_corrected_img = img - bg_img
# cosmic_cleaned_img = median_remove_cosmics(img_list, main_index=main_index, scales=scaled_texp, ronmask=ronmask, debug_level=1, timit=True)
# (3) fit and remove background (ERRORS REMAIN UNCHANGED)
# bg_corrected_img = remove_background(cosmic_cleaned_img, P_id, obsname, path, degpol=5, slit_height=slit_height, save_bg=True, savefile=True, save_err=False,
# exclude_top_and_bottom=True, verbose=True, timit=True) # [e-]
# bg_corrected_img = remove_background(img, P_id, obsname, path, degpol=5, slit_height=slit_height, save_bg=False, savefile=True, save_err=False,
# exclude_top_and_bottom=True, verbose=True, timit=True) # [e-]
# adjust errors?
# (4) remove pixel-to-pixel sensitivity variations (2-dim)
#XXXXXXXXXXXXXXXXXXXXXXXXXXX
#TEMPFIX
final_img = bg_corrected_img.copy() # [e-]
# final_img = img.copy() # [e-]
#adjust errors?
# (5) extract stripes
if not from_indices:
stripes,stripe_indices = extract_stripes(final_img, P_id, return_indices=True, slit_height=slit_height, savefiles=saveall, obsname=obsname, path=path, timit=True)
err_stripes = extract_stripes(err_img, P_id, return_indices=False, slit_height=slit_height, savefiles=saveall, obsname=obsname+'_err', path=path, timit=True)
if stripe_indices is None:
# this is just to get the stripe indices in case we forgot to provide them (DONE ONLY ONCE, if at all...)
stripes,stripe_indices = extract_stripes(final_img, P_id, return_indices=True, slit_height=slit_height, savefiles=False, obsname=obsname, path=path, timit=True)
# (6) perform extraction of 1-dim spectrum
if from_indices:
pix,flux,err = extract_spectrum_from_indices(final_img, err_img, quick_indices, method='quick', slit_height=qsh, ronmask=ronmask, savefile=True,
filetype='fits', obsname=obsname, date=date, path=path, timit=True)
pix,flux,err = extract_spectrum_from_indices(final_img, err_img, stripe_indices, method=ext_method, slope=slope, offset=offset, fibs=fibs, slit_height=slit_height,
ronmask=ronmask, savefile=True, filetype='fits', obsname=obsname, date=date, path=path, timit=True)
else:
pix,flux,err = extract_spectrum(stripes, err_stripes=err_stripes, ron_stripes=ron_stripes, method='quick', slit_height=qsh, ronmask=ronmask, savefile=True,
filetype='fits', obsname=obsname, date=date, path=path, timit=True)
pix,flux,err = extract_spectrum(stripes, err_stripes=err_stripes, ron_stripes=ron_stripes, method=ext_method, slope=slope, offset=offset, fibs=fibs,
slit_height=slit_height, ronmask=ronmask, savefile=True, filetype='fits', obsname=obsname, date=date, path=path, timit=True)
# # (7) get relative intensities of different fibres
# if from_indices:
# relints = get_relints_from_indices(P_id, final_img, err_img, stripe_indices, mask=mask, sampling_size=sampling_size, slit_height=slit_height, return_full=False, timit=True)
# else:
# relints = get_relints(P_id, stripes, err_stripes, mask=mask, sampling_size=sampling_size, slit_height=slit_height, return_full=False, timit=True)
#
#
# # (8) get wavelength solution
# #XXXXX
# # (9) get barycentric correction
# if obstype == 'stellar':
# bc = get_barycentric_correction(filename)
# bc = np.round(bc,2)
# if np.isnan(bc):
# bc = ''
# # write the barycentric correction into the FITS header of both the quick-extracted and the optimal-extracted reduced spectrum files
# outfn_list = glob.glob(path + '*' + obsname + '*extracted*')
# for outfn in outfn_list:
# pyfits.setval(outfn, 'BARYCORR', value=bc, comment='barycentric velocity correction [m/s]')
# #now append relints, wl-solution, and barycorr to extracted FITS file header
# outfn = path + obsname + '_extracted.fits'
# if os.path.isfile(outfn):
# #relative fibre intensities
# dum = append_relints_to_FITS(relints, outfn, nfib=19)
# #wavelength solution
# #pyfits.setval(fn, 'RELINT' + str(i + 1).zfill(2), value=relints[i], comment='fibre #' + str(fibnums[i]) + ' - ' + fibinfo[i] + ' fibre')
if timit:
print('Total time elapsed: '+str(np.round(time.time() - start_time,1))+' seconds')
return
def process_science_images(imglist,
P_id,
mask=None,
sampling_size=25,
slit_height=25,
gain=[1., 1., 1., 1.],
MB=None,
ronmask=None,
MD=None,
scalable=False,
saveall=False,
path=None,
ext_method='optimal',
from_indices=True,
timit=False):
"""
Process all science images. This includes:
(1) bias and dark subtraction
(2) cosmic ray removal
(3) background extraction and estimation
(4) flat-fielding (ie removal of pixel-to-pixel sensitivity variations)
=============================
(5) extraction of stripes
(6) extraction of 1-dim spectra
(7) get relative intensities of different fibres
(8) wavelength solution
(9) barycentric correction
"""
if timit:
start_time = time.time()
#####################################
### (1) bias and dark subtraction ###
#####################################
#if the darks have a different exposure time than the science images, then we need to re-scale the master dark
texp = pyfits.getval(imglist[0], 'exptime')
#if INPUT arrays are not given, read them from default files
if path is None:
print('WARNING: output file directory not provided!!!')
print('Using same directory as input file...')
dum = imglist[0].split('/')
path = imglist[0][0:-len(dum[-1])]
if MB is None:
#no need to fix orientation, this is already a processed file [ADU]
MB = pyfits.getdata(path + 'master_bias.fits')
if ronmask is None:
#no need to fix orientation, this is already a processed file [e-]
ronmask = pyfits.getdata(path + 'read_noise_mask.fits')
if MD is None:
if scalable:
#no need to fix orientation, this is already a processed file [e-]
MD = pyfits.getdata(path + 'master_dark_scalable.fits', 0)
# err_MD = pyfits.getdata(path+'master_dark_scalable.fits', 1)
else:
#no need to fix orientation, this is already a processed file [e-]
MD = pyfits.getdata(
path + 'master_dark_t' + str(int(np.round(texp, 0))) + '.fits',
0)
# err_MD = pyfits.getdata(path+'master_dark_t'+str(int(np.round(texp,0)))+'.fits', 1)
if not from_indices:
ron_stripes = extract_stripes(ronmask,
P_id,
return_indices=False,
slit_height=slit_height,
savefiles=False,
timit=True)
for filename in imglist:
#do some housekeeping with filenames
dum = filename.split('/')
dum2 = dum[-1].split('.')
obsname = dum2[0]
# (1) call routine that does all the bias and dark correction stuff and proper error treatment
img = correct_for_bias_and_dark_from_filename(filename,
MB,
MD,
gain=gain,
scalable=False,
savefile=saveall,
path=path,
timit=True) #[e-]
#err = np.sqrt(img + ronmask*ronmask) # [e-]
#TEMPFIX:
err_img = np.sqrt(np.clip(img, 0, None) + ronmask * ronmask) # [e-]
# (2) remove cosmic rays (ERRORS REMAIN UNCHANGED)
cosmic_cleaned_img = remove_cosmics(img,
ronmask,
obsname,
path,
Flim=3.0,
siglim=5.0,
maxiter=1,
savemask=True,
savefile=True,
save_err=False,
verbose=True,
timit=True) # [e-]
#adjust errors?
# (3) fit and remove background (ERRORS REMAIN UNCHANGED)
bg_corrected_img = remove_background(cosmic_cleaned_img,
P_id,
obsname,
path,
degpol=5,
slit_height=slit_height,
save_bg=True,
savefile=True,
save_err=False,
exclude_top_and_bottom=True,
verbose=True,
timit=True) # [e-]
#adjust errors?
# (4) remove pixel-to-pixel sensitivity variations (2-dim)
#XXXXXXXXXXXXXXXXXXXXXXXXXXX
#TEMPFIX
final_img = bg_corrected_img.copy() # [e-]
#adjust errors?
# (5) extract stripes
stripes, stripe_indices = extract_stripes(final_img,
P_id,
return_indices=True,
slit_height=slit_height,
savefiles=True,
obsname=obsname,
path=path,
timit=True)
if not from_indices:
err_stripes = extract_stripes(err_img,
P_id,
return_indices=False,
slit_height=slit_height,
savefiles=True,
obsname=obsname + '_err',
path=path,
timit=True)
# (6) perform extraction of 1-dim spectrum
if from_indices:
pix, flux, err = extract_spectrum_from_indices(
final_img,
err_img,
stripe_indices,
method=ext_method,
slit_height=slit_height,
RON=ronmask,
savefile=True,
filetype='fits',
obsname=obsname,
path=path,
timit=True)
else:
pix2, flux2, err2 = extract_spectrum(stripes,
err_stripes=err_stripes,
ron_stripes=ron_stripes,
method=ext_method,
slit_height=slit_height,
RON=ronmask,
savefile=False,
filetype='fits',
obsname=obsname,
path=path,
timit=True)
# (7) get relative intensities of different fibres
if from_indices:
relints = get_relints_from_indices(P_id,
final_img,
err_img,
stripe_indices,
mask=mask,
sampling_size=sampling_size,
slit_height=slit_height,
return_full=False,
timit=True)
else:
relints = get_relints(P_id,
stripes,
err_stripes,
mask=mask,
sampling_size=sampling_size,
slit_height=slit_height,
return_full=False,
timit=True)
# (8) get wavelength solution
#XXXXX
# (9) get barycentric correction
lat, long, alt = get_obs_coords_from_header(fn)
bc = barycorrpy.get_BC_vel(JDUTC=JDUTC,
hip_id=8102,
lat=lat,
longi=long,
alt=float(alt),
ephemeris='de430',
zmeas=0.0)
#bc = barycorrpy.get_BC_vel(JDUTC=JDUTC, hip_id=8102, lat=-31.2755, longi=149.0673, alt=1165.0, ephemeris='de430', zmeas=0.0)
#bc = barycorrpy.get_BC_vel(JDUTC=JDUTC, hip_id=8102, obsname='AAO', ephemeris='de430')
#now append relints, wl-solution, and barycorr to extracted FITS file header
outfn = path + obsname + '_extracted.fits'
if os.path.isfile(outfn):
#relative fibre intensities
dum = append_relints_to_FITS(relints, outfn, nfib=19)
#wavelength solution
#pyfits.setval(fn, 'RELINT' + str(i + 1).zfill(2), value=relints[i], comment='fibre #' + str(fibnums[i]) + ' - ' + fibinfo[i] + ' fibre')
#barycentric correction
pyfits.setval(outfn,
'BARYCORR',
value=np.array(bc[0])[0],
comment='barycentric correction [m/s]')
if timit:
print('Total time elapsed: ' +
str(np.round(time.time() - start_time, 1)) + ' seconds')
return