timit=False) ##################################################################################################################################################### # (3) ORDER TRACING ################################################################################################################################# # find orders roughly P, tempmask = find_stripes(MW, deg_polynomial=2, min_peak=0.05, gauss_filter_sigma=3., simu=False) # assign physical diffraction order numbers (this is only a dummy function for now) to order-fit polynomials and bad-region masks P_id = make_P_id(P) mask = make_mask_dict(tempmask) # extract stripes of user-defined width from the science image, centred on the polynomial fits defined in step (1) MW_stripes, MW_indices = extract_stripes(MW, P_id, return_indices=True, slit_height=5) ##################################################################################################################################################### ### #if we want to determine spatial profiles, then we should remove cosmics and background from MW like so: # cosmic_cleaned_MW = remove_cosmics(MW, ronmask, obsname, path, Flim=3.0, siglim=5.0, maxiter=1, savemask=False, savefile=False, save_err=False, verbose=True, timit=True) # bg_corrected_MW = remove_background(cosmic_cleaned_MW, P_id, obsname, path, degpol=5, slit_height=5, save_bg=False, savefile=False, save_err=False, exclude_top_and_bottom=True, verbose=True, timit=True) #before doing the following: MW_stripes, MW_stripe_indices = extract_stripes(MW, P_id, return_indices=True, slit_height=30) err_MW_stripes = extract_stripes(err_MW, P_id,
for o in P_id.keys(): P_id[o][0] -= 2. np.save(path + 'P_id.npy', P_id) np.save(path + 'mask.npy', mask) # now redo the master white properly, incl background removal, and save to file if choice_mw.lower() == 'r': MW,err_MW = process_whites(flat_list, MB=medbias, ronmask=ronmask, MD=MDS, gain=gain, scalable=True, fancy=False, P_id=P_id, clip=5., savefile=True, saveall=False, diffimg=False, remove_bg=True, path=path, debug_level=1, timit=False) ##################################################################################################################################################### ### (4) CREATE FIBRE PROFILES ####################################################################################################################### slit_height = 30 MW_stripes,MW_indices = extract_stripes(MW, P_id, return_indices=True, slit_height=slit_height) del MW_stripes # save memory # err_MW_stripes = extract_stripes(err_MW, P_id, return_indices=False, slit_height=slit_height) choice_fp = 'r' if os.path.isfile(fibparms_path + 'combined_fibre_profile_fits_' + date + '.npy'): choice_fp = raw_input("FIBRE PROFILES for " + date + " already exists! Do you want to skip this step or recreate it? ['s' / 'r']") if choice_fp.lower() == 's': print('Loading fibre profiles for ' + date + '...') fibparms = np.load(fibparms_path + 'combined_fibre_profile_fits_' + date + '.npy').item() else: sure = 'x' while sure.lower() not in ['y','n']: sure = raw_input("Are you sure you want to create fibre profiles for " + date + "??? This may take several hours!!! ['y' / 'n']") if sure == 'y': fp_in = fit_multiple_profiles_from_indices(P_id, MW, err_MW, MW_indices, slit_height=slit_height, timit=True, debug_level=1)
ny=4096 medbias,coeffs,offsets,rons = get_bias_and_readnoise_from_bias_frames(corrected_bias_list, degpol=5, clip=5., gain=gain, debug_level=0, timit=True) # create MASTER BIAS frame and read-out noise mask (units = electrons) offmask,ronmask = make_offmask_and_ronmask(offsets, rons, nx, ny, gain=gain, savefiles=True, path=path, timit=True) MB = make_master_bias_from_coeffs(coeffs, nx, ny, savefile=True, path=path, timit=True) #we did not have darks, so I did this MD = np.zeros(MB.shape) #create (bias- & dark-subtracted) MASTER WHITE frame and corresponding error array (units = ADUs) MW,err_MW = process_whites(white_list, corrected_white_list, MB=MB, ronmask=ronmask, MD=MD, gain=gain, scalable=False, fancy=False, clip=5., savefile=True, saveall=True, diffimg=False, path=path, timit=False) # find orders P,tempmask = find_stripes(MW, deg_polynomial=2, min_peak=0.05, gauss_filter_sigma=3., simu=False) # assign physical diffraction order numbers (this is only a dummy function for now) to order-fit polynomials and bad-region masks P_id = make_P_id(P) mask = make_mask_dict(tempmask) #loop over all files you want to extract a 1-dim spectrum for for filename in corrected_stellar_list: #do some housekeeping with filenames dum = filename.split('/') dum2 = dum[-1].split('.') obsname = dum2[0] # (1) call routine that does all the bias and dark correction stuff and proper error treatment img = correct_for_bias_and_dark_from_filename(filename, MB, MD, gain=gain, scalable=False, savefile=True, path=path, timit=True) #[e-] err_img = np.sqrt(np.clip(img,0,None) + ronmask*ronmask) # [e-] # (5) extract stripes stripes,stripe_indices = extract_stripes(img, P_id, return_indices=True, slit_height=25, savefiles=True, obsname=obsname, path=path, timit=True) pix,flux,err = extract_spectrum_from_indices(img, err_img, stripe_indices, method="quick", slit_height=25, RON=ronmask, savefile=True, filetype='fits', obsname=obsname, path=path, timit=True)
def make_fibparms_by_fib(savefile=True): path = '/Users/christoph/OneDrive - UNSW/fibre_profiles/' fp_files = glob.glob(path + "sim/" + "fibre_profiles*.npy") #mask_files = glob.glob(path+"masks/"+"mask*.npy") fibparms = {} for file in fp_files: fibnum = file[-6:-4] fib = 'fibre_' + fibnum fp = np.load(file).item() mask = np.load(path + "masks/" + "mask_" + fibnum + ".npy").item() flatname = '/Volumes/BERGRAID/data/simu/veloce_flat_t70000_single_fib' + fibnum + '.fit' flat = pyfits.getdata(flatname) img = flat + 1. #we need the flux later only to weight the fits P, tempmask = find_stripes(flat, deg_polynomial=2) P_id = make_P_id(P) #mask = make_mask_dict(tempmask) stripes = extract_stripes(img, P_id, slit_height=10) fibparms[fib] = {} for ord in sorted(fp.keys()): fibparms[fib][ord] = {} mu = np.array(fp[ord]['mu']) #amp = np.array(fp[ord]['amp']) sigma = np.array(fp[ord]['sigma']) beta = np.array(fp[ord]['beta']) #offset = np.array(fp[ord]['offset']) #slope = np.array(fp[ord]['slope']) onchip = mu >= 0 good = np.logical_and(mu >= 0, mask[ord]) stripe = stripes[ord] sc, sr = flatten_single_stripe(stripe, slit_height=10, timit=False) fluxsum = np.sum(sc, axis=0) w = np.sqrt( fluxsum ) #use relative error so that the order centres receive the largest weight-contribution xx = np.arange(len(mu)) mu_fit = np.poly1d(np.polyfit(xx[good], mu[good], 5, w=w[good])) # xarr = xx*4*np.pi/np.max(xx) - 2*np.pi # amp_popt,amp_pcov = curve_fit(blaze,xarr,amp,sigma=w,p0=(0.2,np.max(amp),0.)) sigma_fit = np.poly1d( np.polyfit(xx[good], sigma[good], 2, w=w[good])) beta_fit = np.poly1d(np.polyfit(xx[good], beta[good], 2, w=w[good])) #offset_fit = np.poly1d(np.polyfit(xx[good], offset[good], 0, w=w[good])) #offset_fit = np.average(offset, weights=w) fibparms[fib][ord]['mu_fit'] = mu_fit fibparms[fib][ord]['sigma_fit'] = sigma_fit fibparms[fib][ord]['beta_fit'] = beta_fit #fibparms[fib][ord]['offset_fit'] = offset_fit fibparms[fib][ord]['onchip'] = onchip if savefile: np.save( '/Users/christoph/OneDrive - UNSW/fibre_profiles/sim/fibparms_by_fib.npy', fibparms) return fibparms
norm=True) ##################################################################################################################################################### # (3) order tracing ################################################################################################################################# # find orders roughly P, tempmask = find_stripes(MW, deg_polynomial=2, min_peak=0.05, gauss_filter_sigma=3., simu=True) # assign physical diffraction order numbers (this is only a dummy function for now) to order-fit polynomials and bad-region masks P_id = make_P_id(P) mask = make_mask_dict(tempmask) # extract stripes of user-defined width from the science image, centred on the polynomial fits defined in step (1) flat_stripes, fs_indices = extract_stripes(MW, P_id, return_indices=True, slit_height=5) ##################################################################################################################################################### # (4) cosmic ray removal ############################################################################################################################ ##################################################################################################################################################### # (5) fit and remove background ##################################################################################################################### # extract and fit background bg, bg_mask = extract_background(MW, P_id, return_mask=True, slit_height=10) bg_coeffs, bg_image = fit_background(bg, deg=3, timit=True, return_full=True) ##################################################################################################################################################### # (6) fibre-profile modelling ####################################################################################################################### # fit fibre profiles
def process_science_images(imglist, P_id, chipmask, mask=None, stripe_indices=None, quick_indices=None, sampling_size=25, slit_height=32, qsh=23, gain=[1.,1.,1.,1.], MB=None, ronmask=None, MD=None, scalable=False, saveall=False, path=None, ext_method='optimal', from_indices=True, slope=True, offset=True, fibs='all', date=None, timit=False): """ Process all science / calibration lamp images. This includes: (1) bias and dark subtraction (2) cosmic ray removal (3) background extraction and estimation (4) flat-fielding (ie removal of pixel-to-pixel sensitivity variations) ============================= (5) extraction of stripes (6) extraction of 1-dim spectra (7) get relative intensities of different fibres (8) wavelength solution (9) barycentric correction (for stellar observations only) """ print('WARNING: I commented out BARCYRORR') # cont = raw_input('Do you still want to continue?') cont='y' assert cont.lower() == 'y', 'You chose to quit!' if timit: start_time = time.time() # sort image list, just in case imglist.sort() # get a list with the object names object_list = [pyfits.getval(file, 'OBJECT').split('+')[0] for file in imglist] if object_list[0] == 'ARC - ThAr': obstype = 'ARC' elif object_list[0].lower() in ["lc", "lc-only", "lfc", "lfc-only", "simlc", "thxe", "thxe-only", "simth", "thxe+lfc", "lfc+thxe", "lc+simthxe", "lc+thxe"]: obstype = 'simcalib' else: obstype = 'stellar' if obstype in ['stellar', 'ARC']: # and the indices where the object changes (to figure out which observations belong to one epoch) changes = np.where(np.array(object_list)[:-1] != np.array(object_list)[1:])[0] + 1 # need the plus one to get the indices of the first occasion of a new object # list of indices for individual epochs - there's gotta be a smarter way to do this... all_epoch_list = [] if len(changes) > 0: all_epoch_list.append(np.arange(0,changes[0])) for j in range(len(changes) - 1): all_epoch_list.append(np.arange(changes[j], changes[j+1])) all_epoch_list.append(np.arange(changes[-1], len(object_list))) else: all_epoch_list.append(np.arange(0, len(object_list))) ##################################### ### (1) bias and dark subtraction ### ##################################### # if INPUT arrays are not given, read them from default files if path is None: print('WARNING: output file directory not provided!!!') print('Using same directory as input file...') dum = imglist[0].split('/') path = imglist[0][0: -len(dum[-1])] if MB is None: # no need to fix orientation, this is already a processed file [ADU] # MB = pyfits.getdata(path + 'master_bias.fits') MB = pyfits.getdata(path + 'median_bias.fits') if ronmask is None: # no need to fix orientation, this is already a processed file [e-] ronmask = pyfits.getdata(path + 'read_noise_mask.fits') if MD is None: if scalable: # no need to fix orientation, this is already a processed file [e-] MD = pyfits.getdata(path + 'master_dark_scalable.fits', 0) # err_MD = pyfits.getdata(path + 'master_dark_scalable.fits', 1) else: # no need to fix orientation, this is already a processed file [e-] print('WARNING: scalable KW not properly implemented (stellar_list can have different exposure times...)') texp = 600. MD = pyfits.getdata(path + 'master_dark_t' + str(int(np.round(texp, 0))) + '.fits', 0) # err_MD = pyfits.getdata(path + 'master_dark_t' + str(int(np.round(texp, 0))) + '.fits', 1) if not from_indices: ron_stripes = extract_stripes(ronmask, P_id, return_indices=False, slit_height=slit_height, savefiles=False, timit=True) # loop over all files for i,filename in enumerate(imglist): # (0) do some housekeeping with filenames, and check if there are multiple exposures for a given epoch of a star dum = filename.split('/') dum2 = dum[-1].split('.') obsname = dum2[0] obsnum = int(obsname[-5:]) object = pyfits.getval(filename, 'OBJECT').split('+')[0] object_indices = np.where(object == np.array(object_list))[0] texp = pyfits.getval(filename, 'ELAPSED') # check if this exposure belongs to the same epoch as the previous one if obstype in ['stellar', 'ARC']: if i > 0: if filename in epoch_list: new_epoch = False else: new_epoch = True # delete existing temp bg files so we don't accidentally load them for a wrong epoch if os.path.isfile(path + 'temp_bg_lfc.fits'): os.remove(path + 'temp_bg_lfc.fits') if os.path.isfile(path + 'temp_bg_thxe.fits'): os.remove(path + 'temp_bgthxe.fits') if os.path.isfile(path + 'temp_bg_both.fits'): os.remove(path + 'temp_bg_both.fits') if os.path.isfile(path + 'temp_bg_neither.fits'): os.remove(path + 'temp_bg_neither.fits') else: new_epoch = True # delete existing temp bg files so we don't accidentally load them for a wrong epoch if os.path.isfile(path + 'temp_bg_lfc.fits'): os.remove(path + 'temp_bg_lfc.fits') if os.path.isfile(path + 'temp_bg_thxe.fits'): os.remove(path + 'temp_bgthxe.fits') if os.path.isfile(path + 'temp_bg_both.fits'): os.remove(path + 'temp_bg_both.fits') if os.path.isfile(path + 'temp_bg_neither.fits'): os.remove(path + 'temp_bg_neither.fits') else: if i == 0: new_epoch = True else: new_epoch = False print('Extracting ' + obstype + ' spectrum ' + str(i + 1) + '/' + str(len(imglist)) + ': ' + obsname) if obstype in ['stellar', 'ARC']: # list of all the observations belonging to this epoch epoch_ix = [sublist for sublist in all_epoch_list if i in sublist] # different from object_indices, as epoch_ix contains only indices for this particular epoch if there are multiple epochs of a target in a given night epoch_list = list(np.array(imglist)[epoch_ix]) # make sublists according to the four possible calibration lamp configurations epoch_sublists = {'lfc':[], 'thxe':[], 'both':[], 'neither':[]} if int(date) < 20190503: # look at the actual 2D image (using chipmasks for LFC and simThXe) to determine which calibration lamps fired for file in epoch_list: img = correct_for_bias_and_dark_from_filename(file, MB, MD, gain=gain, scalable=scalable, savefile=saveall, path=path) lc = laser_on(img, chipmask) thxe = thxe_on(img, chipmask) if (not lc) and (not thxe): epoch_sublists['neither'].append(file) elif (lc) and (thxe): epoch_sublists['both'].append(file) else: if lc: epoch_sublists['lfc'].append(file) elif thxe: epoch_sublists['thxe'].append(file) # now check the calibration lamp configuration for the main observation in question img = correct_for_bias_and_dark_from_filename(filename, MB, MD, gain=gain, scalable=scalable, savefile=saveall, path=path) lc = laser_on(img, chipmask) thxe = thxe_on(img, chipmask) if (not lc) and (not thxe): lamp_config = 'neither' elif (lc) and (thxe): lamp_config = 'both' else: if lc: lamp_config = 'lfc' elif thxe: lamp_config = 'thxe' else: # since May 2019 the header keywords are correct, so check for LFC / ThXe in header, as that is MUCH faster for file in epoch_list: lc = 0 thxe = 0 h = pyfits.getheader(file) if 'LCNEXP' in h.keys(): # this indicates the latest version of the FITS headers (from May 2019 onwards) if ('LCEXP' in h.keys()) or ('LCMNEXP' in h.keys()): # this indicates the LFC actually was actually exposed (either automatically or manually) lc = 1 else: # if not, just go with the OBJECT field if ('LC' in pyfits.getval(filename, 'OBJECT').split('+')) or ('LFC' in pyfits.getval(filename, 'OBJECT').split('+')): lc = 1 if h['SIMCALTT'] > 0: thxe = 1 assert lc+thxe in [0,1,2], 'ERROR: could not establish status of LFC and simultaneous ThXe for ' + obsname + '.fits !!!' if lc+thxe == 0: epoch_sublists['neither'].append(file) elif lc+thxe == 1: if lc == 1: epoch_sublists['lfc'].append(file) else: epoch_sublists['thxe'].append(file) elif lc+thxe == 2: epoch_sublists['both'].append(file) # now check the calibration lamp configuration for the main observation in question lc = 0 thxe = 0 h = pyfits.getheader(filename) if 'LCNEXP' in h.keys(): # this indicates the latest version of the FITS headers (from May 2019 onwards) if ('LCEXP' in h.keys()) or ('LCMNEXP' in h.keys()): # this indicates the LFC actually was actually exposed (either automatically or manually) lc = 1 else: # if not latest header version, just go with the OBJECT field if ('LC' in pyfits.getval(filename, 'OBJECT').split('+')) or ('LFC' in pyfits.getval(filename, 'OBJECT').split('+')): lc = 1 if h['SIMCALTT'] > 0: thxe = 1 if lc + thxe == 0: lamp_config = 'neither' elif lc + thxe == 1: if lc == 1: lamp_config = 'lfc' else: lamp_config = 'thxe' elif lc + thxe == 2: lamp_config = 'both' else: # for sim. calibration images we don't need to check for the calibration lamp configuration for all exposures (done external to this function)! # just for the file in question and then create a dummy copy of the image list so that it is in the same format that ix expected for stellar # observations if int(date) < 20190503: # now check the calibration lamp configuration for the main observation in question img = correct_for_bias_and_dark_from_filename(filename, MB, MD, gain=gain, scalable=scalable, savefile=saveall, path=path) lc = laser_on(img, chipmask) thxe = thxe_on(img, chipmask) if (not lc) and (not thxe): lamp_config = 'neither' elif (lc) and (thxe): lamp_config = 'both' else: if lc: lamp_config = 'lfc' elif thxe: lamp_config = 'thxe' else: # now check the calibration lamp configuration for the main observation in question lc = 0 thxe = 0 h = pyfits.getheader(filename) if 'LCNEXP' in h.keys(): # this indicates the latest version of the FITS headers (from May 2019 onwards) if ('LCEXP' in h.keys()) or ( 'LCMNEXP' in h.keys()): # this indicates the LFC actually was actually exposed (either automatically or manually) lc = 1 else: # if not latest header version, just go with the OBJECT field if ('LC' in pyfits.getval(filename, 'OBJECT').split('+')) or ( 'LFC' in pyfits.getval(filename, 'OBJECT').split('+')): lc = 1 if h['SIMCALTT'] > 0: thxe = 1 if lc + thxe == 0: lamp_config = 'neither' elif lc + thxe == 1: if lc == 1: lamp_config = 'lfc' else: lamp_config = 'thxe' elif lc + thxe == 2: lamp_config = 'both' epoch_sublists = {} epoch_sublists[lamp_config] = imglist[:] # (1) call routine that does all the overscan-, bias- & dark-correction stuff and proper error treatment img = correct_for_bias_and_dark_from_filename(filename, MB, MD, gain=gain, scalable=scalable, savefile=saveall, path=path) # [e-] #err = np.sqrt(img + ronmask*ronmask) # [e-] #TEMPFIX: (how should I be doing this properly???) err_img = np.sqrt(np.clip(img,0,None) + ronmask*ronmask) # [e-] ## (2) remove cosmic rays (ERRORS MUST REMAIN UNCHANGED) ## check if there are multiple exposures for this epoch (if yes, we can do the much simpler "median_remove_cosmics") if len(epoch_sublists[lamp_config]) == 1: # do it the hard way using LACosmic # identify and extract background bg_raw = extract_background(img, chipmask['bg'], timit=timit) # remove cosmics, but only from background cosmic_cleaned_img = remove_cosmics(bg_raw.todense(), ronmask, obsname, path, Flim=3.0, siglim=5.0, maxiter=1, savemask=False, savefile=False, save_err=False, verbose=True, timit=True) # [e-] # identify and extract background from cosmic-cleaned image bg = extract_background(cosmic_cleaned_img, chipmask['bg'], timit=timit) # bg = extract_background_pid(cosmic_cleaned_img, P_id, slit_height=30, exclude_top_and_bottom=True, timit=timit) # fit background bg_coeffs, bg_img = fit_background(bg, clip=10, return_full=True, timit=timit) elif len(epoch_sublists[lamp_config]) == 2: if new_epoch or not os.path.isfile(path + 'temp_bg_' + lamp_config + '.fits'): # list of individual exposure times for this epoch subepoch_texp_list = [pyfits.getval(file, 'ELAPSED') for file in epoch_sublists[lamp_config]] tscale = np.array(subepoch_texp_list) / texp # get background from the element-wise minimum-image of the two images img1 = correct_for_bias_and_dark_from_filename(epoch_sublists[lamp_config][0], MB, MD, gain=gain, scalable=scalable, savefile=False) img2 = correct_for_bias_and_dark_from_filename(epoch_sublists[lamp_config][1], MB, MD, gain=gain, scalable=scalable, savefile=False) min_img = np.minimum(img1/tscale[0], img2/tscale[1]) # identify and extract background from the minimum-image bg = extract_background(min_img, chipmask['bg'], timit=timit) # bg = extract_background_pid(min_img, P_id, slit_height=30, exclude_top_and_bottom=True, timit=timit) del min_img # fit background bg_coeffs, bg_img = fit_background(bg, clip=10, return_full=True, timit=timit) # save background image to temporary file for re-use later (when reducing the next file of this sublist) pyfits.writeto(path + 'temp_bg_' + lamp_config + '.fits', bg_img, clobber=True) else: # no need to re-compute background, just load it from file print('Loading background image for this epoch and lamp configuration...') bg_img = pyfits.getdata(path + 'temp_bg_' + lamp_config + '.fits') else: if new_epoch or not os.path.isfile(path + 'temp_bg_' + lamp_config + '.fits'): # make sure this sublist is not too long (otherwise we might run out of memory in this step) if len(epoch_sublists[lamp_config]) > 10: mainix = epoch_sublists[lamp_config].index(filename) if mainix < 5: epoch_sublists[lamp_config] = epoch_sublists[lamp_config][:11] elif mainix > len(epoch_sublists[lamp_config]) - 6: epoch_sublists[lamp_config] = epoch_sublists[lamp_config][-11:] else: epoch_sublists[lamp_config] = epoch_sublists[lamp_config][mainix-5:mainix+6] # list of individual exposure times for this epoch subepoch_texp_list = [pyfits.getval(file, 'ELAPSED') for file in epoch_sublists[lamp_config]] tscale = np.array(subepoch_texp_list) / texp # make list of actual images img_list = [] for file in epoch_sublists[lamp_config]: img_list.append(correct_for_bias_and_dark_from_filename(file, MB, MD, gain=gain, scalable=scalable, savefile=False)) # # index indicating which one of the files in the epoch list is the "main" one # main_index = np.where(np.array(epoch_ix) == i)[0][0] # take median after scaling to same exposure time as main exposure med_img = np.median(np.array(img_list) / tscale.reshape(len(img_list), 1, 1), axis=0) del img_list # identify and extract background from the median image bg = extract_background(med_img, chipmask['bg'], timit=timit) # bg = extract_background_pid(med_img, P_id, slit_height=30, exclude_top_and_bottom=True, timit=timit) del med_img # fit background bg_coeffs, bg_img = fit_background(bg, clip=10, return_full=True, timit=timit) # save background image to temporary file for re-use later (when reducing the next file of this sublist) pyfits.writeto(path + 'temp_bg_' + lamp_config + '.fits', bg_img, clobber=True) else: # no need to re-compute background, just load it from file print('Loading background image for this epoch and lamp configuration...') bg_img = pyfits.getdata(path + 'temp_bg_' + lamp_config + '.fits') # now actually subtract the background model bg_corrected_img = img - bg_img # cosmic_cleaned_img = median_remove_cosmics(img_list, main_index=main_index, scales=scaled_texp, ronmask=ronmask, debug_level=1, timit=True) # (3) fit and remove background (ERRORS REMAIN UNCHANGED) # bg_corrected_img = remove_background(cosmic_cleaned_img, P_id, obsname, path, degpol=5, slit_height=slit_height, save_bg=True, savefile=True, save_err=False, # exclude_top_and_bottom=True, verbose=True, timit=True) # [e-] # bg_corrected_img = remove_background(img, P_id, obsname, path, degpol=5, slit_height=slit_height, save_bg=False, savefile=True, save_err=False, # exclude_top_and_bottom=True, verbose=True, timit=True) # [e-] # adjust errors? # (4) remove pixel-to-pixel sensitivity variations (2-dim) #XXXXXXXXXXXXXXXXXXXXXXXXXXX #TEMPFIX final_img = bg_corrected_img.copy() # [e-] # final_img = img.copy() # [e-] #adjust errors? # (5) extract stripes if not from_indices: stripes,stripe_indices = extract_stripes(final_img, P_id, return_indices=True, slit_height=slit_height, savefiles=saveall, obsname=obsname, path=path, timit=True) err_stripes = extract_stripes(err_img, P_id, return_indices=False, slit_height=slit_height, savefiles=saveall, obsname=obsname+'_err', path=path, timit=True) if stripe_indices is None: # this is just to get the stripe indices in case we forgot to provide them (DONE ONLY ONCE, if at all...) stripes,stripe_indices = extract_stripes(final_img, P_id, return_indices=True, slit_height=slit_height, savefiles=False, obsname=obsname, path=path, timit=True) # (6) perform extraction of 1-dim spectrum if from_indices: pix,flux,err = extract_spectrum_from_indices(final_img, err_img, quick_indices, method='quick', slit_height=qsh, ronmask=ronmask, savefile=True, filetype='fits', obsname=obsname, date=date, path=path, timit=True) pix,flux,err = extract_spectrum_from_indices(final_img, err_img, stripe_indices, method=ext_method, slope=slope, offset=offset, fibs=fibs, slit_height=slit_height, ronmask=ronmask, savefile=True, filetype='fits', obsname=obsname, date=date, path=path, timit=True) else: pix,flux,err = extract_spectrum(stripes, err_stripes=err_stripes, ron_stripes=ron_stripes, method='quick', slit_height=qsh, ronmask=ronmask, savefile=True, filetype='fits', obsname=obsname, date=date, path=path, timit=True) pix,flux,err = extract_spectrum(stripes, err_stripes=err_stripes, ron_stripes=ron_stripes, method=ext_method, slope=slope, offset=offset, fibs=fibs, slit_height=slit_height, ronmask=ronmask, savefile=True, filetype='fits', obsname=obsname, date=date, path=path, timit=True) # # (7) get relative intensities of different fibres # if from_indices: # relints = get_relints_from_indices(P_id, final_img, err_img, stripe_indices, mask=mask, sampling_size=sampling_size, slit_height=slit_height, return_full=False, timit=True) # else: # relints = get_relints(P_id, stripes, err_stripes, mask=mask, sampling_size=sampling_size, slit_height=slit_height, return_full=False, timit=True) # # # # (8) get wavelength solution # #XXXXX # # (9) get barycentric correction # if obstype == 'stellar': # bc = get_barycentric_correction(filename) # bc = np.round(bc,2) # if np.isnan(bc): # bc = '' # # write the barycentric correction into the FITS header of both the quick-extracted and the optimal-extracted reduced spectrum files # outfn_list = glob.glob(path + '*' + obsname + '*extracted*') # for outfn in outfn_list: # pyfits.setval(outfn, 'BARYCORR', value=bc, comment='barycentric velocity correction [m/s]') # #now append relints, wl-solution, and barycorr to extracted FITS file header # outfn = path + obsname + '_extracted.fits' # if os.path.isfile(outfn): # #relative fibre intensities # dum = append_relints_to_FITS(relints, outfn, nfib=19) # #wavelength solution # #pyfits.setval(fn, 'RELINT' + str(i + 1).zfill(2), value=relints[i], comment='fibre #' + str(fibnums[i]) + ' - ' + fibinfo[i] + ' fibre') if timit: print('Total time elapsed: '+str(np.round(time.time() - start_time,1))+' seconds') return
def process_science_images(imglist, P_id, mask=None, sampling_size=25, slit_height=25, gain=[1., 1., 1., 1.], MB=None, ronmask=None, MD=None, scalable=False, saveall=False, path=None, ext_method='optimal', from_indices=True, timit=False): """ Process all science images. This includes: (1) bias and dark subtraction (2) cosmic ray removal (3) background extraction and estimation (4) flat-fielding (ie removal of pixel-to-pixel sensitivity variations) ============================= (5) extraction of stripes (6) extraction of 1-dim spectra (7) get relative intensities of different fibres (8) wavelength solution (9) barycentric correction """ if timit: start_time = time.time() ##################################### ### (1) bias and dark subtraction ### ##################################### #if the darks have a different exposure time than the science images, then we need to re-scale the master dark texp = pyfits.getval(imglist[0], 'exptime') #if INPUT arrays are not given, read them from default files if path is None: print('WARNING: output file directory not provided!!!') print('Using same directory as input file...') dum = imglist[0].split('/') path = imglist[0][0:-len(dum[-1])] if MB is None: #no need to fix orientation, this is already a processed file [ADU] MB = pyfits.getdata(path + 'master_bias.fits') if ronmask is None: #no need to fix orientation, this is already a processed file [e-] ronmask = pyfits.getdata(path + 'read_noise_mask.fits') if MD is None: if scalable: #no need to fix orientation, this is already a processed file [e-] MD = pyfits.getdata(path + 'master_dark_scalable.fits', 0) # err_MD = pyfits.getdata(path+'master_dark_scalable.fits', 1) else: #no need to fix orientation, this is already a processed file [e-] MD = pyfits.getdata( path + 'master_dark_t' + str(int(np.round(texp, 0))) + '.fits', 0) # err_MD = pyfits.getdata(path+'master_dark_t'+str(int(np.round(texp,0)))+'.fits', 1) if not from_indices: ron_stripes = extract_stripes(ronmask, P_id, return_indices=False, slit_height=slit_height, savefiles=False, timit=True) for filename in imglist: #do some housekeeping with filenames dum = filename.split('/') dum2 = dum[-1].split('.') obsname = dum2[0] # (1) call routine that does all the bias and dark correction stuff and proper error treatment img = correct_for_bias_and_dark_from_filename(filename, MB, MD, gain=gain, scalable=False, savefile=saveall, path=path, timit=True) #[e-] #err = np.sqrt(img + ronmask*ronmask) # [e-] #TEMPFIX: err_img = np.sqrt(np.clip(img, 0, None) + ronmask * ronmask) # [e-] # (2) remove cosmic rays (ERRORS REMAIN UNCHANGED) cosmic_cleaned_img = remove_cosmics(img, ronmask, obsname, path, Flim=3.0, siglim=5.0, maxiter=1, savemask=True, savefile=True, save_err=False, verbose=True, timit=True) # [e-] #adjust errors? # (3) fit and remove background (ERRORS REMAIN UNCHANGED) bg_corrected_img = remove_background(cosmic_cleaned_img, P_id, obsname, path, degpol=5, slit_height=slit_height, save_bg=True, savefile=True, save_err=False, exclude_top_and_bottom=True, verbose=True, timit=True) # [e-] #adjust errors? # (4) remove pixel-to-pixel sensitivity variations (2-dim) #XXXXXXXXXXXXXXXXXXXXXXXXXXX #TEMPFIX final_img = bg_corrected_img.copy() # [e-] #adjust errors? # (5) extract stripes stripes, stripe_indices = extract_stripes(final_img, P_id, return_indices=True, slit_height=slit_height, savefiles=True, obsname=obsname, path=path, timit=True) if not from_indices: err_stripes = extract_stripes(err_img, P_id, return_indices=False, slit_height=slit_height, savefiles=True, obsname=obsname + '_err', path=path, timit=True) # (6) perform extraction of 1-dim spectrum if from_indices: pix, flux, err = extract_spectrum_from_indices( final_img, err_img, stripe_indices, method=ext_method, slit_height=slit_height, RON=ronmask, savefile=True, filetype='fits', obsname=obsname, path=path, timit=True) else: pix2, flux2, err2 = extract_spectrum(stripes, err_stripes=err_stripes, ron_stripes=ron_stripes, method=ext_method, slit_height=slit_height, RON=ronmask, savefile=False, filetype='fits', obsname=obsname, path=path, timit=True) # (7) get relative intensities of different fibres if from_indices: relints = get_relints_from_indices(P_id, final_img, err_img, stripe_indices, mask=mask, sampling_size=sampling_size, slit_height=slit_height, return_full=False, timit=True) else: relints = get_relints(P_id, stripes, err_stripes, mask=mask, sampling_size=sampling_size, slit_height=slit_height, return_full=False, timit=True) # (8) get wavelength solution #XXXXX # (9) get barycentric correction lat, long, alt = get_obs_coords_from_header(fn) bc = barycorrpy.get_BC_vel(JDUTC=JDUTC, hip_id=8102, lat=lat, longi=long, alt=float(alt), ephemeris='de430', zmeas=0.0) #bc = barycorrpy.get_BC_vel(JDUTC=JDUTC, hip_id=8102, lat=-31.2755, longi=149.0673, alt=1165.0, ephemeris='de430', zmeas=0.0) #bc = barycorrpy.get_BC_vel(JDUTC=JDUTC, hip_id=8102, obsname='AAO', ephemeris='de430') #now append relints, wl-solution, and barycorr to extracted FITS file header outfn = path + obsname + '_extracted.fits' if os.path.isfile(outfn): #relative fibre intensities dum = append_relints_to_FITS(relints, outfn, nfib=19) #wavelength solution #pyfits.setval(fn, 'RELINT' + str(i + 1).zfill(2), value=relints[i], comment='fibre #' + str(fibnums[i]) + ' - ' + fibinfo[i] + ' fibre') #barycentric correction pyfits.setval(outfn, 'BARYCORR', value=np.array(bc[0])[0], comment='barycentric correction [m/s]') if timit: print('Total time elapsed: ' + str(np.round(time.time() - start_time, 1)) + ' seconds') return