def normalize_now(filenameblue,
filenamered,
redfile,
plotall=True,
extinct_correct=False):
#Read in the observed spectrum
obs_spectrablue, airmass, exptime, dispersion = st.readspectrum(
filenameblue)
datalistblue = fits.open(filenameblue)
if redfile:
obs_spectrared, airmassred, exptimered, dispersionred = st.readspectrum(
filenamered)
#Extinction correction
if extinct_correct:
print 'Extinction correcting spectra.'
plt.clf()
plt.plot(obs_spectrablue.warr, obs_spectrablue.opfarr)
obs_spectrablue.opfarr = st.extinction_correction(
obs_spectrablue.warr, obs_spectrablue.opfarr, airmass)
obs_spectrablue.farr = st.extinction_correction(
obs_spectrablue.warr, obs_spectrablue.farr, airmass)
obs_spectrablue.sky = st.extinction_correction(obs_spectrablue.warr,
obs_spectrablue.sky,
airmass)
obs_spectrablue.sigma = st.extinction_correction(
obs_spectrablue.warr, obs_spectrablue.sigma, airmass)
plt.plot(obs_spectrablue.warr, obs_spectrablue.opfarr)
plt.show()
if redfile:
plt.clf()
plt.plot(obs_spectrared.warr, obs_spectrared.opfarr)
obs_spectrared.opfarr = st.extinction_correction(
obs_spectrared.warr, obs_spectrared.opfarr, airmassred)
obs_spectrared.farr = st.extinction_correction(
obs_spectrared.warr, obs_spectrared.farr, airmassred)
obs_spectrared.sky = st.extinction_correction(
obs_spectrared.warr, obs_spectrared.sky, airmassred)
obs_spectrared.sigma = st.extinction_correction(
obs_spectrared.warr, obs_spectrared.sigma, airmassred)
plt.plot(obs_spectrared.warr, obs_spectrared.opfarr)
plt.show()
#Read in measured FWHM from header. This is used to convolve the model spectrum.
FWHMpix = datalistblue[0].header['specfwhm']
FWHM = FWHMpix * (obs_spectrablue.warr[-1] -
obs_spectrablue.warr[0]) / len(obs_spectrablue.warr)
#Read in DA model
cwd = os.getcwd()
os.chdir(
'/afs/cas.unc.edu/depts/physics_astronomy/clemens/students/group/modelfitting/Koester_08'
)
dafile = 'da12500_800.dk'
mod_wav, mod_spec = np.genfromtxt(dafile, unpack=True, skip_header=33)
os.chdir(cwd) #Move back to directory with observed spectra
#Convolve the model to match the seeing of the spectrum
intlambda = np.divide(range(31000), 10.) + 3660.0
lowlambda = np.min(np.where(mod_wav > 3600.))
highlambda = np.min(np.where(mod_wav > 6800.))
shortlambdas = mod_wav[lowlambda:highlambda]
shortinten = mod_spec[lowlambda:highlambda]
interp = inter.InterpolatedUnivariateSpline(shortlambdas, shortinten, k=1)
intflux = interp(intlambda)
sig = FWHM / (2. * np.sqrt(2. * np.log(2.)))
gx = np.divide(range(360), 10.)
gauss = (1. / (sig * np.sqrt(2. * np.pi))) * np.exp(-(gx - 18.)**2. /
(2. * sig**2.))
gf = np.divide(np.outer(intflux, gauss), 10.)
length = len(intflux) - 360.
cflux = np.zeros(length)
clambda = intlambda[180:len(intlambda) - 180]
x = 0
while x < length:
cflux[x] = np.sum(np.diagonal(gf, x, axis1=1, axis2=0), dtype='d')
x += 1
interp2 = inter.InterpolatedUnivariateSpline(clambda, cflux, k=1)
cflux2blue = interp2(obs_spectrablue.warr)
cflux2blue /= 10**13. #Divide by 10**13 to scale
if redfile:
cflux2red = interp2(obs_spectrared.warr)
cflux2red /= 10**13. #Divide by 10**13 to scale
#plt.clf()
#plt.plot(obs_spectrablue.warr,obs_spectrablue.opfarr,'b')
#plt.plot(obs_spectrablue.warr,cflux2blue,'r')
#if redfile:
# plt.plot(obs_spectrared.warr,obs_spectrared.opfarr,'b')
# plt.plot(obs_spectrared.warr,cflux2red,'r')
#plt.show()
#The response function is the observed spectrum divided by the model spectrum.
response_blue = obs_spectrablue.opfarr / cflux2blue
if redfile:
response_red = obs_spectrared.opfarr / cflux2red
'''
plt.clf()
plt.plot(obs_spectrablue.warr,response_blue,'k')
if redfile:
plt.plot(obs_spectrared.warr,response_red,'k')
plt.show()
'''
#We want to mask out the Balmer line features, and the telluric line in the red spectrum. Set up the wavelength ranges to mask here.
#balmer_features_blue = [[3745,3757],[3760,3780],[3784,3812],[3816,3856],[3865,3921],[3935,4021],[4040,4191],[4223,4460],[4691,5010]] #Keeping ends
balmer_features_blue = [[3400, 3700], [3745, 3757], [3760, 3780],
[3784, 3812], [3816, 3856], [3865, 3921],
[3935, 4021], [4040, 4191], [4223, 4460],
[4691, 5010], [5140, 5500]] #Discarding ends
balmer_features_red = [[6350, 6780], [6835, 6970]]
balmer_mask_blue = obs_spectrablue.warr == obs_spectrablue.warr
for wavrange in balmer_features_blue:
inds = np.where((obs_spectrablue.warr > wavrange[0])
& (obs_spectrablue.warr < wavrange[1]))
balmer_mask_blue[inds] = False
if redfile:
balmer_mask_red = obs_spectrared.warr == obs_spectrared.warr
for wavrange in balmer_features_red:
indxs = np.where((obs_spectrared.warr > wavrange[0])
& (obs_spectrared.warr < wavrange[1]))
balmer_mask_red[indxs] = False
spec_wav_masked_blue = obs_spectrablue.warr[balmer_mask_blue]
response_masked_blue = response_blue[balmer_mask_blue]
if redfile:
spec_wav_masked_red = obs_spectrared.warr[balmer_mask_red]
response_masked_red = response_red[balmer_mask_red]
#Fit the response function with a polynomial. The order of polynomial is specified first.
response_poly_order_blue = 7.
response_fit_blue_poly = np.polyfit(spec_wav_masked_blue,
response_masked_blue,
response_poly_order_blue)
response_fit_blue = np.poly1d(response_fit_blue_poly)
if redfile:
response_poly_order_red = 3.
response_fit_red_poly = np.polyfit(spec_wav_masked_red,
response_masked_red,
response_poly_order_red)
response_fit_red = np.poly1d(response_fit_red_poly)
#Save response function
#np.savetxt('response_model_no_extinction.txt',np.transpose([obs_spectrablue.warr,response_fit_blue(obs_spectrablue.warr),obs_spectrared.warr,response_fit_red(obs_spectrared.warr)]))
#plt.clf()
#plt.plot(obs_spectrablue.warr,response_fit_blue(obs_spectrablue.warr)/response_fit_blue(obs_spectrablue.warr)[1000])
#plt.show()
#exit()
if plotall:
plt.clf()
plt.plot(obs_spectrablue.warr, response_blue, 'r')
plt.plot(spec_wav_masked_blue, response_masked_blue, 'g.')
plt.plot(obs_spectrablue.warr, response_fit_blue(obs_spectrablue.warr),
'k--')
#plt.show()
#plt.clf()
if redfile:
plt.plot(obs_spectrared.warr, response_red, 'r')
plt.plot(spec_wav_masked_red, response_masked_red, 'g.')
plt.plot(obs_spectrared.warr,
response_fit_red(obs_spectrared.warr), 'k--')
plt.show()
#Divide by the fit to the response function to get the continuum normalized spectra. Divide every extension by the same polynomial
fcorr_wd_blue_opfarr = obs_spectrablue.opfarr / response_fit_blue(
obs_spectrablue.warr)
fcorr_wd_blue_farr = obs_spectrablue.farr / response_fit_blue(
obs_spectrablue.warr)
fcorr_wd_blue_sky = obs_spectrablue.sky / response_fit_blue(
obs_spectrablue.warr)
fcorr_wd_blue_sigma = obs_spectrablue.sigma / response_fit_blue(
obs_spectrablue.warr)
if redfile:
fcorr_wd_red_opfarr = obs_spectrared.opfarr / response_fit_red(
obs_spectrared.warr)
fcorr_wd_red_farr = obs_spectrared.farr / response_fit_red(
obs_spectrared.warr)
fcorr_wd_red_sky = obs_spectrared.sky / response_fit_red(
obs_spectrared.warr)
fcorr_wd_red_sigma = obs_spectrared.sigma / response_fit_red(
obs_spectrared.warr)
if plotall:
plt.clf()
plt.plot(obs_spectrablue.warr, fcorr_wd_blue_opfarr, 'b')
if redfile:
plt.plot(obs_spectrared.warr, fcorr_wd_red_opfarr, 'r')
plt.show()
#exit()
#Save parameters for diagnostics
if redfile:
bigarray = np.zeros([len(obs_spectrablue.warr), 12])
bigarray[0:len(obs_spectrablue.warr), 0] = obs_spectrablue.warr
bigarray[0:len(response_blue), 1] = response_blue
bigarray[0:len(spec_wav_masked_blue), 2] = spec_wav_masked_blue
bigarray[0:len(response_masked_blue), 3] = response_masked_blue
bigarray[0:len(response_fit_blue(obs_spectrablue.warr)),
4] = response_fit_blue(obs_spectrablue.warr)
bigarray[0:len(fcorr_wd_blue_opfarr), 5] = fcorr_wd_blue_opfarr
bigarray[0:len(obs_spectrared.warr), 6] = obs_spectrared.warr
bigarray[0:len(response_red), 7] = response_red
bigarray[0:len(spec_wav_masked_red), 8] = spec_wav_masked_red
bigarray[0:len(response_masked_red), 9] = response_masked_red
bigarray[0:len(response_fit_red(obs_spectrared.warr)),
10] = response_fit_red(obs_spectrared.warr)
bigarray[0:len(fcorr_wd_red_opfarr), 11] = fcorr_wd_red_opfarr
now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
endpoint = '_930'
with open(
'continuum_normalization_' +
filenameblue[5:filenameblue.find(endpoint)] + '_' + now +
'.txt', 'a') as handle:
header = str(filenameblue) + ',' + str(
filenamered
) + ',' + dafile + '\n Columns structured as blue then red. If no red file, only blue data given. Columns are: blue wavelengths, blue response all data, blue masked wavelengths, blue masked response data, blue response fit, blue continuum-normalize flux, red wavelengths, red response all data, red masked wavelengths, red masked response data, red response fit, red continuum-normalized flux'
np.savetxt(handle, bigarray, fmt='%f', header=header)
if not redfile:
bigarray = np.zeros([len(obs_spectrablue.warr), 6])
bigarray[0:len(obs_spectrablue.warr), 0] = obs_spectrablue.warr
bigarray[0:len(response_blue), 1] = response_blue
bigarray[0:len(spec_wav_masked_blue), 2] = spec_wav_masked_blue
bigarray[0:len(response_masked_blue), 3] = response_masked_blue
bigarray[0:len(response_fit_blue(obs_spectrablue.warr)),
4] = response_fit_blue(obs_spectrablue.warr)
bigarray[0:len(fcorr_wd_blue_opfarr), 5] = fcorr_wd_blue_opfarr
now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
endpoint = '_930'
with open(
'continuum_normalization_' +
filenameblue[5:filenameblue.find(endpoint)] + '_' + now +
'.txt', 'a') as handle:
header = str(
filenameblue
) + ',' + ',' + dafile + '\n Columns structured as blue then red. If no red file, only blue data given. Columns are: blue wavelengths, blue response all data, blue masked wavelengths, blue masked response data, blue response fit, blue continuum-normalized flux'
np.savetxt(handle, bigarray, fmt='%f', header=header)
#Save the continuum normalized spectra here.
Ni = 4. #Number of extensions
Nx1 = len(fcorr_wd_blue_opfarr)
if redfile:
Nx2 = len(fcorr_wd_red_opfarr)
Ny = 1. #All 1D spectra
#Update header
header1 = st.readheader(filenameblue)
header1.set('STANDARD', dafile, 'DA Model for Continuum Calibration')
header1.set('RESPPOLY', response_poly_order_blue,
'Polynomial order for Response Function')
header1.set('DATENORM',
datetime.datetime.now().strftime("%Y-%m-%d"),
'Date of Continuum Normalization')
data1 = np.empty(shape=(Ni, Ny, Nx1))
data1[0, :, :] = fcorr_wd_blue_opfarr
data1[1, :, :] = fcorr_wd_blue_farr
data1[2, :, :] = fcorr_wd_blue_sky
data1[3, :, :] = fcorr_wd_blue_sigma
#Check that filename does not already exist. Prompt user for input if it does.
loc1 = filenameblue.find('.ms.fits')
newname1 = filenameblue[0:loc1] + '_flux_model_short.ms.fits'
clob = False
mylist = [True for f in os.listdir('.') if f == newname1]
exists = bool(mylist)
if exists:
print 'File %s already exists.' % newname1
nextstep = raw_input(
'Do you want to overwrite or designate a new name (overwrite/new)? '
)
if nextstep == 'overwrite':
clob = True
exists = False
elif nextstep == 'new':
newname1 = raw_input('New file name: ')
exists = False
else:
exists = False
print 'Writing ', newname1
newim1 = fits.PrimaryHDU(data=data1, header=header1)
newim1.writeto(newname1, clobber=clob)
#Save the red file if it exists.
if redfile:
header2 = st.readheader(filenamered)
header2.set('STANDARD', dafile, 'DA Model for Continuum Calibration')
header2.set('RESPPOLY', response_poly_order_red,
'Polynomial order for Response Function')
header2.set('DATENORM',
datetime.datetime.now().strftime("%Y-%m-%d"),
'Date of Continuum Normalization')
data2 = np.empty(shape=(Ni, Ny, Nx2))
data2[0, :, :] = fcorr_wd_red_opfarr
data2[1, :, :] = fcorr_wd_red_farr
data2[2, :, :] = fcorr_wd_red_sky
data2[3, :, :] = fcorr_wd_red_sigma
loc2 = filenamered.find('.ms.fits')
newname2 = filenamered[0:loc2] + '_flux_model_short.ms.fits'
clob = False
mylist = [True for f in os.listdir('.') if f == newname2]
exists = bool(mylist)
if exists:
print 'File %s already exists.' % newname2
nextstep = raw_input(
'Do you want to overwrite or designate a new name (overwrite/new)? '
)
if nextstep == 'overwrite':
clob = True
exists = False
elif nextstep == 'new':
newname2 = raw_input('New file name: ')
exists = False
else:
exists = False
print 'Writing ', newname2
newim2 = fits.PrimaryHDU(data=data2, header=header2)
newim2.writeto(newname2, clobber=clob)
now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
endpoint = '_930'
with open('continuum_normalization_' + filenameblue[5:filenameblue.find(endpoint)] + '_' + now + '.txt','a') as handle:
header = str(filenameblue) + ',' + ',' + dafile + '\n Columns structured as blue then red. If no red file, only blue data given. Columns are: blue wavelengths, blue response all data, blue masked wavelengths, blue masked response data, blue response fit, blue continuum-normalized flux'
np.savetxt(handle,bigarray,fmt='%f',header=header)
#Save the continuum normalized spectra here.
Ni = 4. #Number of extensions
Nx1 = len(fcorr_wd_blue_opfarr)
if redfile:
Nx2 = len(fcorr_wd_red_opfarr)
Ny = 1. #All 1D spectra
#Update header
header1 = st.readheader(filenameblue)
header1.set('STANDARD',dafile,'DA Model for Continuum Calibration')
header1.set('RESPPOLY',response_poly_order_blue,'Polynomial order for Response Function')
header1.set('DATENORM',datetime.datetime.now().strftime("%Y-%m-%d"),'Date of Continuum Normalization')
data1 = np.empty(shape = (Ni,Ny,Nx1))
data1[0,:,:] = fcorr_wd_blue_opfarr
data1[1,:,:] = fcorr_wd_blue_farr
data1[2,:,:] = fcorr_wd_blue_sky
data1[3,:,:] = fcorr_wd_blue_sigma
#Check that filename does not already exist. Prompt user for input if it does.
loc1 = filenameblue.find('.ms.fits')
newname1 = filenameblue[0:loc1] + '_flux_model.ms.fits'
clob = False
sigma_flux1 = st.cal_spec(WD_spectra1.sigma,sens_wave1,exptime1,dispersion1)
if redfile:
star_opflux2 = st.cal_spec(WD_spectra2.opfarr,sens_wave2,exptime2,dispersion2)
star_flux2 = st.cal_spec(WD_spectra2.farr,sens_wave2,exptime2,dispersion2)
sky_flux2 = st.cal_spec(WD_spectra2.sky,sens_wave2,exptime2,dispersion2)
sigma_flux2 = st.cal_spec(WD_spectra2.sigma,sens_wave2,exptime2,dispersion2)
#plt.clf()
#plt.plot(WD_spectra.warr,star_opflux)
#plt.show()
print 'Saving the final spectrum.'
#Save the flux-calibrated spectrum and update the header
header1 = st.readheader(specfile[avocado])
header1.set('EX-FLAG',-1) #Extiction correction? 0=yes, -1=no
header1.set('CA-FLAG',0) #Calibrated to flux scale? 0=yes, -1=no
header1.set('BUNIT','erg/cm2/s/A') #physical units of the array value
header1.set('STANDARD',str(standards[choice]),'Flux standard used') #flux standard used for flux-calibration
if redfile:
header2 = st.readheader(specfile[avocado+1])
header2.set('EX-FLAG',-1) #Extiction correction? 0=yes, -1=no
header2.set('CA-FLAG',0) #Calibrated to flux scale? 0=yes, -1=no
header2.set('BUNIT','erg/cm2/s/A') #physical units of the array value
header2.set('STANDARD',str(standards[choice+1]),'Flux standard used') #flux standard used for flux-calibration
#Set up size of new fits image
Ni = 4. #Number of extensions
Nx1 = len(star_flux1)
signal = np.mean(shortspec)
noise = np.sqrt(np.sum((shortspec-bestline)**2.) / float(len(bestline))) #Noise from RMS
#noise = np.mean(shortsigma) #noise from sigma spectrum
sn_res_ele = signal/noise * np.sqrt(fwhm)
print 'Signal to Noise is: ', sn_res_ele
#plt.clf()
#plt.plot(shortpix,shortspec,'b')
#plt.plot(shortpix,bestline,'g')
#plt.show()
#exit()
#Get the image header and add keywords
header = st.readheader(specfile)
header.set('BANDID1','Optimally Extracted Spectrum')
header.set('BANDID2','Raw Extracted Spectrum')
header.set('BANDID3','Mean Background')
header.set('BANDID4','Sigma Spectrum')
header.set('DISPCOR',0) #Dispersion axis of image
fwhmsave = np.round(fwhm,decimals=4)
snrsave = np.round(sn_res_ele,decimals=4)
header.set('SPECFWHM',fwhmsave,'FWHM of spectrum in pixels') #FWHM of spectrum in pixels
header.set('SNR',snrsave,'Signal to Noise per resolution element')
header.set('DATEEXTR',datetime.datetime.now().strftime("%Y-%m-%d"),'Date of Spectral Extraction')
#Save the extracted image
Ni = 4. #Number of extensions
Nx = 1. #All 1D spectra
Ny = len(output_spec.spectrum[:,0])
def extract_now(specfile,lamp,FWHMfile,tracefile,trace_exist=False):
#Open file and read gain and readnoise
datalist = fits.open(specfile)
data = datalist[0].data
data = np.transpose(data[0,:,:])
#Since we have combined multiple images, to keep our statistics correct, we need to multiply the values in ADU by the number of images
try:
nimages = float(datalist[0].header['NCOMBINE'])
except:
nimages = 1.
data = nimages * data
#gain = datalist[0].header['GAIN']
gain = 1.33 #from 2017-06-07
rdnoise = np.sqrt(nimages) * datalist[0].header['RDNOISE']
#Calculate the variance of each pixel in ADU
varmodel = ((nimages*rdnoise**2.) + np.absolute(data)*gain)/gain
#Fit a Gaussian every 10 pixels to determine FWHM for convolving in the model fitting, unless this file already exists
fitpixel = np.arange(3,len(data[:,100]),10)
allfwhm = np.zeros(len(fitpixel))
for x in fitpixel:
#forfit = data[x,2:]
forfit = np.median(np.array([data[x-2,2:],data[x-1,2:],data[x,2:],data[x+1,2:],data[x+2,2:]]),axis=0)
guess = np.zeros(4)
guess[0] = np.mean(forfit)
guess[1] = np.amax(forfit)
guess[2] = np.argmax(forfit)
guess[3] = 3.
error_fit = np.ones(len(forfit))
xes = np.linspace(0,len(forfit)-1,num=len(forfit))
fa = {'x':xes,'y':forfit,'err':error_fit}
fitparams = mpfit.mpfit(fitgauss,guess,functkw=fa,quiet=True)
allfwhm[np.argwhere(fitpixel == x)] = 2.*np.sqrt(2.*np.log(2.))*fitparams.params[3]
#plt.clf()
#plt.plot(forfit)
#plt.plot(xes,gauss(xes,fitparams.params))
#plt.show()
fwhmclipped = SigClip(allfwhm,3,3)
if not FWHMfile:
#Fit using a line, but give user the option to fit with a different order
order = 1
repeat = 'yes'
while repeat == 'yes':
fwhmpolyvalues = np.polyfit(fitpixel,fwhmclipped,order)
allpixel = np.arange(0,len(data[:,100]),1)
fwhmpoly = np.poly1d(fwhmpolyvalues)
plt.clf()
plt.plot(fitpixel,fwhmclipped,'^')
plt.plot(allpixel,fwhmpoly(allpixel),'g')
plt.title(specfile)
plt.show()
repeat = raw_input('Do you want to try again (yes/no)? ')
if repeat == 'yes':
order = raw_input('New order for polynomial: ')
locfwhm = specfile.find('.fits')
print '\n Saving FWHM file.'
np.save(specfile[0:locfwhm] + '_poly',fwhmpoly(allpixel))
diagnostics = np.zeros([len(data[:,100]),12])
diagnostics[0:len(allfwhm),0] = fwhmclipped
diagnostics[0:len(fitpixel),1] = fitpixel
if not FWHMfile:
diagnostics[0:len(allpixel),2] = fwhmpoly(allpixel)
diagnostics[0:len(allpixel),3] = allpixel
else:
fwhm_fit = np.load(FWHMfile)
allpixel = np.arange(0,len(data[:,100]),1)
diagnostics[0:len(fwhm_fit),2] = fwhm_fit
diagnostics[0:len(allpixel),3] = allpixel
#===============================
#Section to prepare inputs for extraction
#===============================
#Fit a column of the 2D image to determine the FWHM in pixels
if 'blue' in specfile.lower():
#Average over 5 rows to deal with any remaining cosmic rays
forfit = np.mean(np.array([data[1198,:],data[1199,:],data[1200,:],data[1201,:],data[1202,:]]),axis=0)
elif 'red' in specfile.lower():
forfit = np.mean(np.array([data[998,:],data[999,:],data[1000,:],data[1001,:],data[1002,:]]),axis=0)
guess = np.zeros(4)
guess[0] = np.mean(forfit)
guess[1] = np.amax(forfit)
guess[2] = np.argmax(forfit)
guess[3] = 3.
error_fit = np.ones(len(forfit))
xes = np.linspace(0,len(forfit)-1,num=len(forfit))
fa = {'x':xes,'y':forfit,'err':error_fit}
fitparams = mpfit.mpfit(fitgauss,guess,functkw=fa,quiet=True)
#The guassian gives us sigma, convert to FWHM
fwhm = 2.*np.sqrt(2.*np.log(2.))*fitparams.params[3]
extraction_rad = 5. * np.round(fwhm,decimals=1) #Extract up to 5 times FWHM
#Check to make sure background region does not go within 10 pixels of edge
background_radii = [35,60]
#First check this against the bottom
if fitparams.params[2] - background_radii[1] < 10.:
background_radii[1] = fitparams.params[2] - 10.
background_radii[0] -= 60 - background_radii[1]
#Then check against the top
hold = background_radii[1]
if fitparams.params[2] + background_radii[1] > 190.:
background_radii[1] = 190. - fitparams.params[2]
background_radii[0] -= hold - background_radii[1]
#Ensure that the closest point is at least 20 pixels away.
if background_radii[0] < 20.:
background_radii[0] = 20.
background_radii[0] = np.round(background_radii[0],decimals=1)
background_radii[1] = np.round(background_radii[1],decimals=1)
#plt.plot(data[1200,:])
#plt.plot(xes,gauss(xes,fitparams.params))
#plt.show()
#extraction_rad = 10.
#background_radii = [40,60]
#Extract the spectrum using superextract
print 'Starting extraction.'
if trace_exist:
trace = np.load(tracefile)
output_spec = superextract.superExtract(data,varmodel,gain,rdnoise,trace=trace,pord=2,tord=2,bord=1,bkg_radii=background_radii,bsigma=2.,extract_radius=extraction_rad,dispaxis=1,verbose=False,csigma=5.,polyspacing=1,retall=False)
else:
output_spec = superextract.superExtract(data,varmodel,gain,rdnoise,pord=2,tord=2,bord=1,bkg_radii=background_radii,bsigma=2.,extract_radius=extraction_rad,dispaxis=1,verbose=False,csigma=5.,polyspacing=1,retall=False)
#pord = order of profile polynomial. Default = 2. This seems appropriate, no change for higher or lower order.
#tord = degree of spectral-trace polynomial, 1 = line
#bord = degree of polynomial background fit
#bkg_radii = inner and outer radii to use in computing background. Goes on both sides of aperture.
#bsigma = sigma-clipping thresholf for computing background
#extract_radius: radius for spectral extraction. Setting this to be 5*FWHM
#csigma = sigma-clipping threshold for cleaning & cosmic-ray rejection. Default = 5.
#qmode: how to compute Marsh's Q-matrix. 'fast-linear' default and preferred.
#nreject = number of outlier-pixels to reject at each iteration. Default = 100
#polyspacing = Marsh's S: the spacing between the polynomials. This should be <= 1. Default = 1. Best to leave at 1. S/N decreases dramatically if greater than 1. If less than one, slower but final spectrum is the same. Crossfield note: A few cursory tests suggests that the extraction precision (in the high S/N case) scales as S^-2 -- but the code slows down as S^2.
#Verbose=True if you want lots of output
###########
# In superextract, to plot a 2D frame at any point, use the following
# plt.clf()
# plt.imshow(np.transpose(frame),aspect='auto',interpolation='nearest')
# plt.show()
##########
print 'Done extracting. Starting to save.'
if not trace_exist:
print 'Saving the trace.'
np.save(specfile[0:locfwhm] + '_trace',output_spec.trace)
sigSpectrum = np.sqrt(output_spec.varSpectrum)
#plt.clf()
#plt.imshow(data)
#plt.plot(output_spec.spectrum,'b')
#plt.plot(output_spec.raw,'g')
#plt.plot(output_spec.varSpectrum,'r')
#plt.plot(sigSpectrum,'r')
#plt.plot(output_spec.trace,'m')
#plt.plot(output_spec.tracepos[0],output_spec.tracepos[1],'b^')
#plt.plot(output_spec.background,'k')
#plt.plot(output_spec.profile_map,'b')
#plt.show()
#plt.clf()
#plt.plot(output_spec.extractionApertures)
#plt.show()
#plt.clf()
#plt.plot(output_spec.background_map)
#plt.show()
#Save diagnostic info
diagnostics[0:len(output_spec.tracepos[0]),4] = output_spec.tracepos[0]
diagnostics[0:len(output_spec.tracepos[1]),5] = output_spec.tracepos[1]
diagnostics[0:len(output_spec.trace),6] = output_spec.trace
diagnostics[0:len(output_spec.backgroundcolumnpixels),7] = output_spec.backgroundcolumnpixels
diagnostics[0:len(output_spec.backgroundcolumnvalues),8] = output_spec.backgroundcolumnvalues
diagnostics[0:len(output_spec.backgroundfitpixels),9] = output_spec.backgroundfitpixels
diagnostics[0:len(output_spec.backgroundfitvalues),10] = output_spec.backgroundfitvalues
diagnostics[0:len(output_spec.backgroundfitpolynomial),11] = output_spec.backgroundfitpolynomial
now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
endpoint = '.fits'
with open('extraction_' + specfile[4:specfile.find(endpoint)] + '_' + now + '.txt','a') as handle:
header = 'Columns are: 1) Measured FWHM, 2) pixel value of each FWHM, 3) fit to FWHM measurements, 4) all pixel values, 5) profile pixels, 6) profile position, 7) fit profile positions, 8) Pixels values of cut at pixel 1200, 9) Values along column 1200, 10) Pixels of fit to background, 11) Values used for fit, 12) polynomial fit to background'
np.savetxt(handle,diagnostics,fmt='%f',header=header)
#Compute the extracted signal to noise and save to header
if 'blue' in specfile.lower():
low_pixel, high_pixel = 1125., 1175.
elif 'red' in specfile.lower():
low_pixel, high_pixel = 825., 875.
shortspec = output_spec.spectrum[:,0][low_pixel:high_pixel]
shortsigma = sigSpectrum[:,0][low_pixel:high_pixel]
shortpix = np.linspace(low_pixel,high_pixel,num=(high_pixel-low_pixel),endpoint=False)
guessline = np.zeros(2)
guessline[0] = (shortspec[-1] - shortspec[0]) / (shortpix[-1]-shortpix[0])
guessline[1] = shortspec[0]-guessline[0]*shortpix[0]
par, cov = curve_fit(line,shortpix,shortspec,guessline)
bestline = line(shortpix,par[0],par[1])
signal = np.mean(shortspec)
noise = np.sqrt(np.sum((shortspec-bestline)**2.) / float(len(bestline))) #Noise from RMS
#noise = np.mean(shortsigma) #noise from sigma spectrum
sn_res_ele = signal/noise * np.sqrt(fwhm)
print 'Signal to Noise is: ', sn_res_ele
#plt.clf()
#plt.plot(shortpix,shortspec,'b')
#plt.plot(shortpix,bestline,'g')
#plt.show()
#exit()
#Get the image header and add keywords
header = st.readheader(specfile)
header.set('BANDID1','Optimally Extracted Spectrum')
header.set('BANDID2','Raw Extracted Spectrum')
header.set('BANDID3','Mean Background')
header.set('BANDID4','Sigma Spectrum')
header.set('DISPCOR',0) #Dispersion axis of image
fwhmsave = np.round(fwhm,decimals=4)
snrsave = np.round(sn_res_ele,decimals=4)
header.set('SPECFWHM',fwhmsave,'FWHM of spectrum in pixels') #FWHM of spectrum in pixels
header.set('SNR',snrsave,'Signal to Noise per resolution element')
header.set('DATEEXTR',datetime.datetime.now().strftime("%Y-%m-%d"),'Date of Spectral Extraction')
#Save the extracted image
Ni = 4. #Number of extensions
Nx = 1. #All 1D spectra
Ny = len(output_spec.spectrum[:,0])
spectrum = np.empty(shape = (Ni,Nx,Ny))
spectrum[0,:,:] = output_spec.spectrum[:,0]
spectrum[1,:,:] = output_spec.raw[:,0]
spectrum[2,:,:] = output_spec.background
spectrum[3,:,:] = sigSpectrum[:,0]
#Save the extracted spectra with .ms.fits in the filename
#Ask to overwrite if file already exists or provide new name
loc = specfile.find('.fits')
newname = specfile[0:loc] + '.ms.fits'
clob = False
mylist = [True for f in os.listdir('.') if f == newname]
exists = bool(mylist)
if exists:
print 'File %s already exists.' % newname
nextstep = raw_input('Do you want to overwrite or designate a new name (overwrite/new)? ')
if nextstep == 'overwrite':
clob = True
exists = False
elif nextstep == 'new':
newname = raw_input('New file name: ')
exists = False
else:
exists = False
newim = fits.PrimaryHDU(data=spectrum,header=header)
newim.writeto(newname,clobber=clob)
print 'Wrote %s to file.' % newname
#Save parameters to a file for future reference.
# specfile,date of extraction, extration_rad,background_radii,newname,newname2
f = open('extraction_params.txt','a')
now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
newinfo = specfile + '\t' + now + '\t' + str(extraction_rad) + '\t' + str(background_radii) + '\t' + newname
f.write(newinfo + "\n")
f.close()
###########################
#Extract a lamp spectrum using the trace from above
##########################
if lamp != 'no':
lamplist = fits.open(lamp)
lampdata = lamplist[0].data
lampdata = lampdata[0,:,:]
lampdata = np.transpose(lampdata)
#extraction radius will be the FWHM the star
#But since the Fe lamps are taken binned by 1 in spectral direction, we need to adjust the trace to match.
#We do that by expanding out the trace to match the length of the Fe lamps, then interpolate and read off values for every pixel.
bin2size = np.arange(1,len(output_spec.trace)+1)
bin1size = np.arange(1,len(lampdata)+1)
ratio = float(len(bin1size)) / float(len(bin2size))
interpolates = InterpolatedUnivariateSpline(ratio*bin2size,output_spec.trace,k=1)
newtrace = interpolates(bin1size)
#Do the extraction here.
lamp_radius = np.ceil(fwhm) #Make sure that extraction radius is a whole number, otherwise you'll get odd structures.
lampspec = lampextract(lampdata,newtrace,lamp_radius)
#Save the 1D lamp
lampheader = st.readheader(lamp)
lampheader.set('BANDID2','Raw Extracted Spectrum')
lampheader.set('REF',newname,'Reference Star used for trace')
lampheader.set('DATEEXTR',datetime.datetime.now().strftime("%Y-%m-%d"),'Date of Spectral Extraction')
Ni = 1. #We are writing just 1 1D spectrum
Ny = len(lampspec[:,0])
lampspectrum = np.empty(shape = (Ni,Ny))
lampspectrum[0,:] = lampspec[:,0]
#Save the extracted spectra with .ms.fits in the filename
#Ask to overwrite if file already exists or provide new name
loc2 = lamp.find('.fits')
loc3 = newname.find('_930')
newname2 = lamp[0:loc2] + '_' + newname[5:loc3] + '.ms.fits'
clob = False
mylist = [True for f in os.listdir('.') if f == newname2]
exists = bool(mylist)
if exists:
print 'File %s already exists.' % newname2
nextstep = raw_input('Do you want to overwrite or designate a new name (overwrite/new)? ')
if nextstep == 'overwrite':
clob = True
exists = False
elif nextstep == 'new':
newname2 = raw_input('New file name: ')
exists = False
else:
exists = False
lampim = fits.PrimaryHDU(data=lampspectrum,header=lampheader)
lampim.writeto(newname2,clobber=clob)
print 'Wrote %s to file.' % newname2
#Save parameters to a file for future reference.
# specfile,date of extraction, extration_rad,background_radii,newname,newname2
background_radii2 = [0,0] #We do not extract a background for the lamp
f = open('extraction_params.txt','a')
now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
newinfo2 = lamp + '\t' + now + '\t' + str(extraction_rad) + '\t' + str(background_radii2) + '\t' + newname2
f.write(newinfo2 + "\n")
f.close()
def flux_calibrate_now(stdlist,
fluxlist,
speclist,
extinct_correct=False,
masterresp=False):
if extinct_correct:
extinctflag = 0
else:
extinctflag = -1
if masterresp: #Use the master response function
#Read in master response function and use that.
cwd = os.getcwd()
os.chdir(
'/afs/cas.unc.edu/depts/physics_astronomy/clemens/students/group/standards/response_curves/'
)
standards = sorted(glob('*resp*.npy'))
master_response_blue_in = np.load(standards[0])
master_response_blue_in_pol = np.poly1d(master_response_blue_in)
master_response_blue_out = np.load(standards[1])
master_response_blue_out_pol = np.poly1d(master_response_blue_out)
master_response_red_in = np.load(standards[2])
master_response_red_in_pol = np.poly1d(master_response_red_in)
master_response_red_out = np.load(standards[3])
master_response_red_out_pol = np.poly1d(master_response_red_out)
os.chdir(cwd)
airstd = np.ones([4])
#airstd[0] = 1.1
#For saving files correctly
stdflux = np.array(['mmaster.dat'])
#standards = np.array([masterlist])
allexcluded = [[None] for i in range(len(standards))]
orderused = np.zeros([len(standards)])
size = 0.
#Find shift for each night
#For blue setup: use mean of 4530-4590
#for red setup: use mean of 6090-6190
try:
flux_tonight_list = np.genfromtxt('response_curves.txt', dtype=str)
print 'Found response_curves.txt file.'
print flux_tonight_list
if flux_tonight_list.size == 1:
flux_tonight_list = np.array([flux_tonight_list])
for x in flux_tonight_list:
#print x
if 'blue' in x.lower():
wave_tonight, sens_tonight = np.genfromtxt(x, unpack=True)
blue_low_index = np.min(np.where(wave_tonight > 4530.))
blue_high_index = np.min(np.where(wave_tonight > 4590.))
blue_mean_tonight = np.mean(
sens_tonight[blue_low_index:blue_high_index])
elif 'red' in x.lower():
wave_tonight, sens_tonight = np.genfromtxt(x, unpack=True)
red_low_index = np.min(np.where(wave_tonight > 6090.))
red_high_index = np.min(np.where(wave_tonight > 6190.))
red_mean_tonight = np.mean(
sens_tonight[red_low_index:red_high_index])
except:
print 'No response_curves.txt file found.'
blue_mean_tonight = None
red_mean_tonight = None
flux_tonight_list = ['None', 'None']
else: #Use the standard star fluxes in the typical manner
#Read in each standard star spectrum
standards = np.genfromtxt(stdlist, dtype=str)
if standards.size == 1:
standards = np.array([standards])
stdflux = np.genfromtxt(fluxlist, dtype=str)
if stdflux.size == 1:
stdflux = np.array(
[stdflux]
) #save stdflux explicitly as an array so you can index if only 1 element
#Check that the files are set up correctly to avoid mixing standards.
#This checks that the files in liststandard have similar characters to those in listflux and the correct order. But might break if flux file doesn't match. E.G. mcd32d9927.dat is often called CD-32_9927 in our system.
'''
onion = 0
for stanspec in standards:
quickcheck = stdflux[onion//2].lower()[1:-4] in stanspec.lower()
if not quickcheck:
print 'Check your standard star and flux files. They are mixed up.'
sys.exit()
onion += 1
'''
orderused = np.zeros([len(standards)])
senspolys = []
airstd = np.zeros([len(standards)])
allexcluded = [[None] for i in range(len(standards))]
#Calculating the sensitivity function of each standard star
cucumber = 0
for stdspecfile in standards:
print stdspecfile
#Read in the observed spectrum of the standard star
obs_spectra, airmass, exptime, dispersion = st.readspectrum(
stdspecfile
) #obs_spectra is an object containing opfarr,farr,sky,sigma,warr
airstd[cucumber] = airmass
#plt.clf()
#plt.plot(obs_spectra.warr,obs_spectra.opfarr)
#plt.show()
#Do the extinction correction
if extinct_correct:
print 'Extinction correcting spectra.'
plt.clf()
plt.plot(obs_spectra.warr, obs_spectra.opfarr)
obs_spectra.opfarr = st.extinction_correction(
obs_spectra.warr, obs_spectra.opfarr, airmass)
plt.plot(obs_spectra.warr, obs_spectra.opfarr)
#plt.show()
#Change to the standard star directory
cwd = os.getcwd()
os.chdir(
'/afs/cas.unc.edu/depts/physics_astronomy/clemens/students/group/standards'
)
#read in the standard file
placeholder = cucumber // 2
stdfile = stdflux[placeholder]
std_spectra = st.readstandard(stdfile)
os.chdir(cwd)
#plt.clf()
#plt.plot(std_spectra.warr,std_spectra.magarr,'.')
#plt.show()
#Only keep the part of the standard file that overlaps with observation.
lowwv = np.where(std_spectra.warr >= np.min(obs_spectra.warr))
lowwv = np.asarray(lowwv)
highwv = np.where(std_spectra.warr <= np.max(obs_spectra.warr))
highwv = np.asarray(highwv)
index = np.intersect1d(lowwv, highwv)
std_spectra.warr = std_spectra.warr[index]
std_spectra.magarr = std_spectra.magarr[index]
std_spectra.wbin = std_spectra.wbin[index]
#Convert from AB mag to fnu, then to fwave (ergs/s/cm2/A)
stdzp = 3.68e-20 #The absolute flux per unit frequency at an AB mag of zero
std_spectra.magarr = st.magtoflux(std_spectra.magarr, stdzp)
std_spectra.magarr = st.fnutofwave(std_spectra.warr,
std_spectra.magarr)
#plt.clf()
#plt.plot(std_spectra.warr,std_spectra.magarr,'.')
#plt.show()
#np.savetxt('hz4_stan.txt',np.transpose([std_spectra.warr,std_spectra.magarr]))
#exit()
#We want to rebin the observed spectrum to match with the bins in the standard file. This makes summing up counts significantly easier.
#Set the new binning here.
print 'Starting to rebin: ', stdspecfile
low = np.rint(np.min(obs_spectra.warr)) #Rounds to nearest integer
high = np.rint(np.max(obs_spectra.warr))
size = 0.05 #size in Angstroms you want each bin
num = (
high - low
) / size + 1. #number of bins. Must add one to get correct number.
wavenew = np.linspace(low, high,
num=num) #wavelength of each new bin
#Now do the rebinning using Ian Crossfield's rebinning package
binflux = st.resamplespec(wavenew, obs_spectra.warr,
obs_spectra.opfarr,
200.) #200 is the oversampling factor
print 'Done rebinning. Now summing the spectrum into new bins to match', stdfile
#plt.clf()
#plt.plot(obs_spectra.warr,obs_spectra.opfarr)
#plt.plot(wavenew,binflux)
#plt.show()
#Now sum the rebinned spectra into the same bins as the standard star file
counts = st.sum_std(std_spectra.warr, std_spectra.wbin, wavenew,
binflux)
#plt.clf()
#plt.plot(std_spectra.warr,std_spectra.magarr)
#plt.plot(obs_spectra.warr,obs_spectra.opfarr,'b')
#plt.plot(std_spectra.warr,counts,'g+')
#plt.show()
#Calculate the sensitivity function
sens_function = st.sensfunc(counts, std_spectra.magarr, exptime,
std_spectra.wbin, airmass)
#plt.clf()
#plt.plot(std_spectra.warr,sens_function)
#plt.show()
#sys.exit()
#Fit a low order polynomial to this function so that it is smooth.
#The sensitivity function is in units of 2.5 * log10[counts/sec/Ang / ergs/cm2/sec/Ang]
#Choose regions to not include in fit, first by checking if a mask file exists, and if not the prompt for user interaction.
if 'blue' in stdspecfile.lower():
std_mask = stdfile[0:-4] + '_blue_maskasdf.dat'
if 'red' in stdspecfile.lower():
std_mask = stdfile[0:-4] + '_red_maskasdf.dat'
std_mask2 = glob(std_mask)
if len(std_mask2) == 1.:
print 'Found mask file.\n'
mask = np.ones(len(std_spectra.warr))
excluded_wave = np.genfromtxt(
std_mask) #Read in wavelengths to exclude
#print excluded_wave
#print type(excluded_wave)
#Find index of each wavelength
excluded = []
for x in excluded_wave:
#print x
#print np.where(std_spectra.warr == find_nearest(std_spectra.warr,x))
pix_val = np.where(
std_spectra.warr == find_nearest(std_spectra.warr, x))
excluded.append(pix_val[0][0])
#print excluded
lettuce = 0
while lettuce < len(excluded):
mask[excluded[lettuce]:excluded[lettuce + 1] + 1] = 0
lettuce += 2
excluded = np.array(excluded).tolist()
allexcluded[cucumber] = excluded
indices = np.where(mask != 0.)
lambdasfit = std_spectra.warr[indices]
fluxesfit = sens_function[indices]
else:
print 'No mask found. User interaction required.\n'
global ax, fig, coords
coords = []
plt.clf()
fig = plt.figure(1)
ax = fig.add_subplot(111)
ax.plot(std_spectra.warr, sens_function)
cid = fig.canvas.mpl_connect('button_press_event', onclick)
print 'Please click on both sides of regions you want to exclude. Then close the plot.'
plt.title(
'Click both sides of regions you want to exclude. Then close the plot.'
)
plt.show(1)
#Mask our the regions you don't want to fit
#We need make sure left to right clicking and right to left clicking both work.
mask = np.ones(len(std_spectra.warr))
excluded = np.zeros(len(coords))
lettuce = 0
if len(coords) > 0:
while lettuce < len(coords):
x1 = np.where(std_spectra.warr == (find_nearest(
std_spectra.warr, coords[lettuce][0])))
excluded[lettuce] = np.asarray(x1)
lettuce += 1
x2 = np.where(std_spectra.warr == (find_nearest(
std_spectra.warr, coords[lettuce][0])))
if x2 < x1:
x1, x2 = x2, x1
mask[
x1[0][0]:x2[0][0] +
1] = 0 #have to add 1 here to the second index so that we exclude through that index. Most important for when we need to exclude the last point of the array.
excluded[lettuce - 1] = np.asarray(x1)
excluded[lettuce] = np.asarray(x2)
lettuce += 1
excluded = np.array(excluded).tolist()
allexcluded[cucumber] = excluded
indices = np.where(mask != 0.)
lambdasfit = std_spectra.warr[indices]
fluxesfit = sens_function[indices]
#Save masked wavelengths
lambdasnotfit = std_spectra.warr[excluded]
#print lambdasnotfit
#print stdfile
if 'blue' in stdspecfile.lower():
std_mask_name = stdfile[0:-4] + '_blue_mask.dat'
if 'red' in stdspecfile.lower():
std_mask_name = stdfile[0:-4] + '_red_mask.dat'
np.savetxt(std_mask_name,
np.transpose(np.array(lambdasnotfit)))
#exit()
##Move back to directory with observed spectra
#os.chdir(cwd)
#Make sure they are finite
ind1 = np.isfinite(lambdasfit) & np.isfinite(fluxesfit)
lambdasfit = lambdasfit[ind1]
fluxesfit = fluxesfit[ind1]
print 'Fitting the sensitivity funtion now.'
order = 4
repeat = 'yes'
while repeat == 'yes':
p = np.polyfit(lambdasfit, fluxesfit, order)
f = np.poly1d(p)
smooth_sens = f(lambdasfit)
residual = fluxesfit - smooth_sens
plt.close()
plt.ion()
g, (ax1, ax2) = plt.subplots(2, sharex=True)
ax1.plot(lambdasfit, fluxesfit, 'b+')
ax1.plot(lambdasfit, smooth_sens, 'r', linewidth=2.0)
ax1.set_ylabel('Sensitivity Function')
ax2.plot(lambdasfit, residual, 'k+')
ax2.set_ylabel('Residuals')
ax1.set_title('Current polynomial order: %s' % order)
g.subplots_adjust(hspace=0)
plt.setp([a.get_xticklabels() for a in g.axes[:-1]],
visible=False)
plt.show()
plt.ioff()
#Save this sensitivity curve
'''
try:
temp_file = fits.open(stdspecfile)
ADCstat = temp_file[0].header['ADCSTAT']
except:
ADCstat = 'none'
pass
if 'blue' in stdspecfile.lower():
resp_name = 'senscurve_' + stdfile[1:-4] + '_' + str(np.round(airstd[cucumber],decimals=3)) + '_' + ADCstat + '_' + cwd[60:70] + '_blue.txt'
elif 'red' in stdspecfile.lower():
resp_name = 'senscurve_' + stdfile[1:-4] + '_' + str(np.round(airstd[cucumber],decimals=3)) + '_' + ADCstat + '_' + cwd[60:70] + '_red.txt'
print resp_name
#exit()
np.savetxt(resp_name,np.transpose([lambdasfit,fluxesfit]))
'''
repeat = raw_input('Do you want to try again (yes/no)? ')
if repeat == 'yes':
order = raw_input('New order for polynomial: ')
orderused[cucumber] = order
senspolys.append(f)
#Save arrays for diagnostic plots
if cucumber == 0:
bigarray = np.zeros([len(lambdasfit), 4. * len(standards)])
artichoke = 0
bigarray[0:len(lambdasfit), artichoke] = lambdasfit
bigarray[0:len(fluxesfit), artichoke + 1] = fluxesfit
bigarray[0:len(smooth_sens), artichoke + 2] = smooth_sens
bigarray[0:len(residual), artichoke + 3] = residual
artichoke += 4
cucumber += 1
#Save fit and residuals into text file for diagnostic plotting later.
#Need to save lambdasfit,fluxesfit,smooth_sens,residual for each standard
#List of standards is found as standards
now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
with open('sens_fits_' + now + '.txt', 'a') as handle:
header = str(
standards
) + '\n Set of four columns correspond to wavelength, observed flux, polynomial fit, \n and residuals for each standard listed above. \n You will probably need to strip zeros from the bottoms of some columns.'
np.savetxt(handle, bigarray, fmt='%f', header=header)
#Outline for next steps:
#Read in both red and blue files
#compute airmass and compare to airstd
#choose best standard and flux calibrate both blue and red
#save files and write to sensitivity_params.txt
if speclist[-4:] == 'fits':
specfile = np.array([speclist])
else:
specfile = np.genfromtxt(speclist, dtype=str)
if specfile.size == 1:
specfile = np.array([specfile])
length = len(specfile)
airwd = np.zeros([length])
bean = 0
#if length == 1:
# redfile = False
#else:
# redfile = True
avocado = 0
while avocado < length:
#Read in the blue and red spectra we want to flux calibrate. Save the airmass
WD_spectra1, airmass1, exptime1, dispersion1 = st.readspectrum(
specfile[avocado])
if (len(specfile) >= 1) and (avocado + 1 < length):
if 'red' in specfile[avocado + 1]:
redfile = True
else:
redfile = False
else:
redfile = False
if redfile:
WD_spectra2, airmass2, exptime2, dispersion2 = st.readspectrum(
specfile[avocado + 1])
#Extinction correct WD
if extinct_correct:
print 'Extinction correcting spectra.'
#plt.clf()
#plt.plot(WD_spectra1.warr,WD_spectra1.opfarr)
WD_spectra1.opfarr = st.extinction_correction(
WD_spectra1.warr, WD_spectra1.opfarr, airmass1)
WD_spectra1.farr = st.extinction_correction(
WD_spectra1.warr, WD_spectra1.farr, airmass1)
WD_spectra1.sky = st.extinction_correction(WD_spectra1.warr,
WD_spectra1.sky,
airmass1)
WD_spectra1.sigma = st.extinction_correction(
WD_spectra1.warr, WD_spectra1.sigma, airmass1)
#plt.plot(WD_spectra1.warr,WD_spectra1.opfarr)
#plt.show()
if redfile:
#plt.clf()
#plt.plot(WD_spectra2.warr,WD_spectra2.opfarr)
WD_spectra2.opfarr = st.extinction_correction(
WD_spectra2.warr, WD_spectra2.opfarr, airmass2)
WD_spectra2.farr = st.extinction_correction(
WD_spectra2.warr, WD_spectra2.farr, airmass2)
WD_spectra2.sky = st.extinction_correction(
WD_spectra2.warr, WD_spectra2.sky, airmass2)
WD_spectra2.sigma = st.extinction_correction(
WD_spectra2.warr, WD_spectra2.sigma, airmass2)
#zaplt.plot(WD_spectra2.warr,WD_spectra2.opfarr)
#plt.show()
airwd[avocado] = airmass1
if redfile:
airwd[avocado + 1] = airmass2
#Compare the airmasses to determine the best standard star
tomato = 0
while tomato < len(airstd):
if redfile:
diff = np.absolute(
np.mean([airwd[avocado], airwd[avocado + 1]]) -
np.mean([airstd[tomato], airstd[tomato + 1]]))
else:
diff = np.absolute(airwd[avocado] - airstd[tomato])
if tomato == 0:
difference = diff
choice = tomato
if diff < difference:
difference = diff
choice = tomato
tomato += 2
#To get the flux calibration, perform the following
#Flux = counts / (Exptime * dispersion * 10**(sens/2.5))
#Get the sensitivity function at the correct wavelength spacing
if masterresp:
header_temp = st.readheader(specfile[avocado])
ADCstatus = header_temp['ADCSTAT']
if ADCstatus == 'IN':
sens_wave1_unscale = master_response_blue_in_pol(
WD_spectra1.warr)
blue_low_index = np.min(np.where(WD_spectra1.warr > 4530.))
blue_high_index = np.min(np.where(WD_spectra1.warr > 4590.))
blue_mean_stan = np.mean(
sens_wave1_unscale[blue_low_index:blue_high_index])
if blue_mean_tonight == None:
sens_wave1 = sens_wave1_unscale
else:
sens_wave1 = sens_wave1_unscale + (blue_mean_tonight -
blue_mean_stan)
choice = 0
else:
sens_wave1_unscale = master_response_blue_out_pol(
WD_spectra1.warr)
blue_low_index = np.min(np.where(WD_spectra1.warr > 4530.))
blue_high_index = np.min(np.where(WD_spectra1.warr > 4590.))
blue_mean_stan = np.mean(
sens_wave1_unscale[blue_low_index:blue_high_index])
if blue_mean_tonight == None:
sens_wave1 = sens_wave1_unscale
else:
sens_wave1 = sens_wave1_unscale + (blue_mean_tonight -
blue_mean_stan)
choice = 1
if redfile:
header_temp = st.readheader(specfile[avocado + 1])
ADCstatus = header_temp['ADCSTAT']
if ADCstatus == 'IN':
sens_wave2_unscale = master_response_red_in_pol(
WD_spectra2.warr)
red_low_index = np.min(np.where(WD_spectra2.warr > 6090.))
red_high_index = np.min(np.where(WD_spectra2.warr > 6190.))
red_mean_stan = np.mean(
sens_wave2_unscale[red_low_index:red_high_index])
if red_mean_tonight == None:
sens_wave2 = sens_wave2_unscale
else:
sens_wave2 = sens_wave2_unscale + (red_mean_tonight -
red_mean_stan)
choice2 = 2
else:
sens_wave2_unscale = master_response_red_out_pol(
WD_spectra2.warr)
red_low_index = np.min(np.where(WD_spectra2.warr > 6090.))
red_high_index = np.min(np.where(WD_spectra2.warr > 6190.))
red_mean_stan = np.mean(
sens_wave2_unscale[red_low_index:red_high_index])
if red_mean_tonight == None:
sens_wave2 = sens_wave2_unscale
else:
sens_wave2 = sens_wave2_unscale + (red_mean_tonight -
red_mean_stan)
choice2 = 3
else:
sens_wave1 = senspolys[choice](WD_spectra1.warr)
if redfile:
sens_wave2 = senspolys[choice + 1](WD_spectra2.warr)
#Perform the flux calibration. We do this on the optimal extraction, non-variance weighted aperture, the sky spectrum, and the sigma spectrum.
print 'Doing the final flux calibration.'
#np.savetxt('response_g60-54_extinction_2016-03-17.txt',np.transpose([WD_spectra1.warr,(exptime1 * dispersion1 * 10.**(sens_wave1/2.5))]))#,WD_spectra2.warr,(exptime2 * dispersion2 * 10.**(sens_wave2/2.5))]))
#exit()
star_opflux1 = st.cal_spec(WD_spectra1.opfarr, sens_wave1, exptime1,
dispersion1)
star_flux1 = st.cal_spec(WD_spectra1.farr, sens_wave1, exptime1,
dispersion1)
sky_flux1 = st.cal_spec(WD_spectra1.sky, sens_wave1, exptime1,
dispersion1)
sigma_flux1 = st.cal_spec(WD_spectra1.sigma, sens_wave1, exptime1,
dispersion1)
if redfile:
star_opflux2 = st.cal_spec(WD_spectra2.opfarr, sens_wave2,
exptime2, dispersion2)
star_flux2 = st.cal_spec(WD_spectra2.farr, sens_wave2, exptime2,
dispersion2)
sky_flux2 = st.cal_spec(WD_spectra2.sky, sens_wave2, exptime2,
dispersion2)
sigma_flux2 = st.cal_spec(WD_spectra2.sigma, sens_wave2, exptime2,
dispersion2)
#plt.clf()
#plt.plot(WD_spectra.warr,star_opflux)
#plt.show()
#Save final spectra if using master response
if masterresp:
if avocado == 0:
diagnostic_array = np.zeros(
[len(WD_spectra1.warr), 2 * length])
diagnostic_array[0:len(WD_spectra1.warr), bean] = WD_spectra1.warr
bean += 1
diagnostic_array[0:len(star_opflux1), bean] = star_opflux1
bean += 1
if redfile:
diagnostic_array[0:len(WD_spectra2.warr),
bean] = WD_spectra2.warr
bean += 1
diagnostic_array[0:len(star_opflux2), bean] = star_opflux2
bean += 1
#if avocado == (length -1 ) or (redfile == True and avocado == (length-2)):
# print 'Saveing diagnostic file.'
# now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
# with open('flux_fits_' + now + '.txt','a') as handle:
# header = str(specfile) + '\n Each star is formatted as wavelength, flux'
# np.savetxt(handle,diagnostic_array,fmt='%.10e',header=header)
print 'Saving the final spectrum.'
#Save the flux-calibrated spectrum and update the header
header1 = st.readheader(specfile[avocado])
header1.set('EX-FLAG',
extinctflag) #Extiction correction? 0=yes, -1=no
header1.set('CA-FLAG', 0) #Calibrated to flux scale? 0=yes, -1=no
header1.set('BUNIT', 'erg/cm2/s/A') #physical units of the array value
header1.set(
'STANDARD', str(standards[choice]),
'Flux standard used') #flux standard used for flux-calibration
if masterresp:
header1.set('STDOFF', str(flux_tonight_list[0]),
'Night offset used')
if redfile:
header2 = st.readheader(specfile[avocado + 1])
header2.set('EX-FLAG',
extinctflag) #Extiction correction? 0=yes, -1=no
header2.set('CA-FLAG', 0) #Calibrated to flux scale? 0=yes, -1=no
header2.set('BUNIT',
'erg/cm2/s/A') #physical units of the array value
if masterresp:
header2.set('STANDARD', str(standards[choice2]),
'Flux standard used'
) #flux standard used for flux-calibration
header1.set('STDOFF', str(flux_tonight_list[1]),
'Night offset used')
else:
header2.set('STANDARD', str(standards[choice + 1]),
'Flux standard used'
) #flux standard used for flux-calibration
#Set up size of new fits image
Ni = 4. #Number of extensions
Nx1 = len(star_flux1)
if redfile:
Nx2 = len(star_flux2)
Ny = 1. #All 1D spectra
data1 = np.empty(shape=(Ni, Ny, Nx1))
data1[0, :, :] = star_opflux1
data1[1, :, :] = star_flux1
data1[2, :, :] = sky_flux1
data1[3, :, :] = sigma_flux1
if redfile:
data2 = np.empty(shape=(Ni, Ny, Nx2))
data2[0, :, :] = star_opflux2
data2[1, :, :] = star_flux2
data2[2, :, :] = sky_flux2
data2[3, :, :] = sigma_flux2
#Add '_flux' to the end of the filename
loc1 = specfile[avocado].find('.ms.fits')
if masterresp:
newname1 = specfile[avocado][0:loc1] + '_flux_' + stdflux[0][
1:-4] + '.ms.fits'
else:
newname1 = specfile[avocado][0:loc1] + '_flux_' + stdflux[
choice // 2][1:-4] + '.ms.fits'
clob = False
mylist = [True for f in os.listdir('.') if f == newname1]
exists = bool(mylist)
if exists:
print 'File %s already exists.' % newname1
nextstep = raw_input(
'Do you want to overwrite or designate a new name (overwrite/new)? '
)
if nextstep == 'overwrite':
clob = True
exists = False
elif nextstep == 'new':
newname1 = raw_input('New file name: ')
exists = False
else:
exists = False
print 'Saving: ', newname1
newim1 = fits.PrimaryHDU(data=data1, header=header1)
newim1.writeto(newname1, clobber=clob)
if redfile:
loc2 = specfile[avocado + 1].find('.ms.fits')
if masterresp:
newname2 = specfile[
avocado +
1][0:loc2] + '_flux_' + stdflux[0][1:-4] + '.ms.fits'
else:
newname2 = specfile[avocado + 1][0:loc2] + '_flux_' + stdflux[
choice // 2][1:-4] + '.ms.fits'
clob = False
mylist = [True for f in os.listdir('.') if f == newname2]
exists = bool(mylist)
if exists:
print 'File %s already exists.' % newname2
nextstep = raw_input(
'Do you want to overwrite or designate a new name (overwrite/new)? '
)
if nextstep == 'overwrite':
clob = True
exists = False
elif nextstep == 'new':
newname2 = raw_input('New file name: ')
exists = False
else:
exists = False
newim2 = fits.PrimaryHDU(data=data2, header=header2)
newim2.writeto(newname2, clobber=clob)
print 'Saving: ', newname2
#Finally, save all the used parameters into a file for future reference.
# specfile,current date, stdspecfile,stdfile,order,size,newname
f = open('sensitivity_params.txt', 'a')
now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
if masterresp:
newinfo1 = specfile[avocado] + '\t' + now + '\t' + standards[
choice] + '\t' + stdflux[0] + '\t' + str(
allexcluded[choice]) + '\t' + str(
orderused[choice]) + '\t' + str(size) + '\t' + newname1
else:
newinfo1 = specfile[avocado] + '\t' + now + '\t' + standards[
choice] + '\t' + stdflux[choice // 2] + '\t' + str(
allexcluded[choice]) + '\t' + str(
orderused[choice]) + '\t' + str(size) + '\t' + newname1
if redfile:
if masterresp:
newinfo2 = specfile[
avocado + 1] + '\t' + now + '\t' + standards[
choice2] + '\t' + stdflux[0] + '\t' + str(
allexcluded[choice + 1]) + '\t' + str(
orderused[choice + 1]) + '\t' + str(
size) + '\t' + newname2
else:
newinfo2 = specfile[
avocado + 1] + '\t' + now + '\t' + standards[
choice + 1] + '\t' + stdflux[choice // 2] + '\t' + str(
allexcluded[choice + 1]) + '\t' + str(
orderused[choice + 1]) + '\t' + str(
size) + '\t' + newname2
f.write(newinfo1 + "\n" + newinfo2 + "\n")
else:
f.write(newinfo1 + "\n")
f.close()
if redfile:
avocado += 2
else:
avocado += 1
print 'Done flux calibrating the spectra.'
