def agncalibrate(img, outfile, calfile, specformat='lcogt'):
#set up some files that will be needed
logfile='specext.log'
hdu = fits.open(img)
w1 = hdu[0].header['CRVAL1']
p1 = hdu[0].header['CRPIX1']
dw = hdu[0].header['CD1_1']
f = hdu[0].data[0][0]
e = hdu[0].data[3][0]
xarr = np.arange(len(f))
w = (xarr)*dw+w1
cal_spectra=st.readspectrum(calfile, error=False, ftype='ascii')
airmass=hdu[0].header['AIRMASS']
exptime=hdu[0].header['EXPTIME']
extfile=iraf.osfn("pysalt$data/site/suth_extinct.dat")
ext_spectra=st.readspectrum(extfile, error=False, ftype='ascii')
flux_spec=Spectrum.Spectrum(w, f, e, stype='continuum')
flux_spec=calfunc(flux_spec, cal_spectra, ext_spectra, airmass, exptime, True)
hdu[0].data[0][0] = flux_spec.flux
hdu[0].data[3][0] = flux_spec.var
hdu.writeto(outfile, clobber=True)
def extract_spectra(hdu, yc, dy, outfile, ext=1, minsize=5, thresh=3, grow=0, smooth=False, maskzeros=False,
convert=True, cleanspectra=True, calfile=None, clobber=True, specformat='ascii'):
"""From an image, extract a spectra.
"""
data=hdu[ext].data
#replace the zeros with the average from the frame
if maskzeros:
mean,std=iterstat(data[data>0])
#rdata=mean np.random.normal(mean, std, size=data.shape)
data[data<=0]=mean #rdata[data<=0]
y1=yc-dy
y2=yc+dy
ap_list=extract(hdu, method='normal', section=[(y1,y2)], minsize=minsize, thresh=thresh, convert=convert)
sy1a=y2
sy2a=sy1a+2.0*dy
ska_list=extract(hdu, method='normal', section=[(sy1a,sy2a)], minsize=minsize, thresh=thresh, convert=convert)
sy2b=y1-dy
sy1b=sy2b-2.0*dy
skb_list=extract(hdu, method='normal', section=[(sy1b,sy2b)], minsize=minsize, thresh=thresh, convert=convert)
print sy1b, sy2b
sdata = 0.5*(ska_list[0].ldata/(sy2a-sy1a) + skb_list[0].ldata/(sy2b-sy1b))
#sdata = ska_list[0].ldata/(sy2a-sy1a)
#sdata = skb_list[0].ldata/(sy2b-sy1b)
raw = 1.0 * ap_list[0].ldata
print 'extract:', ap_list[0].ldata[1124]
ap_list[0].ldata=ap_list[0].ldata-float(y2-y1) * sdata
print 'sky:', ap_list[0].ldata[1124]
print ap_list[0].wave[10], ap_list[0].ldata[10], ap_list[0].lvar[10]
flux_spec=Spectrum.Spectrum(ap_list[0].wave, ap_list[0].ldata, abs(ap_list[0].lvar)**0.5, stype='continuum')
print flux_spec.wavelength[10], flux_spec.flux[10], flux_spec.var[10]
if cleanspectra:
clean_spectra(ap_list[0], grow=grow)
print 'clean:', ap_list[0].ldata[1124]
if calfile:
cal_spectra=st.readspectrum(calfile, error=False, ftype='ascii')
airmass=hdu[0].header['AIRMASS']
exptime=hdu[0].header['EXPTIME']
extfile=os.path.dirname(st.__file__)+"/suth_extinct.dat"
print extfile
ext_spectra=st.readspectrum(extfile, error=False, ftype='ascii')
flux_spec=Spectrum.Spectrum(ap_list[0].wave, ap_list[0].ldata, abs(ap_list[0].lvar)**0.5, stype='continuum')
print flux_spec.flux[10], flux_spec.var[10]
flux_spec=calfunc(flux_spec, cal_spectra, ext_spectra, airmass, exptime, True)
print flux_spec.flux[10], flux_spec.var[10]
else:
flux_spec = Spectrum.Spectrum(ap_list[0].wave, ap_list[0].ldata, abs(ap_list[0].lvar)**0.5, stype='continuum')
if specformat == 'ascii':
write_ascii(outfile, flux_spec, clobber=clobber)
elif specformat == 'lcogt':
write_lcogt(outfile, flux_spec, hdu, sky=float(y2-y1) * sdata, raw = raw, clobber=clobber)
def specsens(specfile, outfile, stdfile, extfile, airmass=None, exptime=None,
stdzp=3.68e-20, function='polynomial', order=3, thresh=3, niter=5,
clobber=True, logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
#read in the specfile and create a spectrum object
obs_spectra=st.readspectrum(specfile, error=True, ftype='ascii')
#read in the std file and convert from magnitudes to fnu
#then convert it to fwave (ergs/s/cm2/A)
std_spectra=st.readspectrum(stdfile, error=False, ftype='ascii')
std_spectra.flux=Spectrum.magtoflux(std_spectra.flux, stdzp)
std_spectra.flux=Spectrum.fnutofwave(std_spectra.wavelength, std_spectra.flux)
#read in the extinction file (leave in magnitudes)
ext_spectra=st.readspectrum(extfile, error=False, ftype='ascii')
#determine the airmass if not specified
if saltio.checkfornone(airmass) is None:
message='Airmass was not supplied'
raise SALTSpecError(message)
#determine the exptime if not specified
if saltio.checkfornone(airmass) is None:
message='Exposure Time was not supplied'
raise SALTSpecError(message)
#calculate the calibrated spectra
log.message('Calculating the calibration curve for %s' % specfile)
cal_spectra=sensfunc(obs_spectra, std_spectra, ext_spectra, airmass, exptime)
#fit the spectra--first take a first cut of the spectra
#using the median absolute deviation to throw away bad points
cmed=np.median(cal_spectra.flux)
cmad=saltstat.mad(cal_spectra.flux)
mask=(abs(cal_spectra.flux-cmed)<thresh*cmad)
mask=(cal_spectra.flux>0)
#now fit the data
fit=interfit(cal_spectra.wavelength[mask], cal_spectra.flux[mask], function=function, order=order, thresh=thresh, niter=niter)
fit.interfit()
#print 'plotting...'
#figure()
#plot(cal_spectra.wavelength, cal_spectra.flux)
#plot(obs_spectra.wavelength, obs_spectra.flux*cal_spectra.flux.mean()/obs_spectra.flux.mean())
#plot(std_spectra.wavelength, std_spectra.flux*cal_spectra.flux.mean()/std_spectra.flux.mean())
#plot(cal_spectra.wavelength, fit(cal_spectra.wavelength))
#show()
#write the spectra out
cal_spectra.flux=fit(cal_spectra.wavelength)
st.writespectrum(cal_spectra, outfile, ftype='ascii')
def speccal(specfile, outfile, calfile, extfile, airmass=None, exptime=None,
clobber=True, logfile='salt.log', verbose=True):
with logging(logfile, debug) as log:
# read in the specfile and create a spectrum object
obs_spectra = st.readspectrum(specfile, error=True, ftype='ascii')
# read in the std file and convert from magnitudes to fnu
# then convert it to fwave (ergs/s/cm2/A)
cal_spectra = st.readspectrum(calfile, error=False, ftype='ascii')
# read in the extinction file (leave in magnitudes)
ext_spectra = st.readspectrum(extfile, error=False, ftype='ascii')
# determine the airmass if not specified
if saltio.checkfornone(airmass) is None:
message = 'Airmass was not supplied'
raise SALTSpecError(message)
# determine the exptime if not specified
if saltio.checkfornone(airmass) is None:
message = 'Exposure Time was not supplied'
raise SALTSpecError(message)
# calculate the calibrated spectra
log.message('Calculating the calibration curve for %s' % specfile)
error = False
try:
if obs_spectra.var is not None:
error = True
except:
error = False
flux_spectra = calfunc(
obs_spectra,
cal_spectra,
ext_spectra,
airmass,
exptime,
error)
# write the spectra out
st.writespectrum(flux_spectra, outfile, ftype='ascii', error=error)
def speccal(
specfile, outfile, calfile, extfile, airmass=None, exptime=None, clobber=True, logfile="salt.log", verbose=True
):
with logging(logfile, debug) as log:
# read in the specfile and create a spectrum object
obs_spectra = st.readspectrum(specfile, error=True, ftype="ascii")
# read in the std file and convert from magnitudes to fnu
# then convert it to fwave (ergs/s/cm2/A)
cal_spectra = st.readspectrum(calfile, error=False, ftype="ascii")
# read in the extinction file (leave in magnitudes)
ext_spectra = st.readspectrum(extfile, error=False, ftype="ascii")
# determine the airmass if not specified
if saltio.checkfornone(airmass) is None:
message = "Airmass was not supplied"
raise SALTSpecError(message)
# determine the exptime if not specified
if saltio.checkfornone(airmass) is None:
message = "Exposure Time was not supplied"
raise SALTSpecError(message)
# calculate the calibrated spectra
log.message("Calculating the calibration curve for %s" % specfile)
error = False
try:
if obs_spectra.var is not None:
error = True
except:
error = False
flux_spectra = calfunc(obs_spectra, cal_spectra, ext_spectra, airmass, exptime, error)
# write the spectra out
st.writespectrum(flux_spectra, outfile, ftype="ascii", error=error)
def normalize_now(filenameblue,
filenamered,
redfile,
plotall=True,
extinct_correct=False):
#Read in the observed spectrum
obs_spectrablue, airmass, exptime, dispersion = st.readspectrum(
filenameblue)
datalistblue = fits.open(filenameblue)
if redfile:
obs_spectrared, airmassred, exptimered, dispersionred = st.readspectrum(
filenamered)
#Extinction correction
if extinct_correct:
print 'Extinction correcting spectra.'
plt.clf()
plt.plot(obs_spectrablue.warr, obs_spectrablue.opfarr)
obs_spectrablue.opfarr = st.extinction_correction(
obs_spectrablue.warr, obs_spectrablue.opfarr, airmass)
obs_spectrablue.farr = st.extinction_correction(
obs_spectrablue.warr, obs_spectrablue.farr, airmass)
obs_spectrablue.sky = st.extinction_correction(obs_spectrablue.warr,
obs_spectrablue.sky,
airmass)
obs_spectrablue.sigma = st.extinction_correction(
obs_spectrablue.warr, obs_spectrablue.sigma, airmass)
plt.plot(obs_spectrablue.warr, obs_spectrablue.opfarr)
plt.show()
if redfile:
plt.clf()
plt.plot(obs_spectrared.warr, obs_spectrared.opfarr)
obs_spectrared.opfarr = st.extinction_correction(
obs_spectrared.warr, obs_spectrared.opfarr, airmassred)
obs_spectrared.farr = st.extinction_correction(
obs_spectrared.warr, obs_spectrared.farr, airmassred)
obs_spectrared.sky = st.extinction_correction(
obs_spectrared.warr, obs_spectrared.sky, airmassred)
obs_spectrared.sigma = st.extinction_correction(
obs_spectrared.warr, obs_spectrared.sigma, airmassred)
plt.plot(obs_spectrared.warr, obs_spectrared.opfarr)
plt.show()
#Read in measured FWHM from header. This is used to convolve the model spectrum.
FWHMpix = datalistblue[0].header['specfwhm']
FWHM = FWHMpix * (obs_spectrablue.warr[-1] -
obs_spectrablue.warr[0]) / len(obs_spectrablue.warr)
#Read in DA model
cwd = os.getcwd()
os.chdir(
'/afs/cas.unc.edu/depts/physics_astronomy/clemens/students/group/modelfitting/Koester_08'
)
dafile = 'da12500_800.dk'
mod_wav, mod_spec = np.genfromtxt(dafile, unpack=True, skip_header=33)
os.chdir(cwd) #Move back to directory with observed spectra
#Convolve the model to match the seeing of the spectrum
intlambda = np.divide(range(31000), 10.) + 3660.0
lowlambda = np.min(np.where(mod_wav > 3600.))
highlambda = np.min(np.where(mod_wav > 6800.))
shortlambdas = mod_wav[lowlambda:highlambda]
shortinten = mod_spec[lowlambda:highlambda]
interp = inter.InterpolatedUnivariateSpline(shortlambdas, shortinten, k=1)
intflux = interp(intlambda)
sig = FWHM / (2. * np.sqrt(2. * np.log(2.)))
gx = np.divide(range(360), 10.)
gauss = (1. / (sig * np.sqrt(2. * np.pi))) * np.exp(-(gx - 18.)**2. /
(2. * sig**2.))
gf = np.divide(np.outer(intflux, gauss), 10.)
length = len(intflux) - 360.
cflux = np.zeros(length)
clambda = intlambda[180:len(intlambda) - 180]
x = 0
while x < length:
cflux[x] = np.sum(np.diagonal(gf, x, axis1=1, axis2=0), dtype='d')
x += 1
interp2 = inter.InterpolatedUnivariateSpline(clambda, cflux, k=1)
cflux2blue = interp2(obs_spectrablue.warr)
cflux2blue /= 10**13. #Divide by 10**13 to scale
if redfile:
cflux2red = interp2(obs_spectrared.warr)
cflux2red /= 10**13. #Divide by 10**13 to scale
#plt.clf()
#plt.plot(obs_spectrablue.warr,obs_spectrablue.opfarr,'b')
#plt.plot(obs_spectrablue.warr,cflux2blue,'r')
#if redfile:
# plt.plot(obs_spectrared.warr,obs_spectrared.opfarr,'b')
# plt.plot(obs_spectrared.warr,cflux2red,'r')
#plt.show()
#The response function is the observed spectrum divided by the model spectrum.
response_blue = obs_spectrablue.opfarr / cflux2blue
if redfile:
response_red = obs_spectrared.opfarr / cflux2red
'''
plt.clf()
plt.plot(obs_spectrablue.warr,response_blue,'k')
if redfile:
plt.plot(obs_spectrared.warr,response_red,'k')
plt.show()
'''
#We want to mask out the Balmer line features, and the telluric line in the red spectrum. Set up the wavelength ranges to mask here.
#balmer_features_blue = [[3745,3757],[3760,3780],[3784,3812],[3816,3856],[3865,3921],[3935,4021],[4040,4191],[4223,4460],[4691,5010]] #Keeping ends
balmer_features_blue = [[3400, 3700], [3745, 3757], [3760, 3780],
[3784, 3812], [3816, 3856], [3865, 3921],
[3935, 4021], [4040, 4191], [4223, 4460],
[4691, 5010], [5140, 5500]] #Discarding ends
balmer_features_red = [[6350, 6780], [6835, 6970]]
balmer_mask_blue = obs_spectrablue.warr == obs_spectrablue.warr
for wavrange in balmer_features_blue:
inds = np.where((obs_spectrablue.warr > wavrange[0])
& (obs_spectrablue.warr < wavrange[1]))
balmer_mask_blue[inds] = False
if redfile:
balmer_mask_red = obs_spectrared.warr == obs_spectrared.warr
for wavrange in balmer_features_red:
indxs = np.where((obs_spectrared.warr > wavrange[0])
& (obs_spectrared.warr < wavrange[1]))
balmer_mask_red[indxs] = False
spec_wav_masked_blue = obs_spectrablue.warr[balmer_mask_blue]
response_masked_blue = response_blue[balmer_mask_blue]
if redfile:
spec_wav_masked_red = obs_spectrared.warr[balmer_mask_red]
response_masked_red = response_red[balmer_mask_red]
#Fit the response function with a polynomial. The order of polynomial is specified first.
response_poly_order_blue = 7.
response_fit_blue_poly = np.polyfit(spec_wav_masked_blue,
response_masked_blue,
response_poly_order_blue)
response_fit_blue = np.poly1d(response_fit_blue_poly)
if redfile:
response_poly_order_red = 3.
response_fit_red_poly = np.polyfit(spec_wav_masked_red,
response_masked_red,
response_poly_order_red)
response_fit_red = np.poly1d(response_fit_red_poly)
#Save response function
#np.savetxt('response_model_no_extinction.txt',np.transpose([obs_spectrablue.warr,response_fit_blue(obs_spectrablue.warr),obs_spectrared.warr,response_fit_red(obs_spectrared.warr)]))
#plt.clf()
#plt.plot(obs_spectrablue.warr,response_fit_blue(obs_spectrablue.warr)/response_fit_blue(obs_spectrablue.warr)[1000])
#plt.show()
#exit()
if plotall:
plt.clf()
plt.plot(obs_spectrablue.warr, response_blue, 'r')
plt.plot(spec_wav_masked_blue, response_masked_blue, 'g.')
plt.plot(obs_spectrablue.warr, response_fit_blue(obs_spectrablue.warr),
'k--')
#plt.show()
#plt.clf()
if redfile:
plt.plot(obs_spectrared.warr, response_red, 'r')
plt.plot(spec_wav_masked_red, response_masked_red, 'g.')
plt.plot(obs_spectrared.warr,
response_fit_red(obs_spectrared.warr), 'k--')
plt.show()
#Divide by the fit to the response function to get the continuum normalized spectra. Divide every extension by the same polynomial
fcorr_wd_blue_opfarr = obs_spectrablue.opfarr / response_fit_blue(
obs_spectrablue.warr)
fcorr_wd_blue_farr = obs_spectrablue.farr / response_fit_blue(
obs_spectrablue.warr)
fcorr_wd_blue_sky = obs_spectrablue.sky / response_fit_blue(
obs_spectrablue.warr)
fcorr_wd_blue_sigma = obs_spectrablue.sigma / response_fit_blue(
obs_spectrablue.warr)
if redfile:
fcorr_wd_red_opfarr = obs_spectrared.opfarr / response_fit_red(
obs_spectrared.warr)
fcorr_wd_red_farr = obs_spectrared.farr / response_fit_red(
obs_spectrared.warr)
fcorr_wd_red_sky = obs_spectrared.sky / response_fit_red(
obs_spectrared.warr)
fcorr_wd_red_sigma = obs_spectrared.sigma / response_fit_red(
obs_spectrared.warr)
if plotall:
plt.clf()
plt.plot(obs_spectrablue.warr, fcorr_wd_blue_opfarr, 'b')
if redfile:
plt.plot(obs_spectrared.warr, fcorr_wd_red_opfarr, 'r')
plt.show()
#exit()
#Save parameters for diagnostics
if redfile:
bigarray = np.zeros([len(obs_spectrablue.warr), 12])
bigarray[0:len(obs_spectrablue.warr), 0] = obs_spectrablue.warr
bigarray[0:len(response_blue), 1] = response_blue
bigarray[0:len(spec_wav_masked_blue), 2] = spec_wav_masked_blue
bigarray[0:len(response_masked_blue), 3] = response_masked_blue
bigarray[0:len(response_fit_blue(obs_spectrablue.warr)),
4] = response_fit_blue(obs_spectrablue.warr)
bigarray[0:len(fcorr_wd_blue_opfarr), 5] = fcorr_wd_blue_opfarr
bigarray[0:len(obs_spectrared.warr), 6] = obs_spectrared.warr
bigarray[0:len(response_red), 7] = response_red
bigarray[0:len(spec_wav_masked_red), 8] = spec_wav_masked_red
bigarray[0:len(response_masked_red), 9] = response_masked_red
bigarray[0:len(response_fit_red(obs_spectrared.warr)),
10] = response_fit_red(obs_spectrared.warr)
bigarray[0:len(fcorr_wd_red_opfarr), 11] = fcorr_wd_red_opfarr
now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
endpoint = '_930'
with open(
'continuum_normalization_' +
filenameblue[5:filenameblue.find(endpoint)] + '_' + now +
'.txt', 'a') as handle:
header = str(filenameblue) + ',' + str(
filenamered
) + ',' + dafile + '\n Columns structured as blue then red. If no red file, only blue data given. Columns are: blue wavelengths, blue response all data, blue masked wavelengths, blue masked response data, blue response fit, blue continuum-normalize flux, red wavelengths, red response all data, red masked wavelengths, red masked response data, red response fit, red continuum-normalized flux'
np.savetxt(handle, bigarray, fmt='%f', header=header)
if not redfile:
bigarray = np.zeros([len(obs_spectrablue.warr), 6])
bigarray[0:len(obs_spectrablue.warr), 0] = obs_spectrablue.warr
bigarray[0:len(response_blue), 1] = response_blue
bigarray[0:len(spec_wav_masked_blue), 2] = spec_wav_masked_blue
bigarray[0:len(response_masked_blue), 3] = response_masked_blue
bigarray[0:len(response_fit_blue(obs_spectrablue.warr)),
4] = response_fit_blue(obs_spectrablue.warr)
bigarray[0:len(fcorr_wd_blue_opfarr), 5] = fcorr_wd_blue_opfarr
now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
endpoint = '_930'
with open(
'continuum_normalization_' +
filenameblue[5:filenameblue.find(endpoint)] + '_' + now +
'.txt', 'a') as handle:
header = str(
filenameblue
) + ',' + ',' + dafile + '\n Columns structured as blue then red. If no red file, only blue data given. Columns are: blue wavelengths, blue response all data, blue masked wavelengths, blue masked response data, blue response fit, blue continuum-normalized flux'
np.savetxt(handle, bigarray, fmt='%f', header=header)
#Save the continuum normalized spectra here.
Ni = 4. #Number of extensions
Nx1 = len(fcorr_wd_blue_opfarr)
if redfile:
Nx2 = len(fcorr_wd_red_opfarr)
Ny = 1. #All 1D spectra
#Update header
header1 = st.readheader(filenameblue)
header1.set('STANDARD', dafile, 'DA Model for Continuum Calibration')
header1.set('RESPPOLY', response_poly_order_blue,
'Polynomial order for Response Function')
header1.set('DATENORM',
datetime.datetime.now().strftime("%Y-%m-%d"),
'Date of Continuum Normalization')
data1 = np.empty(shape=(Ni, Ny, Nx1))
data1[0, :, :] = fcorr_wd_blue_opfarr
data1[1, :, :] = fcorr_wd_blue_farr
data1[2, :, :] = fcorr_wd_blue_sky
data1[3, :, :] = fcorr_wd_blue_sigma
#Check that filename does not already exist. Prompt user for input if it does.
loc1 = filenameblue.find('.ms.fits')
newname1 = filenameblue[0:loc1] + '_flux_model_short.ms.fits'
clob = False
mylist = [True for f in os.listdir('.') if f == newname1]
exists = bool(mylist)
if exists:
print 'File %s already exists.' % newname1
nextstep = raw_input(
'Do you want to overwrite or designate a new name (overwrite/new)? '
)
if nextstep == 'overwrite':
clob = True
exists = False
elif nextstep == 'new':
newname1 = raw_input('New file name: ')
exists = False
else:
exists = False
print 'Writing ', newname1
newim1 = fits.PrimaryHDU(data=data1, header=header1)
newim1.writeto(newname1, clobber=clob)
#Save the red file if it exists.
if redfile:
header2 = st.readheader(filenamered)
header2.set('STANDARD', dafile, 'DA Model for Continuum Calibration')
header2.set('RESPPOLY', response_poly_order_red,
'Polynomial order for Response Function')
header2.set('DATENORM',
datetime.datetime.now().strftime("%Y-%m-%d"),
'Date of Continuum Normalization')
data2 = np.empty(shape=(Ni, Ny, Nx2))
data2[0, :, :] = fcorr_wd_red_opfarr
data2[1, :, :] = fcorr_wd_red_farr
data2[2, :, :] = fcorr_wd_red_sky
data2[3, :, :] = fcorr_wd_red_sigma
loc2 = filenamered.find('.ms.fits')
newname2 = filenamered[0:loc2] + '_flux_model_short.ms.fits'
clob = False
mylist = [True for f in os.listdir('.') if f == newname2]
exists = bool(mylist)
if exists:
print 'File %s already exists.' % newname2
nextstep = raw_input(
'Do you want to overwrite or designate a new name (overwrite/new)? '
)
if nextstep == 'overwrite':
clob = True
exists = False
elif nextstep == 'new':
newname2 = raw_input('New file name: ')
exists = False
else:
exists = False
print 'Writing ', newname2
newim2 = fits.PrimaryHDU(data=data2, header=header2)
newim2.writeto(newname2, clobber=clob)
def extract_spectra(hdu, yc, dy, outfile, minsize=5, thresh=3, grow=0, smooth=False, maskzeros=False,
convert=True, cleanspectra=True, calfile=None, clobber=True, specformat='ascii'):
"""From an image, extract a spectra.
"""
data=hdu[1].data
#replace the zeros with the average from the frame
if maskzeros:
mean,std=iterstat(data[data>0])
#rdata=mean np.random.normal(mean, std, size=data.shape)
data[data<=0]=mean #rdata[data<=0]
y1=yc-dy
y2=yc+dy
#sy1=y2-2*dy
#sy2=y2+2*dy
#sdata = 1.0 * data
#y,x = np.indices(sdata.shape)
#for i in range(sdata.shape[1]):
# mask=(hdu[3].data[:,i]==0)
# mask[sy1:sy2] = 0
# if mask.sum()>0:
# sdata[y1:y2,i] = np.interp(y[y1:y2,i], y[:,i][mask], data[:,i][mask])
#hdu[1].data = sdata
#sk_list=extract(hdu, method='normal', section=[(y1,y2)], minsize=minsize, thresh=thresh, convert=True)
#ap_list[0].ldata=ap_list[0].ldata-sk_list[0].ldata
#ap_list[0].ldata=ap_list[0].ldata-float(y2-y1)/(sy2-sy1)*sk_list[0].ldata
ap_list=extract(hdu, method='normal', section=[(y1,y2)], minsize=minsize, thresh=thresh, convert=convert)
sy1a=y2
sy2a=sy1a+2.0*dy
ska_list=extract(hdu, method='normal', section=[(sy1a,sy2a)], minsize=minsize, thresh=thresh, convert=convert)
sy2b=y1-dy
sy1b=sy2b-2.0*dy
skb_list=extract(hdu, method='normal', section=[(sy1b,sy2b)], minsize=minsize, thresh=thresh, convert=convert)
print sy1b, sy2b
sdata = 0.5*(ska_list[0].ldata/(sy2a-sy1a) + skb_list[0].ldata/(sy2b-sy1b))
#sdata = ska_list[0].ldata/(sy2a-sy1a)
#sdata = skb_list[0].ldata/(sy2b-sy1b)
raw = 1.0 * ap_list[0].ldata
ap_list[0].ldata=ap_list[0].ldata-float(y2-y1) * sdata
print ap_list[0].wave[10], ap_list[0].ldata[10], ap_list[0].lvar[10]
flux_spec=Spectrum.Spectrum(ap_list[0].wave, ap_list[0].ldata, abs(ap_list[0].lvar)**0.5, stype='continuum')
print flux_spec.wavelength[10], flux_spec.flux[10], flux_spec.var[10]
if cleanspectra:
clean_spectra(ap_list[0], grow=grow)
if calfile:
cal_spectra=st.readspectrum(calfile, error=False, ftype='ascii')
airmass=hdu[0].header['AIRMASS']
exptime=hdu[0].header['EXPTIME']
extfile=iraf.osfn("pysalt$data/site/suth_extinct.dat")
ext_spectra=st.readspectrum(extfile, error=False, ftype='ascii')
flux_spec=Spectrum.Spectrum(ap_list[0].wave, ap_list[0].ldata, abs(ap_list[0].lvar)**0.5, stype='continuum')
print flux_spec.flux[10], flux_spec.var[10]
flux_spec=calfunc(flux_spec, cal_spectra, ext_spectra, airmass, exptime, True)
print flux_spec.flux[10], flux_spec.var[10]
else:
flux_spec = Spectrum.Spectrum(ap_list[0].wave, ap_list[0].ldata, abs(ap_list[0].lvar)**0.5, stype='continuum')
if specformat == 'ascii':
write_ascii(outfile, flux_spec, clobber=clobber)
elif specformat == 'lcogt':
write_lcogt(outfile, flux_spec, hdu, sky=float(y2-y1) * sdata, raw = raw, clobber=clobber)
def specsens(specfile, outfile, stdfile, extfile, airmass=None, exptime=None,
stdzp=3.68e-20, function='polynomial', order=3, thresh=3, niter=5,
fitter='gaussian', clobber=True, logfile='salt.log', verbose=True):
with logging(logfile, debug) as log:
# read in the specfile and create a spectrum object
obs_spectra = st.readspectrum(specfile.strip(), error=True, ftype='ascii')
# smooth the observed spectrum
# read in the std file and convert from magnitudes to fnu
# then convert it to fwave (ergs/s/cm2/A)
std_spectra = st.readspectrum(stdfile.strip(), error=False, ftype='ascii')
std_spectra.flux = Spectrum.magtoflux(std_spectra.flux, stdzp)
std_spectra.flux = Spectrum.fnutofwave(
std_spectra.wavelength, std_spectra.flux)
# Get the typical bandpass of the standard star,
std_bandpass = np.diff(std_spectra.wavelength).mean()
# Smooth the observed spectrum to that bandpass
obs_spectra.flux = st.boxcar_smooth(obs_spectra, std_bandpass)
# read in the extinction file (leave in magnitudes)
ext_spectra = st.readspectrum(extfile.strip(), error=False, ftype='ascii')
# determine the airmass if not specified
if saltio.checkfornone(airmass) is None:
message = 'Airmass was not supplied'
raise SALTSpecError(message)
# determine the exptime if not specified
if saltio.checkfornone(exptime) is None:
message = 'Exposure Time was not supplied'
raise SALTSpecError(message)
# calculate the calibrated spectra
log.message('Calculating the calibration curve for %s' % specfile)
cal_spectra = sensfunc(
obs_spectra, std_spectra, ext_spectra, airmass, exptime)
# plot(cal_spectra.wavelength, cal_spectra.flux * std_spectra.flux)
# fit the spectra--first take a first cut of the spectra
# using the median absolute deviation to throw away bad points
cmed = np.median(cal_spectra.flux)
cmad = saltstat.mad(cal_spectra.flux)
mask = (abs(cal_spectra.flux - cmed) < thresh * cmad)
mask = np.logical_and(mask, (cal_spectra.flux > 0))
# now fit the data
# Fit using a gaussian process.
if fitter=='gaussian':
from sklearn.gaussian_process import GaussianProcess
#Instanciate a Gaussian Process model
dy = obs_spectra.var[mask] ** 0.5
dy /= obs_spectra.flux[mask] / cal_spectra.flux[mask]
y = cal_spectra.flux[mask]
gp = GaussianProcess(corr='squared_exponential', theta0=1e-2,
thetaL=1e-4, thetaU=0.1, nugget=(dy / y) ** 2.0)
X = np.atleast_2d(cal_spectra.wavelength[mask]).T
# Fit to data using Maximum Likelihood Estimation of the parameters
gp.fit(X, y)
x = np.atleast_2d(cal_spectra.wavelength).T
# Make the prediction on the meshed x-axis (ask for MSE as well)
y_pred = gp.predict(x)
cal_spectra.flux = y_pred
else:
fit=interfit(cal_spectra.wavelength[mask], cal_spectra.flux[mask], function=function, order=order, thresh=thresh, niter=niter)
fit.interfit()
cal_spectra.flux=fit(cal_spectra.wavelength)
# write the spectra out
st.writespectrum(cal_spectra, outfile, ftype='ascii')
import os
import sys
import datetime
if len(sys.argv) == 3:
script, filenameblue, filenamered = sys.argv
redfile = True
elif len(sys.argv) == 2:
script, filenameblue = sys.argv
redfile = False
else:
print '\n Incorrect number of arguments. \n'
#Read in the observed spectrum
obs_spectrablue,airmass,exptime,dispersion = st.readspectrum(filenameblue)
datalistblue = fits.open(filenameblue)
if redfile:
obs_spectrared, airmassred,exptimered,dispersionred = st.readspectrum(filenamered)
#Read in measured FWHM from header. This is used to convolve the model spectrum.
FWHMpix = datalistblue[0].header['specfwhm']
FWHM = FWHMpix * (obs_spectrablue.warr[-1] - obs_spectrablue.warr[0])/len(obs_spectrablue.warr)
#Read in DA model
cwd = os.getcwd()
def specsens(specfile,
outfile,
stdfile,
extfile,
airmass=None,
exptime=None,
stdzp=3.68e-20,
function='polynomial',
order=3,
thresh=3,
niter=5,
fitter='gaussian',
clobber=True,
logfile='salt.log',
verbose=True):
with logging(logfile, debug) as log:
# read in the specfile and create a spectrum object
obs_spectra = st.readspectrum(specfile.strip(),
error=True,
ftype='ascii')
# smooth the observed spectrum
# read in the std file and convert from magnitudes to fnu
# then convert it to fwave (ergs/s/cm2/A)
std_spectra = st.readspectrum(stdfile.strip(),
error=False,
ftype='ascii')
std_spectra.flux = Spectrum.magtoflux(std_spectra.flux, stdzp)
std_spectra.flux = Spectrum.fnutofwave(std_spectra.wavelength,
std_spectra.flux)
# Get the typical bandpass of the standard star,
std_bandpass = np.diff(std_spectra.wavelength).mean()
# Smooth the observed spectrum to that bandpass
obs_spectra.flux = st.boxcar_smooth(obs_spectra, std_bandpass)
# read in the extinction file (leave in magnitudes)
ext_spectra = st.readspectrum(extfile.strip(),
error=False,
ftype='ascii')
# determine the airmass if not specified
if saltio.checkfornone(airmass) is None:
message = 'Airmass was not supplied'
raise SALTSpecError(message)
# determine the exptime if not specified
if saltio.checkfornone(exptime) is None:
message = 'Exposure Time was not supplied'
raise SALTSpecError(message)
# calculate the calibrated spectra
log.message('Calculating the calibration curve for %s' % specfile)
cal_spectra = sensfunc(obs_spectra, std_spectra, ext_spectra, airmass,
exptime)
# plot(cal_spectra.wavelength, cal_spectra.flux * std_spectra.flux)
# fit the spectra--first take a first cut of the spectra
# using the median absolute deviation to throw away bad points
cmed = np.median(cal_spectra.flux)
cmad = saltstat.mad(cal_spectra.flux)
mask = (abs(cal_spectra.flux - cmed) < thresh * cmad)
mask = np.logical_and(mask, (cal_spectra.flux > 0))
# now fit the data
# Fit using a gaussian process.
if fitter == 'gaussian':
from sklearn.gaussian_process import GaussianProcess
#Instanciate a Gaussian Process model
dy = obs_spectra.var[mask]**0.5
dy /= obs_spectra.flux[mask] / cal_spectra.flux[mask]
y = cal_spectra.flux[mask]
gp = GaussianProcess(corr='squared_exponential',
theta0=1e-2,
thetaL=1e-4,
thetaU=0.1,
nugget=(dy / y)**2.0)
X = np.atleast_2d(cal_spectra.wavelength[mask]).T
# Fit to data using Maximum Likelihood Estimation of the parameters
gp.fit(X, y)
x = np.atleast_2d(cal_spectra.wavelength).T
# Make the prediction on the meshed x-axis (ask for MSE as well)
y_pred = gp.predict(x)
cal_spectra.flux = y_pred
else:
fit = interfit(cal_spectra.wavelength[mask],
cal_spectra.flux[mask],
function=function,
order=order,
thresh=thresh,
niter=niter)
fit.interfit()
cal_spectra.flux = fit(cal_spectra.wavelength)
# write the spectra out
st.writespectrum(cal_spectra, outfile, ftype='ascii')
quickcheck = stdflux[onion//2].lower()[1:-4] in stanspec.lower()
if not quickcheck:
print 'Check your standard star and flux files. They are mixed up.'
sys.exit()
onion += 1
'''
orderused = np.zeros([len(standards)])
senspolys = []
airstd = np.zeros([len(standards)])
allexcluded = [[None] for i in range(len(standards))]
#Calculating the sensitivity function of each standard star
cucumber = 0
for stdspecfile in standards:
#Read in the observed spectrum of the standard star
obs_spectra,airmass,exptime,dispersion = st.readspectrum(stdspecfile) #This is an object containing var_farr,farr,sky,sigma,warr
airstd[cucumber] = airmass
#plt.clf()
#plt.plot(obs_spectra.warr,obs_spectra.farr)
#plt.show()
#read in the standard file
placeholder = cucumber // 2
stdfile = stdflux[placeholder]
std_spectra = st.readstandard(stdfile)
#plt.clf()
#plt.plot(std_spectra.warr,std_spectra.magarr)
#plt.show()
#Only keep the part of the standard file that overlaps with observation.
lowwv = np.where(std_spectra.warr >= np.min(obs_spectra.warr))
lowwv = np.asarray(lowwv)
def flux_calibrate_now(stdlist,
fluxlist,
speclist,
extinct_correct=False,
masterresp=False):
if extinct_correct:
extinctflag = 0
else:
extinctflag = -1
if masterresp: #Use the master response function
#Read in master response function and use that.
cwd = os.getcwd()
os.chdir(
'/afs/cas.unc.edu/depts/physics_astronomy/clemens/students/group/standards/response_curves/'
)
standards = sorted(glob('*resp*.npy'))
master_response_blue_in = np.load(standards[0])
master_response_blue_in_pol = np.poly1d(master_response_blue_in)
master_response_blue_out = np.load(standards[1])
master_response_blue_out_pol = np.poly1d(master_response_blue_out)
master_response_red_in = np.load(standards[2])
master_response_red_in_pol = np.poly1d(master_response_red_in)
master_response_red_out = np.load(standards[3])
master_response_red_out_pol = np.poly1d(master_response_red_out)
os.chdir(cwd)
airstd = np.ones([4])
#airstd[0] = 1.1
#For saving files correctly
stdflux = np.array(['mmaster.dat'])
#standards = np.array([masterlist])
allexcluded = [[None] for i in range(len(standards))]
orderused = np.zeros([len(standards)])
size = 0.
#Find shift for each night
#For blue setup: use mean of 4530-4590
#for red setup: use mean of 6090-6190
try:
flux_tonight_list = np.genfromtxt('response_curves.txt', dtype=str)
print 'Found response_curves.txt file.'
print flux_tonight_list
if flux_tonight_list.size == 1:
flux_tonight_list = np.array([flux_tonight_list])
for x in flux_tonight_list:
#print x
if 'blue' in x.lower():
wave_tonight, sens_tonight = np.genfromtxt(x, unpack=True)
blue_low_index = np.min(np.where(wave_tonight > 4530.))
blue_high_index = np.min(np.where(wave_tonight > 4590.))
blue_mean_tonight = np.mean(
sens_tonight[blue_low_index:blue_high_index])
elif 'red' in x.lower():
wave_tonight, sens_tonight = np.genfromtxt(x, unpack=True)
red_low_index = np.min(np.where(wave_tonight > 6090.))
red_high_index = np.min(np.where(wave_tonight > 6190.))
red_mean_tonight = np.mean(
sens_tonight[red_low_index:red_high_index])
except:
print 'No response_curves.txt file found.'
blue_mean_tonight = None
red_mean_tonight = None
flux_tonight_list = ['None', 'None']
else: #Use the standard star fluxes in the typical manner
#Read in each standard star spectrum
standards = np.genfromtxt(stdlist, dtype=str)
if standards.size == 1:
standards = np.array([standards])
stdflux = np.genfromtxt(fluxlist, dtype=str)
if stdflux.size == 1:
stdflux = np.array(
[stdflux]
) #save stdflux explicitly as an array so you can index if only 1 element
#Check that the files are set up correctly to avoid mixing standards.
#This checks that the files in liststandard have similar characters to those in listflux and the correct order. But might break if flux file doesn't match. E.G. mcd32d9927.dat is often called CD-32_9927 in our system.
'''
onion = 0
for stanspec in standards:
quickcheck = stdflux[onion//2].lower()[1:-4] in stanspec.lower()
if not quickcheck:
print 'Check your standard star and flux files. They are mixed up.'
sys.exit()
onion += 1
'''
orderused = np.zeros([len(standards)])
senspolys = []
airstd = np.zeros([len(standards)])
allexcluded = [[None] for i in range(len(standards))]
#Calculating the sensitivity function of each standard star
cucumber = 0
for stdspecfile in standards:
print stdspecfile
#Read in the observed spectrum of the standard star
obs_spectra, airmass, exptime, dispersion = st.readspectrum(
stdspecfile
) #obs_spectra is an object containing opfarr,farr,sky,sigma,warr
airstd[cucumber] = airmass
#plt.clf()
#plt.plot(obs_spectra.warr,obs_spectra.opfarr)
#plt.show()
#Do the extinction correction
if extinct_correct:
print 'Extinction correcting spectra.'
plt.clf()
plt.plot(obs_spectra.warr, obs_spectra.opfarr)
obs_spectra.opfarr = st.extinction_correction(
obs_spectra.warr, obs_spectra.opfarr, airmass)
plt.plot(obs_spectra.warr, obs_spectra.opfarr)
#plt.show()
#Change to the standard star directory
cwd = os.getcwd()
os.chdir(
'/afs/cas.unc.edu/depts/physics_astronomy/clemens/students/group/standards'
)
#read in the standard file
placeholder = cucumber // 2
stdfile = stdflux[placeholder]
std_spectra = st.readstandard(stdfile)
os.chdir(cwd)
#plt.clf()
#plt.plot(std_spectra.warr,std_spectra.magarr,'.')
#plt.show()
#Only keep the part of the standard file that overlaps with observation.
lowwv = np.where(std_spectra.warr >= np.min(obs_spectra.warr))
lowwv = np.asarray(lowwv)
highwv = np.where(std_spectra.warr <= np.max(obs_spectra.warr))
highwv = np.asarray(highwv)
index = np.intersect1d(lowwv, highwv)
std_spectra.warr = std_spectra.warr[index]
std_spectra.magarr = std_spectra.magarr[index]
std_spectra.wbin = std_spectra.wbin[index]
#Convert from AB mag to fnu, then to fwave (ergs/s/cm2/A)
stdzp = 3.68e-20 #The absolute flux per unit frequency at an AB mag of zero
std_spectra.magarr = st.magtoflux(std_spectra.magarr, stdzp)
std_spectra.magarr = st.fnutofwave(std_spectra.warr,
std_spectra.magarr)
#plt.clf()
#plt.plot(std_spectra.warr,std_spectra.magarr,'.')
#plt.show()
#np.savetxt('hz4_stan.txt',np.transpose([std_spectra.warr,std_spectra.magarr]))
#exit()
#We want to rebin the observed spectrum to match with the bins in the standard file. This makes summing up counts significantly easier.
#Set the new binning here.
print 'Starting to rebin: ', stdspecfile
low = np.rint(np.min(obs_spectra.warr)) #Rounds to nearest integer
high = np.rint(np.max(obs_spectra.warr))
size = 0.05 #size in Angstroms you want each bin
num = (
high - low
) / size + 1. #number of bins. Must add one to get correct number.
wavenew = np.linspace(low, high,
num=num) #wavelength of each new bin
#Now do the rebinning using Ian Crossfield's rebinning package
binflux = st.resamplespec(wavenew, obs_spectra.warr,
obs_spectra.opfarr,
200.) #200 is the oversampling factor
print 'Done rebinning. Now summing the spectrum into new bins to match', stdfile
#plt.clf()
#plt.plot(obs_spectra.warr,obs_spectra.opfarr)
#plt.plot(wavenew,binflux)
#plt.show()
#Now sum the rebinned spectra into the same bins as the standard star file
counts = st.sum_std(std_spectra.warr, std_spectra.wbin, wavenew,
binflux)
#plt.clf()
#plt.plot(std_spectra.warr,std_spectra.magarr)
#plt.plot(obs_spectra.warr,obs_spectra.opfarr,'b')
#plt.plot(std_spectra.warr,counts,'g+')
#plt.show()
#Calculate the sensitivity function
sens_function = st.sensfunc(counts, std_spectra.magarr, exptime,
std_spectra.wbin, airmass)
#plt.clf()
#plt.plot(std_spectra.warr,sens_function)
#plt.show()
#sys.exit()
#Fit a low order polynomial to this function so that it is smooth.
#The sensitivity function is in units of 2.5 * log10[counts/sec/Ang / ergs/cm2/sec/Ang]
#Choose regions to not include in fit, first by checking if a mask file exists, and if not the prompt for user interaction.
if 'blue' in stdspecfile.lower():
std_mask = stdfile[0:-4] + '_blue_maskasdf.dat'
if 'red' in stdspecfile.lower():
std_mask = stdfile[0:-4] + '_red_maskasdf.dat'
std_mask2 = glob(std_mask)
if len(std_mask2) == 1.:
print 'Found mask file.\n'
mask = np.ones(len(std_spectra.warr))
excluded_wave = np.genfromtxt(
std_mask) #Read in wavelengths to exclude
#print excluded_wave
#print type(excluded_wave)
#Find index of each wavelength
excluded = []
for x in excluded_wave:
#print x
#print np.where(std_spectra.warr == find_nearest(std_spectra.warr,x))
pix_val = np.where(
std_spectra.warr == find_nearest(std_spectra.warr, x))
excluded.append(pix_val[0][0])
#print excluded
lettuce = 0
while lettuce < len(excluded):
mask[excluded[lettuce]:excluded[lettuce + 1] + 1] = 0
lettuce += 2
excluded = np.array(excluded).tolist()
allexcluded[cucumber] = excluded
indices = np.where(mask != 0.)
lambdasfit = std_spectra.warr[indices]
fluxesfit = sens_function[indices]
else:
print 'No mask found. User interaction required.\n'
global ax, fig, coords
coords = []
plt.clf()
fig = plt.figure(1)
ax = fig.add_subplot(111)
ax.plot(std_spectra.warr, sens_function)
cid = fig.canvas.mpl_connect('button_press_event', onclick)
print 'Please click on both sides of regions you want to exclude. Then close the plot.'
plt.title(
'Click both sides of regions you want to exclude. Then close the plot.'
)
plt.show(1)
#Mask our the regions you don't want to fit
#We need make sure left to right clicking and right to left clicking both work.
mask = np.ones(len(std_spectra.warr))
excluded = np.zeros(len(coords))
lettuce = 0
if len(coords) > 0:
while lettuce < len(coords):
x1 = np.where(std_spectra.warr == (find_nearest(
std_spectra.warr, coords[lettuce][0])))
excluded[lettuce] = np.asarray(x1)
lettuce += 1
x2 = np.where(std_spectra.warr == (find_nearest(
std_spectra.warr, coords[lettuce][0])))
if x2 < x1:
x1, x2 = x2, x1
mask[
x1[0][0]:x2[0][0] +
1] = 0 #have to add 1 here to the second index so that we exclude through that index. Most important for when we need to exclude the last point of the array.
excluded[lettuce - 1] = np.asarray(x1)
excluded[lettuce] = np.asarray(x2)
lettuce += 1
excluded = np.array(excluded).tolist()
allexcluded[cucumber] = excluded
indices = np.where(mask != 0.)
lambdasfit = std_spectra.warr[indices]
fluxesfit = sens_function[indices]
#Save masked wavelengths
lambdasnotfit = std_spectra.warr[excluded]
#print lambdasnotfit
#print stdfile
if 'blue' in stdspecfile.lower():
std_mask_name = stdfile[0:-4] + '_blue_mask.dat'
if 'red' in stdspecfile.lower():
std_mask_name = stdfile[0:-4] + '_red_mask.dat'
np.savetxt(std_mask_name,
np.transpose(np.array(lambdasnotfit)))
#exit()
##Move back to directory with observed spectra
#os.chdir(cwd)
#Make sure they are finite
ind1 = np.isfinite(lambdasfit) & np.isfinite(fluxesfit)
lambdasfit = lambdasfit[ind1]
fluxesfit = fluxesfit[ind1]
print 'Fitting the sensitivity funtion now.'
order = 4
repeat = 'yes'
while repeat == 'yes':
p = np.polyfit(lambdasfit, fluxesfit, order)
f = np.poly1d(p)
smooth_sens = f(lambdasfit)
residual = fluxesfit - smooth_sens
plt.close()
plt.ion()
g, (ax1, ax2) = plt.subplots(2, sharex=True)
ax1.plot(lambdasfit, fluxesfit, 'b+')
ax1.plot(lambdasfit, smooth_sens, 'r', linewidth=2.0)
ax1.set_ylabel('Sensitivity Function')
ax2.plot(lambdasfit, residual, 'k+')
ax2.set_ylabel('Residuals')
ax1.set_title('Current polynomial order: %s' % order)
g.subplots_adjust(hspace=0)
plt.setp([a.get_xticklabels() for a in g.axes[:-1]],
visible=False)
plt.show()
plt.ioff()
#Save this sensitivity curve
'''
try:
temp_file = fits.open(stdspecfile)
ADCstat = temp_file[0].header['ADCSTAT']
except:
ADCstat = 'none'
pass
if 'blue' in stdspecfile.lower():
resp_name = 'senscurve_' + stdfile[1:-4] + '_' + str(np.round(airstd[cucumber],decimals=3)) + '_' + ADCstat + '_' + cwd[60:70] + '_blue.txt'
elif 'red' in stdspecfile.lower():
resp_name = 'senscurve_' + stdfile[1:-4] + '_' + str(np.round(airstd[cucumber],decimals=3)) + '_' + ADCstat + '_' + cwd[60:70] + '_red.txt'
print resp_name
#exit()
np.savetxt(resp_name,np.transpose([lambdasfit,fluxesfit]))
'''
repeat = raw_input('Do you want to try again (yes/no)? ')
if repeat == 'yes':
order = raw_input('New order for polynomial: ')
orderused[cucumber] = order
senspolys.append(f)
#Save arrays for diagnostic plots
if cucumber == 0:
bigarray = np.zeros([len(lambdasfit), 4. * len(standards)])
artichoke = 0
bigarray[0:len(lambdasfit), artichoke] = lambdasfit
bigarray[0:len(fluxesfit), artichoke + 1] = fluxesfit
bigarray[0:len(smooth_sens), artichoke + 2] = smooth_sens
bigarray[0:len(residual), artichoke + 3] = residual
artichoke += 4
cucumber += 1
#Save fit and residuals into text file for diagnostic plotting later.
#Need to save lambdasfit,fluxesfit,smooth_sens,residual for each standard
#List of standards is found as standards
now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
with open('sens_fits_' + now + '.txt', 'a') as handle:
header = str(
standards
) + '\n Set of four columns correspond to wavelength, observed flux, polynomial fit, \n and residuals for each standard listed above. \n You will probably need to strip zeros from the bottoms of some columns.'
np.savetxt(handle, bigarray, fmt='%f', header=header)
#Outline for next steps:
#Read in both red and blue files
#compute airmass and compare to airstd
#choose best standard and flux calibrate both blue and red
#save files and write to sensitivity_params.txt
if speclist[-4:] == 'fits':
specfile = np.array([speclist])
else:
specfile = np.genfromtxt(speclist, dtype=str)
if specfile.size == 1:
specfile = np.array([specfile])
length = len(specfile)
airwd = np.zeros([length])
bean = 0
#if length == 1:
# redfile = False
#else:
# redfile = True
avocado = 0
while avocado < length:
#Read in the blue and red spectra we want to flux calibrate. Save the airmass
WD_spectra1, airmass1, exptime1, dispersion1 = st.readspectrum(
specfile[avocado])
if (len(specfile) >= 1) and (avocado + 1 < length):
if 'red' in specfile[avocado + 1]:
redfile = True
else:
redfile = False
else:
redfile = False
if redfile:
WD_spectra2, airmass2, exptime2, dispersion2 = st.readspectrum(
specfile[avocado + 1])
#Extinction correct WD
if extinct_correct:
print 'Extinction correcting spectra.'
#plt.clf()
#plt.plot(WD_spectra1.warr,WD_spectra1.opfarr)
WD_spectra1.opfarr = st.extinction_correction(
WD_spectra1.warr, WD_spectra1.opfarr, airmass1)
WD_spectra1.farr = st.extinction_correction(
WD_spectra1.warr, WD_spectra1.farr, airmass1)
WD_spectra1.sky = st.extinction_correction(WD_spectra1.warr,
WD_spectra1.sky,
airmass1)
WD_spectra1.sigma = st.extinction_correction(
WD_spectra1.warr, WD_spectra1.sigma, airmass1)
#plt.plot(WD_spectra1.warr,WD_spectra1.opfarr)
#plt.show()
if redfile:
#plt.clf()
#plt.plot(WD_spectra2.warr,WD_spectra2.opfarr)
WD_spectra2.opfarr = st.extinction_correction(
WD_spectra2.warr, WD_spectra2.opfarr, airmass2)
WD_spectra2.farr = st.extinction_correction(
WD_spectra2.warr, WD_spectra2.farr, airmass2)
WD_spectra2.sky = st.extinction_correction(
WD_spectra2.warr, WD_spectra2.sky, airmass2)
WD_spectra2.sigma = st.extinction_correction(
WD_spectra2.warr, WD_spectra2.sigma, airmass2)
#zaplt.plot(WD_spectra2.warr,WD_spectra2.opfarr)
#plt.show()
airwd[avocado] = airmass1
if redfile:
airwd[avocado + 1] = airmass2
#Compare the airmasses to determine the best standard star
tomato = 0
while tomato < len(airstd):
if redfile:
diff = np.absolute(
np.mean([airwd[avocado], airwd[avocado + 1]]) -
np.mean([airstd[tomato], airstd[tomato + 1]]))
else:
diff = np.absolute(airwd[avocado] - airstd[tomato])
if tomato == 0:
difference = diff
choice = tomato
if diff < difference:
difference = diff
choice = tomato
tomato += 2
#To get the flux calibration, perform the following
#Flux = counts / (Exptime * dispersion * 10**(sens/2.5))
#Get the sensitivity function at the correct wavelength spacing
if masterresp:
header_temp = st.readheader(specfile[avocado])
ADCstatus = header_temp['ADCSTAT']
if ADCstatus == 'IN':
sens_wave1_unscale = master_response_blue_in_pol(
WD_spectra1.warr)
blue_low_index = np.min(np.where(WD_spectra1.warr > 4530.))
blue_high_index = np.min(np.where(WD_spectra1.warr > 4590.))
blue_mean_stan = np.mean(
sens_wave1_unscale[blue_low_index:blue_high_index])
if blue_mean_tonight == None:
sens_wave1 = sens_wave1_unscale
else:
sens_wave1 = sens_wave1_unscale + (blue_mean_tonight -
blue_mean_stan)
choice = 0
else:
sens_wave1_unscale = master_response_blue_out_pol(
WD_spectra1.warr)
blue_low_index = np.min(np.where(WD_spectra1.warr > 4530.))
blue_high_index = np.min(np.where(WD_spectra1.warr > 4590.))
blue_mean_stan = np.mean(
sens_wave1_unscale[blue_low_index:blue_high_index])
if blue_mean_tonight == None:
sens_wave1 = sens_wave1_unscale
else:
sens_wave1 = sens_wave1_unscale + (blue_mean_tonight -
blue_mean_stan)
choice = 1
if redfile:
header_temp = st.readheader(specfile[avocado + 1])
ADCstatus = header_temp['ADCSTAT']
if ADCstatus == 'IN':
sens_wave2_unscale = master_response_red_in_pol(
WD_spectra2.warr)
red_low_index = np.min(np.where(WD_spectra2.warr > 6090.))
red_high_index = np.min(np.where(WD_spectra2.warr > 6190.))
red_mean_stan = np.mean(
sens_wave2_unscale[red_low_index:red_high_index])
if red_mean_tonight == None:
sens_wave2 = sens_wave2_unscale
else:
sens_wave2 = sens_wave2_unscale + (red_mean_tonight -
red_mean_stan)
choice2 = 2
else:
sens_wave2_unscale = master_response_red_out_pol(
WD_spectra2.warr)
red_low_index = np.min(np.where(WD_spectra2.warr > 6090.))
red_high_index = np.min(np.where(WD_spectra2.warr > 6190.))
red_mean_stan = np.mean(
sens_wave2_unscale[red_low_index:red_high_index])
if red_mean_tonight == None:
sens_wave2 = sens_wave2_unscale
else:
sens_wave2 = sens_wave2_unscale + (red_mean_tonight -
red_mean_stan)
choice2 = 3
else:
sens_wave1 = senspolys[choice](WD_spectra1.warr)
if redfile:
sens_wave2 = senspolys[choice + 1](WD_spectra2.warr)
#Perform the flux calibration. We do this on the optimal extraction, non-variance weighted aperture, the sky spectrum, and the sigma spectrum.
print 'Doing the final flux calibration.'
#np.savetxt('response_g60-54_extinction_2016-03-17.txt',np.transpose([WD_spectra1.warr,(exptime1 * dispersion1 * 10.**(sens_wave1/2.5))]))#,WD_spectra2.warr,(exptime2 * dispersion2 * 10.**(sens_wave2/2.5))]))
#exit()
star_opflux1 = st.cal_spec(WD_spectra1.opfarr, sens_wave1, exptime1,
dispersion1)
star_flux1 = st.cal_spec(WD_spectra1.farr, sens_wave1, exptime1,
dispersion1)
sky_flux1 = st.cal_spec(WD_spectra1.sky, sens_wave1, exptime1,
dispersion1)
sigma_flux1 = st.cal_spec(WD_spectra1.sigma, sens_wave1, exptime1,
dispersion1)
if redfile:
star_opflux2 = st.cal_spec(WD_spectra2.opfarr, sens_wave2,
exptime2, dispersion2)
star_flux2 = st.cal_spec(WD_spectra2.farr, sens_wave2, exptime2,
dispersion2)
sky_flux2 = st.cal_spec(WD_spectra2.sky, sens_wave2, exptime2,
dispersion2)
sigma_flux2 = st.cal_spec(WD_spectra2.sigma, sens_wave2, exptime2,
dispersion2)
#plt.clf()
#plt.plot(WD_spectra.warr,star_opflux)
#plt.show()
#Save final spectra if using master response
if masterresp:
if avocado == 0:
diagnostic_array = np.zeros(
[len(WD_spectra1.warr), 2 * length])
diagnostic_array[0:len(WD_spectra1.warr), bean] = WD_spectra1.warr
bean += 1
diagnostic_array[0:len(star_opflux1), bean] = star_opflux1
bean += 1
if redfile:
diagnostic_array[0:len(WD_spectra2.warr),
bean] = WD_spectra2.warr
bean += 1
diagnostic_array[0:len(star_opflux2), bean] = star_opflux2
bean += 1
#if avocado == (length -1 ) or (redfile == True and avocado == (length-2)):
# print 'Saveing diagnostic file.'
# now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
# with open('flux_fits_' + now + '.txt','a') as handle:
# header = str(specfile) + '\n Each star is formatted as wavelength, flux'
# np.savetxt(handle,diagnostic_array,fmt='%.10e',header=header)
print 'Saving the final spectrum.'
#Save the flux-calibrated spectrum and update the header
header1 = st.readheader(specfile[avocado])
header1.set('EX-FLAG',
extinctflag) #Extiction correction? 0=yes, -1=no
header1.set('CA-FLAG', 0) #Calibrated to flux scale? 0=yes, -1=no
header1.set('BUNIT', 'erg/cm2/s/A') #physical units of the array value
header1.set(
'STANDARD', str(standards[choice]),
'Flux standard used') #flux standard used for flux-calibration
if masterresp:
header1.set('STDOFF', str(flux_tonight_list[0]),
'Night offset used')
if redfile:
header2 = st.readheader(specfile[avocado + 1])
header2.set('EX-FLAG',
extinctflag) #Extiction correction? 0=yes, -1=no
header2.set('CA-FLAG', 0) #Calibrated to flux scale? 0=yes, -1=no
header2.set('BUNIT',
'erg/cm2/s/A') #physical units of the array value
if masterresp:
header2.set('STANDARD', str(standards[choice2]),
'Flux standard used'
) #flux standard used for flux-calibration
header1.set('STDOFF', str(flux_tonight_list[1]),
'Night offset used')
else:
header2.set('STANDARD', str(standards[choice + 1]),
'Flux standard used'
) #flux standard used for flux-calibration
#Set up size of new fits image
Ni = 4. #Number of extensions
Nx1 = len(star_flux1)
if redfile:
Nx2 = len(star_flux2)
Ny = 1. #All 1D spectra
data1 = np.empty(shape=(Ni, Ny, Nx1))
data1[0, :, :] = star_opflux1
data1[1, :, :] = star_flux1
data1[2, :, :] = sky_flux1
data1[3, :, :] = sigma_flux1
if redfile:
data2 = np.empty(shape=(Ni, Ny, Nx2))
data2[0, :, :] = star_opflux2
data2[1, :, :] = star_flux2
data2[2, :, :] = sky_flux2
data2[3, :, :] = sigma_flux2
#Add '_flux' to the end of the filename
loc1 = specfile[avocado].find('.ms.fits')
if masterresp:
newname1 = specfile[avocado][0:loc1] + '_flux_' + stdflux[0][
1:-4] + '.ms.fits'
else:
newname1 = specfile[avocado][0:loc1] + '_flux_' + stdflux[
choice // 2][1:-4] + '.ms.fits'
clob = False
mylist = [True for f in os.listdir('.') if f == newname1]
exists = bool(mylist)
if exists:
print 'File %s already exists.' % newname1
nextstep = raw_input(
'Do you want to overwrite or designate a new name (overwrite/new)? '
)
if nextstep == 'overwrite':
clob = True
exists = False
elif nextstep == 'new':
newname1 = raw_input('New file name: ')
exists = False
else:
exists = False
print 'Saving: ', newname1
newim1 = fits.PrimaryHDU(data=data1, header=header1)
newim1.writeto(newname1, clobber=clob)
if redfile:
loc2 = specfile[avocado + 1].find('.ms.fits')
if masterresp:
newname2 = specfile[
avocado +
1][0:loc2] + '_flux_' + stdflux[0][1:-4] + '.ms.fits'
else:
newname2 = specfile[avocado + 1][0:loc2] + '_flux_' + stdflux[
choice // 2][1:-4] + '.ms.fits'
clob = False
mylist = [True for f in os.listdir('.') if f == newname2]
exists = bool(mylist)
if exists:
print 'File %s already exists.' % newname2
nextstep = raw_input(
'Do you want to overwrite or designate a new name (overwrite/new)? '
)
if nextstep == 'overwrite':
clob = True
exists = False
elif nextstep == 'new':
newname2 = raw_input('New file name: ')
exists = False
else:
exists = False
newim2 = fits.PrimaryHDU(data=data2, header=header2)
newim2.writeto(newname2, clobber=clob)
print 'Saving: ', newname2
#Finally, save all the used parameters into a file for future reference.
# specfile,current date, stdspecfile,stdfile,order,size,newname
f = open('sensitivity_params.txt', 'a')
now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
if masterresp:
newinfo1 = specfile[avocado] + '\t' + now + '\t' + standards[
choice] + '\t' + stdflux[0] + '\t' + str(
allexcluded[choice]) + '\t' + str(
orderused[choice]) + '\t' + str(size) + '\t' + newname1
else:
newinfo1 = specfile[avocado] + '\t' + now + '\t' + standards[
choice] + '\t' + stdflux[choice // 2] + '\t' + str(
allexcluded[choice]) + '\t' + str(
orderused[choice]) + '\t' + str(size) + '\t' + newname1
if redfile:
if masterresp:
newinfo2 = specfile[
avocado + 1] + '\t' + now + '\t' + standards[
choice2] + '\t' + stdflux[0] + '\t' + str(
allexcluded[choice + 1]) + '\t' + str(
orderused[choice + 1]) + '\t' + str(
size) + '\t' + newname2
else:
newinfo2 = specfile[
avocado + 1] + '\t' + now + '\t' + standards[
choice + 1] + '\t' + stdflux[choice // 2] + '\t' + str(
allexcluded[choice + 1]) + '\t' + str(
orderused[choice + 1]) + '\t' + str(
size) + '\t' + newname2
f.write(newinfo1 + "\n" + newinfo2 + "\n")
else:
f.write(newinfo1 + "\n")
f.close()
if redfile:
avocado += 2
else:
avocado += 1
print 'Done flux calibrating the spectra.'