def construct_KpVsys(rv, ccf, ccf_e, dp, kprange=[0, 300], dkp=1.0):
"""The name says it all. Do good tests."""
import tayph.functions as fun
import tayph.operations as ops
import numpy as np
import tayph.system_parameters as sp
import matplotlib.pyplot as plt
import astropy.io.fits as fits
import tayph.util as ut
import sys
import pdb
from joblib import Parallel, delayed
Kp = fun.findgen((kprange[1] - kprange[0]) / dkp + 1) * dkp + kprange[0]
n_exp = np.shape(ccf)[0]
KpVsys = np.zeros((len(Kp), len(rv)))
KpVsys_e = np.zeros((len(Kp), len(rv)))
transit = sp.transit(dp) - 1.0
transit /= np.nansum(transit)
transitblock = fun.rebinreform(transit, len(rv)).T
def Kp_parallel(i):
dRV = sp.RV(dp, vorb=i) * (-1.0)
ccf_shifted = shift_ccf(rv, ccf, dRV)
ccf_e_shifted = shift_ccf(rv, ccf_e, dRV)
return (np.nansum(transitblock * ccf_shifted, axis=0), (np.nansum(
(transitblock * ccf_e_shifted)**2.0, axis=0))**0.5)
KpVsys, KpVsys_e = zip(*Parallel(n_jobs=-1, verbose=5)(
delayed(Kp_parallel)(i) for i in Kp))
return (Kp, KpVsys, KpVsys_e)
def read_wave_from_e2ds_header(h, mode='HARPS'):
"""
This reads the wavelength solution from the HARPS header keywords that
encode the coefficients as a 4-th order polynomial.
"""
import numpy as np
import tayph.functions as fun
if mode not in ['HARPS', 'HARPSN', 'HARPS-N', 'UVES']:
raise ValueError(
"in read_wave+from_e2ds_header: mode needs to be set to HARPS, HARPSN or UVES."
)
npx = h['NAXIS1']
no = h['NAXIS2']
x = fun.findgen(npx)
wave = np.zeros((npx, no))
if mode == 'HARPS':
coeffkeyword = 'ESO'
if mode in ['HARPSN', 'HARPS-N']:
coeffkeyword = 'TNG'
if mode == 'UVES':
delt = h['CDELT1']
for i in range(no):
keystart = h[f'WSTART{i+1}']
# keyend = h[f'WEND{i+1}']
# wave[:,i] = fun.findgen(npx)*(keyend-keystart)/(npx-1)+keystart
wave[:, i] = fun.findgen(npx) * delt + keystart
#These FITS headers have a start and end, but (end-start)/npx does not equal
#the stepsize provided in CDELT (by far). Turns out that keystart+n*CDELT is the correct
#representation of the wavelength. I do not know why WEND is provided at all and how
#it got to be so wrong...
else:
key_counter = 0
for i in range(no):
l = x * 0.0
for j in range(4):
l += h[coeffkeyword +
' DRS CAL TH COEFF LL%s' % key_counter] * x**j
key_counter += 1
wave[:, i] = l
wave = wave.T
return (wave)
def update_plots(self):
"""
This redraws the plot planels, taking care to reselect axis ranges and such.
"""
import numpy as np
import tayph.drag_colour as dcb
import tayph.functions as fun
import copy
import pdb
import tayph.plotting as plotting
if self.npx != len(self.x_axis):
print('--------- Redrawing to account for order width mismatch')
array1 = copy.deepcopy(self.order)
array2 = copy.deepcopy(self.residual)
array1[np.isnan(array1)] = np.inf#The colobar doesn't eat NaNs, so now set them to inf just for the plot.
array2[np.isnan(array2)] = np.inf#And here too.
self.xrange = [0,self.npx-1]
# self.yrange=[0,self.nexp-1]#Should not be needed as self.y_axis cant change!
self.x_axis=fun.findgen(self.npx).astype(int)
# self.y_axis = fun.findgen(self.nexp).astype(int)
self.x2,self.y2,self.z,self.wl_sel,self.y_axis_sel,self.xticks,self.yticks,void1,void2 = plotting.plotting_scales_2D(self.x_axis,self.y_axis,self.residual,self.xrange,self.yrange,Nxticks=self.Nxticks,Nyticks=self.Nyticks,nsigma=self.nsigma)
self.img1=self.ax[0].pcolormesh(self.x2,self.y2,array1,vmin=0,vmax=self.img_max,cmap='hot')
self.img2=self.ax[1].pcolormesh(self.x2,self.y2,array2,vmin=self.vmin,vmax=self.vmax,cmap='hot')
self.ax[2].set_xlim((min(self.x_axis),max(self.x_axis)))
# self.ax[2].set_ylim(0,self.img_max)
self.ax[2].clear()
self.img3=self.ax[2].plot(self.x_axis,self.meanspec)
# self.cbar.remove()
# self.cbar = self.fig.colorbar(self.img2, ax=self.ax.ravel().tolist(),aspect = 20)
# self.cbarD = dcb.DraggableColorbar_fits(self.cbar,[self.img2],'hot')
# self.cbarD.connect()
else:
array1 = copy.deepcopy(self.order.ravel())
array2 = copy.deepcopy(self.residual.ravel())
array1[np.isnan(array1)] = np.inf#The colobar doesn't eat NaNs, so now set them to inf just for the plot.
array2[np.isnan(array2)] = np.inf#And here too.
self.img1.set_array(array1)
self.img1.set_clim(vmin=0,vmax=self.img_max)
self.img2.set_array(array2)
self.img2.set_clim(vmin=self.vmin,vmax=self.vmax)
self.img3[0].set_ydata(self.meanspec)
self.ax[0].set_title('Spectral order %s (%s - %s nm)' % (self.N,round(np.min(self.wl),1),round(np.max(self.wl),1)))
self.ax[2].set_ylim(0,self.img_max)
self.draw_masked_areas()
self.fig.canvas.draw_idle()
def applytotal(self,event):
"""
This is an event handler for pressing the Mask entire order button in the GUI.
It has 2 behaviours depending on whether anything is masked or not.
If nothing was masked, the entire order will be masked.
If something was, it will all be unmasked.
"""
import tayph.functions as fun
ncol = len(self.list_of_orders[self.N][0])
self.list_of_selected_columns[self.N] = list(fun.findgen(ncol))
# print(self.list_of_selected_columns[self.N])
self.draw_masked_areas()#Update the green areas.
self.fig.canvas.draw()
self.fig.canvas.draw()
def __init__(self,fig,ax,rv,ccf,primer,dp,outname):
"""We initialize with a figure object, three axis objects (in a list)
an rv axis, the CCF, the user-generated prior on the doppler shadow (the two)
points, and the pointer to the data in order to load physics."""
import tayph.system_parameters as sp
import numpy as np
import pdb
import scipy.interpolate as interpol
import tayph.functions as fun
import tayph.util as ut
import sys
#Upon initialization, we pass the keywords onto self.
nexp = np.shape(ccf)[0]
nrv = np.shape(ccf)[1]
self.rv = rv
self.ccf = ccf
self.p = sp.phase(dp)
if len(self.p) != nexp:
print('ERROR IN FIT_DOPPLER_MODEL __INIT__:')
print('The height of the CCF does not match nexp.')
sys.exit()
transit = sp.transit(dp)-1.0
self.T = abs(transit/max(abs(transit)))
self.ax = ax
self.aRstar = sp.paramget('aRstar',dp)
self.vsys = sp.paramget('vsys',dp)
self.RVp = sp.RV(dp) + self.vsys
self.inclination = sp.paramget('inclination',dp)
self.n_components = 1
self.maskHW = 10.0 #Default masking half-width
self.offset = 0.0
self.D = 0#Initial degree of polynomial fitting.
self.S = 0
self.dp=ut.check_path(dp,exists=True)
self.outpath=self.dp/(outname+'.pkl')
#Translate the pivot to RV-phase points.
#Need to interpolate on phase only, because the primer was already defined in RV space.
p_i = interpol.interp1d(fun.findgen(nexp),self.p)
p1 = float(p_i(primer[0][1]))
p2 = float(p_i(primer[1][1]))
v1 = primer[0][0]
v2 = primer[1][0]
self.primer = [[v1,p1],[v2,p2]]
self.fit_spinorbit()
# self.primer = a*self.rv+b
self.v_star_primer = fun.local_v_star(self.p,self.aRstar,self.inclination,self.vsini_fit,self.l_fit)+self.vsys
# ax[0].plot(v_star_primer,fun.findgen(nexp),'--',color='black',label='Spin-orbit fit to primer')
self.mask_ccf()
self.fit_model()
def construct_KpVsys(rv, ccf, ccf_e, dp, kprange=[0, 300], dkp=1.0):
"""The name says it all. Do good tests."""
import tayph.functions as fun
import tayph.operations as ops
import numpy as np
import tayph.system_parameters as sp
import matplotlib.pyplot as plt
import astropy.io.fits as fits
import tayph.util as ut
import sys
import pdb
Kp = fun.findgen((kprange[1] - kprange[0]) / dkp + 1) * dkp + kprange[0]
n_exp = np.shape(ccf)[0]
KpVsys = np.zeros((len(Kp), len(rv)))
KpVsys_e = np.zeros((len(Kp), len(rv)))
transit = sp.transit(dp) - 1.0
transit /= np.nansum(transit)
transitblock = fun.rebinreform(transit, len(rv)).T
j = 0
ccfs = []
for i in Kp:
dRV = sp.RV(dp, vorb=i) * (-1.0)
ccf_shifted = shift_ccf(rv, ccf, dRV)
ccf_e_shifted = shift_ccf(rv, ccf_e, dRV)
ccfs.append(ccf_shifted)
KpVsys[j, :] = np.nansum(transitblock * ccf_shifted, axis=0)
KpVsys_e[j, :] = (np.nansum((transitblock * ccf_e_shifted)**2.0,
axis=0))**0.5
# plt.plot(rv,KpVsys[j,:])
# plt.fill_between(rv, KpVsys[j,:]-KpVsys_e[j,:], KpVsys[j,:]+KpVsys_e[j,:],alpha=0.5)
# plt.show()
# pdb.set_trace()
j += 1
ut.statusbar(i, Kp)
return (Kp, KpVsys, KpVsys_e)
def construct_doppler_model(rv,ccf,dp,shadowname,xrange=[-200,200],Nxticks=20.0,Nyticks=10.0):
"""This is the the main function to construct a doppler model. The above are mostly dependencies."""
import numpy as np
import matplotlib.pyplot as plt
import tayph.drag_colour as dcb
import tayph.functions as fun
import tayph.system_parameters as sp
import tayph.plotting as fancyplots
import sys
from matplotlib.widgets import Slider, Button, RadioButtons, CheckButtons
#This is for setting plot axes in the call to plotting_scales_2D below.
nexp = np.shape(ccf)[0]
yrange=[0,nexp-1]
y_axis = fun.findgen(nexp)
#And for adding the planet line:
vsys = sp.paramget('vsys',dp)
vsini = sp.paramget('vsini',dp)
RVp = sp.RV(dp)+vsys
transit = sp.transit(dp)
sel_transit = y_axis[(transit < 1.0)]
transit_start = min(sel_transit)
transit_end = max(sel_transit)
fig,ax,cbar = fancyplots.plot_ccf(rv,ccf,dp,xrange = xrange,Nxticks = Nxticks,Nyticks = Nyticks,i_legend=False,show=False)#Initiates the plot for the primer.
primer = prime_doppler_model(fig,ax,cbar)#We use the active Figure
#to let the user indicate where the doppler shadow is located (the primer). This calls the plt.show()
#which was not called by plot_ccf. To proceed, we would like to model the shadow using the
#primer, along with fancy visualization of the ccf.
#We first re-instate a plot. This plot is a bit more complex than the primer so we don't
#use plot_ccf anymore; though the philosophy is similar.
x2,y2,z,rv_sel,y_sel,xticks,yticks,vmin,vmax = fancyplots.plotting_scales_2D(rv,y_axis,ccf,xrange,yrange,Nxticks=Nxticks,Nyticks=Nyticks,nsigma=3.0)
#Create empty plots.
fig,ax = plt.subplots(3,1,sharex=True,figsize=(13,6))
plt.subplots_adjust(right=0.75)
#Here we initiate the model instance that does the fitting and handles the GUI.
#This does an initial fit based on the primer.
model_callback = fit_doppler_model(fig,ax,rv_sel,z,primer,dp,shadowname)
s_init = model_callback.maskHW#Masking half-width around the planet RV to start the slider with.
#We continue by overplotting the velocty traces and transit markers onto the 3 subplots, saving the references to the lines.
#These lines can be set to be visible/invisible using the checkbuttons below, and are set to be invisible
#when the plot opens.
l_planet = []
t_start = []
t_end = []
l_primer = []
l_vfit = []
for sub_ax in ax:
sub_ax.axis([x2.min(),x2.max(),y2.min(),y2.max()])
sub_ax.set_xticks(xticks)
sub_ax.set_yticks(yticks)
sub_ax.set_ylabel('Exposure')
l1 = sub_ax.plot(RVp,y_axis,'--',color='black',label='Planet rest-frame',visible=False)[0]
l2 = sub_ax.plot(rv,rv*0.0+transit_start,'--',color='white',label='Transit start',visible=False)[0]
l3 = sub_ax.plot(rv,rv*0.0+transit_end,'--',color='white',label='Transit end',visible=False)[0]
l4 = sub_ax.plot(model_callback.v_star_primer,y_axis,'--',color='black',label='Local velocity (primer)',visible=False)[0]
l5 = sub_ax.plot(model_callback.v_star_fit,y_axis,'--',color='black',label='Local velocity (fit)',visible=False)[0]
l_planet.append(l1)
t_start.append(l2)
t_end.append(l3)
l_primer.append(l4)
l_vfit.append(l5)
ax[0].set_title('Data')
ax[1].set_title('Model shadow')
ax[2].set_title('Residual')
ax[2].set_xlabel('Radial velocity (km/s)')
#Here we actually plot the initial fit, which will be modified each time the parameters are changed
#using the GUI buttons/sliders.
img1=ax[0].pcolormesh(x2,y2,z,vmin=vmin,vmax=vmax,cmap='hot')
img2=ax[1].pcolormesh(x2,y2,model_callback.model,vmin=vmin,vmax=vmax,cmap='hot')
img3=ax[2].pcolormesh(x2,y2,z-model_callback.model,vmax=vmax,cmap='hot')
#This trick to associate a single CB to multiple axes comes from
#https://stackoverflow.com/questions/13784201/matplotlib-2-subplots-1-colorbar
cbar = fig.colorbar(img1, ax=ax.ravel().tolist(),format='%05.4f',aspect = 15)
cbar = dcb.DraggableColorbar_fits(cbar,[img1,img2,img3],'hot')
cbar.connect()
#We define the interface and the bahaviour of button/slider callbacks.
#First the check buttons for showing the lines defined above.
rax_top = plt.axes([0.8, 0.65, 0.15, 0.25])
rax_top.set_title('Plot:')
labels = ['Planet velocity','Transit start/end','Shadow v$_c$ primer','Shadow $v_c$ fit','Masked area']
start = [False,False,False,False,False,False]#Start with none of these actually visible.
check = CheckButtons(rax_top, labels, start)
def func(label):
index = labels.index(label)
lines = [l_planet,np.append(t_end,t_start),l_primer,l_vfit]
if index < len(lines):
for l in lines[index]:
l.set_visible(not l.get_visible())
if index == len(lines):#I.e. if we are on the last element, which is not a line an option for SHOWING the masked area:
status = check.get_status()[-1]
if status == True:#If true, mask the image.
data = z*model_callback.ccf_mask
data[np.isnan(data)] = np.inf#The colobar doesn't eat NaNs, set them to inf instead for the plot. Makes them white, too.
img1.set_array((data).ravel())
img3.set_array((data-model_callback.model).ravel())
if status == False:#If false (unclicked), then just the data w/o mask.
img1.set_array(z.ravel())
img3.set_array((z-model_callback.model).ravel())
plt.draw()
check.on_clicked(func)
#Then the choice for 1 or 2 Gaussian fitting components:
rax_middle = plt.axes([0.8, 0.53, 0.15, 0.10])
clabels = ['1 component', '2 components']
radio = RadioButtons(rax_middle,clabels)
def cfunc(label):
index = clabels.index(label)
model_callback.n_components = index+1
model_callback.fit_model()#Each time we change the choice, refit.
status = check.get_status()[-1]
if status == True:#If true, mask the image.
data = z*model_callback.ccf_mask
data[np.isnan(data)] = np.inf#The colobar doesn't eat NaNs, set them to inf instead for the plot.
img2.set_array(model_callback.model.ravel())
img3.set_array((data-model_callback.model).ravel())
if status == False:#If false (unclicked), then just the data w/o mask.
img2.set_array(model_callback.model.ravel())
img3.set_array((z-model_callback.model).ravel())
plt.draw()
radio.on_clicked(cfunc)
rax_deg = plt.axes([0.8,0.35,0.07,0.13])
plt.title('Polynomial degree',fontsize=8)
rax_poly = plt.axes([0.88,0.35,0.07,0.13])
dlabels = ['0 (off)','2','4','6']
plabels = ['Single','Even','Full']
dradio = RadioButtons(rax_deg,dlabels)
pradio = RadioButtons(rax_poly,plabels)
def update_degree(label):
model_callback.D = dlabels.index(label) * 2
model_callback.fit_model()#Each time we change the choice, refit.
status = check.get_status()[-1]
if status == True:#If true, mask the image.
data = z*model_callback.ccf_mask
data[np.isnan(data)] = np.inf#The colobar doesn't eat NaNs, set them to inf instead for the plot.
img2.set_array(model_callback.model.ravel())
img3.set_array((data-model_callback.model).ravel())
if status == False:#If false (unclicked), then just the data w/o mask.
img2.set_array(model_callback.model.ravel())
img3.set_array((z-model_callback.model).ravel())
plt.draw()
def update_poly(label):
model_callback.S = plabels.index(label)
model_callback.fit_model()#Each time we change the choice, refit.
status = check.get_status()[-1]
if status == True:#If true, mask the image.
data = z*model_callback.ccf_mask
data[np.isnan(data)] = np.inf#The colobar doesn't eat NaNs, set them to inf instead for the plot.
img2.set_array(model_callback.model.ravel())
img3.set_array((data-model_callback.model).ravel())
if status == False:#If false (unclicked), then just the data w/o mask.
img2.set_array(model_callback.model.ravel())
img3.set_array((z-model_callback.model).ravel())
plt.draw()
dradio.on_clicked(update_degree)
pradio.on_clicked(update_poly)
#Then the offset slider:
rax_slider2 = plt.axes([0.8, 0.25, 0.15, 0.02])
rax_slider2.set_title('Offset 2nd component')
offset_slider = Slider(rax_slider2,'',-1.0*np.ceil(vsini),np.ceil(vsini),valinit=0.0,valstep=1.0)
def update_offset(val):
model_callback.offset = offset_slider.val
status = radio.value_selected
if status == clabels[1]:#Only update the fit if we are actually asked to do 2 components.
model_callback.fit_model()
# data = z*model_callback.ccf_mask
# data[np.isnan(data)] = np.inf#The colobar doesn't eat NaNs...
# img1.set_array((data).ravel())
img2.set_array(model_callback.model.ravel())
img3.set_array((z-model_callback.model).ravel())
# if status == False:#If false (unclicked), then just the data w/o mask.
# img1.set_array(z.ravel())
# img2.set_array(model_callback.model.ravel())
# img3.set_array((z-model_callback.model).ravel())
plt.draw()
offset_slider.on_changed(update_offset)
#Then the slider:
rax_slider = plt.axes([0.8, 0.18, 0.15, 0.02])
rax_slider.set_title('Mask width (km/s)')
mask_slider = Slider(rax_slider,'', 0.0,30.0,valinit=s_init,valstep=1.0)
def update(val):
model_callback.maskHW = mask_slider.val
model_callback.mask_ccf()
model_callback.fit_model()
status = check.get_status()[-1]
if status == True:#If true, mask the image.
data = z*model_callback.ccf_mask
data[np.isnan(data)] = np.inf#The colobar doesn't eat NaNs...
img1.set_array((data).ravel())
img2.set_array(model_callback.model.ravel())
img3.set_array((data-model_callback.model).ravel())
if status == False:#If false (unclicked), then just the data w/o mask.
img1.set_array(z.ravel())
img2.set_array(model_callback.model.ravel())
img3.set_array((z-model_callback.model).ravel())
plt.draw()
mask_slider.on_changed(update)
#And finally the save button.
rax_save = plt.axes([0.875, 0.1, 0.06, 0.05])
bsave = Button(rax_save, 'Save')
bsave.on_clicked(model_callback.save)
rax_cancel = plt.axes([0.8, 0.1, 0.06, 0.05])
bcancel = Button(rax_cancel, 'Cancel')
bcancel.on_clicked(model_callback.cancel)
plt.show()#All fitting is done before this line through event handling.
def xcor(list_of_wls,
list_of_orders,
wlm,
fxm,
drv,
RVrange,
plot=False,
list_of_errors=None):
"""
This routine takes a combined dataset (in the form of lists of wl spaces,
spectral orders and possible a matching list of errors on those spectal orders),
as well as a template (wlm,fxm) to cross-correlate with, and the cross-correlation
parameters (drv,RVrange). The code takes on the order of ~10 minutes for an entire
HARPS dataset, which appears to be superior to my old IDL pipe.
The CCF used is the Geneva-style weighted average; not the Pearson CCF. Therefore
it measures true 'average' planet lines, with flux on the y-axis of the CCF.
The template must therefore be (something close to) a binary mask, with values
inside spectral lines (the CCF is scale-invariant so their overall scaling
doesn't matter),
It returns the RV axis and the resulting CCF in a tuple.
Thanks to Brett Morris (bmorris3), this code now implements a clever numpy broadcasting trick to
instantly apply and interpolate the wavelength shifts of the model template onto
the data grid in 2 dimensions. The matrix multiplication operator (originally
recommended to me by Matteo Brogi) allowed this 2D template matrix to be multiplied
with a 2D spectral order. np.hstack() is used to concatenate all orders end to end,
effectively making a giant single spectral order (with NaNs in between due to masking).
All these steps have eliminated ALL the forloops from the equation, and effectuated a
speed gain of a factor between 2,000 and 3,000. The time to do cross correlations is now
typically measured in 100s of milliseconds rather than minutes.
This way of calculation does impose some strict rules on NaNs, though. To keep things fast,
NaNs are now used to set the interpolated template matrix to zero wherever there are NaNs in the data.
These NaNs are found by looking at the first spectrum in the stack, with the assumption that
every NaN is in an all-NaN column. In the standard cross-correlation work-flow, isolated
NaNs are interpolated over (healed), after all.
The places where there are NaN columns in the data are therefore set to 0 in the template matrix.
The NaN values themselves are then set to to an arbitrary value, since they will never
weigh into the average by construction.
Parameters
----------
list_of_wls : list
List of wavelength axes of the data.
list_of_orders : list
List of corresponding 2D orders.
list_of_errors : list
Optional, list of corresponding 2D error matrices.
wlm : np.ndarray
Wavelength axis of the template.
fxm : np.ndarray
Weight-axis of the template.
drv : int,float
The velocity step onto which the CCF is computed. Typically ~1 km/s.
RVrange : int,float
The velocity range in the positive and negative direction over which to
evaluate the CCF. Typically >100 km/s.
plot : bool
Set to True for diagnostic plotting.
Returns
-------
RV : np.ndarray
The radial velocity grid over which the CCF is evaluated.
CCF : np.ndarray
The weighted average flux in the spectrum as a function of radial velocity.
CCF_E : np.ndarray
Optional. The error on each CCF point propagated from the error on the spectral values.
Tsums : np.ndarray
The sum of the template for each velocity step. Used for normalising the CCFs.
"""
import tayph.functions as fun
import astropy.constants as const
import tayph.util as ut
from tayph.vartests import typetest, dimtest, postest, nantest
import numpy as np
import scipy.interpolate
import astropy.io.fits as fits
import matplotlib.pyplot as plt
import sys
import pdb
#===FIRST ALL SORTS OF TESTS ON THE INPUT===
if len(list_of_wls) != len(list_of_orders):
raise ValueError(
f'In xcor(): List of wls and list of orders have different length ({len(list_of_wls)} & {len(list_of_orders)}).'
)
dimtest(wlm, [len(fxm)], 'wlm in ccf.xcor()')
typetest(wlm, np.ndarray, 'wlm in ccf.xcor')
typetest(fxm, np.ndarray, 'fxm in ccf.xcor')
typetest(drv, [int, float], 'drv in ccf.xcor')
typetest(
RVrange,
float,
'RVrange in ccf.xcor()',
)
postest(RVrange, 'RVrange in ccf.xcor()')
postest(drv, 'drv in ccf.xcor()')
nantest(wlm, 'fxm in ccf.xcor()')
nantest(fxm, 'fxm in ccf.xcor()')
nantest(drv, 'drv in ccf.xcor()')
nantest(RVrange, 'RVrange in ccf.xcor()')
drv = float(drv)
N = len(list_of_wls) #Number of orders.
if np.ndim(list_of_orders[0]) == 1.0:
n_exp = 1
else:
n_exp = len(list_of_orders[0][:, 0]) #Number of exposures.
#===Then check that all orders indeed have n_exp exposures===
for i in range(N):
if len(list_of_orders[i][:, 0]) != n_exp:
raise ValueError(
f'In xcor(): Not all orders have {n_exp} exposures.')
#===END OF TESTS. NOW DEFINE CONSTANTS===
c = const.c.to('km/s').value #In km/s
RV = fun.findgen(
2.0 * RVrange / drv +
1) * drv - RVrange #..... CONTINUE TO DEFINE THE VELOCITY GRID
beta = 1.0 - RV / c #The doppler factor with which each wavelength is to be shifted.
n_rv = len(RV)
#===STACK THE ORDERS IN MASSIVE CONCATENATIONS===
stack_of_orders = np.hstack(list_of_orders)
stack_of_wls = np.concatenate(list_of_wls)
if list_of_errors is not None:
stack_of_errors = np.hstack(list_of_errors) #Stack them horizontally
#Check that the number of NaNs is the same in the orders as in the errors on the orders;
#and that they are in the same place; meaning that if I add the errors to the orders, the number of
#NaNs does not increase (NaN+value=NaN).
if (np.sum(np.isnan(stack_of_orders)) != np.sum(
np.isnan(stack_of_errors + stack_of_orders))) and (np.sum(
np.isnan(stack_of_orders)) != np.sum(
np.isnan(stack_of_errors))):
raise ValueError(
f"in CCF: The number of NaNs in list_of_orders and list_of_errors is not equal ({np.sum(np.isnan(list_of_orders))},{np.sum(np.isnan(list_of_errors))})"
)
#===HERE IS THE JUICY BIT===
shifted_wavelengths = stack_of_wls * beta[:, np.
newaxis] #2D broadcast of wl_data, each row shifted by beta[i].
T = scipy.interpolate.interp1d(wlm, fxm, bounds_error=False, fill_value=0)(
shifted_wavelengths) #...making this a 2D thing.
T[:, np.isnan(
stack_of_orders[0]
)] = 0.0 #All NaNs are assumed to be in all-NaN columns. If that is not true, the below nantest will fail.
T_sums = np.sum(T, axis=1)
#We check whether there are isolated NaNs:
n_nans = np.sum(np.isnan(stack_of_orders),
axis=0) #This is the total number of NaNs in each column.
n_nans[n_nans == len(
stack_of_orders
)] = 0 #Whenever the number of NaNs equals the length of a column, set the flag to zero.
if np.max(
n_nans
) > 0: #If there are any columns which still have NaNs in them, we need to crash.
raise ValueError(
f"in CCF: Not all NaN values are purely in columns. There are still isolated NaNs. Remove those."
)
stack_of_orders[np.isnan(
stack_of_orders)] = 47e20 #Set NaNs to arbitrarily high values.
CCF = stack_of_orders @ T.T / T_sums #Here it the entire cross-correlation. Over all orders and velocity steps. No forloops.
CCF_E = CCF * 0.0
#If the errors were provided, we do the same to those:
if list_of_errors is not None:
stack_of_errors[np.isnan(
stack_of_errors
)] = 42e20 #we have already tested that these NaNs are in the same place.
CCF_E = stack_of_errors**2 @ (
T.T / T_sums)**2 #This has been mathematically proven.
#===THAT'S ALL. TEST INTEGRITY AND RETURN THE RESULT===
nantest(
CCF, 'CCF in ccf.xcor()'
) #If anything went wrong with NaNs in the data, these tests will fail because the matrix operation @ is non NaN-friendly.
nantest(CCF_E, 'CCF_E in ccf.xcor()')
if list_of_errors != None:
return (RV, CCF, np.sqrt(CCF_E), T_sums)
return (RV, CCF, T_sums)
def plot_ccf(rv,
ccf,
dp,
xrange=[-200, 200],
yrange=[0, 0],
Nxticks=10.0,
Nyticks=10.0,
title='',
doppler_model=[],
i_legend=True,
show=True):
"""
This is a routine that does all the plotting of the cleaned 2D CCF. It overplots
the expected planet velocity, modified by the systemic velocity given in the config file.
Optionally, a trace of the doppler model is added as a list (of y-values) in the
doppler_model parameter. Set i_legend to False if you wish to remove the interactive legend.
"""
import numpy as np
import matplotlib.pyplot as plt
import pdb
import tayph.drag_colour as dcb
import tayph.functions as fun
import pylab as pl
import tayph.system_parameters as sp
#Load necessary physics for overplotting planet velocity.
vsys = sp.paramget('vsys', dp)
RVp = sp.RV(dp) + vsys
nexp = np.shape(ccf)[0]
#Default to the entire y-axis if yrange = [0,0]
if all(v == 0 for v in yrange):
yrange = [0, nexp - 1]
x2, y2, z, rv_sel, y_sel, xticks, yticks, vmin, vmax = plotting_scales_2D(
rv,
fun.findgen(nexp),
ccf,
xrange,
yrange,
Nxticks=Nxticks,
Nyticks=Nyticks,
nsigma=3.0)
#The plotting
fig, ax = plt.subplots(figsize=(12, 6))
img = ax.pcolormesh(x2, y2, z, vmin=vmin, vmax=vmax, cmap='hot')
ax.axis([x2.min(), x2.max(), y2.min(), y2.max()])
line1, = ax.plot(RVp,
fun.findgen(nexp),
'--',
color='black',
label='Planet rest-frame')
if len(doppler_model) > 0:
line2, = ax.plot(doppler_model + vsys,
fun.findgen(nexp),
'--',
color='black',
label='Doppler shadow')
ax.set_xticks(xticks)
ax.set_yticks(yticks)
ax.set_title(title)
ax.set_xlabel('Radial velocity (km/s)')
ax.set_ylabel('Exposure')
#The colourbar
cbar = plt.colorbar(img, format='%05.4f', aspect=15)
# cbar.set_norm(dcb.MyNormalize(vmin=vmin,vmax=vmax,stretch='linear'))
cbar = dcb.DraggableColorbar_fits(cbar, img, 'hot')
cbar.connect()
#The clickable legend.
if len(doppler_model) > 0:
lines = [line1, line2]
else:
lines = [line1]
if i_legend == True:
interactive_legend(fig, ax, lines)
if show == True:
plt.show()
return (fig, ax, cbar)
def blur_rotate(wl, order, dv, Rp, P, inclination, status=False, fast=False):
"""This function takes a spectrum and blurs it using a rotation x Gaussian
kernel which has a FWHM width of dv km/s everywhere. Meaning that its width changes
dynamically.
Because the kernel needs to be recomputed on each element of the wavelength axis
individually, this operation is much slower than convolution with a constant kernel,
in which a simple shifting of the array, rather than a recomputation of the rotation
profile is sufficient. By setting the fast keyword, the input array will first
be oversampled onto a constant-velocity grid to enable the usage of a constant kernel,
after which the result is interpolated back to the original grid.
Input:
The wavelength axis wl.
The spectral axis order.
The FHWM width of the resolution element in km/s.
The Radius of the rigid body in Rj.
The periodicity of the rigid body rotation in days.
The inclination of the spin axis in degrees.
Wavelength and order need to be numpy arrays and have the same number of elements.
Rp, P and i need to be scalar floats.
Output:
The blurred spectral axis, with the same dimensions as wl and order.
WARNING: THIS FUNCTION HANDLES NANS POORLY. I HAVE THEREFORE DECIDED CURRENTLY
TO REQUIRE NON-NAN INPUT.
This computes the simple numerical derivative of x by convolving with kernel [-1,0,1].
Parameters
----------
wl : list, np.ndarray
The wavelength array.
order : list, np.ndarray.
The spectral axis.
dv: float
The FWHM of a resolution element in km/s.
Rp: float
The radius of the planet in jupiter radii.
P: float
The rotation period of the planet. For tidally locked planets, this is equal
to the orbital period.
inclination:
The inclination of the spin axis in degrees. Presumed to be close to 90 degrees
for transiting planets
status: bool
Output a statusbar, but only if fast == False.
fast: bool
Re-interpolate the input on a constant-v grid in order to speed up the computation
of the convolution by eliminating the need to re-interpolate the kernel every step.
Returns
-------
order_blurred : np.array
The rotation-broadened spectrum on the same wavelength grid as the input.
Example
-------
>>> import tayph.functions as fun
>>> wl = fun.findgen(4000)*0.001+500.0
>>> fx = wl*0.0
>>> fx[2000] = 1.0
>>> fx_blurred1 = blur_rotate(wl,fx,3.0,1.5,0.8,90.0,status=False,fast=False)
>>> fx_blurred2 = blur_rotate(wl,fx,3.0,1.5,0.8,90.0,status=False,fast=True)
"""
import numpy as np
import tayph.util as ut
import tayph.functions as fun
from tayph.vartests import typetest, nantest, dimtest
from matplotlib import pyplot as plt
import astropy.constants as const
import astropy.units as u
import time
import sys
import pdb
from scipy import interpolate
typetest(dv, float, 'dv in blur_rotate()')
typetest(wl, [list, np.ndarray], 'wl in blur_rotate()')
typetest(order, [list, np.ndarray], 'order in blur_rotate()')
typetest(P, float, 'P in blur_rotate()')
typetest(Rp, float, 'Rp in blur_rotate()')
typetest(inclination, float, 'inclination in blur_rotate()')
typetest(status, bool, 'status in blur_rotate()')
typetest(fast, bool, 'fast in blur_rotate()')
nantest(wl, 'dv in blur_rotate()')
nantest(order, 'order in blur_rotate()')
dimtest(wl, [0], 'wl in blur_rotate()')
dimtest(order, [len(wl)],
'order in blur_rotate()') #Test that wl and order are 1D, and that
#they have the same length.
if np.min(np.array([dv, P, Rp])) <= 0.0:
raise Exception(
"ERROR in blur_rotate: dv, P and Rp should be strictly positive.")
#ut.typetest_array('wl',wl,np.float64)
#ut.typetest_array('order',order,np.float64)
#This is not possible because order may be 2D...
#And besides, you can have floats, np.float32 and np.float64... All of these would
#need to pass. Need to fix typetest_array some day.
order_blurred = order * 0.0 #init the output.
truncsize = 5.0 #The gaussian is truncated at 5 sigma from the extremest points of the RV amplitude.
sig_dv = dv / (2 * np.sqrt(2.0 * np.log(2))
) #Transform FWHM to Gaussian sigma. In km/s.
deriv = derivative(wl)
if max(deriv) < 0:
raise Exception(
"ERROR in ops.blur_rotate: WL derivative is smaller than 1.0. Sort wl in ascending order."
)
sig_wl = wl * sig_dv / (const.c.to('km/s').value) #in nm
sig_px = sig_wl / deriv
n = 1000.0
a = fun.findgen(n) / (n - 1) * np.pi
rv = np.cos(a) * np.sin(
np.radians(inclination)) * (2.0 * np.pi * Rp * const.R_jup /
(P * u.day)).to('km/s').value #in km/s
trunc_dist = np.round(sig_px * truncsize + np.max(rv) * wl /
(const.c.to('km/s').value) / deriv).astype(int)
# print('Maximum rotational rv: %s' % max(rv))
# print('Sigma_px: %s' % np.nanmean(np.array(sig_px)))
rvgrid_max = (np.max(trunc_dist) + 1.0) * sig_dv + np.max(rv)
rvgrid_n = rvgrid_max / dv * 100.0 #100 samples per lsf fwhm.
rvgrid = (
fun.findgen(2 * rvgrid_n + 1) - rvgrid_n
) / rvgrid_n * rvgrid_max #Need to make sure that this is wider than the truncation bin and more finely sampled than wl - everywhere.
lsf = rvgrid * 0.0
#We loop through velocities in the velocity grid to build up the sum of Gaussians
#that is the LSF.
for v in rv:
lsf += fun.gaussian(
rvgrid, 1.0, v, sig_dv
) #This defines the LSF on a velocity grid wih high fidelity.
if fast:
wlt, fxt, dv = constant_velocity_wl_grid(wl, order, 4)
dv_grid = rvgrid[1] - rvgrid[0]
len_rv_grid_low = int(max(rvgrid) / dv * 2 - 2)
# print(len_rv_grid_low)
# print(len(fun.findgen(len_rv_grid_low)))
# print(len_rv_grid_low%2)
if len_rv_grid_low % 2 == 0:
len_rv_grid_low -= 1
rvgrid_low = fun.findgen(
len_rv_grid_low) * dv #Slightly smaller than the original grid.
rvgrid_low -= 0.5 * np.max(rvgrid_low)
lsf_low = interpolate.interp1d(rvgrid, lsf)(rvgrid_low)
lsf_low /= np.sum(
lsf_low
) #This is now an LSF on a grid with the same spacing as the data has.
#This means I can use it directly as a convolution kernel:
fxt_blurred = convolve(fxt, lsf_low, edge_degree=1, fit_width=1)
#And interpolate back to where it came from:
order_blurred = interpolate.interp1d(wlt,
fxt_blurred,
bounds_error=False)(wl)
#I can use interp1d because after blurring, we are now oversampled.
# order_blurred2 = bin_avg(wlt,fxt_blurred,wl)
return (order_blurred)
#Now we loop through the wavelength grid to place this LSF at each wavelength position.
for i in range(0, len(wl)):
binstart = max([0, i - trunc_dist[i]])
binend = i + trunc_dist[i]
wlbin = wl[binstart:binend]
wlgrid = wl[i] * rvgrid / (const.c.to('km/s').value) + wl[
i] #This converts the velocity grid to a d-wavelength grid centered on wk[i]
#print([np.min(wlbin),np.min(wlgrid),np.max(wlbin),np.max(wlgrid)])
i_wl = interpolate.interp1d(
wlgrid, lsf, bounds_error=False, fill_value='extrapolate'
) #Extrapolate should not be necessary but sometimes there is a minute mismatch between the
#start and end wavelengths of the constructed grid and the bin.
try:
lsf_wl = i_wl(wlbin)
except:
ut.tprint(
'Error in interpolating LSF onto wlbin. Pausing to debug.')
pdb.set_trace()
k_n = lsf_wl / np.sum(
lsf_wl
) #Normalize at each instance of the interpolation to make sure flux is conserved exactly.
order_blurred[i] = np.sum(k_n * order[binstart:binend])
if status == True:
ut.statusbar(i, len(wl))
return (order_blurred)
def smooth(fx, w, mode='box', edge_degree=1):
"""
This function takes a spectrum, and blurs it using either a
Gaussian kernel or a box kernel, which have a FWHM width of w px everywhere.
Meaning that the width changes dynamically on a constant d-lambda grid.
Set the mode to gaussian or box. Because in box, care is taken to correctly
interpolate the edges, it is about twice slower than the Gaussian.
This interpolation is done manually in the fun.box function.
"""
import numpy as np
import tayph.util as ut
from tayph.vartests import typetest, postest
import tayph.functions as fun
from matplotlib import pyplot as plt
typetest(w, [int, float], 'w in ops.smooth()')
typetest(fx, np.ndarray, 'fx in ops.smooth()')
typetest(mode, str, 'mode in ops.smooth()')
typetest(edge_degree, int, 'edge_degree in ops.smooth()')
postest(w, 'w in ops.smooth()')
if mode not in ['box', 'gaussian']:
raise Exception(
f'RuntimeError in ops.smooth(): Mode should be set to "top" or "bottom" ({mode}).'
)
truncsize = 4.0 #The gaussian is truncated at 4 sigma.
shape = np.shape(fx)
sig_w = w / 2 * np.sqrt(
2.0 * np.log(2)) #Transform FWHM to Gaussian sigma. In km/s.
trunc_dist = np.round(sig_w * truncsize).astype(int)
#First define the kernel.
kw = int(np.round(truncsize * sig_w * 2.0))
if kw % 2.0 != 1.0: #This is to make sure that the kernel has an odd number of
#elements, and that it is symmetric around zero.
kw += 1
kx = fun.findgen(kw)
kx -= np.mean(
kx) #This must be centered around zero. Doing a hardcoded check:
if (-1.0) * kx[-1] != kx[0]:
print(kx)
raise Exception(
"ERROR in box_smooth: Kernel could not be made symmetric somehow. Attempted kernel grid is printed above. Kernel width is %s pixels."
% kw)
if mode == 'gaussian':
k = fun.gaussian(kx, 1.0, 0.0, sig_w)
if mode == 'box':
k = fun.box(kx, 1.0, 0.0, w)
kx = kx[k > 0.0]
k = k[k > 0.0]
if (-1.0) * kx[-1] != kx[0]:
print(kx)
raise Exception(
"ERROR in box_smooth: Kernel could not be made symmetric AFTER CROPPING OUT THE BOX, somehow. Attempted kernel grid is printed above. Kernel width is %s pixels."
% kw)
k /= np.sum(k)
return (convolve(fx, k, edge_degree))
def write_file_to_molecfit(molecfit_file_root,
name,
headers,
spectra,
ii,
mode='HARPS',
wave=[]):
"""This is a wrapper for writing a spectrum from a list to molecfit format.
name is the filename of the fits file that is the output.
headers is the list of astropy header objects associated with the list of spectra
in the spectra variable. ii is the number from that list that needs to be written.
The wave keyword is set for when the s1d headers do not contain wavelength information like HARPS does.
(for instance, ESPRESSO). The wave keyword needs to be set in this case, to the wavelength array as extracted from FITS files or smth.
If you do that for HARPS and set the wave keyword, this code will still grab it from the header, and overwrite it. So dont bother.
"""
import astropy.io.fits as fits
from scipy import stats
import copy
import tayph.functions as fun
import astropy.constants as const
import astropy.units as u
import numpy as np
from tayph.vartests import typetest
import tayph.util as ut
import sys
typetest(ii, int, 'ii write_file_to_molecfit')
molecfit_file_root = ut.check_path(molecfit_file_root, exists=True)
spectrum = spectra[int(ii)]
npx = len(spectrum)
if mode == 'HARPS':
berv = headers[ii][
'HIERARCH ESO DRS BERV'] #Need to un-correct the s1d spectra to go back to the frame of the Earths atmosphere.
wave = (headers[ii]['CDELT1'] * fun.findgen(len(spectra[ii])) +
headers[ii]['CRVAL1']) * (
1.0 - (berv * u.km / u.s / const.c).decompose().value)
elif mode == 'HARPSN':
berv = headers[ii][
'HIERARCH TNG DRS BERV'] #Need to un-correct the s1d spectra to go back to the frame of the Earths atmosphere.
wave = (headers[ii]['CDELT1'] * fun.findgen(len(spectra[ii])) +
headers[ii]['CRVAL1']) * (
1.0 - (berv * u.km / u.s / const.c).decompose().value)
elif mode in ['ESPRESSO', 'UVES-red', 'UVES-blue']:
if len(wave) == 0:
raise ValueError(
'in write_file_to_molecfit(): When mode in [ESPRESSO,UVES-red,UVES-blue], the 1D wave axis needs to be provided.'
)
#WAVE VARIABLE NEEDS TO BE PASSED NOW.
berv = headers[ii][
'HIERARCH ESO QC BERV'] #Need to un-correct the s1d spectra to go back to the frame of the Earths atmosphere.
wave = copy.deepcopy(
wave * (1.0 - (berv * u.km / u.s / const.c).decompose().value))
err = np.sqrt(spectrum)
#Write out the s1d spectrum in a format that molecfit eats.
#This is a fits file with an empty primary extension that contains the header of the original s1d file.
#Plus an extension that contains a binary table with 3 columns.
#The names of these columns need to be indicated in the molecfit parameter file,
#as well as the name of the file itself. This is currently hardcoded.
col1 = fits.Column(name='wavelength', format='1D', array=wave)
col2 = fits.Column(name='flux', format='1D', array=spectrum)
col3 = fits.Column(name='err_flux', format='1D', array=err)
cols = fits.ColDefs([col1, col2, col3])
tbhdu = fits.BinTableHDU.from_columns(cols)
prihdr = fits.Header()
prihdr = copy.deepcopy(headers[ii])
prihdu = fits.PrimaryHDU(header=prihdr)
thdulist = fits.HDUList([prihdu, tbhdu])
thdulist.writeto(molecfit_file_root / name, overwrite=True)
print(f'Spectrum {ii} written')
return (0)
def read_e2ds(inpath,
outname,
config,
nowave=False,
molecfit=False,
mode='HARPS',
ignore_exp=[]):
"""This is the workhorse for reading in a time-series of archival 2D echelle
spectra and formatting these into the order-wise FITS format that Tayph uses.
The user should point this script to a folder (located at inpath) that contains
their pipeline-reduced echelle spectra. The script expects a certain data
format, depending on the instrument in question. It is designed to accept
pipeline products of the HARPS, HARPS-N, ESPRESSO and UVES instruments. In the
case of HARPS, HARPS-N and ESPRESSO these may be downloaded from the archive.
UVES is a bit special, because the 2D echelle spectra are not a standard
pipeline output. Typical use cases are explained further below.
This script formats the time series of 2D echelle spectra into 2D FITS images,
where each FITS file is the time-series of a single echelle order. If the
spectrograph has N orders, an order spans npx pixels, and M exposures
were taken during the time-series, there will be N FITS files, each measuring
M rows by npx columns. This script will read the headers of the pipeline
reduced data files to determine the date/time of each, the exposure time, the
barycentric correction (without applying it) and the airmass, and writes
these to an ASCII table along with the FITS files containing the spectral
orders.
A crucial functionality of this script is that it also acts as a wrapper
for the Molecfit telluric correction software. If installed properly, the
user can call this script with the molecfit keyword to let Molecfit loop
over the entire timeseries. To enable this functionality, the script
reads the full-width, 1D spectra that are output by the instrument pipelines
as well. Molecfit is applied to this time-series of 1D spectra, creating a
time-series of models of the telluric absorption spectrum that is saved along
with the 2D fits files. Tayph later interpolates these models onto the 2D
spectra. Molecfit is called once in GUI-mode, allowing the user to select the
relevant fitting regions and parameters, after which it is repeated
automatically for the entire time series.
Without Molecfit, this script finishes in a matter of seconds. However with
molecfit enabled, it can take many hours (so if I wish to telluric-correct
my data, I run this script overnight).
The processing of HARPS, HARPS-N and ESPRESSO data is executed in an almost
identical manner, because the pipeline-reduced products are almost identical.
To run on either of these instruments, the user simply downloads all pipeline
products of a given time-series, and extracts these in the same folder (meaning
ccfs, e2ds/s2d, s1d, blaze, wave files, etc.) This happens to be the standard
format when downloading pipeline-reduced data from the archive.
For UVES, the functionality is much more constricted because the pipeline
reduced data in the ESO archive is generally not of sufficient stability to
enable precise time-resolved spectroscopy. I designed this function therefore
to run on the pipeline-products produced by the Reflex (GUI) software. For this,
a user should download the raw UVES data of their time series, letting ESO's
calselector tool find the associated calibration files. This can easily be
many GBs worth of data for a given observing program. The user should then
reduce these data with the Reflex software. Reflex creates resampled, stitched
1D spectra as its primary output. However, we will elect to use the intermediate
pipeline products, which include the 2D extracted orders, located in Reflex's
working directory after the reduction process is completed.
A further complication of UVES data is that it can be used with different
dichroics and 'arms', leading to spectral coverage on the blue, redu and/or redl
chips. The user should take care that their time series contains only one of these
types at any time. If they are mixed, this script will throw an exception.
Set the nowave keyword to True if the dataset is HARPS or HARPSN, but it has
no wave files associated with it. This may happen if you downloaded ESO
Advanced Data Products, which include reduced science e2ds's but not reduced
wave e2ds's. The wavelength solution is still encoded in the fits header however,
so we take it from there, instead.
Set the ignore_exp keyword to a list of exposures (start counting at 0) that
need to be ignored when reading, e.g. because they are bad for some reason.
If you have set molecfit to True, this becomes an expensive parameter to
play with in terms of computing time, so its better to figure out which
exposures you'd wish to ignore first (by doing most of your analysis),
before actually running Molecfit, which is icing on the cake in many use-
cases in the optical.
The config parameter points to a configuration file (usually your generic
run definition file) that is only used to point the Molecfit wrapper to the
Molecfit installation on your system. If you are not using molecfit, you may
pass an empty string here.
"""
import os
import pdb
from astropy.io import fits
import astropy.constants as const
import numpy as np
import matplotlib.pyplot as plt
import sys
import tayph.util as ut
from tayph.vartests import typetest, dimtest
import tayph.tellurics as mol
import tayph.system_parameters as sp
import tayph.functions as fun
import copy
import scipy.interpolate as interp
import pickle
from pathlib import Path
import warnings
import glob
# molecfit = False
#First check the input:
inpath = ut.check_path(inpath, exists=True)
typetest(outname, str, 'outname in read_HARPS_e2ds()')
typetest(nowave, bool, 'nowave switch in read_HARPS_e2ds()')
typetest(molecfit, bool, 'molecfit switch in read_HARPS_e2ds()')
typetest(ignore_exp, list, 'ignore_exp in read_HARPS_e2ds()')
typetest(mode, str, 'mode in read_HARPS_e2ds()')
if molecfit:
config = ut.check_path(config, exists=True)
if mode not in [
'HARPS', 'HARPSN', 'HARPS-N', 'ESPRESSO', 'UVES-red', 'UVES-blue'
]:
raise ValueError(
"in read_HARPS_e2ds: mode needs to be set to HARPS, HARPSN, UVES-red, UVES-blue or ESPRESSO."
)
filelist = os.listdir(
inpath
) #If mode == UVES, these are folders. Else, they are fits files.
N = len(filelist)
if len(filelist) == 0:
raise FileNotFoundError(
f" in read_e2ds: input folder {str(inpath)} is empty.")
#The following variables define lists in which all the necessary data will be stored.
framename = []
header = []
s1dhdr = []
type = []
texp = np.array([])
date = []
mjd = np.array([])
s1dmjd = np.array([])
npx = np.array([])
norders = np.array([])
e2ds = []
s1d = []
wave1d = []
airmass = np.array([])
berv = np.array([])
wave = []
blaze = []
wavefile_used = []
outpath = Path('data/' + outname)
if os.path.exists(outpath) != True:
os.makedirs(outpath)
e2ds_count = 0
sci_count = 0
wave_count = 0
blaze_count = 0
s1d_count = 0
if mode == 'HARPS-N': mode = 'HARPSN'
#MODE SWITCHING HERE:
if mode in ['HARPS', 'UVES-red', 'UVES-blue']:
catkeyword = 'HIERARCH ESO DPR CATG'
bervkeyword = 'HIERARCH ESO DRS BERV'
thfilekeyword = 'HIERARCH ESO DRS CAL TH FILE'
Zstartkeyword = 'HIERARCH ESO TEL AIRM START'
Zendkeyword = 'HIERARCH ESO TEL AIRM END'
if mode == 'HARPSN':
catkeyword = 'OBS-TYPE'
bervkeyword = 'HIERARCH TNG DRS BERV'
thfilekeyword = 'HIERARCH TNG DRS CAL TH FILE'
Zstartkeyword = 'AIRMASS'
Zendkeyword = 'AIRMASS' #These are the same because HARPSN doesnt have start and end keywords.
#Down there, the airmass is averaged, so there is no problem in taking the average of the same number.
#Here is the actual parsing of the list of files that were read above. The
#behaviour is different depending on whether this is HARPS, UVES or ESPRESSO
#data, so it switches with a big if-statement in which there is a forloop
#over the filelist in each case. The result is lists or np.arrays containing
#the 2D spectra, the 1D spectra, their 2D and 1D wavelength solutions, the
#headers, the MJDs, the BERVs and the airmasses, as well as (optionally) CCFs
#and blaze files, though these are not really used.
print(f'Read_e2ds is attempting to read a {mode} datafolder.')
if mode == 'UVES-red' or mode == 'UVES-blue': #IF we are UVES-like
for i in range(N):
print(filelist[i])
if (inpath / Path(filelist[i])).is_dir():
tmp_products = [
i for i in (
inpath /
Path(filelist[i])).glob('resampled_science_*.fits')
]
tmp_products1d = [
i for i in (inpath /
Path(filelist[i])).glob('red_science_*.fits')
]
if mode == 'UVES-red' and len(tmp_products) != 2:
raise ValueError(
f"in read_e2ds: When mode=UVES-red there should be 2 resampled_science files (redl and redu), but {len(tmp_products)} were detected."
)
if mode == 'UVES-blue' and len(tmp_products) != 1:
raise ValueError(
f"in read_e2ds: When mode=UVES-rblue there should be 1 resampled_science files (blue), but {len(tmp_products)} were detected."
)
if mode == 'UVES-red' and len(tmp_products1d) != 2:
raise ValueError(
f"in read_e2ds: When mode=UVES-red there should be 2 red_science files (redl and redu), but {len(tmp_products1d)} were detected."
)
if mode == 'UVES-blue' and len(tmp_products1d) != 1:
raise ValueError(
f"in read_e2ds: When mode=UVES-rblue there should be 1 red_science files (blue), but {len(tmp_products1d)} were detected."
)
data_combined = [
] #This will store the two chips (redu and redl) in case of UVES_red, or simply the blue chip if otherwise.
wave_combined = []
wave1d_combined = []
data1d_combined = []
norders_tmp = 0
for tmp_product in tmp_products:
hdul = fits.open(tmp_product)
data = copy.deepcopy(hdul[0].data)
hdr = hdul[0].header
hdul.close()
del hdul[0].data
if not hdr[
'HIERARCH ESO PRO SCIENCE']: #Only add if it's actually a science product:#I force the user to supply only science exposures in the input folder. No BS allowed... UVES is hard enough as it is.
raise ValueError(
f' in read_e2ds: UVES file {tmp_product} is not classified as a SCIENCE file, but should be. Remove it from the folder?'
)
wavedata = ut.read_wave_from_e2ds_header(hdr, mode='UVES')
data_combined.append(data)
wave_combined.append(wavedata)
norders_tmp += np.shape(data)[0]
for tmp_product in tmp_products1d:
hdul = fits.open(tmp_product)
data_1d = copy.deepcopy(hdul[0].data)
hdr1d = hdul[0].header
hdul.close()
del hdul[0].data
if not hdr1d[
'HIERARCH ESO PRO SCIENCE']: #Only add if it's actually a science product:#I force the user to supply only science exposures in the input folder. No BS allowed... UVES is hard enough as it is.
raise ValueError(
f' in read_e2ds: UVES file {tmp_product} is not classified as a SCIENCE file, but should be. Remove it from the folder?'
)
npx1d = hdr1d['NAXIS1']
wavedata = fun.findgen(
npx1d) * hdr1d['CDELT1'] + hdr1d['CRVAL1']
data1d_combined.append(data_1d)
wave1d_combined.append(wavedata)
if len(data_combined) < 1 or len(
data_combined
) > 2: #Double-checking that length here...
raise ValueError(
f'in read_e2ds(): Expected 1 or 2 chips, but {len(data_combined)} files were somehow read.'
)
#The chips generally don't give the same size. Therefore I will pad the smaller one with NaNs to make it fit:
if len(data_combined) != len(data1d_combined):
raise ValueError(
f'in read_e2ds(): The number of chips in the 1d and 2d spectra is not the same {len(data1d_combined)} vs {len(data_combined)}.'
)
if len(data_combined) == 2:
chip1 = data_combined[0]
chip2 = data_combined[1]
wave1 = wave_combined[0]
wave2 = wave_combined[1]
npx_1 = np.shape(chip1)[1]
npx_2 = np.shape(chip2)[1]
no_1 = np.shape(chip1)[0]
no_2 = np.shape(chip2)[0]
npx_max = np.max([npx_1, npx_2])
npx_min = np.min([npx_1, npx_2])
diff = npx_max - npx_min
#Pad the smaller one with NaNs to match the wider one:
if npx_1 < npx_2:
chip1 = np.hstack(
[chip1, np.zeros((no_1, diff)) * np.nan])
wave1 = np.hstack(
[wave1, np.zeros((no_1, diff)) * np.nan])
else:
chip2 = np.hstack(
[chip2, np.zeros((no_2, diff)) * np.nan])
wave2 = np.hstack(
[wave2, np.zeros((no_2, diff)) * np.nan])
#So now they can be stacked:
e2ds_stacked = np.vstack((chip1, chip2))
npx = np.append(npx, np.shape(e2ds_stacked)[1])
e2ds.append(e2ds_stacked)
wave.append(np.vstack((wave1, wave2)))
chip1_1d = data1d_combined[0]
chip2_1d = data1d_combined[1]
wave1_1d = wave1d_combined[0]
wave2_1d = wave1d_combined[1]
if np.nanmean(wave1_1d) < np.nanmean(wave2_1d):
combined_data_1d = np.concatenate((chip1_1d, chip2_1d))
combined_wave_1d = np.concatenate((wave1_1d, wave2_1d))
else:
combined_data_1d = np.concatenate((chip2_1d, chip1_1d))
combined_wave_1d = np.concatenate((wave2_1d, wave1_1d))
wave1d.append(combined_wave_1d)
s1d.append(combined_data_1d)
else:
e2ds.append(data_combined[0])
wave.append(wave_combined[0])
npx = np.append(npx, np.shape(data_combined[0])[1])
wave1d.append(wave1d_combined[0])
s1d.append(data1d_combined[0])
#Only using the keyword from the second header in case of redl,redu.
s1dmjd = np.append(s1dmjd, hdr1d['MJD-OBS'])
framename.append(hdr['ARCFILE'])
header.append(hdr)
type.append('SCIENCE')
texp = np.append(texp, hdr['EXPTIME'])
date.append(hdr['DATE-OBS'])
mjd = np.append(mjd, hdr['MJD-OBS'])
norders = np.append(norders, norders_tmp)
airmass = np.append(
airmass, 0.5 * (hdr[Zstartkeyword] + hdr[Zendkeyword])
) #This is an approximation where we take the mean airmass.
berv_i = sp.calculateberv(hdr['MJD-OBS'],
hdr['HIERARCH ESO TEL GEOLAT'],
hdr['HIERARCH ESO TEL GEOLON'],
hdr['HIERARCH ESO TEL GEOELEV'],
hdr['RA'], hdr['DEC'])
berv = np.append(berv, berv_i)
hdr1d[
'HIERARCH ESO QC BERV'] = berv_i #Append the berv here using the ESPRESSO berv keyword, so that it can be used in molecfit later.
s1dhdr.append(hdr1d)
sci_count += 1
s1d_count += 1
e2ds_count += 1
elif mode == 'ESPRESSO':
catkeyword = 'EXTNAME'
bervkeyword = 'HIERARCH ESO QC BERV'
airmass_keyword1 = 'HIERARCH ESO TEL'
airmass_keyword2 = ' AIRM '
airmass_keyword3_start = 'START'
airmass_keyword3_end = 'END'
for i in range(N):
if filelist[i].endswith('S2D_A.fits'):
e2ds_count += 1
print(filelist[i])
hdul = fits.open(inpath / filelist[i])
data = copy.deepcopy(hdul[1].data)
hdr = hdul[0].header
hdr2 = hdul[1].header
wavedata = copy.deepcopy(hdul[5].data)
hdul.close()
del hdul[1].data
if hdr2[catkeyword] == 'SCIDATA':
print('science keyword found')
framename.append(filelist[i])
header.append(hdr)
type.append('SCIENCE')
texp = np.append(texp, hdr['EXPTIME'])
date.append(hdr['DATE-OBS'])
mjd = np.append(mjd, hdr['MJD-OBS'])
npx = np.append(npx, hdr2['NAXIS1'])
norders = np.append(norders, hdr2['NAXIS2'])
e2ds.append(data)
sci_count += 1
berv = np.append(berv, hdr[bervkeyword] * 1000.0)
telescope = hdr['TELESCOP'][-1]
airmass = np.append(
airmass, 0.5 *
(hdr[airmass_keyword1 + telescope + ' AIRM START'] +
hdr[airmass_keyword1 + telescope + ' AIRM END']))
wave.append(wavedata * (1.0 -
(hdr[bervkeyword] * u.km / u.s /
const.c).decompose().value))
#Ok.! So unlike HARPS, ESPRESSO wavelengths are BERV corrected in the S2Ds.
#WHY!!!?. WELL SO BE IT. IN ORDER TO HAVE E2DSes THAT ARE ON THE SAME GRID, AS REQUIRED, WE UNDO THE BERV CORRECTION HERE.
#WHEN COMPARING WAVE[0] WITH WAVE[1], YOU SHOULD SEE THAT THE DIFFERENCE IS NILL.
#THATS WHY LATER WE JUST USE WAVE[0] AS THE REPRESENTATIVE GRID FOR ALL.
if filelist[i].endswith('CCF_A.fits'):
#ccf,hdr=fits.getdata(inpath+filelist[i],header=True)
hdul = fits.open(inpath / filelist[i])
ccf = copy.deepcopy(hdul[1].data)
hdr = hdul[0].header
hdr2 = hdul[1].header
hdul.close()
del hdul[1].data
if hdr2[catkeyword] == 'SCIDATA':
print('CCF ADDED')
#ccftotal+=ccf
ccfs.append(ccf)
ccfmjd = np.append(ccfmjd, hdr['MJD-OBS'])
nrv = np.append(nrv, hdr2['NAXIS1'])
ccf_count += 1
if filelist[i].endswith('S1D_A.fits'):
hdul = fits.open(inpath / filelist[i])
data_table = copy.deepcopy(hdul[1].data)
hdr = hdul[0].header
hdr2 = hdul[1].header
hdul.close()
del hdul[1].data
if hdr['HIERARCH ESO PRO SCIENCE'] == True:
s1d.append(data_table.field(2))
wave1d.append(data_table.field(1))
s1dhdr.append(hdr)
s1dmjd = np.append(s1dmjd, hdr['MJD-OBS'])
s1d_count += 1
else: #IF we are HARPS-like:
for i in range(N):
if filelist[i].endswith('e2ds_A.fits'):
e2ds_count += 1
print(filelist[i])
hdul = fits.open(inpath / filelist[i])
data = copy.deepcopy(hdul[0].data)
hdr = hdul[0].header
hdul.close()
del hdul[0].data
if hdr[catkeyword] == 'SCIENCE':
framename.append(filelist[i])
header.append(hdr)
type.append(hdr[catkeyword])
texp = np.append(texp, hdr['EXPTIME'])
date.append(hdr['DATE-OBS'])
mjd = np.append(mjd, hdr['MJD-OBS'])
npx = np.append(npx, hdr['NAXIS1'])
norders = np.append(norders, hdr['NAXIS2'])
e2ds.append(data)
sci_count += 1
berv = np.append(berv, hdr[bervkeyword])
airmass = np.append(
airmass, 0.5 * (hdr[Zstartkeyword] + hdr[Zendkeyword])
) #This is an approximation where we take the mean airmass.
if nowave == True:
#Record which wavefile was used by the pipeline to
#create the wavelength solution.
wavefile_used.append(hdr[thfilekeyword])
wavedata = ut.read_wave_from_e2ds_header(hdr,
mode=mode)
wave.append(wavedata)
# else:
# berv=np.append(berv,np.nan)
# airmass=np.append(airmass,np.nan)
if filelist[i].endswith('wave_A.fits'):
print(filelist[i] + ' (wave)')
if nowave == True:
warnings.warn(
" in read_e2ds: nowave was set to True but a wave_A file was detected. This wave file is now ignored in favor of the header.",
RuntimeWarning)
else:
wavedata = fits.getdata(inpath / filelist[i])
wave.append(wavedata)
wave_count += 1
if filelist[i].endswith('blaze_A.fits'):
print(filelist[i] + ' (blaze)')
blazedata = fits.getdata(inpath / filelist[i])
blaze.append(blazedata)
blaze_count += 1
if filelist[i].endswith('s1d_A.fits'):
hdul = fits.open(inpath / filelist[i])
data_1d = copy.deepcopy(hdul[0].data)
hdr = hdul[0].header
hdul.close()
del hdul[0].data
if hdr[catkeyword] == 'SCIENCE':
s1d.append(data_1d)
s1dhdr.append(hdr)
s1dmjd = np.append(s1dmjd, hdr['MJD-OBS'])
s1d_count += 1
#Now we catch some errors:
#-The above should have read a certain number of e2ds files.
#-A certain number of these should be SCIENCE frames.
#-There should be at least one WAVE file.
#-All exposures should have the same number of spectral orders.
#-All orders should have the same number of pixels (this is true for HARPS).
#-The wave frame should have the same dimensions as the order frames.
#-If nowave is set, test that all frames used the same wave_A calibrator.
#-The blaze file needs to have the same shape as the e2ds files.
#-The number of s1d files should be the same as the number of e2ds files.
if e2ds_count == 0:
raise FileNotFoundError(
f"in read_e2ds: The input folder {str(inpath)} does not contain files ending in e2ds.fits."
)
if sci_count == 0:
print('')
print('')
print('')
print("These are the files and their types:")
for i in range(len(type)):
print(' ' + framename[i] + ' %s' % type[i])
raise ValueError(
"in read_e2ds: The input folder (%2) contains e2ds files, but none of them are classified as SCIENCE frames with the HIERARCH ESO DPR CATG/OBS-TYPE keyword or HIERARCH ESO PRO SCIENCE keyword. The list of frames is printed above."
)
if np.max(np.abs(norders - norders[0])) == 0:
norders = int(norders[0])
else:
print('')
print('')
print('')
print("These are the files and their number of orders:")
for i in range(len(type)):
print(' ' + framename[i] + ' %s' % norders[i])
raise ValueError(
"in read_e2ds: Not all files have the same number of orders. The list of frames is printed above."
)
if np.max(np.abs(npx - npx[0])) == 0:
npx = int(npx[0])
else:
print('')
print('')
print('')
print("These are the files and their number of pixels:")
for i in range(len(type)):
print(' ' + framename[i] + ' %s' % npx[i])
raise ValueError(
"in read_HARPS_e2ds: Not all files have the same number of pixels. The list of frames is printed above."
)
if wave_count >= 1:
wave = wave[0] #SELECT ONLY THE FIRST WAVE FRAME. The rest is ignored.
else:
if nowave == False and mode not in ['UVES-red', 'UVES-blue']:
print('')
print('')
print('')
print("ERROR in read_e2ds: No wave_A.fits file was detected.")
print("These are the files in the folder:")
for i in range(N):
print(filelist[i])
print(
"This may have happened if you downloaded the HARPS data from the"
)
print(
"ADP query form, which doesn't include wave_A files (as far as I"
)
print(
"have seen). Set the /nowave keyword in your call to read_HARPS_e2ds"
)
print("if you indeed do not expect a wave_A file to be present.")
raise FileNotFoundError(
"No wave_A.fits file was detected. More details are printed above."
)
if nowave == True and mode not in [
'UVES-red', 'UVES-blue', 'ESPRESSO'
]: #This here is peculiar to HARPS/HARPSN.
if all(x == wavefile_used[0] for x in wavefile_used):
print(
"Nowave is set, and simple wavelength calibration extraction")
print(
"works, as all files in the dataset used the same wave_A file."
)
wave = wave[0]
else:
print('')
print('')
print('')
print("These are the filenames and their wave_A file used:")
for i in range(N - 1):
print(' ' + framename[i] + ' %s' % wavefile_used[0])
warnings.warn(
"in read_e2ds: Nowave is set, but not all files in the dataset used the same wave_A file when the pipeline was run. This script will continue using only the first wavelength solution. Theoretically, this may affect the quality of the data if the solution is wrong (in which case interpolation would be needed), but due to the stability of HARPS this is probably not an issue worth interpolating for.",
RuntimeWarning)
wave = wave[0]
if mode == 'ESPRESSO':
dimtest(wave1d, np.shape(s1d), 'wave and s1d in read_e2ds()')
dimtest(wave, np.shape(e2ds), 'wave and e2ds in read_e2ds()')
diff = wave - wave[0]
if np.abs(np.nanmax(diff)) > (np.nanmin(wave[0]) / 1e6):
warnings.warn(
" in read_e2ds: The wavelength solution over the time series is not constant. I continue by interpolating all the data onto the wavelength solution of the first frame. This will break if the wavelength solutions of your time-series are vastly different, in which case the output will be garbage.",
RuntimeWarning)
for ii in range(len(e2ds)):
if ii > 0:
for jj in range(len(e2ds[ii])):
e2ds[ii][jj] = interp.interp1d(
wave[ii][jj],
e2ds[ii][jj],
fill_value='extrapolate')(wave[0][jj])
wave = wave[0]
diff1d = wave1d - wave1d[0]
if np.abs(np.nanmax(diff1d)) > (
np.nanmin(wave1d) / 1e6
): #if this is true, all the wavelength solutions are the same. Great!
warnings.warn(
" in read_e2ds: The wavelength solution over the time series of the 1D spectra is not constant. I continue by interpolating all the 1D spectra onto the wavelength solution of the first frame. This will break if the wavelength solutions of your time-series are vastly different, in which case the output will be garbage.",
RuntimeWarning)
for ii in range(len(s1d)):
if ii > 0:
s1d[ii] = interp.interp1d(wave1d[ii],
s1d[ii],
fill_value='extrapolate')(
wave1d[0])
wave1d = wave1d[0]
#We are going to test whether the wavelength solution is the same for the
#entire time series in the case of UVES. In normal use cases this should be true.
if mode in ['UVES-red', 'UVES-blue']:
dimtest(wave1d, np.shape(s1d), 'wave and s1d in read_e2ds()')
dimtest(wave, np.shape(e2ds), 'wave and e2ds in read_e2ds()')
diff = wave - wave[0]
if np.abs(
np.nanmax(diff)
) == 0: #if this is true, all the wavelength solutions are the same. Great!
wave = wave[0]
else:
warnings.warn(
" in read_e2ds: The wavelength solution over the time series is not constant. I continue by interpolating all the data onto the wavelength solution of the first frame. This will break if the wavelength solutions of your time-series are vastly different, in which case the output will be garbage.",
RuntimeWarning)
for ii in range(len(e2ds)):
if ii > 0:
for jj in range(len(e2ds[ii])):
e2ds[ii][jj] = interp.interp1d(
wave[ii][jj],
e2ds[ii][jj],
fill_value='extrapolate')(wave[0][jj])
wave = wave[0]
diff1d = wave1d - wave1d[0]
if np.abs(
np.nanmax(diff1d)
) == 0: #if this is true, all the wavelength solutions are the same. Great!
wave1d = wave1d[0]
else:
warnings.warn(
" in read_e2ds: The wavelength solution over the time series of the 1D spectra is not constant. I continue by interpolating all the 1D spectra onto the wavelength solution of the first frame. This will break if the wavelength solutions of your time-series are vastly different, in which case the output will be garbage.",
RuntimeWarning)
for ii in range(len(s1d)):
if ii > 0:
s1d[ii] = interp.interp1d(wave1d[ii],
s1d[ii],
fill_value='extrapolate')(
wave1d[0])
wave1d = wave1d[0]
if blaze_count >= 1: #This will only be triggered for HARPS/HARPSN/ESPRESSO
blaze = blaze[
0] #SELECT ONLY THE FIRST blaze FRAME. The rest is ignored.
if np.shape(wave) != np.shape(
e2ds[0]): #If UVES/ESPRESSO, this was already checked implicitly.
raise ValueError(
f"in read_e2ds: A wave file was detected but its shape ({np.shape(wave)[0]},{np.shape(wave)[1]}) does not match that of the orders ({np.shape(e2ds[0])[0]},{np.shape(e2ds[0])[1]})"
)
if np.shape(blaze) != np.shape(e2ds[0]) and blaze_count > 0:
raise ValueError(
f"in read_e2ds: A blaze file was detected but its shape ({np.shape(blaze)[0]},{np.shape(wave)[1]}) does not match that of the orders ({np.shape(e2ds[0])[0]},{np.shape(e2ds[0])[1]})"
)
if len(s1dhdr) != len(
e2ds
) and molecfit == True: #Only a problem if we actually run Molecfit.
raise ValueError(
f'in read_e2ds: The number of s1d SCIENCE files and e2ds SCIENCE files is not the same. ({len(s1dhdr)} vs {len(e2ds)})'
)
#Ok, so now we should have ended up with a number of lists that contain all
#the relevant science data and associated information.
#We determine how to sort the resulting lists in time:
sorting = np.argsort(mjd)
s1dsorting = np.argsort(s1dmjd)
if len(ignore_exp) > 0:
sorting = [x for i, x in enumerate(sorting) if i not in ignore_exp]
s1dsorting = [
x for i, x in enumerate(s1dsorting) if i not in ignore_exp
]
if mode == 'HARPSN': #In the case of HARPS-N we need to convert the units of the elevation and provide a UTC keyword.
for i in range(len(header)):
s1dhdr[i]['TELALT'] = np.degrees(float(s1dhdr[i]['EL']))
s1dhdr[i]['UTC'] = (float(s1dhdr[i]['MJD-OBS']) % 1.0) * 86400.0
#Sort the s1d files for application of molecfit.
if molecfit == True:
if len(sorting) != len(s1dsorting):
raise ValueError(
"in read_HARPS_e2ds: Sorted science frames and sorted s1d frames are not of the same length. Telluric correction can't proceed."
)
s1dhdr_sorted = []
s1d_sorted = []
for i in range(len(s1dsorting)):
s1dhdr_sorted.append(s1dhdr[s1dsorting[i]])
s1d_sorted.append(s1d[s1dsorting[i]])
print("")
print("")
print("")
print(
'Molecfit will be executed onto the files with dates in this order:'
)
for x in s1dhdr_sorted:
print(x['DATE-OBS'])
print("")
print("")
print("")
if mode in ['ESPRESSO', 'UVES-red', 'UVES-blue']:
list_of_wls, list_of_trans = mol.do_molecfit(s1dhdr_sorted,
s1d_sorted,
config,
load_previous=False,
mode=mode,
wave=wave1d)
else:
list_of_wls, list_of_trans = mol.do_molecfit(s1dhdr_sorted,
s1d_sorted,
config,
load_previous=False,
mode=mode)
if len(list_of_trans) != len(sorting):
raise ValueError(
"in read_e2ds(): Molecfit did not produce the same number of spectra as there are in the e2ds spectra."
)
mol.write_telluric_transmission_to_file(
list_of_wls, list_of_trans,
outpath / 'telluric_transmission_spectra.pkl')
#Now we loop over all exposures and collect the i-th order from each exposure,
#put these into a new matrix and save them to FITS images:
f = open(outpath / 'obs_times', 'w', newline='\n')
headerline = 'MJD' + '\t' + 'DATE' + '\t' + 'EXPTIME' + '\t' + 'MEAN AIRMASS' + '\t' + 'BERV (km/s)' + '\t' + 'FILE NAME'
for i in range(norders):
order = np.zeros((len(sorting), npx))
wave_axis = wave[i, :] / 10.0 #Convert to nm.
print('CONSTRUCTING ORDER %s' % i)
c = 0 #To count the number of science frames that have passed. The counter
# c is not equal to j because the list of files contains not only SCIENCE
# frames.
for j in range(len(sorting)): #Loop over exposures
if i == 0:
print('---' + type[sorting[j]] + ' ' + date[sorting[j]])
if type[sorting[j]] == 'SCIENCE': #This check may be redundant.
exposure = e2ds[sorting[j]]
order[c, :] = exposure[i, :]
#T_i = interp.interp1d(list_of_wls[j],list_of_trans[j])#This should be time-sorted, just as the e2ds files.
#Do a manual check here that the MJDs are identical.
#Also, determiine what to do with airtovac.
#tel_order[c,:] = T_i[wave_axis]
#Now I also need to write it to file.
if i == 0: #Only do it the first time, not for every order.
line = str(mjd[sorting[j]]) + '\t' + date[sorting[
j]] + '\t' + str(texp[sorting[j]]) + '\t' + str(
np.round(airmass[sorting[j]], 3)) + '\t' + str(
np.round(
berv[sorting[j]],
5)) + '\t' + framename[sorting[j]] + '\n'
f.write(line)
c += 1
fits.writeto(outpath / ('order_' + str(i) + '.fits'),
order,
overwrite=True)
fits.writeto(outpath / ('wave_' + str(i) + '.fits'),
wave_axis,
overwrite=True)
f.close()
print(f'Time-table written to {outpath/"obs_times"}.')
def run_instance(p):
"""This runs the entire cross correlation analysis cascade."""
import numpy as np
from astropy.io import fits
import astropy.constants as const
import astropy.units as u
from matplotlib import pyplot as plt
import os.path
import scipy.interpolate as interp
import pylab
import pdb
import os.path
import os
import sys
import glob
import distutils.util
import pickle
import copy
from pathlib import Path
import tayph.util as ut
import tayph.operations as ops
import tayph.functions as fun
import tayph.system_parameters as sp
import tayph.tellurics as telcor
import tayph.masking as masking
import tayph.models as models
from tayph.ccf import xcor, clean_ccf, filter_ccf, construct_KpVsys
from tayph.vartests import typetest, notnegativetest, nantest, postest, typetest_array, dimtest
import tayph.shadow as shadow
# from lib import analysis
# from lib import cleaning
# from lib import masking as masking
# from lib import shadow as shadow
# from lib import molecfit as telcor
#First parse the parameter dictionary into required variables and test them.
typetest(p, dict, 'params in run_instance()')
dp = Path(p['dp'])
ut.check_path(dp, exists=True)
modellist = p['modellist']
templatelist = p['templatelist']
model_library = p['model_library']
template_library = p['template_library']
typetest(modellist, [str, list], 'modellist in run_instance()')
typetest(templatelist, [str, list], 'templatelist in run_instance()')
typetest(model_library, str, 'model_library in run_instance()')
typetest(template_library, str, 'template_library in run_instance()')
ut.check_path(model_library, exists=True)
ut.check_path(template_library, exists=True)
if type(modellist) == str:
modellist = [modellist] #Force this to be a list
if type(templatelist) == str:
templatelist = [templatelist] #Force this to be a list
typetest_array(modellist, str, 'modellist in run_instance()')
typetest_array(templatelist, str, 'modellist in run_instance()')
shadowname = p['shadowname']
maskname = p['maskname']
typetest(shadowname, str, 'shadowname in run_instance()')
typetest(maskname, str, 'shadowname in run_instance()')
RVrange = p['RVrange']
drv = p['drv']
f_w = p['f_w']
resolution = sp.paramget('resolution', dp)
typetest(RVrange, [int, float], 'RVrange in run_instance()')
typetest(drv, [int, float], 'drv in run_instance()')
typetest(f_w, [int, float], 'f_w in run_instance()')
typetest(resolution, [int, float], 'resolution in run_instance()')
nantest(RVrange, 'RVrange in run_instance()')
nantest(drv, 'drv in run_instance()')
nantest(f_w, 'f_w in run_instance()')
nantest(resolution, 'resolution in run_instance()')
postest(RVrange, 'RVrange in run_instance()')
postest(drv, 'drv in run_instance()')
postest(resolution, 'resolution in run_instance()')
notnegativetest(f_w, 'f_w in run_instance()')
do_colour_correction = p['do_colour_correction']
do_telluric_correction = p['do_telluric_correction']
do_xcor = p['do_xcor']
plot_xcor = p['plot_xcor']
make_mask = p['make_mask']
apply_mask = p['apply_mask']
c_subtract = p['c_subtract']
do_berv_correction = p['do_berv_correction']
do_keplerian_correction = p['do_keplerian_correction']
make_doppler_model = p['make_doppler_model']
skip_doppler_model = p['skip_doppler_model']
typetest(do_colour_correction, bool,
'do_colour_correction in run_instance()')
typetest(do_telluric_correction, bool,
'do_telluric_correction in run_instance()')
typetest(do_xcor, bool, 'do_xcor in run_instance()')
typetest(plot_xcor, bool, 'plot_xcor in run_instance()')
typetest(make_mask, bool, 'make_mask in run_instance()')
typetest(apply_mask, bool, 'apply_mask in run_instance()')
typetest(c_subtract, bool, 'c_subtract in run_instance()')
typetest(do_berv_correction, bool, 'do_berv_correction in run_instance()')
typetest(do_keplerian_correction, bool,
'do_keplerian_correction in run_instance()')
typetest(make_doppler_model, bool, 'make_doppler_model in run_instance()')
typetest(skip_doppler_model, bool, 'skip_doppler_model in run_instance()')
#We start by defining constants and preparing for generating output.
c = const.c.value / 1000.0 #in km/s
colourdeg = 3 #A fitting degree for the colour correction.
print(
f'---Passed parameter input tests. Initiating output folder tree in {Path("output")/dp}.'
)
libraryname = str(template_library).split('/')[-1]
if str(dp).split('/')[0] == 'data':
dataname = str(dp).replace('data/', '')
print(
f'------Data is located in data/ folder. Assuming output name for this dataset as {dataname}'
)
else:
dataname = dp
print(
f'------Data is NOT located in data/ folder. Assuming output name for this dataset as {dataname}'
)
list_of_wls = [] #This will store all the data.
list_of_orders = [] #All of it needs to be loaded into your memory.
list_of_sigmas = []
trigger2 = 0 #These triggers are used to limit the generation of output in the forloop.
trigger3 = 0
n_negative_total = 0 #This will hold the total number of pixels that were set to NaN because they were zero when reading in the data.
air = sp.paramget('air', dp) #Read bool from str in config file.
typetest(air, bool, 'air in run_instance()')
filelist_orders = [str(i) for i in Path(dp).glob('order_*.fits')]
if len(filelist_orders) == 0:
raise Exception(
f'Runtime error: No orders_*.fits files were found in {dp}.')
try:
order_numbers = [
int(i.split('order_')[1].split('.')[0]) for i in filelist_orders
]
except:
raise Exception(
'Runtime error: Failed casting fits filename numerals to ints. Are the filenames of the spectral orders correctly formatted?'
)
order_numbers.sort() #This is the ordered list of numerical order IDs.
n_orders = len(order_numbers)
if n_orders == 0:
raise Exception(
f'Runtime error: n_orders may not have ended up as zero. ({n_orders})'
)
#Loading the data from the datafolder.
if do_xcor == True or plot_xcor == True or make_mask == True:
print(f'---Loading orders from {dp}.')
# for i in range(startorder,endorder+1):
for i in order_numbers:
wavepath = dp / f'wave_{i}.fits'
orderpath = dp / f'order_{i}.fits'
sigmapath = dp / f'sigma_{i}.fits'
ut.check_path(wavepath, exists=True)
ut.check_path(orderpath, exists=True)
ut.check_path(sigmapath, exists=False)
wave_axis = fits.getdata(wavepath)
dimtest(wave_axis, [0], 'wavelength grid in run_instance()')
n_px = len(wave_axis) #Pixel width of the spectral order.
if air == False:
if i == np.min(order_numbers):
print("------Assuming wavelengths are in vaccuum.")
list_of_wls.append(1.0 * wave_axis)
else:
if i == np.min(order_numbers):
print("------Applying airtovac correction.")
list_of_wls.append(ops.airtovac(wave_axis))
order_i = fits.getdata(orderpath)
if i == np.min(order_numbers):
dimtest(
order_i, [0, n_px], f'order {i} in run_instance()'
) #For the first order, check that it is 2D and that is has a width equal to n_px.
n_exp = np.shape(
order_i
)[0] #then fix n_exp. All other orders should have the same n_exp.
print(f'------{n_exp} exposures recognised.')
else:
dimtest(order_i, [n_exp, n_px], f'order {i} in run_instance()')
#Now test for negatives, set them to NaN and track them.
n_negative = len(order_i[order_i <= 0])
if trigger3 == 0 and n_negative > 0:
print("------Setting negative values to NaN.")
trigger3 = -1
n_negative_total += n_negative
order_i[order_i <= 0] = np.nan
postest(order_i, f'order {i} in run_instance().'
) #make sure whatever comes out here is strictly positive.
list_of_orders.append(order_i)
try: #Try to get a sigma file. If it doesn't exist, we raise a warning. If it does, we test its dimensions and append it.
sigma_i = fits.getdata(sigmapath)
dimtest(sigma_i, [n_exp, n_px],
f'order {i} in run_instance().')
list_of_sigmas.append(sigma_i)
except FileNotFoundError:
if trigger2 == 0:
print(
'------WARNING: Sigma (flux error) files not provided. Assuming sigma = sqrt(flux). This is standard practise for HARPS data, but e.g. ESPRESSO has a pipeline that computes standard errors on each pixel for you.'
)
trigger2 = -1
list_of_sigmas.append(np.sqrt(order_i))
print(
f"------{n_negative_total} negative values set to NaN ({np.round(100.0*n_negative_total/n_exp/n_px/len(order_numbers),2)}% of total spectral pixels in dataset.)"
)
if len(list_of_orders) != n_orders:
raise Exception(
'Runtime error: n_orders is not equal to the length of list_of_orders. Something went wrong when reading them in?'
)
print('---Finished loading dataset to memory.')
#Apply telluric correction file or not.
# plt.plot(list_of_wls[60],list_of_orders[60][10],color='red')
# plt.plot(list_of_wls[60],list_of_orders[60][10]+list_of_sigmas[60][10],color='red',alpha=0.5)#plot corrected spectra
# plt.plot(list_of_wls[60],list_of_orders[60][10]/list_of_sigmas[60][10],color='red',alpha=0.5)#plot SNR
if do_telluric_correction == True and n_orders > 0:
print('---Applying telluric correction')
telpath = dp / 'telluric_transmission_spectra.pkl'
list_of_orders, list_of_sigmas = telcor.apply_telluric_correction(
telpath, list_of_wls, list_of_orders, list_of_sigmas)
# plt.plot(list_of_wls[60],list_of_orders[60][10],color='blue')
# plt.plot(list_of_wls[60],list_of_orders[60][10]+list_of_sigmas[60][10],color='blue',alpha=0.5)#plot corrected spectra
# plt.plot(list_of_wls[60],list_of_orders[60][10]/list_of_sigmas[60][10],color='blue',alpha=0.5) #plot SNR
# plt.show()
# pdb.set_trace()
#Do velocity correction of wl-solution. Explicitly after telluric correction
#but before masking. Because the cross-correlation relies on columns being masked.
#Then if you start to move the CCFs around before removing the time-average,
#each masked column becomes slanted. Bad deal.
rv_cor = 0
if do_berv_correction == True:
rv_cor += sp.berv(dp)
if do_keplerian_correction == True:
rv_cor -= sp.RV_star(dp) * (1.0)
if type(rv_cor) != int and len(list_of_orders) > 0:
print('---Reinterpolating data to correct velocities')
list_of_orders_cor = []
list_of_sigmas_cor = []
for i in range(len(list_of_wls)):
order = list_of_orders[i]
sigma = list_of_sigmas[i]
order_cor = order * 0.0
sigma_cor = sigma * 0.0
for j in range(len(list_of_orders[0])):
wl_i = interp.interp1d(list_of_wls[i],
order[j],
bounds_error=False)
si_i = interp.interp1d(list_of_wls[i],
sigma[j],
bounds_error=False)
wl_cor = list_of_wls[i] * (
1.0 - (rv_cor[j] * u.km / u.s / const.c)
) #The minus sign was tested on a slow-rotator.
order_cor[j] = wl_i(wl_cor)
sigma_cor[j] = si_i(
wl_cor
) #I checked that this works because it doesn't affect the SNR, apart from wavelength-shifting it.
list_of_orders_cor.append(order_cor)
list_of_sigmas_cor.append(sigma_cor)
ut.statusbar(i, fun.findgen(len(list_of_wls)))
# plt.plot(list_of_wls[60],list_of_orders[60][10]/list_of_sigmas[60][10],color='blue')
# plt.plot(list_of_wls[60],list_of_orders_cor[60][10]/list_of_sigmas_cor[60][10],color='red')
# plt.show()
# sys.exit()
list_of_orders = list_of_orders_cor
list_of_sigmas = list_of_sigmas_cor
if len(list_of_orders) != n_orders:
raise RuntimeError(
'n_orders is no longer equal to the length of list_of_orders, though it was before. Something went wrong during telluric correction or velocity correction.'
)
#Compute / create a mask and save it to file (or not)
if make_mask == True and len(list_of_orders) > 0:
if do_colour_correction == True:
print(
'---Constructing mask with intra-order colour correction applied'
)
masking.mask_orders(list_of_wls,
ops.normalize_orders(list_of_orders,
list_of_sigmas,
colourdeg)[0],
dp,
maskname,
40.0,
5.0,
manual=True)
else:
print(
'---Constructing mask WITHOUT intra-order colour correction applied.'
)
print(
'---Switch on colour correction if you see colour variations in the 2D spectra.'
)
masking.mask_orders(list_of_wls,
list_of_orders,
dp,
maskname,
40.0,
5.0,
manual=True)
if apply_mask == False:
print(
'---WARNING in run_instance: Mask was made but is not applied to data (apply_mask == False)'
)
#Apply the mask that was previously created and saved to file.
if apply_mask == True:
print('---Applying mask')
list_of_orders = masking.apply_mask_from_file(dp, maskname,
list_of_orders)
list_of_sigmas = masking.apply_mask_from_file(dp, maskname,
list_of_sigmas)
#Interpolate over all isolated NaNs and set bad columns to NaN (so that they are ignored in the CCF)
if do_xcor == True:
print('---Healing NaNs')
list_of_orders = masking.interpolate_over_NaNs(
list_of_orders
) #THERE IS AN ISSUE HERE: INTERPOLATION SHOULD ALSO HAPPEN ON THE SIGMAS ARRAY!
list_of_sigmas = masking.interpolate_over_NaNs(list_of_sigmas)
#Normalize the orders to their average flux in order to effectively apply a broad-band colour correction (colour is typically a function of airmass and seeing).
if do_colour_correction == True:
print('---Normalizing orders to common flux level')
# plt.plot(list_of_wls[60],list_of_orders[60][10]/list_of_sigmas[60][10],color='blue',alpha=0.4)
list_of_orders_normalised, list_of_sigmas_normalised, meanfluxes = ops.normalize_orders(
list_of_orders, list_of_sigmas, colourdeg
) #I tested that this works because it doesn't alter the SNR.
meanfluxes_norm = meanfluxes / np.nanmean(meanfluxes)
else:
meanfluxes_norm = fun.findgen(len(
list_of_orders[0])) * 0.0 + 1.0 #All unity.
# plt.plot(list_of_wls[60],list_of_orders_normalised[60][10]/list_of_sigmas[60][10],color='red',alpha=0.4)
# plt.show()
# sys.exit()
if len(list_of_orders) != n_orders:
raise RuntimeError(
'n_orders is no longer equal to the length of list_of_orders, though it was before. Something went wrong during masking or colour correction.'
)
#Construct the cross-correlation templates in case we will be computing or plotting the CCF.
if do_xcor == True or plot_xcor == True:
list_of_wlts = []
list_of_templates = []
outpaths = []
for templatename in templatelist:
print(f'---Building template {templatename}')
wlt, T = models.build_template(templatename,
binsize=0.5,
maxfrac=0.01,
resolution=resolution,
template_library=template_library,
c_subtract=c_subtract)
T *= (-1.0)
if np.mean(wlt) < 50.0: #This is likely in microns:
print(
'------WARNING: The loaded template has a mean wavelength less than 50.0, meaning that it is very likely not in nm, but in microns. I have divided by 1,000 now and hope for the best...'
)
wlt *= 1000.0
list_of_wlts.append(wlt)
list_of_templates.append(T)
outpath = Path('output') / Path(dataname) / Path(
libraryname) / Path(templatename)
if not os.path.exists(outpath):
print(
f"------The output location ({outpath}) didn't exist, I made it now."
)
os.makedirs(outpath)
outpaths.append(outpath)
#Perform the cross-correlation on the entire list of orders.
for i in range(len(list_of_wlts)):
templatename = templatelist[i]
wlt = list_of_wlts[i]
T = list_of_templates[i]
outpath = outpaths[i]
if do_xcor == True:
print(
f'---Cross-correlating spectra with template {templatename}.')
t1 = ut.start()
rv, ccf, ccf_e, Tsums = xcor(
list_of_wls,
list_of_orders_normalised,
np.flipud(np.flipud(wlt)),
T,
drv,
RVrange,
list_of_errors=list_of_sigmas_normalised)
ut.end(t1)
print(f'------Writing CCFs to {str(outpath)}')
ut.writefits(outpath / 'ccf.fits', ccf)
ut.writefits(outpath / 'ccf_e.fits', ccf_e)
ut.writefits(outpath / 'RV.fits', rv)
ut.writefits(outpath / 'Tsum.fits', Tsums)
else:
print(
f'---Reading CCFs with template {templatename} from {str(outpath)}.'
)
if os.path.isfile(outpath / 'ccf.fits') == False:
raise FileNotFoundError(
f'CCF output not located at {outpath}. Rerun with do_xcor=True to create these files?'
)
rv = fits.getdata(outpath / 'rv.fits')
ccf = fits.getdata(outpath / 'ccf.fits')
ccf_e = fits.getdata(outpath / 'ccf_e.fits')
Tsums = fits.getdata(outpath / 'Tsum.fits')
ccf_cor = ccf * 1.0
ccf_e_cor = ccf_e * 1.0
print('---Cleaning CCFs')
ccf_n, ccf_ne, ccf_nn, ccf_nne = clean_ccf(rv, ccf_cor, ccf_e_cor, dp)
if make_doppler_model == True and skip_doppler_model == False:
shadow.construct_doppler_model(rv,
ccf_nn,
dp,
shadowname,
xrange=[-200, 200],
Nxticks=20.0,
Nyticks=10.0)
make_doppler_model = False # This sets it to False after it's been run once, for the first template.
if skip_doppler_model == False:
print('---Reading doppler shadow model from ' + shadowname)
doppler_model, dsmask = shadow.read_shadow(
dp, shadowname, rv, ccf
) #This returns both the model evaluated on the rv,ccf grid, as well as the mask that blocks the planet trace.
ccf_clean, matched_ds_model = shadow.match_shadow(
rv, ccf_nn, dsmask, dp, doppler_model
) #THIS IS AN ADDITIVE CORRECTION, SO CCF_NNE DOES NOT NEED TO BE ALTERED AND IS STILL VALID VOOR CCF_CLEAN
else:
print('---Not performing shadow correction')
ccf_clean = ccf_nn * 1.0
matched_ds_model = ccf_clean * 0.0
if f_w > 0.0:
print('---Performing high-pass filter on the CCF')
ccf_clean_filtered, wiggles = filter_ccf(
rv, ccf_clean, v_width=f_w
) #THIS IS ALSO AN ADDITIVE CORRECTION, SO CCF_NNE IS STILL VALID.
else:
print('---Skipping high-pass filter')
ccf_clean_filtered = ccf_clean * 1.0
wiggles = ccf_clean * 0.0 #This filtering is additive so setting to zero is accurate.
print('---Weighing CCF rows by mean fluxes that were normalised out')
ccf_clean_weighted = np.transpose(
np.transpose(ccf_clean_filtered) * meanfluxes_norm
) #MULTIPLYING THE AVERAGE FLUXES BACK IN! NEED TO CHECK THAT THIS ALSO GOES PROPERLY WITH THE ERRORS!
ccf_nne = np.transpose(np.transpose(ccf_nne) * meanfluxes_norm)
ut.save_stack(outpath / 'cleaning_steps.fits', [
ccf, ccf_cor, ccf_nn, ccf_clean, matched_ds_model,
ccf_clean_filtered, wiggles, ccf_clean_weighted
])
ut.writefits(outpath / 'ccf_cleaned.fits', ccf_clean_weighted)
ut.writefits(outpath / 'ccf_cleaned_error.fits', ccf_nne)
print('---Constructing KpVsys')
Kp, KpVsys, KpVsys_e = construct_KpVsys(rv, ccf_clean_weighted,
ccf_nne, dp)
ut.writefits(outpath / 'KpVsys.fits', KpVsys)
ut.writefits(outpath / 'KpVsys_e.fits', KpVsys_e)
ut.writefits(outpath / 'Kp.fits', Kp)
return
sys.exit()
if plot_xcor == True and inject_model == False:
print('---Plotting KpVsys')
analysis.plot_KpVsys(rv, Kp, KpVsys, dp)
#Now repeat it all for the model injection.
if inject_model == True:
for modelname in modellist:
outpath_i = outpath + modelname + '/'
if do_xcor == True:
print('---Injecting model ' + modelname)
list_of_orders_injected = models.inject_model(
list_of_wls,
list_of_orders,
dp,
modelname,
model_library=model_library
) #Start with the unnormalised orders from before.
#Normalize the orders to their average flux in order to effectively apply
#a broad-band colour correction (colour is a function of airmass and seeing).
if do_colour_correction == True:
print(
'------Normalizing injected orders to common flux level'
)
list_of_orders_injected, list_of_sigmas_injected, meanfluxes_injected = ops.normalize_orders(
list_of_orders_injected, list_of_sigmas, colourdeg)
meanfluxes_norm_injected = meanfluxes_injected / np.mean(
meanfluxes_injected)
else:
meanfluxes_norm_injected = fun.findgen(
len(list_of_orders_injected[0])
) * 0.0 + 1.0 #All unity.
print('------Cross-correlating injected orders')
rv_i, ccf_i, ccf_e_i, Tsums_i = analysis.xcor(
list_of_wls,
list_of_orders_injected,
np.flipud(np.flipud(wlt)),
T,
drv,
RVrange,
list_of_errors=list_of_sigmas_injected)
print('------Writing injected CCFs to ' + outpath_i)
if not os.path.exists(outpath_i):
print("---------That path didn't exist, I made it now.")
os.makedirs(outpath_i)
ut.writefits(outpath_i + '/' + 'ccf_i_' + modelname + '.fits',
ccf_i)
ut.writefits(
outpath_i + '/' + 'ccf_e_i_' + modelname + '.fits',
ccf_e_i)
else:
print('---Reading injected CCFs from ' + outpath_i)
if os.path.isfile(outpath_i + 'ccf_i_' + modelname +
'.fits') == False:
print('------ERROR: Injected CCF not located at ' +
outpath_i + 'ccf_i_' + modelname + '.fits' +
'. Set do_xcor and inject_model to True?')
sys.exit()
if os.path.isfile(outpath_i + 'ccf_e_i_' + modelname +
'.fits') == False:
print('------ERROR: Injected CCF error not located at ' +
outpath_i + 'ccf_e_i_' + modelname + '.fits' +
'. Set do_xcor and inject_model to True?')
sys.exit()
# f.close()
# f2.close()
ccf_i = fits.getdata(outpath_i + 'ccf_i_' + modelname +
'.fits')
ccf_e_i = fits.getdata(outpath_i + 'ccf_e_i_' + modelname +
'.fits')
print('---Cleaning injected CCFs')
ccf_n_i, ccf_ne_i, ccf_nn_i, ccf_nne_i = cleaning.clean_ccf(
rv, ccf_i, ccf_e_i, dp)
ut.writefits(outpath_i + 'ccf_normalized_i.fits', ccf_nn_i)
ut.writefits(outpath_i + 'ccf_ne_i.fits', ccf_ne_i)
# if make_doppler_model == True and skip_doppler_model == False:
# shadow.construct_doppler_model(rv,ccf_nn,dp,shadowname,xrange=[-200,200],Nxticks=20.0,Nyticks=10.0)
if skip_doppler_model == False:
# print('---Reading doppler shadow model from '+shadowname)
# doppler_model,maskHW = shadow.read_shadow(dp,shadowname,rv,ccf)
ccf_clean_i, matched_ds_model_i = shadow.match_shadow(
rv, ccf_nn_i, dp, doppler_model, maskHW)
else:
print(
'---Not performing shadow correction on injected spectra either.'
)
ccf_clean_i = ccf_nn_i * 1.0
matched_ds_model_i = ccf_clean_i * 0.0
if f_w > 0.0:
ccf_clean_i_filtered, wiggles_i = cleaning.filter_ccf(
rv, ccf_clean_i, v_width=f_w)
else:
ccf_clean_i_filtered = ccf_clean_i * 1.0
ut.writefits(outpath_i + 'ccf_cleaned_i.fits',
ccf_clean_i_filtered)
ut.writefits(outpath + 'ccf_cleaned_i_error.fits', ccf_nne)
print(
'---Weighing injected CCF rows by mean fluxes that were normalised out'
)
ccf_clean_i_filtered = np.transpose(
np.transpose(ccf_clean_i_filtered) * meanfluxes_norm_injected
) #MULTIPLYING THE AVERAGE FLUXES BACK IN! NEED TO CHECK THAT THIS ALSO GOES PROPERLY WITH THE ERRORS!
ccf_nne_i = np.transpose(
np.transpose(ccf_nne_i) * meanfluxes_norm_injected)
print('---Constructing injected KpVsys')
Kp, KpVsys_i, KpVsys_e_i = analysis.construct_KpVsys(
rv, ccf_clean_i_filtered, ccf_nne_i, dp)
ut.writefits(outpath_i + 'KpVsys_i.fits', KpVsys_i)
# ut.writefits(outpath+'KpVsys_e_i.fits',KpVsys_e_i)
if plot_xcor == True:
print('---Plotting KpVsys with ' + modelname + ' injected.')
analysis.plot_KpVsys(rv, Kp, KpVsys, dp, injected=KpVsys_i)
def check_fit_gui(wls, fxc, trans):
"""This code initializes the GUI that plots the telluric-corrected spectra
from Molecfit. The user may select spectra to be re-fit manually via the Molecfit GUI. Note that
since molecfit takes between a few and 10 minutes to run on a single spectrum,
this becomes arduous when more than a few spectra are selected in this way.
It quickly becomes worthwile to redo the entire sequence with different inclusion
regions overnight. The code returns the list of spectra that need to be done
manually via the Molecfit GUI.
Input: The list of wl axes, each of which was returned by a call to molecfit;
and similarly the corrected spectra fxc and the transmission spectra.
"""
import sys
import matplotlib.pyplot as plt
from matplotlib.widgets import Slider, Button, RadioButtons, CheckButtons
import tayph.functions as fun
import numpy as np
print('Starting visual inspection GUI')
M = molecfit_gui(wls, fxc, trans)
#The slider to cycle through orders:
rax_slider = plt.axes([0.8, 0.2, 0.1, 0.02])
rax_slider.set_title('Exposure #')
M.spectrum_slider = Slider(rax_slider,
'',
0,
M.N - 1,
valinit=0,
valstep=1) #Store the slider in the model class
M.spectrum_slider.on_changed(M.slide_spectrum)
#The Previous order button:
rax_prev = plt.axes([0.8, 0.1, 0.04, 0.05])
bprev = Button(rax_prev, ' <<< ')
bprev.on_clicked(M.previous)
#The Next order button:
rax_next = plt.axes([0.86, 0.1, 0.04, 0.05])
bnext = Button(rax_next, ' >>> ')
bnext.on_clicked(M.next)
#The save button:
rax_save = plt.axes([0.92, 0.1, 0.07, 0.05])
bsave = Button(rax_save, 'Continue')
bsave.on_clicked(M.save)
#The cancel button:
rax_cancel = plt.axes([0.92, 0.025, 0.07, 0.05])
bcancel = Button(rax_cancel, 'Cancel')
bcancel.on_clicked(M.cancel)
#This is to rescale the x-size of the checkboxes so that they are squares.
bbox = M.fig.get_window_extent().transformed(
M.fig.dpi_scale_trans.inverted())
width, height = bbox.width * M.fig.dpi, bbox.height * M.fig.dpi
M.selec = plt.axes([0.05, 0.03, 0.7, 0.05 * M.nrows])
M.selec.spines['bottom'].set_color('white')
M.selec.spines['top'].set_color('white')
M.selec.spines['left'].set_color('white')
M.selec.spines['right'].set_color('white')
vlines = fun.findgen(M.N - 1) + 0.5
row = M.nrows
offset = 0
for i in range(M.N):
#print(i,float(i)-offset)
if float(i) - offset > M.maxboxes - 1.0:
row -= 1
offset += M.maxboxes
M.selec.plot(float(i) - offset +
np.array([-0.5, -0.5, 0.5, 0.5, -0.5]),
[row, row - 1, row - 1, row, row],
color='black')
M.selec.text(float(i) - offset,
row - 0.5,
'%s' % i,
color='black',
horizontalalignment='center',
verticalalignment='center')
M.selec.set_xlim(
-0.55, M.maxboxes - 1.0 + 0.55
) #A little margin to make sure that the line thickness is included.
M.selec.set_ylim(-0.05, 1.0 * M.nrows + 0.05)
M.selec.xaxis.set_tick_params(labelsize=8)
M.selec.yaxis.set_tick_params(labelsize=8)
def select_spectrum_box(event):
#This handles with a mouseclick in either of the three plots while in add mode.
if event.inaxes in [M.selec
]: #Check that it occurs in one of the subplots.
cc = event.xdata * 1.0 #xdata is the column that is selected.
cr = event.ydata * 1.0
spectrum = np.round(cc) + np.round(
(M.nrows - cr - 0.5)) * M.maxboxes
if spectrum < M.N:
if spectrum in M.selected:
M.selected.remove(spectrum)
print('---Removed spectrum %s from manual' % spectrum)
else:
M.selected.append(spectrum)
print('---Added spectrum %s to manual' % spectrum)
M.draw_crosses()
M.click_connector = M.fig.canvas.mpl_connect(
'button_press_event',
select_spectrum_box) #This is the connector that registers clicks
plt.show()
print('Closed GUI, returning.')
return (M.selected)
def apply_telluric_correction(inpath, list_of_wls, list_of_orders,
list_of_sigmas):
"""
This applies a set of telluric spectra (computed by molecfit) for each exposure
in our time series that were written to a pickle file by write_telluric_transmission_to_file.
List of errors are provided to propagate telluric correction into the error array as well.
Parameters
----------
inpath : str, path like
The path to the pickled transmission spectra.
list_of_wls : list
List of wavelength axes.
list_of_orders :
List of 2D spectral orders, matching to the wavelength axes in dimensions and in number.
list_of_isgmas :
List of 2D error matrices, matching dimensions and number of list_of_orders.
Returns
-------
list_of_orders_corrected : list
List of 2D spectral orders, telluric corrected.
list_of_sigmas_corrected : list
List of 2D error matrices, telluric corrected.
"""
import scipy.interpolate as interp
import numpy as np
import tayph.util as ut
import tayph.functions as fun
from tayph.vartests import dimtest, postest, typetest, nantest
wlT, fxT = read_telluric_transmission_from_file(inpath)
typetest(list_of_wls, list, 'list_of_wls in apply_telluric_correction()')
typetest(list_of_orders, list,
'list_of_orders in apply_telluric_correction()')
typetest(list_of_sigmas, list,
'list_of_sigmas in apply_telluric_correction()')
typetest(wlT, list,
'list of telluric wave-axes in apply_telluric_correction()')
typetest(
fxT, list,
'list of telluric transmission spectra in apply_telluric_correction()')
No = len(list_of_wls)
x = fun.findgen(No)
if No != len(list_of_orders):
raise Exception(
'Runtime error in telluric correction: List of data wls and List of orders do not have the same length.'
)
Nexp = len(wlT)
if Nexp != len(fxT):
raise Exception(
'Runtime error in telluric correction: List of telluric wls and telluric spectra read from file do not have the same length.'
)
if Nexp != len(list_of_orders[0]):
raise Exception(
f'Runtime error in telluric correction: List of telluric spectra and data spectra read from file do not have the same length ({Nexp} vs {len(list_of_orders[0])}).'
)
list_of_orders_cor = []
list_of_sigmas_cor = []
# ut.save_stack('test.fits',list_of_orders)
# pdb.set_trace()
for i in range(No): #Do the correction order by order.
order = list_of_orders[i]
order_cor = order * 0.0
error = list_of_sigmas[i]
error_cor = error * 0.0
wl = list_of_wls[i]
dimtest(order, [0, len(wl)],
f'order {i}/{No} in apply_telluric_correction()')
dimtest(error, np.shape(order),
f'errors {i}/{No} in apply_telluric_correction()')
for j in range(Nexp):
T_i = interp.interp1d(wlT[j], fxT[j], fill_value="extrapolate")(wl)
postest(T_i,
f'T-spec of exposure {j} in apply_telluric_correction()')
nantest(T_i,
f'T-spec of exposure {j} in apply_telluric_correction()')
order_cor[j] = order[j] / T_i
error_cor[j] = error[
j] / T_i #I checked that this works because the SNR before and after telluric correction is identical.
list_of_orders_cor.append(order_cor)
list_of_sigmas_cor.append(error_cor)
ut.statusbar(i, x)
return (list_of_orders_cor, list_of_sigmas_cor)
def read_uves(inpath, filelist, mode):
"""This reads a folder of UVES-blue or UVES-red intermediate pipeline products. Input is a list of filepaths and the mode."""
#The following variables define lists in which all the necessary data will be stored.
framename = []
header = []
s1dhdr = []
obstype = []
texp = np.array([])
date = []
mjd = np.array([])
s1dmjd = np.array([])
npx = np.array([])
npx1d = np.array([])
norders = np.array([])
e2ds = []
s1d = []
wave1d = []
airmass = np.array([])
berv = np.array([])
wave = []
catkeyword = 'HIERARCH ESO DPR CATG'
bervkeyword = 'HIERARCH ESO DRS BERV'
thfilekeyword = 'HIERARCH ESO DRS CAL TH FILE'
Zstartkeyword = 'HIERARCH ESO TEL AIRM START'
Zendkeyword = 'HIERARCH ESO TEL AIRM END'
for i in range(len(filelist)):
print(f'------{filelist[i]}', end="\r")
if (inpath / Path(filelist[i])).is_dir():
tmp_products = [
i for i in (inpath /
Path(filelist[i])).glob('resampled_science_*.fits')
]
tmp_products1d = [
i for i in (inpath /
Path(filelist[i])).glob('red_science_*.fits')
]
if mode == 'UVES-red' and len(tmp_products) != 2:
raise ValueError(
f"in read_e2ds: When mode=UVES-red there should be 2 resampled_science files (redl and redu), but {len(tmp_products)} were detected in {str(inpath/Path(filelist[i]))}."
)
if mode == 'UVES-blue' and len(tmp_products) != 1:
raise ValueError(
f"in read_e2ds: When mode=UVES-rblue there should be 1 resampled_science files (blue), but {len(tmp_products)} were detected in {str(inpath/Path(filelist[i]))}."
)
if mode == 'UVES-red' and len(tmp_products1d) != 2:
raise ValueError(
f"in read_e2ds: When mode=UVES-red there should be 2 red_science files (redl and redu), but {len(tmp_products1d)} were detected in {str(inpath/Path(filelist[i]))}."
)
if mode == 'UVES-blue' and len(tmp_products1d) != 1:
raise ValueError(
f"in read_e2ds: When mode=UVES-rblue there should be 1 red_science files (blue), but {len(tmp_products1d)} were detected in {str(inpath/Path(filelist[i]))}."
)
data_combined = [
] #This will store the two chips (redu and redl) in case of UVES_red, or simply the blue chip if otherwise.
wave_combined = []
wave1d_combined = []
data1d_combined = []
norders_tmp = 0
for tmp_product in tmp_products:
hdul = fits.open(tmp_product)
data = copy.deepcopy(hdul[0].data)
hdr = hdul[0].header
hdul.close()
del hdul[0].data
if not hdr[
'HIERARCH ESO PRO SCIENCE']: #Only add if it's actually a science product:#I force the user to supply only science exposures in the input folder. No BS allowed... UVES is hard enough as it is.
raise ValueError(
f' in read_e2ds: UVES file {tmp_product} is not classified as a SCIENCE file, but should be. Remove it from the folder?'
)
wavedata = ut.read_wave_from_e2ds_header(
hdr, mode='UVES') / 10.0 #Convert to nm.
data_combined.append(data)
wave_combined.append(wavedata)
norders_tmp += np.shape(data)[0]
for tmp_product in tmp_products1d:
hdul = fits.open(tmp_product)
data_1d = copy.deepcopy(hdul[0].data)
hdr1d = hdul[0].header
hdul.close()
del hdul[0].data
if not hdr1d[
'HIERARCH ESO PRO SCIENCE']: #Only add if it's actually a science product:#I force the user to supply only science exposures in the input folder. No BS allowed... UVES is hard enough as it is.
raise ValueError(
f' in read_e2ds: UVES file {tmp_product} is not classified as a SCIENCE file, but should be. Remove it from the folder?'
)
npx_1d = hdr1d['NAXIS1']
wavedata = fun.findgen(
npx_1d) * hdr1d['CDELT1'] + hdr1d['CRVAL1']
data1d_combined.append(data_1d)
wave1d_combined.append(wavedata)
if len(data_combined) < 1 or len(
data_combined) > 2: #Double-checking that length here...
raise ValueError(
f'in read_e2ds(): Expected 1 or 2 chips, but {len(data_combined)} files were somehow read.'
)
#The chips generally don't give the same size. Therefore I will pad the smaller one with NaNs to make it fit:
if len(data_combined) != len(data1d_combined):
raise ValueError(
f'in read_e2ds(): The number of chips in the 1d and 2d spectra is not the same {len(data1d_combined)} vs {len(data_combined)}.'
)
if len(data_combined) == 2:
chip1 = data_combined[0]
chip2 = data_combined[1]
wave1 = wave_combined[0]
wave2 = wave_combined[1]
npx_1 = np.shape(chip1)[1]
npx_2 = np.shape(chip2)[1]
no_1 = np.shape(chip1)[0]
no_2 = np.shape(chip2)[0]
npx_max = np.max([npx_1, npx_2])
npx_min = np.min([npx_1, npx_2])
diff = npx_max - npx_min
#Pad the smaller one with NaNs to match the wider one:
if npx_1 < npx_2:
chip1 = np.hstack([chip1, np.zeros((no_1, diff)) * np.nan])
wave1 = np.hstack([wave1, np.zeros((no_1, diff)) * np.nan])
else:
chip2 = np.hstack([chip2, np.zeros((no_2, diff)) * np.nan])
wave2 = np.hstack([wave2, np.zeros((no_2, diff)) * np.nan])
#So now they can be stacked:
e2ds_stacked = np.vstack((chip1, chip2))
wave_stacked = np.vstack((wave1, wave2))
if np.shape(e2ds_stacked)[1] != np.shape(wave_stacked)[1]:
raise ValueError(
"Width of stacked e2ds and stacked wave frame are not the same. Is the wavelength solution in the header of this file correct?"
)
npx = np.append(npx, np.shape(e2ds_stacked)[1])
e2ds.append(e2ds_stacked)
wave.append(wave_stacked)
chip1_1d = data1d_combined[0]
chip2_1d = data1d_combined[1]
wave1_1d = wave1d_combined[0]
wave2_1d = wave1d_combined[1]
if np.nanmean(wave1_1d) < np.nanmean(wave2_1d):
combined_data_1d = np.concatenate((chip1_1d, chip2_1d))
combined_wave_1d = np.concatenate((wave1_1d, wave2_1d))
else:
combined_data_1d = np.concatenate((chip2_1d, chip1_1d))
combined_wave_1d = np.concatenate((wave2_1d, wave1_1d))
wave1d.append(combined_wave_1d)
s1d.append(combined_data_1d)
npx1d = np.append(npx1d, len(combined_wave_1d))
else:
e2ds.append(data_combined[0])
wave.append(wave_combined[0])
npx = np.append(npx, np.shape(data_combined[0])[1])
wave1d.append(wave1d_combined[0])
s1d.append(data1d_combined[0])
npx1d = np.append(npx1d, len(combined_wave_1d))
#Only using the keyword from the second header in case of redl,redu.
s1dmjd = np.append(s1dmjd, hdr1d['MJD-OBS'])
framename.append(hdr['ARCFILE'])
header.append(hdr)
obstype.append('SCIENCE')
texp = np.append(texp, hdr['EXPTIME'])
date.append(hdr['DATE-OBS'])
mjd = np.append(mjd, hdr['MJD-OBS'])
norders = np.append(norders, norders_tmp)
airmass = np.append(
airmass, 0.5 * (hdr[Zstartkeyword] + hdr[Zendkeyword])
) #This is an approximation where we take the mean airmass.
berv_i = sp.calculateberv(hdr['MJD-OBS'],
hdr['HIERARCH ESO TEL GEOLAT'],
hdr['HIERARCH ESO TEL GEOLON'],
hdr['HIERARCH ESO TEL GEOELEV'],
hdr['RA'], hdr['DEC'])
berv = np.append(berv, berv_i)
hdr1d[
'HIERARCH ESO QC BERV'] = berv_i #Append the berv here using the ESPRESSO berv keyword, so that it can be used in molecfit later.
s1dhdr.append(hdr1d)
#Check that all exposures have the same number of pixels, and clip orders if needed.
min_npx = int(np.min(np.array(npx)))
min_npx1d = int(np.min(np.array(npx1d)))
if np.sum(np.abs(np.array(npx) - npx[0])) != 0:
warnings.warn(
"in read_e2ds when reading UVES data: Not all e2ds files have the same number of pixels. This could have happened if the pipeline has extracted one or two extra pixels in some exposures but not others. The e2ds files will be clipped to the smallest width.",
RuntimeWarning)
for i in range(len(e2ds)):
wave[i] = wave[i][:, 0:min_npx]
e2ds[i] = e2ds[i][:, 0:min_npx]
npx[i] = min_npx
if np.sum(np.abs(np.array(npx1d) - npx1d[0])) != 0:
warnings.warn(
"in read_e2ds when reading UVES data: Not all s1d files have the same number of pixels. This could have happened if the pipeline has extracted one or two extra pixels in some exposures but not others. The s1d files will be clipped to the smallest width.",
RuntimeWarning)
for i in range(len(s1d)):
wave1d[i] = wave1d[i][0:min_npx1d]
s1d[i] = s1d[i][0:min_npx1d]
npx1d[i] = min_npx1d
output = {
'wave': wave,
'e2ds': e2ds,
'header': header,
'wave1d': wave1d,
's1d': s1d,
's1dhdr': s1dhdr,
'mjd': mjd,
'date': date,
'texp': texp,
'obstype': obstype,
'framename': framename,
'npx': npx,
'npx1d': npx1d,
'norders': norders,
'berv': berv,
'airmass': airmass,
's1dmjd': s1dmjd
}
return (output)
def read_harpslike(inpath, filelist, mode, read_s1d=True):
"""
This reads a folder of HARPS or HARPSN data. Input is a list of filepaths and the mode (HARPS
or HARPSN).
"""
if mode == 'HARPS':
catkeyword = 'HIERARCH ESO DPR CATG'
bervkeyword = 'HIERARCH ESO DRS BERV'
thfilekeyword = 'HIERARCH ESO DRS CAL TH FILE'
Zstartkeyword = 'HIERARCH ESO TEL AIRM START'
Zendkeyword = 'HIERARCH ESO TEL AIRM END'
elif mode == 'HARPSN':
catkeyword = 'OBS-TYPE'
bervkeyword = 'HIERARCH TNG DRS BERV'
thfilekeyword = 'HIERARCH TNG DRS CAL TH FILE'
Zstartkeyword = 'AIRMASS'
Zendkeyword = 'AIRMASS' #These are the same because HARPSN doesnt have start and end keywords.
#Down there, the airmass is averaged, so there is no problem in taking the average of the same number.
else:
raise ValueError(
f"Error in read_harpslike: mode should be set to HARPS or HARPSN ({mode})"
)
#The following variables define lists in which all the necessary data will be stored.
framename = []
header = []
s1dhdr = []
obstype = []
texp = np.array([])
date = []
mjd = np.array([])
s1dmjd = np.array([])
npx = np.array([])
norders = np.array([])
e2ds = []
s1d = []
wave1d = []
airmass = np.array([])
berv = np.array([])
wave = []
# wavefile_used = []
for i in range(len(filelist)):
if filelist[i].endswith('e2ds_A.fits'):
print(f'------{filelist[i]}', end="\r")
hdul = fits.open(inpath / filelist[i])
data = copy.deepcopy(hdul[0].data)
hdr = hdul[0].header
hdul.close()
del hdul[0].data
if hdr[catkeyword] == 'SCIENCE':
framename.append(filelist[i])
header.append(hdr)
obstype.append(hdr[catkeyword])
texp = np.append(texp, hdr['EXPTIME'])
date.append(hdr['DATE-OBS'])
mjd = np.append(mjd, hdr['MJD-OBS'])
npx = np.append(npx, hdr['NAXIS1'])
norders = np.append(norders, hdr['NAXIS2'])
e2ds.append(data)
berv = np.append(berv, hdr[bervkeyword])
airmass = np.append(
airmass, 0.5 * (hdr[Zstartkeyword] + hdr[Zendkeyword])
) #This is an approximation where we take the mean airmass.
# if nowave == True:
# wavefile_used.append(hdr[thfilekeyword])
#Record which wavefile was used by the pipeline to
#create the wavelength solution.
wavedata = ut.read_wave_from_e2ds_header(
hdr, mode=mode) / 10.0 #convert to nm.
wave.append(wavedata)
# if filelist[i].endswith('wave_A.fits'):
# print(filelist[i]+' (wave)')
# if nowave == True:
# warnings.warn(" in read_e2ds: nowave was set to True but a wave_A file was detected. This wave file is now ignored in favor of the header.",RuntimeWarning)
# else:
# wavedata=fits.getdata(inpath/filelist[i])
# wave.append(wavedata)
if read_s1d:
s1d_path = inpath / Path(
str(filelist[i]).replace('e2ds_A.fits', 's1d_A.fits'))
ut.check_path(
s1d_path,
exists=True) #Crash if the S1D doesn't exist.
# if filelist[i].endswith('s1d_A.fits'):
hdul = fits.open(s1d_path)
data_1d = copy.deepcopy(hdul[0].data)
hdr1d = hdul[0].header
hdul.close()
del hdul
# if hdr[catkeyword] == 'SCIENCE':
s1d.append(data_1d)
if mode == 'HARPSN': #In the case of HARPS-N we need to convert the units of the
#elevation and provide a UTC keyword.
hdr1d['TELALT'] = np.degrees(float(hdr1d['EL']))
hdr1d['UTC'] = (float(hdr1d['MJD-OBS']) %
1.0) * 86400.0
s1dhdr.append(hdr1d)
s1dmjd = np.append(s1dmjd, hdr1d['MJD-OBS'])
berv1d = hdr1d[bervkeyword]
if berv1d != hdr[bervkeyword]:
wrn_msg = (
'WARNING in read_harpslike(): BERV correction of s1d file is not'
f'equal to that of the e2ds file. {berv1d} vs {hdr[bervkeyword]}'
)
ut.tprint(wrn_msg)
gamma = (1.0 -
(berv1d * u.km / u.s / const.c).decompose().value
) #Doppler factor BERV.
wave1d.append(
(hdr1d['CDELT1'] * fun.findgen(len(data_1d)) +
hdr1d['CRVAL1']) * gamma)
#Check that all exposures have the same number of pixels, and clip s1ds if needed.
# min_npx1d = int(np.min(np.array(npx1d)))
# if np.sum(np.abs(np.array(npx1d)-npx1d[0])) != 0:
# warnings.warn("in read_e2ds when reading HARPS data: Not all s1d files have the same number of pixels. This could have happened if the pipeline has extracted one or two extra pixels in some exposures but not others. The s1d files will be clipped to the smallest length.",RuntimeWarning)
# for i in range(len(s1d)):
# wave1d[i]=wave1d[i][0:min_npx1d]
# s1d[i]=s1d[i][0:min_npx1d]
# npx1d[i]=min_npx1d
output = {
'wave': wave,
'e2ds': e2ds,
'header': header,
'wave1d': wave1d,
's1d': s1d,
's1dhdr': s1dhdr,
'mjd': mjd,
'date': date,
'texp': texp,
'obstype': obstype,
'framename': framename,
'npx': npx,
'norders': norders,
'berv': berv,
'airmass': airmass,
's1dmjd': s1dmjd
}
return (output)
def apply_telluric_correction(inpath, list_of_wls, list_of_orders,
list_of_sigmas):
"""
This applies a set of telluric spectra (computed by molecfit) for each exposure
in our time series that were written to a pickle file by write_telluric_transmission_to_file.
List of errors are provided to propagate telluric correction into the error array as well.
Parameters
----------
inpath : str, path like
The path to the pickled transmission spectra.
list_of_wls : list
List of wavelength axes.
list_of_orders :
List of 2D spectral orders, matching to the wavelength axes in dimensions and in number.
list_of_isgmas :
List of 2D error matrices, matching dimensions and number of list_of_orders.
Returns
-------
list_of_orders_corrected : list
List of 2D spectral orders, telluric corrected.
list_of_sigmas_corrected : list
List of 2D error matrices, telluric corrected.
"""
import scipy.interpolate as interp
import numpy as np
import tayph.util as ut
import tayph.functions as fun
from tayph.vartests import dimtest, postest, typetest, nantest
import copy
T = read_telluric_transmission_from_file(inpath)
wlT = T[0]
fxT = T[1]
typetest(list_of_wls, list, 'list_of_wls in apply_telluric_correction()')
typetest(list_of_orders, list,
'list_of_orders in apply_telluric_correction()')
typetest(list_of_sigmas, list,
'list_of_sigmas in apply_telluric_correction()')
typetest(wlT, list,
'list of telluric wave-axes in apply_telluric_correction()')
typetest(
fxT, list,
'list of telluric transmission spectra in apply_telluric_correction()')
No = len(list_of_wls) #Number of orders.
x = fun.findgen(No)
Nexp = len(wlT)
#Test dimensions
if No != len(list_of_orders):
raise Exception(
'Runtime error in telluric correction: List of wavelength axes and List '
'of orders do not have the same length.')
if Nexp != len(fxT):
raise Exception(
'Runtime error in telluric correction: List of telluric wls and telluric '
'spectra read from file do not have the same length.')
if Nexp != len(list_of_orders[0]):
raise Exception(
f'Runtime error in telluric correction: List of telluric spectra and data'
f'spectra read from file do not have the same length ({Nexp} vs {len(list_of_orders[0])}).'
)
#Corrected orders will be stored here.
list_of_orders_cor = []
list_of_sigmas_cor = []
#Do the correction order by order:
for i in range(No):
order = list_of_orders[i]
order_cor = order * 0.0
error = list_of_sigmas[i]
error_cor = error * 0.0
wl = copy.deepcopy(list_of_wls[i]) #input wl axis, either 1D or 2D.
#If it is 1D, we make it 2D by tiling it vertically:
if wl.ndim == 1: wl = np.tile(wl, (Nexp, 1)) #Tile it into a 2D thing.
#If read (2D) or tiled (1D) correctly, wl and order should have the same shape:
dimtest(wl, np.shape(order),
f'Wl axis of order {i}/{No} in apply_telluric_correction()')
dimtest(error, np.shape(order),
f'errors {i}/{No} in apply_telluric_correction()')
for j in range(Nexp):
T_i = interp.interp1d(wlT[j], fxT[j],
fill_value="extrapolate")(wl[j])
postest(T_i,
f'T-spec of exposure {j} in apply_telluric_correction()')
nantest(T_i,
f'T-spec of exposure {j} in apply_telluric_correction()')
order_cor[j] = order[j] / T_i
error_cor[j] = error[
j] / T_i #I checked that this works because the SNR before and after
#telluric correction is identical.
list_of_orders_cor.append(order_cor)
list_of_sigmas_cor.append(error_cor)
ut.statusbar(i, x)
return (list_of_orders_cor, list_of_sigmas_cor)
def normalize_orders(list_of_orders, list_of_sigmas, deg=1, nsigma=4):
"""
If deg is set to 1, this function will normalise based on the mean flux in each order.
If set higher, it will remove the average spectrum in each order and fit a polynomial
to the residual. This means that in the presence of spectral lines, the fluxes will be
slightly lower than if def=1 is used. nsigma is only used if deg > 1, and is used to
throw away outliers from the polynomial fit. The program also computes the total
mean flux of each exposure in the time series - totalled over all orders. These
are important to correctly weigh the cross-correlation functions later. The
inter-order colour correction is assumed to be an insignificant modification to
these weights.
Parameters
----------
list_of_orders : list
The list of 2D orders that need to be normalised.
list_of_sigmas : list
The list of 2D error matrices corresponding to the 2D orders that need to be normalised.
deg : int
The polynomial degree to remove. If set to 1, only the average flux is removed. If higher,
polynomial fits are made to the residuals after removal of the average spectrum.
nsigma : int, float
The number of sigmas beyond which outliers are rejected from the polynomial fit.
Only used when deg > 1.
Returns
-------
out_list_of_orders : list
The normalised 2D orders.
out_list_of_sigmas : list
The corresponding errors.
meanfluxes : np.array
The mean flux of each exposure in the time series, averaged over all orders.
"""
import numpy as np
import tayph.functions as fun
from tayph.vartests import dimtest, postest, typetest
import warnings
typetest(list_of_orders, list, 'list_of_orders in ops.normalize_orders()')
typetest(list_of_sigmas, list, 'list_of_sigmas in ops.normalize_orders()')
dimtest(list_of_orders[0], [0, 0]) #Test that the first order is 2D.
dimtest(list_of_sigmas[0],
[0, 0]) #And that the corresponding sigma array is, as well.
n_exp = np.shape(list_of_orders[0])[0] #Get the number of exposures.
for i in range(len(list_of_orders)): #Should be the same for all orders.
dimtest(list_of_orders[i], [n_exp, 0])
dimtest(list_of_sigmas[i], np.shape(list_of_orders[i]))
typetest(deg, int, 'degree in ops.normalize_orders()')
typetest(nsigma, [int, float], 'nsigma in ops.normalize_orders()')
postest(deg, 'degree in ops.normalize_orders()')
postest(nsigma, 'degree in ops.normalize_orders()')
N = len(list_of_orders)
out_list_of_orders = []
out_list_of_sigmas = []
#First compute the exposure-to-exposure flux variations to be used as weights.
meanfluxes = fun.findgen(n_exp) * 0.0
N_i = 0
for i in range(N):
m = np.nanmedian(list_of_orders[i], axis=1) #Median or mean?
if np.sum(np.isnan(m)) > 0:
print(
'---Warning in normalise_orders: Skipping order %s because many nans are present.'
% i)
else:
N_i += 1
meanfluxes += m #These contain the exposure-to-exposure variability of the time-series.
meanfluxes /= N_i #These are the weights.
if deg == 1:
for i in range(N):
#What I'm doing here is probably stupid and numpy division will probably work just fine without
#IDL-relics.
n_px = np.shape(list_of_orders[i])[1]
meanflux = np.nanmedian(
list_of_orders[i],
axis=1) #Average flux in each order. Median or mean?
meanblock = fun.rebinreform(
meanflux / np.nanmean(meanflux), n_px
).T #This is a slow operation. Row-by-row division is better done using a double-transpose...
out_list_of_orders.append(list_of_orders[i] / meanblock)
out_list_of_sigmas.append(list_of_sigmas[i] / meanblock)
else:
for i in range(N):
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=RuntimeWarning)
meanspec = np.nanmean(list_of_orders[i],
axis=0) #Average spectrum in each order.
x = np.array(range(len(meanspec)))
poly_block = list_of_orders[
i] * 0.0 #Array that will host the polynomial fits.
colour = list_of_orders[
i] / meanspec #What if there are zeroes? I.e. padding around the edges of the order?
for j, s in enumerate(list_of_orders[i]):
idx = np.isfinite(colour[j])
if np.sum(idx) > 0:
p = np.poly1d(np.polyfit(x[idx], colour[j][idx], deg))(
x) #Polynomial fit to the colour variation.
res = colour[
j] / p - 1.0 #The residual, which is flat around zero if it's a good fit. This has all sorts of line residuals that we need to throw out.
#We do that using the weight keyword of polyfit, and just set all those weights to zero.
sigma = np.nanstd(res)
w = x * 0.0 + 1.0 #Start with a weight function that is 1.0 everywhere.
with warnings.catch_warnings():
warnings.simplefilter("ignore",
category=RuntimeWarning)
w[np.abs(res) > nsigma * sigma] = 0.0
w = x * 0.0 + 1.0 #Start with a weight function that is 1.0 everywhere.
p2 = np.poly1d(
np.polyfit(x[idx], colour[j][idx], deg, w=w[idx])
)(
x
) #Second, weighted polynomial fit to the colour variation.
poly_block[j] = p2
out_list_of_orders.append(list_of_orders[i] / poly_block)
out_list_of_sigmas.append(list_of_sigmas[i] / poly_block)
return (out_list_of_orders, out_list_of_sigmas, meanfluxes)
def __init__(self,list_of_wls,list_of_orders,list_of_saved_selected_columns,Nxticks,Nyticks,nsigma=3.0):
"""
We initialize with a figure object, three axis objects (in a list)
the wls, the orders, the masks already made; and we do the first plot.
NOTICE: Anything that is potted in these things as INF actually used to be
a NaN that was masked out before.
"""
import numpy as np
import pdb
import tayph.functions as fun
import tayph.plotting as plotting
import tayph.drag_colour as dcb
import matplotlib.pyplot as plt
import itertools
from matplotlib.widgets import MultiCursor
import tayph.util as ut
from tayph.vartests import typetest,postest
import copy
#Upon initialization, we raise the keywords onto self.
self.N_orders = len(list_of_wls)
if len(list_of_wls) < 1 or len(list_of_orders) < 1:# or len(list_of_masks) <1:
raise Exception('Runtime Error in mask_maker init: lists of WLs, orders and/or masks have less than 1 element.')
if len(list_of_wls) != len(list_of_orders):# or len(list_of_wls) != len(list_of_masks):
raise Exception('Runtime Error in mask_maker init: List of wls and list of orders have different length (%s & %s).' % (len(list_of_wls),len(list_of_orders)))
typetest(Nxticks,int,'Nxticks in mask_maker init',)
typetest(Nyticks,int,'Nyticks in mask_maker init',)
typetest(nsigma,float,'Nsigma in mask_maker init',)
postest(Nxticks,varname='Nxticks in mask_maker init')
postest(Nyticks,varname='Nyticks in mask_maker init')
postest(nsigma,varname='Nsigma in mask_maker init')
self.list_of_wls = list_of_wls
self.list_of_orders = list_of_orders
self.list_of_selected_columns = list(list_of_saved_selected_columns)
#Normally, if there are no saved columns to load, list_of_saved_selected_columns is an empty list. However if
#it is set, then its automatically loaded into self.list_of_selected_columns upon init.
#Below there is a check to determine whether it was empty or not, and whether the list of columns
#has the same length as the list of orders.
if len(self.list_of_selected_columns) == 0:
for i in range(self.N_orders):
self.list_of_selected_columns.append([])#Make a list of empty lists.
#This will contain all columns masked by the user, on top of the things
#that are already masked by the program.
else:
if len(self.list_of_selected_columns) != self.N_orders:
raise Exception('Runtime Error in mask_maker init: Trying to restore previously saved columns but the number of orders in the saved column file does not match the number of orders provided.')
print('------Restoring previously saved columns in mask-maker')
#All checks are now complete. Lets prepare to do the masking.
# self.N = min([56,self.N_orders-1])#We start on order 56, or the last order if order 56 doesn't exist.
self.N=0
#Set the current active order to order , and calculate the meanspec
#and residuals to be plotted, which are saved in self.
self.set_order(self.N)
#Sorry for the big self.spaghetti of code. This initializes the plot.
#Functions and vars further down in the class will deal with updating the plots
#as buttons are pressed. Some of this is copied from the construct_doppler_model
#function; but this time I made it part of the class.
#First define plotting and axis parameters for the colormesh below.
self.Nxticks = Nxticks
self.Nyticks = Nyticks
self.nsigma = nsigma
self.xrange = [0,self.npx-1]
self.yrange=[0,self.nexp-1]
self.x_axis=fun.findgen(self.npx).astype(int)
self.y_axis = fun.findgen(self.nexp).astype(int)
self.x2,self.y2,self.z,self.wl_sel,self.y_axis_sel,self.xticks,self.yticks,void1,void2= plotting.plotting_scales_2D(self.x_axis,self.y_axis,self.residual,self.xrange,self.yrange,Nxticks=self.Nxticks,Nyticks=self.Nyticks,nsigma=self.nsigma)
self.fig,self.ax = plt.subplots(3,1,sharex=True,figsize=(14,6))#Initialize the figure and 3 axes.
plt.subplots_adjust(left=0.05)#Make them more tight, we need all the space we can get.
plt.subplots_adjust(right=0.85)
self.ax[0].set_title('Spectral order %s (%s - %s nm)' % (self.N,round(np.min(self.wl),1),round(np.max(self.wl),1)))
self.ax[1].set_title('Residual of time-average')
self.ax[2].set_title('Time average 1D spectrum')
array1 = copy.deepcopy(self.order)
array2 = copy.deepcopy(self.residual)
array1[np.isnan(array1)] = np.inf#The colobar doesn't eat NaNs, so now set them to inf just for the plot.
array2[np.isnan(array2)] = np.inf#And here too.
#The previous three lines are repeated in self.update_plots()
self.img1=self.ax[0].pcolormesh(self.x2,self.y2,array1,vmin=0,vmax=self.img_max,cmap='hot')
self.img2=self.ax[1].pcolormesh(self.x2,self.y2,array2,vmin=self.vmin,vmax=self.vmax,cmap='hot')
self.img3=self.ax[2].plot(self.x_axis,self.meanspec)
self.ax[2].set_xlim((min(self.x_axis),max(self.x_axis)))
self.ax[2].set_ylim(0,self.img_max)
#This trick to associate a single CB to multiple axes comes from
#https://stackoverflow.com/questions/13784201/matplotlib-2-subplots-1-colorbar
self.cbar = self.fig.colorbar(self.img2, ax=self.ax.ravel().tolist(),aspect = 20)
self.cbar = dcb.DraggableColorbar_fits(self.cbar,[self.img2],'hot')
self.cbar.connect()
#The rest is for dealing with the masking itself; the behaviour of the
#add/subtact buttons, the cursor and the saving of the masked columns.
self.col_active = ['coral','mistyrose']#The colours for the ADD and SUBTRACT buttons that
#can be activated.
self.col_passive = ['lightgrey','whitesmoke']#Colours when they are not active.
self.MW = 50#The default masking width.
self.addstatus = 0#The status for adding-to-mask mode starts as zero; i.e. it starts inactive.
self.substatus = 0#Same for subtraction.
self.list_of_polygons = []#This stores the polygons that are currently plotted. When initializing, none are plotted.
#However when proceeding through draw_masked_areas below, this variable could become populated if previously selected
#column were loaded from file.
self.multi = MultiCursor(self.fig.canvas, (self.ax[0],self.ax[1],self.ax[2]), color='g', lw=1, horizOn=False, vertOn=True)
self.multi.set_active(False)#The selection cursor starts deactivated as well, and is activated and deactivated
#further down as the buttons are pressed.
self.apply_to_all = False
self.apply_to_all_future = False
#The following show the z value of the plotted arrays in the statusbar,
#taken from https://matplotlib.org/examples/api/image_zcoord.html
numrows, numcols = self.order.shape
def format_coord_order(x, y):
col = int(x + 0.5)
row = int(y + 0.5)
if col >= 0 and col < numcols and row >= 0 and row < numrows:
z = self.order[row, col]
return 'x=%1.4f, y=%1.4f, z=%1.4f' % (x, y, z)
else:
return 'x=%1.4f, y=%1.4f' % (x, y)
def format_coord_res(x, y):
col = int(x + 0.5)
row = int(y + 0.5)
if col >= 0 and col < numcols and row >= 0 and row < numrows:
z = self.residual[row, col]
return 'x=%1.4f, y=%1.4f, z=%1.4f' % (x, y, z)
else:
return 'x=%1.4f, y=%1.4f' % (x, y)
self.ax[0].format_coord = format_coord_order
self.ax[1].format_coord = format_coord_res
self.draw_masked_areas()
def convolve(array, kernel, edge_degree=1, fit_width=2):
"""It's unbelievable, but I could not find the python equivalent of IDL's
/edge_truncate keyword, which truncates the kernel at the edge of the convolution.
Therefore, unfortunately, I need to code a convolution operation myself.
Stand by to be slowed down by an order of magnitude #thankspython.
Nope! Because I can just use np.convolve for most of the array, just not the edge...
So the strategy is to extrapolate the edge of the array using a polynomial fit
to the edge elements. By default, I fit over a range that is twice the length of the kernel; but
this value can be modified using the fit_width parameter.
Parameters
----------
array : list, np.ndarray
The horizontal axis.
kernel : list, np.ndarray
The convolution kernel. It is required to have a length that is less than 25% of the size of the array.
edge_degree : int
The polynomial degree by which the array is extrapolated in order to
fit_width : int
The length of the area at the edges of array used to fit the polynomial, in units of the length of the kernel.
Increase this number for small kernels or noisy arrays.
Returns
-------
array_convolved : np.array
The input array convolved with the kernel
Example
-------
>>> import numpy as np
>>> a=[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]
>>> b=[-0.5,0,0.5]
>>> c=convolve(a,b,edge_degree=1)
"""
import numpy as np
import tayph.functions as fun
from tayph.vartests import typetest, postest, dimtest
typetest(edge_degree, int, 'edge_degree in ops.convolve()')
typetest(fit_width, int, 'edge_degree in ops.convolve()')
typetest(array, [list, np.ndarray])
typetest(kernel, [list, np.ndarray])
dimtest(array, [0], 'array in ops.convolve()')
dimtest(kernel, [0], 'array in ops.convolve()')
postest(edge_degree, 'edge_degree in ops.convolve()')
postest(fit_width, 'edge_degree in ops.convolve()')
array = np.array(array)
kernel = np.array(kernel)
if len(kernel) >= len(array) / 4.0:
raise Exception(
f"Error in ops.convolve(): Kernel length is larger than a quarter of the array ({len(kernel)}, {len(array)}). Can't extrapolate over that length. And you probably don't want to be doing a convolution like that, anyway."
)
if len(kernel) % 2 != 1:
raise Exception(
'Error in ops.convolve(): Kernel needs to have an odd number of elements.'
)
#Perform polynomial fits at the edges.
x = fun.findgen(len(array))
fit_left = np.polyfit(x[0:len(kernel) * 2], array[0:len(kernel) * 2],
edge_degree)
fit_right = np.polyfit(x[-2 * len(kernel) - 1:-1],
array[-2 * len(kernel) - 1:-1], edge_degree)
#Pad both the x-grid (onto which the polynomial is defined)
#and the data array.
pad = fun.findgen((len(kernel) - 1) / 2)
left_pad = pad - (len(kernel) - 1) / 2
right_pad = np.max(x) + pad + 1
left_array_pad = np.polyval(fit_left, left_pad)
right_array_pad = np.polyval(fit_right, right_pad)
#Perform the padding.
x_padded = np.append(left_pad, x)
x_padded = np.append(
x_padded, right_pad
) #Pad the array with the missing elements of the kernel at the edge.
array_padded = np.append(left_array_pad, array)
array_padded = np.append(array_padded, right_array_pad)
#Reverse the kernel because np.convol does that automatically and I don't want that.
#(Imagine doing a derivative with a kernel [-1,0,1] and it gets reversed...)
kr = kernel[::-1]
#The valid keyword effectively undoes the padding, leaving only those values for which the kernel was entirely in the padded array.
#This thus again has length equal to len(array).
return np.convolve(array_padded, kr, 'valid')
def mask_orders(list_of_wls,list_of_orders,dp,maskname,w,c_thresh,manual=False):
"""
This code takes the list of orders and masks out bad pixels.
It combines two steps, a simple sigma clipping step and a manual step, where
the user can interactively identify bad pixels in each order. The sigma
clipping is done on a threshold of c_thresh, using a rolling standard dev.
with a width of w pixels. Manual masking is a big routine needed to support
a nice GUI to do that.
If c_thresh is set to zero, sigma clipping is skipped. If manual=False, the
manual selection of masking regions (which is manual labour) is turned off.
If both are turned off, the list_of_orders is returned unchanged.
If either or both are active, the routine will output 1 or 2 FITS files that
contain a stack (cube) of the masks for each order. The first file is the mask
that was computed automatically, the second is the mask that was constructed
manually. This is done so that the manual mask can be transplanted onto another
dataset, or saved under a different file-name, to limit repetition of work.
At the end of the routine, the two masks are merged into a single list, and
applied to the list of orders.
"""
import tayph.operations as ops
import numpy as np
import tayph.functions as fun
import tayph.plotting as plotting
import sys
import matplotlib.pyplot as plt
import tayph.util as ut
import warnings
from tayph.vartests import typetest,dimtest,postest
ut.check_path(dp)
typetest(maskname,str,'maskname in mask_orders()')
typetest(w,[int,float],'w in mask_orders()')
typetest(c_thresh,[int,float],'c_thresh in mask_orders()')
postest(w,'w in mask_orders()')
postest(c_thresh,'c_thresh in mask_orders()')
typetest(list_of_wls,list,'list_of_wls in mask_orders()')
typetest(list_of_orders,list,'list_of_orders in mask_orders()')
typetest(manual,bool,'manual keyword in mask_orders()')
dimtest(list_of_wls,[0,0],'list_of_wls in mask_orders()')
dimtest(list_of_orders,[len(list_of_wls),0,0],'list_of_orders in mask_orders()')
if c_thresh <= 0 and manual == False:
print('---WARNING in mask_orders: c_thresh is set to zero and manual masking is turned off.')
print('---Returning orders unmasked.')
return(list_of_orders)
N = len(list_of_orders)
void = fun.findgen(N)
list_of_orders = ops.normalize_orders(list_of_orders,list_of_orders)[0]#first normalize. Dont want outliers to
#affect the colour correction later on, so colour correction cant be done before masking, meaning
#that this needs to be done twice; as colour correction is also needed for proper maskng. The second variable is
#a dummy to replace the expected list_of_sigmas input.
N_NaN = 0
list_of_masked_orders = []
for i in range(N):
list_of_masked_orders.append(list_of_orders[i])
list_of_masks = []
if c_thresh > 0:#Check that c_thresh is positive. If not, skip sigma clipping.
print('------Sigma-clipping mask')
for i in range(N):
order = list_of_orders[i]
N_exp = np.shape(order)[0]
N_px = np.shape(order)[1]
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=RuntimeWarning)
meanspec = np.nanmean(order,axis = 0)
meanblock = fun.rebinreform(meanspec,N_exp)
res = order / meanblock - 1.0
sigma = fun.running_MAD_2D(res,w)
with np.errstate(invalid='ignore'):#https://stackoverflow.com/questions/25345843/inequality-comparison-of-numpy-array-with-nan-to-a-scalar
sel = np.abs(res) >= c_thresh*sigma
N_NaN += np.sum(sel)#This is interesting because True values count as 1, and False as zero.
order[sel] = np.nan
list_of_masks.append(order*0.0)
ut.statusbar(i,void)
print(f'%s outliers identified and set to NaN ({N_NaN}/{round(N_NaN/np.size(list_of_masks)*100.0,3)}).')
else:
print('------Skipping sigma-clipping (c_thres <= 0)')
#Do nothing to list_of_masks. It is now an empty list.
#We now automatically proceed to manual masking, because at this point
#it has already been established that it must have been turned on.
list_of_masks_manual = []
if manual == True:
previous_list_of_masked_columns = load_columns_from_file(dp,maskname,mode='relaxed')
list_of_masked_columns = manual_masking(list_of_wls,list_of_orders,list_of_masks,saved = previous_list_of_masked_columns)
print('------Successfully concluded manual mask.')
write_columns_to_file(dp,maskname,list_of_masked_columns)
print('------Building manual mask from selected columns')
for i in range(N):
order = list_of_orders[i]
N_exp = np.shape(order)[0]
N_px = np.shape(order)[1]
list_of_masks_manual.append(np.zeros((N_exp,N_px)))
for j in list_of_masked_columns[i]:
list_of_masks_manual[i][:,int(j)] = np.nan
#We write 1 or 2 mask files here. The list of manual masks
#and list_of_masks (auto) are either filled, or either is an emtpy list if
#c_thresh was set to zero or manual was set to False (because they were defined
#as empty lists initially, and then not filled with anything).
write_mask_to_file(dp,maskname,list_of_masks,list_of_masks_manual)
return(0)
def constant_velocity_wl_grid(wl, fx, oversampling=1.0):
"""This function will define a constant-velocity grid that is (optionally)
sampled a number of times finer than the SMALLEST velocity difference that is
currently in the grid.
Example: wl_cv,fx_cv = constant_velocity_wl_grid(wl,fx,oversampling=1.5).
This function is hardcoded to raise an exception if wl or fx contain NaNs,
because interp1d does not handle NaNs.
Parameters
----------
wl : list, np.ndarray
The wavelength array to be resampled.
fx : list, np.ndarray
The flux array to be resampled.
oversampling : float
The factor by which the wavelength array is *minimally* oversampled.
Returns
-------
wl : np.array
The new wavelength grid.
fx : np.array
The interpolated flux values.
a : float
The velocity step in km/s.
"""
import astropy.constants as consts
import numpy as np
import tayph.functions as fun
from tayph.vartests import typetest, nantest, dimtest, postest
from scipy import interpolate
import pdb
import matplotlib.pyplot as plt
typetest(
oversampling,
[int, float],
'oversampling in constant_velocity_wl_grid()',
)
typetest(wl, [list, np.ndarray], 'wl in constant_velocity_wl_grid()')
typetest(fx, [list, np.ndarray], 'fx in constant_velocity_wl_grid()')
nantest(wl, 'wl in in constant_velocity_wl_grid()')
nantest(fx, 'fx in constant_velocity_wl_grid()')
dimtest(wl, [0], 'wl in constant_velocity_wl_grid()')
dimtest(fx, [len(wl)], 'fx in constant_velocity_wl_grid()')
postest(oversampling, 'oversampling in constant_velocity_wl_grid()')
oversampling = float(oversampling)
wl = np.array(wl)
fx = np.array(fx)
c = consts.c.to('km/s').value
dl = derivative(wl)
dv = dl / wl * c
a = np.min(dv) / oversampling
wl_new = 0.0
#The following while loop will define the new pixel grid.
#It starts trying 100,000 points, and if that's not enough to cover the entire
#range from min(wl) to max(wl), it will add 100,000 more; until it's enough.
n = len(wl)
while np.max(wl_new) < np.max(wl):
x = fun.findgen(n)
wl_new = np.exp(a / c * x) * np.min(wl)
n += len(wl)
wl_new[0] = np.min(
wl) #Artificially set to zero to avoid making a small round
#off error in that exponent.
#Then at the end we crop the part that goes too far:
wl_new_cropped = wl_new[(wl_new <= np.max(wl))]
x_cropped = x[(wl_new <= np.max(wl))]
i_fx = interpolate.interp1d(wl, fx)
fx_new_cropped = i_fx(wl_new_cropped)
return (wl_new_cropped, fx_new_cropped, a)
