def envelope(wlm, fxm, binsize, selfrac=0.05, mode='top', threshold=''):
"""
This program measures the top or bottom envelope of a spectrum (wl,fx), by
chopping it up into bins of size binsze (unit of wl), and measuring the mean
of the top n % of values in that bin. Setting the mode to 'bottom' will do the
oppiste: The mean of the bottom n% of values. The output is the resulting wl
and flux points of these bins.
Example: wle,fxe = envelope(wl,fx,1.0,selfrac=3.0,mode='top')
"""
import numpy as np
import tayph.util as ut
import tayph.functions as fun
from tayph.vartests import typetest, dimtest, nantest, postest
from matplotlib import pyplot as plt
typetest(wlm, np.ndarray, 'wlm in ops.envelope()')
typetest(fxm, np.ndarray, 'fxm in ops.envelope()')
dimtest(wlm, [len(fxm)])
typetest(binsize, [int, float], 'binsize in ops.envelope()')
typetest(selfrac, float, 'percentage in ops.envelope()')
typetest(mode, str, 'mode in envelope')
nantest(fxm, 'fxm in ops.envelope()')
nantest(wlm, 'wlm in ops.envelope()')
postest(wlm, 'wlm in ops.envelope()')
postest(binsize, 'binsize in ops.envelope()')
if mode not in ['top', 'bottom']:
raise Exception(
f'RuntimeError in ops.envelope(): Mode should be set to "top" or "bottom" ({mode}).'
)
binsize = float(binsize)
if mode == 'bottom':
fxm *= -1.0
wlcs = np.array([]) #Center wavelengths
fxcs = np.array([]) #Flux at center wavelengths
i_start = 0
wlm_start = wlm[i_start]
for i in range(0, len(wlm) - 1):
if wlm[i] - wlm_start >= binsize:
wlsel = wlm[i_start:i]
fxsel = fxm[i_start:i]
maxsel = fun.selmax(fxsel, selfrac)
wlcs = np.append(wlcs, np.mean(wlsel[maxsel]))
fxcs = np.append(fxcs, np.mean(fxsel[maxsel]))
i_start = i + 1
wlm_start = wlm[i + 1]
if isinstance(threshold, float) == True:
#This means that the threshold value is set, and we set all bins less than
#that threshold to the threshold value:
if mode == 'bottom':
threshold *= -1.0
fxcs[(fxcs < threshold)] = threshold
if mode == 'bottom':
fxcs *= -1.0
fxm *= -1.0
return wlcs, fxcs
def running_MAD_2D(z, w, verbose=False, parallel=False):
"""Computers a running standard deviation of a 2-dimensional array z.
The stddev is evaluated over the vertical block with width w pixels.
The output is a 1D array with length equal to the width of z.
This is very slow on arrays that are wide in x (hundreds of thousands of points)."""
import astropy.stats as stats
import numpy as np
from tayph.vartests import typetest, dimtest, postest
import tayph.util as ut
if parallel: from joblib import Parallel, delayed
typetest(z, np.ndarray, 'z in fun.running_MAD_2D()')
dimtest(z, [0, 0], 'z in fun.running_MAD_2D()')
typetest(w, [int, float], 'w in fun.running_MAD_2D()')
postest(w, 'w in fun.running_MAD_2D()')
size = np.shape(z)
ny = size[0]
nx = size[1]
s = np.arange(0, nx, dtype=float) * 0.0
dx1 = int(0.5 * w)
dx2 = int(int(0.5 * w) + (w % 2)) #To deal with odd windows.
for i in range(nx):
minx = max([0, i - dx1]) #This here is only a 3% slowdown.
maxx = min([nx, i + dx2])
s[i] = stats.mad_std(
z[:, minx:maxx],
ignore_nan=True) #This is what takes 97% of the time.
if verbose: ut.statusbar(i, nx)
return (s)
def nan_helper(y):
"""
Helper function to handle indices and logical indices of NaNs.
Input:
- y, 1D (!!) numpy array with possible NaNs
Output:
- nans, logical indices of NaNs
- index, a function, with signature indices= index(logical_indices),
to convert logical indices of NaNs to 'equivalent' indices
Example:
>>> # linear interpolation of NaNs
>>> import numpy as np
>>> y = np.array([0,0.1,0.2,np.nan,0.4,0.5])
>>> nans, x= nan_helper(y)
>>> y[nans]= np.interp(x(nans), x(~nans), y[~nans])
"""
#This comes from https://stackoverflow.com/questions/6518811/interpolate-nan-values-in-a-numpy-array
# Lets define first a simple helper function in order to make it more straightforward to handle indices and logical indices of NaNs:
# Now the nan_helper(.) can now be utilized like:
#
# >>> y= array([1, 1, 1, NaN, NaN, 2, 2, NaN, 0])
# >>>
# >>> nans, x= nan_helper(y)
# >>> y[nans]= np.interp(x(nans), x(~nans), y[~nans])
# >>>
# >>> print y.round(2)
# [ 1. 1. 1. 1.33 1.67 2. 2. 1. 0. ]
import numpy as np
from tayph.vartests import dimtest
dimtest(y,[0],'y in fun.nan_helper()')
return np.isnan(y), lambda z: z.nonzero()[0]
def derivative(x):
"""
This computes the simple numerical derivative of x by convolving with kernel [-1,0,1].
Parameters
----------
x : list, np.ndarray
The array from which the derivative is required.
Returns
-------
derivative : np.array
The numerical derivative of x.
Example
-------
>>> import numpy as np
>>> x=[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]
>>> dx=derivative(x)
"""
import numpy as np
from tayph.vartests import typetest, dimtest
typetest(x, [list, np.ndarray], 'x in derivative()')
dimtest(x, [0], 'x in derivative()')
x = np.array(x)
d_kernel = np.array([-1, 0, 1]) / 2.0
return (convolve(x, d_kernel, fit_width=3))
def selmax(y_in,p,s=0.0):
"""This program returns the p (fraction btw 0 and 1) highest points in y,
ignoring the very top s % (default zero, i.e. no points ignored), for the
purpose of outlier rejection."""
import tayph.util as ut
from tayph.vartests import postest,dimtest
import numpy as np
import copy
postest(p)
y=copy.deepcopy(y_in)#Copy itself, or there are pointer-related troubles...
if s < 0.0:
raise Exception("ERROR in selmax: s should be zero or positive.")
if p >= 1.0:
raise Exception("ERROR in selmax: p should be strictly between 0.0 and 1.0.")
if s >= 1.0:
raise Exception("ERROR in selmax: s should be strictly less than 1.0.")
postest(-1.0*p+1.0)
#ut.nantest('y in selmax',y)
dimtest(y,[0])#Test that it is one-dimensional.
y[np.isnan(y)]=np.nanmin(y)#set all nans to the minimum value such that they will not be selected.
y_sorting = np.flipud(np.argsort(y))#These are the indices in descending order (thats why it gets a flip)
N=len(y)
if s == 0.0:
max_index = np.max([int(round(N*p)),1])#Max because if the fraction is 0 elements, then at least it should contain 1.0
return y_sorting[0:max_index]
if s > 0.0:
min_index = np.max([int(round(N*s)),1])#If 0, then at least it should be 1.0
max_index = np.max([int(round(N*(p+s))),2]) #If 0, then at least it should be 1+1.
return y_sorting[min_index:max_index]
def telluric_correction_order(i):
order = list_of_orders[i]
order_cor = order * 0.0
error = list_of_sigmas[i]
error_cor = error * 0.0
wl = copy.deepcopy(list_of_wls[i]) #input wl axis, either 1D or 2D.
#If it is 1D, we make it 2D by tiling it vertically:
if wl.ndim == 1: wl = np.tile(wl, (Nexp, 1)) #Tile it into a 2D thing.
#If read (2D) or tiled (1D) correctly, wl and order should have the same shape:
dimtest(wl, np.shape(order),
f'Wl axis of order {i}/{No} in apply_telluric_correction()')
dimtest(error, np.shape(order),
f'errors {i}/{No} in apply_telluric_correction()')
for j in range(Nexp):
T_i = interp.interp1d(wlT[j], fxT[j],
fill_value="extrapolate")(wl[j])
postest(T_i,
f'T-spec of exposure {j} in apply_telluric_correction()')
nantest(T_i,
f'T-spec of exposure {j} in apply_telluric_correction()')
order_cor[j] = order[j] / T_i
error_cor[j] = error[
j] / T_i #I checked that this works because the SNR before and after
#telluric correction is identical.
return (order_cor, error_cor)
def write_mask_to_file(dp,maskname,list_of_masks_auto,list_of_masks_manual=[]):
import sys
from pathlib import Path
import pickle
import tayph.util as ut
from tayph.vartests import typetest,dimtest,lentest
import pdb
import numpy as np
ut.check_path(dp)
typetest(maskname,str,'maskname in write_mask_to_file()')
typetest(list_of_masks_auto,list,'list_of_masks_auto in write_mask_to_file()')
typetest(list_of_masks_manual,list,'list_of_masks_manual in write_mask_to_file()')
lentest(list_of_masks_auto,len(list_of_masks_manual),'list_of_masks_auto in '
'write_mask_to_file()')
for i in range(len(list_of_masks_auto)):
dimtest(list_of_masks_auto[i],np.shape(list_of_masks_manual[i]),'list_of_masks_auto in '
'write_mask_to_file()')
outpath=Path(dp)/maskname
if len(list_of_masks_auto) == 0 and len(list_of_masks_manual) == 0:
print('RuntimeError in write_mask_to_file: Both lists of masks are emtpy!')
sys.exit()
print(f'---Saving lists of auto and manual masks to {str(outpath)}')
if len(list_of_masks_auto) > 0:
with open(str(outpath)+'_auto.pkl', 'wb') as f: pickle.dump(list_of_masks_auto,f)
# ut.save_stack(str(outpath)+'_auto.fits',list_of_masks_auto)
if len(list_of_masks_manual) > 0:
with open(str(outpath)+'_manual.pkl', 'wb') as f: pickle.dump(list_of_masks_manual,f)
def get_model(name, library='models/library', root='models'):
"""This program queries a model from a library file, with predefined models
for use in model injection, cross-correlation and plotting. These models have
a standard format. They are 2 rows by N points, where N corresponds to the
number of wavelength samples. The first row contains the wavelengths, the
second the associated flux values. The library file has two columns that are
formatted as follows:
modelname modelpath
modelname modelpath
modelname modelpath
modelpath starts in the models/ subdirectory.
Set the root variable if a location is required other than the ./models directory.
Example call:
wlm,fxm = get_model('WASP-121_TiO',library='models/testlib')
"""
from tayph.vartests import typetest, dimtest
from tayph.util import check_path
from astropy.io import fits
from pathlib import Path
import errno
import os
typetest(name, str, 'name in mod.get_model()')
library = check_path(library, exists=True)
#First open the library file.
f = open(library, 'r')
x = f.read().splitlines() #Read everything into a big string array.
f.close()
n_lines = len(x) #Number of models in the library.
models = {} #This will contain the model names.
for i in range(0, n_lines):
line = x[i].split()
value = (line[1])
models[line[0]] = value
try:
modelpath = Path(models[name])
except KeyError:
raise KeyError(
f'Model {name} is not present in library at {str(library)}'
) from None
if str(modelpath)[0] != '/': #Test if this is an absolute path.
root = check_path(root, exists=True)
modelpath = root / modelpath
try:
modelarray = fits.getdata(
modelpath
) #Two-dimensional array with wl on the first row and flux on the second row.
except FileNotFoundError:
raise FileNotFoundError(
f'Model file {modelpath} from library {str(library)} does not exist.'
)
dimtest(modelarray, [2, 0])
return (modelarray[0, :], modelarray[1, :])
def save_stack(filename, list_of_2D_frames):
"""This code saves a stack of fits-files to a 3D cube, that you can play
through in DS9. For diagnostic purposes.
Parameters
----------
filename : str, Path
Output filename/path.
list_of_2D-frames : list
A list with 2D arrays
Returns
-------
elapsed : float
The time elapsed since start.
"""
import astropy.io.fits as fits
import numpy as np
import pathlib
from tayph.vartests import typetest
from tayph.vartests import dimtest
import warnings
filename = check_path(filename, 'filename in save_stack()')
typetest(list_of_2D_frames, list,
'list_of_2D_frames in save_stack()') #Test that its a list
typetest(list_of_2D_frames[0], [list, np.ndarray],
'list_of_2D_frames in save_stack()')
for i, f in enumerate(list_of_2D_frames):
typetest(f, [list, np.ndarray],
'frame %s of list_of_2D_frames in save_stack()' % i)
base = np.shape(list_of_2D_frames[0])
N = len(list_of_2D_frames)
dimtest(base, [2], 'shape of list_of_2D_frames in save_stack()'
) #Test that its 2-dimensional
for i, f in enumerate(list_of_2D_frames):
dimtest(f,
base,
varname='frame %s of list_of_2D_frames in save_stack()' %
i) #Test that all have the same shape.
N = len(list_of_2D_frames)
if N > 0:
out = np.zeros((base[0], base[1], N))
for i in range(N):
out[:, :, i] = list_of_2D_frames[i]
fits.writeto(filename,
np.swapaxes(np.swapaxes(out, 2, 0), 1, 2),
overwrite=True)
else:
warnings.warn(
"List_of_2D_frames has length zero. No output was generated by save_stack().",
RuntimeWarning)
def bin(x, y, n, err=[]):
"""
A simple function to quickly bin a spectrum by a certain number of points.
Parameters
----------
x : list, np.ndarray
The horizontal axis.
y : list, np.ndarray
The array corresponding to x.
n : int, float
The number of points by which to bin. Int is recommended, and it is converted
to int if a float is provided, with the consequence of the rounding that int() does.
err : list, np.array, optional
If set, errors corresponding to the flux points.
Returns
-------
x_bin : np.array
The binned x-axis.
y_bin : np.array
The binned y-axis.
e_bin : np.array
Only if err is set to an array with non-zero length.
"""
from tayph.vartests import typetest, dimtest, minlength
import numpy as np
typetest(x, [list, np.ndarray], 'x in ops.bin()')
typetest(y, [list, np.ndarray], 'y in ops.bin()')
typetest(n, [int, float], 'n in ops.bin()')
dimtest(x, [0], 'x in ops.bin()')
dimtest(y, [len(x)], 'y in ops.bin()')
minlength(x, 0, 'x in ops.bin()')
minlength(x, 100, 'x in ops.bin()', warning_only=True)
x_bin = []
y_bin = []
et = False
if len(err) > 0:
e_bin = []
et = True
for i in range(0, int(len(x) - n - 1), n):
x_bin.append(np.nanmean(x[i:i + n]))
y_bin.append(np.nanmean(y[i:i + n]))
if et:
e_bin.append(
np.sqrt(np.nansum(err[i:i + n]**2)) / len(err[i:i + n]))
if et:
return (np.array(x_bin), np.array(y_bin), np.array(e_bin))
else:
return (np.array(x_bin), np.array(y_bin))
def running_MAD_2D(z,w):
"""Computers a running standard deviation of a 2-dimensional array z.
The stddev is evaluated over the vertical block with width w pixels.
The output is a 1D array with length equal to the width of z."""
import astropy.stats as stats
import numpy as np
from tayph.vartests import typetest,dimtest,postest
typetest(z,np.ndarray,'z in fun.running_MAD_2D()')
dimtest(z,[0,0],'z in fun.running_MAD_2D()')
typetest(w,[int,float],'w in fun.running_MAD_2D()')
postest(w,'w in fun.running_MAD_2D()')
size = np.shape(z)
ny = size[0]
nx = size[1]
s = findgen(nx)*0.0
for i in range(nx):
minx = max([0,i-int(0.5*w)])
maxx = min([nx-1,i+int(0.5*w)])
s[i] = stats.mad_std(z[:,minx:maxx],ignore_nan=True)
return(s)
def write_mask_to_file(dp,maskname,list_of_masks_auto,list_of_masks_manual=[]):
import sys
from pathlib import Path
import tayph.util as ut
from tayph.vartests import typetest,dimtest
import pdb
ut.check_path(dp)
typetest(maskname,str,'maskname in write_mask_to_file()')
typetest(list_of_masks_auto,list,'list_of_masks_auto in write_mask_to_file()')
typetest(list_of_masks_manual,list,'list_of_masks_manual in write_mask_to_file()')
dimtest(list_of_masks_auto,[len(list_of_masks_manual),0,0],'list_of_masks_auto in write_mask_to_file()')
outpath=Path(dp)/maskname
if len(list_of_masks_auto) == 0 and len(list_of_masks_manual) == 0:
print('RuntimeError in write_mask_to_file: Both lists of masks are emtpy!')
sys.exit()
print(f'---Saving lists of auto and manual masks to {str(outpath)}')
if len(list_of_masks_auto) > 0:
ut.save_stack(str(outpath)+'_auto.fits',list_of_masks_auto)
if len(list_of_masks_manual) > 0:
ut.save_stack(str(outpath)+'_manual.fits',list_of_masks_manual)
def read_e2ds(inpath,
outname,
config,
nowave=False,
molecfit=False,
mode='HARPS',
ignore_exp=[]):
"""This is the workhorse for reading in a time-series of archival 2D echelle
spectra and formatting these into the order-wise FITS format that Tayph uses.
The user should point this script to a folder (located at inpath) that contains
their pipeline-reduced echelle spectra. The script expects a certain data
format, depending on the instrument in question. It is designed to accept
pipeline products of the HARPS, HARPS-N, ESPRESSO and UVES instruments. In the
case of HARPS, HARPS-N and ESPRESSO these may be downloaded from the archive.
UVES is a bit special, because the 2D echelle spectra are not a standard
pipeline output. Typical use cases are explained further below.
This script formats the time series of 2D echelle spectra into 2D FITS images,
where each FITS file is the time-series of a single echelle order. If the
spectrograph has N orders, an order spans npx pixels, and M exposures
were taken during the time-series, there will be N FITS files, each measuring
M rows by npx columns. This script will read the headers of the pipeline
reduced data files to determine the date/time of each, the exposure time, the
barycentric correction (without applying it) and the airmass, and writes
these to an ASCII table along with the FITS files containing the spectral
orders.
A crucial functionality of this script is that it also acts as a wrapper
for the Molecfit telluric correction software. If installed properly, the
user can call this script with the molecfit keyword to let Molecfit loop
over the entire timeseries. To enable this functionality, the script
reads the full-width, 1D spectra that are output by the instrument pipelines
as well. Molecfit is applied to this time-series of 1D spectra, creating a
time-series of models of the telluric absorption spectrum that is saved along
with the 2D fits files. Tayph later interpolates these models onto the 2D
spectra. Molecfit is called once in GUI-mode, allowing the user to select the
relevant fitting regions and parameters, after which it is repeated
automatically for the entire time series.
Without Molecfit, this script finishes in a matter of seconds. However with
molecfit enabled, it can take many hours (so if I wish to telluric-correct
my data, I run this script overnight).
The processing of HARPS, HARPS-N and ESPRESSO data is executed in an almost
identical manner, because the pipeline-reduced products are almost identical.
To run on either of these instruments, the user simply downloads all pipeline
products of a given time-series, and extracts these in the same folder (meaning
ccfs, e2ds/s2d, s1d, blaze, wave files, etc.) This happens to be the standard
format when downloading pipeline-reduced data from the archive.
For UVES, the functionality is much more constricted because the pipeline
reduced data in the ESO archive is generally not of sufficient stability to
enable precise time-resolved spectroscopy. I designed this function therefore
to run on the pipeline-products produced by the Reflex (GUI) software. For this,
a user should download the raw UVES data of their time series, letting ESO's
calselector tool find the associated calibration files. This can easily be
many GBs worth of data for a given observing program. The user should then
reduce these data with the Reflex software. Reflex creates resampled, stitched
1D spectra as its primary output. However, we will elect to use the intermediate
pipeline products, which include the 2D extracted orders, located in Reflex's
working directory after the reduction process is completed.
A further complication of UVES data is that it can be used with different
dichroics and 'arms', leading to spectral coverage on the blue, redu and/or redl
chips. The user should take care that their time series contains only one of these
types at any time. If they are mixed, this script will throw an exception.
Set the nowave keyword to True if the dataset is HARPS or HARPSN, but it has
no wave files associated with it. This may happen if you downloaded ESO
Advanced Data Products, which include reduced science e2ds's but not reduced
wave e2ds's. The wavelength solution is still encoded in the fits header however,
so we take it from there, instead.
Set the ignore_exp keyword to a list of exposures (start counting at 0) that
need to be ignored when reading, e.g. because they are bad for some reason.
If you have set molecfit to True, this becomes an expensive parameter to
play with in terms of computing time, so its better to figure out which
exposures you'd wish to ignore first (by doing most of your analysis),
before actually running Molecfit, which is icing on the cake in many use-
cases in the optical.
The config parameter points to a configuration file (usually your generic
run definition file) that is only used to point the Molecfit wrapper to the
Molecfit installation on your system. If you are not using molecfit, you may
pass an empty string here.
"""
import os
import pdb
from astropy.io import fits
import astropy.constants as const
import numpy as np
import matplotlib.pyplot as plt
import sys
import tayph.util as ut
from tayph.vartests import typetest, dimtest
import tayph.tellurics as mol
import tayph.system_parameters as sp
import tayph.functions as fun
import copy
import scipy.interpolate as interp
import pickle
from pathlib import Path
import warnings
import glob
# molecfit = False
#First check the input:
inpath = ut.check_path(inpath, exists=True)
typetest(outname, str, 'outname in read_HARPS_e2ds()')
typetest(nowave, bool, 'nowave switch in read_HARPS_e2ds()')
typetest(molecfit, bool, 'molecfit switch in read_HARPS_e2ds()')
typetest(ignore_exp, list, 'ignore_exp in read_HARPS_e2ds()')
typetest(mode, str, 'mode in read_HARPS_e2ds()')
if molecfit:
config = ut.check_path(config, exists=True)
if mode not in [
'HARPS', 'HARPSN', 'HARPS-N', 'ESPRESSO', 'UVES-red', 'UVES-blue'
]:
raise ValueError(
"in read_HARPS_e2ds: mode needs to be set to HARPS, HARPSN, UVES-red, UVES-blue or ESPRESSO."
)
filelist = os.listdir(
inpath
) #If mode == UVES, these are folders. Else, they are fits files.
N = len(filelist)
if len(filelist) == 0:
raise FileNotFoundError(
f" in read_e2ds: input folder {str(inpath)} is empty.")
#The following variables define lists in which all the necessary data will be stored.
framename = []
header = []
s1dhdr = []
type = []
texp = np.array([])
date = []
mjd = np.array([])
s1dmjd = np.array([])
npx = np.array([])
norders = np.array([])
e2ds = []
s1d = []
wave1d = []
airmass = np.array([])
berv = np.array([])
wave = []
blaze = []
wavefile_used = []
outpath = Path('data/' + outname)
if os.path.exists(outpath) != True:
os.makedirs(outpath)
e2ds_count = 0
sci_count = 0
wave_count = 0
blaze_count = 0
s1d_count = 0
if mode == 'HARPS-N': mode = 'HARPSN'
#MODE SWITCHING HERE:
if mode in ['HARPS', 'UVES-red', 'UVES-blue']:
catkeyword = 'HIERARCH ESO DPR CATG'
bervkeyword = 'HIERARCH ESO DRS BERV'
thfilekeyword = 'HIERARCH ESO DRS CAL TH FILE'
Zstartkeyword = 'HIERARCH ESO TEL AIRM START'
Zendkeyword = 'HIERARCH ESO TEL AIRM END'
if mode == 'HARPSN':
catkeyword = 'OBS-TYPE'
bervkeyword = 'HIERARCH TNG DRS BERV'
thfilekeyword = 'HIERARCH TNG DRS CAL TH FILE'
Zstartkeyword = 'AIRMASS'
Zendkeyword = 'AIRMASS' #These are the same because HARPSN doesnt have start and end keywords.
#Down there, the airmass is averaged, so there is no problem in taking the average of the same number.
#Here is the actual parsing of the list of files that were read above. The
#behaviour is different depending on whether this is HARPS, UVES or ESPRESSO
#data, so it switches with a big if-statement in which there is a forloop
#over the filelist in each case. The result is lists or np.arrays containing
#the 2D spectra, the 1D spectra, their 2D and 1D wavelength solutions, the
#headers, the MJDs, the BERVs and the airmasses, as well as (optionally) CCFs
#and blaze files, though these are not really used.
print(f'Read_e2ds is attempting to read a {mode} datafolder.')
if mode == 'UVES-red' or mode == 'UVES-blue': #IF we are UVES-like
for i in range(N):
print(filelist[i])
if (inpath / Path(filelist[i])).is_dir():
tmp_products = [
i for i in (
inpath /
Path(filelist[i])).glob('resampled_science_*.fits')
]
tmp_products1d = [
i for i in (inpath /
Path(filelist[i])).glob('red_science_*.fits')
]
if mode == 'UVES-red' and len(tmp_products) != 2:
raise ValueError(
f"in read_e2ds: When mode=UVES-red there should be 2 resampled_science files (redl and redu), but {len(tmp_products)} were detected."
)
if mode == 'UVES-blue' and len(tmp_products) != 1:
raise ValueError(
f"in read_e2ds: When mode=UVES-rblue there should be 1 resampled_science files (blue), but {len(tmp_products)} were detected."
)
if mode == 'UVES-red' and len(tmp_products1d) != 2:
raise ValueError(
f"in read_e2ds: When mode=UVES-red there should be 2 red_science files (redl and redu), but {len(tmp_products1d)} were detected."
)
if mode == 'UVES-blue' and len(tmp_products1d) != 1:
raise ValueError(
f"in read_e2ds: When mode=UVES-rblue there should be 1 red_science files (blue), but {len(tmp_products1d)} were detected."
)
data_combined = [
] #This will store the two chips (redu and redl) in case of UVES_red, or simply the blue chip if otherwise.
wave_combined = []
wave1d_combined = []
data1d_combined = []
norders_tmp = 0
for tmp_product in tmp_products:
hdul = fits.open(tmp_product)
data = copy.deepcopy(hdul[0].data)
hdr = hdul[0].header
hdul.close()
del hdul[0].data
if not hdr[
'HIERARCH ESO PRO SCIENCE']: #Only add if it's actually a science product:#I force the user to supply only science exposures in the input folder. No BS allowed... UVES is hard enough as it is.
raise ValueError(
f' in read_e2ds: UVES file {tmp_product} is not classified as a SCIENCE file, but should be. Remove it from the folder?'
)
wavedata = ut.read_wave_from_e2ds_header(hdr, mode='UVES')
data_combined.append(data)
wave_combined.append(wavedata)
norders_tmp += np.shape(data)[0]
for tmp_product in tmp_products1d:
hdul = fits.open(tmp_product)
data_1d = copy.deepcopy(hdul[0].data)
hdr1d = hdul[0].header
hdul.close()
del hdul[0].data
if not hdr1d[
'HIERARCH ESO PRO SCIENCE']: #Only add if it's actually a science product:#I force the user to supply only science exposures in the input folder. No BS allowed... UVES is hard enough as it is.
raise ValueError(
f' in read_e2ds: UVES file {tmp_product} is not classified as a SCIENCE file, but should be. Remove it from the folder?'
)
npx1d = hdr1d['NAXIS1']
wavedata = fun.findgen(
npx1d) * hdr1d['CDELT1'] + hdr1d['CRVAL1']
data1d_combined.append(data_1d)
wave1d_combined.append(wavedata)
if len(data_combined) < 1 or len(
data_combined
) > 2: #Double-checking that length here...
raise ValueError(
f'in read_e2ds(): Expected 1 or 2 chips, but {len(data_combined)} files were somehow read.'
)
#The chips generally don't give the same size. Therefore I will pad the smaller one with NaNs to make it fit:
if len(data_combined) != len(data1d_combined):
raise ValueError(
f'in read_e2ds(): The number of chips in the 1d and 2d spectra is not the same {len(data1d_combined)} vs {len(data_combined)}.'
)
if len(data_combined) == 2:
chip1 = data_combined[0]
chip2 = data_combined[1]
wave1 = wave_combined[0]
wave2 = wave_combined[1]
npx_1 = np.shape(chip1)[1]
npx_2 = np.shape(chip2)[1]
no_1 = np.shape(chip1)[0]
no_2 = np.shape(chip2)[0]
npx_max = np.max([npx_1, npx_2])
npx_min = np.min([npx_1, npx_2])
diff = npx_max - npx_min
#Pad the smaller one with NaNs to match the wider one:
if npx_1 < npx_2:
chip1 = np.hstack(
[chip1, np.zeros((no_1, diff)) * np.nan])
wave1 = np.hstack(
[wave1, np.zeros((no_1, diff)) * np.nan])
else:
chip2 = np.hstack(
[chip2, np.zeros((no_2, diff)) * np.nan])
wave2 = np.hstack(
[wave2, np.zeros((no_2, diff)) * np.nan])
#So now they can be stacked:
e2ds_stacked = np.vstack((chip1, chip2))
npx = np.append(npx, np.shape(e2ds_stacked)[1])
e2ds.append(e2ds_stacked)
wave.append(np.vstack((wave1, wave2)))
chip1_1d = data1d_combined[0]
chip2_1d = data1d_combined[1]
wave1_1d = wave1d_combined[0]
wave2_1d = wave1d_combined[1]
if np.nanmean(wave1_1d) < np.nanmean(wave2_1d):
combined_data_1d = np.concatenate((chip1_1d, chip2_1d))
combined_wave_1d = np.concatenate((wave1_1d, wave2_1d))
else:
combined_data_1d = np.concatenate((chip2_1d, chip1_1d))
combined_wave_1d = np.concatenate((wave2_1d, wave1_1d))
wave1d.append(combined_wave_1d)
s1d.append(combined_data_1d)
else:
e2ds.append(data_combined[0])
wave.append(wave_combined[0])
npx = np.append(npx, np.shape(data_combined[0])[1])
wave1d.append(wave1d_combined[0])
s1d.append(data1d_combined[0])
#Only using the keyword from the second header in case of redl,redu.
s1dmjd = np.append(s1dmjd, hdr1d['MJD-OBS'])
framename.append(hdr['ARCFILE'])
header.append(hdr)
type.append('SCIENCE')
texp = np.append(texp, hdr['EXPTIME'])
date.append(hdr['DATE-OBS'])
mjd = np.append(mjd, hdr['MJD-OBS'])
norders = np.append(norders, norders_tmp)
airmass = np.append(
airmass, 0.5 * (hdr[Zstartkeyword] + hdr[Zendkeyword])
) #This is an approximation where we take the mean airmass.
berv_i = sp.calculateberv(hdr['MJD-OBS'],
hdr['HIERARCH ESO TEL GEOLAT'],
hdr['HIERARCH ESO TEL GEOLON'],
hdr['HIERARCH ESO TEL GEOELEV'],
hdr['RA'], hdr['DEC'])
berv = np.append(berv, berv_i)
hdr1d[
'HIERARCH ESO QC BERV'] = berv_i #Append the berv here using the ESPRESSO berv keyword, so that it can be used in molecfit later.
s1dhdr.append(hdr1d)
sci_count += 1
s1d_count += 1
e2ds_count += 1
elif mode == 'ESPRESSO':
catkeyword = 'EXTNAME'
bervkeyword = 'HIERARCH ESO QC BERV'
airmass_keyword1 = 'HIERARCH ESO TEL'
airmass_keyword2 = ' AIRM '
airmass_keyword3_start = 'START'
airmass_keyword3_end = 'END'
for i in range(N):
if filelist[i].endswith('S2D_A.fits'):
e2ds_count += 1
print(filelist[i])
hdul = fits.open(inpath / filelist[i])
data = copy.deepcopy(hdul[1].data)
hdr = hdul[0].header
hdr2 = hdul[1].header
wavedata = copy.deepcopy(hdul[5].data)
hdul.close()
del hdul[1].data
if hdr2[catkeyword] == 'SCIDATA':
print('science keyword found')
framename.append(filelist[i])
header.append(hdr)
type.append('SCIENCE')
texp = np.append(texp, hdr['EXPTIME'])
date.append(hdr['DATE-OBS'])
mjd = np.append(mjd, hdr['MJD-OBS'])
npx = np.append(npx, hdr2['NAXIS1'])
norders = np.append(norders, hdr2['NAXIS2'])
e2ds.append(data)
sci_count += 1
berv = np.append(berv, hdr[bervkeyword] * 1000.0)
telescope = hdr['TELESCOP'][-1]
airmass = np.append(
airmass, 0.5 *
(hdr[airmass_keyword1 + telescope + ' AIRM START'] +
hdr[airmass_keyword1 + telescope + ' AIRM END']))
wave.append(wavedata * (1.0 -
(hdr[bervkeyword] * u.km / u.s /
const.c).decompose().value))
#Ok.! So unlike HARPS, ESPRESSO wavelengths are BERV corrected in the S2Ds.
#WHY!!!?. WELL SO BE IT. IN ORDER TO HAVE E2DSes THAT ARE ON THE SAME GRID, AS REQUIRED, WE UNDO THE BERV CORRECTION HERE.
#WHEN COMPARING WAVE[0] WITH WAVE[1], YOU SHOULD SEE THAT THE DIFFERENCE IS NILL.
#THATS WHY LATER WE JUST USE WAVE[0] AS THE REPRESENTATIVE GRID FOR ALL.
if filelist[i].endswith('CCF_A.fits'):
#ccf,hdr=fits.getdata(inpath+filelist[i],header=True)
hdul = fits.open(inpath / filelist[i])
ccf = copy.deepcopy(hdul[1].data)
hdr = hdul[0].header
hdr2 = hdul[1].header
hdul.close()
del hdul[1].data
if hdr2[catkeyword] == 'SCIDATA':
print('CCF ADDED')
#ccftotal+=ccf
ccfs.append(ccf)
ccfmjd = np.append(ccfmjd, hdr['MJD-OBS'])
nrv = np.append(nrv, hdr2['NAXIS1'])
ccf_count += 1
if filelist[i].endswith('S1D_A.fits'):
hdul = fits.open(inpath / filelist[i])
data_table = copy.deepcopy(hdul[1].data)
hdr = hdul[0].header
hdr2 = hdul[1].header
hdul.close()
del hdul[1].data
if hdr['HIERARCH ESO PRO SCIENCE'] == True:
s1d.append(data_table.field(2))
wave1d.append(data_table.field(1))
s1dhdr.append(hdr)
s1dmjd = np.append(s1dmjd, hdr['MJD-OBS'])
s1d_count += 1
else: #IF we are HARPS-like:
for i in range(N):
if filelist[i].endswith('e2ds_A.fits'):
e2ds_count += 1
print(filelist[i])
hdul = fits.open(inpath / filelist[i])
data = copy.deepcopy(hdul[0].data)
hdr = hdul[0].header
hdul.close()
del hdul[0].data
if hdr[catkeyword] == 'SCIENCE':
framename.append(filelist[i])
header.append(hdr)
type.append(hdr[catkeyword])
texp = np.append(texp, hdr['EXPTIME'])
date.append(hdr['DATE-OBS'])
mjd = np.append(mjd, hdr['MJD-OBS'])
npx = np.append(npx, hdr['NAXIS1'])
norders = np.append(norders, hdr['NAXIS2'])
e2ds.append(data)
sci_count += 1
berv = np.append(berv, hdr[bervkeyword])
airmass = np.append(
airmass, 0.5 * (hdr[Zstartkeyword] + hdr[Zendkeyword])
) #This is an approximation where we take the mean airmass.
if nowave == True:
#Record which wavefile was used by the pipeline to
#create the wavelength solution.
wavefile_used.append(hdr[thfilekeyword])
wavedata = ut.read_wave_from_e2ds_header(hdr,
mode=mode)
wave.append(wavedata)
# else:
# berv=np.append(berv,np.nan)
# airmass=np.append(airmass,np.nan)
if filelist[i].endswith('wave_A.fits'):
print(filelist[i] + ' (wave)')
if nowave == True:
warnings.warn(
" in read_e2ds: nowave was set to True but a wave_A file was detected. This wave file is now ignored in favor of the header.",
RuntimeWarning)
else:
wavedata = fits.getdata(inpath / filelist[i])
wave.append(wavedata)
wave_count += 1
if filelist[i].endswith('blaze_A.fits'):
print(filelist[i] + ' (blaze)')
blazedata = fits.getdata(inpath / filelist[i])
blaze.append(blazedata)
blaze_count += 1
if filelist[i].endswith('s1d_A.fits'):
hdul = fits.open(inpath / filelist[i])
data_1d = copy.deepcopy(hdul[0].data)
hdr = hdul[0].header
hdul.close()
del hdul[0].data
if hdr[catkeyword] == 'SCIENCE':
s1d.append(data_1d)
s1dhdr.append(hdr)
s1dmjd = np.append(s1dmjd, hdr['MJD-OBS'])
s1d_count += 1
#Now we catch some errors:
#-The above should have read a certain number of e2ds files.
#-A certain number of these should be SCIENCE frames.
#-There should be at least one WAVE file.
#-All exposures should have the same number of spectral orders.
#-All orders should have the same number of pixels (this is true for HARPS).
#-The wave frame should have the same dimensions as the order frames.
#-If nowave is set, test that all frames used the same wave_A calibrator.
#-The blaze file needs to have the same shape as the e2ds files.
#-The number of s1d files should be the same as the number of e2ds files.
if e2ds_count == 0:
raise FileNotFoundError(
f"in read_e2ds: The input folder {str(inpath)} does not contain files ending in e2ds.fits."
)
if sci_count == 0:
print('')
print('')
print('')
print("These are the files and their types:")
for i in range(len(type)):
print(' ' + framename[i] + ' %s' % type[i])
raise ValueError(
"in read_e2ds: The input folder (%2) contains e2ds files, but none of them are classified as SCIENCE frames with the HIERARCH ESO DPR CATG/OBS-TYPE keyword or HIERARCH ESO PRO SCIENCE keyword. The list of frames is printed above."
)
if np.max(np.abs(norders - norders[0])) == 0:
norders = int(norders[0])
else:
print('')
print('')
print('')
print("These are the files and their number of orders:")
for i in range(len(type)):
print(' ' + framename[i] + ' %s' % norders[i])
raise ValueError(
"in read_e2ds: Not all files have the same number of orders. The list of frames is printed above."
)
if np.max(np.abs(npx - npx[0])) == 0:
npx = int(npx[0])
else:
print('')
print('')
print('')
print("These are the files and their number of pixels:")
for i in range(len(type)):
print(' ' + framename[i] + ' %s' % npx[i])
raise ValueError(
"in read_HARPS_e2ds: Not all files have the same number of pixels. The list of frames is printed above."
)
if wave_count >= 1:
wave = wave[0] #SELECT ONLY THE FIRST WAVE FRAME. The rest is ignored.
else:
if nowave == False and mode not in ['UVES-red', 'UVES-blue']:
print('')
print('')
print('')
print("ERROR in read_e2ds: No wave_A.fits file was detected.")
print("These are the files in the folder:")
for i in range(N):
print(filelist[i])
print(
"This may have happened if you downloaded the HARPS data from the"
)
print(
"ADP query form, which doesn't include wave_A files (as far as I"
)
print(
"have seen). Set the /nowave keyword in your call to read_HARPS_e2ds"
)
print("if you indeed do not expect a wave_A file to be present.")
raise FileNotFoundError(
"No wave_A.fits file was detected. More details are printed above."
)
if nowave == True and mode not in [
'UVES-red', 'UVES-blue', 'ESPRESSO'
]: #This here is peculiar to HARPS/HARPSN.
if all(x == wavefile_used[0] for x in wavefile_used):
print(
"Nowave is set, and simple wavelength calibration extraction")
print(
"works, as all files in the dataset used the same wave_A file."
)
wave = wave[0]
else:
print('')
print('')
print('')
print("These are the filenames and their wave_A file used:")
for i in range(N - 1):
print(' ' + framename[i] + ' %s' % wavefile_used[0])
warnings.warn(
"in read_e2ds: Nowave is set, but not all files in the dataset used the same wave_A file when the pipeline was run. This script will continue using only the first wavelength solution. Theoretically, this may affect the quality of the data if the solution is wrong (in which case interpolation would be needed), but due to the stability of HARPS this is probably not an issue worth interpolating for.",
RuntimeWarning)
wave = wave[0]
if mode == 'ESPRESSO':
dimtest(wave1d, np.shape(s1d), 'wave and s1d in read_e2ds()')
dimtest(wave, np.shape(e2ds), 'wave and e2ds in read_e2ds()')
diff = wave - wave[0]
if np.abs(np.nanmax(diff)) > (np.nanmin(wave[0]) / 1e6):
warnings.warn(
" in read_e2ds: The wavelength solution over the time series is not constant. I continue by interpolating all the data onto the wavelength solution of the first frame. This will break if the wavelength solutions of your time-series are vastly different, in which case the output will be garbage.",
RuntimeWarning)
for ii in range(len(e2ds)):
if ii > 0:
for jj in range(len(e2ds[ii])):
e2ds[ii][jj] = interp.interp1d(
wave[ii][jj],
e2ds[ii][jj],
fill_value='extrapolate')(wave[0][jj])
wave = wave[0]
diff1d = wave1d - wave1d[0]
if np.abs(np.nanmax(diff1d)) > (
np.nanmin(wave1d) / 1e6
): #if this is true, all the wavelength solutions are the same. Great!
warnings.warn(
" in read_e2ds: The wavelength solution over the time series of the 1D spectra is not constant. I continue by interpolating all the 1D spectra onto the wavelength solution of the first frame. This will break if the wavelength solutions of your time-series are vastly different, in which case the output will be garbage.",
RuntimeWarning)
for ii in range(len(s1d)):
if ii > 0:
s1d[ii] = interp.interp1d(wave1d[ii],
s1d[ii],
fill_value='extrapolate')(
wave1d[0])
wave1d = wave1d[0]
#We are going to test whether the wavelength solution is the same for the
#entire time series in the case of UVES. In normal use cases this should be true.
if mode in ['UVES-red', 'UVES-blue']:
dimtest(wave1d, np.shape(s1d), 'wave and s1d in read_e2ds()')
dimtest(wave, np.shape(e2ds), 'wave and e2ds in read_e2ds()')
diff = wave - wave[0]
if np.abs(
np.nanmax(diff)
) == 0: #if this is true, all the wavelength solutions are the same. Great!
wave = wave[0]
else:
warnings.warn(
" in read_e2ds: The wavelength solution over the time series is not constant. I continue by interpolating all the data onto the wavelength solution of the first frame. This will break if the wavelength solutions of your time-series are vastly different, in which case the output will be garbage.",
RuntimeWarning)
for ii in range(len(e2ds)):
if ii > 0:
for jj in range(len(e2ds[ii])):
e2ds[ii][jj] = interp.interp1d(
wave[ii][jj],
e2ds[ii][jj],
fill_value='extrapolate')(wave[0][jj])
wave = wave[0]
diff1d = wave1d - wave1d[0]
if np.abs(
np.nanmax(diff1d)
) == 0: #if this is true, all the wavelength solutions are the same. Great!
wave1d = wave1d[0]
else:
warnings.warn(
" in read_e2ds: The wavelength solution over the time series of the 1D spectra is not constant. I continue by interpolating all the 1D spectra onto the wavelength solution of the first frame. This will break if the wavelength solutions of your time-series are vastly different, in which case the output will be garbage.",
RuntimeWarning)
for ii in range(len(s1d)):
if ii > 0:
s1d[ii] = interp.interp1d(wave1d[ii],
s1d[ii],
fill_value='extrapolate')(
wave1d[0])
wave1d = wave1d[0]
if blaze_count >= 1: #This will only be triggered for HARPS/HARPSN/ESPRESSO
blaze = blaze[
0] #SELECT ONLY THE FIRST blaze FRAME. The rest is ignored.
if np.shape(wave) != np.shape(
e2ds[0]): #If UVES/ESPRESSO, this was already checked implicitly.
raise ValueError(
f"in read_e2ds: A wave file was detected but its shape ({np.shape(wave)[0]},{np.shape(wave)[1]}) does not match that of the orders ({np.shape(e2ds[0])[0]},{np.shape(e2ds[0])[1]})"
)
if np.shape(blaze) != np.shape(e2ds[0]) and blaze_count > 0:
raise ValueError(
f"in read_e2ds: A blaze file was detected but its shape ({np.shape(blaze)[0]},{np.shape(wave)[1]}) does not match that of the orders ({np.shape(e2ds[0])[0]},{np.shape(e2ds[0])[1]})"
)
if len(s1dhdr) != len(
e2ds
) and molecfit == True: #Only a problem if we actually run Molecfit.
raise ValueError(
f'in read_e2ds: The number of s1d SCIENCE files and e2ds SCIENCE files is not the same. ({len(s1dhdr)} vs {len(e2ds)})'
)
#Ok, so now we should have ended up with a number of lists that contain all
#the relevant science data and associated information.
#We determine how to sort the resulting lists in time:
sorting = np.argsort(mjd)
s1dsorting = np.argsort(s1dmjd)
if len(ignore_exp) > 0:
sorting = [x for i, x in enumerate(sorting) if i not in ignore_exp]
s1dsorting = [
x for i, x in enumerate(s1dsorting) if i not in ignore_exp
]
if mode == 'HARPSN': #In the case of HARPS-N we need to convert the units of the elevation and provide a UTC keyword.
for i in range(len(header)):
s1dhdr[i]['TELALT'] = np.degrees(float(s1dhdr[i]['EL']))
s1dhdr[i]['UTC'] = (float(s1dhdr[i]['MJD-OBS']) % 1.0) * 86400.0
#Sort the s1d files for application of molecfit.
if molecfit == True:
if len(sorting) != len(s1dsorting):
raise ValueError(
"in read_HARPS_e2ds: Sorted science frames and sorted s1d frames are not of the same length. Telluric correction can't proceed."
)
s1dhdr_sorted = []
s1d_sorted = []
for i in range(len(s1dsorting)):
s1dhdr_sorted.append(s1dhdr[s1dsorting[i]])
s1d_sorted.append(s1d[s1dsorting[i]])
print("")
print("")
print("")
print(
'Molecfit will be executed onto the files with dates in this order:'
)
for x in s1dhdr_sorted:
print(x['DATE-OBS'])
print("")
print("")
print("")
if mode in ['ESPRESSO', 'UVES-red', 'UVES-blue']:
list_of_wls, list_of_trans = mol.do_molecfit(s1dhdr_sorted,
s1d_sorted,
config,
load_previous=False,
mode=mode,
wave=wave1d)
else:
list_of_wls, list_of_trans = mol.do_molecfit(s1dhdr_sorted,
s1d_sorted,
config,
load_previous=False,
mode=mode)
if len(list_of_trans) != len(sorting):
raise ValueError(
"in read_e2ds(): Molecfit did not produce the same number of spectra as there are in the e2ds spectra."
)
mol.write_telluric_transmission_to_file(
list_of_wls, list_of_trans,
outpath / 'telluric_transmission_spectra.pkl')
#Now we loop over all exposures and collect the i-th order from each exposure,
#put these into a new matrix and save them to FITS images:
f = open(outpath / 'obs_times', 'w', newline='\n')
headerline = 'MJD' + '\t' + 'DATE' + '\t' + 'EXPTIME' + '\t' + 'MEAN AIRMASS' + '\t' + 'BERV (km/s)' + '\t' + 'FILE NAME'
for i in range(norders):
order = np.zeros((len(sorting), npx))
wave_axis = wave[i, :] / 10.0 #Convert to nm.
print('CONSTRUCTING ORDER %s' % i)
c = 0 #To count the number of science frames that have passed. The counter
# c is not equal to j because the list of files contains not only SCIENCE
# frames.
for j in range(len(sorting)): #Loop over exposures
if i == 0:
print('---' + type[sorting[j]] + ' ' + date[sorting[j]])
if type[sorting[j]] == 'SCIENCE': #This check may be redundant.
exposure = e2ds[sorting[j]]
order[c, :] = exposure[i, :]
#T_i = interp.interp1d(list_of_wls[j],list_of_trans[j])#This should be time-sorted, just as the e2ds files.
#Do a manual check here that the MJDs are identical.
#Also, determiine what to do with airtovac.
#tel_order[c,:] = T_i[wave_axis]
#Now I also need to write it to file.
if i == 0: #Only do it the first time, not for every order.
line = str(mjd[sorting[j]]) + '\t' + date[sorting[
j]] + '\t' + str(texp[sorting[j]]) + '\t' + str(
np.round(airmass[sorting[j]], 3)) + '\t' + str(
np.round(
berv[sorting[j]],
5)) + '\t' + framename[sorting[j]] + '\n'
f.write(line)
c += 1
fits.writeto(outpath / ('order_' + str(i) + '.fits'),
order,
overwrite=True)
fits.writeto(outpath / ('wave_' + str(i) + '.fits'),
wave_axis,
overwrite=True)
f.close()
print(f'Time-table written to {outpath/"obs_times"}.')
def apply_telluric_correction(inpath, list_of_wls, list_of_orders,
list_of_sigmas):
"""
This applies a set of telluric spectra (computed by molecfit) for each exposure
in our time series that were written to a pickle file by write_telluric_transmission_to_file.
List of errors are provided to propagate telluric correction into the error array as well.
Parameters
----------
inpath : str, path like
The path to the pickled transmission spectra.
list_of_wls : list
List of wavelength axes.
list_of_orders :
List of 2D spectral orders, matching to the wavelength axes in dimensions and in number.
list_of_isgmas :
List of 2D error matrices, matching dimensions and number of list_of_orders.
Returns
-------
list_of_orders_corrected : list
List of 2D spectral orders, telluric corrected.
list_of_sigmas_corrected : list
List of 2D error matrices, telluric corrected.
"""
import scipy.interpolate as interp
import numpy as np
import tayph.util as ut
import tayph.functions as fun
from tayph.vartests import dimtest, postest, typetest, nantest
wlT, fxT = read_telluric_transmission_from_file(inpath)
typetest(list_of_wls, list, 'list_of_wls in apply_telluric_correction()')
typetest(list_of_orders, list,
'list_of_orders in apply_telluric_correction()')
typetest(list_of_sigmas, list,
'list_of_sigmas in apply_telluric_correction()')
typetest(wlT, list,
'list of telluric wave-axes in apply_telluric_correction()')
typetest(
fxT, list,
'list of telluric transmission spectra in apply_telluric_correction()')
No = len(list_of_wls)
x = fun.findgen(No)
if No != len(list_of_orders):
raise Exception(
'Runtime error in telluric correction: List of data wls and List of orders do not have the same length.'
)
Nexp = len(wlT)
if Nexp != len(fxT):
raise Exception(
'Runtime error in telluric correction: List of telluric wls and telluric spectra read from file do not have the same length.'
)
if Nexp != len(list_of_orders[0]):
raise Exception(
f'Runtime error in telluric correction: List of telluric spectra and data spectra read from file do not have the same length ({Nexp} vs {len(list_of_orders[0])}).'
)
list_of_orders_cor = []
list_of_sigmas_cor = []
# ut.save_stack('test.fits',list_of_orders)
# pdb.set_trace()
for i in range(No): #Do the correction order by order.
order = list_of_orders[i]
order_cor = order * 0.0
error = list_of_sigmas[i]
error_cor = error * 0.0
wl = list_of_wls[i]
dimtest(order, [0, len(wl)],
f'order {i}/{No} in apply_telluric_correction()')
dimtest(error, np.shape(order),
f'errors {i}/{No} in apply_telluric_correction()')
for j in range(Nexp):
T_i = interp.interp1d(wlT[j], fxT[j], fill_value="extrapolate")(wl)
postest(T_i,
f'T-spec of exposure {j} in apply_telluric_correction()')
nantest(T_i,
f'T-spec of exposure {j} in apply_telluric_correction()')
order_cor[j] = order[j] / T_i
error_cor[j] = error[
j] / T_i #I checked that this works because the SNR before and after telluric correction is identical.
list_of_orders_cor.append(order_cor)
list_of_sigmas_cor.append(error_cor)
ut.statusbar(i, x)
return (list_of_orders_cor, list_of_sigmas_cor)
def inject_model(list_of_wls,
list_of_orders,
dp,
modelname,
model_library='library/models'):
"""This function takes a list of spectral orders and injects a model with library
identifier modelname, and system parameters as defined in dp. The model is blurred taking into
account spectral resolution and rotation broadening (with an LSF as per Brogi et al.) and
finite-exposure broadening (with a box LSF).
It returns a copy of the list of orders with the model injected."""
import tayph.util as ut
import tayph.system_parameters as sp
import tayph.models
import astropy.constants as const
import numpy as np
import scipy
import tayph.operations as ops
from tayph.vartests import typetest, dimtest
import pdb
import copy
import matplotlib.pyplot as plt
# dimtest(order,[0,len(wld)])
dp = ut.check_path(dp)
typetest(modelname, str, 'modelname in models.inject_model()')
typetest(model_library, str, 'model_library in models.inject_model()')
c = const.c.to('km/s').value #In km/s
Rd = sp.paramget('resolution', dp)
planet_radius = sp.paramget('Rp', dp)
inclination = sp.paramget('inclination', dp)
P = sp.paramget('P', dp)
transit = sp.transit(dp)
n_exp = len(transit)
vsys = sp.paramget('vsys', dp)
rv = sp.RV(dp) + vsys
dRV = sp.dRV(dp)
phi = sp.phase(dp)
dimtest(transit, [n_exp])
dimtest(rv, [n_exp])
dimtest(phi, [n_exp])
dimtest(dRV, [n_exp])
mask = (transit - 1.0) / (np.min(transit - 1.0))
wlm, fxm = get_model(modelname, library=model_library)
if wlm[-1] <= wlm[0]: #Reverse the wl axis if its sorted the wrong way.
wlm = np.flipud(wlm)
fxm = np.flipud(fxm)
#With the model and the revelant parameters in hand, now only select that
#part of the model that covers the wavelengths of the order provided.
#A larger wavelength range would take much extra time because the convolution
#is a slow operation.
N_orders = len(list_of_wls)
if N_orders != len(list_of_orders):
raise RuntimeError(
f'in models.inject_model: List_of_wls and list_of_orders are not of the '
f'same length ({N_orders} vs {len(list_of_orders)})')
if np.min(wlm) > np.min(list_of_wls) - 1.0 or np.max(
wlm) < np.max(list_of_wls) + 1.0:
ut.tprint(
'WARNING in model injection: Data grid falls (partly) outside of model range. '
'Setting missing area to 1.0. (meaning, no planet absorption.)')
modelsel = [
(wlm >= np.min(list_of_wls) - 1.0) & (wlm <= np.max(list_of_wls) + 1.0)
]
wlm = wlm[tuple(modelsel)]
fxm = fxm[tuple(modelsel)]
fxm_b = ops.blur_rotate(wlm, fxm, c / Rd, planet_radius, P,
inclination) #Only do this once per dataset.
oversampling = 2.5
wlm_cv, fxm_bcv, vstep = ops.constant_velocity_wl_grid(
wlm, fxm_b, oversampling=oversampling)
if np.min(dRV) < c / Rd / 10.0:
dRV_min = c / Rd / 10.0 #If the minimum dRV is less than 10% of the spectral
#resolution, we introduce a lower limit to when we are going to blur, because the effect
#becomes insignificant.
else:
dRV_min = np.min(dRV)
if dRV_min / vstep < 3: #Meaning, if the smoothing is going to be undersampled by this choice
#in v_step, it means that the oversampling parameter in ops.constant_velocity_wl_grid was
#not high enough. Then we adjust it. I choose a value of 3 here to be safe, even though
#ops.smooth below accepts values as low as 2.
oversampling_new = 3.0 / (
dRV_min / vstep) * oversampling #scale up the oversampling factor.
wlm_cv, fxm_bcv, vstep = ops.constant_velocity_wl_grid(
wlm, fxm_b, oversampling=oversampling_new)
list_of_orders_injected = copy.deepcopy(list_of_orders)
for i in range(n_exp):
if dRV[i] >= c / Rd / 10.0:
fxm_b2 = ops.smooth(fxm_bcv, dRV[i] / vstep, mode='box')
else:
fxm_b2 = copy.deepcopy(fxm_bcv)
shift = 1.0 + rv[i] / c
fxm_i = scipy.interpolate.interp1d(
wlm_cv * shift, fxm_b2, fill_value=1.0,
bounds_error=False) #This is a class that can be called.
#Fill_value = 1 because if the model does not fully cover the order, it will be padded with 1.0s,
#assuming that we are dealing with a model that is in transmission.
for j in range(len(list_of_orders)):
list_of_orders_injected[j][i] *= (
1.0 + mask[i] * (fxm_i(list_of_wls[j]) - 1.0)) #This assumes
#that the model is in transit radii. This can definitely be vectorised!
#These are for checking that the broadening worked as expected:
# injection_total[i,:]= scipy.interpolate.interp1d(wlm_cv*shift,fxm_b2)(wld)
# injection_rot_only[i,:]=scipy.interpolate.interp1d(wlm*shift,fxm_b)(wld)
# injection_pure[i,:]=scipy.interpolate.interp1d(wlm*shift,fxm)(wld)
# ut.save_stack('test.fits',[injection_pure,injection_rot_only,injection_total])
# pdb.set_trace()
# ut.writefits('test.fits',injection)
# pdb.set_trace()
return (list_of_orders_injected)
def normalize_orders(list_of_orders, list_of_sigmas, deg=1, nsigma=4):
"""
If deg is set to 1, this function will normalise based on the mean flux in each order.
If set higher, it will remove the average spectrum in each order and fit a polynomial
to the residual. This means that in the presence of spectral lines, the fluxes will be
slightly lower than if def=1 is used. nsigma is only used if deg > 1, and is used to
throw away outliers from the polynomial fit. The program also computes the total
mean flux of each exposure in the time series - totalled over all orders. These
are important to correctly weigh the cross-correlation functions later. The
inter-order colour correction is assumed to be an insignificant modification to
these weights.
Parameters
----------
list_of_orders : list
The list of 2D orders that need to be normalised.
list_of_sigmas : list
The list of 2D error matrices corresponding to the 2D orders that need to be normalised.
deg : int
The polynomial degree to remove. If set to 1, only the average flux is removed. If higher,
polynomial fits are made to the residuals after removal of the average spectrum.
nsigma : int, float
The number of sigmas beyond which outliers are rejected from the polynomial fit.
Only used when deg > 1.
Returns
-------
out_list_of_orders : list
The normalised 2D orders.
out_list_of_sigmas : list
The corresponding errors.
meanfluxes : np.array
The mean flux of each exposure in the time series, averaged over all orders.
"""
import numpy as np
import tayph.functions as fun
from tayph.vartests import dimtest, postest, typetest
import warnings
typetest(list_of_orders, list, 'list_of_orders in ops.normalize_orders()')
typetest(list_of_sigmas, list, 'list_of_sigmas in ops.normalize_orders()')
dimtest(list_of_orders[0], [0, 0]) #Test that the first order is 2D.
dimtest(list_of_sigmas[0],
[0, 0]) #And that the corresponding sigma array is, as well.
n_exp = np.shape(list_of_orders[0])[0] #Get the number of exposures.
for i in range(len(list_of_orders)): #Should be the same for all orders.
dimtest(list_of_orders[i], [n_exp, 0])
dimtest(list_of_sigmas[i], np.shape(list_of_orders[i]))
typetest(deg, int, 'degree in ops.normalize_orders()')
typetest(nsigma, [int, float], 'nsigma in ops.normalize_orders()')
postest(deg, 'degree in ops.normalize_orders()')
postest(nsigma, 'degree in ops.normalize_orders()')
N = len(list_of_orders)
out_list_of_orders = []
out_list_of_sigmas = []
#First compute the exposure-to-exposure flux variations to be used as weights.
meanfluxes = fun.findgen(n_exp) * 0.0
N_i = 0
for i in range(N):
m = np.nanmedian(list_of_orders[i], axis=1) #Median or mean?
if np.sum(np.isnan(m)) > 0:
print(
'---Warning in normalise_orders: Skipping order %s because many nans are present.'
% i)
else:
N_i += 1
meanfluxes += m #These contain the exposure-to-exposure variability of the time-series.
meanfluxes /= N_i #These are the weights.
if deg == 1:
for i in range(N):
#What I'm doing here is probably stupid and numpy division will probably work just fine without
#IDL-relics.
n_px = np.shape(list_of_orders[i])[1]
meanflux = np.nanmedian(
list_of_orders[i],
axis=1) #Average flux in each order. Median or mean?
meanblock = fun.rebinreform(
meanflux / np.nanmean(meanflux), n_px
).T #This is a slow operation. Row-by-row division is better done using a double-transpose...
out_list_of_orders.append(list_of_orders[i] / meanblock)
out_list_of_sigmas.append(list_of_sigmas[i] / meanblock)
else:
for i in range(N):
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=RuntimeWarning)
meanspec = np.nanmean(list_of_orders[i],
axis=0) #Average spectrum in each order.
x = np.array(range(len(meanspec)))
poly_block = list_of_orders[
i] * 0.0 #Array that will host the polynomial fits.
colour = list_of_orders[
i] / meanspec #What if there are zeroes? I.e. padding around the edges of the order?
for j, s in enumerate(list_of_orders[i]):
idx = np.isfinite(colour[j])
if np.sum(idx) > 0:
p = np.poly1d(np.polyfit(x[idx], colour[j][idx], deg))(
x) #Polynomial fit to the colour variation.
res = colour[
j] / p - 1.0 #The residual, which is flat around zero if it's a good fit. This has all sorts of line residuals that we need to throw out.
#We do that using the weight keyword of polyfit, and just set all those weights to zero.
sigma = np.nanstd(res)
w = x * 0.0 + 1.0 #Start with a weight function that is 1.0 everywhere.
with warnings.catch_warnings():
warnings.simplefilter("ignore",
category=RuntimeWarning)
w[np.abs(res) > nsigma * sigma] = 0.0
w = x * 0.0 + 1.0 #Start with a weight function that is 1.0 everywhere.
p2 = np.poly1d(
np.polyfit(x[idx], colour[j][idx], deg, w=w[idx])
)(
x
) #Second, weighted polynomial fit to the colour variation.
poly_block[j] = p2
out_list_of_orders.append(list_of_orders[i] / poly_block)
out_list_of_sigmas.append(list_of_sigmas[i] / poly_block)
return (out_list_of_orders, out_list_of_sigmas, meanfluxes)
def gaussfit(x,y,nparams=3,startparams=None,yerr=None,fixparams=None,minvalues=None,maxvalues=None,verbose=False):
"""
This is a wrapper around lmfit to fit a Gaussian plus a polynomial continuum
using the LM least-squares minimization algorithm. It is designed to be
interacted with in a similar way as IDL's Gaussfit routine, though not
exactly the same.
The user provides data as np.arrays that form x and y pairs. The Gaussian model
has a minimum number of 3 parameters: The amplitude, mean and standard deviation.
In addition, the user can request additional parameters that describe the continuum
as a polynomial with increasing degree. The model is therefore defined as
follows:
y = A*exp(-(x - mu)**2/(sqrt(2)sigma)**2) + c1 + c2*x + c3*x**2 + ...
If initial guesses for the parameters are unknown or omitted, this function
will guess start parameters albeit coarsely. The code will determine whether
the peak is positive or negative by comparing the distance between the maximum and
the median with the distance between the minimum and the median. If the former is larger,
the Gaussian is considered to be positive.
The amplitude is then guessed as the maximum - the minimum of the data y and the
mean is guessed as the location of that maximum. Vice versa if the peak was
established to be a minimum. The standard deviation is estimated as a fifth
of the length of the data array.
The nparams keyword will determine the number of free parameters to fit
(3 for the Gaussian plus however many for the continuum).
If startparams is set, the nparams keyword is overruled; and initial parameters
need to be applied in the order [A,mu,sigma,c1,c2,...].
By default, the least-squares optimizer assumes unweighted data points.
If error bars on y are known, provide them with the yerr keyword (in
the same way as you would supply them to plt.errorbar(x,y,yerr=...)).
Parameter values can also be fixed (if startparams is set) or bounded.
To fix parameters, set fixparams to a list (with same length as startparams)
filled with 0's and 1's. Each parameter that is 1 will be fixed.
For putting minimum and/or maximum bounds on the parameters, similarly
provide lists with the same length as startparams, with the minimum/maximum
of each parameter. To set a bound on only one parameter, the others that you
wish to remain unbounded need to be set to np.inf or -np.inf.
NaNs in y are omitted by the lmfit.minimizer routine. NaNs in x are not accepted.
Lmfit is a powerful package for all of your curvefitting needs, and
includes emcee functionality as well. Do check it out if you wish to fit
models that are more complex than simple Gaussians, or wish to have
access to more advanced statistics: https://lmfit.github.io.
If you want to add more advanced functionality or build variations on this
wrapper, a very useful page is their examples page:
https://lmfit.github.io/lmfit-py/examples/index.html
Parameters
----------
x : np.ndarray
The grid onto which the data is defined. NaNs are not allowed.
y : np.ndarray
The data. NaNs are allowed and will be omitted by the minimizer.
nparams : int, optional.
The number of parameters of the model. The model is defined as above, with
a minimum of 3 parameters.
startparmas : list, optional.
The starting parameters at which the fit is initialized, in the order
[A,mu,sigma,c1,c2,...]. This overrules the nparams keyword.
yerr : np.ndarray, optional.
The standard deviation on each of the values of y, by which the least-squares
minimization will be weighted.
fixparams : list, optional
The parameters to fix, in the form of [0,1,0,1,1]; in the same order as
startparams.
minvalues : list, optional
The minimum bounds of the fitting parameters, in the form of [-np.inf,10,5,-np.inf,-np.inf].
maxvalues : list, optional
The maximum bounds of the fitting parameters, in the form of [np.inf,10,5,np.inf,np.inf].
verbose : bool, optional
If set to True, the lmfit package will talk to you.
Returns
-------
r : list
The best-fit parameters, in the same order as startparams.
re: list
The 1-sigma confidence intervals on the best-fit parameters.
Example
-------
>>> import numpy as np
>>> x=findgen(100)
>>> p=[10,50,6.0,0.0,0.05]
>>> y=gaussian(x,*p)
>>> f,e=gaussfit(x,y,startparams=[8.0,47.0,5.0,0.0,0.0],fixparams=[0,0,0,1,0.0])
"""
import tayph.util as ut
from tayph import typetest,minlength,dimtest,nantest
from lmfit import Minimizer, Parameters, report_fit
import sys
import numpy as np
import matplotlib.pyplot as plt
typetest(x,np.ndarray,'x in fun.gaussfit()')
typetest(y,np.ndarray,'y in fun.gaussfit()')
typetest(nparams,int,'nparams in fun.gaussfit()')
dimtest(x,[0],'x in fun.gaussfit()')
dimtest(y,[len(x)],'y in fun.gaussfit()')
nantest(x,'x in fun.gaussfit()')
typetest(verbose,bool,'verbose in fun.gaussfit()')
if startparams is None:
if fixparams:#If startparams is not set, fixparams would fix the initial guesses which are likely not accurate, so there is no reason to this and I hereby protect the user against doing this to themselves.
raise InputError("in fun.gaussfit(): startparams needs to be provided if you wish to fix any fitting parameters.")
N = nparams
if nparams <3:
raise ValueError(f"in fun.gaussfit: nparams needs to be 3 or greater ({nparams}).")
A = np.nanmax(y)-np.nanmin(y)
if np.abs(np.nanmax(y)-np.median(y)) > np.abs(np.nanmin(y)-np.median(y)):#then we have a positive signal
mu = x[np.nanargmax(y)]
else:#or a negative signal.
mu = x[np.nanargmin(y)]
A*=(-1.0)
sig= np.std(x)/5.0
startparams = [A,mu,sig]
paramnames = ['A','mu','sigma']
if nparams > 3:
for i in range(3,nparams):
paramnames.append(f"c{i-2}")
startparams.append(0.0)
else:
typetest(startparams,[list,np.ndarray],'startparams in fun.gaussfit()')
minlength(startparams,3,varname='startparams in fun.gaussfit()',warning_only=False)
nparams = len(startparams)#Startparams overrides the nparams keyword.
paramnames = ['A','mu','sigma']
if nparams > 3:
for i in range(3,nparams):
paramnames.append(f"c{i-2}")
if isinstance(startparams,list):
startparams=np.array(startparams)
if yerr is not None:
typetest(yerr,np.ndarray,'yerr in fun.gaussfit()')
dimtest(yerr,[len(x)],'yerr in fun.gaussfit()')
nantest(yerr,'yerr in fun.gaussfit()')
weight = 1.0/yerr#Weigh by the variance as per https://en.wikipedia.org/wiki/Weighted_least_squares
#Note that there is no square here, because the minimizer function squares.
else:
weight = x*0.0+1.0#Force weight to 1.0, i.e. unweighted least-sq.
if fixparams is None:
fixparams = startparams*0.0#Set all fix parameters to zero. I.e. none will be fixed.
else:
dimtest(fixparams,[len(startparams)],'fixparams in fun.gaussfit()')
nantest(fixparams,'fixparams in fun.gaussfit()')
if minvalues is None:
minvalues = startparams*0.0-np.inf
minvalues[2] = 0 #stddev needs to be positive.
else:
dimtest(fixparams,[len(startparams)],'minvalues in fun.gaussfit()')
nantest(fixparams,'minvalues in fun.gaussfit()')
if maxvalues is None:
maxvalues = startparams*0.0+np.inf
else:
dimtest(fixparams,[len(startparams)],'maxvalues in fun.gaussfit()')
nantest(fixparams,'maxvalues in fun.gaussfit()')
def fcn2min(params, x, y, w):
"""
Model the gaussian and subtract data. The square of the outcome of this
will be minimized.
"""
A = params['A']
mu = params['mu']
s = params['sigma']
trigger = 0
i=1
p = [A,mu,s]
while trigger == 0:
try:
p.append(params[f'c{i}'])
except KeyError:
trigger = 1
i+=1
model = gaussian(x,*p)
return((model - y)*w)
# pass the Parameters
params = Parameters()
for i in range(len(paramnames)):
params.add(paramnames[i],value=startparams[i])
params[paramnames[i]].min=minvalues[i]#These are +/- np.inf by default
params[paramnames[i]].max=maxvalues[i]#So they will trivially be overwritten by infs if min/maxvalues are not set.
if fixparams[i] == 1:
params[paramnames[i]].vary = False
minner = Minimizer(fcn2min, params, fcn_args=(x, y, weight),nan_policy='omit')
result = minner.minimize()
# write error report
if verbose:
report_fit(result)
# calculate final result
final = y + result.residual
out_res = []
out_err = []
for i in range(len(paramnames)):
out_res.append(result.params[paramnames[i]].value)
out_err.append(result.params[paramnames[i]].stderr)
return(out_res,out_err)
def apply_telluric_correction(inpath, list_of_wls, list_of_orders,
list_of_sigmas):
"""
This applies a set of telluric spectra (computed by molecfit) for each exposure
in our time series that were written to a pickle file by write_telluric_transmission_to_file.
List of errors are provided to propagate telluric correction into the error array as well.
Parameters
----------
inpath : str, path like
The path to the pickled transmission spectra.
list_of_wls : list
List of wavelength axes.
list_of_orders :
List of 2D spectral orders, matching to the wavelength axes in dimensions and in number.
list_of_isgmas :
List of 2D error matrices, matching dimensions and number of list_of_orders.
Returns
-------
list_of_orders_corrected : list
List of 2D spectral orders, telluric corrected.
list_of_sigmas_corrected : list
List of 2D error matrices, telluric corrected.
"""
import scipy.interpolate as interp
import numpy as np
import tayph.util as ut
import tayph.functions as fun
from tayph.vartests import dimtest, postest, typetest, nantest
import copy
T = read_telluric_transmission_from_file(inpath)
wlT = T[0]
fxT = T[1]
typetest(list_of_wls, list, 'list_of_wls in apply_telluric_correction()')
typetest(list_of_orders, list,
'list_of_orders in apply_telluric_correction()')
typetest(list_of_sigmas, list,
'list_of_sigmas in apply_telluric_correction()')
typetest(wlT, list,
'list of telluric wave-axes in apply_telluric_correction()')
typetest(
fxT, list,
'list of telluric transmission spectra in apply_telluric_correction()')
No = len(list_of_wls) #Number of orders.
x = fun.findgen(No)
Nexp = len(wlT)
#Test dimensions
if No != len(list_of_orders):
raise Exception(
'Runtime error in telluric correction: List of wavelength axes and List '
'of orders do not have the same length.')
if Nexp != len(fxT):
raise Exception(
'Runtime error in telluric correction: List of telluric wls and telluric '
'spectra read from file do not have the same length.')
if Nexp != len(list_of_orders[0]):
raise Exception(
f'Runtime error in telluric correction: List of telluric spectra and data'
f'spectra read from file do not have the same length ({Nexp} vs {len(list_of_orders[0])}).'
)
#Corrected orders will be stored here.
list_of_orders_cor = []
list_of_sigmas_cor = []
#Do the correction order by order:
for i in range(No):
order = list_of_orders[i]
order_cor = order * 0.0
error = list_of_sigmas[i]
error_cor = error * 0.0
wl = copy.deepcopy(list_of_wls[i]) #input wl axis, either 1D or 2D.
#If it is 1D, we make it 2D by tiling it vertically:
if wl.ndim == 1: wl = np.tile(wl, (Nexp, 1)) #Tile it into a 2D thing.
#If read (2D) or tiled (1D) correctly, wl and order should have the same shape:
dimtest(wl, np.shape(order),
f'Wl axis of order {i}/{No} in apply_telluric_correction()')
dimtest(error, np.shape(order),
f'errors {i}/{No} in apply_telluric_correction()')
for j in range(Nexp):
T_i = interp.interp1d(wlT[j], fxT[j],
fill_value="extrapolate")(wl[j])
postest(T_i,
f'T-spec of exposure {j} in apply_telluric_correction()')
nantest(T_i,
f'T-spec of exposure {j} in apply_telluric_correction()')
order_cor[j] = order[j] / T_i
error_cor[j] = error[
j] / T_i #I checked that this works because the SNR before and after
#telluric correction is identical.
list_of_orders_cor.append(order_cor)
list_of_sigmas_cor.append(error_cor)
ut.statusbar(i, x)
return (list_of_orders_cor, list_of_sigmas_cor)
def bin_avg(wl, fx, wlm):
"""
A simple function to bin a high-res model spectrum (wl,fx) to a lower resolution wavelength grid (wlm),
by averaging the points fx inside the lower-resolution wavelength bins set by wlm.
Parameters
----------
wl : list, np.ndarray
The horizontal axis.
fx : list, np.ndarray
The array corresponding to x.
wlm : list, np.ndarray
The lower resolution wlm array. wlm should have fewer points than wl and fx.
Returns
-------
y_bin : np.array
The binned flux points.
"""
import numpy as np
import sys
from tayph.vartests import typetest, dimtest, minlength
import matplotlib.pyplot as plt
import tayph.util as ut
typetest(wl, [list, np.ndarray], 'wl in ops.bin_avg()')
typetest(fx, [list, np.ndarray], 'fx in ops.bin_avg()')
typetest(wlm, [list, np.ndarray], 'wlm in ops.bin_avg()')
dimtest(wl, [0], 'wl in ops.bin_avg()')
dimtest(fx, [len(fx)], 'fx in ops.bin_avg()')
minlength(wl, 0, 'wl in ops.bin_avg()')
minlength(wl, 50, 'wl in ops.bin_avg()', warning_only=True)
minlength(wlm, 0, 'wlm in ops.bin_avg()')
minlength(wlm, 100, 'wlm in ops.bin_avg()', warning_only=True)
if len(wl) < len(wlm):
raise ValueError(
f"Error in bin_avg: The input grid should have (many) more points than the requested grid of interpolates ({len(wl)}, {len(wlm)})."
)
if max(wl) < min(wlm) or min(wl) > max(wlm):
raise ValueError(
f"Error in bin_avg: the supplied wl and wlm arrays have no overlap. Are the wavelength units the same? (mean(wl)={np.mean(wlm)}, mean(wlm)={np.mean(wlm)})."
)
dwl_start = wlm[1] - wlm[0]
wlm_borders = [wlm[0] - dwl_start / 2.0]
for i in range(len(wlm) - 1):
dwl = wlm[i + 1] - wlm[i]
if i == 0:
wlm_borders = [wlm[0] - dwl / 2.0]
wlm_borders.append(wlm[i] + dwl / 2.0)
wlm_borders.append(wlm[-1] + dwl / 2.0)
fxm = []
for i in range(len(wlm_borders) - 1):
#This indexing is slow. But I always only need to compute it once!
fx_sel = fx[(wl > wlm_borders[i]) & (wl <= wlm_borders[i + 1])]
if len(fx_sel) == 0:
raise RuntimeError(
f"Error in bin_avg: No points were selected in step {i}. Wlm_borders has a smaller step size than fx?"
)
else:
fxbin = np.mean(fx[(wl > wlm_borders[i])
& (wl <= wlm_borders[i + 1])])
fxm.append(fxbin)
return (fxm)
def convolve(array, kernel, edge_degree=1, fit_width=2):
"""It's unbelievable, but I could not find the python equivalent of IDL's
/edge_truncate keyword, which truncates the kernel at the edge of the convolution.
Therefore, unfortunately, I need to code a convolution operation myself.
Stand by to be slowed down by an order of magnitude #thankspython.
Nope! Because I can just use np.convolve for most of the array, just not the edge...
So the strategy is to extrapolate the edge of the array using a polynomial fit
to the edge elements. By default, I fit over a range that is twice the length of the kernel; but
this value can be modified using the fit_width parameter.
Parameters
----------
array : list, np.ndarray
The horizontal axis.
kernel : list, np.ndarray
The convolution kernel. It is required to have a length that is less than 25% of the size of the array.
edge_degree : int
The polynomial degree by which the array is extrapolated in order to
fit_width : int
The length of the area at the edges of array used to fit the polynomial, in units of the length of the kernel.
Increase this number for small kernels or noisy arrays.
Returns
-------
array_convolved : np.array
The input array convolved with the kernel
Example
-------
>>> import numpy as np
>>> a=[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]
>>> b=[-0.5,0,0.5]
>>> c=convolve(a,b,edge_degree=1)
"""
import numpy as np
import tayph.functions as fun
from tayph.vartests import typetest, postest, dimtest
typetest(edge_degree, int, 'edge_degree in ops.convolve()')
typetest(fit_width, int, 'edge_degree in ops.convolve()')
typetest(array, [list, np.ndarray])
typetest(kernel, [list, np.ndarray])
dimtest(array, [0], 'array in ops.convolve()')
dimtest(kernel, [0], 'array in ops.convolve()')
postest(edge_degree, 'edge_degree in ops.convolve()')
postest(fit_width, 'edge_degree in ops.convolve()')
array = np.array(array)
kernel = np.array(kernel)
if len(kernel) >= len(array) / 4.0:
raise Exception(
f"Error in ops.convolve(): Kernel length is larger than a quarter of the array ({len(kernel)}, {len(array)}). Can't extrapolate over that length. And you probably don't want to be doing a convolution like that, anyway."
)
if len(kernel) % 2 != 1:
raise Exception(
'Error in ops.convolve(): Kernel needs to have an odd number of elements.'
)
#Perform polynomial fits at the edges.
x = fun.findgen(len(array))
fit_left = np.polyfit(x[0:len(kernel) * 2], array[0:len(kernel) * 2],
edge_degree)
fit_right = np.polyfit(x[-2 * len(kernel) - 1:-1],
array[-2 * len(kernel) - 1:-1], edge_degree)
#Pad both the x-grid (onto which the polynomial is defined)
#and the data array.
pad = fun.findgen((len(kernel) - 1) / 2)
left_pad = pad - (len(kernel) - 1) / 2
right_pad = np.max(x) + pad + 1
left_array_pad = np.polyval(fit_left, left_pad)
right_array_pad = np.polyval(fit_right, right_pad)
#Perform the padding.
x_padded = np.append(left_pad, x)
x_padded = np.append(
x_padded, right_pad
) #Pad the array with the missing elements of the kernel at the edge.
array_padded = np.append(left_array_pad, array)
array_padded = np.append(array_padded, right_array_pad)
#Reverse the kernel because np.convol does that automatically and I don't want that.
#(Imagine doing a derivative with a kernel [-1,0,1] and it gets reversed...)
kr = kernel[::-1]
#The valid keyword effectively undoes the padding, leaving only those values for which the kernel was entirely in the padded array.
#This thus again has length equal to len(array).
return np.convolve(array_padded, kr, 'valid')
def constant_velocity_wl_grid(wl, fx, oversampling=1.0):
"""This function will define a constant-velocity grid that is (optionally)
sampled a number of times finer than the SMALLEST velocity difference that is
currently in the grid.
Example: wl_cv,fx_cv = constant_velocity_wl_grid(wl,fx,oversampling=1.5).
This function is hardcoded to raise an exception if wl or fx contain NaNs,
because interp1d does not handle NaNs.
Parameters
----------
wl : list, np.ndarray
The wavelength array to be resampled.
fx : list, np.ndarray
The flux array to be resampled.
oversampling : float
The factor by which the wavelength array is *minimally* oversampled.
Returns
-------
wl : np.array
The new wavelength grid.
fx : np.array
The interpolated flux values.
a : float
The velocity step in km/s.
"""
import astropy.constants as consts
import numpy as np
import tayph.functions as fun
from tayph.vartests import typetest, nantest, dimtest, postest
from scipy import interpolate
import pdb
import matplotlib.pyplot as plt
typetest(
oversampling,
[int, float],
'oversampling in constant_velocity_wl_grid()',
)
typetest(wl, [list, np.ndarray], 'wl in constant_velocity_wl_grid()')
typetest(fx, [list, np.ndarray], 'fx in constant_velocity_wl_grid()')
nantest(wl, 'wl in in constant_velocity_wl_grid()')
nantest(fx, 'fx in constant_velocity_wl_grid()')
dimtest(wl, [0], 'wl in constant_velocity_wl_grid()')
dimtest(fx, [len(wl)], 'fx in constant_velocity_wl_grid()')
postest(oversampling, 'oversampling in constant_velocity_wl_grid()')
oversampling = float(oversampling)
wl = np.array(wl)
fx = np.array(fx)
c = consts.c.to('km/s').value
dl = derivative(wl)
dv = dl / wl * c
a = np.min(dv) / oversampling
wl_new = 0.0
#The following while loop will define the new pixel grid.
#It starts trying 100,000 points, and if that's not enough to cover the entire
#range from min(wl) to max(wl), it will add 100,000 more; until it's enough.
n = len(wl)
while np.max(wl_new) < np.max(wl):
x = fun.findgen(n)
wl_new = np.exp(a / c * x) * np.min(wl)
n += len(wl)
wl_new[0] = np.min(
wl) #Artificially set to zero to avoid making a small round
#off error in that exponent.
#Then at the end we crop the part that goes too far:
wl_new_cropped = wl_new[(wl_new <= np.max(wl))]
x_cropped = x[(wl_new <= np.max(wl))]
i_fx = interpolate.interp1d(wl, fx)
fx_new_cropped = i_fx(wl_new_cropped)
return (wl_new_cropped, fx_new_cropped, a)
def xcor(list_of_wls,
list_of_orders,
wlm,
fxm,
drv,
RVrange,
plot=False,
list_of_errors=None):
"""
This routine takes a combined dataset (in the form of lists of wl spaces,
spectral orders and possible a matching list of errors on those spectal orders),
as well as a template (wlm,fxm) to cross-correlate with, and the cross-correlation
parameters (drv,RVrange). The code takes on the order of ~10 minutes for an entire
HARPS dataset, which appears to be superior to my old IDL pipe.
The CCF used is the Geneva-style weighted average; not the Pearson CCF. Therefore
it measures true 'average' planet lines, with flux on the y-axis of the CCF.
The template must therefore be (something close to) a binary mask, with values
inside spectral lines (the CCF is scale-invariant so their overall scaling
doesn't matter),
It returns the RV axis and the resulting CCF in a tuple.
Thanks to Brett Morris (bmorris3), this code now implements a clever numpy broadcasting trick to
instantly apply and interpolate the wavelength shifts of the model template onto
the data grid in 2 dimensions. The matrix multiplication operator (originally
recommended to me by Matteo Brogi) allowed this 2D template matrix to be multiplied
with a 2D spectral order. np.hstack() is used to concatenate all orders end to end,
effectively making a giant single spectral order (with NaNs in between due to masking).
All these steps have eliminated ALL the forloops from the equation, and effectuated a
speed gain of a factor between 2,000 and 3,000. The time to do cross correlations is now
typically measured in 100s of milliseconds rather than minutes.
This way of calculation does impose some strict rules on NaNs, though. To keep things fast,
NaNs are now used to set the interpolated template matrix to zero wherever there are NaNs in the data.
These NaNs are found by looking at the first spectrum in the stack, with the assumption that
every NaN is in an all-NaN column. In the standard cross-correlation work-flow, isolated
NaNs are interpolated over (healed), after all.
The places where there are NaN columns in the data are therefore set to 0 in the template matrix.
The NaN values themselves are then set to to an arbitrary value, since they will never
weigh into the average by construction.
Parameters
----------
list_of_wls : list
List of wavelength axes of the data.
list_of_orders : list
List of corresponding 2D orders.
list_of_errors : list
Optional, list of corresponding 2D error matrices.
wlm : np.ndarray
Wavelength axis of the template.
fxm : np.ndarray
Weight-axis of the template.
drv : int,float
The velocity step onto which the CCF is computed. Typically ~1 km/s.
RVrange : int,float
The velocity range in the positive and negative direction over which to
evaluate the CCF. Typically >100 km/s.
plot : bool
Set to True for diagnostic plotting.
Returns
-------
RV : np.ndarray
The radial velocity grid over which the CCF is evaluated.
CCF : np.ndarray
The weighted average flux in the spectrum as a function of radial velocity.
CCF_E : np.ndarray
Optional. The error on each CCF point propagated from the error on the spectral values.
Tsums : np.ndarray
The sum of the template for each velocity step. Used for normalising the CCFs.
"""
import tayph.functions as fun
import astropy.constants as const
import tayph.util as ut
from tayph.vartests import typetest, dimtest, postest, nantest
import numpy as np
import scipy.interpolate
import astropy.io.fits as fits
import matplotlib.pyplot as plt
import sys
import pdb
#===FIRST ALL SORTS OF TESTS ON THE INPUT===
if len(list_of_wls) != len(list_of_orders):
raise ValueError(
f'In xcor(): List of wls and list of orders have different length ({len(list_of_wls)} & {len(list_of_orders)}).'
)
dimtest(wlm, [len(fxm)], 'wlm in ccf.xcor()')
typetest(wlm, np.ndarray, 'wlm in ccf.xcor')
typetest(fxm, np.ndarray, 'fxm in ccf.xcor')
typetest(drv, [int, float], 'drv in ccf.xcor')
typetest(
RVrange,
float,
'RVrange in ccf.xcor()',
)
postest(RVrange, 'RVrange in ccf.xcor()')
postest(drv, 'drv in ccf.xcor()')
nantest(wlm, 'fxm in ccf.xcor()')
nantest(fxm, 'fxm in ccf.xcor()')
nantest(drv, 'drv in ccf.xcor()')
nantest(RVrange, 'RVrange in ccf.xcor()')
drv = float(drv)
N = len(list_of_wls) #Number of orders.
if np.ndim(list_of_orders[0]) == 1.0:
n_exp = 1
else:
n_exp = len(list_of_orders[0][:, 0]) #Number of exposures.
#===Then check that all orders indeed have n_exp exposures===
for i in range(N):
if len(list_of_orders[i][:, 0]) != n_exp:
raise ValueError(
f'In xcor(): Not all orders have {n_exp} exposures.')
#===END OF TESTS. NOW DEFINE CONSTANTS===
c = const.c.to('km/s').value #In km/s
RV = fun.findgen(
2.0 * RVrange / drv +
1) * drv - RVrange #..... CONTINUE TO DEFINE THE VELOCITY GRID
beta = 1.0 - RV / c #The doppler factor with which each wavelength is to be shifted.
n_rv = len(RV)
#===STACK THE ORDERS IN MASSIVE CONCATENATIONS===
stack_of_orders = np.hstack(list_of_orders)
stack_of_wls = np.concatenate(list_of_wls)
if list_of_errors is not None:
stack_of_errors = np.hstack(list_of_errors) #Stack them horizontally
#Check that the number of NaNs is the same in the orders as in the errors on the orders;
#and that they are in the same place; meaning that if I add the errors to the orders, the number of
#NaNs does not increase (NaN+value=NaN).
if (np.sum(np.isnan(stack_of_orders)) != np.sum(
np.isnan(stack_of_errors + stack_of_orders))) and (np.sum(
np.isnan(stack_of_orders)) != np.sum(
np.isnan(stack_of_errors))):
raise ValueError(
f"in CCF: The number of NaNs in list_of_orders and list_of_errors is not equal ({np.sum(np.isnan(list_of_orders))},{np.sum(np.isnan(list_of_errors))})"
)
#===HERE IS THE JUICY BIT===
shifted_wavelengths = stack_of_wls * beta[:, np.
newaxis] #2D broadcast of wl_data, each row shifted by beta[i].
T = scipy.interpolate.interp1d(wlm, fxm, bounds_error=False, fill_value=0)(
shifted_wavelengths) #...making this a 2D thing.
T[:, np.isnan(
stack_of_orders[0]
)] = 0.0 #All NaNs are assumed to be in all-NaN columns. If that is not true, the below nantest will fail.
T_sums = np.sum(T, axis=1)
#We check whether there are isolated NaNs:
n_nans = np.sum(np.isnan(stack_of_orders),
axis=0) #This is the total number of NaNs in each column.
n_nans[n_nans == len(
stack_of_orders
)] = 0 #Whenever the number of NaNs equals the length of a column, set the flag to zero.
if np.max(
n_nans
) > 0: #If there are any columns which still have NaNs in them, we need to crash.
raise ValueError(
f"in CCF: Not all NaN values are purely in columns. There are still isolated NaNs. Remove those."
)
stack_of_orders[np.isnan(
stack_of_orders)] = 47e20 #Set NaNs to arbitrarily high values.
CCF = stack_of_orders @ T.T / T_sums #Here it the entire cross-correlation. Over all orders and velocity steps. No forloops.
CCF_E = CCF * 0.0
#If the errors were provided, we do the same to those:
if list_of_errors is not None:
stack_of_errors[np.isnan(
stack_of_errors
)] = 42e20 #we have already tested that these NaNs are in the same place.
CCF_E = stack_of_errors**2 @ (
T.T / T_sums)**2 #This has been mathematically proven.
#===THAT'S ALL. TEST INTEGRITY AND RETURN THE RESULT===
nantest(
CCF, 'CCF in ccf.xcor()'
) #If anything went wrong with NaNs in the data, these tests will fail because the matrix operation @ is non NaN-friendly.
nantest(CCF_E, 'CCF_E in ccf.xcor()')
if list_of_errors != None:
return (RV, CCF, np.sqrt(CCF_E), T_sums)
return (RV, CCF, T_sums)
def run_instance(p):
"""This runs the entire cross correlation analysis cascade."""
import numpy as np
from astropy.io import fits
import astropy.constants as const
import astropy.units as u
from matplotlib import pyplot as plt
import os.path
import scipy.interpolate as interp
import pylab
import pdb
import os.path
import os
import sys
import glob
import distutils.util
import pickle
import copy
from pathlib import Path
import tayph.util as ut
import tayph.operations as ops
import tayph.functions as fun
import tayph.system_parameters as sp
import tayph.tellurics as telcor
import tayph.masking as masking
import tayph.models as models
from tayph.ccf import xcor, clean_ccf, filter_ccf, construct_KpVsys
from tayph.vartests import typetest, notnegativetest, nantest, postest, typetest_array, dimtest
import tayph.shadow as shadow
# from lib import analysis
# from lib import cleaning
# from lib import masking as masking
# from lib import shadow as shadow
# from lib import molecfit as telcor
#First parse the parameter dictionary into required variables and test them.
typetest(p, dict, 'params in run_instance()')
dp = Path(p['dp'])
ut.check_path(dp, exists=True)
modellist = p['modellist']
templatelist = p['templatelist']
model_library = p['model_library']
template_library = p['template_library']
typetest(modellist, [str, list], 'modellist in run_instance()')
typetest(templatelist, [str, list], 'templatelist in run_instance()')
typetest(model_library, str, 'model_library in run_instance()')
typetest(template_library, str, 'template_library in run_instance()')
ut.check_path(model_library, exists=True)
ut.check_path(template_library, exists=True)
if type(modellist) == str:
modellist = [modellist] #Force this to be a list
if type(templatelist) == str:
templatelist = [templatelist] #Force this to be a list
typetest_array(modellist, str, 'modellist in run_instance()')
typetest_array(templatelist, str, 'modellist in run_instance()')
shadowname = p['shadowname']
maskname = p['maskname']
typetest(shadowname, str, 'shadowname in run_instance()')
typetest(maskname, str, 'shadowname in run_instance()')
RVrange = p['RVrange']
drv = p['drv']
f_w = p['f_w']
resolution = sp.paramget('resolution', dp)
typetest(RVrange, [int, float], 'RVrange in run_instance()')
typetest(drv, [int, float], 'drv in run_instance()')
typetest(f_w, [int, float], 'f_w in run_instance()')
typetest(resolution, [int, float], 'resolution in run_instance()')
nantest(RVrange, 'RVrange in run_instance()')
nantest(drv, 'drv in run_instance()')
nantest(f_w, 'f_w in run_instance()')
nantest(resolution, 'resolution in run_instance()')
postest(RVrange, 'RVrange in run_instance()')
postest(drv, 'drv in run_instance()')
postest(resolution, 'resolution in run_instance()')
notnegativetest(f_w, 'f_w in run_instance()')
do_colour_correction = p['do_colour_correction']
do_telluric_correction = p['do_telluric_correction']
do_xcor = p['do_xcor']
plot_xcor = p['plot_xcor']
make_mask = p['make_mask']
apply_mask = p['apply_mask']
c_subtract = p['c_subtract']
do_berv_correction = p['do_berv_correction']
do_keplerian_correction = p['do_keplerian_correction']
make_doppler_model = p['make_doppler_model']
skip_doppler_model = p['skip_doppler_model']
typetest(do_colour_correction, bool,
'do_colour_correction in run_instance()')
typetest(do_telluric_correction, bool,
'do_telluric_correction in run_instance()')
typetest(do_xcor, bool, 'do_xcor in run_instance()')
typetest(plot_xcor, bool, 'plot_xcor in run_instance()')
typetest(make_mask, bool, 'make_mask in run_instance()')
typetest(apply_mask, bool, 'apply_mask in run_instance()')
typetest(c_subtract, bool, 'c_subtract in run_instance()')
typetest(do_berv_correction, bool, 'do_berv_correction in run_instance()')
typetest(do_keplerian_correction, bool,
'do_keplerian_correction in run_instance()')
typetest(make_doppler_model, bool, 'make_doppler_model in run_instance()')
typetest(skip_doppler_model, bool, 'skip_doppler_model in run_instance()')
#We start by defining constants and preparing for generating output.
c = const.c.value / 1000.0 #in km/s
colourdeg = 3 #A fitting degree for the colour correction.
print(
f'---Passed parameter input tests. Initiating output folder tree in {Path("output")/dp}.'
)
libraryname = str(template_library).split('/')[-1]
if str(dp).split('/')[0] == 'data':
dataname = str(dp).replace('data/', '')
print(
f'------Data is located in data/ folder. Assuming output name for this dataset as {dataname}'
)
else:
dataname = dp
print(
f'------Data is NOT located in data/ folder. Assuming output name for this dataset as {dataname}'
)
list_of_wls = [] #This will store all the data.
list_of_orders = [] #All of it needs to be loaded into your memory.
list_of_sigmas = []
trigger2 = 0 #These triggers are used to limit the generation of output in the forloop.
trigger3 = 0
n_negative_total = 0 #This will hold the total number of pixels that were set to NaN because they were zero when reading in the data.
air = sp.paramget('air', dp) #Read bool from str in config file.
typetest(air, bool, 'air in run_instance()')
filelist_orders = [str(i) for i in Path(dp).glob('order_*.fits')]
if len(filelist_orders) == 0:
raise Exception(
f'Runtime error: No orders_*.fits files were found in {dp}.')
try:
order_numbers = [
int(i.split('order_')[1].split('.')[0]) for i in filelist_orders
]
except:
raise Exception(
'Runtime error: Failed casting fits filename numerals to ints. Are the filenames of the spectral orders correctly formatted?'
)
order_numbers.sort() #This is the ordered list of numerical order IDs.
n_orders = len(order_numbers)
if n_orders == 0:
raise Exception(
f'Runtime error: n_orders may not have ended up as zero. ({n_orders})'
)
#Loading the data from the datafolder.
if do_xcor == True or plot_xcor == True or make_mask == True:
print(f'---Loading orders from {dp}.')
# for i in range(startorder,endorder+1):
for i in order_numbers:
wavepath = dp / f'wave_{i}.fits'
orderpath = dp / f'order_{i}.fits'
sigmapath = dp / f'sigma_{i}.fits'
ut.check_path(wavepath, exists=True)
ut.check_path(orderpath, exists=True)
ut.check_path(sigmapath, exists=False)
wave_axis = fits.getdata(wavepath)
dimtest(wave_axis, [0], 'wavelength grid in run_instance()')
n_px = len(wave_axis) #Pixel width of the spectral order.
if air == False:
if i == np.min(order_numbers):
print("------Assuming wavelengths are in vaccuum.")
list_of_wls.append(1.0 * wave_axis)
else:
if i == np.min(order_numbers):
print("------Applying airtovac correction.")
list_of_wls.append(ops.airtovac(wave_axis))
order_i = fits.getdata(orderpath)
if i == np.min(order_numbers):
dimtest(
order_i, [0, n_px], f'order {i} in run_instance()'
) #For the first order, check that it is 2D and that is has a width equal to n_px.
n_exp = np.shape(
order_i
)[0] #then fix n_exp. All other orders should have the same n_exp.
print(f'------{n_exp} exposures recognised.')
else:
dimtest(order_i, [n_exp, n_px], f'order {i} in run_instance()')
#Now test for negatives, set them to NaN and track them.
n_negative = len(order_i[order_i <= 0])
if trigger3 == 0 and n_negative > 0:
print("------Setting negative values to NaN.")
trigger3 = -1
n_negative_total += n_negative
order_i[order_i <= 0] = np.nan
postest(order_i, f'order {i} in run_instance().'
) #make sure whatever comes out here is strictly positive.
list_of_orders.append(order_i)
try: #Try to get a sigma file. If it doesn't exist, we raise a warning. If it does, we test its dimensions and append it.
sigma_i = fits.getdata(sigmapath)
dimtest(sigma_i, [n_exp, n_px],
f'order {i} in run_instance().')
list_of_sigmas.append(sigma_i)
except FileNotFoundError:
if trigger2 == 0:
print(
'------WARNING: Sigma (flux error) files not provided. Assuming sigma = sqrt(flux). This is standard practise for HARPS data, but e.g. ESPRESSO has a pipeline that computes standard errors on each pixel for you.'
)
trigger2 = -1
list_of_sigmas.append(np.sqrt(order_i))
print(
f"------{n_negative_total} negative values set to NaN ({np.round(100.0*n_negative_total/n_exp/n_px/len(order_numbers),2)}% of total spectral pixels in dataset.)"
)
if len(list_of_orders) != n_orders:
raise Exception(
'Runtime error: n_orders is not equal to the length of list_of_orders. Something went wrong when reading them in?'
)
print('---Finished loading dataset to memory.')
#Apply telluric correction file or not.
# plt.plot(list_of_wls[60],list_of_orders[60][10],color='red')
# plt.plot(list_of_wls[60],list_of_orders[60][10]+list_of_sigmas[60][10],color='red',alpha=0.5)#plot corrected spectra
# plt.plot(list_of_wls[60],list_of_orders[60][10]/list_of_sigmas[60][10],color='red',alpha=0.5)#plot SNR
if do_telluric_correction == True and n_orders > 0:
print('---Applying telluric correction')
telpath = dp / 'telluric_transmission_spectra.pkl'
list_of_orders, list_of_sigmas = telcor.apply_telluric_correction(
telpath, list_of_wls, list_of_orders, list_of_sigmas)
# plt.plot(list_of_wls[60],list_of_orders[60][10],color='blue')
# plt.plot(list_of_wls[60],list_of_orders[60][10]+list_of_sigmas[60][10],color='blue',alpha=0.5)#plot corrected spectra
# plt.plot(list_of_wls[60],list_of_orders[60][10]/list_of_sigmas[60][10],color='blue',alpha=0.5) #plot SNR
# plt.show()
# pdb.set_trace()
#Do velocity correction of wl-solution. Explicitly after telluric correction
#but before masking. Because the cross-correlation relies on columns being masked.
#Then if you start to move the CCFs around before removing the time-average,
#each masked column becomes slanted. Bad deal.
rv_cor = 0
if do_berv_correction == True:
rv_cor += sp.berv(dp)
if do_keplerian_correction == True:
rv_cor -= sp.RV_star(dp) * (1.0)
if type(rv_cor) != int and len(list_of_orders) > 0:
print('---Reinterpolating data to correct velocities')
list_of_orders_cor = []
list_of_sigmas_cor = []
for i in range(len(list_of_wls)):
order = list_of_orders[i]
sigma = list_of_sigmas[i]
order_cor = order * 0.0
sigma_cor = sigma * 0.0
for j in range(len(list_of_orders[0])):
wl_i = interp.interp1d(list_of_wls[i],
order[j],
bounds_error=False)
si_i = interp.interp1d(list_of_wls[i],
sigma[j],
bounds_error=False)
wl_cor = list_of_wls[i] * (
1.0 - (rv_cor[j] * u.km / u.s / const.c)
) #The minus sign was tested on a slow-rotator.
order_cor[j] = wl_i(wl_cor)
sigma_cor[j] = si_i(
wl_cor
) #I checked that this works because it doesn't affect the SNR, apart from wavelength-shifting it.
list_of_orders_cor.append(order_cor)
list_of_sigmas_cor.append(sigma_cor)
ut.statusbar(i, fun.findgen(len(list_of_wls)))
# plt.plot(list_of_wls[60],list_of_orders[60][10]/list_of_sigmas[60][10],color='blue')
# plt.plot(list_of_wls[60],list_of_orders_cor[60][10]/list_of_sigmas_cor[60][10],color='red')
# plt.show()
# sys.exit()
list_of_orders = list_of_orders_cor
list_of_sigmas = list_of_sigmas_cor
if len(list_of_orders) != n_orders:
raise RuntimeError(
'n_orders is no longer equal to the length of list_of_orders, though it was before. Something went wrong during telluric correction or velocity correction.'
)
#Compute / create a mask and save it to file (or not)
if make_mask == True and len(list_of_orders) > 0:
if do_colour_correction == True:
print(
'---Constructing mask with intra-order colour correction applied'
)
masking.mask_orders(list_of_wls,
ops.normalize_orders(list_of_orders,
list_of_sigmas,
colourdeg)[0],
dp,
maskname,
40.0,
5.0,
manual=True)
else:
print(
'---Constructing mask WITHOUT intra-order colour correction applied.'
)
print(
'---Switch on colour correction if you see colour variations in the 2D spectra.'
)
masking.mask_orders(list_of_wls,
list_of_orders,
dp,
maskname,
40.0,
5.0,
manual=True)
if apply_mask == False:
print(
'---WARNING in run_instance: Mask was made but is not applied to data (apply_mask == False)'
)
#Apply the mask that was previously created and saved to file.
if apply_mask == True:
print('---Applying mask')
list_of_orders = masking.apply_mask_from_file(dp, maskname,
list_of_orders)
list_of_sigmas = masking.apply_mask_from_file(dp, maskname,
list_of_sigmas)
#Interpolate over all isolated NaNs and set bad columns to NaN (so that they are ignored in the CCF)
if do_xcor == True:
print('---Healing NaNs')
list_of_orders = masking.interpolate_over_NaNs(
list_of_orders
) #THERE IS AN ISSUE HERE: INTERPOLATION SHOULD ALSO HAPPEN ON THE SIGMAS ARRAY!
list_of_sigmas = masking.interpolate_over_NaNs(list_of_sigmas)
#Normalize the orders to their average flux in order to effectively apply a broad-band colour correction (colour is typically a function of airmass and seeing).
if do_colour_correction == True:
print('---Normalizing orders to common flux level')
# plt.plot(list_of_wls[60],list_of_orders[60][10]/list_of_sigmas[60][10],color='blue',alpha=0.4)
list_of_orders_normalised, list_of_sigmas_normalised, meanfluxes = ops.normalize_orders(
list_of_orders, list_of_sigmas, colourdeg
) #I tested that this works because it doesn't alter the SNR.
meanfluxes_norm = meanfluxes / np.nanmean(meanfluxes)
else:
meanfluxes_norm = fun.findgen(len(
list_of_orders[0])) * 0.0 + 1.0 #All unity.
# plt.plot(list_of_wls[60],list_of_orders_normalised[60][10]/list_of_sigmas[60][10],color='red',alpha=0.4)
# plt.show()
# sys.exit()
if len(list_of_orders) != n_orders:
raise RuntimeError(
'n_orders is no longer equal to the length of list_of_orders, though it was before. Something went wrong during masking or colour correction.'
)
#Construct the cross-correlation templates in case we will be computing or plotting the CCF.
if do_xcor == True or plot_xcor == True:
list_of_wlts = []
list_of_templates = []
outpaths = []
for templatename in templatelist:
print(f'---Building template {templatename}')
wlt, T = models.build_template(templatename,
binsize=0.5,
maxfrac=0.01,
resolution=resolution,
template_library=template_library,
c_subtract=c_subtract)
T *= (-1.0)
if np.mean(wlt) < 50.0: #This is likely in microns:
print(
'------WARNING: The loaded template has a mean wavelength less than 50.0, meaning that it is very likely not in nm, but in microns. I have divided by 1,000 now and hope for the best...'
)
wlt *= 1000.0
list_of_wlts.append(wlt)
list_of_templates.append(T)
outpath = Path('output') / Path(dataname) / Path(
libraryname) / Path(templatename)
if not os.path.exists(outpath):
print(
f"------The output location ({outpath}) didn't exist, I made it now."
)
os.makedirs(outpath)
outpaths.append(outpath)
#Perform the cross-correlation on the entire list of orders.
for i in range(len(list_of_wlts)):
templatename = templatelist[i]
wlt = list_of_wlts[i]
T = list_of_templates[i]
outpath = outpaths[i]
if do_xcor == True:
print(
f'---Cross-correlating spectra with template {templatename}.')
t1 = ut.start()
rv, ccf, ccf_e, Tsums = xcor(
list_of_wls,
list_of_orders_normalised,
np.flipud(np.flipud(wlt)),
T,
drv,
RVrange,
list_of_errors=list_of_sigmas_normalised)
ut.end(t1)
print(f'------Writing CCFs to {str(outpath)}')
ut.writefits(outpath / 'ccf.fits', ccf)
ut.writefits(outpath / 'ccf_e.fits', ccf_e)
ut.writefits(outpath / 'RV.fits', rv)
ut.writefits(outpath / 'Tsum.fits', Tsums)
else:
print(
f'---Reading CCFs with template {templatename} from {str(outpath)}.'
)
if os.path.isfile(outpath / 'ccf.fits') == False:
raise FileNotFoundError(
f'CCF output not located at {outpath}. Rerun with do_xcor=True to create these files?'
)
rv = fits.getdata(outpath / 'rv.fits')
ccf = fits.getdata(outpath / 'ccf.fits')
ccf_e = fits.getdata(outpath / 'ccf_e.fits')
Tsums = fits.getdata(outpath / 'Tsum.fits')
ccf_cor = ccf * 1.0
ccf_e_cor = ccf_e * 1.0
print('---Cleaning CCFs')
ccf_n, ccf_ne, ccf_nn, ccf_nne = clean_ccf(rv, ccf_cor, ccf_e_cor, dp)
if make_doppler_model == True and skip_doppler_model == False:
shadow.construct_doppler_model(rv,
ccf_nn,
dp,
shadowname,
xrange=[-200, 200],
Nxticks=20.0,
Nyticks=10.0)
make_doppler_model = False # This sets it to False after it's been run once, for the first template.
if skip_doppler_model == False:
print('---Reading doppler shadow model from ' + shadowname)
doppler_model, dsmask = shadow.read_shadow(
dp, shadowname, rv, ccf
) #This returns both the model evaluated on the rv,ccf grid, as well as the mask that blocks the planet trace.
ccf_clean, matched_ds_model = shadow.match_shadow(
rv, ccf_nn, dsmask, dp, doppler_model
) #THIS IS AN ADDITIVE CORRECTION, SO CCF_NNE DOES NOT NEED TO BE ALTERED AND IS STILL VALID VOOR CCF_CLEAN
else:
print('---Not performing shadow correction')
ccf_clean = ccf_nn * 1.0
matched_ds_model = ccf_clean * 0.0
if f_w > 0.0:
print('---Performing high-pass filter on the CCF')
ccf_clean_filtered, wiggles = filter_ccf(
rv, ccf_clean, v_width=f_w
) #THIS IS ALSO AN ADDITIVE CORRECTION, SO CCF_NNE IS STILL VALID.
else:
print('---Skipping high-pass filter')
ccf_clean_filtered = ccf_clean * 1.0
wiggles = ccf_clean * 0.0 #This filtering is additive so setting to zero is accurate.
print('---Weighing CCF rows by mean fluxes that were normalised out')
ccf_clean_weighted = np.transpose(
np.transpose(ccf_clean_filtered) * meanfluxes_norm
) #MULTIPLYING THE AVERAGE FLUXES BACK IN! NEED TO CHECK THAT THIS ALSO GOES PROPERLY WITH THE ERRORS!
ccf_nne = np.transpose(np.transpose(ccf_nne) * meanfluxes_norm)
ut.save_stack(outpath / 'cleaning_steps.fits', [
ccf, ccf_cor, ccf_nn, ccf_clean, matched_ds_model,
ccf_clean_filtered, wiggles, ccf_clean_weighted
])
ut.writefits(outpath / 'ccf_cleaned.fits', ccf_clean_weighted)
ut.writefits(outpath / 'ccf_cleaned_error.fits', ccf_nne)
print('---Constructing KpVsys')
Kp, KpVsys, KpVsys_e = construct_KpVsys(rv, ccf_clean_weighted,
ccf_nne, dp)
ut.writefits(outpath / 'KpVsys.fits', KpVsys)
ut.writefits(outpath / 'KpVsys_e.fits', KpVsys_e)
ut.writefits(outpath / 'Kp.fits', Kp)
return
sys.exit()
if plot_xcor == True and inject_model == False:
print('---Plotting KpVsys')
analysis.plot_KpVsys(rv, Kp, KpVsys, dp)
#Now repeat it all for the model injection.
if inject_model == True:
for modelname in modellist:
outpath_i = outpath + modelname + '/'
if do_xcor == True:
print('---Injecting model ' + modelname)
list_of_orders_injected = models.inject_model(
list_of_wls,
list_of_orders,
dp,
modelname,
model_library=model_library
) #Start with the unnormalised orders from before.
#Normalize the orders to their average flux in order to effectively apply
#a broad-band colour correction (colour is a function of airmass and seeing).
if do_colour_correction == True:
print(
'------Normalizing injected orders to common flux level'
)
list_of_orders_injected, list_of_sigmas_injected, meanfluxes_injected = ops.normalize_orders(
list_of_orders_injected, list_of_sigmas, colourdeg)
meanfluxes_norm_injected = meanfluxes_injected / np.mean(
meanfluxes_injected)
else:
meanfluxes_norm_injected = fun.findgen(
len(list_of_orders_injected[0])
) * 0.0 + 1.0 #All unity.
print('------Cross-correlating injected orders')
rv_i, ccf_i, ccf_e_i, Tsums_i = analysis.xcor(
list_of_wls,
list_of_orders_injected,
np.flipud(np.flipud(wlt)),
T,
drv,
RVrange,
list_of_errors=list_of_sigmas_injected)
print('------Writing injected CCFs to ' + outpath_i)
if not os.path.exists(outpath_i):
print("---------That path didn't exist, I made it now.")
os.makedirs(outpath_i)
ut.writefits(outpath_i + '/' + 'ccf_i_' + modelname + '.fits',
ccf_i)
ut.writefits(
outpath_i + '/' + 'ccf_e_i_' + modelname + '.fits',
ccf_e_i)
else:
print('---Reading injected CCFs from ' + outpath_i)
if os.path.isfile(outpath_i + 'ccf_i_' + modelname +
'.fits') == False:
print('------ERROR: Injected CCF not located at ' +
outpath_i + 'ccf_i_' + modelname + '.fits' +
'. Set do_xcor and inject_model to True?')
sys.exit()
if os.path.isfile(outpath_i + 'ccf_e_i_' + modelname +
'.fits') == False:
print('------ERROR: Injected CCF error not located at ' +
outpath_i + 'ccf_e_i_' + modelname + '.fits' +
'. Set do_xcor and inject_model to True?')
sys.exit()
# f.close()
# f2.close()
ccf_i = fits.getdata(outpath_i + 'ccf_i_' + modelname +
'.fits')
ccf_e_i = fits.getdata(outpath_i + 'ccf_e_i_' + modelname +
'.fits')
print('---Cleaning injected CCFs')
ccf_n_i, ccf_ne_i, ccf_nn_i, ccf_nne_i = cleaning.clean_ccf(
rv, ccf_i, ccf_e_i, dp)
ut.writefits(outpath_i + 'ccf_normalized_i.fits', ccf_nn_i)
ut.writefits(outpath_i + 'ccf_ne_i.fits', ccf_ne_i)
# if make_doppler_model == True and skip_doppler_model == False:
# shadow.construct_doppler_model(rv,ccf_nn,dp,shadowname,xrange=[-200,200],Nxticks=20.0,Nyticks=10.0)
if skip_doppler_model == False:
# print('---Reading doppler shadow model from '+shadowname)
# doppler_model,maskHW = shadow.read_shadow(dp,shadowname,rv,ccf)
ccf_clean_i, matched_ds_model_i = shadow.match_shadow(
rv, ccf_nn_i, dp, doppler_model, maskHW)
else:
print(
'---Not performing shadow correction on injected spectra either.'
)
ccf_clean_i = ccf_nn_i * 1.0
matched_ds_model_i = ccf_clean_i * 0.0
if f_w > 0.0:
ccf_clean_i_filtered, wiggles_i = cleaning.filter_ccf(
rv, ccf_clean_i, v_width=f_w)
else:
ccf_clean_i_filtered = ccf_clean_i * 1.0
ut.writefits(outpath_i + 'ccf_cleaned_i.fits',
ccf_clean_i_filtered)
ut.writefits(outpath + 'ccf_cleaned_i_error.fits', ccf_nne)
print(
'---Weighing injected CCF rows by mean fluxes that were normalised out'
)
ccf_clean_i_filtered = np.transpose(
np.transpose(ccf_clean_i_filtered) * meanfluxes_norm_injected
) #MULTIPLYING THE AVERAGE FLUXES BACK IN! NEED TO CHECK THAT THIS ALSO GOES PROPERLY WITH THE ERRORS!
ccf_nne_i = np.transpose(
np.transpose(ccf_nne_i) * meanfluxes_norm_injected)
print('---Constructing injected KpVsys')
Kp, KpVsys_i, KpVsys_e_i = analysis.construct_KpVsys(
rv, ccf_clean_i_filtered, ccf_nne_i, dp)
ut.writefits(outpath_i + 'KpVsys_i.fits', KpVsys_i)
# ut.writefits(outpath+'KpVsys_e_i.fits',KpVsys_e_i)
if plot_xcor == True:
print('---Plotting KpVsys with ' + modelname + ' injected.')
analysis.plot_KpVsys(rv, Kp, KpVsys, dp, injected=KpVsys_i)
def blur_rotate(wl, order, dv, Rp, P, inclination, status=False, fast=False):
"""This function takes a spectrum and blurs it using a rotation x Gaussian
kernel which has a FWHM width of dv km/s everywhere. Meaning that its width changes
dynamically.
Because the kernel needs to be recomputed on each element of the wavelength axis
individually, this operation is much slower than convolution with a constant kernel,
in which a simple shifting of the array, rather than a recomputation of the rotation
profile is sufficient. By setting the fast keyword, the input array will first
be oversampled onto a constant-velocity grid to enable the usage of a constant kernel,
after which the result is interpolated back to the original grid.
Input:
The wavelength axis wl.
The spectral axis order.
The FHWM width of the resolution element in km/s.
The Radius of the rigid body in Rj.
The periodicity of the rigid body rotation in days.
The inclination of the spin axis in degrees.
Wavelength and order need to be numpy arrays and have the same number of elements.
Rp, P and i need to be scalar floats.
Output:
The blurred spectral axis, with the same dimensions as wl and order.
WARNING: THIS FUNCTION HANDLES NANS POORLY. I HAVE THEREFORE DECIDED CURRENTLY
TO REQUIRE NON-NAN INPUT.
This computes the simple numerical derivative of x by convolving with kernel [-1,0,1].
Parameters
----------
wl : list, np.ndarray
The wavelength array.
order : list, np.ndarray.
The spectral axis.
dv: float
The FWHM of a resolution element in km/s.
Rp: float
The radius of the planet in jupiter radii.
P: float
The rotation period of the planet. For tidally locked planets, this is equal
to the orbital period.
inclination:
The inclination of the spin axis in degrees. Presumed to be close to 90 degrees
for transiting planets
status: bool
Output a statusbar, but only if fast == False.
fast: bool
Re-interpolate the input on a constant-v grid in order to speed up the computation
of the convolution by eliminating the need to re-interpolate the kernel every step.
Returns
-------
order_blurred : np.array
The rotation-broadened spectrum on the same wavelength grid as the input.
Example
-------
>>> import tayph.functions as fun
>>> wl = fun.findgen(4000)*0.001+500.0
>>> fx = wl*0.0
>>> fx[2000] = 1.0
>>> fx_blurred1 = blur_rotate(wl,fx,3.0,1.5,0.8,90.0,status=False,fast=False)
>>> fx_blurred2 = blur_rotate(wl,fx,3.0,1.5,0.8,90.0,status=False,fast=True)
"""
import numpy as np
import tayph.util as ut
import tayph.functions as fun
from tayph.vartests import typetest, nantest, dimtest
from matplotlib import pyplot as plt
import astropy.constants as const
import astropy.units as u
import time
import sys
import pdb
from scipy import interpolate
typetest(dv, float, 'dv in blur_rotate()')
typetest(wl, [list, np.ndarray], 'wl in blur_rotate()')
typetest(order, [list, np.ndarray], 'order in blur_rotate()')
typetest(P, float, 'P in blur_rotate()')
typetest(Rp, float, 'Rp in blur_rotate()')
typetest(inclination, float, 'inclination in blur_rotate()')
typetest(status, bool, 'status in blur_rotate()')
typetest(fast, bool, 'fast in blur_rotate()')
nantest(wl, 'dv in blur_rotate()')
nantest(order, 'order in blur_rotate()')
dimtest(wl, [0], 'wl in blur_rotate()')
dimtest(order, [len(wl)],
'order in blur_rotate()') #Test that wl and order are 1D, and that
#they have the same length.
if np.min(np.array([dv, P, Rp])) <= 0.0:
raise Exception(
"ERROR in blur_rotate: dv, P and Rp should be strictly positive.")
#ut.typetest_array('wl',wl,np.float64)
#ut.typetest_array('order',order,np.float64)
#This is not possible because order may be 2D...
#And besides, you can have floats, np.float32 and np.float64... All of these would
#need to pass. Need to fix typetest_array some day.
order_blurred = order * 0.0 #init the output.
truncsize = 5.0 #The gaussian is truncated at 5 sigma from the extremest points of the RV amplitude.
sig_dv = dv / (2 * np.sqrt(2.0 * np.log(2))
) #Transform FWHM to Gaussian sigma. In km/s.
deriv = derivative(wl)
if max(deriv) < 0:
raise Exception(
"ERROR in ops.blur_rotate: WL derivative is smaller than 1.0. Sort wl in ascending order."
)
sig_wl = wl * sig_dv / (const.c.to('km/s').value) #in nm
sig_px = sig_wl / deriv
n = 1000.0
a = fun.findgen(n) / (n - 1) * np.pi
rv = np.cos(a) * np.sin(
np.radians(inclination)) * (2.0 * np.pi * Rp * const.R_jup /
(P * u.day)).to('km/s').value #in km/s
trunc_dist = np.round(sig_px * truncsize + np.max(rv) * wl /
(const.c.to('km/s').value) / deriv).astype(int)
# print('Maximum rotational rv: %s' % max(rv))
# print('Sigma_px: %s' % np.nanmean(np.array(sig_px)))
rvgrid_max = (np.max(trunc_dist) + 1.0) * sig_dv + np.max(rv)
rvgrid_n = rvgrid_max / dv * 100.0 #100 samples per lsf fwhm.
rvgrid = (
fun.findgen(2 * rvgrid_n + 1) - rvgrid_n
) / rvgrid_n * rvgrid_max #Need to make sure that this is wider than the truncation bin and more finely sampled than wl - everywhere.
lsf = rvgrid * 0.0
#We loop through velocities in the velocity grid to build up the sum of Gaussians
#that is the LSF.
for v in rv:
lsf += fun.gaussian(
rvgrid, 1.0, v, sig_dv
) #This defines the LSF on a velocity grid wih high fidelity.
if fast:
wlt, fxt, dv = constant_velocity_wl_grid(wl, order, 4)
dv_grid = rvgrid[1] - rvgrid[0]
len_rv_grid_low = int(max(rvgrid) / dv * 2 - 2)
# print(len_rv_grid_low)
# print(len(fun.findgen(len_rv_grid_low)))
# print(len_rv_grid_low%2)
if len_rv_grid_low % 2 == 0:
len_rv_grid_low -= 1
rvgrid_low = fun.findgen(
len_rv_grid_low) * dv #Slightly smaller than the original grid.
rvgrid_low -= 0.5 * np.max(rvgrid_low)
lsf_low = interpolate.interp1d(rvgrid, lsf)(rvgrid_low)
lsf_low /= np.sum(
lsf_low
) #This is now an LSF on a grid with the same spacing as the data has.
#This means I can use it directly as a convolution kernel:
fxt_blurred = convolve(fxt, lsf_low, edge_degree=1, fit_width=1)
#And interpolate back to where it came from:
order_blurred = interpolate.interp1d(wlt,
fxt_blurred,
bounds_error=False)(wl)
#I can use interp1d because after blurring, we are now oversampled.
# order_blurred2 = bin_avg(wlt,fxt_blurred,wl)
return (order_blurred)
#Now we loop through the wavelength grid to place this LSF at each wavelength position.
for i in range(0, len(wl)):
binstart = max([0, i - trunc_dist[i]])
binend = i + trunc_dist[i]
wlbin = wl[binstart:binend]
wlgrid = wl[i] * rvgrid / (const.c.to('km/s').value) + wl[
i] #This converts the velocity grid to a d-wavelength grid centered on wk[i]
#print([np.min(wlbin),np.min(wlgrid),np.max(wlbin),np.max(wlgrid)])
i_wl = interpolate.interp1d(
wlgrid, lsf, bounds_error=False, fill_value='extrapolate'
) #Extrapolate should not be necessary but sometimes there is a minute mismatch between the
#start and end wavelengths of the constructed grid and the bin.
try:
lsf_wl = i_wl(wlbin)
except:
ut.tprint(
'Error in interpolating LSF onto wlbin. Pausing to debug.')
pdb.set_trace()
k_n = lsf_wl / np.sum(
lsf_wl
) #Normalize at each instance of the interpolation to make sure flux is conserved exactly.
order_blurred[i] = np.sum(k_n * order[binstart:binend])
if status == True:
ut.statusbar(i, len(wl))
return (order_blurred)
def manual_masking(list_of_wls,list_of_orders,list_of_masks,Nxticks = 20,Nyticks = 10,saved = []):
import numpy as np
import matplotlib.pyplot as plt
import pdb
import tayph.drag_colour as dcb
import tayph.util as ut
from tayph.vartests import typetest,postest,dimtest
import tayph.functions as fun
import sys
from matplotlib.widgets import Slider, Button, RadioButtons, CheckButtons
"""
This brings the user into a GUI in which he/she/they can both visualize
spectral orders and mask regions of bad data; such as occur at edges
of orders, inside deep spectral lines (stellar or telluric) or other places.
In the standard workflow, this routine succeeds an automatic masking step that has
performed a sigma clipping using a rolling standard deviation. This procedure has
added a certain number of pixels to the mask. If this routine is subsequently
called, the user can mask out bad *columns* by selecting them in the GUI. These
are then added to that mask as well.
"""
typetest(Nxticks,int,'Nxticks in manual_masking()')
typetest(Nyticks,int,'Nyticks in manual_masking()')
postest(Nxticks,'Nxticks in manual_masking()')
postest(Nyticks,'Nyticks in manual_masking()')
typetest(list_of_wls,list,'list_of_wls in manual_masking()')
typetest(list_of_orders,list,'list_of_orders in manual_masking()')
typetest(list_of_masks,list,'list_of_masks in manual_masking()')
typetest(saved,list,'saved list_of_masks in manual_masking()')
dimtest(list_of_wls,[0,0],'list_of_wls in manual_masking()')
dimtest(list_of_orders,[len(list_of_wls),0,0],'list_of_orders in manual_masking()')
dimtest(list_of_masks,[len(list_of_wls),0,0],'list_of_masks in manual_masking()')
print('------Entered manual masking mode')
M = mask_maker(list_of_wls,list_of_orders,saved,Nxticks=Nxticks,Nyticks=Nyticks,nsigma=3.0) #Mask callback
#This initializes all the parameters of the plot. Which order it is plotting, what
#dimensions these have, the actual data arrays to plot; etc. Initialises on order 56.
#I also dumped most of the buttons and callbacks into there; so this thing
#is probably a bit of a sphaghetti to read.
#Here we only need to define the buttons and add them to the plot. In fact,
#I could probably have shoved most of this into the class as well, (see
#me defining all these buttons as class attributes? But, I
#suppose that the structure is a bit more readable this way.
#The button to add a region to the mask:
rax_sub = plt.axes([0.8, 0.4, 0.14, 0.05])
M.bsub = Button(rax_sub, ' Unmask columns ')
M.bsub.on_clicked(M.subtract)
#The button to add a region to the mask:
rax_add = plt.axes([0.8, 0.48, 0.14, 0.05])
M.badd = Button(rax_add, ' Mask columns ')
M.badd.color=M.col_passive[0]
M.badd.hovercolor=M.col_passive[1]
M.add_connector = M.badd.on_clicked(M.add)
#The button to add a region to the mask:
rax_all = plt.axes([0.8, 0.56, 0.14, 0.05])
M.ball = Button(rax_all, ' Apply to all ')
M.ball.on_clicked(M.applyall)
rax_future = plt.axes([0.8, 0.64, 0.14, 0.05])
M.ballf = Button(rax_future, ' Apply to all above')
M.ballf.on_clicked(M.applyallfuture)
rax_clearentire = plt.axes([0.8, 0.72, 0.14, 0.05])
M.bcent = Button(rax_clearentire, ' Unmask order ')
M.bcent.on_clicked(M.cleartotal)
#The button to add a region to the mask:
rax_entire = plt.axes([0.8, 0.8, 0.14, 0.05])
M.bent = Button(rax_entire, ' Mask order ')
M.bent.on_clicked(M.applytotal)
# rax_atv = plt.axes([0.9, 0.45, 0.1, 0.15])
# clabels = ['To air', 'No conversion','To vaccuum']
# radio = RadioButtons(rax_atv,clabels)
# radio.on_clicked(M.atv)
#The mask width:
rax_slider = plt.axes([0.8, 0.3, 0.14, 0.02])
rax_slider.set_title('Mask width')
M.MW_slider = Slider(rax_slider,'', 1,200,valinit=M.MW,valstep=1)#Store the slider in the model class
M.MW_slider.on_changed(M.slide_maskwidth)
#The slider to cycle through orders:
rax_slider = plt.axes([0.8, 0.2, 0.14, 0.02])
rax_slider.set_title('Order')
M.mask_slider = Slider(rax_slider,'', 0,M.N_orders-1,valinit=M.N,valstep=1)#Store the slider in the model class
M.mask_slider.on_changed(M.slide_order)
#The Previous order button:
rax_prev = plt.axes([0.8, 0.1, 0.04, 0.05])
bprev = Button(rax_prev, ' <<< ')
bprev.on_clicked(M.previous)
#The Next order button:
rax_next = plt.axes([0.86, 0.1, 0.04, 0.05])
bnext = Button(rax_next, ' >>> ')
bnext.on_clicked(M.next)
#The save button:
rax_save = plt.axes([0.92, 0.1, 0.07, 0.05])
bsave = Button(rax_save, 'Save')
bsave.on_clicked(M.save)
#The cancel button:
rax_cancel = plt.axes([0.92, 0.025, 0.07, 0.05])
bcancel = Button(rax_cancel, 'Cancel')
bcancel.on_clicked(M.cancel)
plt.show()
#When pressing "save", the figure is closed, the suspesion caused by plt.show() is
#lifted and we continue to exit this GUI function, as its job has been done:
#return all the columns that were selected by the user.
return(M.list_of_selected_columns)
def clean_ccf(rv, ccf, ccf_e, dp):
"""
This routine normalizes the CCF fluxes and subtracts the average out of
transit CCF, using the transit lightcurve as a mask.
Parameters
----------
rv : np.ndarray
The radial velocity axis
ccf : np.ndarray
The CCF with second dimension matching the length of rv.
ccf_e : np.ndarray
The error on ccf.
dp : str or path-like
The datapath of the present dataset, to establish which exposures in ccf
are in or out of transit.
Returns
-------
ccf_n : np.ndarray
The transit-lightcurve normalised CCF.
ccf_ne : np.ndarray
The error on ccf_n
ccf_nn : np.ndarray
The CCF relative to the out-of-transit time-averaged, if sufficient (>25%
of the time-series) out of transit exposures were available. Otherwise, the
average over the entire time-series is used.
ccf_ne : np.array
The error on ccf_nn.
"""
import numpy as np
import tayph.functions as fun
import tayph.util as ut
from matplotlib import pyplot as plt
import pdb
import math
import tayph.system_parameters as sp
import tayph.operations as ops
import astropy.io.fits as fits
import sys
import copy
from tayph.vartests import typetest, dimtest, nantest
typetest(rv, np.ndarray, 'rv in clean_ccf()')
typetest(ccf, np.ndarray, 'ccf in clean_ccf')
typetest(ccf_e, np.ndarray, 'ccf_e in clean_ccf')
dp = ut.check_path(dp)
dimtest(ccf, [0, len(rv)])
dimtest(ccf_e, [0, len(rv)])
nantest(rv, 'rv in clean_ccf()')
nantest(ccf, 'ccf in clean_ccf()')
nantest(ccf_e, 'ccf_e in clean_ccf()')
#ADD PARAMGET DV HERE.
transit = sp.transit(dp)
# transitblock = fun.rebinreform(transit,len(rv))
Nrv = int(math.floor(len(rv)))
baseline_ccf = np.hstack((ccf[:,
0:int(0.25 * Nrv)], ccf[:,
int(0.75 * Nrv):]))
baseline_ccf_e = np.hstack(
(ccf_e[:, 0:int(0.25 * Nrv)], ccf_e[:, int(0.75 * Nrv):]))
baseline_rv = np.hstack((rv[0:int(0.25 * Nrv)], rv[int(0.75 * Nrv):]))
meanflux = np.median(
baseline_ccf, axis=1
) #Normalize the baseline flux, but away from the signal of the planet.
meanflux_e = 1.0 / len(baseline_rv) * np.sqrt(
np.nansum(baseline_ccf_e**2.0, axis=1)) #1/N times sum of squares.
#I validated that this is approximately equal to ccf_e/sqrt(N).
meanblock = fun.rebinreform(meanflux, len(rv))
meanblock_e = fun.rebinreform(meanflux_e, len(rv))
ccf_n = ccf / meanblock.T
ccf_ne = np.abs(ccf_n) * np.sqrt(
(ccf_e / ccf)**2.0 + (meanblock_e.T / meanblock.T)**
2.0) #R=X/Z -> dR = R*sqrt( (dX/X)^2+(dZ/Z)^2 )
#I validated that this is essentially equal to ccf_e/meanblock.T; as expected because the error on the mean spectrum is small compared to ccf_e.
if np.sum(transit == 1) == 0:
print(
'------WARNING in Cleaning: The data contains only in-transit exposures.'
)
print('------The mean ccf is taken over the entire time-series.')
meanccf = np.nanmean(ccf_n, axis=0)
meanccf_e = 1.0 / len(transit) * np.sqrt(
np.nansum(ccf_ne**2.0,
axis=0)) #I validated that this is approximately equal
#to sqrt(N)*ccf_ne, where N is the number of out-of-transit exposures.
elif np.sum(transit == 1) <= 0.25 * len(transit):
print(
'------WARNING in Cleaning: The data contains very few (<25%) out of transit exposures.'
)
print('------The mean ccf is taken over the entire time-series.')
meanccf = np.nanmean(ccf_n, axis=0)
meanccf_e = 1.0 / len(transit) * np.sqrt(
np.nansum(ccf_ne**2.0,
axis=0)) #I validated that this is approximately equal
#to sqrt(N)*ccf_ne, where N is the number of out-of-transit exposures.
if np.min(transit) == 1.0:
print(
'------WARNING in Cleaning: The data is not predicted to contain in-transit exposures.'
)
print(
f'------If you expect to be dealing with transit-data, please check the ephemeris '
f'at {dp}')
print('------The mean ccf is taken over the entire time-series.')
meanccf = np.nanmean(ccf_n, axis=0)
meanccf_e = 1.0 / len(transit) * np.sqrt(
np.nansum(ccf_ne**2.0,
axis=0)) #I validated that this is approximately equal
#to sqrt(N)*ccf_ne, where N is the number of out-of-transit exposures.
else:
meanccf = np.nanmean(ccf_n[transit == 1.0, :], axis=0)
meanccf_e = 1.0 / np.sum(transit == 1) * np.sqrt(
np.nansum(ccf_ne[transit == 1.0, :]**2.0,
axis=0)) #I validated that this is approximately equal
#to sqrt(N)*ccf_ne, where N is the number of out-of-transit exposures.
meanblock2 = fun.rebinreform(meanccf, len(meanflux))
meanblock2_e = fun.rebinreform(meanccf_e, len(meanflux))
ccf_nn = ccf_n / meanblock2 #MAY NEED TO DO SUBTRACTION INSTEAD TOGETHER W. NORMALIZATION OF LIGHTCURVE. SEE ABOVE.
ccf_nne = np.abs(
ccf_n / meanblock2) * np.sqrt((ccf_ne / ccf_n)**2.0 +
(meanblock2_e / meanblock2)**2.0)
#I validated that this error is almost equal to ccf_ne/meanccf
#ONLY WORKS IF LIGHTCURVE MODEL IS ACCURATE, i.e. if Euler observations are available.
# print("---> WARNING IN CLEANING.CLEAN_CCF(): NEED TO ADD A FUNCTION THAT YOU CAN NORMALIZE BY THE LIGHTCURVE AND SUBTRACT INSTEAD OF DIVISION!")
return (ccf_n, ccf_ne, ccf_nn - 1.0, ccf_nne)
def mask_orders(list_of_wls,list_of_orders,dp,maskname,w,c_thresh,manual=False):
"""
This code takes the list of orders and masks out bad pixels.
It combines two steps, a simple sigma clipping step and a manual step, where
the user can interactively identify bad pixels in each order. The sigma
clipping is done on a threshold of c_thresh, using a rolling standard dev.
with a width of w pixels. Manual masking is a big routine needed to support
a nice GUI to do that.
If c_thresh is set to zero, sigma clipping is skipped. If manual=False, the
manual selection of masking regions (which is manual labour) is turned off.
If both are turned off, the list_of_orders is returned unchanged.
If either or both are active, the routine will output 1 or 2 FITS files that
contain a stack (cube) of the masks for each order. The first file is the mask
that was computed automatically, the second is the mask that was constructed
manually. This is done so that the manual mask can be transplanted onto another
dataset, or saved under a different file-name, to limit repetition of work.
At the end of the routine, the two masks are merged into a single list, and
applied to the list of orders.
"""
import tayph.operations as ops
import numpy as np
import tayph.functions as fun
import tayph.plotting as plotting
import sys
import matplotlib.pyplot as plt
import tayph.util as ut
import warnings
from tayph.vartests import typetest,dimtest,postest
ut.check_path(dp)
typetest(maskname,str,'maskname in mask_orders()')
typetest(w,[int,float],'w in mask_orders()')
typetest(c_thresh,[int,float],'c_thresh in mask_orders()')
postest(w,'w in mask_orders()')
postest(c_thresh,'c_thresh in mask_orders()')
typetest(list_of_wls,list,'list_of_wls in mask_orders()')
typetest(list_of_orders,list,'list_of_orders in mask_orders()')
typetest(manual,bool,'manual keyword in mask_orders()')
dimtest(list_of_wls,[0,0],'list_of_wls in mask_orders()')
dimtest(list_of_orders,[len(list_of_wls),0,0],'list_of_orders in mask_orders()')
if c_thresh <= 0 and manual == False:
print('---WARNING in mask_orders: c_thresh is set to zero and manual masking is turned off.')
print('---Returning orders unmasked.')
return(list_of_orders)
N = len(list_of_orders)
void = fun.findgen(N)
list_of_orders = ops.normalize_orders(list_of_orders,list_of_orders)[0]#first normalize. Dont want outliers to
#affect the colour correction later on, so colour correction cant be done before masking, meaning
#that this needs to be done twice; as colour correction is also needed for proper maskng. The second variable is
#a dummy to replace the expected list_of_sigmas input.
N_NaN = 0
list_of_masked_orders = []
for i in range(N):
list_of_masked_orders.append(list_of_orders[i])
list_of_masks = []
if c_thresh > 0:#Check that c_thresh is positive. If not, skip sigma clipping.
print('------Sigma-clipping mask')
for i in range(N):
order = list_of_orders[i]
N_exp = np.shape(order)[0]
N_px = np.shape(order)[1]
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=RuntimeWarning)
meanspec = np.nanmean(order,axis = 0)
meanblock = fun.rebinreform(meanspec,N_exp)
res = order / meanblock - 1.0
sigma = fun.running_MAD_2D(res,w)
with np.errstate(invalid='ignore'):#https://stackoverflow.com/questions/25345843/inequality-comparison-of-numpy-array-with-nan-to-a-scalar
sel = np.abs(res) >= c_thresh*sigma
N_NaN += np.sum(sel)#This is interesting because True values count as 1, and False as zero.
order[sel] = np.nan
list_of_masks.append(order*0.0)
ut.statusbar(i,void)
print(f'%s outliers identified and set to NaN ({N_NaN}/{round(N_NaN/np.size(list_of_masks)*100.0,3)}).')
else:
print('------Skipping sigma-clipping (c_thres <= 0)')
#Do nothing to list_of_masks. It is now an empty list.
#We now automatically proceed to manual masking, because at this point
#it has already been established that it must have been turned on.
list_of_masks_manual = []
if manual == True:
previous_list_of_masked_columns = load_columns_from_file(dp,maskname,mode='relaxed')
list_of_masked_columns = manual_masking(list_of_wls,list_of_orders,list_of_masks,saved = previous_list_of_masked_columns)
print('------Successfully concluded manual mask.')
write_columns_to_file(dp,maskname,list_of_masked_columns)
print('------Building manual mask from selected columns')
for i in range(N):
order = list_of_orders[i]
N_exp = np.shape(order)[0]
N_px = np.shape(order)[1]
list_of_masks_manual.append(np.zeros((N_exp,N_px)))
for j in list_of_masked_columns[i]:
list_of_masks_manual[i][:,int(j)] = np.nan
#We write 1 or 2 mask files here. The list of manual masks
#and list_of_masks (auto) are either filled, or either is an emtpy list if
#c_thresh was set to zero or manual was set to False (because they were defined
#as empty lists initially, and then not filled with anything).
write_mask_to_file(dp,maskname,list_of_masks,list_of_masks_manual)
return(0)
def get_model(name,library='models/library',root='models',is_binary=False):
"""This program queries a model from a library file, with predefined models
for use in model injection, cross-correlation and plotting. These models have
a standard format. They are 2 rows by N points, where N corresponds to the
number of wavelength samples. The first row contains the wavelengths, the
second the associated flux values. The library file has two columns that are
formatted as follows:
modelname modelpath
modelname modelpath
modelname modelpath
modelpath starts in the models/ subdirectory.
Set the root variable if a location is required other than the ./models directory.
Example call:
wlm,fxm = get_model('WASP-121_TiO',library='models/testlib')
Set the is_binary keyword to ask whether the file is a binary .dat file or not, and return a
single boolean. Used to switch between cross-correlation and template construction modes.
"""
from tayph.vartests import typetest,dimtest
from tayph.util import check_path
from astropy.io import fits
from pathlib import Path
import errno
import os
import numpy as np
typetest(name,str,'name in mod.get_model()')
library=check_path(library,exists=True)
#First open the library file.
f = open(library, 'r')
x = f.read().splitlines()#Read everything into a big string array.
f.close()
n_lines=len(x)#Number of models in the library.
models={}#This will contain the model names.
for i in range(0,n_lines):
line=x[i].split()
value=(line[1])
models[line[0]] = value
try:
modelpath=Path(models[name])
except KeyError:
raise KeyError(f'Model {name} is not present in library at {str(library)}') from None
if str(modelpath)[0]!='/':#Test if this is an absolute path.
root=check_path(root,exists=True)
modelpath=root/modelpath
if modelpath.exists():
if modelpath.suffix == '.fits':
if is_binary: return(False)
modelarray=fits.getdata(modelpath)#Two-dimensional array with wl on the first row and
#spectral flux on the second row.
elif modelpath.suffix == '.dat':
if is_binary: return(True)
modelarray = np.loadtxt(modelpath).T#Two-dimensional array with wavelength positions of
#spectral lines on the first column and weights on the second column (transposed).
else:
raise RunTimeError(f'Model file {modelpath} from library {str(library)} is required to '
'have extension .fits or .dat.')
else:
raise FileNotFoundError(f'Model file {modelpath} from library {str(library)} does not '
'exist.')
dimtest(modelarray,[2,0])
return(modelarray[0,:],modelarray[1,:])
