def w2p(dispersion, w, extrapolate=True):
"""
Convert wavelength values to pixels for a "dispersion solution".
Parameters
----------
dispersion : 1D array
The dispersion solution. This is basically just a 1D array of
monotonically increasing (or decreasing) wavelengths.
w : array
Array of wavelength values to convert to pixels.
extrapolate : bool, optional
Extrapolate beyond the dispersion solution, if necessary.
This is True by default.
Returns
-------
x : array
Array of converted pixel values.
Example
-------
.. code-block:: python
x = w2p(disp,w)
"""
x = interp1d(dispersion,
np.arange(len(dispersion)),
kind='cubic',
bounds_error=False,
fill_value=(np.nan, np.nan),
assume_sorted=False)(w)
# Need to extrapolate
if ((np.min(w) < np.min(dispersion)) |
(np.max(w) > np.max(dispersion))) & (extrapolate is True):
win = dispersion
xin = np.arange(len(dispersion))
si = np.argsort(win)
win = win[si]
xin = xin[si]
npix = len(win)
# At the beginning
if (np.min(w) < np.min(dispersion)):
#coef1 = dln.poly_fit(win[0:10], xin[0:10], 2)
coef1 = dln.quadratic_coefficients(win[0:10], xin[0:10])
bd1, nbd1 = dln.where(w < np.min(dispersion))
x[bd1] = dln.poly(w[bd1], coef1)
# At the end
if (np.max(w) > np.max(dispersion)):
#coef2 = dln.poly_fit(win[npix-10:], xin[npix-10:], 2)
coef2 = dln.quadratic_coefficients(win[npix - 10:],
xin[npix - 10:])
bd2, nbd2 = dln.where(w > np.max(dispersion))
x[bd2] = dln.poly(w[bd2], coef2)
return x
def p2w(dispersion, x, extrapolate=True):
"""
Convert pixel values to wavelengths for a "dispersion solution".
Parameters
----------
dispersion : 1D array
The dispersion solution. This is basically just a 1D array of
monotonically increasing (or decreasing) wavelengths.
x : array
Array of pixel values to convert to wavelengths.
extrapolate : bool, optional
Extrapolate beyond the dispersion solution, if necessary.
This is True by default.
Returns
-------
w : array
Array of converted wavelengths.
Example
-------
.. code-block:: python
w = p2w(disp,x)
"""
npix = len(dispersion)
w = interp1d(np.arange(len(dispersion)),
dispersion,
kind='cubic',
bounds_error=False,
fill_value=(np.nan, np.nan),
assume_sorted=False)(x)
# Need to extrapolate
if ((np.min(x) < 0) | (np.max(x) > (npix - 1))) & (extrapolate is True):
xin = np.arange(npix)
win = dispersion
# At the beginning
if (np.min(x) < 0):
#coef1 = dln.poly_fit(xin[0:10], win[0:10], 2)
coef1 = dln.quadratic_coefficients(xin[0:10], win[0:10])
bd1, nbd1 = dln.where(x < 0)
w[bd1] = dln.poly(x[bd1], coef1)
# At the end
if (np.max(x) > (npix - 1)):
#coef2 = dln.poly_fit(xin[npix-10:], win[npix-10:], 2)
coef2 = dln.quadratic_coefficients(xin[npix - 10:],
win[npix - 10:])
bd2, nbd2 = dln.where(x > (npix - 1))
w[bd2] = dln.poly(x[bd2], coef2)
return w
def maskoutliers(spec,nsig=5,verbose=False,logger=None):
"""
Mask large positive outliers and negative flux pixels in the spectrum.
"""
if logger is None:
logger = dln.basiclogger()
spec2 = spec.copy()
wave = spec2.wave.copy().reshape(spec2.npix,spec2.norder) # make 2D
flux = spec2.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D
err = spec2.err.copy().reshape(spec2.npix,spec2.norder) # make 2D
mask = spec2.mask.copy().reshape(spec2.npix,spec2.norder) # make 2D
totnbd = 0
for o in range(spec2.norder):
w = wave[:,o].copy()
x = (w-np.median(w))/(np.max(w*0.5)-np.min(w*0.5)) # -1 to +1
y = flux[:,o].copy()
m = mask[:,o].copy()
# Divide by median
medy = np.nanmedian(y)
y /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x,y,2,robust=True)
sig = dln.mad(y-dln.poly(x,coef))
bd,nbd = dln.where( ((y-dln.poly(x,coef)) > nsig*sig) | (y<0))
totnbd += nbd
if nbd>0:
flux[bd,o] = dln.poly(x[bd],coef)*medy
err[bd,o] = 1e30
mask[bd,o] = True
# Flatten to 1D if norder=1
if spec2.norder==1:
flux = flux.flatten()
err = err.flatten()
mask = mask.flatten()
# Stuff back in
spec2.flux = flux
spec2.err = err
spec2.mask = mask
if verbose is True: logger.info('Masked '+str(totnbd)+' outlier or negative pixels')
return spec2
def maskdiscrepant(spec,model,nsig=10,verbose=False,logger=None):
"""
Mask pixels that are discrepant when compared to a model.
"""
if logger is None:
logger = dln.basiclogger()
spec2 = spec.copy()
wave = spec2.wave.copy().reshape(spec2.npix,spec2.norder) # make 2D
flux = spec2.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D
err = spec2.err.copy().reshape(spec2.npix,spec2.norder) # make 2D
mask = spec2.mask.copy().reshape(spec2.npix,spec2.norder) # make 2D
mflux = model.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D
totnbd = 0
for o in range(spec2.norder):
w = wave[:,o].copy()
x = (w-np.median(w))/(np.max(w*0.5)-np.min(w*0.5)) # -1 to +1
y = flux[:,o].copy()
m = mask[:,o].copy()
my = mflux[:,o].copy()
# Divide by median
medy = np.nanmedian(y)
y /= medy
my /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x,y,2,robust=True)
sig = dln.mad(y-my)
bd,nbd = dln.where( np.abs(y-my) > nsig*sig )
totnbd += nbd
if nbd>0:
flux[bd,o] = dln.poly(x[bd],coef)*medy
err[bd,o] = 1e30
mask[bd,o] = True
# Flatten to 1D if norder=1
if spec2.norder==1:
flux = flux.flatten()
err = err.flatten()
mask = mask.flatten()
# Stuff back in
spec2.flux = flux
spec2.err = err
spec2.mask = mask
if verbose is True: logger.info('Masked '+str(totnbd)+' discrepant pixels')
return spec2
def continuum(spec, norder=6, perclevel=90.0, binsize=0.1, interp=True):
"""
Measure the continuum of a spectrum.
Parameters
----------
spec : Spec1D object
A spectrum object. This at least needs
to have a FLUX and WAVE attribute.
norder : float, optional
Polynomial order to use for the continuum fitting.
The default is 6.
perclevel : float, optional
Percent level to use for the continuum value
in large bins. Default is 90.
binsize : float, optional
Fraction of the wavelength range (scaled to -1 to +1) to bin.
Default is 0.1.
interp : bool, optional
Use interpolation of the binned values instead of a polynomial
fit. Default is True.
Returns
-------
cont : numpy array
The continuum array, in the same shape as the input flux.
Examples
--------
.. code-block:: python
cont = continuum(spec)
"""
wave = spec.wave.copy().reshape(spec.npix, spec.norder) # make 2D
flux = spec.flux.copy().reshape(spec.npix, spec.norder) # make 2D
err = spec.err.copy().reshape(spec.npix, spec.norder) # make 2D
mask = spec.mask.copy().reshape(spec.npix, spec.norder) # make 2D
cont = err.copy() * 0.0 + 1
for o in range(spec.norder):
w = wave[:, o].copy()
wr = [np.min(w), np.max(w)]
x = (w - np.mean(wr)) / dln.valrange(wr) * 2 # -1 to +1
y = flux[:, o].copy()
m = mask[:, o].copy()
# Divide by median
medy = np.nanmedian(y)
if medy <= 0.0:
medy = 1.0
y /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x[~m], y[~m], 2, robust=True)
sig = dln.mad(y - dln.poly(x, coef))
bd, nbd = dln.where((y - dln.poly(x, coef)) > 5 * sig)
if nbd > 0: m[bd] = True
gdmask = (y > 0) & (m == False) # need positive fluxes and no mask set
if np.sum(gdmask) == 0:
continue
# Bin the data points
xr = [np.nanmin(x), np.nanmax(x)]
bins = int(np.ceil((xr[1] - xr[0]) / binsize))
ybin, bin_edges, binnumber = bindata.binned_statistic(
x[gdmask],
y[gdmask],
statistic='percentile',
percentile=perclevel,
bins=bins,
range=None)
xbin = bin_edges[0:-1] + 0.5 * binsize
# Interpolate to full grid
if interp is True:
fnt = np.isfinite(ybin)
cont1 = dln.interp(xbin[fnt],
ybin[fnt],
x,
kind='quadratic',
extrapolate=True,
exporder=1)
else:
coef = dln.poly_fit(xbin, ybin, norder)
cont1 = dln.poly(x, coef)
cont1 *= medy
cont[:, o] = cont1
# Flatten to 1D if norder=1
if spec.norder == 1:
cont = cont.flatten()
return cont
def apstar(filename,badval=20735):
"""
Read an SDSS APOGEE apStar spectrum.
Parameters
----------
filename : string
The name of the spectrum file to load.
Returns
-------
spec : Spec1D object
The spectrum as a Spec1D object.
Examples
--------
spec = apstar('spec.fits')
"""
base, ext = os.path.splitext(os.path.basename(filename))
# APOGEE apStar, combined spectrum
if (base.find("apStar") > -1) | (base.find("asStar") > -1):
# HISTORY APSTAR: HDU0 = Header only
# HISTORY APSTAR: All image extensions have:
# HISTORY APSTAR: row 1: combined spectrum with individual pixel weighting
# HISTORY APSTAR: row 2: combined spectrum with global weighting
# HISTORY APSTAR: row 3-nvisits+2: individual resampled visit spectra
# HISTORY APSTAR: unless nvisits=1, which only have a single row
# HISTORY APSTAR: All spectra shifted to rest (vacuum) wavelength scale
# HISTORY APSTAR: HDU1 - Flux (10^-17 ergs/s/cm^2/Ang)
# HISTORY APSTAR: HDU2 - Error (10^-17 ergs/s/cm^2/Ang)
# HISTORY APSTAR: HDU3 - Flag mask:
# HISTORY APSTAR: row 1: bitwise OR of all visits
# HISTORY APSTAR: row 2: bitwise AND of all visits
# HISTORY APSTAR: row 3-nvisits+2: individual visit masks
# HISTORY APSTAR: HDU4 - Sky (10^-17 ergs/s/cm^2/Ang)
# HISTORY APSTAR: HDU5 - Sky Error (10^-17 ergs/s/cm^2/Ang)
# HISTORY APSTAR: HDU6 - Telluric
# HISTORY APSTAR: HDU7 - Telluric Error
# HISTORY APSTAR: HDU8 - LSF coefficients
# HISTORY APSTAR: HDU9 - RV and CCF structure
# Get number of extensions
hdulist = fits.open(filename)
nhdu = len(hdulist)
hdulist.close()
# Spectrum, error, sky, skyerr are in units of 1e-17
# these are 2D arrays with [Nvisit+2,Npix]
# the first two are combined and the rest are the individual spectra
head1 = fits.getheader(filename,1)
w0 = np.float64(head1["CRVAL1"])
dw = np.float64(head1["CDELT1"])
nw = head1["NAXIS1"]
wave = 10**(np.arange(nw)*dw+w0)
# flux, err, sky, skyerr are in units of 1e-17
flux = fits.getdata(filename,1).T * 1e-17
lsfcoef = fits.getdata(filename,8).T
spec = Spec1D(flux,wave=wave,lsfpars=lsfcoef,lsftype='Gauss-Hermite',lsfxtype='Pixels')
spec.filename = filename
spec.sptype = "apStar"
spec.waveregime = "NIR"
spec.instrument = "APOGEE"
spec.head = fits.getheader(filename,0)
spec.err = fits.getdata(filename,2).T * 1e-17
#bad = (spec.err<=0) # fix bad error values
#if np.sum(bad) > 0:
# spec.err[bad] = 1e30
spec.bitmask = fits.getdata(filename,3)
spec.sky = fits.getdata(filename,4).T * 1e-17
spec.skyerr = fits.getdata(filename,5).T * 1e-17
spec.telluric = fits.getdata(filename,6).T
spec.telerr = fits.getdata(filename,7).T
spec.lsf = fits.getdata(filename,8).T
# Create the bad pixel mask
# "bad" pixels:
# flag = ['BADPIX','CRPIX','SATPIX','UNFIXABLE','BADDARK','BADFLAT','BADERR','NOSKY',
# 'LITTROW_GHOST','PERSIST_HIGH','PERSIST_MED','PERSIST_LOW','SIG_SKYLINE','SIG_TELLURIC','NOT_ENOUGH_PSF','']
# badflag = [1,1,1,1,1,1,1,1,
# 0,0,0,0,0,0,1,0]
mask = (np.bitwise_and(spec.bitmask,badval)!=0) | (np.isfinite(spec.flux)==False)
# Extra masking for bright skylines
x = np.arange(spec.npix)
nsky = 4
medsky = median_filter(spec.sky,201,mode='reflect')
medcoef = dln.poly_fit(x,medsky/np.median(medsky),2)
medsky2 = dln.poly(x,medcoef)*np.median(medsky)
skymask1 = (sky>nsky*medsky2) # pixels Nsig above median sky
mask[:,i] = np.logical_or(mask[:,i],skymask1) # OR combine
spec.mask = mask
# Fix NaN or bad pixels pixels
bd,nbd = dln.where( (np.isfinite(spec.flux[:,i])==False) | (spec.err[:,i] <= 0.0) )
if nbd>0:
spec.flux[bd] = 0.0
spec.err[bd] = 1e30
spec.mask[bd] = True
if nhdu>=9:
spec.meta = fits.getdata(filename,9) # meta-data
spec.snr = spec.head["SNR"]
if base.find("apStar") > -1:
spec.observatory = 'apo'
else:
spec.observatory = 'lco'
spec.wavevac = True
return spec
def normalize(self, ncorder=6, perclevel=0.95):
"""
Normalize the spectrum.
Parameters
----------
ncorder : float, optional
Polynomial order to use for the continuum fitting.
The default is 6.
perclevel : float, optional
Percent level (1.0 for 100%) to use for the continuum value
in large bins. Default is 0.95.
Returns
-------
The flux and err arrays will normalized (divided) by the continuum
and the continuum saved in cont. The normalized property is set to
True.
Examples
--------
spec.normalize()
"""
self._flux = self.flux # Save the original
#nspec, cont, masked = normspec(self,ncorder=ncorder,perclevel=perclevel)
binsize = 0.10
perclevel = 90.0
wave = self.wave.copy().reshape(self.npix, self.norder) # make 2D
flux = self.flux.copy().reshape(self.npix, self.norder) # make 2D
err = self.err.copy().reshape(self.npix, self.norder) # make 2D
mask = self.mask.copy().reshape(self.npix, self.norder) # make 2D
cont = err.copy() * 0.0 + 1
for o in range(self.norder):
w = wave[:, o].copy()
x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5)
) # -1 to +1
y = flux[:, o].copy()
m = mask[:, o].copy()
# Divide by median
medy = np.nanmedian(y)
y /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x, y, 2, robust=True)
sig = dln.mad(y - dln.poly(x, coef))
bd, nbd = dln.where((y - dln.poly(x, coef)) > 5 * sig)
if nbd > 0: m[bd] = True
gdmask = (y > 0) & (m == False
) # need positive fluxes and no mask set
# Bin the data points
xr = [np.nanmin(x), np.nanmax(x)]
bins = np.ceil((xr[1] - xr[0]) / binsize) + 1
ybin, bin_edges, binnumber = bindata.binned_statistic(
x[gdmask],
y[gdmask],
statistic='percentile',
percentile=perclevel,
bins=bins,
range=None)
xbin = bin_edges[0:-1] + 0.5 * binsize
# Interpolate to full grid
fnt = np.isfinite(ybin)
cont1 = dln.interp(xbin[fnt], ybin[fnt], x, extrapolate=True)
cont1 *= medy
flux[:, o] /= cont1
err[:, o] /= cont1
cont[:, o] = cont1
# Flatten to 1D if norder=1
if self.norder == 1:
flux = flux.flatten()
err = err.flatten()
cont = cont.flatten()
# Stuff back in
self.flux = flux
self.err = err
self.cont = cont
self.normalized = True
return
linecat['wave0']))
maxdiff = 0.5 #1.0
if np.abs(tlinecat['wave'][j] -
linecat['wave0'][bestline]) < maxdiff:
match[j] = bestline
dwave[j] = tlinecat['wave'][j] - linecat['wave0'][bestline]
gmatch, = np.where(match > -1)
linecat['wave'][match[gmatch]] = tlinecat['wave'][gmatch]
linecat['match'][match[gmatch]] = 1
# Perform an initial fit
gline1, = np.where(linecat['match'] == 1)
meanwave = np.mean(linecat['wave'][gline1])
coef1 = dln.poly_fit(linecat['center'][gline1],
linecat['wave'][gline1], 3)
dwave = dln.poly(linecat['center'][gline1],
coef1) - linecat['wave'][gline1]
sigwave = dln.mad(dwave)
if sigwave > 0.1:
print('Sigma high. Trying higher order')
coef1 = dln.poly_fit(linecat['center'][gline1],
linecat['wave'][gline1], 4)
dwave = dln.poly(linecat['center'][gline1],
coef1) - linecat['wave'][gline1]
sigwave = dln.mad(dwave)
print('Sigma of initial fit %6.4f A' % sigwave)
bdline, = np.where(np.abs(dwave) > 3 * sigwave)
if len(bdline) > 0:
linecat['outlier'][gline1[bdline]] = True
print('throwing out ' + str(len(bdline)) + ' outlier lines')
#if sigwave>0.1:
def apvisit(filename):
"""
Read a SDSS APOGEE apVisit spectrum.
Parameters
----------
filename : string
The name of the spectrum file to load.
Returns
-------
spec : Spec1D object
The spectrum as a Spec1D object.
Examples
--------
spec = apvisit('spec.fits')
"""
base, ext = os.path.splitext(os.path.basename(filename))
# Get number of extensions
hdulist = fits.open(filename)
nhdu = len(hdulist)
hdulist.close()
# Check that this has the right format
validfile = False
if (base.find("apVisit") == -1) | (base.find("asVisit") == -1):
if nhdu >= 10:
gd, = np.where(
np.char.array(hdulist[0].header['HISTORY']).astype(str).find(
'AP1DVISIT') > -1)
if len(gd) > 0: validfile = True
if validfile is False:
return None
# APOGEE apVisit, visit-level spectrum
# HISTORY AP1DVISIT: HDU0 = Header only
# HISTORY AP1DVISIT: HDU1 - Flux (10^-17 ergs/s/cm^2/Ang)
# HISTORY AP1DVISIT: HDU2 - Error (10^-17 ergs/s/cm^2/Ang)
# HISTORY AP1DVISIT: HDU3 - Flag mask (bitwise OR combined)
# HISTORY AP1DVISIT: HDU4 - Wavelength (Ang)
# HISTORY AP1DVISIT: HDU5 - Sky (10^-17 ergs/s/cm^2/Ang)
# HISTORY AP1DVISIT: HDU6 - Sky Error (10^-17 ergs/s/cm^2/Ang)
# HISTORY AP1DVISIT: HDU7 - Telluric
# HISTORY AP1DVISIT: HDU8 - Telluric Error
# HISTORY AP1DVISIT: HDU9 - Wavelength coefficients
# HISTORY AP1DVISIT: HDU10 - LSF coefficients
# HISTORY AP1DVISIT: HDU11 - RV catalog
# flux, err, sky, skyerr are in units of 1e-17
flux = fits.getdata(filename, 1).T * 1e-17 # [Npix,Norder]
wave = fits.getdata(filename, 4).T
lsfcoef = fits.getdata(filename, 10).T
spec = Spec1D(flux,
wave=wave,
lsfpars=lsfcoef,
lsftype='Gauss-Hermite',
lsfxtype='Pixels')
spec.reader = 'apvisit'
spec.filename = filename
spec.sptype = "apVisit"
spec.waveregime = "NIR"
spec.instrument = "APOGEE"
spec.head = fits.getheader(filename, 0)
spec.err = fits.getdata(filename, 2).T * 1e-17 # [Npix,Norder]
#bad = (spec.err<=0) # fix bad error values
#if np.sum(bad) > 0:
# spec.err[bad] = 1e30
spec.bitmask = fits.getdata(filename, 3).T
spec.sky = fits.getdata(filename, 5).T * 1e-17
spec.skyerr = fits.getdata(filename, 6).T * 1e-17
spec.telluric = fits.getdata(filename, 7).T
spec.telerr = fits.getdata(filename, 8).T
spec.wcoef = fits.getdata(filename, 9).T
# Create the bad pixel mask
# "bad" pixels:
# flag = ['BADPIX','CRPIX','SATPIX','UNFIXABLE','BADDARK','BADFLAT','BADERR','NOSKY',
# 'LITTROW_GHOST','PERSIST_HIGH','PERSIST_MED','PERSIST_LOW','SIG_SKYLINE','SIG_TELLURIC','NOT_ENOUGH_PSF','']
# badflag = [1,1,1,1,1,1,1,1,
# 0,0,0,0,0,0,1,0]
mask = (np.bitwise_and(spec.bitmask, 16639) != 0) | (np.isfinite(spec.flux)
== False)
# Extra masking for bright skylines
x = np.arange(spec.npix)
nsky = 4
for i in range(spec.norder):
sky = spec.sky[:, i]
medsky = median_filter(sky, 201, mode='reflect')
medcoef = dln.poly_fit(x, medsky / np.median(medsky), 2)
medsky2 = dln.poly(x, medcoef) * np.median(medsky)
skymask1 = (sky > nsky * medsky2) # pixels Nsig above median sky
mask[:, i] = np.logical_or(mask[:, i], skymask1) # OR combine
spec.mask = mask
# Fix NaN pixels
for i in range(spec.norder):
bd, nbd = dln.where((np.isfinite(spec.flux[:, i]) == False)
| (spec.err[:, i] <= 0))
if nbd > 0:
spec.flux[bd, i] = 0.0
spec.err[bd, i] = 1e30
spec.mask[bd, i] = True
if (nhdu >= 11):
spec.meta = fits.getdata(filename,
11) # catalog of RV and other meta-data
# Spectrum, error, sky, skyerr are in units of 1e-17
spec.snr = spec.head["SNR"]
if base.find("apVisit") > -1:
spec.observatory = 'apo'
else:
spec.observatory = 'lco'
spec.wavevac = True
return spec
def sigma(self,x=None,xtype='pixels',order=0,extrapolate=True):
"""
Return the Gaussian sigma at specified locations. The sigma
will be in units of lsf.xtype.
Parameters
----------
x : array, optional
The x-values for which to return the Gaussian sigma values.
xtype : string, optional
The type of x-value input, either 'wave' or 'pixels'. Default is 'pixels'.
order : int, optional
The order to use if there are multiple orders.
The default is 0.
extrapolate : bool, optional
Extrapolate beyond the dispersion solution, if necessary.
True by default.
Returns
-------
sigma : array
The array of Gaussian sigma values.
Examples
--------
sigma = lsf.sigma([100,200])
"""
# The sigma will be returned in units given in lsf.xtype
if self._sigma is not None:
_sigma = self._sigma
if self.ndim==2: _sigma = self._sigma[:,order]
if x is None:
return _sigma
else:
# Wavelength input
if xtype.lower().find('wave') > -1:
x0 = np.array(x).copy() # backup
x = self.wave2pix(x0,order=order) # convert to pixels
# Integer, just return the values
if( type(x)==int) | (np.array(x).dtype.kind=='i'):
return _sigma[x]
# Floats, interpolate
else:
sig = interp1d(np.arange(len(_sigma)),_sigma,kind='cubic',bounds_error=False,
fill_value=(np.nan,np.nan),assume_sorted=True)(x)
# Extrapolate
npix = self.npix
if ((np.min(x)<0) | (np.max(x)>(npix-1))) & (extrapolate is True):
xin = np.arange(npix)
# At the beginning
if (np.min(x)<0):
coef1 = dln.poly_fit(xin[0:10], _sigma[0:10], 2)
bd1, nbd1 = dln.where(x <0)
sig[bd1] = dln.poly(x[bd1],coef1)
# At the end
if (np.max(x)>(npix-1)):
coef2 = dln.poly_fit(xin[npix-10:], _sigma[npix-10:], 2)
bd2, nbd2 = dln.where(x > (npix-1))
sig[bd2] = dln.poly(x[bd2],coef2)
return sig
# Need to calculate
else:
if x is None:
x = np.arange(self.npix)
if self.pars is None:
raise Exception("No LSF parameters")
# Get parameters
pars = self.pars
if self.ndim==2: pars=self.pars[:,order]
# Pixels input
if xtype.lower().find('pix') > -1:
# Pixel LSF parameters
if self.xtype.lower().find('pix') > -1:
return np.polyval(pars[::-1],x)
# Wave LSF parameters
else:
w = self.pix2wave(x,order=order)
return np.polyval(pars[::-1],w)
# Wavelengths input
else:
# Wavelength LSF parameters
if self.xtype.lower().find('wave') > -1:
return np.polyval(pars[::-1],x)
# Pixel LSF parameters
else:
x0 = np.array(x).copy()
x = self.wave2pix(x0,order=order)
return np.polyval(pars[::-1],x)
def maskdiscrepant(spec, model, nsig=10, verbose=False):
"""
Mask pixels that are discrepant when compared to a model.
Parameters
----------
spec : Spec1D object
Observed spectrum for which to mask discrepant values.
model : Spec1D object
Reference/model spectrum to use to find discrepant values.
nsig : int, optional
Number of standard deviations to use for the discrepant values.
Default is 10.0.
verbose : boolean, optional
Verbose output. Default is False.
Returns
-------
spec2 : Spec1D object
Spectrum with discrepant values masked.
Example
-------
.. code-block:: python
spec = maskdiscrepant(spec,nsig=5)
"""
print = getprintfunc(
) # Get print function to be used locally, allows for easy logging
spec2 = spec.copy()
wave = spec2.wave.copy().reshape(spec2.npix, spec2.norder) # make 2D
flux = spec2.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D
err = spec2.err.copy().reshape(spec2.npix, spec2.norder) # make 2D
mask = spec2.mask.copy().reshape(spec2.npix, spec2.norder) # make 2D
mflux = model.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D
totnbd = 0
for o in range(spec2.norder):
w = wave[:, o].copy()
x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5)
) # -1 to +1
y = flux[:, o].copy()
m = mask[:, o].copy()
my = mflux[:, o].copy()
# Divide by median
medy = np.nanmedian(y)
if medy <= 0.0:
medy = 1.0
y /= medy
my /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x, y, 2, robust=True)
sig = dln.mad(y - my)
bd, nbd = dln.where(np.abs(y - my) > nsig * sig)
totnbd += nbd
if nbd > 0:
flux[bd, o] = dln.poly(x[bd], coef) * medy
err[bd, o] = 1e30
mask[bd, o] = True
# Flatten to 1D if norder=1
if spec2.norder == 1:
flux = flux.flatten()
err = err.flatten()
mask = mask.flatten()
# Stuff back in
spec2.flux = flux
spec2.err = err
spec2.mask = mask
if verbose is True: print('Masked ' + str(totnbd) + ' discrepant pixels')
return spec2
def maskoutliers(spec, nsig=5, verbose=False):
"""
Mask large positive outliers and negative flux pixels in the spectrum.
Parameters
----------
spec : Spec1D object
Observed spectrum to mask outliers on.
nsig : int, optional
Number of standard deviations to use for the outlier rejection.
Default is 5.0.
verbose : boolean, optional
Verbose output. Default is False.
Returns
-------
spec2 : Spec1D object
Spectrum with outliers masked.
Example
-------
.. code-block:: python
spec = maskoutliers(spec,nsig=5)
"""
print = getprintfunc(
) # Get print function to be used locally, allows for easy logging
spec2 = spec.copy()
wave = spec2.wave.copy().reshape(spec2.npix, spec2.norder) # make 2D
flux = spec2.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D
err = spec2.err.copy().reshape(spec2.npix, spec2.norder) # make 2D
mask = spec2.mask.copy().reshape(spec2.npix, spec2.norder) # make 2D
totnbd = 0
for o in range(spec2.norder):
w = wave[:, o].copy()
x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5)
) # -1 to +1
y = flux[:, o].copy()
m = mask[:, o].copy()
# Divide by median
medy = np.nanmedian(y)
if medy <= 0.0:
medy = 1.0
y /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x, y, 2, robust=True)
sig = dln.mad(y - dln.poly(x, coef))
bd, nbd = dln.where(((y - dln.poly(x, coef)) > nsig * sig) | (y < 0))
totnbd += nbd
if nbd > 0:
flux[bd, o] = dln.poly(x[bd], coef) * medy
err[bd, o] = 1e30
mask[bd, o] = True
# Flatten to 1D if norder=1
if spec2.norder == 1:
flux = flux.flatten()
err = err.flatten()
mask = mask.flatten()
# Stuff back in
spec2.flux = flux
spec2.err = err
spec2.mask = mask
if verbose is True:
print('Masked ' + str(totnbd) + ' outlier or negative pixels')
return spec2
def fix_pms(objectid):
""" Correct the proper motions in the healpix object catalog."""
t00 = time.time()
hostname = socket.gethostname()
host = hostname.split('.')[0]
version = 'v3'
radeg = np.float64(180.00) / np.pi
meas = qc.query(sql="select * from nsc_dr2.meas where objectid='"+objectid+"'",fmt='table')
nmeas = len(meas)
print(' '+str(nmeas))
mnra = np.median(meas['ra'].data)
mndec = np.median(meas['dec'].data)
lim = 20.0 # 50.0
gd, = np.where( (np.abs(meas['ra'].data-mnra)/np.cos(np.deg2rad(mndec))*3600 < lim) &
(np.abs(meas['dec'].data-mndec)*3600 < lim))
ngd = len(gd)
nbd = nmeas-ngd
print('bad measurements '+str(nbd))
#if nbd==0:
# return None
meas = meas[gd]
# Make cut on FWHM
# maybe only use values for 0.5*fwhm_chip to 1.5*fwhm_chip
sql = "select chip.* from nsc_dr2.chip as chip join nsc_dr2.meas as meas on chip.exposure=meas.exposure and chip.ccdnum=meas.ccdnum"
sql += " where meas.objectid='"+objectid+"'"
chip = qc.query(sql=sql,fmt='table')
ind3,ind4 = dln.match(chip['exposure'],meas['exposure'])
si = np.argsort(ind4) # sort by input meas catalog
ind3 = ind3[si]
ind4 = ind4[si]
chip = chip[ind3]
meas = meas[ind4]
gdfwhm, = np.where((meas['fwhm'] > 0.2*chip['fwhm']) & (meas['fwhm'] < 2.0*chip['fwhm']))
if len(gdfwhm)==0:
print('All measurements have bad FWHM values')
return
if len(gdfwhm) < len(meas):
print('Removing '+str(len(meas)-len(gdfwhm))+' measurements with bad FWHM values')
meas = meas[gdfwhm]
raerr = np.array(meas['raerr']*1e3,np.float64) # milli arcsec
ra = np.array(meas['ra'],np.float64)
ra -= np.mean(ra)
ra *= 3600*1e3 * np.cos(mndec/radeg) # convert to true angle, milli arcsec
t = np.array(meas['mjd'].copy())
t -= np.mean(t)
t /= 365.2425 # convert to year
# Calculate robust slope
try:
#pmra, pmraerr = dln.robust_slope(t,ra,raerr,reweight=True)
# LADfit
pmra_ladcoef, absdev = dln.ladfit(t,ra)
pmra_lad = pmra_ladcoef[1]
# Run RANSAC
ransac = linear_model.RANSACRegressor()
ransac.fit(t.reshape(-1,1), ra)
inlier_mask = ransac.inlier_mask_
outlier_mask = np.logical_not(inlier_mask)
gdmask = inlier_mask
pmra_ransac = ransac.estimator_.coef_[0]
print(' ransac '+str(np.sum(inlier_mask))+' inliers '+str(np.sum(outlier_mask))+' outliers')
# Robust, weighted linear with with INLIERS
#pmra_coef, pmra_coeferr = dln.poly_fit(t[gdmask],ra[gdmask],1,sigma=raerr[gdmask],robust=True,error=True)
#pmra_coef, pmra_coeferr = dln.poly_fit(t,ra,1,sigma=raerr,robust=True,error=True)
#pmra = pmra_coef[0]
#pmraerr = pmra_coeferr[0]
#radiff = ra-dln.poly(t,pmra_coef)
radiff = ra-t*pmra_lad
radiff -= np.median(radiff)
rasig = dln.mad(radiff)
# Reject outliers
gdsig = (np.abs(radiff) < 2.5*rasig) | (np.abs(radiff) < 2.5*raerr)
print(' '+str(nmeas-np.sum(gdsig))+' 2.5*sigma clip outliers rejected')
#if np.sum(gdsig) < nmeas:
pmra_coef, pmra_coeferr = dln.poly_fit(t[gdsig],ra[gdsig],1,sigma=raerr[gdsig],robust=True,error=True)
pmra = pmra_coef[0]
pmraerr = pmra_coeferr[0]
rasig = dln.mad(ra-dln.poly(t,pmra_coef))
except:
print('problem')
#import pdb; pdb.set_trace()
return np.append(np.zeros(10,float)+np.nan, np.zeros(2,int))
decerr = np.array(meas['decerr']*1e3,np.float64) # milli arcsec
dec = np.array(meas['dec'],np.float64)
dec -= np.mean(dec)
dec *= 3600*1e3 # convert to milli arcsec
# Calculate robust slope
try:
#pmdec, pmdecerr = dln.robust_slope(t,dec,decerr,reweight=True)
# LADfit
pmdec_ladcoef, absdev = dln.ladfit(t,dec)
pmdec_lad = pmdec_ladcoef[1]
# Run RANSAC
ransac = linear_model.RANSACRegressor()
ransac.fit(t.reshape(-1,1), dec)
inlier_mask = ransac.inlier_mask_
outlier_mask = np.logical_not(inlier_mask)
gdmask = inlier_mask
pmdec_ransac = ransac.estimator_.coef_[0]
print(' ransac '+str(np.sum(inlier_mask))+' inliers '+str(np.sum(outlier_mask))+' outliers')
# Robust, weighted linear with with INLIERS
#pmdec_coef, pmdec_coeferr = dln.poly_fit(t[gdmask],dec[gdmask],1,sigma=decerr[gdmask],robust=True,error=True)
#pmdec_coef, pmdec_coeferr = dln.poly_fit(t,dec,1,sigma=decerr,robust=True,error=True)
#pmdec = pmdec_coef[0]
#pmdecerr = pmdec_coeferr[0]
#decdiff = dec-dln.poly(t,pmdec_coef)
decdiff = dec-t*pmdec_lad
decdiff -= np.median(decdiff)
decsig = dln.mad(decdiff)
# Reject outliers
gdsig = (np.abs(decdiff) < 2.5*decsig) | (np.abs(decdiff) < 2.5*decerr)
print(' '+str(nmeas-np.sum(gdsig))+' 2.5*sigma clip outliers rejected')
#if np.sum(gdsig) < nmeas:
pmdec_coef, pmdec_coeferr = dln.poly_fit(t[gdsig],dec[gdsig],1,sigma=decerr[gdsig],robust=True,error=True)
pmdec = pmdec_coef[0]
pmdecerr = pmdec_coeferr[0]
decsig = dln.mad(dec-dln.poly(t,pmdec_coef))
except:
print('problem')
#import pdb; pdb.set_trace()
return np.append(np.zeros(10,float)+np.nan, np.zeros(2,int))
deltamjd = np.max(meas['mjd'])-np.min(meas['mjd'])
out = np.array([pmra,pmraerr,pmra_ransac,pmra_lad,rasig,pmdec,pmdecerr,pmdec_ransac,pmdec_lad,decsig,nmeas,deltamjd])
#print(out[[0,2,3]])
#print(out[[5,7,8]])
#import pdb; pdb.set_trace()
return out
#print('Second fit coefficients ',func2.parameters)
#print('Second fit coefficients ',coef2)
print('Second sigma = %f6.2' % sigwave2)
# 4) outlier rejection
# take sections and robust fit a line, then reject outliers
xx = linecat['center'][gline]
yy = wavecorr1[gline]
si = np.argsort(xx)
step = 256
bad = np.zeros(len(gline),bool)
for i in range(8):
g, = np.where((xx>=i*step) & (xx<(i+1)*step))
cc,absdev = dln.ladfit(xx[g],yy[g])
cc = cc[::-1] # flip
diff = yy[g]-dln.poly(xx[g],cc)
sig = dln.mad(diff)
bd, = np.where(np.abs(diff) > 3*sig)
if len(bd)>0:
bad[g[bd]] = True
#print(i,i*step,(i+1)*step,len(bd))
bd, = np.where(bad==True)
if len(bd)>0:
print('Rejecting '+str(len(bd))+' outlier points')
linecat['outlier'][gline[bd]] = True
# Select new good points
gline, = np.where((linecat['match']==1) & (linecat['outlier']==False))
xx = linecat['center'][gline]
yy = wavecorr1[gline]
si = np.argsort(xx)
if nind>0:
calstr1 = calstr[ind]
zpterm = calstr1['zpterm']
bdzp,nbdzp = dln.where(~np.isfinite(zpterm)) # fix Infinity/NAN
if nbdzp>0:zpterm[bdzp] = 999999.9
am = calstr1['airmass']
mjd = calstr1['mjd']
bdam,nbdam = dln.where(am < 0.9)
if nbdam>0: am[bdam] = np.median(am)
# I GOT TO HERE IN THE TRANSLATING!!!
#glactc,calstr1.ra,calstr1.dec,2000.0,glon,glat,1,/deg
# Measure airmass dependence
gg0,ngg0 = dln.where((np.abs(zpterm)<50) & (am<2.0))
coef0 = dln.poly_fit(am[gg0],zpterm[gg0],1,robust=True)
zpf = dln.poly(am,coef0)
sig0 = np.mad(zpterm[gg0]-zpf[gg0])
gg,ngg = dln.where(np.abs(zpterm-zpf) < (np.maximum(3.5*sig0,0.2)))
coef = dln.poly_fit(am[gg],zpterm[gg],1,robust=True)
print(zpstr['instrument'][i]+'-'+zpstr['filter'][i]+' '+str(coef))
# Trim out bad exposures to determine the correlations and make figures
gg,ngg = dln.where(np.abs(zpterm-zpf) < (3.5*sig0 > 0.2) and calstr1.airmass < 2.0 and calstr1.fwhm < 2.0 and calstr1.rarms < 0.15 &
calstr1.decrms < 0.15 and calstr1.success == 1 and calstr1.wcscal == 'Successful' and calstr1.zptermerr < 0.05 &
calstr1.zptermsig < 0.08 and (calstr1.ngoodchipwcs == calstr1.nchips) &
(calstr1.instrument != 'c4d' or calstr1.zpspatialvar_nccd <= 5 or (calstr1.instrument == 'c4d' and
calstr1.zpspatialvar_nccd > 5 and calstr1.zpspatialvar_rms < 0.1)) and
np.abs(glat) > 10 and calstr1.nrefmatch > 100 and calstr1.exptime >= 30)
# Zpterm with airmass dependence removed
relzpterm = zpterm + 25 # 25 to get "absolute" zpterm
relzpterm -= (zpstr['amcoef'][i])[1]*(am-1)