def w2p(dispersion,w,extrapolate=True):
"""
Convert wavelength values to pixels for a "dispersion solution".
Parameters
----------
dispersion : 1D array
The dispersion solution. This is basically just a 1D array of
monotonically increasing (or decreasing) wavelengths.
w : array
Array of wavelength values to convert to pixels.
extrapolate : bool, optional
Extrapolate beyond the dispersion solution, if necessary.
This is True by default.
Returns
-------
x : array
Array of converted pixel values.
Examples
--------
x = w2p(disp,w)
"""
x = interp1d(dispersion,np.arange(len(dispersion)),kind='cubic',bounds_error=False,fill_value=(np.nan,np.nan),assume_sorted=False)(w)
# Need to extrapolate
if ((np.min(w)<np.min(dispersion)) | (np.max(w)>np.max(dispersion))) & (extrapolate is True):
win = dispersion
xin = np.arange(len(dispersion))
si = np.argsort(win)
win = win[si]
xin = xin[si]
npix = len(win)
# At the beginning
if (np.min(w)<np.min(dispersion)):
coef1 = dln.poly_fit(win[0:10], xin[0:10], 2)
bd1, nbd1 = dln.where(w < np.min(dispersion))
x[bd1] = dln.poly(w[bd1],coef1)
# At the end
if (np.max(w)>np.max(dispersion)):
coef2 = dln.poly_fit(win[npix-10:], xin[npix-10:], 2)
bd2, nbd2 = dln.where(w > np.max(dispersion))
x[bd2] = dln.poly(w[bd2],coef2)
return x
def p2w(dispersion,x,extrapolate=True):
"""
Convert pixel values to wavelengths for a "dispersion solution".
Parameters
----------
dispersion : 1D array
The dispersion solution. This is basically just a 1D array of
monotonically increasing (or decreasing) wavelengths.
x : array
Array of pixel values to convert to wavelengths.
extrapolate : bool, optional
Extrapolate beyond the dispersion solution, if necessary.
This is True by default.
Returns
-------
w : array
Array of converted wavelengths.
Examples
--------
w = p2w(disp,x)
"""
npix = len(dispersion)
w = interp1d(np.arange(len(dispersion)),dispersion,kind='cubic',bounds_error=False,fill_value=(np.nan,np.nan),assume_sorted=False)(x)
# Need to extrapolate
if ((np.min(x)<0) | (np.max(x)>(npix-1))) & (extrapolate is True):
xin = np.arange(npix)
win = dispersion
# At the beginning
if (np.min(x)<0):
coef1 = dln.poly_fit(xin[0:10], win[0:10], 2)
bd1, nbd1 = dln.where(x < 0)
w[bd1] = dln.poly(x[bd1],coef1)
# At the end
if (np.max(x)>(npix-1)):
coef2 = dln.poly_fit(xin[npix-10:], win[npix-10:], 2)
bd2, nbd2 = dln.where(x > (npix-1))
w[bd2] = dln.poly(x[bd2],coef2)
return w
def maskdiscrepant(spec,model,nsig=10,verbose=False,logger=None):
"""
Mask pixels that are discrepant when compared to a model.
"""
if logger is None:
logger = dln.basiclogger()
spec2 = spec.copy()
wave = spec2.wave.copy().reshape(spec2.npix,spec2.norder) # make 2D
flux = spec2.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D
err = spec2.err.copy().reshape(spec2.npix,spec2.norder) # make 2D
mask = spec2.mask.copy().reshape(spec2.npix,spec2.norder) # make 2D
mflux = model.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D
totnbd = 0
for o in range(spec2.norder):
w = wave[:,o].copy()
x = (w-np.median(w))/(np.max(w*0.5)-np.min(w*0.5)) # -1 to +1
y = flux[:,o].copy()
m = mask[:,o].copy()
my = mflux[:,o].copy()
# Divide by median
medy = np.nanmedian(y)
y /= medy
my /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x,y,2,robust=True)
sig = dln.mad(y-my)
bd,nbd = dln.where( np.abs(y-my) > nsig*sig )
totnbd += nbd
if nbd>0:
flux[bd,o] = dln.poly(x[bd],coef)*medy
err[bd,o] = 1e30
mask[bd,o] = True
# Flatten to 1D if norder=1
if spec2.norder==1:
flux = flux.flatten()
err = err.flatten()
mask = mask.flatten()
# Stuff back in
spec2.flux = flux
spec2.err = err
spec2.mask = mask
if verbose is True: logger.info('Masked '+str(totnbd)+' discrepant pixels')
return spec2
def maskoutliers(spec,nsig=5,verbose=False,logger=None):
"""
Mask large positive outliers and negative flux pixels in the spectrum.
"""
if logger is None:
logger = dln.basiclogger()
spec2 = spec.copy()
wave = spec2.wave.copy().reshape(spec2.npix,spec2.norder) # make 2D
flux = spec2.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D
err = spec2.err.copy().reshape(spec2.npix,spec2.norder) # make 2D
mask = spec2.mask.copy().reshape(spec2.npix,spec2.norder) # make 2D
totnbd = 0
for o in range(spec2.norder):
w = wave[:,o].copy()
x = (w-np.median(w))/(np.max(w*0.5)-np.min(w*0.5)) # -1 to +1
y = flux[:,o].copy()
m = mask[:,o].copy()
# Divide by median
medy = np.nanmedian(y)
y /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x,y,2,robust=True)
sig = dln.mad(y-dln.poly(x,coef))
bd,nbd = dln.where( ((y-dln.poly(x,coef)) > nsig*sig) | (y<0))
totnbd += nbd
if nbd>0:
flux[bd,o] = dln.poly(x[bd],coef)*medy
err[bd,o] = 1e30
mask[bd,o] = True
# Flatten to 1D if norder=1
if spec2.norder==1:
flux = flux.flatten()
err = err.flatten()
mask = mask.flatten()
# Stuff back in
spec2.flux = flux
spec2.err = err
spec2.mask = mask
if verbose is True: logger.info('Masked '+str(totnbd)+' outlier or negative pixels')
return spec2
def continuum(spec, norder=6, perclevel=90.0, binsize=0.1, interp=True):
"""
Measure the continuum of a spectrum.
Parameters
----------
spec : Spec1D object
A spectrum object. This at least needs
to have a FLUX and WAVE attribute.
norder : float, optional
Polynomial order to use for the continuum fitting.
The default is 6.
perclevel : float, optional
Percent level to use for the continuum value
in large bins. Default is 90.
binsize : float, optional
Fraction of the wavelength range (scaled to -1 to +1) to bin.
Default is 0.1.
interp : bool, optional
Use interpolation of the binned values instead of a polynomial
fit. Default is True.
Returns
-------
cont : numpy array
The continuum array, in the same shape as the input flux.
Examples
--------
.. code-block:: python
cont = continuum(spec)
"""
wave = spec.wave.copy().reshape(spec.npix, spec.norder) # make 2D
flux = spec.flux.copy().reshape(spec.npix, spec.norder) # make 2D
err = spec.err.copy().reshape(spec.npix, spec.norder) # make 2D
mask = spec.mask.copy().reshape(spec.npix, spec.norder) # make 2D
cont = err.copy() * 0.0 + 1
for o in range(spec.norder):
w = wave[:, o].copy()
wr = [np.min(w), np.max(w)]
x = (w - np.mean(wr)) / dln.valrange(wr) * 2 # -1 to +1
y = flux[:, o].copy()
m = mask[:, o].copy()
# Divide by median
medy = np.nanmedian(y)
if medy <= 0.0:
medy = 1.0
y /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x[~m], y[~m], 2, robust=True)
sig = dln.mad(y - dln.poly(x, coef))
bd, nbd = dln.where((y - dln.poly(x, coef)) > 5 * sig)
if nbd > 0: m[bd] = True
gdmask = (y > 0) & (m == False) # need positive fluxes and no mask set
if np.sum(gdmask) == 0:
continue
# Bin the data points
xr = [np.nanmin(x), np.nanmax(x)]
bins = int(np.ceil((xr[1] - xr[0]) / binsize))
ybin, bin_edges, binnumber = bindata.binned_statistic(
x[gdmask],
y[gdmask],
statistic='percentile',
percentile=perclevel,
bins=bins,
range=None)
xbin = bin_edges[0:-1] + 0.5 * binsize
# Interpolate to full grid
if interp is True:
fnt = np.isfinite(ybin)
cont1 = dln.interp(xbin[fnt],
ybin[fnt],
x,
kind='quadratic',
extrapolate=True,
exporder=1)
else:
coef = dln.poly_fit(xbin, ybin, norder)
cont1 = dln.poly(x, coef)
cont1 *= medy
cont[:, o] = cont1
# Flatten to 1D if norder=1
if spec.norder == 1:
cont = cont.flatten()
return cont
def apstar(filename,badval=20735):
"""
Read an SDSS APOGEE apStar spectrum.
Parameters
----------
filename : string
The name of the spectrum file to load.
Returns
-------
spec : Spec1D object
The spectrum as a Spec1D object.
Examples
--------
spec = apstar('spec.fits')
"""
base, ext = os.path.splitext(os.path.basename(filename))
# APOGEE apStar, combined spectrum
if (base.find("apStar") > -1) | (base.find("asStar") > -1):
# HISTORY APSTAR: HDU0 = Header only
# HISTORY APSTAR: All image extensions have:
# HISTORY APSTAR: row 1: combined spectrum with individual pixel weighting
# HISTORY APSTAR: row 2: combined spectrum with global weighting
# HISTORY APSTAR: row 3-nvisits+2: individual resampled visit spectra
# HISTORY APSTAR: unless nvisits=1, which only have a single row
# HISTORY APSTAR: All spectra shifted to rest (vacuum) wavelength scale
# HISTORY APSTAR: HDU1 - Flux (10^-17 ergs/s/cm^2/Ang)
# HISTORY APSTAR: HDU2 - Error (10^-17 ergs/s/cm^2/Ang)
# HISTORY APSTAR: HDU3 - Flag mask:
# HISTORY APSTAR: row 1: bitwise OR of all visits
# HISTORY APSTAR: row 2: bitwise AND of all visits
# HISTORY APSTAR: row 3-nvisits+2: individual visit masks
# HISTORY APSTAR: HDU4 - Sky (10^-17 ergs/s/cm^2/Ang)
# HISTORY APSTAR: HDU5 - Sky Error (10^-17 ergs/s/cm^2/Ang)
# HISTORY APSTAR: HDU6 - Telluric
# HISTORY APSTAR: HDU7 - Telluric Error
# HISTORY APSTAR: HDU8 - LSF coefficients
# HISTORY APSTAR: HDU9 - RV and CCF structure
# Get number of extensions
hdulist = fits.open(filename)
nhdu = len(hdulist)
hdulist.close()
# Spectrum, error, sky, skyerr are in units of 1e-17
# these are 2D arrays with [Nvisit+2,Npix]
# the first two are combined and the rest are the individual spectra
head1 = fits.getheader(filename,1)
w0 = np.float64(head1["CRVAL1"])
dw = np.float64(head1["CDELT1"])
nw = head1["NAXIS1"]
wave = 10**(np.arange(nw)*dw+w0)
# flux, err, sky, skyerr are in units of 1e-17
flux = fits.getdata(filename,1).T * 1e-17
lsfcoef = fits.getdata(filename,8).T
spec = Spec1D(flux,wave=wave,lsfpars=lsfcoef,lsftype='Gauss-Hermite',lsfxtype='Pixels')
spec.filename = filename
spec.sptype = "apStar"
spec.waveregime = "NIR"
spec.instrument = "APOGEE"
spec.head = fits.getheader(filename,0)
spec.err = fits.getdata(filename,2).T * 1e-17
#bad = (spec.err<=0) # fix bad error values
#if np.sum(bad) > 0:
# spec.err[bad] = 1e30
spec.bitmask = fits.getdata(filename,3)
spec.sky = fits.getdata(filename,4).T * 1e-17
spec.skyerr = fits.getdata(filename,5).T * 1e-17
spec.telluric = fits.getdata(filename,6).T
spec.telerr = fits.getdata(filename,7).T
spec.lsf = fits.getdata(filename,8).T
# Create the bad pixel mask
# "bad" pixels:
# flag = ['BADPIX','CRPIX','SATPIX','UNFIXABLE','BADDARK','BADFLAT','BADERR','NOSKY',
# 'LITTROW_GHOST','PERSIST_HIGH','PERSIST_MED','PERSIST_LOW','SIG_SKYLINE','SIG_TELLURIC','NOT_ENOUGH_PSF','']
# badflag = [1,1,1,1,1,1,1,1,
# 0,0,0,0,0,0,1,0]
mask = (np.bitwise_and(spec.bitmask,badval)!=0) | (np.isfinite(spec.flux)==False)
# Extra masking for bright skylines
x = np.arange(spec.npix)
nsky = 4
medsky = median_filter(spec.sky,201,mode='reflect')
medcoef = dln.poly_fit(x,medsky/np.median(medsky),2)
medsky2 = dln.poly(x,medcoef)*np.median(medsky)
skymask1 = (sky>nsky*medsky2) # pixels Nsig above median sky
mask[:,i] = np.logical_or(mask[:,i],skymask1) # OR combine
spec.mask = mask
# Fix NaN or bad pixels pixels
bd,nbd = dln.where( (np.isfinite(spec.flux[:,i])==False) | (spec.err[:,i] <= 0.0) )
if nbd>0:
spec.flux[bd] = 0.0
spec.err[bd] = 1e30
spec.mask[bd] = True
if nhdu>=9:
spec.meta = fits.getdata(filename,9) # meta-data
spec.snr = spec.head["SNR"]
if base.find("apStar") > -1:
spec.observatory = 'apo'
else:
spec.observatory = 'lco'
spec.wavevac = True
return spec
def normalize(self, ncorder=6, perclevel=0.95):
"""
Normalize the spectrum.
Parameters
----------
ncorder : float, optional
Polynomial order to use for the continuum fitting.
The default is 6.
perclevel : float, optional
Percent level (1.0 for 100%) to use for the continuum value
in large bins. Default is 0.95.
Returns
-------
The flux and err arrays will normalized (divided) by the continuum
and the continuum saved in cont. The normalized property is set to
True.
Examples
--------
spec.normalize()
"""
self._flux = self.flux # Save the original
#nspec, cont, masked = normspec(self,ncorder=ncorder,perclevel=perclevel)
binsize = 0.10
perclevel = 90.0
wave = self.wave.copy().reshape(self.npix, self.norder) # make 2D
flux = self.flux.copy().reshape(self.npix, self.norder) # make 2D
err = self.err.copy().reshape(self.npix, self.norder) # make 2D
mask = self.mask.copy().reshape(self.npix, self.norder) # make 2D
cont = err.copy() * 0.0 + 1
for o in range(self.norder):
w = wave[:, o].copy()
x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5)
) # -1 to +1
y = flux[:, o].copy()
m = mask[:, o].copy()
# Divide by median
medy = np.nanmedian(y)
y /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x, y, 2, robust=True)
sig = dln.mad(y - dln.poly(x, coef))
bd, nbd = dln.where((y - dln.poly(x, coef)) > 5 * sig)
if nbd > 0: m[bd] = True
gdmask = (y > 0) & (m == False
) # need positive fluxes and no mask set
# Bin the data points
xr = [np.nanmin(x), np.nanmax(x)]
bins = np.ceil((xr[1] - xr[0]) / binsize) + 1
ybin, bin_edges, binnumber = bindata.binned_statistic(
x[gdmask],
y[gdmask],
statistic='percentile',
percentile=perclevel,
bins=bins,
range=None)
xbin = bin_edges[0:-1] + 0.5 * binsize
# Interpolate to full grid
fnt = np.isfinite(ybin)
cont1 = dln.interp(xbin[fnt], ybin[fnt], x, extrapolate=True)
cont1 *= medy
flux[:, o] /= cont1
err[:, o] /= cont1
cont[:, o] = cont1
# Flatten to 1D if norder=1
if self.norder == 1:
flux = flux.flatten()
err = err.flatten()
cont = cont.flatten()
# Stuff back in
self.flux = flux
self.err = err
self.cont = cont
self.normalized = True
return
for j in range(ntlines):
bestline = np.argmin(np.abs(tlinecat['wave'][j] -
linecat['wave0']))
maxdiff = 0.5 #1.0
if np.abs(tlinecat['wave'][j] -
linecat['wave0'][bestline]) < maxdiff:
match[j] = bestline
dwave[j] = tlinecat['wave'][j] - linecat['wave0'][bestline]
gmatch, = np.where(match > -1)
linecat['wave'][match[gmatch]] = tlinecat['wave'][gmatch]
linecat['match'][match[gmatch]] = 1
# Perform an initial fit
gline1, = np.where(linecat['match'] == 1)
meanwave = np.mean(linecat['wave'][gline1])
coef1 = dln.poly_fit(linecat['center'][gline1],
linecat['wave'][gline1], 3)
dwave = dln.poly(linecat['center'][gline1],
coef1) - linecat['wave'][gline1]
sigwave = dln.mad(dwave)
if sigwave > 0.1:
print('Sigma high. Trying higher order')
coef1 = dln.poly_fit(linecat['center'][gline1],
linecat['wave'][gline1], 4)
dwave = dln.poly(linecat['center'][gline1],
coef1) - linecat['wave'][gline1]
sigwave = dln.mad(dwave)
print('Sigma of initial fit %6.4f A' % sigwave)
bdline, = np.where(np.abs(dwave) > 3 * sigwave)
if len(bdline) > 0:
linecat['outlier'][gline1[bdline]] = True
print('throwing out ' + str(len(bdline)) + ' outlier lines')
def apvisit(filename):
"""
Read a SDSS APOGEE apVisit spectrum.
Parameters
----------
filename : string
The name of the spectrum file to load.
Returns
-------
spec : Spec1D object
The spectrum as a Spec1D object.
Examples
--------
spec = apvisit('spec.fits')
"""
base, ext = os.path.splitext(os.path.basename(filename))
# Get number of extensions
hdulist = fits.open(filename)
nhdu = len(hdulist)
hdulist.close()
# Check that this has the right format
validfile = False
if (base.find("apVisit") == -1) | (base.find("asVisit") == -1):
if nhdu >= 10:
gd, = np.where(
np.char.array(hdulist[0].header['HISTORY']).astype(str).find(
'AP1DVISIT') > -1)
if len(gd) > 0: validfile = True
if validfile is False:
return None
# APOGEE apVisit, visit-level spectrum
# HISTORY AP1DVISIT: HDU0 = Header only
# HISTORY AP1DVISIT: HDU1 - Flux (10^-17 ergs/s/cm^2/Ang)
# HISTORY AP1DVISIT: HDU2 - Error (10^-17 ergs/s/cm^2/Ang)
# HISTORY AP1DVISIT: HDU3 - Flag mask (bitwise OR combined)
# HISTORY AP1DVISIT: HDU4 - Wavelength (Ang)
# HISTORY AP1DVISIT: HDU5 - Sky (10^-17 ergs/s/cm^2/Ang)
# HISTORY AP1DVISIT: HDU6 - Sky Error (10^-17 ergs/s/cm^2/Ang)
# HISTORY AP1DVISIT: HDU7 - Telluric
# HISTORY AP1DVISIT: HDU8 - Telluric Error
# HISTORY AP1DVISIT: HDU9 - Wavelength coefficients
# HISTORY AP1DVISIT: HDU10 - LSF coefficients
# HISTORY AP1DVISIT: HDU11 - RV catalog
# flux, err, sky, skyerr are in units of 1e-17
flux = fits.getdata(filename, 1).T * 1e-17 # [Npix,Norder]
wave = fits.getdata(filename, 4).T
lsfcoef = fits.getdata(filename, 10).T
spec = Spec1D(flux,
wave=wave,
lsfpars=lsfcoef,
lsftype='Gauss-Hermite',
lsfxtype='Pixels')
spec.reader = 'apvisit'
spec.filename = filename
spec.sptype = "apVisit"
spec.waveregime = "NIR"
spec.instrument = "APOGEE"
spec.head = fits.getheader(filename, 0)
spec.err = fits.getdata(filename, 2).T * 1e-17 # [Npix,Norder]
#bad = (spec.err<=0) # fix bad error values
#if np.sum(bad) > 0:
# spec.err[bad] = 1e30
spec.bitmask = fits.getdata(filename, 3).T
spec.sky = fits.getdata(filename, 5).T * 1e-17
spec.skyerr = fits.getdata(filename, 6).T * 1e-17
spec.telluric = fits.getdata(filename, 7).T
spec.telerr = fits.getdata(filename, 8).T
spec.wcoef = fits.getdata(filename, 9).T
# Create the bad pixel mask
# "bad" pixels:
# flag = ['BADPIX','CRPIX','SATPIX','UNFIXABLE','BADDARK','BADFLAT','BADERR','NOSKY',
# 'LITTROW_GHOST','PERSIST_HIGH','PERSIST_MED','PERSIST_LOW','SIG_SKYLINE','SIG_TELLURIC','NOT_ENOUGH_PSF','']
# badflag = [1,1,1,1,1,1,1,1,
# 0,0,0,0,0,0,1,0]
mask = (np.bitwise_and(spec.bitmask, 16639) != 0) | (np.isfinite(spec.flux)
== False)
# Extra masking for bright skylines
x = np.arange(spec.npix)
nsky = 4
for i in range(spec.norder):
sky = spec.sky[:, i]
medsky = median_filter(sky, 201, mode='reflect')
medcoef = dln.poly_fit(x, medsky / np.median(medsky), 2)
medsky2 = dln.poly(x, medcoef) * np.median(medsky)
skymask1 = (sky > nsky * medsky2) # pixels Nsig above median sky
mask[:, i] = np.logical_or(mask[:, i], skymask1) # OR combine
spec.mask = mask
# Fix NaN pixels
for i in range(spec.norder):
bd, nbd = dln.where((np.isfinite(spec.flux[:, i]) == False)
| (spec.err[:, i] <= 0))
if nbd > 0:
spec.flux[bd, i] = 0.0
spec.err[bd, i] = 1e30
spec.mask[bd, i] = True
if (nhdu >= 11):
spec.meta = fits.getdata(filename,
11) # catalog of RV and other meta-data
# Spectrum, error, sky, skyerr are in units of 1e-17
spec.snr = spec.head["SNR"]
if base.find("apVisit") > -1:
spec.observatory = 'apo'
else:
spec.observatory = 'lco'
spec.wavevac = True
return spec
def sigma(self,x=None,xtype='pixels',order=0,extrapolate=True):
"""
Return the Gaussian sigma at specified locations. The sigma
will be in units of lsf.xtype.
Parameters
----------
x : array, optional
The x-values for which to return the Gaussian sigma values.
xtype : string, optional
The type of x-value input, either 'wave' or 'pixels'. Default is 'pixels'.
order : int, optional
The order to use if there are multiple orders.
The default is 0.
extrapolate : bool, optional
Extrapolate beyond the dispersion solution, if necessary.
True by default.
Returns
-------
sigma : array
The array of Gaussian sigma values.
Examples
--------
sigma = lsf.sigma([100,200])
"""
# The sigma will be returned in units given in lsf.xtype
if self._sigma is not None:
_sigma = self._sigma
if self.ndim==2: _sigma = self._sigma[:,order]
if x is None:
return _sigma
else:
# Wavelength input
if xtype.lower().find('wave') > -1:
x0 = np.array(x).copy() # backup
x = self.wave2pix(x0,order=order) # convert to pixels
# Integer, just return the values
if( type(x)==int) | (np.array(x).dtype.kind=='i'):
return _sigma[x]
# Floats, interpolate
else:
sig = interp1d(np.arange(len(_sigma)),_sigma,kind='cubic',bounds_error=False,
fill_value=(np.nan,np.nan),assume_sorted=True)(x)
# Extrapolate
npix = self.npix
if ((np.min(x)<0) | (np.max(x)>(npix-1))) & (extrapolate is True):
xin = np.arange(npix)
# At the beginning
if (np.min(x)<0):
coef1 = dln.poly_fit(xin[0:10], _sigma[0:10], 2)
bd1, nbd1 = dln.where(x <0)
sig[bd1] = dln.poly(x[bd1],coef1)
# At the end
if (np.max(x)>(npix-1)):
coef2 = dln.poly_fit(xin[npix-10:], _sigma[npix-10:], 2)
bd2, nbd2 = dln.where(x > (npix-1))
sig[bd2] = dln.poly(x[bd2],coef2)
return sig
# Need to calculate
else:
if x is None:
x = np.arange(self.npix)
if self.pars is None:
raise Exception("No LSF parameters")
# Get parameters
pars = self.pars
if self.ndim==2: pars=self.pars[:,order]
# Pixels input
if xtype.lower().find('pix') > -1:
# Pixel LSF parameters
if self.xtype.lower().find('pix') > -1:
return np.polyval(pars[::-1],x)
# Wave LSF parameters
else:
w = self.pix2wave(x,order=order)
return np.polyval(pars[::-1],w)
# Wavelengths input
else:
# Wavelength LSF parameters
if self.xtype.lower().find('wave') > -1:
return np.polyval(pars[::-1],x)
# Pixel LSF parameters
else:
x0 = np.array(x).copy()
x = self.wave2pix(x0,order=order)
return np.polyval(pars[::-1],x)
def maskdiscrepant(spec, model, nsig=10, verbose=False):
"""
Mask pixels that are discrepant when compared to a model.
Parameters
----------
spec : Spec1D object
Observed spectrum for which to mask discrepant values.
model : Spec1D object
Reference/model spectrum to use to find discrepant values.
nsig : int, optional
Number of standard deviations to use for the discrepant values.
Default is 10.0.
verbose : boolean, optional
Verbose output. Default is False.
Returns
-------
spec2 : Spec1D object
Spectrum with discrepant values masked.
Example
-------
.. code-block:: python
spec = maskdiscrepant(spec,nsig=5)
"""
print = getprintfunc(
) # Get print function to be used locally, allows for easy logging
spec2 = spec.copy()
wave = spec2.wave.copy().reshape(spec2.npix, spec2.norder) # make 2D
flux = spec2.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D
err = spec2.err.copy().reshape(spec2.npix, spec2.norder) # make 2D
mask = spec2.mask.copy().reshape(spec2.npix, spec2.norder) # make 2D
mflux = model.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D
totnbd = 0
for o in range(spec2.norder):
w = wave[:, o].copy()
x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5)
) # -1 to +1
y = flux[:, o].copy()
m = mask[:, o].copy()
my = mflux[:, o].copy()
# Divide by median
medy = np.nanmedian(y)
if medy <= 0.0:
medy = 1.0
y /= medy
my /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x, y, 2, robust=True)
sig = dln.mad(y - my)
bd, nbd = dln.where(np.abs(y - my) > nsig * sig)
totnbd += nbd
if nbd > 0:
flux[bd, o] = dln.poly(x[bd], coef) * medy
err[bd, o] = 1e30
mask[bd, o] = True
# Flatten to 1D if norder=1
if spec2.norder == 1:
flux = flux.flatten()
err = err.flatten()
mask = mask.flatten()
# Stuff back in
spec2.flux = flux
spec2.err = err
spec2.mask = mask
if verbose is True: print('Masked ' + str(totnbd) + ' discrepant pixels')
return spec2
def maskoutliers(spec, nsig=5, verbose=False):
"""
Mask large positive outliers and negative flux pixels in the spectrum.
Parameters
----------
spec : Spec1D object
Observed spectrum to mask outliers on.
nsig : int, optional
Number of standard deviations to use for the outlier rejection.
Default is 5.0.
verbose : boolean, optional
Verbose output. Default is False.
Returns
-------
spec2 : Spec1D object
Spectrum with outliers masked.
Example
-------
.. code-block:: python
spec = maskoutliers(spec,nsig=5)
"""
print = getprintfunc(
) # Get print function to be used locally, allows for easy logging
spec2 = spec.copy()
wave = spec2.wave.copy().reshape(spec2.npix, spec2.norder) # make 2D
flux = spec2.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D
err = spec2.err.copy().reshape(spec2.npix, spec2.norder) # make 2D
mask = spec2.mask.copy().reshape(spec2.npix, spec2.norder) # make 2D
totnbd = 0
for o in range(spec2.norder):
w = wave[:, o].copy()
x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5)
) # -1 to +1
y = flux[:, o].copy()
m = mask[:, o].copy()
# Divide by median
medy = np.nanmedian(y)
if medy <= 0.0:
medy = 1.0
y /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x, y, 2, robust=True)
sig = dln.mad(y - dln.poly(x, coef))
bd, nbd = dln.where(((y - dln.poly(x, coef)) > nsig * sig) | (y < 0))
totnbd += nbd
if nbd > 0:
flux[bd, o] = dln.poly(x[bd], coef) * medy
err[bd, o] = 1e30
mask[bd, o] = True
# Flatten to 1D if norder=1
if spec2.norder == 1:
flux = flux.flatten()
err = err.flatten()
mask = mask.flatten()
# Stuff back in
spec2.flux = flux
spec2.err = err
spec2.mask = mask
if verbose is True:
print('Masked ' + str(totnbd) + ' outlier or negative pixels')
return spec2
def fix_pms(objectid):
""" Correct the proper motions in the healpix object catalog."""
t00 = time.time()
hostname = socket.gethostname()
host = hostname.split('.')[0]
version = 'v3'
radeg = np.float64(180.00) / np.pi
meas = qc.query(sql="select * from nsc_dr2.meas where objectid='"+objectid+"'",fmt='table')
nmeas = len(meas)
print(' '+str(nmeas))
mnra = np.median(meas['ra'].data)
mndec = np.median(meas['dec'].data)
lim = 20.0 # 50.0
gd, = np.where( (np.abs(meas['ra'].data-mnra)/np.cos(np.deg2rad(mndec))*3600 < lim) &
(np.abs(meas['dec'].data-mndec)*3600 < lim))
ngd = len(gd)
nbd = nmeas-ngd
print('bad measurements '+str(nbd))
#if nbd==0:
# return None
meas = meas[gd]
# Make cut on FWHM
# maybe only use values for 0.5*fwhm_chip to 1.5*fwhm_chip
sql = "select chip.* from nsc_dr2.chip as chip join nsc_dr2.meas as meas on chip.exposure=meas.exposure and chip.ccdnum=meas.ccdnum"
sql += " where meas.objectid='"+objectid+"'"
chip = qc.query(sql=sql,fmt='table')
ind3,ind4 = dln.match(chip['exposure'],meas['exposure'])
si = np.argsort(ind4) # sort by input meas catalog
ind3 = ind3[si]
ind4 = ind4[si]
chip = chip[ind3]
meas = meas[ind4]
gdfwhm, = np.where((meas['fwhm'] > 0.2*chip['fwhm']) & (meas['fwhm'] < 2.0*chip['fwhm']))
if len(gdfwhm)==0:
print('All measurements have bad FWHM values')
return
if len(gdfwhm) < len(meas):
print('Removing '+str(len(meas)-len(gdfwhm))+' measurements with bad FWHM values')
meas = meas[gdfwhm]
raerr = np.array(meas['raerr']*1e3,np.float64) # milli arcsec
ra = np.array(meas['ra'],np.float64)
ra -= np.mean(ra)
ra *= 3600*1e3 * np.cos(mndec/radeg) # convert to true angle, milli arcsec
t = np.array(meas['mjd'].copy())
t -= np.mean(t)
t /= 365.2425 # convert to year
# Calculate robust slope
try:
#pmra, pmraerr = dln.robust_slope(t,ra,raerr,reweight=True)
# LADfit
pmra_ladcoef, absdev = dln.ladfit(t,ra)
pmra_lad = pmra_ladcoef[1]
# Run RANSAC
ransac = linear_model.RANSACRegressor()
ransac.fit(t.reshape(-1,1), ra)
inlier_mask = ransac.inlier_mask_
outlier_mask = np.logical_not(inlier_mask)
gdmask = inlier_mask
pmra_ransac = ransac.estimator_.coef_[0]
print(' ransac '+str(np.sum(inlier_mask))+' inliers '+str(np.sum(outlier_mask))+' outliers')
# Robust, weighted linear with with INLIERS
#pmra_coef, pmra_coeferr = dln.poly_fit(t[gdmask],ra[gdmask],1,sigma=raerr[gdmask],robust=True,error=True)
#pmra_coef, pmra_coeferr = dln.poly_fit(t,ra,1,sigma=raerr,robust=True,error=True)
#pmra = pmra_coef[0]
#pmraerr = pmra_coeferr[0]
#radiff = ra-dln.poly(t,pmra_coef)
radiff = ra-t*pmra_lad
radiff -= np.median(radiff)
rasig = dln.mad(radiff)
# Reject outliers
gdsig = (np.abs(radiff) < 2.5*rasig) | (np.abs(radiff) < 2.5*raerr)
print(' '+str(nmeas-np.sum(gdsig))+' 2.5*sigma clip outliers rejected')
#if np.sum(gdsig) < nmeas:
pmra_coef, pmra_coeferr = dln.poly_fit(t[gdsig],ra[gdsig],1,sigma=raerr[gdsig],robust=True,error=True)
pmra = pmra_coef[0]
pmraerr = pmra_coeferr[0]
rasig = dln.mad(ra-dln.poly(t,pmra_coef))
except:
print('problem')
#import pdb; pdb.set_trace()
return np.append(np.zeros(10,float)+np.nan, np.zeros(2,int))
decerr = np.array(meas['decerr']*1e3,np.float64) # milli arcsec
dec = np.array(meas['dec'],np.float64)
dec -= np.mean(dec)
dec *= 3600*1e3 # convert to milli arcsec
# Calculate robust slope
try:
#pmdec, pmdecerr = dln.robust_slope(t,dec,decerr,reweight=True)
# LADfit
pmdec_ladcoef, absdev = dln.ladfit(t,dec)
pmdec_lad = pmdec_ladcoef[1]
# Run RANSAC
ransac = linear_model.RANSACRegressor()
ransac.fit(t.reshape(-1,1), dec)
inlier_mask = ransac.inlier_mask_
outlier_mask = np.logical_not(inlier_mask)
gdmask = inlier_mask
pmdec_ransac = ransac.estimator_.coef_[0]
print(' ransac '+str(np.sum(inlier_mask))+' inliers '+str(np.sum(outlier_mask))+' outliers')
# Robust, weighted linear with with INLIERS
#pmdec_coef, pmdec_coeferr = dln.poly_fit(t[gdmask],dec[gdmask],1,sigma=decerr[gdmask],robust=True,error=True)
#pmdec_coef, pmdec_coeferr = dln.poly_fit(t,dec,1,sigma=decerr,robust=True,error=True)
#pmdec = pmdec_coef[0]
#pmdecerr = pmdec_coeferr[0]
#decdiff = dec-dln.poly(t,pmdec_coef)
decdiff = dec-t*pmdec_lad
decdiff -= np.median(decdiff)
decsig = dln.mad(decdiff)
# Reject outliers
gdsig = (np.abs(decdiff) < 2.5*decsig) | (np.abs(decdiff) < 2.5*decerr)
print(' '+str(nmeas-np.sum(gdsig))+' 2.5*sigma clip outliers rejected')
#if np.sum(gdsig) < nmeas:
pmdec_coef, pmdec_coeferr = dln.poly_fit(t[gdsig],dec[gdsig],1,sigma=decerr[gdsig],robust=True,error=True)
pmdec = pmdec_coef[0]
pmdecerr = pmdec_coeferr[0]
decsig = dln.mad(dec-dln.poly(t,pmdec_coef))
except:
print('problem')
#import pdb; pdb.set_trace()
return np.append(np.zeros(10,float)+np.nan, np.zeros(2,int))
deltamjd = np.max(meas['mjd'])-np.min(meas['mjd'])
out = np.array([pmra,pmraerr,pmra_ransac,pmra_lad,rasig,pmdec,pmdecerr,pmdec_ransac,pmdec_lad,decsig,nmeas,deltamjd])
#print(out[[0,2,3]])
#print(out[[5,7,8]])
#import pdb; pdb.set_trace()
return out
def zscaling(im, contrast=0.25, nsample=500000):
"""
This finds the IRAF display z1,z2 scalings
for an image
Parameters:
im 2D image
=contrast Contrast to use (contrast=0.25 by default)
=nsample The number of points of the image to use for the sample
(nsample=5e4 by default).
Returns:
[z1,z1] The minimum and maximum value to use for display scaling.
Examples:
zcals = zscale(im)
From IRAF display help
If the contrast is not zero the sample pixels are ranked in
brightness to form the function I(i) where i is the rank of the
pixel and I is its value. Generally the midpoint of this function
(the median) is very near the peak of the image histogram and there
is a well defined slope about the midpoint which is related to the
width of the histogram. At the ends of the I(i) function there are
a few very bright and dark pixels due to objects and defects in the
field. To determine the slope a linear function is fit with
iterative rejection;
I(i) = intercept + slope * (i - midpoint)
If more than half of the points are rejected then there is no well
defined slope and the full range of the sample defines z1 and z2.
Otherwise the endpoints of the linear function are used (provided
they are within the original range of the sample):
z1 = I(midpoint) + (slope / contrast) * (1 - midpoint)
z2 = I(midpoint) + (slope / contrast) * (npoints - midpoint)
The actual IRAF program is called zsc_zlimits and is in the file
/net/astro/iraf2.12/iraf/pkg/images/tv/display/zscale.x
By D.Nidever Oct 2007 (using IRAF display zscale algorithm)
"""
if im.ndim != 2:
raise ValueError('The input must be 2 dimensiona')
nx, ny = im.shape
n = nx * ny
nsample = np.minimum(nsample, n)
xind = np.round(np.random.random(nsample) * (nx - 1)).astype(int)
yind = np.round(np.random.random(nsample) * (ny - 1)).astype(int)
f = im[xind, yind]
si = np.argsort(f)
f2 = f[si]
x = np.arange(nsample)
midpoint = np.round(nsample * 0.5)
med = np.median(f)
zmin = np.min(im)
zmax = np.max(im)
zmed = np.median(im)
# Robust fitting program
coef = dln.poly_fit(x, f2, 1, robust=True)
# y = m*x + b
# I = intercept + slope * (i-midpoint)
# I = intercept + slope * i - slope*midpoint
# I = slope * i + (intercept - slope*midpoint)
# b = intercept - slope*midpoint
# intercept = b + slope*midpoint
slope = coef[0]
intercept = coef[1] + slope * midpoint
# z1 = I(midpoint) + (slope / contrast) * (1 - midpoint)
# z2 = I(midpoint) + (slope / contrast) * (npoints - midpoint)
#z1 = f2[midpoint] + (slope/contrast) * (1L - midpoint)
#z2 = f2[midpoint] + (slope/contrast) * (nsample - midpoint)
z1 = zmed + (slope / contrast) * (1 - midpoint)
z2 = zmed + (slope / contrast) * (nsample - midpoint)
z1 = np.maximum(z1, zmin)
z2 = np.minimum(z2, zmax)
return [z1, z2]
print(zpstr['instrument'][i]+'-'+zpstr['filter'][i]+' '+str(nind)+' exposures')
if nind>0:
calstr1 = calstr[ind]
zpterm = calstr1['zpterm']
bdzp,nbdzp = dln.where(~np.isfinite(zpterm)) # fix Infinity/NAN
if nbdzp>0:zpterm[bdzp] = 999999.9
am = calstr1['airmass']
mjd = calstr1['mjd']
bdam,nbdam = dln.where(am < 0.9)
if nbdam>0: am[bdam] = np.median(am)
# I GOT TO HERE IN THE TRANSLATING!!!
#glactc,calstr1.ra,calstr1.dec,2000.0,glon,glat,1,/deg
# Measure airmass dependence
gg0,ngg0 = dln.where((np.abs(zpterm)<50) & (am<2.0))
coef0 = dln.poly_fit(am[gg0],zpterm[gg0],1,robust=True)
zpf = dln.poly(am,coef0)
sig0 = np.mad(zpterm[gg0]-zpf[gg0])
gg,ngg = dln.where(np.abs(zpterm-zpf) < (np.maximum(3.5*sig0,0.2)))
coef = dln.poly_fit(am[gg],zpterm[gg],1,robust=True)
print(zpstr['instrument'][i]+'-'+zpstr['filter'][i]+' '+str(coef))
# Trim out bad exposures to determine the correlations and make figures
gg,ngg = dln.where(np.abs(zpterm-zpf) < (3.5*sig0 > 0.2) and calstr1.airmass < 2.0 and calstr1.fwhm < 2.0 and calstr1.rarms < 0.15 &
calstr1.decrms < 0.15 and calstr1.success == 1 and calstr1.wcscal == 'Successful' and calstr1.zptermerr < 0.05 &
calstr1.zptermsig < 0.08 and (calstr1.ngoodchipwcs == calstr1.nchips) &
(calstr1.instrument != 'c4d' or calstr1.zpspatialvar_nccd <= 5 or (calstr1.instrument == 'c4d' and
calstr1.zpspatialvar_nccd > 5 and calstr1.zpspatialvar_rms < 0.1)) and
np.abs(glat) > 10 and calstr1.nrefmatch > 100 and calstr1.exptime >= 30)
# Zpterm with airmass dependence removed
relzpterm = zpterm + 25 # 25 to get "absolute" zpterm
