def test_iterfit():
import numpy as np
from pydlutils.bspline import iterfit
from pydl import smooth
y0 = np.array([0.661984, 0.134913, 0.0410350, 0.940134, 0.411034,
0.484675, 0.169943, 0.325046, 0.269194, 0.552381,
0.797177, 0.971658, 0.251765, 0.531675, 0.854556,
0.411237, 0.694380, 0.499562, 0.437242, 0.362451,
0.343206, 0.524099, 0.158634, 0.728597, 0.198340,
0.571210, 0.477527, 0.962797, 0.973921, 0.413651,
0.736380, 0.516366, 0.104283, 0.675993, 0.467053,
0.230112, 0.866994, 0.469885, 0.964392, 0.541084,
0.332984, 0.581252, 0.422322, 0.872555, 0.803636,
0.520998, 0.918942, 0.241564, 0.169263, 0.686649,
0.708284, 0.707858, 0.00113957, 0.827920, 0.845985,
0.416961, 0.553842, 0.526549, 0.501051, 0.337514,
0.700873, 0.152816, 0.762935, 0.650039, 0.483321,
0.708600, 0.410033, 0.507671, 0.596956, 0.177692,
0.498112, 0.422037, 0.788333, 0.856578, 0.941245,
0.432411, 0.356469, 0.341916, 0.0331059, 0.641100,
0.690452, 0.168667, 0.915178, 0.158406, 0.701508,
0.841774, 0.434161, 0.153123, 0.420066, 0.0499331,
0.947241, 0.0768818, 0.410540, 0.843788, 0.0640255,
0.513463, 0.511104, 0.680434, 0.762480, 0.0563867])
y = smooth(y0,10)
x = np.arange(y0.size,dtype='d')
sset,outmask = iterfit(x,y,nord=3,maxiter=0,bkspace=10)
# print sset
yfit,mask = sset.value(x)
# print yfit
# pylab.plot(x,y,'k-',x,yfit,'r-')
return (x,y,yfit)
def clean_data():
ns = len(specs)
# only applying check for spikes for first 350 wavelengths
#ns = 300
nt = len(etx)
diff = np.zeros(ns*nt).reshape(ns,nt)
smooth = np.zeros(ns*nt).reshape(ns,nt)
for i in range(ns):
diff[i] = specs[i] - dl.smooth(specs[i], 3)
threshold = 0.03
for i in range(ns):
spectmp = []
ttmp = []
for j in range(nt):
if (diff[i][j] < threshold):
spectmp.append(specs[i][j])
ttmp.append(j)
trange = np.arange(0,len(etx))
specs[i] = np.interp(trange, np.asarray(ttmp), np.asarray(spectmp))
def combine1fiber(inloglam,objflux,objivar=None,**kwargs):
"""Combine several spectra of the same object, or resample a single spectrum.
"""
import numpy as np
import pydl.pydlutils.bspline
import pydl.pydlutils.sdss
from pydl import smooth
from . import aesthetics
#
# Check that dimensions of inputs are valid.
#
if 'newloglam' in kwargs:
newloglam = kwargs['newloglam']
nfinalpix = len(newloglam)
else:
raise ValueError('newloglam is required.')
if objflux.shape != inloglam.shape:
raise ValueError('Dimensions of inloglam and objflux do not agree.')
if objivar is not None:
if objivar.shape != inloglam.shape:
raise ValueError('Dimensions of inloglam and objivar do not agree.')
if 'finalmask' in kwargs:
if kwargs['finalmask'].shape != inloglam.shape:
raise ValueError('Dimensions of inloglam and finalmask do not agree.')
if 'indisp' in kwargs:
if kwargs['indisp'].shape != inloglam.shape:
raise ValueError('Dimensions of inloglam and indisp do not agree.')
#
# Set defaults
#
EPS = np.finfo(np.float32).eps
if 'binsz' in kwargs:
binsz = kwargs['binsz']
else:
binsz = inloglam[1] - inloglam[0]
if 'nord' in kwargs:
nord = kwargs['nord']
else:
nord = 3
if 'bkptbin' in kwargs:
bkptbin = kwargs['bkptbin']
else:
bkptbin = 1.2 * binsz
if 'maxsep' in kwargs:
maxsep = kwargs['maxsep']
else:
maxsep = 2.0 * binsz
if inloglam.ndim == 1:
#
# Set specnum = 0 for all elements
#
npix = inloglam.shape[0]
nspec = 1
specnum = np.zeros(inloglam.shape,dtype=inloglam.dtype)
else:
nspec,npix = inloglam.shape
specnum = np.tile(np.arange(nspec),npix).reshape(npix,nspec).transpose()
#
# Use fullcombmask for modifying the pixel masks in the original input files.
#
fullcombmask = np.zeros(npix)
newflux = np.zeros(nfinalpix,dtype=inloglam.dtype)
newmask = np.zeros(nfinalpix,dtype='i4')
newivar = np.zeros(nfinalpix,dtype=inloglam.dtype)
newdisp = np.zeros(nfinalpix,dtype=inloglam.dtype)
newsky = np.zeros(nfinalpix,dtype=inloglam.dtype)
newdispweight = np.zeros(nfinalpix,dtype=inloglam.dtype)
if objivar is None:
nonzero = np.arange(npix,dtype='i4')
ngood = npix
else:
nonzero = (objivar > 0).nonzero()[0]
ngood = len(nonzero)
#
# ormask is needed to create andmask
#
andmask = np.zeros(nfinalpix,dtype='i4')
ormask = np.zeros(nfinalpix,dtype='i4')
if ngood == 0:
#
# In this case of no good points, set the nodata bit everywhere.
# Also if noplug is set in the first input bit-mask, assume it
# should be set everywhere in the output bit masks. No other bits
# are set.
#
if 'verbose' in kwargs:
print('No good points')
bitval = pydl.pydlutils.sdss.sdss_flagval('SPPIXMASK','NODATA')
if 'finalmask' in kwargs:
bitval = bitval | (pydl.pydlutils.sdss.sdss_flagval('SPPIXMASK','NOPLUG') *
(finalmask[0] & pydl.pydlutils.sdss.sdss_flagval('SPPIXMASK','NODATA')))
andmask = andmask | bitval
ormask = ormask | bitval
return (newflux,newivar)
else:
#
# Now let's break sorted wavelengths into groups where pixel
# separations are larger than maxsep.
#
isort = nonzero[inloglam[nonzero].argsort()]
wavesort = inloglam[isort]
padwave = np.insert(wavesort,0,wavesort.min() - 2.0*maxsep)
padwave = np.append(padwave,wavesort.max() + 2.0*maxsep)
ig1 = ((padwave[1:ngood+1]-padwave[0:ngood]) > maxsep).nonzero()[0]
ig2 = ((padwave[2:ngood+2]-padwave[1:ngood+1]) > maxsep).nonzero()[0]
if ig1.size != ig2.size:
raise ValueError('Grouping tricks did not work!')
for igrp in range(ig1.size):
ss = isort[ig1[igrp]:ig2[igrp]+1]
if ss.size > 2:
if objivar is None:
#
# Fit without variance
#
sset,bmask = pydl.pydlutils.bspline.iterfit(inloglam[ss],objflux[ss],
nord=nord,groupbadpix=True,requiren=1,bkspace=bkptbin,
silent=True)
else:
#
# Fit with variance
#
sset,bmask = pydl.pydlutils.bspline.iterfit(inloglam[ss],objflux[ss],
invvar=objivar[ss],
nord=nord,groupbadpix=True,requiren=1,bkspace=bkptbin,
silent=True)
if np.sum(np.absolute(sset.coeff)) == 0:
sset = None
bmask = np.zeros(len(ss))
if 'verbose' in kwargs:
print('WARNING: All B-spline coefficients have been set to zero!')
else:
bmask = np.zeros(len(ss))
sset = None
if 'verbose' in kwargs:
print('WARNING: Not enough data for B-spline fit!')
inside = ((newloglam >= inloglam[ss].min()-EPS) &
(newloglam <= inloglam[ss].max()+EPS)).nonzero()[0]
#
# It is possible for numinside to be zero, if the input data points
# span an extremely small wavelength range, within which there are
# no output wavelengths.
#
if sset is not None and len(inside) > 0:
newflux[inside],bvalumask = sset.value(newloglam[inside])
if bvalumask.any():
newmask[inside[bvalumask]] = 1
if 'verbose' in kwargs:
print('Masked {0:d} of {1:d} pixels.'.format(bmask.sum()-bmask.size,bmask.size))
#
# Determine which pixels should be masked based upon the spline
# fit. Set the combinerej bit.
#
ireplace = ~bmask
if ireplace.any():
#
# The following would replace the original flux values of
# masked pixels with b-spline evaluations.
#
# objflux[ss[ireplace]] = sset.value(inloglam[ss[ireplace]])
#
# Set the inverse variance of these pixels to zero.
#
if objivar is not None:
objivar[ss[ireplace]] = 0.0
if 'verbose' in kwargs:
print('Replaced {0:d} pixels in objivar.'.format(len(ss[ireplace])))
if 'finalmask' in kwargs:
finalmask[ss[ireplace]] = (finalmask[ss[ireplace]] |
pydl.pydlutils.sdss.sdss_flagval('SPPIXMASK','COMBINEREJ'))
fullcombmask[ss] = bmask
#
# Combine inverse variance and pixel masks.
#
# Start with all bits set in andmask
#
andmask[:] = -1
for j in range(int(specnum.max())+1):
these = specnum == j
if these.any():
inbetween = ((newloglam >= inloglam[these].min()) &
(newloglam <= inloglam[these].max()))
if inbetween.any():
jnbetween = inbetween.nonzero()[0]
#
# Conserve inverse variance by doing a linear interpolation
# on that quantity.
#
result = np.interp(newloglam[jnbetween],
inloglam[these],objivar[these]*fullcombmask[these])
#
# Grow the fullcombmask below to reject any new sampling
# containing even a partial masked pixel.
#
smask = np.interp(newloglam[jnbetween],inloglam[these],
fullcombmask[these].astype(inloglam.dtype))
result *= smask >= (1.0 - EPS)
newivar[jnbetween] += result*newmask[jnbetween]
lowside = np.floor((inloglam[these]-newloglam[0])/binsz).astype('i4')
highside = lowside + 1
if 'finalmask' in kwargs:
andmask[lowside] &= finalmask[these]
andmask[highside] &= finalmask[these]
ormask[lowside] |= finalmask[these]
ormask[highside] |= finalmask[these]
#
# Combine the dispersions + skies in the dumbest way possible
# [sic].
#
if 'indisp' in kwargs:
newdispweight[jnbetween] += result
newdisp[jnbetween] += result * np.interp(newloglam[jnbetween],
inloglam[these],indisp[these])
newsky[jnbetween] += result * np.interp(newloglam[jnbetween],
inloglam[these],skyflux[these])
if 'indisp' in kwargs:
newdisp /= newdispweight + (newdispweight == 0)
newsky /= newdispweight + (newdispweight == 0)
#
# Grow regions where 3 or more pixels are rejected together ???
#
# print newivar
foo = smooth(newivar,3)
# print foo
# sys.exit(1)
badregion = np.absolute(foo) < EPS
if badregion.any():
if 'verbose' in kwargs:
print('WARNING: Growing bad pixel region, {0:d} pixels found.'.format(badregion.sum()))
ibad = badregion.nonzero()[0]
lowerregion = np.where(ibad-2 < 0,0,ibad-2)
upperregion = np.where(ibad+2 > nfinalpix-1,nfinalpix-1,ibad+2)
newivar[lowerregion] = 0.0
newivar[upperregion] = 0.0
#
# Replace NaNs in combined spectra; this should really never happen.
#
inff = ((~np.isfinite(newflux)) | (~np.isfinite(newivar)))
if inff.any():
print('WARNING: {0:d} NaNs in combined spectra.'.format(inff.sum()))
newflux[inff] = 0.0
newivar[inff] = 0.0
#
# Interpolate over masked pixels, just for aesthetic purposoes.
#
goodpts = newivar > 0
if 'aesthetics' in kwargs:
amethod = kwargs['aesthetics']
else:
amethod = 'traditional'
newflux = aesthetics(newflux,newivar,method=amethod)
# if 'interpolate' in kwargs:
# newflux = pydlutils.image.djs_maskinterp(newflux,~goodpts,const=True)
# else:
# newflux[~goodpts] = newflux[goodpts].mean()
if goodpts.any():
minglam = newloglam[goodpts].min()
maxglam = newloglam[goodpts].max()
ibad = ((newloglam < minglam) | (newloglam > maxglam))
if ibad.any():
ormask[ibad] |= pydl.pydlutils.sdss.sdss_flagval('SPPIXMASK','NODATA')
#
# Replace values of -1 in the andmask with 0.
#
andmask *= (andmask != -1)
#
# Copy the nodata bad pixels from the ormask to the andmask.
#
andmask |= ormask & pydl.pydlutils.sdss.sdss_flagval('SPPIXMASK','NODATA')
return (newflux,newivar)
