def w2p(dispersion, w, extrapolate=True): """ Convert wavelength values to pixels for a "dispersion solution". Parameters ---------- dispersion : 1D array The dispersion solution. This is basically just a 1D array of monotonically increasing (or decreasing) wavelengths. w : array Array of wavelength values to convert to pixels. extrapolate : bool, optional Extrapolate beyond the dispersion solution, if necessary. This is True by default. Returns ------- x : array Array of converted pixel values. Example ------- .. code-block:: python x = w2p(disp,w) """ x = interp1d(dispersion, np.arange(len(dispersion)), kind='cubic', bounds_error=False, fill_value=(np.nan, np.nan), assume_sorted=False)(w) # Need to extrapolate if ((np.min(w) < np.min(dispersion)) | (np.max(w) > np.max(dispersion))) & (extrapolate is True): win = dispersion xin = np.arange(len(dispersion)) si = np.argsort(win) win = win[si] xin = xin[si] npix = len(win) # At the beginning if (np.min(w) < np.min(dispersion)): #coef1 = dln.poly_fit(win[0:10], xin[0:10], 2) coef1 = dln.quadratic_coefficients(win[0:10], xin[0:10]) bd1, nbd1 = dln.where(w < np.min(dispersion)) x[bd1] = dln.poly(w[bd1], coef1) # At the end if (np.max(w) > np.max(dispersion)): #coef2 = dln.poly_fit(win[npix-10:], xin[npix-10:], 2) coef2 = dln.quadratic_coefficients(win[npix - 10:], xin[npix - 10:]) bd2, nbd2 = dln.where(w > np.max(dispersion)) x[bd2] = dln.poly(w[bd2], coef2) return x
def p2w(dispersion, x, extrapolate=True): """ Convert pixel values to wavelengths for a "dispersion solution". Parameters ---------- dispersion : 1D array The dispersion solution. This is basically just a 1D array of monotonically increasing (or decreasing) wavelengths. x : array Array of pixel values to convert to wavelengths. extrapolate : bool, optional Extrapolate beyond the dispersion solution, if necessary. This is True by default. Returns ------- w : array Array of converted wavelengths. Example ------- .. code-block:: python w = p2w(disp,x) """ npix = len(dispersion) w = interp1d(np.arange(len(dispersion)), dispersion, kind='cubic', bounds_error=False, fill_value=(np.nan, np.nan), assume_sorted=False)(x) # Need to extrapolate if ((np.min(x) < 0) | (np.max(x) > (npix - 1))) & (extrapolate is True): xin = np.arange(npix) win = dispersion # At the beginning if (np.min(x) < 0): #coef1 = dln.poly_fit(xin[0:10], win[0:10], 2) coef1 = dln.quadratic_coefficients(xin[0:10], win[0:10]) bd1, nbd1 = dln.where(x < 0) w[bd1] = dln.poly(x[bd1], coef1) # At the end if (np.max(x) > (npix - 1)): #coef2 = dln.poly_fit(xin[npix-10:], win[npix-10:], 2) coef2 = dln.quadratic_coefficients(xin[npix - 10:], win[npix - 10:]) bd2, nbd2 = dln.where(x > (npix - 1)) w[bd2] = dln.poly(x[bd2], coef2) return w
def maskoutliers(spec,nsig=5,verbose=False,logger=None): """ Mask large positive outliers and negative flux pixels in the spectrum. """ if logger is None: logger = dln.basiclogger() spec2 = spec.copy() wave = spec2.wave.copy().reshape(spec2.npix,spec2.norder) # make 2D flux = spec2.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D err = spec2.err.copy().reshape(spec2.npix,spec2.norder) # make 2D mask = spec2.mask.copy().reshape(spec2.npix,spec2.norder) # make 2D totnbd = 0 for o in range(spec2.norder): w = wave[:,o].copy() x = (w-np.median(w))/(np.max(w*0.5)-np.min(w*0.5)) # -1 to +1 y = flux[:,o].copy() m = mask[:,o].copy() # Divide by median medy = np.nanmedian(y) y /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x,y,2,robust=True) sig = dln.mad(y-dln.poly(x,coef)) bd,nbd = dln.where( ((y-dln.poly(x,coef)) > nsig*sig) | (y<0)) totnbd += nbd if nbd>0: flux[bd,o] = dln.poly(x[bd],coef)*medy err[bd,o] = 1e30 mask[bd,o] = True # Flatten to 1D if norder=1 if spec2.norder==1: flux = flux.flatten() err = err.flatten() mask = mask.flatten() # Stuff back in spec2.flux = flux spec2.err = err spec2.mask = mask if verbose is True: logger.info('Masked '+str(totnbd)+' outlier or negative pixels') return spec2
def maskdiscrepant(spec,model,nsig=10,verbose=False,logger=None): """ Mask pixels that are discrepant when compared to a model. """ if logger is None: logger = dln.basiclogger() spec2 = spec.copy() wave = spec2.wave.copy().reshape(spec2.npix,spec2.norder) # make 2D flux = spec2.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D err = spec2.err.copy().reshape(spec2.npix,spec2.norder) # make 2D mask = spec2.mask.copy().reshape(spec2.npix,spec2.norder) # make 2D mflux = model.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D totnbd = 0 for o in range(spec2.norder): w = wave[:,o].copy() x = (w-np.median(w))/(np.max(w*0.5)-np.min(w*0.5)) # -1 to +1 y = flux[:,o].copy() m = mask[:,o].copy() my = mflux[:,o].copy() # Divide by median medy = np.nanmedian(y) y /= medy my /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x,y,2,robust=True) sig = dln.mad(y-my) bd,nbd = dln.where( np.abs(y-my) > nsig*sig ) totnbd += nbd if nbd>0: flux[bd,o] = dln.poly(x[bd],coef)*medy err[bd,o] = 1e30 mask[bd,o] = True # Flatten to 1D if norder=1 if spec2.norder==1: flux = flux.flatten() err = err.flatten() mask = mask.flatten() # Stuff back in spec2.flux = flux spec2.err = err spec2.mask = mask if verbose is True: logger.info('Masked '+str(totnbd)+' discrepant pixels') return spec2
def continuum(spec, norder=6, perclevel=90.0, binsize=0.1, interp=True): """ Measure the continuum of a spectrum. Parameters ---------- spec : Spec1D object A spectrum object. This at least needs to have a FLUX and WAVE attribute. norder : float, optional Polynomial order to use for the continuum fitting. The default is 6. perclevel : float, optional Percent level to use for the continuum value in large bins. Default is 90. binsize : float, optional Fraction of the wavelength range (scaled to -1 to +1) to bin. Default is 0.1. interp : bool, optional Use interpolation of the binned values instead of a polynomial fit. Default is True. Returns ------- cont : numpy array The continuum array, in the same shape as the input flux. Examples -------- .. code-block:: python cont = continuum(spec) """ wave = spec.wave.copy().reshape(spec.npix, spec.norder) # make 2D flux = spec.flux.copy().reshape(spec.npix, spec.norder) # make 2D err = spec.err.copy().reshape(spec.npix, spec.norder) # make 2D mask = spec.mask.copy().reshape(spec.npix, spec.norder) # make 2D cont = err.copy() * 0.0 + 1 for o in range(spec.norder): w = wave[:, o].copy() wr = [np.min(w), np.max(w)] x = (w - np.mean(wr)) / dln.valrange(wr) * 2 # -1 to +1 y = flux[:, o].copy() m = mask[:, o].copy() # Divide by median medy = np.nanmedian(y) if medy <= 0.0: medy = 1.0 y /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x[~m], y[~m], 2, robust=True) sig = dln.mad(y - dln.poly(x, coef)) bd, nbd = dln.where((y - dln.poly(x, coef)) > 5 * sig) if nbd > 0: m[bd] = True gdmask = (y > 0) & (m == False) # need positive fluxes and no mask set if np.sum(gdmask) == 0: continue # Bin the data points xr = [np.nanmin(x), np.nanmax(x)] bins = int(np.ceil((xr[1] - xr[0]) / binsize)) ybin, bin_edges, binnumber = bindata.binned_statistic( x[gdmask], y[gdmask], statistic='percentile', percentile=perclevel, bins=bins, range=None) xbin = bin_edges[0:-1] + 0.5 * binsize # Interpolate to full grid if interp is True: fnt = np.isfinite(ybin) cont1 = dln.interp(xbin[fnt], ybin[fnt], x, kind='quadratic', extrapolate=True, exporder=1) else: coef = dln.poly_fit(xbin, ybin, norder) cont1 = dln.poly(x, coef) cont1 *= medy cont[:, o] = cont1 # Flatten to 1D if norder=1 if spec.norder == 1: cont = cont.flatten() return cont
def apstar(filename,badval=20735): """ Read an SDSS APOGEE apStar spectrum. Parameters ---------- filename : string The name of the spectrum file to load. Returns ------- spec : Spec1D object The spectrum as a Spec1D object. Examples -------- spec = apstar('spec.fits') """ base, ext = os.path.splitext(os.path.basename(filename)) # APOGEE apStar, combined spectrum if (base.find("apStar") > -1) | (base.find("asStar") > -1): # HISTORY APSTAR: HDU0 = Header only # HISTORY APSTAR: All image extensions have: # HISTORY APSTAR: row 1: combined spectrum with individual pixel weighting # HISTORY APSTAR: row 2: combined spectrum with global weighting # HISTORY APSTAR: row 3-nvisits+2: individual resampled visit spectra # HISTORY APSTAR: unless nvisits=1, which only have a single row # HISTORY APSTAR: All spectra shifted to rest (vacuum) wavelength scale # HISTORY APSTAR: HDU1 - Flux (10^-17 ergs/s/cm^2/Ang) # HISTORY APSTAR: HDU2 - Error (10^-17 ergs/s/cm^2/Ang) # HISTORY APSTAR: HDU3 - Flag mask: # HISTORY APSTAR: row 1: bitwise OR of all visits # HISTORY APSTAR: row 2: bitwise AND of all visits # HISTORY APSTAR: row 3-nvisits+2: individual visit masks # HISTORY APSTAR: HDU4 - Sky (10^-17 ergs/s/cm^2/Ang) # HISTORY APSTAR: HDU5 - Sky Error (10^-17 ergs/s/cm^2/Ang) # HISTORY APSTAR: HDU6 - Telluric # HISTORY APSTAR: HDU7 - Telluric Error # HISTORY APSTAR: HDU8 - LSF coefficients # HISTORY APSTAR: HDU9 - RV and CCF structure # Get number of extensions hdulist = fits.open(filename) nhdu = len(hdulist) hdulist.close() # Spectrum, error, sky, skyerr are in units of 1e-17 # these are 2D arrays with [Nvisit+2,Npix] # the first two are combined and the rest are the individual spectra head1 = fits.getheader(filename,1) w0 = np.float64(head1["CRVAL1"]) dw = np.float64(head1["CDELT1"]) nw = head1["NAXIS1"] wave = 10**(np.arange(nw)*dw+w0) # flux, err, sky, skyerr are in units of 1e-17 flux = fits.getdata(filename,1).T * 1e-17 lsfcoef = fits.getdata(filename,8).T spec = Spec1D(flux,wave=wave,lsfpars=lsfcoef,lsftype='Gauss-Hermite',lsfxtype='Pixels') spec.filename = filename spec.sptype = "apStar" spec.waveregime = "NIR" spec.instrument = "APOGEE" spec.head = fits.getheader(filename,0) spec.err = fits.getdata(filename,2).T * 1e-17 #bad = (spec.err<=0) # fix bad error values #if np.sum(bad) > 0: # spec.err[bad] = 1e30 spec.bitmask = fits.getdata(filename,3) spec.sky = fits.getdata(filename,4).T * 1e-17 spec.skyerr = fits.getdata(filename,5).T * 1e-17 spec.telluric = fits.getdata(filename,6).T spec.telerr = fits.getdata(filename,7).T spec.lsf = fits.getdata(filename,8).T # Create the bad pixel mask # "bad" pixels: # flag = ['BADPIX','CRPIX','SATPIX','UNFIXABLE','BADDARK','BADFLAT','BADERR','NOSKY', # 'LITTROW_GHOST','PERSIST_HIGH','PERSIST_MED','PERSIST_LOW','SIG_SKYLINE','SIG_TELLURIC','NOT_ENOUGH_PSF',''] # badflag = [1,1,1,1,1,1,1,1, # 0,0,0,0,0,0,1,0] mask = (np.bitwise_and(spec.bitmask,badval)!=0) | (np.isfinite(spec.flux)==False) # Extra masking for bright skylines x = np.arange(spec.npix) nsky = 4 medsky = median_filter(spec.sky,201,mode='reflect') medcoef = dln.poly_fit(x,medsky/np.median(medsky),2) medsky2 = dln.poly(x,medcoef)*np.median(medsky) skymask1 = (sky>nsky*medsky2) # pixels Nsig above median sky mask[:,i] = np.logical_or(mask[:,i],skymask1) # OR combine spec.mask = mask # Fix NaN or bad pixels pixels bd,nbd = dln.where( (np.isfinite(spec.flux[:,i])==False) | (spec.err[:,i] <= 0.0) ) if nbd>0: spec.flux[bd] = 0.0 spec.err[bd] = 1e30 spec.mask[bd] = True if nhdu>=9: spec.meta = fits.getdata(filename,9) # meta-data spec.snr = spec.head["SNR"] if base.find("apStar") > -1: spec.observatory = 'apo' else: spec.observatory = 'lco' spec.wavevac = True return spec
def normalize(self, ncorder=6, perclevel=0.95): """ Normalize the spectrum. Parameters ---------- ncorder : float, optional Polynomial order to use for the continuum fitting. The default is 6. perclevel : float, optional Percent level (1.0 for 100%) to use for the continuum value in large bins. Default is 0.95. Returns ------- The flux and err arrays will normalized (divided) by the continuum and the continuum saved in cont. The normalized property is set to True. Examples -------- spec.normalize() """ self._flux = self.flux # Save the original #nspec, cont, masked = normspec(self,ncorder=ncorder,perclevel=perclevel) binsize = 0.10 perclevel = 90.0 wave = self.wave.copy().reshape(self.npix, self.norder) # make 2D flux = self.flux.copy().reshape(self.npix, self.norder) # make 2D err = self.err.copy().reshape(self.npix, self.norder) # make 2D mask = self.mask.copy().reshape(self.npix, self.norder) # make 2D cont = err.copy() * 0.0 + 1 for o in range(self.norder): w = wave[:, o].copy() x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5) ) # -1 to +1 y = flux[:, o].copy() m = mask[:, o].copy() # Divide by median medy = np.nanmedian(y) y /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x, y, 2, robust=True) sig = dln.mad(y - dln.poly(x, coef)) bd, nbd = dln.where((y - dln.poly(x, coef)) > 5 * sig) if nbd > 0: m[bd] = True gdmask = (y > 0) & (m == False ) # need positive fluxes and no mask set # Bin the data points xr = [np.nanmin(x), np.nanmax(x)] bins = np.ceil((xr[1] - xr[0]) / binsize) + 1 ybin, bin_edges, binnumber = bindata.binned_statistic( x[gdmask], y[gdmask], statistic='percentile', percentile=perclevel, bins=bins, range=None) xbin = bin_edges[0:-1] + 0.5 * binsize # Interpolate to full grid fnt = np.isfinite(ybin) cont1 = dln.interp(xbin[fnt], ybin[fnt], x, extrapolate=True) cont1 *= medy flux[:, o] /= cont1 err[:, o] /= cont1 cont[:, o] = cont1 # Flatten to 1D if norder=1 if self.norder == 1: flux = flux.flatten() err = err.flatten() cont = cont.flatten() # Stuff back in self.flux = flux self.err = err self.cont = cont self.normalized = True return
linecat['wave0'])) maxdiff = 0.5 #1.0 if np.abs(tlinecat['wave'][j] - linecat['wave0'][bestline]) < maxdiff: match[j] = bestline dwave[j] = tlinecat['wave'][j] - linecat['wave0'][bestline] gmatch, = np.where(match > -1) linecat['wave'][match[gmatch]] = tlinecat['wave'][gmatch] linecat['match'][match[gmatch]] = 1 # Perform an initial fit gline1, = np.where(linecat['match'] == 1) meanwave = np.mean(linecat['wave'][gline1]) coef1 = dln.poly_fit(linecat['center'][gline1], linecat['wave'][gline1], 3) dwave = dln.poly(linecat['center'][gline1], coef1) - linecat['wave'][gline1] sigwave = dln.mad(dwave) if sigwave > 0.1: print('Sigma high. Trying higher order') coef1 = dln.poly_fit(linecat['center'][gline1], linecat['wave'][gline1], 4) dwave = dln.poly(linecat['center'][gline1], coef1) - linecat['wave'][gline1] sigwave = dln.mad(dwave) print('Sigma of initial fit %6.4f A' % sigwave) bdline, = np.where(np.abs(dwave) > 3 * sigwave) if len(bdline) > 0: linecat['outlier'][gline1[bdline]] = True print('throwing out ' + str(len(bdline)) + ' outlier lines') #if sigwave>0.1:
def apvisit(filename): """ Read a SDSS APOGEE apVisit spectrum. Parameters ---------- filename : string The name of the spectrum file to load. Returns ------- spec : Spec1D object The spectrum as a Spec1D object. Examples -------- spec = apvisit('spec.fits') """ base, ext = os.path.splitext(os.path.basename(filename)) # Get number of extensions hdulist = fits.open(filename) nhdu = len(hdulist) hdulist.close() # Check that this has the right format validfile = False if (base.find("apVisit") == -1) | (base.find("asVisit") == -1): if nhdu >= 10: gd, = np.where( np.char.array(hdulist[0].header['HISTORY']).astype(str).find( 'AP1DVISIT') > -1) if len(gd) > 0: validfile = True if validfile is False: return None # APOGEE apVisit, visit-level spectrum # HISTORY AP1DVISIT: HDU0 = Header only # HISTORY AP1DVISIT: HDU1 - Flux (10^-17 ergs/s/cm^2/Ang) # HISTORY AP1DVISIT: HDU2 - Error (10^-17 ergs/s/cm^2/Ang) # HISTORY AP1DVISIT: HDU3 - Flag mask (bitwise OR combined) # HISTORY AP1DVISIT: HDU4 - Wavelength (Ang) # HISTORY AP1DVISIT: HDU5 - Sky (10^-17 ergs/s/cm^2/Ang) # HISTORY AP1DVISIT: HDU6 - Sky Error (10^-17 ergs/s/cm^2/Ang) # HISTORY AP1DVISIT: HDU7 - Telluric # HISTORY AP1DVISIT: HDU8 - Telluric Error # HISTORY AP1DVISIT: HDU9 - Wavelength coefficients # HISTORY AP1DVISIT: HDU10 - LSF coefficients # HISTORY AP1DVISIT: HDU11 - RV catalog # flux, err, sky, skyerr are in units of 1e-17 flux = fits.getdata(filename, 1).T * 1e-17 # [Npix,Norder] wave = fits.getdata(filename, 4).T lsfcoef = fits.getdata(filename, 10).T spec = Spec1D(flux, wave=wave, lsfpars=lsfcoef, lsftype='Gauss-Hermite', lsfxtype='Pixels') spec.reader = 'apvisit' spec.filename = filename spec.sptype = "apVisit" spec.waveregime = "NIR" spec.instrument = "APOGEE" spec.head = fits.getheader(filename, 0) spec.err = fits.getdata(filename, 2).T * 1e-17 # [Npix,Norder] #bad = (spec.err<=0) # fix bad error values #if np.sum(bad) > 0: # spec.err[bad] = 1e30 spec.bitmask = fits.getdata(filename, 3).T spec.sky = fits.getdata(filename, 5).T * 1e-17 spec.skyerr = fits.getdata(filename, 6).T * 1e-17 spec.telluric = fits.getdata(filename, 7).T spec.telerr = fits.getdata(filename, 8).T spec.wcoef = fits.getdata(filename, 9).T # Create the bad pixel mask # "bad" pixels: # flag = ['BADPIX','CRPIX','SATPIX','UNFIXABLE','BADDARK','BADFLAT','BADERR','NOSKY', # 'LITTROW_GHOST','PERSIST_HIGH','PERSIST_MED','PERSIST_LOW','SIG_SKYLINE','SIG_TELLURIC','NOT_ENOUGH_PSF',''] # badflag = [1,1,1,1,1,1,1,1, # 0,0,0,0,0,0,1,0] mask = (np.bitwise_and(spec.bitmask, 16639) != 0) | (np.isfinite(spec.flux) == False) # Extra masking for bright skylines x = np.arange(spec.npix) nsky = 4 for i in range(spec.norder): sky = spec.sky[:, i] medsky = median_filter(sky, 201, mode='reflect') medcoef = dln.poly_fit(x, medsky / np.median(medsky), 2) medsky2 = dln.poly(x, medcoef) * np.median(medsky) skymask1 = (sky > nsky * medsky2) # pixels Nsig above median sky mask[:, i] = np.logical_or(mask[:, i], skymask1) # OR combine spec.mask = mask # Fix NaN pixels for i in range(spec.norder): bd, nbd = dln.where((np.isfinite(spec.flux[:, i]) == False) | (spec.err[:, i] <= 0)) if nbd > 0: spec.flux[bd, i] = 0.0 spec.err[bd, i] = 1e30 spec.mask[bd, i] = True if (nhdu >= 11): spec.meta = fits.getdata(filename, 11) # catalog of RV and other meta-data # Spectrum, error, sky, skyerr are in units of 1e-17 spec.snr = spec.head["SNR"] if base.find("apVisit") > -1: spec.observatory = 'apo' else: spec.observatory = 'lco' spec.wavevac = True return spec
def sigma(self,x=None,xtype='pixels',order=0,extrapolate=True): """ Return the Gaussian sigma at specified locations. The sigma will be in units of lsf.xtype. Parameters ---------- x : array, optional The x-values for which to return the Gaussian sigma values. xtype : string, optional The type of x-value input, either 'wave' or 'pixels'. Default is 'pixels'. order : int, optional The order to use if there are multiple orders. The default is 0. extrapolate : bool, optional Extrapolate beyond the dispersion solution, if necessary. True by default. Returns ------- sigma : array The array of Gaussian sigma values. Examples -------- sigma = lsf.sigma([100,200]) """ # The sigma will be returned in units given in lsf.xtype if self._sigma is not None: _sigma = self._sigma if self.ndim==2: _sigma = self._sigma[:,order] if x is None: return _sigma else: # Wavelength input if xtype.lower().find('wave') > -1: x0 = np.array(x).copy() # backup x = self.wave2pix(x0,order=order) # convert to pixels # Integer, just return the values if( type(x)==int) | (np.array(x).dtype.kind=='i'): return _sigma[x] # Floats, interpolate else: sig = interp1d(np.arange(len(_sigma)),_sigma,kind='cubic',bounds_error=False, fill_value=(np.nan,np.nan),assume_sorted=True)(x) # Extrapolate npix = self.npix if ((np.min(x)<0) | (np.max(x)>(npix-1))) & (extrapolate is True): xin = np.arange(npix) # At the beginning if (np.min(x)<0): coef1 = dln.poly_fit(xin[0:10], _sigma[0:10], 2) bd1, nbd1 = dln.where(x <0) sig[bd1] = dln.poly(x[bd1],coef1) # At the end if (np.max(x)>(npix-1)): coef2 = dln.poly_fit(xin[npix-10:], _sigma[npix-10:], 2) bd2, nbd2 = dln.where(x > (npix-1)) sig[bd2] = dln.poly(x[bd2],coef2) return sig # Need to calculate else: if x is None: x = np.arange(self.npix) if self.pars is None: raise Exception("No LSF parameters") # Get parameters pars = self.pars if self.ndim==2: pars=self.pars[:,order] # Pixels input if xtype.lower().find('pix') > -1: # Pixel LSF parameters if self.xtype.lower().find('pix') > -1: return np.polyval(pars[::-1],x) # Wave LSF parameters else: w = self.pix2wave(x,order=order) return np.polyval(pars[::-1],w) # Wavelengths input else: # Wavelength LSF parameters if self.xtype.lower().find('wave') > -1: return np.polyval(pars[::-1],x) # Pixel LSF parameters else: x0 = np.array(x).copy() x = self.wave2pix(x0,order=order) return np.polyval(pars[::-1],x)
def maskdiscrepant(spec, model, nsig=10, verbose=False): """ Mask pixels that are discrepant when compared to a model. Parameters ---------- spec : Spec1D object Observed spectrum for which to mask discrepant values. model : Spec1D object Reference/model spectrum to use to find discrepant values. nsig : int, optional Number of standard deviations to use for the discrepant values. Default is 10.0. verbose : boolean, optional Verbose output. Default is False. Returns ------- spec2 : Spec1D object Spectrum with discrepant values masked. Example ------- .. code-block:: python spec = maskdiscrepant(spec,nsig=5) """ print = getprintfunc( ) # Get print function to be used locally, allows for easy logging spec2 = spec.copy() wave = spec2.wave.copy().reshape(spec2.npix, spec2.norder) # make 2D flux = spec2.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D err = spec2.err.copy().reshape(spec2.npix, spec2.norder) # make 2D mask = spec2.mask.copy().reshape(spec2.npix, spec2.norder) # make 2D mflux = model.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D totnbd = 0 for o in range(spec2.norder): w = wave[:, o].copy() x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5) ) # -1 to +1 y = flux[:, o].copy() m = mask[:, o].copy() my = mflux[:, o].copy() # Divide by median medy = np.nanmedian(y) if medy <= 0.0: medy = 1.0 y /= medy my /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x, y, 2, robust=True) sig = dln.mad(y - my) bd, nbd = dln.where(np.abs(y - my) > nsig * sig) totnbd += nbd if nbd > 0: flux[bd, o] = dln.poly(x[bd], coef) * medy err[bd, o] = 1e30 mask[bd, o] = True # Flatten to 1D if norder=1 if spec2.norder == 1: flux = flux.flatten() err = err.flatten() mask = mask.flatten() # Stuff back in spec2.flux = flux spec2.err = err spec2.mask = mask if verbose is True: print('Masked ' + str(totnbd) + ' discrepant pixels') return spec2
def maskoutliers(spec, nsig=5, verbose=False): """ Mask large positive outliers and negative flux pixels in the spectrum. Parameters ---------- spec : Spec1D object Observed spectrum to mask outliers on. nsig : int, optional Number of standard deviations to use for the outlier rejection. Default is 5.0. verbose : boolean, optional Verbose output. Default is False. Returns ------- spec2 : Spec1D object Spectrum with outliers masked. Example ------- .. code-block:: python spec = maskoutliers(spec,nsig=5) """ print = getprintfunc( ) # Get print function to be used locally, allows for easy logging spec2 = spec.copy() wave = spec2.wave.copy().reshape(spec2.npix, spec2.norder) # make 2D flux = spec2.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D err = spec2.err.copy().reshape(spec2.npix, spec2.norder) # make 2D mask = spec2.mask.copy().reshape(spec2.npix, spec2.norder) # make 2D totnbd = 0 for o in range(spec2.norder): w = wave[:, o].copy() x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5) ) # -1 to +1 y = flux[:, o].copy() m = mask[:, o].copy() # Divide by median medy = np.nanmedian(y) if medy <= 0.0: medy = 1.0 y /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x, y, 2, robust=True) sig = dln.mad(y - dln.poly(x, coef)) bd, nbd = dln.where(((y - dln.poly(x, coef)) > nsig * sig) | (y < 0)) totnbd += nbd if nbd > 0: flux[bd, o] = dln.poly(x[bd], coef) * medy err[bd, o] = 1e30 mask[bd, o] = True # Flatten to 1D if norder=1 if spec2.norder == 1: flux = flux.flatten() err = err.flatten() mask = mask.flatten() # Stuff back in spec2.flux = flux spec2.err = err spec2.mask = mask if verbose is True: print('Masked ' + str(totnbd) + ' outlier or negative pixels') return spec2
def fix_pms(objectid): """ Correct the proper motions in the healpix object catalog.""" t00 = time.time() hostname = socket.gethostname() host = hostname.split('.')[0] version = 'v3' radeg = np.float64(180.00) / np.pi meas = qc.query(sql="select * from nsc_dr2.meas where objectid='"+objectid+"'",fmt='table') nmeas = len(meas) print(' '+str(nmeas)) mnra = np.median(meas['ra'].data) mndec = np.median(meas['dec'].data) lim = 20.0 # 50.0 gd, = np.where( (np.abs(meas['ra'].data-mnra)/np.cos(np.deg2rad(mndec))*3600 < lim) & (np.abs(meas['dec'].data-mndec)*3600 < lim)) ngd = len(gd) nbd = nmeas-ngd print('bad measurements '+str(nbd)) #if nbd==0: # return None meas = meas[gd] # Make cut on FWHM # maybe only use values for 0.5*fwhm_chip to 1.5*fwhm_chip sql = "select chip.* from nsc_dr2.chip as chip join nsc_dr2.meas as meas on chip.exposure=meas.exposure and chip.ccdnum=meas.ccdnum" sql += " where meas.objectid='"+objectid+"'" chip = qc.query(sql=sql,fmt='table') ind3,ind4 = dln.match(chip['exposure'],meas['exposure']) si = np.argsort(ind4) # sort by input meas catalog ind3 = ind3[si] ind4 = ind4[si] chip = chip[ind3] meas = meas[ind4] gdfwhm, = np.where((meas['fwhm'] > 0.2*chip['fwhm']) & (meas['fwhm'] < 2.0*chip['fwhm'])) if len(gdfwhm)==0: print('All measurements have bad FWHM values') return if len(gdfwhm) < len(meas): print('Removing '+str(len(meas)-len(gdfwhm))+' measurements with bad FWHM values') meas = meas[gdfwhm] raerr = np.array(meas['raerr']*1e3,np.float64) # milli arcsec ra = np.array(meas['ra'],np.float64) ra -= np.mean(ra) ra *= 3600*1e3 * np.cos(mndec/radeg) # convert to true angle, milli arcsec t = np.array(meas['mjd'].copy()) t -= np.mean(t) t /= 365.2425 # convert to year # Calculate robust slope try: #pmra, pmraerr = dln.robust_slope(t,ra,raerr,reweight=True) # LADfit pmra_ladcoef, absdev = dln.ladfit(t,ra) pmra_lad = pmra_ladcoef[1] # Run RANSAC ransac = linear_model.RANSACRegressor() ransac.fit(t.reshape(-1,1), ra) inlier_mask = ransac.inlier_mask_ outlier_mask = np.logical_not(inlier_mask) gdmask = inlier_mask pmra_ransac = ransac.estimator_.coef_[0] print(' ransac '+str(np.sum(inlier_mask))+' inliers '+str(np.sum(outlier_mask))+' outliers') # Robust, weighted linear with with INLIERS #pmra_coef, pmra_coeferr = dln.poly_fit(t[gdmask],ra[gdmask],1,sigma=raerr[gdmask],robust=True,error=True) #pmra_coef, pmra_coeferr = dln.poly_fit(t,ra,1,sigma=raerr,robust=True,error=True) #pmra = pmra_coef[0] #pmraerr = pmra_coeferr[0] #radiff = ra-dln.poly(t,pmra_coef) radiff = ra-t*pmra_lad radiff -= np.median(radiff) rasig = dln.mad(radiff) # Reject outliers gdsig = (np.abs(radiff) < 2.5*rasig) | (np.abs(radiff) < 2.5*raerr) print(' '+str(nmeas-np.sum(gdsig))+' 2.5*sigma clip outliers rejected') #if np.sum(gdsig) < nmeas: pmra_coef, pmra_coeferr = dln.poly_fit(t[gdsig],ra[gdsig],1,sigma=raerr[gdsig],robust=True,error=True) pmra = pmra_coef[0] pmraerr = pmra_coeferr[0] rasig = dln.mad(ra-dln.poly(t,pmra_coef)) except: print('problem') #import pdb; pdb.set_trace() return np.append(np.zeros(10,float)+np.nan, np.zeros(2,int)) decerr = np.array(meas['decerr']*1e3,np.float64) # milli arcsec dec = np.array(meas['dec'],np.float64) dec -= np.mean(dec) dec *= 3600*1e3 # convert to milli arcsec # Calculate robust slope try: #pmdec, pmdecerr = dln.robust_slope(t,dec,decerr,reweight=True) # LADfit pmdec_ladcoef, absdev = dln.ladfit(t,dec) pmdec_lad = pmdec_ladcoef[1] # Run RANSAC ransac = linear_model.RANSACRegressor() ransac.fit(t.reshape(-1,1), dec) inlier_mask = ransac.inlier_mask_ outlier_mask = np.logical_not(inlier_mask) gdmask = inlier_mask pmdec_ransac = ransac.estimator_.coef_[0] print(' ransac '+str(np.sum(inlier_mask))+' inliers '+str(np.sum(outlier_mask))+' outliers') # Robust, weighted linear with with INLIERS #pmdec_coef, pmdec_coeferr = dln.poly_fit(t[gdmask],dec[gdmask],1,sigma=decerr[gdmask],robust=True,error=True) #pmdec_coef, pmdec_coeferr = dln.poly_fit(t,dec,1,sigma=decerr,robust=True,error=True) #pmdec = pmdec_coef[0] #pmdecerr = pmdec_coeferr[0] #decdiff = dec-dln.poly(t,pmdec_coef) decdiff = dec-t*pmdec_lad decdiff -= np.median(decdiff) decsig = dln.mad(decdiff) # Reject outliers gdsig = (np.abs(decdiff) < 2.5*decsig) | (np.abs(decdiff) < 2.5*decerr) print(' '+str(nmeas-np.sum(gdsig))+' 2.5*sigma clip outliers rejected') #if np.sum(gdsig) < nmeas: pmdec_coef, pmdec_coeferr = dln.poly_fit(t[gdsig],dec[gdsig],1,sigma=decerr[gdsig],robust=True,error=True) pmdec = pmdec_coef[0] pmdecerr = pmdec_coeferr[0] decsig = dln.mad(dec-dln.poly(t,pmdec_coef)) except: print('problem') #import pdb; pdb.set_trace() return np.append(np.zeros(10,float)+np.nan, np.zeros(2,int)) deltamjd = np.max(meas['mjd'])-np.min(meas['mjd']) out = np.array([pmra,pmraerr,pmra_ransac,pmra_lad,rasig,pmdec,pmdecerr,pmdec_ransac,pmdec_lad,decsig,nmeas,deltamjd]) #print(out[[0,2,3]]) #print(out[[5,7,8]]) #import pdb; pdb.set_trace() return out
#print('Second fit coefficients ',func2.parameters) #print('Second fit coefficients ',coef2) print('Second sigma = %f6.2' % sigwave2) # 4) outlier rejection # take sections and robust fit a line, then reject outliers xx = linecat['center'][gline] yy = wavecorr1[gline] si = np.argsort(xx) step = 256 bad = np.zeros(len(gline),bool) for i in range(8): g, = np.where((xx>=i*step) & (xx<(i+1)*step)) cc,absdev = dln.ladfit(xx[g],yy[g]) cc = cc[::-1] # flip diff = yy[g]-dln.poly(xx[g],cc) sig = dln.mad(diff) bd, = np.where(np.abs(diff) > 3*sig) if len(bd)>0: bad[g[bd]] = True #print(i,i*step,(i+1)*step,len(bd)) bd, = np.where(bad==True) if len(bd)>0: print('Rejecting '+str(len(bd))+' outlier points') linecat['outlier'][gline[bd]] = True # Select new good points gline, = np.where((linecat['match']==1) & (linecat['outlier']==False)) xx = linecat['center'][gline] yy = wavecorr1[gline] si = np.argsort(xx)
if nind>0: calstr1 = calstr[ind] zpterm = calstr1['zpterm'] bdzp,nbdzp = dln.where(~np.isfinite(zpterm)) # fix Infinity/NAN if nbdzp>0:zpterm[bdzp] = 999999.9 am = calstr1['airmass'] mjd = calstr1['mjd'] bdam,nbdam = dln.where(am < 0.9) if nbdam>0: am[bdam] = np.median(am) # I GOT TO HERE IN THE TRANSLATING!!! #glactc,calstr1.ra,calstr1.dec,2000.0,glon,glat,1,/deg # Measure airmass dependence gg0,ngg0 = dln.where((np.abs(zpterm)<50) & (am<2.0)) coef0 = dln.poly_fit(am[gg0],zpterm[gg0],1,robust=True) zpf = dln.poly(am,coef0) sig0 = np.mad(zpterm[gg0]-zpf[gg0]) gg,ngg = dln.where(np.abs(zpterm-zpf) < (np.maximum(3.5*sig0,0.2))) coef = dln.poly_fit(am[gg],zpterm[gg],1,robust=True) print(zpstr['instrument'][i]+'-'+zpstr['filter'][i]+' '+str(coef)) # Trim out bad exposures to determine the correlations and make figures gg,ngg = dln.where(np.abs(zpterm-zpf) < (3.5*sig0 > 0.2) and calstr1.airmass < 2.0 and calstr1.fwhm < 2.0 and calstr1.rarms < 0.15 & calstr1.decrms < 0.15 and calstr1.success == 1 and calstr1.wcscal == 'Successful' and calstr1.zptermerr < 0.05 & calstr1.zptermsig < 0.08 and (calstr1.ngoodchipwcs == calstr1.nchips) & (calstr1.instrument != 'c4d' or calstr1.zpspatialvar_nccd <= 5 or (calstr1.instrument == 'c4d' and calstr1.zpspatialvar_nccd > 5 and calstr1.zpspatialvar_rms < 0.1)) and np.abs(glat) > 10 and calstr1.nrefmatch > 100 and calstr1.exptime >= 30) # Zpterm with airmass dependence removed relzpterm = zpterm + 25 # 25 to get "absolute" zpterm relzpterm -= (zpstr['amcoef'][i])[1]*(am-1)