def extract_spectrum(cube,pixels,wvslice=None): from linetools.spectra.xspectrum1d import XSpectrum1D if isinstance(cube,str): cube = DataCube(cube) if wvslice is not None: cube = kt.slice_cube(cube,wvslice[0],wvslice[1]) dat = cube.data # Get rid of NaNs dc = dat.copy() nans = np.where(np.isnan(dc)) dc[nans]=0 # Perform the extraction spaxels = [] for i,px in enumerate(pixels): thisspec = dc[:,px[0],px[1]] spaxels.append(thisspec) spaxels=np.array(spaxels) medspec = np.median(spaxels, axis=0) # Get the wavelength array newwavearr = cube.wavelength # Create the spectrum try: spec = XSpectrum1D(wave = newwavearr,flux=medspec) except: import pdb; pdb.set_trace() return spec
def parse_DESI_brick(hdulist, select=0, **kwargs): """ Read a spectrum from a DESI brick format HDU list Parameters ---------- hdulist : FITS HDU list select : int, optional Spectrum selected. Default is 0 Returns ------- xspec1d : XSpectrum1D Parsed spectrum """ fx = hdulist[0].data # Sig if hdulist[1].name in ['ERROR', 'SIG']: sig = hdulist[1].data else: ivar = hdulist[1].data sig = np.zeros_like(ivar) gdi = ivar > 0. sig[gdi] = np.sqrt(1. / ivar[gdi]) # Wave wave = hdulist[2].data wave = give_wv_units(wave) if wave.shape != fx.shape: wave = np.tile(wave, (fx.shape[0], 1)) # Finish xspec1d = XSpectrum1D(wave, fx, sig, select=select, **kwargs) return xspec1d
def grab_specmeta(self, rows, verbose=None, masking='edges', use_XSpec=True, **kwargs): """ Grab the spectra and meta data for an input set of rows Aligned to the rows input Parameters ---------- rows : int or ndarray verbose kwargs Returns ------- spec : XSpectrum1D or ndarray Spectra requested, ordered by the input rows meta : Table -- THIS MAY BE DEPRECATED Meta table, ordered by the input rows """ if isinstance(rows, (int, np.int64)): rows = np.array([rows]) # Insures meta and other arrays are proper if verbose is None: verbose = self.verbose # Check spectra even exist! (can be only meta data) if 'spec' not in list(self.hdf[self.group].keys()): warnings.warn("No spectra in group: {:s}".format(self.group)) return None, None # Check memory if self.stage_data(rows, **kwargs): if verbose: print("Loaded spectra") # Load msk = np.array([False] * len(self.meta)) msk[rows] = True tmp_data = self.hdf[self.group]['spec'][msk] # Replicate and sort according to input rows idx = match_ids(rows, np.where(msk)[0]) data = tmp_data[idx] else: print("Staging failed.. Not returning spectra") return # Generate XSpectrum1D if 'co' in data.dtype.names: co = data['co'] else: co = None if use_XSpec: spec = XSpectrum1D(data['wave'], data['flux'], sig=data['sig'], co=co, masking=masking) else: spec = data # Return return spec, self.meta[rows]
def loop_grab_spec(self, survey, IDs, verbose=None, **kwargs): """ Grab spectra using staged IDs All IDs must occur in each of the surveys listed Order of spectra and meta tables will match the input IDs Parameters ---------- survey : str or list IDs : int or intarr Returns ------- spec : XSpectrum1D Spectra requested, ordered by the IDs meta : Table Meta table, ordered by the IDs """ if verbose is None: verbose = self.verbose if isinstance(survey, list): all_spec = [] all_meta = [] for isurvey in survey: spec, meta = self.grab_spec(isurvey, IDs, **kwargs) if spec is not None: all_spec.append(spec.copy()) all_meta.append(meta.copy()) return all_spec, all_meta # Grab IDs if self.stage_data(survey, IDs, **kwargs): if np.sum(self.survey_bool) == 0: if verbose: print("No spectra matching in survey {:s}".format(survey)) return None, None else: if verbose: print("Loaded spectra") tmp_data = self.hdf[survey]['spec'][self.survey_bool] # Replicate and sort according to input IDs data = tmp_data[self.indices] else: print("Staging failed.. Not returning spectra") return # Generate XSpectrum1D if 'co' in data.dtype.names: co = data['co'] else: co = None spec = XSpectrum1D(data['wave'], data['flux'], sig=data['sig'], co=co, masking='edges') # Return return spec, self.meta
def parse_hdf5(inp, close=True, **kwargs): """ Read a spectrum from HDF5 written in XSpectrum1D format Expects: meta, data, units Parameters ---------- inp : str or hdf5 Returns ------- """ import json import h5py # Path path = kwargs.pop('path', '/') # Open if isinstance(inp, basestring): hdf5 = h5py.File(inp, 'r') else: hdf5 = inp # Data data = hdf5[path+'data'].value # Meta if 'meta' in hdf5[path].keys(): meta = json.loads(hdf5[path+'meta'].value) # Headers for jj,heads in enumerate(meta['headers']): try: meta['headers'][jj] = fits.Header.fromstring(meta['headers'][jj]) except TypeError: # dict if not isinstance(meta['headers'][jj], dict): raise IOError("Bad meta type") else: meta = None # Units units = json.loads(hdf5[path+'units'].value) for key,item in units.items(): if item == 'dimensionless_unit': units[key] = u.dimensionless_unscaled else: units[key] = getattr(u, item) # Other arrays try: sig = data['sig'] except (NameError, IndexError): sig = None try: co = data['co'] except (NameError, IndexError): co = None # Finish if close: hdf5.close() return XSpectrum1D(data['wave'], data['flux'], sig=sig, co=co, meta=meta, units=units, **kwargs)
def test_errors(): # from_tuple try: spec = XSpectrum1D.from_tuple('this_is_not_a_tuple') except IOError: pass try: n_tuple = np.array([np.ones(5), np.ones(5)]), np.ones(5) spec = XSpectrum1D.from_tuple(n_tuple) except IOError: pass # wrong instances flux = [1, 2, 3] wave = [1, 2, 3] try: spec = XSpectrum1D(wave, flux) except IOError: pass #wrong shapes flux = np.ones(5) wave = np.array([1, 2, 3]) try: spec = XSpectrum1D(wave, flux) except IOError: pass try: spec = XSpectrum1D(wave, np.ones(len(wave)), sig=np.ones(2)) except IOError: pass try: spec = XSpectrum1D(wave, np.ones(len(wave)), co=np.ones(2), verbose=True) # test verbose here too except IOError: pass # wrong masking try: spec = XSpectrum1D(wave, np.ones(len(wave)), masking='wrong_masking') except IOError: pass #wrong units input try: spec = XSpectrum1D(wave, np.ones(len(wave)), units='not_a_dict') except IOError: pass try: spec = XSpectrum1D(wave, np.ones(len(wave)), units=dict(wrong_key=2)) except IOError: pass
def smash_spectra(spec, method='average', debug=False): """ Collapse the data in XSpectrum1D.data One might call this 'stacking' Note: This works on the unmasked data array and returns unmasked spectra Parameters ---------- spec : XSpectrum1D method : str, optional Approach to the smash ['smash'] debug : bool, optional Returns ------- new_spec : XSpectrum1D """ from linetools.spectra.xspectrum1d import XSpectrum1D # Checks if spec.nspec <= 1: raise IOError( "This method smashes an XSpectrum1D instance with multiple spectra" ) np.testing.assert_allclose(spec.data['wave'][0], spec.data['wave'][1]) # Generate mask stack_msk = spec.data['sig'] > 0. if method == 'average': tot_flx = np.sum(spec.data['flux'] * stack_msk, 0) navg = np.sum(stack_msk, 0) fin_flx = tot_flx / np.maximum(navg, 1.) elif method == 'median': fin_flx = np.median(spec.data['flux'], 0) else: raise IOError("Not prepared for this smash method") # Finish new_spec = XSpectrum1D(spec.data['wave'][0], fin_flx, masking='none', units=spec.units.copy()) new_spec.meta = spec.meta.copy() # Return return new_spec
def clean(spec, red): """ :param spec: (numpy array) XSpectrum1D objects :param red: (numpy array) their redshift values (z) :returns: rest_spec: (numpy array) truncatted, normalized and restframed XSpectrum1D objects """ import numpy as np import astropy.units as u from linetools.spectra import utils as ltsu from linetools.spectra.xspectrum1d import XSpectrum1D r = range(len(spec)) temp = [np.asarray(spec[i].wavelength / (1 + red[i])) for i in r] # truncating each spectra to only include wavelength values from 1000 to 1450 wv_coverage = [(1000 < entry) & (entry < 1450) for entry in temp] wave = np.asarray([spec[i].wavelength[wv_coverage[i]] for i in r]) flux = np.asarray([spec[i].flux[wv_coverage[i]] for i in r]) error = np.asarray([spec[i].sig[wv_coverage[i]] for i in r]) temp = [np.asarray(wave[i] / (1 + red[i])) for i in r] # normalizing just between the SiII lines wv_norm = [(1260 < entry) & (entry < 1304) for entry in temp] # just between the SiII lines flux_range = np.asarray([flux[i][wv_norm[i]] for i in r]) medians = np.asarray([np.median(flux_range[i]) for i in r]) norm_flux = np.asarray([(flux[i] / medians[i]) for i in r]) # getting a single XSpec object to feed into the stack spec = [XSpectrum1D(wave[i], norm_flux[i], error[i]) for i in r] collate = ltsu.collate(spec) rest_spec = ltsu.rebin_to_rest(collate, red, 300 * u.km / u.s, grow_bad_sig=True) return rest_spec
def convert_wavelength(cube, outfile=None): from linetools.spectra.xspectrum1d import XSpectrum1D dims = np.shape(cube.data[0]) lastwaveidx = np.max(dims) - 1 wave1 = cube.wcs.wcs_pix2world([[0, 0, 0]], 1)[0][2] wave2 = cube.wcs.wcs_pix2world([[0, 0, lastwaveidx]], 1)[0][2] import pdb pdb.set_trace() if wave1 > 1000.: inc = 1. else: inc = 1e-10 newwavearr = np.arange(wave1, wave2 + inc, inc) newspec = XSpectrum1D(newwavearr * inc, np.zeros_like(newwavearr)) newspec.meta['airvac'] = 'air' newspec.airtovac() diffs = newspec.wavelength[1:] - newspec.wavelength[0:-1] # TODO: Come up with a way to keep the header info and WCS synced cube.wcs.wcs.crval[2] = newspec.wvmin.value cube.wcs.wcs.crpix[2] = 1. cube.wcs.wcs.pc[2][2] = np.median(diffs.value) newhdulist = cube.wcs.to_fits() newhdulist[0].data = cube.data newhdulist[0].header.set('CTYPE3', 'VAC', 'Vacuum wavelengths') newhdulist[0].header['CUNIT3'] = 'Angstrom' newhdulist[0].header['CD3_3'] = np.median(diffs.value) # cube.header['CRPIX3'] = 1 # cube.header['PC3_3'] = np.median(diffs.value) # cube.header['CRVAL3'] = newspec.wvmin.value cube.header = newhdulist[0].header #import pdb; pdb.set_trace() if outfile is not None: newhdulist.writeto(outfile, overwrite=True)
def median_coadd(cubes, stackwcs, outfile=None, air_to_vac=True): nc_arr = np.array(cubes) nc_med = np.median(nc_arr, axis=0) newhdulist = stackwcs.to_fits() newhdulist[0].data = nc_med if air_to_vac is True: dims = np.shape(cubes[0]) lastwaveidx = np.max(dims) - 1 wave1 = stackwcs.wcs_pix2world([[0, 0, 0]], 1)[0][2] wave2 = stackwcs.wcs_pix2world([[0, 0, lastwaveidx]], 1)[0][2] newwavearr = np.arange(wave1, wave2 + 1e-10, 1e-10) newspec = XSpectrum1D(newwavearr * 1e10, np.zeros_like(newwavearr)) newspec.meta['airvac'] = 'air' newspec.airtovac() diffs = newspec.wavelength[1:] - newspec.wavelength[0:-1] newhdulist[0].header['CRVAL3'] = newspec.wvmin.value newhdulist[0].header['CRPIX3'] = 1 newhdulist[0].header['CD3_3'] = np.median(diffs.value) newhdulist[0].header['PC3_3'] = np.median(diffs.value) newhdulist[0].header.set('CTYPE3', 'VAC', 'Vacuum wavelengths') newhdulist[0].header['CUNIT3'] = 'Angstrom' if outfile is not None: newhdulist.writeto(outfile, overwrite=True) return newhdulist
def rtModel_conv_rb_spec(input, outfil=None, specR=1800., zgal=0.6942, restlam=2796.35, dlam_rb=None, fmt=['lam', 'y', 'x'], return_cube=True): '''Convolve and rebin a radiative transfer model cube in the spectral direction Parameters ---------- input : str or DataCube Input filename or DataCube to convolve/rebin spectrally outfil : str, optional Output filename specR : float, optional Resolving power of spectrum zgal : float, optional Redshift of object restlam : float, optional Reference rest-frame wavelength for calculating kernel sigma dlam_rb : float, optional New wavelength bin size; if None, do not rebin fmt : list, optional Axis order of data cube, must have 'x', 'y', and 'lam' in some order return_cube : bool, optional If True, return DataCube object Returns ------- newwave : array Wavelength vector (same as that of input cube if dlam_rb is None) specconv_cube : 3D array Datacube with convolved spaxels (spatial, spatial, spectral) ''' from linetools.spectra.xspectrum1d import XSpectrum1D dlam_obs = restlam * (1.0 + zgal) / specR dlam_rest = dlam_obs / (1.0 + zgal ) # FWHM of resolution element in Angstroms if isinstance(input, str): ### Assume data format is as the output of ### KR's read_spec_output.read_fullfits_conv_rebin() hdu = fits.open(input) data_pad = hdu[0].data # Original data, just padded cubedat = hdu[1].data # Spatially convolved data wave = hdu[2].data # Wavelength array nx, ny, nlam = cubedat.shape cubewcs = None else: wave = input.wavelength / (1. + zgal) cubedat = input.data cubewcs = input.wcs ### Establish dimensions of input cube xs = fmt.index('x') ys = fmt.index('y') sp = fmt.index('lam') cdims = cubedat.shape nlam = cdims[sp] nx = cdims[xs] ny = cdims[ys] ### Rearrange the cube (temporarily) : (y,x,lam) cubedat_tp = np.transpose(cubedat, axes=[ys, xs, sp]) fmt_internal = ['y', 'x', 'lam'] ### Set up kernel to convolve spaxels dwv = np.abs(wave[1] - wave[0]) fwhm_specpix = dlam_rest / dwv # FWHM in number of pixels stddev_specpix = fwhm_specpix / (2.0 * (2.0 * np.log(2.0))**0.5) spec_kernel = Gaussian1DKernel(stddev_specpix) ### Rebin, if desired, and convolve if dlam_rb is None: specconv_cube = np.zeros((ny, nx, nlam)) newwave = wave for spaxx in range(nx): for spaxy in range(ny): spax_spec = cubedat_tp[spaxy, spaxx, :] conv_spec = convolve(spax_spec, spec_kernel, normalize_kernel=True) specconv_cube[spaxy, spaxx, :] = conv_spec else: waveq = wave * u.Angstrom newwave = np.arange(int(wave[0]), int(wave[-1]), dlam_rb) * u.Angstrom specconv_cube = np.zeros((ny, nx, len(newwave))) for spaxx in range(nx): for spaxy in range(ny): spax_spec = cubedat_tp[spaxy, spaxx, :] conv_spec = convolve(spax_spec, spec_kernel, normalize_kernel=True) # Use the linetools rebinning method csX1d = XSpectrum1D(wave=waveq, flux=conv_spec) cs_rb = csX1d.rebin(newwave) try: specconv_cube[spaxy, spaxx, :] = cs_rb.flux except: import pdb pdb.set_trace() if cubewcs is not None: cubewcs.wcs.pc[2, 2] = dlam_rb ### Probably shouldn't be hard-coded ### Transpose back to input dimensions axs = ['a'] * 3 for cc in fmt_internal: axs[fmt.index(cc)] = fmt_internal.index(cc) newdat = np.transpose(specconv_cube, axes=axs) if return_cube: newcube = DataCube(wavelength=newwave, data=newdat, include_wcs=cubewcs) return newwave, newcube else: return newwave, specconv_cube
def stacks(spec, red, N, zbin, plot=False): """ :param spec: (numpy array) XSpectrum1D objects :param red: (numpy array) their redshift values (z) :param N: (int) number of bootstrap iterations :param zbin: (str) redshift bin "lowz" or "hiz" """ import numpy as np from astropy.io import fits from numpy import random as ran from linetools.spectra import utils as ltsu from linetools.spectra.xspectrum1d import XSpectrum1D print("Number of spectra provided =", spec.nspec) # the actual stack stack = ltsu.smash_spectra(spec) z_med = np.median(red) # the bootstrap R = (spec.nspec) # restraints on the random variable N_stack = [] N_z_med = [] for i in np.arange(N): # this might take a while if i % 10 == 0: print(i) choice = np.asarray(ran.randint(0, R, R)) # the shuffle rand_spec = np.asarray([spec[index] for index in choice]) rand_red = np.asarray([red[index] for index in choice]) rand_collate = ltsu.collate(rand_spec) # collate and stack N_stack.append(ltsu.smash_spectra(rand_collate)) N_z_med.append(np.median(rand_red)) # matrix math to create the error array N_flux = np.array([entry.flux for entry in N_stack]) N_matrix = np.array([N_flux[i] - stack.flux for i in range(N)]) tranpose = np.transpose(N_matrix) # matrix math covariance = np.dot(tranpose, N_matrix) sigma = np.sqrt(np.diagonal(covariance) / (N - 1)) # the final error array composite = XSpectrum1D(stack.wavelength, stack.flux, sig=sigma) # plot with bootstrap generated error if plot == True: composite.plot() # writing out the stack hdr = fits.Header() hdr["REDSHIFT"] = z_med hdr["NSPEC"] = spec.nspec header = fits.PrimaryHDU(header=hdr) c_wave = fits.Column(name='WAVELENGTH', array=stack.wavelength, unit="angstroms", format="E") c_flux = fits.Column(name='FLUX', array=stack.flux, unit="relative flux", format="E") c_noise = fits.Column(name='FLUX_ERR', array=sigma, unit="relative flux", format="E") table = fits.BinTableHDU.from_columns([c_wave, c_flux, c_noise]) full_hdu = fits.HDUList([header, table]) full_hdu.writeto("../../fits/composites/" + zbin + "/composite.fits") # writing out the boostrap hdr = fits.Header() hdr["NSPEC"] = spec.nspec hdr["NBOOT"] = N header = fits.PrimaryHDU(header=hdr) c_wave = fits.Column(name='WAVELENGTH', array=stack.wavelength, unit="angstroms", format="E") c_noise = fits.Column(name='FLUX_ERR', array=sigma, unit="relative flux", format="E") table = fits.BinTableHDU.from_columns([c_wave, c_noise]) flux_data = fits.ImageHDU(N_flux, name="FLUX_ITERATIONS") full_hdu = fits.HDUList([header, table, flux_data]) full_hdu.writeto("../../fits/bootstrap/" + zbin + "/stacks.fits")
def reading_data(path, resol, zlow, zhigh): ''' function for requesting the spectra from desi mocks :param resol float: dleta v for rebin :param zlow float: lower limit for request redshift :param zhigh float: higher limit for request redshift :return: two class object: 1.meta data and unbinned spectra 2. xspectrum object ''' #define the variable dataz = [] z = [] zerr = [] fluxt = [] wavetotal = [] ivar = [] ra = [] dec = [] id = [] filename = [] wvmin = [] wvmax = [] npix = [] date = [] #loop through the whole directory for requesting spectra specf = [] mag = [] item1 = os.listdir(str(path)) for k in item1: item = os.listdir(str(path) + str(k)) # print (item) for j in item: if os.listdir(str(path) + str(k) + '/' + str(j) + '/'): dataz = fits.open( str(path) + str(k) + '/' + str(j) + '/zbest-16-' + str(j) + '.fits') data = fits.open( str(path) + str(k) + '/' + str(j) + '/spectra-16-' + str(j) + '.fits') # k j j zcut = (zlow < np.array(dataz[1].data['Z'])) & (np.array( dataz[1].data['Z']) < zhigh) wavecut = (np.array(data[2].data) < 5730) wavecut2 = ((np.array(data[7].data) > 5730) & (np.array(data[7].data) < 7560)) wavecut3 = (np.array(data[12].data) > 7560) z.append(dataz[1].data['Z'][zcut]) zerr.append(dataz[1].data['ZERR'][zcut]) ra.append(data[1].data['TARGET_RA'][zcut]) dec.append(data[1].data['TARGET_DEC'][zcut]) id.append(data[1].data['TARGETID'][zcut]) filename.append( np.tile([k, j, j], (len(dataz[1].data['Z'][zcut]), 1))) mag.append(1.0) date.append(data[1].data['NIGHT'][zcut]) #combining the spectra from three channels wavetotal.append( np.repeat([ np.concatenate([ data[2].data[wavecut], data[7].data[wavecut2], data[12].data[wavecut3] ]) ], len(dataz[1].data['Z']), axis=0)[zcut]) fluxt.append( np.concatenate( (data[3].data[:, wavecut], data[8].data[:, wavecut2], data[13].data[:, wavecut3]), axis=1)[zcut]) ivar.append( np.concatenate( (data[4].data[:, wavecut], data[9].data[:, wavecut2], data[14].data[:, wavecut3]), axis=1)[zcut]) sp = XSpectrum1D(np.vstack(np.array(wavetotal)), np.vstack(np.array(fluxt)), np.sqrt(1 / np.vstack(np.array(ivar))), verbose=False) #ra,dec,id,filename,wvmin,wvmax,npix, date, mag,z, zerr binspec = ltsu.rebin_to_rest(sp, np.zeros(len(np.array(np.concatenate(z)))), resol * u.km / u.s) specf = spec(np.concatenate(ra), np.concatenate(dec), np.concatenate(id), np.concatenate(filename), np.min(binspec.wavelength), np.max(binspec.wavelength), len(binspec.wavelength), np.concatenate(date), np.array(mag), np.concatenate(z), np.concatenate(zerr)) # newwave = np.linspace(np.min(wavetotal[0]),np.max(wavetotal[0]),reso) return specf, binspec
def extinction(spec, red, coord): """ :param spec: (numpy array) XSpectrum1D objects: use clamato_read.py :param red: (numpy array) redshift values :param coord: (numpy array) coordinates :return: unred_spec: (numpy array) de-reddened spec """ import numpy as np from numpy.lib.polynomial import poly1d import astropy.units as u from astropy.coordinates import SkyCoord from dustmaps.bayestar import BayestarQuery from astropy.cosmology import WMAP9 as cosmo from linetools.spectra.xspectrum1d import XSpectrum1D r = range(len(spec)) Mpc = cosmo.comoving_distance(red) bayestar = BayestarQuery() coords = [ SkyCoord(coord[i][0] * u.deg, coord[i][1] * u.deg, distance=Mpc[i], frame='fk5') for i in r ] ebv = [bayestar(i) for i in coords] #to get the ebv values for each galaxy unred_spec = [] for i in r: x = 10000. / np.array(spec[i].wavelength) # Convert to inverse microns npts = x.size a = np.zeros(npts, dtype=np.float) b = np.zeros(npts, dtype=np.float) r_v = 3.1 good = np.where((x >= 0.3) & (x < 1.1)) if len(good[0]) > 0: a[good] = 0.574 * x[good]**(1.61) b[good] = -0.527 * x[good]**(1.61) good = np.where((x >= 1.1) & (x < 3.3)) if len(good[0]) > 0: # Use new constants from O'Donnell (1994) y = x[good] - 1.82 c1 = np.array([ 1., 0.104, -0.609, 0.701, 1.137, -1.718, -0.827, 1.647, -0.505 ]) # from O'Donnell c2 = np.array([ 0., 1.952, 2.908, -3.989, -7.985, 11.102, 5.491, -10.805, 3.347 ]) a[good] = poly1d(c1[::-1])(y) b[good] = poly1d(c2[::-1])(y) good = np.where((x >= 3.3) & (x < 8)) if len(good[0]) > 0: y = x[good] a[good] = 1.752 - 0.316 * y - (0.104 / ((y - 4.67)**2 + 0.341)) # + f_a b[good] = -3.090 + 1.825 * y + (1.206 / ((y - 4.62)**2 + 0.263)) # + f_b good = np.where((x >= 8) & (x <= 11)) if len(good[0]) > 0: y = x[good] - 8. c1 = np.array([-1.073, -0.628, 0.137, -0.070]) c2 = np.array([13.670, 4.257, -0.420, 0.374]) a[good] = poly1d(c1[::-1])(y) b[good] = poly1d(c2[::-1])(y) # Now apply extinction correction to input flux vector a_v = r_v * ebv[i] a_lambda = a_v * (a + b / r_v) funred = spec[i].flux * 10.**(0.4 * a_lambda) # Derive unreddened flux funred = np.asarray(funred) unred_spec.append(XSpectrum1D(spec[i].wavelength, funred, spec[i].sig)) return np.asarray(unred_spec)
def rebin_to_rest(spec, zarr, dv, debug=False, **kwargs): """ Shuffle an XSpectrum1D dataset to an array of observed wavelengths and rebin to dv pixels. Note: This works on the unmasked data array and returns unmasked spectra Parameters ---------- spec : XSpectrum1D zarr : nd.array Array of redshifts dv : Quantity Velocity width of the new pixels **kwargs : Passed to spec.rebin() Returns ------- new_spec : XSpectrum1D Not masked """ from linetools.spectra.xspectrum1d import XSpectrum1D # Error checking if spec.nspec <= 1: raise IOError("Use spec.rebin instead") if spec.nspec != len(zarr): raise IOError("Input redshift array must have same dimension as nspec") # Generate final wave array dlnlamb = np.log(1 + dv / const.c) z2d = np.outer(zarr, np.ones(spec.totpix)) wvmin, wvmax = (np.min(spec.data['wave'] / (1 + z2d)) * spec.units['wave'], np.max(spec.data['wave'] / (1 + z2d)) * spec.units['wave']) npix = int(np.round(np.log(wvmax / wvmin) / dlnlamb)) + 1 new_wv = wvmin * np.exp(dlnlamb * np.arange(npix)) # Final image f_flux = np.zeros((spec.nspec, npix)) f_sig = np.zeros((spec.nspec, npix)) f_wv = np.outer(np.ones(spec.nspec), new_wv.value) # Let's loop for ispec in range(spec.nspec): if debug: print("ispec={:d}".format(ispec)) # Select spec.select = ispec # Rebin in obs frame tspec = spec.rebin(new_wv * (1 + zarr[ispec]), do_sig=True, masking='none', **kwargs) # Save in rest-frame (worry about flambda) f_flux[ispec, :] = tspec.flux.value f_sig[ispec, :] = tspec.sig.value # Finish new_spec = XSpectrum1D(f_wv, f_flux, sig=f_sig, masking='none', units=spec.units.copy()) new_spec.meta = spec.meta.copy() # Return return new_spec
def collate(spectra, **kwargs): """ Generate a single XSpectrum1D instance containing an array of spectra from a list of individual XSpectrum1D spectra. Each spectrum is padded with extra pixels so that the wavelength ranges of all spectra are covered. Padded pixels are masked. Also note that masked pixels in the original data are ignored! Parameters ---------- spectra : list of XSpectrum1D **kwargs : optional Passed to the XSpectrum1D object generated Returns ------- new_spec : XSpectrum1D """ from linetools.spectra.xspectrum1d import XSpectrum1D # Init maxpix = 0 flg_co, flg_sig = False, False nspec = 0 units = None # Setup for spec in spectra: nspec += spec.nspec for ii in range(spec.nspec): spec.select = ii maxpix = max(maxpix, spec.npix) if spec.co_is_set: flg_co = True if spec.sig_is_set: flg_sig = True # Check for identical units if units is None: units = spec.units else: assert spec.units == units # Generate data arrays wave = np.zeros((nspec, maxpix), dtype='float64') flux = np.zeros_like(wave, dtype='float32') if flg_sig: sig = np.zeros_like(wave, dtype='float32') else: sig = None if flg_co: co = np.zeros_like(flux) else: co = None # Load meta = dict(headers=[]) idx = 0 for xspec in spectra: # Allow for multiple spectra in the XSpectrum1D object for jj in range(xspec.nspec): xspec.select = jj wave[idx, :xspec.npix] = xspec.wavelength.value flux[idx, :xspec.npix] = xspec.flux.value if flg_sig: try: # This will fail for spectra with fully masked arrays sig[idx, :xspec.npix] = xspec.sig.value except AttributeError: sig[idx, :] = 0. if flg_co: if xspec.co_is_set: # Allow for a mix of continua (mainly for specdb) co[idx, :xspec.npix] = xspec.co.value idx += 1 # Meta meta['headers'] += xspec.meta['headers'] # Finish new_spec = XSpectrum1D(wave, flux, sig=sig, co=co, units=units.copy(), meta=meta, **kwargs) # Return return new_spec
def weight_mean_coadd(cubes, vars, stackwcs, varwcs, outfile=None, outvarfile=None, air_to_vac=True): # Go through each cube and find the brightest pixels buff = 200 levels = np.zeros(len(cubes)) for i, cc in enumerate(cubes): var = vars[i] partdat = cc[buff:-buff, :, :] sumdat = np.sum(partdat, axis=0) partvar = var[buff:-buff, :, :] sumvar = np.sum(partvar, axis=0) datnonan = sumdat[(~np.isnan(sumdat) & (sumdat != 0))] varnonan = sumvar[(~np.isnan(sumdat) & (sumdat != 0))] mednonan = np.median(datnonan) stddev = np.std(datnonan) #bright = np.where((datnonan > (mednonan+stddev))&(datnonan>0))[0] bright = np.where((datnonan > (mednonan)) & (datnonan > 0))[0] sumbright = np.sum(datnonan[bright]) varbright = np.sum(varnonan[bright]) cc[np.isnan(cc)] = 0. var[np.isnan(var)] = 0. sn = sumbright / np.sqrt(varbright) #import pdb; pdb.set_trace() print(sumbright, mednonan, stddev, sn) if i == 0: combined = cc * sn combvar = var * sn**2 else: combined += cc * sn combvar += var * sn**2 levels[i] = sn finaldat = combined / np.sum(levels) finalvar = combvar / np.sum(levels)**2 newhdulist = stackwcs.to_fits() newhdulist[0].data = finaldat if air_to_vac is True: dims = np.shape(cubes[0]) lastwaveidx = np.max(dims) - 1 wave1 = stackwcs.wcs_pix2world([[0, 0, 0]], 1)[0][2] wave2 = stackwcs.wcs_pix2world([[0, 0, lastwaveidx]], 1)[0][2] newwavearr = np.arange(wave1, wave2 + 1e-10, 1e-10) newspec = XSpectrum1D(newwavearr * 1e10, np.zeros_like(newwavearr)) newspec.meta['airvac'] = 'air' newspec.airtovac() diffs = newspec.wavelength[1:] - newspec.wavelength[0:-1] newhdulist[0].header['CRVAL3'] = newspec.wvmin.value newhdulist[0].header['CRPIX3'] = 1 newhdulist[0].header['CD3_3'] = np.median(diffs.value) newhdulist[0].header['PC3_3'] = np.median(diffs.value) newhdulist[0].header.set('CTYPE3', 'VAC', 'Vacuum wavelengths') newhdulist[0].header['CUNIT3'] = 'Angstrom' if outfile is not None: newhdulist.writeto(outfile, overwrite=True) varhdulist = varwcs.to_fits() varhdulist[0].data = finalvar if air_to_vac is True: dims = np.shape(cubes[0]) lastwaveidx = np.max(dims) - 1 wave1 = varwcs.wcs_pix2world([[0, 0, 0]], 1)[0][2] wave2 = varwcs.wcs_pix2world([[0, 0, lastwaveidx]], 1)[0][2] newwavearr = np.arange(wave1, wave2 + 1e-10, 1e-10) newspec = XSpectrum1D(newwavearr * 1e10, np.zeros_like(newwavearr)) newspec.meta['airvac'] = 'air' newspec.airtovac() diffs = newspec.wavelength[1:] - newspec.wavelength[0:-1] varhdulist[0].header['CRVAL3'] = newspec.wvmin.value newhdulist[0].header['CRPIX3'] = 1 varhdulist[0].header['CD3_3'] = np.median(diffs.value) varhdulist[0].header['PC3_3'] = np.median(diffs.value) varhdulist[0].header.set('CTYPE3', 'VAC', 'Vacuum wavelengths') varhdulist[0].header['CUNIT3'] = 'Angstrom' if outvarfile is not None: varhdulist.writeto(outvarfile, overwrite=True) return newhdulist
def extract_spectrum(cube,pixels,wvslice=None,method='median',varcube=None, spatial_scale_xy=None): ## KHRR: if spatial_scale_xy is not None, flux in units of cgs / sq. arcsec are assumed ## This is now only implemented with the 'sum' method from linetools.spectra.xspectrum1d import XSpectrum1D if isinstance(cube,str): cube = DataCube(cube) if wvslice is not None: cube = kt.slice_cube(cube,wvslice[0],wvslice[1]) if varcube is not None: varcube = kt.slice_cube(varcube, wvslice[0], wvslice[1]) ##import pdb; #pdb.set_trace() dat = cube.data # Get rid of NaNs dc = dat.copy() nans = np.where(np.isnan(dc)) dc[nans]=0 # Perform the extraction if method == 'median': if np.shape(np.array(pixels))==(2,): extspec = dc[:,pixels[0],pixels[1]] else: spaxels = [] for i,px in enumerate(pixels): thisspec = dc[:,px[0],px[1]] spaxels.append(thisspec) spaxels=np.array(spaxels) extspec = np.median(spaxels, axis=0) sigspec = -99. * np.ones(np.shape(dc)[0]) elif method == 'mean': if np.shape(np.array(pixels))==(2,): extspec = dc[:,pixels[0],pixels[1]] else: spaxels = [] for i,px in enumerate(pixels): thisspec = dc[:,px[0],px[1]] spaxels.append(thisspec) spaxels=np.array(spaxels) extspec = np.mean(spaxels, axis=0) sigspec = -99. * np.ones(np.shape(dc)[0]) elif method == 'sum': if np.shape(np.array(pixels))==(2,): extspec = dc[:,pixels[0],pixels[1]] else: spaxels = [] for i,px in enumerate(pixels): thisspec = dc[:,px[0],px[1]] spaxels.append(thisspec) spaxels=np.array(spaxels) sigspec = -99. * np.ones(np.shape(dc)[0]) if spatial_scale_xy is None: extspec = np.sum(spaxels, axis=0) else: spxsz = spatial_scale_xy[0] * spatial_scale_xy[1] extspec = spxsz * np.sum(spaxels, axis=0) else: # do the weighted mean vardat = varcube.data vdc = vardat.copy() if np.shape(np.array(pixels))==(2,): extspec = dc[:,pixels[0],pixels[1]] else: spaxels = [] varspaxels = [] for i, px in enumerate(pixels): thisspec = dc[:, px[0], px[1]] thisvar = vdc[:, px[0], px[1]] spaxels.append(thisspec) varspaxels.append(thisvar) spaxels = np.array(spaxels) ivarspaxels = 1./np.array(varspaxels) extspec = np.sum(spaxels*ivarspaxels, axis=0)/np.sum(ivarspaxels,axis=0) sigspec = 1./np.sqrt(np.sum(ivarspaxels,axis=0)) # Get the wavelength array newwavearr = cube.wavelength # Create the spectrum try: spec = XSpectrum1D(wave = newwavearr,flux=extspec,sig=sigspec) except: import pdb; pdb.set_trace() return spec
def miss_id_check(path, idt, resol): ''' Function for requesting spectra and meta for undetected DLAs :param path object: path for the spectra :param idt float: undetected id :param resol: resolution for spectr :return: two class object: 1. meta for spectra 2. spectra for undetected DLAs ''' # define the variable dataz = [] z = [] zerr = [] fluxt = [] wavetotal = [] ivar = [] mockid = [] ra = [] dec = [] mockid = [] filename = [] wvmin = [] wvmax = [] npix = [] date = [] # loop through the whole directory for requesting spectra specf = [] mag = [] item1 = os.listdir(str(path)) for k in item1: item = os.listdir(str(path) + str(k)) # print (item) for j in item: if os.listdir(str(path) + str(k) + '/' + str(j) + '/'): dataz = fits.open( str(path) + str(k) + '/' + str(j) + '/truth-16-' + str( j) + '.fits') data = fits.open(str(path) + str(k) + '/' + str( j) + '/spectra-16-' + str(j) + '.fits') # k j j indexcut = np.isin(data[1].data['TARGETID'], idt) wavecut = (np.array(data[2].data) < 5730) wavecut2 = ((np.array(data[7].data) > 5730) & (np.array(data[7].data) < 7560)) wavecut3 = (np.array(data[12].data) > 7560) z.append(dataz[1].data['Z'][indexcut]) zerr.append(dataz[1].data['TRUEZ'][indexcut]) ra.append(data[1].data['TARGET_RA'][indexcut]) dec.append(data[1].data['TARGET_DEC'][indexcut]) mockid.append(data[1].data['TARGETID'][indexcut]) # combining the spectra from three channels wavetotal.append(np.repeat( [np.concatenate([data[2].data[wavecut], data[7].data[wavecut2], data[12].data[wavecut3]])], len(dataz[1].data['Z']), axis=0)[indexcut]) fluxt.append( np.concatenate((data[3].data[:, wavecut], data[8].data[:, wavecut2], data[13].data[:, wavecut3]), axis=1)[indexcut]) ivar.append( np.concatenate((data[4].data[:, wavecut], data[9].data[:, wavecut2], data[14].data[:, wavecut3]), axis=1)[indexcut]) sp = XSpectrum1D(np.vstack(np.array(wavetotal)), np.vstack(np.array(fluxt)), np.sqrt(1 / np.vstack(np.array(ivar))), verbose=False) # ra,dec,id,filename,wvmin,wvmax,npix, date, mag,z, zerr binspec = ltsu.rebin_to_rest(sp, np.zeros(len(np.array(np.concatenate(z)))), resol * u.km / u.s) specf = SPEC.spec_meta(np.concatenate(ra), np.concatenate(dec), np.concatenate(mockid), np.min(binspec.wavelength), np.max(binspec.wavelength), len(binspec.wavelength), np.concatenate(z), np.concatenate(zerr)) # newwave = np.linspace(np.min(wavetotal[0]),np.max(wavetotal[0]),reso) return specf, binspec