def read_from_pix(in_dir, pix, thid, ra, dec, zqso, plate, mjd, fid, order, log=None): try: fin = in_dir + "/pix_{}.fits.gz".format(pix) h = fitsio.FITS(fin) except IOError: try: fin = in_dir + "/pix_{}.fits".format(pix) h = fitsio.FITS(fin) except IOError: print("error reading {}".format(pix)) return None ## fill log if log is not None: for t in thid: if t not in h[0][:]: log.write("{} missing from pixel {}\n".format(t, pix)) print("{} missing from pixel {}".format(t, pix)) pix_data = [] thid_list = list(h[0][:]) thid2idx = {t: thid_list.index(t) for t in thid if t in thid_list} loglam = h[1][:] flux = h[2].read() ivar = h[3].read() andmask = h[4].read() for (t, r, d, z, p, m, f) in zip(thid, ra, dec, zqso, plate, mjd, fid): try: idx = thid2idx[t] except: ## fill log if log is not None: log.write("{} missing from pixel {}\n".format(t, pix)) print("{} missing from pixel {}".format(t, pix)) continue d = forest(loglam, flux[:, idx], ivar[:, idx] * (andmask[:, idx] == 0), t, r, d, z, p, m, f, order) if log is not None: log.write("{} read\n".format(t)) pix_data.append(d) h.close() return pix_data
def read_from_spec(in_dir,thid,ra,dec,zqso,plate,mjd,fid,order,mode): pix_data = [] for t,r,d,z,p,m,f in zip(thid,ra,dec,zqso,plate,mjd,fid): try: fin = in_dir + '/spec-%d-%d-%04d.fits'%(p, m, f) h = fitsio.FITS(fin) except IOError: print("error reading {}\n".format(fin)) continue print("{} read\n".format(fin)) ll = h[1]["loglam"][:] fl = h[1]["flux"][:] iv = h[1]["ivar"][:]*(h[1]["and_mask"][:]==0) d = forest(ll,fl,iv, t, r, d, z, p, m, f,order) pix_data.append(d) h.close() return pix_data
def read_from_mock_1D(in_dir, thid, ra, dec, zqso, plate, mjd, fid, order, mode, log=None): pix_data = [] try: fin = in_dir hdu = fitsio.FITS(fin) except IOError: log.write("error reading {}\n".format(fin)) for t, r, d, z, p, m, f in zip(thid, ra, dec, zqso, plate, mjd, fid): h = hdu["{}".format(t)] log.write("file: {} hdu {} read \n".format(fin, h)) lamb = h["wavelength"][:] ll = np.log10(lamb) fl = h["flux"][:] error = h["error"][:] iv = 1.0 / error**2 # compute difference between exposure diff = np.zeros(len(lamb)) # compute spectral resolution wdisp = h["psf"][:] reso = spectral_resolution(wdisp) # compute the mean expected flux f_mean_tr = h.read_header()["MEANFLUX"] cont = h["continuum"][:] mef = f_mean_tr * cont d = forest(ll, fl, iv, t, r, d, z, p, m, f, order, diff, reso, mef) pix_data.append(d) hdu.close() return pix_data
for d in data: d.cont_fit() if d.bad_cont is not None: print 'bad cont: ' + str(T) continue else: newdata.append(d) data = N.asarray(newdata) # Define g in order to retrieve thid, etc for stats, continuum fitting and plot g = data[0] # Stack the weighted flux ll1, st1, wst1 = stack_flux(data, 0) # Fit the continuum for the stack d = forest(ll1, st1, wst1, g.thid, g.ra, g.dec, g.zqso, g.plate, g.mjd, g.fid, g.order) d.cont_fit() # Create continuum in same bin widths as before nstack = int((forest.lmax - forest.lmin) / forest.dll) + 1 co1 = sp.zeros(nstack) bins=((d.ll - d.lmin) / d.dll + 0.5).astype(int) c = sp.bincount(bins, weights = d.co) co1[:len(c)] += c # Get mean flux for each epoch epochflux = [] for f in data: epochflux.append(N.sum(f.fl * f.iv)/N.sum(f.iv)) # Find mean, standard deviations, median flux = N.asarray(epochflux)
def read_from_desi(nside, in_dir, thid, ra, dec, zqso, plate, mjd, fid, order, pk1d=None): in_nside = int(in_dir.split('spectra-')[-1].replace('/', '')) nest = True data = {} ndata = 0 ztable = {t: z for t, z in zip(thid, zqso)} in_pixs = healpy.ang2pix(in_nside, sp.pi / 2. - dec, ra, nest=nest) fi = np.unique(in_pixs) for i, f in enumerate(fi): path = in_dir + "/" + str(int(f / 100)) + "/" + str( f) + "/spectra-" + str(in_nside) + "-" + str(f) + ".fits" print("\rread {} of {}. ndata: {}".format(i, len(fi), ndata)) try: h = fitsio.FITS(path) except IOError: print("Error reading pix {}\n".format(f)) continue ## get the quasars tid_qsos = thid[(in_pixs == f)] plate_qsos = plate[(in_pixs == f)] mjd_qsos = mjd[(in_pixs == f)] fid_qsos = fid[(in_pixs == f)] if 'TARGET_RA' in h["FIBERMAP"].get_colnames(): ra = h["FIBERMAP"]["TARGET_RA"][:] * sp.pi / 180. de = h["FIBERMAP"]["TARGET_DEC"][:] * sp.pi / 180. elif 'RA_TARGET' in h["FIBERMAP"].get_colnames(): ## TODO: These lines are for backward compatibility ## Should be removed at some point ra = h["FIBERMAP"]["RA_TARGET"][:] * sp.pi / 180. de = h["FIBERMAP"]["DEC_TARGET"][:] * sp.pi / 180. pixs = healpy.ang2pix(nside, sp.pi / 2 - de, ra) #exp = h["FIBERMAP"]["EXPID"][:] #night = h["FIBERMAP"]["NIGHT"][:] #fib = h["FIBERMAP"]["FIBER"][:] in_tids = h["FIBERMAP"]["TARGETID"][:] specData = {} for spec in ['B', 'R', 'Z']: dic = {} try: dic['LL'] = np.log10(h['{}_WAVELENGTH'.format(spec)].read()) dic['FL'] = h['{}_FLUX'.format(spec)].read() dic['IV'] = h['{}_IVAR'.format(spec)].read() * ( h['{}_MASK'.format(spec)].read() == 0) w = sp.isnan(dic['FL']) | sp.isnan(dic['IV']) for k in ['FL', 'IV']: dic[k][w] = 0. dic['RESO'] = h['{}_RESOLUTION'.format(spec)].read() specData[spec] = dic except OSError: pass h.close() for t, p, m, f in zip(tid_qsos, plate_qsos, mjd_qsos, fid_qsos): wt = in_tids == t if wt.sum() == 0: print("\nError reading thingid {}\n".format(t)) continue d = None for tspecData in specData.values(): iv = tspecData['IV'][wt] fl = (iv * tspecData['FL'][wt]).sum(axis=0) iv = iv.sum(axis=0) w = iv > 0. fl[w] /= iv[w] if not pk1d is None: reso_sum = tspecData['RESO'][wt].sum(axis=0) reso_in_km_per_s = spectral_resolution_desi( reso_sum, tspecData['LL']) diff = np.zeros(tspecData['LL'].shape) else: reso_in_km_per_s = None diff = None td = forest(tspecData['LL'], fl, iv, t, ra[wt][0], de[wt][0], ztable[t], p, m, f, order, diff, reso_in_km_per_s) if d is None: d = copy.deepcopy(td) else: d += td pix = pixs[wt][0] if pix not in data: data[pix] = [] data[pix].append(d) ndata += 1 print("found {} quasars in input files\n".format(ndata)) return data
def read_from_spplate(in_dir, thid, ra, dec, zqso, plate, mjd, fid, order, log=None, best_obs=False): drq_dict = {t: (r, d, z) for t, r, d, z in zip(thid, ra, dec, zqso)} ## if using multiple observations, ## then replace thid, plate, mjd, fid ## by what's available in spAll if not best_obs: fi = glob.glob(in_dir + "/spAll-*.fits") if len(fi) > 1: print("ERROR: found multiple spAll files") print( "ERROR: try running with --bestobs option (but you will lose reobservations)" ) for f in fi: print("found: ", fi) sys.exit(1) if len(fi) == 0: print("ERROR: can't find required spAll file in {}".format(in_dir)) print( "ERROR: try runnint with --best-obs option (but you will lose reobservations)" ) sys.exit(1) spAll = fitsio.FITS(fi[0]) print("INFO: reading spAll from {}".format(fi[0])) thid_spall = spAll[1]["THING_ID"][:] plate_spall = spAll[1]["PLATE"][:] mjd_spall = spAll[1]["MJD"][:] fid_spall = spAll[1]["FIBERID"][:] qual_spall = spAll[1]["PLATEQUALITY"][:].astype(str) zwarn_spall = spAll[1]["ZWARNING"][:] w = np.in1d(thid_spall, thid) print("INFO: Found {} spectra with required THING_ID".format(w.sum())) w &= qual_spall == "good" print("INFO: Found {} spectra with 'good' plate".format(w.sum())) ## Removing spectra with the following ZWARNING bits set: ## SKY, LITTLE_COVERAGE, UNPLUGGED, BAD_TARGET, NODATA ## https://www.sdss.org/dr14/algorithms/bitmasks/#ZWARNING bad_zwarnbit = { 0: 'SKY', 1: 'LITTLE_COVERAGE', 7: 'UNPLUGGED', 8: 'BAD_TARGET', 9: 'NODATA' } for zwarnbit, zwarnbit_str in bad_zwarnbit.items(): w &= (zwarn_spall & 2**zwarnbit) == 0 print("INFO: Found {} spectra without {} bit set: {}".format( w.sum(), zwarnbit, zwarnbit_str)) print("INFO: # unique objs: ", len(thid)) print("INFO: # spectra: ", w.sum()) thid = thid_spall[w] plate = plate_spall[w] mjd = mjd_spall[w] fid = fid_spall[w] spAll.close() ## to simplify, use a list of all metadata allmeta = [] for t, p, m, f in zip(thid, plate, mjd, fid): r, d, z = drq_dict[t] meta = metadata() meta.thid = t meta.ra = r meta.dec = d meta.zqso = z meta.plate = p meta.mjd = m meta.fid = f meta.order = order allmeta.append(meta) pix_data = {} platemjd = {} for p, m, meta in zip(plate, mjd, allmeta): pm = (p, m) if not pm in platemjd: platemjd[pm] = [] platemjd[pm].append(meta) print("reading {} plates".format(len(platemjd))) for pm in platemjd: p, m = pm spplate = in_dir + "/{0}/spPlate-{0}-{1}.fits".format( str(p).zfill(4), m) try: h = fitsio.FITS(spplate) head0 = h[0].read_header() except IOError: log.write("error reading {}\n".format(spplate)) continue t0 = time.time() coeff0 = head0["COEFF0"] coeff1 = head0["COEFF1"] flux = h[0].read() ivar = h[1].read() * (h[2].read() == 0) llam = coeff0 + coeff1 * np.arange(flux.shape[1]) ## now convert all those fluxes into forest objects for meta in platemjd[pm]: t = meta.thid r = meta.ra d = meta.dec z = meta.zqso f = meta.fid o = meta.order i = meta.fid - 1 d = forest(llam, flux[i], ivar[i], t, r, d, z, p, m, f, o) if t in pix_data: pix_data[t] += d else: pix_data[t] = d if log is not None: log.write("{} read from file {} and mjd {}\n".format( t, spplate, m)) nread = len(platemjd[pm]) print("INFO: read {} from {} in {} per spec. Progress: {} of {} \n". format(nread, os.path.basename(spplate), (time.time() - t0) / (nread + 1e-3), len(pix_data), len(thid))) h.close() data = list(pix_data.values()) return data
def read_from_spcframe(in_dir, thid, ra, dec, zqso, plate, mjd, fid, order, mode=None, log=None, best_obs=False, single_exp=False): if not best_obs: print( "ERROR: multiple observations not (yet) compatible with spframe option" ) print("ERROR: rerun with the --best-obs option") sys.exit(1) allmeta = [] for t, r, d, z, p, m, f in zip(thid, ra, dec, zqso, plate, mjd, fid): meta = metadata() meta.thid = t meta.ra = r meta.dec = d meta.zqso = z meta.plate = p meta.mjd = m meta.fid = f meta.order = order allmeta.append(meta) platemjd = {} for i in range(len(thid)): pm = (plate[i], mjd[i]) if not pm in platemjd: platemjd[pm] = [] platemjd[pm].append(allmeta[i]) pix_data = {} print("reading {} plates".format(len(platemjd))) for pm in platemjd: p, m = pm exps = [] spplate = in_dir + "/{0}/spPlate-{0}-{1}.fits".format(p, m) print("INFO: reading plate {}".format(spplate)) h = fitsio.FITS(spplate) head = h[0].read_header() h.close() iexp = 1 for c in ["B1", "B2", "R1", "R2"]: card = "NEXP_{}".format(c) if card in head: nexp = head["NEXP_{}".format(c)] else: continue for _ in range(nexp): str_iexp = str(iexp) if iexp < 10: str_iexp = '0' + str_iexp card = "EXPID" + str_iexp if not card in head: continue exps.append(head["EXPID" + str_iexp][:11]) iexp += 1 print("INFO: found {} exposures in plate {}-{}".format( len(exps), p, m)) if len(exps) == 0: continue exp_num = [e[3:] for e in exps] exp_num = np.unique(exp_num) sp.random.shuffle(exp_num) exp_num = exp_num[0] for exp in exps: if single_exp: if not exp_num in exp: continue t0 = time.time() ## find the spectrograph number: spectro = int(exp[1]) assert spectro == 1 or spectro == 2 spcframe = fitsio.FITS(in_dir + "/{}/spCFrame-{}.fits".format(p, exp)) flux = spcframe[0].read() ivar = spcframe[1].read() * (spcframe[2].read() == 0) llam = spcframe[3].read() ## now convert all those fluxes into forest objects for meta in platemjd[pm]: if spectro == 1 and meta.fid > 500: continue if spectro == 2 and meta.fid <= 500: continue index = (meta.fid - 1) % 500 t = meta.thid r = meta.ra d = meta.dec z = meta.zqso f = meta.fid order = meta.order d = forest(llam[index], flux[index], ivar[index], t, r, d, z, p, m, f, order) if t in pix_data: pix_data[t] += d else: pix_data[t] = d if log is not None: log.write("{} read from exp {} and mjd {}\n".format( t, exp, m)) nread = len(platemjd[pm]) print( "INFO: read {} from {} in {} per spec. Progress: {} of {} \n". format(nread, exp, (time.time() - t0) / (nread + 1e-3), len(pix_data), len(thid))) spcframe.close() data = list(pix_data.values()) return data
def read_from_spec(in_dir, thid, ra, dec, zqso, plate, mjd, fid, order, mode, log=None, pk1d=None, best_obs=None): drq_dict = {t: (r, d, z) for t, r, d, z in zip(thid, ra, dec, zqso)} ## if using multiple observations, ## then replace thid, plate, mjd, fid ## by what's available in spAll if not best_obs: fi = glob.glob( in_dir.replace("spectra/", "").replace("lite", "").replace( "full", "") + "/spAll-*.fits") if len(fi) > 1: print("ERROR: found multiple spAll files") print( "ERROR: try running with --bestobs option (but you will lose reobservations)" ) for f in fi: print("found: ", fi) sys.exit(1) if len(fi) == 0: print("ERROR: can't find required spAll file in {}".format(in_dir)) print( "ERROR: try runnint with --best-obs option (but you will lose reobservations)" ) sys.exit(1) spAll = fitsio.FITS(fi[0]) print("INFO: reading spAll from {}".format(fi[0])) thid_spall = spAll[1]["THING_ID"][:] plate_spall = spAll[1]["PLATE"][:] mjd_spall = spAll[1]["MJD"][:] fid_spall = spAll[1]["FIBERID"][:] qual_spall = spAll[1]["PLATEQUALITY"][:].astype(str) zwarn_spall = spAll[1]["ZWARNING"][:] w = np.in1d(thid_spall, thid) & (qual_spall == "good") ## Removing spectra with the following ZWARNING bits set: ## SKY, LITTLE_COVERAGE, UNPLUGGED, BAD_TARGET, NODATA ## https://www.sdss.org/dr14/algorithms/bitmasks/#ZWARNING for zwarnbit in [0, 1, 7, 8, 9]: w &= (zwarn_spall & 2**zwarnbit) == 0 print("INFO: # unique objs: ", len(thid)) print("INFO: # spectra: ", w.sum()) thid = thid_spall[w] plate = plate_spall[w] mjd = mjd_spall[w] fid = fid_spall[w] spAll.close() ## to simplify, use a list of all metadata allmeta = [] ## Used to preserve original order and pass unit tests. t_list = [] t_set = set() for t, p, m, f in zip(thid, plate, mjd, fid): if t not in t_set: t_list.append(t) t_set.add(t) r, d, z = drq_dict[t] meta = metadata() meta.thid = t meta.ra = r meta.dec = d meta.zqso = z meta.plate = p meta.mjd = m meta.fid = f meta.order = order allmeta.append(meta) pix_data = [] thids = {} for meta in allmeta: t = meta.thid if not t in thids: thids[t] = [] thids[t].append(meta) print("reading {} thids".format(len(thids))) for t in t_list: t_delta = None for meta in thids[t]: r, d, z, p, m, f = meta.ra, meta.dec, meta.zqso, meta.plate, meta.mjd, meta.fid try: fin = in_dir + "/{}/{}-{}-{}-{:04d}.fits".format( p, mode, p, m, f) h = fitsio.FITS(fin) except IOError: log.write("error reading {}\n".format(fin)) continue log.write("{} read\n".format(fin)) ll = h[1]["loglam"][:] fl = h[1]["flux"][:] iv = h[1]["ivar"][:] * (h[1]["and_mask"][:] == 0) if pk1d is not None: # compute difference between exposure diff = exp_diff(h, ll) # compute spectral resolution wdisp = h[1]["wdisp"][:] reso = spectral_resolution(wdisp, True, f, ll) else: diff = None reso = None deltas = forest(ll, fl, iv, t, r, d, z, p, m, f, order, diff=diff, reso=reso) if t_delta is None: t_delta = deltas else: t_delta += deltas h.close() if t_delta is not None: pix_data.append(t_delta) return pix_data
if j != i: l.append(data[j]) resample_data = N.asarray(l) # Fit continuum for all spectra for d in resample_data: d.cont_fit() # Create a stack of all the deltas ll2, de2, wst2 = stack_flux(resample_data, 1) ## Create deltas of the stack (of weighted flux) # Stack the weighted flux ll1, st1, wst1 = stack_flux(resample_data, 0) # Fit the continuum for the stack d = forest(ll1, st1, wst1, d.thid, d.ra, d.dec, d.zqso, d.plate, d.mjd, d.fid, d.order) d.cont_fit() # Create deltas and weights in same bin widths as before nstack = int((forest.lmax - forest.lmin) / forest.dll) + 1 de1 = sp.zeros(nstack) bins=((d.ll - d.lmin) / d.dll + 0.5).astype(int) c = sp.bincount(bins, weights = d.fl / d.co - 1) de1[:len(c)] += c eta = forest.eta(d.ll) var_lss = forest.var_lss(d.ll) iv = d.iv / eta we = iv * d.co**2 / (iv * d.co**2 * var_lss + 1) c = sp.bincount(bins, weights = we) wst1[:len(c)] += c