def read_from_pix(in_dir,
pix,
thid,
ra,
dec,
zqso,
plate,
mjd,
fid,
order,
log=None):
try:
fin = in_dir + "/pix_{}.fits.gz".format(pix)
h = fitsio.FITS(fin)
except IOError:
try:
fin = in_dir + "/pix_{}.fits".format(pix)
h = fitsio.FITS(fin)
except IOError:
print("error reading {}".format(pix))
return None
## fill log
if log is not None:
for t in thid:
if t not in h[0][:]:
log.write("{} missing from pixel {}\n".format(t, pix))
print("{} missing from pixel {}".format(t, pix))
pix_data = []
thid_list = list(h[0][:])
thid2idx = {t: thid_list.index(t) for t in thid if t in thid_list}
loglam = h[1][:]
flux = h[2].read()
ivar = h[3].read()
andmask = h[4].read()
for (t, r, d, z, p, m, f) in zip(thid, ra, dec, zqso, plate, mjd, fid):
try:
idx = thid2idx[t]
except:
## fill log
if log is not None:
log.write("{} missing from pixel {}\n".format(t, pix))
print("{} missing from pixel {}".format(t, pix))
continue
d = forest(loglam, flux[:, idx], ivar[:, idx] * (andmask[:, idx] == 0),
t, r, d, z, p, m, f, order)
if log is not None:
log.write("{} read\n".format(t))
pix_data.append(d)
h.close()
return pix_data
def read_from_spec(in_dir,thid,ra,dec,zqso,plate,mjd,fid,order,mode):
pix_data = []
for t,r,d,z,p,m,f in zip(thid,ra,dec,zqso,plate,mjd,fid):
try:
fin = in_dir + '/spec-%d-%d-%04d.fits'%(p, m, f)
h = fitsio.FITS(fin)
except IOError:
print("error reading {}\n".format(fin))
continue
print("{} read\n".format(fin))
ll = h[1]["loglam"][:]
fl = h[1]["flux"][:]
iv = h[1]["ivar"][:]*(h[1]["and_mask"][:]==0)
d = forest(ll,fl,iv, t, r, d, z, p, m, f,order)
pix_data.append(d)
h.close()
return pix_data
def read_from_mock_1D(in_dir,
thid,
ra,
dec,
zqso,
plate,
mjd,
fid,
order,
mode,
log=None):
pix_data = []
try:
fin = in_dir
hdu = fitsio.FITS(fin)
except IOError:
log.write("error reading {}\n".format(fin))
for t, r, d, z, p, m, f in zip(thid, ra, dec, zqso, plate, mjd, fid):
h = hdu["{}".format(t)]
log.write("file: {} hdu {} read \n".format(fin, h))
lamb = h["wavelength"][:]
ll = np.log10(lamb)
fl = h["flux"][:]
error = h["error"][:]
iv = 1.0 / error**2
# compute difference between exposure
diff = np.zeros(len(lamb))
# compute spectral resolution
wdisp = h["psf"][:]
reso = spectral_resolution(wdisp)
# compute the mean expected flux
f_mean_tr = h.read_header()["MEANFLUX"]
cont = h["continuum"][:]
mef = f_mean_tr * cont
d = forest(ll, fl, iv, t, r, d, z, p, m, f, order, diff, reso, mef)
pix_data.append(d)
hdu.close()
return pix_data
for d in data:
d.cont_fit()
if d.bad_cont is not None:
print 'bad cont: ' + str(T)
continue
else:
newdata.append(d)
data = N.asarray(newdata)
# Define g in order to retrieve thid, etc for stats, continuum fitting and plot
g = data[0]
# Stack the weighted flux
ll1, st1, wst1 = stack_flux(data, 0)
# Fit the continuum for the stack
d = forest(ll1, st1, wst1, g.thid, g.ra, g.dec, g.zqso, g.plate, g.mjd, g.fid, g.order)
d.cont_fit()
# Create continuum in same bin widths as before
nstack = int((forest.lmax - forest.lmin) / forest.dll) + 1
co1 = sp.zeros(nstack)
bins=((d.ll - d.lmin) / d.dll + 0.5).astype(int)
c = sp.bincount(bins, weights = d.co)
co1[:len(c)] += c
# Get mean flux for each epoch
epochflux = []
for f in data:
epochflux.append(N.sum(f.fl * f.iv)/N.sum(f.iv))
# Find mean, standard deviations, median
flux = N.asarray(epochflux)
def read_from_desi(nside,
in_dir,
thid,
ra,
dec,
zqso,
plate,
mjd,
fid,
order,
pk1d=None):
in_nside = int(in_dir.split('spectra-')[-1].replace('/', ''))
nest = True
data = {}
ndata = 0
ztable = {t: z for t, z in zip(thid, zqso)}
in_pixs = healpy.ang2pix(in_nside, sp.pi / 2. - dec, ra, nest=nest)
fi = np.unique(in_pixs)
for i, f in enumerate(fi):
path = in_dir + "/" + str(int(f / 100)) + "/" + str(
f) + "/spectra-" + str(in_nside) + "-" + str(f) + ".fits"
print("\rread {} of {}. ndata: {}".format(i, len(fi), ndata))
try:
h = fitsio.FITS(path)
except IOError:
print("Error reading pix {}\n".format(f))
continue
## get the quasars
tid_qsos = thid[(in_pixs == f)]
plate_qsos = plate[(in_pixs == f)]
mjd_qsos = mjd[(in_pixs == f)]
fid_qsos = fid[(in_pixs == f)]
if 'TARGET_RA' in h["FIBERMAP"].get_colnames():
ra = h["FIBERMAP"]["TARGET_RA"][:] * sp.pi / 180.
de = h["FIBERMAP"]["TARGET_DEC"][:] * sp.pi / 180.
elif 'RA_TARGET' in h["FIBERMAP"].get_colnames():
## TODO: These lines are for backward compatibility
## Should be removed at some point
ra = h["FIBERMAP"]["RA_TARGET"][:] * sp.pi / 180.
de = h["FIBERMAP"]["DEC_TARGET"][:] * sp.pi / 180.
pixs = healpy.ang2pix(nside, sp.pi / 2 - de, ra)
#exp = h["FIBERMAP"]["EXPID"][:]
#night = h["FIBERMAP"]["NIGHT"][:]
#fib = h["FIBERMAP"]["FIBER"][:]
in_tids = h["FIBERMAP"]["TARGETID"][:]
specData = {}
for spec in ['B', 'R', 'Z']:
dic = {}
try:
dic['LL'] = np.log10(h['{}_WAVELENGTH'.format(spec)].read())
dic['FL'] = h['{}_FLUX'.format(spec)].read()
dic['IV'] = h['{}_IVAR'.format(spec)].read() * (
h['{}_MASK'.format(spec)].read() == 0)
w = sp.isnan(dic['FL']) | sp.isnan(dic['IV'])
for k in ['FL', 'IV']:
dic[k][w] = 0.
dic['RESO'] = h['{}_RESOLUTION'.format(spec)].read()
specData[spec] = dic
except OSError:
pass
h.close()
for t, p, m, f in zip(tid_qsos, plate_qsos, mjd_qsos, fid_qsos):
wt = in_tids == t
if wt.sum() == 0:
print("\nError reading thingid {}\n".format(t))
continue
d = None
for tspecData in specData.values():
iv = tspecData['IV'][wt]
fl = (iv * tspecData['FL'][wt]).sum(axis=0)
iv = iv.sum(axis=0)
w = iv > 0.
fl[w] /= iv[w]
if not pk1d is None:
reso_sum = tspecData['RESO'][wt].sum(axis=0)
reso_in_km_per_s = spectral_resolution_desi(
reso_sum, tspecData['LL'])
diff = np.zeros(tspecData['LL'].shape)
else:
reso_in_km_per_s = None
diff = None
td = forest(tspecData['LL'], fl, iv, t, ra[wt][0], de[wt][0],
ztable[t], p, m, f, order, diff, reso_in_km_per_s)
if d is None:
d = copy.deepcopy(td)
else:
d += td
pix = pixs[wt][0]
if pix not in data:
data[pix] = []
data[pix].append(d)
ndata += 1
print("found {} quasars in input files\n".format(ndata))
return data
def read_from_spplate(in_dir,
thid,
ra,
dec,
zqso,
plate,
mjd,
fid,
order,
log=None,
best_obs=False):
drq_dict = {t: (r, d, z) for t, r, d, z in zip(thid, ra, dec, zqso)}
## if using multiple observations,
## then replace thid, plate, mjd, fid
## by what's available in spAll
if not best_obs:
fi = glob.glob(in_dir + "/spAll-*.fits")
if len(fi) > 1:
print("ERROR: found multiple spAll files")
print(
"ERROR: try running with --bestobs option (but you will lose reobservations)"
)
for f in fi:
print("found: ", fi)
sys.exit(1)
if len(fi) == 0:
print("ERROR: can't find required spAll file in {}".format(in_dir))
print(
"ERROR: try runnint with --best-obs option (but you will lose reobservations)"
)
sys.exit(1)
spAll = fitsio.FITS(fi[0])
print("INFO: reading spAll from {}".format(fi[0]))
thid_spall = spAll[1]["THING_ID"][:]
plate_spall = spAll[1]["PLATE"][:]
mjd_spall = spAll[1]["MJD"][:]
fid_spall = spAll[1]["FIBERID"][:]
qual_spall = spAll[1]["PLATEQUALITY"][:].astype(str)
zwarn_spall = spAll[1]["ZWARNING"][:]
w = np.in1d(thid_spall, thid)
print("INFO: Found {} spectra with required THING_ID".format(w.sum()))
w &= qual_spall == "good"
print("INFO: Found {} spectra with 'good' plate".format(w.sum()))
## Removing spectra with the following ZWARNING bits set:
## SKY, LITTLE_COVERAGE, UNPLUGGED, BAD_TARGET, NODATA
## https://www.sdss.org/dr14/algorithms/bitmasks/#ZWARNING
bad_zwarnbit = {
0: 'SKY',
1: 'LITTLE_COVERAGE',
7: 'UNPLUGGED',
8: 'BAD_TARGET',
9: 'NODATA'
}
for zwarnbit, zwarnbit_str in bad_zwarnbit.items():
w &= (zwarn_spall & 2**zwarnbit) == 0
print("INFO: Found {} spectra without {} bit set: {}".format(
w.sum(), zwarnbit, zwarnbit_str))
print("INFO: # unique objs: ", len(thid))
print("INFO: # spectra: ", w.sum())
thid = thid_spall[w]
plate = plate_spall[w]
mjd = mjd_spall[w]
fid = fid_spall[w]
spAll.close()
## to simplify, use a list of all metadata
allmeta = []
for t, p, m, f in zip(thid, plate, mjd, fid):
r, d, z = drq_dict[t]
meta = metadata()
meta.thid = t
meta.ra = r
meta.dec = d
meta.zqso = z
meta.plate = p
meta.mjd = m
meta.fid = f
meta.order = order
allmeta.append(meta)
pix_data = {}
platemjd = {}
for p, m, meta in zip(plate, mjd, allmeta):
pm = (p, m)
if not pm in platemjd:
platemjd[pm] = []
platemjd[pm].append(meta)
print("reading {} plates".format(len(platemjd)))
for pm in platemjd:
p, m = pm
spplate = in_dir + "/{0}/spPlate-{0}-{1}.fits".format(
str(p).zfill(4), m)
try:
h = fitsio.FITS(spplate)
head0 = h[0].read_header()
except IOError:
log.write("error reading {}\n".format(spplate))
continue
t0 = time.time()
coeff0 = head0["COEFF0"]
coeff1 = head0["COEFF1"]
flux = h[0].read()
ivar = h[1].read() * (h[2].read() == 0)
llam = coeff0 + coeff1 * np.arange(flux.shape[1])
## now convert all those fluxes into forest objects
for meta in platemjd[pm]:
t = meta.thid
r = meta.ra
d = meta.dec
z = meta.zqso
f = meta.fid
o = meta.order
i = meta.fid - 1
d = forest(llam, flux[i], ivar[i], t, r, d, z, p, m, f, o)
if t in pix_data:
pix_data[t] += d
else:
pix_data[t] = d
if log is not None:
log.write("{} read from file {} and mjd {}\n".format(
t, spplate, m))
nread = len(platemjd[pm])
print("INFO: read {} from {} in {} per spec. Progress: {} of {} \n".
format(nread, os.path.basename(spplate),
(time.time() - t0) / (nread + 1e-3), len(pix_data),
len(thid)))
h.close()
data = list(pix_data.values())
return data
def read_from_spcframe(in_dir,
thid,
ra,
dec,
zqso,
plate,
mjd,
fid,
order,
mode=None,
log=None,
best_obs=False,
single_exp=False):
if not best_obs:
print(
"ERROR: multiple observations not (yet) compatible with spframe option"
)
print("ERROR: rerun with the --best-obs option")
sys.exit(1)
allmeta = []
for t, r, d, z, p, m, f in zip(thid, ra, dec, zqso, plate, mjd, fid):
meta = metadata()
meta.thid = t
meta.ra = r
meta.dec = d
meta.zqso = z
meta.plate = p
meta.mjd = m
meta.fid = f
meta.order = order
allmeta.append(meta)
platemjd = {}
for i in range(len(thid)):
pm = (plate[i], mjd[i])
if not pm in platemjd:
platemjd[pm] = []
platemjd[pm].append(allmeta[i])
pix_data = {}
print("reading {} plates".format(len(platemjd)))
for pm in platemjd:
p, m = pm
exps = []
spplate = in_dir + "/{0}/spPlate-{0}-{1}.fits".format(p, m)
print("INFO: reading plate {}".format(spplate))
h = fitsio.FITS(spplate)
head = h[0].read_header()
h.close()
iexp = 1
for c in ["B1", "B2", "R1", "R2"]:
card = "NEXP_{}".format(c)
if card in head:
nexp = head["NEXP_{}".format(c)]
else:
continue
for _ in range(nexp):
str_iexp = str(iexp)
if iexp < 10:
str_iexp = '0' + str_iexp
card = "EXPID" + str_iexp
if not card in head:
continue
exps.append(head["EXPID" + str_iexp][:11])
iexp += 1
print("INFO: found {} exposures in plate {}-{}".format(
len(exps), p, m))
if len(exps) == 0:
continue
exp_num = [e[3:] for e in exps]
exp_num = np.unique(exp_num)
sp.random.shuffle(exp_num)
exp_num = exp_num[0]
for exp in exps:
if single_exp:
if not exp_num in exp:
continue
t0 = time.time()
## find the spectrograph number:
spectro = int(exp[1])
assert spectro == 1 or spectro == 2
spcframe = fitsio.FITS(in_dir +
"/{}/spCFrame-{}.fits".format(p, exp))
flux = spcframe[0].read()
ivar = spcframe[1].read() * (spcframe[2].read() == 0)
llam = spcframe[3].read()
## now convert all those fluxes into forest objects
for meta in platemjd[pm]:
if spectro == 1 and meta.fid > 500: continue
if spectro == 2 and meta.fid <= 500: continue
index = (meta.fid - 1) % 500
t = meta.thid
r = meta.ra
d = meta.dec
z = meta.zqso
f = meta.fid
order = meta.order
d = forest(llam[index], flux[index], ivar[index], t, r, d, z,
p, m, f, order)
if t in pix_data:
pix_data[t] += d
else:
pix_data[t] = d
if log is not None:
log.write("{} read from exp {} and mjd {}\n".format(
t, exp, m))
nread = len(platemjd[pm])
print(
"INFO: read {} from {} in {} per spec. Progress: {} of {} \n".
format(nread, exp, (time.time() - t0) / (nread + 1e-3),
len(pix_data), len(thid)))
spcframe.close()
data = list(pix_data.values())
return data
def read_from_spec(in_dir,
thid,
ra,
dec,
zqso,
plate,
mjd,
fid,
order,
mode,
log=None,
pk1d=None,
best_obs=None):
drq_dict = {t: (r, d, z) for t, r, d, z in zip(thid, ra, dec, zqso)}
## if using multiple observations,
## then replace thid, plate, mjd, fid
## by what's available in spAll
if not best_obs:
fi = glob.glob(
in_dir.replace("spectra/", "").replace("lite", "").replace(
"full", "") + "/spAll-*.fits")
if len(fi) > 1:
print("ERROR: found multiple spAll files")
print(
"ERROR: try running with --bestobs option (but you will lose reobservations)"
)
for f in fi:
print("found: ", fi)
sys.exit(1)
if len(fi) == 0:
print("ERROR: can't find required spAll file in {}".format(in_dir))
print(
"ERROR: try runnint with --best-obs option (but you will lose reobservations)"
)
sys.exit(1)
spAll = fitsio.FITS(fi[0])
print("INFO: reading spAll from {}".format(fi[0]))
thid_spall = spAll[1]["THING_ID"][:]
plate_spall = spAll[1]["PLATE"][:]
mjd_spall = spAll[1]["MJD"][:]
fid_spall = spAll[1]["FIBERID"][:]
qual_spall = spAll[1]["PLATEQUALITY"][:].astype(str)
zwarn_spall = spAll[1]["ZWARNING"][:]
w = np.in1d(thid_spall, thid) & (qual_spall == "good")
## Removing spectra with the following ZWARNING bits set:
## SKY, LITTLE_COVERAGE, UNPLUGGED, BAD_TARGET, NODATA
## https://www.sdss.org/dr14/algorithms/bitmasks/#ZWARNING
for zwarnbit in [0, 1, 7, 8, 9]:
w &= (zwarn_spall & 2**zwarnbit) == 0
print("INFO: # unique objs: ", len(thid))
print("INFO: # spectra: ", w.sum())
thid = thid_spall[w]
plate = plate_spall[w]
mjd = mjd_spall[w]
fid = fid_spall[w]
spAll.close()
## to simplify, use a list of all metadata
allmeta = []
## Used to preserve original order and pass unit tests.
t_list = []
t_set = set()
for t, p, m, f in zip(thid, plate, mjd, fid):
if t not in t_set:
t_list.append(t)
t_set.add(t)
r, d, z = drq_dict[t]
meta = metadata()
meta.thid = t
meta.ra = r
meta.dec = d
meta.zqso = z
meta.plate = p
meta.mjd = m
meta.fid = f
meta.order = order
allmeta.append(meta)
pix_data = []
thids = {}
for meta in allmeta:
t = meta.thid
if not t in thids:
thids[t] = []
thids[t].append(meta)
print("reading {} thids".format(len(thids)))
for t in t_list:
t_delta = None
for meta in thids[t]:
r, d, z, p, m, f = meta.ra, meta.dec, meta.zqso, meta.plate, meta.mjd, meta.fid
try:
fin = in_dir + "/{}/{}-{}-{}-{:04d}.fits".format(
p, mode, p, m, f)
h = fitsio.FITS(fin)
except IOError:
log.write("error reading {}\n".format(fin))
continue
log.write("{} read\n".format(fin))
ll = h[1]["loglam"][:]
fl = h[1]["flux"][:]
iv = h[1]["ivar"][:] * (h[1]["and_mask"][:] == 0)
if pk1d is not None:
# compute difference between exposure
diff = exp_diff(h, ll)
# compute spectral resolution
wdisp = h[1]["wdisp"][:]
reso = spectral_resolution(wdisp, True, f, ll)
else:
diff = None
reso = None
deltas = forest(ll,
fl,
iv,
t,
r,
d,
z,
p,
m,
f,
order,
diff=diff,
reso=reso)
if t_delta is None:
t_delta = deltas
else:
t_delta += deltas
h.close()
if t_delta is not None:
pix_data.append(t_delta)
return pix_data
if j != i:
l.append(data[j])
resample_data = N.asarray(l)
# Fit continuum for all spectra
for d in resample_data:
d.cont_fit()
# Create a stack of all the deltas
ll2, de2, wst2 = stack_flux(resample_data, 1)
## Create deltas of the stack (of weighted flux)
# Stack the weighted flux
ll1, st1, wst1 = stack_flux(resample_data, 0)
# Fit the continuum for the stack
d = forest(ll1, st1, wst1, d.thid, d.ra, d.dec, d.zqso, d.plate, d.mjd, d.fid, d.order)
d.cont_fit()
# Create deltas and weights in same bin widths as before
nstack = int((forest.lmax - forest.lmin) / forest.dll) + 1
de1 = sp.zeros(nstack)
bins=((d.ll - d.lmin) / d.dll + 0.5).astype(int)
c = sp.bincount(bins, weights = d.fl / d.co - 1)
de1[:len(c)] += c
eta = forest.eta(d.ll)
var_lss = forest.var_lss(d.ll)
iv = d.iv / eta
we = iv * d.co**2 / (iv * d.co**2 * var_lss + 1)
c = sp.bincount(bins, weights = we)
wst1[:len(c)] += c
