def main_aux(
pathlist=None,
reqnum=None,
band=None,
ccdnum=None,
nproc=None,
chunk=None,
px_side=None,
label=None,
):
''' Main auxiliaty function to call the code in parallel
'''
if (pathlist is None):
# If no set of pull paths is provided, go to the DB and retrieve them,
# based on reqnum and band
fpath = db_red_pixcor(reqnum, band, ccdnum)
elif (pathlist is not None):
# Load the table
exp = np.genfromtxt(pathlist,
dtype=[
('path', '|S200'),
],
comments='#',
missing_values=np.nan,
usecols=0)
fpath = exp['path']
# Remove duplicates using np.unique capabilities
uarr, uidx, uinverse, ucounts = np.unique(
fpath,
return_index=True,
return_inverse=True,
return_counts=True,
)
if False:
# Save table for simplicity of testing
tmp = pd.DataFrame({'path': fpath})
tmp.to_csv('path_r3437_g.txt', index=False, header=False)
#
# NOTE
# The goal is to read the same section, from a bunch of images, in
# parallel. Then stack the set of section and calculate the median
# of each pixel.
#
# Get shape of the CCD
try:
aux_fits = fitsio.read(fpath[0])
f_dim = aux_fits.shape
logging.info('CCD shape: {0}'.format(f_dim))
except:
logging.error('File cannot be read. Using 2048x4096 as CCD dimensions')
f_dim = (4096, 2048)
# Define squared sections to be used for calculation
if (px_side is None):
px_side = 512
if ((f_dim[0] % px_side != 0) or (f_dim[1] % px_side != 0)):
logging.warning('{0}x{0} des not fit exactly on CCD'.format(px_side))
# idx_d0 = np.arange(0, fits_dim[0] + 1, px_side)
# idx_d1 = np.arange(0, fits_dim[1] + 1, px_side)
yx = np.mgrid[0:f_dim[0] + 1:px_side, 0:f_dim[1] + 1:px_side]
# coo_list contains [x0, y0, x1, y1]
coo_list = []
for iy in range(f_dim[0] / px_side): #yx.shape[1]):
for ix in range(f_dim[1] / px_side): #(yx.shape[2]):
y0, x0 = (yx[0, iy, ix], yx[1, iy, ix])
y1, x1 = (yx[0, iy + 1, ix], yx[1, iy, ix + 1])
coo_list.append((x0, y0, x1, y1))
# Create an array to harbor the results
tmp_res = np.zeros((f_dim[0], f_dim[1]))
# Setup the parallel call
# The value for chunk will help prevent memory errors. "Chops the iterable
# into a number of chunks which it submits to the process pool as separate
# tasks"
if (nproc is None):
nproc = mp.cpu_count()
if (chunk is None):
chunk = int(np.ceil(fpath.size / nproc))
logging.info('Launch {0} parallel processes'.format(nproc))
P1 = mp.Pool(processes=nproc)
# Here I can also parallelize in terms of coordinates, but need to be
# careful to not mix results from different sections of the CCD, and
# manage the Pool (2 in fact)
for idx_c, coo in enumerate(coo_list):
logging.info('Box {0} of {1}'.format(idx_c + 1, len(coo_list)))
# coo: coordinates, ext: extension to read from the FITS file,
# delta: side size (pixel) of the square used to sample the FITS file
kw_section = {
'coo': coo,
'ext': 0,
'delta': px_side,
}
# Call using partial() or constructed list
try:
partial_aux = partial(
fits_section,
[kw_section['coo'], kw_section['ext'], kw_section['delta']])
t0 = time.time()
# map_async does not block the processes, and executes non-ordered
boxi = P1.map_async(partial_aux, fpath, chunk)
boxi.wait()
t1 = time.time()
except:
aux_list = [(fnm, kw_section['coo'], kw_section['ext'],
kw_section['delta']) for fnm in fpath]
t0 = time.time()
boxi = P1.map_async(fits_section, aux_list, chunk)
boxi.wait()
t1 = time.time()
boxi = boxi.get()
# At this point the Pool has returned a list, containig the stamps
# for all the CCDs
# Call the median image generator, using np.nditer(), with a 3D array
x3d = np.dstack(boxi)
z_median = stat_cube(x3d, (lambda: np.median)())
# Put into the auxiliary array
x0, y0, x1, y1 = coo
tmp_res[y0:y1, x0:x1] = z_median
# The results looks good!
# Save in FITS format
if (label is None):
label = str(uuid.uuid4())
ores = 'medImg_{0}.fits'.format(label)
while os.path.exists(ores):
logging.info('File {0} exists. Changing output name'.format(ores))
ores = 'medImg_{0}.fits'.format(str(uuid.uuid4()))
fits = fitsio.FITS(ores, 'rw')
fits.write(tmp_res)
fits[-1].write_checksum()
fits.close()
logging.info('Median image {0} saved'.format(ores))
return True
def read_mock_spectra(truthfile, targetids, mockdir=None):
'''
Reads mock spectra from a truth file
Args:
truthfile (str): full path to a mocks spectra_truth*.fits file
targetids (array-like): targetids to load from that file
mockdir: ???
Returns (flux, wave, truth) tuples:
flux[nspec, nwave]: flux in 1e-17 erg/s/cm2/Angstrom
wave[nwave]: wavelengths in Angstroms
truth[nspec]: metadata truth table
'''
if len(targetids) != len(np.unique(targetids)):
from lvmutil.log import get_logger
log = get_logger()
log.error("Requested TARGETIDs for {} are not unique".format(
os.path.basename(truthfile)))
#- astropy.io.fits doesn't return a real ndarray, causing problems
#- with the reordering downstream so use fitsio instead
# with fits.open(truthfile, memmap=False) as fx:
# truth = fx['TRUTH'].data
# wave = fx['WAVE'].data
# flux = fx['FLUX'].data
with fitsio.FITS(truthfile) as fx:
truth = fx['TRUTH'].read()
wave = fx['WAVE'].read()
flux = fx['FLUX'].read()
missing = np.in1d(targetids, truth['TARGETID'], invert=True)
if np.any(missing):
missingids = targetids[missing]
raise ValueError('Targets missing from {}: {}'.format(
truthfile, missingids))
#- Trim to just the spectra for these targetids
ii = np.in1d(truth['TARGETID'], targetids)
flux = flux[ii]
truth = truth[ii]
assert set(targetids) == set(truth['TARGETID'])
#- sort truth to match order of input targetids
if len(targetids) == len(truth['TARGETID']):
i = np.argsort(targetids)
j = np.argsort(truth['TARGETID'])
k = np.argsort(i)
reordered_truth = truth[j[k]]
reordered_flux = flux[j[k]]
else:
#- Slower, but works even with repeated TARGETIDs
ii = np.argsort(truth['TARGETID'])
sorted_truthids = truth['TARGETID'][ii]
reordered_flux = np.empty(shape=(len(targetids), flux.shape[1]),
dtype=flux.dtype)
reordered_truth = np.empty(shape=(len(targetids), ), dtype=truth.dtype)
for j, tx in enumerate(targetids):
k = np.searchsorted(sorted_truthids, tx)
reordered_flux[j] = flux[ii[k]]
reordered_truth[j] = truth[ii[k]]
assert np.all(reordered_truth['TARGETID'] == targetids)
wave = lvmspec.io.util.native_endian(wave).astype(np.float64)
reordered_flux = lvmspec.io.util.native_endian(reordered_flux).astype(
np.float64)
return reordered_flux, wave, reordered_truth
import argparse
import os
import pylab as plt
plt.switch_backend('agg')
parser = argparse.ArgumentParser()
parser.add_argument('-f', '--filename', action='store', type=str)
parser.add_argument('--nofits', action='store_true')
parser.add_argument('--nofig', action='store_true')
args = parser.parse_args()
files = glob.glob(args.filename)
print 'found %d files to process' % len(files)
baryons1 = fi.FITS(
'/home/ssamurof/massive_black_ii/subhalo_cat-nthreshold5.fits'
)['baryons'][:]
dm1 = fi.FITS(
'/home/ssamurof/massive_black_ii/subhalo_cat-nthreshold5.fits')['dm'][:]
select = (dm1['npart'] > 1000) & (baryons1['npart'] > 0) & (np.isfinite(
baryons1['x']) & np.isfinite(baryons1['y']) & np.isfinite(baryons1['z']))
dat = copy.deepcopy(baryons1[select])
for f in files:
d = fi.FITS(f)['baryons'].read()
for comp in ['x', 'y', 'z']:
dat[comp][(d[comp] != baryons1[comp][select])] = d[comp][(
d[comp] != baryons1[comp][select])]
print f
def main():
import argparse
parser = argparse.ArgumentParser(
description='This script creates small self-contained data sets that '
'are useful for test cases of the pipeline codes.')
parser.add_argument('ccds', help='CCDs table describing region to grab')
parser.add_argument('outdir', help='Output directory name')
parser.add_argument('brick', help='Brick containing these images')
parser.add_argument('--survey-dir', type=str, default=None)
parser.add_argument('--cache-dir', type=str, default=None,
help='Directory to search for cached files')
parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image')
parser.add_argument('--wise-wcs-hdu', help='For WISE outputs, the HDU to read the WCS from in the file given by --wise.', type=int, default=0)
parser.add_argument('--fpack', action='store_true', default=False)
parser.add_argument('--gzip', action='store_true', default=False)
parser.add_argument('--pad', action='store_true', default=False,
help='Keep original image size, but zero out pixels outside ROI')
args = parser.parse_args()
v = 'SKY_TEMPLATE_DIR'
if v in os.environ:
del os.environ[v]
C = fits_table(args.ccds)
print(len(C), 'CCDs in', args.ccds)
C.camera = np.array([c.strip() for c in C.camera])
survey = LegacySurveyData(cache_dir=args.cache_dir, survey_dir=args.survey_dir)
if ',' in args.brick:
ra,dec = args.brick.split(',')
ra = float(ra)
dec = float(dec)
fakebricks = fits_table()
fakebricks.brickname = np.array([('custom-%06i%s%05i' %
(int(1000*ra), 'm' if dec < 0 else 'p',
int(1000*np.abs(dec))))])
fakebricks.ra = np.array([ra])
fakebricks.dec = np.array([dec])
bricks = fakebricks
outbricks = bricks
else:
bricks = survey.get_bricks_readonly()
outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])]
assert(len(outbricks) == 1)
outsurvey = LegacySurveyData(survey_dir = args.outdir)
trymakedirs(args.outdir)
outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz'))
targetwcs = wcs_for_brick(outbricks[0])
H,W = targetwcs.shape
tycho2fn = survey.find_file('tycho2')
kd = tree_open(tycho2fn, 'stars')
radius = 1.
rc,dc = targetwcs.radec_center()
I = tree_search_radec(kd, rc, dc, radius)
print(len(I), 'Tycho-2 stars within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (rc,dc))
# Read only the rows within range.
tycho = fits_table(tycho2fn, rows=I)
del kd
print('Read', len(tycho), 'Tycho-2 stars')
ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec)
#margin = 100
#tycho.cut(ok * (tx > -margin) * (tx < W+margin) *
# (ty > -margin) * (ty < H+margin))
print('Cut to', len(tycho), 'Tycho-2 stars within brick')
del ok,tx,ty
#tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz'))
f,tfn = tempfile.mkstemp(suffix='.fits')
os.close(f)
tycho.writeto(tfn)
outfn = os.path.join(args.outdir, 'tycho2.kd.fits')
cmd = 'startree -i %s -o %s -P -k -n stars -T' % (tfn, outfn)
print(cmd)
rtn = os.system(cmd)
assert(rtn == 0)
os.unlink(tfn)
from legacypipe.gaiacat import GaiaCatalog
gcat = GaiaCatalog()
# from ps1cat.py:
wcs = targetwcs
step=100.
margin=10.
# Grid the CCD in pixel space
W,H = wcs.get_width(), wcs.get_height()
xx,yy = np.meshgrid(
np.linspace(1-margin, W+margin, 2+int((W+2*margin)/step)),
np.linspace(1-margin, H+margin, 2+int((H+2*margin)/step)))
# Convert to RA,Dec and then to unique healpixes
ra,dec = wcs.pixelxy2radec(xx.ravel(), yy.ravel())
healpixes = set()
for r,d in zip(ra,dec):
healpixes.add(gcat.healpix_for_radec(r, d))
for hp in healpixes:
hpcat = gcat.get_healpix_catalog(hp)
ok,xx,yy = wcs.radec2pixelxy(hpcat.ra, hpcat.dec)
onccd = np.flatnonzero((xx >= 1.-margin) * (xx <= W+margin) *
(yy >= 1.-margin) * (yy <= H+margin))
hpcat.cut(onccd)
if len(hpcat):
outfn = os.path.join(args.outdir, 'gaia', 'chunk-%05d.fits' % hp)
trymakedirs(os.path.join(args.outdir, 'gaia'))
hpcat.writeto(outfn)
outccds = C.copy()
cols = outccds.get_columns()
for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1',
'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1',
'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver',
'psfplver' ]:
if c in cols:
outccds.delete_column(c)
outccds.image_hdu[:] = 1
# Convert to list to avoid truncating filenames
outccds.image_filename = [fn for fn in outccds.image_filename]
for iccd,ccd in enumerate(C):
decam = (ccd.camera.strip() == 'decam')
bok = (ccd.camera.strip() == '90prime')
im = survey.get_image_object(ccd)
print('Got', im)
if survey.cache_dir is not None:
im.check_for_cached_files(survey)
slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1))
psfkwargs = dict(pixPsf=True, gaussPsf=False, hybridPsf=False,
normalizePsf=False)
tim = im.get_tractor_image(slc, pixPsf=True,
subsky=False, nanomaggies=False,
no_remap_invvar=True, old_calibs_ok=True)
print('Tim:', tim.shape)
psfrow = psfhdr = None
if args.pad:
psf = im.read_psf_model(0, 0, w=im.width, h=im.height, **psfkwargs)
psfex = psf.psfex
else:
psf = tim.getPsf()
psfex = psf.psfex
# Did the PSF model come from a merged file?
for fn in [im.merged_psffn, im.psffn] + im.old_merged_psffns:
if not os.path.exists(fn):
continue
T = fits_table(fn)
I, = np.nonzero((T.expnum == im.expnum) *
np.array([c.strip() == im.ccdname for c in T.ccdname]))
if len(I) != 1:
continue
psfrow = T[I]
x0 = ccd.ccd_x0
y0 = ccd.ccd_y0
psfrow.polzero1[0] -= x0
psfrow.polzero2[0] -= y0
#psfhdr = fitsio.read_header(im.merged_psffn)
break
psfex.fwhm = tim.psf_fwhm
#### HACK
#psfrow = None
assert(psfrow is not None)
if psfrow is not None:
print('PSF row:', psfrow)
#else:
# print('PSF:', psf)
# print('PsfEx:', psfex)
skyrow = skyhdr = None
if args.pad:
primhdr = fitsio.read_header(im.imgfn)
imghdr = fitsio.read_header(im.imgfn, hdu=im.hdu)
sky = im.read_sky_model(splinesky=True, primhdr=primhdr, imghdr=imghdr)
#skyhdr = fitsio.read_header(im.splineskyfn)
#msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True)
else:
sky = tim.getSky()
# Did the sky model come from a merged file?
#msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True)
print('merged skyfn:', im.merged_skyfn)
print('single skyfn:', im.skyfn)
print('old merged skyfns:', im.old_merged_skyfns)
for fn in [im.merged_skyfn, im.skyfn] + im.old_merged_skyfns:
if not os.path.exists(fn):
continue
T = fits_table(fn)
I, = np.nonzero((T.expnum == im.expnum) *
np.array([c.strip() == im.ccdname for c in T.ccdname]))
skyrow = T[I]
skyrow.x0[0] = ccd.ccd_x0
skyrow.y0[0] = ccd.ccd_y0
# s_med = skyrow.sky_med[0]
# s_john = skyrow.sky_john[0]
# skyhdr = fitsio.read_header(fn)
assert(skyrow is not None)
### HACK
#skyrow = None
if skyrow is not None:
print('Sky row:', skyrow)
else:
print('Sky:', sky)
# Output filename format:
fn = ccd.image_filename.strip()
ccd.image_filename = os.path.join(os.path.dirname(fn),
'%s.%s.fits' % (os.path.basename(fn).split('.')[0], ccd.ccdname.strip()))
outim = outsurvey.get_image_object(ccd)
print('Output image:', outim)
print('Image filename:', outim.imgfn)
trymakedirs(outim.imgfn, dir=True)
imgdata = tim.getImage()
ivdata = tim.getInvvar()
# Since we remap DQ codes (always with Mosaic and Bok, sometimes with DECam),
# re-read from the FITS file rather than using tim.dq.
print('Reading data quality from', im.dqfn, 'hdu', im.hdu)
dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice)
print('Tim shape:', tim.shape, 'Slice', tim.slice)
print('image shape:', imgdata.shape, 'iv', ivdata.shape, 'DQ', dqdata.shape)
from collections import Counter
dqvals = Counter(dqdata.ravel())
print('DQ pixel counts:')
for k,n in dqvals.most_common():
print(' 0x%x' % k, ':', n)
if args.pad:
# Create zero image of full size, copy in data.
fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype)
fullsize[slc] = imgdata
imgdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype)
fullsize[slc] = dqdata
dqdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype)
fullsize[slc] = ivdata
ivdata = fullsize
else:
# Adjust the header WCS by x0,y0
crpix1 = tim.hdr['CRPIX1']
crpix2 = tim.hdr['CRPIX2']
tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0
tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0
# Add image extension to filename
# fitsio doesn't compress .fz by default, so drop .fz suffix
#outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits')
if args.gzip:
outim.imgfn = outim.imgfn.replace('.fits', '.fits.gz')
#outim.wtfn = outim.wtfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.wtfn = outim.wtfn.replace('.fits.fz', '.fits')
if args.gzip:
outim.wtfn = outim.wtfn.replace('.fits', '.fits.gz')
if outim.dqfn is not None:
#outim.dqfn = outim.dqfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.dqfn = outim.dqfn.replace('.fits.fz', '.fits')
if args.gzip:
outim.dqfn = outim.dqfn.replace('.fits', '.fits.gz')
if bok:
outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname)
ccdfn = outim.imgfn
ccdfn = ccdfn.replace(outsurvey.get_image_dir(), '')
if ccdfn.startswith('/'):
ccdfn = ccdfn[1:]
outccds.image_filename[iccd] = ccdfn
print('Changed output filenames to:')
print(outim.imgfn)
print(outim.dqfn)
ofn = outim.imgfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fits = fitsio.FITS(ofn, 'rw', clobber=True)
fits.write(None, header=tim.primhdr)
fits.write(imgdata, header=tim.hdr, extname=ccd.ccdname)
fits.close()
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
h,w = tim.shape
if not args.pad:
outccds.width[iccd] = w
outccds.height[iccd] = h
outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0
outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0
wcs = Tan(*[float(x) for x in
[ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2,
ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]])
if args.pad:
subwcs = wcs
else:
subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h)
outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center()
print('Weight filename:', outim.wtfn)
wfn = outim.wtfn
trymakedirs(wfn, dir=True)
ofn = wfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fits = fitsio.FITS(ofn, 'rw', clobber=True)
fits.write(None, header=tim.primhdr)
fits.write(ivdata, header=tim.hdr, extname=ccd.ccdname)
fits.close()
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
if outim.dqfn is not None:
print('DQ filename', outim.dqfn)
trymakedirs(outim.dqfn, dir=True)
ofn = outim.dqfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fits = fitsio.FITS(ofn, 'rw', clobber=True)
fits.write(None, header=tim.primhdr)
fits.write(dqdata, header=tim.hdr, extname=ccd.ccdname)
fits.close()
if args.fpack:
cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
psfout = outim.psffn
#if psfrow:
# psfout = outim.merged_psffn
print('PSF output filename:', psfout)
trymakedirs(psfout, dir=True)
if psfrow:
psfrow.writeto(psfout, primhdr=psfhdr)
else:
print('Writing PsfEx:', psfout)
psfex.writeto(psfout)
# update header
F = fitsio.FITS(psfout, 'rw')
F[0].write_keys([dict(name='EXPNUM', value=ccd.expnum),
dict(name='PLVER', value=psf.plver),
dict(name='PROCDATE', value=psf.procdate),
dict(name='PLPROCID', value=psf.plprocid),])
F.close()
skyout = outim.skyfn
#if skyrow:
# skyout = outim.merged_splineskyfn
print('Sky output filename:', skyout)
trymakedirs(skyout, dir=True)
if skyrow is not None:
skyrow.writeto(skyout, primhdr=skyhdr)
else:
primhdr = fitsio.FITSHDR()
primhdr['PLVER'] = sky.plver
primhdr['PLPROCID'] = sky.plprocid
primhdr['PROCDATE'] = sky.procdate
primhdr['EXPNUM'] = ccd.expnum
primhdr['IMGDSUM'] = sky.datasum
primhdr['S_MED'] = s_med
primhdr['S_JOHN'] = s_john
sky.write_fits(skyout, primhdr=primhdr)
# HACK -- check result immediately.
outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))
outsurvey.ccds = None
outC = outsurvey.get_ccds_readonly()
occd = outC[iccd]
outim = outsurvey.get_image_object(occd)
print('Got output image:', outim)
otim = outim.get_tractor_image(pixPsf=True,
hybridPsf=True, old_calibs_ok=True)
print('Got output tim:', otim)
outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))
# WISE
if args.wise is not None:
from wise.forcedphot import unwise_tiles_touching_wcs
from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs,
get_unwise_tractor_image, get_unwise_tile_dir)
# Read WCS...
print('Reading TAN wcs header from', args.wise, 'HDU', args.wise_wcs_hdu)
targetwcs = Tan(args.wise, args.wise_wcs_hdu)
tiles = unwise_tiles_touching_wcs(targetwcs)
print('Cut to', len(tiles), 'unWISE tiles')
H,W = targetwcs.shape
r,d = targetwcs.pixelxy2radec(np.array([1, W, W/2, W/2]),
np.array([H/2, H/2, 1, H ]))
roiradec = [r[0], r[1], d[2], d[3]]
unwise_dir = os.environ['UNWISE_COADDS_DIR']
wise_out = os.path.join(args.outdir, 'images', 'unwise')
print('Will write WISE outputs to', wise_out)
unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR']
wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr')
print('Will write WISE time-resolved outputs to', wise_tr_out)
trymakedirs(wise_tr_out)
W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_atlas.fits'))
print('Read', len(W), 'time-resolved WISE coadd tiles')
W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id]))
print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth')
# Write the time-resolved index subset.
W.writeto(os.path.join(wise_tr_out, 'time_resolved_atlas.fits'))
# this ought to be enough for anyone =)
_,Nepochs = W.epoch_bitmask.shape
print('N epochs in time-resolved atlas:', Nepochs)
wisedata = []
# full depth
for band in [1,2,3,4]:
wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band, True))
# time-resolved
for band in [1,2]:
# W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2)
bitmask = (1 << (band-1))
for e in range(Nepochs):
# Which tiles have images for this epoch?
I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask)
if len(I) == 0:
continue
print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I])
edir = os.path.join(unwise_tr_dir, 'e%03i' % e)
eoutdir = os.path.join(wise_tr_out, 'e%03i' % e)
wisedata.append((edir, eoutdir, tiles.coadd_id[I], band, False))
wrote_masks = set()
model_dir = os.environ.get('UNWISE_MODEL_SKY_DIR')
if model_dir is not None:
model_dir_out = os.path.join(args.outdir, 'images', 'unwise-mod')
trymakedirs(model_dir_out)
for indir, outdir, tiles, band, fulldepth in wisedata:
for tile in tiles:
wanyband = 'w'
tim = get_unwise_tractor_image(indir, tile, band,
bandname=wanyband, roiradecbox=roiradec)
print('Got unWISE tim', tim)
print(tim.shape)
if model_dir is not None and fulldepth and band in [1,2]:
print('ROI', tim.roi)
#0387p575.1.mod.fits
fn = '%s.%i.mod.fits' % (tile, band)
print('Filename', fn)
F = fitsio.FITS(os.path.join(model_dir, fn))
x0,x1,y0,y1 = tim.roi
slc = slice(y0,y1),slice(x0,x1)
phdr = F[0].read_header()
outfn = os.path.join(model_dir_out, fn)
for e,extname in [(1,'MODEL'), (2,'SKY')]:
pix = F[e][slc]
hdr = F[e].read_header()
crpix1 = hdr['CRPIX1']
crpix2 = hdr['CRPIX2']
hdr['CRPIX1'] -= x0
hdr['CRPIX2'] -= y0
#print('mod', mod)
#print('Model', mod.shape)
if e == 1:
fitsio.write(outfn, None, clobber=True, header=phdr)
fitsio.write(outfn, pix, header=hdr, extname=extname)
print('Wrote', outfn)
thisdir = get_unwise_tile_dir(outdir, tile)
print('Directory for this WISE tile:', thisdir)
base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band))
print('Base filename:', base)
masked = True
mu = 'm' if masked else 'u'
imfn = base + 'img-%s.fits' % mu
ivfn = base + 'invvar-%s.fits.gz' % mu
nifn = base + 'n-%s.fits.gz' % mu
nufn = base + 'n-u.fits.gz'
#print('WISE image header:', tim.hdr)
# Adjust the header WCS by x0,y0
wcs = tim.wcs.wcs
tim.hdr['CRPIX1'] = wcs.crpix[0]
tim.hdr['CRPIX2'] = wcs.crpix[1]
H,W = tim.shape
tim.hdr['IMAGEW'] = W
tim.hdr['IMAGEH'] = H
print('WCS:', wcs)
print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2'])
trymakedirs(imfn, dir=True)
fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True)
print('Wrote', imfn)
fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True)
print('Wrote', ivfn)
fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True)
print('Wrote', nifn)
fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True)
print('Wrote', nufn)
if not (indir,tile) in wrote_masks:
print('Looking for mask file for', indir, tile)
# record that we tried this dir/tile combo
wrote_masks.add((indir,tile))
for idir in indir.split(':'):
tdir = get_unwise_tile_dir(idir, tile)
maskfn = 'unwise-%s-msk.fits.gz' % tile
fn = os.path.join(tdir, maskfn)
print('Mask file:', fn)
if os.path.exists(fn):
print('Reading', fn)
(x0,x1,y0,y1) = tim.roi
roislice = (slice(y0,y1), slice(x0,x1))
F = fitsio.FITS(fn)[0]
hdr = F.read_header()
M = F[roislice]
outfn = os.path.join(thisdir, maskfn)
fitsio.write(outfn, M, header=tim.hdr, clobber=True)
print('Wrote', outfn)
break
outC = outsurvey.get_ccds_readonly()
for iccd,ccd in enumerate(outC):
outim = outsurvey.get_image_object(ccd)
print('Got output image:', outim)
otim = outim.get_tractor_image(pixPsf=True,
hybridPsf=True, old_calibs_ok=True)
print('Got output tim:', otim)
import fitsio
from time import sleep
import gc
fnames = [
'/Users/ch/Downloads/ktwo200002562-c02_lpd-targ.fits.gz',
'/Users/ch/Downloads/ktwo200069997-c92_lpd-targ.fits.gz',
'/Users/ch/Downloads/ktwo200071159-c91_spd-targ.fits.gz',
'/Users/ch/Downloads/ktwo200071208-c91_lpd-targ.fits.gz',
'/Users/ch/Downloads/ktwo200083102-c102_lpd-targ.fits.gz',
'/Users/ch/Downloads/ktwo200083127-c102_lpd-targ.fits.gz'
]
for f in fnames:
tpf = fitsio.FITS(f)
sleep(1)
cadencelist = tpf[1]['CADENCENO'].read()
sleep(1)
tpf.close()
sleep(1)
def readFrame(self, run, camcol, field, band, filename=None):
'''
http://data.sdss3.org/datamodel/files/BOSS_PHOTOOBJ/frames/RERUN/RUN/CAMCOL/frame.html
'''
f = Frame(run, camcol, field, band)
# ...
if filename is None:
fn = self.getPath('frame', run, camcol, field, band)
else:
fn = filename
# optionally bunzip2 the frame file.
tempfn,keep = self._unzip_frame(fn, run, camcol)
if tempfn is not None:
fn = tempfn
if fitsio:
print 'Frame filename', fn
# eg /clusterfs/riemann/raid006/dr10/boss/photoObj/frames/301/2825/1/frame-u-002825-1-0126.fits.bz2
F = fitsio.FITS(fn, lower=True)
f.header = F[0].read_header()
# Allow later reading of just the pixels of interest.
f.image_proxy = F[0]
f.calib = F[1].read()
sky = F[2].read_columns(['allsky', 'xinterp', 'yinterp'])
#print 'sky', type(sky)
# ... supposed to be a recarray, but it's not...
f.sky, f.skyxi, f.skyyi = sky.tolist()[0]
tab = fits_table(F[3].read())
if not keep and tempfn is not None:
os.remove(tempfn)
else:
p = pyfits.open(fn)
# in nanomaggies
f.image = p[0].data
f.header = p[0].header
# converts counts -> nanomaggies
f.calib = p[1].data
# table with val,x,y -- binned; use bilinear interpolation to expand
sky = p[2].data
# table -- asTrans structure
tab = fits_table(p[3].data)
f.sky = sky.field('allsky')[0]
f.skyxi = sky.field('xinterp')[0]
f.skyyi = sky.field('yinterp')[0]
#print 'sky shape', f.sky.shape
if len(f.sky.shape) != 2:
f.sky = f.sky.reshape((-1, 256))
assert(len(tab) == 1)
tab = tab[0]
# DR7 has NODE, INCL in radians...
f.astrans = AsTrans(run, camcol, field, band,
node=np.deg2rad(tab.node), incl=np.deg2rad(tab.incl),
astrans=tab, cut_to_band=False)
return f
exposure_dir_list += glob.glob(
os.path.join(redux_dir, 'exposures', obsdate, '*'))
cframe_list = []
# Get a list of all science exposures.
for exposure_dir in exposure_dir_list:
cframe_list_tmp = glob.glob(os.path.join(exposure_dir, 'cframe-*'))
if len(cframe_list_tmp) > 0:
if tileid_list is None:
cframe_list += cframe_list_tmp
else:
# only need to check one cframe file in the exposure
with fitsio.FITS(cframe_list_tmp[0]) as f:
if f[0].read_header()['TILEID'] in tileid_list:
cframe_list += cframe_list_tmp
cframe_list = sorted(cframe_list)
# Gather exposure/petal information
cframes = Table()
cframes['cframe'] = np.array(cframe_list)
cframes['night'] = ' '
cframes['mjd'] = np.zeros(len(cframes), dtype=np.float) - 1.0
cframes['lat'] = np.zeros(len(cframes), dtype=np.float)
cframes['lon'] = np.zeros(len(cframes), dtype=np.float)
cframes['elv'] = np.zeros(len(cframes), dtype=np.float)
cframes['tileid'] = np.zeros(len(cframes), dtype=int)
def write_to(self,
fn,
columns=None,
header='default',
primheader=None,
use_fitsio=True,
append=False,
append_to_hdu=None):
fitsio = None
if use_fitsio:
try:
import fitsio
except:
pass
if columns is None:
columns = self.get_columns()
if fitsio:
arrays = [self.get(c) for c in columns]
fits = fitsio.FITS(fn, 'rw', clobber=(not append))
arrays = [
np.array(a) if isinstance(a, list) else a for a in arrays
]
if header == 'default':
header = None
try:
if append and append_to_hdu is not None:
fits[append_to_hdu].append(arrays,
names=columns,
header=header)
else:
if primheader is not None:
fits.write(None, header=primheader)
fits.write(arrays, names=columns, header=header)
fits.close()
except:
print('Failed to write FITS table')
print('Columns:')
for c, a in zip(columns, arrays):
print(' ', c, type(a), end='')
try:
print(a.dtype, a.shape, end='')
except:
pass
print()
raise
return
fc = self.to_fits_columns(columns)
T = pyfits.new_table(fc)
if header == 'default':
header = self._header
if header is not None:
add_nonstructural_headers(header, T.header)
if primheader is not None:
P = pyfits.PrimaryHDU()
add_nonstructural_headers(primheader, P.header)
pyfits.HDUList([P, T]).writeto(fn, clobber=True)
else:
pyfits_writeto(T, fn)
def fits_table(dataorfn=None,
rows=None,
hdunum=1,
hdu=None,
ext=None,
header='default',
columns=None,
column_map=None,
lower=True,
mmap=True,
normalize=True,
use_fitsio=True,
tabledata_class=tabledata):
'''
If 'columns' (a list of strings) is passed, only those columns
will be read; otherwise all columns will be read.
'''
if dataorfn is None:
return tabledata_class(header=header)
fitsio = None
if use_fitsio:
try:
import fitsio
except:
pass
pf = None
hdr = None
# aliases
if hdu is not None:
hdunum = hdu
if ext is not None:
hdunum = ext
if isinstance(dataorfn, str):
if fitsio:
F = fitsio.FITS(dataorfn)
data = F[hdunum]
hdr = data.read_header()
else:
global pyfits
pf = pyfits.open(dataorfn, memmap=mmap)
data = pf[hdunum].data
if header == 'default':
hdr = pf[hdunum].header
del pf
pf = None
else:
data = dataorfn
if data is None:
return None
T = tabledata_class(header=hdr)
T._columns = []
if fitsio:
isrecarray = False
try:
import pyfits.core
# in a try/catch in case pyfits isn't available
isrecarray = (type(data) == pyfits.core.FITS_rec)
except:
try:
from astropy.io import fits as pyfits
isrecarray = (type(data) == pyfits.core.FITS_rec)
except:
#import traceback
#traceback.print_exc()
pass
if not isrecarray:
try:
import pyfits.fitsrec
isrecarray = (type(data) == pyfits.fitsrec.FITS_rec)
except:
try:
from astropy.io import fits as pyfits
isrecarray = (type(data) == pyfits.fitsrec.FITS_rec)
except:
import traceback
traceback.print_exc()
pass
#if not isrecarray:
# if type(data) == np.recarray:
# isrecarray = True
if fitsio and not isrecarray:
# fitsio sorts the rows and de-duplicates them, so compute
# permutation vector 'I' to undo that.
I = None
if rows is not None:
rows, I = np.unique(rows, return_inverse=True)
if type(data) == np.ndarray:
dd = data
if columns is None:
columns = data.dtype.fields.keys()
else:
if data.get_exttype() == 'IMAGE_HDU':
# This can happen on empty tables (eg, empty SDSS photoObjs)
return None
try:
dd = data.read(rows=rows, columns=columns, lower=True)
except:
import sys
print('Error reading from FITS object',
type(data),
data,
'dataorfn',
dataorfn,
file=sys.stderr)
raise
if dd is None:
return None
if columns is None:
try:
columns = data.get_colnames()
except:
columns = data.colnames
if lower:
columns = [c.lower() for c in columns]
for c in columns:
X = dd[c.lower()]
if I is not None:
# apply permutation
X = X[I]
if column_map is not None:
c = column_map.get(c, c)
if lower:
c = c.lower()
T.set(c, X)
else:
if columns is None:
columns = data.dtype.names
for c in columns:
col = data.field(c)
if rows is not None:
col = col[rows]
if normalize:
col = normalize_column(col)
if column_map is not None:
c = column_map.get(c, c)
if lower:
c = c.lower()
T.set(c, col)
return T
def produce_forests(self):
"""
"""
userprint("\n")
nside = 8
### Load DRQ
vac = fitsio.FITS(self._branchFiles + "/Products/cat.fits")
ra = vac[1]["RA"][:] * np.pi / 180.
dec = vac[1]["DEC"][:] * np.pi / 180.
thid = vac[1]["THING_ID"][:]
plate = vac[1]["PLATE"][:]
mjd = vac[1]["MJD"][:]
fiberid = vac[1]["FIBERID"][:]
vac.close()
### Get Healpy pixels
pixs = healpy.ang2pix(nside, np.pi / 2. - dec, ra)
### Save master file
path = self._branchFiles + "/Products/Spectra/master.fits"
head = {}
head['NSIDE'] = nside
cols = [thid, pixs, plate, mjd, fiberid]
names = ['THING_ID', 'PIX', 'PLATE', 'MJD', 'FIBER']
out = fitsio.FITS(path, 'rw', clobber=True)
out.write(cols, names=names, header=head, extname="MASTER TABLE")
out.close()
### Log lambda grid
logl_min = 3.550
logl_max = 4.025
logl_step = 1.e-4
log_lambda = np.arange(logl_min, logl_max, logl_step)
###
for p in np.unique(pixs):
###
p_thid = thid[(pixs == p)]
p_fl = numpy.random.normal(loc=1.,
scale=1.,
size=(log_lambda.size, p_thid.size))
p_iv = numpy.random.lognormal(mean=0.1,
sigma=0.1,
size=(log_lambda.size, p_thid.size))
p_am = np.zeros((log_lambda.size, p_thid.size)).astype(int)
p_am[numpy.random.random(size=(log_lambda.size,
p_thid.size)) > 0.90] = 1
p_om = np.zeros((log_lambda.size, p_thid.size)).astype(int)
###
p_path = self._branchFiles + "/Products/Spectra/pix_" + str(
p) + ".fits"
out = fitsio.FITS(p_path, 'rw', clobber=True)
out.write(p_thid, header={}, extname="THING_ID_MAP")
out.write(log_lambda, header={}, extname="LOGLAM_MAP")
out.write(p_fl, header={}, extname="FLUX")
out.write(p_iv, header={}, extname="IVAR")
out.write(p_am, header={}, extname="ANDMASK")
out.write(p_om, header={}, extname="ORMASK")
out.close()
return
#- make sure an import can't accidentally trigger this
if __name__ == "__main__":
import os, sys
import numpy as np
import fitsio
truthfile = "/global/cfs/cdirs/desi/datachallenge/reference_runs/20.4/targets/52/5299/dark/truth-dark-64-5299.fits"
targetfile = "/global/cfs/cdirs/desi/datachallenge/reference_runs/20.4/targets/52/5299/dark/targets-dark-64-5299.fits"
if not os.path.exists(truthfile):
print(f"ERROR: Unable to find {truthfile}")
print("you must run this script at NERSC")
sys.exit(1)
truth_data = dict()
with fitsio.FITS(truthfile) as fx:
keep_targetids = list()
for objtype in ['BGS', 'ELG', 'LRG', 'QSO', 'STAR', 'WD']:
extname = 'TRUTH_' + objtype
truth = fx[extname].read()[0:3] #- keep 3 targets per class
keep_targetids.extend(truth['TARGETID'])
truth_data[extname] = truth
truth_hdr = fx['TRUTH'].read_header()
truth = fx['TRUTH'].read()
flux = fx['FLUX'].read()
wave = fx['WAVE'].read()
hdr = fx['TRUTH'].read_header()
keep = np.in1d(truth['TARGETID'], keep_targetids)
import pylab as plt
import numpy as np
import sys
import os
import fitsio
import optparse
parser = optparse.OptionParser()
#parser.add_option('--prefix', help='Plot prefix', default='edge')
opt,args = parser.parse_args()
for fn in args:
#'c4d_131028_014102_ooi_r_v1.fits.fz')
print 'Reading', fn
F = fitsio.FITS(fn)
mfn = fn.replace('_ooi_','_ood_')
print 'Reading', mfn
M = fitsio.FITS(mfn)
print len(F), 'hdus'
hdr = F[0].read_header()
name = os.path.basename(fn)
name = name.replace('.fits','').replace('.fz','')
expnum = int(hdr['EXPNUM'])
print 'Exposure number', expnum
tt = 'Exp %i, %s' % (expnum, name)
textargs = dict(ha='left', va='center', fontsize=8)
for fig in [1,2,3,4]:
def __init__(self, band,
filename = None,
fits_file_template = None,
timg = None,
exposure_num = 0,
calib = None,
gain = None,
darkvar = None,
sky = None,
frame = None,
fits_table = None):
self.band = band
if fits_file_template:
self.band_file = fits_file_template%band
self.img = fitsio.FITS(self.band_file)[exposure_num].read()
header = fitsio.read_header(self.band_file, ext=exposure_num)
elif filename is not None:
self.band_file = filename
self.img = fitsio.FITS(self.band_file)[exposure_num].read()
header = fitsio.read_header(self.band_file, ext=exposure_num)
elif timg:
self.band_file = None
self.img = timg[0].getImage()
header = timg[1]['hdr']
self.timg = timg[0]
self.invvar = self.timg.getInvvar()
else:
pass
self.header = header
self.frame = frame
self.fits_table = fits_table
# Compute the number of electrons, resource:
# http://data.sdss3.org/datamodel/files/BOSS_PHOTOOBJ/frames/RERUN/RUN/CAMCOL/frame.html
# (Neither of these look like integers)
if fits_file_template or filename:
self.dn = self.img / header["CALIB"] + header["SKY"]
self.nelec = np.round(self.dn * header["GAIN"])
else:
# TODO(awu): what are CALIB and GAIN?
self.dn = self.img / calib + sky #timg[0].getSky().val
self.nelec = np.round(self.dn * gain)
# make nelec immutable - it is constant data!!
self.nelec.flags.writeable = False
self.shape = self.nelec.shape
self.pixel_grid = self.make_pixel_grid() # keep pixel grid around
# reference points
# TODO: Does CRPIX1 refer to the first axis of self.img ??
self.rho_n = np.array([header['CRPIX1'], header['CRPIX2']]) - 1 # PIXEL REFERENCE POINT (fits stores it 1-based indexing)
self.phi_n = np.array([header['CRVAL1'], header['CRVAL2']]) # EQUA REFERENCE POINT
self.Ups_n = np.array([[header['CD1_1'], header['CD1_2']], # MATRIX takes you into EQUA TANGENT PLANE
[header['CD2_1'], header['CD2_2']]])
self.Ups_n_inv = np.linalg.inv(self.Ups_n)
#astrometry wcs object for "exact" x,y to equa ra,dec conversion
import astropy.wcs as wcs
self.wcs = wcs.WCS(self.header)
self.use_wcs = False
# set image specific KAPPA and epsilon
if fits_file_template:
self.kappa = header['GAIN'] # TODO is this right??
self.epsilon = header['SKY'] * self.kappa # background rate
self.epsilon0 = self.epsilon # background rate copy (for debuggin)
self.darkvar = header['DARKVAR'] # also eventually contributes to mean?
self.calib = header['CALIB'] # dn = nmaggies / calib, calib is NMGY
else:
self.kappa = gain
self.epsilon = timg[0].sky.val * self.kappa
self.epsilon0 = self.epsilon
self.darkvar = darkvar
self.calib = calib
# point spread function
if fits_file_template:
psfvec = [header['PSF_P%d'%i] for i in range(18)]
else:
psfvec = [psf for psf in timg[0].getPsf()]
self.weights = np.array(psfvec[0:3])
self.means = np.array(psfvec[3:9]).reshape(3, 2) # one comp mean per row
covars = np.array(psfvec[9:]).reshape(3, 3) # [var_k(x), var_k(y), cov_k(x,y)] per row
self.covars = np.zeros((3, 2, 2))
self.invcovars = np.zeros((3, 2, 2))
self.logdets = np.zeros(3)
for i in range(3):
self.covars[i,:,:] = np.array([[ covars[i,0], covars[i,2]],
[ covars[i,2], covars[i,1]]])
# cache inverse covariance
self.invcovars[i,:,:] = np.linalg.inv(self.covars[i,:,:])
# cache log determinant
sign, logdet = np.linalg.slogdet(self.covars[i,:,:])
self.logdets[i] = logdet
self.psf_mog = MixtureOfGaussians(means = self.means, covs = self.covars, pis = self.weights)
# for a point source in this image, calculate the radius such that
# at least 99% of photons from that source will fall within
ERROR = 0.001
self.R = calc_bounding_radius(self.weights,
self.means,
self.covars,
ERROR)
def compute_inertia_tensors_projected(snap,
reduced=False,
inclusion_threshold=1):
""" Compute the intertia tensors for all subhalos for
the dark matter and stellar components and saves the output for
later integration within the database
"""
print 'Using reduced (distance weighted) tensors', 'yes' * int(
reduced), 'no' * int(np.invert(reduced))
# Read the subhalo information
h = snap.readsubhalo()
# Load the positions and masses of the constituent particles
print 'Loading dark matter particles'
x = snap.load(1, 'pos', h)
m = snap.load(1, 'mass', h)
print 'Loading baryon particles'
xb = snap.load(4, 'pos', h)
mb = snap.load(4, 'mass', h)
eigvectors = np.zeros((2, len(h), 2, 2))
eigvalues = np.zeros((2, len(h), 2))
centroids = np.zeros((2, len(h), 3))
spher_pos = np.zeros((2, len(h), 3))
length = np.zeros((2, len(h)))
# Will compute for each halo the inertia tensor
for i in range(len(h)):
if i % 100 == 0:
print "Done %d samples" % i
# Reject subhalos with less than some threshold occupation number
if len(x[i]) < inclusion_threshold:
pass
else:
# Decide whether to weight by the particle mass
weights = np.ones(len(x[i]))
normFactor = np.double(len(x[i].T[0]))
x0 = x[i].T[0] - np.dot(x[i].T[0], weights) / normFactor
x1 = x[i].T[1] - np.dot(x[i].T[1], weights) / normFactor
# Beware the ambiguous notation here -
# These weights define the difference between reduced and standard inertia tensors.
# i.e. downweighting particles at the fringes of the subhalo
# They have nothing to do with whether the mass weighting is applied.
if reduced:
wt = (x0 * x0 + x1 * x1)
select = (wt != 0)
else:
wt = np.ones(x0.size)
select = wt.astype(bool)
normFactor = np.double(len(x[i].T[0][select]))
tens = np.zeros((2, 2))
tens[0, 0] = np.dot(weights[select] * x0[select],
x0[select] / wt[select]) / normFactor
tens[1, 1] = np.dot(weights[select] * x1[select],
x1[select] / wt[select]) / normFactor
tens[1, 0] = np.dot(weights[select] * x1[select],
x0[select] / wt[select]) / normFactor
tens[0, 1] = tens[1, 0]
# Evaluate the mass-weighted centroid along each axis
X = np.trapz(m[i] * x[i].T[0], x[i].T[0]) / np.trapz(
m[i], x[i].T[0])
Y = np.trapz(m[i] * x[i].T[1], x[i].T[1]) / np.trapz(
m[i], x[i].T[1])
Z = np.trapz(m[i] * x[i].T[2], x[i].T[2]) / np.trapz(
m[i], x[i].T[2])
phi = np.arctan2(Y, X)
r = np.sqrt(X * X + Y * Y + Z * Z)
theta = np.arccos(Z / r)
# Compute the eigenvalues of the halos and store the outputs
w, v = np.linalg.eigh(tens)
eigvalues[0, i] = w
eigvectors[0, i] = v
spher_pos[0, i] = np.array([r, theta, phi])
centroids[0, i] = np.array([X, Y, Z])
length[0, i] = len(x[i])
if (len(xb[i]) < inclusion_threshold):
pass
else:
weights = np.ones(len(xb[i]))
normFactor = np.double(len(xb[i].T[0]))
x0 = xb[i].T[0] - np.dot(xb[i].T[0], weights) / normFactor
x1 = xb[i].T[1] - np.dot(xb[i].T[1], weights) / normFactor
if reduced:
wt = (x0 * x0 + x1 * x1)
select = (wt != 0)
else:
wt = np.ones(x0.size)
select = wt.astype(bool)
normFactor = np.double(len(xb[i].T[0][select]))
tens = np.zeros((2, 2))
tens[0, 0] = np.dot(weights[select] * x0[select],
x0[select] / wt[select]) / normFactor
tens[1, 1] = np.dot(weights[select] * x1[select],
x1[select] / wt[select]) / normFactor
tens[1, 0] = np.dot(weights[select] * x1[select],
x0[select] / wt[select]) / normFactor
tens[0, 1] = tens[1, 0]
# Evaluate the mass-weighted centroid along each axis
X = np.trapz(mb[i] * xb[i].T[0], xb[i].T[0]) / np.trapz(
mb[i], xb[i].T[0])
Y = np.trapz(mb[i] * xb[i].T[1], xb[i].T[1]) / np.trapz(
mb[i], xb[i].T[1])
Z = np.trapz(mb[i] * xb[i].T[2], xb[i].T[2]) / np.trapz(
mb[i], xb[i].T[2])
phi = np.arctan2(Y, X)
r = np.sqrt(X * X + Y * Y + Z * Z)
theta = np.arccos(Z / r)
# Compute the eigenvalues of the halos
w, v = np.linalg.eigh(tens)
eigvalues[1, i] = w
eigvectors[1, i] = v
spher_pos[1, i] = np.array([r, theta, phi])
centroids[1, i] = np.array([X, Y, Z])
length[1, i] = len(xb[i])
print "Saving output"
if reduced:
out = fi.FITS(
'/home/ssamurof/massive_black_ii/subhalo_cat_reduced_projected-nthreshold%d.fits'
% inclusion_threshold, 'rw')
else:
out = fi.FITS(
'/home/ssamurof/massive_black_ii/subhalo_cat_projected-nthreshold%d.fits'
% inclusion_threshold, 'rw')
dat = np.zeros(len(h),
dtype=[('x', float), ('y', float), ('z', float),
('npart', float), ('lambda1', float),
('lambda2', float), ('a1', float), ('a2', float),
('b1', float), ('b2', float)])
dat['lambda1'] = eigvalues[0].T[0]
dat['lambda2'] = eigvalues[0].T[1]
dat['a1'] = eigvectors[0].T[0, 0]
dat['a2'] = eigvectors[0].T[0, 1]
dat['b1'] = eigvectors[0].T[1, 0]
dat['b2'] = eigvectors[0].T[1, 1]
dat['x'] = centroids[0].T[0]
dat['y'] = centroids[0].T[1]
dat['z'] = centroids[0].T[2]
dat['npart'] = length[0]
out.write(dat)
out[-1].write_key('EXTNAME', 'dm')
dat2 = np.zeros(len(h),
dtype=[('x', float), ('y', float), ('z', float),
('npart', float), ('lambda1', float),
('lambda2', float), ('a1', float), ('a2', float),
('b1', float), ('b2', float)])
dat2['lambda1'] = eigvalues[1].T[0]
dat2['lambda2'] = eigvalues[1].T[1]
dat2['a1'] = eigvectors[1].T[0, 0]
dat2['a2'] = eigvectors[1].T[0, 1]
dat2['b1'] = eigvectors[1].T[1, 0]
dat2['b2'] = eigvectors[1].T[1, 1]
dat2['x'] = centroids[1].T[0]
dat2['y'] = centroids[1].T[1]
dat2['z'] = centroids[1].T[2]
dat2['npart'] = length[1]
out.write(dat2)
out[-1].write_key('EXTNAME', 'baryons')
out.close()
def get_column_names(fname):
with fitsio.FITS(fname) as infile:
hdu = infile[1]
return hdu.get_colnames()
def __init__(self,
spectrum_file,
metadata,
sky_mask,
mask_jpas=False,
mask_jpas_alt=False,
rebin_pixels_width=0,
extend_pixels=0,
noise_increase=1,
forbidden_wavelenghts=None):
""" Initialize class instance
Parameters
----------
spectrum_file : str
Name of the fits files containing the spectrum
metadata : dict
A dictionary with the metadata. Keys should be strings
sky_mask : (np.array, float)
A tuple containing the array of the wavelengths to mask and the margin
used in the masking. Wavelengths separated to wavelength given in the array
by less than the margin will be masked
mask_jpas : bool - Default: False
If set, mask pixels corresponding to filters in trays T3 and T4. Only works if
the bin size is 100 Angstroms
mask_jpas_alt : bool - Default: False
If set, mask pixels corresponding to filters in trays T3* and T4. Only works if
the bin size is 100 Angstroms
rebin_pixels_width : float, >0 - Default: 0
Width of the new pixel (in Angstroms)
extend_pixels : float, >0 - Default: 0
Pixel overlap region (in Angstroms)
noise_increase : int, >0 - Default: 1
Adds noise to the spectrum by adding a gaussian random number of width
equal to the (noise_amount-1) times the given variance. Then increase the
variance by a factor of sqrt(noise_amount)
forbidden_wavelengths : list of tuples or None - Default: None
If not None, a list containing tuples specifying ranges of wavelengths that will
be masked (both ends included). Each tuple must contain the initial and final range
of wavelenghts. This is intended to be complementary to the sky mask to limit the
wavelength coverage, and hard cuts will be applied
"""
# check that "specid" is present in metadata
if "SPECID" not in metadata.keys():
raise Error("""The property "SPECID" must be present in metadata""")
# open fits file
spectrum_hdul = fitsio.FITS(spectrum_file)
# intialize arrays
# The 1.0 mulitplying is added to change type from >4f to np.float
# this is required by numba later on
wave = 10**spectrum_hdul[1]["LOGLAM"][:]
flux = 1.0 * spectrum_hdul[1]["FLUX"][:]
ivar = 1.0 * spectrum_hdul[1]["IVAR"][:]
super().__init__(flux, ivar, wave, metadata)
# compute sky mask
masklambda = sky_mask[0]
margin = sky_mask[1]
self.__skymask = None
self.__find_skymask(masklambda, margin)
# mask forbidden lines
if forbidden_wavelenghts is not None:
self.__filter_wavelengths(forbidden_wavelenghts)
# store the wavelength, flux and inverse variance as arrays
# mask pixels
self._ivar[self.__skymask] = 0.0
if noise_increase > 1:
self.__add_noise(noise_increase)
if rebin_pixels_width > 0:
self._flux, self._ivar, self._wave = self.rebin(
rebin_pixels_width, extend_pixels=extend_pixels)
# JPAS mask
if mask_jpas:
pos = np.where(~((np.isin(
self._wave, [3900, 4000, 4300, 4400, 4700, 4800, 5100, 5200])) |
(self._wave >= 7300)))
self._wave = self._wave[pos].copy()
self._ivar = self._ivar[pos].copy()
self._flux = self._flux[pos].copy()
elif mask_jpas_alt:
pos = np.where(~((np.isin(
self._wave, [3800, 4000, 4200, 4400, 4600, 4800, 5000, 5200])) |
(self._wave >= 7300)))
self._wave = self._wave[pos].copy()
self._ivar = self._ivar[pos].copy()
self._flux = self._flux[pos].copy()
spectrum_hdul.close()
parser.add_argument("--out", type=str, required=True, help="output file")
parser.add_argument("--no-dmat",
action='store_true',
default=False,
required=False,
help='Use an identity matrix as the distortion matrix.')
args = parser.parse_args()
for f in args.data:
if not os.path.isfile(f):
args.data.remove(f)
h = fitsio.FITS(args.data[0])
head = h[1].read_header()
rp = h[1]['RP'][:] * 0
rt = h[1]['RT'][:] * 0
nb = h[1]['NB'][:] * 0
z = h[1]['Z'][:] * 0
wet = rp * 0
h.close()
da = {}
we = {}
if not args.no_dmat:
h = fitsio.FITS(args.data[0].replace('cf', 'dmat'))
dm = h[1]['DM'][:] * 0
def read_spec_spplate(p,
m,
fiber=None,
path_spec=None,
lambda_min=None,
lambda_max=None,
cutANDMASK=True,
veto_lines=None,
flux_calib=None,
ivar_calib=None):
"""
"""
path = path_spec + '/{}/spPlate-{}-{}.fits'.format(
str(p).zfill(4),
str(p).zfill(4), m)
h = fitsio.FITS(path)
fl = h[0].read()
iv = h[1].read()
iv *= iv > 0.
an = h[2].read()
head = h[0].read_header()
h.close()
if cutANDMASK:
iv *= an == 0
ll = head['CRVAL1'] + head['CD1_1'] * sp.arange(head['NAXIS1'])
if head['DC-FLAG']:
ll = 10**ll
w = sp.ones(ll.size, dtype=bool)
if not lambda_min is None:
w &= ll >= lambda_min
if not lambda_max is None:
w &= ll <= lambda_max
if not veto_lines is None:
for lmin, lmax in veto_lines:
w &= (ll < lmin) | (ll > lmax)
ll = ll[w]
fl = fl[:, w]
iv = iv[:, w]
if not flux_calib is None:
correction = flux_calib(ll)
fl /= correction[None, :]
iv *= correction[None, :]**2
if not ivar_calib is None:
correction = ivar_calib(ll)
iv /= correction[None, :]
if not fiber is None:
w = iv[fiber - 1] > 0.
ll = ll[w]
fl = fl[fiber - 1, w]
iv = iv[fiber - 1, w]
return ll, fl, iv
y_size = 990 # mm
plane_limits = [-x_size, x_size, -y_size, y_size]
extent = [
-detector_size / 2 + plane_limits[0], detector_size / 2 + plane_limits[1],
-detector_size / 2 + plane_limits[2], detector_size / 2 + plane_limits[3]
]
data = np.loadtxt('relative.out', delimiter=' ')
dimension = data.shape
x = np.linspace(plane_limits[0], plane_limits[1], dimension[1])
y = np.linspace(plane_limits[2], plane_limits[3], dimension[0])
distance = 5.6 * 1e3
# File with the mean template of all events (53551 events)
pixels_waveforms = '/Users/lonewolf/Desktop/test_output.fits'
with fitsio.FITS(pixels_waveforms, 'r') as file:
#print(file['PULSE_TEMPLATE'])
time = file['PULSE_TEMPLATE']['time'].read()
templates = file['PULSE_TEMPLATE']['original_amplitude'].read()
templates = templates.T
maximum_array = []
charge_array = []
for pixel in range(0, 1296):
arg_max = np.argmax(templates[pixel])
max_in_template = np.max(templates[pixel])
charge = np.sum((templates[pixel])[arg_max - 3:arg_max + 4])
def read_cat(pathData,
zmin=None,
zmax=None,
zkey='Z_VI',
extinction=True,
stack_obs=False,
in_dir=None,
nspec=None,
rvextinction=3.793,
nside=None):
"""
"""
dic = {}
h = fitsio.FITS(pathData)
if 'PLATE' in h[1].get_colnames():
if 'MJD' in h[1].get_colnames():
lst = {
'PLATE': 'PLATE',
'MJD': 'MJD',
'FIBERID': 'FIBERID',
'THING_ID': 'THING_ID',
'Z': zkey
}
else:
lst = {
'PLATE': 'PLATE',
'MJD': 'SMJD',
'FIBERID': 'FIBER',
'THING_ID': 'BESTID',
'Z': zkey
}
for k, v in lst.items():
dic[k] = h[1][v][:]
else:
p, m, f = targetid2platemjdfiber(h[1]['TARGETID'][:])
dic['PLATE'] = p
dic['MJD'] = m
dic['FIBERID'] = f
dic['Z'] = h[1][zkey][:]
dic['THING_ID'] = -1 * sp.ones(p.size, dtype=sp.int64)
if extinction:
dic['G_EXTINCTION'] = h[1]['EXTINCTION'][:][:, 1]
w = dic['G_EXTINCTION'] <= 0.
if w.sum() != 0.:
print('WARNING: some G_EXTINCTION<=0.: {}, {}'.format(
w.sum(), sp.unique(dic['G_EXTINCTION'][w])))
dic['G_EXTINCTION'][w] = 0.
dic['G_EXTINCTION'] /= rvextinction
h.close()
dic['TARGETID'] = platemjdfiber2targetid(dic['PLATE'].astype('int64'),
dic['MJD'].astype('int64'),
dic['FIBERID'].astype('int64'))
w = sp.argsort(dic['TARGETID'])
for k in dic.keys():
dic[k] = dic[k][w]
w = sp.ones(dic['Z'].size, dtype=bool)
print('Found {} quasars'.format(w.sum()))
w &= dic['Z'] != -1.
print('z!=-1: {}'.format(w.sum()))
if not zmin is None:
w &= dic['Z'] > zmin
print('z>zmin: {}'.format(w.sum()))
if not zmax is None:
w &= dic['Z'] < zmax
print('z<zmax: {}'.format(w.sum()))
for k in dic.keys():
dic[k] = dic[k][w]
print('Found {} quasars'.format(dic['Z'].size))
if not nspec is None and nspec < dic['Z'].size:
for k in dic.keys():
dic[k] = dic[k][:nspec]
print('Limit to {} quasars'.format(dic['Z'].size))
if stack_obs:
spall = glob.glob(os.path.expandvars(in_dir + '/spAll-*.fits'))
if len(spall) != 1:
print('WARNING: found {} spAll'.format(len(spall)))
dic['ALLOBS'] = [[t] for t in dic['TARGETID']]
return dic
h = fitsio.FITS(spall[0])
print('INFO: reading spAll from {}'.format(spall[0]))
thid_spall = h[1][lst['THING_ID']][:]
plate_spall = h[1]['PLATE'][:]
mjd_spall = h[1]['MJD'][:]
fid_spall = h[1]['FIBERID'][:]
qual_spall = sp.char.strip(h[1]['PLATEQUALITY'][:].astype(str))
zwarn_spall = h[1]['ZWARNING'][:]
h.close()
w = dic['THING_ID'] > 0
for k in dic.keys():
dic[k] = dic[k][w]
_, w = sp.unique(dic['THING_ID'], return_index=True)
for k in dic.keys():
dic[k] = dic[k][w]
print('Get unique THING_ID: {}'.format(dic['Z'].size))
w = sp.argsort(dic['THING_ID'])
for k in dic.keys():
dic[k] = dic[k][w]
w = sp.in1d(thid_spall, dic['THING_ID'])
print("INFO: Found {} spectra with required THING_ID".format(w.sum()))
w &= qual_spall == 'good'
print("INFO: Found {} spectra with 'good' plate".format(w.sum()))
## Removing spectra with the following ZWARNING bits set:
## SKY, LITTLE_COVERAGE, UNPLUGGED, BAD_TARGET, NODATA
## https://www.sdss.org/dr14/algorithms/bitmasks/#ZWARNING
bad_zwarnbit = {
0: 'SKY',
1: 'LITTLE_COVERAGE',
7: 'UNPLUGGED',
8: 'BAD_TARGET',
9: 'NODATA'
}
for zwarnbit, zwarnbit_str in bad_zwarnbit.items():
w &= (zwarn_spall & 2**zwarnbit) == 0
print("INFO: Found {} spectra without {} bit set: {}".format(
w.sum(), zwarnbit, zwarnbit_str))
thid = thid_spall[w]
plate = plate_spall[w]
mjd = mjd_spall[w]
fid = fid_spall[w]
targetid = platemjdfiber2targetid(plate.astype('int64'),
mjd.astype('int64'),
fid.astype('int64'))
print('INFO: # unique objs: ', dic['THING_ID'].size)
print('INFO: # unique objs in spAll: ', sp.unique(thid).size)
print('INFO: # spectra: ', w.sum())
if w.sum() == 0:
print('INFO: no spectra, exit')
sys.exit()
w = sp.argsort(thid)
thid = thid[w]
targetid = targetid[w]
dic['ALLOBS'] = [sp.sort(targetid[thid == t]) for t in dic['THING_ID']]
w = np.array([len(v) for v in dic['ALLOBS']]) == 0
if sp.any(w):
print('WARNING: Some objects have no valid observation')
for el in dic['THING_ID'][w]:
print('WARNING: {}'.format(el))
return dic
pool.close()
if match:
pres, mres, pres_ex, mres_ex = zip(*outputs)
pres, mres = cut_nones(pres, mres)
pres_ex, mres_ex = cut_nones(pres_ex, mres_ex)
if rank == 0:
dt = [('g1p', 'f8'), ('g1m', 'f8'), ('g1', 'f8'),
('g2p', 'f8'), ('g2m', 'f8'), ('g2', 'f8')]
dplus = np.array(pres, dtype=dt)
dminus = np.array(mres, dtype=dt)
dplus_ex = np.array(pres_ex, dtype=dt)
dminus_ex = np.array(mres_ex, dtype=dt)
with fitsio.FITS('data.fits', 'rw') as fits:
fits.write(dplus, extname='plus')
fits.write(dminus, extname='minus')
fits.write(dplus_ex, extname='plus_ex')
fits.write(dminus_ex, extname='minus_ex')
m, msd, c, csd = estimate_m_and_c(pres, mres, 0.02, swap12=SWAP12)
m_ex, msd_ex, c_ex, csd_ex = estimate_m_and_c(pres_ex, mres_ex,
0.02, swap12=SWAP12)
print("""\
# of sims: {n_sims}
noise cancel m : {m:f} +/- {msd:f}
noise cancel c : {c:f} +/- {csd:f}""".format(
n_sims=len(pres),
m=m,
def read_image_header(row, img_file):
"""Read some information from the image header and write into the df row.
"""
hdu = 1
# Note: The next line usually works, but fitsio doesn't support CONTINUE lines, which DES
# image headers sometimes include.
#h = fitsio.read_header(img_file, hdu)
# I don't care about any of the lines the sometimes use CONITNUE (e.g. OBSERVER), so I
# just remove them and make the header with the rest of the entries.
f = fitsio.FITS(img_file)
header_list = f[hdu].read_header_list()
header_list = [ d for d in header_list if 'CONTINUE' not in d['name'] ]
h = fitsio.FITSHDR(header_list)
try:
date = h['DATE-OBS']
date, time = date.strip().split('T',1)
filter = h['FILTER']
filter = filter.split()[0]
band = h['BAND']
sat = h['SATURATE']
fwhm = h['FWHM']
ccdnum = int(h['CCDNUM'])
detpos = h['DETPOS'].strip()
telra = h['TELRA']
teldec = h['TELDEC']
telha = h['HA']
telra = galsim.Angle.from_hms(telra) / galsim.degrees
teldec = galsim.Angle.from_dms(teldec) / galsim.degrees
telha = galsim.Angle.from_hms(telha) / galsim.degrees
dimmseeing = float(h.get('DIMMSEE',-999))
airmass = float(h.get('AIRMASS',-999))
pressure = float(h.get('PRESSURE',-999))
sky = float(h.get('SKYBRITE',-999))
sigsky = float(h.get('SKYSIGMA',-999))
humidity = float(h.get('HUMIDITY',-999))
windspd = float(h.get('WINDSPD',-999))
winddir = float(h.get('WINDDIR',-999))
tiling = int(h.get('TILING',0))
outtemp = float( h.get('OUTTEMP', - 999) )
msurtemp = float(h.get('MSURTEMP',-999))
dT = float(h.get('MSURTEMP',-999)) - float(h.get('OUTTEMP',0) )
hex = int(h.get('HEX',0))
except:
print("Cannot read header information from " + img_file)
raise
row['sat'] = sat
row['fits_filter'] = filter
row['fwhm'] = fwhm
row['fits_ccdnum'] = ccdnum
row['airmass'] = airmass
row['sky'] = sky
row['sigsky'] = sigsky
row['humidity'] = humidity
row['tiling'] = tiling
row['hex'] = hex
row['band'] = band
row['telra'] = telra
row['teldec'] = teldec
row['telha'] = telha
row['dimmseeing'] = dimmseeing
row['dT'] = dT
row['outtemp'] = outtemp
row['msurtemp'] = msurtemp
row['windspd'] = windspd
row['winddir'] = winddir
def main():
# pylint: disable-msg=too-many-locals,too-many-branches,too-many-statements
"""Computes delta field"""
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description=('Compute the delta field '
'from a list of spectra'))
parser.add_argument('--out-dir',
type=str,
default=None,
required=True,
help='Output directory')
parser.add_argument('--drq',
type=str,
default=None,
required=True,
help='Catalog of objects in DRQ format')
parser.add_argument('--in-dir',
type=str,
default=None,
required=True,
help='Directory to spectra files')
parser.add_argument('--log',
type=str,
default='input.log',
required=False,
help='Log input data')
parser.add_argument('--iter-out-prefix',
type=str,
default='iter',
required=False,
help='Prefix of the iteration file')
parser.add_argument('--mode',
type=str,
default='pix',
required=False,
help=('Open mode of the spectra files: pix, spec, '
'spcframe, spplate, desi'))
parser.add_argument('--best-obs',
action='store_true',
required=False,
help=('If mode == spcframe, then use only the best '
'observation'))
parser.add_argument(
'--single-exp',
action='store_true',
required=False,
help=('If mode == spcframe, then use only one of the '
'available exposures. If best-obs then choose it '
'among those contributing to the best obs'))
parser.add_argument('--zqso-min',
type=float,
default=None,
required=False,
help='Lower limit on quasar redshift from drq')
parser.add_argument('--zqso-max',
type=float,
default=None,
required=False,
help='Upper limit on quasar redshift from drq')
parser.add_argument('--keep-bal',
action='store_true',
required=False,
help='Do not reject BALs in drq')
parser.add_argument('--bi-max',
type=float,
required=False,
default=None,
help=('Maximum CIV balnicity index in drq (overrides '
'--keep-bal)'))
parser.add_argument('--lambda-min',
type=float,
default=3600.,
required=False,
help='Lower limit on observed wavelength [Angstrom]')
parser.add_argument('--lambda-max',
type=float,
default=5500.,
required=False,
help='Upper limit on observed wavelength [Angstrom]')
parser.add_argument('--lambda-rest-min',
type=float,
default=1040.,
required=False,
help='Lower limit on rest frame wavelength [Angstrom]')
parser.add_argument('--lambda-rest-max',
type=float,
default=1200.,
required=False,
help='Upper limit on rest frame wavelength [Angstrom]')
parser.add_argument('--rebin',
type=int,
default=3,
required=False,
help=('Rebin wavelength grid by combining this number '
'of adjacent pixels (ivar weight)'))
parser.add_argument('--npix-min',
type=int,
default=50,
required=False,
help='Minimum of rebined pixels')
parser.add_argument('--dla-vac',
type=str,
default=None,
required=False,
help='DLA catalog file')
parser.add_argument('--dla-mask',
type=float,
default=0.8,
required=False,
help=('Lower limit on the DLA transmission. '
'Transmissions below this number are masked'))
parser.add_argument('--absorber-vac',
type=str,
default=None,
required=False,
help='Absorber catalog file')
parser.add_argument(
'--absorber-mask',
type=float,
default=2.5,
required=False,
help=('Mask width on each side of the absorber central '
'observed wavelength in units of '
'1e4*dlog10(lambda)'))
parser.add_argument('--mask-file',
type=str,
default=None,
required=False,
help=('Path to file to mask regions in lambda_OBS and '
'lambda_RF. In file each line is: region_name '
'region_min region_max (OBS or RF) [Angstrom]'))
parser.add_argument('--optical-depth',
type=str,
default=None,
required=False,
nargs='*',
help=('Correct for the optical depth: tau_1 gamma_1 '
'absorber_1 tau_2 gamma_2 absorber_2 ...'))
parser.add_argument('--dust-map',
type=str,
default=None,
required=False,
help=('Path to DRQ catalog of objects for dust map to '
'apply the Schlegel correction'))
parser.add_argument(
'--flux-calib',
type=str,
default=None,
required=False,
help=('Path to previously produced picca_delta.py file '
'to correct for multiplicative errors in the '
'pipeline flux calibration'))
parser.add_argument(
'--ivar-calib',
type=str,
default=None,
required=False,
help=('Path to previously produced picca_delta.py file '
'to correct for multiplicative errors in the '
'pipeline inverse variance calibration'))
parser.add_argument('--eta-min',
type=float,
default=0.5,
required=False,
help='Lower limit for eta')
parser.add_argument('--eta-max',
type=float,
default=1.5,
required=False,
help='Upper limit for eta')
parser.add_argument('--vlss-min',
type=float,
default=0.,
required=False,
help='Lower limit for variance LSS')
parser.add_argument('--vlss-max',
type=float,
default=0.3,
required=False,
help='Upper limit for variance LSS')
parser.add_argument('--delta-format',
type=str,
default=None,
required=False,
help='Format for Pk 1D: Pk1D')
parser.add_argument('--use-ivar-as-weight',
action='store_true',
default=False,
help=('Use ivar as weights (implemented as eta = 1, '
'sigma_lss = fudge = 0)'))
parser.add_argument('--use-constant-weight',
action='store_true',
default=False,
help=('Set all the delta weights to one (implemented '
'as eta = 0, sigma_lss = 1, fudge = 0)'))
parser.add_argument('--order',
type=int,
default=1,
required=False,
help=('Order of the log10(lambda) polynomial for the '
'continuum fit, by default 1.'))
parser.add_argument('--nit',
type=int,
default=5,
required=False,
help=('Number of iterations to determine the mean '
'continuum shape, LSS variances, etc.'))
parser.add_argument('--nproc',
type=int,
default=None,
required=False,
help='Number of processors')
parser.add_argument('--nspec',
type=int,
default=None,
required=False,
help='Maximum number of spectra to read')
parser.add_argument('--use-mock-continuum',
action='store_true',
default=False,
help='use the mock continuum for computing the deltas')
parser.add_argument('--spall',
type=str,
default=None,
required=False,
help=('Path to spAll file'))
parser.add_argument('--metadata',
type=str,
default=None,
required=False,
help=('Name for table containing forests metadata'))
t0 = time.time()
args = parser.parse_args()
# setup forest class variables
Forest.log_lambda_min = np.log10(args.lambda_min)
Forest.log_lambda_max = np.log10(args.lambda_max)
Forest.log_lambda_min_rest_frame = np.log10(args.lambda_rest_min)
Forest.log_lambda_max_rest_frame = np.log10(args.lambda_rest_max)
Forest.rebin = args.rebin
Forest.delta_log_lambda = args.rebin * 1e-4
# minumum dla transmission
Forest.dla_mask_limit = args.dla_mask
Forest.absorber_mask_width = args.absorber_mask
# Find the redshift range
if args.zqso_min is None:
args.zqso_min = max(0., args.lambda_min / args.lambda_rest_max - 1.)
userprint("zqso_min = {}".format(args.zqso_min))
if args.zqso_max is None:
args.zqso_max = max(0., args.lambda_max / args.lambda_rest_min - 1.)
userprint("zqso_max = {}".format(args.zqso_max))
#-- Create interpolators for mean quantities, such as
#-- Large-scale structure variance : var_lss
#-- Pipeline ivar correction error: eta
#-- Pipeline ivar correction term : fudge
#-- Mean continuum : mean_cont
log_lambda_temp = (Forest.log_lambda_min + np.arange(2) *
(Forest.log_lambda_max - Forest.log_lambda_min))
log_lambda_rest_frame_temp = (
Forest.log_lambda_min_rest_frame + np.arange(2) *
(Forest.log_lambda_max_rest_frame - Forest.log_lambda_min_rest_frame))
Forest.get_var_lss = interp1d(log_lambda_temp,
0.2 + np.zeros(2),
fill_value="extrapolate",
kind="nearest")
Forest.get_eta = interp1d(log_lambda_temp,
np.ones(2),
fill_value="extrapolate",
kind="nearest")
Forest.get_fudge = interp1d(log_lambda_temp,
np.zeros(2),
fill_value="extrapolate",
kind="nearest")
Forest.get_mean_cont = interp1d(log_lambda_rest_frame_temp,
1 + np.zeros(2))
#-- Check that the order of the continuum fit is 0 (constant) or 1 (linear).
if args.order:
if (args.order != 0) and (args.order != 1):
userprint(("ERROR : invalid value for order, must be eqal to 0 or"
"1. Here order = {:d}").format(args.order))
sys.exit(12)
#-- Correct multiplicative pipeline flux calibration
if args.flux_calib is not None:
hdu = fitsio.read(args.flux_calib, ext=1)
stack_log_lambda = hdu['loglam']
stack_delta = hdu['stack']
w = (stack_delta != 0.)
Forest.correct_flux = interp1d(stack_log_lambda[w],
stack_delta[w],
fill_value="extrapolate",
kind="nearest")
#-- Correct multiplicative pipeline inverse variance calibration
if args.ivar_calib is not None:
hdu = fitsio.read(args.ivar_calib, ext=2)
log_lambda = hdu['loglam']
eta = hdu['eta']
Forest.correct_ivar = interp1d(log_lambda,
eta,
fill_value="extrapolate",
kind="nearest")
### Apply dust correction
if not args.dust_map is None:
userprint("applying dust correction")
Forest.extinction_bv_map = io.read_dust_map(args.dust_map)
log_file = open(os.path.expandvars(args.log), 'w')
# Read data
(data, num_data, nside,
healpy_pix_ordering) = io.read_data(os.path.expandvars(args.in_dir),
args.drq,
args.mode,
z_min=args.zqso_min,
z_max=args.zqso_max,
max_num_spec=args.nspec,
log_file=log_file,
keep_bal=args.keep_bal,
bi_max=args.bi_max,
best_obs=args.best_obs,
single_exp=args.single_exp,
pk1d=args.delta_format,
spall=args.spall)
#-- Add order info
for pix in data:
for forest in data[pix]:
if not forest is None:
forest.order = args.order
### Read masks
if args.mask_file is not None:
args.mask_file = os.path.expandvars(args.mask_file)
try:
mask = Table.read(args.mask_file,
names=('type', 'wave_min', 'wave_max', 'frame'),
format='ascii')
mask['log_wave_min'] = np.log10(mask['wave_min'])
mask['log_wave_max'] = np.log10(mask['wave_max'])
except (OSError, ValueError):
userprint(("ERROR: Error while reading mask_file "
"file {}").format(args.mask_file))
sys.exit(1)
else:
mask = Table(names=('type', 'wave_min', 'wave_max', 'frame',
'log_wave_min', 'log_wave_max'))
### Mask lines
for healpix in data:
for forest in data[healpix]:
forest.mask(mask)
### Mask absorbers
if not args.absorber_vac is None:
userprint("INFO: Adding absorbers")
absorbers = io.read_absorbers(args.absorber_vac)
num_absorbers = 0
for healpix in data:
for forest in data[healpix]:
if forest.thingid in absorbers:
for lambda_absorber in absorbers[forest.thingid]:
forest.add_absorber(lambda_absorber)
num_absorbers += 1
log_file.write("Found {} absorbers in forests\n".format(num_absorbers))
### Add optical depth contribution
if not args.optical_depth is None:
userprint(("INFO: Adding {} optical"
"depths").format(len(args.optical_depth) // 3))
assert len(args.optical_depth) % 3 == 0
for index in range(len(args.optical_depth) // 3):
tau = float(args.optical_depth[3 * index])
gamma = float(args.optical_depth[3 * index + 1])
lambda_rest_frame = constants.ABSORBER_IGM[args.optical_depth[
3 * index + 2]]
userprint(
("INFO: Adding optical depth for tau = {}, gamma = {}, "
"lambda_rest_frame = {} A").format(tau, gamma,
lambda_rest_frame))
for healpix in data:
for forest in data[healpix]:
forest.add_optical_depth(tau, gamma, lambda_rest_frame)
### Mask DLAs
if not args.dla_vac is None:
userprint("INFO: Adding DLAs")
np.random.seed(0)
dlas = io.read_dlas(args.dla_vac)
num_dlas = 0
for healpix in data:
for forest in data[healpix]:
if forest.thingid in dlas:
for dla in dlas[forest.thingid]:
forest.add_dla(dla[0], dla[1], mask)
num_dlas += 1
log_file.write("Found {} DLAs in forests\n".format(num_dlas))
## Apply cuts
log_file.write(
("INFO: Input sample has {} "
"forests\n").format(np.sum([len(forest)
for forest in data.values()])))
remove_keys = []
for healpix in data:
forests = []
for forest in data[healpix]:
if ((forest.log_lambda is None)
or len(forest.log_lambda) < args.npix_min):
log_file.write(("INFO: Rejected {} due to forest too "
"short\n").format(forest.thingid))
continue
if np.isnan((forest.flux * forest.ivar).sum()):
log_file.write(("INFO: Rejected {} due to nan "
"found\n").format(forest.thingid))
continue
if (args.use_constant_weight
and (forest.flux.mean() <= 0.0 or forest.mean_snr <= 1.0)):
log_file.write(("INFO: Rejected {} due to negative mean or "
"too low SNR found\n").format(forest.thingid))
continue
forests.append(forest)
log_file.write("{} {}-{}-{} accepted\n".format(
forest.thingid, forest.plate, forest.mjd, forest.fiberid))
data[healpix][:] = forests
if len(data[healpix]) == 0:
remove_keys += [healpix]
for healpix in remove_keys:
del data[healpix]
num_forests = np.sum([len(forest) for forest in data.values()])
log_file.write(("INFO: Remaining sample has {} "
"forests\n").format(num_forests))
userprint(f"Remaining sample has {num_forests} forests")
# Sanity check: all forests must have the attribute log_lambda
for healpix in data:
for forest in data[healpix]:
assert forest.log_lambda is not None
t1 = time.time()
tmin = (t1 - t0) / 60
userprint('INFO: time elapsed to read data', tmin, 'minutes')
# compute fits to the forests iteratively
# (see equations 2 to 4 in du Mas des Bourboux et al. 2020)
num_iterations = args.nit
for iteration in range(num_iterations):
context = multiprocessing.get_context('fork')
pool = context.Pool(processes=args.nproc)
userprint(
f"Continuum fitting: starting iteration {iteration} of {num_iterations}"
)
#-- Sorting healpix pixels before giving to pool (for some reason)
pixels = np.array([k for k in data])
sort = pixels.argsort()
sorted_data = [data[k] for k in pixels[sort]]
data_fit_cont = pool.map(cont_fit, sorted_data)
for index, healpix in enumerate(pixels[sort]):
data[healpix] = data_fit_cont[index]
userprint(
f"Continuum fitting: ending iteration {iteration} of {num_iterations}"
)
pool.close()
if iteration < num_iterations - 1:
#-- Compute mean continuum (stack in rest-frame)
(log_lambda_rest_frame, mean_cont,
mean_cont_weight) = prep_del.compute_mean_cont(data)
w = mean_cont_weight > 0.
log_lambda_cont = log_lambda_rest_frame[w]
new_cont = Forest.get_mean_cont(log_lambda_cont) * mean_cont[w]
Forest.get_mean_cont = interp1d(log_lambda_cont,
new_cont,
fill_value="extrapolate")
#-- Compute observer-frame mean quantities (var_lss, eta, fudge)
if not (args.use_ivar_as_weight or args.use_constant_weight):
(log_lambda, eta, var_lss, fudge, num_pixels, var_pipe_values,
var_delta, var2_delta, count, num_qso, chi2_in_bin, error_eta,
error_var_lss, error_fudge) = prep_del.compute_var_stats(
data, (args.eta_min, args.eta_max),
(args.vlss_min, args.vlss_max))
w = num_pixels > 0
Forest.get_eta = interp1d(log_lambda[w],
eta[w],
fill_value="extrapolate",
kind="nearest")
Forest.get_var_lss = interp1d(log_lambda[w],
var_lss[w],
fill_value="extrapolate",
kind="nearest")
Forest.get_fudge = interp1d(log_lambda[w],
fudge[w],
fill_value="extrapolate",
kind="nearest")
else:
num_bins = 10 # this value is arbitrary
log_lambda = (
Forest.log_lambda_min + (np.arange(num_bins) + .5) *
(Forest.log_lambda_max - Forest.log_lambda_min) / num_bins)
if args.use_ivar_as_weight:
userprint(("INFO: using ivar as weights, skipping eta, "
"var_lss, fudge fits"))
eta = np.ones(num_bins)
var_lss = np.zeros(num_bins)
fudge = np.zeros(num_bins)
else:
userprint(("INFO: using constant weights, skipping eta, "
"var_lss, fudge fits"))
eta = np.zeros(num_bins)
var_lss = np.ones(num_bins)
fudge = np.zeros(num_bins)
error_eta = np.zeros(num_bins)
error_var_lss = np.zeros(num_bins)
error_fudge = np.zeros(num_bins)
chi2_in_bin = np.zeros(num_bins)
num_pixels = np.zeros(num_bins)
var_pipe_values = np.zeros(num_bins)
var_delta = np.zeros((num_bins, num_bins))
var2_delta = np.zeros((num_bins, num_bins))
count = np.zeros((num_bins, num_bins))
num_qso = np.zeros((num_bins, num_bins))
Forest.get_eta = interp1d(log_lambda,
eta,
fill_value='extrapolate',
kind='nearest')
Forest.get_var_lss = interp1d(log_lambda,
var_lss,
fill_value='extrapolate',
kind='nearest')
Forest.get_fudge = interp1d(log_lambda,
fudge,
fill_value='extrapolate',
kind='nearest')
### Read metadata from forests and export it
if not args.metadata is None:
tab_cont = get_metadata(data)
tab_cont.write(args.metadata, format="fits", overwrite=True)
stack_log_lambda, stack_delta, stack_weight = prep_del.stack(data)
### Save iter_out_prefix
results = fitsio.FITS(args.iter_out_prefix + ".fits.gz",
'rw',
clobber=True)
header = {}
header["NSIDE"] = nside
header["PIXORDER"] = healpy_pix_ordering
header["FITORDER"] = args.order
results.write([stack_log_lambda, stack_delta, stack_weight],
names=['loglam', 'stack', 'weight'],
header=header,
extname='STACK')
results.write([log_lambda, eta, var_lss, fudge, num_pixels],
names=['loglam', 'eta', 'var_lss', 'fudge', 'nb_pixels'],
extname='WEIGHT')
results.write([
log_lambda_rest_frame,
Forest.get_mean_cont(log_lambda_rest_frame), mean_cont_weight
],
names=['loglam_rest', 'mean_cont', 'weight'],
extname='CONT')
var_pipe_values = np.broadcast_to(var_pipe_values.reshape(1, -1),
var_delta.shape)
results.write(
[var_pipe_values, var_delta, var2_delta, count, num_qso, chi2_in_bin],
names=['var_pipe', 'var_del', 'var2_del', 'count', 'nqsos', 'chi2'],
extname='VAR')
results.close()
### Compute deltas and format them
get_stack_delta = interp1d(stack_log_lambda[stack_weight > 0.],
stack_delta[stack_weight > 0.],
kind="nearest",
fill_value="extrapolate")
deltas = {}
data_bad_cont = []
for healpix in sorted(data.keys()):
for forest in data[healpix]:
if not forest.bad_cont is None:
continue
#-- Compute delta field from flux, continuum and various quantites
get_delta_from_forest(forest, get_stack_delta, Forest.get_var_lss,
Forest.get_eta, Forest.get_fudge,
args.use_mock_continuum)
if healpix in deltas:
deltas[healpix].append(forest)
else:
deltas[healpix] = [forest]
data_bad_cont = data_bad_cont + [
forest for forest in data[healpix] if forest.bad_cont is not None
]
for forest in data_bad_cont:
log_file.write("INFO: Rejected {} due to {}\n".format(
forest.thingid, forest.bad_cont))
log_file.write(
("INFO: Accepted sample has {}"
"forests\n").format(np.sum([len(p) for p in deltas.values()])))
t2 = time.time()
tmin = (t2 - t1) / 60
userprint('INFO: time elapsed to fit continuum', tmin, 'minutes')
### Save delta
for healpix in sorted(deltas.keys()):
if args.delta_format == 'Pk1D_ascii':
results = open(args.out_dir + "/delta-{}".format(healpix) + ".txt",
'w')
for delta in deltas[healpix]:
num_pixels = len(delta.delta)
if args.mode == 'desi':
delta_log_lambda = (
(delta.log_lambda[-1] - delta.log_lambda[0]) /
float(len(delta.log_lambda) - 1))
else:
delta_log_lambda = delta.delta_log_lambda
line = '{} {} {} '.format(delta.plate, delta.mjd,
delta.fiberid)
line += '{} {} {} '.format(delta.ra, delta.dec, delta.z_qso)
line += '{} {} {} {} {} '.format(delta.mean_z, delta.mean_snr,
delta.mean_reso,
delta_log_lambda, num_pixels)
for index in range(num_pixels):
line += '{} '.format(delta.delta[index])
for index in range(num_pixels):
line += '{} '.format(delta.log_lambda[index])
for index in range(num_pixels):
line += '{} '.format(delta.ivar[index])
for index in range(num_pixels):
line += '{} '.format(delta.exposures_diff[index])
line += ' \n'
results.write(line)
results.close()
else:
results = fitsio.FITS(args.out_dir + "/delta-{}".format(healpix) +
".fits.gz",
'rw',
clobber=True)
for delta in deltas[healpix]:
header = [
{
'name': 'RA',
'value': delta.ra,
'comment': 'Right Ascension [rad]'
},
{
'name': 'DEC',
'value': delta.dec,
'comment': 'Declination [rad]'
},
{
'name': 'Z',
'value': delta.z_qso,
'comment': 'Redshift'
},
{
'name':
'PMF',
'value':
'{}-{}-{}'.format(delta.plate, delta.mjd,
delta.fiberid)
},
{
'name': 'THING_ID',
'value': delta.thingid,
'comment': 'Object identification'
},
{
'name': 'PLATE',
'value': delta.plate
},
{
'name': 'MJD',
'value': delta.mjd,
'comment': 'Modified Julian date'
},
{
'name': 'FIBERID',
'value': delta.fiberid
},
{
'name': 'ORDER',
'value': delta.order,
'comment': 'Order of the continuum fit'
},
]
if args.delta_format == 'Pk1D':
header += [
{
'name': 'MEANZ',
'value': delta.mean_z,
'comment': 'Mean redshift'
},
{
'name': 'MEANRESO',
'value': delta.mean_reso,
'comment': 'Mean resolution'
},
{
'name': 'MEANSNR',
'value': delta.mean_snr,
'comment': 'Mean SNR'
},
]
if args.mode == 'desi':
delta_log_lambda = (
(delta.log_lambda[-1] - delta.log_lambda[0]) /
float(len(delta.log_lambda) - 1))
else:
delta_log_lambda = delta.delta_log_lambda
header += [{
'name': 'DLL',
'value': delta_log_lambda,
'comment': 'Loglam bin size [log Angstrom]'
}]
exposures_diff = delta.exposures_diff
if exposures_diff is None:
exposures_diff = delta.log_lambda * 0
cols = [
delta.log_lambda, delta.delta, delta.ivar,
exposures_diff
]
names = ['LOGLAM', 'DELTA', 'IVAR', 'DIFF']
units = ['log Angstrom', '', '', '']
comments = [
'Log lambda', 'Delta field', 'Inverse variance',
'Difference'
]
else:
cols = [
delta.log_lambda, delta.delta, delta.weights,
delta.cont
]
names = ['LOGLAM', 'DELTA', 'WEIGHT', 'CONT']
units = ['log Angstrom', '', '', '']
comments = [
'Log lambda', 'Delta field', 'Pixel weights',
'Continuum'
]
results.write(cols,
names=names,
header=header,
comment=comments,
units=units,
extname=str(delta.thingid))
results.close()
t3 = time.time()
tmin = (t3 - t2) / 60
userprint('INFO: time elapsed to write deltas', tmin, 'minutes')
ttot = (t3 - t0) / 60
userprint('INFO: total elapsed time', ttot, 'minutes')
log_file.close()
def go(
*,
seed,
gal_imag,
gal_type,
gal_hlr,
ngmix_model,
ntrial,
output,
show=False,
layout='grid',
loglevel='info',
):
"""
seed: int
Seed for a random number generator
ntrial: int
Number of trials to run, paired by simulation plus and minus shear
output: string
Output file path. If output is None, this is a dry
run and no output is written.
show: bool
If True, show some images. Default False
loglevel: string
Log level, default 'info'
"""
rng = np.random.RandomState(seed)
g1, g2 = 0.0, 0.0
# only over ride bands
sim_config = {
'bands': ['r', 'i'],
'psf_type': 'moffat',
'layout': layout,
}
sim_config = get_sim_config(config=sim_config)
logging.basicConfig(stream=sys.stdout)
logger = logging.getLogger('mdet_lsst_sim')
logger.setLevel(getattr(logging, loglevel.upper()))
logger.info(str(sim_config))
dlist = []
for trial in range(ntrial):
logger.info('-' * 70)
logger.info('trial: %d/%d' % (trial + 1, ntrial))
galaxy_catalog = ColorGalaxyCatalog(
rng=rng,
coadd_dim=sim_config['coadd_dim'],
buff=sim_config['buff'],
layout=sim_config['layout'],
imag=gal_imag,
hlr=gal_hlr,
gal_type=gal_type,
)
if sim_config['psf_type'] == 'ps':
se_dim = get_se_dim(coadd_dim=sim_config['coadd_dim'])
psf = make_ps_psf(rng=rng, dim=se_dim)
else:
psf = make_psf(psf_type=sim_config["psf_type"])
sim_data = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
coadd_dim=sim_config['coadd_dim'],
g1=g1,
g2=g2,
psf=psf,
psf_dim=sim_config['psf_dim'],
dither=sim_config['dither'],
rotate=sim_config['rotate'],
bands=sim_config['bands'],
epochs_per_band=sim_config['epochs_per_band'],
noise_factor=sim_config['noise_factor'],
cosmic_rays=sim_config['cosmic_rays'],
bad_columns=sim_config['bad_columns'],
star_bleeds=sim_config['star_bleeds'],
)
if show:
show_sim(sim_data['band_data'])
mbobs = make_mbobs(band_data=sim_data['band_data'], rng=rng)
toutput = run_ngmix(
mbobs=mbobs,
rng=rng,
model=ngmix_model,
)
dlist.append(toutput)
truth = make_truth(cat=galaxy_catalog, bands=sim_config['bands'])
data = eu.numpy_util.combine_arrlist(dlist)
logger.info('writing: %s' % output)
with fitsio.FITS(output, 'rw', clobber=True) as fits:
fits.write(data, extname='model_fits')
fits.write(truth, extname='truth')
Array of stellar masses for galaxies
"""
#read in relevant template info
sfh_tot = fitsio.read(template_path, 12)
sfh_met = fitsio.read(template_path, 13)
mass_tot = fitsio.read(template_path, 17)
#get angular diameter distances
cosmo = FlatLambdaCDM(100, 0.286)
da = cosmo.angular_diameter_distance(z).value
smass = np.dot(mass_tot, coeff.T) * (da * 1e6 / 10) ** 2
ssfr = np.dot(coeff, sfh_tot)
met = np.dot(coeff, sfh_tot * sfh_met) / ssfr
sfr = ssfr * smass[:,np.newaxis]
#get the values at z_galaxy
met = met[:,-1]
sfr = sfr[:,-1]
return numpy.vstack((sfr, met, smass))
if __name__=='__main__':
kfile = sys.argv[1] #name of file containing
filename = sys.argv[2] #name of galaxy catalog file
galaxies = fitsio.FITS(filename)[-1].read(columns=['COEFFS','Z','DELTAM']) # read relevant info from files
coeffs = galaxies['COEFFS']*10**(galaxies['DELTAM'].reshape(-1,1)/2.5)
sfr, met, smass = get_derived_quantities(kfile,coeffs,galaxies['Z'])
def __init__(self,
haspairs,
npatch,
nrbin,
oname=None,
zcens=None,
zmin=None,
zmax=None,
nzbin=None,
rcens=None,
**kwargs):
"""
Container for P(Z) PDF information, including JK-regions
Parameters
----------
haspairs : np.array, bool
which JK patch has P(Z) PDF extracted
npatch : int
number of JK-regions
nrbin : int
number of radial bins
oname : str
output name for pairs FITS file (LEGACY PARAMETER)
zcens : np.array
centers for the redshift grid
zmin : double
minimum redshift of the redshift grid
zmax : double
maximum redshift of the redshift grid
nzbin : int
number of redshift bins
rcens : np.array
radial bin centers
Notes
-----
In the simplest scenario initialization is done as ::
pcont = PDFContainer(haspairs=has_pairsfile, npatch=npatch,
nrbin=nrbins, zcens=zcens, pdf_paths=pdf_paths)
pcont.nsources = nsources
pcont.pdf_subs = pdf_subs
pcont.normsub()
That is some of the data have to be added after the object is created (placeholder values for these exist
alread).
To extract P(z) decomposition boost factors continue with e.g.::
bcont = pzboost.PDFContainer.from_file(bname)
bcont.decomp_gauss(refprof=paths.params['pzpars']['boost']['refprof'],
force_rbmin=paths.params['pzpars']['boost']['rbmin'],
force_rbmax=paths.params['pzpars']['boost']['rbmax'])
boostdict = bcont.to_boostdict()
"""
self.oname = oname
self.haspairs = haspairs.astype(bool)
self.npatch = npatch
self.pindexes = np.arange(self.npatch)[self.haspairs]
self.nrbin = nrbin
self.rbins = np.arange(self.nrbin)
self.rcens = rcens
if self.rcens is None:
self.rcens = shearops.redges(
paths.params["radial_bins"]["rmin"],
paths.params["radial_bins"]["rmax"],
paths.params["radial_bins"]["nbin"])[0]
self.pexts = None
# FITS data file
if self.oname is not None:
self.pfits = fio.FITS(self.oname)
self.pexts = self._get_patch_exts()
# print "5"
# pdf binning parameters
if zcens is not None and zmin is None and zmax is None and nzbin is None:
self.zcens = zcens
self.zmin, self.zmax, self.zedges = self._get_zedges()
self.nbin = len(self.zcens)
elif zcens is None and zmin is not None and zmax is not None and nzbin is not None:
self.zmin = zmin
self.zmax = zmax
self.nbin = nzbin
self.zedges, self.zcens = self._get_zarr()
else:
raise TypeError
# print "6"
# pdf averaging parameters
self.rkey = 'rbin'
self.wkey = 'source_weight'
self.zkey = 'z_sample'
# Data containers
self.nsources = np.zeros(shape=(self.nrbin, self.npatch))
self.pdf_subs = np.zeros(shape=(self.nrbin, self.npatch,
len(self.zcens)))
self.pdfs = None
self.pdf_errs = None
# WARNING this is really just the values in each patch, not the usual JK-notation (i.e except-th patch)
self.pwsum_subs = np.zeros(shape=(self.nrbin, self.npatch,
len(self.zcens)))
self.wsum_subs = np.zeros(shape=(self.nrbin, self.npatch))
# Additional calculated values
self.bw = np.mean(np.diff(self.zedges))
# boost factor initial values
self.mean_init = 0.5
self.sigma_init = 0.1
self.amp_init = 0.5
self.mean = BADVAL
self.mean_err = BADVAL
self.sigma = BADVAL
self.sigma_err = BADVAL
self.amps = None
self.amps_err = None
self.amps_cov = None
self.boost_rvals = None
self.boost = None
self.boost_err = None
self.boost_cov = None
self.point = None
self.point_subs = None
self.point_cov = None
self.goodinds = None
# Boost factor bounds
self.mean_bounds = (0., np.inf)
self.sigma_bounds = (0., np.inf)
self.amp_bounds_single = (0., 1.)
self.verbose_prefix = ''
def get_spectra(lyafile,
nqso=None,
wave=None,
templateid=None,
normfilter='sdss2010-g',
seed=None,
rand=None,
qso=None,
add_dlas=False,
debug=False,
nocolorcuts=False):
"""Generate a QSO spectrum which includes Lyman-alpha absorption.
Args:
lyafile (str): name of the Lyman-alpha spectrum file to read.
nqso (int, optional): number of spectra to generate (starting from the
first spectrum; if more flexibility is needed use TEMPLATEID).
wave (numpy.ndarray, optional): desired output wavelength vector.
templateid (int numpy.ndarray, optional): indices of the spectra
(0-indexed) to read from LYAFILE (default is to read everything). If
provided together with NQSO, TEMPLATEID wins.
normfilter (str, optional): normalization filter
seed (int, optional): Seed for random number generator.
rand (numpy.RandomState, optional): RandomState object used for the
random number generation. If provided together with SEED, this
optional input superseeds the numpy.RandomState object instantiated by
SEED.
qso (desisim.templates.QSO, optional): object with which to generate
individual spectra/templates.
add_dlas (bool): Inject damped Lya systems into the Lya forest
These are done according to the current best estimates for the incidence dN/dz
(Prochaska et al. 2008, ApJ, 675, 1002)
Set in calc_lz
These are *not* inserted according to overdensity along the sightline
nocolorcuts (bool, optional): Do not apply the fiducial rzW1W2 color-cuts
cuts (default False).
Returns (flux, wave, meta, dla_meta) where:
* flux (numpy.ndarray): Array [nmodel, npix] of observed-frame spectra
(erg/s/cm2/A).
* wave (numpy.ndarray): Observed-frame [npix] wavelength array (Angstrom).
* meta (astropy.Table): Table of meta-data [nmodel] for each output spectrum
with columns defined in desisim.io.empty_metatable *plus* RA, DEC.
* dla_meta (astropy.Table): Table of meta-data [ndla] for the DLAs injected
into the spectra. Only returned if add_dlas=True
Note: `dla_meta` is only included if add_dlas=True.
"""
from scipy.interpolate import interp1d
import fitsio
from speclite.filters import load_filters
from desisim.templates import QSO
from desisim.io import empty_metatable
h = fitsio.FITS(lyafile)
if templateid is None:
if nqso is None:
nqso = len(h) - 1
templateid = np.arange(nqso)
else:
templateid = np.asarray(templateid)
nqso = len(np.atleast_1d(templateid))
if rand is None:
rand = np.random.RandomState(seed)
templateseed = rand.randint(2**32, size=nqso)
#heads = [head.read_header() for head in h[templateid + 1]]
heads = []
for indx in templateid:
heads.append(h[indx + 1].read_header())
zqso = np.array([head['ZQSO'] for head in heads])
ra = np.array([head['RA'] for head in heads])
dec = np.array([head['DEC'] for head in heads])
mag_g = np.array([head['MAG_G'] for head in heads])
# Hard-coded filtername!
normfilt = load_filters(normfilter)
if qso is None:
qso = QSO(normfilter=normfilter, wave=wave)
wave = qso.wave
flux = np.zeros([nqso, len(wave)], dtype='f4')
meta = empty_metatable(objtype='QSO', nmodel=nqso)
meta['TEMPLATEID'] = templateid
meta['REDSHIFT'] = zqso
meta['MAG'] = mag_g
meta['SEED'] = templateseed
meta['RA'] = ra
meta['DEC'] = dec
# Lists for DLA meta data
if add_dlas:
dla_NHI, dla_z, dla_id = [], [], []
# Loop on quasars
for ii, indx in enumerate(templateid):
flux1, _, meta1 = qso.make_templates(nmodel=1,
redshift=np.atleast_1d(zqso[ii]),
mag=np.atleast_1d(mag_g[ii]),
seed=templateseed[ii],
nocolorcuts=nocolorcuts,
lyaforest=False)
flux1 *= 1e-17
for col in meta1.colnames:
meta[col][ii] = meta1[col][0]
# read lambda and forest transmission
data = h[indx + 1].read()
la = data['LAMBDA'][:]
tr = data['FLUX'][:]
if len(tr):
# Interpolate the transmission at the spectral wavelengths, if
# outside the forest, the transmission is 1.
itr = interp1d(la, tr, bounds_error=False, fill_value=1.0)
flux1 *= itr(wave)
# Inject a DLA?
if add_dlas:
if np.min(wave / 1215.67 - 1) < zqso[ii]: # Any forest?
dlas, dla_model = insert_dlas(wave,
zqso[ii],
seed=templateseed[ii])
ndla = len(dlas)
if ndla > 0:
flux1 *= dla_model
# Meta
dla_z += [idla['z'] for idla in dlas]
dla_NHI += [idla['N'] for idla in dlas]
dla_id += [indx] * ndla
padflux, padwave = normfilt.pad_spectrum(flux1, wave, method='edge')
normmaggies = np.array(
normfilt.get_ab_maggies(padflux, padwave,
mask_invalid=True)[normfilter])
factor = 10**(-0.4 * mag_g[ii]) / normmaggies
flux1 *= factor
for key in ('FLUX_G', 'FLUX_R', 'FLUX_Z', 'FLUX_W1', 'FLUX_W2'):
meta[key][ii] *= factor
flux[ii, :] = flux1[:]
h.close()
# Finish
if add_dlas:
ndla = len(dla_id)
if ndla > 0:
from astropy.table import Table
dla_meta = Table()
dla_meta['NHI'] = dla_NHI # log NHI values
dla_meta['z'] = dla_z
dla_meta['ID'] = dla_id
else:
dla_meta = None
return flux * 1e17, wave, meta, dla_meta
else:
return flux * 1e17, wave, meta
import numpy as np
import matplotlib.pyplot as plt
import fitsio
plt.rc("font", **{"family": "serif", "serif": ["Computer Modern"]})
plt.rc("text", usetex=True)
g_xlabel = "$\lambda (\AA)$"
g_ylabel = "$F (units)$"
g_fontsize_xlabel = 30
g_fontsize_ylabel = 28
g_file_dpi = 200
g_plot_dpi = 75
fin = fitsio.FITS('/init_directory_spectra_with_BALs/spectra_with_BAL.fits')
wave = fin[2].read()
flux = fin[3].read()
plt.xlabel(g_xlabel, fontsize=g_fontsize_xlabel)
plt.ylabel(g_ylabel, fontsize=g_fontsize_ylabel)
plt.plot(wave,flux[28,:],color='blue',lw=1,zorder=10)
plt.plot(wave,flux[29,:],color='orange',lw=1,zorder=5)
plt.savefig("directory/name_plot.png", bbox_inches="tight", dpi=g_file_dpi)
plt.show()
def download_single_ccd(path, sdir, expnum, ccd):
url_base = 'https://*****:*****@desar2.cosmology.illinois.edu/DESFiles/desarchive/'%ps()
"""
# need;
image_file_name
expnum
zonenum
# no idea what they are yet
path
In [10]: arr['path'][0].rsplit('/', 3)
Out[10]:
['OPS/finalcut/Y2A1/Y3-2379/20160108/D00509375/p01',
'red',
'immask',
'D00509375_z_c39_r2379p01_immasked.fits.fz']
can load the exp_psf_cat_509375.fits blahblah file, go for the path array, which will have the path at desdm
"""
# outname
outname = '{0}/psf_im_{1}_{2}.fits'.format(sdir, expnum, ccd)
if os.path.exists(outname):
print('skipping {0} because it exists!'.format(outname))
return
# Download the files we need:
base_path, _, _, image_file_name = path.rsplit('/',3)
# strip any spaces on end
image_file_name = image_file_name.strip()
root, ext = image_file_name.rsplit('_',1)
print('root, ext = |%s| |%s|'%(root,ext))
image_file = wget(url_base, base_path + '/red/immask/', sdir, root + '_' + ext)
print('image_file = ',image_file)
print(time.ctime())
bkg_file = wget(url_base, base_path + '/red/bkg/', sdir, root + '_bkg.fits.fz')
print('bkg_file = ',bkg_file)
print(time.ctime())
# Unpack the image file if necessary
# change the image file name so that it makes life easier
unpack_image_file = unpack_file(image_file, outname)
print('unpacked to ',unpack_image_file)
print(time.ctime())
# Subtract off the background right from the start
with fitsio.FITS(unpack_image_file, 'rw') as f:
bkg = fitsio.read(bkg_file)
#print('after read bkg')
img = f[0].read()
img -= bkg
#print('after subtract bkg')
f[0].write(img)
#print('after write new img')
print('subtracted off background image')
print(time.ctime())
pack_file(unpack_image_file) # saves to unpack_image_file + .fz
print('packing image')
print(time.ctime())
# delete non subtracted files
for file_to_remove in [sdir + '/' + root + '_' + ext, sdir + '/' + root + '_bkg.fits.fz', unpack_image_file]:
os.remove(file_to_remove)
def cornerplot(theory1, theory2, data1, data2, show_cuts=False):
plt.switch_backend("pdf")
plt.style.use("y1a1")
matplotlib.rcParams["ytick.minor.visible"]=False
matplotlib.rcParams["ytick.minor.width"]=0.1
ni,nj=np.genfromtxt("%s/shear_xi/values.txt"%theory1[0]).T[2]
ni = int(ni)
nj = int(nj)
if data1 is not None:
data1 = fi.FITS(data1)
if data2 is not None:
data2 = fi.FITS(data2)
rows, cols = ni+1, nj+2
count = 0
for i in xrange(ni):
for j in xrange(nj):
count+=1
if j>i:
continue
print i,j
xip_a,xim_a = get_real_spectra(i,j,data1)
xip_b,xim_b = get_real_spectra(i,j,data2)
posp = positions[(i+1,j+1,"+")]
ax = plt.subplot(rows,cols,posp)
ax.annotate("(%d, %d)"%(i+1,j+1), (2,0.65), textcoords='data', fontsize=11, )
ax.yaxis.set_tick_params(which='minor', left='off', right='off')
plt.ylim(-0.4,0.6)
#plt.yscale("log")
plt.xscale("log")
if posp==19:
plt.ylabel(r"$\Delta \xi^{ij}_\mathrm{-,GI}(\theta)/\xi^{ij}_\mathrm{-,GG}(\theta)$", fontsize=12)
plt.xlabel(r"Angular Separation $\theta$", fontsize=12)
plt.yticks(fontsize=12)
else:
plt.yticks(visible=False)
plt.xlabel(r"$\theta$ /arcmin", fontsize=12)
plt.xlim(1,210)
plt.xticks([10,100],["10", "100"])
#plt.yticks([-2,0,2,4,6,8],['-2', '0', '2', '4', '6', '8'])
plt.axhline(0, color='k')
xlower,xupper = lims['-'][(i+1,j+1)]
plt.axvspan(1e-6, xlower, color='gray',alpha=0.2)
plt.axvspan(xupper, 500, color='gray',alpha=0.2)
xta,xip_theory_a_ref, xim_theory_a_ref = get_theory_spectra(i,j,theory1[0], xitype='gg')
xtb,xip_theory_b_ref, xim_theory_b_ref = get_theory_spectra(i,j,theory2[0], xitype='gg')
xta,xip_theory_a_ref_ia, xim_theory_a_ref_ia = get_theory_spectra(i,j,theory1[0], xitype='gi')
xtb,xip_theory_b_ref_ia, xim_theory_b_ref_ia = get_theory_spectra(i,j,theory2[0], xitype='gi')
linestyles=['-',':','--','-']
for iline,(t1, t2) in enumerate(zip(theory1,theory2)):
if iline==0:
continue
xta,xip_theory_a, xim_theory_a = get_theory_spectra(i,j,t1, xitype='gi')
xtb,xip_theory_b, xim_theory_b = get_theory_spectra(i,j,t2, xitype='gi')
plt.plot(xta, (xim_theory_a-xim_theory_a_ref_ia)/xim_theory_a_ref, ls=linestyles[iline], color="red")
plt.plot(xtb, (xim_theory_b-xim_theory_b_ref_ia)/xim_theory_b_ref, ls=linestyles[iline], color="royalblue")
posm = positions[(i+1,j+1,"-")]
ax = plt.subplot(rows,cols,posm)
ax.annotate("(%d, %d)"%(i+1,j+1), (20,0.25), textcoords='data', fontsize=11, )
ax.yaxis.set_tick_params(which='minor', left='off', right='off')
plt.ylim(-0.5,0.55)
if posm==30:
ax.yaxis.set_label_position("right")
ax.yaxis.set_ticks_position("right")
plt.yticks(fontsize=12)
plt.ylabel(r"$\Delta \xi^{ij}_\mathrm{-,II}(\theta)/\xi^{ij}_\mathrm{-,GG}(\theta)$", fontsize=12)
else:
plt.yticks(visible=False)
#plt.yscale("log")
plt.xscale("log")
plt.xlim(1,210)
plt.xticks([10,100],["10", "100"])
#plt.yticks([-2,0,2,4,6,8],['-2', '0', '2', '4', '6', '8'])
plt.axhline(0, color='k')
xta,xip_theory_a_ref, xim_theory_a_ref = get_theory_spectra(i,j,theory1[0], xitype='gg')
xtb,xip_theory_b_ref, xim_theory_b_ref = get_theory_spectra(i,j,theory2[0], xitype='gg')
xta,xip_theory_a_ref_ia, xim_theory_a_ref_ia = get_theory_spectra(i,j,theory1[0], xitype='ii')
xtb,xip_theory_b_ref_ia, xim_theory_b_ref_ia = get_theory_spectra(i,j,theory2[0], xitype='ii')
xlower,xupper = lims['-'][(i+1, j+1)]
plt.axvspan(1e-6, xlower, color='gray',alpha=0.2)
plt.axvspan(xupper, 500, color='gray',alpha=0.2)
linestyles=['-',':','--','-']
for iline,(t1, t2) in enumerate(zip(theory1,theory2)):
if iline==0:
continue
xta,xip_theory_a, xim_theory_a = get_theory_spectra(i,j,t1, xitype='ii')
xtb,xip_theory_b, xim_theory_b = get_theory_spectra(i,j,t2, xitype='ii')
plt.plot(xta, (xim_theory_a-xim_theory_a_ref_ia)/xim_theory_a_ref, ls=linestyles[iline], color="red")
plt.plot(xtb, (xim_theory_b-xim_theory_b_ref_ia)/xim_theory_b_ref, ls=linestyles[iline], color="royalblue")
plt.subplots_adjust(hspace=0,wspace=0)
plt.savefig("/home/samuroff/tmp3.pdf")
