def main_aux( pathlist=None, reqnum=None, band=None, ccdnum=None, nproc=None, chunk=None, px_side=None, label=None, ): ''' Main auxiliaty function to call the code in parallel ''' if (pathlist is None): # If no set of pull paths is provided, go to the DB and retrieve them, # based on reqnum and band fpath = db_red_pixcor(reqnum, band, ccdnum) elif (pathlist is not None): # Load the table exp = np.genfromtxt(pathlist, dtype=[ ('path', '|S200'), ], comments='#', missing_values=np.nan, usecols=0) fpath = exp['path'] # Remove duplicates using np.unique capabilities uarr, uidx, uinverse, ucounts = np.unique( fpath, return_index=True, return_inverse=True, return_counts=True, ) if False: # Save table for simplicity of testing tmp = pd.DataFrame({'path': fpath}) tmp.to_csv('path_r3437_g.txt', index=False, header=False) # # NOTE # The goal is to read the same section, from a bunch of images, in # parallel. Then stack the set of section and calculate the median # of each pixel. # # Get shape of the CCD try: aux_fits = fitsio.read(fpath[0]) f_dim = aux_fits.shape logging.info('CCD shape: {0}'.format(f_dim)) except: logging.error('File cannot be read. Using 2048x4096 as CCD dimensions') f_dim = (4096, 2048) # Define squared sections to be used for calculation if (px_side is None): px_side = 512 if ((f_dim[0] % px_side != 0) or (f_dim[1] % px_side != 0)): logging.warning('{0}x{0} des not fit exactly on CCD'.format(px_side)) # idx_d0 = np.arange(0, fits_dim[0] + 1, px_side) # idx_d1 = np.arange(0, fits_dim[1] + 1, px_side) yx = np.mgrid[0:f_dim[0] + 1:px_side, 0:f_dim[1] + 1:px_side] # coo_list contains [x0, y0, x1, y1] coo_list = [] for iy in range(f_dim[0] / px_side): #yx.shape[1]): for ix in range(f_dim[1] / px_side): #(yx.shape[2]): y0, x0 = (yx[0, iy, ix], yx[1, iy, ix]) y1, x1 = (yx[0, iy + 1, ix], yx[1, iy, ix + 1]) coo_list.append((x0, y0, x1, y1)) # Create an array to harbor the results tmp_res = np.zeros((f_dim[0], f_dim[1])) # Setup the parallel call # The value for chunk will help prevent memory errors. "Chops the iterable # into a number of chunks which it submits to the process pool as separate # tasks" if (nproc is None): nproc = mp.cpu_count() if (chunk is None): chunk = int(np.ceil(fpath.size / nproc)) logging.info('Launch {0} parallel processes'.format(nproc)) P1 = mp.Pool(processes=nproc) # Here I can also parallelize in terms of coordinates, but need to be # careful to not mix results from different sections of the CCD, and # manage the Pool (2 in fact) for idx_c, coo in enumerate(coo_list): logging.info('Box {0} of {1}'.format(idx_c + 1, len(coo_list))) # coo: coordinates, ext: extension to read from the FITS file, # delta: side size (pixel) of the square used to sample the FITS file kw_section = { 'coo': coo, 'ext': 0, 'delta': px_side, } # Call using partial() or constructed list try: partial_aux = partial( fits_section, [kw_section['coo'], kw_section['ext'], kw_section['delta']]) t0 = time.time() # map_async does not block the processes, and executes non-ordered boxi = P1.map_async(partial_aux, fpath, chunk) boxi.wait() t1 = time.time() except: aux_list = [(fnm, kw_section['coo'], kw_section['ext'], kw_section['delta']) for fnm in fpath] t0 = time.time() boxi = P1.map_async(fits_section, aux_list, chunk) boxi.wait() t1 = time.time() boxi = boxi.get() # At this point the Pool has returned a list, containig the stamps # for all the CCDs # Call the median image generator, using np.nditer(), with a 3D array x3d = np.dstack(boxi) z_median = stat_cube(x3d, (lambda: np.median)()) # Put into the auxiliary array x0, y0, x1, y1 = coo tmp_res[y0:y1, x0:x1] = z_median # The results looks good! # Save in FITS format if (label is None): label = str(uuid.uuid4()) ores = 'medImg_{0}.fits'.format(label) while os.path.exists(ores): logging.info('File {0} exists. Changing output name'.format(ores)) ores = 'medImg_{0}.fits'.format(str(uuid.uuid4())) fits = fitsio.FITS(ores, 'rw') fits.write(tmp_res) fits[-1].write_checksum() fits.close() logging.info('Median image {0} saved'.format(ores)) return True
def read_mock_spectra(truthfile, targetids, mockdir=None): ''' Reads mock spectra from a truth file Args: truthfile (str): full path to a mocks spectra_truth*.fits file targetids (array-like): targetids to load from that file mockdir: ??? Returns (flux, wave, truth) tuples: flux[nspec, nwave]: flux in 1e-17 erg/s/cm2/Angstrom wave[nwave]: wavelengths in Angstroms truth[nspec]: metadata truth table ''' if len(targetids) != len(np.unique(targetids)): from lvmutil.log import get_logger log = get_logger() log.error("Requested TARGETIDs for {} are not unique".format( os.path.basename(truthfile))) #- astropy.io.fits doesn't return a real ndarray, causing problems #- with the reordering downstream so use fitsio instead # with fits.open(truthfile, memmap=False) as fx: # truth = fx['TRUTH'].data # wave = fx['WAVE'].data # flux = fx['FLUX'].data with fitsio.FITS(truthfile) as fx: truth = fx['TRUTH'].read() wave = fx['WAVE'].read() flux = fx['FLUX'].read() missing = np.in1d(targetids, truth['TARGETID'], invert=True) if np.any(missing): missingids = targetids[missing] raise ValueError('Targets missing from {}: {}'.format( truthfile, missingids)) #- Trim to just the spectra for these targetids ii = np.in1d(truth['TARGETID'], targetids) flux = flux[ii] truth = truth[ii] assert set(targetids) == set(truth['TARGETID']) #- sort truth to match order of input targetids if len(targetids) == len(truth['TARGETID']): i = np.argsort(targetids) j = np.argsort(truth['TARGETID']) k = np.argsort(i) reordered_truth = truth[j[k]] reordered_flux = flux[j[k]] else: #- Slower, but works even with repeated TARGETIDs ii = np.argsort(truth['TARGETID']) sorted_truthids = truth['TARGETID'][ii] reordered_flux = np.empty(shape=(len(targetids), flux.shape[1]), dtype=flux.dtype) reordered_truth = np.empty(shape=(len(targetids), ), dtype=truth.dtype) for j, tx in enumerate(targetids): k = np.searchsorted(sorted_truthids, tx) reordered_flux[j] = flux[ii[k]] reordered_truth[j] = truth[ii[k]] assert np.all(reordered_truth['TARGETID'] == targetids) wave = lvmspec.io.util.native_endian(wave).astype(np.float64) reordered_flux = lvmspec.io.util.native_endian(reordered_flux).astype( np.float64) return reordered_flux, wave, reordered_truth
import argparse import os import pylab as plt plt.switch_backend('agg') parser = argparse.ArgumentParser() parser.add_argument('-f', '--filename', action='store', type=str) parser.add_argument('--nofits', action='store_true') parser.add_argument('--nofig', action='store_true') args = parser.parse_args() files = glob.glob(args.filename) print 'found %d files to process' % len(files) baryons1 = fi.FITS( '/home/ssamurof/massive_black_ii/subhalo_cat-nthreshold5.fits' )['baryons'][:] dm1 = fi.FITS( '/home/ssamurof/massive_black_ii/subhalo_cat-nthreshold5.fits')['dm'][:] select = (dm1['npart'] > 1000) & (baryons1['npart'] > 0) & (np.isfinite( baryons1['x']) & np.isfinite(baryons1['y']) & np.isfinite(baryons1['z'])) dat = copy.deepcopy(baryons1[select]) for f in files: d = fi.FITS(f)['baryons'].read() for comp in ['x', 'y', 'z']: dat[comp][(d[comp] != baryons1[comp][select])] = d[comp][( d[comp] != baryons1[comp][select])] print f
def main(): import argparse parser = argparse.ArgumentParser( description='This script creates small self-contained data sets that ' 'are useful for test cases of the pipeline codes.') parser.add_argument('ccds', help='CCDs table describing region to grab') parser.add_argument('outdir', help='Output directory name') parser.add_argument('brick', help='Brick containing these images') parser.add_argument('--survey-dir', type=str, default=None) parser.add_argument('--cache-dir', type=str, default=None, help='Directory to search for cached files') parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image') parser.add_argument('--wise-wcs-hdu', help='For WISE outputs, the HDU to read the WCS from in the file given by --wise.', type=int, default=0) parser.add_argument('--fpack', action='store_true', default=False) parser.add_argument('--gzip', action='store_true', default=False) parser.add_argument('--pad', action='store_true', default=False, help='Keep original image size, but zero out pixels outside ROI') args = parser.parse_args() v = 'SKY_TEMPLATE_DIR' if v in os.environ: del os.environ[v] C = fits_table(args.ccds) print(len(C), 'CCDs in', args.ccds) C.camera = np.array([c.strip() for c in C.camera]) survey = LegacySurveyData(cache_dir=args.cache_dir, survey_dir=args.survey_dir) if ',' in args.brick: ra,dec = args.brick.split(',') ra = float(ra) dec = float(dec) fakebricks = fits_table() fakebricks.brickname = np.array([('custom-%06i%s%05i' % (int(1000*ra), 'm' if dec < 0 else 'p', int(1000*np.abs(dec))))]) fakebricks.ra = np.array([ra]) fakebricks.dec = np.array([dec]) bricks = fakebricks outbricks = bricks else: bricks = survey.get_bricks_readonly() outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])] assert(len(outbricks) == 1) outsurvey = LegacySurveyData(survey_dir = args.outdir) trymakedirs(args.outdir) outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz')) targetwcs = wcs_for_brick(outbricks[0]) H,W = targetwcs.shape tycho2fn = survey.find_file('tycho2') kd = tree_open(tycho2fn, 'stars') radius = 1. rc,dc = targetwcs.radec_center() I = tree_search_radec(kd, rc, dc, radius) print(len(I), 'Tycho-2 stars within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (rc,dc)) # Read only the rows within range. tycho = fits_table(tycho2fn, rows=I) del kd print('Read', len(tycho), 'Tycho-2 stars') ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec) #margin = 100 #tycho.cut(ok * (tx > -margin) * (tx < W+margin) * # (ty > -margin) * (ty < H+margin)) print('Cut to', len(tycho), 'Tycho-2 stars within brick') del ok,tx,ty #tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz')) f,tfn = tempfile.mkstemp(suffix='.fits') os.close(f) tycho.writeto(tfn) outfn = os.path.join(args.outdir, 'tycho2.kd.fits') cmd = 'startree -i %s -o %s -P -k -n stars -T' % (tfn, outfn) print(cmd) rtn = os.system(cmd) assert(rtn == 0) os.unlink(tfn) from legacypipe.gaiacat import GaiaCatalog gcat = GaiaCatalog() # from ps1cat.py: wcs = targetwcs step=100. margin=10. # Grid the CCD in pixel space W,H = wcs.get_width(), wcs.get_height() xx,yy = np.meshgrid( np.linspace(1-margin, W+margin, 2+int((W+2*margin)/step)), np.linspace(1-margin, H+margin, 2+int((H+2*margin)/step))) # Convert to RA,Dec and then to unique healpixes ra,dec = wcs.pixelxy2radec(xx.ravel(), yy.ravel()) healpixes = set() for r,d in zip(ra,dec): healpixes.add(gcat.healpix_for_radec(r, d)) for hp in healpixes: hpcat = gcat.get_healpix_catalog(hp) ok,xx,yy = wcs.radec2pixelxy(hpcat.ra, hpcat.dec) onccd = np.flatnonzero((xx >= 1.-margin) * (xx <= W+margin) * (yy >= 1.-margin) * (yy <= H+margin)) hpcat.cut(onccd) if len(hpcat): outfn = os.path.join(args.outdir, 'gaia', 'chunk-%05d.fits' % hp) trymakedirs(os.path.join(args.outdir, 'gaia')) hpcat.writeto(outfn) outccds = C.copy() cols = outccds.get_columns() for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1', 'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1', 'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver', 'psfplver' ]: if c in cols: outccds.delete_column(c) outccds.image_hdu[:] = 1 # Convert to list to avoid truncating filenames outccds.image_filename = [fn for fn in outccds.image_filename] for iccd,ccd in enumerate(C): decam = (ccd.camera.strip() == 'decam') bok = (ccd.camera.strip() == '90prime') im = survey.get_image_object(ccd) print('Got', im) if survey.cache_dir is not None: im.check_for_cached_files(survey) slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1)) psfkwargs = dict(pixPsf=True, gaussPsf=False, hybridPsf=False, normalizePsf=False) tim = im.get_tractor_image(slc, pixPsf=True, subsky=False, nanomaggies=False, no_remap_invvar=True, old_calibs_ok=True) print('Tim:', tim.shape) psfrow = psfhdr = None if args.pad: psf = im.read_psf_model(0, 0, w=im.width, h=im.height, **psfkwargs) psfex = psf.psfex else: psf = tim.getPsf() psfex = psf.psfex # Did the PSF model come from a merged file? for fn in [im.merged_psffn, im.psffn] + im.old_merged_psffns: if not os.path.exists(fn): continue T = fits_table(fn) I, = np.nonzero((T.expnum == im.expnum) * np.array([c.strip() == im.ccdname for c in T.ccdname])) if len(I) != 1: continue psfrow = T[I] x0 = ccd.ccd_x0 y0 = ccd.ccd_y0 psfrow.polzero1[0] -= x0 psfrow.polzero2[0] -= y0 #psfhdr = fitsio.read_header(im.merged_psffn) break psfex.fwhm = tim.psf_fwhm #### HACK #psfrow = None assert(psfrow is not None) if psfrow is not None: print('PSF row:', psfrow) #else: # print('PSF:', psf) # print('PsfEx:', psfex) skyrow = skyhdr = None if args.pad: primhdr = fitsio.read_header(im.imgfn) imghdr = fitsio.read_header(im.imgfn, hdu=im.hdu) sky = im.read_sky_model(splinesky=True, primhdr=primhdr, imghdr=imghdr) #skyhdr = fitsio.read_header(im.splineskyfn) #msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True) else: sky = tim.getSky() # Did the sky model come from a merged file? #msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True) print('merged skyfn:', im.merged_skyfn) print('single skyfn:', im.skyfn) print('old merged skyfns:', im.old_merged_skyfns) for fn in [im.merged_skyfn, im.skyfn] + im.old_merged_skyfns: if not os.path.exists(fn): continue T = fits_table(fn) I, = np.nonzero((T.expnum == im.expnum) * np.array([c.strip() == im.ccdname for c in T.ccdname])) skyrow = T[I] skyrow.x0[0] = ccd.ccd_x0 skyrow.y0[0] = ccd.ccd_y0 # s_med = skyrow.sky_med[0] # s_john = skyrow.sky_john[0] # skyhdr = fitsio.read_header(fn) assert(skyrow is not None) ### HACK #skyrow = None if skyrow is not None: print('Sky row:', skyrow) else: print('Sky:', sky) # Output filename format: fn = ccd.image_filename.strip() ccd.image_filename = os.path.join(os.path.dirname(fn), '%s.%s.fits' % (os.path.basename(fn).split('.')[0], ccd.ccdname.strip())) outim = outsurvey.get_image_object(ccd) print('Output image:', outim) print('Image filename:', outim.imgfn) trymakedirs(outim.imgfn, dir=True) imgdata = tim.getImage() ivdata = tim.getInvvar() # Since we remap DQ codes (always with Mosaic and Bok, sometimes with DECam), # re-read from the FITS file rather than using tim.dq. print('Reading data quality from', im.dqfn, 'hdu', im.hdu) dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice) print('Tim shape:', tim.shape, 'Slice', tim.slice) print('image shape:', imgdata.shape, 'iv', ivdata.shape, 'DQ', dqdata.shape) from collections import Counter dqvals = Counter(dqdata.ravel()) print('DQ pixel counts:') for k,n in dqvals.most_common(): print(' 0x%x' % k, ':', n) if args.pad: # Create zero image of full size, copy in data. fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype) fullsize[slc] = imgdata imgdata = fullsize fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype) fullsize[slc] = dqdata dqdata = fullsize fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype) fullsize[slc] = ivdata ivdata = fullsize else: # Adjust the header WCS by x0,y0 crpix1 = tim.hdr['CRPIX1'] crpix2 = tim.hdr['CRPIX2'] tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0 tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0 # Add image extension to filename # fitsio doesn't compress .fz by default, so drop .fz suffix #outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits') if args.gzip: outim.imgfn = outim.imgfn.replace('.fits', '.fits.gz') #outim.wtfn = outim.wtfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.wtfn = outim.wtfn.replace('.fits.fz', '.fits') if args.gzip: outim.wtfn = outim.wtfn.replace('.fits', '.fits.gz') if outim.dqfn is not None: #outim.dqfn = outim.dqfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.dqfn = outim.dqfn.replace('.fits.fz', '.fits') if args.gzip: outim.dqfn = outim.dqfn.replace('.fits', '.fits.gz') if bok: outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname) ccdfn = outim.imgfn ccdfn = ccdfn.replace(outsurvey.get_image_dir(), '') if ccdfn.startswith('/'): ccdfn = ccdfn[1:] outccds.image_filename[iccd] = ccdfn print('Changed output filenames to:') print(outim.imgfn) print(outim.dqfn) ofn = outim.imgfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fits = fitsio.FITS(ofn, 'rw', clobber=True) fits.write(None, header=tim.primhdr) fits.write(imgdata, header=tim.hdr, extname=ccd.ccdname) fits.close() if args.fpack: cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) h,w = tim.shape if not args.pad: outccds.width[iccd] = w outccds.height[iccd] = h outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0 outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0 wcs = Tan(*[float(x) for x in [ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2, ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]]) if args.pad: subwcs = wcs else: subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h) outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center() print('Weight filename:', outim.wtfn) wfn = outim.wtfn trymakedirs(wfn, dir=True) ofn = wfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fits = fitsio.FITS(ofn, 'rw', clobber=True) fits.write(None, header=tim.primhdr) fits.write(ivdata, header=tim.hdr, extname=ccd.ccdname) fits.close() if args.fpack: cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) if outim.dqfn is not None: print('DQ filename', outim.dqfn) trymakedirs(outim.dqfn, dir=True) ofn = outim.dqfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fits = fitsio.FITS(ofn, 'rw', clobber=True) fits.write(None, header=tim.primhdr) fits.write(dqdata, header=tim.hdr, extname=ccd.ccdname) fits.close() if args.fpack: cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) psfout = outim.psffn #if psfrow: # psfout = outim.merged_psffn print('PSF output filename:', psfout) trymakedirs(psfout, dir=True) if psfrow: psfrow.writeto(psfout, primhdr=psfhdr) else: print('Writing PsfEx:', psfout) psfex.writeto(psfout) # update header F = fitsio.FITS(psfout, 'rw') F[0].write_keys([dict(name='EXPNUM', value=ccd.expnum), dict(name='PLVER', value=psf.plver), dict(name='PROCDATE', value=psf.procdate), dict(name='PLPROCID', value=psf.plprocid),]) F.close() skyout = outim.skyfn #if skyrow: # skyout = outim.merged_splineskyfn print('Sky output filename:', skyout) trymakedirs(skyout, dir=True) if skyrow is not None: skyrow.writeto(skyout, primhdr=skyhdr) else: primhdr = fitsio.FITSHDR() primhdr['PLVER'] = sky.plver primhdr['PLPROCID'] = sky.plprocid primhdr['PROCDATE'] = sky.procdate primhdr['EXPNUM'] = ccd.expnum primhdr['IMGDSUM'] = sky.datasum primhdr['S_MED'] = s_med primhdr['S_JOHN'] = s_john sky.write_fits(skyout, primhdr=primhdr) # HACK -- check result immediately. outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz')) outsurvey.ccds = None outC = outsurvey.get_ccds_readonly() occd = outC[iccd] outim = outsurvey.get_image_object(occd) print('Got output image:', outim) otim = outim.get_tractor_image(pixPsf=True, hybridPsf=True, old_calibs_ok=True) print('Got output tim:', otim) outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz')) # WISE if args.wise is not None: from wise.forcedphot import unwise_tiles_touching_wcs from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs, get_unwise_tractor_image, get_unwise_tile_dir) # Read WCS... print('Reading TAN wcs header from', args.wise, 'HDU', args.wise_wcs_hdu) targetwcs = Tan(args.wise, args.wise_wcs_hdu) tiles = unwise_tiles_touching_wcs(targetwcs) print('Cut to', len(tiles), 'unWISE tiles') H,W = targetwcs.shape r,d = targetwcs.pixelxy2radec(np.array([1, W, W/2, W/2]), np.array([H/2, H/2, 1, H ])) roiradec = [r[0], r[1], d[2], d[3]] unwise_dir = os.environ['UNWISE_COADDS_DIR'] wise_out = os.path.join(args.outdir, 'images', 'unwise') print('Will write WISE outputs to', wise_out) unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR'] wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr') print('Will write WISE time-resolved outputs to', wise_tr_out) trymakedirs(wise_tr_out) W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_atlas.fits')) print('Read', len(W), 'time-resolved WISE coadd tiles') W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id])) print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth') # Write the time-resolved index subset. W.writeto(os.path.join(wise_tr_out, 'time_resolved_atlas.fits')) # this ought to be enough for anyone =) _,Nepochs = W.epoch_bitmask.shape print('N epochs in time-resolved atlas:', Nepochs) wisedata = [] # full depth for band in [1,2,3,4]: wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band, True)) # time-resolved for band in [1,2]: # W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2) bitmask = (1 << (band-1)) for e in range(Nepochs): # Which tiles have images for this epoch? I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask) if len(I) == 0: continue print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I]) edir = os.path.join(unwise_tr_dir, 'e%03i' % e) eoutdir = os.path.join(wise_tr_out, 'e%03i' % e) wisedata.append((edir, eoutdir, tiles.coadd_id[I], band, False)) wrote_masks = set() model_dir = os.environ.get('UNWISE_MODEL_SKY_DIR') if model_dir is not None: model_dir_out = os.path.join(args.outdir, 'images', 'unwise-mod') trymakedirs(model_dir_out) for indir, outdir, tiles, band, fulldepth in wisedata: for tile in tiles: wanyband = 'w' tim = get_unwise_tractor_image(indir, tile, band, bandname=wanyband, roiradecbox=roiradec) print('Got unWISE tim', tim) print(tim.shape) if model_dir is not None and fulldepth and band in [1,2]: print('ROI', tim.roi) #0387p575.1.mod.fits fn = '%s.%i.mod.fits' % (tile, band) print('Filename', fn) F = fitsio.FITS(os.path.join(model_dir, fn)) x0,x1,y0,y1 = tim.roi slc = slice(y0,y1),slice(x0,x1) phdr = F[0].read_header() outfn = os.path.join(model_dir_out, fn) for e,extname in [(1,'MODEL'), (2,'SKY')]: pix = F[e][slc] hdr = F[e].read_header() crpix1 = hdr['CRPIX1'] crpix2 = hdr['CRPIX2'] hdr['CRPIX1'] -= x0 hdr['CRPIX2'] -= y0 #print('mod', mod) #print('Model', mod.shape) if e == 1: fitsio.write(outfn, None, clobber=True, header=phdr) fitsio.write(outfn, pix, header=hdr, extname=extname) print('Wrote', outfn) thisdir = get_unwise_tile_dir(outdir, tile) print('Directory for this WISE tile:', thisdir) base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band)) print('Base filename:', base) masked = True mu = 'm' if masked else 'u' imfn = base + 'img-%s.fits' % mu ivfn = base + 'invvar-%s.fits.gz' % mu nifn = base + 'n-%s.fits.gz' % mu nufn = base + 'n-u.fits.gz' #print('WISE image header:', tim.hdr) # Adjust the header WCS by x0,y0 wcs = tim.wcs.wcs tim.hdr['CRPIX1'] = wcs.crpix[0] tim.hdr['CRPIX2'] = wcs.crpix[1] H,W = tim.shape tim.hdr['IMAGEW'] = W tim.hdr['IMAGEH'] = H print('WCS:', wcs) print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2']) trymakedirs(imfn, dir=True) fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True) print('Wrote', imfn) fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True) print('Wrote', ivfn) fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True) print('Wrote', nifn) fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True) print('Wrote', nufn) if not (indir,tile) in wrote_masks: print('Looking for mask file for', indir, tile) # record that we tried this dir/tile combo wrote_masks.add((indir,tile)) for idir in indir.split(':'): tdir = get_unwise_tile_dir(idir, tile) maskfn = 'unwise-%s-msk.fits.gz' % tile fn = os.path.join(tdir, maskfn) print('Mask file:', fn) if os.path.exists(fn): print('Reading', fn) (x0,x1,y0,y1) = tim.roi roislice = (slice(y0,y1), slice(x0,x1)) F = fitsio.FITS(fn)[0] hdr = F.read_header() M = F[roislice] outfn = os.path.join(thisdir, maskfn) fitsio.write(outfn, M, header=tim.hdr, clobber=True) print('Wrote', outfn) break outC = outsurvey.get_ccds_readonly() for iccd,ccd in enumerate(outC): outim = outsurvey.get_image_object(ccd) print('Got output image:', outim) otim = outim.get_tractor_image(pixPsf=True, hybridPsf=True, old_calibs_ok=True) print('Got output tim:', otim)
import fitsio from time import sleep import gc fnames = [ '/Users/ch/Downloads/ktwo200002562-c02_lpd-targ.fits.gz', '/Users/ch/Downloads/ktwo200069997-c92_lpd-targ.fits.gz', '/Users/ch/Downloads/ktwo200071159-c91_spd-targ.fits.gz', '/Users/ch/Downloads/ktwo200071208-c91_lpd-targ.fits.gz', '/Users/ch/Downloads/ktwo200083102-c102_lpd-targ.fits.gz', '/Users/ch/Downloads/ktwo200083127-c102_lpd-targ.fits.gz' ] for f in fnames: tpf = fitsio.FITS(f) sleep(1) cadencelist = tpf[1]['CADENCENO'].read() sleep(1) tpf.close() sleep(1)
def readFrame(self, run, camcol, field, band, filename=None): ''' http://data.sdss3.org/datamodel/files/BOSS_PHOTOOBJ/frames/RERUN/RUN/CAMCOL/frame.html ''' f = Frame(run, camcol, field, band) # ... if filename is None: fn = self.getPath('frame', run, camcol, field, band) else: fn = filename # optionally bunzip2 the frame file. tempfn,keep = self._unzip_frame(fn, run, camcol) if tempfn is not None: fn = tempfn if fitsio: print 'Frame filename', fn # eg /clusterfs/riemann/raid006/dr10/boss/photoObj/frames/301/2825/1/frame-u-002825-1-0126.fits.bz2 F = fitsio.FITS(fn, lower=True) f.header = F[0].read_header() # Allow later reading of just the pixels of interest. f.image_proxy = F[0] f.calib = F[1].read() sky = F[2].read_columns(['allsky', 'xinterp', 'yinterp']) #print 'sky', type(sky) # ... supposed to be a recarray, but it's not... f.sky, f.skyxi, f.skyyi = sky.tolist()[0] tab = fits_table(F[3].read()) if not keep and tempfn is not None: os.remove(tempfn) else: p = pyfits.open(fn) # in nanomaggies f.image = p[0].data f.header = p[0].header # converts counts -> nanomaggies f.calib = p[1].data # table with val,x,y -- binned; use bilinear interpolation to expand sky = p[2].data # table -- asTrans structure tab = fits_table(p[3].data) f.sky = sky.field('allsky')[0] f.skyxi = sky.field('xinterp')[0] f.skyyi = sky.field('yinterp')[0] #print 'sky shape', f.sky.shape if len(f.sky.shape) != 2: f.sky = f.sky.reshape((-1, 256)) assert(len(tab) == 1) tab = tab[0] # DR7 has NODE, INCL in radians... f.astrans = AsTrans(run, camcol, field, band, node=np.deg2rad(tab.node), incl=np.deg2rad(tab.incl), astrans=tab, cut_to_band=False) return f
exposure_dir_list += glob.glob( os.path.join(redux_dir, 'exposures', obsdate, '*')) cframe_list = [] # Get a list of all science exposures. for exposure_dir in exposure_dir_list: cframe_list_tmp = glob.glob(os.path.join(exposure_dir, 'cframe-*')) if len(cframe_list_tmp) > 0: if tileid_list is None: cframe_list += cframe_list_tmp else: # only need to check one cframe file in the exposure with fitsio.FITS(cframe_list_tmp[0]) as f: if f[0].read_header()['TILEID'] in tileid_list: cframe_list += cframe_list_tmp cframe_list = sorted(cframe_list) # Gather exposure/petal information cframes = Table() cframes['cframe'] = np.array(cframe_list) cframes['night'] = ' ' cframes['mjd'] = np.zeros(len(cframes), dtype=np.float) - 1.0 cframes['lat'] = np.zeros(len(cframes), dtype=np.float) cframes['lon'] = np.zeros(len(cframes), dtype=np.float) cframes['elv'] = np.zeros(len(cframes), dtype=np.float) cframes['tileid'] = np.zeros(len(cframes), dtype=int)
def write_to(self, fn, columns=None, header='default', primheader=None, use_fitsio=True, append=False, append_to_hdu=None): fitsio = None if use_fitsio: try: import fitsio except: pass if columns is None: columns = self.get_columns() if fitsio: arrays = [self.get(c) for c in columns] fits = fitsio.FITS(fn, 'rw', clobber=(not append)) arrays = [ np.array(a) if isinstance(a, list) else a for a in arrays ] if header == 'default': header = None try: if append and append_to_hdu is not None: fits[append_to_hdu].append(arrays, names=columns, header=header) else: if primheader is not None: fits.write(None, header=primheader) fits.write(arrays, names=columns, header=header) fits.close() except: print('Failed to write FITS table') print('Columns:') for c, a in zip(columns, arrays): print(' ', c, type(a), end='') try: print(a.dtype, a.shape, end='') except: pass print() raise return fc = self.to_fits_columns(columns) T = pyfits.new_table(fc) if header == 'default': header = self._header if header is not None: add_nonstructural_headers(header, T.header) if primheader is not None: P = pyfits.PrimaryHDU() add_nonstructural_headers(primheader, P.header) pyfits.HDUList([P, T]).writeto(fn, clobber=True) else: pyfits_writeto(T, fn)
def fits_table(dataorfn=None, rows=None, hdunum=1, hdu=None, ext=None, header='default', columns=None, column_map=None, lower=True, mmap=True, normalize=True, use_fitsio=True, tabledata_class=tabledata): ''' If 'columns' (a list of strings) is passed, only those columns will be read; otherwise all columns will be read. ''' if dataorfn is None: return tabledata_class(header=header) fitsio = None if use_fitsio: try: import fitsio except: pass pf = None hdr = None # aliases if hdu is not None: hdunum = hdu if ext is not None: hdunum = ext if isinstance(dataorfn, str): if fitsio: F = fitsio.FITS(dataorfn) data = F[hdunum] hdr = data.read_header() else: global pyfits pf = pyfits.open(dataorfn, memmap=mmap) data = pf[hdunum].data if header == 'default': hdr = pf[hdunum].header del pf pf = None else: data = dataorfn if data is None: return None T = tabledata_class(header=hdr) T._columns = [] if fitsio: isrecarray = False try: import pyfits.core # in a try/catch in case pyfits isn't available isrecarray = (type(data) == pyfits.core.FITS_rec) except: try: from astropy.io import fits as pyfits isrecarray = (type(data) == pyfits.core.FITS_rec) except: #import traceback #traceback.print_exc() pass if not isrecarray: try: import pyfits.fitsrec isrecarray = (type(data) == pyfits.fitsrec.FITS_rec) except: try: from astropy.io import fits as pyfits isrecarray = (type(data) == pyfits.fitsrec.FITS_rec) except: import traceback traceback.print_exc() pass #if not isrecarray: # if type(data) == np.recarray: # isrecarray = True if fitsio and not isrecarray: # fitsio sorts the rows and de-duplicates them, so compute # permutation vector 'I' to undo that. I = None if rows is not None: rows, I = np.unique(rows, return_inverse=True) if type(data) == np.ndarray: dd = data if columns is None: columns = data.dtype.fields.keys() else: if data.get_exttype() == 'IMAGE_HDU': # This can happen on empty tables (eg, empty SDSS photoObjs) return None try: dd = data.read(rows=rows, columns=columns, lower=True) except: import sys print('Error reading from FITS object', type(data), data, 'dataorfn', dataorfn, file=sys.stderr) raise if dd is None: return None if columns is None: try: columns = data.get_colnames() except: columns = data.colnames if lower: columns = [c.lower() for c in columns] for c in columns: X = dd[c.lower()] if I is not None: # apply permutation X = X[I] if column_map is not None: c = column_map.get(c, c) if lower: c = c.lower() T.set(c, X) else: if columns is None: columns = data.dtype.names for c in columns: col = data.field(c) if rows is not None: col = col[rows] if normalize: col = normalize_column(col) if column_map is not None: c = column_map.get(c, c) if lower: c = c.lower() T.set(c, col) return T
def produce_forests(self): """ """ userprint("\n") nside = 8 ### Load DRQ vac = fitsio.FITS(self._branchFiles + "/Products/cat.fits") ra = vac[1]["RA"][:] * np.pi / 180. dec = vac[1]["DEC"][:] * np.pi / 180. thid = vac[1]["THING_ID"][:] plate = vac[1]["PLATE"][:] mjd = vac[1]["MJD"][:] fiberid = vac[1]["FIBERID"][:] vac.close() ### Get Healpy pixels pixs = healpy.ang2pix(nside, np.pi / 2. - dec, ra) ### Save master file path = self._branchFiles + "/Products/Spectra/master.fits" head = {} head['NSIDE'] = nside cols = [thid, pixs, plate, mjd, fiberid] names = ['THING_ID', 'PIX', 'PLATE', 'MJD', 'FIBER'] out = fitsio.FITS(path, 'rw', clobber=True) out.write(cols, names=names, header=head, extname="MASTER TABLE") out.close() ### Log lambda grid logl_min = 3.550 logl_max = 4.025 logl_step = 1.e-4 log_lambda = np.arange(logl_min, logl_max, logl_step) ### for p in np.unique(pixs): ### p_thid = thid[(pixs == p)] p_fl = numpy.random.normal(loc=1., scale=1., size=(log_lambda.size, p_thid.size)) p_iv = numpy.random.lognormal(mean=0.1, sigma=0.1, size=(log_lambda.size, p_thid.size)) p_am = np.zeros((log_lambda.size, p_thid.size)).astype(int) p_am[numpy.random.random(size=(log_lambda.size, p_thid.size)) > 0.90] = 1 p_om = np.zeros((log_lambda.size, p_thid.size)).astype(int) ### p_path = self._branchFiles + "/Products/Spectra/pix_" + str( p) + ".fits" out = fitsio.FITS(p_path, 'rw', clobber=True) out.write(p_thid, header={}, extname="THING_ID_MAP") out.write(log_lambda, header={}, extname="LOGLAM_MAP") out.write(p_fl, header={}, extname="FLUX") out.write(p_iv, header={}, extname="IVAR") out.write(p_am, header={}, extname="ANDMASK") out.write(p_om, header={}, extname="ORMASK") out.close() return
#- make sure an import can't accidentally trigger this if __name__ == "__main__": import os, sys import numpy as np import fitsio truthfile = "/global/cfs/cdirs/desi/datachallenge/reference_runs/20.4/targets/52/5299/dark/truth-dark-64-5299.fits" targetfile = "/global/cfs/cdirs/desi/datachallenge/reference_runs/20.4/targets/52/5299/dark/targets-dark-64-5299.fits" if not os.path.exists(truthfile): print(f"ERROR: Unable to find {truthfile}") print("you must run this script at NERSC") sys.exit(1) truth_data = dict() with fitsio.FITS(truthfile) as fx: keep_targetids = list() for objtype in ['BGS', 'ELG', 'LRG', 'QSO', 'STAR', 'WD']: extname = 'TRUTH_' + objtype truth = fx[extname].read()[0:3] #- keep 3 targets per class keep_targetids.extend(truth['TARGETID']) truth_data[extname] = truth truth_hdr = fx['TRUTH'].read_header() truth = fx['TRUTH'].read() flux = fx['FLUX'].read() wave = fx['WAVE'].read() hdr = fx['TRUTH'].read_header() keep = np.in1d(truth['TARGETID'], keep_targetids)
import pylab as plt import numpy as np import sys import os import fitsio import optparse parser = optparse.OptionParser() #parser.add_option('--prefix', help='Plot prefix', default='edge') opt,args = parser.parse_args() for fn in args: #'c4d_131028_014102_ooi_r_v1.fits.fz') print 'Reading', fn F = fitsio.FITS(fn) mfn = fn.replace('_ooi_','_ood_') print 'Reading', mfn M = fitsio.FITS(mfn) print len(F), 'hdus' hdr = F[0].read_header() name = os.path.basename(fn) name = name.replace('.fits','').replace('.fz','') expnum = int(hdr['EXPNUM']) print 'Exposure number', expnum tt = 'Exp %i, %s' % (expnum, name) textargs = dict(ha='left', va='center', fontsize=8) for fig in [1,2,3,4]:
def __init__(self, band, filename = None, fits_file_template = None, timg = None, exposure_num = 0, calib = None, gain = None, darkvar = None, sky = None, frame = None, fits_table = None): self.band = band if fits_file_template: self.band_file = fits_file_template%band self.img = fitsio.FITS(self.band_file)[exposure_num].read() header = fitsio.read_header(self.band_file, ext=exposure_num) elif filename is not None: self.band_file = filename self.img = fitsio.FITS(self.band_file)[exposure_num].read() header = fitsio.read_header(self.band_file, ext=exposure_num) elif timg: self.band_file = None self.img = timg[0].getImage() header = timg[1]['hdr'] self.timg = timg[0] self.invvar = self.timg.getInvvar() else: pass self.header = header self.frame = frame self.fits_table = fits_table # Compute the number of electrons, resource: # http://data.sdss3.org/datamodel/files/BOSS_PHOTOOBJ/frames/RERUN/RUN/CAMCOL/frame.html # (Neither of these look like integers) if fits_file_template or filename: self.dn = self.img / header["CALIB"] + header["SKY"] self.nelec = np.round(self.dn * header["GAIN"]) else: # TODO(awu): what are CALIB and GAIN? self.dn = self.img / calib + sky #timg[0].getSky().val self.nelec = np.round(self.dn * gain) # make nelec immutable - it is constant data!! self.nelec.flags.writeable = False self.shape = self.nelec.shape self.pixel_grid = self.make_pixel_grid() # keep pixel grid around # reference points # TODO: Does CRPIX1 refer to the first axis of self.img ?? self.rho_n = np.array([header['CRPIX1'], header['CRPIX2']]) - 1 # PIXEL REFERENCE POINT (fits stores it 1-based indexing) self.phi_n = np.array([header['CRVAL1'], header['CRVAL2']]) # EQUA REFERENCE POINT self.Ups_n = np.array([[header['CD1_1'], header['CD1_2']], # MATRIX takes you into EQUA TANGENT PLANE [header['CD2_1'], header['CD2_2']]]) self.Ups_n_inv = np.linalg.inv(self.Ups_n) #astrometry wcs object for "exact" x,y to equa ra,dec conversion import astropy.wcs as wcs self.wcs = wcs.WCS(self.header) self.use_wcs = False # set image specific KAPPA and epsilon if fits_file_template: self.kappa = header['GAIN'] # TODO is this right?? self.epsilon = header['SKY'] * self.kappa # background rate self.epsilon0 = self.epsilon # background rate copy (for debuggin) self.darkvar = header['DARKVAR'] # also eventually contributes to mean? self.calib = header['CALIB'] # dn = nmaggies / calib, calib is NMGY else: self.kappa = gain self.epsilon = timg[0].sky.val * self.kappa self.epsilon0 = self.epsilon self.darkvar = darkvar self.calib = calib # point spread function if fits_file_template: psfvec = [header['PSF_P%d'%i] for i in range(18)] else: psfvec = [psf for psf in timg[0].getPsf()] self.weights = np.array(psfvec[0:3]) self.means = np.array(psfvec[3:9]).reshape(3, 2) # one comp mean per row covars = np.array(psfvec[9:]).reshape(3, 3) # [var_k(x), var_k(y), cov_k(x,y)] per row self.covars = np.zeros((3, 2, 2)) self.invcovars = np.zeros((3, 2, 2)) self.logdets = np.zeros(3) for i in range(3): self.covars[i,:,:] = np.array([[ covars[i,0], covars[i,2]], [ covars[i,2], covars[i,1]]]) # cache inverse covariance self.invcovars[i,:,:] = np.linalg.inv(self.covars[i,:,:]) # cache log determinant sign, logdet = np.linalg.slogdet(self.covars[i,:,:]) self.logdets[i] = logdet self.psf_mog = MixtureOfGaussians(means = self.means, covs = self.covars, pis = self.weights) # for a point source in this image, calculate the radius such that # at least 99% of photons from that source will fall within ERROR = 0.001 self.R = calc_bounding_radius(self.weights, self.means, self.covars, ERROR)
def compute_inertia_tensors_projected(snap, reduced=False, inclusion_threshold=1): """ Compute the intertia tensors for all subhalos for the dark matter and stellar components and saves the output for later integration within the database """ print 'Using reduced (distance weighted) tensors', 'yes' * int( reduced), 'no' * int(np.invert(reduced)) # Read the subhalo information h = snap.readsubhalo() # Load the positions and masses of the constituent particles print 'Loading dark matter particles' x = snap.load(1, 'pos', h) m = snap.load(1, 'mass', h) print 'Loading baryon particles' xb = snap.load(4, 'pos', h) mb = snap.load(4, 'mass', h) eigvectors = np.zeros((2, len(h), 2, 2)) eigvalues = np.zeros((2, len(h), 2)) centroids = np.zeros((2, len(h), 3)) spher_pos = np.zeros((2, len(h), 3)) length = np.zeros((2, len(h))) # Will compute for each halo the inertia tensor for i in range(len(h)): if i % 100 == 0: print "Done %d samples" % i # Reject subhalos with less than some threshold occupation number if len(x[i]) < inclusion_threshold: pass else: # Decide whether to weight by the particle mass weights = np.ones(len(x[i])) normFactor = np.double(len(x[i].T[0])) x0 = x[i].T[0] - np.dot(x[i].T[0], weights) / normFactor x1 = x[i].T[1] - np.dot(x[i].T[1], weights) / normFactor # Beware the ambiguous notation here - # These weights define the difference between reduced and standard inertia tensors. # i.e. downweighting particles at the fringes of the subhalo # They have nothing to do with whether the mass weighting is applied. if reduced: wt = (x0 * x0 + x1 * x1) select = (wt != 0) else: wt = np.ones(x0.size) select = wt.astype(bool) normFactor = np.double(len(x[i].T[0][select])) tens = np.zeros((2, 2)) tens[0, 0] = np.dot(weights[select] * x0[select], x0[select] / wt[select]) / normFactor tens[1, 1] = np.dot(weights[select] * x1[select], x1[select] / wt[select]) / normFactor tens[1, 0] = np.dot(weights[select] * x1[select], x0[select] / wt[select]) / normFactor tens[0, 1] = tens[1, 0] # Evaluate the mass-weighted centroid along each axis X = np.trapz(m[i] * x[i].T[0], x[i].T[0]) / np.trapz( m[i], x[i].T[0]) Y = np.trapz(m[i] * x[i].T[1], x[i].T[1]) / np.trapz( m[i], x[i].T[1]) Z = np.trapz(m[i] * x[i].T[2], x[i].T[2]) / np.trapz( m[i], x[i].T[2]) phi = np.arctan2(Y, X) r = np.sqrt(X * X + Y * Y + Z * Z) theta = np.arccos(Z / r) # Compute the eigenvalues of the halos and store the outputs w, v = np.linalg.eigh(tens) eigvalues[0, i] = w eigvectors[0, i] = v spher_pos[0, i] = np.array([r, theta, phi]) centroids[0, i] = np.array([X, Y, Z]) length[0, i] = len(x[i]) if (len(xb[i]) < inclusion_threshold): pass else: weights = np.ones(len(xb[i])) normFactor = np.double(len(xb[i].T[0])) x0 = xb[i].T[0] - np.dot(xb[i].T[0], weights) / normFactor x1 = xb[i].T[1] - np.dot(xb[i].T[1], weights) / normFactor if reduced: wt = (x0 * x0 + x1 * x1) select = (wt != 0) else: wt = np.ones(x0.size) select = wt.astype(bool) normFactor = np.double(len(xb[i].T[0][select])) tens = np.zeros((2, 2)) tens[0, 0] = np.dot(weights[select] * x0[select], x0[select] / wt[select]) / normFactor tens[1, 1] = np.dot(weights[select] * x1[select], x1[select] / wt[select]) / normFactor tens[1, 0] = np.dot(weights[select] * x1[select], x0[select] / wt[select]) / normFactor tens[0, 1] = tens[1, 0] # Evaluate the mass-weighted centroid along each axis X = np.trapz(mb[i] * xb[i].T[0], xb[i].T[0]) / np.trapz( mb[i], xb[i].T[0]) Y = np.trapz(mb[i] * xb[i].T[1], xb[i].T[1]) / np.trapz( mb[i], xb[i].T[1]) Z = np.trapz(mb[i] * xb[i].T[2], xb[i].T[2]) / np.trapz( mb[i], xb[i].T[2]) phi = np.arctan2(Y, X) r = np.sqrt(X * X + Y * Y + Z * Z) theta = np.arccos(Z / r) # Compute the eigenvalues of the halos w, v = np.linalg.eigh(tens) eigvalues[1, i] = w eigvectors[1, i] = v spher_pos[1, i] = np.array([r, theta, phi]) centroids[1, i] = np.array([X, Y, Z]) length[1, i] = len(xb[i]) print "Saving output" if reduced: out = fi.FITS( '/home/ssamurof/massive_black_ii/subhalo_cat_reduced_projected-nthreshold%d.fits' % inclusion_threshold, 'rw') else: out = fi.FITS( '/home/ssamurof/massive_black_ii/subhalo_cat_projected-nthreshold%d.fits' % inclusion_threshold, 'rw') dat = np.zeros(len(h), dtype=[('x', float), ('y', float), ('z', float), ('npart', float), ('lambda1', float), ('lambda2', float), ('a1', float), ('a2', float), ('b1', float), ('b2', float)]) dat['lambda1'] = eigvalues[0].T[0] dat['lambda2'] = eigvalues[0].T[1] dat['a1'] = eigvectors[0].T[0, 0] dat['a2'] = eigvectors[0].T[0, 1] dat['b1'] = eigvectors[0].T[1, 0] dat['b2'] = eigvectors[0].T[1, 1] dat['x'] = centroids[0].T[0] dat['y'] = centroids[0].T[1] dat['z'] = centroids[0].T[2] dat['npart'] = length[0] out.write(dat) out[-1].write_key('EXTNAME', 'dm') dat2 = np.zeros(len(h), dtype=[('x', float), ('y', float), ('z', float), ('npart', float), ('lambda1', float), ('lambda2', float), ('a1', float), ('a2', float), ('b1', float), ('b2', float)]) dat2['lambda1'] = eigvalues[1].T[0] dat2['lambda2'] = eigvalues[1].T[1] dat2['a1'] = eigvectors[1].T[0, 0] dat2['a2'] = eigvectors[1].T[0, 1] dat2['b1'] = eigvectors[1].T[1, 0] dat2['b2'] = eigvectors[1].T[1, 1] dat2['x'] = centroids[1].T[0] dat2['y'] = centroids[1].T[1] dat2['z'] = centroids[1].T[2] dat2['npart'] = length[1] out.write(dat2) out[-1].write_key('EXTNAME', 'baryons') out.close()
def get_column_names(fname): with fitsio.FITS(fname) as infile: hdu = infile[1] return hdu.get_colnames()
def __init__(self, spectrum_file, metadata, sky_mask, mask_jpas=False, mask_jpas_alt=False, rebin_pixels_width=0, extend_pixels=0, noise_increase=1, forbidden_wavelenghts=None): """ Initialize class instance Parameters ---------- spectrum_file : str Name of the fits files containing the spectrum metadata : dict A dictionary with the metadata. Keys should be strings sky_mask : (np.array, float) A tuple containing the array of the wavelengths to mask and the margin used in the masking. Wavelengths separated to wavelength given in the array by less than the margin will be masked mask_jpas : bool - Default: False If set, mask pixels corresponding to filters in trays T3 and T4. Only works if the bin size is 100 Angstroms mask_jpas_alt : bool - Default: False If set, mask pixels corresponding to filters in trays T3* and T4. Only works if the bin size is 100 Angstroms rebin_pixels_width : float, >0 - Default: 0 Width of the new pixel (in Angstroms) extend_pixels : float, >0 - Default: 0 Pixel overlap region (in Angstroms) noise_increase : int, >0 - Default: 1 Adds noise to the spectrum by adding a gaussian random number of width equal to the (noise_amount-1) times the given variance. Then increase the variance by a factor of sqrt(noise_amount) forbidden_wavelengths : list of tuples or None - Default: None If not None, a list containing tuples specifying ranges of wavelengths that will be masked (both ends included). Each tuple must contain the initial and final range of wavelenghts. This is intended to be complementary to the sky mask to limit the wavelength coverage, and hard cuts will be applied """ # check that "specid" is present in metadata if "SPECID" not in metadata.keys(): raise Error("""The property "SPECID" must be present in metadata""") # open fits file spectrum_hdul = fitsio.FITS(spectrum_file) # intialize arrays # The 1.0 mulitplying is added to change type from >4f to np.float # this is required by numba later on wave = 10**spectrum_hdul[1]["LOGLAM"][:] flux = 1.0 * spectrum_hdul[1]["FLUX"][:] ivar = 1.0 * spectrum_hdul[1]["IVAR"][:] super().__init__(flux, ivar, wave, metadata) # compute sky mask masklambda = sky_mask[0] margin = sky_mask[1] self.__skymask = None self.__find_skymask(masklambda, margin) # mask forbidden lines if forbidden_wavelenghts is not None: self.__filter_wavelengths(forbidden_wavelenghts) # store the wavelength, flux and inverse variance as arrays # mask pixels self._ivar[self.__skymask] = 0.0 if noise_increase > 1: self.__add_noise(noise_increase) if rebin_pixels_width > 0: self._flux, self._ivar, self._wave = self.rebin( rebin_pixels_width, extend_pixels=extend_pixels) # JPAS mask if mask_jpas: pos = np.where(~((np.isin( self._wave, [3900, 4000, 4300, 4400, 4700, 4800, 5100, 5200])) | (self._wave >= 7300))) self._wave = self._wave[pos].copy() self._ivar = self._ivar[pos].copy() self._flux = self._flux[pos].copy() elif mask_jpas_alt: pos = np.where(~((np.isin( self._wave, [3800, 4000, 4200, 4400, 4600, 4800, 5000, 5200])) | (self._wave >= 7300))) self._wave = self._wave[pos].copy() self._ivar = self._ivar[pos].copy() self._flux = self._flux[pos].copy() spectrum_hdul.close()
parser.add_argument("--out", type=str, required=True, help="output file") parser.add_argument("--no-dmat", action='store_true', default=False, required=False, help='Use an identity matrix as the distortion matrix.') args = parser.parse_args() for f in args.data: if not os.path.isfile(f): args.data.remove(f) h = fitsio.FITS(args.data[0]) head = h[1].read_header() rp = h[1]['RP'][:] * 0 rt = h[1]['RT'][:] * 0 nb = h[1]['NB'][:] * 0 z = h[1]['Z'][:] * 0 wet = rp * 0 h.close() da = {} we = {} if not args.no_dmat: h = fitsio.FITS(args.data[0].replace('cf', 'dmat')) dm = h[1]['DM'][:] * 0
def read_spec_spplate(p, m, fiber=None, path_spec=None, lambda_min=None, lambda_max=None, cutANDMASK=True, veto_lines=None, flux_calib=None, ivar_calib=None): """ """ path = path_spec + '/{}/spPlate-{}-{}.fits'.format( str(p).zfill(4), str(p).zfill(4), m) h = fitsio.FITS(path) fl = h[0].read() iv = h[1].read() iv *= iv > 0. an = h[2].read() head = h[0].read_header() h.close() if cutANDMASK: iv *= an == 0 ll = head['CRVAL1'] + head['CD1_1'] * sp.arange(head['NAXIS1']) if head['DC-FLAG']: ll = 10**ll w = sp.ones(ll.size, dtype=bool) if not lambda_min is None: w &= ll >= lambda_min if not lambda_max is None: w &= ll <= lambda_max if not veto_lines is None: for lmin, lmax in veto_lines: w &= (ll < lmin) | (ll > lmax) ll = ll[w] fl = fl[:, w] iv = iv[:, w] if not flux_calib is None: correction = flux_calib(ll) fl /= correction[None, :] iv *= correction[None, :]**2 if not ivar_calib is None: correction = ivar_calib(ll) iv /= correction[None, :] if not fiber is None: w = iv[fiber - 1] > 0. ll = ll[w] fl = fl[fiber - 1, w] iv = iv[fiber - 1, w] return ll, fl, iv
y_size = 990 # mm plane_limits = [-x_size, x_size, -y_size, y_size] extent = [ -detector_size / 2 + plane_limits[0], detector_size / 2 + plane_limits[1], -detector_size / 2 + plane_limits[2], detector_size / 2 + plane_limits[3] ] data = np.loadtxt('relative.out', delimiter=' ') dimension = data.shape x = np.linspace(plane_limits[0], plane_limits[1], dimension[1]) y = np.linspace(plane_limits[2], plane_limits[3], dimension[0]) distance = 5.6 * 1e3 # File with the mean template of all events (53551 events) pixels_waveforms = '/Users/lonewolf/Desktop/test_output.fits' with fitsio.FITS(pixels_waveforms, 'r') as file: #print(file['PULSE_TEMPLATE']) time = file['PULSE_TEMPLATE']['time'].read() templates = file['PULSE_TEMPLATE']['original_amplitude'].read() templates = templates.T maximum_array = [] charge_array = [] for pixel in range(0, 1296): arg_max = np.argmax(templates[pixel]) max_in_template = np.max(templates[pixel]) charge = np.sum((templates[pixel])[arg_max - 3:arg_max + 4])
def read_cat(pathData, zmin=None, zmax=None, zkey='Z_VI', extinction=True, stack_obs=False, in_dir=None, nspec=None, rvextinction=3.793, nside=None): """ """ dic = {} h = fitsio.FITS(pathData) if 'PLATE' in h[1].get_colnames(): if 'MJD' in h[1].get_colnames(): lst = { 'PLATE': 'PLATE', 'MJD': 'MJD', 'FIBERID': 'FIBERID', 'THING_ID': 'THING_ID', 'Z': zkey } else: lst = { 'PLATE': 'PLATE', 'MJD': 'SMJD', 'FIBERID': 'FIBER', 'THING_ID': 'BESTID', 'Z': zkey } for k, v in lst.items(): dic[k] = h[1][v][:] else: p, m, f = targetid2platemjdfiber(h[1]['TARGETID'][:]) dic['PLATE'] = p dic['MJD'] = m dic['FIBERID'] = f dic['Z'] = h[1][zkey][:] dic['THING_ID'] = -1 * sp.ones(p.size, dtype=sp.int64) if extinction: dic['G_EXTINCTION'] = h[1]['EXTINCTION'][:][:, 1] w = dic['G_EXTINCTION'] <= 0. if w.sum() != 0.: print('WARNING: some G_EXTINCTION<=0.: {}, {}'.format( w.sum(), sp.unique(dic['G_EXTINCTION'][w]))) dic['G_EXTINCTION'][w] = 0. dic['G_EXTINCTION'] /= rvextinction h.close() dic['TARGETID'] = platemjdfiber2targetid(dic['PLATE'].astype('int64'), dic['MJD'].astype('int64'), dic['FIBERID'].astype('int64')) w = sp.argsort(dic['TARGETID']) for k in dic.keys(): dic[k] = dic[k][w] w = sp.ones(dic['Z'].size, dtype=bool) print('Found {} quasars'.format(w.sum())) w &= dic['Z'] != -1. print('z!=-1: {}'.format(w.sum())) if not zmin is None: w &= dic['Z'] > zmin print('z>zmin: {}'.format(w.sum())) if not zmax is None: w &= dic['Z'] < zmax print('z<zmax: {}'.format(w.sum())) for k in dic.keys(): dic[k] = dic[k][w] print('Found {} quasars'.format(dic['Z'].size)) if not nspec is None and nspec < dic['Z'].size: for k in dic.keys(): dic[k] = dic[k][:nspec] print('Limit to {} quasars'.format(dic['Z'].size)) if stack_obs: spall = glob.glob(os.path.expandvars(in_dir + '/spAll-*.fits')) if len(spall) != 1: print('WARNING: found {} spAll'.format(len(spall))) dic['ALLOBS'] = [[t] for t in dic['TARGETID']] return dic h = fitsio.FITS(spall[0]) print('INFO: reading spAll from {}'.format(spall[0])) thid_spall = h[1][lst['THING_ID']][:] plate_spall = h[1]['PLATE'][:] mjd_spall = h[1]['MJD'][:] fid_spall = h[1]['FIBERID'][:] qual_spall = sp.char.strip(h[1]['PLATEQUALITY'][:].astype(str)) zwarn_spall = h[1]['ZWARNING'][:] h.close() w = dic['THING_ID'] > 0 for k in dic.keys(): dic[k] = dic[k][w] _, w = sp.unique(dic['THING_ID'], return_index=True) for k in dic.keys(): dic[k] = dic[k][w] print('Get unique THING_ID: {}'.format(dic['Z'].size)) w = sp.argsort(dic['THING_ID']) for k in dic.keys(): dic[k] = dic[k][w] w = sp.in1d(thid_spall, dic['THING_ID']) print("INFO: Found {} spectra with required THING_ID".format(w.sum())) w &= qual_spall == 'good' print("INFO: Found {} spectra with 'good' plate".format(w.sum())) ## Removing spectra with the following ZWARNING bits set: ## SKY, LITTLE_COVERAGE, UNPLUGGED, BAD_TARGET, NODATA ## https://www.sdss.org/dr14/algorithms/bitmasks/#ZWARNING bad_zwarnbit = { 0: 'SKY', 1: 'LITTLE_COVERAGE', 7: 'UNPLUGGED', 8: 'BAD_TARGET', 9: 'NODATA' } for zwarnbit, zwarnbit_str in bad_zwarnbit.items(): w &= (zwarn_spall & 2**zwarnbit) == 0 print("INFO: Found {} spectra without {} bit set: {}".format( w.sum(), zwarnbit, zwarnbit_str)) thid = thid_spall[w] plate = plate_spall[w] mjd = mjd_spall[w] fid = fid_spall[w] targetid = platemjdfiber2targetid(plate.astype('int64'), mjd.astype('int64'), fid.astype('int64')) print('INFO: # unique objs: ', dic['THING_ID'].size) print('INFO: # unique objs in spAll: ', sp.unique(thid).size) print('INFO: # spectra: ', w.sum()) if w.sum() == 0: print('INFO: no spectra, exit') sys.exit() w = sp.argsort(thid) thid = thid[w] targetid = targetid[w] dic['ALLOBS'] = [sp.sort(targetid[thid == t]) for t in dic['THING_ID']] w = np.array([len(v) for v in dic['ALLOBS']]) == 0 if sp.any(w): print('WARNING: Some objects have no valid observation') for el in dic['THING_ID'][w]: print('WARNING: {}'.format(el)) return dic
pool.close() if match: pres, mres, pres_ex, mres_ex = zip(*outputs) pres, mres = cut_nones(pres, mres) pres_ex, mres_ex = cut_nones(pres_ex, mres_ex) if rank == 0: dt = [('g1p', 'f8'), ('g1m', 'f8'), ('g1', 'f8'), ('g2p', 'f8'), ('g2m', 'f8'), ('g2', 'f8')] dplus = np.array(pres, dtype=dt) dminus = np.array(mres, dtype=dt) dplus_ex = np.array(pres_ex, dtype=dt) dminus_ex = np.array(mres_ex, dtype=dt) with fitsio.FITS('data.fits', 'rw') as fits: fits.write(dplus, extname='plus') fits.write(dminus, extname='minus') fits.write(dplus_ex, extname='plus_ex') fits.write(dminus_ex, extname='minus_ex') m, msd, c, csd = estimate_m_and_c(pres, mres, 0.02, swap12=SWAP12) m_ex, msd_ex, c_ex, csd_ex = estimate_m_and_c(pres_ex, mres_ex, 0.02, swap12=SWAP12) print("""\ # of sims: {n_sims} noise cancel m : {m:f} +/- {msd:f} noise cancel c : {c:f} +/- {csd:f}""".format( n_sims=len(pres), m=m,
def read_image_header(row, img_file): """Read some information from the image header and write into the df row. """ hdu = 1 # Note: The next line usually works, but fitsio doesn't support CONTINUE lines, which DES # image headers sometimes include. #h = fitsio.read_header(img_file, hdu) # I don't care about any of the lines the sometimes use CONITNUE (e.g. OBSERVER), so I # just remove them and make the header with the rest of the entries. f = fitsio.FITS(img_file) header_list = f[hdu].read_header_list() header_list = [ d for d in header_list if 'CONTINUE' not in d['name'] ] h = fitsio.FITSHDR(header_list) try: date = h['DATE-OBS'] date, time = date.strip().split('T',1) filter = h['FILTER'] filter = filter.split()[0] band = h['BAND'] sat = h['SATURATE'] fwhm = h['FWHM'] ccdnum = int(h['CCDNUM']) detpos = h['DETPOS'].strip() telra = h['TELRA'] teldec = h['TELDEC'] telha = h['HA'] telra = galsim.Angle.from_hms(telra) / galsim.degrees teldec = galsim.Angle.from_dms(teldec) / galsim.degrees telha = galsim.Angle.from_hms(telha) / galsim.degrees dimmseeing = float(h.get('DIMMSEE',-999)) airmass = float(h.get('AIRMASS',-999)) pressure = float(h.get('PRESSURE',-999)) sky = float(h.get('SKYBRITE',-999)) sigsky = float(h.get('SKYSIGMA',-999)) humidity = float(h.get('HUMIDITY',-999)) windspd = float(h.get('WINDSPD',-999)) winddir = float(h.get('WINDDIR',-999)) tiling = int(h.get('TILING',0)) outtemp = float( h.get('OUTTEMP', - 999) ) msurtemp = float(h.get('MSURTEMP',-999)) dT = float(h.get('MSURTEMP',-999)) - float(h.get('OUTTEMP',0) ) hex = int(h.get('HEX',0)) except: print("Cannot read header information from " + img_file) raise row['sat'] = sat row['fits_filter'] = filter row['fwhm'] = fwhm row['fits_ccdnum'] = ccdnum row['airmass'] = airmass row['sky'] = sky row['sigsky'] = sigsky row['humidity'] = humidity row['tiling'] = tiling row['hex'] = hex row['band'] = band row['telra'] = telra row['teldec'] = teldec row['telha'] = telha row['dimmseeing'] = dimmseeing row['dT'] = dT row['outtemp'] = outtemp row['msurtemp'] = msurtemp row['windspd'] = windspd row['winddir'] = winddir
def main(): # pylint: disable-msg=too-many-locals,too-many-branches,too-many-statements """Computes delta field""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description=('Compute the delta field ' 'from a list of spectra')) parser.add_argument('--out-dir', type=str, default=None, required=True, help='Output directory') parser.add_argument('--drq', type=str, default=None, required=True, help='Catalog of objects in DRQ format') parser.add_argument('--in-dir', type=str, default=None, required=True, help='Directory to spectra files') parser.add_argument('--log', type=str, default='input.log', required=False, help='Log input data') parser.add_argument('--iter-out-prefix', type=str, default='iter', required=False, help='Prefix of the iteration file') parser.add_argument('--mode', type=str, default='pix', required=False, help=('Open mode of the spectra files: pix, spec, ' 'spcframe, spplate, desi')) parser.add_argument('--best-obs', action='store_true', required=False, help=('If mode == spcframe, then use only the best ' 'observation')) parser.add_argument( '--single-exp', action='store_true', required=False, help=('If mode == spcframe, then use only one of the ' 'available exposures. If best-obs then choose it ' 'among those contributing to the best obs')) parser.add_argument('--zqso-min', type=float, default=None, required=False, help='Lower limit on quasar redshift from drq') parser.add_argument('--zqso-max', type=float, default=None, required=False, help='Upper limit on quasar redshift from drq') parser.add_argument('--keep-bal', action='store_true', required=False, help='Do not reject BALs in drq') parser.add_argument('--bi-max', type=float, required=False, default=None, help=('Maximum CIV balnicity index in drq (overrides ' '--keep-bal)')) parser.add_argument('--lambda-min', type=float, default=3600., required=False, help='Lower limit on observed wavelength [Angstrom]') parser.add_argument('--lambda-max', type=float, default=5500., required=False, help='Upper limit on observed wavelength [Angstrom]') parser.add_argument('--lambda-rest-min', type=float, default=1040., required=False, help='Lower limit on rest frame wavelength [Angstrom]') parser.add_argument('--lambda-rest-max', type=float, default=1200., required=False, help='Upper limit on rest frame wavelength [Angstrom]') parser.add_argument('--rebin', type=int, default=3, required=False, help=('Rebin wavelength grid by combining this number ' 'of adjacent pixels (ivar weight)')) parser.add_argument('--npix-min', type=int, default=50, required=False, help='Minimum of rebined pixels') parser.add_argument('--dla-vac', type=str, default=None, required=False, help='DLA catalog file') parser.add_argument('--dla-mask', type=float, default=0.8, required=False, help=('Lower limit on the DLA transmission. ' 'Transmissions below this number are masked')) parser.add_argument('--absorber-vac', type=str, default=None, required=False, help='Absorber catalog file') parser.add_argument( '--absorber-mask', type=float, default=2.5, required=False, help=('Mask width on each side of the absorber central ' 'observed wavelength in units of ' '1e4*dlog10(lambda)')) parser.add_argument('--mask-file', type=str, default=None, required=False, help=('Path to file to mask regions in lambda_OBS and ' 'lambda_RF. In file each line is: region_name ' 'region_min region_max (OBS or RF) [Angstrom]')) parser.add_argument('--optical-depth', type=str, default=None, required=False, nargs='*', help=('Correct for the optical depth: tau_1 gamma_1 ' 'absorber_1 tau_2 gamma_2 absorber_2 ...')) parser.add_argument('--dust-map', type=str, default=None, required=False, help=('Path to DRQ catalog of objects for dust map to ' 'apply the Schlegel correction')) parser.add_argument( '--flux-calib', type=str, default=None, required=False, help=('Path to previously produced picca_delta.py file ' 'to correct for multiplicative errors in the ' 'pipeline flux calibration')) parser.add_argument( '--ivar-calib', type=str, default=None, required=False, help=('Path to previously produced picca_delta.py file ' 'to correct for multiplicative errors in the ' 'pipeline inverse variance calibration')) parser.add_argument('--eta-min', type=float, default=0.5, required=False, help='Lower limit for eta') parser.add_argument('--eta-max', type=float, default=1.5, required=False, help='Upper limit for eta') parser.add_argument('--vlss-min', type=float, default=0., required=False, help='Lower limit for variance LSS') parser.add_argument('--vlss-max', type=float, default=0.3, required=False, help='Upper limit for variance LSS') parser.add_argument('--delta-format', type=str, default=None, required=False, help='Format for Pk 1D: Pk1D') parser.add_argument('--use-ivar-as-weight', action='store_true', default=False, help=('Use ivar as weights (implemented as eta = 1, ' 'sigma_lss = fudge = 0)')) parser.add_argument('--use-constant-weight', action='store_true', default=False, help=('Set all the delta weights to one (implemented ' 'as eta = 0, sigma_lss = 1, fudge = 0)')) parser.add_argument('--order', type=int, default=1, required=False, help=('Order of the log10(lambda) polynomial for the ' 'continuum fit, by default 1.')) parser.add_argument('--nit', type=int, default=5, required=False, help=('Number of iterations to determine the mean ' 'continuum shape, LSS variances, etc.')) parser.add_argument('--nproc', type=int, default=None, required=False, help='Number of processors') parser.add_argument('--nspec', type=int, default=None, required=False, help='Maximum number of spectra to read') parser.add_argument('--use-mock-continuum', action='store_true', default=False, help='use the mock continuum for computing the deltas') parser.add_argument('--spall', type=str, default=None, required=False, help=('Path to spAll file')) parser.add_argument('--metadata', type=str, default=None, required=False, help=('Name for table containing forests metadata')) t0 = time.time() args = parser.parse_args() # setup forest class variables Forest.log_lambda_min = np.log10(args.lambda_min) Forest.log_lambda_max = np.log10(args.lambda_max) Forest.log_lambda_min_rest_frame = np.log10(args.lambda_rest_min) Forest.log_lambda_max_rest_frame = np.log10(args.lambda_rest_max) Forest.rebin = args.rebin Forest.delta_log_lambda = args.rebin * 1e-4 # minumum dla transmission Forest.dla_mask_limit = args.dla_mask Forest.absorber_mask_width = args.absorber_mask # Find the redshift range if args.zqso_min is None: args.zqso_min = max(0., args.lambda_min / args.lambda_rest_max - 1.) userprint("zqso_min = {}".format(args.zqso_min)) if args.zqso_max is None: args.zqso_max = max(0., args.lambda_max / args.lambda_rest_min - 1.) userprint("zqso_max = {}".format(args.zqso_max)) #-- Create interpolators for mean quantities, such as #-- Large-scale structure variance : var_lss #-- Pipeline ivar correction error: eta #-- Pipeline ivar correction term : fudge #-- Mean continuum : mean_cont log_lambda_temp = (Forest.log_lambda_min + np.arange(2) * (Forest.log_lambda_max - Forest.log_lambda_min)) log_lambda_rest_frame_temp = ( Forest.log_lambda_min_rest_frame + np.arange(2) * (Forest.log_lambda_max_rest_frame - Forest.log_lambda_min_rest_frame)) Forest.get_var_lss = interp1d(log_lambda_temp, 0.2 + np.zeros(2), fill_value="extrapolate", kind="nearest") Forest.get_eta = interp1d(log_lambda_temp, np.ones(2), fill_value="extrapolate", kind="nearest") Forest.get_fudge = interp1d(log_lambda_temp, np.zeros(2), fill_value="extrapolate", kind="nearest") Forest.get_mean_cont = interp1d(log_lambda_rest_frame_temp, 1 + np.zeros(2)) #-- Check that the order of the continuum fit is 0 (constant) or 1 (linear). if args.order: if (args.order != 0) and (args.order != 1): userprint(("ERROR : invalid value for order, must be eqal to 0 or" "1. Here order = {:d}").format(args.order)) sys.exit(12) #-- Correct multiplicative pipeline flux calibration if args.flux_calib is not None: hdu = fitsio.read(args.flux_calib, ext=1) stack_log_lambda = hdu['loglam'] stack_delta = hdu['stack'] w = (stack_delta != 0.) Forest.correct_flux = interp1d(stack_log_lambda[w], stack_delta[w], fill_value="extrapolate", kind="nearest") #-- Correct multiplicative pipeline inverse variance calibration if args.ivar_calib is not None: hdu = fitsio.read(args.ivar_calib, ext=2) log_lambda = hdu['loglam'] eta = hdu['eta'] Forest.correct_ivar = interp1d(log_lambda, eta, fill_value="extrapolate", kind="nearest") ### Apply dust correction if not args.dust_map is None: userprint("applying dust correction") Forest.extinction_bv_map = io.read_dust_map(args.dust_map) log_file = open(os.path.expandvars(args.log), 'w') # Read data (data, num_data, nside, healpy_pix_ordering) = io.read_data(os.path.expandvars(args.in_dir), args.drq, args.mode, z_min=args.zqso_min, z_max=args.zqso_max, max_num_spec=args.nspec, log_file=log_file, keep_bal=args.keep_bal, bi_max=args.bi_max, best_obs=args.best_obs, single_exp=args.single_exp, pk1d=args.delta_format, spall=args.spall) #-- Add order info for pix in data: for forest in data[pix]: if not forest is None: forest.order = args.order ### Read masks if args.mask_file is not None: args.mask_file = os.path.expandvars(args.mask_file) try: mask = Table.read(args.mask_file, names=('type', 'wave_min', 'wave_max', 'frame'), format='ascii') mask['log_wave_min'] = np.log10(mask['wave_min']) mask['log_wave_max'] = np.log10(mask['wave_max']) except (OSError, ValueError): userprint(("ERROR: Error while reading mask_file " "file {}").format(args.mask_file)) sys.exit(1) else: mask = Table(names=('type', 'wave_min', 'wave_max', 'frame', 'log_wave_min', 'log_wave_max')) ### Mask lines for healpix in data: for forest in data[healpix]: forest.mask(mask) ### Mask absorbers if not args.absorber_vac is None: userprint("INFO: Adding absorbers") absorbers = io.read_absorbers(args.absorber_vac) num_absorbers = 0 for healpix in data: for forest in data[healpix]: if forest.thingid in absorbers: for lambda_absorber in absorbers[forest.thingid]: forest.add_absorber(lambda_absorber) num_absorbers += 1 log_file.write("Found {} absorbers in forests\n".format(num_absorbers)) ### Add optical depth contribution if not args.optical_depth is None: userprint(("INFO: Adding {} optical" "depths").format(len(args.optical_depth) // 3)) assert len(args.optical_depth) % 3 == 0 for index in range(len(args.optical_depth) // 3): tau = float(args.optical_depth[3 * index]) gamma = float(args.optical_depth[3 * index + 1]) lambda_rest_frame = constants.ABSORBER_IGM[args.optical_depth[ 3 * index + 2]] userprint( ("INFO: Adding optical depth for tau = {}, gamma = {}, " "lambda_rest_frame = {} A").format(tau, gamma, lambda_rest_frame)) for healpix in data: for forest in data[healpix]: forest.add_optical_depth(tau, gamma, lambda_rest_frame) ### Mask DLAs if not args.dla_vac is None: userprint("INFO: Adding DLAs") np.random.seed(0) dlas = io.read_dlas(args.dla_vac) num_dlas = 0 for healpix in data: for forest in data[healpix]: if forest.thingid in dlas: for dla in dlas[forest.thingid]: forest.add_dla(dla[0], dla[1], mask) num_dlas += 1 log_file.write("Found {} DLAs in forests\n".format(num_dlas)) ## Apply cuts log_file.write( ("INFO: Input sample has {} " "forests\n").format(np.sum([len(forest) for forest in data.values()]))) remove_keys = [] for healpix in data: forests = [] for forest in data[healpix]: if ((forest.log_lambda is None) or len(forest.log_lambda) < args.npix_min): log_file.write(("INFO: Rejected {} due to forest too " "short\n").format(forest.thingid)) continue if np.isnan((forest.flux * forest.ivar).sum()): log_file.write(("INFO: Rejected {} due to nan " "found\n").format(forest.thingid)) continue if (args.use_constant_weight and (forest.flux.mean() <= 0.0 or forest.mean_snr <= 1.0)): log_file.write(("INFO: Rejected {} due to negative mean or " "too low SNR found\n").format(forest.thingid)) continue forests.append(forest) log_file.write("{} {}-{}-{} accepted\n".format( forest.thingid, forest.plate, forest.mjd, forest.fiberid)) data[healpix][:] = forests if len(data[healpix]) == 0: remove_keys += [healpix] for healpix in remove_keys: del data[healpix] num_forests = np.sum([len(forest) for forest in data.values()]) log_file.write(("INFO: Remaining sample has {} " "forests\n").format(num_forests)) userprint(f"Remaining sample has {num_forests} forests") # Sanity check: all forests must have the attribute log_lambda for healpix in data: for forest in data[healpix]: assert forest.log_lambda is not None t1 = time.time() tmin = (t1 - t0) / 60 userprint('INFO: time elapsed to read data', tmin, 'minutes') # compute fits to the forests iteratively # (see equations 2 to 4 in du Mas des Bourboux et al. 2020) num_iterations = args.nit for iteration in range(num_iterations): context = multiprocessing.get_context('fork') pool = context.Pool(processes=args.nproc) userprint( f"Continuum fitting: starting iteration {iteration} of {num_iterations}" ) #-- Sorting healpix pixels before giving to pool (for some reason) pixels = np.array([k for k in data]) sort = pixels.argsort() sorted_data = [data[k] for k in pixels[sort]] data_fit_cont = pool.map(cont_fit, sorted_data) for index, healpix in enumerate(pixels[sort]): data[healpix] = data_fit_cont[index] userprint( f"Continuum fitting: ending iteration {iteration} of {num_iterations}" ) pool.close() if iteration < num_iterations - 1: #-- Compute mean continuum (stack in rest-frame) (log_lambda_rest_frame, mean_cont, mean_cont_weight) = prep_del.compute_mean_cont(data) w = mean_cont_weight > 0. log_lambda_cont = log_lambda_rest_frame[w] new_cont = Forest.get_mean_cont(log_lambda_cont) * mean_cont[w] Forest.get_mean_cont = interp1d(log_lambda_cont, new_cont, fill_value="extrapolate") #-- Compute observer-frame mean quantities (var_lss, eta, fudge) if not (args.use_ivar_as_weight or args.use_constant_weight): (log_lambda, eta, var_lss, fudge, num_pixels, var_pipe_values, var_delta, var2_delta, count, num_qso, chi2_in_bin, error_eta, error_var_lss, error_fudge) = prep_del.compute_var_stats( data, (args.eta_min, args.eta_max), (args.vlss_min, args.vlss_max)) w = num_pixels > 0 Forest.get_eta = interp1d(log_lambda[w], eta[w], fill_value="extrapolate", kind="nearest") Forest.get_var_lss = interp1d(log_lambda[w], var_lss[w], fill_value="extrapolate", kind="nearest") Forest.get_fudge = interp1d(log_lambda[w], fudge[w], fill_value="extrapolate", kind="nearest") else: num_bins = 10 # this value is arbitrary log_lambda = ( Forest.log_lambda_min + (np.arange(num_bins) + .5) * (Forest.log_lambda_max - Forest.log_lambda_min) / num_bins) if args.use_ivar_as_weight: userprint(("INFO: using ivar as weights, skipping eta, " "var_lss, fudge fits")) eta = np.ones(num_bins) var_lss = np.zeros(num_bins) fudge = np.zeros(num_bins) else: userprint(("INFO: using constant weights, skipping eta, " "var_lss, fudge fits")) eta = np.zeros(num_bins) var_lss = np.ones(num_bins) fudge = np.zeros(num_bins) error_eta = np.zeros(num_bins) error_var_lss = np.zeros(num_bins) error_fudge = np.zeros(num_bins) chi2_in_bin = np.zeros(num_bins) num_pixels = np.zeros(num_bins) var_pipe_values = np.zeros(num_bins) var_delta = np.zeros((num_bins, num_bins)) var2_delta = np.zeros((num_bins, num_bins)) count = np.zeros((num_bins, num_bins)) num_qso = np.zeros((num_bins, num_bins)) Forest.get_eta = interp1d(log_lambda, eta, fill_value='extrapolate', kind='nearest') Forest.get_var_lss = interp1d(log_lambda, var_lss, fill_value='extrapolate', kind='nearest') Forest.get_fudge = interp1d(log_lambda, fudge, fill_value='extrapolate', kind='nearest') ### Read metadata from forests and export it if not args.metadata is None: tab_cont = get_metadata(data) tab_cont.write(args.metadata, format="fits", overwrite=True) stack_log_lambda, stack_delta, stack_weight = prep_del.stack(data) ### Save iter_out_prefix results = fitsio.FITS(args.iter_out_prefix + ".fits.gz", 'rw', clobber=True) header = {} header["NSIDE"] = nside header["PIXORDER"] = healpy_pix_ordering header["FITORDER"] = args.order results.write([stack_log_lambda, stack_delta, stack_weight], names=['loglam', 'stack', 'weight'], header=header, extname='STACK') results.write([log_lambda, eta, var_lss, fudge, num_pixels], names=['loglam', 'eta', 'var_lss', 'fudge', 'nb_pixels'], extname='WEIGHT') results.write([ log_lambda_rest_frame, Forest.get_mean_cont(log_lambda_rest_frame), mean_cont_weight ], names=['loglam_rest', 'mean_cont', 'weight'], extname='CONT') var_pipe_values = np.broadcast_to(var_pipe_values.reshape(1, -1), var_delta.shape) results.write( [var_pipe_values, var_delta, var2_delta, count, num_qso, chi2_in_bin], names=['var_pipe', 'var_del', 'var2_del', 'count', 'nqsos', 'chi2'], extname='VAR') results.close() ### Compute deltas and format them get_stack_delta = interp1d(stack_log_lambda[stack_weight > 0.], stack_delta[stack_weight > 0.], kind="nearest", fill_value="extrapolate") deltas = {} data_bad_cont = [] for healpix in sorted(data.keys()): for forest in data[healpix]: if not forest.bad_cont is None: continue #-- Compute delta field from flux, continuum and various quantites get_delta_from_forest(forest, get_stack_delta, Forest.get_var_lss, Forest.get_eta, Forest.get_fudge, args.use_mock_continuum) if healpix in deltas: deltas[healpix].append(forest) else: deltas[healpix] = [forest] data_bad_cont = data_bad_cont + [ forest for forest in data[healpix] if forest.bad_cont is not None ] for forest in data_bad_cont: log_file.write("INFO: Rejected {} due to {}\n".format( forest.thingid, forest.bad_cont)) log_file.write( ("INFO: Accepted sample has {}" "forests\n").format(np.sum([len(p) for p in deltas.values()]))) t2 = time.time() tmin = (t2 - t1) / 60 userprint('INFO: time elapsed to fit continuum', tmin, 'minutes') ### Save delta for healpix in sorted(deltas.keys()): if args.delta_format == 'Pk1D_ascii': results = open(args.out_dir + "/delta-{}".format(healpix) + ".txt", 'w') for delta in deltas[healpix]: num_pixels = len(delta.delta) if args.mode == 'desi': delta_log_lambda = ( (delta.log_lambda[-1] - delta.log_lambda[0]) / float(len(delta.log_lambda) - 1)) else: delta_log_lambda = delta.delta_log_lambda line = '{} {} {} '.format(delta.plate, delta.mjd, delta.fiberid) line += '{} {} {} '.format(delta.ra, delta.dec, delta.z_qso) line += '{} {} {} {} {} '.format(delta.mean_z, delta.mean_snr, delta.mean_reso, delta_log_lambda, num_pixels) for index in range(num_pixels): line += '{} '.format(delta.delta[index]) for index in range(num_pixels): line += '{} '.format(delta.log_lambda[index]) for index in range(num_pixels): line += '{} '.format(delta.ivar[index]) for index in range(num_pixels): line += '{} '.format(delta.exposures_diff[index]) line += ' \n' results.write(line) results.close() else: results = fitsio.FITS(args.out_dir + "/delta-{}".format(healpix) + ".fits.gz", 'rw', clobber=True) for delta in deltas[healpix]: header = [ { 'name': 'RA', 'value': delta.ra, 'comment': 'Right Ascension [rad]' }, { 'name': 'DEC', 'value': delta.dec, 'comment': 'Declination [rad]' }, { 'name': 'Z', 'value': delta.z_qso, 'comment': 'Redshift' }, { 'name': 'PMF', 'value': '{}-{}-{}'.format(delta.plate, delta.mjd, delta.fiberid) }, { 'name': 'THING_ID', 'value': delta.thingid, 'comment': 'Object identification' }, { 'name': 'PLATE', 'value': delta.plate }, { 'name': 'MJD', 'value': delta.mjd, 'comment': 'Modified Julian date' }, { 'name': 'FIBERID', 'value': delta.fiberid }, { 'name': 'ORDER', 'value': delta.order, 'comment': 'Order of the continuum fit' }, ] if args.delta_format == 'Pk1D': header += [ { 'name': 'MEANZ', 'value': delta.mean_z, 'comment': 'Mean redshift' }, { 'name': 'MEANRESO', 'value': delta.mean_reso, 'comment': 'Mean resolution' }, { 'name': 'MEANSNR', 'value': delta.mean_snr, 'comment': 'Mean SNR' }, ] if args.mode == 'desi': delta_log_lambda = ( (delta.log_lambda[-1] - delta.log_lambda[0]) / float(len(delta.log_lambda) - 1)) else: delta_log_lambda = delta.delta_log_lambda header += [{ 'name': 'DLL', 'value': delta_log_lambda, 'comment': 'Loglam bin size [log Angstrom]' }] exposures_diff = delta.exposures_diff if exposures_diff is None: exposures_diff = delta.log_lambda * 0 cols = [ delta.log_lambda, delta.delta, delta.ivar, exposures_diff ] names = ['LOGLAM', 'DELTA', 'IVAR', 'DIFF'] units = ['log Angstrom', '', '', ''] comments = [ 'Log lambda', 'Delta field', 'Inverse variance', 'Difference' ] else: cols = [ delta.log_lambda, delta.delta, delta.weights, delta.cont ] names = ['LOGLAM', 'DELTA', 'WEIGHT', 'CONT'] units = ['log Angstrom', '', '', ''] comments = [ 'Log lambda', 'Delta field', 'Pixel weights', 'Continuum' ] results.write(cols, names=names, header=header, comment=comments, units=units, extname=str(delta.thingid)) results.close() t3 = time.time() tmin = (t3 - t2) / 60 userprint('INFO: time elapsed to write deltas', tmin, 'minutes') ttot = (t3 - t0) / 60 userprint('INFO: total elapsed time', ttot, 'minutes') log_file.close()
def go( *, seed, gal_imag, gal_type, gal_hlr, ngmix_model, ntrial, output, show=False, layout='grid', loglevel='info', ): """ seed: int Seed for a random number generator ntrial: int Number of trials to run, paired by simulation plus and minus shear output: string Output file path. If output is None, this is a dry run and no output is written. show: bool If True, show some images. Default False loglevel: string Log level, default 'info' """ rng = np.random.RandomState(seed) g1, g2 = 0.0, 0.0 # only over ride bands sim_config = { 'bands': ['r', 'i'], 'psf_type': 'moffat', 'layout': layout, } sim_config = get_sim_config(config=sim_config) logging.basicConfig(stream=sys.stdout) logger = logging.getLogger('mdet_lsst_sim') logger.setLevel(getattr(logging, loglevel.upper())) logger.info(str(sim_config)) dlist = [] for trial in range(ntrial): logger.info('-' * 70) logger.info('trial: %d/%d' % (trial + 1, ntrial)) galaxy_catalog = ColorGalaxyCatalog( rng=rng, coadd_dim=sim_config['coadd_dim'], buff=sim_config['buff'], layout=sim_config['layout'], imag=gal_imag, hlr=gal_hlr, gal_type=gal_type, ) if sim_config['psf_type'] == 'ps': se_dim = get_se_dim(coadd_dim=sim_config['coadd_dim']) psf = make_ps_psf(rng=rng, dim=se_dim) else: psf = make_psf(psf_type=sim_config["psf_type"]) sim_data = make_sim( rng=rng, galaxy_catalog=galaxy_catalog, coadd_dim=sim_config['coadd_dim'], g1=g1, g2=g2, psf=psf, psf_dim=sim_config['psf_dim'], dither=sim_config['dither'], rotate=sim_config['rotate'], bands=sim_config['bands'], epochs_per_band=sim_config['epochs_per_band'], noise_factor=sim_config['noise_factor'], cosmic_rays=sim_config['cosmic_rays'], bad_columns=sim_config['bad_columns'], star_bleeds=sim_config['star_bleeds'], ) if show: show_sim(sim_data['band_data']) mbobs = make_mbobs(band_data=sim_data['band_data'], rng=rng) toutput = run_ngmix( mbobs=mbobs, rng=rng, model=ngmix_model, ) dlist.append(toutput) truth = make_truth(cat=galaxy_catalog, bands=sim_config['bands']) data = eu.numpy_util.combine_arrlist(dlist) logger.info('writing: %s' % output) with fitsio.FITS(output, 'rw', clobber=True) as fits: fits.write(data, extname='model_fits') fits.write(truth, extname='truth')
Array of stellar masses for galaxies """ #read in relevant template info sfh_tot = fitsio.read(template_path, 12) sfh_met = fitsio.read(template_path, 13) mass_tot = fitsio.read(template_path, 17) #get angular diameter distances cosmo = FlatLambdaCDM(100, 0.286) da = cosmo.angular_diameter_distance(z).value smass = np.dot(mass_tot, coeff.T) * (da * 1e6 / 10) ** 2 ssfr = np.dot(coeff, sfh_tot) met = np.dot(coeff, sfh_tot * sfh_met) / ssfr sfr = ssfr * smass[:,np.newaxis] #get the values at z_galaxy met = met[:,-1] sfr = sfr[:,-1] return numpy.vstack((sfr, met, smass)) if __name__=='__main__': kfile = sys.argv[1] #name of file containing filename = sys.argv[2] #name of galaxy catalog file galaxies = fitsio.FITS(filename)[-1].read(columns=['COEFFS','Z','DELTAM']) # read relevant info from files coeffs = galaxies['COEFFS']*10**(galaxies['DELTAM'].reshape(-1,1)/2.5) sfr, met, smass = get_derived_quantities(kfile,coeffs,galaxies['Z'])
def __init__(self, haspairs, npatch, nrbin, oname=None, zcens=None, zmin=None, zmax=None, nzbin=None, rcens=None, **kwargs): """ Container for P(Z) PDF information, including JK-regions Parameters ---------- haspairs : np.array, bool which JK patch has P(Z) PDF extracted npatch : int number of JK-regions nrbin : int number of radial bins oname : str output name for pairs FITS file (LEGACY PARAMETER) zcens : np.array centers for the redshift grid zmin : double minimum redshift of the redshift grid zmax : double maximum redshift of the redshift grid nzbin : int number of redshift bins rcens : np.array radial bin centers Notes ----- In the simplest scenario initialization is done as :: pcont = PDFContainer(haspairs=has_pairsfile, npatch=npatch, nrbin=nrbins, zcens=zcens, pdf_paths=pdf_paths) pcont.nsources = nsources pcont.pdf_subs = pdf_subs pcont.normsub() That is some of the data have to be added after the object is created (placeholder values for these exist alread). To extract P(z) decomposition boost factors continue with e.g.:: bcont = pzboost.PDFContainer.from_file(bname) bcont.decomp_gauss(refprof=paths.params['pzpars']['boost']['refprof'], force_rbmin=paths.params['pzpars']['boost']['rbmin'], force_rbmax=paths.params['pzpars']['boost']['rbmax']) boostdict = bcont.to_boostdict() """ self.oname = oname self.haspairs = haspairs.astype(bool) self.npatch = npatch self.pindexes = np.arange(self.npatch)[self.haspairs] self.nrbin = nrbin self.rbins = np.arange(self.nrbin) self.rcens = rcens if self.rcens is None: self.rcens = shearops.redges( paths.params["radial_bins"]["rmin"], paths.params["radial_bins"]["rmax"], paths.params["radial_bins"]["nbin"])[0] self.pexts = None # FITS data file if self.oname is not None: self.pfits = fio.FITS(self.oname) self.pexts = self._get_patch_exts() # print "5" # pdf binning parameters if zcens is not None and zmin is None and zmax is None and nzbin is None: self.zcens = zcens self.zmin, self.zmax, self.zedges = self._get_zedges() self.nbin = len(self.zcens) elif zcens is None and zmin is not None and zmax is not None and nzbin is not None: self.zmin = zmin self.zmax = zmax self.nbin = nzbin self.zedges, self.zcens = self._get_zarr() else: raise TypeError # print "6" # pdf averaging parameters self.rkey = 'rbin' self.wkey = 'source_weight' self.zkey = 'z_sample' # Data containers self.nsources = np.zeros(shape=(self.nrbin, self.npatch)) self.pdf_subs = np.zeros(shape=(self.nrbin, self.npatch, len(self.zcens))) self.pdfs = None self.pdf_errs = None # WARNING this is really just the values in each patch, not the usual JK-notation (i.e except-th patch) self.pwsum_subs = np.zeros(shape=(self.nrbin, self.npatch, len(self.zcens))) self.wsum_subs = np.zeros(shape=(self.nrbin, self.npatch)) # Additional calculated values self.bw = np.mean(np.diff(self.zedges)) # boost factor initial values self.mean_init = 0.5 self.sigma_init = 0.1 self.amp_init = 0.5 self.mean = BADVAL self.mean_err = BADVAL self.sigma = BADVAL self.sigma_err = BADVAL self.amps = None self.amps_err = None self.amps_cov = None self.boost_rvals = None self.boost = None self.boost_err = None self.boost_cov = None self.point = None self.point_subs = None self.point_cov = None self.goodinds = None # Boost factor bounds self.mean_bounds = (0., np.inf) self.sigma_bounds = (0., np.inf) self.amp_bounds_single = (0., 1.) self.verbose_prefix = ''
def get_spectra(lyafile, nqso=None, wave=None, templateid=None, normfilter='sdss2010-g', seed=None, rand=None, qso=None, add_dlas=False, debug=False, nocolorcuts=False): """Generate a QSO spectrum which includes Lyman-alpha absorption. Args: lyafile (str): name of the Lyman-alpha spectrum file to read. nqso (int, optional): number of spectra to generate (starting from the first spectrum; if more flexibility is needed use TEMPLATEID). wave (numpy.ndarray, optional): desired output wavelength vector. templateid (int numpy.ndarray, optional): indices of the spectra (0-indexed) to read from LYAFILE (default is to read everything). If provided together with NQSO, TEMPLATEID wins. normfilter (str, optional): normalization filter seed (int, optional): Seed for random number generator. rand (numpy.RandomState, optional): RandomState object used for the random number generation. If provided together with SEED, this optional input superseeds the numpy.RandomState object instantiated by SEED. qso (desisim.templates.QSO, optional): object with which to generate individual spectra/templates. add_dlas (bool): Inject damped Lya systems into the Lya forest These are done according to the current best estimates for the incidence dN/dz (Prochaska et al. 2008, ApJ, 675, 1002) Set in calc_lz These are *not* inserted according to overdensity along the sightline nocolorcuts (bool, optional): Do not apply the fiducial rzW1W2 color-cuts cuts (default False). Returns (flux, wave, meta, dla_meta) where: * flux (numpy.ndarray): Array [nmodel, npix] of observed-frame spectra (erg/s/cm2/A). * wave (numpy.ndarray): Observed-frame [npix] wavelength array (Angstrom). * meta (astropy.Table): Table of meta-data [nmodel] for each output spectrum with columns defined in desisim.io.empty_metatable *plus* RA, DEC. * dla_meta (astropy.Table): Table of meta-data [ndla] for the DLAs injected into the spectra. Only returned if add_dlas=True Note: `dla_meta` is only included if add_dlas=True. """ from scipy.interpolate import interp1d import fitsio from speclite.filters import load_filters from desisim.templates import QSO from desisim.io import empty_metatable h = fitsio.FITS(lyafile) if templateid is None: if nqso is None: nqso = len(h) - 1 templateid = np.arange(nqso) else: templateid = np.asarray(templateid) nqso = len(np.atleast_1d(templateid)) if rand is None: rand = np.random.RandomState(seed) templateseed = rand.randint(2**32, size=nqso) #heads = [head.read_header() for head in h[templateid + 1]] heads = [] for indx in templateid: heads.append(h[indx + 1].read_header()) zqso = np.array([head['ZQSO'] for head in heads]) ra = np.array([head['RA'] for head in heads]) dec = np.array([head['DEC'] for head in heads]) mag_g = np.array([head['MAG_G'] for head in heads]) # Hard-coded filtername! normfilt = load_filters(normfilter) if qso is None: qso = QSO(normfilter=normfilter, wave=wave) wave = qso.wave flux = np.zeros([nqso, len(wave)], dtype='f4') meta = empty_metatable(objtype='QSO', nmodel=nqso) meta['TEMPLATEID'] = templateid meta['REDSHIFT'] = zqso meta['MAG'] = mag_g meta['SEED'] = templateseed meta['RA'] = ra meta['DEC'] = dec # Lists for DLA meta data if add_dlas: dla_NHI, dla_z, dla_id = [], [], [] # Loop on quasars for ii, indx in enumerate(templateid): flux1, _, meta1 = qso.make_templates(nmodel=1, redshift=np.atleast_1d(zqso[ii]), mag=np.atleast_1d(mag_g[ii]), seed=templateseed[ii], nocolorcuts=nocolorcuts, lyaforest=False) flux1 *= 1e-17 for col in meta1.colnames: meta[col][ii] = meta1[col][0] # read lambda and forest transmission data = h[indx + 1].read() la = data['LAMBDA'][:] tr = data['FLUX'][:] if len(tr): # Interpolate the transmission at the spectral wavelengths, if # outside the forest, the transmission is 1. itr = interp1d(la, tr, bounds_error=False, fill_value=1.0) flux1 *= itr(wave) # Inject a DLA? if add_dlas: if np.min(wave / 1215.67 - 1) < zqso[ii]: # Any forest? dlas, dla_model = insert_dlas(wave, zqso[ii], seed=templateseed[ii]) ndla = len(dlas) if ndla > 0: flux1 *= dla_model # Meta dla_z += [idla['z'] for idla in dlas] dla_NHI += [idla['N'] for idla in dlas] dla_id += [indx] * ndla padflux, padwave = normfilt.pad_spectrum(flux1, wave, method='edge') normmaggies = np.array( normfilt.get_ab_maggies(padflux, padwave, mask_invalid=True)[normfilter]) factor = 10**(-0.4 * mag_g[ii]) / normmaggies flux1 *= factor for key in ('FLUX_G', 'FLUX_R', 'FLUX_Z', 'FLUX_W1', 'FLUX_W2'): meta[key][ii] *= factor flux[ii, :] = flux1[:] h.close() # Finish if add_dlas: ndla = len(dla_id) if ndla > 0: from astropy.table import Table dla_meta = Table() dla_meta['NHI'] = dla_NHI # log NHI values dla_meta['z'] = dla_z dla_meta['ID'] = dla_id else: dla_meta = None return flux * 1e17, wave, meta, dla_meta else: return flux * 1e17, wave, meta
import numpy as np import matplotlib.pyplot as plt import fitsio plt.rc("font", **{"family": "serif", "serif": ["Computer Modern"]}) plt.rc("text", usetex=True) g_xlabel = "$\lambda (\AA)$" g_ylabel = "$F (units)$" g_fontsize_xlabel = 30 g_fontsize_ylabel = 28 g_file_dpi = 200 g_plot_dpi = 75 fin = fitsio.FITS('/init_directory_spectra_with_BALs/spectra_with_BAL.fits') wave = fin[2].read() flux = fin[3].read() plt.xlabel(g_xlabel, fontsize=g_fontsize_xlabel) plt.ylabel(g_ylabel, fontsize=g_fontsize_ylabel) plt.plot(wave,flux[28,:],color='blue',lw=1,zorder=10) plt.plot(wave,flux[29,:],color='orange',lw=1,zorder=5) plt.savefig("directory/name_plot.png", bbox_inches="tight", dpi=g_file_dpi) plt.show()
def download_single_ccd(path, sdir, expnum, ccd): url_base = 'https://*****:*****@desar2.cosmology.illinois.edu/DESFiles/desarchive/'%ps() """ # need; image_file_name expnum zonenum # no idea what they are yet path In [10]: arr['path'][0].rsplit('/', 3) Out[10]: ['OPS/finalcut/Y2A1/Y3-2379/20160108/D00509375/p01', 'red', 'immask', 'D00509375_z_c39_r2379p01_immasked.fits.fz'] can load the exp_psf_cat_509375.fits blahblah file, go for the path array, which will have the path at desdm """ # outname outname = '{0}/psf_im_{1}_{2}.fits'.format(sdir, expnum, ccd) if os.path.exists(outname): print('skipping {0} because it exists!'.format(outname)) return # Download the files we need: base_path, _, _, image_file_name = path.rsplit('/',3) # strip any spaces on end image_file_name = image_file_name.strip() root, ext = image_file_name.rsplit('_',1) print('root, ext = |%s| |%s|'%(root,ext)) image_file = wget(url_base, base_path + '/red/immask/', sdir, root + '_' + ext) print('image_file = ',image_file) print(time.ctime()) bkg_file = wget(url_base, base_path + '/red/bkg/', sdir, root + '_bkg.fits.fz') print('bkg_file = ',bkg_file) print(time.ctime()) # Unpack the image file if necessary # change the image file name so that it makes life easier unpack_image_file = unpack_file(image_file, outname) print('unpacked to ',unpack_image_file) print(time.ctime()) # Subtract off the background right from the start with fitsio.FITS(unpack_image_file, 'rw') as f: bkg = fitsio.read(bkg_file) #print('after read bkg') img = f[0].read() img -= bkg #print('after subtract bkg') f[0].write(img) #print('after write new img') print('subtracted off background image') print(time.ctime()) pack_file(unpack_image_file) # saves to unpack_image_file + .fz print('packing image') print(time.ctime()) # delete non subtracted files for file_to_remove in [sdir + '/' + root + '_' + ext, sdir + '/' + root + '_bkg.fits.fz', unpack_image_file]: os.remove(file_to_remove)
def cornerplot(theory1, theory2, data1, data2, show_cuts=False): plt.switch_backend("pdf") plt.style.use("y1a1") matplotlib.rcParams["ytick.minor.visible"]=False matplotlib.rcParams["ytick.minor.width"]=0.1 ni,nj=np.genfromtxt("%s/shear_xi/values.txt"%theory1[0]).T[2] ni = int(ni) nj = int(nj) if data1 is not None: data1 = fi.FITS(data1) if data2 is not None: data2 = fi.FITS(data2) rows, cols = ni+1, nj+2 count = 0 for i in xrange(ni): for j in xrange(nj): count+=1 if j>i: continue print i,j xip_a,xim_a = get_real_spectra(i,j,data1) xip_b,xim_b = get_real_spectra(i,j,data2) posp = positions[(i+1,j+1,"+")] ax = plt.subplot(rows,cols,posp) ax.annotate("(%d, %d)"%(i+1,j+1), (2,0.65), textcoords='data', fontsize=11, ) ax.yaxis.set_tick_params(which='minor', left='off', right='off') plt.ylim(-0.4,0.6) #plt.yscale("log") plt.xscale("log") if posp==19: plt.ylabel(r"$\Delta \xi^{ij}_\mathrm{-,GI}(\theta)/\xi^{ij}_\mathrm{-,GG}(\theta)$", fontsize=12) plt.xlabel(r"Angular Separation $\theta$", fontsize=12) plt.yticks(fontsize=12) else: plt.yticks(visible=False) plt.xlabel(r"$\theta$ /arcmin", fontsize=12) plt.xlim(1,210) plt.xticks([10,100],["10", "100"]) #plt.yticks([-2,0,2,4,6,8],['-2', '0', '2', '4', '6', '8']) plt.axhline(0, color='k') xlower,xupper = lims['-'][(i+1,j+1)] plt.axvspan(1e-6, xlower, color='gray',alpha=0.2) plt.axvspan(xupper, 500, color='gray',alpha=0.2) xta,xip_theory_a_ref, xim_theory_a_ref = get_theory_spectra(i,j,theory1[0], xitype='gg') xtb,xip_theory_b_ref, xim_theory_b_ref = get_theory_spectra(i,j,theory2[0], xitype='gg') xta,xip_theory_a_ref_ia, xim_theory_a_ref_ia = get_theory_spectra(i,j,theory1[0], xitype='gi') xtb,xip_theory_b_ref_ia, xim_theory_b_ref_ia = get_theory_spectra(i,j,theory2[0], xitype='gi') linestyles=['-',':','--','-'] for iline,(t1, t2) in enumerate(zip(theory1,theory2)): if iline==0: continue xta,xip_theory_a, xim_theory_a = get_theory_spectra(i,j,t1, xitype='gi') xtb,xip_theory_b, xim_theory_b = get_theory_spectra(i,j,t2, xitype='gi') plt.plot(xta, (xim_theory_a-xim_theory_a_ref_ia)/xim_theory_a_ref, ls=linestyles[iline], color="red") plt.plot(xtb, (xim_theory_b-xim_theory_b_ref_ia)/xim_theory_b_ref, ls=linestyles[iline], color="royalblue") posm = positions[(i+1,j+1,"-")] ax = plt.subplot(rows,cols,posm) ax.annotate("(%d, %d)"%(i+1,j+1), (20,0.25), textcoords='data', fontsize=11, ) ax.yaxis.set_tick_params(which='minor', left='off', right='off') plt.ylim(-0.5,0.55) if posm==30: ax.yaxis.set_label_position("right") ax.yaxis.set_ticks_position("right") plt.yticks(fontsize=12) plt.ylabel(r"$\Delta \xi^{ij}_\mathrm{-,II}(\theta)/\xi^{ij}_\mathrm{-,GG}(\theta)$", fontsize=12) else: plt.yticks(visible=False) #plt.yscale("log") plt.xscale("log") plt.xlim(1,210) plt.xticks([10,100],["10", "100"]) #plt.yticks([-2,0,2,4,6,8],['-2', '0', '2', '4', '6', '8']) plt.axhline(0, color='k') xta,xip_theory_a_ref, xim_theory_a_ref = get_theory_spectra(i,j,theory1[0], xitype='gg') xtb,xip_theory_b_ref, xim_theory_b_ref = get_theory_spectra(i,j,theory2[0], xitype='gg') xta,xip_theory_a_ref_ia, xim_theory_a_ref_ia = get_theory_spectra(i,j,theory1[0], xitype='ii') xtb,xip_theory_b_ref_ia, xim_theory_b_ref_ia = get_theory_spectra(i,j,theory2[0], xitype='ii') xlower,xupper = lims['-'][(i+1, j+1)] plt.axvspan(1e-6, xlower, color='gray',alpha=0.2) plt.axvspan(xupper, 500, color='gray',alpha=0.2) linestyles=['-',':','--','-'] for iline,(t1, t2) in enumerate(zip(theory1,theory2)): if iline==0: continue xta,xip_theory_a, xim_theory_a = get_theory_spectra(i,j,t1, xitype='ii') xtb,xip_theory_b, xim_theory_b = get_theory_spectra(i,j,t2, xitype='ii') plt.plot(xta, (xim_theory_a-xim_theory_a_ref_ia)/xim_theory_a_ref, ls=linestyles[iline], color="red") plt.plot(xtb, (xim_theory_b-xim_theory_b_ref_ia)/xim_theory_b_ref, ls=linestyles[iline], color="royalblue") plt.subplots_adjust(hspace=0,wspace=0) plt.savefig("/home/samuroff/tmp3.pdf")