def __init__( self, expnum=None, ccdname=None, ccdwcs=None, pattern='/project/projectdirs/cosmo/work/ps1/cats/chunks-qz-star-v3/ps1-%(hp)05d.fits' ): """Read PS1 or gaia sources for an exposure number + CCD name or CCD WCS Args: expnum, ccdname: select catalogue with these ccdwcs: or select catalogue with this pattern: absolute path and wildcard for PS1 or Gaia catalogues dr: /project/projectdirs/cosmo/work/ PS1: ${dr}/ps1/cats/chunks-qz-star-v3/ps1-%(hp)05d.fits PS1-Gaia: ${dr}/gaia/chunks-ps1-gaia/chunk-%(hp)05d.fits """ assert ('ps1' in pattern or 'gaia' in pattern) #assert(ps1_or_gaia in ['ps1','ps1_gaia']) #if ps1_or_gaia == 'ps1': # # PS1 "qz" directory # # e.g. /project/projectdirs/cosmo/work/ps1/cats/chunks-qz-star-v2 # self.catdir= os.getenv('PS1CAT_DIR') #elif ps1_or_gaia == 'ps1_gaia': # # PS1-Gaia "qz" matches-only directory # # e.g. /project/projectdirs/cosmo/work/gaia/chunks-ps1-gaia # self.catdir= os.getenv('PS1_GAIA_MATCHES') #fnpattern = os.path.join(self.catdir, prefix + '-%(hp)05d.fits') super(ps1cat, self).__init__(pattern) if ccdwcs is None: from legacypipe.survey import LegacySurveyData survey = LegacySurveyData() ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0] im = survey.get_image_object(ccd) self.ccdwcs = im.get_wcs() else: self.ccdwcs = ccdwcs
def __init__(self, expnum=None, ccdname=None, ccdwcs=None): """Read PS1 or gaia sources for an exposure number + CCD name or CCD WCS Args: expnum, ccdname: select catalogue with these ccdwcs: or select catalogue with this """ self.ps1catdir = os.getenv('PS1CAT_DIR') if self.ps1catdir is None: raise ValueError( 'You must have the PS1CAT_DIR environment variable set to point to healpixed PS1 catalogs' ) fnpattern = os.path.join(self.ps1catdir, 'ps1-%(hp)05d.fits') super(ps1cat, self).__init__(fnpattern) if ccdwcs is None: from legacypipe.survey import LegacySurveyData survey = LegacySurveyData() ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0] im = survey.get_image_object(ccd) self.ccdwcs = im.get_wcs() else: self.ccdwcs = ccdwcs
def __init__(self, expnum=None, ccdname=None, ccdwcs=None): """Initialize the class with either the exposure number *and* CCD name, or directly with the WCS of the CCD of interest. """ # GAIA and PS1 info, gaia for astrometry, ps1 for photometry self.gaiadir = os.getenv('GAIACAT_DIR') # PS1 only self.ps1dir = os.getenv('PS1CAT_DIR') # PS1 only if self.ps1dir is None: raise ValueError('Need PS1CAT_DIR environment variable to be set.') if self.gaiadir is None: print( 'WARNING: GAIACAT_DIR environment variable not set: using Pan-STARRS1 for astrometry' ) self.nside = 32 if ccdwcs is None: from legacypipe.survey import LegacySurveyData survey = LegacySurveyData() ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0] im = survey.get_image_object(ccd) self.ccdwcs = im.get_wcs() else: self.ccdwcs = ccdwcs
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument( '--expnum', type=str, help='Run specified exposure numbers (can be comma-separated list') parser.add_argument( '--all-found', action='store_true', default=False, help='Only write output if all required input files are found') parser.add_argument('--ccds', help='Set ccds.fits file to load, default is all') parser.add_argument('--continue', dest='con', help='Continue even if one exposure is bad', action='store_true', default=False) parser.add_argument('--outdir', help='Output directory, default %(default)s', default='calib') opt = parser.parse_args() survey = LegacySurveyData() if opt.ccds: ccds = fits_table(opt.ccds) ccds = survey.cleanup_ccds_table(ccds) survey.ccds = ccds if opt.expnum is not None: expnums = [(None, int(x, 10)) for x in opt.expnum.split(',')] else: ccds = survey.get_ccds() expnums = set(zip(ccds.camera, ccds.expnum)) print(len(expnums), 'unique camera+expnums') for i, (camera, expnum) in enumerate(expnums): print() print('Exposure', i + 1, 'of', len(expnums), ':', camera, 'expnum', expnum) if camera is None: C = survey.find_ccds(expnum=expnum) print(len(C), 'CCDs with expnum', expnum) camera = C.camera[0] print('Set camera to', camera) C = survey.find_ccds(expnum=expnum, camera=camera) print(len(C), 'CCDs with expnum', expnum, 'and camera', camera) im0 = survey.get_image_object(C[0]) skyoutfn = im0.merged_skyfn psfoutfn = im0.merged_psffn print('Checking for', skyoutfn) print('Checking for', psfoutfn) if os.path.exists(skyoutfn) and os.path.exists(psfoutfn): print('Exposure', expnum, 'is done already') continue if not os.path.exists(skyoutfn): try: merge_splinesky(survey, expnum, C, skyoutfn, opt) except: if not opt.con: raise import traceback traceback.print_exc() print('Exposure failed:', expnum, '. Continuing...') if not os.path.exists(psfoutfn): try: merge_psfex(survey, expnum, C, psfoutfn, opt) except: if not opt.con: raise import traceback traceback.print_exc() print('Exposure failed:', expnum, '. Continuing...')
def main(survey=None, opt=None): print(' '.join(sys.argv)) '''Driver function for forced photometry of individual Legacy Survey images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = tlast = Time() if opt.skip and os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if opt.derivs and opt.agn: print('Sorry, can\'t do --derivs AND --agn') sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0, x1, y0, y1) = opt.zoom zoomslice = (slice(y0, y1), slice(x0, x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing first arg as exposure number (otherwise, it's a filename) try: expnum = int(opt.expnum) filename = None except: # make this 'None' for survey.find_ccds() expnum = None filename = opt.expnum # Try parsing HDU: "all" or HDU name or HDU number. all_hdus = (opt.ccdname == 'all') hdu = -1 ccdname = None if not all_hdus: try: hdu = int(opt.ccdname) except: ccdname = opt.ccdname if survey is None: survey = LegacySurveyData(survey_dir=opt.survey_dir) catsurvey_north = survey catsurvey_south = None if opt.catalog_dir_north is not None: assert (opt.catalog_dir_south is not None) assert (opt.catalog_resolve_dec_ngc is not None) catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north) catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south) if opt.catalog_dir is not None: catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir) if filename is not None and hdu >= 0: # FIXME -- try looking up in CCDs file? # Read metadata from file print('Warning: faking metadata from file contents') T = exposure_metadata([filename], hdus=[hdu]) print('Metadata:') T.about() if not 'ccdzpt' in T.columns(): phdr = fitsio.read_header(filename) T.ccdzpt = np.array([phdr['MAGZERO']]) print('WARNING: using header MAGZERO') T.ccdraoff = np.array([0.]) T.ccddecoff = np.array([0.]) print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.') else: # Read metadata from survey-ccds.fits table T = survey.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and ccdname', ccdname) if hdu >= 0: T.cut(T.image_hdu == hdu) print(len(T), 'with HDU', hdu) if filename is not None: T.cut(np.array([f.strip() == filename for f in T.image_filename])) print(len(T), 'with filename', filename) if opt.camera is not None: T.cut(T.camera == opt.camera) print(len(T), 'with camera', opt.camera) if not all_hdus: assert (len(T) == 1) args = [] for ccd in T: args.append((survey, catsurvey_north, catsurvey_south, opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps)) if opt.threads: from astrometry.util.multiproc import multiproc from astrometry.util.timingpool import TimingPool, TimingPoolMeas pool = TimingPool(opt.threads) poolmeas = TimingPoolMeas(pool, pickleTraffic=False) Time.add_measurement(poolmeas) mp = multiproc(None, pool=pool) tm = Time() FF = mp.map(bounce_one_ccd, args) print('Multi-processing forced-phot:', Time() - tm) else: FF = map(bounce_one_ccd, args) FF = [F for F in FF if F is not None] if len(FF) == 0: print('No photometry results to write.') return 0 # Keep only the first header _, version_hdr = FF[0] FF = [F for F, hdr in FF] F = merge_tables(FF) if all_hdus: version_hdr.delete('CPHDU') version_hdr.delete('CCDNAME') units = { 'exptime': 'sec', 'flux': 'nanomaggy', 'flux_ivar': '1/nanomaggy^2', 'apflux': 'nanomaggy', 'apflux_ivar': '1/nanomaggy^2', 'psfdepth': '1/nanomaggy^2', 'galdepth': '1/nanomaggy^2', 'sky': 'nanomaggy/arcsec^2', 'psfsize': 'arcsec' } if opt.derivs: units.update({ 'dra': 'arcsec', 'ddec': 'arcsec', 'dra_ivar': '1/arcsec^2', 'ddec_ivar': '1/arcsec^2' }) columns = F.get_columns() order = [ 'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum', 'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'ccd_cuts', 'airmass', 'sky', 'psfdepth', 'galdepth', 'ra', 'dec', 'flux', 'flux_ivar', 'fracflux', 'rchisq', 'fracmasked', 'apflux', 'apflux_ivar', 'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar', 'ddec_ivar' ] columns = [c for c in order if c in columns] # Set units headers (must happen after column ordering is set!) hdr = fitsio.FITSHDR() for i, col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.outfn)) fitsio.write(tmpfn, None, header=version_hdr, clobber=True) F.writeto(tmpfn, header=hdr, append=True, columns=columns) os.rename(tmpfn, opt.outfn) print('Wrote', opt.outfn) tnow = Time() print('Total:', tnow - t0) return 0
def main(survey=None, opt=None, args=None): '''Driver function for forced photometry of individual Legacy Survey images. ''' if args is None: args = sys.argv[1:] print('forced_photom.py', ' '.join(args)) if opt is None: parser = get_parser() opt = parser.parse_args(args) import logging if opt.verbose == 0: lvl = logging.INFO else: lvl = logging.DEBUG logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) # tractor logging is *soooo* chatty logging.getLogger('tractor.engine').setLevel(lvl + 10) t0 = Time() if survey is None: survey = LegacySurveyData(survey_dir=opt.survey_dir, cache_dir=opt.cache_dir, output_dir=opt.out_dir) if opt.skip: if opt.out is not None: outfn = opt.out else: outfn = survey.find_file('forced', output=True, camera=opt.camera, expnum=opt.expnum) if os.path.exists(outfn): print('Ouput file exists:', outfn) return 0 if opt.derivs and opt.agn: print('Sorry, can\'t do --derivs AND --agn') return -1 if opt.out is None and opt.out_dir is None: print('Must supply either --out or --out-dir') return -1 if opt.expnum is None and opt.out is None: print('If no --expnum is given, must supply --out filename') return -1 if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0, x1, y0, y1) = opt.zoom zoomslice = (slice(y0, y1), slice(x0, x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Cache CCDs files before the find_ccds call... # Copy required files into the cache? if opt.pre_cache: def copy_files_to_cache(fns): for fn in fns: cachefn = fn.replace(survey.survey_dir, survey.cache_dir) if not cachefn.startswith(survey.cache_dir): print('Skipping', fn) continue outdir = os.path.dirname(cachefn) trymakedirs(outdir) print('Copy', fn) print(' to', cachefn) shutil.copyfile(fn, cachefn) assert (survey.cache_dir is not None) fnset = set() fn = survey.find_file('bricks') fnset.add(fn) fns = survey.find_file('ccd-kds') fnset.update(fns) copy_files_to_cache(fnset) # Read metadata from survey-ccds.fits table ccds = survey.find_ccds(camera=opt.camera, expnum=opt.expnum, ccdname=opt.ccdname) print(len(ccds), 'with camera', opt.camera, 'and expnum', opt.expnum, 'and ccdname', opt.ccdname) # sort CCDs ccds.cut(np.lexsort((ccds.ccdname, ccds.expnum, ccds.camera))) # If there is only one catalog survey_dir, we pass it to get_catalog_in_wcs # as the northern survey. catsurvey_north = survey catsurvey_south = None if opt.catalog_dir_north is not None: assert (opt.catalog_dir_south is not None) assert (opt.catalog_resolve_dec_ngc is not None) catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north) catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south) elif opt.catalog_dir is not None: catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir) # Copy required CCD & calib files into the cache? if opt.pre_cache: assert (survey.cache_dir is not None) fnset = set() for ccd in ccds: im = survey.get_image_object(ccd) for key in im.get_cacheable_filename_variables(): fn = getattr(im, key) if fn is None or not (os.path.exists(fn)): continue fnset.add(fn) copy_files_to_cache(fnset) args = [] for ccd in ccds: args.append((survey, catsurvey_north, catsurvey_south, opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps)) if opt.threads: from astrometry.util.multiproc import multiproc from astrometry.util.timingpool import TimingPool, TimingPoolMeas pool = TimingPool(opt.threads) poolmeas = TimingPoolMeas(pool, pickleTraffic=False) Time.add_measurement(poolmeas) mp = multiproc(None, pool=pool) tm = Time() FF = mp.map(bounce_one_ccd, args) print('Multi-processing forced-phot:', Time() - tm) del mp Time.measurements.remove(poolmeas) del poolmeas pool.close() pool.join() del pool else: FF = map(bounce_one_ccd, args) FF = [F for F in FF if F is not None] if len(FF) == 0: print('No photometry results to write.') return 0 # Keep only the first header _, version_hdr, _, _ = FF[0] # unpack results outlier_masks = [m for _, _, m, _ in FF] outlier_hdrs = [h for _, _, _, h in FF] FF = [F for F, _, _, _ in FF] F = merge_tables(FF) if len(ccds): version_hdr.delete('CPHDU') version_hdr.delete('CCDNAME') from legacypipe.utils import add_bits from legacypipe.bits import DQ_BITS add_bits(version_hdr, DQ_BITS, 'DQMASK', 'DQ', 'D') from legacyzpts.psfzpt_cuts import CCD_CUT_BITS add_bits(version_hdr, CCD_CUT_BITS, 'CCD_CUTS', 'CC', 'C') for i, ap in enumerate(apertures_arcsec): version_hdr.add_record( dict(name='APRAD%i' % i, value=ap, comment='(optical) Aperture radius, in arcsec')) unitmap = { 'exptime': 'sec', 'flux': 'nanomaggy', 'flux_ivar': '1/nanomaggy^2', 'apflux': 'nanomaggy', 'apflux_ivar': '1/nanomaggy^2', 'psfdepth': '1/nanomaggy^2', 'galdepth': '1/nanomaggy^2', 'sky': 'nanomaggy/arcsec^2', 'psfsize': 'arcsec', 'fwhm': 'pixels', 'ccdrarms': 'arcsec', 'ccddecrms': 'arcsec', 'ra': 'deg', 'dec': 'deg', 'skyrms': 'counts/sec', 'dra': 'arcsec', 'ddec': 'arcsec', 'dra_ivar': '1/arcsec^2', 'ddec_ivar': '1/arcsec^2' } columns = F.get_columns() order = [ 'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum', 'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'fwhm', 'ccd_cuts', 'airmass', 'sky', 'skyrms', 'psfdepth', 'galdepth', 'ccdzpt', 'ccdrarms', 'ccddecrms', 'ccdphrms', 'ra', 'dec', 'flux', 'flux_ivar', 'fracflux', 'rchisq', 'fracmasked', 'fracin', 'apflux', 'apflux_ivar', 'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar', 'ddec_ivar' ] columns = [c for c in order if c in columns] units = [unitmap.get(c, '') for c in columns] if opt.out is not None: outdir = os.path.dirname(opt.out) if len(outdir): trymakedirs(outdir) tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.out)) fitsio.write(tmpfn, None, header=version_hdr, clobber=True) F.writeto(tmpfn, units=units, append=True, columns=columns) os.rename(tmpfn, opt.out) print('Wrote', opt.out) else: with survey.write_output('forced', camera=opt.camera, expnum=opt.expnum) as out: F.writeto(None, fits_object=out.fits, primheader=version_hdr, units=units, columns=columns) print('Wrote', out.real_fn) if opt.outlier_mask is not None: # Add outlier bit meanings to the primary header version_hdr.add_record( dict(name='COMMENT', value='Outlier mask bit meanings')) version_hdr.add_record( dict(name='OUTL_POS', value=1, comment='Outlier mask bit for Positive outlier')) version_hdr.add_record( dict(name='OUTL_NEG', value=2, comment='Outlier mask bit for Negative outlier')) if opt.outlier_mask == 'default': outdir = os.path.join(opt.out_dir, 'outlier-masks') camexp = set(zip(ccds.camera, ccds.expnum)) for c, e in camexp: I = np.flatnonzero((ccds.camera == c) * (ccds.expnum == e)) ccd = ccds[I[0]] imfn = ccd.image_filename.strip() outfn = os.path.join(outdir, imfn.replace('.fits', '-outlier.fits')) trymakedirs(outfn, dir=True) tempfn = outfn.replace('.fits', '-tmp.fits') with fitsio.FITS(tempfn, 'rw', clobber=True) as fits: fits.write(None, header=version_hdr) for i in I: mask = outlier_masks[i] _, _, _, meth, tile = survey.get_compression_args( 'outliers_mask', shape=mask.shape) fits.write(mask, header=outlier_hdrs[i], extname=ccds.ccdname[i], compress=meth, tile_dims=tile) os.rename(tempfn, outfn) print('Wrote', outfn) elif opt.outlier_mask is not None: with fitsio.FITS(opt.outlier_mask, 'rw', clobber=True) as F: F.write(None, header=version_hdr) for i, (hdr, mask) in enumerate(zip(outlier_hdrs, outlier_masks)): _, _, _, meth, tile = survey.get_compression_args( 'outliers_mask', shape=mask.shape) F.write(mask, header=hdr, extname=ccds.ccdname[i], compress=meth, tile_dims=tile) print('Wrote', opt.outlier_mask) tnow = Time() print('Total:', tnow - t0) return 0
def psf_residuals(expnum, ccdname, stampsize=35, nstar=30, magrange=(13, 17), verbose=0, splinesky=False): # Set the debugging level. if verbose == 0: lvl = logging.INFO else: lvl = logging.DEBUG logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) pngprefix = 'qapsf-{}-{}'.format(expnum, ccdname) # Gather all the info we need about this CCD. survey = LegacySurveyData() ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0] band = ccd.filter ps1band = dict(g=0, r=1, i=2, z=3, Y=4) print('Band {}'.format(band)) #scales = dict(g=0.0066, r=0.01, z=0.025) #vmin, vmax = np.arcsinh(-1), np.arcsinh(100) #print(scales[band]) im = survey.get_image_object(ccd) iminfo = im.get_image_info() H, W = iminfo['dims'] wcs = im.get_wcs() # Choose a uniformly selected subset of PS1 stars on this CCD. ps1 = ps1cat(ccdwcs=wcs) cat = ps1.get_stars(band=band, magrange=magrange) rand = np.random.RandomState(seed=expnum * ccd.ccdnum) these = rand.choice(len(cat) - 1, nstar, replace=False) #these = rand.random_integers(0,len(cat)-1,nstar) cat = cat[these] cat = cat[np.argsort(cat.median[:, ps1band[band]])] # sort by magnitude #print(cat.nmag_ok) get_tim_kwargs = dict(pixPsf=True, splinesky=splinesky) # Make a QAplot of the positions of all the stars. tim = im.get_tractor_image(**get_tim_kwargs) img = tim.getImage() #img = tim.getImage()/scales[band] fig = plt.figure(figsize=(5, 10)) ax = fig.gca() ax.get_xaxis().get_major_formatter().set_useOffset(False) #ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest', # origin='lower',vmin=vmax,vmax=vmax) ax.imshow(img, **tim.ima) ax.axis('off') ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.format( band, expnum, ccdname, ccd.airmass, ccd.seeing)) for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ax.text(xpos, ypos, '{:2d}'.format(istar + 1), color='red', horizontalalignment='left') circ = plt.Circle((xpos, ypos), radius=30, color='g', fill=False, lw=1) ax.add_patch(circ) #radec = wcs.radec_bounds() #ax.scatter(cat.ra,cat.dec) #ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998]) #ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015]) #ax.set_xlabel('$RA\ (deg)$',fontsize=18) #ax.set_ylabel('$Dec\ (deg)$',fontsize=18) fig.savefig(pngprefix + '-ccd.png', bbox_inches='tight') # Initialize the many-stamp QAplot ncols = 3 nrows = np.ceil(nstar / ncols).astype('int') inchperstamp = 2.0 fig = plt.figure(figsize=(inchperstamp * 3 * ncols, inchperstamp * nrows)) irow = 0 icol = 0 for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) mag = ps1star.median[ps1band[band]] # r-band ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ix, iy = int(xpos), int(ypos) # create a little tractor Image object around the star slc = (slice(max(iy - stampsize, 0), min(iy + stampsize + 1, H)), slice(max(ix - stampsize, 0), min(ix + stampsize + 1, W))) # The PSF model 'const2Psf' is the one used in DR1: a 2-component # Gaussian fit to PsfEx instantiated in the image center. tim = im.get_tractor_image(slc=slc, **get_tim_kwargs) stamp = tim.getImage() ivarstamp = tim.getInvvar() # Initialize a tractor PointSource from PS1 measurements flux = NanoMaggies.magToNanomaggies(mag) star = PointSource(RaDecPos(ra, dec), NanoMaggies(**{band: flux})) # Fit just the source RA,Dec,flux. tractor = Tractor([tim], [star]) tractor.freezeParam('images') print('2-component MOG:', tim.psf) tractor.printThawedParams() for step in range(50): dlnp, X, alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_mog = tractor.getModelImage(0) chi2_mog = -2.0 * tractor.getLogLikelihood() mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0] # Now change the PSF model to a pixelized PSF model from PsfEx instantiated # at this place in the image. psf = PixelizedPsfEx(im.psffn) tim.psf = psf.constantPsfAt(xpos, ypos) #print('PSF model:', tim.psf) #tractor.printThawedParams() for step in range(50): dlnp, X, alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_psfex = tractor.getModelImage(0) chi2_psfex = -2.0 * tractor.getLogLikelihood() mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0] #mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95]) sig = np.std((stamp - model_psfex)[ivarstamp > 0]) mn, mx = [-2.0 * sig, 5 * sig] # Generate a QAplot. if (istar > 0) and (istar % (ncols) == 0): irow = irow + 1 icol = 3 * istar - 3 * ncols * irow #print(istar, irow, icol, icol+1, icol+2) ax1 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol), aspect='equal') ax1.axis('off') #ax1.imshow(stamp, **tim.ima) ax1.imshow(stamp, cmap='gray', interpolation='nearest', origin='lower', vmin=mn, vmax=mx) ax1.text(0.1, 0.9, '{:2d}'.format(istar + 1), color='white', horizontalalignment='left', verticalalignment='top', transform=ax1.transAxes) ax2 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 1), aspect='equal') ax2.axis('off') #ax2.imshow(stamp-model_mog, **tim.ima) ax2.imshow(stamp - model_mog, cmap='gray', interpolation='nearest', origin='lower', vmin=mn, vmax=mx) ax2.text(0.1, 0.9, 'MoG', color='white', horizontalalignment='left', verticalalignment='top', transform=ax2.transAxes) ax2.text(0.08, 0.08, '{:.3f}'.format(mag_mog), color='white', horizontalalignment='left', verticalalignment='bottom', transform=ax2.transAxes) #ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14) #ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex)) ax3 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 2), aspect='equal') ax3.axis('off') #ax3.imshow(stamp-model_psfex, **tim.ima) ax3.imshow(stamp - model_psfex, cmap='gray', interpolation='nearest', origin='lower', vmin=mn, vmax=mx) ax3.text(0.1, 0.9, 'PSFEx', color='white', horizontalalignment='left', verticalalignment='top', transform=ax3.transAxes) ax3.text(0.08, 0.08, '{:.3f}'.format(mag_psfex), color='white', horizontalalignment='left', verticalalignment='bottom', transform=ax3.transAxes) if istar == (nstar - 1): break fig.savefig(pngprefix + '-stargrid.png', bbox_inches='tight')
from astrometry.util.plotutils import * from legacyanalysis.ps1cat import ps1cat from legacypipe.survey import LegacySurveyData from tractor import Image, PointSource, PixPos, NanoMaggies, Tractor ps = PlotSequence('rewcs') expnum, ccdname = 431109, 'N14' cat = ps1cat(expnum=expnum, ccdname=ccdname) stars = cat.get_stars() print len(stars), 'stars' survey = LegacySurveyData() ccd = survey.find_ccds(expnum=expnum,ccdname=ccdname)[0] im = survey.get_image_object(ccd) wcs = im.get_wcs() tim = im.get_tractor_image(pixPsf=True, splinesky=True) margin = 15 ok,stars.xx,stars.yy = wcs.radec2pixelxy(stars.ra, stars.dec) stars.xx -= 1. stars.yy -= 1. W,H = wcs.get_width(), wcs.get_height() stars.ix = np.round(stars.xx).astype(int) stars.iy = np.round(stars.yy).astype(int) stars.cut((stars.ix >= margin) * (stars.ix < (W-margin)) * (stars.iy >= margin) * (stars.iy < (H-margin))) plt.clf()
def psf_residuals(expnum,ccdname,stampsize=35,nstar=30, magrange=(13,17),verbose=0, splinesky=False): # Set the debugging level. if verbose==0: lvl = logging.INFO else: lvl = logging.DEBUG logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout) pngprefix = 'qapsf-{}-{}'.format(expnum,ccdname) # Gather all the info we need about this CCD. survey = LegacySurveyData() ccd = survey.find_ccds(expnum=expnum,ccdname=ccdname)[0] band = ccd.filter ps1band = dict(g=0,r=1,i=2,z=3,Y=4) print('Band {}'.format(band)) #scales = dict(g=0.0066, r=0.01, z=0.025) #vmin, vmax = np.arcsinh(-1), np.arcsinh(100) #print(scales[band]) im = survey.get_image_object(ccd) iminfo = im.get_image_info() H,W = iminfo['dims'] wcs = im.get_wcs() # Choose a uniformly selected subset of PS1 stars on this CCD. ps1 = ps1cat(ccdwcs=wcs) cat = ps1.get_stars(band=band,magrange=magrange) rand = np.random.RandomState(seed=expnum*ccd.ccdnum) these = rand.choice(len(cat)-1,nstar,replace=False) #these = rand.random_integers(0,len(cat)-1,nstar) cat = cat[these] cat = cat[np.argsort(cat.median[:,ps1band[band]])] # sort by magnitude #print(cat.nmag_ok) get_tim_kwargs = dict(pixPsf=True, splinesky=splinesky) # Make a QAplot of the positions of all the stars. tim = im.get_tractor_image(**get_tim_kwargs) img = tim.getImage() #img = tim.getImage()/scales[band] fig = plt.figure(figsize=(5,10)) ax = fig.gca() ax.get_xaxis().get_major_formatter().set_useOffset(False) #ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest', # origin='lower',vmin=vmax,vmax=vmax) ax.imshow(img, **tim.ima) ax.axis('off') ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'. format(band,expnum,ccdname,ccd.airmass,ccd.seeing)) for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ax.text(xpos,ypos,'{:2d}'.format(istar+1),color='red', horizontalalignment='left') circ = plt.Circle((xpos,ypos),radius=30,color='g',fill=False,lw=1) ax.add_patch(circ) #radec = wcs.radec_bounds() #ax.scatter(cat.ra,cat.dec) #ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998]) #ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015]) #ax.set_xlabel('$RA\ (deg)$',fontsize=18) #ax.set_ylabel('$Dec\ (deg)$',fontsize=18) fig.savefig(pngprefix+'-ccd.png',bbox_inches='tight') # Initialize the many-stamp QAplot ncols = 3 nrows = np.ceil(nstar/ncols).astype('int') inchperstamp = 2.0 fig = plt.figure(figsize=(inchperstamp*3*ncols,inchperstamp*nrows)) irow = 0 icol = 0 for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) mag = ps1star.median[ps1band[band]] # r-band ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ix,iy = int(xpos), int(ypos) # create a little tractor Image object around the star slc = (slice(max(iy-stampsize, 0), min(iy+stampsize+1, H)), slice(max(ix-stampsize, 0), min(ix+stampsize+1, W))) # The PSF model 'const2Psf' is the one used in DR1: a 2-component # Gaussian fit to PsfEx instantiated in the image center. tim = im.get_tractor_image(slc=slc, **get_tim_kwargs) stamp = tim.getImage() ivarstamp = tim.getInvvar() # Initialize a tractor PointSource from PS1 measurements flux = NanoMaggies.magToNanomaggies(mag) star = PointSource(RaDecPos(ra,dec), NanoMaggies(**{band: flux})) # Fit just the source RA,Dec,flux. tractor = Tractor([tim], [star]) tractor.freezeParam('images') print('2-component MOG:', tim.psf) tractor.printThawedParams() for step in range(50): dlnp,X,alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_mog = tractor.getModelImage(0) chi2_mog = -2.0*tractor.getLogLikelihood() mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0] # Now change the PSF model to a pixelized PSF model from PsfEx instantiated # at this place in the image. psf = PixelizedPsfEx(im.psffn) tim.psf = psf.constantPsfAt(xpos, ypos) #print('PSF model:', tim.psf) #tractor.printThawedParams() for step in range(50): dlnp,X,alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_psfex = tractor.getModelImage(0) chi2_psfex = -2.0*tractor.getLogLikelihood() mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0] #mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95]) sig = np.std((stamp-model_psfex)[ivarstamp>0]) mn, mx = [-2.0*sig,5*sig] # Generate a QAplot. if (istar>0) and (istar%(ncols)==0): irow = irow+1 icol = 3*istar - 3*ncols*irow #print(istar, irow, icol, icol+1, icol+2) ax1 = plt.subplot2grid((nrows,3*ncols), (irow,icol), aspect='equal') ax1.axis('off') #ax1.imshow(stamp, **tim.ima) ax1.imshow(stamp,cmap='gray',interpolation='nearest', origin='lower',vmin=mn,vmax=mx) ax1.text(0.1,0.9,'{:2d}'.format(istar+1),color='white', horizontalalignment='left',verticalalignment='top', transform=ax1.transAxes) ax2 = plt.subplot2grid((nrows,3*ncols), (irow,icol+1), aspect='equal') ax2.axis('off') #ax2.imshow(stamp-model_mog, **tim.ima) ax2.imshow(stamp-model_mog,cmap='gray',interpolation='nearest', origin='lower',vmin=mn,vmax=mx) ax2.text(0.1,0.9,'MoG',color='white', horizontalalignment='left',verticalalignment='top', transform=ax2.transAxes) ax2.text(0.08,0.08,'{:.3f}'.format(mag_mog),color='white', horizontalalignment='left',verticalalignment='bottom', transform=ax2.transAxes) #ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14) #ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex)) ax3 = plt.subplot2grid((nrows,3*ncols), (irow,icol+2), aspect='equal') ax3.axis('off') #ax3.imshow(stamp-model_psfex, **tim.ima) ax3.imshow(stamp-model_psfex,cmap='gray',interpolation='nearest', origin='lower',vmin=mn,vmax=mx) ax3.text(0.1,0.9,'PSFEx',color='white', horizontalalignment='left',verticalalignment='top', transform=ax3.transAxes) ax3.text(0.08,0.08,'{:.3f}'.format(mag_psfex),color='white', horizontalalignment='left',verticalalignment='bottom', transform=ax3.transAxes) if istar==(nstar-1): break fig.savefig(pngprefix+'-stargrid.png',bbox_inches='tight')
def main(): fn = '/global/cscratch1/sd/dstn/c4d_190730_024955_ori/c4d_190730_024955_ori.52.fits' survey_dir = '/global/cscratch1/sd/dstn/subtractor-survey-dir' imagedir = os.path.join(survey_dir, 'images') trymakedirs(imagedir) calibdir = os.path.join(survey_dir, 'calib') psfexdir = os.path.join(calibdir, 'decam', 'psfex-merged') trymakedirs(psfexdir) skydir = os.path.join(calibdir, 'decam', 'splinesky-merged') trymakedirs(skydir) basename = os.path.basename(fn) basename = basename.replace('.fits', '') # Output filenames for legacyzpts calibration/zeropoint files f, photfn = tempfile.mkstemp() os.close(f) surveyfn = os.path.join(survey_dir, 'survey-ccds-%s.fits.gz' % basename) annfn = os.path.join(survey_dir, 'annotated-%s.fits' % basename) mp = multiproc() survey = LegacySurveyData(survey_dir) # Use the subclass above to handle DECam images! survey.image_typemap.update(decam=GoldsteinDecamImage) # Grab the exposure number and CCD name hdr = fitsio.read_header(fn) expnum = hdr['EXPNUM'] ccdname = hdr['EXTNAME'].strip() print('Exposure', expnum, 'CCD', ccdname) import logging lvl = logging.INFO logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) # tractor logging is *soooo* chatty logging.getLogger('tractor.engine').setLevel(lvl + 10) if not os.path.exists(surveyfn): # Run calibrations and zeropoints runit(fn, photfn, surveyfn, annfn, mp, survey=survey, camera='decam', debug=False, choose_ccd=ccdname, splinesky=True, calibdir=calibdir, measureclass=GoldsteinDecamMeasurer) # Find catalog sources touching this CCD ccds = survey.find_ccds(expnum=expnum, ccdname=ccdname) assert (len(ccds) == 1) ccd = ccds[0] print('Got CCD', ccd) # Create Tractor image im = survey.get_image_object(ccd) print('Got image:', im) # Look at this sub-image, or the whole chip? #zoomslice=None zoomslice = (slice(0, 1000), slice(0, 1000)) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True, hybridPsf=True, normalizePsf=True, old_calibs_ok=True) print('Got tim:', tim) # Read catalog files touching this CCD catsurvey = LegacySurveyData( '/global/project/projectdirs/cosmo/work/legacysurvey/dr8/south') T = get_catalog_in_wcs(tim.subwcs, catsurvey) print('Got', len(T), 'DR8 catalog sources within CCD') # Gaia stars: move RA,Dec to the epoch of this image. I = np.flatnonzero(T.ref_epoch > 0) if len(I): from legacypipe.survey import radec_at_mjd print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd()) ra, dec = radec_at_mjd(T.ra[I], T.dec[I], T.ref_epoch[I].astype(float), T.pmra[I], T.pmdec[I], T.parallax[I], tim.time.toMjd()) T.ra[I] = ra T.dec[I] = dec # Create Tractor Source objects from the catalog cat = read_fits_catalog(T, bands=tim.band) print('Created', len(cat), 'source objects') # Render model image! tr = Tractor([tim], cat) mod = tr.getModelImage(0) # plots ima = dict(interpolation='nearest', origin='lower', vmin=-2 * tim.sig1, vmax=10 * tim.sig1, cmap='gray') plt.clf() plt.imshow(tim.getImage(), **ima) plt.title('Image') plt.savefig('img.jpg') plt.clf() plt.imshow(mod, **ima) plt.title('Model') plt.savefig('mod.jpg') plt.clf() plt.imshow(tim.getImage() - mod, **ima) plt.title('Residual') plt.savefig('res.jpg')
def main(): """Main program. """ import argparse parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( '--force', action='store_true', help='Run calib processes even if files already exist?') parser.add_argument('--survey-dir', help='Override LEGACY_SURVEY_DIR') parser.add_argument( '--expnum', type=str, help='Cut to a single or set of exposures; comma-separated list') parser.add_argument('--extname', '--ccdname', help='Cut to a single extension/CCD name') parser.add_argument('--no-psf', dest='psfex', action='store_false', help='Do not compute PsfEx calibs') parser.add_argument('--no-sky', dest='sky', action='store_false', help='Do not compute sky models') parser.add_argument('--run-se', action='store_true', help='Run SourceExtractor') parser.add_argument('--no-splinesky', dest='splinesky', default=True, action='store_false', help='Use constant, not splinesky') parser.add_argument('--threads', type=int, help='Run multi-threaded', default=None) parser.add_argument('--continue', dest='cont', default=False, action='store_true', help='Continue even if one file fails?') parser.add_argument('--plot-base', help='Make plots with this base filename') parser.add_argument( '--blob-mask-dir', type=str, default=None, help= 'The base directory to search for blob masks during sky model construction' ) parser.add_argument('-v', '--verbose', dest='verbose', action='count', default=0, help='Make more verbose') parser.add_argument('args', nargs=argparse.REMAINDER) opt = parser.parse_args() import logging if opt.verbose: lvl = logging.DEBUG else: lvl = logging.INFO logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) # tractor logging is *soooo* chatty logging.getLogger('tractor.engine').setLevel(lvl + 10) survey = LegacySurveyData(survey_dir=opt.survey_dir) T = None if len(opt.args) == 0: if opt.expnum is not None: expnums = set([int(e) for e in opt.expnum.split(',')]) T = merge_tables([ survey.find_ccds(expnum=e, ccdname=opt.extname) for e in expnums ]) print('Cut to', len(T), 'with expnum in', expnums, 'and extname', opt.extname) opt.args = range(len(T)) else: parser.print_help() return 0 ps = None if opt.plot_base is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plot_base) survey_blob_mask = None if opt.blob_mask_dir is not None: survey_blob_mask = LegacySurveyData(opt.blob_mask_dir) args = [] for a in opt.args: # Check for "expnum-ccdname" format. if '-' in str(a): words = a.split('-') assert (len(words) == 2) expnum = int(words[0]) ccdname = words[1] T = survey.find_ccds(expnum=expnum, ccdname=ccdname) if len(T) != 1: print('Found', len(I), 'CCDs for expnum', expnum, 'CCDname', ccdname, ':', I) print('WARNING: skipping this expnum,ccdname') continue t = T[0] else: i = int(a) print('Index', i) t = T[i] im = survey.get_image_object(t) print('Running', im.name) kwargs = dict(psfex=opt.psfex, sky=opt.sky, ps=ps, survey=survey, survey_blob_mask=survey_blob_mask) if opt.force: kwargs.update(force=True) if opt.run_se: kwargs.update(se=True) if opt.splinesky: kwargs.update(splinesky=True) if opt.cont: kwargs.update(noraise=True) if opt.threads: args.append((im, kwargs)) else: run_calibs((im, kwargs)) if opt.threads: from astrometry.util.multiproc import multiproc mp = multiproc(opt.threads) mp.map(time_run_calibs, args) return 0
from astrometry.util.plotutils import * from legacyanalysis.ps1cat import ps1cat from legacypipe.survey import LegacySurveyData from tractor import Image, PointSource, PixPos, NanoMaggies, Tractor ps = PlotSequence('rewcs') expnum, ccdname = 431109, 'N14' cat = ps1cat(expnum=expnum, ccdname=ccdname) stars = cat.get_stars() print len(stars), 'stars' survey = LegacySurveyData() ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0] im = survey.get_image_object(ccd) wcs = im.get_wcs() tim = im.get_tractor_image(pixPsf=True, splinesky=True) margin = 15 ok, stars.xx, stars.yy = wcs.radec2pixelxy(stars.ra, stars.dec) stars.xx -= 1. stars.yy -= 1. W, H = wcs.get_width(), wcs.get_height() stars.ix = np.round(stars.xx).astype(int) stars.iy = np.round(stars.yy).astype(int) stars.cut((stars.ix >= margin) * (stars.ix < (W - margin)) * (stars.iy >= margin) * (stars.iy < (H - margin))) plt.clf()
def main(survey=None, opt=None): '''Driver function for forced photometry of individual DECam images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0,x1,y0,y1) = opt.zoom zoomslice = (slice(y0,y1), slice(x0,x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.filename) opt.filename = None except: # make this 'None' for survey.find_ccds() expnum = None # Try parsing HDU number try: opt.hdu = int(opt.hdu) ccdname = None except: ccdname = opt.hdu opt.hdu = -1 if survey is None: survey = LegacySurveyData() if opt.filename is not None and opt.hdu >= 0: # Read metadata from file T = exposure_metadata([opt.filename], hdus=[opt.hdu]) print('Metadata:') T.about() else: # Read metadata from survey-ccds.fits table T = survey.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if opt.hdu >= 0: T.cut(T.image_hdu == opt.hdu) print(len(T), 'with HDU', opt.hdu) if opt.filename is not None: T.cut(np.array([f.strip() == opt.filename for f in T.image_filename])) print(len(T), 'with filename', opt.filename) assert(len(T) == 1) ccd = T[0] im = survey.get_image_object(ccd) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True, constant_invvar=opt.constant_invvar) print('Got tim:', tim) print('Read image:', Time()-t0) if opt.catfn in ['DR1', 'DR2', 'DR3']: margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, survey=survey) for b in bricks: # there is some overlap with this brick... read the catalog. fn = survey.find_file('tractor', brick=b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec) W,H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) * (yy >= -margin) * (yy <= (H+margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') T.cut((T.out_of_bounds == False) * (T.left_blob == False)) print('Cut to', len(T), 'on out_of_bounds and left_blob') if len(T): TT.append(T) if len(TT) == 0: print('No sources to photometer.') return 0 T = merge_tables(TT, columns='fillzero') T._header = TT[0]._header del TT # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) surveydir = survey.get_survey_dir() del survey cat = read_fits_catalog(T) # print('Got cat:', cat) print('Read catalog:', Time()-t0) print('Forced photom...') opti = None forced_kwargs = {} if opt.ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 opti = CeresOptimizer(BW=B, BH=B) #forced_kwargs.update(verbose=True) for src in cat: # Limit sizes of huge models from tractor.galaxy import ProfileGalaxy if isinstance(src, ProfileGalaxy): px,py = tim.wcs.positionToPixel(src.getPosition()) h = src._getUnitFluxPatchSize(tim, px, py, tim.modelMinval) MAXHALF = 128 if h > MAXHALF: print('halfsize', h,'for',src,'-> setting to',MAXHALF) src.halfsize = MAXHALF tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') for src in cat: src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) disable_galaxy_cache() F = fits_table() F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32) ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x-1).astype(np.float32) F.y = (y-1).astype(np.float32) if opt.forced: if opt.plots is None: forced_kwargs.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, priors=False, **forced_kwargs) if opt.plots: (data,mod,ie,chi,roi) = R.ims1[0] ima = tim.ima imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5) plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() F.flux = np.array([src.getBrightness().getFlux(tim.band) for src in cat]).astype(np.float32) F.flux_ivar = R.IV.astype(np.float32) F.fracflux = R.fitstats.profracflux.astype(np.float32) F.rchi2 = R.fitstats.prochi2 .astype(np.float32) print('Forced photom:', Time()-t0) if opt.apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() print('Apertures:', apertures, 'pixels') for rad in apertures: aper = photutils.CircularAperture(apxy, rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = ap.astype(np.float32) ap = 1./(np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux_ivar = ap.astype(np.float32) print('Aperture photom:', Time()-t0) program_name = sys.argv[0] version_hdr = get_version_header(program_name, surveydir) filename = getattr(ccd, 'image_filename') if filename is None: # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) filename = os.path.join(d2, d1, fname) print('Trimmed filename to', filename) version_hdr.add_record(dict(name='CPFILE', value=filename, comment='CP file')) version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext')) version_hdr.add_record(dict(name='CAMERA', value=ccd.camera, comment='Camera')) version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='Exposure num')) version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='CCD name')) version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record(dict(name='EXPOSURE', value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname), comment='Name of this image')) keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV', 'INSTRUME'] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = {'exptime':'sec', 'flux':'nanomaggy', 'flux_ivar':'1/nanomaggy^2'} columns = F.get_columns() for i,col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) if opt.save_model or opt.save_data: hdr = fitsio.FITSHDR() tim.getWcs().wcs.add_to_header(hdr) if opt.save_model: print('Getting model image...') mod = tr.getModelImage(tim) fitsio.write(opt.save_model, mod, header=hdr, clobber=True) print('Wrote', opt.save_model) if opt.save_data: fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True) print('Wrote', opt.save_data) print('Finished forced phot:', Time()-t0) return 0
def main(): """Main program. """ import argparse parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( '--force', action='store_true', help='Run calib processes even if files already exist?') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument( '--expnum', type=str, help='Cut to a single or set of exposures; comma-separated list') parser.add_argument('--extname', '--ccdname', help='Cut to a single extension/CCD name') parser.add_argument('--no-psf', dest='psfex', action='store_false', help='Do not compute PsfEx calibs') parser.add_argument('--no-sky', dest='sky', action='store_false', help='Do not compute sky models') parser.add_argument('--run-se', action='store_true', help='Run SourceExtractor') parser.add_argument('--splinesky', action='store_true', help='Spline sky, not constant') parser.add_argument('--threads', type=int, help='Run multi-threaded', default=None) parser.add_argument('--continue', dest='cont', default=False, action='store_true', help='Continue even if one file fails?') parser.add_argument('--plot-base', help='Make plots with this base filename') # actually this doesn't work for calibs... #parser.add_argument('--outdir', dest='output_dir', default=None, # help='Set output base directory') parser.add_argument('args', nargs=argparse.REMAINDER) opt = parser.parse_args() survey = LegacySurveyData() #output_dir=opt.output_dir) T = None if opt.ccds is not None: T = fits_table(opt.ccds) T = survey.cleanup_ccds_table(T) print('Read', len(T), 'from', opt.ccds) #else: # T = survey.get_ccds() # #print len(T), 'CCDs' if len(opt.args) == 0: if opt.expnum is not None: expnums = set([int(e) for e in opt.expnum.split(',')]) #T.cut(np.array([e in expnums for e in T.expnum])) T = merge_tables([ survey.find_ccds(expnum=e, ccdname=opt.extname) for e in expnums ]) print('Cut to', len(T), 'with expnum in', expnums, 'and extname', opt.extname) #if opt.extname is not None: # T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname])) # print('Cut to', len(T), 'with extname =', opt.extname) opt.args = range(len(T)) ps = None if opt.plot_base is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plot_base) args = [] for a in opt.args: # Check for "expnum-ccdname" format. if '-' in str(a): words = a.split('-') assert (len(words) == 2) expnum = int(words[0]) ccdname = words[1] T = survey.find_ccds(expnum=expnum, ccdname=ccdname) if len(T) != 1: print('Found', len(I), 'CCDs for expnum', expnum, 'CCDname', ccdname, ':', I) print('WARNING: skipping this expnum,ccdname') continue t = T[0] else: i = int(a) print('Index', i) t = T[i] #print('CCDnmatch', t.ccdnmatch) #if t.ccdnmatch < 20 and not opt.force: # print('Skipping ccdnmatch = %i' % t.ccdnmatch) # continue im = survey.get_image_object(t) print('Running', im.name) kwargs = dict(psfex=opt.psfex, sky=opt.sky, ps=ps, survey=survey) if opt.force: kwargs.update(force=True) if opt.run_se: kwargs.update(se=True) if opt.splinesky: kwargs.update(splinesky=True) if opt.cont: kwargs.update(noraise=True) if opt.threads: args.append((im, kwargs)) else: run_calibs((im, kwargs)) if opt.threads: from astrometry.util.multiproc import multiproc mp = multiproc(opt.threads) mp.map(time_run_calibs, args) return 0
def main(survey=None, opt=None): '''Driver function for forced photometry of individual Legacy Survey images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if opt.derivs and opt.agn: print('Sorry, can\'t do --derivs AND --agn') sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0, x1, y0, y1) = opt.zoom zoomslice = (slice(y0, y1), slice(x0, x1)) ps = None if opt.plots is not None: import pylab as plt from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.expnum) filename = None except: # make this 'None' for survey.find_ccds() expnum = None filename = opt.expnum # Try parsing HDU number try: hdu = int(opt.ccdname) ccdname = None except: hdu = -1 ccdname = opt.ccdname if survey is None: survey = LegacySurveyData() catsurvey = survey if opt.catalog_dir is not None: catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir) if filename is not None and hdu >= 0: # FIXME -- try looking up in CCDs file? # Read metadata from file print('Warning: faking metadata from file contents') T = exposure_metadata([filename], hdus=[hdu]) print('Metadata:') T.about() if not 'ccdzpt' in T.columns(): phdr = fitsio.read_header(filename) T.ccdzpt = np.array([phdr['MAGZERO']]) print('WARNING: using header MAGZERO') T.ccdraoff = np.array([0.]) T.ccddecoff = np.array([0.]) print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.') else: # Read metadata from survey-ccds.fits table T = survey.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if hdu >= 0: T.cut(T.image_hdu == hdu) print(len(T), 'with HDU', hdu) if filename is not None: T.cut(np.array([f.strip() == filename for f in T.image_filename])) print(len(T), 'with filename', filename) if opt.camera is not None: T.cut(T.camera == opt.camera) print(len(T), 'with camera', opt.camera) assert (len(T) == 1) ccd = T[0] im = survey.get_image_object(ccd) if opt.do_calib: #from legacypipe.survey import run_calibs #kwa = dict(splinesky=True) #run_calibs((im, kwa)) im.run_calibs(splinesky=True) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True, constant_invvar=opt.constant_invvar, hybridPsf=opt.hybrid_psf, normalizePsf=opt.normalize_psf) print('Got tim:', tim) print('Read image:', Time() - t0) if opt.catfn in ['DR1', 'DR2', 'DR3', 'DR5', 'DR']: margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, survey=catsurvey) for b in bricks: # there is some overlap with this brick... read the catalog. fn = catsurvey.find_file('tractor', brick=b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec) W, H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) * (yy >= -margin) * (yy <= (H + margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') for col in ['out_of_bounds', 'left_blob']: if col in T.get_columns(): T.cut(T.get(col) == False) print('Cut to', len(T), 'on', col) if len(T): TT.append(T) if len(TT) == 0: print('No sources to photometer.') return 0 T = merge_tables(TT, columns='fillzero') T._header = TT[0]._header del TT print('Total of', len(T), 'catalog sources') # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) surveydir = survey.get_survey_dir() del survey kwargs = {} cols = T.get_columns() if 'flux_r' in cols and not 'decam_flux_r' in cols: kwargs.update(fluxPrefix='') cat = read_fits_catalog(T, **kwargs) # Replace the brightness (which will be a NanoMaggies with g,r,z) # with a NanoMaggies with this image's band only. for src in cat: src.brightness = NanoMaggies(**{tim.band: 1.}) print('Read catalog:', Time() - t0) print('Forced photom...') F = run_forced_phot(cat, tim, ceres=opt.ceres, derivs=opt.derivs, fixed_also=True, agn=opt.agn, do_forced=opt.forced, do_apphot=opt.apphot, ps=ps) t0 = Time() F.release = T.release F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.camera = np.array([ccd.camera] * len(F)) F.expnum = np.array([im.expnum] * len(F)).astype(np.int32) F.ccdname = np.array([im.ccdname] * len(F)) # "Denormalizing" F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32) F.ra = T.ra F.dec = T.dec ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x - 1).astype(np.float32) F.y = (y - 1).astype(np.float32) h, w = tim.shape F.mask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h - 1), np.clip(np.round(F.x).astype(int), 0, w - 1)] program_name = sys.argv[0] version_hdr = get_version_header(program_name, surveydir) filename = getattr(ccd, 'image_filename') if filename is None: # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) filename = os.path.join(d2, d1, fname) print('Trimmed filename to', filename) version_hdr.add_record( dict(name='CPFILE', value=filename, comment='CP file')) version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext')) version_hdr.add_record( dict(name='CAMERA', value=ccd.camera, comment='Camera')) version_hdr.add_record( dict(name='EXPNUM', value=im.expnum, comment='Exposure num')) version_hdr.add_record( dict(name='CCDNAME', value=im.ccdname, comment='CCD name')) version_hdr.add_record( dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record( dict(name='EXPOSURE', value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname), comment='Name of this image')) keys = [ 'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME' ] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = { 'exptime': 'sec', 'flux': 'nanomaggy', 'flux_ivar': '1/nanomaggy^2' } columns = F.get_columns() for i, col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) if opt.save_model or opt.save_data: hdr = fitsio.FITSHDR() tim.getWcs().wcs.add_to_header(hdr) if opt.save_model: print('Getting model image...') tr = Tractor([tim], cat) mod = tr.getModelImage(tim) fitsio.write(opt.save_model, mod, header=hdr, clobber=True) print('Wrote', opt.save_model) if opt.save_data: fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True) print('Wrote', opt.save_data) print('Finished forced phot:', Time() - t0) return 0