예제 #1
0
 def compare_problematic(self):
     # Assumes j results from self.rename_john_zptfile() 
     j=fits_table('/project/projectdirs/desi/users/burleigh/test_data/old/john_combined_renamed.fits')
     a=fits_table('/project/projectdirs/desi/users/burleigh/test_data/old/arjun_combined.fits')
     # Compare
     discrep_keys=['zpt','zptavg',\
               'skycounts','skyrms','skymag',\
               'transp','phoff','rarms','decrms','raoff','decoff']
            
     panels=len(discrep_keys)
     cols=3
     if panels % cols == 0:
         rows=panels/cols
     else:
         rows=panels/cols+1
     fig,axes= plt.subplots(rows,cols,figsize=(20,10))
     ax=axes.flatten()
     plt.subplots_adjust(hspace=0.4,wspace=0.3)
     cnt=-1
     for key in discrep_keys:
         cnt+=1
         arjun= a.get( self.trans.j2a[key] )
         john= j.get( self.trans.j2a[key] )
         if key == 'skymag':
             arjun=arjun[ j.get('exptime') > 50]
             john=john[ j.get('exptime') > 50]
         ax[cnt].scatter(john,arjun)
         ylab=ax[cnt].set_ylabel('Arjun',fontsize='large')
         xlab=ax[cnt].set_xlabel('%s (John)' % key,fontsize='large')
예제 #2
0
    def setHdus(self, p):
        self.hdus = p
        t = fits_table(p[6].data)
        # the table has only one row...
        assert(len(t) == 1)
        t = t[0]
        #self.table = t
        self.gain = t.gain
        self.dark_variance = t.dark_variance
        self.sky = t.sky
        self.skyerr = t.skyerr
        self.psp_status = t.status
        # Double-Gaussian PSF params
        self.dgpsf_s1 = t.psf_sigma1_2g
        self.dgpsf_s2 = t.psf_sigma2_2g
        self.dgpsf_b  = t.psf_b_2g
        # summary PSF width (sigmas)
        self.psf_fwhm = t.psf_width * (2.*np.sqrt(2.*np.log(2.)))

        # 2-gaussian plus power-law PSF params
        self.plpsf_s1 = t.psf_sigma1
        self.plpsf_s2 = t.psf_sigma2
        self.plpsf_b = t.psf_b
        self.plpsf_p0 = t.psf_p0
        self.plpsf_beta = t.psf_beta
        self.plpsf_sigmap = t.psf_sigmap


        t = fits_table(p[8].data)
        self.per_run_apcorrs = t.ap_corr_run
예제 #3
0
def cat_vcc(req, ver):
    import json
    tag = 'ngc'
    ralo = float(req.GET['ralo'])
    rahi = float(req.GET['rahi'])
    declo = float(req.GET['declo'])
    dechi = float(req.GET['dechi'])

    ver = int(ver)
    if not ver in catversions[tag]:
        raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))

    from astrometry.util.fits import fits_table, merge_tables
    import numpy as np
    from decals import settings

    TT = []
    T = fits_table(os.path.join(settings.DATA_DIR, 'virgo-cluster-cat-2.fits'))
    print(len(T), 'in VCC 2; ra', ralo, rahi, 'dec', declo, dechi)
    T.cut((T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi))
    print(len(T), 'in cut')
    TT.append(T)

    T = fits_table(os.path.join(settings.DATA_DIR, 'virgo-cluster-cat-3.fits'))
    print(len(T), 'in VCC 3; ra', ralo, rahi, 'dec', declo, dechi)
    T.cut((T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi))
    print(len(T), 'in cut')
    T.evcc_id = np.array(['-']*len(T))
    T.rename('id', 'vcc_id')
    TT.append(T)
    T = merge_tables(TT)

    rd = list((float(r),float(d)) for r,d in zip(T.ra, T.dec))
    names = []

    for t in T:
        evcc = t.evcc_id.strip()
        vcc = t.vcc_id.strip()
        ngc = t.ngc.strip()
        nms = []
        if evcc != '-':
            nms.append('EVCC ' + evcc)
        if vcc != '-':
            nms.append('VCC ' + vcc)
        if ngc != '-':
            nms.append('NGC ' + ngc)
        names.append(' / '.join(nms))

    return HttpResponse(json.dumps(dict(rd=rd, name=names)),
                        content_type='application/json')
예제 #4
0
def decals_dr3_check_wcsfailed():
    import fitsio
    basedir = os.environ['LEGACY_SURVEY_DIR']
    image_basedir = os.path.join(basedir, 'images')
    for fn in [#'survey-ccds-decals.fits.gz',
               'survey-ccds-nondecals.fits.gz',
               'survey-ccds-extra.fits.gz',
               ]:
        T = fits_table(fn)
        T.wcscal = np.zeros(len(T), bool)
        fns = np.unique(np.array([f.strip() for f in T.image_filename]))

        H = fits_table()
        H.image_filename = fns
        H.primary_header = []
        
        for ifn,f in enumerate(fns):
            imgfn = os.path.join(image_basedir, f)
            print('Reading', imgfn)
            
            ff = open(imgfn, 'r')
            h = ff.read(32768)
            ff.close()
            hdr = fitsio.FITSHDR()
            hdrstring = ''
            while True:
                line = h[:80]
                h = h[80:]
                # fitsio apparently can't handle CONTINUE
                if line[:8] != 'CONTINUE':
                    hdr.add_record(line)
                hdrstring += line
                if line == ('END' + ' '*77):
                    break
            H.primary_header.append(hdrstring)
                
            #hdr = fitsio.read_header(imgfn)
            expnum = hdr['EXPNUM']
            wcscal = hdr['WCSCAL']
            wcsok = (wcscal == 'Successful')
            print('File', f, 'expnum', expnum, 'WCS cal', wcscal, 'ok', wcsok)
            I = np.flatnonzero(T.expnum == expnum)
            T.wcscal[I] = wcsok

        T.wcsok = T.wcscal.astype(np.uint8)
        T.writeto('new-' + fn)

        H.primary_header = np.array(H.primary_header)
        H.writeto('headers-' + fn)
예제 #5
0
    def getMaskPlane(self, name):
        # Mask planes are described in HDU 11 (the last HDU)
        if self.maskmap is None:
            self.maskmap = {}
            T = fits_table(self.hdus[-1].data)
            T.cut(T.defname == 'S_MASKTYPE')
            for k,v in zip(T.attributename, T.value):
                k = k.replace('S_MASK_', '')
                if k == 'S_NMASK_TYPES':
                    continue
                self.maskmap[k] = v
        if not name in self.maskmap:
            raise RuntimeError('Unknown mask plane \"%s\"' % name)

        return fits_table(self.hdus[1 + self.maskmap[name]].data)
예제 #6
0
def mzls_to_20160315():
    basedir = os.environ['LEGACY_SURVEY_DIR']
    cam = 'mosaic'
    image_basedir = os.path.join(basedir, 'images')
    TT = []

    for fn,dirnms in [
        ('/global/homes/a/arjundey/ZeroPoints/mzls-zpt-all.fits',
         ['CP20160202','CP20160203','CP20160204','CP20160205','CP20160206','CP20160208',
          'CP20160209','CP20160210','CP20160211','CP20160212','CP20160213','CP20160214',
          'CP20160215','CP20160216','CP20160217','CP20160219','CP20160224','CP20160225',
          'CP20160226','CP20160227','CP20160228','CP20160229','CP20160301','CP20160302',
          'CP20160303','CP20160304','CP20160305','CP20160306','CP20160308','CP20160309',
          'CP20160310','CP20160311','CP20160312','CP20160313','CP20160314','CP20160315',
          'CP20160316','CP20160317','CP20160318','CP20160319','CP20160320','CP20160325',
          'CP20160326','CP20160327','CP20160328','CP20160330','CP20160331','CP20160401',
          'CP20160402','CP20160403','CP20160404','CP20160408',]),
        ]:
        T = fits_table(fn)
        normalize_zeropoints(fn, dirnms, image_basedir, cam, T=T)
        TT.append(T)
    T = merge_tables(TT)

    I = np.flatnonzero(T.fwhm == 0)
    if len(I):
        T.fwhm[I] = T.seeing[I] / 0.262

    outfn = 'survey-ccds-mzls-to-20160315.fits'
    T.writeto(outfn)
    print('Wrote', outfn)

    for fn in [outfn]:
        os.system('gzip --best ' + fn)
예제 #7
0
def mzls_dr4(v2=True): 
    if v2:
        zpname='arjuns-ccds-mzls-v2thruMarch19.fits'
        savefn='survey-ccds-mzls-v2thruMarch19.fits.gz'
    else:
        zpname='arjuns-ccds-mzls-v3.fits'
        savefn='survey-ccds-mzls-v3.fits.gz'
    basedir = os.environ['LEGACY_SURVEY_DIR']
    cam = 'mosaic'
    image_basedir = os.path.join(basedir, 'images')
    TT = []

    # mzls_cpdirs_...txt is list of all possible CP directories
    for fn,dirnms in [
        (zpname,
         list(np.loadtxt('mzls_cpdirs_thruMarch19.txt',dtype=str))),
        ]:
        T = fits_table(fn)
        normalize_zeropoints(fn, dirnms, image_basedir, cam, T=T)
        TT.append(T)
    T = merge_tables(TT)

    I = np.flatnonzero(T.fwhm == 0)
    if len(I):
        T.fwhm[I] = T.seeing[I] / 0.262

    T.writeto(savefn)
    print('Wrote', savefn)
예제 #8
0
def decals_dr3_plus():
    basedir = os.environ['LEGACY_SURVEY_DIR']
    cam = 'decam'
    image_basedir = os.path.join(basedir, 'images')

    TT = []

    expnums = [547257, 535154, 535106]
    for fn,dirnms in [
        ('/global/homes/a/arjundey/ZeroPoints/decals-zpt-20160407.fits',
         ['CP20160407',]),
        ('/global/homes/a/arjundey/ZeroPoints/decals-zpt-20160606.fits',
         ['CP20160606',]),
        ]:
        T = fits_table(fn)
        T.cut(np.nonzero([e not in expnums for e in T.expnum])[0])
        normalize_zeropoints(fn, dirnms, image_basedir, cam, T=T)
        TT.append(T)

    dirnms = ['CP20160407','CP20160606']
    for fn in [
        '/global/cscratch1/sd/arjundey/ZeroPoints/zeropoint-c4d_160408_023844_oki_r_v1.fits',
        '/global/cscratch1/sd/arjundey/ZeroPoints/zeropoint-c4d_160408_001343_oki_r_v1.fits',
        '/global/cscratch1/sd/arjundey/ZeroPoints/zeropoint-c4d_160607_023641_oki_g_v1.fits',
        ]:
        T = normalize_zeropoints(fn, dirnms, image_basedir, cam)
        TT.append(T)

    T = merge_tables(TT)
    outfn = 'survey-ccds-dr3plus.fits'
    T.writeto(outfn)
    print('Wrote', outfn)

    for fn in [outfn]:
        os.system('gzip --best ' + fn)
예제 #9
0
def cat_phat_clusters(req, ver):
    import json
    from astrometry.util.fits import fits_table, merge_tables

    tag = 'phat-clusters'
    ralo = float(req.GET['ralo'])
    rahi = float(req.GET['rahi'])
    declo = float(req.GET['declo'])
    dechi = float(req.GET['dechi'])

    ver = int(ver)
    if not ver in catversions[tag]:
        raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))

    cat = fits_table(os.path.join(settings.DATA_DIR, 'phat-clusters.fits'))
    cat.cut((cat.ra  >= ralo ) * (cat.ra  <= rahi) *
            (cat.dec >= declo) * (cat.dec <= dechi))

    return HttpResponse(json.dumps(dict(
        name=[str(s.strip()) for s in cat.name],
        rd=[(float(o.ra),float(o.dec)) for o in cat],
        mag=[float(o.mag) for o in cat],
        young=[bool(o.young) for o in cat],
        velocity=[float(o.velocity) for o in cat],
        metallicity=[float(o.metallicity) for o in cat],
    )),
                        content_type='application/json')
예제 #10
0
def catalog_to_fits(cat):
    from astrometry.util.fits import fits_table
    import numpy as np

    cols = ['ra','dec','g','r','z','w1','w2','w3','w4',
            'brickid', 'brickname', 'objid', 'type',]
    values = cat.values_list(*cols)

    def convert_nan(x):
        if x is None:
            return np.nan
        return x

    T = fits_table()
    for i,c in enumerate(cols):
        v = values[0][i]
        convert = None
        if isinstance(v, unicode):
            convert = str
        if isinstance(v, float):
            convert = convert_nan
        #print('Type of column', c, 'is', type(v), 'eg', v)
        cname = c
        if convert is None:
            T.set(cname, np.array([v[i] for v in values]))
        else:
            T.set(cname, np.array([convert(v[i]) for v in values]))

    return T
예제 #11
0
파일: psfex.py 프로젝트: lyczek/tractor
 def __init__(self, fn=None, ext=1):
      if fn is not None:
         from astrometry.util.fits import fits_table
         T = fits_table(fn, ext=ext)
         ims = T.psf_mask[0]
         print('Got', ims.shape, 'PSF images')
         hdr = T.get_header()
         # PSF distortion bases are polynomials of x,y
         assert(hdr['POLNAME1'].strip() == 'X_IMAGE')
         assert(hdr['POLNAME2'].strip() == 'Y_IMAGE')
         assert(hdr['POLGRP1'] == 1)
         assert(hdr['POLGRP2'] == 1)
         assert(hdr['POLNGRP' ] == 1)
         x0     = hdr.get('POLZERO1')
         xscale = hdr.get('POLSCAL1')
         y0     = hdr.get('POLZERO2')
         yscale = hdr.get('POLSCAL2')
         degree = hdr.get('POLDEG1')
         self.sampling = hdr.get('PSF_SAMP')
         print('PsfEx sampling:', self.sampling)
         # number of terms in polynomial
         ne = (degree + 1) * (degree + 2) / 2
         assert(hdr['PSFAXIS3'] == ne)
         assert(len(ims.shape) == 3)
         assert(ims.shape[0] == ne)
         self.psfbases = ims
         self.xscale, self.yscale = xscale, yscale
         self.degree = degree
         print('PsfEx degree:', self.degree)
         bh,bw = self.psfbases[0].shape
         self.radius = (bh+1)/2.
         self.x0,self.y0 = x0,y0
예제 #12
0
    def get_ccds(self):
        '''
        Returns the table of CCDs.
        '''
        fn = os.path.join(self.decals_dir, 'decals-ccds.fits')
        if not os.path.exists(fn):
            fn += '.gz'
        print('Reading CCDs from', fn)
        T = fits_table(fn)
        print('Got', len(T), 'CCDs')

        cols = T.columns()
        # Make DR1 CCDs table somewhat compatible with DR2
        if 'extname' in cols and not 'ccdname' in cols:
            T.ccdname = T.extname
        if not 'camera' in cols:
            T.camera = np.array(['decam'] * len(T))
        if 'cpimage' in cols and not 'image_filename' in cols:
            T.image_filename = T.cpimage
        if 'cpimage_hdu' in cols and not 'image_hdu' in cols:
            T.image_hdu = T.cpimage_hdu

        if 'ccdname' in T.columns():
            # "N4 " -> "N4"
            T.ccdname = np.array([s.strip() for s in T.ccdname])

        return T
예제 #13
0
def add_depth_tag(decals, brick, outdir, overwrite=False):
    outfn = os.path.join(outdir, 'tractor', brick[:3], 'tractor-%s.fits' % brick)
    if os.path.exists(outfn) and not overwrite:
        print 'Exists:', outfn
        return
    fn = decals.find_file('tractor', brick=brick)
    if not os.path.exists(fn):
        print 'Does not exist:', fn
        return
    T = fits_table(fn, lower=False)
    primhdr = fitsio.read_header(fn)
    hdr = fitsio.read_header(fn, ext=1)
    print 'Read', len(T), 'from', fn
    T.decam_depth    = np.zeros((len(T), len(decals.allbands)), np.float32)
    T.decam_galdepth = np.zeros((len(T), len(decals.allbands)), np.float32)
    bands = 'grz'
    ibands = [decals.index_of_band(b) for b in bands]
    ix = np.clip(np.round(T.bx).astype(int), 0, 3599)
    iy = np.clip(np.round(T.by).astype(int), 0, 3599)
    for iband,band in zip(ibands, bands):
        fn = decals.find_file('depth', brick=brick, band=band)
        if os.path.exists(fn):
            print 'Reading', fn
            img = fitsio.read(fn)
            T.decam_depth[:,iband] = img[iy, ix]

        fn = decals.find_file('galdepth', brick=brick, band=band)
        if os.path.exists(fn):
            print 'Reading', fn
            img = fitsio.read(fn)
            T.decam_galdepth[:,iband] = img[iy, ix]
    outfn = os.path.join(outdir, 'tractor', brick[:3], 'tractor-%s.fits' % brick)
    trymakedirs(outfn, dir=True)

    for s in [
        'Data product of the DECam Legacy Survey (DECaLS)',
        'Full documentation at http://legacysurvey.org',
        ]:
        primhdr.add_record(dict(name='COMMENT', value=s, comment=s))

    # print 'Header:', hdr
    # T.writeto(outfn, header=hdr, primheader=primhdr)

    # Yuck, all this to get the units right
    tmpfn = outfn + '.tmp'
    fits = fitsio.FITS(tmpfn, 'rw', clobber=True)
    fits.write(None, header=primhdr)
    cols = T.get_columns()
    units = []
    for i in range(1, len(cols)+1):
        u = hdr.get('TUNIT%i' % i, '')
        units.append(u)
    # decam_depth units
    fluxiv = '1/nanomaggy^2'
    units[-2] = fluxiv
    units[-1] = fluxiv
    fits.write([T.get(c) for c in cols], names=cols, header=hdr, units=units)
    fits.close()
    os.rename(tmpfn, outfn)
    print 'Wrote', outfn
예제 #14
0
def make_fits_images(run, camcol, field):
    """gets field files from local cache (or sdss), returns UGRIZ dict of 
    fits images"""
    print """==================================================\n\n
            Grabbing image files from the cache.
            TODO: turn off the tractor printing... """

    imgs = {}
    for band in BANDS:
        print "reading in band %s" % band
        imgs[band] = sdss.get_tractor_image_dr9(run, camcol, field, band)

    fn = asdss.DR9().retrieve('photoField', run, camcol, field)
    F = aufits.fits_table(fn)

    # convert to FitsImage's
    imgfits = {}
    for iband,band in enumerate(BANDS):
        print "converting images %s" % band
        frame   = asdss.DR9().readFrame(run, camcol, field, band)
        calib   = np.median(frame.getCalibVec())
        gain    = F[0].gain[iband]
        darkvar = F[0].dark_variance[iband]
        sky     = np.median(frame.getSky())

        imgfits[band] = celeste.FitsImage(band,
                                          timg=imgs[band],
                                          calib=calib,
                                          gain=gain,
                                          darkvar=darkvar,
                                          sky=sky)
    return imgfits
예제 #15
0
    def write_fits(self, filename, hdr=None, primhdr=None):
        tt = type(self)
        sky_type = '%s.%s' % (tt.__module__, tt.__name__)
        if hdr is None:
            import fitsio
            hdr = fitsio.FITSHDR()

        if primhdr is None:
            import fitsio
            primhdr = fitsio.FITSHDR()
        hdr.add_record(dict(name='SKY', value=sky_type,
                            comment='Sky class'))

        primhdr.add_record(dict(name='SKY', value=sky_type,
                            comment='Sky class'))
        #hdr.add_record(dict(name='SPL_ORD', value=self.order,
        #                    comment='Spline sky order'))
        # this writes all params as header cards
        #self.toFitsHeader(hdr, prefix='SKY_')

        #fits = fitsio.FITS(filename, 'rw')
        #fits.write(None, header=primhdr, clobber=True)
        #fits.write(self.c, header=hdr)
        #fits.close()
        from astrometry.util.fits import fits_table
        T = fits_table()
        T.xgrid = np.atleast_2d(self.xgrid).astype(np.int32)
        T.ygrid = np.atleast_2d(self.ygrid).astype(np.int32)
        T.x0 = np.atleast_1d(self.x0).astype(np.int32)
        T.y0 = np.atleast_1d(self.y0).astype(np.int32)
        gridvals = self.spl(self.xgrid, self.ygrid).T
        T.gridvals = np.array([gridvals]).astype(np.float32)
        T.order = np.atleast_1d(self.order).astype(np.uint8)
        assert(len(T) == 1)
        T.writeto(filename, header=hdr, primheader=primhdr)
예제 #16
0
 def from_fits(cls, filename, header, row=0):
     from astrometry.util.fits import fits_table
     T = fits_table(filename)
     T = T[row]
     sky = cls(T.xgrid, T.ygrid, T.gridvals, order=T.order)
     sky.shift(T.x0, T.y0)
     return sky
예제 #17
0
파일: psfex.py 프로젝트: eddienko/tractor
    def __init__(self, fn=None, ext=1):
        if fn is not None:
            from astrometry.util.fits import fits_table

            T = fits_table(fn, ext=ext)
            ims = T.psf_mask[0]
            print("Got", ims.shape, "PSF images")
            hdr = T.get_header()
            # PSF distortion bases are polynomials of x,y
            assert hdr["POLNAME1"].strip() == "X_IMAGE"
            assert hdr["POLNAME2"].strip() == "Y_IMAGE"
            assert hdr["POLGRP1"] == 1
            assert hdr["POLGRP2"] == 1
            assert hdr["POLNGRP"] == 1
            x0 = hdr.get("POLZERO1")
            xscale = hdr.get("POLSCAL1")
            y0 = hdr.get("POLZERO2")
            yscale = hdr.get("POLSCAL2")
            degree = hdr.get("POLDEG1")
            self.sampling = hdr.get("PSF_SAMP")
            print("PsfEx sampling:", self.sampling)
            # number of terms in polynomial
            ne = (degree + 1) * (degree + 2) / 2
            assert hdr["PSFAXIS3"] == ne
            assert len(ims.shape) == 3
            assert ims.shape[0] == ne
            self.psfbases = ims
            self.xscale, self.yscale = xscale, yscale
            self.degree = degree
            print("PsfEx degree:", self.degree)
            bh, bw = self.psfbases[0].shape
            self.radius = (bh + 1) / 2.0
            self.x0, self.y0 = x0, y0
예제 #18
0
 def read(fn, F=None, primhdr=None, table=None):
     '''
     F: fitsio.FITS object to use an already-open file.
     primhdr: FITS header object for the primary HDU.
     table: astrometry.util.fits table
     '''
     if F is None:
         import fitsio
         F = fitsio.FITS(fn)
     if primhdr is None:
         primhdr = F[0].read_header()
     band   = primhdr['FILTER'].strip()
     run    = primhdr['RUN']
     camcol = primhdr['CAMCOL']
     field  = 0  # 'FRAME' != field
     if table is None:
         tab = fits_table(F[3].read(lower=True))
     else:
         tab = table
     assert(len(tab) == 1)
     tab = tab[0]
     return AsTrans(run, camcol, field, band,
                    node=np.deg2rad(tab.node),
                    incl=np.deg2rad(tab.incl),
                    astrans=tab, cut_to_band=False)
예제 #19
0
파일: fits.py 프로젝트: kaylanb/thesis_code
def tractor_cat(fn):
	'''uses Dustin's fits_table() function to read his tractor catalog then returns it as dictionary of np arrays
	his fits_table handles booleans with weird values like 84 for "brick_primary", and converts that to True/False'''
	temp = fits_table(fn)
	data={}
	for key in temp.columns(): data[key]= temp.get(key)
	return data
예제 #20
0
    def read_intermediate_catalog(self, brick, **kwargs):
        '''
        Reads the intermediate tractor catalog for the given brickname.

        *kwargs*: passed to self.find_file()
        
        Returns (T, hdr, primhdr)
        '''
        fn = self.find_file('tractor-intermediate', brick=brick, **kwargs)
        T = fits_table(fn)
        hdr = T.get_header()
        primhdr = fitsio.read_header(fn)

        in_flux_prefix = ''
        # Ensure flux arrays are 2d (N x 1)
        keys = ['flux', 'flux_ivar', 'rchi2', 'fracflux', 'fracmasked',
                'fracin', 'nobs', 'anymask', 'allmask', 'psfsize', 'depth',
                'galdepth']
        for k in keys:
            incol = '%s%s' % (in_flux_prefix, k)
            X = T.get(incol)
            # Hmm, if we need to reshape one of these arrays, we will
            # need to do all of them.
            if len(X.shape) == 1:
                X = X[:, np.newaxis]
                T.set(incol, X)
        
        return T, hdr, primhdr
예제 #21
0
 def setHdus(self, p):
     self.hdus = p
     self.table = fits_table(self.hdus[1].data)[0]
     T = self.table
     self.aa = T.aa.astype(float)
     self.kk = T.kk.astype(float)
     self.airmass = T.airmass
예제 #22
0
def main():
    """Main program.
    """
    import argparse
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--force', action='store_true',
                      help='Run calib processes even if files already exist?')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')

    parser.add_argument('--expnum', type=int, help='Cut to a single exposure')
    parser.add_argument('--extname', '--ccdname', help='Cut to a single extension/CCD name')

    parser.add_argument('--no-astrom', dest='astrom', action='store_false',
                      help='Do not compute astrometric calibs')
    parser.add_argument('--no-psf', dest='psfex', action='store_false',
                      help='Do not compute PsfEx calibs')
    parser.add_argument('--no-sky', dest='sky', action='store_false',
                      help='Do not compute sky models')
    parser.add_argument('--run-se', action='store_true', help='Run SourceExtractor')

    parser.add_argument('--splinesky', action='store_true', help='Spline sky, not constant')
    parser.add_argument('args',nargs=argparse.REMAINDER)
    opt = parser.parse_args()

    D = Decals()
    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        print('Read', len(T), 'from', opt.ccds)
    else:
        T = D.get_ccds()
        #print len(T), 'CCDs'

    if len(opt.args) == 0:
        if opt.expnum is not None:
            T.cut(T.expnum == opt.expnum)
            print('Cut to', len(T), 'with expnum =', opt.expnum)
        if opt.extname is not None:
            T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname]))
            print('Cut to', len(T), 'with extname =', opt.extname)

        opt.args = range(len(T))

    for a in opt.args:
        i = int(a)
        t = T[i]

        im = D.get_image_object(t)
        print('Running', im.calname)

        kwargs = dict(pvastrom=opt.astrom, psfex=opt.psfex, sky=opt.sky)
        if opt.force:
            kwargs.update(force=True)
        if opt.run_se:
            kwargs.update(se=True)
        if opt.splinesky:
            kwargs.update(splinesky=True)

        run_calibs((im, kwargs))
    return 0
예제 #23
0
def main():
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("--name1", help="Name for first data set")
    parser.add_argument("--name2", help="Name for second data set")
    parser.add_argument("--plot-prefix", default="compare", help='Prefix for plot filenames; default "%default"')
    parser.add_argument("--match", default=1.0, help="Astrometric cross-match distance in arcsec")
    parser.add_argument("dir1", help="First directory to compare")
    parser.add_argument("dir2", help="Second directory to compare")

    opt = parser.parse_args()

    ps = PlotSequence(opt.plot_prefix)

    name1 = opt.name1
    if name1 is None:
        name1 = os.path.basename(opt.dir1)
        if not len(name1):
            name1 = os.path.basename(os.path.dirname(opt.dir1))
    name2 = opt.name2
    if name2 is None:
        name2 = os.path.basename(opt.dir2)
        if not len(name2):
            name2 = os.path.basename(os.path.dirname(opt.dir2))
    tt = "Comparing %s to %s" % (name1, name2)

    # regex for tractor-*.fits catalog filename
    catre = re.compile("tractor-.*.fits")

    cat1, cat2 = [], []
    for basedir, cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
        for dirpath, dirnames, filenames in os.walk(basedir, followlinks=True):
            for fn in filenames:
                if not catre.match(fn):
                    print("Skipping", fn, "due to filename")
                    continue
                fn = os.path.join(dirpath, fn)
                t = fits_table(fn)
                print(len(t), "from", fn)
                cat.append(t)
    cat1 = merge_tables(cat1, columns="fillzero")
    cat2 = merge_tables(cat2, columns="fillzero")
    print("Total of", len(cat1), "from", name1)
    print("Total of", len(cat2), "from", name2)
    cat1.cut(cat1.brick_primary)
    cat2.cut(cat2.brick_primary)
    print("Total of", len(cat1), "BRICK_PRIMARY from", name1)
    print("Total of", len(cat2), "BRICK_PRIMARY from", name2)

    cat1.cut((cat1.decam_anymask[:, 1] == 0) * (cat1.decam_anymask[:, 2] == 0) * (cat1.decam_anymask[:, 4] == 0))
    cat2.cut((cat2.decam_anymask[:, 1] == 0) * (cat2.decam_anymask[:, 2] == 0) * (cat2.decam_anymask[:, 4] == 0))
    print("Total of", len(cat1), "unmasked from", name1)
    print("Total of", len(cat2), "unmasked from", name2)

    I, J, d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match / 3600.0, nearest=True)
    print(len(I), "matched")
    matched1 = cat1[I]
    matched2 = cat2[J]
예제 #24
0
파일: fits.py 프로젝트: kaylanb/thesis_code
def bricks_near_radec(ra,dec,d_ra=1.,d_dec=1.):
	'''given bricks table, return all bricknames with brick centers [ra-d_ra,ra+d_ra],[dec-d_dec,dec+d_dec]'''
	b=fits_table('survey-bricks.fits.gz')
	ra1,ra2= ra-d_ra, ra+d_ra
	dec1,dec2= dec-d_dec, dec+d_dec
	ind= np.all((b.get('ra') >= ra1,b.get('ra') <= ra2,b.get('dec') >= dec1,b.get('dec') <= dec2),axis=0)
	print 'these bricks have centers between %.1f < ra < %.1f and %.1f < dec < %.1f' % (ra1,ra2,dec1,dec2)
	return b.get('brickname')[ind]
예제 #25
0
    def get_bricks(self):
        '''
        Returns a table of bricks.  The caller owns the table.

        For read-only purposes, see *get_bricks_readonly()*, which
        uses a cached version.
        '''
        return fits_table(self.find_file('bricks'))
예제 #26
0
    def get_bricks(self):
        '''
        Returns a table of bricks.  The caller owns the table.

        For read-only purposes, see *get_bricks_readonly()*, which
        uses a cached version.
        '''
        return fits_table(os.path.join(self.decals_dir, 'decals-bricks.fits'))
예제 #27
0
 def readPhotoObj(self, run, camcol, field, filename=None):
     obj = PhotoObj(run, camcol, field)
     if filename is None:
         fn = self.getPath('photoObj', run, camcol, field)
     else:
         fn = filename
     obj.table = fits_table(fn)
     return obj
예제 #28
0
파일: fits.py 프로젝트: kaylanb/thesis_code
def overlap_bricks(bricks_fn,ra,dec,dx):
    '''given bricks table, return all bricknames with brick centers [ra-dx,ra+dx],[dec-dx,dec+dx]'''
    b=fits_table(bricks_fn)
    ra1,ra2= ra-dx, ra+dx
    dec1,dec2= dec-dx, dec+dx
    ind= np.all((b.get('ra') >= ra1,b.get('ra') <= ra2,b.get('dec') >= dec1,b.get('dec') <= dec2),axis=0)
    print 'these bricks have centers between %.1f < ra < %.1f and %.1f < dec < %.1f' % (ra1,ra2,dec1,dec2)
    for name,ra,dec in zip(b.get('brickname')[ind],b.get('ra')[ind],b.get('dec')[ind]): print name,ra,dec
예제 #29
0
def upload_cat(req):
    import tempfile
    from astrometry.util.fits import fits_table
    from django.http import HttpResponseRedirect
    from map.views import index

    if req.method != 'POST':
        return HttpResponse('POST only')
    print('Files:', req.FILES)
    cat = req.FILES['catalog']

    dirnm = settings.USER_QUERY_DIR
    if not os.path.exists(dirnm):
        try:
            os.makedirs(dirnm)
        except:
            pass
    f,tmpfn = tempfile.mkstemp(suffix='.fits', dir=dirnm)
    os.close(f)
    os.unlink(tmpfn)
    print('Saving to', tmpfn)
    with open(tmpfn, 'wb+') as destination:
        for chunk in cat.chunks():
            destination.write(chunk)
    print('Wrote', tmpfn)

    try:
        T = fits_table(tmpfn)
    except:
        return HttpResponse('Must upload FITS format catalog including "RA", "Dec", optionally "Name" columns')
    
    # Rename and resave columns if necessary
    if rename_cols(T):
        T.write_to(tmpfn)

    cols = T.columns()
    if not (('ra' in cols) and ('dec' in cols)):
        return HttpResponse('Must upload catalog including "RA", "Dec", optionally "Name" columns')

    ra,dec = T.ra[0], T.dec[0]
    catname = tmpfn.replace(dirnm, '').replace('.fits', '')
    if catname.startswith('/'):
        catname = catname[1:]

    try:
        import fitsio
        primhdr = fitsio.read_header(tmpfn)
        name = primhdr.get('CATNAME', '')
        color = primhdr.get('CATCOLOR', '')
        if len(name):
            catname = catname + '-n%s' % name.strip().replace(' ','_')
        if len(color):
            catname = catname + '-c%s' % color.strip()
    except:
        pass
    
    return HttpResponseRedirect(reverse(index) +
                                '?ra=%.4f&dec=%.4f&catalog=%s' % (ra, dec, catname))
예제 #30
0
def main():

    brickname = '2428p117'
    objtype = 'STAR'
    lobjtype = objtype.lower()

    decals_sim_dir = '/Users/ioannis/decals_sim_dir/star/2428p117'

    zoom = [1800,2400,1800,2400]
    #zoom = [1800,2000,1800,2000]
    #zoom = None

    decals = Decals()
    brickinfo = decals.get_brick_by_name(brickname)
    brickwcs = wcs_for_brick(brickinfo)
    W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale()

    if zoom is not None:
        # See also runbrick.stage_tims()
        #(x0,x1,y0,y1) = args.zoom
        (x0,x1,y0,y1) = zoom
        W = x1-x0
        H = y1-y0
        targetwcs = brickwcs.get_subimage(x0, y0, W, H)

    bounds = brickwcs.radec_bounds()
    ra_range = bounds[1]-bounds[0]
    dec_range = bounds[3]-bounds[2]
    radec_center = brickwcs.radec_center()
    print(radec_center, ra_range, dec_range)
        
    corners = np.array([brickwcs.pixelxy2radec(x,y) for x,y in
                        [(1,1),(W,1),(W,H),(1,H),(1,1)]])
    

    # Identify the CCDs in the region of interest.
    ccdinfo = decals.ccds_touching_wcs(brickwcs)
    ccdinfo.about()
    log.info('Got {} CCDs'.format(len(ccdinfo)))

    # Generate the catalog of simulated sources here!
    simcat = fits_table('simcat-{}-{}-00.fits'.format(brickname,lobjtype))
    #simcat = fits.getdata('simcat-{}-{}-00.fits'.format(brickname,lobjtype))
    simcat.about()
    blobxy = zip(simcat.x,simcat.y)

    simdecals = SimDecals(sim_sources=simcat)

    # For testing!
    #sim = SimImage(simdecals,ccdinfo[0])
    #tim = sim.get_tractor_image(const2psf=True)

    run_brick(brickname, decals=simdecals, outdir=decals_sim_dir,
              threads=8, zoom=zoom, wise=False, sdssInit=False,
              forceAll=True, writePickles=False, blobxy=blobxy,
              pixPsf=False, stages=['tims','srcs','fitblobs',
                                    'fitblobs_finish','coadds',
                                    'writecat'])
예제 #31
0
def main(args):
    """Main program.
    """
    import argparse

    parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).")
    parser.add_argument('--calibs', action='store_true',
                      help='Output CCDs that need to be calibrated.')

    parser.add_argument('--nper', type=int, default=None,
                      help='Batch N calibs per line')
    parser.add_argument('--byexp', action='store_true', default=False,
                        help='Run one whole exposure per job (not one CCD per job)')

    parser.add_argument('--forced', action='store_true',
                      help='Output forced-photometry commands')

    parser.add_argument('--lsb', action='store_true',
                      help='Output Low-Surface-Brightness commands')

    parser.add_argument('--stage', help='Stage image files to given directory')

    parser.add_argument('--touching', action='store_true',
                      help='Cut to only CCDs touching selected bricks')
    parser.add_argument('--near', action='store_true',
                      help='Quick cut to only CCDs near selected bricks')

    parser.add_argument('--check-coadd', action='store_true',
                      help='Check which coadds actually need to run.')
    parser.add_argument('--out', help='Output filename for calibs, default %(default)s',
                      default='jobs')
    parser.add_argument('--command', action='store_true',
                      help='Write out full command-line to run calib')
    parser.add_argument('--opt', help='With --command, extra options to add')

    parser.add_argument('--maxra', type=float, help='Maximum RA to run')
    parser.add_argument('--minra', type=float, help='Minimum RA to run')
    parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
    parser.add_argument('--mindec', type=float, help='Minimum Dec to run')

    parser.add_argument('--region', help='Region to select')

    parser.add_argument('--bricks', help='Set bricks.fits file to load')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')
    parser.add_argument('--ignore_cuts', action='store_true',default=False,help='no photometric cuts')
    parser.add_argument('--save_to_fits', action='store_true',default=False,help='save cut brick,ccd to fits table')
    parser.add_argument('--name', action='store',default='dr3',help='save with this suffix, e.g. refers to ccds table')

    parser.add_argument('--delete-sky', action='store_true',
                      help='Delete any existing sky calibration files')

    parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')

    parser.add_argument('--nccds', action='store_true', default=False, help='Prints number of CCDs per brick')

    parser.add_argument('--bands', default='g,r,z', help='Set bands to keep: comma-separated list.')


    opt = parser.parse_args(args)

    want_ccds = (opt.calibs or opt.forced or opt.lsb)
    want_bricks = not want_ccds



    survey = LegacySurveyData()
    if opt.bricks is not None:
        B = fits_table(opt.bricks)
        log('Read', len(B), 'from', opt.bricks)
    else:
        B = survey.get_bricks()

    log('Bricks Dec range:', B.dec.min(), B.dec.max())

    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        log('Read', len(T), 'from', opt.ccds)
    else:
        T = survey.get_ccds()
        log(len(T), 'CCDs')
    T.index = np.arange(len(T))

    if opt.ignore_cuts == False:
        log('Applying CCD cuts...')
        if 'ccd_cuts' in T.columns():
            T.cut(T.ccd_cuts == 0)
            log(len(T), 'CCDs survive cuts')

    bands = opt.bands.split(',')
    log('Filters:', np.unique(T.filter))
    T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
    log('Cut to', len(T), 'CCDs in filters', bands)

    log('CCDs Dec range:', T.dec.min(), T.dec.max())

    # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
    # plt.clf()
    # plt.hist(counts, counts.max()+1)
    # plt.savefig('bricks.png')
    # B.cut(I[counts >= 9])
    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # #plt.scatter(B.ra[I], B.dec[I], c=counts)
    # plt.savefig('bricks2.png')


    # DES Stripe82
    #rlo,rhi = 350.,360.
    # rlo,rhi = 300., 10.
    # dlo,dhi = -6., 4.
    # TINY bit
    #rlo,rhi = 350.,351.1
    #dlo,dhi = 0., 1.1

    # EDR+
    # 860 bricks
    # ~10,000 CCDs
    #rlo,rhi = 239,246
    #dlo,dhi =   5, 13

    # DR1
    #rlo,rhi = 0, 360
    # part 1
    #dlo,dhi = 25, 40
    # part 2
    #dlo,dhi = 20,25
    # part 3
    #dlo,dhi = 15,20
    # part 4
    #dlo,dhi = 10,15
    # part 5
    #dlo,dhi = 5,10
    # the rest
    #dlo,dhi = -11, 5
    #dlo,dhi = 15,25.5

    dlo,dhi = -25, 40
    rlo,rhi = 0, 360

    # Arjun says 3x3 coverage area is roughly
    # RA=240-252 DEC=6-12 (but not completely rectangular)

    # COSMOS
    #rlo,rhi = 148.9, 151.2
    #dlo,dhi = 0.9, 3.5

    # A nice well-behaved region (EDR2/3)
    # rlo,rhi = 243.6, 244.6
    # dlo,dhi = 8.1, 8.6

    # 56 bricks, ~725 CCDs
    #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
    # 240 bricks, ~3000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
    # 535 bricks, ~7000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))


    if opt.region in ['test1', 'test2', 'test3', 'test4']:
        nm = dict(test1='2446p115', # weird stuff around bright star
                  test2='1183p292', # faint sources around bright galaxy
                  test3='3503p005', # DES
                  test4='1163p277', # Pollux
                  )[opt.region]

        B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
        log('Cut to', len(B), 'bricks')
        log(B.ra, B.dec)
        dlo,dhi = -90,90
        rlo,rhi = 0, 360

    elif opt.region == 'edr':
        # EDR:
        # 535 bricks, ~7000 CCDs
        rlo,rhi = 240,245
        dlo,dhi =   5, 12

    elif opt.region == 'dr8-decam':
        rlo,rhi =   0, 360
        dlo,dhi = -70,  40
        log('DR8-DECam region')

    elif opt.region == 'edrplus':
        rlo,rhi = 235,248
        dlo,dhi =   5, 15

    elif opt.region == 'edr-south':
        rlo,rhi = 240,245
        dlo,dhi =   5, 10

    elif opt.region == 'cosmos1':
        # 16 bricks in the core of the COSMOS field.
        rlo,rhi = 149.75, 150.75
        dlo,dhi = 1.6, 2.6

    elif opt.region == 'pristine':
        # Stream?
        rlo,rhi = 240,250
        dlo,dhi = 10,15

    elif opt.region == 'des':
        dlo, dhi = -6., 4.
        rlo, rhi = 317., 7.

        T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
        log('Cut to', len(T), 'CCDs with "CPDES82" in filename')

    elif opt.region == 'subdes':
        rlo,rhi = 320., 360.
        dlo,dhi = -1.25, 1.25

    elif opt.region == 'northwest':
        rlo,rhi = 240,360
        dlo,dhi = 20,40
    elif opt.region == 'north':
        rlo,rhi = 120,240
        dlo,dhi = 20,40
    elif opt.region == 'northeast':
        rlo,rhi = 0,120
        dlo,dhi = 20,40
    elif opt.region == 'southwest':
        rlo,rhi = 240,360
        dlo,dhi = -20,0
    elif opt.region == 'south':
        rlo,rhi = 120,240
        dlo,dhi = -20,0
    elif opt.region == 'southeast':
        rlo,rhi = 0,120
        dlo,dhi = -20,0
    elif opt.region == 'southsoutheast':
        rlo,rhi = 0,120
        dlo,dhi = -20,-10
    elif opt.region == 'midwest':
        rlo,rhi = 240,360
        dlo,dhi = 0,20
    elif opt.region == 'middle':
        rlo,rhi = 120,240
        dlo,dhi = 0,20
    elif opt.region == 'mideast':
        rlo,rhi = 0,120
        dlo,dhi = 0,20

    elif opt.region == 'grz':
        # Bricks with grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
        log('Cut to', len(B), 'bricks with grz coverage')

    elif opt.region == 'nogrz':
        # Bricks without grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
        log('Cut to', len(B), 'bricks withOUT grz coverage')

    elif opt.region == 'deep2':
        rlo,rhi = 250,260
        dlo,dhi = 30,35

    elif opt.region == 'deep2f2':
        rlo,rhi = 251.4, 254.4
        dlo,dhi =  34.6,  35.3

    elif opt.region == 'deep2f3':
        rlo,rhi = 351.25, 353.75
        dlo,dhi = 0, 0.5

    elif opt.region == 'deep3':
        rlo,rhi = 214,216
        dlo,dhi = 52.25,53.25

    elif opt.region == 'virgo':
        rlo,rhi = 185,190
        dlo,dhi =  10, 15

    elif opt.region == 'virgo2':
        rlo,rhi = 182,192
        dlo,dhi =   8, 18

    elif opt.region == 'coma':
        # van Dokkum et al Coma cluster ultra-diffuse galaxies: 3x3 field centered on Coma cluster
        rc,dc = 195., 28.
        dd = 1.5
        cosdec = np.cos(np.deg2rad(dc))
        rlo,rhi = rc - dd/cosdec, rc + dd/cosdec
        dlo,dhi = dc - dd, dc + dd

    elif opt.region == 'lsb':
        rlo,rhi = 147.2, 147.8
        dlo,dhi = -0.4, 0.4

    elif opt.region == 'eboss-sgc':
        # generous boundaries to make sure get all relevant images
        # RA -45 to +45
        # Dec -5 to +7
        rlo,rhi = 310., 50.
        dlo,dhi = -6., 6.

    elif opt.region == 'eboss-ngc':
        # generous boundaries to make sure get all relevant images
        # NGC ELGs
        # RA 115 to 175
        # Dec 15 to  30
        # rlo,rhi = 122., 177.
        # dlo,dhi =  12.,  32.
        rlo,rhi = 126., 168.
        dlo,dhi =  18.,  33.

    elif opt.region == 'mzls':
        dlo,dhi = -10., 90. # -10: pull in Stripe 82 data too

    elif opt.region == 'dr4-bootes':
        # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
        #dlo,dhi = 34., 35.
        #rlo,rhi = 209.5, 210.5
        dlo,dhi = 33., 36.
        rlo,rhi = 216.5, 219.5

    elif opt.region == 'des-sn-x3':
        #rlo,rhi = 36., 37.
        #dlo,dhi = -5., -4.
        rlo,rhi = 36., 36.5
        dlo,dhi = -4.5, -4.

    elif opt.region == 'ngc2632':
        # open cluster
        rlo,rhi = 129.0, 131.0
        dlo,dhi = 19.0, 20.5

    elif opt.region == 'dr8sky':
        rlo,rhi = 35.0, 37.0
        dlo,dhi = -3.0, -1.0

    # ADM DR8 test regions, see, e.g.:
    # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR8#Testregions
    elif opt.region == 'dr8-test-s82':
        rlo, rhi = 0, 45
        dlo, dhi = -1.25, 1.25
    elif opt.region == 'dr8-test-hsc-sgc':
        rlo, rhi = 30, 40
        dlo, dhi = -6.5, -1.25
    elif opt.region == 'dr8-test-hsc-ngc':
        rlo, rhi = 177.5, 182.5
        dlo, dhi = -1, 1
    elif opt.region == 'dr8-test-edr':
        rlo, rhi = 240, 245
        dlo, dhi = 5, 12
    elif opt.region == 'dr8-test-hsc-north':
        rlo, rhi = 240, 250
        dlo, dhi = 42, 45
    elif opt.region == 'dr8-test-deep2-egs':
        rlo, rhi = 213, 216.5
        dlo, dhi = 52, 54
    elif opt.region == 'dr8-test-overlap':
        rlo, rhi = 132, 140.5
        dlo, dhi = 31.5, 35

    if opt.mindec is not None:
        dlo = opt.mindec
    if opt.maxdec is not None:
        dhi = opt.maxdec
    if opt.minra is not None:
        rlo = opt.minra
    if opt.maxra is not None:
        rhi = opt.maxra

    if rlo < rhi:
        B.cut((B.ra >= rlo) * (B.ra <= rhi) *
              (B.dec >= dlo) * (B.dec <= dhi))
    else: # RA wrap
        B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) *
              (B.dec >= dlo) * (B.dec <= dhi))
    log(len(B), 'bricks in range; cut Dec range', B.dec.min(), B.dec.max())
    #for name in B.get('brickname'):
    #    print(name)
    #B.writeto('bricks-cut.fits')

    bricksize = 0.25
    # A bit more than 0.25-degree brick radius + Bok image radius ~ 0.57
    search_radius = 1.05 * np.sqrt(2.) * (bricksize +
                                          (0.455 * 4096 / 3600.))/2.

    log(len(T), 'CCDs')
    log(len(B), 'Bricks')
    I,J,_ = match_radec(B.ra, B.dec, T.ra, T.dec, search_radius,
                        nearest=True)
    B.cut(I)
    log('Cut to', len(B), 'bricks near CCDs')
    log('Bricks Dec range:', B.dec.min(), B.dec.max())


    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # plt.title('DR3 bricks')
    # plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
    # plt.savefig('bricks.png')

    if opt.touching:
        I,J,_ = match_radec(T.ra, T.dec, B.ra, B.dec, search_radius,
                            nearest=True)
        # list the ones that will be cut
        # drop = np.ones(len(T))
        # drop[I] = False
        # for i in np.flatnonzero(drop):
        #     from astrometry.util.starutil_numpy import degrees_between
        #     dists = degrees_between(B.ra, B.dec, T.ra[i], T.dec[i])
        #     mindist = min(dists)
        #     print('Dropping:', T.ra[i], T.dec[i], 'min dist', mindist, 'search_radius', search_radius)


        T.cut(I)
        log('Cut to', len(T), 'CCDs near bricks')

    if opt.forced:
        log('Writing forced-photometry commands to', opt.out)
        f = open(opt.out,'w')
        log('Total of', len(T), 'CCDs')
        #T.cut(allI)
        camexp = set([(c,e) for c,e in zip(T.camera, T.expnum)])
        print(len(camexp), 'unique camera/exposure pairs')
        for cam,exp in camexp:
            #expstr = '%08i' % exp
            #outfn = os.path.join('forced', cam, expstr[:5], 'forced-%s-%s.fits' % (cam, exp))
            #f.write('%s %s all %s\n' % (cam, exp, outfn))
            f.write('%s %s\n' % (cam, exp))
        f.close()
        log('Wrote', opt.out)
        return 0

    # sort by RA increasing
    B.cut(np.argsort(B.ra))

    if opt.save_to_fits:
        assert(opt.touching)
        # Write cut tables to file
        for tab,typ in zip([B,T],['bricks','ccds']):
            fn='%s-%s-cut.fits' % (typ,opt.region)
            if os.path.exists(fn):
                os.remove(fn)
            tab.writeto(fn)
            log('Wrote %s' % fn)
        # Write text files listing ccd and filename names
        # nm1,nm2= 'ccds-%s.txt'% opt.region,'filenames-%s.txt' % opt.region
        # if os.path.exists(nm1):
        #     os.remove(nm1)
        # if os.path.exists(nm2):
        #     os.remove(nm2)
        # f1,f2=open(nm1,'w'),open(nm2,'w')
        # fns= list(set(T.get('image_filename')))
        # for fn in fns:
        #     f2.write('%s\n' % fn.strip())
        # for ti in T:
        #     f1.write('%s\n' % ti.get('image_filename').strip())
        # f1.close()
        # f2.close()
        # log('Wrote *-names.txt')

    if opt.touching:

        if want_bricks:
            # Shortcut the list of bricks that are definitely touching CCDs --
            # a brick-ccd pair within this radius must be touching.
            closest_radius = 0.95 * (bricksize + 0.262 * 2048 / 3600.) / 2.

            J1,_,_ = match_radec(B.ra, B.dec, T.ra, T.dec, closest_radius, nearest=True)
            log(len(J1), 'of', len(B), 'bricks definitely touch CCDs')
            tocheck = np.ones(len(B), bool)
            tocheck[J1] = False
            J2 = []
            for j in np.flatnonzero(tocheck):
                b = B[j]
                wcs = wcs_for_brick(b)
                I = ccds_touching_wcs(wcs, T)
                log(len(I), 'CCDs for brick', b.brickname)
                if len(I) == 0:
                    continue
                J2.append(j)
            J = np.hstack((J1, J2))
            J = np.sort(J).astype(int)
            B.cut(J)
            log('Cut to', len(B), 'bricks touching CCDs')

        else:
            J = []
            allI = set()
            for j,b in enumerate(B):
                wcs = wcs_for_brick(b)
                I = ccds_touching_wcs(wcs, T)
                log(len(I), 'CCDs for brick', b.brickname)
                if len(I) == 0:
                    continue
                allI.update(I)
                J.append(j)
            allI = list(allI)
            allI.sort()
            B.cut(np.array(J))
            log('Cut to', len(B), 'bricks touching CCDs')

    elif opt.near:
        # Find CCDs near bricks
        allI,_,_ = match_radec(T.ra, T.dec, B.ra, B.dec, search_radius, nearest=True)
        # Find bricks near CCDs
        J,_,_ = match_radec(B.ra, B.dec, T.ra, T.dec, search_radius, nearest=True)
        B.cut(J)
        log('Cut to', len(B), 'bricks near CCDs')

    else:
        allI = np.arange(len(T))

    if opt.byexp:
        _,eI = np.unique(T.expnum[allI], return_index=True)
        allI = allI[eI]
        print('Cut to', len(allI), 'expnums')

    if opt.nccds:
        from queue import Queue
        from threading import Thread

        log('Checking number of CCDs per brick')

        def worker():
            while True:
                i = q.get()
                if i is None:
                    break
                b = B[i]
                wcs = wcs_for_brick(b)
                I = ccds_touching_wcs(wcs, T)
                log(b.brickname, len(I))
                q.task_done()

        q = Queue()
        num_threads = 24
        threads = []

        for i in range(num_threads):
            t = Thread(target=worker)
            t.start()
            threads.append(t)

        for i in range(len(B)):
            q.put(i)

        q.join()
        for i in range(num_threads):
            q.put(None)
        for t in threads:
            t.join()

    if opt.write_ccds:
        T[allI].writeto(opt.write_ccds)
        log('Wrote', opt.write_ccds)

    if want_bricks:
        # Print the list of bricks and exit.
        for b in B:
            print(b.brickname)
        if opt.save_to_fits:
            B.writeto('bricks-%s-touching.fits' % opt.region)
        if not want_ccds:
            return 0

    ## Be careful here -- T has been cut; we want to write out T.index.
    ## 'allI' contains indices into T.

    if opt.stage is not None:
        cmd_pat = 'rsync -LRarv %s %s'
        fns = set()
        for iccd in allI:
            im = survey.get_image_object(T[iccd])
            fns.update([im.imgfn, im.wtfn, im.dqfn, im.psffn, im.merged_psffn,
                   im.merged_skyfn, im.skyfn])
        for i,fn in enumerate(fns):
            print('File', i+1, 'of', len(fns), ':', fn)
            if not os.path.exists(fn):
                print('No such file:', fn)
                continue
            base = survey.get_survey_dir()
            if base.endswith('/'):
                base = base[:-1]

            rel = os.path.relpath(fn, base)

            dest = os.path.join(opt.stage, rel)
            print('Dest:', dest)
            if os.path.exists(dest):
                print('Exists:', dest)
                continue

            cmd = cmd_pat % ('%s/./%s' % (base, rel), opt.stage)
            print(cmd)
            rtn = os.system(cmd)
            assert(rtn == 0)
        return 0

    if opt.lsb:
        log('Writing LSB commands to', opt.out)
        f = open(opt.out,'w')
        log('Total of', len(allI), 'CCDs')
        for j,i in enumerate(allI):
            exp = T.expnum[i]
            ext = T.ccdname[i].strip()
            outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
            f.write('python legacyanalysis/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext))
        f.close()
        log('Wrote', opt.out)
        return 0

    log('Writing calibs to', opt.out)
    f = open(opt.out,'w')
    log('Total of', len(allI), 'CCDs')

    batch = []

    def write_batch(f, batch, cmd):
        if cmd is None:
            cmd = ''
        f.write(cmd + ' '.join(batch) + '\n')

    cmd = None
    if opt.command:
        cmd = 'python legacypipe/run-calib.py '
        if opt.opt is not None:
            cmd += opt.opt + ' '

    for j,i in enumerate(allI):

        if opt.delete_sky:
            log(j+1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if opt.delete_sky and os.path.exists(im.skyfn):
                log('  deleting:', im.skyfn)
                os.unlink(im.skyfn)

        if opt.command:
            if opt.byexp:
                s = '--expnum %i' % (T.expnum[i])
            else:
                s = '%i-%s' % (T.expnum[i], T.ccdname[i])
            prefix = 'python legacypipe/run-calib.py '
            if opt.opt is not None:
                prefix = prefix + opt.opt
            #('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
            #     (T.expnum[i], T.ccdname[i]))
        else:
            s = '%i' % T.index[i]
            prefix = ''

        if j < 10:
            print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i],
                  'filename', T.image_filename[i])

        if not opt.nper:
            f.write(prefix + s + '\n')
        else:
            batch.append(s)
            if len(batch) >= opt.nper:
                write_batch(f, batch, cmd)
                batch = []

    if len(batch):
        write_batch(f, batch, cmd)

    f.close()
    log('Wrote', opt.out)
    return 0
예제 #32
0
    print(len(C), 'CCDs in', region)

    C.cut(np.array([f in bands for f in C.filter]))
    print(len(C), 'in', bands)

    C.cut(C.exptime >= 50.)
    print(len(C), 'with exptime >= 50 sec')

    C.cut(survey.photometric_ccds(C))
    print(len(C), 'photometric')

    C.cut(np.lexsort((C.expnum, C.filter)))
    C.writeto(cfn)

else:
    C = fits_table(cfn)

plt.clf()
plt.plot(C.ra, C.dec, 'bo', alpha=0.2)
plt.savefig('rd.png')

# Find the unique exposures (not CCDs), save as E.
C.galnorm = C.galnorm_mean
nil, I = np.unique(C.expnum, return_index=True)
E = C[I]
print(len(E), 'exposures')

if False:
    # Find the extinction in the center of the COSMOS region and apply it
    # as a correction to our target depths (so that we reach that depth
    # for de-reddened mags).
예제 #33
0
    plot.plot('outline')


mjd0 = 57789.0

db = ComputedExptime.objects.filter(starttime__gt=mjd0,
                                    starttime__lt=mjd0 + 0.8)
print(db.count(), 'entries from', mjd0)
adj = db.filter(adjfactor__gt=1.)
print(adj.count(), 'entries with adjfactor > 1')

#adj = db_to_fits(adj)

ps = PlotSequence('adjust', format='%03i')

tiles = fits_table('obstatus/mosaic-tiles_obstatus.fits')
tiles.cut(tiles.get('pass') <= 3)
tiles.cut(tiles.dec >= 30)

for a in adj[:10]:
    tileid = a.tileid
    print()
    print('tile id', tileid)
    i = np.flatnonzero(tiles.tileid == tileid)
    tile = tiles[i[0]]
    ra, dec = tile.ra, tile.dec
    tilepass = tile.get('pass')

    others = a.otherpasses_set.all()
    print(others.count(), 'other passes involved')
    others = others.filter(depth__gt=1, depth__lt=30)
예제 #34
0
    def __init__(self, fn=None, ext=1, Ti=None):
        '''
        If *fn* is not None, reads a PsfEx file from the given filename and *ext*.

        Else if *Ti* is not None, reads from one row of a merged PsfEx table.
        '''
        if fn is not None:
            from astrometry.util.fits import fits_table
            T = fits_table(fn, ext=ext)
            ims = T.psf_mask[0]
            ims = ims.astype(np.float32)
            #print('Got', ims.shape, 'PSF images')
            hdr = T.get_header()

            ngrp = hdr['POLNGRP']
            assert (ngrp in [0, 1])
            if ngrp == 0:
                # Constant PSF.
                degree = 0
                x0 = y0 = 0.
                xscale = yscale = 1.
            else:
                assert (hdr['POLNGRP'] == 1)
                # PSF distortion bases are polynomials of x,y
                assert (hdr['POLNAME1'].strip() == 'X_IMAGE')
                assert (hdr['POLNAME2'].strip() == 'Y_IMAGE')
                assert (hdr['POLGRP1'] == 1)
                assert (hdr['POLGRP2'] == 1)
                x0 = hdr.get('POLZERO1')
                xscale = hdr.get('POLSCAL1')
                y0 = hdr.get('POLZERO2')
                yscale = hdr.get('POLSCAL2')
                degree = hdr.get('POLDEG1')
            self.sampling = hdr.get('PSF_SAMP')
            #print('PsfEx sampling:', self.sampling)
            # number of terms in polynomial
            ne = (degree + 1) * (degree + 2) // 2
            assert (hdr['PSFAXIS3'] == ne)
            assert (len(ims.shape) == 3)
            assert (ims.shape[0] == ne)
            self.psfbases = ims
            self.xscale, self.yscale = xscale, yscale
            self.degree = degree
            #print('PsfEx degree:', self.degree)
            bh, bw = self.psfbases[0].shape
            self.radius = (bh + 1) / 2.
            self.x0, self.y0 = x0, y0
            self.fwhm = hdr.get('PSF_FWHM')

        elif Ti is not None:
            self.sampling = Ti.psf_samp
            degree = Ti.poldeg1
            # PSF distortion bases are polynomials of x,y
            if degree > 0:
                assert (Ti.polname1.strip() == 'X_IMAGE')
                assert (Ti.polname2.strip() == 'Y_IMAGE')
                assert (Ti.polgrp1 == 1)
                assert (Ti.polgrp2 == 1)
                assert (Ti.polngrp == 1)
            self.x0 = Ti.polzero1
            self.y0 = Ti.polzero2
            self.xscale = Ti.polscal1
            self.yscale = Ti.polscal2
            self.degree = degree
            # number of terms in polynomial
            ne = (degree + 1) * (degree + 2) / 2
            assert (Ti.psfaxis3 == ne)
            ims = Ti.psf_mask
            ims = ims.astype(np.float32)
            assert (len(ims.shape) == 3)
            assert (ims.shape[0] == ne)
            self.psfbases = ims
            bh, bw = self.psfbases[0].shape
            self.radius = (bh + 1) / 2.
            self.fwhm = Ti.psf_fwhm
예제 #35
0
            ps.savefig()
        
        sys.exit(0)

    # Test RexGalaxy

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
    outdir = 'out-testcase6-rex'
    main(args=['--brick', '1102p240', '--zoom', '500', '600', '650', '750',
               '--force-all', '--no-write', '--no-wise',
            #'--rex', #'--plots',
               '--survey-dir', surveydir,
               '--outdir', outdir] + extra_args)
    fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
    assert(os.path.exists(fn))
    T = fits_table(fn)
    assert(len(T) == 2)
    print('Types:', T.type)
    # Since there is a Tycho-2 star in the blob, forced to be PSF.
    assert(T.type[0] == 'PSF ')
    cmd = ('(cd %s && sha256sum -c %s)' %
           (outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum')))
    print(cmd)
    rtn = os.system(cmd)
    assert(rtn == 0)

    # Test with a Tycho-2 star in the blob.

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
    outdir = 'out-testcase6'
    main(args=['--brick', '1102p240', '--zoom', '500', '600', '650', '750',
예제 #36
0
def main():
    # I read a DESI DR8 target catalog, cut to ELGs, then took a narrow
    # r-mag slice around the peak of that distribution;
    # desi/target/catalogs/dr8/0.31.1/targets/main/resolve/targets-dr8-hp-1,5,11,50,55,60,83,84,86,89,91,98,119,155,158,174,186,187,190.fits')
    # Then took the median g and z mags
    # And opened the coadd invvar mags for a random brick in that footprint
    # (0701p000) to find the median per-pixel invvars.

    r = 23.0
    g = 23.4
    z = 22.2

    # Selecting EXPs, the peak of the shapeexp_r was ~ 0.75".
    r_e = 0.75

    # Image properties:

    giv = 77000.
    riv = 27000.
    ziv = 8000.

    # PSF sizes were roughly equal, 1.16, 1.17, 1.19"
    # -> sigma = 1.9 DECam pixels
    psf_sigma = 1.9

    H, W = 1000, 1000

    seed = 42
    np.random.seed(seed)

    ra, dec = 70., 1.
    brick = BrickDuck(ra, dec, 'custom-0700p010')
    wcs = wcs_for_brick(brick, W=W, H=H)
    bands = 'grz'

    tims = []
    for band, iv in zip(bands, [giv, riv, ziv]):
        img = np.zeros((H, W), np.float32)
        tiv = np.zeros_like(img) + iv
        s = psf_sigma**2
        psf = GaussianMixturePSF(1., 0., 0., s, s, 0.)
        twcs = ConstantFitsWcs(wcs)
        sky = ConstantSky(0.)
        photocal = LinearPhotoCal(1., band=band)

        tai = TAITime(None, mjd=55700.)

        tim = tractor.Image(data=img,
                            invvar=tiv,
                            psf=psf,
                            wcs=twcs,
                            sky=sky,
                            photocal=photocal,
                            name='fake %s' % band,
                            time=tai)
        tim.skyver = ('1', '1')
        tim.psfver = ('1', '1')
        tim.plver = '1'
        tim.x0 = tim.y0 = 0
        tim.subwcs = wcs
        tim.psfnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
        tim.galnorm = tim.psfnorm
        tim.propid = '2020A-000'
        tim.band = band
        tim.dq = None
        tim.sig1 = 1. / np.sqrt(iv)
        tim.psf_sigma = psf_sigma
        tim.primhdr = fitsio.FITSHDR()

        tims.append(tim)

    # Simulated catalog
    gflux = NanoMaggies.magToNanomaggies(g)
    rflux = NanoMaggies.magToNanomaggies(r)
    zflux = NanoMaggies.magToNanomaggies(z)
    box = 50
    CX, CY = np.meshgrid(np.arange(box // 2, W, box),
                         np.arange(box // 2, H, box))
    ny, nx = CX.shape
    BA, PHI = np.meshgrid(np.linspace(0.1, 1.0, nx), np.linspace(0., 180., ny))
    cat = []
    for cx, cy, ba, phi in zip(CX.ravel(), CY.ravel(), BA.ravel(),
                               PHI.ravel()):
        #print('x,y %.1f,%.1f, ba %.2f, phi %.2f' % (cx, cy, ba, phi))
        r, d = wcs.pixelxy2radec(cx + 1, cy + 1)
        src = ExpGalaxy(RaDecPos(r, d),
                        NanoMaggies(order=bands, g=gflux, r=rflux, z=zflux),
                        EllipseE.fromRAbPhi(r_e, ba, phi))
        cat.append(src)

    from legacypipe.catalog import prepare_fits_catalog
    TT = fits_table()
    TT.bx = CX.ravel()
    TT.by = CY.ravel()
    TT.ba = BA.ravel()
    TT.phi = PHI.ravel()

    tcat = tractor.Catalog(*cat)
    T2 = prepare_fits_catalog(tcat, None, TT, bands, save_invvars=False)
    T2.writeto('sim-cat.fits')

    tr = Tractor(tims, cat)

    for band, tim in zip(bands, tims):
        mod = tr.getModelImage(tim)
        mod += np.random.standard_normal(
            size=tim.shape) * 1. / tim.getInvError()
        fitsio.write('sim-%s.fits' % band, mod, clobber=True)
        tim.data = mod

    ccds = fits_table()
    ccds.filter = np.array([f for f in bands])
    ccds.ccd_cuts = np.zeros(len(ccds), np.int16)
    ccds.imgfn = np.array([tim.name for tim in tims])
    ccds.propid = np.array(['2020A-000'] * len(ccds))
    ccds.fwhm = np.zeros(len(ccds), np.float32) + psf_sigma * 2.35
    ccds.mjd_obs = np.zeros(len(ccds))
    ccds.camera = np.array(['fake'] * len(ccds))

    survey = FakeLegacySurvey(ccds, tims)

    import logging
    verbose = False
    if verbose == 0:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG
    logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
    # tractor logging is *soooo* chatty
    logging.getLogger('tractor.engine').setLevel(lvl + 10)

    run_brick(None,
              survey,
              radec=(ra, dec),
              width=W,
              height=H,
              do_calibs=False,
              gaia_stars=False,
              large_galaxies=False,
              tycho_stars=False,
              forceall=True,
              outliers=False)  #, stages=['image_coadds'])
예제 #37
0
def unwise_forcedphot(cat,
                      tiles,
                      band=1,
                      roiradecbox=None,
                      use_ceres=True,
                      ceres_block=8,
                      save_fits=False,
                      get_models=False,
                      ps=None,
                      psf_broadening=None,
                      pixelized_psf=False,
                      get_masks=None,
                      move_crpix=False,
                      modelsky_dir=None,
                      tag=None):
    '''
    Given a list of tractor sources *cat*
    and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id)
    runs forced photometry, returning a FITS table the same length as *cat*.

    *get_masks*: the WCS to resample mask bits into.
    '''
    from tractor import PointSource, Tractor, ExpGalaxy, DevGalaxy
    from tractor.sersic import SersicGalaxy

    if tag is None:
        tag = ''
    else:
        tag = tag + ': '
    if not pixelized_psf and psf_broadening is None:
        # PSF broadening in post-reactivation data, by band.
        # Newer version from Aaron's email to decam-chatter, 2018-06-14.
        broadening = {1: 1.0405, 2: 1.0346, 3: None, 4: None}
        psf_broadening = broadening[band]

    if False:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence('wise-forced-w%i' % band)
    plots = (ps is not None)
    if plots:
        import pylab as plt

    wantims = (plots or save_fits or get_models)
    wanyband = 'w'
    if get_models:
        models = []

    wband = 'w%i' % band

    Nsrcs = len(cat)
    phot = fits_table()
    # Filled in based on unique tile overlap
    phot.wise_coadd_id = np.array(['        '] * Nsrcs, dtype='U8')
    phot.wise_x = np.zeros(Nsrcs, np.float32)
    phot.wise_y = np.zeros(Nsrcs, np.float32)
    phot.set('psfdepth_%s' % wband, np.zeros(Nsrcs, np.float32))
    nexp = np.zeros(Nsrcs, np.int16)
    mjd = np.zeros(Nsrcs, np.float64)
    central_flux = np.zeros(Nsrcs, np.float32)

    ra = np.array([src.getPosition().ra for src in cat])
    dec = np.array([src.getPosition().dec for src in cat])

    fskeys = ['prochi2', 'profracflux']
    fitstats = {}

    if get_masks:
        mh, mw = get_masks.shape
        maskmap = np.zeros((mh, mw), np.uint32)

    tims = []
    for tile in tiles:
        info(tag + 'Reading WISE tile', tile.coadd_id, 'band', band)
        tim = get_unwise_tractor_image(tile.unwise_dir,
                                       tile.coadd_id,
                                       band,
                                       bandname=wanyband,
                                       roiradecbox=roiradecbox)
        if tim is None:
            debug('Actually, no overlap with WISE coadd tile', tile.coadd_id)
            continue

        if plots:
            sig1 = tim.sig1
            plt.clf()
            plt.imshow(tim.getImage(),
                       interpolation='nearest',
                       origin='lower',
                       cmap='gray',
                       vmin=-3 * sig1,
                       vmax=10 * sig1)
            plt.colorbar()
            tag = '%s W%i' % (tile.coadd_id, band)
            plt.title('%s: tim data' % tag)
            ps.savefig()
            plt.clf()
            plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
                     range=(-5, 10),
                     bins=100)
            plt.xlabel('Per-pixel intensity (Sigma)')
            plt.title(tag)
            ps.savefig()

        if move_crpix and band in [1, 2]:
            realwcs = tim.wcs.wcs
            x, y = realwcs.crpix
            tile_crpix = tile.get('crpix_w%i' % band)
            dx = tile_crpix[0] - 1024.5
            dy = tile_crpix[1] - 1024.5
            realwcs.set_crpix(x + dx, y + dy)
            debug('unWISE', tile.coadd_id, 'band', band, 'CRPIX', x, y,
                  'shift by', dx, dy, 'to', realwcs.crpix)

        if modelsky_dir and band in [1, 2]:
            fn = os.path.join(modelsky_dir,
                              '%s.%i.mod.fits' % (tile.coadd_id, band))
            if not os.path.exists(fn):
                raise RuntimeError('WARNING: does not exist:', fn)
            x0, x1, y0, y1 = tim.roi
            bg = fitsio.FITS(fn)[2][y0:y1, x0:x1]
            assert (bg.shape == tim.shape)

            if plots:
                plt.clf()
                plt.subplot(1, 2, 1)
                plt.imshow(tim.getImage(),
                           interpolation='nearest',
                           origin='lower',
                           cmap='gray',
                           vmin=-3 * sig1,
                           vmax=5 * sig1)
                plt.subplot(1, 2, 2)
                plt.imshow(bg,
                           interpolation='nearest',
                           origin='lower',
                           cmap='gray',
                           vmin=-3 * sig1,
                           vmax=5 * sig1)
                tag = '%s W%i' % (tile.coadd_id, band)
                plt.suptitle(tag)
                ps.savefig()
                plt.clf()
                ha = dict(range=(-5, 10), bins=100, histtype='step')
                plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
                         color='b',
                         label='Original',
                         **ha)
                plt.hist(((tim.getImage() - bg) *
                          tim.inverr)[tim.inverr > 0].ravel(),
                         color='g',
                         label='Minus Background',
                         **ha)
                plt.axvline(0, color='k', alpha=0.5)
                plt.xlabel('Per-pixel intensity (Sigma)')
                plt.legend()
                plt.title(tag + ': background')
                ps.savefig()

            # Actually subtract the background!
            tim.data -= bg

        # Floor the per-pixel variances,
        # and add Poisson contribution from sources
        if band in [1, 2]:
            # in Vega nanomaggies per pixel
            floor_sigma = {1: 0.5, 2: 2.0}
            poissons = {1: 0.15, 2: 0.3}
            with np.errstate(divide='ignore'):
                new_ie = 1. / np.sqrt(
                    (1. / tim.inverr)**2 + floor_sigma[band] +
                    poissons[band]**2 * np.maximum(0., tim.data))
            new_ie[tim.inverr == 0] = 0.

            if plots:
                plt.clf()
                plt.plot((1. / tim.inverr[tim.inverr > 0]).ravel(),
                         (1. / new_ie[tim.inverr > 0]).ravel(), 'b.')
                plt.title('unWISE per-pixel error: %s band %i' %
                          (tile.coadd_id, band))
                plt.xlabel('original')
                plt.ylabel('floored')
                ps.savefig()

            assert (np.all(np.isfinite(new_ie)))
            assert (np.all(new_ie >= 0.))
            tim.inverr = new_ie

            # Expand a 3-pixel radius around weight=0 (saturated) pixels
            # from Eddie via crowdsource
            # https://github.com/schlafly/crowdsource/blob/7069da3e7d9d3124be1cbbe1d21ffeb63fc36dcc/python/wise_proc.py#L74
            ## FIXME -- W3/W4 ??
            satlimit = 85000
            msat = ((tim.data > satlimit) | ((tim.nims == 0) &
                                             (tim.nuims > 1)))
            from scipy.ndimage.morphology import binary_dilation
            xx, yy = np.mgrid[-3:3 + 1, -3:3 + 1]
            dilate = xx**2 + yy**2 <= 3**2
            msat = binary_dilation(msat, dilate)
            nbefore = np.sum(tim.inverr == 0)
            tim.inverr[msat] = 0
            nafter = np.sum(tim.inverr == 0)
            debug('Masking an additional', (nafter - nbefore),
                  'near-saturated pixels in unWISE', tile.coadd_id, 'band',
                  band)

        # Read mask file?
        if get_masks:
            from astrometry.util.resample import resample_with_wcs, OverlapError
            # unwise_dir can be a colon-separated list of paths
            tilemask = None
            for d in tile.unwise_dir.split(':'):
                fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id,
                                  'unwise-%s-msk.fits.gz' % tile.coadd_id)
                if os.path.exists(fn):
                    debug('Reading unWISE mask file', fn)
                    x0, x1, y0, y1 = tim.roi
                    tilemask = fitsio.FITS(fn)[0][y0:y1, x0:x1]
                    break
            if tilemask is None:
                info('unWISE mask file for tile', tile.coadd_id,
                     'does not exist')
            else:
                try:
                    tanwcs = tim.wcs.wcs
                    assert (tanwcs.shape == tilemask.shape)
                    Yo, Xo, Yi, Xi, _ = resample_with_wcs(get_masks,
                                                          tanwcs,
                                                          intType=np.int16)
                    # Only deal with mask pixels that are set.
                    I, = np.nonzero(tilemask[Yi, Xi] > 0)
                    # Trim to unique area for this tile
                    rr, dd = get_masks.pixelxy2radec(Xo[I] + 1, Yo[I] + 1)
                    good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2,
                                                tile.dec1, tile.dec2)
                    I = I[good]
                    maskmap[Yo[I], Xo[I]] = tilemask[Yi[I], Xi[I]]
                except OverlapError:
                    # Shouldn't happen by this point
                    print('Warning: no overlap between WISE tile',
                          tile.coadd_id, 'and brick')

            if plots:
                plt.clf()
                plt.imshow(tilemask, interpolation='nearest', origin='lower')
                plt.title('Tile %s: mask' % tile.coadd_id)
                ps.savefig()
                plt.clf()
                plt.imshow(maskmap, interpolation='nearest', origin='lower')
                plt.title('Tile %s: accumulated maskmap' % tile.coadd_id)
                ps.savefig()

        # The tiles have some overlap, so zero out pixels outside the
        # tile's unique area.
        th, tw = tim.shape
        xx, yy = np.meshgrid(np.arange(tw), np.arange(th))
        rr, dd = tim.wcs.wcs.pixelxy2radec(xx + 1, yy + 1)
        unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1,
                                      tile.dec2)
        debug('Tile', tile.coadd_id, '- total of', np.sum(unique),
              'unique pixels out of', len(unique.flat), 'total pixels')
        if get_models:
            # Save the inverr before blanking out non-unique pixels, for making coadds with no gaps!
            # (actually, slightly more subtly, expand unique area by 1 pixel)
            from scipy.ndimage.morphology import binary_dilation
            du = binary_dilation(unique)
            tim.coadd_inverr = tim.inverr * du
        tim.inverr[unique == False] = 0.
        del xx, yy, rr, dd, unique

        if plots:
            sig1 = tim.sig1
            plt.clf()
            plt.imshow(tim.getImage() * (tim.inverr > 0),
                       interpolation='nearest',
                       origin='lower',
                       cmap='gray',
                       vmin=-3 * sig1,
                       vmax=10 * sig1)
            plt.colorbar()
            tag = '%s W%i' % (tile.coadd_id, band)
            plt.title('%s: tim data (unique)' % tag)
            ps.savefig()

        if pixelized_psf:
            from unwise_psf import unwise_psf
            if (band == 1) or (band == 2):
                # we only have updated PSFs for W1 and W2
                psfimg = unwise_psf.get_unwise_psf(band,
                                                   tile.coadd_id,
                                                   modelname='neo6_unwisecat')
            else:
                psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id)

            if band == 4:
                # oversample (the unwise_psf models are at native W4 5.5"/pix,
                # while the unWISE coadds are made at 2.75"/pix.
                ph, pw = psfimg.shape
                subpsf = np.zeros((ph * 2 - 1, pw * 2 - 1), np.float32)
                from astrometry.util.util import lanczos3_interpolate
                xx, yy = np.meshgrid(
                    np.arange(0., pw - 0.51, 0.5, dtype=np.float32),
                    np.arange(0., ph - 0.51, 0.5, dtype=np.float32))
                xx = xx.ravel()
                yy = yy.ravel()
                ix = xx.astype(np.int32)
                iy = yy.astype(np.int32)
                dx = (xx - ix).astype(np.float32)
                dy = (yy - iy).astype(np.float32)
                psfimg = psfimg.astype(np.float32)
                rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat],
                                           [psfimg])

                if plots:
                    plt.clf()
                    plt.imshow(psfimg, interpolation='nearest', origin='lower')
                    plt.title('Original PSF model')
                    ps.savefig()
                    plt.clf()
                    plt.imshow(subpsf, interpolation='nearest', origin='lower')
                    plt.title('Subsampled PSF model')
                    ps.savefig()

                psfimg = subpsf
                del xx, yy, ix, iy, dx, dy

            from tractor.psf import PixelizedPSF
            psfimg /= psfimg.sum()
            fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0}
            psfimg *= fluxrescales[band]
            tim.psf = PixelizedPSF(psfimg)

        if psf_broadening is not None and not pixelized_psf:
            # psf_broadening is a factor by which the PSF FWHMs
            # should be scaled; the PSF is a little wider
            # post-reactivation.
            psf = tim.getPsf()
            from tractor import GaussianMixturePSF
            if isinstance(psf, GaussianMixturePSF):
                debug('Broadening PSF: from', psf)
                p0 = psf.getParams()
                pnames = psf.getParamNames()
                p1 = [
                    p * psf_broadening**2 if 'var' in name else p
                    for (p, name) in zip(p0, pnames)
                ]
                psf.setParams(p1)
                debug('Broadened PSF:', psf)
            else:
                print(
                    'WARNING: cannot apply psf_broadening to WISE PSF of type',
                    type(psf))

        wcs = tim.wcs.wcs
        _, fx, fy = wcs.radec2pixelxy(ra, dec)
        x = np.round(fx - 1.).astype(int)
        y = np.round(fy - 1.).astype(int)
        good = (x >= 0) * (x < tw) * (y >= 0) * (y < th)
        # Which sources are in this brick's unique area?
        usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1,
                                    tile.dec2)
        I, = np.nonzero(good * usrc)

        nexp[I] = tim.nuims[y[I], x[I]]
        if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'):
            mjd[I] = (tim.mjdmin + tim.mjdmax) / 2.
        phot.wise_coadd_id[I] = tile.coadd_id
        phot.wise_x[I] = fx[I] - 1.
        phot.wise_y[I] = fy[I] - 1.

        central_flux[I] = tim.getImage()[y[I], x[I]]
        del x, y, good, usrc

        # PSF norm for depth
        psf = tim.getPsf()
        h, w = tim.shape
        patch = psf.getPointSourcePatch(h // 2, w // 2).patch
        psfnorm = np.sqrt(np.sum(patch**2))
        # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags.
        # In the small empty patches of the sky (eg W4 in 0922p702), we get sig1 = NaN
        if np.isfinite(tim.sig1):
            phot.get('psfdepth_%s' % wband)[I] = 1. / (tim.sig1 / psfnorm)**2

        tim.tile = tile
        tims.append(tim)

    if plots:
        plt.clf()
        mn, mx = 0.1, 20000
        plt.hist(np.log10(np.clip(central_flux, mn, mx)),
                 bins=100,
                 range=(np.log10(mn), np.log10(mx)))
        logt = np.arange(0, 5)
        plt.xticks(logt, ['%i' % i for i in 10.**logt])
        plt.title('Central fluxes (W%i)' % band)
        plt.axvline(np.log10(20000), color='k')
        plt.axvline(np.log10(1000), color='k')
        ps.savefig()

    # Eddie's non-secret recipe:
    #- central pixel <= 1000: 19x19 pix box size
    #- central pixel in 1000 - 20000: 59x59 box size
    #- central pixel > 20000 or saturated: 149x149 box size
    #- object near "bright star": 299x299 box size
    nbig = nmedium = nsmall = 0
    for src, cflux in zip(cat, central_flux):
        if cflux > 20000:
            R = 100
            nbig += 1
        elif cflux > 1000:
            R = 30
            nmedium += 1
        else:
            R = 15
            nsmall += 1
        if isinstance(src, PointSource):
            src.fixedRadius = R
        else:
            ### FIXME -- sizes for galaxies..... can we set PSF size separately?
            galrad = 0
            # RexGalaxy is a subclass of ExpGalaxy
            if isinstance(src, (ExpGalaxy, DevGalaxy, SersicGalaxy)):
                galrad = src.shape.re
            pixscale = 2.75
            src.halfsize = int(np.hypot(R, galrad * 5 / pixscale))
    debug('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall,
          'small')

    tractor = Tractor(tims, cat)
    if use_ceres:
        from tractor.ceres_optimizer import CeresOptimizer
        tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block)
    tractor.freezeParamsRecursive('*')
    tractor.thawPathsTo(wanyband)

    t0 = Time()
    R = tractor.optimize_forced_photometry(fitstats=True,
                                           variance=True,
                                           shared_params=False,
                                           wantims=wantims)
    info(tag + 'unWISE forced photometry took', Time() - t0)

    if use_ceres:
        term = R.ceres_status['termination']
        # Running out of memory can cause failure to converge and term
        # status = 2.  Fail completely in this case.
        if term != 0:
            info(tag + 'Ceres termination status:', term)
            raise RuntimeError('Ceres terminated with status %i' % term)

    if wantims:
        ims1 = R.ims1
        # can happen if empty source list (we still want to generate coadds)
        if ims1 is None:
            ims1 = R.ims0

    flux_invvars = R.IV
    if R.fitstats is not None:
        for k in fskeys:
            x = getattr(R.fitstats, k)
            fitstats[k] = np.array(x).astype(np.float32)

    if save_fits:
        for i, tim in enumerate(tims):
            tile = tim.tile
            (dat, mod, _, chi, _) = ims1[i]
            wcshdr = fitsio.FITSHDR()
            tim.wcs.wcs.add_to_header(wcshdr)
            tag = 'fit-%s-w%i' % (tile.coadd_id, band)
            fitsio.write('%s-data.fits' % tag,
                         dat,
                         clobber=True,
                         header=wcshdr)
            fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr)
            fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr)

    if plots:
        # Create models for just the brightest sources
        bright_cat = [
            src for src in cat if src.getBrightness().getBand(wanyband) > 1000
        ]
        debug('Bright soures:', len(bright_cat))
        btr = Tractor(tims, bright_cat)
        for tim in tims:
            mod = btr.getModelImage(tim)
            tile = tim.tile
            tag = '%s W%i' % (tile.coadd_id, band)
            sig1 = tim.sig1
            plt.clf()
            plt.imshow(mod,
                       interpolation='nearest',
                       origin='lower',
                       cmap='gray',
                       vmin=-3 * sig1,
                       vmax=25 * sig1)
            plt.colorbar()
            plt.title('%s: bright-star models' % tag)
            ps.savefig()

    if get_models:
        for i, tim in enumerate(tims):
            tile = tim.tile
            (dat, mod, _, _, _) = ims1[i]
            models.append(
                (tile.coadd_id, band, tim.wcs.wcs, dat, mod, tim.coadd_inverr))

    if plots:
        for i, tim in enumerate(tims):
            tile = tim.tile
            tag = '%s W%i' % (tile.coadd_id, band)
            (dat, mod, _, chi, _) = ims1[i]
            sig1 = tim.sig1
            plt.clf()
            plt.imshow(dat,
                       interpolation='nearest',
                       origin='lower',
                       cmap='gray',
                       vmin=-3 * sig1,
                       vmax=25 * sig1)
            plt.colorbar()
            plt.title('%s: data' % tag)
            ps.savefig()
            plt.clf()
            plt.imshow(mod,
                       interpolation='nearest',
                       origin='lower',
                       cmap='gray',
                       vmin=-3 * sig1,
                       vmax=25 * sig1)
            plt.colorbar()
            plt.title('%s: model' % tag)
            ps.savefig()

            plt.clf()
            plt.imshow(chi,
                       interpolation='nearest',
                       origin='lower',
                       cmap='gray',
                       vmin=-5,
                       vmax=+5)
            plt.colorbar()
            plt.title('%s: chi' % tag)
            ps.savefig()

    nm = np.array([src.getBrightness().getBand(wanyband) for src in cat])
    nm_ivar = flux_invvars
    # Sources out of bounds, eg, never change from their initial
    # fluxes.  Zero them out instead.
    nm[nm_ivar == 0] = 0.

    phot.set('flux_%s' % wband, nm.astype(np.float32))
    phot.set('flux_ivar_%s' % wband, nm_ivar.astype(np.float32))
    for k in fskeys:
        phot.set(k + '_' + wband,
                 fitstats.get(k, np.zeros(len(phot), np.float32)))
    phot.set('nobs_%s' % wband, nexp)
    phot.set('mjd_%s' % wband, mjd)

    rtn = wphotduck()
    rtn.phot = phot
    rtn.models = None
    rtn.maskmap = None
    if get_models:
        rtn.models = models
    if get_masks:
        rtn.maskmap = maskmap
    return rtn
예제 #38
0
    def toFits(self, fn, data=None, hdr=None, merge=False):
        '''
        If merge: merge params "amp0", "amp1", ... into an "amp" array.
        '''
        from astrometry.util.fits import fits_table
        if hdr is None:
            import fitsio
            hdr = fitsio.FITSHDR()

        hdr.add_record(
            dict(name='PSFEX_T',
                 value=typestring(type(self)),
                 comment='PsfEx type'))
        hdr.add_record(
            dict(name='PSF_TYPE',
                 value=typestring(self.psfclass),
                 comment='PsfEx PSF type'))
        hdr.add_record(dict(name='PSF_W', value=self.W, comment='Image width'))
        hdr.add_record(dict(name='PSF_H', value=self.H,
                            comment='Image height'))
        # hdr.add_record(dict(name='PSF_SCALING',
        hdr.add_record(
            dict(name='PSF_K',
                 value=self.K,
                 comment='Number of PSF components'))
        hdr.add_record(
            dict(name='PSF_NX',
                 value=self.nx,
                 comment='Number of X grid points'))
        hdr.add_record(
            dict(name='PSF_NY',
                 value=self.ny,
                 comment='Number of Y grid points'))

        if data is None:
            data = self.splinedata
        (pp, XX, YY) = data
        ny, nx, nparams = pp.shape
        assert (ny == self.ny)
        assert (nx == self.nx)

        X = self.psfclass(*pp[0, 0])
        names = X.getParamNames()

        hdr.add_record(
            dict(name='PSF_NA',
                 value=len(names),
                 comment='PSF number of params'))
        for i, nm in enumerate(names):
            hdr.add_record(
                dict(name='PSF_A%i' % i, value=nm, comment='PSF param name'))

        T = fits_table()
        T.xx = XX.reshape((1, len(XX)))
        T.yy = YY.reshape((1, len(YY)))
        if merge:
            # find like params and group them together.
            # assume names like "amp0"
            assert (self.K < 10)
            pnames = set()
            for nm in names:
                assert (nm[-1] in '0123456789'[:self.K])
                pnames.add(nm[:-1])
            assert (len(pnames) * self.K == nparams)
            pnames = list(pnames)
            pnames.sort()
            print('Pnames:', pnames)
            namemap = dict([(nm, i) for i, nm in enumerate(names)])
            for i, nm in enumerate(pnames):
                X = np.empty((1, self.K, ny, nx))
                for k in range(self.K):
                    X[0, k, :, :] = pp[:, :, namemap['%s%i' % (nm, k)]]
                T.set(nm, X)
                # X = np.dstack([pp[:,:,namemap['%s%i' % (nm, k)]] for k in range(self.K)])
                # print 'pname', nm, 'array:', X.shape
                # T.set(nm, X.reshape((1,self.K,ny,nx)))
        else:
            for i, nm in enumerate(names):
                T.set(nm, pp[:, :, i].reshape((1, ny, nx)))
        T.writeto(fn, header=hdr)
예제 #39
0
def _bounce_main((i, ccds)):
    try:
        outfn = 'ccds-annotated/ccds-annotated-%03i.fits' % i
        if os.path.exists(outfn):
            print('Already exists:', outfn)
            return
        main(outfn=outfn, ccds=ccds)
    except:
        import traceback
        traceback.print_exc()


if __name__ == '__main__':

    TT = [
        fits_table('ccds-annotated/ccds-annotated-%03i.fits' % i)
        for i in range(515)
    ]
    T = merge_tables(TT)
    T.writeto('ccds-annotated.fits')

    import sys
    sys.exit()
    #sys.exit(main())

    decals = Decals()
    ccds = decals.get_ccds()
    from astrometry.util.multiproc import *
    #mp = multiproc(8)
    mp = multiproc(4)
    N = 1000
def run_tiles(X):
    tiles, tag = X
    print('Running', tag, '-', len(tiles), 'tiles')

    # Aaron's file has all images share the boresight CRVAL, so they have large CRPIX values.
    T = fits_table(
        '/global/cfs/cdirs/desi/users/ameisner/GFA/gfa_reduce_etc/gfa_wcs+focus.bigtan-zenith.fits'
    )

    Nbright = 10
    tiles_ann = fits_table()
    tiles_ann.index = tiles.index
    gfa_regions = []
    maxr = 0.

    #t.cd[0,0], t.cd[0,1], t.cd[1,0], t.cd[1,1],
    for t in T:
        wcs = Tan(0., 0., t.crpix[0], t.crpix[1], t.cd[0, 0], t.cd[1, 0],
                  t.cd[0, 1], t.cd[1, 1], float(t.naxis[0]), float(t.naxis[1]))
        ctype = t.extname[:5]
        cnum = int(t.extname[5])
        h, w = wcs.shape
        x, y = [1, 1, w, w, 1], [1, h, h, 1, 1]
        r, d = wcs.pixelxy2radec(x, y)
        dists = degrees_between(0., 0., r, d)
        maxr = max(maxr, max(dists))

        # x0, y0, x1, y1
        rois = []
        if ctype == 'FOCUS':
            # add the two half-chips.
            # wcs.get_subimage moves the CRPIX, but leave CRVAL unchanged, so tx,ty still work unchanged.
            # Aaron's WCS templates correct for the overscans
            #wcs_subs.append((cstr, cnum, 'a', wcs.get_subimage(0, 0, 1024, h)))
            #wcs_subs.append((cstr, cnum, 'b', wcs.get_subimage(1024, 0, 1024, h)))

            #all_sub_wcs[(cstr, cnum, 1)] = (tx, ty, wcs.get_subimage(50, 0, 1024, 1032))
            #all_sub_wcs[(cstr, cnum, 2)] = (tx, ty, wcs.get_subimage(1174, 0, 1024, 1032))
            # Add (negative) margin for donut size and telescope pointing uncertainty.
            # ~10" for donuts and ~10" for telescope pointing
            #margin = 100
            #wcs_subs.append((cstr, cnum, 'a_margin', wcs.get_subimage(margin, margin, 1024-2*margin, h-2*margin)))
            #wcs_subs.append((cstr, cnum, 'b_margin', wcs.get_subimage(1024+margin, margin, 1024-2*margin, h-2*margin)))

            # Also add a positive margin for bright-star reflections off filters
            #margin = 125
            #wcs_subs.append((cstr, cnum, 'expanded', wcs.get_subimage(-margin, -margin, w+2*margin, h+2*margin)))

            rois.append(('a', 0, 0, 1024, h))
            rois.append(('b', 1024, 0, 2048, h))
            margin = 100
            rois.append(
                ('a_margin', margin, margin, 1024 - margin, h - margin))
            rois.append(
                ('b_margin', 1024 + margin, margin, 2048 - margin, h - margin))

            margin = 125
            rois.append(('expanded', -margin, -margin, w + margin, h + margin))

        else:
            # Guide chips include overscan pixels -- including a blank region in the middle.
            #print(cstr,cnum, 'shape', wcs.shape)
            #wcs_subs.append((cstr, cnum, 'ccd', wcs))

            rois.append(('ccd', 0, 0, w, h))

            # Add expanded GUIDE chips -- 25" margin / 0.2"/pix = 125 pix
            margin = 125
            #wcs_subs.append((cstr, cnum, 'expanded', wcs.get_subimage(-margin, -margin, w+2*margin, h+2*margin)))
            rois.append(('expanded', -margin, -margin, w + margin, h + margin))

        margin = 125
        expwcs = wcs.get_subimage(-margin, -margin, w + 2 * margin,
                                  h + 2 * margin)

        newrois = []
        for tag, x0, y0, x1, y1 in rois:
            name = '%s_%i_%s' % (ctype.lower(), cnum, tag)
            arr = np.zeros(len(tiles), (np.float32, Nbright))
            tiles_ann.set('brightest_' + name, arr)
            # (the rois have zero-indexed x0,y0, and non-inclusive x1,y1!)
            newrois.append((name, arr, 1 + x0, 1 + y0, x1, y1))

        gfa_regions.append((ctype, cnum, wcs, expwcs, newrois))

    # DEBUG WCS
    # s = []
    # for ctype,cnum,wcs,expwcs,rois in gfa_regions:
    #     WCS = wcs
    #     h,w = WCS.shape
    #     #print('Expwcs:', w, 'x', h)
    #     x = [1,1,w,w,1]
    #     y = [1,h,h,1,1]
    #     r,d = WCS.pixelxy2radec(x, y)
    #     p = ','.join(['%.4f,%.4f' % (rr,dd) for rr,dd in zip(r,d)])
    #     s.append(p)
    # s = ';'.join(s)
    # print('http://legacysurvey.org/viewer/?ra=0&dec=0&poly='+s)
    # sys.exit(0)

    gaia = CachingGaiaCatalog(columns=[
        'ra', 'dec', 'phot_g_mean_mag', 'phot_bp_mean_mag', 'phot_rp_mean_mag',
        'astrometric_excess_noise', 'astrometric_params_solved', 'source_id',
        'pmra_error', 'pmdec_error', 'parallax_error', 'ra_error', 'dec_error',
        'pmra', 'pmdec', 'parallax', 'ref_epoch'
    ])

    tyc2fn = '/global/cfs/cdirs/cosmo/staging/tycho2/tycho2.kd.fits'
    tycho_kd = tree_open(tyc2fn)
    tycho_cat = fits_table(tyc2fn)

    maxrad = maxr * 1.05
    for itile, tile in enumerate(tiles):
        #if not tile.in_imaging:
        #    continue
        #if tile.centerid % 10 == 0:
        #print('tile program', tile.program, 'pass', tile.get('pass'), 'id', tile.centerid, gaia.get_healpix_tree.cache_info())

        I = tree_search_radec(tycho_kd, tile.ra, tile.dec, maxrad)
        tycstars = tycho_cat[I]
        fix_tycho(tycstars)

        for cstr, cname, chipwcs, bigwcs, rois in gfa_regions:
            h, w = chipwcs.shape
            chipwcs.set_crval(tile.ra, tile.dec)
            bigwcs.set_crval(tile.ra, tile.dec)
            gstars = gaia.get_catalog_in_wcs(bigwcs, step=1032, margin=0)
            fix_gaia(gstars)

            bh, bw = bigwcs.shape
            ok, x, y = bigwcs.radec2pixelxy(tycstars.ra, tycstars.dec)
            tstars = tycstars[(x >= 1) * (y >= 1) * (x <= bw) * (y <= bh)]

            #print('Tile', tile.program, 'p', tile.get('pass'), tile.centerid,
            #      'GFA', cstr, cname, ':', len(gstars), 'Gaia stars', len(tstars), 'Tycho-2 stars')

            if len(gstars) + len(tstars) == 0:
                print('No stars in tile centerid', tile.centerid, 'chip', name)
                continue

            if len(gstars) > 0 and len(tstars) > 0:
                merge_gaia_tycho(gstars, tstars)
                stars = merge_tables([gstars, tstars], columns='fillzero')
            elif len(tstars) > 0:
                stars = tstars
            else:
                stars = gstars

            ok, x, y = chipwcs.radec2pixelxy(stars.ra, stars.dec)

            for name, arr, x0, y0, x1, y1 in rois:
                J = np.flatnonzero(
                    (x >= x0) * (x <= x1) * (y >= y0) * (y <= y1))
                mags = stars.mag[J]
                #print('  ', len(mags), 'in name')
                K = np.argsort(mags)
                K = K[:Nbright]
                arr[itile, :len(K)] = mags[K]

    #tiles.add_columns_from(tiles_ann)
    return tiles_ann
 def get_healpix_tree(self, healpix):
     from astrometry.util.fits import fits_table
     fname = self.fnpattern % dict(hp=healpix)
     tab = fits_table(fname, columns=self.columns)
     kd = tree_build_radec(tab.ra, tab.dec)
     return (kd, tab)
def main(fn, mp):
    basefn = os.path.basename(fn)
    tiles = fits_table(fn)

    #I = np.flatnonzero(tiles.in_imaging)
    #tiles.cut(I)

    ### Split the tiles into nearby chunks of work for multi-processing.
    from astrometry.util.util import radecdegtohealpix
    nside = 8
    Nhp = 12 * nside**2
    Ihps = [[] for i in range(Nhp)]
    for i, (r, d) in enumerate(zip(tiles.ra, tiles.dec)):
        hp = radecdegtohealpix(r, d, nside)
        assert (hp >= 0)
        Ihps[hp].append(i)

    args = []
    tiles.index = np.arange(len(tiles))
    for hp, I in enumerate(Ihps):
        if len(I) == 0:
            continue
        args.append((tiles[np.array(I)], 'HP %i' % hp))
    print(len(args), 'big healpixes are populated')
    R = mp.map(run_tiles, args)
    tiles_ann = merge_tables(R)
    print(len(tiles), 'tiles')
    print(len(tiles_ann), 'annotated')
    tiles_ann.about()
    # unpermute
    I = np.zeros(len(tiles_ann), np.int32)
    I[tiles_ann.index] = np.arange(len(tiles_ann))
    tiles_ann.cut(I)

    tiles.add_columns_from(tiles_ann)
    tiles.delete_column('index')

    outfn = basefn.replace('.fits', '-brightest.fits')
    tiles.writeto(outfn)

    # Nudge (only inside imaging footprint)

    from functools import reduce
    brightest = reduce(np.minimum, [
        tiles.brightest_guide_0_expanded[:, 0],
        tiles.brightest_guide_2_expanded[:, 0],
        tiles.brightest_guide_3_expanded[:, 0],
        tiles.brightest_guide_5_expanded[:, 0],
        tiles.brightest_guide_7_expanded[:, 0],
        tiles.brightest_guide_8_expanded[:, 0],
        tiles.brightest_focus_1_expanded[:, 0],
        tiles.brightest_focus_4_expanded[:, 0],
        tiles.brightest_focus_6_expanded[:, 0],
        tiles.brightest_focus_9_expanded[:, 0],
    ])

    if 'in_imaging' in tiles.get_columns():
        I = np.flatnonzero((brightest < 6.) * tiles.in_imaging)
    else:
        I = np.flatnonzero((brightest < 6.))
    print(len(I), 'tiles with G<6')

    tiles.nudge_ra = np.zeros(len(tiles), np.float32)
    tiles.nudge_dec = np.zeros(len(tiles), np.float32)
    tiles.nudge_brightest = np.zeros(len(tiles), np.float32)
    tiles.index = np.arange(len(tiles))

    for nudge in range(1, 20):
        if len(I) == 0:
            break

        print('Nudging', len(I), 'by', nudge)

        ddec = 10. / 3600.
        dra = ddec / np.cos(np.deg2rad(tiles.dec[I]))

        # copy tiles
        nudgetiles = tiles[np.repeat(I, 4)]
        nudgetiles.nudge_ra[0::4] = +nudge * dra
        nudgetiles.nudge_ra[1::4] = -nudge * dra
        nudgetiles.nudge_dec[2::4] = +nudge * ddec
        nudgetiles.nudge_dec[3::4] = -nudge * ddec

        nudgetiles.ra += nudgetiles.nudge_ra
        nudgetiles.dec += nudgetiles.nudge_dec

        #ann = run_tiles((nudgetiles, 'nudge%i' % nudge))
        # split into subsets
        args = []
        isplit = np.linspace(0, len(nudgetiles), 33, dtype=int)
        for i0, i1 in zip(isplit, isplit[1:]):
            if i0 == i1:
                continue
            args.append((nudgetiles[i0:i1], 'nudge%i+%i' % (nudge, i0)))
        A = mp.map(run_tiles, args)
        ann = merge_tables(A)

        ann.brightest = reduce(np.minimum, [
            ann.brightest_guide_0_expanded[:, 0],
            ann.brightest_guide_2_expanded[:, 0],
            ann.brightest_guide_3_expanded[:, 0],
            ann.brightest_guide_5_expanded[:, 0],
            ann.brightest_guide_7_expanded[:, 0],
            ann.brightest_guide_8_expanded[:, 0],
            ann.brightest_focus_1_expanded[:, 0],
            ann.brightest_focus_4_expanded[:, 0],
            ann.brightest_focus_6_expanded[:, 0],
            ann.brightest_focus_9_expanded[:, 0],
        ])
        ok = (ann.brightest > 6)

        found = np.zeros(len(I), bool)
        for idir, dirok in enumerate([ok[0::4], ok[1::4], ok[2::4], ok[3::4]]):
            J = np.flatnonzero(np.logical_not(found) * dirok)
            print('Nudge dir', idir, ':', len(J), 'are okay')
            found[J] = True
            tiles.nudge_ra[I[J]] = nudgetiles.nudge_ra[J * 4 + idir]
            tiles.nudge_dec[I[J]] = nudgetiles.nudge_dec[J * 4 + idir]
            tiles.nudge_brightest[I[J]] = ann.brightest[J * 4 + idir]

        I = I[np.flatnonzero(found == False)]

    outfn = basefn.replace('.fits', '-brightest-nudged.fits')
    tiles.writeto(outfn)
예제 #43
0
def main(outfn='ccds-annotated.fits', ccds=None):
    decals = Decals()
    if ccds is None:
        ccds = decals.get_ccds()

    # File from the "observing" svn repo:
    # https://desi.lbl.gov/svn/decam/code/observing/trunk
    tiles = fits_table('decam-tiles_obstatus.fits')

    #ccds.cut(np.arange(100))
    #print("HACK!")
    #ccds.cut(np.array([name in ['N15', 'N16', 'N21', 'N9']
    #                   for name in ccds.ccdname]) *
    #                   ccds.expnum == 229683)

    I = decals.photometric_ccds(ccds)
    ccds.photometric = np.zeros(len(ccds), bool)
    ccds.photometric[I] = True

    I = decals.apply_blacklist(ccds)
    ccds.blacklist_ok = np.zeros(len(ccds), bool)
    ccds.blacklist_ok[I] = True

    ccds.good_region = np.empty((len(ccds), 4), np.int16)
    ccds.good_region[:, :] = -1

    ccds.ra0 = np.zeros(len(ccds), np.float64)
    ccds.dec0 = np.zeros(len(ccds), np.float64)
    ccds.ra1 = np.zeros(len(ccds), np.float64)
    ccds.dec1 = np.zeros(len(ccds), np.float64)
    ccds.ra2 = np.zeros(len(ccds), np.float64)
    ccds.dec2 = np.zeros(len(ccds), np.float64)
    ccds.ra3 = np.zeros(len(ccds), np.float64)
    ccds.dec3 = np.zeros(len(ccds), np.float64)

    ccds.dra = np.zeros(len(ccds), np.float32)
    ccds.ddec = np.zeros(len(ccds), np.float32)
    ccds.ra_center = np.zeros(len(ccds), np.float64)
    ccds.dec_center = np.zeros(len(ccds), np.float64)

    ccds.sig1 = np.zeros(len(ccds), np.float32)

    ccds.meansky = np.zeros(len(ccds), np.float32)
    ccds.stdsky = np.zeros(len(ccds), np.float32)
    ccds.maxsky = np.zeros(len(ccds), np.float32)
    ccds.minsky = np.zeros(len(ccds), np.float32)

    ccds.pixscale_mean = np.zeros(len(ccds), np.float32)
    ccds.pixscale_std = np.zeros(len(ccds), np.float32)
    ccds.pixscale_max = np.zeros(len(ccds), np.float32)
    ccds.pixscale_min = np.zeros(len(ccds), np.float32)

    ccds.psfnorm_mean = np.zeros(len(ccds), np.float32)
    ccds.psfnorm_std = np.zeros(len(ccds), np.float32)
    ccds.galnorm_mean = np.zeros(len(ccds), np.float32)
    ccds.galnorm_std = np.zeros(len(ccds), np.float32)

    gaussgalnorm = np.zeros(len(ccds), np.float32)

    # 2nd moments
    ccds.psf_mx2 = np.zeros(len(ccds), np.float32)
    ccds.psf_my2 = np.zeros(len(ccds), np.float32)
    ccds.psf_mxy = np.zeros(len(ccds), np.float32)
    #
    ccds.psf_a = np.zeros(len(ccds), np.float32)
    ccds.psf_b = np.zeros(len(ccds), np.float32)
    ccds.psf_theta = np.zeros(len(ccds), np.float32)
    ccds.psf_ell = np.zeros(len(ccds), np.float32)

    ccds.humidity = np.zeros(len(ccds), np.float32)
    ccds.outtemp = np.zeros(len(ccds), np.float32)

    ccds.tileid = np.zeros(len(ccds), np.int32)
    ccds.tilepass = np.zeros(len(ccds), np.uint8)
    ccds.tileebv = np.zeros(len(ccds), np.float32)

    plvers = []

    for iccd, ccd in enumerate(ccds):
        im = decals.get_image_object(ccd)
        print('Reading CCD %i of %i:' % (iccd + 1, len(ccds)), im)

        X = im.get_good_image_subregion()
        for i, x in enumerate(X):
            if x is not None:
                ccds.good_region[iccd, i] = x

        W, H = ccd.width, ccd.height

        psf = None
        wcs = None
        sky = None
        try:
            tim = im.get_tractor_image(pixPsf=True,
                                       splinesky=True,
                                       subsky=False,
                                       pixels=False)
        except:
            import traceback
            traceback.print_exc()
            plvers.append('')
            continue

        if tim is None:
            plvers.append('')
            continue

        psf = tim.psf
        wcs = tim.wcs.wcs
        sky = tim.sky
        hdr = tim.primhdr

        # print('Got PSF', psf)
        # print('Got sky', type(sky))
        # print('Got WCS', wcs)

        ccds.humidity[iccd] = hdr.get('HUMIDITY')
        ccds.outtemp[iccd] = hdr.get('OUTTEMP')

        ccds.sig1[iccd] = tim.sig1
        plvers.append(tim.plver)

        obj = hdr.get('OBJECT')
        # parse 'DECaLS_15150_r'
        words = obj.split('_')
        tile = None
        if len(words) == 3 and words[0] == 'DECaLS':
            try:
                tileid = int(words[1])
                tile = tiles[tileid - 1]
                if tile.tileid != tileid:
                    I = np.flatnonzero(tile.tileid == tileid)
                    tile = tiles[I[0]]
            except:
                pass

        if tile is not None:
            ccds.tileid[iccd] = tile.tileid
            ccds.tilepass[iccd] = tile.get('pass')
            ccds.tileebv[iccd] = tile.ebv_med

        # Instantiate PSF on a grid
        S = 32
        xx = np.linspace(1 + S, W - S, 5)
        yy = np.linspace(1 + S, H - S, 5)
        xx, yy = np.meshgrid(xx, yy)
        psfnorms = []
        galnorms = []
        for x, y in zip(xx.ravel(), yy.ravel()):
            p = im.psf_norm(tim, x=x, y=y)
            g = im.galaxy_norm(tim, x=x, y=y)
            psfnorms.append(p)
            galnorms.append(g)
        ccds.psfnorm_mean[iccd] = np.mean(psfnorms)
        ccds.psfnorm_std[iccd] = np.std(psfnorms)
        ccds.galnorm_mean[iccd] = np.mean(galnorms)
        ccds.galnorm_std[iccd] = np.std(galnorms)

        # PSF in center of field
        cx, cy = (W + 1) / 2., (H + 1) / 2.
        p = psf.getPointSourcePatch(cx, cy).patch
        ph, pw = p.shape
        px, py = np.meshgrid(np.arange(pw), np.arange(ph))
        psum = np.sum(p)
        # print('psum', psum)
        p /= psum
        # centroids
        cenx = np.sum(p * px)
        ceny = np.sum(p * py)
        # print('cenx,ceny', cenx,ceny)
        # second moments
        x2 = np.sum(p * (px - cenx)**2)
        y2 = np.sum(p * (py - ceny)**2)
        xy = np.sum(p * (px - cenx) * (py - ceny))
        # semi-major/minor axes and position angle
        theta = np.rad2deg(np.arctan2(2 * xy, x2 - y2) / 2.)
        theta = np.abs(theta) * np.sign(xy)
        s = np.sqrt(((x2 - y2) / 2.)**2 + xy**2)
        a = np.sqrt((x2 + y2) / 2. + s)
        b = np.sqrt((x2 + y2) / 2. - s)
        ell = 1. - b / a

        # print('PSF second moments', x2, y2, xy)
        # print('PSF position angle', theta)
        # print('PSF semi-axes', a, b)
        # print('PSF ellipticity', ell)

        ccds.psf_mx2[iccd] = x2
        ccds.psf_my2[iccd] = y2
        ccds.psf_mxy[iccd] = xy
        ccds.psf_a[iccd] = a
        ccds.psf_b[iccd] = b
        ccds.psf_theta[iccd] = theta
        ccds.psf_ell[iccd] = ell

        # Galaxy norm using Gaussian approximation of PSF.
        realpsf = tim.psf
        tim.psf = im.read_psf_model(0,
                                    0,
                                    gaussPsf=True,
                                    psf_sigma=tim.psf_sigma)
        gaussgalnorm[iccd] = im.galaxy_norm(tim, x=cx, y=cy)
        tim.psf = realpsf

        # Sky
        mod = np.zeros((ccd.height, ccd.width), np.float32)
        sky.addTo(mod)
        ccds.meansky[iccd] = np.mean(mod)
        ccds.stdsky[iccd] = np.std(mod)
        ccds.maxsky[iccd] = mod.max()
        ccds.minsky[iccd] = mod.min()

        # WCS
        ccds.ra0[iccd], ccds.dec0[iccd] = wcs.pixelxy2radec(1, 1)
        ccds.ra1[iccd], ccds.dec1[iccd] = wcs.pixelxy2radec(1, H)
        ccds.ra2[iccd], ccds.dec2[iccd] = wcs.pixelxy2radec(W, H)
        ccds.ra3[iccd], ccds.dec3[iccd] = wcs.pixelxy2radec(W, 1)

        midx, midy = (W + 1) / 2., (H + 1) / 2.
        rc, dc = wcs.pixelxy2radec(midx, midy)
        ra, dec = wcs.pixelxy2radec([1, W, midx, midx], [midy, midy, 1, H])
        ccds.dra[iccd] = max(
            degrees_between(ra, dc + np.zeros_like(ra), rc, dc))
        ccds.ddec[iccd] = max(
            degrees_between(rc + np.zeros_like(dec), dec, rc, dc))
        ccds.ra_center[iccd] = rc
        ccds.dec_center[iccd] = dc

        # Compute scale change across the chip
        # how many pixels to step
        step = 10
        xx = np.linspace(1 + step, W - step, 5)
        yy = np.linspace(1 + step, H - step, 5)
        xx, yy = np.meshgrid(xx, yy)
        pixscale = []
        for x, y in zip(xx.ravel(), yy.ravel()):
            sx = [x - step, x - step, x + step, x + step, x - step]
            sy = [y - step, y + step, y + step, y - step, y - step]
            sr, sd = wcs.pixelxy2radec(sx, sy)
            rc, dc = wcs.pixelxy2radec(x, y)
            # project around a tiny little TAN WCS at (x,y), with 1" pixels
            locwcs = Tan(rc, dc, 0., 0., 1. / 3600, 0., 0., 1. / 3600, 1., 1.)
            ok, lx, ly = locwcs.radec2pixelxy(sr, sd)
            #print('local x,y:', lx, ly)
            A = polygon_area((lx, ly))
            pixscale.append(np.sqrt(A / (2 * step)**2))
        # print('Pixel scales:', pixscale)
        ccds.pixscale_mean[iccd] = np.mean(pixscale)
        ccds.pixscale_min[iccd] = min(pixscale)
        ccds.pixscale_max[iccd] = max(pixscale)
        ccds.pixscale_std[iccd] = np.std(pixscale)

    ccds.plver = np.array(plvers)

    sfd = tractor.sfd.SFDMap()
    allbands = 'ugrizY'
    filts = ['%s %s' % ('DES', f) for f in allbands]
    wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4']
    ebv, ext = sfd.extinction(filts + wisebands,
                              ccds.ra_center,
                              ccds.dec_center,
                              get_ebv=True)
    ext = ext.astype(np.float32)
    ccds.ebv = ebv.astype(np.float32)
    ccds.decam_extinction = ext[:, :len(allbands)]
    ccds.wise_extinction = ext[:, len(allbands):]

    # Depth
    detsig1 = ccds.sig1 / ccds.psfnorm_mean
    depth = 5. * detsig1
    # that's flux in nanomaggies -- convert to mag
    ccds.psfdepth = -2.5 * (np.log10(depth) - 9)

    detsig1 = ccds.sig1 / ccds.galnorm_mean
    depth = 5. * detsig1
    # that's flux in nanomaggies -- convert to mag
    ccds.galdepth = -2.5 * (np.log10(depth) - 9)

    # Depth using Gaussian FWHM.
    psf_sigma = ccds.fwhm / 2.35
    gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
    detsig1 = ccds.sig1 / gnorm
    depth = 5. * detsig1
    # that's flux in nanomaggies -- convert to mag
    ccds.gausspsfdepth = -2.5 * (np.log10(depth) - 9)

    # Gaussian galaxy depth
    detsig1 = ccds.sig1 / gaussgalnorm
    depth = 5. * detsig1
    # that's flux in nanomaggies -- convert to mag
    ccds.gaussgaldepth = -2.5 * (np.log10(depth) - 9)

    ccds.writeto(outfn)
예제 #44
0
def prepare_fits_catalog(cat,
                         invvars,
                         T,
                         bands,
                         allbands=None,
                         prefix='',
                         save_invvars=True,
                         force_keep=None):
    if T is None:
        from astrometry.util.fits import fits_table
        T = fits_table()
    if allbands is None:
        allbands = bands

    params0 = cat.getParams()

    flux = np.zeros((len(T), len(allbands)), np.float32)
    flux_ivar = np.zeros((len(T), len(allbands)), np.float32)

    for band in bands:
        i = allbands.index(band)
        for j, src in enumerate(cat):
            if src is not None:
                flux[j,
                     i] = sum(b.getFlux(band) for b in src.getBrightnesses())

        if invvars is None:
            continue
        # Oh my, this is tricky... set parameter values to the variance
        # vector so that we can read off the parameter variances via the
        # python object apis.
        cat.setParams(invvars)

        for j, src in enumerate(cat):
            if src is not None:
                flux_ivar[j, i] = sum(
                    b.getFlux(band) for b in src.getBrightnesses())

        cat.setParams(params0)

    T.set('%sflux' % prefix, flux)
    if save_invvars:
        T.set('%sflux_ivar' % prefix, flux_ivar)

    _get_tractor_fits_values(T, cat, '%s%%s' % prefix)

    if save_invvars:
        if invvars is not None:
            cat.setParams(invvars)
        else:
            cat.setParams(np.zeros(cat.numberOfParams()))
        _get_tractor_fits_values(T, cat, '%s%%s_ivar' % prefix)
        # Heh, "no uncertainty here!"
        T.delete_column('%stype_ivar' % prefix)
    cat.setParams(params0)

    # mod RA
    ra = T.get('%sra' % prefix)
    ra += (ra < 0) * 360.
    ra -= (ra > 360) * 360.

    # Downconvert RA,Dec invvars to float32
    if save_invvars:
        for c in ['ra', 'dec']:
            col = '%s%s_ivar' % (prefix, c)
            T.set(col, T.get(col).astype(np.float32))

        # Zero out unconstrained values
        flux = T.get('%s%s' % (prefix, 'flux'))
        iv = T.get('%s%s' % (prefix, 'flux_ivar'))
        if force_keep is not None:
            flux[(iv == 0) * np.logical_not(force_keep[:, np.newaxis])] = 0.
        else:
            flux[iv == 0] = 0.

    return T
예제 #45
0
def main():
    import optparse
    import sys

    parser = optparse.OptionParser(usage='%prog <json>')
    parser.add_option('--base', default='plan', help='Plot base filename')
    parser.add_option('-t', '--obstatus', help='Show already-observed tiles?')
    parser.add_option(
        '--bands',
        help='Plot only already-observed tiles in the given bands',
        default='g,r,z')
    parser.add_option('--sgc', action='store_true', help='Center on SGC?')

    parser.add_option('--ralo', type=float, default=None)
    parser.add_option('--rahi', type=float, default=None)
    parser.add_option('--declo', type=float, default=None)
    parser.add_option('--dechi', type=float, default=None)
    parser.add_option(
        '--scaled',
        action='store_true',
        default=False,
        help='Scale plot so that 1 deg RA = 1 deg Dec (no COS term)')
    parser.add_option('--wide',
                      action='store_true',
                      default=False,
                      help='Make wider plots?')

    parser.add_option('--also',
                      action='append',
                      default=[],
                      help='Also plot the plan from the given filename.')

    parser.add_option('--mosaic',
                      action='store_true',
                      help='Set defaults for Mosaic survey')

    parser.add_option(
        '--start-time',
        help=
        'Start time for this plan, HH:MM:SS UTC.  Default: 12-degree twilight tonight.'
    )
    parser.add_option('--start-date',
                      help='Start date for this plan, YYYY-MM-DD UTC.')

    parser.add_option(
        '--stop-time',
        help='Stop time for this plan, HH:MM:SS UTC.  Default: no limit.')

    parser.add_option(
        '--second-half',
        action='store_true',
        help='This plan starts at the start of the second half-night.')

    parser.add_option('--skip',
                      type=int,
                      default=1,
                      help='Write every Nth plot only')

    parser.add_option('--threads', type=int, help='Multi-processing?')

    opt, args = parser.parse_args()
    if len(args) != 1:
        parser.print_help()
        sys.exit(-1)

    if opt.mosaic:
        dd = dict(ralo=0, rahi=360, declo=30, dechi=88)
    else:
        dd = dict(ralo=0, rahi=360, declo=-10, dechi=35)
    for k in dd.keys():
        if getattr(opt, k, None) is None:
            setattr(opt, k, dd[k])

    start_date_specified = (opt.start_date is not None)

    if opt.start_date is None:
        # Get date at start of night, where we define a new day as
        # starting at noon UTC.
        now = datetime.datetime.utcnow()
        # noon
        nightstart = now - datetime.timedelta(0, 12 * 3600)
        d = nightstart.date()
        opt.start_date = '%04i-%02i-%02i' % (d.year, d.month, d.day)
        print('Set start date to', opt.start_date)

    if opt.mosaic:
        from camera_mosaic import ephem_observer
    else:
        from camera_decam import ephem_observer
    obs = ephem_observer()

    obs.temp = 10.0  # deg celsius; average temp for August
    obs.pressure = 780.0  # mbar

    ### HACK
    obs.date = ephem.Date(opt.start_date + ' 8:00:00')
    #print('Obs date:', obs.date)
    daystart = obs.date
    obs.horizon = -ephem.degrees('12:00:00.0')
    sun = ephem.Sun()
    eve_twi = obs.next_setting(sun)
    obs.date = eve_twi
    morn_twi = obs.next_rising(sun)
    print('Evening twilight:', eve_twi)
    print('Morning twilight:', morn_twi)
    assert (morn_twi > eve_twi)
    obs.horizon = 0.
    print('Eve twi:', eve_twi, 'Morning:', morn_twi)

    if opt.second_half:
        # Set start-time to the midpoint between 12-degree twilights.
        obs.date = ephem.Date((eve_twi + morn_twi) / 2.)
        print('Second half starts at', obs.date)

    elif opt.start_time is None:
        # 12-degree twilight on start_date
        obs.date = eve_twi

    else:
        obs.date = ephem.Date(opt.start_date + ' ' + opt.start_time)
        if not start_date_specified and obs.date < daystart:
            # If --start-date is, eg, 2am, assume it's during the night starting on daystart.
            obs.date = ephem.Date(float(obs.date) + 1.)
        print('Start date:', obs.date)

    if opt.stop_time is not None:
        # The date should be unambiguous -- try the same as obs.date =
        # start time, add one day if necessary.
        date = obs.date.datetime()
        stopdate = ephem.Date('%04i-%02i-%02i' %
                              (date.year, date.month, date.day) + ' ' +
                              opt.stop_time)
        if stopdate < obs.date:
            stopdate = ephem.Date(float(stopdate) + 1.)
        print('Stop date:', stopdate)

    jfn = args[0]
    print('Reading JSON file', jfn)
    J = json.loads(open(jfn, 'rb').read())
    print(len(J), 'entries')

    Jalso = [json.loads(open(fn, 'rb').read()) for fn in opt.also]

    # Get times when exposures should occur.
    times = []
    LSTs = []

    # If the JSON files include estimated times, use those
    if 'approx_datetime' in J[0]:
        for i, j in enumerate(J):
            obs.date = ephem.Date(str(j['approx_datetime']))
            if opt.stop_time is not None and obs.date > stopdate:
                print('Tile', i, 'is after --stopdate')
                J = J[:i]
                assert (len(J) == len(times))
                break
            times.append(ephem.Date(obs.date))
            LSTs.append(np.rad2deg(float(obs.sidereal_time())))
            print('Date', obs.date)
            print('LST', obs.sidereal_time())
    else:
        # Predict overheads
        lastra, lastdec = None, None
        for i in range(len(J)):
            print('Exposure', i, 'should start at', str(obs.date))
            if opt.stop_time is not None and obs.date > stopdate:
                print('Tile', J[i], 'is after --stopdate')
                break
            times.append(ephem.Date(obs.date))
            LSTs.append(np.rad2deg(float(obs.sidereal_time())))
            overhead = 30.
            if lastra is not None:
                slew = degrees_between(lastra, lastdec, ras[i], decs[i])
                lastra = ras[i]
                lastdec = decs[i]
                # Add 3 seconds per degree for slews longer than 2 degrees
                overhead += np.maximum(0, slew - 2.) * 3.
            # Add overhead
            print('Adding', exptime[i], 'seconds exptime plus', overhead,
                  'seconds overhead')
            obs.date += (exptime[i] + overhead) / (24 * 3600.)

    tiles = None
    if opt.obstatus is not None:
        from astrometry.util.fits import fits_table

        tiles = fits_table(opt.obstatus)
        print('Read', len(tiles), 'tiles')
        tiles = tiles[(tiles.in_des == 0) * np.logical_or(
            (tiles.in_sdss == 1), (tiles.in_sdss == 0) * (tiles.in_desi == 1))]
        print(len(tiles), 'in footprint')

    fcmap = dict(g='g', r='r', z='m', zd='m')
    ddecmap = dict(g=-0.2, r=0, z=0.2, zd=0.2)

    ras = np.array([j['RA'] for j in J])
    decs = np.array([j['dec'] for j in J])
    filts = np.array([j['filter'] for j in J])
    exptime = np.array([j['expTime'] for j in J])
    fieldname = [j['object'] for j in J]
    passnum = np.zeros(len(J), int)

    filtcc = np.array([fcmap[f] for f in filts])
    ddecs = np.array([ddecmap[f] for f in filts])

    # passmap = { 1: dict(marker='.'),
    #             2: dict(marker='o', mfc='none'),
    #             3: dict(marker='x') }
    passmap = {
        1: dict(marker='.'),
        2: dict(marker='.'),
        3: dict(marker='.'),
    }

    opt.bands = opt.bands.split(',')
    if len(opt.bands) == 1:
        filtddec = {'g': 0, 'r': 0, 'z': 0}
    else:
        ddec = 0.4
        filtddec = {'g': -ddec, 'r': 0, 'z': ddec}

    seqmap = ['r', 'y', 'g', 'b', 'm']
    #seqcc = np.array([seqmap[s % len(seqmap)] for s in seqnum])
    #seqcc = np.array([seqmap[s % len(seqmap)] for s in seqid])

    ax = [
        transform_ra(opt.rahi, opt),
        transform_ra(opt.ralo, opt), opt.declo, opt.dechi
    ]

    alsocolors = 'kbr'

    also = []
    for Ja in Jalso:
        # We assume the --also plan files contain approx_datetime...
        atimes = np.array([ephem.Date(str(j['approx_datetime'])) for j in Ja])
        aras = np.array([j['RA'] for j in Ja])
        adecs = np.array([j['dec'] for j in Ja])
        afilts = np.array([j['filter'] for j in Ja])
        aexptime = np.array([j['expTime'] for j in Ja])
        afieldname = [j['object'] for j in Ja]
        apassnum = np.zeros(len(Ja), int)
        if tiles is not None:
            for i, f in enumerate(afieldname):
                tile = get_tile_from_name(f, tiles)
                if tile is None:
                    continue
                pa = tile.get('pass')
                apassnum[i] = pa
        also.append(
            (atimes, aras, adecs, afilts, aexptime, afieldname, apassnum))

    # Try to get the pass number via parsing the field name to get tile id
    # and looking up the pass number in the tiles table.
    if tiles is not None:
        for i, f in enumerate(fieldname):
            tile = get_tile_from_name(f, tiles)
            if tile is None:
                continue
            pa = tile.get('pass')
            passnum[i] = pa
            print('Field', f, 'tileid', tile.tileid, 'pass', pa)

    allargs = []
    for i in reversed(range(0, len(J), opt.skip)):
        #print('Exposure', i, 'of', len(J))
        fn = '%s-%03i.png' % (opt.base, i)
        fn = os.path.join(os.path.dirname(args[0]), fn)
        pargs = (opt, ax, tiles, filtddec, fcmap, passmap, also, LSTs, times,
                 ras, decs, ddecs, fieldname, passnum, exptime, i, filtcc,
                 alsocolors, ddecmap, fn)
        allargs.append(pargs)

    if opt.threads:
        from astrometry.util.multiproc import multiproc
        mp = multiproc(opt.threads, init=plot_init)
        mp.map(plot_one, allargs)
    else:
        plot_init()
        map(plot_one, allargs)
    #plot_one(pargs)

    print()
    cmd = 'avconv -r 4 -i %s-%%03d.png -y %s.mov' % (opt.base, opt.base)
    print(cmd)
    os.system(cmd)
예제 #46
0
def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-b',
        '--brick',
        help='Brick name to run; required unless --radec is given')
    parser.add_argument(
        '--survey-dir',
        type=str,
        default=None,
        help='Override the $LEGACY_SURVEY_DIR environment variable')
    parser.add_argument('-d',
                        '--outdir',
                        dest='output_dir',
                        help='Set output base directory, default "."')
    parser.add_argument(
        '--out',
        help='Output filename -- if not set, defaults to path within --outdir.'
    )
    parser.add_argument('-r',
                        '--run',
                        default=None,
                        help='Set the run type to execute (for images)')

    parser.add_argument(
        '--catalog',
        help=
        'Use the given FITS catalog file, rather than reading from a data release directory'
    )
    parser.add_argument('--catalog-dir',
                        help='Set LEGACY_SURVEY_DIR to use to read catalogs')
    parser.add_argument(
        '--catalog-dir-north',
        help='Set LEGACY_SURVEY_DIR to use to read Northern catalogs')
    parser.add_argument(
        '--catalog-dir-south',
        help='Set LEGACY_SURVEY_DIR to use to read Southern catalogs')
    parser.add_argument(
        '--catalog-resolve-dec-ngc',
        type=float,
        help=
        'Dec at which to switch from Northern to Southern catalogs (NGC only)',
        default=32.375)
    parser.add_argument('-v',
                        '--verbose',
                        dest='verbose',
                        action='count',
                        default=0,
                        help='Make more verbose')

    opt = parser.parse_args()
    if opt.brick is None:
        parser.print_help()
        return -1
    verbose = opt.verbose
    if verbose == 0:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG
    logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
    # tractor logging is *soooo* chatty
    logging.getLogger('tractor.engine').setLevel(lvl + 10)

    from legacypipe.runs import get_survey
    survey = get_survey(opt.run,
                        survey_dir=opt.survey_dir,
                        output_dir=opt.output_dir)

    columns = [
        'release',
        'brickid',
        'objid',
    ]

    cat = None
    catsurvey = survey
    if opt.catalog is not None:
        cat = fits_table(opt.catalog, columns=columns)
        print('Read', len(cat), 'sources from', opt.catalog)
    else:
        from astrometry.util.starutil_numpy import radectolb
        # The "north" and "south" directories often don't have
        # 'survey-bricks" files of their own -- use the 'survey' one
        # instead.
        brick = None
        for s in [survey, catsurvey]:
            try:
                brick = s.get_brick_by_name(opt.brick)
                break
            except:
                import traceback
                traceback.print_exc()
                pass

        l, b = radectolb(brick.ra, brick.dec)
        # NGC and above resolve line? -> north
        if b > 0 and brick.dec >= opt.catalog_resolve_dec_ngc:
            if opt.catalog_dir_north:
                catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir_north)
        else:
            if opt.catalog_dir_south:
                catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir_south)

        fn = catsurvey.find_file('tractor', brick=opt.brick)
        cat = fits_table(fn, columns=columns)
        print('Read', len(cat), 'sources from', fn)

    program_name = sys.argv[0]
    ## FIXME -- from catalog?
    release = 9999
    version_hdr = get_version_header(program_name, opt.survey_dir, release)

    from legacypipe.utils import add_bits
    from legacypipe.bits import DQ_BITS
    add_bits(version_hdr, DQ_BITS, 'DQMASK', 'DQ', 'D')
    from legacyzpts.psfzpt_cuts import CCD_CUT_BITS
    add_bits(version_hdr, CCD_CUT_BITS, 'CCD_CUTS', 'CC', 'C')
    for i, ap in enumerate(apertures_arcsec):
        version_hdr.add_record(
            dict(name='APRAD%i' % i,
                 value=ap,
                 comment='(optical) Aperture radius, in arcsec'))

    cat, forced = merge_forced(survey, opt.brick, cat)
    units = []
    for i, col in enumerate(forced.get_columns()):
        units.append(forced._header.get('TUNIT%i' % (i + 1), ''))
    cols = forced.get_columns()

    if opt.out:
        cat.writeto(opt.out, primheader=version_hdr)
        forced.writeto(opt.out, append=True, units=units, columns=cols)
    else:
        with survey.write_output('forced-brick', brick=opt.brick) as out:
            cat.writeto(None, fits_object=out.fits, primheader=version_hdr)
            forced.writeto(None,
                           fits_object=out.fits,
                           append=True,
                           units=units,
                           columns=cols)
예제 #47
0
def main():
    """Main program.
    """
    import argparse
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--force', action='store_true',
                      help='Run calib processes even if files already exist?')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')

    parser.add_argument('--expnum', type=int, help='Cut to a single exposure')
    parser.add_argument('--extname', '--ccdname', help='Cut to a single extension/CCD name')

    parser.add_argument('--no-psf', dest='psfex', action='store_false',
                      help='Do not compute PsfEx calibs')
    parser.add_argument('--no-sky', dest='sky', action='store_false',
                      help='Do not compute sky models')
    parser.add_argument('--run-se', action='store_true', help='Run SourceExtractor')

    parser.add_argument('--splinesky', action='store_true', help='Spline sky, not constant')
    parser.add_argument('--threads', type=int, help='Run multi-threaded', default=None)
    parser.add_argument('args',nargs=argparse.REMAINDER)
    opt = parser.parse_args()

    survey = LegacySurveyData()
    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        print('Read', len(T), 'from', opt.ccds)
    else:
        T = survey.get_ccds()
        #print len(T), 'CCDs'

    if len(opt.args) == 0:
        if opt.expnum is not None:
            T.cut(T.expnum == opt.expnum)
            print('Cut to', len(T), 'with expnum =', opt.expnum)
        if opt.extname is not None:
            T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname]))
            print('Cut to', len(T), 'with extname =', opt.extname)

        opt.args = range(len(T))

    args = []
    for a in opt.args:
        # Check for "expnum-ccdname" format.
        if '-' in str(a):
            words = a.split('-')
            assert(len(words) == 2)
            expnum = int(words[0])
            ccdname = words[1]
            I = np.flatnonzero((T.expnum == expnum) * (T.ccdname == ccdname))
            if len(I) != 1:
                print('Found', len(I), 'CCDs for expnum', expnum, 'CCDname', ccdname, ':', I)
            assert(len(I) == 1)
            t = T[I[0]]
        else:
            i = int(a)
            print('Index', i)
            t = T[i]

        print('CCDnmatch', t.ccdnmatch)
        if t.ccdnmatch < 20 and not opt.force:
            print('Skipping ccdnmatch = %i' % t.ccdnmatch)
            continue
            
        im = survey.get_image_object(t)
        print('Running', im.calname)
        
        kwargs = dict(psfex=opt.psfex, sky=opt.sky)
        if opt.force:
            kwargs.update(force=True)
        if opt.run_se:
            kwargs.update(se=True)
        if opt.splinesky:
            kwargs.update(splinesky=True)
            
        if opt.threads:
            args.append((im, kwargs))
        else:
            run_calibs((im, kwargs))

    if opt.threads:
        from astrometry.util.multiproc import multiproc
        mp = multiproc(opt.threads)
        mp.map(run_calibs, args)
        
    return 0
예제 #48
0
    def get_footprint_object(self):
        """Returns footprint object 'sfd'"""
        # work with SFD map and Decals/Mzls tiles
        # lonlat from SFD healpix is in galactic coords, convert this to Celestial
        hdu = fitsio.FITS(os.path.join(self.map_dir, 'lambda_sfd_ebv.fits'))
        sfd = EmptyClass()
        temp = hdu[1].read()
        sfd.temp = temp['TEMPERATURE']
        npix = Healpix().get_nside(len(sfd.temp))
        assert (npix == 512)
        sfd.l_indeg, sfd.b_indeg = hp.pix2ang(512,
                                              np.where(sfd.temp > 0)[0],
                                              nest=True,
                                              lonlat=True)
        #inPlane= np.where((sfd_gal_dec > -20) & (sfd_gal_dec < 20))[0]
        trans = Galactic(l=sfd.l_indeg * units.degree,
                         b=sfd.b_indeg * units.degree)
        radec = trans.transform_to(ICRS)
        sfd.ra, sfd.dec = radec.ra.value, radec.dec.value

        all_tiles = fits_table(
            os.path.join(self.tile_dir, 'mosaic-tiles_obstatus.fits'))
        wdes_tiles = fits_table(
            os.path.join(self.tile_dir, 'decam-tiles_obstatus.fits'))

        inDESI = ((all_tiles.in_desi_orig == 1) | (all_tiles.in_desi == 1))
        inDecals = ((inDESI) & (all_tiles.dec <= 30.))
        #(mzls_decals.in_des == 0))
        inMzls = ((inDESI) & (all_tiles.dec > 30.))
        #(mzls_decals.in_des == 0))
        inDes = ((wdes_tiles.in_desi_orig == 1) | (wdes_tiles.in_desi == 1))
        inDes = ((inDes) & (wdes_tiles.in_des == 1))
        #above30= mzls.dec > 30.
        #inDESI= ( (mzls.in_desi_orig == 1) |
        #          (mzls.in_desi == 1))
        #inMzls= ( (above30) &
        #          (inDESI))

        #desi= merge_tables([mzls,decals],columns='fillzero')
        des = wdes_tiles.copy()
        del wdes_tiles
        des.cut(inDes)
        mzls = all_tiles.copy()
        decals = all_tiles.copy()
        del all_tiles
        mzls.cut(inMzls)
        decals.cut(inDecals)

        ps = Healpix().get_pixscale(len(sfd.temp), unit='deg')
        # match_radec(ref,obs): for each point in ref, return matching point in obs
        print('matching tiles to healpix centers')
        I, J, d = match_radec(mzls.ra, mzls.dec, sfd.ra, sfd.dec, ps * 8)
        sfd.ipix_mzls = list(set(J))

        I, J, d = match_radec(decals.ra, decals.dec, sfd.ra, sfd.dec, ps * 8)
        sfd.ipix_decals = list(set(J))

        I, J, d = match_radec(des.ra, des.dec, sfd.ra, sfd.dec, ps * 8)
        sfd.ipix_des = list(set(J))

        return sfd

        # legasurvey pts fill in in ps*3
        I, J, d = match_radec(legsurvey.ra, legsurvey.dec, sfd.ra, sfd.dec,
                              ps * 3)
        sfd.ipix_legsurvey = set(J)
        # des fills in with ps*8
        I, J, d = match_radec(des.ra, des.dec, sfd.ra, sfd.dec, ps * 8)
        sfd.ipix_legsurvey.union(set(J))
        sfd.ipix_legsurvey = list(sfd.ipix_legsurvey)
        return sfd
예제 #49
0
                    action='store',
                    default=True)
parser.add_argument('--dowhat',
                    choices=[
                        'bricks_notdone', 'time_per_brick', 'nersc_time',
                        'sanity_tractors'
                    ],
                    action='store',
                    default=True)
parser.add_argument('--fn', action='store', default=False)
args = parser.parse_args()

if args.dowhat == 'bricks_notdone':
    if args.therun == 'obiwan':
        b = fits_table(
            os.path.join(os.environ['LEGACY_SURVEY_DIR'],
                         'survey-bricks-eboss-ngc.fits.gz'))
    elif args.therun == 'dr4':
        b = fits_table(
            os.path.join(os.environ['LEGACY_SURVEY_DIR'],
                         'survey-bricks-dr4.fits.gz'))
    don = np.loadtxt(args.fn, dtype=str)
    fout = args.fn.replace('_done.tmp', '_notdone.tmp')
    if os.path.exists(fout):
        os.remove(fout)
    # Bricks not finished
    with open(fout, 'w') as fil:
        for brick in list(set(b.brickname).difference(set(don))):
            fil.write('%s\n' % brick)
    print('Wrote %s' % fout)
    # All Bricks
예제 #50
0
 def from_fits(cls, filename, header, row=0):
     from astrometry.util.fits import fits_table
     T = fits_table(filename)
     T = T[row]
     return cls.from_fits_row(T)
예제 #51
0
        cmd += "%d)" % ids[-1]
    else:
        # Slower
        vals = ""
        for i in ids[:-1]:
            vals += "(%d)," % i
        vals += "(%d)" % ids[-1]
        cmd = (cmd + " FROM %s as db RIGHT JOIN (values %s) " %
               (db_randoms_table, vals) + "as v(id) on (db.id=v.id)")
    #print('cmd= %s' % cmd)
    db.cur.execute(cmd)
    # List of tuples [(id,reshift,...),(id,reshift,...)]
    a = db.cur.fetchall()
    # Tuple of lists (ids,reshifts,...)
    tup = zip(*a)
    #tup[ith_col])
    return {col: np.array(vals) for col, vals in zip(columns, tup)}
    #return np.array(sql_ids),np.array(sql_redshift)


if __name__ == '__main__':
    T = getSrcsInBrick('1765p247', 'elg', db_table='obiwan_elg_ra175')

    simcat = fits_table(
        "/global/cscratch1/sd/kaylanb/obiwan_out/elg_9deg2_ra175/elg/176/1765p247/rs0/obiwan/simcat-elg-1765p247.fits"
    )
    data_dict = all_psqlcols_for_ids(simcat.id,
                                     db_randoms_table='obiwan_elg_ra175')
    for i in range(10):
        print(data_dict['id'][i], data_dict['redshift'][i])
예제 #52
0
def main():
    print("Running with args %s" % sys.argv)
    import optparse

    parser = optparse.OptionParser()
    parser.add_option(
        "--server",
        dest="server",
        default=Client.default_url,
        help="Set server base URL (eg, %default)",
    )
    parser.add_option(
        "--apikey",
        "-k",
        dest="apikey",
        help="API key for Astrometry.net web service; if not given will check AN_API_KEY environment variable",
    )
    parser.add_option("--upload", "-u", dest="upload", help="Upload a file")
    parser.add_option(
        "--upload-xy", dest="upload_xy", help="Upload a FITS x,y table as JSON"
    )
    parser.add_option(
        "--wait",
        "-w",
        dest="wait",
        action="store_true",
        help="After submitting, monitor job status",
    )
    parser.add_option(
        "--wcs",
        dest="wcs",
        help="Download resulting wcs.fits file, saving to given filename; implies --wait if --urlupload or --upload",
    )
    parser.add_option(
        "--newfits",
        dest="newfits",
        help="Download resulting new-image.fits file, saving to given filename; implies --wait if --urlupload or --upload",
    )
    parser.add_option(
        "--kmz",
        dest="kmz",
        help="Download resulting kmz file, saving to given filename; implies --wait if --urlupload or --upload",
    )
    parser.add_option(
        "--annotate",
        "-a",
        dest="annotate",
        help="store information about annotations in give file, JSON format; implies --wait if --urlupload or --upload",
    )
    parser.add_option(
        "--urlupload", "-U", dest="upload_url", help="Upload a file at specified url"
    )
    parser.add_option(
        "--scale-units",
        dest="scale_units",
        choices=("arcsecperpix", "arcminwidth", "degwidth", "focalmm"),
        help="Units for scale estimate",
    )
    # parser.add_option('--scale-type', dest='scale_type',
    #                  choices=('ul', 'ev'), help='Scale bounds: lower/upper or estimate/error')
    parser.add_option(
        "--scale-lower", dest="scale_lower", type=float, help="Scale lower-bound"
    )
    parser.add_option(
        "--scale-upper", dest="scale_upper", type=float, help="Scale upper-bound"
    )
    parser.add_option(
        "--scale-est", dest="scale_est", type=float, help="Scale estimate"
    )
    parser.add_option(
        "--scale-err",
        dest="scale_err",
        type=float,
        help='Scale estimate error (in PERCENT), eg "10" if you estimate can be off by 10%',
    )
    parser.add_option("--ra", dest="center_ra", type=float, help="RA center")
    parser.add_option("--dec", dest="center_dec", type=float, help="Dec center")
    parser.add_option(
        "--radius", dest="radius", type=float, help="Search radius around RA,Dec center"
    )
    parser.add_option(
        "--downsample",
        dest="downsample_factor",
        type=int,
        help="Downsample image by this factor",
    )
    parser.add_option(
        "--positional_error",
        dest="positional_error",
        type=float,
        help="How many pixels a star may be from where it should be.",
    )
    parser.add_option(
        "--parity", dest="parity", choices=("0", "1"), help="Parity (flip) of image"
    )
    parser.add_option(
        "--tweak-order",
        dest="tweak_order",
        type=int,
        help="SIP distortion order (default: 2)",
    )
    parser.add_option(
        "--crpix-center",
        dest="crpix_center",
        action="store_true",
        default=None,
        help="Set reference point to center of image?",
    )
    parser.add_option(
        "--invert",
        action="store_true",
        default=None,
        help="Invert image before detecting sources -- for white-sky, black-stars images",
    )
    parser.add_option("--image-width", type=int, help="Set image width for x,y lists")
    parser.add_option("--image-height", type=int, help="Set image height for x,y lists")
    parser.add_option(
        "--sdss",
        dest="sdss_wcs",
        nargs=2,
        help="Plot SDSS image for the given WCS file; write plot to given PNG filename",
    )
    parser.add_option(
        "--galex",
        dest="galex_wcs",
        nargs=2,
        help="Plot GALEX image for the given WCS file; write plot to given PNG filename",
    )
    parser.add_option(
        "--jobid",
        "-i",
        dest="solved_id",
        type=int,
        help="retrieve result for jobId instead of submitting new image",
    )
    parser.add_option(
        "--substatus", "-s", dest="sub_id", help="Get status of a submission"
    )
    parser.add_option("--jobstatus", "-j", dest="job_id", help="Get status of a job")
    parser.add_option(
        "--jobs", "-J", dest="myjobs", action="store_true", help="Get all my jobs"
    )
    parser.add_option(
        "--jobsbyexacttag",
        "-T",
        dest="jobs_by_exact_tag",
        help="Get a list of jobs associated with a given tag--exact match",
    )
    parser.add_option(
        "--jobsbytag",
        "-t",
        dest="jobs_by_tag",
        help="Get a list of jobs associated with a given tag",
    )
    parser.add_option(
        "--private",
        "-p",
        dest="public",
        action="store_const",
        const="n",
        default="y",
        help="Hide this submission from other users",
    )
    parser.add_option(
        "--allow_mod_sa",
        "-m",
        dest="allow_mod",
        action="store_const",
        const="sa",
        default="d",
        help="Select license to allow derivative works of submission, but only if shared under same conditions of original license",
    )
    parser.add_option(
        "--no_mod",
        "-M",
        dest="allow_mod",
        action="store_const",
        const="n",
        default="d",
        help="Select license to disallow derivative works of submission",
    )
    parser.add_option(
        "--no_commercial",
        "-c",
        dest="allow_commercial",
        action="store_const",
        const="n",
        default="d",
        help="Select license to disallow commercial use of submission",
    )
    opt, args = parser.parse_args()

    if opt.apikey is None:
        # try the environment
        opt.apikey = os.environ.get("AN_API_KEY", None)
    if opt.apikey is None:
        parser.print_help()
        print()
        print("You must either specify --apikey or set AN_API_KEY")
        sys.exit(-1)

    args = {}
    args["apiurl"] = opt.server
    c = Client(**args)
    c.login(opt.apikey)

    if opt.upload or opt.upload_url or opt.upload_xy:
        if opt.wcs or opt.kmz or opt.newfits or opt.annotate:
            opt.wait = True

        kwargs = dict(
            allow_commercial_use=opt.allow_commercial,
            allow_modifications=opt.allow_mod,
            publicly_visible=opt.public,
        )
        if opt.scale_lower and opt.scale_upper:
            kwargs.update(
                scale_lower=opt.scale_lower,
                scale_upper=opt.scale_upper,
                scale_type="ul",
            )
        elif opt.scale_est and opt.scale_err:
            kwargs.update(
                scale_est=opt.scale_est, scale_err=opt.scale_err, scale_type="ev"
            )
        elif opt.scale_lower or opt.scale_upper:
            kwargs.update(scale_type="ul")
            if opt.scale_lower:
                kwargs.update(scale_lower=opt.scale_lower)
            if opt.scale_upper:
                kwargs.update(scale_upper=opt.scale_upper)

        for key in [
            "scale_units",
            "center_ra",
            "center_dec",
            "radius",
            "downsample_factor",
            "positional_error",
            "tweak_order",
            "crpix_center",
        ]:
            if getattr(opt, key) is not None:
                kwargs[key] = getattr(opt, key)
        if opt.parity is not None:
            kwargs.update(parity=int(opt.parity))

        if opt.upload:
            upres = c.upload(opt.upload, **kwargs)
        if opt.upload_xy:
            from astrometry.util.fits import fits_table

            T = fits_table(opt.upload_xy)
            kwargs.update(x=[float(x) for x in T.x], y=[float(y) for y in T.y])
            upres = c.upload(**kwargs)
        if opt.upload_url:
            upres = c.url_upload(opt.upload_url, **kwargs)

        stat = upres["status"]
        if stat != "success":
            print("Upload failed: status", stat)
            print(upres)
            sys.exit(-1)

        opt.sub_id = upres["subid"]

    if opt.wait:
        if opt.solved_id is None:
            if opt.sub_id is None:
                print("Can't --wait without a submission id or job id!")
                sys.exit(-1)

            while True:
                stat = c.sub_status(opt.sub_id, justdict=True)
                print("Got status:", stat)
                jobs = stat.get("jobs", [])
                if len(jobs):
                    for j in jobs:
                        if j is not None:
                            break
                    if j is not None:
                        print("Selecting job id", j)
                        opt.solved_id = j
                        break
                time.sleep(5)

        while True:
            stat = c.job_status(opt.solved_id, justdict=True)
            print("Got job status:", stat)
            if stat.get("status", "") in ["success"]:
                success = stat["status"] == "success"
                break
            time.sleep(5)

    if opt.solved_id:
        # we have a jobId for retrieving results
        retrieveurls = []
        if opt.wcs:
            # We don't need the API for this, just construct URL
            url = opt.server.replace("/api/", "/wcs_file/%i" % opt.solved_id)
            retrieveurls.append((url, opt.wcs))
        if opt.kmz:
            url = opt.server.replace("/api/", "/kml_file/%i/" % opt.solved_id)
            retrieveurls.append((url, opt.kmz))
        if opt.newfits:
            url = opt.server.replace("/api/", "/new_fits_file/%i/" % opt.solved_id)
            retrieveurls.append((url, opt.newfits))

        for url, fn in retrieveurls:
            print("Retrieving file from", url, "to", fn)
            f = urlopen(url)
            txt = f.read()
            w = open(fn, "wb")
            w.write(txt)
            w.close()
            print("Wrote to", fn)

        if opt.annotate:
            result = c.annotate_data(opt.solved_id)
            with open(opt.annotate, "w") as f:
                f.write(python2json(result))

    if opt.wait:
        # behaviour as in old implementation
        opt.sub_id = None

    if opt.sdss_wcs:
        (wcsfn, outfn) = opt.sdss_wcs
        c.sdss_plot(outfn, wcsfn)
    if opt.galex_wcs:
        (wcsfn, outfn) = opt.galex_wcs
        c.galex_plot(outfn, wcsfn)

    if opt.sub_id:
        print(c.sub_status(opt.sub_id))
    if opt.job_id:
        print(c.job_status(opt.job_id))

    if opt.jobs_by_tag:
        tag = opt.jobs_by_tag
        print(c.jobs_by_tag(tag, None))
    if opt.jobs_by_exact_tag:
        tag = opt.jobs_by_exact_tag
        print(c.jobs_by_tag(tag, "yes"))

    if opt.myjobs:
        jobs = c.myjobs()
        print(jobs)
예제 #53
0
                      help='Search radius, in pixels (default 50)')
    parser.set_defaults(xcol='X',
                        ycol='Y',
                        nimage=0,
                        cells=0,
                        cellsize=0,
                        rxcol='X',
                        rycol='Y',
                        nref=0)
    opt, args = parser.parse_args()

    if len(args) != 3:
        parser.print_help()
        sys.exit(-1)

    imxy = fits_table(args[0])
    refxy = fits_table(args[1])
    outfn = args[2]

    kwargs = {}
    if opt.cells:
        kwargs['ncells'] = opt.cells
    if opt.cellsize:
        kwargs['dcell'] = opt.cellsize

    ix = imxy.getcolumn(opt.xcol)
    iy = imxy.getcolumn(opt.ycol)
    ixy = vstack((ix, iy)).T
    if opt.nimage:
        ixy = ixy[:opt.nimage, :]
예제 #54
0
def galex_coadds(onegal,
                 galaxy=None,
                 radius_mosaic=30,
                 radius_mask=None,
                 pixscale=1.5,
                 ref_pixscale=0.262,
                 output_dir=None,
                 galex_dir=None,
                 log=None,
                 centrals=True,
                 verbose=False):
    '''Generate custom GALEX cutouts.
    
    radius_mosaic and radius_mask in arcsec
    
    pixscale: GALEX pixel scale in arcsec/pixel.

    '''
    import fitsio
    import matplotlib.pyplot as plt

    from astrometry.libkd.spherematch import match_radec
    from astrometry.util.resample import resample_with_wcs, OverlapError
    from tractor import (Tractor, NanoMaggies, Image, LinearPhotoCal,
                         NCircularGaussianPSF, ConstantFitsWcs, ConstantSky)

    from legacypipe.survey import imsave_jpeg
    from legacypipe.catalog import read_fits_catalog

    if galaxy is None:
        galaxy = 'galaxy'

    if galex_dir is None:
        galex_dir = os.environ.get('GALEX_DIR')

    if output_dir is None:
        output_dir = '.'

    if radius_mask is None:
        radius_mask = radius_mosaic
        radius_search = 5.0  # [arcsec]
    else:
        radius_search = radius_mask

    W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int')  # [pixels]
    targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0,
                    -pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W),
                    float(H))

    # Read the custom Tractor catalog
    tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy))
    if not os.path.isfile(tractorfile):
        print('Missing Tractor catalog {}'.format(tractorfile))
        return 0

    cat = fits_table(tractorfile)
    print('Read {} sources from {}'.format(len(cat), tractorfile),
          flush=True,
          file=log)

    keep = np.ones(len(cat)).astype(bool)
    if centrals:
        # Find the large central galaxy and mask out (ignore) all the models
        # which are within its elliptical mask.

        # This algorithm will have to change for mosaics not centered on large
        # galaxies, e.g., in galaxy groups.
        m1, m2, d12 = match_radec(cat.ra,
                                  cat.dec,
                                  onegal['RA'],
                                  onegal['DEC'],
                                  radius_search / 3600.0,
                                  nearest=False)
        if len(m1) == 0:
            print('No central galaxies found at the central coordinates!',
                  flush=True,
                  file=log)
        else:
            pixfactor = ref_pixscale / pixscale  # shift the optical Tractor positions
            for mm in m1:
                morphtype = cat.type[mm].strip()
                if morphtype == 'EXP' or morphtype == 'COMP':
                    e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[
                        mm], cat.shapeexp_r[mm]  # [arcsec]
                elif morphtype == 'DEV' or morphtype == 'COMP':
                    e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[
                        mm], cat.shapedev_r[mm]  # [arcsec]
                else:
                    r50 = None

                if r50:
                    majoraxis = r50 * 5 / pixscale  # [pixels]
                    ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2)
                    these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis,
                                                  ba * majoraxis,
                                                  np.radians(phi),
                                                  cat.bx * pixfactor,
                                                  cat.by * pixfactor)
                    if np.sum(these) > 0:
                        #keep[these] = False
                        pass
                print('Hack!')
                keep[mm] = False

            #srcs = read_fits_catalog(cat)
            #_srcs = np.array(srcs)[~keep].tolist()
            #mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0)
            #import matplotlib.pyplot as plt
            ##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png')
            #plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png')
            #pdb.set_trace()

    srcs = read_fits_catalog(cat)
    for src in srcs:
        src.freezeAllBut('brightness')
    #srcs_nocentral = np.array(srcs)[keep].tolist()

    # Find all overlapping GALEX tiles and then read the tims.
    galex_tiles = _read_galex_tiles(targetwcs,
                                    galex_dir,
                                    log=log,
                                    verbose=verbose)

    gbands = ['n', 'f']
    nicegbands = ['NUV', 'FUV']

    zps = dict(n=20.08, f=18.82)

    coimgs, comods, coresids, coimgs_central, comods_nocentral = [], [], [], [], []
    for niceband, band in zip(nicegbands, gbands):
        J = np.flatnonzero(galex_tiles.get('has_' + band))
        print(len(J), 'GALEX tiles have coverage in band', band)

        coimg = np.zeros((H, W), np.float32)
        comod = np.zeros((H, W), np.float32)
        cowt = np.zeros((H, W), np.float32)

        comod_nocentral = np.zeros((H, W), np.float32)

        for src in srcs:
            src.setBrightness(NanoMaggies(**{band: 1}))

        for j in J:
            brick = galex_tiles[j]
            fn = os.path.join(
                galex_dir, brick.tilename.strip(),
                '%s-%sd-intbgsub.fits.gz' % (brick.brickname, band))
            #print(fn)

            gwcs = Tan(*[
                float(f) for f in [
                    brick.crval1, brick.crval2, brick.crpix1, brick.crpix2,
                    brick.cdelt1, 0., 0., brick.cdelt2, 3840., 3840.
                ]
            ])
            img = fitsio.read(fn)
            #print('Read', img.shape)

            try:
                Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, gwcs, [], 3)
            except OverlapError:
                continue

            K = np.flatnonzero(img[Yi, Xi] != 0.)
            if len(K) == 0:
                continue
            Yo, Xo, Yi, Xi = Yo[K], Xo[K], Yi[K], Xi[K]

            wt = brick.get(band + 'exptime')
            coimg[Yo, Xo] += wt * img[Yi, Xi]
            cowt[Yo, Xo] += wt

            x0, x1, y0, y1 = min(Xi), max(Xi), min(Yi), max(Yi)
            subwcs = gwcs.get_subimage(x0, y0, x1 - x0 + 1, y1 - y0 + 1)
            twcs = ConstantFitsWcs(subwcs)
            timg = img[y0:y1 + 1, x0:x1 + 1]

            tie = np.ones_like(timg)  ## HACK!
            #hdr = fitsio.read_header(fn)
            #zp = hdr['']
            zp = zps[band]
            photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp),
                                      band=band)
            tsky = ConstantSky(0.0)

            # HACK -- circular Gaussian PSF of fixed size...
            # in arcsec
            #fwhms = dict(NUV=6.0, FUV=6.0)
            # -> sigma in pixels
            #sig = fwhms[band] / 2.35 / twcs.pixel_scale()
            sig = 6.0 / np.sqrt(8 * np.log(2)) / twcs.pixel_scale()
            tpsf = NCircularGaussianPSF([sig], [1.])

            tim = Image(data=timg,
                        inverr=tie,
                        psf=tpsf,
                        wcs=twcs,
                        sky=tsky,
                        photocal=photocal,
                        name='GALEX ' + band + brick.brickname)

            ## Build the model image with and without the central galaxy model.
            tractor = Tractor([tim], srcs)
            mod = tractor.getModelImage(0)
            tractor.freezeParam('images')
            tractor.optimize_forced_photometry(priors=False,
                                               shared_params=False)
            mod = tractor.getModelImage(0)

            srcs_nocentral = np.array(srcs)[keep].tolist()
            #srcs_nocentral = np.array(srcs)[nocentral].tolist()
            tractor_nocentral = Tractor([tim], srcs_nocentral)
            mod_nocentral = tractor_nocentral.getModelImage(0)

            comod[Yo, Xo] += wt * mod[Yi - y0, Xi - x0]
            comod_nocentral[Yo, Xo] += wt * mod_nocentral[Yi - y0, Xi - x0]

        coimg /= np.maximum(cowt, 1e-18)
        comod /= np.maximum(cowt, 1e-18)
        comod_nocentral /= np.maximum(cowt, 1e-18)

        coresid = coimg - comod

        # Subtract the model image which excludes the central (comod_nocentral)
        # from the data (coimg) to isolate the light of the central
        # (coimg_central).
        coimg_central = coimg - comod_nocentral

        coimgs.append(coimg)
        comods.append(comod)
        coresids.append(coresid)

        comods_nocentral.append(comod_nocentral)
        coimgs_central.append(coimg_central)

        # Write out the final images with and without the central, making sure
        # to apply the zeropoint to go from counts/s to AB nanomaggies.
        # https://asd.gsfc.nasa.gov/archive/galex/FAQ/counts_background.html
        for thisimg, imtype in zip((coimg, comod, comod_nocentral),
                                   ('image', 'model', 'model-nocentral')):
            fitsfile = os.path.join(
                output_dir, '{}-{}-{}.fits'.format(galaxy, imtype, niceband))
            if verbose:
                print('Writing {}'.format(fitsfile))
            fitsio.write(fitsfile,
                         thisimg * 10**(-0.4 * (zp - 22.5)),
                         clobber=True)

    # Build a color mosaic (but note that the images here are in units of
    # background-subtracted counts/s).

    #_galex_rgb = _galex_rgb_moustakas
    #_galex_rgb = _galex_rgb_dstn
    _galex_rgb = _galex_rgb_official

    for imgs, imtype in zip(
        (coimgs, comods, coresids, comods_nocentral, coimgs_central),
        ('image', 'model', 'resid', 'model-nocentral', 'image-central')):
        rgb = _galex_rgb(imgs)
        jpgfile = os.path.join(output_dir,
                               '{}-{}-FUVNUV.jpg'.format(galaxy, imtype))
        if verbose:
            print('Writing {}'.format(jpgfile))
        imsave_jpeg(jpgfile, rgb, origin='lower')

    return 1
예제 #55
0
    #summary_plots(summaryfn, ps)
    #sys.exit(0)

    #fns = glob('/project/projectdirs/cosmo/work/legacysurvey/dr3/coadd/*/*/*-depth.fits')
    fns = glob(
        '/project/projectdirs/cosmo/data/legacysurvey/dr3/coadd/*/*/*-depth.fits'
    )
    fns.sort()
    print(len(fns), 'depth files')

    fn = fns.pop(0)
    print('Reading', fn)
    # We'll keep all files for merging...
    TT = []

    T = fits_table(fn)
    # Create / upgrade the count columns to int64.
    for band in 'grz':
        for pro in ['ptsrc', 'gal']:
            col = 'counts_%s_%s' % (pro, band)
            if not col in T.columns():
                v = np.zeros(len(T), np.int64)
            else:
                v = T.get(col).astype(np.int64)
            T.set(col, v)
    T.brickname = np.array([brickname_from_filename(fn)] * len(T))
    TT.append(T.copy())

    for ifn, fn in enumerate(fns):
        print('Reading', ifn, 'of', len(fns), ':', fn)
        t = fits_table(fn)
예제 #56
0
def run_sed_matched_filters(SEDs,
                            bands,
                            detmaps,
                            detivs,
                            omit_xy,
                            targetwcs,
                            nsigma=5,
                            saturated_pix=None,
                            plots=False,
                            ps=None,
                            mp=None):
    '''
    Runs a given set of SED-matched filters.

    Parameters
    ----------
    SEDs : list of (name, sed) tuples
        The SEDs to run.  The `sed` values are lists the same length
        as `bands`.
    bands : list of string
        The band names of `detmaps` and `detivs`.
    detmaps : numpy array, float
        Detection maps for each of the listed `bands`.
    detivs : numpy array, float
        Inverse-variances of the `detmaps`.
    omit_xy : None, or (xx,yy) tuple
        Existing sources to avoid.
    targetwcs : WCS object
        WCS object to use to convert pixel values into RA,Decs for the
        returned Tractor PointSource objects.
    nsigma : float, optional
        Detection threshold
    saturated_pix : None or numpy array, boolean
        Passed through to sed_matched_detection.
        A map of pixels that are always considered "hot" when
        determining whether a new source touches hot pixels of an
        existing source.
    plots : boolean, optional
        Create plots?
    ps : PlotSequence object
        Create plots?
    mp : multiproc object
        Multiprocessing
        
    Returns
    -------
    Tnew : fits_table
        Table of new sources detected
    newcat : list of PointSource objects
        Newly detected objects, with positions and fluxes, as Tractor
        PointSource objects.
    hot : numpy array of bool
        "Hot pixels" containing sources.
        
    See also
    --------
    sed_matched_detection : run a single SED-matched filter.
    
    '''
    if omit_xy is not None:
        xx, yy = omit_xy
        n0 = len(xx)
    else:
        xx, yy = [], []
        n0 = 0

    H, W = detmaps[0].shape
    hot = np.zeros((H, W), bool)

    peaksn = []
    apsn = []

    for sedname, sed in SEDs:
        print('SED', sedname)
        if plots:
            pps = ps
        else:
            pps = None
        t0 = Time()
        sedhot, px, py, peakval, apval = sed_matched_detection(
            sedname,
            sed,
            detmaps,
            detivs,
            bands,
            xx,
            yy,
            nsigma=nsigma,
            saturated_pix=saturated_pix,
            ps=pps)
        print('SED took', Time() - t0)
        if sedhot is None:
            continue
        print(len(px), 'new peaks')
        hot |= sedhot
        # With an empty xx, np.append turns it into a double!
        xx = np.append(xx, px).astype(int)
        yy = np.append(yy, py).astype(int)

        peaksn.extend(peakval)
        apsn.extend(apval)

    # New peaks:
    peakx = xx[n0:]
    peaky = yy[n0:]

    if len(peakx) == 0:
        return None, None, None

    # Add sources for the new peaks we found
    pr, pd = targetwcs.pixelxy2radec(peakx + 1, peaky + 1)
    print('Adding', len(pr), 'new sources')
    # Also create FITS table for new sources
    Tnew = fits_table()
    Tnew.ra = pr
    Tnew.dec = pd
    Tnew.tx = peakx
    Tnew.ty = peaky
    assert (len(peaksn) == len(Tnew))
    assert (len(apsn) == len(Tnew))
    Tnew.peaksn = np.array(peaksn)
    Tnew.apsn = np.array(apsn)

    Tnew.itx = np.clip(np.round(Tnew.tx), 0, W - 1).astype(int)
    Tnew.ity = np.clip(np.round(Tnew.ty), 0, H - 1).astype(int)
    newcat = []
    for i, (r, d, x, y) in enumerate(zip(pr, pd, peakx, peaky)):
        fluxes = dict([(band, detmap[Tnew.ity[i], Tnew.itx[i]])
                       for band, detmap in zip(bands, detmaps)])
        newcat.append(
            PointSource(RaDecPos(r, d), NanoMaggies(order=bands, **fluxes)))

    return Tnew, newcat, hot
예제 #57
0
 def __init__(self, targz_dir, decals=True):
     self.bricks = fits_table(
         os.path.join(targz_dir, 'legacysurveydir',
                      'survey-bricks.fits.gz'))
     if decals:
         self.bricks.cut((self.bricks.dec > -30) & (self.bricks.dec < 30))
예제 #58
0
def orig_code(data, nmatch):
    nside = Healpix().get_nside(len(data))

    _, lo, hi = sigmaclip(data[data != 0], low=3, high=3)
    flag = np.logical_or(data < lo, data > hi)
    flag *= (nmatch > 20)
    ra, dec = hp.pix2ang(nside, np.where(flag)[0], lonlat=True)

    # PLOTTING
    ralim = [ra.min(), ra.max()]
    declim = [dec.min(), dec.max()]
    my_mollzoom(ra,
                dec,
                data[flag],
                'outliers',
                ralim=ralim,
                declim=declim,
                vlim=(lo, hi))
    temp_ra, temp_dec = hp.pix2ang(nside,
                                   np.where(np.ones(len(data), bool))[0],
                                   lonlat=True)
    keep= (temp_ra >= ralim[0])*\
          (temp_ra <= ralim[1])*\
          (temp_dec >= declim[0])*\
          (temp_dec <= declim[1])
    my_mollzoom(temp_ra[keep],
                temp_dec[keep],
                data[keep],
                'all',
                ralim=ralim,
                declim=declim,
                vlim=(lo, hi))
    keep *= (nmatch > 20)
    my_mollzoom(temp_ra[keep],
                temp_dec[keep],
                data[keep],
                'nmatch_gt20',
                ralim=ralim,
                declim=declim,
                vlim=(lo, hi))

    # Match bricks
    #heal= fits_table()
    #for col,arr in zip(['ra','dec'],[ra,dec]):
    #    heal.set(col, arr)
    brick = fits_table(
        os.path.join(args.targz_dir, 'legacysurveydir',
                     'survey-bricks.fits.gz'))
    brick.cut( (brick.dec > -40)*\
               (brick.dec < 40))
    #deg_per_healpix= get_pixscale(npix,unit='deg')
    deg_per_brick = 0.25
    #imatch,imiss,d2d= Matcher().match_within(heal,brick, dist= deg_per_brick/2)
    I, J, d = match_radec(ra,
                          dec,
                          brick.ra,
                          brick.dec,
                          deg_per_brick / 2,
                          nearest=True)
    #raise ValueError
    brick.cut(imatch['obs'])
    my_scatter(brick.ra, brick.dec, 'bricks', ralim=ralim, declim=declim)

    id = fn.replace('/', '').replace('.fits', '')
    savenm = os.path.join(args.outdir, 'brick_table_%s.fits' % id)
    brick.writeto(savenm)
    print('Wrote %s' % savenm)
 def get_healpix_catalog(self, healpix):
     #return super().get_healpix_catalog(healpix)
     from astrometry.util.fits import fits_table
     fname = self.fnpattern % dict(hp=healpix)
     #print('Reading', fname)
     return fits_table(fname, columns=self.columns)
예제 #60
0
def merge_forced(survey, brickname, cat, bands='grz'):
    ccdfn = survey.find_file('ccds-table', brick=brickname)
    CCDs = fits_table(ccdfn)
    print('Read', len(CCDs), 'CCDs')

    # objects in the catalog: (release,brickid,objid)
    catobjs = set([(r, b, o)
                   for r, b, o in zip(cat.release, cat.brickid, cat.objid)])

    # (release, brickid, objid, band) -> [ index in forced-phot table ]
    phot_index = {}

    camexp = set()
    FF = []
    for ccd in CCDs:
        cam = ccd.camera.strip()
        key = (cam, ccd.expnum)
        if key in camexp:
            # already read this camera-expnum
            continue
        camexp.add(key)
        ffn = survey.find_file('forced', camera=cam, expnum=ccd.expnum)
        print('Forced phot filename:', ffn)
        F = fits_table(ffn)
        print('Read', len(F), 'forced-phot entries for CCD')
        ikeep = []
        for i, (r, b, o) in enumerate(zip(F.release, F.brickid, F.objid)):
            if (r, b, o) in catobjs:
                ikeep.append(i)
        if len(ikeep) == 0:
            print('No catalog objects found in this forced-phot table.')
            continue
        F.cut(np.array(ikeep))
        print('Cut to', len(F), 'phot entries matching catalog')
        FF.append(F)
    F = merge_tables(FF)
    F._header = FF[0]._header
    del FF

    I = np.lexsort(
        (F.expnum, F.camera, F.filter, F.objid, F.brickid, F.release))
    F.cut(I)

    for i, (r, brick, o,
            band) in enumerate(zip(F.release, F.brickid, F.objid, F.filter)):
        key = (r, brick, o, band)
        if not key in phot_index:
            phot_index[key] = []
        phot_index[key].append(i)

    # find largest number of photometry measurements!
    Nmax = max([len(inds) for inds in phot_index.values()])
    print('Maximum number of photometry entries:', Nmax)

    for band in bands:
        nobs = np.zeros(len(cat), np.int32)
        indx = np.empty(len(cat), np.int32)
        indx[:] = -1
        cat.set('nobs_%s' % band, nobs)
        cat.set('index_%s' % band, indx)

        for i, (r, brick,
                o) in enumerate(zip(cat.release, cat.brickid, cat.objid)):
            key = (r, brick, o, band)
            try:
                inds = phot_index[key]
            except KeyError:
                continue
            nobs[i] = len(inds)
            # Indices are all contiguous (so we only need store the first one)
            assert (np.all(np.array(inds) == (np.arange(len(inds)) + inds[0])))
            indx[i] = inds[0]
    return cat, F