def getbrickfiles(brickname=None):

    survey = LegacySurveyData()
    brickinfo = survey.get_brick_by_name(brickname)
    brickwcs = wcs_for_brick(brickinfo)
    ccdinfo = survey.ccds_touching_wcs(brickwcs)
    nccd = len(ccdinfo)

    calibdir = survey.get_calib_dir()
    imagedir = survey.survey_dir

    # Construct image file names and the calibration file names.
    expnum = ccdinfo.expnum
    ccdname = ccdinfo.ccdname

    psffiles = list()
    skyfiles = list()
    imagefiles = list()
    for ccd in ccdinfo:
        info = survey.get_image_object(ccd)
        for attr in ['imgfn', 'dqfn', 'wtfn']:
            fn = getattr(info, attr).replace(imagedir+'/', '')
            #if '160108_073601' in fn:
            #    pdb.set_trace()
            imagefiles.append(fn)
        psffiles.append(info.psffn.replace(calibdir, 'calib'))
        skyfiles.append(info.splineskyfn.replace(calibdir, 'calib'))
        
    #for ii in range(nccd):
        #exp = '{0:08d}'.format(expnum[ii])
        #rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits')
        #psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile))
        #skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile))
        #imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii]))))

    #print(np.array(imagefiles))
    #print(np.array(psffiles))
    #print(np.array(skyfiles))
    return imagefiles, psffiles, skyfiles
def getbrickfiles(brickname=None):

    survey = LegacySurveyData()
    brickinfo = survey.get_brick_by_name(brickname)
    brickwcs = wcs_for_brick(brickinfo)
    ccdinfo = survey.ccds_touching_wcs(brickwcs)
    nccd = len(ccdinfo)

    calibdir = survey.get_calib_dir()
    imagedir = survey.survey_dir

    # Construct image file names and the calibration file names.
    expnum = ccdinfo.expnum
    ccdname = ccdinfo.ccdname

    psffiles = list()
    skyfiles = list()
    imagefiles = list()
    for ccd in ccdinfo:
        info = survey.get_image_object(ccd)
        for attr in ['imgfn', 'dqfn', 'wtfn']:
            fn = getattr(info, attr).replace(imagedir + '/', '')
            #if '160108_073601' in fn:
            #    pdb.set_trace()
            imagefiles.append(fn)
        psffiles.append(info.psffn.replace(calibdir, 'calib'))
        skyfiles.append(info.splineskyfn.replace(calibdir, 'calib'))

    #for ii in range(nccd):
    #exp = '{0:08d}'.format(expnum[ii])
    #rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits')
    #psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile))
    #skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile))
    #imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii]))))

    #print(np.array(imagefiles))
    #print(np.array(psffiles))
    #print(np.array(skyfiles))
    return imagefiles, psffiles, skyfiles
Example #3
0
#ra,dec = 357.3060, 2.3957
#ccd1 = ccds[(ccds.expnum == 563212) * (ccds.ccdname == 'N17')]
ra, dec = 124.0317, 1.3028
expnum, ccdname = 393203, 'N11'

survey = LegacySurveyData()

W, H = 200, 200
pixscale = 0.262
cd = pixscale / 3600.
targetwcs = Tan(ra, dec, W / 2., H / 2., -cd, 0., 0., cd, float(W), float(H))

rr, dd = targetwcs.pixelxy2radec([1, W, W, 1, 1], [1, 1, H, H, 1])
targetrd = np.vstack((rr, dd)).T

ccds = survey.ccds_touching_wcs(targetwcs)
print(len(ccds), 'CCDs touching WCS')

print('MJDs', ccds.mjd_obs)
ccds.writeto('test-ccds.fits')

ccd1 = ccds[(ccds.expnum == expnum) * (ccds.ccdname == ccdname)]
print('CCD1:', ccd1)
im1 = survey.get_image_object(ccd1[0])
print('Im:', im1)

#wcs = im1.get_wcs()
#x0,x1,y0,y1,slc = im1.get_image_extent(wcs=wcs, radecpoly=targetrd)

tim1 = im1.get_tractor_image(radecpoly=targetrd,
                             pixPsf=True,
Example #4
0
def run_one_brick(X):
    brick, ibrick, nbricks, plots, kwargs = X

    survey = LegacySurveyData()

    print()
    print()
    print('Brick', (ibrick + 1), 'of', nbricks, ':', brick.brickname)

    dirnm = os.path.join('depthcuts', brick.brickname[:3])
    outfn = os.path.join(dirnm, 'ccds-%s.fits' % brick.brickname)
    if os.path.exists(outfn):
        print('Exists:', outfn)
        return 0

    H, W = 3600, 3600
    pixscale = 0.262
    bands = ['g', 'r', 'z']

    # Get WCS object describing brick
    targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale)
    targetrd = np.array([
        targetwcs.pixelxy2radec(x, y)
        for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
    ])
    gitver = get_git_version()

    ccds = survey.ccds_touching_wcs(targetwcs)
    if ccds is None:
        print('No CCDs actually touching brick')
        return 0
    print(len(ccds), 'CCDs actually touching brick')

    ccds.cut(np.in1d(ccds.filter, bands))
    print('Cut on filter:', len(ccds), 'CCDs remain.')

    if 'ccd_cuts' in ccds.get_columns():
        norig = len(ccds)
        ccds.cut(ccds.ccd_cuts == 0)
        print(len(ccds), 'of', norig, 'CCDs pass cuts')
    else:
        print('No CCD cuts')

    if len(ccds) == 0:
        print('No CCDs left')
        return 0

    ps = None
    if plots:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence('depth-%s' % brick.brickname)

    splinesky = True
    gaussPsf = False
    pixPsf = True
    do_calibs = False
    normalizePsf = True

    get_depth_maps = kwargs.pop('get_depth_maps', False)

    try:
        D = make_depth_cut(survey,
                           ccds,
                           bands,
                           targetrd,
                           brick,
                           W,
                           H,
                           pixscale,
                           plots,
                           ps,
                           splinesky,
                           gaussPsf,
                           pixPsf,
                           normalizePsf,
                           do_calibs,
                           gitver,
                           targetwcs,
                           get_depth_maps=get_depth_maps,
                           **kwargs)
        if get_depth_maps:
            keep, overlapping, depthmaps = D
        else:
            keep, overlapping = D
    except:
        print('Failed to make_depth_cut():')
        import traceback
        traceback.print_exc()
        return -1

    print(np.sum(overlapping), 'CCDs overlap the brick')
    print(np.sum(keep), 'CCDs passed depth cut')
    ccds.overlapping = overlapping
    ccds.passed_depth_cut = keep

    if not os.path.exists(dirnm):
        try:
            os.makedirs(dirnm)
        except:
            pass

    if get_depth_maps:
        for band, depthmap in depthmaps:
            doutfn = os.path.join(dirnm,
                                  'depth-%s-%s.fits' % (brick.brickname, band))
            hdr = fitsio.FITSHDR()
            # Plug the WCS header cards into these images
            targetwcs.add_to_header(hdr)
            hdr.delete('IMAGEW')
            hdr.delete('IMAGEH')
            hdr.add_record(dict(name='EQUINOX', value=2000.))
            hdr.add_record(dict(name='FILTER', value=band))
            fitsio.write(doutfn, depthmap, header=hdr)
            print('Wrote', doutfn)

    tmpfn = os.path.join(os.path.dirname(outfn),
                         'tmp-' + os.path.basename(outfn))
    ccds.writeto(tmpfn)
    os.rename(tmpfn, outfn)
    print('Wrote', outfn)

    return 0
def main():
    import argparse
    parser = argparse.ArgumentParser()

    parser.add_argument(
        '--dr',
        '--drdir',
        dest='drdir',
        default='/project/projectdirs/cosmo/data/legacysurvey/dr5',
        help='Directory containing data release w/ tar-gzipped calibs')
    parser.add_argument(
        '-b',
        '--brick',
        help='Brick name to run; required unless --radec is given')
    parser.add_argument(
        '--radec',
        nargs=2,
        help='RA,Dec center for a custom location (not a brick)')
    parser.add_argument('--pixscale',
                        type=float,
                        default=0.262,
                        help='Pixel scale of the output coadds (arcsec/pixel)')
    parser.add_argument('-W',
                        '--width',
                        type=int,
                        default=3600,
                        help='Target image width, default %(default)i')
    parser.add_argument('-H',
                        '--height',
                        type=int,
                        default=3600,
                        help='Target image height, default %(default)i')
    parser.add_argument(
        '--zoom',
        type=int,
        nargs=4,
        help='Set target image extent (default "0 3600 0 3600")')
    parser.add_argument('--no-psf',
                        dest='do_psf',
                        default=True,
                        action='store_false',
                        help='Do not extract PsfEx files')
    parser.add_argument('--no-sky',
                        dest='do_sky',
                        default=True,
                        action='store_false',
                        help='Do not extract SplineSky files')

    opt = parser.parse_args()
    if opt.brick is None and opt.radec is None:
        parser.print_help()
        return -1
    optdict = vars(opt)

    drdir = opt.drdir
    W = opt.width
    H = opt.height
    pixscale = opt.pixscale
    target_extent = opt.zoom

    do_psf = opt.do_psf
    do_sky = opt.do_sky

    #brickname = '1501p020'

    custom = (opt.radec is not None)
    #ra,dec = 216.03, 34.86
    if custom:
        ra, dec = opt.radec  #27.30, -10.43
        ra = float(ra)
        dec = float(dec)
        #W,H = 1000,1000
        #W,H = 1500,1500
        brickname = 'custom_%.3f_%.3f' % (ra, dec)
        #do_sky = False

    survey = LegacySurveyData()

    if custom:
        from legacypipe.survey import BrickDuck
        # Custom brick; create a fake 'brick' object
        brick = BrickDuck(ra, dec, brickname)
    else:
        brickname = opt.brick
        brick = survey.get_brick_by_name(brickname)

    # Get WCS object describing brick
    targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale)
    if target_extent is not None:
        (x0, x1, y0, y1) = target_extent
        W = x1 - x0
        H = y1 - y0
        targetwcs = targetwcs.get_subimage(x0, y0, W, H)

    # Find CCDs
    ccds = survey.ccds_touching_wcs(targetwcs, ccdrad=None)
    if ccds is None:
        raise NothingToDoError('No CCDs touching brick')
    print(len(ccds), 'CCDs touching target WCS')

    for ccd in ccds:
        im = survey.get_image_object(ccd)
        print('CCD', im)

        expnum = '%08i' % im.expnum

        if do_psf:
            if os.path.exists(im.psffn) or os.path.exists(im.merged_psffn):
                print('PSF file exists')
            else:
                print('Need PSF', im.psffn, im.merged_psffn)

                tarfn = os.path.join(
                    drdir, 'calib', im.camera, 'psfex-merged',
                    'legacysurvey_dr5_calib_decam_psfex-merged_%s.tar.gz' %
                    expnum[:5])
                print(tarfn)
                if os.path.exists(tarfn):
                    outfn = '%s/%s-%s.fits' % (expnum[:5], im.camera, expnum)
                    cmd = 'cd %s/%s/psfex-merged && tar xvzf %s %s' % (
                        survey.get_calib_dir(), im.camera, tarfn, outfn)
                    print(cmd)
                    os.system(cmd)

        if do_sky:
            if os.path.exists(im.splineskyfn) or os.path.exists(
                    im.merged_splineskyfn):
                print('Sky file exists')
            else:
                print('Need sky', im.splineskyfn, im.merged_splineskyfn)

                tarfn = os.path.join(
                    drdir, 'calib', im.camera, 'splinesky-merged',
                    'legacysurvey_dr5_calib_decam_splinesky-merged_%s.tar.gz' %
                    expnum[:5])
                print(tarfn)
                if os.path.exists(tarfn):
                    outfn = '%s/%s-%s.fits' % (expnum[:5], im.camera, expnum)
                    cmd = 'cd %s/%s/splinesky-merged && tar xvzf %s %s' % (
                        survey.get_calib_dir(), im.camera, tarfn, outfn)
                    print(cmd)
                    os.system(cmd)
Example #6
0
def one_brick(X):
    (ibrick, brick) = X
    bands = ['g', 'r', 'z']

    print('Brick', brick.brickname)
    wcs = wcs_for_brick(brick, W=94, H=94, pixscale=10.)
    BH, BW = wcs.shape
    targetrd = np.array([
        wcs.pixelxy2radec(x, y)
        for x, y in [(1, 1), (BW, 1), (BW, BH), (1, BH), (1, 1)]
    ])
    survey = LegacySurveyData()
    C = survey.ccds_touching_wcs(wcs)
    if C is None:
        print('No CCDs touching brick')
        return None
    I = np.flatnonzero(C.ccd_cuts == 0)
    if len(I) == 0:
        print('No good CCDs touching brick')
        return None
    C.cut(I)
    print(len(C), 'CCDs touching brick')

    depths = {}
    for band in bands:
        d = np.zeros((BH, BW), np.float32)
        depths[band] = d

    npix = dict([(band, 0) for band in bands])
    nexps = dict([(band, 0) for band in bands])

    # survey.get_approx_wcs(ccd)
    for ccd in C:
        #im = survey.get_image_object(ccd)
        awcs = survey.get_approx_wcs(ccd)

        imh, imw = ccd.height, ccd.width
        x0, y0 = 0, 0
        x1 = x0 + imw
        y1 = y0 + imh
        imgpoly = [(1, 1), (1, imh), (imw, imh), (imw, 1)]
        ok, tx, ty = awcs.radec2pixelxy(targetrd[:-1, 0], targetrd[:-1, 1])
        tpoly = list(zip(tx, ty))
        clip = clip_polygon(imgpoly, tpoly)
        clip = np.array(clip)
        if len(clip) == 0:
            continue
        x0, y0 = np.floor(clip.min(axis=0)).astype(int)
        x1, y1 = np.ceil(clip.max(axis=0)).astype(int)
        #slc = slice(y0,y1+1), slice(x0,x1+1)
        awcs = awcs.get_subimage(x0, y0, x1 - x0, y1 - y0)
        ah, aw = awcs.shape

        #print('Image', ccd.expnum, ccd.ccdname, ccd.filter, 'overlap', x0,x1, y0,y1, '->', (1+x1-x0),'x',(1+y1-y0))

        # Find bbox in brick space
        r, d = awcs.pixelxy2radec([1, 1, aw, aw], [1, ah, ah, 1])
        ok, bx, by = wcs.radec2pixelxy(r, d)
        bx0 = np.clip(np.round(bx.min()).astype(int) - 1, 0, BW - 1)
        bx1 = np.clip(np.round(bx.max()).astype(int) - 1, 0, BW - 1)
        by0 = np.clip(np.round(by.min()).astype(int) - 1, 0, BH - 1)
        by1 = np.clip(np.round(by.max()).astype(int) - 1, 0, BH - 1)

        #print('Brick', bx0,bx1,by0,by1)

        band = ccd.filter[0]
        assert (band in bands)

        ccdzpt = ccd.ccdzpt + 2.5 * np.log10(ccd.exptime)

        psf_sigma = ccd.fwhm / 2.35
        psfnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
        orig_zpscale = zpscale = NanoMaggies.zeropointToScale(ccdzpt)
        sig1 = ccd.sig1 / orig_zpscale
        detsig1 = sig1 / psfnorm

        # print('Image', ccd.expnum, ccd.ccdname, ccd.filter,
        #       'PSF depth', -2.5 * (np.log10(5.*detsig1) - 9), 'exptime', ccd.exptime,
        #       'sig1', ccd.sig1, 'zpt', ccd.ccdzpt, 'fwhm', ccd.fwhm,
        #       'filename', ccd.image_filename.strip())

        depths[band][by0:by1 + 1, bx0:bx1 + 1] += (1. / detsig1**2)
        npix[band] += (y1 + 1 - y0) * (x1 + 1 - x0)
        nexps[band] += 1

    for band in bands:
        det = np.median(depths[band])
        # compute stats for 5-sigma detection
        with np.errstate(divide='ignore'):
            depth = 5. / np.sqrt(det)
        # that's flux in nanomaggies -- convert to mag
        depth = -2.5 * (np.log10(depth) - 9)
        if not np.isfinite(depth):
            depth = 0.

        depths[band] = depth
        #bricks.get('psfdepth_' + band)[ibrick] = depth
        print(brick.brickname, 'median PSF depth', band, ':', depth, 'npix',
              npix[band], 'nexp', nexps[band])
        #'npix', bricks.get('npix_'+band)[ibrick],
        #'nexp', bricks.get('nexp_'+band)[ibrick])

    return (npix, nexps, depths)
Example #7
0
def rbmain():
    from legacypipe.catalog import read_fits_catalog
    from legacypipe.survey import LegacySurveyData, wcs_for_brick
    from tractor.galaxy import DevGalaxy
    from tractor import PointSource, Catalog
    from tractor import GaussianMixturePSF
    from legacypipe.survey import BrickDuck
    from legacypipe.forced_photom import main as forced_main
    from astrometry.util.file import trymakedirs
    import shutil

    ceres = 'ceres' in sys.argv
    psfex = 'psfex' in sys.argv

    for v in [
            'UNWISE_COADDS_TIMERESOLVED_DIR', 'SKY_TEMPLATE_DIR',
            'LARGEGALAXIES_CAT', 'GAIA_CAT_DIR', 'TYCHO2_KD_DIR'
    ]:
        if v in os.environ:
            del os.environ[v]

    oldargs = sys.argv
    sys.argv = [sys.argv[0]]
    main()
    sys.argv = oldargs

    # Test create_kdtree and (reading CCD kd-tree)!
    indir = os.path.join(os.path.dirname(__file__), 'testcase6')
    with tempfile.TemporaryDirectory() as surveydir:
        files = [
            'calib', 'gaia', 'images', 'survey-bricks.fits.gz',
            'tycho2.kd.fits'
        ]
        for fn in files:
            src = os.path.join(indir, fn)
            dst = os.path.join(surveydir, fn)
            #trymakedirs(dst, dir=True)
            print('Copy', src, dst)
            if os.path.isfile(src):
                shutil.copy(src, dst)
            else:
                shutil.copytree(src, dst)

        from legacypipe.create_kdtrees import create_kdtree
        infn = os.path.join(indir, 'survey-ccds-1.fits.gz')
        outfn = os.path.join(surveydir, 'survey-ccds-1.kd.fits')
        create_kdtree(infn, outfn, False)

        os.environ['TYCHO2_KD_DIR'] = surveydir
        outdir = 'out-testcase6-kd'
        main(args=[
            '--brick', '1102p240', '--zoom', '500', '600', '650', '750',
            '--force-all', '--no-write', '--no-wise', '--no-gaia',
            '--survey-dir', surveydir, '--outdir', outdir
        ])
        fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
        assert (os.path.exists(fn))
        T = fits_table(fn)
        assert (len(T) == 2)
        # Since there is a Tycho-2 star in the blob, forced to be PSF.
        assert (T.type[0].strip() == 'PSF')
        assert (T.type[1].strip() == 'PSF')
        # There is a Tycho-2 star in the blob.
        I = np.flatnonzero(T.ref_cat == 'T2')
        assert (len(I) == 1)
        assert (T.ref_id[I][0] == 1909016711)

        cat = read_fits_catalog(T)
        assert (len(cat) == 2)
        assert (isinstance(cat[0], PointSource))
        assert (isinstance(cat[1], PointSource))
        cat, ivs = read_fits_catalog(T, invvars=True)
        assert (len(cat) == 2)
        assert (isinstance(cat[0], PointSource))
        assert (isinstance(cat[1], PointSource))
        cat2 = Catalog(*cat)
        assert (len(ivs) == len(cat2.getParams()))

        # test --fit-on-coadds
        outdir = 'out-testcase6-coadds'
        main(args=[
            '--brick', '1102p240', '--zoom', '500', '600', '650', '750',
            '--force-all', '--no-write', '--no-wise', '--no-gaia',
            '--survey-dir', surveydir, '--fit-on-coadds', '--outdir', outdir
        ])

        fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
        assert (os.path.exists(fn))
        T = fits_table(fn)
        assert (len(T) == 2)
        # Since there is a Tycho-2 star in the blob, forced to be PSF.
        assert (T.type[0].strip() == 'PSF')
        assert (T.type[1].strip() == 'PSF')
        # There is a Tycho-2 star in the blob.
        I = np.flatnonzero(T.ref_cat == 'T2')
        assert (len(I) == 1)
        assert (T.ref_id[I][0] == 1909016711)
        del os.environ['TYCHO2_KD_DIR']

        # test --skip-coadds
        r = main(args=[
            '--brick', '1102p240', '--zoom', '500', '600', '650', '750',
            '--force-all', '--no-write', '--no-wise', '--no-gaia',
            '--survey-dir', surveydir, '--outdir', outdir, '--skip-coadd'
        ])
        assert (r == 0)

        # test --skip
        r = main(args=[
            '--brick', '1102p240', '--zoom', '500', '600', '650', '750',
            '--force-all', '--no-write', '--no-wise', '--no-gaia',
            '--survey-dir', surveydir, '--outdir', outdir, '--skip'
        ])
        assert (r == 0)

        # NothingToDoError (neighbouring brick)
        r = main(args=[
            '--brick', '1102p240', '--zoom', '0', '100', '0', '100',
            '--force-all', '--no-write', '--no-wise', '--no-gaia',
            '--survey-dir', surveydir, '--outdir', outdir
        ])
        assert (r == 0)

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase9')

    # Test for some get_tractor_image kwargs
    survey = LegacySurveyData(surveydir)
    fakebrick = BrickDuck(9.1228, 3.3975, 'quack')
    wcs = wcs_for_brick(fakebrick, W=100, H=100)
    ccds = survey.ccds_touching_wcs(wcs)
    ccd = ccds[0]
    im = survey.get_image_object(ccd)
    H, W = wcs.shape
    targetrd = np.array([
        wcs.pixelxy2radec(x, y)
        for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
    ])
    tim = im.get_tractor_image(radecpoly=targetrd)
    assert (tim.getImage() is not None)
    assert (tim.getInvError() is not None)
    assert (tim.dq is not None)
    tim2 = im.get_tractor_image(radecpoly=targetrd, pixels=False)
    assert (np.all(tim2.getImage() == 0.))
    tim4 = im.get_tractor_image(radecpoly=targetrd, invvar=False)
    u = np.unique(tim4.inverr)
    assert (len(u) == 1)
    u = u[0]
    target = tim4.zpscale / tim4.sig1
    assert (np.abs(u / target - 1.) < 0.001)
    tim3 = im.get_tractor_image(radecpoly=targetrd, invvar=False, dq=False)
    assert (not hasattr(tim3, 'dq'))
    tim5 = im.get_tractor_image(radecpoly=targetrd, gaussPsf=True)
    print(tim5.getPsf())
    assert (isinstance(tim5.getPsf(), GaussianMixturePSF))

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase12')
    os.environ['TYCHO2_KD_DIR'] = surveydir
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'
    os.environ['UNWISE_MODEL_SKY_DIR'] = os.path.join(surveydir, 'images',
                                                      'unwise-mod')
    #python legacypipe/runbrick.py --radec  --width 100 --height 100 --outdir dup5b --survey-dir test/testcase12 --force-all --no-wise
    unwdir = os.path.join(surveydir, 'images', 'unwise')
    main(args=[
        '--radec', '346.684', '12.791', '--width', '100', '--height', '100',
        '--no-wise-ceres', '--unwise-dir', unwdir, '--survey-dir', surveydir,
        '--outdir', 'out-testcase12', '--skip-coadd', '--force-all'
    ])

    # --plots for stage_wise_forced
    main(args=[
        '--radec', '346.684', '12.791', '--width', '100', '--height', '100',
        '--no-wise-ceres', '--unwise-dir', unwdir, '--survey-dir', surveydir,
        '--outdir', 'out-testcase12', '--stage', 'wise_forced', '--plots'
    ])
    del os.environ['GAIA_CAT_DIR']
    del os.environ['GAIA_CAT_VER']
    del os.environ['TYCHO2_KD_DIR']
    del os.environ['UNWISE_MODEL_SKY_DIR']

    M = fitsio.read(
        'out-testcase12/coadd/cus/custom-346684p12791/legacysurvey-custom-346684p12791-maskbits.fits.fz'
    )
    # Count masked & unmasked bits (the cluster splits this 100x100 field)
    from collections import Counter
    c = Counter(M.ravel())
    from legacypipe.bits import MASKBITS
    assert (c[0] >= 4000)
    assert (c[MASKBITS['CLUSTER']] >= 4000)

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase9')
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'
    os.environ['LARGEGALAXIES_CAT'] = os.path.join(surveydir,
                                                   'sga-sub.kd.fits')
    main(args=[
        '--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
        '--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir',
        surveydir, '--outdir', 'out-testcase9', '--skip', '--force-all',
        '--ps', 'tc9-ps.fits', '--ps-t0',
        str(int(time.time()))
    ])
    # (omit --force-all --no-write... reading from pickles below!)

    # Test with --apodize
    main(args=[
        '--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
        '--old-calibs-ok', '--no-wise', '--force-all', '--no-write',
        '--survey-dir', surveydir, '--outdir', 'out-testcase9-ap', '--apodize'
    ])

    main(args=[
        '--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
        '--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir',
        surveydir, '--outdir', 'out-testcase9', '--plots', '--stage', 'halos'
    ])

    main(args=[
        '--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
        '--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir',
        surveydir, '--outdir', 'out-testcase9-coadds', '--stage',
        'image_coadds', '--blob-image'
    ])

    T = fits_table(
        'out-testcase9/tractor/cus/tractor-custom-009122p03397.fits')
    assert (len(T) == 4)
    # Gaia star becomes a DUP!
    assert (np.sum([t == 'DUP' for t in T.type]) == 1)
    # LSLGA galaxy exists!
    Igal = np.flatnonzero([r == 'L3' for r in T.ref_cat])
    assert (len(Igal) == 1)
    assert (np.all(T.ref_id[Igal] > 0))
    assert (T.type[Igal[0]] == 'SER')

    # --brick and --zoom rather than --radec --width --height
    main(args=[
        '--survey-dir', surveydir, '--outdir', 'out-testcase9b', '--zoom',
        '1950', '2050', '340', '440', '--brick', '0091p035', '--force-all'
    ])

    # test forced phot??
    shutil.copy('test/testcase9/survey-bricks.fits.gz', 'out-testcase9b')

    forced_main(args=[
        '--survey-dir', surveydir, '--no-ceres', '--catalog-dir',
        'out-testcase9b', '372546', 'N26', 'forced1.fits'
    ])
    assert (os.path.exists('forced1.fits'))
    _ = fits_table('forced1.fits')
    # ... more tests...!

    forced_main(args=[
        '--survey-dir', surveydir, '--no-ceres', '--catalog-dir',
        'out-testcase9b', '--derivs', '--threads', '2', '--apphot', '372547',
        'N26', 'forced2.fits'
    ])
    assert (os.path.exists('forced2.fits'))
    _ = fits_table('forced2.fits')

    forced_main(args=[
        '--survey-dir', surveydir, '--no-ceres', '--catalog-dir',
        'out-testcase9b', '--agn', '257266', 'S21', 'forced3.fits'
    ])
    assert (os.path.exists('forced3.fits'))
    _ = fits_table('forced3.fits')

    if ceres:
        forced_main(args=[
            '--survey-dir', surveydir, '--catalog-dir', 'out-testcase9b',
            '--derivs', '--threads', '2', '--apphot', '372546', 'N26',
            'forced4.fits'
        ])
        assert (os.path.exists('forced4.fits'))
        _ = fits_table('forced4.fits')

    # Test cache_dir
    with tempfile.TemporaryDirectory() as cachedir, \
        tempfile.TemporaryDirectory() as tempsurveydir:
        files = []
        for dirpath, _, filenames in os.walk(surveydir):
            for fn in filenames:
                path = os.path.join(dirpath, fn)
                relpath = os.path.relpath(path, surveydir)
                files.append(relpath)
        # cache or no?
        files.sort()
        files_cache = files[::2]
        files_nocache = files[1::2]
        # Survey-ccds *must* be in nocache.
        fn = 'survey-ccds-1.kd.fits'
        if fn in files_cache:
            files_cache.remove(fn)
            files_nocache.append(fn)

        for fn in files_cache:
            src = os.path.join(surveydir, fn)
            dst = os.path.join(cachedir, fn)
            trymakedirs(dst, dir=True)
            print('Copy', src, dst)
            shutil.copy(src, dst)

        for fn in files_nocache:
            src = os.path.join(surveydir, fn)
            dst = os.path.join(tempsurveydir, fn)
            trymakedirs(dst, dir=True)
            print('Copy', src, dst)
            shutil.copy(src, dst)

        main(args=[
            '--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
            '--no-wise', '--survey-dir', tempsurveydir, '--cache-dir',
            cachedir, '--outdir', 'out-testcase9cache', '--force-all'
        ])

    del os.environ['GAIA_CAT_DIR']
    del os.environ['GAIA_CAT_VER']
    del os.environ['LARGEGALAXIES_CAT']

    # if ceres:
    #     surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
    #     main(args=['--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
    #                '--no-wise', '--force-all', '--no-write', '--ceres',
    #                '--survey-dir', surveydir,
    #                '--outdir', 'out-testcase3-ceres',
    #                '--no-depth-cut'])

    # MzLS + BASS data
    # python legacypipe/runbrick.py --run north --brick 1773p595 --zoom 1300 1500 700 900 --survey-dir dr9-north -s coadds
    # fitscopy coadd/177/1773p595/legacysurvey-1773p595-ccds.fits"[#row<3 || #row==12]" cx.fits
    # python legacyanalysis/create_testcase.py cx.fits test/mzlsbass2 1773p595 --survey-dir dr9-north/ --fpack
    surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass2')
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir2, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'
    main(args=[
        '--brick', '1773p595', '--zoom', '1300', '1500', '700', '900',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir2,
        '--outdir', 'out-mzlsbass2'
    ])

    T = fits_table('out-mzlsbass2/tractor/177/tractor-1773p595.fits')
    assert (np.sum(T.ref_cat == 'G2') == 3)
    assert (np.sum(T.ref_id > 0) == 3)

    # Test --max-blobsize, --checkpoint, --bail-out

    outdir = 'out-mzlsbass2b'
    chk = 'checkpoint-mzb2b.p'
    if os.path.exists(chk):
        os.unlink(chk)
    main(args=[
        '--brick', '1773p595', '--zoom', '1300', '1500', '700', '900',
        '--no-wise', '--force-all', '--stage', 'fitblobs', '--write-stage',
        'srcs', '--survey-dir', surveydir2, '--outdir', outdir, '--checkpoint',
        chk, '--nblobs', '3'
    ])
    # err... --max-blobsize does not result in bailed-out blobs masked,
    # because it treats large blobs as *completed*...
    #'--max-blobsize', '3000',

    outdir = 'out-mzlsbass2c'
    main(args=[
        '--brick', '1773p595', '--zoom', '1300', '1500', '700', '900',
        '--no-wise', '--force-all', '--survey-dir', surveydir2, '--outdir',
        outdir, '--bail-out', '--checkpoint', chk, '--no-write'
    ])

    del os.environ['GAIA_CAT_DIR']
    del os.environ['GAIA_CAT_VER']

    M = fitsio.read(
        os.path.join(outdir, 'coadd', '177', '1773p595',
                     'legacysurvey-1773p595-maskbits.fits.fz'))
    assert (np.sum((M & MASKBITS['BAILOUT']) > 0) >= 1000)

    # Test RexGalaxy

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
    outdir = 'out-testcase6-rex'
    the_args = [
        '--brick',
        '1102p240',
        '--zoom',
        '500',
        '600',
        '650',
        '750',
        '--force-all',
        '--no-write',
        '--no-wise',
        '--skip-calibs',
        #'--rex', #'--plots',
        '--survey-dir',
        surveydir,
        '--outdir',
        outdir
    ]
    print('python legacypipe/runbrick.py', ' '.join(the_args))
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'
    main(args=the_args)
    fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 2)
    print('Types:', T.type)
    # Since there is a Tycho-2 star in the blob, forced to be PSF.
    assert (T.type[0].strip() == 'PSF')
    cmd = (
        '(cd %s && sha256sum -c %s)' %
        (outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum')))
    print(cmd)
    rtn = os.system(cmd)
    assert (rtn == 0)

    # Test with a Tycho-2 star in the blob.

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
    outdir = 'out-testcase6'
    main(args=[
        '--brick', '1102p240', '--zoom', '500', '600', '650', '750',
        '--force-all', '--no-write', '--no-wise', '--survey-dir', surveydir,
        '--outdir', outdir
    ])
    fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 2)
    print('Types:', T.type)
    # Since there is a Tycho-2 star in the blob, forced to be PSF.
    assert (T.type[0].strip() == 'PSF')
    del os.environ['GAIA_CAT_DIR']
    del os.environ['GAIA_CAT_VER']

    # Test that we can run splinesky calib if required...

    from legacypipe.decam import DecamImage
    DecamImage.splinesky_boxsize = 128

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')

    survey = LegacySurveyData(surveydir)
    # get brick by id
    brickid = 473357
    brick = survey.get_brick(brickid)
    assert (brick.brickname == '1867p255')
    assert (brick.brickid == brickid)

    outdir = 'out-testcase4'
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'

    fn = os.path.join(surveydir, 'calib', 'sky-single', 'decam', 'CP',
                      'V4.8.2', 'CP20170315', 'c4d_170316_062107_ooi_z_ls9',
                      'c4d_170316_062107_ooi_z_ls9-N2-splinesky.fits')
    if os.path.exists(fn):
        os.unlink(fn)

    main(args=[
        '--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
        '--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
        os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
        os.path.join(surveydir, 'images',
                     'unwise-tr'), '--blob-image', '--no-hybrid-psf',
        '--survey-dir', surveydir, '--outdir', outdir, '-v', '--no-wise-ceres'
    ])
    print('Checking for calib file', fn)
    assert (os.path.exists(fn))

    # Test with blob-masking when creating sky calib.
    os.unlink(fn)

    main(args=[
        '--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
        '--force-all', '--no-write', '--coadd-bw', '--blob-mask-dir',
        surveydir, '--survey-dir', surveydir, '--stage', 'image_coadds',
        '--outdir', 'out-testcase4b', '--plots'
    ])
    print('Checking for calib file', fn)
    assert (os.path.exists(fn))

    if ceres:
        main(args=[
            '--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
            '--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
            os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
            os.path.join(surveydir, 'images', 'unwise-tr'), '--survey-dir',
            surveydir, '--outdir', outdir
        ])
    if psfex:
        # Check that we can regenerate PsfEx files if necessary.
        fn = os.path.join(surveydir, 'calib', 'psfex', 'decam', 'CP', 'V4.8.2',
                          'CP20170315',
                          'c4d_170316_062107_ooi_z_ls9-psfex.fits')
        if os.path.exists(fn):
            os.unlink(fn)

        main(args=[
            '--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
            '--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
            os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
            os.path.join(surveydir, 'images', 'unwise-tr'), '--blob-image',
            '--survey-dir', surveydir, '--outdir', outdir, '-v'
        ])
        print('After generating PsfEx calib:')
        os.system('find %s' % (os.path.join(surveydir, 'calib')))

    # Wrap-around, hybrid PSF
    surveydir = os.path.join(os.path.dirname(__file__), 'testcase8')
    outdir = 'out-testcase8'
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'

    main(args=[
        '--brick',
        '1209p050',
        '--zoom',
        '720',
        '1095',
        '3220',
        '3500',
        '--force-all',
        '--no-write',
        '--no-wise',  #'--plots',
        '--survey-dir',
        surveydir,
        '--outdir',
        outdir
    ])

    # Test with a Tycho-2 star + another saturated star in the blob.

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase7')
    outdir = 'out-testcase7'
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'
    main(args=[
        '--brick',
        '1102p240',
        '--zoom',
        '250',
        '350',
        '1550',
        '1650',
        '--force-all',
        '--no-write',
        '--no-wise',  #'--plots',
        '--survey-dir',
        surveydir,
        '--outdir',
        outdir
    ])
    del os.environ['GAIA_CAT_DIR']
    del os.environ['GAIA_CAT_VER']
    fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 4)

    # Check skipping blobs outside the brick's unique area.
    # (this now doesn't detect any sources at all, reasonably)
    # surveydir = os.path.join(os.path.dirname(__file__), 'testcase5')
    # outdir = 'out-testcase5'
    #
    # fn = os.path.join(outdir, 'tractor', '186', 'tractor-1867p255.fits')
    # if os.path.exists(fn):
    #     os.unlink(fn)
    #
    # main(args=['--brick', '1867p255', '--zoom', '0', '150', '0', '150',
    #            '--force-all', '--no-write', '--coadd-bw',
    #            '--survey-dir', surveydir,
    #            '--early-coadds',
    #            '--outdir', outdir] + extra_args)
    #
    # assert(os.path.exists(fn))
    # T = fits_table(fn)
    # assert(len(T) == 1)

    # Custom RA,Dec; blob ra,dec.
    surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
    outdir = 'out-testcase4b'
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    # Catalog written with one entry (--blobradec)
    fn = os.path.join(outdir, 'tractor', 'cus',
                      'tractor-custom-186743p25461.fits')
    if os.path.exists(fn):
        os.unlink(fn)
    main(args=[
        '--radec', '186.743965', '25.461788', '--width', '250', '--height',
        '250', '--force-all', '--no-write', '--no-wise', '--blobradec',
        '186.740369', '25.453855', '--survey-dir', surveydir, '--outdir',
        outdir
    ])

    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 1)

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
    outdir = 'out-testcase3'
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'
    checkpoint_fn = os.path.join(outdir, 'checkpoint.pickle')
    if os.path.exists(checkpoint_fn):
        os.unlink(checkpoint_fn)
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', outdir, '--checkpoint', checkpoint_fn,
        '--checkpoint-period', '1', '--threads', '2'
    ])

    # Read catalog into Tractor sources to test read_fits_catalog
    survey = LegacySurveyData(survey_dir=outdir)
    fn = survey.find_file('tractor', brick='2447p120')
    print('Checking', fn)
    T = fits_table(fn)
    cat = read_fits_catalog(T)
    print('Read catalog:', cat)

    assert (len(cat) == 2)
    src = cat[1]
    print('Source0', src)
    from tractor.sersic import SersicGalaxy
    assert (type(src) in [DevGalaxy, SersicGalaxy])
    assert (np.abs(src.pos.ra - 244.77973) < 0.00001)
    assert (np.abs(src.pos.dec - 12.07234) < 0.00002)
    src = cat[0]
    print('Source1', src)
    assert (type(src) == PointSource)
    assert (np.abs(src.pos.ra - 244.77828) < 0.00001)
    assert (np.abs(src.pos.dec - 12.07250) < 0.00001)

    # Check that we can run again, using that checkpoint file.
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', outdir, '--checkpoint', checkpoint_fn,
        '--checkpoint-period', '1', '--threads', '2'
    ])
    # Assert...... something?

    # Test --checkpoint without --threads
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', outdir, '--checkpoint', checkpoint_fn,
        '--checkpoint-period', '1'
    ])

    # From Kaylan's Bootes pre-DR4 run
    # surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass3')
    # main(args=['--brick', '2173p350', '--zoom', '100', '200', '100', '200',
    #            '--no-wise', '--force-all', '--no-write',
    #            '--survey-dir', surveydir2,
    #            '--outdir', 'out-mzlsbass3'] + extra_args)

    # With plots!
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', 'out-testcase3', '--plots', '--nblobs', '1'
    ])