def getbrickfiles(brickname=None): survey = LegacySurveyData() brickinfo = survey.get_brick_by_name(brickname) brickwcs = wcs_for_brick(brickinfo) ccdinfo = survey.ccds_touching_wcs(brickwcs) nccd = len(ccdinfo) calibdir = survey.get_calib_dir() imagedir = survey.survey_dir # Construct image file names and the calibration file names. expnum = ccdinfo.expnum ccdname = ccdinfo.ccdname psffiles = list() skyfiles = list() imagefiles = list() for ccd in ccdinfo: info = survey.get_image_object(ccd) for attr in ['imgfn', 'dqfn', 'wtfn']: fn = getattr(info, attr).replace(imagedir+'/', '') #if '160108_073601' in fn: # pdb.set_trace() imagefiles.append(fn) psffiles.append(info.psffn.replace(calibdir, 'calib')) skyfiles.append(info.splineskyfn.replace(calibdir, 'calib')) #for ii in range(nccd): #exp = '{0:08d}'.format(expnum[ii]) #rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits') #psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile)) #skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile)) #imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii])))) #print(np.array(imagefiles)) #print(np.array(psffiles)) #print(np.array(skyfiles)) return imagefiles, psffiles, skyfiles
def getbrickfiles(brickname=None): survey = LegacySurveyData() brickinfo = survey.get_brick_by_name(brickname) brickwcs = wcs_for_brick(brickinfo) ccdinfo = survey.ccds_touching_wcs(brickwcs) nccd = len(ccdinfo) calibdir = survey.get_calib_dir() imagedir = survey.survey_dir # Construct image file names and the calibration file names. expnum = ccdinfo.expnum ccdname = ccdinfo.ccdname psffiles = list() skyfiles = list() imagefiles = list() for ccd in ccdinfo: info = survey.get_image_object(ccd) for attr in ['imgfn', 'dqfn', 'wtfn']: fn = getattr(info, attr).replace(imagedir + '/', '') #if '160108_073601' in fn: # pdb.set_trace() imagefiles.append(fn) psffiles.append(info.psffn.replace(calibdir, 'calib')) skyfiles.append(info.splineskyfn.replace(calibdir, 'calib')) #for ii in range(nccd): #exp = '{0:08d}'.format(expnum[ii]) #rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits') #psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile)) #skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile)) #imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii])))) #print(np.array(imagefiles)) #print(np.array(psffiles)) #print(np.array(skyfiles)) return imagefiles, psffiles, skyfiles
#ra,dec = 357.3060, 2.3957 #ccd1 = ccds[(ccds.expnum == 563212) * (ccds.ccdname == 'N17')] ra, dec = 124.0317, 1.3028 expnum, ccdname = 393203, 'N11' survey = LegacySurveyData() W, H = 200, 200 pixscale = 0.262 cd = pixscale / 3600. targetwcs = Tan(ra, dec, W / 2., H / 2., -cd, 0., 0., cd, float(W), float(H)) rr, dd = targetwcs.pixelxy2radec([1, W, W, 1, 1], [1, 1, H, H, 1]) targetrd = np.vstack((rr, dd)).T ccds = survey.ccds_touching_wcs(targetwcs) print(len(ccds), 'CCDs touching WCS') print('MJDs', ccds.mjd_obs) ccds.writeto('test-ccds.fits') ccd1 = ccds[(ccds.expnum == expnum) * (ccds.ccdname == ccdname)] print('CCD1:', ccd1) im1 = survey.get_image_object(ccd1[0]) print('Im:', im1) #wcs = im1.get_wcs() #x0,x1,y0,y1,slc = im1.get_image_extent(wcs=wcs, radecpoly=targetrd) tim1 = im1.get_tractor_image(radecpoly=targetrd, pixPsf=True,
def run_one_brick(X): brick, ibrick, nbricks, plots, kwargs = X survey = LegacySurveyData() print() print() print('Brick', (ibrick + 1), 'of', nbricks, ':', brick.brickname) dirnm = os.path.join('depthcuts', brick.brickname[:3]) outfn = os.path.join(dirnm, 'ccds-%s.fits' % brick.brickname) if os.path.exists(outfn): print('Exists:', outfn) return 0 H, W = 3600, 3600 pixscale = 0.262 bands = ['g', 'r', 'z'] # Get WCS object describing brick targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale) targetrd = np.array([ targetwcs.pixelxy2radec(x, y) for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)] ]) gitver = get_git_version() ccds = survey.ccds_touching_wcs(targetwcs) if ccds is None: print('No CCDs actually touching brick') return 0 print(len(ccds), 'CCDs actually touching brick') ccds.cut(np.in1d(ccds.filter, bands)) print('Cut on filter:', len(ccds), 'CCDs remain.') if 'ccd_cuts' in ccds.get_columns(): norig = len(ccds) ccds.cut(ccds.ccd_cuts == 0) print(len(ccds), 'of', norig, 'CCDs pass cuts') else: print('No CCD cuts') if len(ccds) == 0: print('No CCDs left') return 0 ps = None if plots: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('depth-%s' % brick.brickname) splinesky = True gaussPsf = False pixPsf = True do_calibs = False normalizePsf = True get_depth_maps = kwargs.pop('get_depth_maps', False) try: D = make_depth_cut(survey, ccds, bands, targetrd, brick, W, H, pixscale, plots, ps, splinesky, gaussPsf, pixPsf, normalizePsf, do_calibs, gitver, targetwcs, get_depth_maps=get_depth_maps, **kwargs) if get_depth_maps: keep, overlapping, depthmaps = D else: keep, overlapping = D except: print('Failed to make_depth_cut():') import traceback traceback.print_exc() return -1 print(np.sum(overlapping), 'CCDs overlap the brick') print(np.sum(keep), 'CCDs passed depth cut') ccds.overlapping = overlapping ccds.passed_depth_cut = keep if not os.path.exists(dirnm): try: os.makedirs(dirnm) except: pass if get_depth_maps: for band, depthmap in depthmaps: doutfn = os.path.join(dirnm, 'depth-%s-%s.fits' % (brick.brickname, band)) hdr = fitsio.FITSHDR() # Plug the WCS header cards into these images targetwcs.add_to_header(hdr) hdr.delete('IMAGEW') hdr.delete('IMAGEH') hdr.add_record(dict(name='EQUINOX', value=2000.)) hdr.add_record(dict(name='FILTER', value=band)) fitsio.write(doutfn, depthmap, header=hdr) print('Wrote', doutfn) tmpfn = os.path.join(os.path.dirname(outfn), 'tmp-' + os.path.basename(outfn)) ccds.writeto(tmpfn) os.rename(tmpfn, outfn) print('Wrote', outfn) return 0
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument( '--dr', '--drdir', dest='drdir', default='/project/projectdirs/cosmo/data/legacysurvey/dr5', help='Directory containing data release w/ tar-gzipped calibs') parser.add_argument( '-b', '--brick', help='Brick name to run; required unless --radec is given') parser.add_argument( '--radec', nargs=2, help='RA,Dec center for a custom location (not a brick)') parser.add_argument('--pixscale', type=float, default=0.262, help='Pixel scale of the output coadds (arcsec/pixel)') parser.add_argument('-W', '--width', type=int, default=3600, help='Target image width, default %(default)i') parser.add_argument('-H', '--height', type=int, default=3600, help='Target image height, default %(default)i') parser.add_argument( '--zoom', type=int, nargs=4, help='Set target image extent (default "0 3600 0 3600")') parser.add_argument('--no-psf', dest='do_psf', default=True, action='store_false', help='Do not extract PsfEx files') parser.add_argument('--no-sky', dest='do_sky', default=True, action='store_false', help='Do not extract SplineSky files') opt = parser.parse_args() if opt.brick is None and opt.radec is None: parser.print_help() return -1 optdict = vars(opt) drdir = opt.drdir W = opt.width H = opt.height pixscale = opt.pixscale target_extent = opt.zoom do_psf = opt.do_psf do_sky = opt.do_sky #brickname = '1501p020' custom = (opt.radec is not None) #ra,dec = 216.03, 34.86 if custom: ra, dec = opt.radec #27.30, -10.43 ra = float(ra) dec = float(dec) #W,H = 1000,1000 #W,H = 1500,1500 brickname = 'custom_%.3f_%.3f' % (ra, dec) #do_sky = False survey = LegacySurveyData() if custom: from legacypipe.survey import BrickDuck # Custom brick; create a fake 'brick' object brick = BrickDuck(ra, dec, brickname) else: brickname = opt.brick brick = survey.get_brick_by_name(brickname) # Get WCS object describing brick targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale) if target_extent is not None: (x0, x1, y0, y1) = target_extent W = x1 - x0 H = y1 - y0 targetwcs = targetwcs.get_subimage(x0, y0, W, H) # Find CCDs ccds = survey.ccds_touching_wcs(targetwcs, ccdrad=None) if ccds is None: raise NothingToDoError('No CCDs touching brick') print(len(ccds), 'CCDs touching target WCS') for ccd in ccds: im = survey.get_image_object(ccd) print('CCD', im) expnum = '%08i' % im.expnum if do_psf: if os.path.exists(im.psffn) or os.path.exists(im.merged_psffn): print('PSF file exists') else: print('Need PSF', im.psffn, im.merged_psffn) tarfn = os.path.join( drdir, 'calib', im.camera, 'psfex-merged', 'legacysurvey_dr5_calib_decam_psfex-merged_%s.tar.gz' % expnum[:5]) print(tarfn) if os.path.exists(tarfn): outfn = '%s/%s-%s.fits' % (expnum[:5], im.camera, expnum) cmd = 'cd %s/%s/psfex-merged && tar xvzf %s %s' % ( survey.get_calib_dir(), im.camera, tarfn, outfn) print(cmd) os.system(cmd) if do_sky: if os.path.exists(im.splineskyfn) or os.path.exists( im.merged_splineskyfn): print('Sky file exists') else: print('Need sky', im.splineskyfn, im.merged_splineskyfn) tarfn = os.path.join( drdir, 'calib', im.camera, 'splinesky-merged', 'legacysurvey_dr5_calib_decam_splinesky-merged_%s.tar.gz' % expnum[:5]) print(tarfn) if os.path.exists(tarfn): outfn = '%s/%s-%s.fits' % (expnum[:5], im.camera, expnum) cmd = 'cd %s/%s/splinesky-merged && tar xvzf %s %s' % ( survey.get_calib_dir(), im.camera, tarfn, outfn) print(cmd) os.system(cmd)
def one_brick(X): (ibrick, brick) = X bands = ['g', 'r', 'z'] print('Brick', brick.brickname) wcs = wcs_for_brick(brick, W=94, H=94, pixscale=10.) BH, BW = wcs.shape targetrd = np.array([ wcs.pixelxy2radec(x, y) for x, y in [(1, 1), (BW, 1), (BW, BH), (1, BH), (1, 1)] ]) survey = LegacySurveyData() C = survey.ccds_touching_wcs(wcs) if C is None: print('No CCDs touching brick') return None I = np.flatnonzero(C.ccd_cuts == 0) if len(I) == 0: print('No good CCDs touching brick') return None C.cut(I) print(len(C), 'CCDs touching brick') depths = {} for band in bands: d = np.zeros((BH, BW), np.float32) depths[band] = d npix = dict([(band, 0) for band in bands]) nexps = dict([(band, 0) for band in bands]) # survey.get_approx_wcs(ccd) for ccd in C: #im = survey.get_image_object(ccd) awcs = survey.get_approx_wcs(ccd) imh, imw = ccd.height, ccd.width x0, y0 = 0, 0 x1 = x0 + imw y1 = y0 + imh imgpoly = [(1, 1), (1, imh), (imw, imh), (imw, 1)] ok, tx, ty = awcs.radec2pixelxy(targetrd[:-1, 0], targetrd[:-1, 1]) tpoly = list(zip(tx, ty)) clip = clip_polygon(imgpoly, tpoly) clip = np.array(clip) if len(clip) == 0: continue x0, y0 = np.floor(clip.min(axis=0)).astype(int) x1, y1 = np.ceil(clip.max(axis=0)).astype(int) #slc = slice(y0,y1+1), slice(x0,x1+1) awcs = awcs.get_subimage(x0, y0, x1 - x0, y1 - y0) ah, aw = awcs.shape #print('Image', ccd.expnum, ccd.ccdname, ccd.filter, 'overlap', x0,x1, y0,y1, '->', (1+x1-x0),'x',(1+y1-y0)) # Find bbox in brick space r, d = awcs.pixelxy2radec([1, 1, aw, aw], [1, ah, ah, 1]) ok, bx, by = wcs.radec2pixelxy(r, d) bx0 = np.clip(np.round(bx.min()).astype(int) - 1, 0, BW - 1) bx1 = np.clip(np.round(bx.max()).astype(int) - 1, 0, BW - 1) by0 = np.clip(np.round(by.min()).astype(int) - 1, 0, BH - 1) by1 = np.clip(np.round(by.max()).astype(int) - 1, 0, BH - 1) #print('Brick', bx0,bx1,by0,by1) band = ccd.filter[0] assert (band in bands) ccdzpt = ccd.ccdzpt + 2.5 * np.log10(ccd.exptime) psf_sigma = ccd.fwhm / 2.35 psfnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma) orig_zpscale = zpscale = NanoMaggies.zeropointToScale(ccdzpt) sig1 = ccd.sig1 / orig_zpscale detsig1 = sig1 / psfnorm # print('Image', ccd.expnum, ccd.ccdname, ccd.filter, # 'PSF depth', -2.5 * (np.log10(5.*detsig1) - 9), 'exptime', ccd.exptime, # 'sig1', ccd.sig1, 'zpt', ccd.ccdzpt, 'fwhm', ccd.fwhm, # 'filename', ccd.image_filename.strip()) depths[band][by0:by1 + 1, bx0:bx1 + 1] += (1. / detsig1**2) npix[band] += (y1 + 1 - y0) * (x1 + 1 - x0) nexps[band] += 1 for band in bands: det = np.median(depths[band]) # compute stats for 5-sigma detection with np.errstate(divide='ignore'): depth = 5. / np.sqrt(det) # that's flux in nanomaggies -- convert to mag depth = -2.5 * (np.log10(depth) - 9) if not np.isfinite(depth): depth = 0. depths[band] = depth #bricks.get('psfdepth_' + band)[ibrick] = depth print(brick.brickname, 'median PSF depth', band, ':', depth, 'npix', npix[band], 'nexp', nexps[band]) #'npix', bricks.get('npix_'+band)[ibrick], #'nexp', bricks.get('nexp_'+band)[ibrick]) return (npix, nexps, depths)
def rbmain(): from legacypipe.catalog import read_fits_catalog from legacypipe.survey import LegacySurveyData, wcs_for_brick from tractor.galaxy import DevGalaxy from tractor import PointSource, Catalog from tractor import GaussianMixturePSF from legacypipe.survey import BrickDuck from legacypipe.forced_photom import main as forced_main from astrometry.util.file import trymakedirs import shutil ceres = 'ceres' in sys.argv psfex = 'psfex' in sys.argv for v in [ 'UNWISE_COADDS_TIMERESOLVED_DIR', 'SKY_TEMPLATE_DIR', 'LARGEGALAXIES_CAT', 'GAIA_CAT_DIR', 'TYCHO2_KD_DIR' ]: if v in os.environ: del os.environ[v] oldargs = sys.argv sys.argv = [sys.argv[0]] main() sys.argv = oldargs # Test create_kdtree and (reading CCD kd-tree)! indir = os.path.join(os.path.dirname(__file__), 'testcase6') with tempfile.TemporaryDirectory() as surveydir: files = [ 'calib', 'gaia', 'images', 'survey-bricks.fits.gz', 'tycho2.kd.fits' ] for fn in files: src = os.path.join(indir, fn) dst = os.path.join(surveydir, fn) #trymakedirs(dst, dir=True) print('Copy', src, dst) if os.path.isfile(src): shutil.copy(src, dst) else: shutil.copytree(src, dst) from legacypipe.create_kdtrees import create_kdtree infn = os.path.join(indir, 'survey-ccds-1.fits.gz') outfn = os.path.join(surveydir, 'survey-ccds-1.kd.fits') create_kdtree(infn, outfn, False) os.environ['TYCHO2_KD_DIR'] = surveydir outdir = 'out-testcase6-kd' main(args=[ '--brick', '1102p240', '--zoom', '500', '600', '650', '750', '--force-all', '--no-write', '--no-wise', '--no-gaia', '--survey-dir', surveydir, '--outdir', outdir ]) fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits') assert (os.path.exists(fn)) T = fits_table(fn) assert (len(T) == 2) # Since there is a Tycho-2 star in the blob, forced to be PSF. assert (T.type[0].strip() == 'PSF') assert (T.type[1].strip() == 'PSF') # There is a Tycho-2 star in the blob. I = np.flatnonzero(T.ref_cat == 'T2') assert (len(I) == 1) assert (T.ref_id[I][0] == 1909016711) cat = read_fits_catalog(T) assert (len(cat) == 2) assert (isinstance(cat[0], PointSource)) assert (isinstance(cat[1], PointSource)) cat, ivs = read_fits_catalog(T, invvars=True) assert (len(cat) == 2) assert (isinstance(cat[0], PointSource)) assert (isinstance(cat[1], PointSource)) cat2 = Catalog(*cat) assert (len(ivs) == len(cat2.getParams())) # test --fit-on-coadds outdir = 'out-testcase6-coadds' main(args=[ '--brick', '1102p240', '--zoom', '500', '600', '650', '750', '--force-all', '--no-write', '--no-wise', '--no-gaia', '--survey-dir', surveydir, '--fit-on-coadds', '--outdir', outdir ]) fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits') assert (os.path.exists(fn)) T = fits_table(fn) assert (len(T) == 2) # Since there is a Tycho-2 star in the blob, forced to be PSF. assert (T.type[0].strip() == 'PSF') assert (T.type[1].strip() == 'PSF') # There is a Tycho-2 star in the blob. I = np.flatnonzero(T.ref_cat == 'T2') assert (len(I) == 1) assert (T.ref_id[I][0] == 1909016711) del os.environ['TYCHO2_KD_DIR'] # test --skip-coadds r = main(args=[ '--brick', '1102p240', '--zoom', '500', '600', '650', '750', '--force-all', '--no-write', '--no-wise', '--no-gaia', '--survey-dir', surveydir, '--outdir', outdir, '--skip-coadd' ]) assert (r == 0) # test --skip r = main(args=[ '--brick', '1102p240', '--zoom', '500', '600', '650', '750', '--force-all', '--no-write', '--no-wise', '--no-gaia', '--survey-dir', surveydir, '--outdir', outdir, '--skip' ]) assert (r == 0) # NothingToDoError (neighbouring brick) r = main(args=[ '--brick', '1102p240', '--zoom', '0', '100', '0', '100', '--force-all', '--no-write', '--no-wise', '--no-gaia', '--survey-dir', surveydir, '--outdir', outdir ]) assert (r == 0) surveydir = os.path.join(os.path.dirname(__file__), 'testcase9') # Test for some get_tractor_image kwargs survey = LegacySurveyData(surveydir) fakebrick = BrickDuck(9.1228, 3.3975, 'quack') wcs = wcs_for_brick(fakebrick, W=100, H=100) ccds = survey.ccds_touching_wcs(wcs) ccd = ccds[0] im = survey.get_image_object(ccd) H, W = wcs.shape targetrd = np.array([ wcs.pixelxy2radec(x, y) for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)] ]) tim = im.get_tractor_image(radecpoly=targetrd) assert (tim.getImage() is not None) assert (tim.getInvError() is not None) assert (tim.dq is not None) tim2 = im.get_tractor_image(radecpoly=targetrd, pixels=False) assert (np.all(tim2.getImage() == 0.)) tim4 = im.get_tractor_image(radecpoly=targetrd, invvar=False) u = np.unique(tim4.inverr) assert (len(u) == 1) u = u[0] target = tim4.zpscale / tim4.sig1 assert (np.abs(u / target - 1.) < 0.001) tim3 = im.get_tractor_image(radecpoly=targetrd, invvar=False, dq=False) assert (not hasattr(tim3, 'dq')) tim5 = im.get_tractor_image(radecpoly=targetrd, gaussPsf=True) print(tim5.getPsf()) assert (isinstance(tim5.getPsf(), GaussianMixturePSF)) surveydir = os.path.join(os.path.dirname(__file__), 'testcase12') os.environ['TYCHO2_KD_DIR'] = surveydir os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia') os.environ['GAIA_CAT_VER'] = '2' os.environ['UNWISE_MODEL_SKY_DIR'] = os.path.join(surveydir, 'images', 'unwise-mod') #python legacypipe/runbrick.py --radec --width 100 --height 100 --outdir dup5b --survey-dir test/testcase12 --force-all --no-wise unwdir = os.path.join(surveydir, 'images', 'unwise') main(args=[ '--radec', '346.684', '12.791', '--width', '100', '--height', '100', '--no-wise-ceres', '--unwise-dir', unwdir, '--survey-dir', surveydir, '--outdir', 'out-testcase12', '--skip-coadd', '--force-all' ]) # --plots for stage_wise_forced main(args=[ '--radec', '346.684', '12.791', '--width', '100', '--height', '100', '--no-wise-ceres', '--unwise-dir', unwdir, '--survey-dir', surveydir, '--outdir', 'out-testcase12', '--stage', 'wise_forced', '--plots' ]) del os.environ['GAIA_CAT_DIR'] del os.environ['GAIA_CAT_VER'] del os.environ['TYCHO2_KD_DIR'] del os.environ['UNWISE_MODEL_SKY_DIR'] M = fitsio.read( 'out-testcase12/coadd/cus/custom-346684p12791/legacysurvey-custom-346684p12791-maskbits.fits.fz' ) # Count masked & unmasked bits (the cluster splits this 100x100 field) from collections import Counter c = Counter(M.ravel()) from legacypipe.bits import MASKBITS assert (c[0] >= 4000) assert (c[MASKBITS['CLUSTER']] >= 4000) surveydir = os.path.join(os.path.dirname(__file__), 'testcase9') os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia') os.environ['GAIA_CAT_VER'] = '2' os.environ['LARGEGALAXIES_CAT'] = os.path.join(surveydir, 'sga-sub.kd.fits') main(args=[ '--radec', '9.1228', '3.3975', '--width', '100', '--height', '100', '--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir', surveydir, '--outdir', 'out-testcase9', '--skip', '--force-all', '--ps', 'tc9-ps.fits', '--ps-t0', str(int(time.time())) ]) # (omit --force-all --no-write... reading from pickles below!) # Test with --apodize main(args=[ '--radec', '9.1228', '3.3975', '--width', '100', '--height', '100', '--old-calibs-ok', '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir, '--outdir', 'out-testcase9-ap', '--apodize' ]) main(args=[ '--radec', '9.1228', '3.3975', '--width', '100', '--height', '100', '--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir', surveydir, '--outdir', 'out-testcase9', '--plots', '--stage', 'halos' ]) main(args=[ '--radec', '9.1228', '3.3975', '--width', '100', '--height', '100', '--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir', surveydir, '--outdir', 'out-testcase9-coadds', '--stage', 'image_coadds', '--blob-image' ]) T = fits_table( 'out-testcase9/tractor/cus/tractor-custom-009122p03397.fits') assert (len(T) == 4) # Gaia star becomes a DUP! assert (np.sum([t == 'DUP' for t in T.type]) == 1) # LSLGA galaxy exists! Igal = np.flatnonzero([r == 'L3' for r in T.ref_cat]) assert (len(Igal) == 1) assert (np.all(T.ref_id[Igal] > 0)) assert (T.type[Igal[0]] == 'SER') # --brick and --zoom rather than --radec --width --height main(args=[ '--survey-dir', surveydir, '--outdir', 'out-testcase9b', '--zoom', '1950', '2050', '340', '440', '--brick', '0091p035', '--force-all' ]) # test forced phot?? shutil.copy('test/testcase9/survey-bricks.fits.gz', 'out-testcase9b') forced_main(args=[ '--survey-dir', surveydir, '--no-ceres', '--catalog-dir', 'out-testcase9b', '372546', 'N26', 'forced1.fits' ]) assert (os.path.exists('forced1.fits')) _ = fits_table('forced1.fits') # ... more tests...! forced_main(args=[ '--survey-dir', surveydir, '--no-ceres', '--catalog-dir', 'out-testcase9b', '--derivs', '--threads', '2', '--apphot', '372547', 'N26', 'forced2.fits' ]) assert (os.path.exists('forced2.fits')) _ = fits_table('forced2.fits') forced_main(args=[ '--survey-dir', surveydir, '--no-ceres', '--catalog-dir', 'out-testcase9b', '--agn', '257266', 'S21', 'forced3.fits' ]) assert (os.path.exists('forced3.fits')) _ = fits_table('forced3.fits') if ceres: forced_main(args=[ '--survey-dir', surveydir, '--catalog-dir', 'out-testcase9b', '--derivs', '--threads', '2', '--apphot', '372546', 'N26', 'forced4.fits' ]) assert (os.path.exists('forced4.fits')) _ = fits_table('forced4.fits') # Test cache_dir with tempfile.TemporaryDirectory() as cachedir, \ tempfile.TemporaryDirectory() as tempsurveydir: files = [] for dirpath, _, filenames in os.walk(surveydir): for fn in filenames: path = os.path.join(dirpath, fn) relpath = os.path.relpath(path, surveydir) files.append(relpath) # cache or no? files.sort() files_cache = files[::2] files_nocache = files[1::2] # Survey-ccds *must* be in nocache. fn = 'survey-ccds-1.kd.fits' if fn in files_cache: files_cache.remove(fn) files_nocache.append(fn) for fn in files_cache: src = os.path.join(surveydir, fn) dst = os.path.join(cachedir, fn) trymakedirs(dst, dir=True) print('Copy', src, dst) shutil.copy(src, dst) for fn in files_nocache: src = os.path.join(surveydir, fn) dst = os.path.join(tempsurveydir, fn) trymakedirs(dst, dir=True) print('Copy', src, dst) shutil.copy(src, dst) main(args=[ '--radec', '9.1228', '3.3975', '--width', '100', '--height', '100', '--no-wise', '--survey-dir', tempsurveydir, '--cache-dir', cachedir, '--outdir', 'out-testcase9cache', '--force-all' ]) del os.environ['GAIA_CAT_DIR'] del os.environ['GAIA_CAT_VER'] del os.environ['LARGEGALAXIES_CAT'] # if ceres: # surveydir = os.path.join(os.path.dirname(__file__), 'testcase3') # main(args=['--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815', # '--no-wise', '--force-all', '--no-write', '--ceres', # '--survey-dir', surveydir, # '--outdir', 'out-testcase3-ceres', # '--no-depth-cut']) # MzLS + BASS data # python legacypipe/runbrick.py --run north --brick 1773p595 --zoom 1300 1500 700 900 --survey-dir dr9-north -s coadds # fitscopy coadd/177/1773p595/legacysurvey-1773p595-ccds.fits"[#row<3 || #row==12]" cx.fits # python legacyanalysis/create_testcase.py cx.fits test/mzlsbass2 1773p595 --survey-dir dr9-north/ --fpack surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass2') os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir2, 'gaia') os.environ['GAIA_CAT_VER'] = '2' main(args=[ '--brick', '1773p595', '--zoom', '1300', '1500', '700', '900', '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir2, '--outdir', 'out-mzlsbass2' ]) T = fits_table('out-mzlsbass2/tractor/177/tractor-1773p595.fits') assert (np.sum(T.ref_cat == 'G2') == 3) assert (np.sum(T.ref_id > 0) == 3) # Test --max-blobsize, --checkpoint, --bail-out outdir = 'out-mzlsbass2b' chk = 'checkpoint-mzb2b.p' if os.path.exists(chk): os.unlink(chk) main(args=[ '--brick', '1773p595', '--zoom', '1300', '1500', '700', '900', '--no-wise', '--force-all', '--stage', 'fitblobs', '--write-stage', 'srcs', '--survey-dir', surveydir2, '--outdir', outdir, '--checkpoint', chk, '--nblobs', '3' ]) # err... --max-blobsize does not result in bailed-out blobs masked, # because it treats large blobs as *completed*... #'--max-blobsize', '3000', outdir = 'out-mzlsbass2c' main(args=[ '--brick', '1773p595', '--zoom', '1300', '1500', '700', '900', '--no-wise', '--force-all', '--survey-dir', surveydir2, '--outdir', outdir, '--bail-out', '--checkpoint', chk, '--no-write' ]) del os.environ['GAIA_CAT_DIR'] del os.environ['GAIA_CAT_VER'] M = fitsio.read( os.path.join(outdir, 'coadd', '177', '1773p595', 'legacysurvey-1773p595-maskbits.fits.fz')) assert (np.sum((M & MASKBITS['BAILOUT']) > 0) >= 1000) # Test RexGalaxy surveydir = os.path.join(os.path.dirname(__file__), 'testcase6') outdir = 'out-testcase6-rex' the_args = [ '--brick', '1102p240', '--zoom', '500', '600', '650', '750', '--force-all', '--no-write', '--no-wise', '--skip-calibs', #'--rex', #'--plots', '--survey-dir', surveydir, '--outdir', outdir ] print('python legacypipe/runbrick.py', ' '.join(the_args)) os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia') os.environ['GAIA_CAT_VER'] = '2' main(args=the_args) fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits') assert (os.path.exists(fn)) T = fits_table(fn) assert (len(T) == 2) print('Types:', T.type) # Since there is a Tycho-2 star in the blob, forced to be PSF. assert (T.type[0].strip() == 'PSF') cmd = ( '(cd %s && sha256sum -c %s)' % (outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum'))) print(cmd) rtn = os.system(cmd) assert (rtn == 0) # Test with a Tycho-2 star in the blob. surveydir = os.path.join(os.path.dirname(__file__), 'testcase6') outdir = 'out-testcase6' main(args=[ '--brick', '1102p240', '--zoom', '500', '600', '650', '750', '--force-all', '--no-write', '--no-wise', '--survey-dir', surveydir, '--outdir', outdir ]) fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits') assert (os.path.exists(fn)) T = fits_table(fn) assert (len(T) == 2) print('Types:', T.type) # Since there is a Tycho-2 star in the blob, forced to be PSF. assert (T.type[0].strip() == 'PSF') del os.environ['GAIA_CAT_DIR'] del os.environ['GAIA_CAT_VER'] # Test that we can run splinesky calib if required... from legacypipe.decam import DecamImage DecamImage.splinesky_boxsize = 128 surveydir = os.path.join(os.path.dirname(__file__), 'testcase4') survey = LegacySurveyData(surveydir) # get brick by id brickid = 473357 brick = survey.get_brick(brickid) assert (brick.brickname == '1867p255') assert (brick.brickid == brickid) outdir = 'out-testcase4' os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia') os.environ['GAIA_CAT_VER'] = '2' fn = os.path.join(surveydir, 'calib', 'sky-single', 'decam', 'CP', 'V4.8.2', 'CP20170315', 'c4d_170316_062107_ooi_z_ls9', 'c4d_170316_062107_ooi_z_ls9-N2-splinesky.fits') if os.path.exists(fn): os.unlink(fn) main(args=[ '--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400', '--force-all', '--no-write', '--coadd-bw', '--unwise-dir', os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir', os.path.join(surveydir, 'images', 'unwise-tr'), '--blob-image', '--no-hybrid-psf', '--survey-dir', surveydir, '--outdir', outdir, '-v', '--no-wise-ceres' ]) print('Checking for calib file', fn) assert (os.path.exists(fn)) # Test with blob-masking when creating sky calib. os.unlink(fn) main(args=[ '--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400', '--force-all', '--no-write', '--coadd-bw', '--blob-mask-dir', surveydir, '--survey-dir', surveydir, '--stage', 'image_coadds', '--outdir', 'out-testcase4b', '--plots' ]) print('Checking for calib file', fn) assert (os.path.exists(fn)) if ceres: main(args=[ '--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400', '--force-all', '--no-write', '--coadd-bw', '--unwise-dir', os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir', os.path.join(surveydir, 'images', 'unwise-tr'), '--survey-dir', surveydir, '--outdir', outdir ]) if psfex: # Check that we can regenerate PsfEx files if necessary. fn = os.path.join(surveydir, 'calib', 'psfex', 'decam', 'CP', 'V4.8.2', 'CP20170315', 'c4d_170316_062107_ooi_z_ls9-psfex.fits') if os.path.exists(fn): os.unlink(fn) main(args=[ '--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400', '--force-all', '--no-write', '--coadd-bw', '--unwise-dir', os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir', os.path.join(surveydir, 'images', 'unwise-tr'), '--blob-image', '--survey-dir', surveydir, '--outdir', outdir, '-v' ]) print('After generating PsfEx calib:') os.system('find %s' % (os.path.join(surveydir, 'calib'))) # Wrap-around, hybrid PSF surveydir = os.path.join(os.path.dirname(__file__), 'testcase8') outdir = 'out-testcase8' os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia') os.environ['GAIA_CAT_VER'] = '2' main(args=[ '--brick', '1209p050', '--zoom', '720', '1095', '3220', '3500', '--force-all', '--no-write', '--no-wise', #'--plots', '--survey-dir', surveydir, '--outdir', outdir ]) # Test with a Tycho-2 star + another saturated star in the blob. surveydir = os.path.join(os.path.dirname(__file__), 'testcase7') outdir = 'out-testcase7' os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia') os.environ['GAIA_CAT_VER'] = '2' main(args=[ '--brick', '1102p240', '--zoom', '250', '350', '1550', '1650', '--force-all', '--no-write', '--no-wise', #'--plots', '--survey-dir', surveydir, '--outdir', outdir ]) del os.environ['GAIA_CAT_DIR'] del os.environ['GAIA_CAT_VER'] fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits') assert (os.path.exists(fn)) T = fits_table(fn) assert (len(T) == 4) # Check skipping blobs outside the brick's unique area. # (this now doesn't detect any sources at all, reasonably) # surveydir = os.path.join(os.path.dirname(__file__), 'testcase5') # outdir = 'out-testcase5' # # fn = os.path.join(outdir, 'tractor', '186', 'tractor-1867p255.fits') # if os.path.exists(fn): # os.unlink(fn) # # main(args=['--brick', '1867p255', '--zoom', '0', '150', '0', '150', # '--force-all', '--no-write', '--coadd-bw', # '--survey-dir', surveydir, # '--early-coadds', # '--outdir', outdir] + extra_args) # # assert(os.path.exists(fn)) # T = fits_table(fn) # assert(len(T) == 1) # Custom RA,Dec; blob ra,dec. surveydir = os.path.join(os.path.dirname(__file__), 'testcase4') outdir = 'out-testcase4b' os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia') # Catalog written with one entry (--blobradec) fn = os.path.join(outdir, 'tractor', 'cus', 'tractor-custom-186743p25461.fits') if os.path.exists(fn): os.unlink(fn) main(args=[ '--radec', '186.743965', '25.461788', '--width', '250', '--height', '250', '--force-all', '--no-write', '--no-wise', '--blobradec', '186.740369', '25.453855', '--survey-dir', surveydir, '--outdir', outdir ]) assert (os.path.exists(fn)) T = fits_table(fn) assert (len(T) == 1) surveydir = os.path.join(os.path.dirname(__file__), 'testcase3') outdir = 'out-testcase3' os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia') os.environ['GAIA_CAT_VER'] = '2' checkpoint_fn = os.path.join(outdir, 'checkpoint.pickle') if os.path.exists(checkpoint_fn): os.unlink(checkpoint_fn) main(args=[ '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815', '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir, '--outdir', outdir, '--checkpoint', checkpoint_fn, '--checkpoint-period', '1', '--threads', '2' ]) # Read catalog into Tractor sources to test read_fits_catalog survey = LegacySurveyData(survey_dir=outdir) fn = survey.find_file('tractor', brick='2447p120') print('Checking', fn) T = fits_table(fn) cat = read_fits_catalog(T) print('Read catalog:', cat) assert (len(cat) == 2) src = cat[1] print('Source0', src) from tractor.sersic import SersicGalaxy assert (type(src) in [DevGalaxy, SersicGalaxy]) assert (np.abs(src.pos.ra - 244.77973) < 0.00001) assert (np.abs(src.pos.dec - 12.07234) < 0.00002) src = cat[0] print('Source1', src) assert (type(src) == PointSource) assert (np.abs(src.pos.ra - 244.77828) < 0.00001) assert (np.abs(src.pos.dec - 12.07250) < 0.00001) # Check that we can run again, using that checkpoint file. main(args=[ '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815', '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir, '--outdir', outdir, '--checkpoint', checkpoint_fn, '--checkpoint-period', '1', '--threads', '2' ]) # Assert...... something? # Test --checkpoint without --threads main(args=[ '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815', '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir, '--outdir', outdir, '--checkpoint', checkpoint_fn, '--checkpoint-period', '1' ]) # From Kaylan's Bootes pre-DR4 run # surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass3') # main(args=['--brick', '2173p350', '--zoom', '100', '200', '100', '200', # '--no-wise', '--force-all', '--no-write', # '--survey-dir', surveydir2, # '--outdir', 'out-mzlsbass3'] + extra_args) # With plots! main(args=[ '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815', '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir, '--outdir', 'out-testcase3', '--plots', '--nblobs', '1' ])