def getbrickfiles(brickname=None):
survey = LegacySurveyData()
brickinfo = survey.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
ccdinfo = survey.ccds_touching_wcs(brickwcs)
nccd = len(ccdinfo)
calibdir = survey.get_calib_dir()
imagedir = survey.survey_dir
# Construct image file names and the calibration file names.
expnum = ccdinfo.expnum
ccdname = ccdinfo.ccdname
psffiles = list()
skyfiles = list()
imagefiles = list()
for ccd in ccdinfo:
info = survey.get_image_object(ccd)
for attr in ['imgfn', 'dqfn', 'wtfn']:
fn = getattr(info, attr).replace(imagedir+'/', '')
#if '160108_073601' in fn:
# pdb.set_trace()
imagefiles.append(fn)
psffiles.append(info.psffn.replace(calibdir, 'calib'))
skyfiles.append(info.splineskyfn.replace(calibdir, 'calib'))
#for ii in range(nccd):
#exp = '{0:08d}'.format(expnum[ii])
#rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits')
#psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile))
#skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile))
#imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii]))))
#print(np.array(imagefiles))
#print(np.array(psffiles))
#print(np.array(skyfiles))
return imagefiles, psffiles, skyfiles
def getbrickfiles(brickname=None):
survey = LegacySurveyData()
brickinfo = survey.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
ccdinfo = survey.ccds_touching_wcs(brickwcs)
nccd = len(ccdinfo)
calibdir = survey.get_calib_dir()
imagedir = survey.survey_dir
# Construct image file names and the calibration file names.
expnum = ccdinfo.expnum
ccdname = ccdinfo.ccdname
psffiles = list()
skyfiles = list()
imagefiles = list()
for ccd in ccdinfo:
info = survey.get_image_object(ccd)
for attr in ['imgfn', 'dqfn', 'wtfn']:
fn = getattr(info, attr).replace(imagedir + '/', '')
#if '160108_073601' in fn:
# pdb.set_trace()
imagefiles.append(fn)
psffiles.append(info.psffn.replace(calibdir, 'calib'))
skyfiles.append(info.splineskyfn.replace(calibdir, 'calib'))
#for ii in range(nccd):
#exp = '{0:08d}'.format(expnum[ii])
#rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits')
#psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile))
#skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile))
#imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii]))))
#print(np.array(imagefiles))
#print(np.array(psffiles))
#print(np.array(skyfiles))
return imagefiles, psffiles, skyfiles
#ra,dec = 357.3060, 2.3957
#ccd1 = ccds[(ccds.expnum == 563212) * (ccds.ccdname == 'N17')]
ra, dec = 124.0317, 1.3028
expnum, ccdname = 393203, 'N11'
survey = LegacySurveyData()
W, H = 200, 200
pixscale = 0.262
cd = pixscale / 3600.
targetwcs = Tan(ra, dec, W / 2., H / 2., -cd, 0., 0., cd, float(W), float(H))
rr, dd = targetwcs.pixelxy2radec([1, W, W, 1, 1], [1, 1, H, H, 1])
targetrd = np.vstack((rr, dd)).T
ccds = survey.ccds_touching_wcs(targetwcs)
print(len(ccds), 'CCDs touching WCS')
print('MJDs', ccds.mjd_obs)
ccds.writeto('test-ccds.fits')
ccd1 = ccds[(ccds.expnum == expnum) * (ccds.ccdname == ccdname)]
print('CCD1:', ccd1)
im1 = survey.get_image_object(ccd1[0])
print('Im:', im1)
#wcs = im1.get_wcs()
#x0,x1,y0,y1,slc = im1.get_image_extent(wcs=wcs, radecpoly=targetrd)
tim1 = im1.get_tractor_image(radecpoly=targetrd,
pixPsf=True,
def run_one_brick(X):
brick, ibrick, nbricks, plots, kwargs = X
survey = LegacySurveyData()
print()
print()
print('Brick', (ibrick + 1), 'of', nbricks, ':', brick.brickname)
dirnm = os.path.join('depthcuts', brick.brickname[:3])
outfn = os.path.join(dirnm, 'ccds-%s.fits' % brick.brickname)
if os.path.exists(outfn):
print('Exists:', outfn)
return 0
H, W = 3600, 3600
pixscale = 0.262
bands = ['g', 'r', 'z']
# Get WCS object describing brick
targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale)
targetrd = np.array([
targetwcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
gitver = get_git_version()
ccds = survey.ccds_touching_wcs(targetwcs)
if ccds is None:
print('No CCDs actually touching brick')
return 0
print(len(ccds), 'CCDs actually touching brick')
ccds.cut(np.in1d(ccds.filter, bands))
print('Cut on filter:', len(ccds), 'CCDs remain.')
if 'ccd_cuts' in ccds.get_columns():
norig = len(ccds)
ccds.cut(ccds.ccd_cuts == 0)
print(len(ccds), 'of', norig, 'CCDs pass cuts')
else:
print('No CCD cuts')
if len(ccds) == 0:
print('No CCDs left')
return 0
ps = None
if plots:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('depth-%s' % brick.brickname)
splinesky = True
gaussPsf = False
pixPsf = True
do_calibs = False
normalizePsf = True
get_depth_maps = kwargs.pop('get_depth_maps', False)
try:
D = make_depth_cut(survey,
ccds,
bands,
targetrd,
brick,
W,
H,
pixscale,
plots,
ps,
splinesky,
gaussPsf,
pixPsf,
normalizePsf,
do_calibs,
gitver,
targetwcs,
get_depth_maps=get_depth_maps,
**kwargs)
if get_depth_maps:
keep, overlapping, depthmaps = D
else:
keep, overlapping = D
except:
print('Failed to make_depth_cut():')
import traceback
traceback.print_exc()
return -1
print(np.sum(overlapping), 'CCDs overlap the brick')
print(np.sum(keep), 'CCDs passed depth cut')
ccds.overlapping = overlapping
ccds.passed_depth_cut = keep
if not os.path.exists(dirnm):
try:
os.makedirs(dirnm)
except:
pass
if get_depth_maps:
for band, depthmap in depthmaps:
doutfn = os.path.join(dirnm,
'depth-%s-%s.fits' % (brick.brickname, band))
hdr = fitsio.FITSHDR()
# Plug the WCS header cards into these images
targetwcs.add_to_header(hdr)
hdr.delete('IMAGEW')
hdr.delete('IMAGEH')
hdr.add_record(dict(name='EQUINOX', value=2000.))
hdr.add_record(dict(name='FILTER', value=band))
fitsio.write(doutfn, depthmap, header=hdr)
print('Wrote', doutfn)
tmpfn = os.path.join(os.path.dirname(outfn),
'tmp-' + os.path.basename(outfn))
ccds.writeto(tmpfn)
os.rename(tmpfn, outfn)
print('Wrote', outfn)
return 0
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
'--dr',
'--drdir',
dest='drdir',
default='/project/projectdirs/cosmo/data/legacysurvey/dr5',
help='Directory containing data release w/ tar-gzipped calibs')
parser.add_argument(
'-b',
'--brick',
help='Brick name to run; required unless --radec is given')
parser.add_argument(
'--radec',
nargs=2,
help='RA,Dec center for a custom location (not a brick)')
parser.add_argument('--pixscale',
type=float,
default=0.262,
help='Pixel scale of the output coadds (arcsec/pixel)')
parser.add_argument('-W',
'--width',
type=int,
default=3600,
help='Target image width, default %(default)i')
parser.add_argument('-H',
'--height',
type=int,
default=3600,
help='Target image height, default %(default)i')
parser.add_argument(
'--zoom',
type=int,
nargs=4,
help='Set target image extent (default "0 3600 0 3600")')
parser.add_argument('--no-psf',
dest='do_psf',
default=True,
action='store_false',
help='Do not extract PsfEx files')
parser.add_argument('--no-sky',
dest='do_sky',
default=True,
action='store_false',
help='Do not extract SplineSky files')
opt = parser.parse_args()
if opt.brick is None and opt.radec is None:
parser.print_help()
return -1
optdict = vars(opt)
drdir = opt.drdir
W = opt.width
H = opt.height
pixscale = opt.pixscale
target_extent = opt.zoom
do_psf = opt.do_psf
do_sky = opt.do_sky
#brickname = '1501p020'
custom = (opt.radec is not None)
#ra,dec = 216.03, 34.86
if custom:
ra, dec = opt.radec #27.30, -10.43
ra = float(ra)
dec = float(dec)
#W,H = 1000,1000
#W,H = 1500,1500
brickname = 'custom_%.3f_%.3f' % (ra, dec)
#do_sky = False
survey = LegacySurveyData()
if custom:
from legacypipe.survey import BrickDuck
# Custom brick; create a fake 'brick' object
brick = BrickDuck(ra, dec, brickname)
else:
brickname = opt.brick
brick = survey.get_brick_by_name(brickname)
# Get WCS object describing brick
targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale)
if target_extent is not None:
(x0, x1, y0, y1) = target_extent
W = x1 - x0
H = y1 - y0
targetwcs = targetwcs.get_subimage(x0, y0, W, H)
# Find CCDs
ccds = survey.ccds_touching_wcs(targetwcs, ccdrad=None)
if ccds is None:
raise NothingToDoError('No CCDs touching brick')
print(len(ccds), 'CCDs touching target WCS')
for ccd in ccds:
im = survey.get_image_object(ccd)
print('CCD', im)
expnum = '%08i' % im.expnum
if do_psf:
if os.path.exists(im.psffn) or os.path.exists(im.merged_psffn):
print('PSF file exists')
else:
print('Need PSF', im.psffn, im.merged_psffn)
tarfn = os.path.join(
drdir, 'calib', im.camera, 'psfex-merged',
'legacysurvey_dr5_calib_decam_psfex-merged_%s.tar.gz' %
expnum[:5])
print(tarfn)
if os.path.exists(tarfn):
outfn = '%s/%s-%s.fits' % (expnum[:5], im.camera, expnum)
cmd = 'cd %s/%s/psfex-merged && tar xvzf %s %s' % (
survey.get_calib_dir(), im.camera, tarfn, outfn)
print(cmd)
os.system(cmd)
if do_sky:
if os.path.exists(im.splineskyfn) or os.path.exists(
im.merged_splineskyfn):
print('Sky file exists')
else:
print('Need sky', im.splineskyfn, im.merged_splineskyfn)
tarfn = os.path.join(
drdir, 'calib', im.camera, 'splinesky-merged',
'legacysurvey_dr5_calib_decam_splinesky-merged_%s.tar.gz' %
expnum[:5])
print(tarfn)
if os.path.exists(tarfn):
outfn = '%s/%s-%s.fits' % (expnum[:5], im.camera, expnum)
cmd = 'cd %s/%s/splinesky-merged && tar xvzf %s %s' % (
survey.get_calib_dir(), im.camera, tarfn, outfn)
print(cmd)
os.system(cmd)
def one_brick(X):
(ibrick, brick) = X
bands = ['g', 'r', 'z']
print('Brick', brick.brickname)
wcs = wcs_for_brick(brick, W=94, H=94, pixscale=10.)
BH, BW = wcs.shape
targetrd = np.array([
wcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (BW, 1), (BW, BH), (1, BH), (1, 1)]
])
survey = LegacySurveyData()
C = survey.ccds_touching_wcs(wcs)
if C is None:
print('No CCDs touching brick')
return None
I = np.flatnonzero(C.ccd_cuts == 0)
if len(I) == 0:
print('No good CCDs touching brick')
return None
C.cut(I)
print(len(C), 'CCDs touching brick')
depths = {}
for band in bands:
d = np.zeros((BH, BW), np.float32)
depths[band] = d
npix = dict([(band, 0) for band in bands])
nexps = dict([(band, 0) for band in bands])
# survey.get_approx_wcs(ccd)
for ccd in C:
#im = survey.get_image_object(ccd)
awcs = survey.get_approx_wcs(ccd)
imh, imw = ccd.height, ccd.width
x0, y0 = 0, 0
x1 = x0 + imw
y1 = y0 + imh
imgpoly = [(1, 1), (1, imh), (imw, imh), (imw, 1)]
ok, tx, ty = awcs.radec2pixelxy(targetrd[:-1, 0], targetrd[:-1, 1])
tpoly = list(zip(tx, ty))
clip = clip_polygon(imgpoly, tpoly)
clip = np.array(clip)
if len(clip) == 0:
continue
x0, y0 = np.floor(clip.min(axis=0)).astype(int)
x1, y1 = np.ceil(clip.max(axis=0)).astype(int)
#slc = slice(y0,y1+1), slice(x0,x1+1)
awcs = awcs.get_subimage(x0, y0, x1 - x0, y1 - y0)
ah, aw = awcs.shape
#print('Image', ccd.expnum, ccd.ccdname, ccd.filter, 'overlap', x0,x1, y0,y1, '->', (1+x1-x0),'x',(1+y1-y0))
# Find bbox in brick space
r, d = awcs.pixelxy2radec([1, 1, aw, aw], [1, ah, ah, 1])
ok, bx, by = wcs.radec2pixelxy(r, d)
bx0 = np.clip(np.round(bx.min()).astype(int) - 1, 0, BW - 1)
bx1 = np.clip(np.round(bx.max()).astype(int) - 1, 0, BW - 1)
by0 = np.clip(np.round(by.min()).astype(int) - 1, 0, BH - 1)
by1 = np.clip(np.round(by.max()).astype(int) - 1, 0, BH - 1)
#print('Brick', bx0,bx1,by0,by1)
band = ccd.filter[0]
assert (band in bands)
ccdzpt = ccd.ccdzpt + 2.5 * np.log10(ccd.exptime)
psf_sigma = ccd.fwhm / 2.35
psfnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
orig_zpscale = zpscale = NanoMaggies.zeropointToScale(ccdzpt)
sig1 = ccd.sig1 / orig_zpscale
detsig1 = sig1 / psfnorm
# print('Image', ccd.expnum, ccd.ccdname, ccd.filter,
# 'PSF depth', -2.5 * (np.log10(5.*detsig1) - 9), 'exptime', ccd.exptime,
# 'sig1', ccd.sig1, 'zpt', ccd.ccdzpt, 'fwhm', ccd.fwhm,
# 'filename', ccd.image_filename.strip())
depths[band][by0:by1 + 1, bx0:bx1 + 1] += (1. / detsig1**2)
npix[band] += (y1 + 1 - y0) * (x1 + 1 - x0)
nexps[band] += 1
for band in bands:
det = np.median(depths[band])
# compute stats for 5-sigma detection
with np.errstate(divide='ignore'):
depth = 5. / np.sqrt(det)
# that's flux in nanomaggies -- convert to mag
depth = -2.5 * (np.log10(depth) - 9)
if not np.isfinite(depth):
depth = 0.
depths[band] = depth
#bricks.get('psfdepth_' + band)[ibrick] = depth
print(brick.brickname, 'median PSF depth', band, ':', depth, 'npix',
npix[band], 'nexp', nexps[band])
#'npix', bricks.get('npix_'+band)[ibrick],
#'nexp', bricks.get('nexp_'+band)[ibrick])
return (npix, nexps, depths)
def rbmain():
from legacypipe.catalog import read_fits_catalog
from legacypipe.survey import LegacySurveyData, wcs_for_brick
from tractor.galaxy import DevGalaxy
from tractor import PointSource, Catalog
from tractor import GaussianMixturePSF
from legacypipe.survey import BrickDuck
from legacypipe.forced_photom import main as forced_main
from astrometry.util.file import trymakedirs
import shutil
ceres = 'ceres' in sys.argv
psfex = 'psfex' in sys.argv
for v in [
'UNWISE_COADDS_TIMERESOLVED_DIR', 'SKY_TEMPLATE_DIR',
'LARGEGALAXIES_CAT', 'GAIA_CAT_DIR', 'TYCHO2_KD_DIR'
]:
if v in os.environ:
del os.environ[v]
oldargs = sys.argv
sys.argv = [sys.argv[0]]
main()
sys.argv = oldargs
# Test create_kdtree and (reading CCD kd-tree)!
indir = os.path.join(os.path.dirname(__file__), 'testcase6')
with tempfile.TemporaryDirectory() as surveydir:
files = [
'calib', 'gaia', 'images', 'survey-bricks.fits.gz',
'tycho2.kd.fits'
]
for fn in files:
src = os.path.join(indir, fn)
dst = os.path.join(surveydir, fn)
#trymakedirs(dst, dir=True)
print('Copy', src, dst)
if os.path.isfile(src):
shutil.copy(src, dst)
else:
shutil.copytree(src, dst)
from legacypipe.create_kdtrees import create_kdtree
infn = os.path.join(indir, 'survey-ccds-1.fits.gz')
outfn = os.path.join(surveydir, 'survey-ccds-1.kd.fits')
create_kdtree(infn, outfn, False)
os.environ['TYCHO2_KD_DIR'] = surveydir
outdir = 'out-testcase6-kd'
main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--no-gaia',
'--survey-dir', surveydir, '--outdir', outdir
])
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0].strip() == 'PSF')
assert (T.type[1].strip() == 'PSF')
# There is a Tycho-2 star in the blob.
I = np.flatnonzero(T.ref_cat == 'T2')
assert (len(I) == 1)
assert (T.ref_id[I][0] == 1909016711)
cat = read_fits_catalog(T)
assert (len(cat) == 2)
assert (isinstance(cat[0], PointSource))
assert (isinstance(cat[1], PointSource))
cat, ivs = read_fits_catalog(T, invvars=True)
assert (len(cat) == 2)
assert (isinstance(cat[0], PointSource))
assert (isinstance(cat[1], PointSource))
cat2 = Catalog(*cat)
assert (len(ivs) == len(cat2.getParams()))
# test --fit-on-coadds
outdir = 'out-testcase6-coadds'
main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--no-gaia',
'--survey-dir', surveydir, '--fit-on-coadds', '--outdir', outdir
])
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0].strip() == 'PSF')
assert (T.type[1].strip() == 'PSF')
# There is a Tycho-2 star in the blob.
I = np.flatnonzero(T.ref_cat == 'T2')
assert (len(I) == 1)
assert (T.ref_id[I][0] == 1909016711)
del os.environ['TYCHO2_KD_DIR']
# test --skip-coadds
r = main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--no-gaia',
'--survey-dir', surveydir, '--outdir', outdir, '--skip-coadd'
])
assert (r == 0)
# test --skip
r = main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--no-gaia',
'--survey-dir', surveydir, '--outdir', outdir, '--skip'
])
assert (r == 0)
# NothingToDoError (neighbouring brick)
r = main(args=[
'--brick', '1102p240', '--zoom', '0', '100', '0', '100',
'--force-all', '--no-write', '--no-wise', '--no-gaia',
'--survey-dir', surveydir, '--outdir', outdir
])
assert (r == 0)
surveydir = os.path.join(os.path.dirname(__file__), 'testcase9')
# Test for some get_tractor_image kwargs
survey = LegacySurveyData(surveydir)
fakebrick = BrickDuck(9.1228, 3.3975, 'quack')
wcs = wcs_for_brick(fakebrick, W=100, H=100)
ccds = survey.ccds_touching_wcs(wcs)
ccd = ccds[0]
im = survey.get_image_object(ccd)
H, W = wcs.shape
targetrd = np.array([
wcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
tim = im.get_tractor_image(radecpoly=targetrd)
assert (tim.getImage() is not None)
assert (tim.getInvError() is not None)
assert (tim.dq is not None)
tim2 = im.get_tractor_image(radecpoly=targetrd, pixels=False)
assert (np.all(tim2.getImage() == 0.))
tim4 = im.get_tractor_image(radecpoly=targetrd, invvar=False)
u = np.unique(tim4.inverr)
assert (len(u) == 1)
u = u[0]
target = tim4.zpscale / tim4.sig1
assert (np.abs(u / target - 1.) < 0.001)
tim3 = im.get_tractor_image(radecpoly=targetrd, invvar=False, dq=False)
assert (not hasattr(tim3, 'dq'))
tim5 = im.get_tractor_image(radecpoly=targetrd, gaussPsf=True)
print(tim5.getPsf())
assert (isinstance(tim5.getPsf(), GaussianMixturePSF))
surveydir = os.path.join(os.path.dirname(__file__), 'testcase12')
os.environ['TYCHO2_KD_DIR'] = surveydir
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
os.environ['UNWISE_MODEL_SKY_DIR'] = os.path.join(surveydir, 'images',
'unwise-mod')
#python legacypipe/runbrick.py --radec --width 100 --height 100 --outdir dup5b --survey-dir test/testcase12 --force-all --no-wise
unwdir = os.path.join(surveydir, 'images', 'unwise')
main(args=[
'--radec', '346.684', '12.791', '--width', '100', '--height', '100',
'--no-wise-ceres', '--unwise-dir', unwdir, '--survey-dir', surveydir,
'--outdir', 'out-testcase12', '--skip-coadd', '--force-all'
])
# --plots for stage_wise_forced
main(args=[
'--radec', '346.684', '12.791', '--width', '100', '--height', '100',
'--no-wise-ceres', '--unwise-dir', unwdir, '--survey-dir', surveydir,
'--outdir', 'out-testcase12', '--stage', 'wise_forced', '--plots'
])
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
del os.environ['TYCHO2_KD_DIR']
del os.environ['UNWISE_MODEL_SKY_DIR']
M = fitsio.read(
'out-testcase12/coadd/cus/custom-346684p12791/legacysurvey-custom-346684p12791-maskbits.fits.fz'
)
# Count masked & unmasked bits (the cluster splits this 100x100 field)
from collections import Counter
c = Counter(M.ravel())
from legacypipe.bits import MASKBITS
assert (c[0] >= 4000)
assert (c[MASKBITS['CLUSTER']] >= 4000)
surveydir = os.path.join(os.path.dirname(__file__), 'testcase9')
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
os.environ['LARGEGALAXIES_CAT'] = os.path.join(surveydir,
'sga-sub.kd.fits')
main(args=[
'--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
'--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir',
surveydir, '--outdir', 'out-testcase9', '--skip', '--force-all',
'--ps', 'tc9-ps.fits', '--ps-t0',
str(int(time.time()))
])
# (omit --force-all --no-write... reading from pickles below!)
# Test with --apodize
main(args=[
'--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
'--old-calibs-ok', '--no-wise', '--force-all', '--no-write',
'--survey-dir', surveydir, '--outdir', 'out-testcase9-ap', '--apodize'
])
main(args=[
'--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
'--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir',
surveydir, '--outdir', 'out-testcase9', '--plots', '--stage', 'halos'
])
main(args=[
'--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
'--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir',
surveydir, '--outdir', 'out-testcase9-coadds', '--stage',
'image_coadds', '--blob-image'
])
T = fits_table(
'out-testcase9/tractor/cus/tractor-custom-009122p03397.fits')
assert (len(T) == 4)
# Gaia star becomes a DUP!
assert (np.sum([t == 'DUP' for t in T.type]) == 1)
# LSLGA galaxy exists!
Igal = np.flatnonzero([r == 'L3' for r in T.ref_cat])
assert (len(Igal) == 1)
assert (np.all(T.ref_id[Igal] > 0))
assert (T.type[Igal[0]] == 'SER')
# --brick and --zoom rather than --radec --width --height
main(args=[
'--survey-dir', surveydir, '--outdir', 'out-testcase9b', '--zoom',
'1950', '2050', '340', '440', '--brick', '0091p035', '--force-all'
])
# test forced phot??
shutil.copy('test/testcase9/survey-bricks.fits.gz', 'out-testcase9b')
forced_main(args=[
'--survey-dir', surveydir, '--no-ceres', '--catalog-dir',
'out-testcase9b', '372546', 'N26', 'forced1.fits'
])
assert (os.path.exists('forced1.fits'))
_ = fits_table('forced1.fits')
# ... more tests...!
forced_main(args=[
'--survey-dir', surveydir, '--no-ceres', '--catalog-dir',
'out-testcase9b', '--derivs', '--threads', '2', '--apphot', '372547',
'N26', 'forced2.fits'
])
assert (os.path.exists('forced2.fits'))
_ = fits_table('forced2.fits')
forced_main(args=[
'--survey-dir', surveydir, '--no-ceres', '--catalog-dir',
'out-testcase9b', '--agn', '257266', 'S21', 'forced3.fits'
])
assert (os.path.exists('forced3.fits'))
_ = fits_table('forced3.fits')
if ceres:
forced_main(args=[
'--survey-dir', surveydir, '--catalog-dir', 'out-testcase9b',
'--derivs', '--threads', '2', '--apphot', '372546', 'N26',
'forced4.fits'
])
assert (os.path.exists('forced4.fits'))
_ = fits_table('forced4.fits')
# Test cache_dir
with tempfile.TemporaryDirectory() as cachedir, \
tempfile.TemporaryDirectory() as tempsurveydir:
files = []
for dirpath, _, filenames in os.walk(surveydir):
for fn in filenames:
path = os.path.join(dirpath, fn)
relpath = os.path.relpath(path, surveydir)
files.append(relpath)
# cache or no?
files.sort()
files_cache = files[::2]
files_nocache = files[1::2]
# Survey-ccds *must* be in nocache.
fn = 'survey-ccds-1.kd.fits'
if fn in files_cache:
files_cache.remove(fn)
files_nocache.append(fn)
for fn in files_cache:
src = os.path.join(surveydir, fn)
dst = os.path.join(cachedir, fn)
trymakedirs(dst, dir=True)
print('Copy', src, dst)
shutil.copy(src, dst)
for fn in files_nocache:
src = os.path.join(surveydir, fn)
dst = os.path.join(tempsurveydir, fn)
trymakedirs(dst, dir=True)
print('Copy', src, dst)
shutil.copy(src, dst)
main(args=[
'--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
'--no-wise', '--survey-dir', tempsurveydir, '--cache-dir',
cachedir, '--outdir', 'out-testcase9cache', '--force-all'
])
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
del os.environ['LARGEGALAXIES_CAT']
# if ceres:
# surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
# main(args=['--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
# '--no-wise', '--force-all', '--no-write', '--ceres',
# '--survey-dir', surveydir,
# '--outdir', 'out-testcase3-ceres',
# '--no-depth-cut'])
# MzLS + BASS data
# python legacypipe/runbrick.py --run north --brick 1773p595 --zoom 1300 1500 700 900 --survey-dir dr9-north -s coadds
# fitscopy coadd/177/1773p595/legacysurvey-1773p595-ccds.fits"[#row<3 || #row==12]" cx.fits
# python legacyanalysis/create_testcase.py cx.fits test/mzlsbass2 1773p595 --survey-dir dr9-north/ --fpack
surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass2')
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir2, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
main(args=[
'--brick', '1773p595', '--zoom', '1300', '1500', '700', '900',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir2,
'--outdir', 'out-mzlsbass2'
])
T = fits_table('out-mzlsbass2/tractor/177/tractor-1773p595.fits')
assert (np.sum(T.ref_cat == 'G2') == 3)
assert (np.sum(T.ref_id > 0) == 3)
# Test --max-blobsize, --checkpoint, --bail-out
outdir = 'out-mzlsbass2b'
chk = 'checkpoint-mzb2b.p'
if os.path.exists(chk):
os.unlink(chk)
main(args=[
'--brick', '1773p595', '--zoom', '1300', '1500', '700', '900',
'--no-wise', '--force-all', '--stage', 'fitblobs', '--write-stage',
'srcs', '--survey-dir', surveydir2, '--outdir', outdir, '--checkpoint',
chk, '--nblobs', '3'
])
# err... --max-blobsize does not result in bailed-out blobs masked,
# because it treats large blobs as *completed*...
#'--max-blobsize', '3000',
outdir = 'out-mzlsbass2c'
main(args=[
'--brick', '1773p595', '--zoom', '1300', '1500', '700', '900',
'--no-wise', '--force-all', '--survey-dir', surveydir2, '--outdir',
outdir, '--bail-out', '--checkpoint', chk, '--no-write'
])
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
M = fitsio.read(
os.path.join(outdir, 'coadd', '177', '1773p595',
'legacysurvey-1773p595-maskbits.fits.fz'))
assert (np.sum((M & MASKBITS['BAILOUT']) > 0) >= 1000)
# Test RexGalaxy
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6-rex'
the_args = [
'--brick',
'1102p240',
'--zoom',
'500',
'600',
'650',
'750',
'--force-all',
'--no-write',
'--no-wise',
'--skip-calibs',
#'--rex', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
]
print('python legacypipe/runbrick.py', ' '.join(the_args))
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
main(args=the_args)
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0].strip() == 'PSF')
cmd = (
'(cd %s && sha256sum -c %s)' %
(outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum')))
print(cmd)
rtn = os.system(cmd)
assert (rtn == 0)
# Test with a Tycho-2 star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6'
main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--survey-dir', surveydir,
'--outdir', outdir
])
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0].strip() == 'PSF')
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
# Test that we can run splinesky calib if required...
from legacypipe.decam import DecamImage
DecamImage.splinesky_boxsize = 128
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
survey = LegacySurveyData(surveydir)
# get brick by id
brickid = 473357
brick = survey.get_brick(brickid)
assert (brick.brickname == '1867p255')
assert (brick.brickid == brickid)
outdir = 'out-testcase4'
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
fn = os.path.join(surveydir, 'calib', 'sky-single', 'decam', 'CP',
'V4.8.2', 'CP20170315', 'c4d_170316_062107_ooi_z_ls9',
'c4d_170316_062107_ooi_z_ls9-N2-splinesky.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
os.path.join(surveydir, 'images',
'unwise-tr'), '--blob-image', '--no-hybrid-psf',
'--survey-dir', surveydir, '--outdir', outdir, '-v', '--no-wise-ceres'
])
print('Checking for calib file', fn)
assert (os.path.exists(fn))
# Test with blob-masking when creating sky calib.
os.unlink(fn)
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--blob-mask-dir',
surveydir, '--survey-dir', surveydir, '--stage', 'image_coadds',
'--outdir', 'out-testcase4b', '--plots'
])
print('Checking for calib file', fn)
assert (os.path.exists(fn))
if ceres:
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
os.path.join(surveydir, 'images', 'unwise-tr'), '--survey-dir',
surveydir, '--outdir', outdir
])
if psfex:
# Check that we can regenerate PsfEx files if necessary.
fn = os.path.join(surveydir, 'calib', 'psfex', 'decam', 'CP', 'V4.8.2',
'CP20170315',
'c4d_170316_062107_ooi_z_ls9-psfex.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
os.path.join(surveydir, 'images', 'unwise-tr'), '--blob-image',
'--survey-dir', surveydir, '--outdir', outdir, '-v'
])
print('After generating PsfEx calib:')
os.system('find %s' % (os.path.join(surveydir, 'calib')))
# Wrap-around, hybrid PSF
surveydir = os.path.join(os.path.dirname(__file__), 'testcase8')
outdir = 'out-testcase8'
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
main(args=[
'--brick',
'1209p050',
'--zoom',
'720',
'1095',
'3220',
'3500',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
])
# Test with a Tycho-2 star + another saturated star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase7')
outdir = 'out-testcase7'
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
main(args=[
'--brick',
'1102p240',
'--zoom',
'250',
'350',
'1550',
'1650',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
])
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 4)
# Check skipping blobs outside the brick's unique area.
# (this now doesn't detect any sources at all, reasonably)
# surveydir = os.path.join(os.path.dirname(__file__), 'testcase5')
# outdir = 'out-testcase5'
#
# fn = os.path.join(outdir, 'tractor', '186', 'tractor-1867p255.fits')
# if os.path.exists(fn):
# os.unlink(fn)
#
# main(args=['--brick', '1867p255', '--zoom', '0', '150', '0', '150',
# '--force-all', '--no-write', '--coadd-bw',
# '--survey-dir', surveydir,
# '--early-coadds',
# '--outdir', outdir] + extra_args)
#
# assert(os.path.exists(fn))
# T = fits_table(fn)
# assert(len(T) == 1)
# Custom RA,Dec; blob ra,dec.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
outdir = 'out-testcase4b'
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
# Catalog written with one entry (--blobradec)
fn = os.path.join(outdir, 'tractor', 'cus',
'tractor-custom-186743p25461.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--radec', '186.743965', '25.461788', '--width', '250', '--height',
'250', '--force-all', '--no-write', '--no-wise', '--blobradec',
'186.740369', '25.453855', '--survey-dir', surveydir, '--outdir',
outdir
])
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 1)
surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
outdir = 'out-testcase3'
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
checkpoint_fn = os.path.join(outdir, 'checkpoint.pickle')
if os.path.exists(checkpoint_fn):
os.unlink(checkpoint_fn)
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
])
# Read catalog into Tractor sources to test read_fits_catalog
survey = LegacySurveyData(survey_dir=outdir)
fn = survey.find_file('tractor', brick='2447p120')
print('Checking', fn)
T = fits_table(fn)
cat = read_fits_catalog(T)
print('Read catalog:', cat)
assert (len(cat) == 2)
src = cat[1]
print('Source0', src)
from tractor.sersic import SersicGalaxy
assert (type(src) in [DevGalaxy, SersicGalaxy])
assert (np.abs(src.pos.ra - 244.77973) < 0.00001)
assert (np.abs(src.pos.dec - 12.07234) < 0.00002)
src = cat[0]
print('Source1', src)
assert (type(src) == PointSource)
assert (np.abs(src.pos.ra - 244.77828) < 0.00001)
assert (np.abs(src.pos.dec - 12.07250) < 0.00001)
# Check that we can run again, using that checkpoint file.
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
])
# Assert...... something?
# Test --checkpoint without --threads
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1'
])
# From Kaylan's Bootes pre-DR4 run
# surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass3')
# main(args=['--brick', '2173p350', '--zoom', '100', '200', '100', '200',
# '--no-wise', '--force-all', '--no-write',
# '--survey-dir', surveydir2,
# '--outdir', 'out-mzlsbass3'] + extra_args)
# With plots!
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', 'out-testcase3', '--plots', '--nblobs', '1'
])
