def organize_by_brick(sample_fns,
sbricks,
outdir=None,
seed=None,
prefix=None):
'''
For each sample_fn, split into bricks
get brick sample fn for that brick and sample
write it if does not exist
sample_fn -- sample_seed.fits file assigned to that mpi task
sbricks -- survey bricks table cut to radec region
'''
dr = get_bybrick_dir(outdir=outdir)
for sample_fn in sample_fns:
# Skip if already looped over bricks for this sample
check_done = os.path.join(dr, get_sample_fn(seed=seed, prefix=prefix))
check_done = check_done.replace('.fits', '_done.txt')
if os.path.exists(check_done):
print('check_done exists: %s' % check_done)
continue
#
sample = fits_table(sample_fn)
print('sample min,max ra,dec= %f %f %f %f' % (sample.ra.min(),sample.ra.max(),\
sample.dec.min(),sample.dec.max()))
# Loop over survey bricks
survey = LegacySurveyData()
for sbrick in sbricks:
# Get output fn for this brick and sample
fn = os.path.join(
dr,
get_brick_sample_fn(brickname=sbrick.brickname,
seed=seed,
prefix=prefix))
if os.path.exists(fn):
continue
# Cut sample by brick's bounds
brickinfo = survey.get_brick_by_name(sbrick.brickname)
brickwcs = wcs_for_brick(brickinfo)
ra1, ra2, dec1, dec2 = brickwcs.radec_bounds()
keep= (sample.ra >= ra1)*(sample.ra <= ra2)*\
(sample.dec >= dec1)*(sample.dec <= dec2)
sample2 = sample.copy()
if np.where(keep)[0].size > 0:
sample2.cut(keep)
sample2.writeto(fn)
print('Wrote %s' % fn)
else:
print('WARNING: sample=%s has no ra,dec in brick=%s' %
(sample_fn, sbrick.brickname))
# This sample is done
with open(check_done, 'w') as foo:
foo.write('done')
def getbrickfiles(brickname=None):
survey = LegacySurveyData()
brickinfo = survey.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
ccdinfo = survey.ccds_touching_wcs(brickwcs)
nccd = len(ccdinfo)
calibdir = survey.get_calib_dir()
imagedir = survey.survey_dir
# Construct image file names and the calibration file names.
expnum = ccdinfo.expnum
ccdname = ccdinfo.ccdname
psffiles = list()
skyfiles = list()
imagefiles = list()
for ccd in ccdinfo:
info = survey.get_image_object(ccd)
for attr in ['imgfn', 'dqfn', 'wtfn']:
fn = getattr(info, attr).replace(imagedir+'/', '')
#if '160108_073601' in fn:
# pdb.set_trace()
imagefiles.append(fn)
psffiles.append(info.psffn.replace(calibdir, 'calib'))
skyfiles.append(info.splineskyfn.replace(calibdir, 'calib'))
#for ii in range(nccd):
#exp = '{0:08d}'.format(expnum[ii])
#rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits')
#psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile))
#skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile))
#imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii]))))
#print(np.array(imagefiles))
#print(np.array(psffiles))
#print(np.array(skyfiles))
return imagefiles, psffiles, skyfiles
def getbrickfiles(brickname=None):
survey = LegacySurveyData()
brickinfo = survey.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
ccdinfo = survey.ccds_touching_wcs(brickwcs)
nccd = len(ccdinfo)
calibdir = survey.get_calib_dir()
imagedir = survey.survey_dir
# Construct image file names and the calibration file names.
expnum = ccdinfo.expnum
ccdname = ccdinfo.ccdname
psffiles = list()
skyfiles = list()
imagefiles = list()
for ccd in ccdinfo:
info = survey.get_image_object(ccd)
for attr in ['imgfn', 'dqfn', 'wtfn']:
fn = getattr(info, attr).replace(imagedir + '/', '')
#if '160108_073601' in fn:
# pdb.set_trace()
imagefiles.append(fn)
psffiles.append(info.psffn.replace(calibdir, 'calib'))
skyfiles.append(info.splineskyfn.replace(calibdir, 'calib'))
#for ii in range(nccd):
#exp = '{0:08d}'.format(expnum[ii])
#rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits')
#psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile))
#skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile))
#imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii]))))
#print(np.array(imagefiles))
#print(np.array(psffiles))
#print(np.array(skyfiles))
return imagefiles, psffiles, skyfiles
if task is None:
break
try:
print('Task:', task.task)
brickname = task.task
print('Checking for existing out file')
# shortcut
dirnm = os.path.join('depthcuts', brickname[:3])
outfn = os.path.join(dirnm, 'ccds-%s.fits' % brickname)
if os.path.exists(outfn):
print('Exists:', outfn)
task.set_state(qdo.Task.SUCCEEDED)
continue
print('Getting brick', brickname)
brick = survey.get_brick_by_name(brickname)
print('Got brick, running depth cut')
rtn = run_one_brick((brick, 0, 1, False, kwargs))
if rtn != 0:
allgood = rtn
print('Done, result', rtn)
if rtn == 0:
task.set_state(qdo.Task.SUCCEEDED)
else:
task.set_state(qdo.Task.FAILED, err=1)
except:
import traceback
traceback.print_exc()
task.set_state(qdo.Task.FAILED, err=1)
sys.exit(0)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--plots', action='store_true')
parser.add_argument('--brick', help='Brick name to run')
parser.add_argument(
'--input-dir',
default='/global/projecta/projectdirs/cosmo/work/legacysurvey/dr7')
#/global/cscratch1/sd/desiproc/dr7out')
parser.add_argument('--survey-dir',
default='/global/cscratch1/sd/dstn/dr7-depthcut')
parser.add_argument('--output-dir',
default='/global/cscratch1/sd/dstn/bright')
opt = parser.parse_args()
plots = opt.plots
ps = PlotSequence('bright')
brickname = opt.brick
insurvey = LegacySurveyData(opt.input_dir, cache_dir=opt.survey_dir)
outsurvey = LegacySurveyData(opt.output_dir, output_dir=opt.output_dir)
bfn = insurvey.find_file('blobmap', brick=brickname)
print('Found blob map', bfn)
blobs = fitsio.read(bfn)
h, w = blobs.shape
brick = insurvey.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brick)
radius = np.sqrt(2.) * 0.25 * 1.01
neighbors = insurvey.get_bricks_near(brick.ra, brick.dec, radius)
print(len(neighbors), 'bricks nearby')
def showbool(X):
d = downsample_max(X, 8)
h, w = X.shape
plt.imshow(d,
interpolation='nearest',
origin='lower',
vmin=0,
vmax=1,
extent=[0, w, 0, h],
cmap='gray')
brightblobs = set()
for nb in neighbors:
if nb.brickname == brickname:
# ignore myself!
continue
print('Neighbor:', nb.brickname)
mfn = insurvey.find_file('maskbits', brick=nb.brickname)
if not os.path.exists(mfn):
print('No maskbits file:', mfn)
continue
maskbits = fitsio.read(mfn)
bright = ((maskbits & MASKBITS['BRIGHT']) > 0)
print(np.sum(bright > 0), 'BRIGHT pixels set')
primary = (maskbits & MASKBITS['NPRIMARY'] == 0)
print(np.sum(primary), 'PRIMARY pixels set')
edge = binary_dilation(primary, structure=np.ones((3, 3), bool))
edge = edge * np.logical_not(primary)
brightedge = edge & bright
if plots:
plt.clf()
showbool(bright)
plt.title('bright: brick %s' % nb.brickname)
ps.savefig()
# plt.clf()
# showbool(primary)
# plt.title('PRIMARY, brick %s' % nb.brickname)
# ps.savefig()
#
# plt.clf()
# showbool(edge)
# plt.title('boundary, brick %s' % nb.brickname)
# ps.savefig()
plt.clf()
showbool(brightedge)
plt.title('bright at edge, brick %s' % nb.brickname)
ps.savefig()
nwcs = wcs_for_brick(nb)
yy, xx = np.nonzero(brightedge)
print(len(yy), 'bright edge pixels')
if len(yy) == 0:
continue
rr, dd = nwcs.pixelxy2radec(xx + 1, yy + 1)
print('RA range', rr.min(), rr.max(), 'vs brick', brick.ra1, brick.ra2)
print('Dec range', dd.min(), dd.max(), 'vs brick', brick.dec1,
brick.dec2)
# Find pixels that are within this brick's unique area
I, = np.nonzero((rr >= brick.ra1) * (rr <= brick.ra2) *
(dd >= brick.dec1) * (dd <= brick.dec2))
if plots:
plt.clf()
plt.plot(
[brick.ra1, brick.ra1, brick.ra2, brick.ra2, brick.ra1],
[brick.dec1, brick.dec2, brick.dec2, brick.dec1, brick.dec1],
'b-')
plt.plot(rr, dd, 'k.')
plt.plot(rr[I], dd[I], 'r.')
plt.title('Bright pixels from %s' % nb.brickname)
ps.savefig()
if len(I) == 0:
print('No edge pixels touch')
#plt.plot(br,bd, 'b-')
continue
#print('Edge pixels touch!')
#plt.plot(br,bd, 'r-', zorder=20)
ok, x, y = brickwcs.radec2pixelxy(rr[I], dd[I])
x = np.round(x).astype(int) - 1
y = np.round(y).astype(int) - 1
print('Pixel ranges X', x.min(), x.max(), 'Y', y.min(), y.max())
assert (np.all((x >= 0) * (x < w) * (y >= 0) * (y < h)))
print('Adding blobs:', np.unique(blobs[y, x]))
brightblobs.update(blobs[y, x])
print('Blobs touching bright pixels:', brightblobs)
print()
brightblobs.discard(-1)
if len(brightblobs) == 0:
print('No neighboring bright blobs to update!')
return
print('Updating', len(brightblobs), 'blobs:', brightblobs)
tmap = np.zeros(blobs.max() + 2, bool)
for b in brightblobs:
tmap[b + 1] = True
touching = tmap[blobs + 1]
if plots:
plt.clf()
showbool(touching)
plt.title('Blobs touching bright, brick %s' % brickname)
ps.savefig()
mfn = insurvey.find_file('maskbits', brick=brickname)
maskbits, hdr = fitsio.read(mfn, header=True)
updated = maskbits | (MASKBITS['BRIGHT'] * touching)
if np.all(maskbits == updated):
print('No bits updated! (Bright stars were already masked)')
return
maskbits = updated
if plots:
plt.clf()
showbool((maskbits & MASKBITS['BRIGHT']) > 0)
plt.title('New maskbits map for BRIGHT, brick %s' % brickname)
ps.savefig()
with outsurvey.write_output('maskbits', brick=brickname) as out:
out.fits.write(maskbits, hdr=hdr)
tfn = insurvey.find_file('tractor', brick=brickname)
phdr = fitsio.read_header(tfn, ext=0)
hdr = fitsio.read_header(tfn, ext=1)
T = fits_table(tfn)
print('Read', len(T), 'sources')
print('Bright:', Counter(T.brightstarinblob))
iby = np.clip(np.round(T.by), 0, h - 1).astype(int)
ibx = np.clip(np.round(T.bx), 0, w - 1).astype(int)
if plots:
before = np.flatnonzero(T.brightstarinblob)
T.brightstarinblob |= touching[iby, ibx]
print('Bright:', Counter(T.brightstarinblob))
# yuck -- copy the TUNIT headers from input to output.
units = [
hdr.get('TUNIT%i' % (i + 1), '') for i in range(len(T.get_columns()))
]
if plots:
plt.clf()
showbool((maskbits & MASKBITS['BRIGHT']) > 0)
ax = plt.axis()
after = np.flatnonzero(T.brightstarinblob)
plt.plot(T.bx[before], T.by[before], 'gx')
plt.plot(T.bx[after], T.by[after], 'r.')
plt.axis(ax)
plt.title('sources with brightstarinblob, brick %s' % brickname)
ps.savefig()
with outsurvey.write_output('tractor', brick=brickname) as out:
T.writeto(None,
fits_object=out.fits,
primheader=phdr,
header=hdr,
units=units)
class SimStamps(object):
"""Object for exracting sim cutouts
Args:
ls_dir: LEGACY_SURVEY_DIR, like 'tests/end_to_end/testcase_DR5_grz'
outdir: path to dir containing obiwan,coadd,tractor dirs
"""
def __init__(self, ls_dir=None, outdir=None, savedir=None, jpeg=False):
"""outdir: required
ls_dir: not needed if env var LEGACY_SURVEY_DIR already set
save_dir: where write hdf5 files, outdir if None
"""
self.outdir = outdir
self.jpeg = jpeg
if ls_dir:
os.environ["LEGACY_SURVEY_DIR"] = ls_dir
self.savedir = savedir
if self.savedir is None:
self.savedir = self.outdir
self.survey = LegacySurveyData()
def get_brickwcs(self, brick):
brickinfo = self.survey.get_brick_by_name(brick)
self.brickwcs = wcs_for_brick(brickinfo)
def load_data(self, brick, cat_fn, coadd_dir):
"""loads coadd and catalogue data
Args:
brick:
coadd_dir: path/to/rs0, rs300, rs300_skipid, etc
"""
print('Loading ra,dec from %s' % (os.path.dirname))
self.cat = fits_table(cat_fn)
print('Loading from %s' % coadd_dir)
self.img_fits, self.ivar_fits = {}, {}
for b in self.bands:
self.img_fits[b] = readImage(
os.path.join(coadd_dir,
'legacysurvey-%s-image-%s.fits.fz' % (brick, b)))
self.ivar_fits[b] = readImage(
os.path.join(coadd_dir,
'legacysurvey-%s-invvar-%s.fits.fz' % (brick, b)))
self.img_jpeg = readImage(os.path.join(
coadd_dir, 'legacysurvey-%s-image.jpg' % (brick)),
jpeg=True)
# galsim.Image() so can determine overlap w/cutouts
self.img_gs, self.ivar_gs, self.jpeg_gs = {}, {}, {}
for ib, b in enumerate(self.bands):
self.img_gs[b] = galsim.Image(self.img_fits[b])
self.ivar_gs[b] = galsim.Image(self.ivar_fits[b])
def extract(self, hw=32):
"""For each id,x,y in self.cat, extracts image cutout
Args:
hw: half-width, pixels, (hw*2) x (hw*2) image cutout
"""
for cat in self.cat:
xslc = slice(cat.x - hw, cat.x + hw)
yslc = slice(cat.y - hw, cat.y + hw)
# N x N x Number of bands
test_img = galsim.Image(np.zeros((2 * hw + 1, 2 * hw + 1)))
# y,x because numpy indexing
test_img.setCenter(cat.x, cat.y)
olap = test_img.bounds & self.img_gs[self.bands[0]].bounds
assert (olap.area() > 0)
if olap.numpyShape() == test_img.array.shape:
# Grab from fits image b/c aligned better
_ = self.hdf5_obj.create_dataset(str(cat.id)+'/img',
chunks=True, \
data= np.array([sliceImage(self.img_fits[band],
xslice=xslc,yslice=yslc)
for band in self.bands_str]).T)
_ = self.hdf5_obj.create_dataset(str(cat.id)+'/ivar',
chunks=True, \
data= np.array([sliceImage(self.ivar_fits[band],
xslice=xslc,yslice=yslc)
for band in self.bands_str]).T)
_ = self.hdf5_jpeg.create_dataset(str(cat.id)+'/img',
chunks=True,dtype=np.uint8, \
data= sliceImage(self.img_jpeg,
xslice=xslc,yslice=yslc))
# _ = self.hdf5_obj.create_dataset(str(cat.id)+'/img',
# chunks=True, \
# data= np.array([self.img_fits[band][olap].array
# for band in self.bands_str]).T)
# _ = self.hdf5_obj.create_dataset(str(cat.id)+'/ivar',
# chunks=True, \
# data= np.array([self.ivar_fits[band][olap].array
# for band in self.bands_str]).T)
else:
# On edge
# Note, galsim.Image() cannot be 3D
img = [test_img.copy()] * len(self.bands)
ivar = [test_img.copy()] * len(self.bands)
for i, band in enumerate(self.bands_str):
img[i][olap] += self.img_gs[band][olap]
ivar[i][olap] += self.ivar_gs[band][olap]
_ = self.hdf5_obj_onedge.create_dataset(str(cat.id)+'/img',
chunks=True, \
data= np.array([d.array for d in img]).T)
_ = self.hdf5_obj_onedge.create_dataset(str(cat.id)+'/ivar',
chunks=True, \
data= np.array([d.array for d in ivar]).T)
def run(self, brick, stampSize=64, applyCuts=True, zoom=None):
"""Write the hdf5 image files for all rs/* in this brick
Args:
brick: brickname
stampSize: height and width in pixes of training image
zoom: if legacypipe was run with zoom option
"""
self.get_brickwcs(brick)
# coadd fits images must exist
self.set_paths_to_data(brick)
coadd_fns = glob(os.path.join(self.coadd_dirs[0], '*-image-*.fits.fz'))
if len(coadd_fns) == 0:
raise IOError('no image.fits.fz file here: %s' %
self.coadd_dirs[0])
# set of bands in this brick
self.bands = (pd.Series(coadd_fns).str.replace('.fits.fz',
'').str[-1].values)
assert (self.bands.size > 0)
# One hdf5 file for this brick, like '_rz.hdf5'
self.bands_str = ''.join(sorted(self.bands))
assert (self.bands_str in HDF5_KEYS)
self.set_output_fns(brick, self.bands_str)
if os.path.exists(self.hdf5_fn):
try:
f = h5py.File(self.hdf5_fn, 'r')
f2 = h5py.File(self.hdf5_fn_onedge, 'r')
f3 = h5py.File(self.hdf5_fn_jpeg, 'r')
if (len(f.keys()) == 0) & (len(f2.keys()) == 0):
# remove empty files then make them
for fn in [
self.hdf5_fn, self.hdf5_fn_onedge,
self.hdf5_fn_jpeg
]:
dobash('rm %s' % fn)
else:
# processing done, skip this brick
print('Skipping %s, hdf5 already filled: %s' %
(brick, self.hdf5_fn))
return None
except OSError:
# One of these got messed up, redo it
for fn in [
self.hdf5_fn, self.hdf5_fn_onedge, self.hdf5_fn_jpeg
]:
os.remove(fn)
print('removed ', fn)
self.hdf5_obj = h5py.File(self.hdf5_fn, "w")
self.hdf5_obj_onedge = h5py.File(self.hdf5_fn_onedge, "w")
self.hdf5_jpeg = h5py.File(self.hdf5_fn_jpeg, "w")
# Many rs*/ dirs per brick
for cat_fn, coadd_dir in zip(self.cat_fns, self.coadd_dirs):
if (self.jpeg) & (os.path.basename(coadd_dir) != 'rs0'):
print('jpeg=True, so skipping %s' % coadd_dir)
continue
self.load_data(brick, cat_fn, coadd_dir)
self.set_xyid(zoom=zoom)
if applyCuts:
self.apply_cuts()
self.write_mag_sorted_ids()
self.extract(hw=int(stampSize / 2))
self.hdf5_obj.close()
self.hdf5_obj_onedge.close()
self.hdf5_jpeg.close()
print('Wrote %s' % self.hdf5_fn)
print('Wrote %s' % self.hdf5_fn_onedge)
print('Wrote %s' % self.hdf5_fn_jpeg)
def set_paths_to_data(self, brick):
"""lists of catalogues filenames and coadd dirs"""
search = os.path.join(self.outdir, 'coadd', brick[:3], brick, '*rs*',
'legacysurvey-%s-ccds.fits' % brick)
rs_dirs = glob(search)
rs_dirs = [os.path.basename(os.path.dirname(a)) for a in rs_dirs]
if len(rs_dirs) == 0:
raise IOError('No rs dirs here: %s' % search)
self.cat_fns,self.coadd_dirs= \
zip(*[(os.path.join(self.outdir,'obiwan',
brick[:3],brick,rs_dir,
'simcat-elg-%s.fits' % brick),
os.path.join(self.outdir,'coadd',
brick[:3],brick,rs_dir)
)
for rs_dir in rs_dirs])
def set_output_fns(self, brick, bands_str):
dr = os.path.join(self.savedir, 'hdf5', brick[:3], brick)
# hdf5
self.hdf5_fn = os.path.join(dr, 'img_ivar_%s.hdf5' % bands_str)
self.hdf5_fn_onedge = self.hdf5_fn.replace('.hdf5', '_onedge.hdf5')
self.hdf5_fn_jpeg = self.hdf5_fn.replace('img_ivar_', 'jpeg_')
# table of mag sorted ids
self.sorted_ids_fn = self.hdf5_fn.replace(
os.path.basename(self.hdf5_fn), 'sorted_ids.fits')
try:
dobash('mkdir -p %s' % dr)
except ValueError:
print('hdf5 dir already exists: ', dr)
def set_xyid(self, zoom=None):
_, x, y = self.brickwcs.radec2pixelxy(self.cat.ra, self.cat.dec)
if zoom:
x -= zoom[0]
y -= zoom[2]
self.cat.set('x', x.astype(int))
self.cat.set('y', y.astype(int))
assert ('id' in self.cat.get_columns())
def apply_cuts(self):
len_bef = len(self.cat)
print('After cut, have %d/%d' % (len(self.cat), len_bef))
def write_mag_sorted_ids(self, band='g'):
mag = self.get_mag(band)
inds = np.argsort(
mag) # small to large (mag, so brightest to faintest)
T = fits_table()
T.set('id', self.cat.id)
T.set('mag_' + band, mag)
T = T[inds]
T.writeto(self.sorted_ids_fn)
print('Wrote %s' % self.sorted_ids_fn)
def get_mag(self, band='g'):
return flux2mag(self.cat.get(band + 'flux'))
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
'--dr',
'--drdir',
dest='drdir',
default='/project/projectdirs/cosmo/data/legacysurvey/dr5',
help='Directory containing data release w/ tar-gzipped calibs')
parser.add_argument(
'-b',
'--brick',
help='Brick name to run; required unless --radec is given')
parser.add_argument(
'--radec',
nargs=2,
help='RA,Dec center for a custom location (not a brick)')
parser.add_argument('--pixscale',
type=float,
default=0.262,
help='Pixel scale of the output coadds (arcsec/pixel)')
parser.add_argument('-W',
'--width',
type=int,
default=3600,
help='Target image width, default %(default)i')
parser.add_argument('-H',
'--height',
type=int,
default=3600,
help='Target image height, default %(default)i')
parser.add_argument(
'--zoom',
type=int,
nargs=4,
help='Set target image extent (default "0 3600 0 3600")')
parser.add_argument('--no-psf',
dest='do_psf',
default=True,
action='store_false',
help='Do not extract PsfEx files')
parser.add_argument('--no-sky',
dest='do_sky',
default=True,
action='store_false',
help='Do not extract SplineSky files')
opt = parser.parse_args()
if opt.brick is None and opt.radec is None:
parser.print_help()
return -1
optdict = vars(opt)
drdir = opt.drdir
W = opt.width
H = opt.height
pixscale = opt.pixscale
target_extent = opt.zoom
do_psf = opt.do_psf
do_sky = opt.do_sky
#brickname = '1501p020'
custom = (opt.radec is not None)
#ra,dec = 216.03, 34.86
if custom:
ra, dec = opt.radec #27.30, -10.43
ra = float(ra)
dec = float(dec)
#W,H = 1000,1000
#W,H = 1500,1500
brickname = 'custom_%.3f_%.3f' % (ra, dec)
#do_sky = False
survey = LegacySurveyData()
if custom:
from legacypipe.survey import BrickDuck
# Custom brick; create a fake 'brick' object
brick = BrickDuck(ra, dec, brickname)
else:
brickname = opt.brick
brick = survey.get_brick_by_name(brickname)
# Get WCS object describing brick
targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale)
if target_extent is not None:
(x0, x1, y0, y1) = target_extent
W = x1 - x0
H = y1 - y0
targetwcs = targetwcs.get_subimage(x0, y0, W, H)
# Find CCDs
ccds = survey.ccds_touching_wcs(targetwcs, ccdrad=None)
if ccds is None:
raise NothingToDoError('No CCDs touching brick')
print(len(ccds), 'CCDs touching target WCS')
for ccd in ccds:
im = survey.get_image_object(ccd)
print('CCD', im)
expnum = '%08i' % im.expnum
if do_psf:
if os.path.exists(im.psffn) or os.path.exists(im.merged_psffn):
print('PSF file exists')
else:
print('Need PSF', im.psffn, im.merged_psffn)
tarfn = os.path.join(
drdir, 'calib', im.camera, 'psfex-merged',
'legacysurvey_dr5_calib_decam_psfex-merged_%s.tar.gz' %
expnum[:5])
print(tarfn)
if os.path.exists(tarfn):
outfn = '%s/%s-%s.fits' % (expnum[:5], im.camera, expnum)
cmd = 'cd %s/%s/psfex-merged && tar xvzf %s %s' % (
survey.get_calib_dir(), im.camera, tarfn, outfn)
print(cmd)
os.system(cmd)
if do_sky:
if os.path.exists(im.splineskyfn) or os.path.exists(
im.merged_splineskyfn):
print('Sky file exists')
else:
print('Need sky', im.splineskyfn, im.merged_splineskyfn)
tarfn = os.path.join(
drdir, 'calib', im.camera, 'splinesky-merged',
'legacysurvey_dr5_calib_decam_splinesky-merged_%s.tar.gz' %
expnum[:5])
print(tarfn)
if os.path.exists(tarfn):
outfn = '%s/%s-%s.fits' % (expnum[:5], im.camera, expnum)
cmd = 'cd %s/%s/splinesky-merged && tar xvzf %s %s' % (
survey.get_calib_dir(), im.camera, tarfn, outfn)
print(cmd)
os.system(cmd)
def main(args=None):
"""Main routine which parses the optional inputs."""
# Command line options
parser= get_parser()
args = parser.parse_args(args=args)
# Setup loggers
if args.verbose:
lvl = logging.DEBUG
else:
lvl = logging.INFO
logging.basicConfig(level=lvl, stream=sys.stdout) #,format='%(message)s')
log = logging.getLogger('decals_sim')
# Sort through args
log.info('decals_sim.py args={}'.format(args))
max_nobj=500
max_nchunk=1000
if args.ith_chunk is not None: assert(args.ith_chunk <= max_nchunk-1)
assert(args.nchunk <= max_nchunk)
assert(args.nobj <= max_nobj)
if args.ith_chunk is not None:
assert(args.nchunk == 1) #if choose a chunk, only doing 1 chunk
if args.nobj is None:
parser.print_help()
sys.exit(1)
brickname = args.brick
objtype = args.objtype.upper()
lobjtype = objtype.lower()
for obj in ('LRG', 'LSB', 'QSO'):
if objtype == obj:
log.warning('{} objtype not yet supported!'.format(objtype))
return 0
# Deal with the paths.
if 'DECALS_SIM_DIR' in os.environ:
decals_sim_dir = os.getenv('DECALS_SIM_DIR')
else:
decals_sim_dir = '.'
nobj = args.nobj
nchunk = args.nchunk
rand = np.random.RandomState(args.seed) # determines seed for all chunks
seeds = rand.random_integers(0,2**18, max_nchunk)
log.info('Object type = {}'.format(objtype))
log.info('Number of objects = {}'.format(nobj))
log.info('Number of chunks = {}'.format(nchunk))
# Optionally zoom into a portion of the brick
survey = LegacySurveyData()
brickinfo = survey.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale()
log.info('Brick = {}'.format(brickname))
if args.zoom is not None: # See also runbrick.stage_tims()
(x0, x1, y0, y1) = args.zoom
W = x1 - x0
H = y1 - y0
brickwcs = brickwcs.get_subimage(x0, y0, W, H)
log.info('Zoom (pixel boundaries) = {}'.format(args.zoom))
targetrd = np.array([brickwcs.pixelxy2radec(x, y) for x, y in
[(1,1), (W,1), (W,H), (1,H), (1,1)]])
radec_center = brickwcs.radec_center()
log.info('RA, Dec center = {}'.format(radec_center))
log.info('Brick = {}'.format(brickname))
if args.ith_chunk is not None:
chunk_list= [args.ith_chunk]
else:
chunk_list= range(nchunk)
# Store args in dict for easy func passing
kwargs=dict(seeds=seeds,\
brickname=brickname, \
decals_sim_dir= decals_sim_dir,\
brickwcs= brickwcs, \
objtype=objtype,\
lobjtype=lobjtype,\
nobj=nobj,\
nchunk=nchunk,\
args=args)
# Create simulated catalogues and run Tractor
create_metadata(kwargs=kwargs)
# do chunks
for ith_chunk in chunk_list:
log.info('Working on chunk {:02d}/{:02d}'.format(ith_chunk,kwargs['nchunk']-1))
# Random ra,dec and source properties
create_ith_simcat(ith_chunk, d=kwargs)
# Run tractor
do_one_chunk(d=kwargs)
# Clean up output
do_ith_cleanup(ith_chunk=ith_chunk, d=kwargs)
log.info('All done!')
from legacypipe.survey import LegacySurveyData, wcs_for_brick, SimpleGalaxy
from tractor import NanoMaggies
import numpy as np
import pylab as plt
survey = LegacySurveyData()
brick = survey.get_brick_by_name('0006m062')
print('Brick', brick)
targetwcs = wcs_for_brick(brick)
H, W = targetwcs.shape
targetrd = np.array([
targetwcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
print('Target Ra,Dec:', targetrd)
ccds = survey.ccds_touching_wcs(targetwcs)
#ccds = survey.find_ccds(expnum=239662, ccdname='S18')
ccds = ccds[(ccds.expnum == 239662) * (ccds.ccdname == 'S18')]
assert (len(ccds) == 1)
ccd = ccds[0]
im = survey.get_image_object(ccd)
tim = im.get_tractor_image(pixPsf=True,
hybridPsf=True,
splinesky=True,
radecpoly=targetrd)
fullpsf = tim.psf
th, tw = tim.shape
