def main():
reffn = 'tmp/ps1-decals-3.fits'
splitpat = 'tmp/ps1-hp%02i.fits'
if False:
cmd = 'hpsplit %s -o %s -n 2' % (reffn, splitpat)
print cmd
os.system(cmd)
# hpsplit data/decam/sdss-indexes/calibObj-merge-both{,-2}.fits -o data/decam/sdss-indexes/sdss-hp%02i-ns2.fits -n 2
# build-astrometry-index -o data/decam/sdss-indexes/index-sdss-z-hp00-2.fits -P 2 -i data/decam/sdss-indexes/sdss-stars-hp00-ns2.fits -S z_psf -H 0 -s 2 -L 20 -I 1408120 -t data/tmp
cmds = []
scale = 2
for hp in range(48):
indfn = 'tmp/index-ps1-hp%02i-%i.fits' % (hp, scale)
if os.path.exists(indfn):
print 'Exists:', indfn
continue
catfn = splitpat % hp
if not os.path.exists(catfn):
print 'No input catalog:', catfn
continue
nstars = 30
cmd = ('build-astrometry-index -o %s -P %i -i %s -S rmag -H %i -s 2 -L 20 -I 1409260 -t tmp -n %i'
% (indfn, scale, catfn, hp, nstars))
print cmd
cmds.append((cmd,indfn))
mp = multiproc(8)
mp.map(_run_one, cmds)
def main(opt, cs82field): T,F = getTables(cs82field) mp = multiproc(opt.threads) results = [] args = [] alldone = False for r,c,f,r0,r1,d0,d1 in zip(F.run, F.camcol, F.field, F.ra0, F.ra1, F.dec0, F.dec1): dobands = [] for band in 'ugriz': basename = 'cs82-%s-%04i-%i-%04i-%s' % (cs82field, r, c, f, band) outfn = 'mags-%s.fits' % basename if opt.skipExisting and os.path.exists(outfn): print 'File', outfn, 'exists, skipping.' continue dobands.append(band) if len(dobands) == 0: continue margin = 10. / 3600. print 'Cutting to sources in range of image.' Ti = T[(T.ra + margin >= r0) * (T.ra - margin <= r1) * (T.dec + margin >= d0) * (T.dec - margin <= d1)] print len(Ti), 'CS82 sources in range' print 'Creating Tractor sources...' maglim = 24 cat,icat = get_cs82_sources(Ti, maglim=maglim) print 'Got', len(cat), 'sources' realinds = Ti.index[icat] for band in dobands: basename = 'cs82-%s-%04i-%i-%04i-%s' % (cs82field, r, c, f, band) res = mp.apply(runfield, ((r,c,f,band,basename, r0,r1,d0,d1, opt, cat,realinds),)) results.append(res) if opt.nfields and len(results) >= opt.nfields: alldone = True break if alldone: break print len(results), 'async jobs' for r in results: print ' waiting for', r if r is None: continue r.wait() print 'Done!'
def main():
decals = FakeDecals()
# Our list of fake sources to insert into images...
decals.fake_sources = [PointSource(RaDecPos(0., 0.),
NanoMaggies(g=1000, r=1000, z=1000))]
# Now, we can either call run_brick()....
run_brick(None, decals=decals, radec=(0.,0.), width=100, height=100,
stages=['image_coadds'])
# ... or the individual stages in the pipeline:
if False:
mp = multiproc()
kwa = dict(W = 100, H=100, ra=0., dec=0., mp=mp, brickname='fakey')
kwa.update(decals=decals)
R = stage_tims(**kwa)
kwa.update(R)
R = stage_image_coadds(**kwa)
def _run_one((dataset, band, rlo, rhi, dlo, dhi, T, sfn)):
opt = myopts()
opt.wisedatadirs = wisedatadirs
opt.minflux = None
opt.sources = sfn
opt.nonsdss = True
opt.wsources = os.path.join(datadir, 'wise-objs-%s.fits' % dataset)
opt.osources = None
opt.minsb = 0.005
opt.ptsrc = False
opt.pixpsf = False
opt.force = []
#opt.force = [104]
opt.write = True
opt.ri = None
opt.di = None
opt.bandnum = band
opt.name = '%s-w%i' % (dataset, band)
opt.picklepat = os.path.join(datadir, opt.name + '-stage%0i.pickle')
# Plots?
opt.ps = opt.name
#opt.ps = None
mp = multiproc()
try:
#runtostage(110, opt, mp, rlo,rhi,dlo,dhi)
runtostage(108, opt, mp, rlo,rhi,dlo,dhi, indexfn=indexfn)
#runtostage(700, opt, mp, rlo,rhi,dlo,dhi)
except:
import traceback
print
traceback.print_exc()
print
return None
def main():
F = fits_table('window_flist-cut.fits')
print len(F), 'SDSS fields'
reffn = 'data/decam/sdss-indexes/calibObj-merge-both-2.fits'
if not os.path.exists(reffn):
TT = []
bright = photo_flags1_map.get('BRIGHT')
fns = glob('/clusterfs/riemann/raid006/bosswork/boss/sweeps/eboss.v5b/301/calibObj-??????-?-stargal-primary.fits.gz')
fns.sort()
for fn in fns:
T = fits_table(fn, columns=['ra','dec','psfflux', 'objc_type', 'objc_flags'])
print 'Read', fn,
#T.cut(T.dec < 31.)
T.cut((T.dec > 31.) * (T.dec < 35))
print len(T), 'in Dec range',
if len(T) == 0:
print
continue
# I don't think the BRIGHT cut is necessary -- already done by Tinker
T.cut((T.objc_flags & bright) == 0)
print len(T), 'non-BRIGHT'
T.objc_type = T.objc_type.astype(np.int8)
T.u_psfflux = T.psfflux[:,0]
T.g_psfflux = T.psfflux[:,1]
T.r_psfflux = T.psfflux[:,2]
T.i_psfflux = T.psfflux[:,3]
T.z_psfflux = T.psfflux[:,4]
T.delete_column('psfflux')
T.delete_column('objc_flags')
TT.append(T)
T = merge_tables(TT)
del TT
T.writeto(reffn)
# hpsplit data/decam/sdss-indexes/calibObj-merge-both{,-2}.fits -o data/decam/sdss-indexes/sdss-hp%02i-ns2.fits -n 2
# build-astrometry-index -o data/decam/sdss-indexes/index-sdss-z-hp00-2.fits -P 2 -i data/decam/sdss-indexes/sdss-stars-hp00-ns2.fits -S z_psf -H 0 -s 2 -L 20 -I 1408120 -t data/tmp
cmds = []
#scale = 2
scale = 3
#for band in ['r','z']:
for band in ['r']:
for hp in range(48):
#indfn = 'data/decam/sdss-indexes/index-sdss-%s-hp%02i-%i.fits' % (band, hp, scale)
indfn = 'data/decam/sdss-indexes/index-sdss2-%s-hp%02i-%i.fits' % (band, hp, scale)
if os.path.exists(indfn):
print 'Exists:', indfn
continue
catfn = 'data/decam/sdss-indexes/sdss-hp%02i-ns2.fits' % hp
if not os.path.exists(catfn):
print 'No input catalog:', catfn
continue
nstars = 30
cmd = ('build-astrometry-index -o %s -P %i -i %s -S %s_psfflux -f -H %i -s 2 -L 20 -I 1408130 -t data/tmp -n %i'
% (indfn, scale, catfn, band, hp, nstars))
print cmd
cmds.append((cmd,indfn))
mp = multiproc(8)
mp.map(_run_one, cmds)
ps = PlotSequence('hennawi')
r0,r1 = T.ra.min(), T.ra.max()
d0,d1 = T.dec.min(), T.dec.max()
print 'RA range', r0,r1
print 'Dec range', d0,d1
margin = 0.003
dr = margin / np.cos(np.deg2rad((d0+d1)/2.))
rlo = r0 - dr
rhi = r1 + dr
dlo = d0 - margin
dhi = d1 + margin
#mp = multiproc(8)
mp = multiproc(1)
#lvl = logging.INFO
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
if do_sdss:
sdss_forced_phot(r0,r1,d0,d1, rlo, rhi, dlo, dhi, T)
opt = myopts()
opt.wisedatadirs = wisedatadirs
opt.minflux = None
opt.sources = sfn
opt.nonsdss = True
opt.wsources = wsources
if opt.empty:
# log file found
allfailed = False
break
if not allfailed:
continue
print('All jobs failed for sub', sub.id) #, 'via ssh failure')
failedsubs.append(sub)
failedjobs.append(failedjob)
print('Found total of', len(failedsubs), 'failed Submissions')
if opt.rerun:
from process_submissions import try_dosub, try_dojob
if opt.threads is not None:
mp = multiproc(opt.threads)
args = []
for j in failedjobs:
if j is None:
continue
args.append(j.id) #, j.user_image))
mp.map(bounce_try_dojob, args)
else:
for sub in failedsubs:
print('Re-trying sub', sub.id)
try_dosub(sub, 1)
if opt.sub:
sub = Submission.objects.all().get(id=opt.sub)
opt.errfrac = 0.
opt.blocks = 1
opt.minsig1 = 0.1
opt.minsig2 = 0.1
opt.nonneg = False
opt.pickle2 = False
opt.sky = False
opt.cells = []
opt.bright1 = None
opt.minradius = 2
opt.photoObjsOnly = False
opt.output = 'cosmos-phot-%s.fits'
opt.outdir = '.'
tiledir = 'wise-coadds'
outdir = '.'
tempoutdir = 'phot-temp'
#pobjoutdir = '%s-pobj'
Time.add_measurement(MemMeas)
ps = PlotSequence('cosmos')
mp = multiproc(8)
mp.map(_bounce_one_tile,
[((tile, opt, False, ps, A, tiledir, tempoutdir), dict(T=T))
for tile in A])
#for i,tile in enumerate(A):
# one_tile(tile, opt, False, ps, A, tiledir, tempoutdir, T=T)
def runstage(stage, force=[], threads=1, doplots=True, opt=None):
if threads > 1:
if False:
global dpool
import debugpool
dpool = debugpool.DebugPool(threads)
Time.add_measurement(debugpool.DebugPoolMeas(dpool))
mp = multiproc(pool=dpool)
else:
mp = multiproc(pool=multiprocessing.Pool(threads))
else:
mp = multiproc(threads)
print 'Runstage', stage
pfn = 'tractor%02i.pickle' % stage
if os.path.exists(pfn):
if stage in force:
print 'Ignoring pickle', pfn, 'and forcing stage', stage
else:
print 'Reading pickle', pfn
R = unpickle_from_file(pfn)
return R
if stage > 0:
prereqs = {
100: 0,
200: 0,
}
# Get prereq: from dict, or stage-1
prereq = prereqs.get(stage, stage-1)
P = runstage(prereq, force=force, threads=threads, doplots=doplots, opt=opt)
else:
P = {}
print 'Running stage', stage
F = eval('stage%02i' % stage)
P.update(mp=mp, doplots=doplots)
if 'tractor' in P:
tractor = P['tractor']
tractor.mp = mp
tractor.modtype = np.float32
#P.update(RA = 334.4, DEC = 0.3, sz = 2.*60.) # arcsec
#P.update(RA = 334.4, DEC = 0.3, sz = 15.*60.) # arcsec
# "sz" is the square side length of the ROI, in arcsec
# This is a full SDSS frame (2048 x 2048)
P.update(RA = 334.32, DEC = 0.315, sz = 0.24 * 3600.)
#P.update(RA = 334.32, DEC = 0.315, sz = 0.12 * 3600.)
#plotims = [0,1,2,3, 7,8,9]
# CFHT, W[1234], ugriz
plotims = [0, 1,2,3,4, 5,6,7,8,9]
plotsa = dict(imis=plotims, mp=mp)
P.update(plotsa=plotsa)
P.update(opt=opt)
R = F(**P)
print 'Stage', stage, 'finished'
if 'tractor' in R:
try:
tractor = R['tractor']
tractor.pickleCache = False
except:
pass
print 'Saving pickle', pfn
pickle_to_file(R, pfn)
print 'Saved', pfn
return R
lines = [l.split() for l in lines]
T = fits_table()
types = dict(projcell=np.int16, subcell=np.int16, ra=np.float64, dec=np.float64, mjd=None)
for i,col in enumerate(cols.split()):
tt = types.get(col, str)
if tt is None:
continue
vals = [words[i] for words in lines]
#print('Values for', col, ':', vals)
# HACK -- base-10 parsing for integer subcell
if col == 'subcell':
vals = [int(v, 10) for v in vals]
T.set(col, np.array([tt(v) for v in vals], dtype=tt))
return T
mp = multiproc(8)
TT = []
for skycell in range(635, 2643+1):
fn = 'ps1skycells-%i.fits' % skycell
if os.path.exists(fn):
print('Reading cached', fn)
TT.append(fits_table(fn))
continue
args = []
for subcell in range(100):
args.append((skycell, subcell))
TTi = mp.map(get_cell, args)
Ti = merge_tables(TTi)
subh, subw = tim.shape
tim.subwcs = targetwcs.get_subimage(tim.x0, tim.y0, subw, subh)
tr = tractor.Tractor([tim], [src])
mod = tr.getModelImage(0)
tim.data = tim.data + noise.data + mod.data
tims.append(tim)
print(
'Appended {} image: sky {:.4f} [nanomaggies]; psf fwhm {:.4f}; zpt {:.4f}'
.format(band, sky_level_sig, psf_fwhm, zpt))
##
mp = multiproc()
##
SEDs = sed_matched_filters(bands)
detmaps, detivs, satmaps = detection_maps(tims,
targetwcs,
bands,
mp=mp,
apodize=None)
Tnew, newcat, hot = run_sed_matched_filters(SEDs,
bands,
detmaps,
detivs,
omit_xy=None,
targetwcs=targetwcs,
def main():
import optparse
parser = optparse.OptionParser()
parser.add_option('--zoom', '-z', type=int, action='append', default=[],
help='Add zoom level; default 13')
parser.add_option('--threads', type=int, default=1, help='Number of threads')
parser.add_option('--y0', type=int, default=0, help='Start row')
parser.add_option('--y1', type=int, default=None, help='End row (non-inclusive)')
parser.add_option('--x0', type=int, default=None)
parser.add_option('--x1', type=int, default=None)
parser.add_option('-x', type=int)
parser.add_option('-y', type=int)
parser.add_option('--mindec', type=float, default=None, help='Minimum Dec to run')
parser.add_option('--maxdec', type=float, default=None, help='Maximum Dec to run')
parser.add_option('--minra', type=float, default=0., help='Minimum RA to run')
parser.add_option('--maxra', type=float, default=360., help='Maximum RA to run')
parser.add_option('--near', action='store_true', help='Only run tiles near bricks')
parser.add_option('--near-ccds', action='store_true', help='Only run tiles near CCDs')
parser.add_option('--queue', action='store_true', default=False,
help='Print qdo commands')
parser.add_option('--all', action='store_true', help='Render all tiles')
parser.add_option('--ignore', action='store_true', help='Ignore cached tile files',
default=False)
parser.add_option('--top', action='store_true', help='Top levels of the pyramid')
parser.add_option('--kind', default='image')
parser.add_option('--scale', action='store_true', help='Scale images?')
parser.add_option('--coadd', action='store_true', help='Create SDSS coadd images?')
parser.add_option('--grass', action='store_true', help='progress plot')
opt,args = parser.parse_args()
if len(opt.zoom) == 0:
opt.zoom = [13]
mp = multiproc(opt.threads)
if opt.kind == 'sdss':
if opt.maxdec is None:
opt.maxdec = 90
if opt.mindec is None:
opt.mindec = -25
elif opt.kind in ['halpha', 'unwise-neo1']:
if opt.maxdec is None:
opt.maxdec = 90
if opt.mindec is None:
opt.mindec = -90
else:
if opt.maxdec is None:
opt.maxdec = 40
if opt.mindec is None:
opt.mindec = -20
if opt.top:
top_levels(mp, opt)
sys.exit(0)
from legacypipe.common import Decals
decals = Decals()
if opt.near:
if opt.kind == 'sdss':
B = fits_table(os.path.join(settings.DATA_DIR, 'bricks-sdssco.fits'))
else:
B = decals.get_bricks()
print len(B), 'bricks'
if opt.scale:
opt.near_ccds = True
if opt.near_ccds:
if opt.kind == 'sdss':
C = fits_table(os.path.join(settings.DATA_DIR, 'sdss', 'window_flist.fits'),
columns=['rerun','ra','dec', 'run', 'camcol', 'field', 'score'])
C.cut(C.rerun == '301')
C.cut(C.score >= 0.6)
#C.delete_column('rerun')
# SDSS field size
radius = 1.01 * np.hypot(10., 14.)/2. / 60.
ccdsize = radius
print len(C), 'SDSS fields'
else:
C = decals.get_ccds()
print len(C), 'CCDs'
ccdsize = 0.2
if opt.x is not None:
opt.x0 = opt.x
opt.x1 = opt.x + 1
if opt.y is not None:
opt.y0 = opt.y
opt.y1 = opt.y + 1
if opt.coadd and opt.kind == 'sdss':
from legacypipe.common import wcs_for_brick
from map.views import trymakedirs
B = decals.get_bricks()
print len(B), 'bricks'
B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec))
print len(B), 'in Dec range'
B.cut((B.ra >= opt.minra) * (B.ra < opt.maxra))
print len(B), 'in RA range'
if opt.queue:
# ~ square-degree tiles
# RA slices
rr = np.arange(opt.minra , opt.maxra +1)
dd = np.arange(opt.mindec, opt.maxdec+1)
for rlo,rhi in zip(rr, rr[1:]):
for dlo,dhi in zip(dd, dd[1:]):
print 'time python render-tiles.py --kind sdss --coadd --minra %f --maxra %f --mindec %f --maxdec %f' % (rlo, rhi, dlo, dhi)
sys.exit(0)
if opt.grass:
basedir = settings.DATA_DIR
codir = os.path.join(basedir, 'coadd', 'sdssco')
rr,dd = [],[]
exist = []
for i,b in enumerate(B):
print 'Brick', b.brickname,
fn = os.path.join(codir, b.brickname[:3], 'sdssco-%s-%s.fits' % (b.brickname, 'r'))
print '-->', fn,
if not os.path.exists(fn):
print
continue
print 'found'
rr.append(b.ra)
dd.append(b.dec)
exist.append(i)
exist = np.array(exist)
B.cut(exist)
B.writeto('bricks-sdssco-exist.fits')
import pylab as plt
plt.clf()
plt.plot(rr, dd, 'k.')
plt.title('SDSS coadd tiles')
plt.savefig('sdss.png')
sys.exit(0)
basedir = settings.DATA_DIR
codir = os.path.join(basedir, 'coadd', 'sdssco')
for b in B:
print 'Brick', b.brickname
wcs = wcs_for_brick(b, W=2400, H=2400, pixscale=0.396)
bands = 'gri'
dirnm = os.path.join(codir, b.brickname[:3])
fns = [os.path.join(dirnm, 'sdssco-%s-%s.fits' % (b.brickname, band))
for band in bands]
hdr = fitsio.FITSHDR()
hdr['SURVEY'] = 'SDSS'
wcs.add_to_header(hdr)
if all([os.path.exists(fn) for fn in fns]):
print 'Already exist'
continue
ims = map_sdss(req, 1, 0, 0, 0, get_images=True, wcs=wcs, ignoreCached=True,
forcescale=0)
if ims is None:
print 'No overlap'
continue
trymakedirs(os.path.join(dirnm, 'xxx'))
for fn,band,im in zip(fns,bands, ims):
fitsio.write(fn, im, header=hdr, clobber=True)
print 'Wrote', fn
# Also write scaled versions
dirnm = os.path.join(basedir, 'scaled', 'sdssco')
scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(brickname).3s', 'sdssco-%(brickname)s-%(band)s.fits')
for im,band in zip(ims,bands):
scalekwargs = dict(band=band, brick=b.brickid, brickname=b.brickname)
imwcs = wcs
for scale in range(1, 7):
print 'Writing scale level', scale
im,imwcs,sfn = get_scaled(scalepat, scalekwargs, scale, None,
wcs=imwcs, img=im, return_data=True)
sys.exit(0)
if opt.scale:
if opt.kind == 'sdss':
C.cut((C.dec >= opt.mindec) * (C.dec < opt.maxdec))
print len(C), 'in Dec range'
C.cut((C.ra >= opt.minra) * (C.ra < opt.maxra))
print len(C), 'in RA range'
from astrometry.sdss import AsTransWrapper, DR9
sdss = DR9(basedir=settings.SDSS_DIR)
sdss.saveUnzippedFiles(settings.SDSS_DIR)
sdss.setFitsioReadBZ2()
if settings.SDSS_PHOTOOBJS:
sdss.useLocalTree(photoObjs=settings.SDSS_PHOTOOBJS,
resolve=settings.SDSS_RESOLVE)
basedir = settings.DATA_DIR
scaledir = 'sdss'
dirnm = os.path.join(basedir, 'scaled', scaledir)
for im in C:
from map.views import _read_sip_wcs
#if im.rerun != '301':
# continue
for band in 'gri':
scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(rerun)s', '%(run)i', '%(camcol)i', 'sdss-%(run)i-%(camcol)i-%(field)i-%(band)s.fits')
tmpsuff = '.tmp%08i' % np.random.randint(100000000)
basefn = sdss.retrieve('frame', im.run, im.camcol, field=im.field,
band=band, rerun=im.rerun, tempsuffix=tmpsuff)
fnargs = dict(band=band, rerun=im.rerun, run=im.run,
camcol=im.camcol, field=im.field)
scaled = 1
fn = get_scaled(scalepat, fnargs, scaled, basefn,
read_base_wcs=read_astrans, read_wcs=_read_sip_wcs)
print 'get_scaled:', fn
return 0
else:
assert(False)
for zoom in opt.zoom:
N = 2**zoom
if opt.y1 is None:
y1 = N
else:
y1 = opt.y1
if opt.x0 is None:
opt.x0 = 0
x1 = opt.x1
if x1 is None:
x1 = N
# Find grid of Ra,Dec tile centers and select the ones near DECaLS bricks.
rr,dd = [],[]
yy = np.arange(opt.y0, y1)
xx = np.arange(opt.x0, x1)
if opt.grass:
import pylab as plt
tileexists = np.zeros((len(yy),len(xx)), bool)
basedir = settings.DATA_DIR
ver = tileversions[opt.kind][-1]
tiledir = os.path.join(basedir, 'tiles', opt.kind, '%i'%ver, '%i'%zoom)
for dirpath,dirnames,filenames in os.walk(tiledir):
# change walk order
dirnames.sort()
if len(filenames) == 0:
continue
print 'Dirpath', dirpath
#print 'Dirnames', dirnames
#print 'Filenames', filenames
# check for symlinks
if False:
fns = []
for fn in filenames:
fullfn = os.path.join(tiledir, dirpath, fn)
if os.path.isfile(fullfn) and not os.path.islink(fullfn):
fns.append(fn)
print len(fns), 'of', len(filenames), 'are files (not symlinks)'
filenames = fns
x = os.path.basename(dirpath)
x = int(x)
#print 'x', x
yy = [int(fn.replace('.jpg','')) for fn in filenames]
#print 'yy', yy
print len(yy), 'tiles'
for y in yy:
tileexists[y - opt.y0, x - opt.x0] = True
plt.clf()
plt.imshow(tileexists, interpolation='nearest', origin='upper',
vmin=0, vmax=1, cmap='gray')
fn = 'exist-%s-z%02i' % (opt.kind, zoom)
plt.savefig(fn+'.png')
fitsio.write(fn+'.fits', tileexists, clobber=True)
print 'Wrote', fn+'.png and', fn+'.fits'
continue
if not opt.all:
for y in yy:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, y)
r,d = wcs.get_center()
dd.append(d)
for x in xx:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, x, 0)
r,d = wcs.get_center()
rr.append(r)
dd = np.array(dd)
rr = np.array(rr)
if len(dd) > 1:
tilesize = max(np.abs(np.diff(dd)))
print 'Tile size:', tilesize
else:
if opt.near_ccds or opt.near:
try:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0+1)
r2,d2 = wcs.get_center()
except:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0-1)
r2,d2 = wcs.get_center()
tilesize = np.abs(dd[0] - d2)
print 'Tile size:', tilesize
else:
tilesize = 180.
I = np.flatnonzero((dd >= opt.mindec) * (dd <= opt.maxdec))
print 'Keeping', len(I), 'Dec points between', opt.mindec, 'and', opt.maxdec
dd = dd[I]
yy = yy[I]
I = np.flatnonzero((rr >= opt.minra) * (rr <= opt.maxra))
print 'Keeping', len(I), 'RA points between', opt.minra, 'and', opt.maxra
rr = rr[I]
xx = xx[I]
print len(rr), 'RA points x', len(dd), 'Dec points'
print 'x tile range:', xx.min(), xx.max(), 'y tile range:', yy.min(), yy.max()
for iy,y in enumerate(yy):
print
print 'Y row', y
if opt.near:
d = dd[iy]
I,J,dist = match_radec(rr, d+np.zeros_like(rr), B.ra, B.dec, 0.25 + tilesize, nearest=True)
if len(I) == 0:
print 'No matches to bricks'
continue
keep = np.zeros(len(rr), bool)
keep[I] = True
print 'Keeping', sum(keep), 'tiles in row', y, 'Dec', d
x = xx[keep]
elif opt.near_ccds:
d = dd[iy]
print 'RA range of tiles:', rr.min(), rr.max()
print 'Dec of tile row:', d
I,J,dist = match_radec(rr, d+np.zeros_like(rr), C.ra, C.dec, ccdsize + tilesize, nearest=True)
if len(I) == 0:
print 'No matches to CCDs'
continue
keep = np.zeros(len(rr), bool)
keep[I] = True
print 'Keeping', sum(keep), 'tiles in row', y, 'Dec', d
x = xx[keep]
else:
x = xx
if opt.queue:
cmd = 'python -u render-tiles.py --zoom %i --y0 %i --y1 %i --kind %s' % (zoom, y, y+1, opt.kind)
if opt.near_ccds:
cmd += ' --near-ccds'
if opt.all:
cmd += ' --all'
if opt.ignore:
cmd += ' --ignore'
print cmd
continue
# if opt.grass:
# for xi in x:
# basedir = settings.DATA_DIR
# ver = tileversions[opt.kind][-1]
# tilefn = os.path.join(basedir, 'tiles', opt.kind,
# '%i/%i/%i/%i.jpg' % (ver, zoom, xi, y))
# print 'Checking for', tilefn
# if os.path.exists(tilefn):
# print 'EXISTS'
# tileexists[yi-opt.y0, xi-opt.x0]
# continue
args = []
for xi in x:
args.append((opt.kind,zoom,xi,y, opt.ignore))
print 'Rendering', len(args), 'tiles in row y =', y
mp.map(_bounce_one_tile, args, chunksize=min(100, max(1, len(args)/opt.threads)))
print 'Rendered', len(args), 'tiles'
def main():
cutToPrimary = True
if False:
stars = [
# David's nearby pairs of F stars
(3900., 0., 118.37066, 52.527073),
(3705., 0., 130.17654, 52.750081),
# High stellar density
#(0., 0., 270.0, 0.003),
]
# Dustin's stars
# (4472.001, 0.02514649, 246.47016, 19.066909),
# (5196.53, 0.02490235, 240.09403, 37.404078),
# (6179.05, 0.6324392, 310.47791, 57.523221),
# (6021.875, 0.7000019, 150.52443, -0.478836),
# (7757.096, 0.06507664, 305.11144, -12.957655),
# (8088.685, 0.2436366, 253.11475, 11.60716),
# (8395.096, 0.7563477, 188.34439, 63.442057),
# (9201.74, 178, 93.971719, 0.56302169),
# ]
T = fits_table('stars2.fits')
print 'Read stars:'
T.about()
stars.extend(zip(T.teff, T.teff_sigma, T.ra, T.dec))
# reformat
stars = [(ra, dec, [('T_EFF', teff, 'Effective temperature'),
('DT_EFF', dteff, 'Effective temperate error')],
cutToPrimary) for teff, dteff, ra, dec in stars]
elif False:
sdss = DR9(basedir=tempdir)
sdss.useLocalTree()
# near M87 / Virgo cluster
run, camcol, field = 3836, 2, 258
pofn = sdss.retrieve('photoObj', run, camcol, field)
T = fits_table(
pofn,
columns=[
'parent', 'objid', 'ra', 'dec', 'fracdev', 'objc_type',
'modelflux', 'theta_dev', 'theta_deverr', 'ab_dev',
'ab_deverr', 'phi_dev_deg', 'theta_exp', 'theta_experr',
'ab_exp', 'ab_experr', 'phi_exp_deg', 'resolve_status',
'nchild', 'flags', 'objc_flags', 'run', 'camcol', 'field', 'id'
])
print len(T), 'objects'
T.cut(T.objc_type == 3)
print len(T), 'galaxies'
T.cut(T.nchild == 0)
print len(T), 'children'
T.cut(np.argsort(-T.modelflux[:, 2]))
# keep only one child in each blend family
parents = set()
keepi = []
for i in range(len(T)):
if T.parent[i] in parents:
continue
keepi.append(i)
parents.add(T.parent[i])
T.cut(np.array(keepi))
print len(T), 'unique blend families'
T = T[:25]
stars = [(ra, dec, [], cutToPrimary) for ra, dec in zip(T.ra, T.dec)]
else:
# Objects in the Stripe82 coadd context in CAS:
# select * into mydb.s82 from PhotoPrimary where
# ra between 5 and 6
# and dec between 0 and 1
# and (run = 106 or run = 206) -> s82.fits
T = fits_table('s82.fits')
print 'Read', len(T), 'objects'
T.cut(T.nchild == 0)
print len(T), 'children'
T.cut(T.insidemask == 0)
print len(T), 'not in mask'
#T.cut(np.hypot(T.ra - 5.0562, T.dec - 0.0643) < 0.001)
# http://skyserver.sdss.org/dr12/en/help/browser/browser.aspx#&&history=enum+PhotoFlags+E
for flagname, flagval in [
('BRIGHT', 0x2),
('EDGE', 0x4),
('NODEBLEND', 0x40),
('DEBLEND_TOO_MANY_PEAKS', 0x800),
('NOTCHECKED', 0x80000),
('TOO_LARGE', 0x1000000),
('BINNED2', 0x20000000),
('BINNED4', 0x40000000),
('SATUR_CENTER', 0x80000000000),
('INTERP_CENTER', 0x100000000000),
('MAYBE_CR', 0x100000000000000),
('MAYBE_EGHOST', 0x200000000000000),
]:
T.cut(T.flags & flagval == 0)
print len(T), 'without', flagname, 'bit set'
#pass
# Cut to objects that are likely to appear in the individual images
T.cut(T.psfmag_r < 22.)
print 'Cut to', len(T), 'with psfmag_r < 22 in coadd'
# Select "interesting" objects...
# for i in range(len(T)):
#
# t = T[i]
# ra,dec = t.ra, t.dec
#
# radius = 2. * 0.396 / 3600.
# ddec = radius
# dra = radius / np.cos(np.deg2rad(dec))
# r0,r1 = ra - dra, ra + dra
# d0,d1 = dec - ddec, dec + ddec
#
# wlistfn = sdss.filenames.get('window_flist', 'window_flist.fits')
# RCF = radec_to_sdss_rcf(ra, dec, tablefn=wlistfn)
# print 'Found', len(RCF), 'fields in range.'
# keepRCF = []
# for run,camcol,field,r,d in RCF:
# rr = sdss.get_rerun(run, field)
# print 'Rerun:', rr
# if rr == '157':
# continue
# keepRCF.append((run,camcol,field))
# RCF = keepRCF
# for ifield,(run,camcol,field) in enumerate(RCF):
# objfn = sdss.getPath('photoObj', run, camcol, field)
# objs = fits_table(objfn)
# objs.cut((objs.ra > r0) * (objs.ra < r1) *
# (objs.dec > d0) * (objs.dec < d1))
# bright = photo_flags1_map.get('BRIGHT')
# objs.cut((objs.nchild == 0) * ((objs.objc_flags & bright) == 0))
# Write out Stripe82 measurements...
pixscale = 0.396
pixradius = 25
radius = np.sqrt(2.) * pixradius * pixscale / 3600.
Nkeep = 1000
outdir = 'stamps'
T.tag = np.array(
['%.4f-%.4f.fits' % (r, d) for r, d in zip(T.ra, T.dec)])
T[:Nkeep].writeto(
os.path.join(outdir, 'stamps.fits'),
columns=
'''tag objid run camcol field ra dec psfmag_u psfmag_g psfmag_r
psfmag_i psfmag_z modelmag_u modelmag_g modelmag_r
modelmag_i modelmag_z'''.split())
for i in range(len(T[:Nkeep])):
# t = T[np.array([i])]
# print 't:', t
# t.about()
# assert(len(t) == 1)
I, J, d = match_radec(np.array([T.ra[i]]), np.array([T.dec[i]]),
T.ra, T.dec, radius)
print len(J), 'matched within', radius * 3600., 'arcsec'
t = T[J]
print len(t), 'matched within', radius * 3600., 'arcsec'
tt = fits_table()
cols = [
'ra',
'dec',
'run',
'camcol',
'field', #'probpsf',
#'flags', #'type',
'fracdev_r', #'probpsf_r',
'devrad_r',
'devraderr_r',
'devab_r',
'devaberr_r',
'devphi_r',
'devphierr_r',
'exprad_r',
'expraderr_r',
'expab_r',
'expaberr_r',
'expphi_r',
'expphierr_r',
]
for c in cols:
cout = c
# drop "_r" from dev/exp shapes
if cout.endswith('_r'):
cout = cout[:-2]
coutmap = dict(devrad='theta_dev',
devphi='phi_dev',
devab='ab_dev',
devraderr='theta_dev_err',
devphierr='phi_dev_err',
devaberr='ab_dev_err',
exprad='theta_exp',
expphi='phi_exp',
expab='ab_exp',
expraderr='theta_exp_err',
expphierr='phi_exp_err',
expaberr='ab_exp_err',
fracdev='frac_dev')
cout = coutmap.get(cout, cout)
tt.set(cout, t.get(c))
tt.is_star = (t.type == 6)
for magname in ['psf', 'dev', 'exp']:
for band in 'ugriz':
mag = t.get('%smag_%s' % (magname, band))
magerr = t.get('%smagerr_%s' % (magname, band))
### FIXME -- arcsinh mags??
flux = NanoMaggies.magToNanomaggies(mag)
dflux = np.abs(flux * np.log(10.) / -2.5 * magerr)
tt.set('%sflux_%s' % (magname, band), flux)
tt.set('%sfluxerr_%s' % (magname, band), dflux)
for band in 'ugriz':
# http://www.sdss3.org/dr10/algorithms/magnitudes.php#cmodel
fexp = tt.get('expflux_%s' % band)
fdev = tt.get('expflux_%s' % band)
fracdev = t.get('fracdev_%s' % band)
tt.set('cmodelflux_%s' % band,
fracdev * fdev + (1. - fracdev) * fexp)
catfn = os.path.join(
outdir, 'cat-s82-%.4f-%.4f.fits' % (t.ra[0], t.dec[0]))
tt.writeto(catfn)
print 'Wrote', catfn
cutToPrimary = False
### HACK -- move old files into place.
for ra, dec in zip(T.ra, T.dec)[:Nkeep]:
plotfn = 'stamps-%.4f-%.4f.png' % (ra, dec)
if os.path.exists(plotfn):
fns = [plotfn]
for band in 'ugriz':
stampfn = 'stamp-%s-%.4f-%.4f.fits' % (band, ra, dec)
fns.append(stampfn)
catfn = 'cat-%.4f-%.4f.fits' % (ra, dec)
fns.append(catfn)
for fn in fns:
cmd = 'mv %s %s' % (fn, outdir)
print cmd
os.system(cmd)
stars = [(ra, dec, [], cutToPrimary, outdir)
for ra, dec in zip(T.ra, T.dec)[:Nkeep]]
plots = True
if True:
from astrometry.util.multiproc import *
mp = multiproc(8)
#mp = multiproc()
mp.map(_bounce_one_blob, stars)
else:
# stars = [ (0.,0., 131.59054, 0.66408610),
# (0.,0., 147.34576, 0.51657783 ),
# ]
for args in stars:
ra, dec = stars[:2]
try:
fns = oneblob(*stars)
except:
import traceback
traceback.print_exc()
continue
if plots:
stamp_pattern = 'stamp-%%s-%.4f-%.4f.fits' % (ra, dec)
bands = 'ugriz'
fns = ['cat'] + [stamp_pattern % band for band in bands]
for j, fn in enumerate(fns[1:]):
print 'Filename', fn
F = fitsio.FITS(fn)
n = len(F) / 2
print 'n ext:', n
cols = int(np.ceil(np.sqrt(n)))
rows = int(np.ceil(n / float(cols)))
plt.clf()
for i, ext in enumerate(range(0, len(F), 2)):
plt.subplot(rows, cols, i + 1)
hdr = F[ext].read_header()
dimshow(F[ext].read(), ticks=False)
plt.title('RCF %i/%i/%i' %
(hdr['RUN'], hdr['CAMCOL'], hdr['FIELD']))
plt.suptitle('%s band' % bands[j])
plt.savefig(fn.replace('.fits', '.png'))
F.close()
del F
def main():
import optparse
parser = optparse.OptionParser()
parser.add_option('--zoom', '-z', type=int, action='append', default=[],
help='Add zoom level; default 13')
parser.add_option('--threads', type=int, default=1, help='Number of threads')
parser.add_option('--y0', type=int, default=0, help='Start row')
parser.add_option('--y1', type=int, default=None, help='End row (non-inclusive)')
parser.add_option('--x0', type=int, default=None)
parser.add_option('--x1', type=int, default=None)
parser.add_option('-x', type=int)
parser.add_option('-y', type=int)
parser.add_option('--mindec', type=float, default=None, help='Minimum Dec to run')
parser.add_option('--maxdec', type=float, default=None, help='Maximum Dec to run')
parser.add_option('--minra', type=float, default=None, help='Minimum RA to run')
parser.add_option('--maxra', type=float, default=None, help='Maximum RA to run')
parser.add_option('--near', action='store_true', help='Only run tiles near bricks')
parser.add_option('--near-ccds', action='store_true', help='Only run tiles near CCDs')
parser.add_option('--queue', action='store_true', default=False,
help='Print qdo commands')
parser.add_option('--all', action='store_true', help='Render all tiles')
parser.add_option('--ignore', action='store_true', help='Ignore cached tile files',
default=False)
parser.add_option('--top', action='store_true', help='Top levels of the pyramid')
parser.add_option('--split', action='store_true', help='For split layers (DR6+DR7), only compute one y strip per zoom level')
parser.add_option('--bricks-exist', action='store_true', help='Create table of bricks that exist')
parser.add_option('--kind', default='image')
parser.add_option('--scale', action='store_true', help='Scale images?')
parser.add_option('--bricks', action='store_true', help='Compute scaled brick tables?')
parser.add_option('--coadd', action='store_true', help='Create SDSS coadd images?')
parser.add_option('--grass', action='store_true', help='progress plot')
parser.add_option('--bands', default=None)
parser.add_option('-v', '--verbose', dest='verbose', action='count',
default=0, help='Make more verbose')
opt,args = parser.parse_args()
if opt.verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
mp = multiproc(opt.threads)
if opt.kind in ['sdss', 'sdss2']:
if opt.maxdec is None:
opt.maxdec = 90
if opt.mindec is None:
opt.mindec = -25
elif opt.kind in ['halpha', 'unwise-neo1', 'unwise-neo2', 'unwise-neo3', 'unwise-neo4', 'unwise-cat-model',
'galex', 'wssa', 'vlass', 'hsc']:
if opt.maxdec is None:
opt.maxdec = 90.
if opt.mindec is None:
opt.mindec = -90.
if opt.maxra is None:
opt.maxra = 360.
if opt.minra is None:
opt.minra = 0.
if opt.kind == 'galex' and opt.bands is None:
opt.bands = 'nf'
if 'unwise' in opt.kind and opt.bands is None:
opt.bands = '12'
if 'vlass' in opt.kind:
opt.bands = [1]
elif opt.kind == 'm33':
if opt.mindec is None:
opt.mindec = 30.40
if opt.maxdec is None:
opt.maxdec = 30.90
if opt.minra is None:
opt.minra = 23.29
if opt.maxra is None:
opt.maxra = 23.73
elif opt.kind in ['des-dr1']:
if opt.maxdec is None:
opt.maxdec = 6
if opt.mindec is None:
opt.mindec = -68
if opt.maxra is None:
opt.maxra = 360
if opt.minra is None:
opt.minra = 0
elif opt.kind in ['mzls+bass-dr4', 'mzls+bass-dr4-model', 'mzls+bass-dr4-resid']:
if opt.maxdec is None:
opt.maxdec = 90
if opt.mindec is None:
opt.mindec = 30
if opt.maxra is None:
opt.maxra = 54
if opt.minra is None:
opt.minra = 301
elif opt.kind in ['mzls+bass-dr6', 'mzls+bass-dr6-model', 'mzls+bass-dr6-resid']:
if opt.maxdec is None:
opt.maxdec = 90
if opt.mindec is None:
opt.mindec = -20
if opt.maxra is None:
opt.maxra = 360
if opt.minra is None:
opt.minra = 0
elif opt.kind in ['decaps2', 'decaps2-model', 'decaps2-resid']:
if opt.maxdec is None:
opt.maxdec = -20
if opt.mindec is None:
opt.mindec = -70
if opt.maxra is None:
opt.maxra = 280
if opt.minra is None:
opt.minra = 90
else:
if opt.maxdec is None:
opt.maxdec = 40
if opt.mindec is None:
opt.mindec = -25
if opt.maxra is None:
opt.maxra = 360
if opt.minra is None:
opt.minra = 0
if opt.bands is None:
opt.bands = 'grz'
if opt.top:
top_levels(mp, opt)
sys.exit(0)
if opt.bricks:
from map.views import get_layer
layer = get_layer(opt.kind)
for scale in range(1,8):
B = layer.get_bricks_for_scale(scale)
sys.exit(0)
if opt.scale:
if opt.kind == 'ps1':
from map.views import get_layer
fns = glob('data/ps1/skycells/*/ps1-*.fits')
fns.sort()
#ps1-1561-021-r.fits
if len(opt.zoom) == 0:
opt.zoom = [1,2,3,4,5,6,7]
print(len(fns), 'PS1 image files')
layer = get_layer(opt.kind)
B = layer.get_bricks()
for i,brick in enumerate(B):
for band in opt.bands:
fn0 = layer.get_filename(brick, band, 0)
if not os.path.exists(fn0):
continue
for scale in opt.zoom:
fn = layer.get_filename(brick, band, scale)
layer.create_scaled_image(brick, band, scale, fn)
sys.exit(0)
# Rebricked
if opt.kind in ['decals-dr5', 'decals-dr5-model', 'decals-dr7', 'decals-dr7-model',
'eboss',
'mzls+bass-dr6', 'mzls+bass-dr6-model',
'unwise-neo3', 'unwise-neo4', 'unwise-cat-model',
'galex', 'wssa', 'des-dr1', 'hsc',
] or opt.kind.startswith('dr8-test'): # or True:
from map.views import get_layer
layer = get_layer(opt.kind)
if opt.queue:
if len(opt.zoom) == 0:
opt.zoom = [1,2,3,4,5,6,7]
#step = 0.1
#ras = np.arange(opt.minra, opt.maxra+step, step)
step = 3.
#ras = np.arange(opt.minra, opt.maxra+step, step)
decs = np.arange(opt.mindec, opt.maxdec+step, step)
for zoom in opt.zoom:
for declo,dechi in zip(decs, np.clip(decs[1:], opt.mindec, opt.maxdec)):
rstep = step / np.maximum(0.05, np.cos(np.deg2rad((declo+dechi)/2.)))
ras = np.arange(opt.minra, opt.maxra+rstep, rstep)
for ralo,rahi in zip(ras, np.clip(ras[1:], opt.minra, opt.maxra)):
cmd = ('python render-tiles.py --kind %s --scale --minra %f --maxra %f --mindec %f --maxdec %f -z %i' %
(opt.kind, ralo, rahi, declo, dechi, zoom))
print(cmd)
sys.exit(0)
if len(opt.zoom) == 0:
opt.zoom = [1]
for scale in opt.zoom:
B = layer.get_bricks_for_scale(scale)
print(len(B), 'bricks for scale', scale)
B.cut((B.dec >= opt.mindec) * (B.dec <= opt.maxdec))
print(len(B), 'in Dec range')
B.cut((B.ra >= opt.minra) * (B.ra <= opt.maxra))
print(len(B), 'in RA range')
bands = opt.bands
has = {}
for band in bands:
if 'has_%s' % band in B.get_columns():
has[band] = B.get('has_%s' % band)
else:
# assume yes
has[band] = np.ones(len(B), bool)
# Run one scale at a time
args = []
for ibrick,brick in enumerate(B):
for band in bands:
if has[band][ibrick]:
args.append((layer, brick, band, scale, opt.ignore))
print(len(args), 'bricks for scale', scale)
mp.map(_layer_get_filename, args)
sys.exit(0)
if (opt.kind in ['decals-dr3', 'decals-dr3-model',
'mzls+bass-dr4', 'mzls+bass-dr4-model',
'decaps2', 'decaps2-model', 'eboss', 'ps1']
or 'dr8b' in opt.kind
or 'dr8c' in opt.kind):
from map.views import get_survey, get_layer
surveyname = opt.kind
# *-model -> *
# for prefix in ['decals-dr3', 'mzls+bass-dr4', 'decaps2', 'decals-dr5']:
# if prefix in surveyname:
# surveyname = prefix
for suffix in ['-model', '-resid']:
if surveyname.endswith(suffix):
surveyname = surveyname[:-len(suffix)]
survey = get_survey(surveyname)
print('Survey:', survey)
print(' cache_dir:', survey.cache_dir)
B = survey.get_bricks()
print(len(B), 'bricks')
B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec))
print(len(B), 'in Dec range')
B.cut((B.ra >= opt.minra) * (B.ra < opt.maxra))
print(len(B), 'in RA range')
# find all image files
filetype = 'image'
model = False
if '-model' in opt.kind:
model = True
filetype = 'model'
bands = opt.bands
layer = get_layer(opt.kind)
has = {}
for band in bands:
if 'has_%s' % band in B.get_columns():
has[band] = B.get('has_%s' % band)
else:
# assume yes
has[band] = np.ones(len(B), bool)
for scale in opt.zoom:
args = []
for ibrick,brick in enumerate(B):
for band in bands:
if not has[band][ibrick]:
print('Brick', brick.brickname, 'does not have', band)
continue
args.append((layer, brick, band, scale, opt.ignore))
mp.map(_layer_get_filename, args)
sys.exit(0)
elif opt.kind == 'sdss2':
layer = get_layer(opt.kind)
bands = 'gri'
scales = opt.zoom
if len(scales) == 0:
scales = list(range(1,8))
for scale in scales:
#maxscale = 7
bricks = layer.get_bricks_for_scale(scale)
print('Got', len(bricks), 'bricks for scale', scale)
bricks.cut((bricks.dec > opt.mindec) * (bricks.dec <= opt.maxdec) *
(bricks.ra > opt.minra ) * (bricks.ra <= opt.maxra))
print('Cut to', len(bricks), 'bricks within RA,Dec box')
for ibrick,brick in enumerate(bricks):
for band in bands:
print('Scaling brick', ibrick, 'of', len(bricks), 'scale', scale, 'brick', brick.brickname, 'band', band)
fn = layer.get_filename(brick, band, scale)
sys.exit(0)
if opt.kind in ['unwise-w1w2', 'unwise-neo2']:
# scaledir = opt.kind
# basedir = settings.DATA_DIR
# dirnm = os.path.join(basedir, 'scaled', scaledir)
# B = fits_table(os.path.join(basedir, 'unwise-bricks.fits'))
layer = get_layer(opt.kind)
B = layer.get_bricks()
print(len(B), 'unWISE tiles')
for b in B:
for band in ['1','2']:
for scale in [1,2,3,4,5,6,7]:
print('Get brick', b.brickname, 'band', band, 'scale', scale)
layer.get_filename(b, band, scale)
else:
assert(False)
if opt.bricks_exist:
from map.views import get_survey
surveyname = opt.kind
filetype = 'image'
survey = get_survey(surveyname)
B = survey.get_bricks()
print(len(B), 'bricks')
B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec))
print(len(B), 'in Dec range')
B.cut((B.ra >= opt.minra) * (B.ra < opt.maxra))
print(len(B), 'in RA range')
# find all image files
bands = opt.bands
has_band = {}
for b in bands:
B.set('has_%s' % b, np.zeros(len(B), bool))
has_band[b] = B.get('has_%s' % b)
exists = np.zeros(len(B), bool)
for i,brick in enumerate(B.brickname):
found = False
for band in bands:
fn = survey.find_file(filetype, brick=brick, band=band)
ex = os.path.exists(fn)
print('Brick', brick, 'band', band, 'exists?', ex)
has_band[band][i] = ex
if ex:
found = True
exists[i] = found
B.cut(exists)
B.writeto('bricks-exist-%s.fits' % opt.kind)
sys.exit(0)
from map.views import get_survey
surveyname = opt.kind
if surveyname.endswith('-model'):
surveyname = surveyname.replace('-model','')
if surveyname.endswith('-resid'):
surveyname = surveyname.replace('-resid','')
survey = get_survey(surveyname)
if len(opt.zoom) == 0:
opt.zoom = [13]
if opt.near:
if opt.kind == 'sdss':
B = fits_table(os.path.join(settings.DATA_DIR, 'bricks-sdssco.fits'))
else:
B = survey.get_bricks()
print(len(B), 'bricks')
#if opt.scale:
# opt.near_ccds = True
if opt.near_ccds:
if opt.kind == 'sdss':
C = fits_table(os.path.join(settings.DATA_DIR, 'sdss', 'window_flist.fits'),
columns=['rerun','ra','dec', 'run', 'camcol', 'field', 'score'])
C.cut(C.rerun == '301')
C.cut(C.score >= 0.6)
#C.delete_column('rerun')
# SDSS field size
radius = 1.01 * np.hypot(10., 14.)/2. / 60.
ccdsize = radius
print(len(C), 'SDSS fields')
else:
C = survey.get_ccds()
print(len(C), 'CCDs')
ccdsize = 0.2
if opt.x is not None:
opt.x0 = opt.x
opt.x1 = opt.x + 1
if opt.y is not None:
opt.y0 = opt.y
opt.y1 = opt.y + 1
if opt.coadd and opt.kind == 'galex':
layer = GalexLayer('galex')
# base-level (coadd) bricks
B = layer.get_bricks()
print(len(B), 'bricks')
B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec))
print(len(B), 'in Dec range')
B.cut((B.ra >= opt.minra) * (B.ra < opt.maxra))
print(len(B), 'in RA range')
pat = layer.get_scaled_pattern()
tempfiles = []
for b in B:
for band in ['n','f']:
fn = pat % dict(scale=0, band=band, brickname=b.brickname)
layer.create_coadd_image(b, band, 0, fn, tempfiles=tempfiles)
for fn in tempfiles:
os.unlink(fn)
sys.exit(0)
if opt.coadd and opt.kind == 'sdss':
from legacypipe.survey import wcs_for_brick
from map.views import trymakedirs
#B = survey.get_bricks()
B = fits_table(os.path.join(settings.DATA_DIR, 'sdss2', 'bricks-sdssco.fits'))
print(len(B), 'bricks')
B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec))
print(len(B), 'in Dec range')
B.cut((B.ra >= opt.minra) * (B.ra < opt.maxra))
print(len(B), 'in RA range')
if opt.queue:
# ~ square-degree tiles
# RA slices
rr = np.arange(opt.minra , opt.maxra +1)
dd = np.arange(opt.mindec, opt.maxdec+1)
for rlo,rhi in zip(rr, rr[1:]):
for dlo,dhi in zip(dd, dd[1:]):
print('time python render-tiles.py --kind sdss --coadd --minra %f --maxra %f --mindec %f --maxdec %f' % (rlo, rhi, dlo, dhi))
sys.exit(0)
if opt.grass:
basedir = settings.DATA_DIR
codir = os.path.join(basedir, 'coadd', 'sdssco')
rr,dd = [],[]
exist = []
for i,b in enumerate(B):
print('Brick', b.brickname,)
fn = os.path.join(codir, b.brickname[:3], 'sdssco-%s-%s.fits' % (b.brickname, 'r'))
print('-->', fn,)
if not os.path.exists(fn):
print()
continue
print('found')
rr.append(b.ra)
dd.append(b.dec)
exist.append(i)
exist = np.array(exist)
B.cut(exist)
B.writeto('bricks-sdssco-exist.fits')
import pylab as plt
plt.clf()
plt.plot(rr, dd, 'k.')
plt.title('SDSS coadd tiles')
plt.savefig('sdss.png')
sys.exit(0)
basedir = settings.DATA_DIR
codir = os.path.join(basedir, 'coadd', 'sdssco')
for b in B:
print('Brick', b.brickname)
wcs = wcs_for_brick(b, W=2400, H=2400, pixscale=0.396)
#bands = 'gri'
bands = 'z'
dirnm = os.path.join(codir, b.brickname[:3])
fns = [os.path.join(dirnm, 'sdssco-%s-%s.fits' % (b.brickname, band))
for band in bands]
hdr = fitsio.FITSHDR()
hdr['SURVEY'] = 'SDSS'
wcs.add_to_header(hdr)
if all([os.path.exists(fn) for fn in fns]):
print('Already exist')
continue
from map.oldviews import map_sdss
ims = map_sdss(req, 1, 0, 0, 0, get_images=True, wcs=wcs, ignoreCached=True,
forcescale=0, bands=bands)
if ims is None:
print('No overlap')
continue
trymakedirs(os.path.join(dirnm, 'xxx'))
for fn,band,im in zip(fns,bands, ims):
fitsio.write(fn, im, header=hdr, clobber=True)
print('Wrote', fn)
# Also write scaled versions
# dirnm = os.path.join(basedir, 'scaled', 'sdssco')
# scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(brickname).3s', 'sdssco-%(brickname)s-%(band)s.fits')
# for im,band in zip(ims,bands):
# scalekwargs = dict(band=band, brick=b.brickid, brickname=b.brickname)
# imwcs = wcs
# for scale in range(1, 7):
# print('Writing scale level', scale)
# im,imwcs,sfn = get_scaled(scalepat, scalekwargs, scale, None,
# wcs=imwcs, img=im, return_data=True)
sys.exit(0)
for zoom in opt.zoom:
N = 2**zoom
if opt.y1 is None:
y1 = N
else:
y1 = opt.y1
if opt.x0 is None:
opt.x0 = 0
x1 = opt.x1
if x1 is None:
x1 = N
if opt.split:
decsplit = 32.
y = 2.**zoom/(2.*np.pi) * (np.pi - np.log(np.tan(np.pi/4. + np.deg2rad(decsplit)/2.)))
y = int(y)
opt.y0 = y
y1 = y+1
# Find grid of Ra,Dec tile centers and select the ones near DECaLS bricks.
rr,dd = [],[]
yy = np.arange(opt.y0, y1)
xx = np.arange(opt.x0, x1)
if opt.grass:
import pylab as plt
tileexists = np.zeros((len(yy),len(xx)), bool)
basedir = settings.DATA_DIR
ver = tileversions[opt.kind][-1]
tiledir = os.path.join(basedir, 'tiles', opt.kind, '%i'%ver, '%i'%zoom)
for dirpath,dirnames,filenames in os.walk(tiledir):
# change walk order
dirnames.sort()
if len(filenames) == 0:
continue
print('Dirpath', dirpath)
#print 'Dirnames', dirnames
#print 'Filenames', filenames
# check for symlinks
if False:
fns = []
for fn in filenames:
fullfn = os.path.join(tiledir, dirpath, fn)
if os.path.isfile(fullfn) and not os.path.islink(fullfn):
fns.append(fn)
print(len(fns), 'of', len(filenames), 'are files (not symlinks)')
filenames = fns
x = os.path.basename(dirpath)
x = int(x)
#print 'x', x
yy = [int(fn.replace('.jpg','')) for fn in filenames]
#print 'yy', yy
print(len(yy), 'tiles')
for y in yy:
tileexists[y - opt.y0, x - opt.x0] = True
plt.clf()
plt.imshow(tileexists, interpolation='nearest', origin='upper',
vmin=0, vmax=1, cmap='gray')
fn = 'exist-%s-z%02i' % (opt.kind, zoom)
plt.savefig(fn+'.png')
fitsio.write(fn+'.fits', tileexists, clobber=True)
print('Wrote', fn+'.png and', fn+'.fits')
continue
if not opt.all:
for y in yy:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, y)
r,d = wcs.get_center()
dd.append(d)
for x in xx:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, x, 0)
r,d = wcs.get_center()
rr.append(r)
dd = np.array(dd)
rr = np.array(rr)
if len(dd) > 1:
tilesize = max(np.abs(np.diff(dd)))
print('Tile size:', tilesize)
else:
if opt.near_ccds or opt.near:
try:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0+1)
r2,d2 = wcs.get_center()
except:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0-1)
r2,d2 = wcs.get_center()
tilesize = np.abs(dd[0] - d2)
print('Tile size:', tilesize)
else:
tilesize = 180.
I = np.flatnonzero((dd >= opt.mindec) * (dd <= opt.maxdec))
print('Keeping', len(I), 'Dec points between', opt.mindec, 'and', opt.maxdec)
dd = dd[I]
yy = yy[I]
if opt.near_ccds:
margin = tilesize + ccdsize
I = np.flatnonzero((dd > C.dec.min()-margin) * (dd < C.dec.max()+margin))
if len(I) == 0:
print('No Dec points within range of CCDs')
continue
dd = dd[I]
yy = yy[I]
print('Keeping', len(I), 'Dec points within range of CCDs: Dec',
dd.min(), dd.max())
I = np.flatnonzero((rr >= opt.minra) * (rr <= opt.maxra))
print('Keeping', len(I), 'RA points between', opt.minra, 'and', opt.maxra)
rr = rr[I]
xx = xx[I]
print(len(rr), 'RA points x', len(dd), 'Dec points')
print('x tile range:', xx.min(), xx.max(), 'y tile range:', yy.min(), yy.max())
for iy,y in enumerate(yy):
print()
print('Y row', y)
if opt.queue:
if 'decaps2' in opt.kind:
layer = get_layer(opt.kind)
if zoom >= layer.nativescale:
oldscale = 0
else:
oldscale = (layer.nativescale - zoom)
oldscale = np.clip(oldscale, layer.minscale, layer.maxscale)
x = 0
wcs, W, H, zoomscale, z,xi,yi = get_tile_wcs(zoom, x, y)
newscale = layer.get_scale(zoom, 0, y, wcs)
if oldscale == newscale:
print('Oldscale = newscale = ', oldscale)
continue
cmd = 'python -u render-tiles.py --zoom %i --y0 %i --y1 %i --kind %s --mindec %f --maxdec %f' % (zoom, y, y+1, opt.kind, opt.mindec, opt.maxdec)
cmd += ' --threads 32'
if opt.near_ccds:
cmd += ' --near-ccds'
if opt.all:
cmd += ' --all'
if opt.ignore:
cmd += ' --ignore'
print(cmd)
continue
if opt.near:
d = dd[iy]
I,J,dist = match_radec(rr, d+np.zeros_like(rr), B.ra, B.dec, 0.25 + tilesize, nearest=True)
if len(I) == 0:
print('No matches to bricks')
continue
keep = np.zeros(len(rr), bool)
keep[I] = True
print('Keeping', sum(keep), 'tiles in row', y, 'Dec', d)
x = xx[keep]
elif opt.near_ccds:
d = dd[iy]
print('RA range of tiles:', rr.min(), rr.max())
print('Dec of tile row:', d)
I,J,dist = match_radec(rr, d+np.zeros_like(rr), C.ra, C.dec, ccdsize + tilesize, nearest=True)
if len(I) == 0:
print('No matches to CCDs')
continue
keep = np.zeros(len(rr), bool)
keep[I] = True
print('Keeping', sum(keep), 'tiles in row', y, 'Dec', d)
x = xx[keep]
else:
x = xx
# if opt.grass:
# for xi in x:
# basedir = settings.DATA_DIR
# ver = tileversions[opt.kind][-1]
# tilefn = os.path.join(basedir, 'tiles', opt.kind,
# '%i/%i/%i/%i.jpg' % (ver, zoom, xi, y))
# print 'Checking for', tilefn
# if os.path.exists(tilefn):
# print 'EXISTS'
# tileexists[yi-opt.y0, xi-opt.x0]
# continue
args = []
for xi in x:
args.append((opt.kind,zoom,xi,y, opt.ignore, False))
#args.append((opt.kind,zoom,xi,y, opt.ignore, True))
print('Rendering', len(args), 'tiles in row y =', y)
mp.map(_bounce_one_tile, args, chunksize=min(100, max(1, int(len(args)/opt.threads))))
#mp.map(_one_tile, args, chunksize=min(100, max(1, int(len(args)/opt.threads))))
print('Rendered', len(args), 'tiles')
ps = PlotSequence('hennawi')
r0,r1 = T.ra.min(), T.ra.max()
d0,d1 = T.dec.min(), T.dec.max()
print('RA range', r0,r1)
print('Dec range', d0,d1)
margin = 0.003
dr = margin / np.cos(np.deg2rad((d0+d1)/2.))
rlo = r0 - dr
rhi = r1 + dr
dlo = d0 - margin
dhi = d1 + margin
#mp = multiproc(8)
mp = multiproc(1)
#lvl = logging.INFO
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
if do_sdss:
sdss_forced_phot(r0,r1,d0,d1, rlo, rhi, dlo, dhi, T)
opt = myopts()
opt.wisedatadirs = wisedatadirs
opt.minflux = None
opt.sources = sfn
opt.nonsdss = True
opt.wsources = wsources
def redqsos():
# W4 detections without SDSS matches.
T = fits_table('w4targets.fits')
ps = None
#ps = PlotSequence('redqso')
arr = os.environ.get('PBS_ARRAYID')
tag = '-b'
if arr is not None:
arr = int(arr)
#chunk = 100
chunk = 50
T = T[arr * chunk: (arr+1) * chunk]
print('Cut to chunk', (arr * chunk))
tag = '-%03i' % arr
sdss = DR9()
sdss.useLocalTree()
sdss.saveUnzippedFiles('data/unzip')
mp = multiproc(1)
#lvl = logging.DEBUG
lvl = logging.INFO
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
newcols = {}
origcols = T.get_columns()
T.done = np.zeros(len(T), np.uint8)
version = get_svn_version()
print('SVN version info:', version)
hdr = fitsio.FITSHDR()
hdr.add_record(dict(name='PHOT_VER', value=version['Revision'],
comment='SVN revision'))
hdr.add_record(dict(name='PHOT_URL', value=version['URL'], comment='SVN URL'))
hdr.add_record(dict(name='PHOT_DAT', value=datetime.datetime.now().isoformat(),
comment='forced phot run time'))
for i,(ra,dec) in enumerate(zip(T.ra, T.dec)):
print()
print(i)
print('RA,Dec', ra, dec)
r0,r1 = ra,ra
d0,d1 = dec,dec
#margin = 0.003
# ~9 SDSS pixel half-box
margin = 0.001
dr = margin / np.cos(np.deg2rad((d0+d1)/2.))
rlo = r0 - dr
rhi = r1 + dr
dlo = d0 - margin
dhi = d1 + margin
sky = True
#sky = False
t = T[np.array([i])]
sdss_forced_phot(r0,r1,d0,d1, rlo, rhi, dlo, dhi, t, ps,
sdss=sdss, fitsky=sky)
#print 'Columns:', t.get_columns()
#t.about()
for key in t.get_columns():
if key in origcols:
continue
val = t.get(key)
if not key in newcols:
newcols[key] = np.zeros(len(T), val.dtype)
T.set(key, newcols[key])
print('set', key, i, '=', val[0])
newcols[key][i] = val[0]
T.done[i] = 1
#if i and i % 100 == 0:
if False:
fn = 'wisew4phot-interim%s.fits' % tag
T.writeto(fn, header=hdr)
print('Wrote', fn)
T.writeto('wisew4phot%s.fits' % tag, header=hdr)
psf = None
twcs = ConstantFitsWcs(wcs)
photocal = LinearPhotoCal(1.0, band=band)
sig1 = 1.0
data = np.zeros((H, W), np.float32)
tim = Image(data=data, invvar=np.ones_like(data) / sig1 ** 2, psf=psf, wcs=twcs, photocal=photocal)
tim.subwcs = wcs
tim.band = band
tim.sig1 = sig1
# Render synthetic source into image
re = 0.45
mp = multiproc()
ps = PlotSequence("galdet")
sourcetypes = ["E", "D", "C", "P", "-", "+"]
ccmap = dict(E="r", D="b", C="m", P="g")
ccmap["+"] = "k"
ccmap["-"] = "0.5"
namemap = {"E": "Exp", "D": "deVauc", "C": "composite", "P": "PSF", "+": ">1 src", "-": "No detection"}
for flux in [300.0]: # , 150. ]:
S = fits_table()
S.psffwhm = []
for t in sourcetypes:
def stage_sims(**kwargs):
W,H = 40,40
ra,dec = 0.,0.
band = 'r'
pixscale = 0.262
ps = pixscale / 3600.
wcs = Tan(ra, dec, W/2.+0.5, H/2.+0.5,
-ps, 0., 0., ps, float(W), float(H))
psf = None
twcs = ConstantFitsWcs(wcs)
photocal = LinearPhotoCal(1., band=band)
np.random.seed(1000042)
# survey target limiting mag: r=23.6, let's say 5-sig point source
limit = 23.6
data = np.zeros((H,W), np.float32)
tim = Image(data=data, inverr=np.ones_like(data), psf=psf, wcs=twcs,
photocal=photocal)
tim.subwcs = wcs
tim.band = band
tim.skyver = (0,0)
tim.wcsver = (0,0)
tim.psfver = (0,0)
tim.plver = 0
tim.dq = np.zeros(tim.shape, np.int16)
tim.dq_bits = dict(satur=1)
mp = multiproc()
sourcetypes = [ 'E','D','S','C','P','-','+' ]
ccmap = dict(E='r', D='b', S='c', C='m', P='g',
exp='r', dev='b', simp='c', comp='m', psf='g')
ccmap['+'] = 'k'
ccmap['-'] = '0.5'
namemap = { 'E': 'Exp', 'D': 'deVauc', 'C': 'composite', 'P':'PSF', 'S': 'simple', '+':'>1 src', '-':'No detection' }
#
#mag = 23.0
#re = 0.45
mag = np.arange(19.0, 23.5 + 1e-3, 0.5)
#mag = np.array([19.])
re = np.arange(0.1, 0.9 + 1e-3, 0.1)
#re = np.array([0.9])
psfsig = 2.0
N = 20
#N = 1
S = fits_table()
for t in sourcetypes:
S.set('frac_%s' % t, [])
S.frac_U = []
S.mag = []
S.re = []
S.imag = []
S.ire = []
S.psffwhm = []
S.mods = []
S.sig1 = []
S.dchisq = []
S.exp_re = []
simk = 0
for imag,mag_i in enumerate(mag):
for ire,re_i in enumerate(re):
#simk += 1
#print 'simk', simk
simk = 9
np.random.seed(1000 + simk)
S.mag.append(mag_i)
S.re.append(re_i)
S.imag.append(imag)
S.ire.append(ire)
flux = NanoMaggies.magToNanomaggies(mag_i)
gal = ExpGalaxy(RaDecPos(ra, dec), NanoMaggies(**{ band: flux }),
EllipseESoft(np.log(re_i), 0., 0.))
var1 = psfsig**2
psf = GaussianMixturePSF(1.0, 0., 0., var1, var1, 0.)
tim.psf = psf
tim.psf_sigma = psfsig
# Set the per-pixel noise based on the PSF size!?!
limitflux = NanoMaggies.magToNanomaggies(limit)
psfnorm = 1./(2. * np.sqrt(np.pi) * tim.psf_sigma)
sig1 = limitflux / 5. * psfnorm
tim.sig1 = sig1
tim.inverr[:,:] = 1./sig1
print
print 'Running mag=', mag_i, 're=', re_i
print 'psf norm', psfnorm
print 'limiting flux', limitflux
print 'sig1:', sig1
print 'flux:', flux
tr = Tractor([tim], [gal])
mod = tr.getModelImage(0)
S.mods.append(mod + sig1 * np.random.normal(size=mod.shape))
S.sig1.append(sig1)
res = run_sim([tim], [gal], N, mods=[mod],
W=W, H=H, ra=ra, dec=dec, mp=mp, bands=[band])
T = res.T
T.flux = T.flux_r
T.fluxiv = T.fluxiv_r
#catalog = res.TT
print 'T.nsrcs:', T.nsrcs
S.psffwhm.append(psfsig * 2.35 * pixscale)
for t in sourcetypes:
S.get('frac_%s' % t).append(
100. * np.count_nonzero(T.type == t) / float(len(T)))
S.frac_U.append(100. * np.count_nonzero(T.nsrcs == 0) / float(len(T)))
TT = res.TT
if 'dchisq' in TT.columns():
dchisq = TT.dchisq
S.dchisq.append(np.mean(dchisq, axis=0))
else:
S.dchisq.append(np.zeros(5))
I = []
if 'exp_shape_r' in TT.columns():
I = np.flatnonzero(TT.exp_shape_r > 0)
if len(I):
S.exp_re.append(np.mean(TT.exp_shape_r[I]))
else:
S.exp_re.append(0.)
S.to_np_arrays()
print 'S:', len(S)
rtn = {}
for k in ['S', 'mag', 're', 'psfsig']:
rtn[k] = locals()[k]
return rtn
def main():
import optparse
parser = optparse.OptionParser('%(prog)')
parser.add_option('-s',
'--sub',
type=int,
dest='sub',
help='Submission ID')
parser.add_option('-j', '--job', type=int, dest='job', help='Job ID')
parser.add_option('-u',
'--userimage',
type=int,
dest='uimage',
help='UserImage ID')
parser.add_option('-e',
'--email',
help='Find user id with given email address')
parser.add_option('-U',
'--userid',
help='Find user with given numerical ID',
type=int)
parser.add_option('-i', '--image', type=int, dest='image', help='Image ID')
parser.add_option('-d',
'--disk-file',
type=str,
dest='df',
help='DiskFile id')
parser.add_option('-r',
'--rerun',
dest='rerun',
action='store_true',
help='Re-run this submission/job?')
parser.add_option(
'--threads',
type=int,
help=
'Re-run failed jobs within this process using N threads; else submit to process_submissions process.'
)
parser.add_option(
'--chown',
dest='chown',
type=int,
default=0,
help='Change owner of userimage or submission by user id #')
parser.add_option(
'--solve-command',
help='Command to run instead of ssh to actually solve image')
parser.add_option(
'--solve-locally',
help='Command to run astrometry-engine on this machine, not via ssh')
parser.add_option('--ssh',
action='store_true',
default=False,
help='Find submissions whose jobs have ssh errors')
parser.add_option('--minsub',
type='int',
default=0,
help='Minimum submission id to look at')
parser.add_option('--empty',
action='store_true',
default=False,
help='Find submissions whose jobs have no log files')
parser.add_option('--delete',
action='store_true',
default=False,
help='Delete everything associated with the given image')
parser.add_option('--deluser',
action='store_true',
default=False,
help='Delete everything associated with the given user')
parser.add_option('--delextra',
action='store_true',
default=False,
help='Delete extraneous duplicate jobs?')
parser.add_option('--hide',
action='store_true',
default=False,
help='For a UserImage, set publicly_visible=False')
opt, args = parser.parse_args()
if opt.email:
users = User.objects.filter(email__contains=opt.email)
print('Users with email containing "%s":' % opt.email)
for u in users:
print(u.id, u, u.email)
if opt.deluser:
delete_user(u)
sys.exit(0)
if opt.userid:
users = User.objects.filter(id=opt.userid)
print('Users with ID', opt.userid)
for u in users:
print(u.id, u, u.email)
print(u.profile)
print(u.profile)
for k in [
'email', 'first_name', 'last_name', 'profile',
'social_auth', 'username'
]:
print(' ', k, getattr(u, k))
for f in [
'get_full_name',
'get_short_name',
'get_username',
]:
print(' ', f, getattr(u, f)())
if opt.deluser:
delete_user(u)
sys.exit(0)
if not (opt.sub or opt.job or opt.uimage or opt.image or opt.ssh
or opt.empty or opt.df):
print(
'Must specify one of --sub, --job, or --userimage or --image (or --ssh or --empty)'
)
parser.print_help()
sys.exit(-1)
if opt.threads is not None:
mp = multiproc(opt.threads)
else:
mp = None
if opt.ssh or opt.empty or opt.delextra:
subs = Submission.objects.all()
if opt.minsub:
subs = subs.filter(id__gt=opt.minsub)
subs = subs.order_by('-id')
failedsubs = []
failedjobs = []
for sub in subs:
print('Checking submission', sub.id)
allfailed = True
# last failed Job
failedjob = None
uis = sub.user_images.all()
for ui in uis:
jobs = ui.jobs.all()
for job in jobs:
print(' job', job.id)
if job.status == 'S':
print(' -> succeeded')
allfailed = False
break
print(' error msg', job.error_message)
logfn = job.get_log_file()
if not os.path.exists(logfn):
failedjob = job
continue
if opt.ssh:
log = read_file(logfn)
# 'Connection refused'
# 'Connection timed out'
if not 'ssh: connect to host astro.cs.toronto.edu port 22:' in log:
allfailed = False
break
print('SSH failed')
failedjob = job
if opt.empty:
# log file found
allfailed = False
break
if opt.delextra:
print('Delextra:', len(jobs), 'jobs', len(uis),
'uis; failedjob:', failedjob)
if len(jobs) > 1 and failedjob is not None:
print('Delextra: delete', failedjob)
if not allfailed:
continue
print('All jobs failed for sub', sub.id) #, 'via ssh failure')
#failedsubs.append(sub)
failedjobs.append(failedjob)
print('Found total of', len(failedsubs), 'failed Submissions and',
len(failedjobs), 'failed Jobs')
if opt.rerun:
from process_submissions import try_dosub, try_dojob
if opt.threads is not None:
args = []
for j in failedjobs:
if j is None:
continue
args.append((j.id, opt.solve_command, opt.solve_locally))
mp.map(bounce_try_dojob, args)
else:
for job in failedjobs:
if job is None:
continue
print('Re-trying job', job.id)
try_dojob(job, job.user_image, opt.solve_command,
opt.solve_locally)
# FIXME -- failed subs??
#
# else:
# for sub in failedsubs:
# print('Re-trying sub', sub.id)
# try_dosub(sub, 1)
if opt.df:
dfs = DiskFile.objects.filter(file_hash=opt.df)
print('Found', dfs.count(), 'matching DiskFiles')
for df in dfs:
print(' ', df)
print(' ', df.get_path())
ims = Image.objects.filter(disk_file=df)
print(' Used by', ims.count(), 'Image objects')
for im in ims:
print(' ', im, im.id)
print(' size %i x %i' % (im.width, im.height))
print(' thumb Image', im.thumbnail,
(im.thumbnail and im.thumbnail.id or ''))
print(' display Image', im.display_image,
(im.display_image and im.display_image.id or ''))
ims = Image.objects.filter(disk_file=df,
display_image__isnull=False,
thumbnail__isnull=False)
print(' found', ims.count(),
'Images for this df w/ display and thumbnails')
if opt.sub:
sub = Submission.objects.all().get(id=opt.sub)
print('Submission', sub)
if sub.disk_file is None:
print(' no disk file')
else:
print('Path', sub.disk_file.get_path())
print('Is fits image:', sub.disk_file.is_fits_image())
print('Is fits image:', sub.disk_file.file_type)
print('User:'******'UserImages:', len(uis))
for ui in uis:
print(' ', ui)
print(' user', ui.user)
print(' with Jobs:', len(ui.jobs.all()))
for j in ui.jobs.all():
print(' ', j)
if opt.chown:
newuser = User.objects.all().get(id=opt.chown)
for ui in uis:
ui.user = newuser
ui.save()
if opt.rerun:
from process_submissions import try_dosub, dosub
print('Re-trying sub', sub.id)
#try_dosub(sub, 1)
dosub(sub)
if opt.delete:
print('Deleting submission', sub)
sub.delete()
if opt.job:
job = Job.objects.all().get(id=opt.job)
print('Job', job)
print(job.get_dir())
print('Status:', job.status)
print('Error message:', job.error_message)
ui = job.user_image
print('UserImage:', ui)
print('User', ui.user)
im = ui.image
print('Image', im)
sub = ui.submission
print('Submission', sub)
print(sub.disk_file.get_path())
if opt.rerun:
from astrometry.net.process_submissions import try_dojob
print('Re-trying job', job.id)
try_dojob(job, ui, opt.solve_command, opt.solve_locally)
if opt.delete:
print('Deleting job', job)
job.delete()
if opt.uimage:
ui = UserImage.objects.all().get(id=opt.uimage)
print('UserImage', ui)
im = ui.image
print('Image', im)
sub = ui.submission
print('User', ui.user)
print('Submission', sub)
print(sub.disk_file.get_path())
if opt.chown:
user = User.objects.all().get(id=opt.chown)
print('User:'******'chowning', ui, 'to', user)
ui.user = user
ui.save()
if opt.delete:
print('Deleting ui', ui)
ui.delete()
if opt.hide:
print('Hiding ui', ui)
ui.hide()
if opt.image:
im = Image.objects.all().get(id=opt.image)
# thumbnail
# display_image
print('Image:', im, im.id)
#uis = im.userimage_set()
uis = UserImage.objects.all().filter(image=im.id)
print('UserImages:', uis)
print('Thumbnail:', im.thumbnail)
print('Display:', im.display_image)
if opt.delete:
print('Deleting...')
im.delete()
if im.thumbnail:
im.thumbnail.delete()
if im.display_image:
im.display_image.delete()
if __name__ == '__main__':
TT = [fits_table('ccds-annotated/ccds-annotated-%03i.fits' % i) for i in range(515)]
T = merge_tables(TT)
T.writeto('ccds-annotated.fits')
import sys
sys.exit()
#sys.exit(main())
decals = Decals()
ccds = decals.get_ccds()
from astrometry.util.multiproc import *
#mp = multiproc(8)
mp = multiproc(4)
N = 1000
args = []
i = 0
while len(ccds):
c = ccds[:N]
ccds = ccds[N:]
args.append((i, c))
i += 1
print('Split CCDs file into', len(args), 'pieces')
print('sizes:', [len(c) for i,c in args])
mp.map(_bounce_main, args)
# reassemble outputs
TT = [fits_table('ccds-annotated/ccds-annotated-%03i.fits' % i) for i,nil in args]
T = merge_tables(TT)
def main():
cutToPrimary = True
if True:
stars = [
# David's nearby pairs of F stars
(3900., 0., 118.37066, 52.527073),
(3705., 0., 130.17654, 52.750081),
# High stellar density
#(0., 0., 270.0, 0.003),
]
# Dustin's stars
# (4472.001, 0.02514649, 246.47016, 19.066909),
# (5196.53, 0.02490235, 240.09403, 37.404078),
# (6179.05, 0.6324392, 310.47791, 57.523221),
# (6021.875, 0.7000019, 150.52443, -0.478836),
# (7757.096, 0.06507664, 305.11144, -12.957655),
# (8088.685, 0.2436366, 253.11475, 11.60716),
# (8395.096, 0.7563477, 188.34439, 63.442057),
# (9201.74, 178, 93.971719, 0.56302169),
# ]
####### (awu) CHANGE THIS TO READ IN ANOTHER FITS FILE
T = fits_table('stars2.fits')
print 'Read stars:'
T.about()
stars.extend(zip(T.teff, T.teff_sigma, T.ra, T.dec))
# reformat
stars = [(ra,dec,[('T_EFF',teff,'Effective temperature'),
('DT_EFF',dteff,'Effective temperate error')],
cutToPrimary) for teff,dteff,ra,dec in stars]
elif False:
sdss = DR9(basedir=tempdir)
sdss.useLocalTree()
# near M87 / Virgo cluster
run,camcol,field = 3836,2,258
pofn = sdss.retrieve('photoObj', run, camcol, field)
T = fits_table(pofn, columns=[
'parent', 'objid', 'ra', 'dec', 'fracdev', 'objc_type', 'modelflux',
'theta_dev', 'theta_deverr', 'ab_dev', 'ab_deverr', 'phi_dev_deg',
'theta_exp', 'theta_experr', 'ab_exp', 'ab_experr', 'phi_exp_deg',
'resolve_status', 'nchild', 'flags', 'objc_flags',
'run','camcol','field','id'])
print len(T), 'objects'
T.cut(T.objc_type == 3)
print len(T), 'galaxies'
T.cut(T.nchild == 0)
print len(T), 'children'
T.cut(np.argsort(-T.modelflux[:,2]))
# keep only one child in each blend family
parents = set()
keepi = []
for i in range(len(T)):
if T.parent[i] in parents:
continue
keepi.append(i)
parents.add(T.parent[i])
T.cut(np.array(keepi))
print len(T), 'unique blend families'
T = T[:25]
stars = [(ra,dec,[],cutToPrimary) for ra,dec in zip(T.ra, T.dec)]
else:
# Objects in the Stripe82 coadd context in CAS:
# select * into mydb.s82 from PhotoPrimary where
# ra between 5 and 6
# and dec between 0 and 1
# and (run = 106 or run = 206) -> s82.fits
T = fits_table('s82.fits')
print 'Read', len(T), 'objects'
T.cut(T.nchild == 0)
print len(T), 'children'
T.cut(T.insidemask == 0)
print len(T), 'not in mask'
#T.cut(np.hypot(T.ra - 5.0562, T.dec - 0.0643) < 0.001)
# http://skyserver.sdss.org/dr12/en/help/browser/browser.aspx#&&history=enum+PhotoFlags+E
for flagname,flagval in [('BRIGHT', 0x2),
('EDGE', 0x4),
('NODEBLEND', 0x40),
('DEBLEND_TOO_MANY_PEAKS' , 0x800),
('NOTCHECKED', 0x80000),
('TOO_LARGE', 0x1000000),
('BINNED2', 0x20000000),
('BINNED4', 0x40000000),
('SATUR_CENTER', 0x80000000000),
('INTERP_CENTER', 0x100000000000),
('MAYBE_CR', 0x100000000000000),
('MAYBE_EGHOST', 0x200000000000000),
]:
T.cut(T.flags & flagval == 0)
print len(T), 'without', flagname, 'bit set'
#pass
# Cut to objects that are likely to appear in the individual images
T.cut(T.psfmag_r < 22.)
print 'Cut to', len(T), 'with psfmag_r < 22 in coadd'
# Select "interesting" objects...
# for i in range(len(T)):
#
# t = T[i]
# ra,dec = t.ra, t.dec
#
# radius = 2. * 0.396 / 3600.
# ddec = radius
# dra = radius / np.cos(np.deg2rad(dec))
# r0,r1 = ra - dra, ra + dra
# d0,d1 = dec - ddec, dec + ddec
#
# wlistfn = sdss.filenames.get('window_flist', 'window_flist.fits')
# RCF = radec_to_sdss_rcf(ra, dec, tablefn=wlistfn)
# print 'Found', len(RCF), 'fields in range.'
# keepRCF = []
# for run,camcol,field,r,d in RCF:
# rr = sdss.get_rerun(run, field)
# print 'Rerun:', rr
# if rr == '157':
# continue
# keepRCF.append((run,camcol,field))
# RCF = keepRCF
# for ifield,(run,camcol,field) in enumerate(RCF):
# objfn = sdss.getPath('photoObj', run, camcol, field)
# objs = fits_table(objfn)
# objs.cut((objs.ra > r0) * (objs.ra < r1) *
# (objs.dec > d0) * (objs.dec < d1))
# bright = photo_flags1_map.get('BRIGHT')
# objs.cut((objs.nchild == 0) * ((objs.objc_flags & bright) == 0))
# Write out Stripe82 measurements...
pixscale = 0.396
pixradius = 25
radius = np.sqrt(2.) * pixradius * pixscale / 3600.
Nkeep = 1000
outdir = 'stamps'
T.tag = np.array(['%.4f-%.4f.fits' % (r,d) for r,d in zip(T.ra, T.dec)])
T[:Nkeep].writeto(os.path.join(outdir, 'stamps.fits'), columns=
'''tag objid run camcol field ra dec psfmag_u psfmag_g psfmag_r
psfmag_i psfmag_z modelmag_u modelmag_g modelmag_r
modelmag_i modelmag_z'''.split())
for i in range(len(T[:Nkeep])):
# t = T[np.array([i])]
# print 't:', t
# t.about()
# assert(len(t) == 1)
I,J,d = match_radec(np.array([T.ra[i]]), np.array([T.dec[i]]),
T.ra, T.dec, radius)
print len(J), 'matched within', radius*3600., 'arcsec'
t = T[J]
print len(t), 'matched within', radius*3600., 'arcsec'
tt = fits_table()
cols = ['ra','dec','run','camcol','field',#'probpsf',
#'flags', #'type',
'fracdev_r', #'probpsf_r',
'devrad_r','devraderr_r', 'devab_r', 'devaberr_r',
'devphi_r', 'devphierr_r',
'exprad_r','expraderr_r', 'expab_r', 'expaberr_r',
'expphi_r', 'expphierr_r',
]
for c in cols:
cout = c
# drop "_r" from dev/exp shapes
if cout.endswith('_r'):
cout = cout[:-2]
coutmap = dict(devrad='theta_dev',
devphi='phi_dev',
devab ='ab_dev',
devraderr='theta_dev_err',
devphierr='phi_dev_err',
devaberr ='ab_dev_err',
exprad='theta_exp',
expphi='phi_exp',
expab ='ab_exp',
expraderr='theta_exp_err',
expphierr='phi_exp_err',
expaberr ='ab_exp_err',
fracdev='frac_dev')
cout = coutmap.get(cout, cout)
tt.set(cout, t.get(c))
tt.is_star = (t.type == 6)
for magname in ['psf', 'dev', 'exp']:
for band in 'ugriz':
mag = t.get('%smag_%s' % (magname, band))
magerr = t.get('%smagerr_%s' % (magname, band))
### FIXME -- arcsinh mags??
flux = NanoMaggies.magToNanomaggies(mag)
dflux = np.abs(flux * np.log(10.)/-2.5 * magerr)
tt.set('%sflux_%s' % (magname, band), flux)
tt.set('%sfluxerr_%s' % (magname, band), dflux)
for band in 'ugriz':
# http://www.sdss3.org/dr10/algorithms/magnitudes.php#cmodel
fexp = tt.get('expflux_%s' % band)
fdev = tt.get('expflux_%s' % band)
fracdev = t.get('fracdev_%s' % band)
tt.set('cmodelflux_%s' % band, fracdev * fdev + (1.-fracdev) * fexp)
catfn = os.path.join(outdir, 'cat-s82-%.4f-%.4f.fits' % (t.ra[0], t.dec[0]))
tt.writeto(catfn)
print 'Wrote', catfn
cutToPrimary = False
### HACK -- move old files into place.
for ra,dec in zip(T.ra, T.dec)[:Nkeep]:
plotfn = 'stamps-%.4f-%.4f.png' % (ra, dec)
if os.path.exists(plotfn):
fns = [plotfn]
for band in 'ugriz':
stampfn = 'stamp-%s-%.4f-%.4f.fits' % (band, ra, dec)
fns.append(stampfn)
catfn = 'cat-%.4f-%.4f.fits' % (ra,dec)
fns.append(catfn)
for fn in fns:
cmd = 'mv %s %s' % (fn, outdir)
print cmd
os.system(cmd)
stars = [(ra,dec,[],cutToPrimary,outdir) for ra,dec in zip(T.ra, T.dec)[:Nkeep]]
plots = True
###### (awu) CHANGE TO ACHIEVE PARALLEL
if False:
from astrometry.util.multiproc import *
mp = multiproc(8)
mp = multiproc()
mp.map(_bounce_one_blob, stars)
else:
# stars = [ (0.,0., 131.59054, 0.66408610),
# (0.,0., 147.34576, 0.51657783 ),
# ]
for args in stars:
ra,dec = args[:2]
try:
fns = oneblob(*args)
except:
import traceback
traceback.print_exc()
continue
if plots:
stamp_pattern = 'stamps/stamp-%%s-%.4f-%.4f.fits' % (ra, dec)
bands = 'ugriz'
fns = ['cat'] + [stamp_pattern % band for band in bands]
for j,fn in enumerate(fns[1:]):
print 'Filename', fn
F = fitsio.FITS(fn)
n = len(F) / 2
print 'n ext:', n
cols = int(np.ceil(np.sqrt(n)))
rows = int(np.ceil(n / float(cols)))
plt.clf()
#for i,ext in enumerate(range(0, len(F), 2)):
#plt.subplot(rows, cols, i+1)
#hdr = F[ext].read_header()
dimshow(F[0].read(), ticks=False)
#plt.title('RCF %i/%i/%i' % (hdr['RUN'], hdr['CAMCOL'], hdr['FIELD']))
plt.suptitle('%s band' % bands[j])
plt.savefig(fn.replace('.fits','.png'))
F.close()
del F
def main(opt, ps):
#ralo = 36
#rahi = 42
#declo = -1.25
#dechi = 1.25
#width = 7
ralo = 37.5
rahi = 41.5
declo = -1.5
dechi = 2.5
width = 2.5
rl,rh = 39,40
dl,dh = 0,1
roipoly = np.array([(rl,dl),(rl,dh),(rh,dh),(rh,dl)])
ra = (ralo + rahi ) / 2.
dec = (declo + dechi) / 2.
bandnum = 1
band = 'w%i' % bandnum
plt.figure(figsize=(12,12))
#basedir = '/project/projectdirs/bigboss'
#wisedatadir = os.path.join(basedir, 'data', 'wise')
wisedatadirs = ['/clusterfs/riemann/raid007/bosswork/boss/wise_level1b',
'/clusterfs/riemann/raid000/bosswork/boss/wise1ext']
wisecatdir = '/home/boss/products/NULL/wise/trunk/fits/'
ofn = 'wise-images-overlapping.fits'
if os.path.exists(ofn):
print 'File exists:', ofn
T = fits_table(ofn)
print 'Found', len(T), 'images overlapping'
print 'Reading WCS headers...'
wcses = []
T.filename = [fn.strip() for fn in T.filename]
for fn in T.filename:
wcs = anwcs(fn, 0)
wcses.append(wcs)
else:
TT = []
for d in wisedatadirs:
ifn = os.path.join(d, 'WISE-index-L1b.fits') #'index-allsky-astr-L1b.fits')
T = fits_table(ifn, columns=['ra','dec','scan_id','frame_num'])
print 'Read', len(T), 'from WISE index', ifn
I = np.flatnonzero((T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi))
print len(I), 'overlap RA,Dec box'
T.cut(I)
fns = []
for sid,fnum in zip(T.scan_id, T.frame_num):
print 'scan,frame', sid, fnum
fn = get_l1b_file(d, sid, fnum, bandnum)
print '-->', fn
assert(os.path.exists(fn))
fns.append(fn)
T.filename = np.array(fns)
TT.append(T)
T = merge_tables(TT)
wcses = []
corners = []
ii = []
for i in range(len(T)):
wcs = anwcs(T.filename[i], 0)
W,H = wcs.get_width(), wcs.get_height()
rd = []
for x,y in [(1,1),(1,H),(W,H),(W,1)]:
rd.append(wcs.pixelxy2radec(x,y))
rd = np.array(rd)
if polygons_intersect(roipoly, rd):
wcses.append(wcs)
corners.append(rd)
ii.append(i)
print 'Found', len(wcses), 'overlapping'
I = np.array(ii)
T.cut(I)
outlines = corners
corners = np.vstack(corners)
nin = sum([1 if point_in_poly(ra,dec,ol) else 0 for ol in outlines])
print 'Number of images containing RA,Dec,', ra,dec, 'is', nin
r0,r1 = corners[:,0].min(), corners[:,0].max()
d0,d1 = corners[:,1].min(), corners[:,1].max()
print 'RA,Dec extent', r0,r1, d0,d1
T.writeto(ofn)
print 'Wrote', ofn
# MAGIC 2.75: approximate pixel scale, "/pix
S = int(3600. / 2.75)
print 'Coadd size', S
cowcs = anwcs_create_box(ra, dec, 1., S, S)
if False:
print 'Plotting map...'
plot = Plotstuff(outformat='png', ra=ra, dec=dec, width=width, size=(800,800))
out = plot.outline
plot.color = 'white'
plot.alpha = 0.07
plot.apply_settings()
for wcs in wcses:
out.wcs = wcs
out.fill = False
plot.plot('outline')
out.fill = True
plot.plot('outline')
plot.color = 'gray'
plot.alpha = 1.0
plot.lw = 1
plot.plot_grid(1, 1, 1, 1)
plot.color = 'red'
plot.lw = 3
plot.alpha = 0.75
out.wcs = cowcs
out.fill = False
plot.plot('outline')
if opt.sources:
rd = plot.radec
plot_radec_set_filename(rd, opt.sources)
plot.plot('radec')
pfn = ps.getnext()
plot.write(pfn)
print 'Wrote', pfn
# Re-sort by distance to RA,Dec center...
#I = np.argsort(np.hypot(T.ra - ra, T.dec - dec))
#T.cut(I)
# IF YOU DO THIS, MUST ALSO RE-SORT 'wcses'!
if opt.sources:
# Look at a radius this big, in arcsec, around each source position.
# 15" = about 6 WISE pixels
Wrad = 15. / 3600.
# Look for SDSS objects within this radius; Wrad + a margin
Srad = Wrad + 5./3600.
S = fits_table(opt.sources)
print 'Read', len(S), 'sources from', opt.sources
groups,singles = cluster_radec(S.ra, S.dec, Wrad, singles=True)
print 'Source clusters:', groups
print 'Singletons:', singles
tractors = []
sdss = DR9(basedir='data-dr9')
sband = 'r'
for i in singles:
r,d = S.ra[i],S.dec[i]
print 'Source', i, 'at', r,d
fn = sdss.retrieve('photoObj', S.run[i], S.camcol[i], S.field[i], band=sband)
print 'Reading', fn
oo = fits_table(fn)
print 'Got', len(oo)
cat1,obj1,I = get_tractor_sources_dr9(None, None, None, bandname=sband,
objs=oo, radecrad=(r,d,Srad), bands=[],
nanomaggies=True, extrabands=[band],
fixedComposites=True,
getobjs=True, getobjinds=True)
print 'Got', len(cat1), 'SDSS sources nearby'
# Find images that overlap?
ims = []
for j,wcs in enumerate(wcses):
print 'Filename', T.filename[j]
ok,x,y = wcs.radec2pixelxy(r,d)
print 'WCS', j, '-> x,y:', x,y
if not anwcs_radec_is_inside_image(wcs, r, d):
continue
tim = wise.read_wise_level1b(
T.filename[j].replace('-int-1b.fits',''),
nanomaggies=True, mask_gz=True, unc_gz=True,
sipwcs=True, constantInvvar=True, radecrad=(r,d,Wrad))
ims.append(tim)
print 'Found', len(ims), 'images containing this source'
tr = Tractor(ims, cat1)
tractors.append(tr)
if len(groups):
# TODO!
assert(False)
sys.exit(0)
# Find additional SDSS sources nearby = within R pixels radius.
R = 30.
#R = 50.
rad = R * 0.396 / 3600.
cats = []
objs = []
for run,camcol,field,r,d in zip(S.run, S.camcol, S.field, S.ra, S.dec):
fn = sdss.retrieve('photoObj', run, camcol, field, band=sband)
print 'Reading', fn
oo = fits_table(fn)
print 'Got', len(oo)
cat1,obj1,I = get_tractor_sources_dr9(None, None, None, bandname=sband,
objs=oo, radecrad=(r,d,rad), bands=[],
nanomaggies=True, extrabands=[band],
fixedComposites=True,
getobjs=True, getobjinds=True)
print 'Got', len(cat1), 'SDSS sources nearby'
cats.append(cat1)
objs.append(obj1[I])
# Merge into one big catalog.
cat = Catalog()
for c in cats:
for src in c:
cat.append(src)
S = merge_tables(objs)
print 'Merged catalog has', len(cat), 'entries'
print 'S table has', len(S)
assert(len(S) == len(cat))
if opt.ptsrc:
print 'Converting all sources to PointSources'
pcat = Catalog()
for src in cat:
ps = PointSource(src.getPosition(), src.getBrightness())
pcat.append(ps)
print 'PointSource catalog:', pcat
cat = pcat
# ??
WW = S
#WW = tabledata()
# cat = get_tractor_sources_dr9(None, None, None, bandname=sband,
# objs=S, bands=[], nanomaggies=True,
# extrabands=[band])
print 'Got', len(cat), 'tractor sources'
#cat = Catalog(*cat)
print cat
for src in cat:
print ' ', src
### FIXME -- match to WISE catalog to initialize mags?
# Initialize WISE mags to be at least detectable
# so that we identify the right pixel ROIs below.
#minbright = NanoMaggies.magToNanomaggies()
#minbright = 50.
minbright = 250.
cat.freezeParamsRecursive('*')
cat.thawPathsTo(band)
p0 = cat.getParams()
cat.setParams(np.maximum(minbright, p0))
print 'Set minimum W1 brightness:'
for src in cat:
print ' ', src
# Cut images that don't overlap.
ii = []
for i,wcs in enumerate(wcses):
isin = False
for r,d in zip(S.ra, S.dec):
if anwcs_radec_is_inside_image(wcs, r, d):
isin = True
break
if isin:
ii.append(i)
T.cut(np.array(ii))
print 'Cut to', len(T), 'images containing sources'
else:
wfn = 'wise-sources-nearby.fits'
if os.path.exists(wfn):
print 'Reading existing file', wfn
W = fits_table(wfn)
print 'Got', len(W), 'with range RA', W.ra.min(), W.ra.max(), ', Dec', W.dec.min(), W.dec.max()
else:
# Range of WISE slices (inclusive) containing this Dec range.
ws0, ws1 = 26,27
WW = []
for w in range(ws0, ws1+1):
fn = os.path.join(wisecatdir, 'wise-allsky-cat-part%02i-radec.fits' % w)
print 'Searching for sources in', fn
W = fits_table(fn)
I = np.flatnonzero((W.ra >= r0) * (W.ra <= r1) * (W.dec >= d0) * (W.dec <= d1))
fn = os.path.join(wisecatdir, 'wise-allsky-cat-part%02i.fits' % w)
print 'Reading', len(I), 'rows from', fn
W = fits_table(fn, rows=I)
print 'Cut to', len(W), 'sources in range'
WW.append(W)
W = merge_tables(WW)
del WW
print 'Total of', len(W)
W.writeto(wfn)
print 'wrote', wfn
# DEBUG
W.cut((W.ra >= rl) * (W.ra <= rh) * (W.dec >= dl) * (W.dec <= dh))
print 'Cut to', len(W), 'in the central region'
print 'Creating', len(W), 'Tractor sources'
cat = Catalog()
for i in range(len(W)):
w1 = W.w1mpro[i]
nm = NanoMaggies.magToNanomaggies(w1)
cat.append(PointSource(RaDecPos(W.ra[i], W.dec[i]), NanoMaggies(w1=nm)))
WW = W
cat.freezeParamsRecursive('*')
cat.thawPathsTo(band)
cat0 = cat.getParams()
br0 = [src.getBrightness().copy() for src in cat]
nm0 = np.array([b.getBand(band) for b in br0])
WW.nm0 = nm0
w1psf = wise.get_psf_model(bandnum, opt.pixpsf)
# Create fake image in the "coadd" footprint in order to find overlapping
# sources.
H,W = int(cowcs.imageh), int(cowcs.imagew)
# MAGIC -- sigma a bit smaller than typical images (4.0-ish)
sig = 3.5
# typical zeropoint
zp = 20.752
faketim = Image(data=np.zeros((H,W), np.float32),
invvar=np.zeros((H,W), np.float32) + (1./sig**2),
psf=w1psf, wcs=ConstantFitsWcs(cowcs), sky=ConstantSky(0.),
photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp),
band=band),
#photocal=LinearPhotoCal(1., band=band),
name='fake')
minsb = 0.1 * sig
#minsb = 0.
# pc = faketim.getPhotoCal()
# print 'Source counts:'
# for src in cat:
# print ' ', src
# print '-->', pc.brightnessToCounts(src.getBrightness())
# print ' -->', [pc.brightnessToCounts(br) for br in src.getBrightnesses()]
# print 'Source pixel positions:'
# wcs = faketim.getWcs()
# for src in cat:
# print ' ', src
# print '--> x,y', wcs.positionToPixel(src.getPosition())
print 'Finding overlapping sources...'
t0 = Time()
tractor = Tractor([faketim], cat)
groups,L,fakemod = tractor.getOverlappingSources(0, minsb=minsb)
print 'Overlapping sources took', Time()-t0
print 'Got', len(groups), 'groups of sources'
nl = L.max()
gslices = find_objects(L, nl)
print 'unique labels:', np.unique(L)
# plt.clf()
# plt.imshow(fakemod, interpolation='nearest', origin='lower',
# vmin=0, vmax=sig*3.)
# plt.title('Fakemod')
# ps.savefig()
#
# for IM in [L, (L>0)]:
# plt.clf()
# plt.imshow(IM, interpolation='nearest', origin='lower')
# plt.gray()
# wcs = faketim.getWcs()
# xy = []
# for src in cat:
# x,y = wcs.positionToPixel(src.getPosition())
# xy.append((x,y))
# xy = np.array(xy)
# ax = plt.axis()
# plt.plot(xy[:,0], xy[:,1], 'r+')
# plt.title('Source groups')
# ps.savefig()
# Find sources touching each group's (rectangular) ROI
tgroups = {}
for i,gslice in enumerate(gslices):
gl = i+1
tg = np.unique(L[gslice])
tsrcs = []
for g in tg:
if not g in [gl,0]:
if g in groups:
tsrcs.extend(groups[g])
tgroups[gl] = tsrcs
# for i,gslice in enumerate(gslices):
# if not (i+1) in groups:
# continue
#
# plt.clf()
# plt.imshow(IM[gslice], interpolation='nearest', origin='lower')
# plt.gray()
# wcs = faketim.getWcs()
# xy = []
# y0,x0 = gslice[0].start, gslice[1].start
# for src in cat:
# x,y = wcs.positionToPixel(src.getPosition())
# xy.append((x-x0,y-y0))
# xy = np.array(xy)
#
# ax = plt.axis()
#
# plt.plot(xy[:,0], xy[:,1], 'r+')
#
# I = np.array(groups[i+1])
# if len(I):
# plt.plot(xy[I,0], xy[I,1], 'g.')
#
# I = np.array(tgroups[i+1])
# if len(I):
# plt.plot(xy[I,0], xy[I,1], 'gx')
#
# ps.savefig()
#
# plt.axis(ax)
# ps.savefig()
print 'Group size histogram:'
ng = Counter()
for g in groups.values():
ng[len(g)] += 1
kk = ng.keys()
kk.sort()
for k in kk:
print ' ', k, 'sources:', ng[k], 'groups'
nms = []
tims = []
allrois = {}
badrois = {}
if opt.threads:
mp = multiproc(opt.threads)
else:
mp = multiproc(1)
tims = mp.map(_read_l1b, T.filename)
for imi,tim in enumerate(tims):
tim.psf = w1psf
H,W = tim.shape
nin = 0
for src in cat:
x,y = tim.getWcs().positionToPixel(src.getPosition())
if x >= 0 and y >= 0 and x < W and y < H:
nin += 1
print 'Number of sources inside image:', nin
tractor = Tractor([tim], cat)
tractor.freezeParam('images')
### ??
cat.setParams(cat0)
pgroups = 0
pobjs = 0
for gi in range(len(gslices)):
gl = gi
# note, gslices is zero-indexed
gslice = gslices[gl]
gl += 1
if not gl in groups:
print 'Group', gl, 'not in groups array; skipping'
continue
gsrcs = groups[gl]
tsrcs = tgroups[gl]
# print 'Group number', (gi+1), 'of', len(Gorder), ', id', gl, ': sources', gsrcs
# print 'sources in groups touching slice:', tsrcs
# Convert from 'canonical' ROI to this image.
yl,yh = gslice[0].start, gslice[0].stop
xl,xh = gslice[1].start, gslice[1].stop
x0,y0 = W-1,H-1
x1,y1 = 0,0
for x,y in [(xl,yl),(xh-1,yl),(xh-1,yh-1),(xl,yh-1)]:
r,d = cowcs.pixelxy2radec(x+1, y+1)
x,y = tim.getWcs().positionToPixel(RaDecPos(r,d))
x = int(np.round(x))
y = int(np.round(y))
x = np.clip(x, 0, W-1)
y = np.clip(y, 0, H-1)
x0 = min(x0, x)
y0 = min(y0, y)
x1 = max(x1, x)
y1 = max(y1, y)
if x1 == x0 or y1 == y0:
print 'Gslice', gslice, 'is completely outside this image'
continue
gslice = (slice(y0,y1+1), slice(x0, x1+1))
if np.all(tim.getInvError()[gslice] == 0):
print 'This whole object group has invvar = 0.'
if not gl in badrois:
badrois[gl] = {}
badrois[gl][imi] = gslice
continue
if not gl in allrois:
allrois[gl] = {}
allrois[gl][imi] = gslice
if not opt.individual:
continue
fullcat = tractor.catalog
subcat = Catalog(*[fullcat[i] for i in gsrcs + tsrcs])
for i in range(len(tsrcs)):
subcat.freezeParam(len(gsrcs) + i)
tractor.catalog = subcat
print len(gsrcs), 'sources unfrozen; total', len(subcat)
pgroups += 1
pobjs += len(gsrcs)
t0 = Time()
tractor.optimize_forced_photometry(minsb=minsb, mindlnp=1.,
rois=[gslice])
print 'optimize_forced_photometry took', Time()-t0
tractor.catalog = fullcat
# mod = tractor.getModelImage(0, minsb=minsb)
# noise = np.random.normal(size=mod.shape)
# noise[tim.getInvError() == 0] = 0.
# nz = (tim.getInvError() > 0)
# noise[nz] *= (1./tim.getInvError()[nz])
# ima = dict(interpolation='nearest', origin='lower',
# vmin=tim.zr[0], vmax=tim.zr[1])
# imchi = dict(interpolation='nearest', origin='lower',
# vmin=-5, vmax=5)
# plt.clf()
# plt.subplot(2,2,1)
# plt.imshow(tim.getImage(), **ima)
# plt.gray()
# plt.subplot(2,2,2)
# plt.imshow(mod, **ima)
# plt.gray()
# plt.subplot(2,2,3)
# plt.imshow((tim.getImage() - mod) * tim.getInvError(), **imchi)
# plt.gray()
# plt.subplot(2,2,4)
# plt.imshow(mod + noise, **ima)
# plt.gray()
# plt.suptitle('W1, scan %s, frame %i' % (sid, fnum))
# ps.savefig()
if opt.individual:
print 'Photometered', pgroups, 'groups containing', pobjs, 'objects'
cat.thawPathsTo(band)
nm1 = np.array([src.getBrightness().getBand(band) for src in cat])
nms.append(nm1)
WW.nms = np.array(nms).T
fn = opt.output % imi
WW.writeto(fn)
print 'Wrote', fn
return dict(cat0=cat0, WW=WW, band=band, tims=tims,
allrois=allrois, badrois=badrois, groups=groups,
tgroups=tgroups, minsb=minsb,
gslices=gslices, cat=cat)
def main():
if False:
stars = [
# David's nearby pairs of F stars
(3900., 0., 118.37066, 52.527073),
(3705., 0., 130.17654, 52.750081),
# High stellar density
#(0., 0., 270.0, 0.003),
]
# Dustin's stars
# (4472.001, 0.02514649, 246.47016, 19.066909),
# (5196.53, 0.02490235, 240.09403, 37.404078),
# (6179.05, 0.6324392, 310.47791, 57.523221),
# (6021.875, 0.7000019, 150.52443, -0.478836),
# (7757.096, 0.06507664, 305.11144, -12.957655),
# (8088.685, 0.2436366, 253.11475, 11.60716),
# (8395.096, 0.7563477, 188.34439, 63.442057),
# (9201.74, 178, 93.971719, 0.56302169),
# ]
T = fits_table('stars2.fits')
print 'Read stars:'
T.about()
stars.extend(zip(T.teff, T.teff_sigma, T.ra, T.dec))
else:
# This photoObj is unknown
sdss = DR9(basedir=tempdir)
sdss.useLocalTree()
# near M87 / Virgo cluster
run,camcol,field = 3836,2,258
pofn = sdss.retrieve('photoObj', run, camcol, field)
T = fits_table(pofn, columns=[
'parent', 'objid', 'ra', 'dec', 'fracdev', 'objc_type', 'modelflux',
'theta_dev', 'theta_deverr', 'ab_dev', 'ab_deverr', 'phi_dev_deg',
'theta_exp', 'theta_experr', 'ab_exp', 'ab_experr', 'phi_exp_deg',
'resolve_status', 'nchild', 'flags', 'objc_flags',
'run','camcol','field','id'])
print len(T), 'objects'
T.cut(T.objc_type == 3)
print len(T), 'galaxies'
T.cut(T.nchild == 0)
print len(T), 'children'
T.cut(np.argsort(-T.modelflux[:,2]))
# keep only one child in each blend family
parents = set()
keepi = []
for i in range(len(T)):
if T.parent[i] in parents:
continue
keepi.append(i)
parents.add(T.parent[i])
T.cut(np.array(keepi))
print len(T), 'unique blend families'
T = T[:25]
stars = [(0,0,ra,dec) for ra,dec in zip(T.ra, T.dec)]
plots = True
if True:
from astrometry.util.multiproc import *
mp = multiproc(4)
mp.map(_bounce_one_blob, stars)
else:
# stars = [ (0.,0., 131.59054, 0.66408610),
# (0.,0., 147.34576, 0.51657783 ),
# ]
for teff, dteff, ra,dec in stars:
try:
fns = oneblob(ra,dec, teff, dteff)
except:
import traceback
traceback.print_exc()
continue
if plots:
stamp_pattern = 'stamp-%%s-%.4f-%.4f.fits' % (ra, dec)
bands = 'ugriz'
fns = ['cat'] + [stamp_pattern % band for band in bands]
for j,fn in enumerate(fns[1:]):
print 'Filename', fn
F = fitsio.FITS(fn)
n = len(F) / 2
print 'n ext:', n
cols = int(np.ceil(np.sqrt(n)))
rows = int(np.ceil(n / float(cols)))
plt.clf()
for i,ext in enumerate(range(0, len(F), 2)):
plt.subplot(rows, cols, i+1)
hdr = F[ext].read_header()
dimshow(F[ext].read(), ticks=False)
plt.title('RCF %i/%i/%i' % (hdr['RUN'], hdr['CAMCOL'], hdr['FIELD']))
plt.suptitle('%s band' % bands[j])
plt.savefig(fn.replace('.fits','.png'))
F.close()
del F
def create_detmaps(bands):
mp = multiproc(8)
#mp = multiproc()
tile = '3524p000'
# 1/4 x 1/4 subimage
nsub = 4
subx, suby = 0, 3
chips = [30, 29, 24, 23, 22, 21, 43, 42]
# 1/16 x 1/16 for faster testing
nsub = 16
subx, suby = 0, 12
chips = [22, 43]
import desi_common
desi_common.N_subtiles = nsub
cowcs = get_subtile_wcs(tile, subx, suby)
# Insert imaging database here...
paths = []
for dirpath,dirs,files in os.walk('data/desi', followlinks=True):
for fn in files:
path = os.path.join(dirpath, fn)
if path.endswith('.p.w.fits'):
if any([('/C%02i/' % chip) in path for chip in chips]):
paths.append(path)
print 'Found', len(paths), 'images'
# Plug the WCS header cards into the output coadd files.
f,tmpfn = tempfile.mkstemp()
os.close(f)
cowcs.write_to(tmpfn)
hdr = fitsio.read_header(tmpfn)
os.remove(tmpfn)
for band in bands:
print
print 'Band', band
wise = band.startswith('W')
if not wise:
fns = [fn for fn in paths if '%sband' % band in fn]
wcsdir = 'data/decam/astrom'
wcsfns = [os.path.join(wcsdir, os.path.basename(fn).replace('.fits','.wcs'))
for fn in fns]
else:
# HACK
wisefn = 'unwise/352/3524p000/unwise-3524p000-%s-img-m.fits' % band.lower()
fns = [wisefn]
wcsfns = fns
# resample image too (not just detection map?)
do_img = True
coH,coW = cowcs.get_height(), cowcs.get_width()
codet = np.zeros((coH,coW))
codet_iv = np.zeros((coH,coW))
if do_img:
coadd = np.zeros((coH, coW))
coadd_iv = np.zeros((coH,coW))
args = [(cowcs, fn, wcsfn, do_img, wise) for fn,wcsfn in zip(fns,wcsfns)]
for i,A in enumerate(mp.map(_det_one, args)):
if A is None:
print 'Skipping input', fns[i]
continue
Yo,Xo,rdetmap,detmap_iv,rimg = A
codet[Yo,Xo] += rdetmap * detmap_iv
codet_iv[Yo,Xo] += detmap_iv
if do_img:
coadd[Yo,Xo] += rimg * detmap_iv
coadd_iv[Yo,Xo] += detmap_iv
codet /= np.maximum(codet_iv, 1e-16)
fitsio.write('detmap-%s.fits' % band, codet.astype(np.float32), header=hdr, clobber=True)
# no clobber -- append to file
fitsio.write('detmap-%s.fits' % band, codet_iv.astype(np.float32), header=hdr)
if do_img:
coadd /= np.maximum(coadd_iv, 1e-16)
fitsio.write('coadd-%s.fits' % band, coadd.astype(np.float32), header=hdr, clobber=True)
# no clobber -- append to file
fitsio.write('coadd-%s.fits' % band, coadd_iv.astype(np.float32), header=hdr)
def stage_sims(**kwargs):
W, H = 40, 40
ra, dec = 0., 0.
band = 'r'
pixscale = 0.262
ps = pixscale / 3600.
wcs = Tan(ra, dec, W / 2. + 0.5, H / 2. + 0.5, -ps, 0., 0., ps, float(W),
float(H))
psf = None
twcs = ConstantFitsWcs(wcs)
photocal = LinearPhotoCal(1., band=band)
np.random.seed(1000042)
# survey target limiting mag: r=23.6, let's say 5-sig point source
limit = 23.6
data = np.zeros((H, W), np.float32)
tim = Image(data=data,
inverr=np.ones_like(data),
psf=psf,
wcs=twcs,
photocal=photocal)
tim.subwcs = wcs
tim.band = band
tim.skyver = (0, 0)
tim.wcsver = (0, 0)
tim.psfver = (0, 0)
tim.plver = 0
tim.dq = np.zeros(tim.shape, np.int16)
tim.dq_bits = dict(satur=1)
mp = multiproc()
sourcetypes = ['E', 'D', 'S', 'C', 'P', '-', '+']
ccmap = dict(E='r',
D='b',
S='c',
C='m',
P='g',
exp='r',
dev='b',
simp='c',
comp='m',
psf='g')
ccmap['+'] = 'k'
ccmap['-'] = '0.5'
namemap = {
'E': 'Exp',
'D': 'deVauc',
'C': 'composite',
'P': 'PSF',
'S': 'simple',
'+': '>1 src',
'-': 'No detection'
}
#
#mag = 23.0
#re = 0.45
mag = np.arange(19.0, 23.5 + 1e-3, 0.5)
#mag = np.array([19.])
re = np.arange(0.1, 0.9 + 1e-3, 0.1)
#re = np.array([0.9])
psfsig = 2.0
N = 20
#N = 1
S = fits_table()
for t in sourcetypes:
S.set('frac_%s' % t, [])
S.frac_U = []
S.mag = []
S.re = []
S.imag = []
S.ire = []
S.psffwhm = []
S.mods = []
S.sig1 = []
S.dchisq = []
S.exp_re = []
simk = 0
for imag, mag_i in enumerate(mag):
for ire, re_i in enumerate(re):
#simk += 1
#print 'simk', simk
simk = 9
np.random.seed(1000 + simk)
S.mag.append(mag_i)
S.re.append(re_i)
S.imag.append(imag)
S.ire.append(ire)
flux = NanoMaggies.magToNanomaggies(mag_i)
gal = ExpGalaxy(RaDecPos(ra, dec), NanoMaggies(**{band: flux}),
EllipseESoft(np.log(re_i), 0., 0.))
var1 = psfsig**2
psf = GaussianMixturePSF(1.0, 0., 0., var1, var1, 0.)
tim.psf = psf
tim.psf_sigma = psfsig
# Set the per-pixel noise based on the PSF size!?!
limitflux = NanoMaggies.magToNanomaggies(limit)
psfnorm = 1. / (2. * np.sqrt(np.pi) * tim.psf_sigma)
sig1 = limitflux / 5. * psfnorm
tim.sig1 = sig1
tim.inverr[:, :] = 1. / sig1
print
print 'Running mag=', mag_i, 're=', re_i
print 'psf norm', psfnorm
print 'limiting flux', limitflux
print 'sig1:', sig1
print 'flux:', flux
tr = Tractor([tim], [gal])
mod = tr.getModelImage(0)
S.mods.append(mod + sig1 * np.random.normal(size=mod.shape))
S.sig1.append(sig1)
res = run_sim([tim], [gal],
N,
mods=[mod],
W=W,
H=H,
ra=ra,
dec=dec,
mp=mp,
bands=[band])
T = res.T
T.flux = T.flux_r
T.fluxiv = T.fluxiv_r
#catalog = res.TT
print 'T.nsrcs:', T.nsrcs
S.psffwhm.append(psfsig * 2.35 * pixscale)
for t in sourcetypes:
S.get('frac_%s' % t).append(
100. * np.count_nonzero(T.type == t) / float(len(T)))
S.frac_U.append(100. * np.count_nonzero(T.nsrcs == 0) /
float(len(T)))
TT = res.TT
if 'dchisq' in TT.columns():
dchisq = TT.dchisq
S.dchisq.append(np.mean(dchisq, axis=0))
else:
S.dchisq.append(np.zeros(5))
I = []
if 'exp_shape_r' in TT.columns():
I = np.flatnonzero(TT.exp_shape_r > 0)
if len(I):
S.exp_re.append(np.mean(TT.exp_shape_r[I]))
else:
S.exp_re.append(0.)
S.to_np_arrays()
print 'S:', len(S)
rtn = {}
for k in ['S', 'mag', 're', 'psfsig']:
rtn[k] = locals()[k]
return rtn
def redqsos():
# W4 detections without SDSS matches.
T = fits_table('w4targets.fits')
ps = None
#ps = PlotSequence('redqso')
arr = os.environ.get('PBS_ARRAYID')
tag = '-b'
if arr is not None:
arr = int(arr)
#chunk = 100
chunk = 50
T = T[arr * chunk: (arr+1) * chunk]
print 'Cut to chunk', (arr * chunk)
tag = '-%03i' % arr
sdss = DR9()
sdss.useLocalTree()
sdss.saveUnzippedFiles('data/unzip')
mp = multiproc(1)
#lvl = logging.DEBUG
lvl = logging.INFO
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
newcols = {}
origcols = T.get_columns()
T.done = np.zeros(len(T), np.uint8)
version = get_svn_version()
print 'SVN version info:', version
hdr = fitsio.FITSHDR()
hdr.add_record(dict(name='PHOT_VER', value=version['Revision'],
comment='SVN revision'))
hdr.add_record(dict(name='PHOT_URL', value=version['URL'], comment='SVN URL'))
hdr.add_record(dict(name='PHOT_DAT', value=datetime.datetime.now().isoformat(),
comment='forced phot run time'))
for i,(ra,dec) in enumerate(zip(T.ra, T.dec)):
print
print i
print 'RA,Dec', ra, dec
r0,r1 = ra,ra
d0,d1 = dec,dec
#margin = 0.003
# ~9 SDSS pixel half-box
margin = 0.001
dr = margin / np.cos(np.deg2rad((d0+d1)/2.))
rlo = r0 - dr
rhi = r1 + dr
dlo = d0 - margin
dhi = d1 + margin
sky = True
#sky = False
t = T[np.array([i])]
sdss_forced_phot(r0,r1,d0,d1, rlo, rhi, dlo, dhi, t, ps,
sdss=sdss, fitsky=sky)
#print 'Columns:', t.get_columns()
#t.about()
for key in t.get_columns():
if key in origcols:
continue
val = t.get(key)
if not key in newcols:
newcols[key] = np.zeros(len(T), val.dtype)
T.set(key, newcols[key])
print 'set', key, i, '=', val[0]
newcols[key][i] = val[0]
T.done[i] = 1
#if i and i % 100 == 0:
if False:
fn = 'wisew4phot-interim%s.fits' % tag
T.writeto(fn, header=hdr)
print 'Wrote', fn
T.writeto('wisew4phot%s.fits' % tag, header=hdr)
if opt.empty:
# log file found
allfailed = False
break
if not allfailed:
continue
print 'All jobs failed for sub', sub.id #, 'via ssh failure'
failedsubs.append(sub)
failedjobs.append(failedjob)
print 'Found total of', len(failedsubs), 'failed Submissions'
if opt.rerun:
from process_submissions import try_dosub, try_dojob
if opt.threads is not None:
mp = multiproc(opt.threads)
args = []
for j in failedjobs:
if j is None:
continue
args.append(j.id) #, j.user_image))
mp.map(bounce_try_dojob, args)
else:
for sub in failedsubs:
print 'Re-trying sub', sub.id
try_dosub(sub, 1)
if opt.sub:
sub = Submission.objects.all().get(id=opt.sub)
print('Submission', sub)
def qsocuts(SW):
in1 = ( ((SW.gpsf - SW.ipsf) < 1.5) *
(SW.optpsf > 17.) *
(SW.optpsf < 22.) *
((SW.optmod - SW.wise) > ((SW.gpsf - SW.ipsf) + 3)) *
np.logical_or(SW.ispsf, (SW.optpsf - SW.optmod) < 0.1) )
I = np.flatnonzero(in1)
print('Selected', len(I))
in2 = in1 * (SW.w1mag < 25.) * (SW.w2mag < 25.)
I2 = np.flatnonzero(in2)
print('With w1,w2 < 25:', len(I2))
SW.w1rchi2 = SW.w1_prochi2 / SW.w1_pronpix
in3 = in2 * (SW.w1rchi2 < 10.)
I3 = np.flatnonzero(in3)
print('And chi2/pix < 10:', len(I3))
I = I2
# Check against WISE catalog
#wfn = 'w3-wise.fits'
#WC = fits_table(wfn)
# SDSS matched to WISE catalog
swfn = 'eboss-w3-wise-cat-dr9.fits'
#SWC = fits_table(swfn)
S = fits_table('objs-eboss-w3-dr9.fits')
print('Read', len(S), 'SDSS objects')
wfn = 'w3-wise.fits'
WC = fits_table(wfn)
print('Read', len(WC), 'WISE catalog objects')
R = 4./3600.
I,J,d = match_radec(S.ra, S.dec, WC.ra, WC.dec, R, nearest=True)
print(len(I), 'matches of SDSS to WISE catalog')
SWC = S[I]
for k in WC.columns():
if k in ['ra','dec']:
outkey = k+'_wise'
else:
outkey = k
X = WC.get(k)
SWC.set(outkey, X[J])
SWC.writeto(swfn)
SWC.gpsf = fluxtomag(SWC.psfflux[:,1])
SWC.rpsf = fluxtomag(SWC.psfflux[:,2])
SWC.ipsf = fluxtomag(SWC.psfflux[:,3])
SWC.ispsf = (SWC.objc_type == 6)
SWC.isgal = (SWC.objc_type == 3)
SWC.optpsf = fluxtomag((SWC.psfflux[:,1] * 0.8 +
SWC.psfflux[:,2] * 0.6 +
SWC.psfflux[:,3] * 1.0) / 2.4)
SWC.optmod = fluxtomag((SWC.modelflux[:,1] * 0.8 +
SWC.modelflux[:,2] * 0.6 +
SWC.modelflux[:,3] * 1.0) / 2.4)
SWC.wise = fluxtomag((SWC.w1mpro * 1.0 +
SWC.w2mpro * 0.5) / 1.5)
in1 = ( ((SWC.gpsf - SWC.ipsf) < 1.5) *
(SWC.optpsf > 17.) *
(SWC.optpsf < 22.) *
((SWC.optpsf - SWC.wise) > ((SWC.gpsf - SWC.ipsf) + 3)) *
np.logical_or(SWC.ispsf, (SWC.optpsf - SWC.optmod) < 0.1) )
I = np.flatnonzero(in1)
print('Selected', len(I), 'from WISE catalog')
sys.exit(0)
worstI = I[np.argsort(-SW.w1rchi2[I])]
print('Worst:')
mp = multiproc(1)
for wi,i in enumerate(worstI):
print(' %8.3f, %8.3f, chi2/npix %g' % (SW.ra[i], SW.dec[i], SW.w1rchi2[i]))
ra, dec = SW.ra[i], SW.dec[i]
ri = int((ra - r0) / (rr[1]-rr[0]))
di = int((dec - d0) / (dd[1]-dd[0]))
print(' ri %i, di %i' % (ri,di))
if SW.w1rchi2[i] > 25:
continue
opt = myopts()
wbasefn = 'eboss-w3-worst%04i-r%02i-d%02i' % (wi, ri,di)
opt.picklepat = '%s-stage%%0i.pickle' % wbasefn
opt.ps = wbasefn
opt.minflux = None
opt.bandnum = 1
opt.osources = None
opt.sources = 'objs-eboss-w3-dr9.fits'
opt.ptsrc = False
opt.pixpsf = False
# opt.minsb = 0.05
opt.minsb = 0.005
opt.write = True
opt.force = [205]
opt.ri = ri
opt.di = di
import wise3
import tractor
pcat = []
pcat.append(tractor.PointSource(RaDecPos(ra, dec), None))
R = wise3.runtostage(205, opt, mp, rr[ri],rr[ri+1],dd[di],dd[di+1],
ttsuf='chi2/npix %g' % SW.w1rchi2[i],
pcat=pcat, addSky=True)
### Try to use the stored solved values -- actually doesn't make things
### much faster.
# R = wise3.runtostage(103, opt, mp, rr[ri],rr[ri+1],dd[di],dd[di+1],
# ttsuf='chi2/npix %g' % SW.w1rchi2[i],
# pcat=pcat)
# t = R['tractor']
# T = fits_table('ebossw3-v4-r%02i-d%02i-w1.fits' % (ri, di))
# assert(len(t.catalog) == len(T))
# for i,src in enumerate(t.catalog):
# print('Source', src)
# assert(src.getPosition().ra == T.ra [i])
# assert(src.getPosition().dec == T.dec[i])
# t.catalog.freezeParamsRecursive('*')
# t.catalog.thawPathsTo('w1')
# assert(len(t.catalog.getParams()) == len(T))
# t.catalog.setParams(T.w1)
# R2 = wise3.stage204(opt=opt, mp=mp, ri=opt.ri, di=opt.di, **R)
# R2['ims1'] = R2['ims0']
# ps = PlotSequence(wbasefn)
# R3 = wise3.stage205(opt=opt, mp=mp, ri=opt.ri, di=opt.di, ps=ps, **R2)
plt.clf()
plt.hist(SW.w1mag, 100, range=(10,30), histtype='step', color='b', log=True)
plt.hist(SW.w1mag[I], 100, range=(10,30), histtype='step', color='r', log=True)
plt.xlabel('W1 mag')
ylo,yhi = plt.ylim()
plt.ylim(0.3, yhi)
ps.savefig()
plt.clf()
plt.hist(SW.w2mag, 100, range=(10,30), histtype='step', color='b', log=True)
plt.hist(SW.w2mag[I], 100, range=(10,30), histtype='step', color='r', log=True)
plt.xlabel('W2 mag')
ylo,yhi = plt.ylim()
plt.ylim(0.3, yhi)
ps.savefig()
plt.clf()
plt.hist(np.log10(SW.w1_prochi2 / SW.w1_pronpix), 100, range=(0,3),
log=True, histtype='step', color='b')
plt.hist(np.log10(SW.w1_prochi2[I] / SW.w1_pronpix[I]), 100, range=(0,3),
log=True, histtype='step', color='r')
plt.xlabel('log chi2/npix')
ylo,yhi = plt.ylim()
plt.ylim(0.3, yhi)
ps.savefig()
gz = (100*len(bad)/len(T))
if max(pg, pr, pz, gg, gr, gz) < 10:
continue
print '%3i' % pg, '% bad g ptsrc'
print '%3i' % pr, '% bad r ptsrc'
print '%3i' % pz, '% bad z ptsrc'
print '%3i' % gg, '% bad g gal'
print '%3i' % gr, '% bad r gal'
print '%3i' % gz, '% bad z gal'
# -> p202 through p242
sys.exit(0)
bricks.cut((bricks.dec > 20.1) * (bricks.dec < 24.3))
print len(bricks), 'bricks to re-run'
for brick in bricks.brickname:
add_depth_tag(decals, brick, outdir, overwrite=True)
if True:
for brick in bricks.brickname:
add_depth_tag(decals, brick, outdir)
else:
# totally I/O-bound; this doesn't help.
from astrometry.util.multiproc import *
mp = multiproc(24)
mp.map(bounce_add_depth_tag,
[(decals, brick, outdir) for brick in bricks.brickname])