def main():
# load in parsed args
object_file = results.source_file
outdir = results.output_dir
idx_keep = int(results.idx_keep)
Nproc = results.num_proc
print """
=================================================================
BUILD DATASET CALLED:
object_file = {object_file}
outdir = {outdir}
idx_keep = {idx_keep}
Nproc = {Nproc}
=================================================================
""".format(object_file = object_file,
outdir = outdir,
idx_keep = idx_keep,
Nproc = Nproc)
# load and clean objects
T = load_and_clean_objects(object_file, outdir=outdir, idx_keep=idx_keep)
# write object cat files - will correspond to each stamp
#write_cat_files(T, outdir=outdir)
cutToPrimary = False
# grab the necessary information for each source
stars = [(ra,dec,[],cutToPrimary,outdir) for ra,dec in zip(T.ra, T.dec)]
# Write out stamps and cat files
mp = amultiproc.multiproc(1)
mp.map(_bounce_one_blob, stars)
def main():
T = fits_table('data/wssa/tiles/wisetile-index-allsky.fits.gz')
print(len(T), 'tiles')
args = T.fname
mp = multiproc(16)
mp.map(ext_one, args)
sys.exit(0)
T.magzp = np.zeros(len(T), np.float32)
T.ra1 = np.zeros(len(T), np.float32)
T.ra2 = np.zeros(len(T), np.float32)
ddec = 5.
T.dec1 = np.maximum(-90., T.dec - ddec)
T.dec2 = np.minimum(+90., T.dec + ddec)
decs = np.unique(T.dec)
for d in decs:
print('Dec', d)
I = np.flatnonzero(T.dec == d)
print('RAs:', T.ra[I])
print('dRA:', np.diff(T.ra[I]))
if len(I) == 1:
T.ra1[I] = 0.
T.ra2[I] = 360.
else:
dra = np.diff(T.ra[I])[0]
T.ra1[I] = T.ra[I] - dra/2.
T.ra2[I] = T.ra[I] + dra/2.
T.ra1 += (T.ra1 < 0)*360.
T.ra2 += (T.ra2 > 360)*(-360.)
for ii,t in enumerate(T):
fn = os.path.join('data','wssa','tiles',t.fname+'.gz')
print('File', fn)
cmd = 'imcopy data/wssa/tiles/%s.gz"+1" data/wssa/%s' % (t.fname, t.fname)
print(cmd)
if os.system(cmd):
break
fn = 'data/wssa/'+t.fname.strip()
hdr = fitsio.read_header(fn)
zp = hdr['MAGZP']
print('ZP', zp)
T.magzp[ii] = zp
T.writeto('data/wssa/wssa-bricks.fits')
def refit_galaxies_1():
import optparse
parser = optparse.OptionParser(usage='%prog [options] <NGC-number>')
parser.add_option('--threads', dest='threads', type=int, default=None,
help='use multiprocessing')
opt,args = parser.parse_args()
mp = multiproc(nthreads=opt.threads)
'''
select fracdev_i,exprad_i,expab_i,expmag_i,expphi_i,run,camcol,field,ra,dec,flags
into mydb.exp5 from Galaxy where
exprad_i > 3
and expmag_i < 20
and clean=1 and probpsf=0
and fracdev_i < 0.5
select * from mydb.exp5 into mydb.exp5b
where (flags & 0x10000000) = 0x10000000
and (flags & 0x3002000a0020008) = 0
'''
#T = fits_table('exp4_dstn.fit')
Ti = fits_table('exp5b_dstn.fit')
rlo,rhi = 4.1, 4.4
Ti = T[(T.exprad_i > rlo) * (T.exprad_i < rhi)]
Ti = Ti[Ti.expmag_i < 19]
I = np.argsort(Ti.expmag_i)
###
#I = I[:4]
#I = I[7:8]
Ti = Ti[I]
print 'Cut to', len(Ti), 'galaxies in radius and mag cuts'
refit_galaxies(Ti, intermediate_fn='mye4-%06i.fits', mp=mp,
modswitch=True)
Ti.writeto('mye4.fits')
def main(survey=None, opt=None):
print(' '.join(sys.argv))
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = tlast = Time()
if opt.skip and os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0,x1,y0,y1) = opt.zoom
zoomslice = (slice(y0,y1), slice(x0,x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing first arg as exposure number (otherwise, it's a filename)
try:
expnum = int(opt.expnum)
filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
filename = opt.expnum
# Try parsing HDU: "all" or HDU name or HDU number.
all_hdus = (opt.ccdname == 'all')
hdu = -1
ccdname = None
if not all_hdus:
try:
hdu = int(opt.ccdname)
except:
ccdname = opt.ccdname
if survey is None:
survey = LegacySurveyData()
catsurvey = survey
if opt.catalog_dir is not None:
catsurvey = LegacySurveyData(survey_dir = opt.catalog_dir)
if filename is not None and hdu >= 0:
# FIXME -- try looking up in CCDs file?
# Read metadata from file
print('Warning: faking metadata from file contents')
T = exposure_metadata([filename], hdus=[hdu])
print('Metadata:')
T.about()
if not 'ccdzpt' in T.columns():
phdr = fitsio.read_header(filename)
T.ccdzpt = np.array([phdr['MAGZERO']])
print('WARNING: using header MAGZERO')
T.ccdraoff = np.array([0.])
T.ccddecoff = np.array([0.])
print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.')
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and ccdname', ccdname)
if hdu >= 0:
T.cut(T.image_hdu == hdu)
print(len(T), 'with HDU', hdu)
if filename is not None:
T.cut(np.array([f.strip() == filename for f in T.image_filename]))
print(len(T), 'with filename', filename)
if opt.camera is not None:
T.cut(T.camera == opt.camera)
print(len(T), 'with camera', opt.camera)
if not all_hdus:
assert(len(T) == 1)
args = []
for ccd in T:
args.append((survey, catsurvey, ccd, opt, zoomslice, ps))
if opt.threads:
from astrometry.util.multiproc import multiproc
from astrometry.util.timingpool import TimingPool, TimingPoolMeas
pool = TimingPool(opt.threads)
poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
Time.add_measurement(poolmeas)
mp = multiproc(None, pool=pool)
tm = Time()
FF = mp.map(bounce_one_ccd, args)
print('Multi-processing forced-phot:', Time()-tm)
else:
FF = map(bounce_one_ccd, args)
FF = [F for F in FF if F is not None]
if len(FF) == 0:
print('No photometry results to write.')
return 0
# Keep only the first header
_,version_hdr = FF[0]
FF = [F for F,hdr in FF]
F = merge_tables(FF)
if all_hdus:
version_hdr.delete('CPHDU')
version_hdr.delete('CCDNAME')
version_hdr.delete('EXPOSURE')
hdr = fitsio.FITSHDR()
units = {'exptime':'sec',
'flux':'nanomaggy', 'flux_ivar':'1/nanomaggy^2',
'apflux':'nanomaggy', 'apflux_ivar':'1/nanomaggy^2',
'psfdepth':'1/nanomaggy^2', 'galdepth':'1/nanomaggy^2',
'sky':'nanomaggy/arcsec^2', 'psfsize':'arcsec' }
if opt.derivs:
units.update({'dra':'arcsec', 'ddec':'arcsec',
'dra_ivar':'1/arcsec^2', 'ddec_ivar':'1/arcsec^2'})
columns = F.get_columns()
for i,col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.outfn))
fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
F.writeto(tmpfn, header=hdr, append=True)
os.rename(tmpfn, opt.outfn)
print('Wrote', opt.outfn)
tnow = Time()
print('Total:', tnow-t0)
return 0
def general(name,ra,dec,remradius,fieldradius,threads=None,itune1=5,itune2=5,ntune=0,nocache=False,scale=1,ab=1.,angle=0.):
#Radius should be in arcminutes
if threads:
mp = multiproc(nthreads=threads)
else:
mp = multiproc()
IRLS_scale = 25.
dr9 = True
dr8 = False
noarcsinh = False
print name
prefix = 'swapCG_%s' % (name.replace(' ', '_'))
print 'Removal Radius', remradius
print 'Field Radius', fieldradius
print 'RA,Dec', ra, dec
print os.getcwd()
print ra,dec,math.hypot(fieldradius,13./2.)
rcfs = radec_to_sdss_rcf(ra,dec,radius=math.hypot(fieldradius,13./2.),tablefn="dr9fields.fits")
print 'RCFS:', rcfs
print len(rcfs)
assert(len(rcfs)>0)
if 10 <= len(rcfs) < 20:
scale = 2
elif 20 <= len(rcfs) < 40:
scale = 4
elif 40 <= len(rcfs) < 80:
scale = 8
assert(len(rcfs)<80)
sras, sdecs, smags = tychoMatch(ra,dec,(fieldradius*1.5)/60.)
imkw = dict(psf='kl-gm')
if dr9:
getim = st.get_tractor_image_dr9
getsrc = st.get_tractor_sources_dr9
imkw.update(zrange=[-3,100])
elif dr8:
getim = st.get_tractor_image_dr8
getsrc = st.get_tractor_sources_dr8
imkw.update(zrange=[-3,100])
else:
getim = st.get_tractor_image
getsrc = st.get_tractor_sources_dr8
imkw.update(useMags=True)
bands=['u','g','r','i','z']
#bands=['r']
bandname = 'r'
flipBands = ['r']
print rcfs
imsrcs = mp.map(get_ims_and_srcs, [(rcf + (bands, ra, dec, fieldradius*60./0.396, imkw, getim, getsrc))
for rcf in rcfs])
timgs = []
sources = []
allsources = []
for ims,s in imsrcs:
if ims is None:
continue
if s is None:
continue
if scale > 1:
for im in ims:
timgs.append(st.scale_sdss_image(im,scale))
else:
timgs.extend(ims)
allsources.extend(s)
sources.append(s)
#rds = [rcf[3:5] for rcf in rcfs]
#plotarea(ra, dec, fieldradius, name, prefix, timgs) #, rds)
#lvl = logging.DEBUG
#logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout)
tractor = st.Tractor(timgs, allsources, mp=mp)
sa = dict(debug=True, plotAll=False,plotBands=False)
if noarcsinh:
sa.update(nlscale=0)
elif dr8 or dr9:
sa.update(chilo=-8.,chihi=8.)
if nocache:
tractor.cache = NullCache()
sg.disable_galaxy_cache()
zr = timgs[0].zr
print "zr is: ",zr
print bands
print "Number of images: ", len(timgs)
#for timg,band in zip(timgs,bands):
# data = timg.getImage()/np.sqrt(timg.getInvvar())
# plt.hist(data,bins=100)
# plt.savefig('hist-%s.png' % (band))
saveAll('initial-'+prefix, tractor,**sa)
#plotInvvar('initial-'+prefix,tractor)
for sra,sdec,smag in zip(sras,sdecs,smags):
for img in tractor.getImages():
wcs = img.getWcs()
starx,stary = wcs.positionToPixel(RaDecPos(sra,sdec))
starr=25*(2**(max(11-smag,0.)))
if starx+starr<0. or starx-starr>img.getWidth() or stary+starr <0. or stary-starr>img.getHeight():
continue
X,Y = np.meshgrid(np.arange(img.getWidth()), np.arange(img.getHeight()))
R2 = (X - starx)**2 + (Y - stary)**2
img.getStarMask()[R2 < starr**2] = 0
for timgs,sources in imsrcs:
timg = timgs[0]
wcs = timg.getWcs()
xtr,ytr = wcs.positionToPixel(RaDecPos(ra,dec))
xt = xtr
yt = ytr
r = ((remradius*60.))/.396 #radius in pixels
for src in sources:
xs,ys = wcs.positionToPixel(src.getPosition(),src)
if (xs-xt)**2+(ys-yt)**2 <= r**2:
#print "Removed:", src
#print xs,ys
tractor.removeSource(src)
#saveAll('removed-'+prefix, tractor,**sa)
newShape = sg.GalaxyShape((remradius*60.)/10.,ab,angle)
newBright = ba.Mags(r=15.0,g=15.0,u=15.0,z=15.0,i=15.0,order=['u','g','r','i','z'])
EG = st.ExpGalaxy(RaDecPos(ra,dec),newBright,newShape)
print EG
tractor.addSource(EG)
saveAll('added-'+prefix,tractor,**sa)
#print 'Tractor has', tractor.getParamNames()
for im in tractor.images:
im.freezeAllParams()
im.thawParam('sky')
tractor.catalog.freezeAllBut(EG)
#print 'Tractor has', tractor.getParamNames()
#print 'values', tractor.getParams()
for i in range(itune1):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('itune1-%d-' % (i+1)+prefix,tractor,**sa)
tractor.clearCache()
sg.get_galaxy_cache().clear()
gc.collect()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
CGPos = EG.getPosition()
CGShape1 = EG.getShape().copy()
CGShape2 = EG.getShape().copy()
EGBright = EG.getBrightness()
CGu = EGBright[0] + 0.75
CGg = EGBright[1] + 0.75
CGr = EGBright[2] + 0.75
CGi = EGBright[3] + 0.75
CGz = EGBright[4] + 0.75
CGBright1 = ba.Mags(r=CGr,g=CGg,u=CGu,z=CGz,i=CGi,order=['u','g','r','i','z'])
CGBright2 = ba.Mags(r=CGr,g=CGg,u=CGu,z=CGz,i=CGi,order=['u','g','r','i','z'])
print EGBright
print CGBright1
CG = st.CompositeGalaxy(CGPos,CGBright1,CGShape1,CGBright2,CGShape2)
tractor.removeSource(EG)
tractor.addSource(CG)
tractor.catalog.freezeAllBut(CG)
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
for i in range(itune2):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('itune2-%d-' % (i+1)+prefix,tractor,**sa)
tractor.clearCache()
sg.get_galaxy_cache().clear()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.catalog.thawAllParams()
for i in range(ntune):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('ntune-%d-' % (i+1)+prefix,tractor,**sa)
#plotInvvar('final-'+prefix,tractor)
sa.update(plotBands=True)
saveAll('allBands-' + prefix,tractor,**sa)
print "end of first round of optimization:", tractor.getLogLikelihood()
print CG
print CG.getPosition()
print CGBright1
print CGBright2
print CGShape1
print CGShape2
print CGBright1+CGBright2
print CG.getBrightness()
pfn = '%s.pickle' % prefix
pickle_to_file(CG,pfn)
makeflipbook(prefix,len(tractor.getImages()),itune1,itune2,ntune)
# now SWAP exp and dev and DO IT AGAIN
newCG = st.CompositeGalaxy(CG.getPosition, CG.brightnessDev.copy(),
CG.shapeDev.copy(), CG.brightnessExp.copy(),
CG.shapeExp.copy())
tractor.removeSource(CG)
tractor.addSource(newCG)
tractor.catalog.thawAllParams()
for i in range(ntune):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('ntune-swap-%d-' % (i+1)+prefix,tractor,**sa)
#plotInvvar('final-'+prefix,tractor)
sa.update(plotBands=True)
saveAll('allBands-swap-' + prefix,tractor,**sa)
print "end of second (swapped) round of optimization:", tractor.getLogLikelihood()
print newCG
print newCG.getPosition()
print newCG.getBrightness()
pfn = '%s-swap.pickle' % prefix
pickle_to_file(newCG,pfn)
makeflipbook(prefix+"-swap",len(tractor.getImages()),itune1,itune2,ntune)
def find_alignments(fns, wcsfns, gaia_fn, aff_fn, aligned_fn):
from astrometry.libkd.spherematch import tree_build_radec, trees_match
from astrometry.libkd.spherematch import match_radec
from astrometry.util.plotutils import plothist
from astrometry.util.util import Tan
import fitsio
from astrom_common import getwcsoutline
from singles import find_overlaps
if True:
WCS = []
for fn in wcsfns:
wcs = Tan(fn)
WCS.append(wcs)
names = [fn.replace('-bright.fits', '') for fn in fns]
outlines = [getwcsoutline(wcs) for wcs in WCS]
overlaps, areas = find_overlaps(outlines)
print('Reading tables...')
TT = [fits_table(fn) for fn in fns]
print('Building trees...')
kds = [tree_build_radec(T.ra, T.dec) for T in TT]
for T, name in zip(TT, names):
T.name = np.array([name] * len(T))
allra = np.hstack([T.ra for T in TT])
alldec = np.hstack([T.dec for T in TT])
minra = np.min(allra)
maxra = np.max(allra)
mindec = np.min(alldec)
maxdec = np.max(alldec)
print('RA,Dec range:', minra, maxra, mindec, maxdec)
plothist(allra, alldec)
plt.axis([maxra, minra, mindec, maxdec])
plt.xlabel('RA (deg)')
plt.ylabel('Dec (deg)')
plt.savefig('match-all.png')
Tref = fits_table(gaia_fn)
r_arcsec = 0.2
I, J, d = match_radec(Tref.ra, Tref.dec, allra, alldec, r_arcsec / 3600.)
dec = alldec[J]
cosdec = np.cos(np.deg2rad(dec))
dr = (Tref.ra[I] - allra[J]) * cosdec * 3600.
dd = (Tref.dec[I] - alldec[J]) * 3600.
plt.clf()
rr = (-r_arcsec * 1000, +r_arcsec * 1000)
plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr))
plt.xlabel('dRA (milli-arcsec)')
plt.ylabel('dDec (milli-arcsec)')
plt.savefig('match-all-ref-before.png')
# Initial matching of all stars
r_arcsec = 0.2
I, J, d = match_radec(allra,
alldec,
allra,
alldec,
r_arcsec / 3600.,
notself=True)
dec = alldec[I]
cosdec = np.cos(np.deg2rad(dec))
dr = (allra[I] - allra[J]) * cosdec * 3600.
dd = (alldec[I] - alldec[J]) * 3600.
plt.clf()
rr = (-r_arcsec * 1000, +r_arcsec * 1000)
plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr))
plt.xlabel('dRA (milli-arcsec)')
plt.ylabel('dDec (milli-arcsec)')
plt.savefig('match-all-before.png')
hulls = []
from scipy.spatial import ConvexHull
for T in TT:
hull = ConvexHull(np.vstack((T.ra, T.dec)).T)
ra = T.ra[hull.vertices]
ra = np.append(ra, ra[0])
dec = T.dec[hull.vertices]
dec = np.append(dec, dec[0])
hulls.append((ra, dec))
aligns = {}
#for i in []:
for i in range(len(kds)):
for j in range(i + 1, len(kds)):
print('Matching trees', i, 'and', j)
r_arcsec = 0.2
radius = np.deg2rad(r_arcsec / 3600)
I, J, d2 = trees_match(kds[i], kds[j], radius)
print(len(I), 'matches')
if len(I) == 0:
continue
Ti = TT[i]
Tj = TT[j]
dec = Ti[I].dec
cosdec = np.cos(np.deg2rad(dec))
dr = (Ti[I].ra - Tj[J].ra) * cosdec * 3600.
dd = (Ti[I].dec - Tj[J].dec) * 3600.
if False:
al = Alignment(Ti, Tj, searchradius=r_arcsec)
print('Aligning...')
if not al.shift():
print('Failed to find Alignment between fields')
continue
aligns[(i, j)] = al
plt.clf()
plotalignment(al)
plt.savefig('match-align-%02i-%02i.png' % (i, j))
plt.clf()
#plothist(np.append(Ti.ra, Tj.ra), np.append(Ti.dec, Tj.dec), docolorbar=False, doclf=False, dohot=False,
# imshowargs=dict(cmap=antigray))
plothist(Ti.ra[I], Ti.dec[I], docolorbar=False, doclf=False)
r, d = hulls[i]
plt.plot(r, d, 'r-')
r, d = hulls[j]
plt.plot(r, d, 'b-')
mra = Ti.ra[I]
mdec = Ti.dec[I]
mnra = np.min(mra)
mxra = np.max(mra)
mndec = np.min(mdec)
mxdec = np.max(mdec)
plt.plot([mnra, mnra, mxra, mxra, mnra],
[mndec, mxdec, mxdec, mndec, mndec], 'g-')
plt.axis([maxra, minra, mindec, maxdec])
plt.xlabel('RA (deg)')
plt.ylabel('Dec (deg)')
plt.savefig('match-radec-%02i-%02i.png' % (i, j))
plt.clf()
rr = (-r_arcsec, +r_arcsec)
plothist(dr, dd, nbins=100, range=(rr, rr))
plt.xlabel('dRA (arcsec)')
plt.ylabel('dDec (arcsec)')
plt.savefig('match-dradec-%02i-%02i.png' % (i, j))
#for roundi,(Nk,R) in enumerate(NkeepRads):
refrad = 0.15
targetrad = 0.005
ps = PlotSequence('shift')
from astrom_intra import intrabrickshift
from singles import plot_all_alignments
#Rads = [0.25, 0.1]
Rads = [0.2, 0.050, 0.020]
#Rads = [0.1]
affs = None
# this is the reference point around which rotations take place, NOT reference catalog stars.
refrd = None
for roundi, R in enumerate(Rads):
if roundi > 0:
refrad = 0.050
TT1 = TT
nb = int(np.ceil(R / targetrad))
nb = max(nb, 5)
if nb % 2 == 0:
nb += 1
print('Round', roundi + 1, ': matching with radius', R)
print('Nbins:', nb)
# kwargs to pass to intrabrickshift
ikwargs = {}
minoverlap = 0.01
tryoverlaps = (overlaps > minoverlap)
ikwargs.update(
do_affine=True, #mp=mp,
#alignplotargs=dict(bins=25),
alignplotargs=dict(bins=50),
overlaps=tryoverlaps)
ikwargs.update(ref=Tref, refrad=refrad)
# kwargs to pass to Alignment
akwargs = {}
i1 = intrabrickshift(TT1,
matchradius=R,
refradecs=refrd,
align_kwargs=dict(histbins=nb, **akwargs),
**ikwargs)
refrd = i1.get_reference_radecs()
filts = ['' for n in names]
ap = i1.alplotgrid
Nk = 100000
plot_all_alignments(ap, R * 1000, refrad * 1000, roundi + 1, names,
filts, ps, overlaps, outlines, Nk)
for T, aff in zip(TT, i1.affines):
T.ra, T.dec = aff.apply(T.ra, T.dec)
if affs is None:
affs = i1.affines
else:
for a, a2 in zip(affs, i1.affines):
a.add(a2)
from astrom_common import Affine
T = Affine.toTable(affs)
T.filenames = fns
#T.flt = fltfns
#T.gst = gstfns
#T.chip = chips
# FAKE -- used as a name in alignment_plots
T.gst = np.array([n + '.gst.fits' for n in names])
T.writeto(aff_fn)
# Final matching of all stars
allra = np.hstack([T.ra for T in TT])
alldec = np.hstack([T.dec for T in TT])
r_arcsec = 0.2
I, J, d = match_radec(allra,
alldec,
allra,
alldec,
r_arcsec / 3600.,
notself=True)
dec = alldec[I]
cosdec = np.cos(np.deg2rad(dec))
dr = (allra[I] - allra[J]) * cosdec * 3600.
dd = (alldec[I] - alldec[J]) * 3600.
plt.clf()
rr = (-r_arcsec * 1000, +r_arcsec * 1000)
plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr))
plt.xlabel('dRA (milli-arcsec)')
plt.ylabel('dDec (milli-arcsec)')
plt.savefig('match-all-after.png')
I, J, d = match_radec(Tref.ra, Tref.dec, allra, alldec, r_arcsec / 3600.)
dec = alldec[J]
cosdec = np.cos(np.deg2rad(dec))
dr = (Tref.ra[I] - allra[J]) * cosdec * 3600.
dd = (Tref.dec[I] - alldec[J]) * 3600.
plt.clf()
rr = (-r_arcsec * 1000, +r_arcsec * 1000)
plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr))
plt.xlabel('dRA (milli-arcsec)')
plt.ylabel('dDec (milli-arcsec)')
plt.savefig('match-all-ref-after.png')
r_arcsec = 0.02
I, J, d = match_radec(allra,
alldec,
allra,
alldec,
r_arcsec / 3600.,
notself=True)
dec = alldec[I]
cosdec = np.cos(np.deg2rad(dec))
dr = (allra[I] - allra[J]) * cosdec * 3600.
dd = (alldec[I] - alldec[J]) * 3600.
plt.clf()
rr = (-r_arcsec * 1000, +r_arcsec * 1000)
plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr))
plt.xlabel('dRA (milli-arcsec)')
plt.ylabel('dDec (milli-arcsec)')
plt.savefig('match-all-after2.png')
T = fits_table()
T.ra = allra
T.dec = alldec
for col in [
'f814w_vega', 'f475w_vega', 'f336w_vega', 'f275w_vega',
'f110w_vega', 'f160w_vega', 'name'
]:
T.set(col, np.hstack([t.get(col) for t in TT]))
T.writeto(aligned_fn)
if False:
from singles import alignment_plots
dataset = 'M31'
Nkeep = 100000
R = 0.1
minoverlap = 0.01
perfield = False
nocache = True
from astrometry.util.multiproc import multiproc
mp = multiproc()
filts = ['F475W' for n in names]
chips = [-1] * len(names)
exptimes = [1] * len(names)
Nall = [0] * len(names)
rd = (minra, maxra, mindec, maxdec)
cnames = names
meta = (chips, names, cnames, filts, exptimes, Nall, rd)
alignment_plots(afffn,
dataset,
Nkeep,
0,
R,
minoverlap,
perfield,
nocache,
mp,
0,
tables=(TT, outlines, meta),
lexsort=False)
def main():
import optparse
parser = optparse.OptionParser(usage="%prog [options] <NGC-number>")
parser.add_option("--threads", dest="threads", type=int, help="use multiprocessing")
parser.add_option("--itune1", dest="itune1", type=int, help="Individual tuning, first stage", default=5)
parser.add_option("--itune2", dest="itune2", type=int, help="Individual tuning, second stage", default=5)
parser.add_option("--ntune", dest="ntune", type=int, help="All objects tuning", default=0)
parser.add_option(
"--nocache", dest="nocache", action="store_true", default=False, help="Disable caching for memory reasons"
)
opt, args = parser.parse_args()
if len(args) != 1:
parser.print_help()
sys.exit(-1)
if opt.threads:
mp = multiproc(nthreads=opt.threads)
else:
mp = multiproc()
ngc = int(args[0])
j = getNGC(ngc)
print j
ra = float(j["RA"][0])
dec = float(j["DEC"][0])
itune1 = opt.itune1
itune2 = opt.itune2
ntune = opt.ntune
IRLS_scale = 25.0
radius = (10.0 ** j["LOG_D25"][0]) / 10.0
dr8 = True
noarcsinh = False
print "Radius", radius
print "RA,Dec", ra, dec
sras, sdecs, smags = tychoMatch(ra, dec, (radius * 1.5) / 60.0)
for sra, sdec, smag in zip(sras, sdecs, smags):
print sra, sdec, smag
rcfs = radec_to_sdss_rcf(ra, dec, radius=math.hypot(radius, 13.0 / 2.0), tablefn="dr8fields.fits")
print rcfs
# fieldPlot(ra,dec,radius,ngc)
imkw = dict(psf="dg")
if dr8:
getim = st.get_tractor_image_dr8
getsrc = st.get_tractor_sources_dr8
imkw.update(zrange=[-3, 100])
else:
getim = st.get_tractor_image
getsrc = st.get_tractor_sources_dr8
imkw.update(useMags=True)
bands = ["u", "g", "r", "i", "z"]
# bands=['r']
bandname = "r"
flipBands = ["r"]
imsrcs = mp.map(
get_ims_and_srcs, [(rcf + (bands, ra, dec, radius * 60.0 / 0.396, imkw, getim, getsrc)) for rcf in rcfs]
)
timgs = []
sources = []
allsources = []
for ims, s in imsrcs:
if ims is None:
continue
if s is None:
continue
timgs.extend(ims)
allsources.extend(s)
sources.append(s)
# rds = [rcf[3:5] for rcf in rcfs]
plotarea(ra, dec, radius, ngc, timgs) # , rds)
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format="%(message)s", stream=sys.stdout)
tractor = st.Tractor(timgs, allsources, mp=mp)
sa = dict(debug=True, plotAll=False, plotBands=False)
if noarcsinh:
sa.update(nlscale=0)
elif dr8:
sa.update(chilo=-8.0, chihi=8.0)
if opt.nocache:
tractor.cache = NullCache()
sg.disable_galaxy_cache()
zr = timgs[0].zr
print "zr is: ", zr
print bands
print "Number of images: ", len(timgs)
# for timg,band in zip(timgs,bands):
# data = timg.getImage()/np.sqrt(timg.getInvvar())
# plt.hist(data,bins=100)
# plt.savefig('hist-%s.png' % (band))
prefix = "ngc%d" % (ngc)
saveAll("initial-" + prefix, tractor, **sa)
# plotInvvar('initial-'+prefix,tractor)
for sra, sdec, smag in zip(sras, sdecs, smags):
print sra, sdec, smag
for img in tractor.getImages():
wcs = img.getWcs()
starx, stary = wcs.positionToPixel(RaDecPos(sra, sdec))
starr = 25 * (2 ** (max(11 - smag, 0.0)))
if (
starx + starr < 0.0
or starx - starr > img.getWidth()
or stary + starr < 0.0
or stary - starr > img.getHeight()
):
continue
X, Y = np.meshgrid(np.arange(img.getWidth()), np.arange(img.getHeight()))
R2 = (X - starx) ** 2 + (Y - stary) ** 2
img.getStarMask()[R2 < starr ** 2] = 0
# star = [(x,y) for x in range(img.getWidth()) for y in range(img.getHeight()) if (x-starx)**2+(y-stary)**2 <= starr**2]
# for (x,y) in star:
# img.getStarMask()[y][x] = 0
for timgs, sources in imsrcs:
timg = timgs[0]
wcs = timg.getWcs()
xtr, ytr = wcs.positionToPixel(RaDecPos(ra, dec))
print xtr, ytr
xt = xtr
yt = ytr
r = ((radius * 60.0)) / 0.396 # radius in pixels
for src in sources:
xs, ys = wcs.positionToPixel(src.getPosition(), src)
if (xs - xt) ** 2 + (ys - yt) ** 2 <= r ** 2:
print "Removed:", src
print xs, ys
tractor.removeSource(src)
# saveAll('removed-'+prefix, tractor,**sa)
newShape = sg.GalaxyShape(30.0, 1.0, 0.0)
newBright = ba.Mags(r=15.0, g=15.0, u=15.0, z=15.0, i=15.0, order=["u", "g", "r", "i", "z"])
EG = st.ExpGalaxy(RaDecPos(ra, dec), newBright, newShape)
print EG
tractor.addSource(EG)
saveAll("added-" + prefix, tractor, **sa)
print "Tractor has", tractor.getParamNames()
for im in tractor.images:
im.freezeAllParams()
im.thawParam("sky")
tractor.catalog.freezeAllBut(EG)
print "Tractor has", tractor.getParamNames()
print "values", tractor.getParams()
for i in range(itune1):
tractor.optimize()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.changeInvvar(IRLS_scale)
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
saveAll("itune1-%d-" % (i + 1) + prefix, tractor, **sa)
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.clearCache()
sg.get_galaxy_cache().clear()
gc.collect()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
CGPos = EG.getPosition()
CGShape1 = EG.getShape().copy()
CGShape2 = EG.getShape().copy()
EGBright = EG.getBrightness()
CGu = EGBright[0] + 0.75
CGg = EGBright[1] + 0.75
CGr = EGBright[2] + 0.75
CGi = EGBright[3] + 0.75
CGz = EGBright[4] + 0.75
CGBright1 = ba.Mags(r=CGr, g=CGg, u=CGu, z=CGz, i=CGi, order=["u", "g", "r", "i", "z"])
CGBright2 = ba.Mags(r=CGr, g=CGg, u=CGu, z=CGz, i=CGi, order=["u", "g", "r", "i", "z"])
print EGBright
print CGBright1
CG = st.CompositeGalaxy(CGPos, CGBright1, CGShape1, CGBright2, CGShape2)
tractor.removeSource(EG)
tractor.addSource(CG)
tractor.catalog.freezeAllBut(CG)
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
for i in range(itune2):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll("itune2-%d-" % (i + 1) + prefix, tractor, **sa)
print resource.getpagesize()
print "RUsage is: ", resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.clearCache()
sg.get_galaxy_cache().clear()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.catalog.thawAllParams()
for i in range(ntune):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll("ntune-%d-" % (i + 1) + prefix, tractor, **sa)
# plotInvvar('final-'+prefix,tractor)
sa.update(plotBands=True)
saveAll("allBands-" + prefix, tractor, **sa)
print CG
print CG.getPosition()
print CGBright1
print CGBright2
print CGShape1
print CGShape2
print CGBright1 + CGBright2
print CG.getBrightness()
pfn = "ngc-%d.pickle" % ngc
pickle_to_file(CG, pfn)
makeflipbook(prefix, len(tractor.getImages()), itune1, itune2, ntune)
def next(self):
return self.y.next()
def __len__(self):
return self.n
def yielder(n):
for i in range(n):
print 'Yielding', i
yield i
N = 20
y = yielder(N)
args = ywrapper(y, N)
dpool = DebugPool(4, taskqueuesize=4)
dmup = multiproc.multiproc(pool=dpool)
Time.add_measurement(DebugPoolMeas(dpool))
t0 = Time()
res = dmup.map(work, args)
print Time()-t0
print 'Got result:', res
sys.exit(0)
from tractor import *
from tractor import sdss as st
from tractor.engine import getmodelimagefunc2
if not (("BOSS_PHOTOOBJ" in os.environ) and ("PHOTO_RESOLVE" in os.environ)):
print(
"""$BOSS_PHOTOOBJ and $PHOTO_RESOLVE not set -- on NERSC, you can do:
export BOSS_PHOTOOBJ=/project/projectdirs/cosmo/data/sdss/pre13/eboss/photoObj.v5b
export PHOTO_RESOLVE=/project/projectdirs/cosmo/data/sdss/pre13/eboss/resolve/2013-07-29
To read SDSS files from the local filesystem rather than downloading them.
"""
)
opt, args = parser.parse_args()
brickid = opt.brick
bands = opt.bands
if opt.threads and opt.threads > 1:
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads)
else:
mp = multiproc()
ps = None
plots = False
decals = Decals()
brick = decals.get_brick(brickid)
print("Chosen brick:")
brick.about()
targetwcs = wcs_for_brick(brick, W=opt.W, H=opt.H)
W, H = targetwcs.get_width(), targetwcs.get_height()
# Read SDSS sources
cat, T = get_sdss_sources(bands, targetwcs)
def general(name,
ra,
dec,
remradius,
fieldradius,
threads=None,
itune1=5,
itune2=5,
ntune=0,
nocache=False,
scale=1,
ab=1.,
angle=0.):
#Radius should be in arcminutes
if threads:
mp = multiproc(nthreads=threads)
else:
mp = multiproc()
IRLS_scale = 25.
dr9 = True
dr8 = False
noarcsinh = False
print name
prefix = 'swapCG_%s' % (name.replace(' ', '_'))
print 'Removal Radius', remradius
print 'Field Radius', fieldradius
print 'RA,Dec', ra, dec
print os.getcwd()
print ra, dec, math.hypot(fieldradius, 13. / 2.)
rcfs = radec_to_sdss_rcf(ra,
dec,
radius=math.hypot(fieldradius, 13. / 2.),
tablefn="dr9fields.fits")
print 'RCFS:', rcfs
print len(rcfs)
assert (len(rcfs) > 0)
if 10 <= len(rcfs) < 20:
scale = 2
elif 20 <= len(rcfs) < 40:
scale = 4
elif 40 <= len(rcfs) < 80:
scale = 8
assert (len(rcfs) < 80)
sras, sdecs, smags = tychoMatch(ra, dec, (fieldradius * 1.5) / 60.)
imkw = dict(psf='kl-gm')
if dr9:
getim = st.get_tractor_image_dr9
getsrc = st.get_tractor_sources_dr9
imkw.update(zrange=[-3, 100])
elif dr8:
getim = st.get_tractor_image_dr8
getsrc = st.get_tractor_sources_dr8
imkw.update(zrange=[-3, 100])
else:
getim = st.get_tractor_image
getsrc = st.get_tractor_sources_dr8
imkw.update(useMags=True)
bands = ['u', 'g', 'r', 'i', 'z']
#bands=['r']
bandname = 'r'
flipBands = ['r']
print rcfs
imsrcs = mp.map(
get_ims_and_srcs,
[(rcf +
(bands, ra, dec, fieldradius * 60. / 0.396, imkw, getim, getsrc))
for rcf in rcfs])
timgs = []
sources = []
allsources = []
for ims, s in imsrcs:
if ims is None:
continue
if s is None:
continue
if scale > 1:
for im in ims:
timgs.append(st.scale_sdss_image(im, scale))
else:
timgs.extend(ims)
allsources.extend(s)
sources.append(s)
#rds = [rcf[3:5] for rcf in rcfs]
#plotarea(ra, dec, fieldradius, name, prefix, timgs) #, rds)
#lvl = logging.DEBUG
#logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout)
tractor = st.Tractor(timgs, allsources, mp=mp)
sa = dict(debug=True, plotAll=False, plotBands=False)
if noarcsinh:
sa.update(nlscale=0)
elif dr8 or dr9:
sa.update(chilo=-8., chihi=8.)
if nocache:
tractor.cache = NullCache()
sg.disable_galaxy_cache()
zr = timgs[0].zr
print "zr is: ", zr
print bands
print "Number of images: ", len(timgs)
#for timg,band in zip(timgs,bands):
# data = timg.getImage()/np.sqrt(timg.getInvvar())
# plt.hist(data,bins=100)
# plt.savefig('hist-%s.png' % (band))
saveAll('initial-' + prefix, tractor, **sa)
#plotInvvar('initial-'+prefix,tractor)
for sra, sdec, smag in zip(sras, sdecs, smags):
for img in tractor.getImages():
wcs = img.getWcs()
starx, stary = wcs.positionToPixel(RaDecPos(sra, sdec))
starr = 25 * (2**(max(11 - smag, 0.)))
if starx + starr < 0. or starx - starr > img.getWidth(
) or stary + starr < 0. or stary - starr > img.getHeight():
continue
X, Y = np.meshgrid(np.arange(img.getWidth()),
np.arange(img.getHeight()))
R2 = (X - starx)**2 + (Y - stary)**2
img.getStarMask()[R2 < starr**2] = 0
for timgs, sources in imsrcs:
timg = timgs[0]
wcs = timg.getWcs()
xtr, ytr = wcs.positionToPixel(RaDecPos(ra, dec))
xt = xtr
yt = ytr
r = ((remradius * 60.)) / .396 #radius in pixels
for src in sources:
xs, ys = wcs.positionToPixel(src.getPosition(), src)
if (xs - xt)**2 + (ys - yt)**2 <= r**2:
#print "Removed:", src
#print xs,ys
tractor.removeSource(src)
#saveAll('removed-'+prefix, tractor,**sa)
newShape = sg.GalaxyShape((remradius * 60.) / 10., ab, angle)
newBright = ba.Mags(r=15.0,
g=15.0,
u=15.0,
z=15.0,
i=15.0,
order=['u', 'g', 'r', 'i', 'z'])
EG = st.ExpGalaxy(RaDecPos(ra, dec), newBright, newShape)
print EG
tractor.addSource(EG)
saveAll('added-' + prefix, tractor, **sa)
#print 'Tractor has', tractor.getParamNames()
for im in tractor.images:
im.freezeAllParams()
im.thawParam('sky')
tractor.catalog.freezeAllBut(EG)
#print 'Tractor has', tractor.getParamNames()
#print 'values', tractor.getParams()
for i in range(itune1):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('itune1-%d-' % (i + 1) + prefix, tractor, **sa)
tractor.clearCache()
sg.get_galaxy_cache().clear()
gc.collect()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
CGPos = EG.getPosition()
CGShape1 = EG.getShape().copy()
CGShape2 = EG.getShape().copy()
EGBright = EG.getBrightness()
CGu = EGBright[0] + 0.75
CGg = EGBright[1] + 0.75
CGr = EGBright[2] + 0.75
CGi = EGBright[3] + 0.75
CGz = EGBright[4] + 0.75
CGBright1 = ba.Mags(r=CGr,
g=CGg,
u=CGu,
z=CGz,
i=CGi,
order=['u', 'g', 'r', 'i', 'z'])
CGBright2 = ba.Mags(r=CGr,
g=CGg,
u=CGu,
z=CGz,
i=CGi,
order=['u', 'g', 'r', 'i', 'z'])
print EGBright
print CGBright1
CG = st.CompositeGalaxy(CGPos, CGBright1, CGShape1, CGBright2, CGShape2)
tractor.removeSource(EG)
tractor.addSource(CG)
tractor.catalog.freezeAllBut(CG)
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
for i in range(itune2):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('itune2-%d-' % (i + 1) + prefix, tractor, **sa)
tractor.clearCache()
sg.get_galaxy_cache().clear()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.catalog.thawAllParams()
for i in range(ntune):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('ntune-%d-' % (i + 1) + prefix, tractor, **sa)
#plotInvvar('final-'+prefix,tractor)
sa.update(plotBands=True)
saveAll('allBands-' + prefix, tractor, **sa)
print "end of first round of optimization:", tractor.getLogLikelihood()
print CG
print CG.getPosition()
print CGBright1
print CGBright2
print CGShape1
print CGShape2
print CGBright1 + CGBright2
print CG.getBrightness()
pfn = '%s.pickle' % prefix
pickle_to_file(CG, pfn)
makeflipbook(prefix, len(tractor.getImages()), itune1, itune2, ntune)
# now SWAP exp and dev and DO IT AGAIN
newCG = st.CompositeGalaxy(CG.getPosition, CG.brightnessDev.copy(),
CG.shapeDev.copy(), CG.brightnessExp.copy(),
CG.shapeExp.copy())
tractor.removeSource(CG)
tractor.addSource(newCG)
tractor.catalog.thawAllParams()
for i in range(ntune):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('ntune-swap-%d-' % (i + 1) + prefix, tractor, **sa)
#plotInvvar('final-'+prefix,tractor)
sa.update(plotBands=True)
saveAll('allBands-swap-' + prefix, tractor, **sa)
print "end of second (swapped) round of optimization:", tractor.getLogLikelihood(
)
print newCG
print newCG.getPosition()
print newCG.getBrightness()
pfn = '%s-swap.pickle' % prefix
pickle_to_file(newCG, pfn)
makeflipbook(prefix + "-swap", len(tractor.getImages()), itune1, itune2,
ntune)
upccds = ccds[I]
annotate(upccds,
survey,
mzls=opt.mzls,
bass=opt.bass,
normalizePsf=opt.normalizePsf)
ccds[I] = upccds
fn = 'updated-' + os.path.basename(opt.update)
ccds.writeto(fn)
print('Wrote', fn)
sys.exit(0)
from astrometry.util.multiproc import *
mp = multiproc(opt.threads)
N = 100
if len(opt.part) == 0 and len(opt.ccds) == 0:
opt.part.append('decals')
opt.part.append('nondecals')
opt.part.append('extra')
ccdfns = opt.ccds
for p in opt.part:
ccdfns.append(
os.path.join(survey.survey_dir, 'survey-ccds-%s.fits.gz' % p))
for fn in ccdfns:
print()
print('Reading', fn)
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--force', action='store_true',
help='Run calib processes even if files already exist?')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--expnum', type=int, help='Cut to a single exposure')
parser.add_argument('--extname', '--ccdname', help='Cut to a single extension/CCD name')
parser.add_argument('--no-psf', dest='psfex', action='store_false',
help='Do not compute PsfEx calibs')
parser.add_argument('--no-sky', dest='sky', action='store_false',
help='Do not compute sky models')
parser.add_argument('--run-se', action='store_true', help='Run SourceExtractor')
parser.add_argument('--splinesky', action='store_true', help='Spline sky, not constant')
parser.add_argument('--threads', type=int, help='Run multi-threaded', default=None)
parser.add_argument('args',nargs=argparse.REMAINDER)
opt = parser.parse_args()
survey = LegacySurveyData()
if opt.ccds is not None:
T = fits_table(opt.ccds)
print('Read', len(T), 'from', opt.ccds)
else:
T = survey.get_ccds()
#print len(T), 'CCDs'
if len(opt.args) == 0:
if opt.expnum is not None:
T.cut(T.expnum == opt.expnum)
print('Cut to', len(T), 'with expnum =', opt.expnum)
if opt.extname is not None:
T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname]))
print('Cut to', len(T), 'with extname =', opt.extname)
opt.args = range(len(T))
args = []
for a in opt.args:
# Check for "expnum-ccdname" format.
if '-' in str(a):
words = a.split('-')
assert(len(words) == 2)
expnum = int(words[0])
ccdname = words[1]
I = np.flatnonzero((T.expnum == expnum) * (T.ccdname == ccdname))
if len(I) != 1:
print('Found', len(I), 'CCDs for expnum', expnum, 'CCDname', ccdname, ':', I)
assert(len(I) == 1)
t = T[I[0]]
else:
i = int(a)
print('Index', i)
t = T[i]
print('CCDnmatch', t.ccdnmatch)
if t.ccdnmatch < 20 and not opt.force:
print('Skipping ccdnmatch = %i' % t.ccdnmatch)
continue
im = survey.get_image_object(t)
print('Running', im.calname)
kwargs = dict(psfex=opt.psfex, sky=opt.sky)
if opt.force:
kwargs.update(force=True)
if opt.run_se:
kwargs.update(se=True)
if opt.splinesky:
kwargs.update(splinesky=True)
if opt.threads:
args.append((im, kwargs))
else:
run_calibs((im, kwargs))
if opt.threads:
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads)
mp.map(run_calibs, args)
return 0
def main():
sbfn = 'skybricks.fits'
SB = fits_table(sbfn)
Bnorth = fits_table(
'/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/north/survey-bricks-dr9-north.fits.gz'
)
Bsouth = fits_table(
'/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/survey-bricks-dr9-south.fits.gz'
)
Bnorth.cut(Bnorth.survey_primary)
Bsouth.cut(Bsouth.survey_primary)
#Bsouth.cut(Bsouth.dec > -30)
Bnorth.hemi = np.array(['north'] * len(Bnorth))
Bsouth.hemi = np.array(['south'] * len(Bsouth))
B = merge_tables([Bnorth, Bsouth])
# Rough cut the skybricks to those near bricks.
I, J, d = match_radec(SB.ra, SB.dec, B.ra, B.dec, 1., nearest=True)
SB.cut(I)
import argparse
parser = argparse.ArgumentParser()
#parser.add_argument('--brick', help='Sky brick name')
parser.add_argument('--minra',
type=float,
help='Cut to a minimum RA range of sky bricks')
parser.add_argument('--maxra',
type=float,
help='Cut to a maximum RA range of sky bricks')
parser.add_argument('--threads',
type=int,
help='Parallelize on this many cores')
opt = parser.parse_args()
if opt.minra:
SB.cut(SB.ra >= opt.minra)
if opt.maxra:
SB.cut(SB.ra <= opt.maxra)
version = get_git_version(os.getcwd())
print('Version string:', version)
# Find bricks near skybricks, as a rough cut.
# (magic 1. degree > hypot(skybrick radius, brick radiu) ~= 0.9)
Inear = match_radec(SB.ra, SB.dec, B.ra, B.dec, 1., indexlist=True)
args = []
k = 0
Isb = []
for isb, (sb, inear) in enumerate(zip(SB, Inear)):
if inear is None:
continue
args.append((k, sb, B[np.array(inear)], version))
k += 1
Isb.append(isb)
print(len(args), 'sky bricks')
SB.cut(np.array(Isb))
if opt.threads:
mp = multiproc(opt.threads)
else:
mp = multiproc()
exist = mp.map(run_one, args)
if opt.minra is None and opt.maxra is None:
exist = np.array(exist)
SB[exist].writeto('skybricks-exist.fits')
return
def __init__(self, mp=None, **kwargs):
super(TractorMultiprocMixin, self).__init__(**kwargs)
if mp is None:
from astrometry.util.multiproc import multiproc
mp = multiproc()
return
if __name__ == '__main__':
#main()
#B = fits_table('bricks-near.fits')
#SB = fits_table('skybricks.fits')
B = fits_table(
'/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/north/survey-bricks-dr9-north.fits.gz'
)
B.cut(B.dec > 77)
B.hemi = np.array(['north'] * len(B))
SB = fits_table(
'/global/cfs/cdirs/desi/target/skybricks/v3/skybricks-exist.fits')
#I = np.flatnonzero(np.isin(SB.brickname, [
# '2187p340', '2175p340',
#'1500p290', '1507p300', '1519p300',
#'1505p310', '1490p320'
#]))
I = np.flatnonzero(SB.dec >= 78.)
version = get_git_version(os.getcwd())
#version = 4
mp = multiproc(32)
args = []
for i in I:
args.append((0, SB[i], B, version))
mp.map(run_one, args)
def main():
os.environ['DESIMODEL'] = '/global/homes/d/dstn/desimodel-data'
global hw
global stuck_x
global stuck_y
global stuck_loc
global starkd
hw = load_hardware()
# From fiberassign/stucksky.py: find X,Y positions of stuck positioners.
# (grab the hw dictionaries once -- these are python wrappers over C++ so not simple accessors)
state = hw.state
devtype = hw.loc_device_type
stuck_loc = [
loc for loc in hw.locations
if (((state[loc] & (FIBER_STATE_STUCK | FIBER_STATE_BROKEN)
) == FIBER_STATE_STUCK) and (devtype[loc] == 'POS'))
]
print(len(stuck_loc), 'stuck positioners')
theta_pos = hw.loc_theta_pos
theta_off = hw.loc_theta_offset
phi_pos = hw.loc_phi_pos
phi_off = hw.loc_phi_offset
stuck_theta = [theta_pos[loc] + theta_off[loc] for loc in stuck_loc]
stuck_phi = [phi_pos[loc] + phi_off[loc] for loc in stuck_loc]
curved_mm = hw.loc_pos_curved_mm
theta_arm = hw.loc_theta_arm
phi_arm = hw.loc_phi_arm
theta_min = hw.loc_theta_min
theta_max = hw.loc_theta_max
phi_min = hw.loc_phi_min
phi_max = hw.loc_phi_max
# Convert positioner angle orientations to curved focal surface X / Y (not CS5)
# Note: we could add some methods to the python bindings to vectorize this or make it less clunky...
stuck_x = np.zeros(len(stuck_loc))
stuck_y = np.zeros(len(stuck_loc))
for iloc, (loc, theta,
phi) in enumerate(zip(stuck_loc, stuck_theta, stuck_phi)):
loc_x, loc_y = hw.thetaphi_to_xy(curved_mm[loc], theta, phi,
theta_arm[loc], phi_arm[loc],
theta_off[loc], phi_off[loc],
theta_min[loc], phi_min[loc],
theta_max[loc], phi_max[loc], True)
stuck_x[iloc] = loc_x
stuck_y[iloc] = loc_y
tiles = Table.read(
'/global/cfs/cdirs/desi/target/surveyops/ops/tiles-main.ecsv')
print(len(tiles), 'tiles')
# Deduplicate tiles with same RA,Dec center
tilera = tiles['RA']
tiledec = tiles['DEC']
tileid = tiles['TILEID']
rdtile = {}
tilemap = {}
for tid, r, d in zip(tileid, tilera, tiledec):
key = r, d
if key in rdtile:
# already seen a tile with this RA,Dec; point to it
tilemap[tid] = rdtile[key]
else:
rdtile[key] = tid
del rdtile
tnow = datetime.now()
tile_obstime = tnow.isoformat(timespec='seconds')
mjd = Time(tnow).mjd
stars = fits_table(
'/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/masking/gaia-mask-dr9.fits.gz'
)
print(len(stars), 'stars for masking')
print('Moving to MJD', mjd)
ra, dec = radec_at_mjd(stars.ra, stars.dec, stars.ref_epoch.astype(float),
stars.pmra, stars.pmdec, stars.parallax, mjd)
assert (np.all(np.isfinite(ra)))
assert (np.all(np.isfinite(dec)))
stars.ra = ra
stars.dec = dec
print('Building kd-tree...')
starkd = tree_build_radec(stars.ra, stars.dec)
match_radius = deg2dist(30. / 3600.)
stuck_loc = np.array(stuck_loc)
allresults = {}
mp = multiproc(32)
print('Building arg lists...')
args = []
for tid, tile_ra, tile_dec, tile_obsha in zip(tileid, tilera, tiledec,
tiles['DESIGNHA']):
# skip duplicate tiles
if tid in tilemap:
continue
# "fieldrot"
tile_theta = field_rotation_angle(tile_ra, tile_dec, mjd)
args.append((tid, tile_ra, tile_dec, tile_obstime, tile_theta,
tile_obsha, match_radius))
print('Matching', len(args), 'unique tile RA,Decs in parallel...')
res = mp.map(_match_tile, args)
print('Organizing results...')
T = fits_table()
T.tileid = []
T.loc = []
T.petal = []
T.device = []
T.fiber = []
T.pos_ra = []
T.pos_dec = []
T.star_ra = []
T.star_dec = []
T.dist_arcsec = []
T.mask_mag = []
loc_to_petal = hw.loc_petal
loc_to_device = hw.loc_device
loc_to_fiber = hw.loc_fiber
for vals in res:
if vals is None:
continue
tileid, I, pos_ra, pos_dec, pos_loc, dists = vals
T.tileid.extend([tileid] * len(I))
T.loc.extend(pos_loc)
for loc in pos_loc:
T.petal.append(loc_to_petal[loc])
T.device.append(loc_to_device[loc])
T.fiber.append(loc_to_fiber[loc])
T.pos_ra.extend(pos_ra)
T.pos_dec.extend(pos_dec)
T.star_ra.extend(stars.ra[I])
T.star_dec.extend(stars.dec[I])
T.dist_arcsec.extend(dists)
T.mask_mag.extend(stars.mask_mag[I])
T.to_np_arrays()
T.writeto('stuck-on-stars.fits')
def main():
batch = False
arr = os.environ.get('PBS_ARRAYID')
if arr is not None:
arr = int(arr)
batch = True
# This gets set when running runslice.py from the command-line within an interactive job...
d = os.environ.get('PBS_O_WORKDIR')
if batch and d is not None:
os.chdir(d)
sys.path.append(os.getcwd())
opt = myopts()
opt.nonsdss = False
ps = None
if True:
# W3 area
if arr is None:
# which slice to do for interactive jobs
#arr = 147
#arr = 148
arr = 149
opt.sources = 'objs-eboss-w3-dr9.fits'
NDEC = 50
NRA = 90
band = int(arr / 100)
ri = arr % 100
print 'Band', band
print 'RA slice', ri
r0,r1 = 210.593, 219.132
d0,d1 = 51.1822, 54.1822
basedir = '/clusterfs/riemann/raid000/bosswork/boss/wise_frames'
opt.wisedatadirs = [(basedir, 'merged'),]
# basedir = '/clusterfs/riemann/raid000/bosswork/boss/wise1test'
# opt.wisedatadirs = [(os.path.join(basedir, 'allsky'), 'cryo'),
# (os.path.join(basedir, 'prelim_postcryo'), 'post-cryo'),]
opt.minflux = None
opt.wsources = 'wise-objs-w3.fits'
opt.bandnum = band
opt.osources = None
opt.minsb = 0.005
opt.ptsrc = False
opt.pixpsf = False
if False:
# eboss w3 v4
basename = 'ebossw3-v4'
opt.ptsrc = False
opt.pixpsf = False
if False:
# eboss w3 v5
basename = 'ebossw3-v5'
opt.ptsrc = True
if True:
# eboss w3 v6 (after the fact)
basename = 'eboss-w3-v6'
if not batch:
basename = 'eboss-w3-tst'
opt.ptsrc = False
opt.pixpsf = False
if False:
# Stripe82 QSO truth-table region
base = '/clusterfs/riemann/raid000/bosswork/boss/wise1ext/sdss_stripe82'
opt.sources = os.path.join(base, 'objs-eboss-stripe82-dr9.fits')
r0, r1 = 317.0, 330.0
d0, d1 = 0., 1.25
NRA = 260
NDEC = 25
if arr is None:
# which slice to do for interactive jobs
arr = 1000
band = int(arr / 1000)
ri = arr % 1000
print 'Band', band
print 'RA slice', ri
basedir = '/clusterfs/riemann/raid000/bosswork/boss/wise1test_stripe82/old'
opt.wisedatadirs = [(os.path.join(basedir, 'allsky'), 'cryo'),
(os.path.join(basedir, 'prelim_postcryo'), 'post-cryo'),]
opt.minflux = None
opt.bandnum = band
opt.osources = None
opt.minsb = 0.005
opt.ptsrc = False
opt.pixpsf = False
# v1
basename = 'eboss-s82-v1'
opt.ptsrc = False
opt.pixpsf = False
if not batch:
basename = 'eboss-s82-tst'
opt.ptsrc = False
opt.pixpsf = False
if not batch:
import matplotlib
matplotlib.use('Agg')
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(basename)
# Has to happen *after* matplotlib init...
import wise3
dd = np.linspace(d0, d1, NDEC + 1)
rr = np.linspace(r0, r1, NRA + 1)
mp = multiproc()
tr0 = Time()
rlo,rhi = rr[ri], rr[ri+1]
for di,(dlo,dhi) in enumerate(zip(dd[:-1], dd[1:])):
fn = '%s-r%02i-d%02i-w%i.fits' % (basename, ri, di, opt.bandnum)
if os.path.exists(fn):
print 'Output file exists:', fn
print 'Skipping'
if batch:
continue
# HACK!!
#if not batch and di != 25:
# continue
td0 = Time()
try:
P = dict(ralo=rlo, rahi=rhi, declo=dlo, dechi=dhi,
opt=opt, mp=mp, ps=ps)
t00 = Time()
for stage in [ 100, 101, 102, 103, 104, 105, 106, 700 ]:
f = eval('wise3.stage%i' % stage)
ts0 = Time()
R = f(**P)
dt = Time() - ts0
P.update(R)
P.update(**{ 't%i' % stage: dt })
print 'stage', stage, ':', dt
dt = Time() - t00
P.update(**{ 'ttotal': dt })
# Grab result FITS structure
R = P['R']
R.writeto(fn)
print 'Wrote', fn
imst = P['imstats']
fn = '%s-r%02i-d%02i-w%i-imstats.fits' % (basename, ri, di, opt.bandnum)
imst.writeto(fn)
print 'Wrote', fn
pfn = '%s-r%02i-d%02i-w%i.pickle' % (basename, ri, di, opt.bandnum)
# tractor = P['tractor']
# ims1 = P['ims1']
# res1 = []
# for tim,(img,mod,ie,chi,roi) in zip(tractor.images, ims1):
# for k in ['origInvvar', 'starMask', 'inverr', 'cinvvar', 'goodmask',
# 'maskplane', 'rdmask', 'mask', 'uncplane', 'vinvvar']:
#
# ### DEBUG
# #continue
# # Debug
# if k == 'rdmask':
# continue
#
# try:
# delattr(tim, k)
# except:
# pass
#
# res1.append((tim, mod, roi))
#
# PP = dict(res1=res1, cat=tractor.getCatalog(), rd=P['rd'], ri=ri, di=di,
# bandnum=opt.bandnum, S=P['S'],
# ralo=rlo, rahi=rhi, declo=dlo, dechi=dhi,
# opt=opt,
# T=P['T'])
pickle_to_file(P, pfn)
print
print 'Dec slice time:', Time() - td0
print
except:
import traceback
print '---------------------------------------------------'
print 'FAILED: dec slice', di, 'ra slice', ri
print rlo,rhi, dlo,dhi
print '---------------------------------------------------'
traceback.print_exc()
print '---------------------------------------------------'
if not batch:
raise
print
print 'RA slice time:', Time() - tr0
print
def __str__(self):
return 'ywrapper: n=%i; ' % self.n + self.y
def __iter__(self):
return self
def next(self):
return self.y.next()
def __len__(self):
return self.n
def yielder(n):
for i in range(n):
print('Yielding', i)
yield i
N = 20
y = yielder(N)
args = ywrapper(y, N)
dpool = TimingPool(4, taskqueuesize=4)
dmup = multiproc.multiproc(pool=dpool)
Time.add_measurement(TimingPoolMeas(dpool))
t0 = Time()
res = dmup.map(work, args)
print(Time() - t0)
print('Got result:', res)
def twogalaxies(name1,ra1,dec1,name2,ra2,dec2):
name = "%s %s" % (name1,name2)
ra = float(ra1)
dec = float(dec1)
ra2 = float(ra2)
dec2 = float(dec2)
remradius = 6.
fieldradius = 6.
threads = None
itune1=5
itune2=5
ntune=0
nocache=True
#Radius should be in arcminutes
if threads:
mp = multiproc(nthreads=threads)
else:
mp = multiproc()
IRLS_scale = 25.
dr9 = True
dr8 = False
noarcsinh = False
print name
prefix = '%s' % (name.replace(' ', '_'))
prefix1 = '%s' % (name1.replace(' ', '_'))
prefix2 = '%s' % (name2.replace(' ', '_'))
print 'Removal Radius', remradius
print 'Field Radius', fieldradius
print 'RA,Dec', ra, dec
print os.getcwd()
rcfs = radec_to_sdss_rcf(ra,dec,radius=math.hypot(fieldradius,13./2.),tablefn="dr9fields.fits")
print rcfs
assert(len(rcfs)>0)
assert(len(rcfs)<15)
sras, sdecs, smags = tychoMatch(ra,dec,(fieldradius*1.5)/60.)
for sra,sdec,smag in zip(sras,sdecs,smags):
print sra,sdec,smag
imkw = dict(psf='dg')
if dr9:
getim = st.get_tractor_image_dr9
getsrc = st.get_tractor_sources_dr9
imkw.update(zrange=[-3,100])
elif dr8:
getim = st.get_tractor_image_dr8
getsrc = st.get_tractor_sources_dr8
imkw.update(zrange=[-3,100])
else:
getim = st.get_tractor_image
getsrc = st.get_tractor_sources_dr8
imkw.update(useMags=True)
bands=['u','g','r','i','z']
bandname = 'r'
flipBands = ['r']
print rcfs
imsrcs = mp.map(get_ims_and_srcs, [(rcf + (bands, ra, dec, fieldradius*60./0.396, imkw, getim, getsrc))
for rcf in rcfs])
timgs = []
sources = []
allsources = []
for ims,s in imsrcs:
if ims is None:
continue
if s is None:
continue
timgs.extend(ims)
allsources.extend(s)
sources.append(s)
#rds = [rcf[3:5] for rcf in rcfs]
plotarea(ra, dec, fieldradius, name, prefix, timgs) #, rds)
lvl = logging.DEBUG
logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout)
tractor = st.Tractor(timgs, allsources, mp=mp)
sa = dict(debug=True, plotAll=False,plotBands=False)
if noarcsinh:
sa.update(nlscale=0)
elif dr8 or dr9:
sa.update(chilo=-8.,chihi=8.)
if nocache:
tractor.cache = NullCache()
sg.disable_galaxy_cache()
zr = timgs[0].zr
print "zr is: ",zr
print bands
print "Number of images: ", len(timgs)
# for timg,band in zip(timgs,bands):
# data = timg.getImage()/np.sqrt(timg.getInvvar())
# plt.hist(data,bins=100)
# plt.savefig('hist-%s.png' % (band))
saveAll('initial-'+prefix, tractor,**sa)
#plotInvvar('initial-'+prefix,tractor)
for sra,sdec,smag in zip(sras,sdecs,smags):
print sra,sdec,smag
for img in tractor.getImages():
wcs = img.getWcs()
starx,stary = wcs.positionToPixel(RaDecPos(sra,sdec))
starr=25*(2**(max(11-smag,0.)))
if starx+starr<0. or starx-starr>img.getWidth() or stary+starr <0. or stary-starr>img.getHeight():
continue
X,Y = np.meshgrid(np.arange(img.getWidth()), np.arange(img.getHeight()))
R2 = (X - starx)**2 + (Y - stary)**2
img.getStarMask()[R2 < starr**2] = 0
for timgs,sources in imsrcs:
timg = timgs[0]
wcs = timg.getWcs()
xtr,ytr = wcs.positionToPixel(RaDecPos(ra,dec))
print xtr,ytr
xt = xtr
yt = ytr
r = ((remradius*60.))/.396 #radius in pixels
for src in sources:
xs,ys = wcs.positionToPixel(src.getPosition(),src)
if (xs-xt)**2+(ys-yt)**2 <= r**2:
#print "Removed:", src
#print xs,ys
tractor.removeSource(src)
#saveAll('removed-'+prefix, tractor,**sa)
newShape = sg.GalaxyShape((remradius*60.)/10.,1.,0.)
newBright = ba.Mags(r=15.0,g=15.0,u=15.0,z=15.0,i=15.0,order=['u','g','r','i','z'])
EG = st.ExpGalaxy(RaDecPos(ra,dec),newBright,newShape)
newShape2 = sg.GalaxyShape((remradius*60.)/10.,1.,0.)
newBright2 = ba.Mags(r=15.0,g=15.0,u=15.0,z=15.0,i=15.0,order=['u','g','r','i','z'])
EG2 = st.ExpGalaxy(RaDecPos(ra2,dec2),newBright2,newShape2)
print EG
print EG2
tractor.addSource(EG)
tractor.addSource(EG2)
saveAll('added-'+prefix,tractor,**sa)
#print 'Tractor has', tractor.getParamNames()
for im in tractor.images:
im.freezeAllParams()
im.thawParam('sky')
tractor.catalog.freezeAllBut(EG)
tractor.catalog.thawParams(EG2)
#print 'Tractor has', tractor.getParamNames()
#print 'values', tractor.getParams()
for i in range(itune1):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('itune1-%d-' % (i+1)+prefix,tractor,**sa)
tractor.clearCache()
sg.get_galaxy_cache().clear()
gc.collect()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
CGPos = EG.getPosition()
CGShape1 = EG.getShape().copy()
CGShape2 = EG.getShape().copy()
EGBright = EG.getBrightness()
CGu = EGBright[0] + 0.75
CGg = EGBright[1] + 0.75
CGr = EGBright[2] + 0.75
CGi = EGBright[3] + 0.75
CGz = EGBright[4] + 0.75
CGBright1 = ba.Mags(r=CGr,g=CGg,u=CGu,z=CGz,i=CGi,order=['u','g','r','i','z'])
CGBright2 = ba.Mags(r=CGr,g=CGg,u=CGu,z=CGz,i=CGi,order=['u','g','r','i','z'])
print EGBright
print CGBright1
CG = st.CompositeGalaxy(CGPos,CGBright1,CGShape1,CGBright2,CGShape2)
CG2Pos = EG2.getPosition()
CG2Shape1 = EG2.getShape().copy()
CG2Shape2 = EG2.getShape().copy()
EG2Bright = EG2.getBrightness()
CG2u = EG2Bright[0] + 0.75
CG2g = EG2Bright[1] + 0.75
CG2r = EG2Bright[2] + 0.75
CG2i = EG2Bright[3] + 0.75
CG2z = EG2Bright[4] + 0.75
CG2Bright1 = ba.Mags(r=CG2r,g=CG2g,u=CG2u,z=CG2z,i=CG2i,order=['u','g','r','i','z'])
CG2Bright2 = ba.Mags(r=CG2r,g=CG2g,u=CG2u,z=CG2z,i=CG2i,order=['u','g','r','i','z'])
CG2 = st.CompositeGalaxy(CG2Pos,CG2Bright1,CG2Shape1,CG2Bright2,CG2Shape2)
tractor.removeSource(EG)
tractor.removeSource(EG2)
tractor.addSource(CG)
tractor.addSource(CG2)
tractor.catalog.freezeAllBut(CG)
tractor.catalog.thawParams(CG2)
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
for i in range(itune2):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('itune2-%d-' % (i+1)+prefix,tractor,**sa)
tractor.clearCache()
sg.get_galaxy_cache().clear()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.catalog.thawAllParams()
for i in range(ntune):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('ntune-%d-' % (i+1)+prefix,tractor,**sa)
#plotInvvar('final-'+prefix,tractor)
sa.update(plotBands=True)
saveAll('allBands-' + prefix,tractor,**sa)
print CG
print CG.getPosition()
print CGBright1
print CGBright2
print CGShape1
print CGShape2
print CGBright1+CGBright2
print CG.getBrightness()
pfn = '%s.pickle' % prefix
pickle_to_file([CG,CG2],pfn)
makeflipbook(prefix,len(tractor.getImages()),itune1,itune2,ntune)
pickle_to_file(CG,"%s.pickle" % prefix1)
pickle_to_file(CG2,"%s.pickle" % prefix2)
os.system('cp %s.pickle RC3_Output' % prefix1)
os.system('cp %s.pickle RC3_Output' % prefix2)
def main():
import optparse
from astrometry.util.stages import CallGlobal, runstage
parser = optparse.OptionParser()
parser.add_option('-f', '--force-stage', dest='force', action='append', default=[],
help="Force re-running the given stage(s) -- don't read from pickle.")
parser.add_option('-s', '--stage', dest='stage', default=[], action='append',
help="Run up to the given stage(s)")
parser.add_option('-n', '--no-write', dest='write', default=True, action='store_false')
parser.add_option('-P', '--pickle', dest='picklepat', help='Pickle filename pattern, with %i, default %default',
default='pickles/tunebrick-%(brick)06i-%%(stage)s.pickle')
parser.add_option('-b', '--brick', type=int, help='Brick ID to run: default %default',
default=377306)
parser.add_option('-p', '--plots', dest='plots', action='store_true')
#parser.add_option('--stamp', action='store_true')
parser.add_option('--zoom', type=int, nargs=4, help='Set target image extent (default "0 3600 0 3600")')
parser.add_option('-W', type=int, default=3600, help='Target image width (default %default)')
parser.add_option('-H', type=int, default=3600, help='Target image height (default %default)')
parser.add_option('--bands', help='Bands to process; default "%default"', default='grz')
parser.add_option('--plot-base', default='plot-%(brick)06i', #'tunebrick/coadd/plot-%(brick)06i',
help='Plot filenames; default %default')
parser.add_option('--threads', type=int, help='Run multi-threaded')
parser.add_option('--base-dir', dest='basedir', default='tunebrick',
help='Base output directory; default %default')
parser.add_option('--mock-psf', dest='mock_psf', action='store_true',
help='Use fake PSF?')
opt,args = parser.parse_args()
Time.add_measurement(MemMeas)
stagefunc = CallGlobal('stage_%s', globals())
if len(opt.stage) == 0:
opt.stage.append('writecat2')
opt.force.extend(opt.stage)
opt.picklepat = opt.picklepat % dict(brick=opt.brick)
prereqs = {'tims': None,
'cat': 'tims',
'tune': 'cat',
'writecat2': 'tune',
'recoadd': 'tims',
'rergb': 'recoadd',
'primage': 'recoadd',
}
ps = PlotSequence(opt.plot_base % dict(brick=opt.brick))
initargs = dict(ps=ps)
initargs.update(W=opt.W, H=opt.H, brickid=opt.brick, target_extent=opt.zoom,
program_name = 'tunebrick.py', pipe=True,
bands=opt.bands,
mock_psf=opt.mock_psf)
kwargs = {}
kwargs.update(basedir=opt.basedir)
if opt.threads and opt.threads > 1:
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads, init=runbrick_global_init, initargs=())
runbrick.mp = mp
else:
runbrick_global_init()
t0 = Time()
for stage in opt.stage:
runstage(stage, opt.picklepat, stagefunc, force=opt.force, write=opt.write,
prereqs=prereqs, initial_args=initargs, **kwargs)
#tune(opt.brick, target_extent=opt.zoom)
print 'Total:', Time()-t0
def __init__(self, mp=None, **kwargs):
super(TractorMultiprocMixin, self).__init__(**kwargs)
if mp is None:
from astrometry.util.multiproc import multiproc
mp = multiproc()
def annotate(ccds,
survey,
mp=None,
mzls=False,
bass=False,
normalizePsf=True,
carryOn=True):
if mp is None:
from astrometry.util.multiproc import multiproc
mp = multiproc()
# File from the "observing" svn repo:
from pkg_resources import resource_filename
if mzls:
tilefile = resource_filename('legacyzpts',
'data/mosaic-tiles_obstatus.fits')
elif bass:
tilefile = None
else:
tilefile = resource_filename('legacyzpts',
'data/decam-tiles_obstatus.fits')
if tilefile is not None:
if os.path.isfile(tilefile):
print('Reading {}'.format(tilefile))
tiles = fits_table(tilefile)
else:
print('Required tile file {} not found!'.format(tilefile))
raise IOError
else:
tiles = None
if tiles is not None:
# Map tile IDs back to index in the obstatus file.
tileid_to_index = np.empty(max(tiles.tileid) + 1, int)
tileid_to_index[:] = -1
tileid_to_index[tiles.tileid] = np.arange(len(tiles))
#assert('ccd_cuts' in ccds.get_columns())
gaussgalnorm = np.zeros(len(ccds), np.float32)
anns = mp.map(annotate_one_ccd,
[(ccd, survey, normalizePsf, carryOn) for ccd in ccds])
for iccd, ann in enumerate(anns):
tileid = ann.pop('tileid', -1)
if tileid > -1:
tile = tiles[tileid_to_index[tileid]]
assert (tile.tileid == tileid)
ccds.tileid[iccd] = tile.tileid
ccds.tilepass[iccd] = tile.get('pass')
ccds.tileebv[iccd] = tile.ebv_med
gaussgalnorm[iccd] = ann.pop('gaussgalnorm', 0)
for k, v in ann.items():
ccds.get(k)[iccd] = v
sfd = tractor.sfd.SFDMap()
allbands = 'ugrizY'
filts = ['%s %s' % ('DES', f) for f in allbands]
wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4']
ebv, ext = sfd.extinction(filts + wisebands,
ccds.ra_center,
ccds.dec_center,
get_ebv=True)
ext[np.logical_not(ccds.annotated), :] = 0.
ebv[np.logical_not(ccds.annotated)] = 0.
ext = ext.astype(np.float32)
ccds.ebv = ebv.astype(np.float32)
ccds.decam_extinction = ext[:, :len(allbands)]
ccds.wise_extinction = ext[:, len(allbands):]
# Depth
with np.errstate(invalid='ignore', divide='ignore'):
detsig1 = ccds.sig1 / ccds.psfnorm_mean
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.psfdepth = -2.5 * (np.log10(depth) - 9)
detsig1 = ccds.sig1 / ccds.galnorm_mean
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.galdepth = -2.5 * (np.log10(depth) - 9)
# Depth using Gaussian FWHM.
psf_sigma = ccds.fwhm / 2.35
gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
detsig1 = ccds.sig1 / gnorm
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.gausspsfdepth = -2.5 * (np.log10(depth) - 9)
# Gaussian galaxy depth
detsig1 = ccds.sig1 / gaussgalnorm
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.gaussgaldepth = -2.5 * (np.log10(depth) - 9)
# NaN depths -> 0
for X in [
ccds.psfdepth, ccds.galdepth, ccds.gausspsfdepth,
ccds.gaussgaldepth
]:
X[np.logical_not(np.isfinite(X))] = 0.
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'--force',
action='store_true',
help='Run calib processes even if files already exist?')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument(
'--expnum',
type=str,
help='Cut to a single or set of exposures; comma-separated list')
parser.add_argument('--extname',
'--ccdname',
help='Cut to a single extension/CCD name')
parser.add_argument('--no-psf',
dest='psfex',
action='store_false',
help='Do not compute PsfEx calibs')
parser.add_argument('--no-sky',
dest='sky',
action='store_false',
help='Do not compute sky models')
parser.add_argument('--run-se',
action='store_true',
help='Run SourceExtractor')
parser.add_argument('--splinesky',
action='store_true',
help='Spline sky, not constant')
parser.add_argument('--threads',
type=int,
help='Run multi-threaded',
default=None)
parser.add_argument('--continue',
dest='cont',
default=False,
action='store_true',
help='Continue even if one file fails?')
parser.add_argument('--plot-base',
help='Make plots with this base filename')
# actually this doesn't work for calibs...
#parser.add_argument('--outdir', dest='output_dir', default=None,
# help='Set output base directory')
parser.add_argument('args', nargs=argparse.REMAINDER)
opt = parser.parse_args()
survey = LegacySurveyData() #output_dir=opt.output_dir)
T = None
if opt.ccds is not None:
T = fits_table(opt.ccds)
T = survey.cleanup_ccds_table(T)
print('Read', len(T), 'from', opt.ccds)
#else:
# T = survey.get_ccds()
# #print len(T), 'CCDs'
if len(opt.args) == 0:
if opt.expnum is not None:
expnums = set([int(e) for e in opt.expnum.split(',')])
#T.cut(np.array([e in expnums for e in T.expnum]))
T = merge_tables([
survey.find_ccds(expnum=e, ccdname=opt.extname)
for e in expnums
])
print('Cut to', len(T), 'with expnum in', expnums, 'and extname',
opt.extname)
#if opt.extname is not None:
# T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname]))
# print('Cut to', len(T), 'with extname =', opt.extname)
opt.args = range(len(T))
ps = None
if opt.plot_base is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plot_base)
args = []
for a in opt.args:
# Check for "expnum-ccdname" format.
if '-' in str(a):
words = a.split('-')
assert (len(words) == 2)
expnum = int(words[0])
ccdname = words[1]
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
if len(T) != 1:
print('Found', len(I), 'CCDs for expnum', expnum, 'CCDname',
ccdname, ':', I)
print('WARNING: skipping this expnum,ccdname')
continue
t = T[0]
else:
i = int(a)
print('Index', i)
t = T[i]
#print('CCDnmatch', t.ccdnmatch)
#if t.ccdnmatch < 20 and not opt.force:
# print('Skipping ccdnmatch = %i' % t.ccdnmatch)
# continue
im = survey.get_image_object(t)
print('Running', im.name)
kwargs = dict(psfex=opt.psfex, sky=opt.sky, ps=ps, survey=survey)
if opt.force:
kwargs.update(force=True)
if opt.run_se:
kwargs.update(se=True)
if opt.splinesky:
kwargs.update(splinesky=True)
if opt.cont:
kwargs.update(noraise=True)
if opt.threads:
args.append((im, kwargs))
else:
run_calibs((im, kwargs))
if opt.threads:
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads)
mp.map(time_run_calibs, args)
return 0
if len(args) != 1:
parser.print_help()
sys.exit(-1)
brick = int(args[0], 10)
Time.add_measurement(MemMeas)
lvl = logging.WARNING
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
if opt.threads and opt.threads > 1:
from astrometry.util.multiproc import multiproc
if True:
mp = multiproc(opt.threads, init=runbrick_global_init, initargs=())
else:
from utils.debugpool import DebugPool, DebugPoolMeas
dpool = DebugPool(opt.threads, taskqueuesize=2*opt.threads,
initializer=runbrick_global_init)
mp = multiproc(pool=dpool)
Time.add_measurement(DebugPoolMeas(dpool))
runbrick.mp = mp
else:
runbrick_global_init()
if opt.no_ceres:
runbrick.useCeres = False
P = dict(W=3600, H=3600, brickid=brick, pipe=True)
def refit_galaxies_2():
import optparse
parser = optparse.OptionParser(usage='%prog [options] <NGC-number>')
parser.add_option('--threads', dest='threads', type=int, default=1,
help='use multiprocessing')
opt,args = parser.parse_args()
mp = multiproc(nthreads=opt.threads)
band = 'r'
T = fits_table('ipe3_dstn_2.fit')
print len(T), 'objects'
binned1 = 268435456
blended = 8
bad = 216207968633487360
bright = 2
T.cut((T.flags & (bad | blended)) == 0)
print 'Removing bad or blended:', len(T)
T.cut((T.flags & bright) == 0)
print 'Removing bright:', len(T)
T.cut((T.flags & binned1) > 0)
print 'Binned1:', len(T)
T1 = T[T.run == 5183]
T2 = T[T.run == 5224]
rascale = np.mean(np.cos(np.deg2rad(np.append(T1.dec, T2.dec))))
X1 = np.vstack((T1.ra * rascale, T1.dec)).T
X2 = np.vstack((T2.ra * rascale, T2.dec)).T
R = 1. / 3600.
inds,d = nearest(X1, X2, R)
J = np.flatnonzero(inds > -1)
I = inds[J]
print len(J), 'matches'
print len(np.unique(I)), 'unique objs in target'
from collections import Counter
tally = Counter(I)
# subtract 1
for k in tally: tally[k] -= 1
# remove zero and negative entries
tally += Counter()
# Now "tally" just contains keys (from I) with >= 2 counts
print 'multiple matches:', len(tally), ':', tally.keys()
multi = set(tally.keys())
K = np.array([k for k,ii in enumerate(I) if not ii in multi])
I = I[K]
J = J[K]
print 'Kept', len(I), 'non-multi-matched pairs'
print len(np.unique(J)), 'unique J'
print len(np.unique(I)), 'unique I'
M1 = T1[I]
M2 = T2[J]
# select both-galaxy subset.
K = ((M1.type == 3) * (M2.type == 3))
M1 = M1[K]
M2 = M2[K]
print 'Both galaxies:', len(M1)
# sort by mag, moving -9999 to the end.
mag = M1.get('modelmag_' + band).copy()
mag[mag < 0] = 9999.
# avoid bright guys too...
mag[mag < 16] = 9000.
I = np.argsort(mag)
M1 = M1[I]
M2 = M2[I]
# interleave them
N = len(M1)
for c in M1.get_columns():
X1 = M1.get(c)
X2 = M2.get(c)
#print 'column', c, 'shape', X1.shape
## can't handle arrays here
assert(len(X1.shape) == 1)
XX = np.zeros(N * 2, dtype=X1.dtype)
XX[0::2] = X1
XX[1::2] = X2
M1.set(c, XX)
MM = M1
# We have to set the length manually.
MM._length = 2*N
print 'Length of MM:', len(MM)
del M1
del M2
#####
#MM = MM[:2]
refit_galaxies(MM, intermediate_fn='my-ipes-%06i.fits', mp=mp,
modswitch=False, errors=True, band=band)
MM.writeto('my-ipes.fits')
continue
if 'mkdir: cannot create directory `job-nova-' in log:
print('Re-running:', j)
#try_dojob(j, j.user_image)
#args.append(j.id)
args.append(j)
continue
wcsfn = j.get_wcs_file()
if os.system('grep -l RA---SIN %s' % wcsfn) == 0:
print('Job', j.id, 'has SIN WCS')
args.append(j)
continue
print('Running Job ids:', args)
from astrometry.util.multiproc import multiproc
mp = multiproc(4)
django.db.connection.close()
mp.map(_dojob, args)
#for j in args:
# _dojob(j)
def x():
oldsubs = Submission.objects.filter(processing_started__isnull=False,
processing_finished__isnull=True)
print(oldsubs.count(), 'submissions started but not finished')
for sub in oldsubs:
print('Resetting the processing status of', sub)
def refit_galaxies(T, band='i', S=100,
intermediate_fn='refit-%06i.fits', mp=None,
modswitch=False, errors=False):
if mp is None:
mp = multiproc()
sdss = DR9(basedir='paper0-data-dr9')
print 'basedir', sdss.basedir
print 'dasurl', sdss.dasurl
N = len(T)
ps = [('my_',''), ('init_',''),]
if modswitch:
ps.append(('sw_',''))
if errors:
ps.append(('my_','_err'))
for prefix,suffix in ps:
for c in ['exprad_', 'expab_', 'expphi_', 'expmag_',
'devrad_', 'devab_', 'devphi_', 'devmag_']:
T.set(prefix + c + band + suffix, np.zeros(N, dtype=np.float32))
for c in ['ra', 'dec']:
if len(suffix):
dt = np.float32
else:
dt = np.float64
T.set(prefix + c + suffix, np.zeros(N, dtype=dt))
# assume suffix implies _error; omit prefix_type_err field
if len(suffix):
continue
T.set(prefix + 'type', np.chararray(len(T), 1))
if modswitch:
T.set('sw_dlnp', np.zeros(len(T), np.float32))
args = []
for gali in range(len(T)):
ti = T[gali]
args.append((ti, band, S, sdss, gali, modswitch, errors))
# Run in blocks.
tinew = []
B = 0
while len(args):
N = 100
#N = 4
B += N
# Pop N args off the front of the list
thisargs = args[:N]
args = args[N:]
# Run on those args
thisres = mp.map(_refit_gal, thisargs)
#tinew.extend(thisres)
#print 'tinew:', tinew
for resi,argi in zip(thisres, thisargs):
###
gali = argi[4]
###
if resi is None:
print 'Result', gali, 'is None'
continue
print 'Saving result', gali
T[gali] = resi
#for gali in range(min(len(T), len(tinew))):
# tin = tinew[gali]
# if tin is None:
# print 'Skipping', gali
# continue
# T[gali] = tin
#Ti.about()
if intermediate_fn:
T.writeto(intermediate_fn % B)
def main(cmdlineargs=None, get_copilot=False):
global gSFD
import optparse
parser = optparse.OptionParser(usage='%prog')
# Mosaic or Decam?
from camera import (nominal_cal, ephem_observer, default_extension,
tile_path)
nom = nominal_cal
obs = ephem_observer()
plotfn_default = 'recent.png'
parser.add_option('--ext', default=default_extension,
help='Extension to read for computing observing conditions: default %default')
parser.add_option('--extnum', type=int, help='Integer extension to read')
parser.add_option('--rawdata', help='Directory to monitor for new images: default $MOS3_DATA if set, else "rawdata"', default=None)
parser.add_option('--n-fwhm', default=None, type=int, help='Number of stars on which to measure FWHM')
parser.add_option('--no-db', dest='db', default=True, action='store_false',
help='Do not append results to database')
parser.add_option('--no-focus', dest='focus', default=True,
action='store_false', help='Do not analyze focus frames')
parser.add_option('--fits', help='Write database to given FITS table')
parser.add_option('--plot', action='store_true',
help='Plot recent data and quit')
parser.add_option('--plot-filename', default=None,
help='Save plot to given file, default %s' % plotfn_default)
parser.add_option('--nightplot', '--night', action='store_true',
help="Plot tonight's data and quit")
parser.add_option('--qa-plots', dest='doplots', default=False,
action='store_true', help='Create QA plots')
parser.add_option('--keep-plots', action='store_true',
help='Do not remove PNG-format plots (normally merged into PDF)')
parser.add_option('--mjdstart', type=float, default=None,
help='MJD (UTC) at which to start plot')
now = mjdnow()
parser.add_option('--mjdend', type=float, default=None,
help='MJD (UTC) at which to end plot (default: now, which is %.3f)' % now)
parser.add_option('--skip', action='store_true',
help='Skip images that already exist in the database')
parser.add_option('--threads', type=int, default=None,
help='Run multi-threaded when processing list of files on command-line')
parser.add_option('--fix-db', action='store_true')
parser.add_option('--tiles', default=tile_path,
help='Tiles table, default %default')
parser.add_option('--no-show', dest='show', default=True, action='store_false',
help='Do not show plot window, just save it.')
if cmdlineargs is None:
opt,args = parser.parse_args()
else:
opt,args = parser.parse_args(cmdlineargs)
if not opt.show:
import matplotlib
matplotlib.use('Agg')
imagedir = opt.rawdata
if imagedir is None:
imagedir = os.environ.get('MOS3_DATA', 'rawdata')
rawext = opt.ext
if opt.extnum is not None:
rawext = opt.extnum
assert(rawext is not None)
from astrometry.util.fits import fits_table
tiles = fits_table(opt.tiles)
from django.conf import settings
import obsdb
import pylab as plt
plt.figure(figsize=(8,10))
markmjds = []
if opt.nightplot:
opt.plot = True
if opt.plot_filename is None:
opt.plot_filename = 'night.png'
# Are we at Tololo or Kitt Peak? Look for latest image.
o = obsdb.MeasuredCCD.objects.all().order_by('-mjd_obs')
cam = o[0].camera
print('Camera:', cam)
if opt.mjdstart is not None:
sdate = ephem.Date(mjdtodate(opt.mjdend))
else:
sdate = ephem.Date(datenow())
(sunset, eve12, eve18, morn18, morn12, sunrise) = get_twilight(
cam, sdate)
if opt.mjdstart is None:
opt.mjdstart = ephemdate_to_mjd(sunset)
print('Set mjd start to sunset:', sunset, opt.mjdstart)
if opt.mjdend is None:
opt.mjdend = ephemdate_to_mjd(sunrise)
print('Set mjd end to sunrise', sunrise, opt.mjdend)
markmjds.append((ephemdate_to_mjd(eve18),'b'))
print('Evening twi18:', eve18, markmjds[-1])
markmjds.append((ephemdate_to_mjd(morn18),'b'))
print('Morning twi18:', morn18, markmjds[-1])
markmjds.append((ephemdate_to_mjd(eve12),'g'))
print('Evening twi12:', eve12, markmjds[-1])
markmjds.append((ephemdate_to_mjd(morn12),'g'))
print('Morning twi12:', morn12, markmjds[-1])
if opt.plot_filename is None:
opt.plot_filename = plotfn_default
if opt.fits:
ccds = obsdb.MeasuredCCD.objects.all()
print(ccds.count(), 'measured CCDs')
T = db_to_fits(ccds)
T.writeto(opt.fits)
print('Wrote', opt.fits)
return 0
if opt.fix_db:
from astrometry.util.fits import fits_table
tiles = fits_table('obstatus/mosaic-tiles_obstatus.fits')
now = mjdnow()
#ccds = obsdb.MeasuredCCD.objects.all()
#ccds = obsdb.MeasuredCCD.objects.all().filter(mjd_obs__gt=now - 0.25)
ccds = obsdb.MeasuredCCD.objects.all().filter(mjd_obs__gt=57434)
print(ccds.count(), 'measured CCDs')
for ccd in ccds:
try:
hdr = fitsio.read_header(ccd.filename, ext=0)
# band = hdr['FILTER']
# band = band.split()[0]
# ccd.band = band
set_tile_fields(ccd, hdr, tiles)
ccd.save()
print('Fixed', ccd.filename)
except:
import traceback
traceback.print_exc()
return 0
if opt.plot:
plot_recent(opt, nom, tiles=tiles, markmjds=markmjds, show_plot=False)
return 0
print('Loading SFD maps...')
sfd = SFDMap()
if len(args) > 0:
mp = None
if opt.threads > 1:
gSFD = sfd
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads)
if opt.skip:
fns = skip_existing_files(args, rawext)
else:
fns = args
if mp is None:
for fn in fns:
process_image(fn, rawext, nom, sfd, opt, obs, tiles)
else:
sfd = None
mp.map(bounce_process_image,
[(fn, rawext, nom, sfd, opt, obs, tiles) for fn in fns])
plot_recent(opt, nom, tiles=tiles, markmjds=markmjds, show_plot=False)
return 0
copilot = Copilot(imagedir, rawext, opt, nom, sfd, obs, tiles)
# for testability
if get_copilot:
return copilot
copilot.run()
return 0
def main():
import optparse
parser = optparse.OptionParser("%prog [options]")
parser.add_option("--threads", dest="threads", type=int, help="Multiproc", default=1)
parser.add_option(
"--outdir", "-o", dest="outdir", default="unwise-coadds", help="Output directory: default %default"
)
parser.add_option(
"--size", dest="size", default=2048, type=int, help="Set output image size in pixels; default %default"
)
parser.add_option(
"--width", dest="width", default=0, type=int, help="Set output image width in pixels; default --size"
)
parser.add_option(
"--height", dest="height", default=0, type=int, help="Set output image height in pixels; default --size"
)
parser.add_option(
"--pixscale",
dest="pixscale",
type=float,
default=2.75,
help="Set coadd pixel scale, default %default arcsec/pixel",
)
# parser.add_option('--cube', dest='cube', action='store_true',
# default=False, help='Save & write out image cube')
parser.add_option("--ra", dest="ra", type=float, default=None, help="Build coadd at given RA center")
parser.add_option("--dec", dest="dec", type=float, default=None, help="Build coadd at given Dec center")
parser.add_option(
"-b", "--band", dest="bands", action="append", type=int, default=[], help="Add WISE band (default: 1,2)"
)
parser.add_option("--year", action="store_true", default=False)
opt, args = parser.parse_args()
mp = multiproc(opt.threads)
radec = opt.ra is not None and opt.dec is not None
if not radec:
print "Must specify --ra and --dec"
sys.exit(-1)
W = H = opt.size
if opt.width:
W = opt.width
if opt.height:
H = opt.height
if len(opt.bands) == 0:
opt.bands = [1, 2]
# Allow specifying bands like "123"
bb = []
for band in opt.bands:
for s in str(band):
bb.append(int(s))
opt.bands = bb
print "Bands", opt.bands
create_animations(
opt.ra, opt.dec, W, H, pixscale=opt.pixscale, bands=opt.bands, yearly=opt.year, mp=mp, outdir=opt.outdir
)
def _setup_queues(self):
self._inqueue = DebugSimpleQueue()
self._outqueue = DebugSimpleQueue()
self._quick_put = self._inqueue._writer.send
self._quick_get = self._outqueue._reader.recv
from tractor import *
from tractor import sdss as st
from astrometry.util import multiproc
if __name__ == '__main__':
#run,camcol,field = 7164,4,273
#band='g'
run,camcol,field = 2662, 4, 111
band='i'
roi=[0,300,0,300]
im,info = st.get_tractor_image(run, camcol, field, band,
useMags=True, roi=roi)
sources = st.get_tractor_sources(run, camcol, field, band, roi=roi)
tractor = Tractor([im], sources)
print tractor
print tractor.getLogProb()
tractor.freezeParam('images')
pool = DebugPool(4)
tractor.mp = multiproc.multiproc(pool=pool)
tractor.opt2()
def create_animations(
ra, dec, W, H, pixscale=2.75, bands=[1, 2], yearly=False, diffim=True, sdiffim=True, mp=None, outdir="."
):
if mp is None:
mp = multiproc(1)
dataset_tag = "%04i%s%03i" % (int(ra * 10.0), "p" if dec >= 0.0 else "m", int(np.abs(dec) * 10.0))
dataset = "custom-%s" % dataset_tag
print "Setting custom dataset", dataset
cosd = np.cos(np.deg2rad(dec))
r0 = ra - (pixscale * W / 2.0) / 3600.0 / cosd
r1 = ra + (pixscale * W / 2.0) / 3600.0 / cosd
d0 = dec - (pixscale * H / 2.0) / 3600.0
d1 = dec + (pixscale * H / 2.0) / 3600.0
WISE = get_wise_frames_for_dataset(dataset, r0, r1, d0, d1)
T = fits_table()
T.coadd_id = np.array([dataset])
T.ra = np.array([ra])
T.dec = np.array([dec])
tile = T[0]
ps = None
randomize = False
force = False
medfilt = False
dsky = False
bgmatch = False
cube = False
cube1 = False
rchi_fraction = 0.01
if yearly:
ebreaks = [56000]
etag = "y%i"
else:
ebreaks = get_epoch_breaks(WISE.mjd)
print len(ebreaks), "epoch breaks"
etag = "e%i"
ebreaks = [0] + ebreaks + [1000000]
gifs = []
eims = []
for ei, (elo, ehi) in enumerate(zip(ebreaks, ebreaks[1:])):
ims = []
for band in bands:
WI = WISE[(WISE.mjd >= elo) * (WISE.mjd < ehi) * (WISE.band == band)]
if len(WI) == 0:
ims.append(None)
continue
out = os.path.join(outdir, etag % ei)
outfn = os.path.join(out, "unwise-%s-w%i-img-u.fits" % (dataset, band))
if not os.path.exists(outfn):
trymakedirs(out, dir=True)
print "Band", band, "Epoch", ei
if one_coadd(
tile,
band,
W,
H,
pixscale,
WI,
ps,
False,
out,
mp,
mp,
cube,
False,
None,
None,
force,
medfilt,
0,
dsky,
False,
bgmatch,
False,
False,
rchi_fraction,
cube1,
None,
None,
None,
force_outdir=True,
):
os.unlink(out)
break
print "read", outfn
ims.append(fitsio.read(outfn))
print ims[-1].shape, ims[-1].min(), ims[-1].max()
eims.append(ims)
if bands == [1, 2]:
w1, w2 = ims
if w1 is None or w2 is None:
continue
lo, hi = -10.0, 100.0
w1 = (w1 - lo) / (hi - lo)
w2 = (w2 - lo) / (hi - lo)
assert w1.shape == w2.shape
h, w = w1.shape
rgb = np.zeros((h, w, 3), np.float32)
rgb[:, :, 0] = w2
rgb[:, :, 1] = (w1 + w2) / 2.0
rgb[:, :, 2] = w1
rgb = np.round(np.clip(rgb, 0.0, 1.0) * 255).astype(np.uint8)
fn = os.path.join(outdir, "%s-%s.jpg" % (dataset_tag, etag % ei))
plt.imsave(fn, rgb, origin="lower")
print "Wrote", fn
giffn = os.path.join(outdir, "%s-%s.gif" % (dataset_tag, etag % ei))
cmd = "jpegtopnm %s | pnmquant 256 | ppmtogif > %s" % (fn, giffn)
print cmd
os.system(cmd)
gifs.append(giffn)
anim = os.path.join(outdir, "anim-%s.gif" % dataset_tag)
cmd = "gifsicle -o %s -d 50 -l %s" % (anim, " ".join(gifs))
print cmd
os.system(cmd)
if diffim:
# Difference image vs coadd of all images
allims = []
out = os.path.join(outdir, "all")
trymakedirs(out, dir=True)
for band in bands:
outfn = os.path.join(out, "unwise-%s-w%i-img-u.fits" % (dataset, band))
if not os.path.exists(outfn):
print "Band", band, "Epoch", ei
one_coadd(
tile,
band,
W,
H,
pixscale,
WISE,
ps,
False,
out,
mp,
mp,
cube,
False,
None,
None,
force,
medfilt,
0,
dsky,
False,
bgmatch,
False,
False,
rchi_fraction,
cube1,
None,
None,
None,
force_outdir=True,
)
print "read", outfn
allims.append(fitsio.read(outfn))
print allims[-1].shape, allims[-1].min(), allims[-1].max()
gifs = []
if bands == [1, 2]:
w1all, w2all = allims
if w1all is None or w2all is None:
return 0
for ei, (w1, w2) in enumerate(eims):
lo, hi = -50.0, 50.0
w1 = (w1 - w1all - lo) / (hi - lo)
w2 = (w2 - w2all - lo) / (hi - lo)
print "diff range", w1.min(), w1.max(), w2.min(), w2.max()
h, w = w1.shape
rgb = np.zeros((h, w, 3), np.float32)
rgb[:, :, 0] = w2
rgb[:, :, 1] = (w1 + w2) / 2.0
rgb[:, :, 2] = w1
rgb = np.round(np.clip(rgb, 0.0, 1.0) * 255).astype(np.uint8)
fn = os.path.join(outdir, "diff-%s-%s.jpg" % (dataset_tag, etag % ei))
plt.imsave(fn, rgb, origin="lower")
print "Wrote", fn
giffn = os.path.join(outdir, "diff-%s-%s.gif" % (dataset_tag, etag % ei))
cmd = "jpegtopnm %s | pnmquant 256 | ppmtogif > %s" % (fn, giffn)
print cmd
os.system(cmd)
gifs.append(giffn)
anim = os.path.join(outdir, "danim-%s.gif" % dataset_tag)
cmd = "gifsicle -o %s -d 50 -l %s" % (anim, " ".join(gifs))
print cmd
os.system(cmd)
# Relative difference images.
for ei, (w1, w2) in enumerate(eims):
sig1 = 30.0
w1 = (w1 - w1all) / np.maximum(sig1, w1all)
w2 = (w2 - w2all) / np.maximum(sig1, w2all)
lo, hi = -3.0, 3.0
w1 = (w1 - lo) / (hi - lo)
w2 = (w2 - lo) / (hi - lo)
print "diff range", w1.min(), w1.max(), w2.min(), w2.max()
h, w = w1.shape
rgb = np.zeros((h, w, 3), np.float32)
rgb[:, :, 0] = w2
rgb[:, :, 1] = (w1 + w2) / 2.0
rgb[:, :, 2] = w1
rgb = np.round(np.clip(rgb, 0.0, 1.0) * 255).astype(np.uint8)
fn = os.path.join(outdir, "reldiff-%s-%s.jpg" % (dataset_tag, etag % ei))
plt.imsave(fn, rgb, origin="lower")
print "Wrote", fn
if sdiffim:
# Sequential difference images
for ei, ((w1z, w2z), (w1, w2)) in enumerate(zip(eims, eims[1:])):
# lo,hi = -50.,50.
# lo,hi = -100.,100.
lo, hi = -200.0, 200.0
w1 = (w1 - w1z - lo) / (hi - lo)
w2 = (w2 - w2z - lo) / (hi - lo)
h, w = w1.shape
rgb = np.zeros((h, w, 3), np.float32)
rgb[:, :, 0] = w2
rgb[:, :, 1] = (w1 + w2) / 2.0
rgb[:, :, 2] = w1
rgb = np.round(np.clip(rgb, 0.0, 1.0) * 255).astype(np.uint8)
fn = os.path.join(outdir, "sdiff-%s-%s.jpg" % (dataset_tag, etag % ei))
plt.imsave(fn, rgb, origin="lower")
print "Wrote", fn
# giffn = os.path.join(outdir, 'diff-%s-%s.gif' % (etag % ei, dataset_tag))
# cmd = 'jpegtopnm %s | pnmquant 256 | ppmtogif > %s' % (fn, giffn)
# print cmd
# os.system(cmd)
# gifs.append(giffn)
return
#if rtn:
# sys.exit(-1)
if __name__ == '__main__':
#T = fits_table('data/galex/galex_dstn.fit')
T = fits_table('data/galex/galex-images.fits')
args = []
for tile,sv,path,nt,ft in zip(T.tilename, T.subvis, T.filenpath,
T.nexptime, T.fexptime):
tile = tile.strip()
if nt > 0:
args.append((tile, sv, path, 'n'))
if ft > 0:
args.append((tile, sv, path, 'f'))
mp = multiproc(8)
mp.map(_get_one, args)
T.have_n = np.zeros(len(T), bool)
T.have_f = np.zeros(len(T), bool)
for i,(tile,sv,path) in enumerate(zip(T.tilename, T.subvis, T.filenpath)):
tile = tile.strip()
band = 'n'
if sv == -999:
fnpart = '%s-%sd-intbgsub.fits.gz' % (tile, band)
else:
fnpart = '%s_sg%02i-%sd-intbgsub.fits.gz' % (tile, sv, band)
fn = os.path.join('data','galex', tile, fnpart)
T.have_n[i] = os.path.exists(fn)
band = 'f'
survey = LegacySurveyData()
for brickname in bricks:
print('Checking for existing out file')
# shortcut
dirnm = os.path.join(args.outdir, brickname[:3])
outfn = os.path.join(dirnm, 'ccds-%s.fits' % brickname)
if os.path.exists(outfn):
print('Exists:', outfn)
continue
print('Getting brick', brickname)
brick = survey.get_brick_by_name(brickname)
bargs.append((brick, 0, 1, plots, kwargs))
if args.threads is not None:
mp = multiproc(args.threads)
rtns = mp.map(run_one_brick, bargs)
for rtn in rtns:
if rtn != 0:
allgood = rtn
else:
for arg in bargs:
rtn = run_one_brick(arg)
if rtn != 0:
allgood = rtn
print('Done, result', rtn)
sys.exit(allgood)
else:
queue()
def main():
import optparse
import logging
import sys
parser = optparse.OptionParser()
parser.add_option('--threads', dest='threads', default=1, type=int, help='Use this many concurrent processors')
parser.add_option('-v', '--verbose', dest='verbose', action='count', default=0,
help='Make more verbose')
parser.add_option('--grid', '-g', dest='gridn', type=int, default=5, help='Dust parameter grid size')
parser.add_option('--steps', '-s', dest='steps', type=int, default=10, help='# Optimization step')
parser.add_option('--suffix', dest='suffix', default='', help='Output file suffix')
parser.add_option('--no-100', dest='no100', action='store_true', default=False,
help='Omit PACS-100 data?')
parser.add_option('--callgrind', dest='callgrind', action='store_true', default=False, help='Turn on callgrind around tractor.optimize()')
parser.add_option('--resume', '-r', dest='resume', type=int, default=-1, help='Resume from a previous run at the given step?')
parser.add_option('--zoom', dest='zoom', type=float, default=1, help='Scale down the model to only touch the (1/zoom x 1/zoom) central region of the images')
parser.add_option('--damp', dest='damp', type=float, default=1., help='LSQR damping')
opt,args = parser.parse_args()
if opt.verbose == 0:
lvl = logging.INFO
log_init(2)
else:
lvl = logging.DEBUG
log_init(3)
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
if opt.threads > 1 and False:
global dpool
import debugpool
dpool = debugpool.DebugPool(opt.threads)
Time.add_measurement(debugpool.DebugPoolMeas(dpool))
mp = multiproc(pool=dpool)
else:
print 'N threads', opt.threads
mp = multiproc(opt.threads)#, wrap_all=True)
if opt.callgrind:
import callgrind
else:
callgrind = None
np.seterrcall(np_err_handler)
np.seterr(all='call')
#np.seterr(all='raise')
if opt.resume > -1:
pfn = 'herschel-%02i%s.pickle' % (opt.resume, opt.suffix)
print 'Reading from', pfn
tractor = unpickle_from_file(pfn)
tractor.mp = mp
ds = tractor.getCatalog()[0]
print 'DustSheet:', ds
# derivs = ds.getParamDerivatives(tim)
# dim = np.zeros(tim.shape)
# #for k,deriv in enumerate(derivs[:40]):
# for k,deriv in enumerate(derivs[::10]):
# dim[:,:] = 0
# deriv.addTo(dim)
# plt.clf()
# plt.imshow(dim, interpolation='nearest', origin='lower')
# plt.savefig('deriv-%04i.png' % k)
#tim = tractor.getImages()[0]
# for it,tim in enumerate(tractor.getImages()):
# X = ds._getTransformation(tim)
# # #print 'X', X
# keys = X.keys()
# keys.sort()
# # for k in keys[::10]:
# # for k in keys[:40]:
# for k in keys[::202]:
# I,G,nil,nil = X[k]
# rim = np.zeros_like(tim.getImage())
# rim.ravel()[I] = G
# plt.clf()
# plt.imshow(rim, interpolation='nearest', origin='lower')
# plt.colorbar()
# plt.savefig('rim-%i-%04i.png' % (it,k))
# print 'pix', k
# sys.exit(0)
makeplots(tractor, opt.resume, opt.suffix)
step0 = opt.resume + 1
else:
step0 = 0
tractor = create_tractor(opt)
tractor.mp = mp
# zero out invvar outside the model bounds.
ds = tractor.getCatalog()[0]
rd = ds.getRaDecCorners()
for i,tim in enumerate(tractor.getImages()):
poly = np.array([tim.getWcs().positionToPixel(RaDecPos(rdi[0], rdi[1])) for rdi in rd])
poly = poly[:-1,:]
print 'Model bounding box in image', tim.name, 'coordinates:'
print poly.shape
print poly
H,W = tim.shape
xx,yy = np.meshgrid(np.arange(W), np.arange(H))
inside = point_in_poly(xx, yy, poly)
iv = tim.getInvvar()
iv[(inside == 0)] = 0.
tim.setInvvar(iv)
print 'Precomputing transformations...'
ds = tractor.getCatalog()[0]
# Split the grid-spread matrix into strips...
async_results = []
for im in tractor.getImages():
args = []
H,W = ds.shape
dy = 10
y = 0
while y <= H:
args.append((ds, im, y, min(H, y+dy)))
y += dy
async_results.append(mp.map_async(_map_trans, args))
# Glue to strips back together...
XX = []
for ar in async_results:
Xblocks = ar.get()
X = Xblocks[0]
for xi in Xblocks[1:]:
X.update(xi)
XX.append(X)
for im,X in zip(tractor.getImages(), XX):
ds._normalizeTransformation(im, X)
ds._setTransformation(im, X)
print 'done precomputing.'
makeplots(tractor, 0, opt.suffix)
pfn = 'herschel-%02i%s.pickle' % (0, opt.suffix)
pickle_to_file(tractor, pfn)
print 'Wrote', pfn
for im in tractor.getImages():
im.freezeAllBut('sky')
for i in range(step0, opt.steps):
if callgrind:
callgrind.callgrind_start_instrumentation()
tractor.optimize(damp=opt.damp, alphas=[1e-3, 1e-2, 0.1, 0.3, 1., 3., 10., 30., 100.])
if callgrind:
callgrind.callgrind_stop_instrumentation()
makeplots(tractor, 1 + i, opt.suffix)
pfn = 'herschel-%02i%s.pickle' % (1 + i, opt.suffix)
pickle_to_file(tractor, pfn)
print 'Wrote', pfn
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-b', '--brick', type=int, metavar='BRICK',
help='Brick ID to run: default %(default)s', default=377306)
parser.add_argument('-s', '--sed-matched', action='store_true',
help='Run SED-matched filter?')
parser.add_argument('-B', '--bands', default='grz', help='Bands to retrieve')
parser.add_argument('-o', '--output', metavar='FILE', help='Output filename for catalog',
default='initial-cat.fits')
parser.add_argument('-t', '--threads', type=int, help='Run multi-threaded')
parser.add_argument('-W', '--width', type=int, dest='W', default=3600,
metavar='PIXELS', help='Target image width (default %(default)s)')
parser.add_argument('-H', '--height', type=int, dest='H', default=3600,
metavar='PIXELS', help='Target image height (default %(default)s)')
if not (('BOSS_PHOTOOBJ' in os.environ) and ('PHOTO_RESOLVE' in os.environ)):
print('''$BOSS_PHOTOOBJ and $PHOTO_RESOLVE not set -- on NERSC, you can do:
export BOSS_PHOTOOBJ=/project/projectdirs/cosmo/data/sdss/pre13/eboss/photoObj.v5b
export PHOTO_RESOLVE=/project/projectdirs/cosmo/data/sdss/pre13/eboss/resolve/2013-07-29
To read SDSS files from the local filesystem rather than downloading them.
''')
opt = parser.parse_args()
brickid = opt.brick
bands = opt.bands
if opt.threads and opt.threads > 1:
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads)
else:
mp = multiproc()
ps = None
plots = False
decals = Decals()
brick = decals.get_brick(brickid)
print('Chosen brick:')
brick.about()
targetwcs = wcs_for_brick(brick, W=opt.W, H=opt.H)
W,H = targetwcs.get_width(), targetwcs.get_height()
# Read SDSS sources
cat,T = get_sdss_sources(bands, targetwcs)
if opt.sed_matched:
# Read images
tims = decals.tims_touching_wcs(targetwcs, mp, mock_psf=True, bands=bands)
print('Rendering detection maps...')
detmaps, detivs = detection_maps(tims, targetwcs, bands, mp)
SEDs = sed_matched_filters(bands)
Tnew,newcat,nil = run_sed_matched_filters(SEDs, bands, detmaps, detivs,
(T.itx,T.ity), targetwcs)
T = merge_tables([T,Tnew], columns='fillzero')
cat.extend(newcat)
from desi_common import prepare_fits_catalog
TT = T.copy()
for k in ['itx','ity','index']:
TT.delete_column(k)
for col in TT.get_columns():
if not col in ['tx', 'ty', 'blob']:
TT.rename(col, 'sdss_%s' % col)
TT.brickid = np.zeros(len(TT), np.int32) + brickid
TT.objid = np.arange(len(TT)).astype(np.int32)
invvars = None
hdr = None
fs = None
cat.thawAllRecursive()
T2,hdr = prepare_fits_catalog(cat, invvars, TT, hdr, bands, fs)
# Unpack shape columns
T2.shapeExp_r = T2.shapeExp[:,0]
T2.shapeExp_e1 = T2.shapeExp[:,1]
T2.shapeExp_e2 = T2.shapeExp[:,2]
T2.shapeDev_r = T2.shapeExp[:,0]
T2.shapeDev_e1 = T2.shapeExp[:,1]
T2.shapeDev_e2 = T2.shapeExp[:,2]
T2.shapeExp_r_ivar = T2.shapeExp_ivar[:,0]
T2.shapeExp_e1_ivar = T2.shapeExp_ivar[:,1]
T2.shapeExp_e2_ivar = T2.shapeExp_ivar[:,2]
T2.shapeDev_r_ivar = T2.shapeExp_ivar[:,0]
T2.shapeDev_e1_ivar = T2.shapeExp_ivar[:,1]
T2.shapeDev_e2_ivar = T2.shapeExp_ivar[:,2]
T2.writeto(opt.output)
print('Wrote', opt.output)
return 0