def run(rerun, # Rerun name
frames, # Frame number
ccds, # CCD number
stack, # Stack identifier
patch, # Patch identifier
filter, # Filter name
config, # Configuration
coords, # Skycell centre coordinates
scale, # Pixel scale
sizes, # Skycell size
ignore=False, # Ignore missing files?
):
io = pipReadWrite.ReadWrite(hsc.HscSimMapper(rerun=rerun), ['visit', 'ccd'], config=config)
roots = config['roots']
basename = os.path.join(roots['output'], rerun)
stackProc = pipStack.Stack(config=config)
identMatrix = list()
for frame in frames:
identList = list()
for ccd in ccds:
dataId = { 'visit': frame, 'ccd': ccd }
identList.append(dataId)
identMatrix.append(identList)
exp = stackProc.run(identMatrix, io.inButler,
coords[0], coords[1], scale, sizes[0], sizes[1], ignore=ignore)
#stack.writeFits(basename + ".fits")
stackProc.write(io.outButler, {'stack': stack, 'patch': patch, 'filter': filter}, {"stack": exp})
def run(
rerun, # Rerun name
stack, # Stack identifier
filter, # Filter name
field, # Field name
scale, # Scale, arcsec/pix
):
io = pipReadWrite.ReadWrite(hsc.HscSimMapper(rerun=rerun),
['visit', 'ccd'])
skyPolicy = io.inButler.get('skypolicy')
print skyPolicy.toString()
sky = skypix.QuadSpherePixelization(
skyPolicy.get('resolutionPix'),
skyPolicy.get('paddingArcsec') / 3600.0)
dataId = {'filter': filter}
if field is not None:
dataId['field'] = field
skytiles = io.inButler.queryMetadata('calexp', None, 'skyTile', dataId)
for tile in skytiles:
dataId['skyTile'] = tile
visits = io.inButler.queryMetadata('calexp', None, 'visit', dataId)
if len(visits) == 0:
continue
geom = sky.getGeometry(tile)
bbox = geom.getBoundingBox()
ra, dec = bbox.getCenter() # Degrees
theta = bbox.getThetaExtent() # Width, degrees
size = int(theta * 3600.0 / scale) # Size, pixels
cmd = "hsc_stack.py --rerun " + rerun
cmd += " --stack %d" % stack
cmd += " --patch %d" % tile
cmd += " --filter %s" % filter
cmd += " --coords %f %f" % (ra, dec)
cmd += " --scale %f" % scale
cmd += " --sizes %d %d" % (size, size)
cmd += " --frames %s" % ":".join(map(str, visits))
cmd += " --ignore"
print cmd
def run(rerun, # Rerun name
frame, # Frame number
ccds, # CCD number
patch, # Patch identifier
config, # Configuration
coords, # Skycell centre coordinates
scale, # Pixel scale
sizes, # Skycell size
):
io = pipReadWrite.ReadWrite(hsc.HscSimMapper(rerun=rerun), ['visit', 'ccd'], config=config)
roots = config['roots']
basename = os.path.join(roots['output'], '%s-%d' % (rerun, frame))
warpProc = pipWarp.Warp(config=config)
identList = list()
for ccd in ccds:
dataId = { 'visit': frame, 'ccd': ccd }
identList.append(dataId)
warp = warpProc.run(identList, io.inButler, coords[0], coords[1], scale, sizes[0], sizes[1])
#warp.writeFits(basename + ".fits")
warpProc.write(io.outButler, {'visit': frame, 'skytile': skytile}, {"warp": warp})
def main(dataDir,
visit,
title="",
outputTxtFileName=None,
showFwhm=False,
minFwhm=None,
maxFwhm=None,
correctDistortion=False,
showEllipticity=False,
ellipticityDirection=False,
showNdataFwhm=False,
showNdataEll=False,
minNdata=None,
maxNdata=None,
gridPoints=30,
verbose=False):
butler = dafPersist.ButlerFactory(mapper=hscSim.HscSimMapper(
root=dataDir)).create()
camera = butler.get("camera")
if not (showFwhm or showEllipticity or showNdataFwhm or showNdataEll
or outputTxtFileName):
showFwhm = True
#
# Get a dict of cameraGeom::Ccd indexed by serial number
#
ccds = {}
for raft in camera:
for ccd in raft:
ccd.setTrimmed(True)
ccds[ccd.getId().getSerial()] = ccd
#
# Read all the tableSeeingMap files, converting their (x, y) to focal plane coordinates
#
xArr = []
yArr = []
ellArr = []
fwhmArr = []
paArr = []
aArr = []
bArr = []
e1Arr = []
e2Arr = []
elle1e2Arr = []
for tab in butler.subset("tableSeeingMap", visit=visit):
# we could use tab.datasetExists() but it prints a rude message
fileName = butler.get("tableSeeingMap_filename", **tab.dataId)[0]
if not os.path.exists(fileName):
continue
with open(fileName) as fd:
ccd = None
for line in fd.readlines():
if re.search(r"^\s*#", line):
continue
fields = [float(_) for _ in line.split()]
if ccd is None:
ccd = ccds[int(fields[0])]
x, y, fwhm, ell, pa, a, b = fields[1:8]
x, y = ccd.getPositionFromPixel(afwGeom.PointD(x, y)).getMm()
xArr.append(x)
yArr.append(y)
ellArr.append(ell)
fwhmArr.append(fwhm)
paArr.append(pa)
aArr.append(a)
bArr.append(b)
if len(fields) == 11:
e1 = fields[8]
e2 = fields[9]
elle1e2 = fields[10]
else:
e1 = -9999.
e2 = -9999.
elle1e2 = -9999.
e1Arr.append(e1)
e2Arr.append(e2)
elle1e2Arr.append(elle1e2)
xArr = np.array(xArr)
yArr = np.array(yArr)
ellArr = np.array(ellArr)
fwhmArr = np.array(fwhmArr) * 0.168 # arcseconds
paArr = np.radians(np.array(paArr))
aArr = np.array(aArr)
bArr = np.array(bArr)
e1Arr = np.array(e1Arr)
e2Arr = np.array(e2Arr)
elle1e2Arr = np.array(elle1e2Arr)
if correctDistortion:
import lsst.afw.geom.ellipses as afwEllipses
dist = camera.getDistortion()
for i in range(len(aArr)):
axes = afwEllipses.Axes(aArr[i], bArr[i], paArr[i])
if False: # testing only!
axes = afwEllipses.Axes(1.0, 1.0, np.arctan2(yArr[i], xArr[i]))
quad = afwEllipses.Quadrupole(axes)
quad = quad.transform(
dist.computeQuadrupoleTransform(
afwGeom.PointD(xArr[i], yArr[i]), False))
axes = afwEllipses.Axes(quad)
aArr[i], bArr[i], paArr[i] = axes.getA(), axes.getB(
), axes.getTheta()
ellArr = 1 - bArr / aArr
if len(xArr) == 0:
gridPoints = 0
xs, ys = [], []
else:
N = gridPoints * 1j
extent = [min(xArr), max(xArr), min(yArr), max(yArr)]
xs, ys = np.mgrid[extent[0]:extent[1]:N, extent[2]:extent[3]:N]
title = [
title,
]
title.append("\n#")
if outputTxtFileName:
f = open(outputTxtFileName, 'w')
f.write("# %s visit %s\n" % (" ".join(title), visit))
for x, y, ell, fwhm, pa, a, b, e1, e2, elle1e2 in zip(
xArr, yArr, ellArr, fwhmArr, paArr, aArr, bArr, e1Arr, e2Arr,
elle1e2Arr):
f.write('%f %f %f %f %f %f %f %f %f %f\n' %
(x, y, ell, fwhm, pa, a, b, e1, e2, elle1e2))
if showFwhm:
title.append("FWHM (arcsec)")
if len(xs) > 0:
fwhmResampled = griddata(xArr, yArr, fwhmArr, xs, ys)
plt.imshow(fwhmResampled.T,
extent=extent,
vmin=minFwhm,
vmax=maxFwhm,
origin='lower')
plt.colorbar()
if outputTxtFileName:
ndataGrids = getNumDataGrids(xArr, yArr, fwhmArr, xs, ys)
f = open(outputTxtFileName + '-fwhm-grid.txt', 'w')
f.write("# %s visit %s\n" % (" ".join(title), visit))
for xline, yline, fwhmline, ndataline in zip(
xs.tolist(), ys.tolist(), fwhmResampled.tolist(),
ndataGrids):
for xx, yy, fwhm, ndata in zip(xline, yline, fwhmline,
ndataline):
if fwhm is None:
fwhm = -9999
f.write('%f %f %f %d\n' % (xx, yy, fwhm, ndata))
elif showEllipticity:
title.append("Ellipticity")
scale = 4
if ellipticityDirection: # we don't care about the magnitude
ellArr = 0.1
u = -ellArr * np.cos(paArr)
v = -ellArr * np.sin(paArr)
if gridPoints > 0:
u = griddata(xArr, yArr, u, xs, ys)
v = griddata(xArr, yArr, v, xs, ys)
x, y = xs, ys
else:
x, y = xArr, yArr
Q = plt.quiver(
x,
y,
u,
v,
scale=scale,
pivot="middle",
headwidth=0,
headlength=0,
headaxislength=0,
)
keyLen = 0.10
if not ellipticityDirection: # we care about the magnitude
plt.quiverkey(Q, 0.20, 0.95, keyLen, "e=%g" % keyLen, labelpos='W')
if outputTxtFileName:
ndataGrids = getNumDataGrids(xArr, yArr, ellArr, xs, ys)
f = open(outputTxtFileName + '-ell-grid.txt', 'w')
f.write("# %s visit %s\n" % (" ".join(title), visit))
#f.write('# %f %f %f %f %f %f %f\n' % (x, y, ell, fwhm, pa, a, b))
for xline, yline, uline, vline, ndataline in zip(
x.tolist(), y.tolist(), u.tolist(), v.tolist(),
ndataGrids):
for xx, yy, uu, vv, ndata in zip(xline, yline, uline, vline,
ndataline):
if uu is None:
uu = -9999
if vv is None:
vv = -9999
f.write('%f %f %f %f %d\n' % (xx, yy, uu, vv, ndata))
elif showNdataFwhm:
title.append("N per fwhm grid")
if len(xs) > 0:
ndataGrids = getNumDataGrids(xArr, yArr, fwhmArr, xs, ys)
plt.imshow(ndataGrids,
interpolation='nearest',
extent=extent,
vmin=minNdata,
vmax=maxNdata,
origin='lower')
plt.colorbar()
else:
pass
elif showNdataEll:
title.append("N per ell grid")
if len(xs) > 0:
ndataGrids = getNumDataGrids(xArr, yArr, ellArr, xs, ys)
plt.imshow(ndataGrids,
interpolation='nearest',
extent=extent,
vmin=minNdata,
vmax=maxNdata,
origin='lower')
plt.colorbar()
else:
pass
#plt.plot(xArr, yArr, "r.")
#plt.plot(xs, ys, "b.")
plt.axes().set_aspect('equal')
plt.axis([-20000, 20000, -20000, 20000])
def frameInfoFrom(filepath):
import pyfits
with pyfits.open(filepath) as hdul:
h = hdul[0].header
'object=ABELL2163 filter=HSC-I exptime=360.0 alt=62.11143274 azm=202.32265181 hst=(23:40:08.363-23:40:48.546)'
return 'object=%s filter=%s exptime=%.1f azm=%.2f hst=%s' % (
h['OBJECT'], h['FILTER01'], h['EXPTIME'], h['AZIMUTH'],
h['HST'])
title.insert(
0,
frameInfoFrom(
butler.get('raw_filename', {
'visit': visit,
'ccd': 0
})[0]))
title.append(r'$\langle$FWHM$\rangle %4.2f$"' % np.median(fwhmArr))
plt.title("%s visit=%s" % (" ".join(title), visit), fontsize=9)
return plt
def run(outName,
rerun,
frame1,
frame2,
config,
matchTol=1.0,
bright=None,
ccd=None):
io = pipReadWrite.ReadWrite(hscSim.HscSimMapper(rerun=rerun), ['visit'],
fileKeys=['visit', 'ccd'],
config=config)
roots = config['roots']
outName = os.path.join(roots['output'], '%s.pdf' %
outName) if outName is not None else None
data1 = {'visit': frame1}
data2 = {'visit': frame2}
if ccd is not None:
data1['ccd'] = ccd
data2['ccd'] = ccd
sources1 = io.read('src', data1, ignore=True)
md1 = io.read('calexp_md', data1, ignore=True)
sources2 = io.read('src', data2, ignore=True)
md2 = io.read('calexp_md', data2, ignore=True)
assert len(sources1) == len(md1)
for i in range(len(sources1)):
sources1[i] = filterSources(sources1[i], md1[i], bright)
sources1 = concatenate(sources1)
print len(sources1), "sources filtered from", frame1
assert len(sources2) == len(md2)
for i in range(len(sources2)):
sources2[i] = filterSources(sources2[i], md2[i], bright)
sources2 = concatenate(sources2)
print len(sources2), "sources filtered from", frame2
comp = pipCompare.Comparisons(sources1, sources2, matchTol=matchTol)
print "%d matches" % comp.num
ra = (comp['ra1'] + comp['ra2']) / 2.0
dec = (comp['dec1'] + comp['dec2']) / 2.0
if False:
psfAvg = (comp['psf1'] + comp['psf2']) / 2.0
psfDiff = comp['psf1'] - comp['psf2']
apAvg = (comp['ap1'] + comp['ap2']) / 2.0
apDiff = comp['ap1'] - comp['ap2']
modelAvg = (comp['model1'] + comp['model2']) / 2.0
modelDiff = comp['model1'] - comp['model2']
else:
psfAvg = (-2.5 * numpy.log10(comp['psf1']) -
2.5 * numpy.log10(comp['psf2'])) / 2.0
psfDiff = -2.5 * numpy.log10(comp['psf1']) + 2.5 * numpy.log10(
comp['psf2'])
apAvg = (-2.5 * numpy.log10(comp['ap1']) -
2.5 * numpy.log10(comp['ap2'])) / 2.0
apDiff = (-2.5 * numpy.log10(comp['ap1']) +
2.5 * numpy.log10(comp['ap2']))
modelAvg = (-2.5 * numpy.log10(comp['model1']) -
2.5 * numpy.log10(comp['model2'])) / 2.0
modelDiff = (-2.5 * numpy.log10(comp['model1']) +
2.5 * numpy.log10(comp['model2']))
plot = plotter.Plotter(outName)
plot.xy(ra, dec, title="Detections")
plot.xy(psfAvg,
psfDiff,
axis=[comp['psf1'].min(), comp['psf1'].max(), -0.25, 0.25],
title="PSF photometry")
plot.xy(apAvg,
apDiff,
axis=[comp['ap1'].min(), comp['ap1'].max(), -0.25, 0.25],
title="Aperture photometry")
plot.histogram(psfDiff, [-0.25, 0.25], title="PSF photometry")
plot.histogram(apDiff, [-0.25, 0.25], title="Aperture photometry")
plot.histogram(modelDiff, [-0.25, 0.25], title="Model photometry")
plot.xy(psfAvg, modelAvg - psfAvg, title="PSF vs Model")
plot.histogram(modelAvg - psfAvg, [-0.6, 0.2],
bins=81,
title="PSF vs Model")
plot.xy2(ra,
comp['distance'],
dec,
comp['distance'],
axis1=[ra.min(), ra.max(), 0, matchTol],
axis2=[dec.min(), dec.max(), 0, matchTol],
title1="Right ascension",
title2="Declination")
plot.quivers(ra,
dec,
comp['ra1'] - comp['ra2'],
comp['dec1'] - comp['dec2'],
title="Astrometry",
addUnitQuiver=1.0 / 3600.0)
plot.close()
def run(
rerun, # Rerun name
frame, # Frame number
ccd, # CCD number
config, # Configuration
log=pexLog.Log.getDefaultLog(), # Log object
):
# Make our own mappers for now
mapperArgs = {'rerun': rerun} # Arguments for mapper instantiation
if config.has_key('roots'):
roots = config['roots']
for key, value in {
'data': 'root',
'calib': 'calibRoot',
'output': 'outRoot'
}.iteritems():
if roots.has_key(key):
mapperArgs[value] = roots[key]
camera = config['camera']
if camera.lower() in ("hsc"):
mapper = obsHsc.HscSimMapper(**mapperArgs)
ccdProc = pipCcd.ProcessCcd(config=config,
Calibrate=CalibrateHscDc2,
log=log)
elif camera.lower() in ("suprimecam-mit", "sc-mit", "scmit",
"suprimecam-old", "sc-old", "scold"):
mapper = obsSc.SuprimecamMapper(mit=True, **mapperArgs)
ccdProc = ProcessCcdSuprimeCam(config=config, log=log)
elif camera.lower() in ("suprimecam", "suprime-cam", "sc"):
mapper = obsSc.SuprimecamMapper(**mapperArgs)
ccdProc = ProcessCcdSuprimeCam(config=config, log=log)
io = pipReadWrite.ReadWrite(mapper, ['visit', 'ccd'], config=config)
oldUmask = os.umask(2)
if oldUmask != 2:
io.log.log(io.log.WARN, "pipette umask started as: %s" % (os.umask(2)))
dataId = {'visit': frame, 'ccd': ccd}
raws = io.readRaw(dataId)
detrends = io.detrends(dataId, config)
if len([x for x in detrends
if x]): # We need to run at least part of the ISR
raws = io.readRaw(dataId)
else:
io.fileKeys = ['visit', 'ccd']
try:
raws = io.read('calexp', dataId)
config['do']['calibrate']['repair']['cosmicray'] = False
except:
raws = io.readRaw(dataId)
detrends = None
exposure, psf, apcorr, brightSources, sources, matches, matchMeta = ccdProc.run(
raws, detrends)
io.write(dataId, exposure=None, psf=psf, sources=None)
catPolicy = os.path.join(os.getenv("PIPETTE_DIR"), "policy", "catalog.paf")
catalog = pipCatalog.Catalog(catPolicy, allowNonfinite=False)
deferredState = DeferredHSCState(dataId, io, matches, matchMeta, sources,
brightSources, exposure)
return deferredState