def testImageToPixels(self):
"""Tests intersection of an image (WCS and dimensions) with a
quad-sphere pixelization.
"""
#metadata taken from CFHT data v695856-e0/v695856-e0-c000-a00.sci_img.fits
metadata = dafBase.PropertySet()
metadata.set("SIMPLE", "T")
metadata.set("BITPIX", -32)
metadata.set("NAXIS", 2)
metadata.set("NAXIS1", 1024)
metadata.set("NAXIS2", 1153)
metadata.set("RADECSYS", "FK5")
metadata.set("EQUINOX", 2000.0)
metadata.set("CRVAL1", 215.604025685476)
metadata.set("CRVAL2", 53.1595451514076)
metadata.set("CRPIX1", 1109.99981456774)
metadata.set("CRPIX2", 560.018167811613)
metadata.set("CTYPE1", "RA---TAN")
metadata.set("CTYPE2", "DEC--TAN")
metadata.set("CD1_1", 5.10808596133527E-05)
metadata.set("CD1_2", 1.85579539217196E-07)
metadata.set("CD2_2", -5.10281493481982E-05)
metadata.set("CD2_1", -8.27440751733828E-07)
wcs = afwImage.makeWcs(metadata)
qs = skypix.createQuadSpherePixelization()
poly = skypix.imageToPolygon(wcs, 1024, 1153)
pixels = qs.intersect(poly)
self.assertEqual(len(pixels), 1)
self.assertEqual(pixels[0], 182720)
self.assertTrue(qs.getGeometry(182720).contains(poly))
def testImageToPixels(self):
"""Tests intersection of an image (WCS and dimensions) with a
quad-sphere pixelization.
"""
# metadata taken from CFHT data v695856-e0/v695856-e0-c000-a00.sci_img.fits
metadata = dafBase.PropertySet()
metadata.set("SIMPLE", "T")
metadata.set("BITPIX", -32)
metadata.set("NAXIS", 2)
metadata.set("NAXIS1", 1024)
metadata.set("NAXIS2", 1153)
metadata.set("RADECSYS", "FK5")
metadata.set("EQUINOX", 2000.0)
metadata.set("CRVAL1", 215.604025685476)
metadata.set("CRVAL2", 53.1595451514076)
metadata.set("CRPIX1", 1109.99981456774)
metadata.set("CRPIX2", 560.018167811613)
metadata.set("CTYPE1", "RA---TAN")
metadata.set("CTYPE2", "DEC--TAN")
metadata.set("CD1_1", 5.10808596133527E-05)
metadata.set("CD1_2", 1.85579539217196E-07)
metadata.set("CD2_2", -5.10281493481982E-05)
metadata.set("CD2_1", -8.27440751733828E-07)
wcs = afwImage.makeWcs(metadata)
qs = skypix.createQuadSpherePixelization()
poly = skypix.imageToPolygon(wcs, 1024, 1153)
pixels = qs.intersect(poly)
self.assertEqual(len(pixels), 1)
self.assertEqual(pixels[0], 182720)
self.assertTrue(qs.getGeometry(182720).contains(poly))
def processRaft(raftDir, conn, done, qsp):
nProcessed = 0
nSkipped = 0
nUnrecognized = 0
for fits in glob.glob(
os.path.join(
raftDir, "S[0-2][0-2]",
"imsim_*_R[0-4][0-4]_S[0-2][0-2]_C[01][0-7]_E00[01].fits*")):
m = re.search(
r'v(\d+)-f(\w)/E00(\d)/R(\d)(\d)/S(\d)(\d)/' +
r'imsim_\1_R\4\5_S\6\7_C(\d)(\d)_E00\3\.fits', fits)
if not m:
print >> sys.stderr, "Warning: Unrecognized file:", fits
nUnrecognized += 1
continue
(visit, filter, snap, raft1, raft2, sensor1, sensor2, channel1,
channel2) = m.groups()
key = "%s_F%s_E%s_R%s,%s_S%s,%s_C%s,%s" % (visit, filter, snap, raft1,
raft2, sensor1, sensor2,
channel1, channel2)
if key in done:
nSkipped += 1
continue
md = afwImage.readMetadata(fits)
expTime = md.get("EXPTIME")
mjdObs = md.get("MJD-OBS")
taiObs = dafBase.DateTime(mjdObs, dafBase.DateTime.MJD,
dafBase.DateTime.TAI).toString(
dafBase.DateTime.UTC)[:-1]
conn.execute(
"""INSERT INTO raw VALUES
(NULL, ?, ?, ?, ?, ?, ?, ?, ?)""",
(visit, filter, snap, "%s,%s" % (raft1, raft2), "%s,%s" %
(sensor1, sensor2), "%s,%s" %
(channel1, channel2), taiObs, expTime))
for row in conn.execute("SELECT last_insert_rowid()"):
id = row[0]
break
wcs = afwImage.makeWcs(md)
poly = skypix.imageToPolygon(wcs,
md.get("NAXIS1"),
md.get("NAXIS2"),
padRad=0.000075) # about 15 arcsec
pix = qsp.intersect(poly)
for skyTileId in pix:
conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)",
(id, skyTileId))
conn.commit()
nProcessed += 1
print >>sys.stderr, raftDir, \
"... %d processed, %d skipped, %d unrecognized" % \
(nProcessed, nSkipped, nUnrecognized)
def processRaft(raftDir, conn, done, qsp):
nProcessed = 0
nSkipped = 0
nUnrecognized = 0
for fits in glob.glob(os.path.join(raftDir, "S[0-2][0-2]",
"imsim_*_R[0-4][0-4]_S[0-2][0-2]_C[01][0-7]_E00[01].fits*")):
import pdb; pdb.set_trace()
m = re.search(r'v(\d+)-f(\w)/E00(\d)/R(\d)(\d)/S(\d)(\d)/' +
r'imsim_\1_R\4\5_S\6\7_C(\d)(\d)_E00\3\.fits', fits)
if not m:
print >>sys.stderr, "Warning: Unrecognized file:", fits
nUnrecognized += 1
continue
(visit, filter, snap, raft1, raft2, sensor1, sensor2,
channel1, channel2) = m.groups()
key = "%s_F%s_E%s_R%s,%s_S%s,%s_C%s,%s" % (visit, filter,
snap, raft1, raft2, sensor1, sensor2, channel1, channel2)
if done.has_key(key):
nSkipped += 1
continue
md = afwImage.readMetadata(fits)
expTime = md.get("EXPTIME")
mjdObs = md.get("MJD-OBS")
taiObs = dafBase.DateTime(mjdObs, dafBase.DateTime.MJD,
dafBase.DateTime.TAI).toString()[:-1]
conn.execute("""INSERT INTO raw VALUES
(NULL, ?, ?, ?, ?, ?, ?, ?, ?)""",
(visit, filter, snap, "%s,%s" % (raft1, raft2),
"%s,%s" % (sensor1, sensor2),
"%s,%s" % (channel1, channel2), taiObs, expTime))
for row in conn.execute("SELECT last_insert_rowid()"):
id = row[0]
break
wcs = afwImage.makeWcs(md)
poly = skypix.imageToPolygon(wcs,
md.get("NAXIS1"), md.get("NAXIS2"),
padRad=0.000075) # about 15 arcsec
pix = qsp.intersect(poly)
for skyTileId in pix:
conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)",
(id, skyTileId))
conn.commit()
nProcessed += 1
print >>sys.stderr, raftDir, \
"... %d processed, %d skipped, %d unrecognized" % \
(nProcessed, nSkipped, nUnrecognized)
def findWcsCoveringSkyTile(skyPixelization, skyTileId, imageRes):
"""Computes and returns a TAN WCS such that a 2D image with the
given WCS and the following properties completely covers the
sky-tile with the given pixel id:
- NAXIS1/NAXIS2 >= imageRes
- CRPIX1 = NAXIS1 / 2 + 0.5
- CRPIX2 = NAXIS2 / 2 + 0.5
"""
if not isinstance(imageRes, (int, long)):
raise TypeError("Image resolution must be an integer")
if imageRes < 1:
raise RuntimeError("Image resolution must be at least 1")
crpix = afwGeom.Point2D(0.5 * (imageRes + 1), 0.5 * (imageRes + 1))
crval = geom.sphericalCoords(skyPixelization.getCenter(skyTileId))
crval = afwCoord.makeCoord(afwCoord.ICRS, crval[0] * afwGeom.degrees,
crval[1] * afwGeom.degrees)
skyTile = skyPixelization.getGeometry(skyTileId)
# Start with a huge TAN image centered at the sky-tile center,
# then shrink it using binary search to determine suitable
# CD matrix coefficients
scale = 1000.0 # deg/pixel, ridiculously large
delta = 0.5 * scale
frac = 0.01 # desired relative accuracy of CD matrix coeffs
wcs = afwImage.makeWcs(crval, crpix, scale, 0.0, 0.0, scale)
imagePoly = skypix.imageToPolygon(wcs, imageRes, imageRes)
# Make sure the initial guess really is too large
if not imagePoly.contains(skyTile):
raise RuntimeError("Failed to construct image WCS covering sky-tile")
# Search for a WCS with a tight fit to the sky-tile. Note that the
# tightness of fit could be further improved by searching for a rotation
# and not just a pixel scale.
while delta >= frac * scale:
tmp = scale - delta
wcs = afwImage.makeWcs(crval, crpix, tmp, 0.0, 0.0, tmp)
imagePoly = skypix.imageToPolygon(wcs, imageRes, imageRes)
delta *= 0.5
if imagePoly.contains(skyTile):
scale = tmp
return afwImage.makeWcs(crval, crpix, scale, 0.0, 0.0, scale)
def processBand(filterDir, conn, done, qsp):
nProcessed = 0
nSkipped = 0
nUnrecognized = 0
print(filterDir, "... started", file=sys.stderr)
for fits in glob.iglob(
os.path.join(filterDir, "fpC*_ts_coaddNorm_NN.fit.gz")):
m = re.search(
r'/([ugriz])/fpC-(\d{6})-\1(\d)-(\d{4})_ts_coaddNorm_NN.fit.gz',
fits)
if not m:
print("Warning: Unrecognized file:", fits, file=sys.stderr)
nUnrecognized += 1
continue
(filter, run, camcol, field) = m.groups()
camcol = int(camcol)
run = int(run)
field = int(field)
key = "%d_B%s_C%d_F%d" % (run, filter, camcol, field)
if key in done:
nSkipped += 1
continue
md = readMetadata(fits)
conn.execute(
"""INSERT INTO raw VALUES
(NULL, ?, ?, ?, ?)""", (run, filter, camcol, field))
for row in conn.execute("SELECT last_insert_rowid()"):
id = row[0]
break
wcs = makeSkyWcs(md)
poly = skypix.imageToPolygon(wcs,
md.get("NAXIS1"),
md.get("NAXIS2"),
padRad=0.000075) # about 15 arcsec
pix = qsp.intersect(poly)
for skyTileId in pix:
conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)",
(id, skyTileId))
nProcessed += 1
if nProcessed % 100 == 0:
conn.commit()
conn.commit()
print(filterDir,
"... %d processed, %d skipped, %d unrecognized" %
(nProcessed, nSkipped, nUnrecognized),
file=sys.stderr)
def findWcsCoveringSkyTile(skyPixelization, skyTileId, imageRes):
"""Computes and returns a TAN WCS such that a 2D image with the
given WCS and the following properties completely covers the
sky-tile with the given pixel id:
- NAXIS1/NAXIS2 >= imageRes
- CRPIX1 = NAXIS1 / 2 + 0.5
- CRPIX2 = NAXIS2 / 2 + 0.5
"""
if not isinstance(imageRes, (int, long)):
raise TypeError("Image resolution must be an integer")
if imageRes < 1:
raise RuntimeError("Image resolution must be at least 1")
crpix = afwGeom.Point2D(0.5*(imageRes + 1), 0.5*(imageRes + 1))
crval = geom.sphericalCoords(skyPixelization.getCenter(skyTileId))
crval = afwCoord.makeCoord(afwCoord.ICRS, crval[0] * afwGeom.degrees, crval[1] * afwGeom.degrees)
skyTile = skyPixelization.getGeometry(skyTileId)
# Start with a huge TAN image centered at the sky-tile center,
# then shrink it using binary search to determine suitable
# CD matrix coefficients
scale = 1000.0 # deg/pixel, ridiculously large
delta = 0.5*scale
frac = 0.01 # desired relative accuracy of CD matrix coeffs
wcs = afwImage.makeWcs(crval, crpix, scale, 0.0, 0.0, scale)
imagePoly = skypix.imageToPolygon(wcs, imageRes, imageRes)
# Make sure the initial guess really is too large
if not imagePoly.contains(skyTile):
raise RuntimeError("Failed to construct image WCS covering sky-tile")
# Search for a WCS with a tight fit to the sky-tile. Note that the
# tightness of fit could be further improved by searching for a rotation
# and not just a pixel scale.
while delta >= frac * scale:
tmp = scale - delta
wcs = afwImage.makeWcs(crval, crpix, tmp, 0.0, 0.0, tmp)
imagePoly = skypix.imageToPolygon(wcs, imageRes, imageRes)
delta *= 0.5
if imagePoly.contains(skyTile):
scale = tmp
return afwImage.makeWcs(crval, crpix, scale, 0.0, 0.0, scale)
def processBand(filterDir, conn, done, qsp):
nProcessed = 0
nSkipped = 0
nUnrecognized = 0
print >>sys.stderr, filterDir, "... started"
for fits in glob.iglob(
os.path.join(filterDir, "fpC*_ts_coaddNorm_NN.fit.gz")):
m = re.search(r'/([ugriz])/fpC-(\d{6})-\1(\d)-(\d{4})_ts_coaddNorm_NN.fit.gz', fits)
if not m:
print >>sys.stderr, "Warning: Unrecognized file:", fits
nUnrecognized += 1
continue
(filter, run, camcol, field) = m.groups()
camcol = int(camcol)
run = int(run)
field = int(field)
key = "%d_B%s_C%d_F%d" % (run, filter, camcol, field)
if done.has_key(key):
nSkipped += 1
continue
md = afwImage.readMetadata(fits)
conn.execute("""INSERT INTO raw VALUES
(NULL, ?, ?, ?, ?)""", (run, filter, camcol, field))
for row in conn.execute("SELECT last_insert_rowid()"):
id = row[0]
break
wcs = afwImage.makeWcs(md)
poly = skypix.imageToPolygon(wcs,
md.get("NAXIS1"), md.get("NAXIS2"),
padRad=0.000075) # about 15 arcsec
pix = qsp.intersect(poly)
for skyTileId in pix:
conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)",
(id, skyTileId))
nProcessed += 1
if nProcessed % 100 == 0:
conn.commit()
conn.commit()
print >>sys.stderr, filterDir, \
"... %d processed, %d skipped, %d unrecognized" % \
(nProcessed, nSkipped, nUnrecognized)
def processRawDir(rawDir, conn, done, qsp):
print(rawDir, "... started")
nProcessed = 0
nSkipped = 0
nUnrecognized = 0
for fitsPath in glob.glob(os.path.join(rawDir, "*.fits*")):
m = re.search(r'raw_v(\d*)_f(.+)\.fits', fitsPath)
if not m:
sys.stderr.write("Warning: Unrecognized file: %r\n" % (fitsPath,))
nUnrecognized += 1
continue
visit, filterName = m.groups()
key = "%s_f%s" % (visit, filterName)
if done.has_key(key):
nSkipped += 1
continue
md = afwImage.readMetadata(fitsPath)
expTime = md.get("EXPTIME")
mjdObs = md.get("MJD-OBS")
taiObs = dafBase.DateTime(mjdObs, dafBase.DateTime.MJD,
dafBase.DateTime.TAI).toString()[:-1]
conn.execute("""INSERT INTO raw VALUES
(NULL, ?, ?, ?, ?)""",
(visit, filterName, taiObs, expTime))
for row in conn.execute("SELECT last_insert_rowid()"):
id = row[0]
break
wcs = afwImage.makeWcs(md)
poly = skypix.imageToPolygon(wcs,
md.get("NAXIS1"), md.get("NAXIS2"),
padRad=0.000075) # about 15 arcsec
pix = qsp.intersect(poly)
for skyTileId in pix:
conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)",
(id, skyTileId))
conn.commit()
nProcessed += 1
print("%s... %d processed, %d skipped, %d unrecognized" %
(rawDir, nProcessed, nSkipped, nUnrecognized))
dafBase.DateTime.UTC).toString()[:-1]
except:
dateObsCard = md.get("DATE-OBS")
if dateObsCard.find('T') == -1:
dateObsCard += "T00:00:00Z"
taiObs = dafBase.DateTime(dateObsCard).toString()[:-1]
if md.get("CTYPE1").strip() == "CPX" and md.get(
"CTYPE2").strip() == "CPY": # HSC pixel WCS
pix = []
else:
wcs = afwImage.makeWcs(fixSubaruHeader(md))
try:
poly = skypix.imageToPolygon(wcs,
md.get("NAXIS1"),
md.get("NAXIS2"),
padRad=math.radians(15.0 / 3600))
except ZeroDivisionError, e:
print >> sys.stderr, "\nFailure finding polygon for %s: %s" % (
fits, e)
poly = None
if poly:
pix = qsp.intersect(poly)
else:
pix = []
cmd = "INSERT INTO raw VALUES (NULL, ?, ?, ?, ?, ?, ?, ?, ?)"
if cursor:
try:
data = (field, visit, filterId, ccd, dateObs, taiObs, expTime,
def processRun(runDir, conn, done, qsp):
nProcessed = 0
nSkipped = 0
nUnrecognized = 0
print(runDir, "... started", file=sys.stderr)
for fits in glob.iglob(
os.path.join(runDir, "*", "corr", "[1-6]", "fpC*.fit.gz")):
m = re.search(
r'(\d+)/corr/([1-6])/fpC-(\d{6})-([ugriz])\2-(\d{4}).fit.gz', fits)
if not m:
print("Warning: Unrecognized file:", fits, file=sys.stderr)
nUnrecognized += 1
continue
(rerun, camcol, run, filter, field) = m.groups()
rerun = int(rerun)
camcol = int(camcol)
run = int(run)
field = int(field)
key = "%d_R%d_B%s_C%d_F%d" % (run, rerun, filter, camcol, field)
if key in done or rerun < 40:
nSkipped += 1
continue
md = afwImage.readMetadata(fits)
date = md.get("DATE-OBS")
if date.find("-") != -1:
(year, month, day) = md.get("DATE-OBS").split("-")
else:
(day, month, year) = md.get("DATE-OBS").split("/")
year = 1900 + int(year)
(hour, minute, second) = md.get("TAIHMS").split(":")
seconds = float(second)
second = int(seconds)
taiObs = dafBase.DateTime(int(year), int(month), int(day), int(hour),
int(minute), second, dafBase.DateTime.TAI)
taiObs = dafBase.DateTime(
taiObs.nsecs() + int((seconds - second) * 1000000000),
dafBase.DateTime.TAI)
taiObs = taiObs.toString(dafBase.DateTime.UTC)[:-1]
strip = "%d%s" % (md.get('STRIPE'), md.get('STRIP'))
conn.execute(
"""INSERT INTO raw VALUES
(NULL, ?, ?, ?, ?, ?, ?, ?)""",
(run, rerun, filter, camcol, field, taiObs, strip))
for row in conn.execute("SELECT last_insert_rowid()"):
id = row[0]
break
wcs = afwImage.makeWcs(md)
poly = skypix.imageToPolygon(wcs,
md.get("NAXIS1"),
md.get("NAXIS2"),
padRad=0.000075) # about 15 arcsec
pix = qsp.intersect(poly)
for skyTileId in pix:
conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)",
(id, skyTileId))
nProcessed += 1
if nProcessed % 100 == 0:
conn.commit()
conn.commit()
print(runDir, \
"... %d processed, %d skipped, %d unrecognized" % \
(nProcessed, nSkipped, nUnrecognized), file=sys.stderr)
mjdObs = md.get("MJD")
taiObs = dafBase.DateTime(mjdObs, dafBase.DateTime.MJD,
dafBase.DateTime.UTC).toString()[:-1]
except:
dateObsCard = md.get("DATE-OBS")
if dateObsCard.find('T') == -1:
dateObsCard += "T00:00:00Z"
taiObs = dafBase.DateTime(dateObsCard).toString()[:-1]
if md.get("CTYPE1").strip() == "CPX" and md.get("CTYPE2").strip() == "CPY": # HSC pixel WCS
pix = []
else:
wcs = afwImage.makeWcs(fixSubaruHeader(md))
try:
poly = skypix.imageToPolygon(wcs, md.get("NAXIS1"), md.get("NAXIS2"),
padRad=math.radians(15.0/3600))
except ZeroDivisionError, e:
print >> sys.stderr, "\nFailure finding polygon for %s: %s" % (fits, e)
poly = None
if poly:
pix = qsp.intersect(poly)
else:
pix = []
cmd = "INSERT INTO raw VALUES (NULL, ?, ?, ?, ?, ?, ?, ?, ?)"
if cursor:
try:
data = (field, visit, filterId, ccd, dateObs, taiObs, expTime, pointing)
cursor.execute(cmd, data)
except Exception, e:
def processRun(runDir, conn, done, qsp):
nProcessed = 0
nSkipped = 0
nUnrecognized = 0
print >>sys.stderr, runDir, "... started"
for fits in glob.iglob(
os.path.join(runDir, "*", "corr", "[1-6]", "fpC*.fit.gz")):
m = re.search(r'(\d+)/corr/([1-6])/fpC-(\d{6})-([ugriz])\2-(\d{4}).fit.gz', fits)
if not m:
print >>sys.stderr, "Warning: Unrecognized file:", fits
nUnrecognized += 1
continue
(rerun, camcol, run, filter, field) = m.groups()
rerun = int(rerun)
camcol = int(camcol)
run = int(run)
field = int(field)
key = "%d_R%d_B%s_C%d_F%d" % (run, rerun, filter, camcol, field)
if done.has_key(key) or rerun < 40:
nSkipped += 1
continue
md = afwImage.readMetadata(fits)
date = md.get("DATE-OBS")
if date.find("-") != -1:
(year, month, day) = md.get("DATE-OBS").split("-")
else:
(day, month, year) = md.get("DATE-OBS").split("/")
year = 1900 + int(year)
(hour, minute, second) = md.get("TAIHMS").split(":")
seconds = float(second)
second = int(seconds)
taiObs = dafBase.DateTime(int(year), int(month), int(day), int(hour),
int(minute), second, dafBase.DateTime.TAI)
taiObs = dafBase.DateTime(taiObs.nsecs() +
long((seconds - second) * 1000000000), dafBase.DateTime.TAI)
taiObs = taiObs.toString()[:-1]
strip = "%d%s" % (md.get('STRIPE'), md.get('STRIP'))
conn.execute("""INSERT INTO raw VALUES
(NULL, ?, ?, ?, ?, ?, ?, ?)""",
(run, rerun, filter, camcol, field, taiObs, strip))
for row in conn.execute("SELECT last_insert_rowid()"):
id = row[0]
break
wcs = afwImage.makeWcs(md)
poly = skypix.imageToPolygon(wcs,
md.get("NAXIS1"), md.get("NAXIS2"),
padRad=0.000075) # about 15 arcsec
pix = qsp.intersect(poly)
for skyTileId in pix:
conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)",
(id, skyTileId))
nProcessed += 1
if nProcessed % 100 == 0:
conn.commit()
conn.commit()
print >>sys.stderr, runDir, \
"... %d processed, %d skipped, %d unrecognized" % \
(nProcessed, nSkipped, nUnrecognized)
