def testImageToPixels(self):
"""Tests intersection of an image (WCS and dimensions) with a
quad-sphere pixelization.
"""
#metadata taken from CFHT data v695856-e0/v695856-e0-c000-a00.sci_img.fits
metadata = dafBase.PropertySet()
metadata.set("SIMPLE", "T")
metadata.set("BITPIX", -32)
metadata.set("NAXIS", 2)
metadata.set("NAXIS1", 1024)
metadata.set("NAXIS2", 1153)
metadata.set("RADECSYS", "FK5")
metadata.set("EQUINOX", 2000.0)
metadata.set("CRVAL1", 215.604025685476)
metadata.set("CRVAL2", 53.1595451514076)
metadata.set("CRPIX1", 1109.99981456774)
metadata.set("CRPIX2", 560.018167811613)
metadata.set("CTYPE1", "RA---TAN")
metadata.set("CTYPE2", "DEC--TAN")
metadata.set("CD1_1", 5.10808596133527E-05)
metadata.set("CD1_2", 1.85579539217196E-07)
metadata.set("CD2_2", -5.10281493481982E-05)
metadata.set("CD2_1", -8.27440751733828E-07)
wcs = afwImage.makeWcs(metadata)
qs = skypix.createQuadSpherePixelization()
poly = skypix.imageToPolygon(wcs, 1024, 1153)
pixels = qs.intersect(poly)
self.assertEqual(len(pixels), 1)
self.assertEqual(pixels[0], 182720)
self.assertTrue(qs.getGeometry(182720).contains(poly))
def testImageToPixels(self):
"""Tests intersection of an image (WCS and dimensions) with a
quad-sphere pixelization.
"""
# metadata taken from CFHT data v695856-e0/v695856-e0-c000-a00.sci_img.fits
metadata = dafBase.PropertySet()
metadata.set("SIMPLE", "T")
metadata.set("BITPIX", -32)
metadata.set("NAXIS", 2)
metadata.set("NAXIS1", 1024)
metadata.set("NAXIS2", 1153)
metadata.set("RADECSYS", "FK5")
metadata.set("EQUINOX", 2000.0)
metadata.set("CRVAL1", 215.604025685476)
metadata.set("CRVAL2", 53.1595451514076)
metadata.set("CRPIX1", 1109.99981456774)
metadata.set("CRPIX2", 560.018167811613)
metadata.set("CTYPE1", "RA---TAN")
metadata.set("CTYPE2", "DEC--TAN")
metadata.set("CD1_1", 5.10808596133527E-05)
metadata.set("CD1_2", 1.85579539217196E-07)
metadata.set("CD2_2", -5.10281493481982E-05)
metadata.set("CD2_1", -8.27440751733828E-07)
wcs = afwImage.makeWcs(metadata)
qs = skypix.createQuadSpherePixelization()
poly = skypix.imageToPolygon(wcs, 1024, 1153)
pixels = qs.intersect(poly)
self.assertEqual(len(pixels), 1)
self.assertEqual(pixels[0], 182720)
self.assertTrue(qs.getGeometry(182720).contains(poly))
def process(dirList, inputRegistry, outputRegistry="registry.sqlite3"):
if os.path.exists(outputRegistry):
print >> sys.stderr, "Output registry exists; will not overwrite."
sys.exit(1)
if inputRegistry is not None:
if not os.path.exists(inputRegistry):
print >> sys.stderr, "Input registry does not exist."
sys.exit(1)
shutil.copy(inputRegistry, outputRegistry)
conn = sqlite3.connect(outputRegistry)
done = {}
if inputRegistry is None:
# Create tables in new output registry.
cmd = """CREATE TABLE raw (id INTEGER PRIMARY KEY AUTOINCREMENT,
visit INT, filter TEXT, snap INT,
raft TEXT, sensor TEXT, channel TEXT,
taiObs TEXT, expTime DOUBLE)"""
# cmd += ", unique(visit, snap, raft, sensor, channel))"
conn.execute(cmd)
cmd = "CREATE TABLE raw_skyTile (id INTEGER, skyTile INTEGER)"
# cmd += ", unique(id, skyTile), foreign key(id) references raw(id))"
conn.execute(cmd)
conn.execute("""CREATE TABLE raw_visit (visit INT, filter TEXT,
taiObs TEXT, expTime DOUBLE, UNIQUE(visit))""")
conn.commit()
else:
cmd = """SELECT visit || '_F' || filter || '_E' || snap ||
'_R' || raft || '_S' || sensor || '_C' || channel FROM raw"""
for row in conn.execute(cmd):
done[row[0]] = True
qsp = skypix.createQuadSpherePixelization()
try:
for dir in dirList:
if os.path.exists(os.path.join(dir, "raw")):
for visitDir in glob.glob(os.path.join(
dir,
"raw",
"v*-f*",
)):
processVisit(visitDir, conn, done, qsp)
else:
processVisit(dir, conn, done, qsp)
finally:
print >> sys.stderr, "Cleaning up..."
conn.execute("DELETE FROM raw_visit")
conn.commit()
conn.execute("""INSERT INTO raw_visit
SELECT DISTINCT visit, filter, taiObs, expTime FROM raw
WHERE snap = 0""")
conn.commit()
conn.execute("""CREATE UNIQUE INDEX uq_raw ON raw
(visit, snap, raft, sensor, channel)""")
conn.execute("CREATE INDEX ix_skyTile_id ON raw_skyTile (id)")
conn.execute("CREATE INDEX ix_skyTile_tile ON raw_skyTile (skyTile)")
conn.close()
def process(dirList, inputRegistry, outputRegistry="registry.sqlite3"):
if os.path.exists(outputRegistry):
print >>sys.stderr, "Output registry exists; will not overwrite."
sys.exit(1)
if inputRegistry is not None:
if not os.path.exists(inputRegistry):
print >>sys.stderr, "Input registry does not exist."
sys.exit(1)
shutil.copy(inputRegistry, outputRegistry)
conn = sqlite3.connect(outputRegistry)
done = {}
if inputRegistry is None:
# Create tables in new output registry.
cmd = """CREATE TABLE raw (id INTEGER PRIMARY KEY AUTOINCREMENT,
visit INT, filter TEXT, snap INT,
raft TEXT, sensor TEXT, channel TEXT,
taiObs TEXT, expTime DOUBLE)"""
# cmd += ", unique(visit, snap, raft, sensor, channel))"
conn.execute(cmd)
cmd = "CREATE TABLE raw_skyTile (id INTEGER, skyTile INTEGER)"
# cmd += ", unique(id, skyTile), foreign key(id) references raw(id))"
conn.execute(cmd)
conn.execute("""CREATE TABLE raw_visit (visit INT, filter TEXT,
taiObs TEXT, expTime DOUBLE, UNIQUE(visit))""")
conn.commit()
else:
cmd = """SELECT visit || '_F' || filter || '_E' || snap ||
'_R' || raft || '_S' || sensor || '_C' || channel FROM raw"""
for row in conn.execute(cmd):
done[row[0]] = True
qsp = skypix.createQuadSpherePixelization()
try:
for dir in dirList:
if os.path.exists(os.path.join(dir, "raw")):
for visitDir in glob.glob(os.path.join(dir, "raw", "v*-f*",)):
processVisit(visitDir, conn, done, qsp)
else:
processVisit(dir, conn, done, qsp)
finally:
print >>sys.stderr, "Cleaning up..."
conn.execute("DELETE FROM raw_visit")
conn.commit()
conn.execute("""INSERT INTO raw_visit
SELECT DISTINCT visit, filter, taiObs, expTime FROM raw
WHERE snap = 0""")
conn.commit()
conn.execute("""CREATE UNIQUE INDEX uq_raw ON raw
(visit, snap, raft, sensor, channel)""")
conn.execute("CREATE INDEX ix_skyTile_id ON raw_skyTile (id)")
conn.execute("CREATE INDEX ix_skyTile_tile ON raw_skyTile (skyTile)")
conn.close()
def process(dirList, inputRegistry=None, outputRegistry="registry.sqlite3"):
print("process(dirList=%s)" % (dirList,))
if os.path.exists(outputRegistry):
sys.stderr.write("Output registry %r exists; will not overwrite.\n" % (outputRegistry,))
sys.exit(1)
if inputRegistry is not None:
if not os.path.exists(inputRegistry):
sys.stderr.write("Input registry %r does not exist.\n" % (inputRegistry,))
sys.exit(1)
shutil.copy(inputRegistry, outputRegistry)
conn = sqlite3.connect(outputRegistry)
done = {}
if inputRegistry is None:
# Create tables in new output registry.
cmd = """CREATE TABLE raw (id INTEGER PRIMARY KEY AUTOINCREMENT,
visit INT, filter TEXT, taiObs TEXT, expTime DOUBLE)"""
conn.execute(cmd)
cmd = "CREATE TABLE raw_skyTile (id INTEGER, skyTile INTEGER)"
conn.execute(cmd)
conn.execute("""CREATE TABLE raw_visit (visit INT, filter TEXT,
taiObs TEXT, expTime DOUBLE, UNIQUE(visit))""")
conn.commit()
else:
cmd = """SELECT visit || '_f' || filter || FROM raw"""
for row in conn.execute(cmd):
done[row[0]] = True
qsp = skypix.createQuadSpherePixelization()
try:
for dirPath in dirList:
rawDir = os.path.join(dirPath, "raw")
if not os.path.exists(rawDir):
sys.stderr.write("Could not find raw data dir %r\n" % (rawDir,))
processRawDir(rawDir, conn, done, qsp)
finally:
print("Cleaning up...")
conn.execute("DELETE FROM raw_visit")
conn.commit()
conn.execute("""INSERT INTO raw_visit
SELECT DISTINCT visit, filter, taiObs, expTime FROM raw""")
conn.commit()
conn.execute("""CREATE UNIQUE INDEX uq_raw ON raw
(visit)""")
conn.execute("CREATE INDEX ix_skyTile_id ON raw_skyTile (id)")
conn.execute("CREATE INDEX ix_skyTile_tile ON raw_skyTile (skyTile)")
conn.close()
print("wrote registry file %r" % (outputRegistry,))
def process(dirList, inputRegistry, outputRegistry="registry.sqlite3"):
if os.path.exists(outputRegistry):
print("Output registry exists; will not overwrite.", file=sys.stderr)
sys.exit(1)
if inputRegistry is not None:
if not os.path.exists(inputRegistry):
print("Input registry does not exist.", file=sys.stderr)
sys.exit(1)
shutil.copy(inputRegistry, outputRegistry)
conn = sqlite3.connect(outputRegistry)
done = {}
if inputRegistry is None:
# Create tables in new output registry.
cmd = """CREATE TABLE raw (id INTEGER PRIMARY KEY AUTOINCREMENT,
run INT, rerun INT, filter TEXT, camcol INT, field INT,
taiObs TEXT, strip TEXT)"""
# cmd += ", unique(run, filter, camcol, field))"
conn.execute(cmd)
cmd = "CREATE TABLE raw_skyTile (id INTEGER, skyTile INTEGER)"
# cmd += ", unique(id, skyTile), foreign key(id) references raw(id))"
conn.execute(cmd)
else:
cmd = """SELECT run || '_R' || rerun || '_B' || filter ||
'_C' || camcol || '_F' || field FROM raw"""
for row in conn.execute(cmd):
done[row[0]] = True
qsp = skypix.createQuadSpherePixelization()
try:
for dir in dirList:
if dir.endswith("runs"):
for runDir in glob.iglob(os.path.join(dir, "*")):
processRun(runDir, conn, done, qsp)
else:
processRun(dir, conn, done, qsp)
finally:
print("Cleaning up...", file=sys.stderr)
conn.execute("""CREATE UNIQUE INDEX uq_raw ON raw
(run, filter, camcol, field)""")
conn.execute("CREATE INDEX ix_skyTile_id ON raw_skyTile (id)")
conn.execute("CREATE INDEX ix_skyTile_tile ON raw_skyTile (skyTile)")
conn.commit()
conn.close()
def process(dirList, inputRegistry, outputRegistry="registry.sqlite3"):
if os.path.exists(outputRegistry):
print >>sys.stderr, "Output registry exists; will not overwrite."
sys.exit(1)
if inputRegistry is not None:
if not os.path.exists(inputRegistry):
print >>sys.stderr, "Input registry does not exist."
sys.exit(1)
shutil.copy(inputRegistry, outputRegistry)
conn = sqlite3.connect(outputRegistry)
done = {}
if inputRegistry is None:
# Create tables in new output registry.
cmd = """CREATE TABLE raw (id INTEGER PRIMARY KEY AUTOINCREMENT,
run INT, rerun INT, filter TEXT, camcol INT, field INT,
taiObs TEXT, strip TEXT)"""
# cmd += ", unique(run, filter, camcol, field))"
conn.execute(cmd)
cmd = "CREATE TABLE raw_skyTile (id INTEGER, skyTile INTEGER)"
# cmd += ", unique(id, skyTile), foreign key(id) references raw(id))"
conn.execute(cmd)
else:
cmd = """SELECT run || '_R' || rerun || '_B' || filter ||
'_C' || camcol || '_F' || field FROM raw"""
for row in conn.execute(cmd):
done[row[0]] = True
qsp = skypix.createQuadSpherePixelization()
try:
for dir in dirList:
if dir.endswith("runs"):
for runDir in glob.iglob(os.path.join(dir, "*")):
processRun(runDir, conn, done, qsp)
else:
processRun(dir, conn, done, qsp)
finally:
print >>sys.stderr, "Cleaning up..."
conn.execute("""CREATE UNIQUE INDEX uq_raw ON raw
(run, filter, camcol, field)""")
conn.execute("CREATE INDEX ix_skyTile_id ON raw_skyTile (id)")
conn.execute("CREATE INDEX ix_skyTile_tile ON raw_skyTile (skyTile)")
conn.commit()
conn.close()
if conn:
conn.execute(cmd)
conn.commit()
else:
print >> sys.stderr, cmd
print >> sys.stderr, "commit"
# Set up regex for parsing directory structure
reField = r'([\w .+-]+)'
reDate = r'(\d{4}-\d{2}-\d{2})'
rePointing = r'(\d+)'
reFilter = r'([\w\-\+]+)'
regex = re.compile('/'.join(
[reField, reDate, rePointing, reFilter, reFilename]))
qsp = skypix.createQuadSpherePixelization(skyPolicy)
if conn:
cursor = conn.cursor()
else:
cursor = None
nfile = len(files)
for fileNo, fits in enumerate(files):
m = re.search(regex, fits)
if not m:
print "Warning: skipping unrecognized filename:", fits
continue
field, dateObs, pointing, filterId, visit, ccd = m.groups()
visit = int(visit)
ccd = int(ccd)
def cluster(self, skyTileId, butler):
"""Cluster sources falling inside the given sky-tile with the OPTICS
algorithm.
@param skyTileId: Integer sky-tile ID
@param butler: Butler responsible for retrieving calibrated
exposure metadata and associated sources
@return A lsst.pipe.base.Struct with the following fields:
- sources: Sources inside the sky-tile with valid
positions and no "bad" flags set.
- badSources: Sources inside the sky-tile with valid
positions and at least one "bad" flag set.
- invalidSources: Sources with invalid positions/centroids.
- clusters: A list of lsst.afw.table.SourceCatalog objects,
one per cluster generated.
- exposures: An lsst.ap.match.ExposureInfoMap,
mapping calibrated exposure IDs to
lsst.ap.match.ExposureInfo objects.
- sourceHistogram: A 2D histogram of source positions.
- badSourceHistogram: A 2D histogram of bad source positions.
Note that any of these return values can legitimately be None
due to lack of inputs, problems reading them in, or the task
configuration.
"""
# sky-tile setup
qsp = skypix.createQuadSpherePixelization(butler.mapper.skypolicy)
root, x, y = qsp.coords(skyTileId);
skyTile = apUtils.PT1SkyTile(qsp.resolution, root, x, y, skyTileId)
del root, x, y
spControl = self.config.sourceProcessing.makeControl()
sourceTable = None
results = pipeBase.Struct(
sources = None,
badSources = None,
invalidSources = None,
clusters = None,
exposures = apMatch.ExposureInfoMap(),
sourceHistogram = None,
badSourceHistogram = None
)
for dataRef in butler.subset(
self.config.inputSourceDataset, self.config.inputLevel, skyTile=skyTileId):
if not dataRef.datasetExists(self.config.inputSourceDataset):
continue
try:
expMd = dataRef.get(self.config.inputCalexpMetadataDataset, immediate=True)
expSources = dataRef.get(self.config.inputSourceDataset, immediate=True)
expConfig = dataRef.get("processCcd_config", immediate=True)
if (self.config.measPrefix != expConfig.measurement.prefix or
self.config.measSlots != expConfig.measurement.slots):
self.log.warn(str.format(
"skipping {} : processCcd measurement prefix and slot configuration "
"do not match sourceAssoc configuration", str(dataRef.dataId)))
continue
except Exception:
self.log.warn(str.format(
"skipping {} : failed to unpersist {}, {}, or processCcd_config dataset: {}",
str(dataRef.dataId), self.config.inputCalexpMetadataDataset,
self.config.inputSourceDataset, traceback.format_exc()))
continue
if sourceTable == None:
# create output source table
sourceTable, schemaMapper = apCluster.makeOutputSourceTable(
expSources.getTable(), spControl)
# create output source catalogs
results.sources = afwTable.SourceCatalog(sourceTable)
results.badSources = afwTable.SourceCatalog(sourceTable)
results.invalidSources = afwTable.SourceCatalog(sourceTable)
# process sources: segregate into "good", "bad", and "invalid"
# sets, discard sources outside sky-tile, and denormalize the
# source schema.
try:
expInfo = apMatch.ExposureInfo(expMd)
except:
self.log.warn(str.format(
"skipping {} : failed to convert {} dataset to ExposureInfo",
str(dataRef.dataId), self.config.inputCalexpMetadataDataset))
continue
results.exposures.insert(expInfo)
apCluster.processSources(
expSources,
expInfo,
skyTile,
spControl,
schemaMapper,
results.sources,
results.badSources,
results.invalidSources)
# in hopes of freeing memory occupied by input sources
del expSources, expMd
if (sourceTable == None):
return results # nothing to do
# create clusters
if self.config.doCluster and len(results.sources) > 0:
results.clusters = apCluster.cluster(
results.sources, self.config.clustering.makeControl())
# create good/bad source histograms
if self.config.doMakeSourceHistogram and len(results.sources) > 0:
results.sourceHistogram, wcs = apUtils.createImageCoveringSkyTile(
qsp, skyTileId, self.config.sourceHistogramResolution)
apUtils.makeSourceHistogram(
results.sourceHistogram.getImage(), results.sources, wcs, False)
if self.config.doMakeBadSourceHistogram and len(results.badSources) > 0:
results.badSourceHistogram, wcs = apUtils.createImageCoveringSkyTile(
qsp, skyTileId, self.config.sourceHistogramResolution)
apUtils.makeSourceHistogram(
results.badSourceHistogram.getImage(), results.badSources, wcs, False)
return results
cmd += ", unique(visit))"
if conn:
conn.execute(cmd)
conn.commit()
else:
print >> sys.stderr, cmd
print >> sys.stderr, "commit"
# Set up regex for parsing directory structure
reField = r'([\w .+-]+)'
reDate = r'(\d{4}-\d{2}-\d{2})'
rePointing = r'(\d+)'
reFilter = r'([\w\-\+]+)'
regex = re.compile('/'.join([reField, reDate, rePointing, reFilter, reFilename]))
qsp = skypix.createQuadSpherePixelization(skyPolicy)
if conn:
cursor = conn.cursor()
else:
cursor = None
nfile = len(files)
for fileNo, fits in enumerate(files):
m = re.search(regex, fits)
if not m:
print "Warning: skipping unrecognized filename:", fits
continue
field, dateObs, pointing, filterId, visit, ccd = m.groups()
visit = int(visit)
ccd = int(ccd)
def cluster(self, skyTileId, butler):
"""Cluster sources falling inside the given sky-tile with the OPTICS
algorithm.
@param skyTileId: Integer sky-tile ID
@param butler: Butler responsible for retrieving calibrated
exposure metadata and associated sources
@return A lsst.pipe.base.Struct with the following fields:
- sources: Sources inside the sky-tile with valid
positions and no "bad" flags set.
- badSources: Sources inside the sky-tile with valid
positions and at least one "bad" flag set.
- invalidSources: Sources with invalid positions/centroids.
- clusters: A list of lsst.afw.table.SourceCatalog objects,
one per cluster generated.
- exposures: An lsst.ap.match.ExposureInfoMap,
mapping calibrated exposure IDs to
lsst.ap.match.ExposureInfo objects.
- sourceHistogram: A 2D histogram of source positions.
- badSourceHistogram: A 2D histogram of bad source positions.
Note that any of these return values can legitimately be None
due to lack of inputs, problems reading them in, or the task
configuration.
"""
# sky-tile setup
qsp = skypix.createQuadSpherePixelization(butler.mapper.skypolicy)
root, x, y = qsp.coords(skyTileId)
skyTile = apUtils.PT1SkyTile(qsp.resolution, root, x, y, skyTileId)
del root, x, y
spControl = self.config.sourceProcessing.makeControl()
sourceTable = None
results = pipeBase.Struct(sources=None,
badSources=None,
invalidSources=None,
clusters=None,
exposures=apMatch.ExposureInfoMap(),
sourceHistogram=None,
badSourceHistogram=None)
for dataRef in butler.subset(self.config.inputSourceDataset,
self.config.inputLevel,
skyTile=skyTileId):
if not dataRef.datasetExists(self.config.inputSourceDataset):
continue
try:
expMd = dataRef.get(self.config.inputCalexpMetadataDataset,
immediate=True)
expSources = dataRef.get(self.config.inputSourceDataset,
immediate=True)
expConfig = dataRef.get("processCcd_config", immediate=True)
if (self.config.measPrefix != expConfig.measurement.prefix or
self.config.measSlots != expConfig.measurement.slots):
self.log.warn(
str.format(
"skipping {} : processCcd measurement prefix and slot configuration "
"do not match sourceAssoc configuration",
str(dataRef.dataId)))
continue
except Exception:
self.log.warn(
str.format(
"skipping {} : failed to unpersist {}, {}, or processCcd_config dataset: {}",
str(dataRef.dataId),
self.config.inputCalexpMetadataDataset,
self.config.inputSourceDataset,
traceback.format_exc()))
continue
if sourceTable == None:
# create output source table
sourceTable, schemaMapper = apCluster.makeOutputSourceTable(
expSources.getTable(), spControl)
# create output source catalogs
results.sources = afwTable.SourceCatalog(sourceTable)
results.badSources = afwTable.SourceCatalog(sourceTable)
results.invalidSources = afwTable.SourceCatalog(sourceTable)
# process sources: segregate into "good", "bad", and "invalid"
# sets, discard sources outside sky-tile, and denormalize the
# source schema.
try:
expInfo = apMatch.ExposureInfo(expMd)
except:
self.log.warn(
str.format(
"skipping {} : failed to convert {} dataset to ExposureInfo",
str(dataRef.dataId),
self.config.inputCalexpMetadataDataset))
continue
results.exposures.insert(expInfo)
apCluster.processSources(expSources, expInfo, skyTile, spControl,
schemaMapper, results.sources,
results.badSources,
results.invalidSources)
# in hopes of freeing memory occupied by input sources
del expSources, expMd
if (sourceTable == None):
return results # nothing to do
# create clusters
if self.config.doCluster and len(results.sources) > 0:
results.clusters = apCluster.cluster(
results.sources, self.config.clustering.makeControl())
# create good/bad source histograms
if self.config.doMakeSourceHistogram and len(results.sources) > 0:
results.sourceHistogram, wcs = apUtils.createImageCoveringSkyTile(
qsp, skyTileId, self.config.sourceHistogramResolution)
apUtils.makeSourceHistogram(results.sourceHistogram.getImage(),
results.sources, wcs, False)
if self.config.doMakeBadSourceHistogram and len(
results.badSources) > 0:
results.badSourceHistogram, wcs = apUtils.createImageCoveringSkyTile(
qsp, skyTileId, self.config.sourceHistogramResolution)
apUtils.makeSourceHistogram(results.badSourceHistogram.getImage(),
results.badSources, wcs, False)
return results
