def setUp(self):
# Set up a Coadd with CoaddInputs tables that have blank filter
# columns to be filled in by later test code.
self.coadd = ExposureF(30, 90)
# WCS is arbitrary, since it'll be the same for all images
wcs = makeSkyWcs(crpix=Point2D(0, 0),
crval=SpherePoint(45.0, 45.0, degrees),
cdMatrix=makeCdMatrix(scale=0.17 * degrees))
self.coadd.setWcs(wcs)
schema = ExposureCatalog.Table.makeMinimalSchema()
self.filterKey = schema.addField("filter", type=str, doc="", size=16)
weightKey = schema.addField("weight", type=float, doc="")
# First input image covers the first 2/3, second covers the last 2/3,
# so they overlap in the middle 1/3.
inputs = ExposureCatalog(schema)
self.input1 = inputs.addNew()
self.input1.setId(1)
self.input1.setBBox(Box2I(Point2I(0, 0), Point2I(29, 59)))
self.input1.setWcs(wcs)
self.input1.set(weightKey, 2.0)
self.input2 = inputs.addNew()
self.input2.setId(2)
self.input2.setBBox(Box2I(Point2I(0, 30), Point2I(29, 89)))
self.input2.setWcs(wcs)
self.input2.set(weightKey, 3.0)
# Use the same catalog for visits and CCDs since the algorithm we're
# testing only cares about CCDs.
self.coadd.getInfo().setCoaddInputs(CoaddInputs(inputs, inputs))
# Set up a catalog with centroids and a FilterFraction plugin.
# We have one record in each region (first input only, both inputs,
# second input only)
schema = SourceCatalog.Table.makeMinimalSchema()
centroidKey = Point2DKey.addFields(schema,
"centroid",
doc="position",
unit="pixel")
schema.getAliasMap().set("slot_Centroid", "centroid")
self.plugin = FilterFractionPlugin(
config=FilterFractionPlugin.ConfigClass(),
schema=schema,
name="subaru_FilterFraction",
metadata=PropertyList())
catalog = SourceCatalog(schema)
self.record1 = catalog.addNew()
self.record1.set(centroidKey, Point2D(14.0, 14.0))
self.record12 = catalog.addNew()
self.record12.set(centroidKey, Point2D(14.0, 44.0))
self.record2 = catalog.addNew()
self.record2.set(centroidKey, Point2D(14.0, 74.0))
def testPut(self):
with lsst.utils.tests.temporaryDirectory() as root:
Butler3.makeRepo(root)
butler3 = Butler3(root, run="three")
butler3.registry.registerDatasetType(
DatasetType("cat", ["htm7"], "SourceCatalog",
universe=butler3.registry.dimensions)
)
butlerShim = ShimButler(butler3)
catIn = SourceCatalog(SourceCatalog.Table.makeMinimalSchema())
catIn.addNew().set("id", 42)
butlerShim.put(catIn, "cat", htm7=131072)
catOut = butlerShim.get("cat", htm7=131072)
self.assertEqual(list(catIn["id"]), list(catOut["id"]))
def _makeDummyCatalog(size, skyFlag=False):
"""Create a trivial catalog for testing source counts.
Parameters
----------
size : `int`
The number of entries in the catalog.
skyFlag : `bool`
If set, the schema is guaranteed to have the ``sky_source`` flag, and
one row has it set to `True`. If not set, the ``sky_source`` flag is
not present.
Returns
-------
catalog : `lsst.afw.table.SourceCatalog`
A new catalog with ``size`` rows.
"""
schema = SourceCatalog.Table.makeMinimalSchema()
if skyFlag:
schema.addField("sky_source", type="Flag", doc="Sky objects.")
catalog = SourceCatalog(schema)
for i in range(size):
record = catalog.addNew()
if skyFlag and size > 0:
record["sky_source"] = True
return catalog
def testPut(self):
with TemporaryDirectory(dir=TESTDIR) as root:
Butler3.makeRepo(root)
butler3 = Butler3(root, run="three")
butler3.registry.registerDatasetType(
DatasetType("cat", ["label"], "SourceCatalog",
universe=butler3.registry.dimensions)
)
butlerShim = ShimButler(butler3)
catIn = SourceCatalog(SourceCatalog.Table.makeMinimalSchema())
catIn.addNew().set("id", 42)
butlerShim.put(catIn, "cat", label="four")
catOut = butlerShim.get("cat", label="four")
self.assertEqual(list(catIn["id"]), list(catOut["id"]))
# Without this the temporary directory can not be removed
# if on NFS because these objects have open SQLite registries.
del butler3
del butlerShim
def _makeDummyCatalog(nParents,
*,
skyFlags=False,
deblendFlags=False,
nChildren=0,
nGrandchildren=0):
"""Create a trivial catalog for testing deblending counts.
Parameters
----------
nParents : `int`
The number of entries in the catalog prior to deblending.
skyFlags : `bool`
If set, the schema includes flags associated with sky sources,
and one top-level source (the deblended one, if it exists) and any
descendents are sky sources.
deblendFlags : `bool`
If set, the schema includes flags associated with the deblender.
nChildren : `int`
If positive, one source is deblended into ``nChildren`` children. This
parameter is ignored if ``deblendFlags`` is `False`.
nGrandchildren : `int`
If positive, one source produced by ``nChildren`` is deblended into
``nGrandchildren`` children. This parameter is ignored if ``nChildren``
is 0 or not applicable.
Returns
-------
catalog : `lsst.afw.table.SourceCatalog`
A new catalog with ``nParents + nChildren + nGrandchildren`` rows.
"""
schema = SourceCatalog.Table.makeMinimalSchema()
if skyFlags:
schema.addField("sky_source", type="Flag", doc="Sky source.")
if deblendFlags:
# See https://community.lsst.org/t/4957 for flag definitions.
# Do not use detect_ flags, as they are defined by a postprocessing
# task and some post-deblend catalogs may not have them.
schema.addField('deblend_nChild', type=np.int32, doc='')
schema.addField('deblend_nPeaks', type=np.int32, doc='')
schema.addField('deblend_parentNPeaks', type=np.int32, doc='')
schema.addField('deblend_parentNChild', type=np.int32, doc='')
catalog = SourceCatalog(schema)
if nParents > 0: # normally anti-pattern, but simplifies nested ifs
for i in range(nParents):
record = catalog.addNew()
if deblendFlags:
record["deblend_nPeaks"] = 1
if skyFlags:
record["sky_source"] = True
if deblendFlags and nChildren > 0:
children = _addChildren(catalog, record, nChildren)
if nGrandchildren > 0:
_addChildren(catalog, children[0], nGrandchildren)
return catalog
def makeRerunCatalog(schema, oldCatalog, idList, fields=None):
""" Creates a catalog prepopulated with ids
This function is used to generate a SourceCatalog containing blank records
with Ids specified in the idList parameter
This function is primarily used when rerunning measurements on a footprint.
Specifying ids in a new measurement catalog which correspond to ids in an
old catalog makes comparing results much easier.
Parameters
----------
schema : lsst.afw.table.Schema
Schema used to describe the fields in the resulting SourceCatalog
oldCatalog : lsst.afw.table.SourceCatalog
Catalog containing previous measurements.
idList : iterable
Python iterable whose values should be numbers corresponding to
measurement ids, ids must exist in the oldCatalog
fields : iterable
Python iterable whose entries should be strings corresponding to schema
keys that exist in both the old catalog and input schema. Fields listed
will be copied from the old catalog into the new catalog.
Returns
-------
measCat : lsst.afw.table.SourceCatalog
SourceCatalog prepopulated with entries corresponding to the ids
specified
"""
if fields is None:
fields = []
if not isinstance(fields, Iterable):
raise RuntimeError("fields list must be an iterable with string"
"elements")
for entry in fields:
if entry not in schema:
schema.addField(oldCatalog.schema.find(entry).field)
measCat = SourceCatalog(schema)
for srcId in idList:
oldSrc = oldCatalog.find(srcId)
src = measCat.addNew()
src.setId(srcId)
src.setFootprint(oldSrc.getFootprint())
src.setParent(oldSrc.getParent())
src.setCoord(oldSrc.getCoord())
for entry in fields:
src[entry] = oldSrc[entry]
return measCat
def create_source_catalog_from_text_and_butler(repo_dir, info, dataset='src'):
butler = dafPersistence.Butler(repo_dir)
schema = butler.get(dataset + "_schema", immediate=True).schema
mapper = SchemaMapper(schema)
mapper.addMinimalSchema(schema)
newSchema = mapper.getOutputSchema()
src_cat = SourceCatalog(newSchema)
for row in info:
record = src_cat.addNew()
record.set('coord_ra', Angle(row['RA']*degrees))
record.set('coord_dec', Angle(row['Dec']*degrees))
print(src_cat['coord_ra'], src_cat['coord_dec'])
return(src_cat)
def makeRerunCatalog(schema, oldCatalog, idList, fields=None,
resetParents=True, addParents=False):
"""Create a catalog prepopulated with IDs.
This function is used to generate a `~lsst.afw.table.SourceCatalog`
containing blank records with IDs as specified in the ``idList``
parameter.
This function is primarily used when re-running measurements on a
particular footprint. Specifying IDs in the new measurement catalog which
correspond to IDs in the old catalog makes comparing results much easier.
The ``resetParents`` and ``addParents`` options are needed because
`SingleFrameMeasurementTask.runPlugins` will skip child
objects whose parents are not in the catalog.
Parameters
----------
schema : `lsst.afw.table.Schema`
Schema used to describe the fields in the resulting catalog.
oldCatalog : `lsst.afw.table.SourceCatalog`
Catalog containing previous measurements.
idList : iterable of `int`
Iterable whose values should be numbers corresponding to measurement
IDs which exist in ``oldCatalog``.
fields : iterable of `str`
Iterable whose entries should be strings corresponding to schema keys
that exist in both the old catalog and input schema. Fields listed
will be copied from the old catalog into the new catalog.
resetParents : `bool`
If `True`, child objects whose parents are not in the ``idList``
will have their parents reset to zero.
addParents : `bool`
If `True`, parents of child objects will be added to ``idList`` if
they are not already present.
Returns
-------
measCat : `lsst.afw.table.SourceCatalog`
Catalog prepopulated with entries with the IDs specified.
"""
if not isinstance(schema, Schema):
raise RuntimeError("schema must be an instance of "
"lsst.afw.table.Schema")
if not isinstance(oldCatalog, SourceCatalog):
raise RuntimeError("oldCatalog must be an instance of "
"lsst.afw.table.SourceCatalogiterable")
if fields is None:
fields = []
if not isinstance(fields, Iterable):
raise RuntimeError("fields list must be an iterable with string"
"elements")
for entry in fields:
if entry not in schema:
schema.addField(oldCatalog.schema.find(entry).field)
if addParents:
newIdList = set()
for srcId in idList:
newIdList.add(srcId)
parentId = oldCatalog.find(srcId).getParent()
if parentId:
newIdList.add(parentId)
idList = newIdList
idList = sorted(idList)
measCat = SourceCatalog(schema)
for srcId in idList:
oldSrc = oldCatalog.find(srcId)
src = measCat.addNew()
src.setId(srcId)
src.setFootprint(oldSrc.getFootprint())
parent = oldSrc.getParent()
if resetParents and parent and parent not in idList:
parent = 0
src.setParent(parent)
src.setCoord(oldSrc.getCoord())
for entry in fields:
src[entry] = oldSrc[entry]
return measCat
def _makeDummyCatalog(size):
catalog = SourceCatalog(SourceCatalog.Table.makeMinimalSchema())
for i in range(size):
catalog.addNew()
return catalog
def _makeDummyCatalog(size):
catalog = SourceCatalog(SourceCatalog.Table.makeMinimalSchema())
for i in range(size):
catalog.addNew()
return catalog
def makeRerunCatalog(schema,
oldCatalog,
idList,
fields=None,
resetParents=True,
addParents=False):
""" Creates a catalog prepopulated with ids
This function is used to generate a SourceCatalog containing blank records
with Ids specified in the idList parameter
This function is primarily used when rerunning measurements on a footprint.
Specifying ids in a new measurement catalog which correspond to ids in an
old catalog makes comparing results much easier.
The resetParents and addParents options are needed because
lsst.meas.base.SingleFrameMeasurementTask.runPlugins() will skip child
objects whose parents are not in the catalog.
Parameters
----------
schema : lsst.afw.table.Schema
Schema used to describe the fields in the resulting SourceCatalog
oldCatalog : lsst.afw.table.SourceCatalog
Catalog containing previous measurements.
idList : iterable
Python iterable whose values should be numbers corresponding to
measurement ids, ids must exist in the oldCatalog
fields : iterable
Python iterable whose entries should be strings corresponding to schema
keys that exist in both the old catalog and input schema. Fields listed
will be copied from the old catalog into the new catalog.
resetParents: boolean
Flag to toggle whether child objects whose parents are not in the
idList should have their parents reset to zero.
addParents: boolean
Flag to toggle whether parents of child objects will be added to the
idList (if not already present).
Returns
-------
measCat : lsst.afw.table.SourceCatalog
SourceCatalog prepopulated with entries corresponding to the ids
specified
"""
if not isinstance(schema, Schema):
raise RuntimeError("schema must be an instance of "
"lsst.afw.table.Schema")
if not isinstance(oldCatalog, SourceCatalog):
raise RuntimeError("oldCatalog must be an instance of "
"lsst.afw.table.SourceCatalogiterable")
if fields is None:
fields = []
if not isinstance(fields, Iterable):
raise RuntimeError("fields list must be an iterable with string"
"elements")
for entry in fields:
if entry not in schema:
schema.addField(oldCatalog.schema.find(entry).field)
if addParents:
newIdList = set()
for srcId in idList:
newIdList.add(srcId)
parentId = oldCatalog.find(srcId).getParent()
if parentId:
newIdList.add(parentId)
idList = newIdList
idList = sorted(idList)
measCat = SourceCatalog(schema)
for srcId in idList:
oldSrc = oldCatalog.find(srcId)
src = measCat.addNew()
src.setId(srcId)
src.setFootprint(oldSrc.getFootprint())
parent = oldSrc.getParent()
if resetParents and parent and parent not in idList:
parent = 0
src.setParent(parent)
src.setCoord(oldSrc.getCoord())
for entry in fields:
src[entry] = oldSrc[entry]
return measCat
def makeRerunCatalog(schema,
oldCatalog,
idList,
fields=None,
resetParents=True,
addParents=False):
"""Create a catalog prepopulated with IDs.
This function is used to generate a `~lsst.afw.table.SourceCatalog`
containing blank records with IDs as specified in the ``idList``
parameter.
This function is primarily used when re-running measurements on a
particular footprint. Specifying IDs in the new measurement catalog which
correspond to IDs in the old catalog makes comparing results much easier.
The ``resetParents`` and ``addParents`` options are needed because
`SingleFrameMeasurementTask.runPlugins` will skip child
objects whose parents are not in the catalog.
Parameters
----------
schema : `lsst.afw.table.Schema`
Schema used to describe the fields in the resulting catalog.
oldCatalog : `lsst.afw.table.SourceCatalog`
Catalog containing previous measurements.
idList : iterable of `int`
Iterable whose values should be numbers corresponding to measurement
IDs which exist in ``oldCatalog``.
fields : iterable of `str`
Iterable whose entries should be strings corresponding to schema keys
that exist in both the old catalog and input schema. Fields listed
will be copied from the old catalog into the new catalog.
resetParents : `bool`
If `True`, child objects whose parents are not in the ``idList``
will have their parents reset to zero.
addParents : `bool`
If `True`, parents of child objects will be added to ``idList`` if
they are not already present.
Returns
-------
measCat : `lsst.afw.table.SourceCatalog`
Catalog prepopulated with entries with the IDs specified.
"""
if not isinstance(schema, Schema):
raise RuntimeError("schema must be an instance of "
"lsst.afw.table.Schema")
if not isinstance(oldCatalog, SourceCatalog):
raise RuntimeError("oldCatalog must be an instance of "
"lsst.afw.table.SourceCatalogiterable")
if fields is None:
fields = []
if not isinstance(fields, Iterable):
raise RuntimeError("fields list must be an iterable with string"
"elements")
for entry in fields:
if entry not in schema:
schema.addField(oldCatalog.schema.find(entry).field)
if addParents:
newIdList = set()
for srcId in idList:
newIdList.add(srcId)
parentId = oldCatalog.find(srcId).getParent()
if parentId:
newIdList.add(parentId)
idList = newIdList
idList = sorted(idList)
measCat = SourceCatalog(schema)
for srcId in idList:
oldSrc = oldCatalog.find(srcId)
src = measCat.addNew()
src.setId(srcId)
src.setFootprint(oldSrc.getFootprint())
parent = oldSrc.getParent()
if resetParents and parent and parent not in idList:
parent = 0
src.setParent(parent)
src.setCoord(oldSrc.getCoord())
for entry in fields:
src[entry] = oldSrc[entry]
return measCat
def getFakeSources(butler, dataId, tol=1.0, extraCols=('zeropoint', 'visit', 'ccd'),
includeMissing=False, footprints=False, radecMatch=None):
"""Get list of sources which agree in pixel position with fake ones with tol
this returns a sourceCatalog of all the matched fake objects,
note, there will be duplicates in this list, since I haven't checked deblend.nchild,
and I'm only doing a tolerance match, which could include extra sources
the outputs can include extraCols as long as they are one of:
zeropoint, visit, ccd, thetaNorth, pixelScale
if includeMissing is true, then the pipeline looks at the fake sources
added in the header and includes an entry in the table for sources without
any measurements, specifically the 'id' column will be 0
radecMatch is the fakes table. if it's not None(default), then do an ra/dec
match with the input catalog instead of looking in the header for where the
sources where added
"""
availExtras = {'zeropoint':{'type':float, 'doc':'zeropoint'},
'visit':{'type':int, 'doc':'visit id'},
'ccd':{'type':int, 'doc':'ccd id'},
'thetaNorth':{'type':lsst.afw.geom.Angle, 'doc':'angle to north'},
'pixelScale':{'type':float, 'doc':'pixelscale in arcsec/pixel'}}
if not np.in1d(extraCols, availExtras.keys()).all():
print "extraCols must be in ",availExtras
try:
if not 'filter' in dataId:
sources = butler.get('src', dataId,
flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
cal = butler.get('calexp', dataId, immediate=True)
cal_md = butler.get('calexp_md', dataId, immediate=True)
else:
sources = butler.get('deepCoadd_src', dataId,
flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
cal = butler.get('deepCoadd', dataId, immediate=True)
cal_md = butler.get('deepCoadd_md', dataId, immediate=True)
except (lsst.pex.exceptions.LsstException, RuntimeError) as e:
print "skipping", dataId
return None
if ('pixelScale' in extraCols) or ('thetaNorth' in extraCols):
wcs = cal.getWcs()
availExtras['pixelScale']['value'] = wcs.pixelScale().asArcseconds()
availExtras['thetaNorth']['value'] = lsst.afw.geom.Angle(
np.arctan2(*tuple(wcs.getLinearTransform().invert()
(lsst.afw.geom.Point2D(1.0,0.0)))))
if 'visit' in extraCols:
availExtras['visit']['value'] = dataId['visit']
if 'ccd' in extraCols:
availExtras['ccd']['value'] = dataId['ccd']
if 'zeropoint' in extraCols:
availExtras['zeropoint']['value'] = 2.5*np.log10(cal_md.get('FLUXMAG0'))
if radecMatch is None:
fakeXY, srcIndex = getFakeMatchesHeader(cal_md, sources, tol=tol)
else:
fakeXY, srcIndex = getFakeMatchesRaDec(sources, radecMatch,
lsst.afw.geom.Box2D(cal.getBBox(lsst.afw.image.PARENT)),
cal.getWcs(),
tol=tol)
mapper = SchemaMapper(sources.schema)
mapper.addMinimalSchema(sources.schema)
newSchema = mapper.getOutputSchema()
newSchema.addField('fakeId', type=int, doc='id of fake source matched to position')
newSchema.addField('fakeOffset', type=lsst.afw.geom.Point2D,
doc='offset from input fake position (pixels)')
for extraName in set(extraCols).intersection(availExtras):
newSchema.addField(extraName, type=availExtras[extraName]['type'],
doc=availExtras[extraName]['doc'])
srcList = SourceCatalog(newSchema)
srcList.reserve(sum([len(s) for s in srcIndex.values()]) +
(0 if not includeMissing else srcIndex.values().count([])))
centroidKey = sources.schema.find('centroid.sdss').getKey()
for ident, sindlist in srcIndex.items():
if includeMissing and (len(sindlist)==0):
newRec = srcList.addNew()
newRec.set('fakeId', ident)
newRec.set('id', 0)
for ss in sindlist:
newRec = srcList.addNew()
newRec.assign(sources[ss], mapper)
newRec.set('fakeId', ident)
newRec.set('fakeOffset',
lsst.afw.geom.Point2D(sources[ss].get(centroidKey).getX() -
fakeXY[ident][0],
sources[ss].get(centroidKey).getY() -
fakeXY[ident][1]))
if includeMissing:
srcList = srcList.copy(deep=True)
for extraName in set(extraCols).intersection(availExtras):
tempCol = srcList.get(extraName)
tempCol.fill(availExtras[extraName]['value'])
return srcList
def run(self, ccdExposure):
"""Mask negative pixels"""
ccd = ccdExposure.getDetector()
ccdExposure = self.convertIntToFloat(ccdExposure)
self.updateVariance(ccdExposure, ccd[0]) # Treating as having only a single amplifier
image = ccdExposure.getMaskedImage().getImage()
mask = ccdExposure.getMaskedImage().getMask()
bad = mask.getPlaneBitMask("BAD")
if False:
mask.getArray()[:] = numpy.where(image <= 0, bad, 0) # XXX this causes bad things to happen
#from lsst.afw.image.utils import clipImage
#clipImage(image,0,10)
#exit()
""" transfer wcs system to TAN """
matches = ReferenceMatchVector()
md = ccdExposure.getMetadata()
wcs = ccdExposure.getWcs()
refSchema = SimpleTable.makeMinimalSchema()
Point2DKey.addFields(refSchema, "centroid", "centroid position", "pixel")
refCatalog = SimpleCatalog(refSchema)
schema = SourceTable.makeMinimalSchema()
centroidKey = Point2DKey.addFields(schema, "centroid", "centroid position", "pixel")
imgCatalog = SourceCatalog(schema)
imgCatalog.defineCentroid("centroid")
# for i in numpy.linspace(10, md.get("ZNAXIS1")-10, 20):
# for j in numpy.linspace(10, md.get("ZNAXIS2")-10, 20):
for i in numpy.linspace(10, 4000, 10):
for j in numpy.linspace(10, 4000, 10):
imgcrd = Point2D(i,j)
skycrd = wcs.pixelToSky(afwGeom.Point2D(i, j))
# Create the reference catalog (with coordinates on the sky)
refSrc = refCatalog.addNew()
refSrc.setCoord(skycrd)
# Create the position catalog (with positions on the image)
imgSrc = imgCatalog.addNew()
imgSrc.set(centroidKey, imgcrd)
# matches
matches.push_back(ReferenceMatch(refSrc, imgSrc, float("NaN")))
# make initial wcs
refPix = afwGeom.Point2D(0.5*ccdExposure.getWidth(), 0.5*ccdExposure.getHeight())
refSky = wcs.pixelToSky(refPix)
xPixelScale = yPixelScale = (0.2*afwGeom.arcseconds).asDegrees()
initanWcs = afwImage.makeWcs(refSky, refPix, xPixelScale, 0.0, 0.0, yPixelScale)
# obtain modified wcs with matched catalogs
fitter = FitTanSipWcsTask()
fitRes = fitter.fitWcs(
matches = matches,
initWcs = initanWcs,
refCat = refCatalog,
sourceCat = imgCatalog,
)
ccdExposure.setWcs(fitRes.wcs)
""" Set zero point, ZP error, exptime """
zp = md.get("MAGZPT")
ccdExposure.getCalib().setFluxMag0(10.0**(0.4*zp))
return lsst.pipe.base.Struct(exposure=ccdExposure)
def getFakeSources(
butler,
dataId,
tol=1.0,
extraCols=("zeropoint", "visit", "ccd"),
includeMissing=False,
footprints=False,
radecMatch=None,
multiband=False,
reffMatch=False,
pix=0.168,
minRad=None,
raCol="RA",
decCol="Dec",
):
"""
Get list of sources which agree in pixel position with fake ones with tol.
This returns a sourceCatalog of all the matched fake objects,
note, there will be duplicates in this list, since I haven't
checked deblend.nchild, and I'm only doing a tolerance match,
which could include extra sources
The outputs can include extraCols as long as they are one of:
zeropoint, visit, ccd, thetaNorth, pixelScale
If includeMissing is true, then the pipeline looks at the fake sources
added in the header and includes an entry in the table for sources without
any measurements, specifically the 'id' column will be 0
radecMatch is the fakes table. if it's not None(default), then do an ra/dec
match with the input catalog instead of looking in the header for where the
sources where added
"""
pipeVersion = dafPersist.eupsVersions.EupsVersions().versions["hscPipe"]
if StrictVersion(pipeVersion) >= StrictVersion("3.9.0"):
coaddData = "deepCoadd_calexp"
coaddMeta = "deepCoadd_calexp_md"
else:
coaddData = "deepCoadd"
coaddMeta = "deepCoadd_md"
availExtras = {
"zeropoint": {"type": float, "doc": "zeropoint"},
"visit": {"type": int, "doc": "visit id"},
"ccd": {"type": int, "doc": "ccd id"},
"thetaNorth": {"type": lsst.afw.geom.Angle, "doc": "angle to north"},
"pixelScale": {"type": float, "doc": "pixelscale in arcsec/pixel"},
}
if not np.in1d(extraCols, availExtras.keys()).all():
print "extraCols must be in ", availExtras
try:
if "filter" not in dataId:
sources = butler.get("src", dataId, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
cal = butler.get("calexp", dataId, immediate=True)
cal_md = butler.get("calexp_md", dataId, immediate=True)
else:
meas = butler.get("deepCoadd_meas", dataId, flags=NO_FOOTPRINT, immediate=True)
force = butler.get("deepCoadd_forced_src", dataId, flags=NO_FOOTPRINT, immediate=True)
sources = combineWithForce(meas, force)
cal = butler.get(coaddData, dataId, immediate=True)
cal_md = butler.get(coaddMeta, dataId, immediate=True)
except (lsst.pex.exceptions.LsstException, RuntimeError):
print "skipping", dataId
return None
if ("pixelScale" in extraCols) or ("thetaNorth" in extraCols):
wcs = cal.getWcs()
availExtras["pixelScale"]["value"] = wcs.pixelScale().asArcseconds()
availExtras["thetaNorth"]["value"] = lsst.afw.geom.Angle(
np.arctan2(*tuple(wcs.getLinearTransform().invert()(lsst.afw.geom.Point2D(1.0, 0.0))))
)
if "visit" in extraCols:
availExtras["visit"]["value"] = dataId["visit"]
if "ccd" in extraCols:
availExtras["ccd"]["value"] = dataId["ccd"]
if "zeropoint" in extraCols:
zeropoint = 2.5 * np.log10(cal_md.get("FLUXMAG0"))
availExtras["zeropoint"]["value"] = zeropoint
if radecMatch is None:
fakeXY, srcIndex = getFakeMatchesHeader(cal_md, sources, tol=tol)
else:
if minRad is not None:
print "# The min matching radius is %4.1f pixel" % minRad
bbox = lsst.afw.geom.Box2D(cal.getBBox(lsst.afw.image.PARENT))
fakeXY, srcIndex, srcClose = getFakeMatchesRaDec(
sources,
radecMatch,
bbox,
cal.getWcs(),
tol=tol,
reffMatch=reffMatch,
pix=pix,
minRad=minRad,
raCol=raCol,
decCol=decCol,
)
mapper = SchemaMapper(sources.schema)
mapper.addMinimalSchema(sources.schema)
newSchema = mapper.getOutputSchema()
newSchema.addField("fakeId", type=int, doc="id of fake source matched to position")
newSchema.addField("nMatched", type=int, doc="Number of matched objects")
newSchema.addField("nPrimary", type=int, doc="Number of unique matched objects")
newSchema.addField("nNoChild", type=int, doc="Number of matched objects with nchild==0")
newSchema.addField("rMatched", type=float, doc="Radius used form atching obects, in pixel")
newSchema.addField("fakeOffX", type=float, doc="offset from input fake position in X (pixels)")
newSchema.addField("fakeOffY", type=float, doc="offset from input fake position in Y (pixels)")
newSchema.addField("fakeOffR", type=float, doc="offset from input fake position in radius")
newSchema.addField("fakeClosest", type="Flag", doc="Is this match the closest one?")
for extraName in set(extraCols).intersection(availExtras):
newSchema.addField(extraName, type=availExtras[extraName]["type"], doc=availExtras[extraName]["doc"])
srcList = SourceCatalog(newSchema)
srcList.reserve(
sum([len(s) for s in srcIndex.values()]) + (0 if not includeMissing else srcIndex.values().count([]))
)
centroidKey = sources.schema.find("centroid.sdss").getKey()
isPrimary = sources.schema.find("detect.is-primary").getKey()
nChild = sources.schema.find("force.deblend.nchild").getKey()
for ident, sindlist in srcIndex.items():
rMatched = fakeXY[ident][2]
if minRad is not None:
if rMatched < minRad:
rMatched = minRad
nMatched = len(sindlist)
nPrimary = np.sum([sources[obj].get(isPrimary) for obj in sindlist])
nNoChild = np.sum([(sources[obj].get(nChild) == 0) for obj in sindlist])
if includeMissing and (nMatched == 0):
newRec = srcList.addNew()
newRec.set("fakeId", ident)
newRec.set("id", 0)
newRec.set("nMatched", 0)
newRec.set("rMatched", rMatched)
for ss in sindlist:
newRec = srcList.addNew()
newRec.assign(sources[ss], mapper)
newRec.set("fakeId", ident)
newRec.set("nMatched", nMatched)
newRec.set("nPrimary", nPrimary)
newRec.set("nNoChild", nNoChild)
newRec.set("rMatched", rMatched)
offsetX = sources[ss].get(centroidKey).getX() - fakeXY[ident][0]
newRec.set("fakeOffX", offsetX)
offsetY = sources[ss].get(centroidKey).getY() - fakeXY[ident][1]
newRec.set("fakeOffY", offsetY)
newRec.set("fakeOffR", np.sqrt(offsetX ** 2.0 + offsetY ** 2.0))
if radecMatch:
if ss == srcClose[ident]:
newRec.set("fakeClosest", True)
else:
newRec.set("fakeClosest", False)
if includeMissing:
srcList = srcList.copy(deep=True)
for extraName in set(extraCols).intersection(availExtras):
tempCol = srcList.get(extraName)
tempCol.fill(availExtras[extraName]["value"])
return srcList
def getFakeSources(butler,
dataId,
tol=1.0,
extraCols=('zeropoint', 'visit', 'ccd'),
includeMissing=False,
footprints=False,
radecMatch=None,
multiband=False,
reffMatch=False,
pix=0.168,
minRad=None,
raCol='RA',
decCol='Dec'):
"""
Get list of sources which agree in pixel position with fake ones with tol.
This returns a sourceCatalog of all the matched fake objects,
note, there will be duplicates in this list, since I haven't
checked deblend.nchild, and I'm only doing a tolerance match,
which could include extra sources
The outputs can include extraCols as long as they are one of:
zeropoint, visit, ccd, thetaNorth, pixelScale
If includeMissing is true, then the pipeline looks at the fake sources
added in the header and includes an entry in the table for sources without
any measurements, specifically the 'id' column will be 0
radecMatch is the fakes table. if it's not None(default), then do an ra/dec
match with the input catalog instead of looking in the header for where the
sources where added
"""
coaddData = "deepCoadd_calexp"
coaddMeta = "deepCoadd_calexp_md"
availExtras = {
'zeropoint': {
'type': float,
'doc': 'zeropoint'
},
'visit': {
'type': int,
'doc': 'visit id'
},
'ccd': {
'type': int,
'doc': 'ccd id'
},
'thetaNorth': {
'type': lsst.afw.geom.Angle,
'doc': 'angle to north'
},
'pixelScale': {
'type': float,
'doc': 'pixelscale in arcsec/pixel'
}
}
if not np.in1d(extraCols, list(availExtras.keys())).all():
print("extraCols must be in ", availExtras)
try:
if 'filter' not in dataId:
sources = butler.get('src',
dataId,
flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
cal = butler.get('calexp', dataId, immediate=True)
cal_md = butler.get('calexp_md', dataId, immediate=True)
else:
meas = butler.get('deepCoadd_meas',
dataId,
flags=NO_FOOTPRINT,
immediate=True)
force = butler.get('deepCoadd_forced_src',
dataId,
flags=NO_FOOTPRINT,
immediate=True)
sources = combineWithForce(meas, force)
cal = butler.get(coaddData, dataId, immediate=True)
cal_md = butler.get(coaddMeta, dataId, immediate=True)
except RuntimeError:
print("skipping", dataId)
return None
if ('pixelScale' in extraCols) or ('thetaNorth' in extraCols):
wcs = cal.getWcs()
availExtras['pixelScale']['value'] = wcs.getPixelScale().asArcseconds()
# The 8 lines of code below find the angle to north, first the mid pixel of the calexp is found,
# then the pixel to sky matrix at this point, the coordinate this gives can then be used to find the
# linearized sky to pixel matrix which can then be used to find the angle.
xMid = cal.getWidth() // 2
yMid = cal.getHeight() // 2
midPoint = lsst.afw.geom.Point2D(xMid, yMid)
midCoord = wcs.pixelToSky(midPoint)
northSkyToPixelMatrix = wcs.linearizeSkyToPixel(
midCoord, lsst.afw.geom.degrees)
northSkyToPixelMatrix = northSkyToPixelMatrix.getLinear()
availExtras['thetaNorth']['value'] = (np.arctan2(
*tuple(northSkyToPixelMatrix(lsst.afw.geom.Point2D(1.0, 0.0))))
) * lsst.afw.geom.radians
if 'visit' in extraCols:
availExtras['visit']['value'] = dataId['visit']
if 'ccd' in extraCols:
availExtras['ccd']['value'] = dataId['ccd']
if 'zeropoint' in extraCols:
zeropoint = 2.5 * np.log10(cal_md.getScalar('FLUXMAG0'))
availExtras['zeropoint']['value'] = zeropoint
if radecMatch is None:
fakeXY, srcIndex = getFakeMatchesHeader(cal_md, sources, tol=tol)
else:
if minRad is not None:
print("# The min matching radius is %4.1f pixel" % minRad)
bbox = lsst.afw.geom.Box2D(cal.getBBox(lsst.afw.image.PARENT))
fakeXY, srcIndex, srcClose = getFakeMatchesRaDec(sources,
radecMatch,
bbox,
cal.getWcs(),
tol=tol,
reffMatch=reffMatch,
pix=pix,
minRad=minRad,
raCol=raCol,
decCol=decCol)
mapper = SchemaMapper(sources.schema)
mapper.addMinimalSchema(sources.schema)
newSchema = mapper.getOutputSchema()
newSchema.addField('fakeId',
type=np.int32,
doc='id of fake source matched to position')
newSchema.addField('nMatched',
type=np.int32,
doc='Number of matched objects')
newSchema.addField('nPrimary',
type=np.int32,
doc='Number of unique matched objects')
newSchema.addField('nNoChild',
type=np.int32,
doc='Number of matched objects with nchild==0')
newSchema.addField('rMatched',
type=float,
doc='Radius used form atching obects, in pixel')
newSchema.addField('fakeOffX',
type=float,
doc='offset from input fake position in X (pixels)')
newSchema.addField('fakeOffY',
type=float,
doc='offset from input fake position in Y (pixels)')
newSchema.addField('fakeOffR',
type=float,
doc='offset from input fake position in radius')
newSchema.addField('fakeClosest',
type="Flag",
doc='Is this match the closest one?')
for extraName in set(extraCols).intersection(availExtras):
newSchema.addField(extraName,
type=availExtras[extraName]['type'],
doc=availExtras[extraName]['doc'])
srcList = SourceCatalog(newSchema)
srcList.reserve(
sum([len(s) for s in srcIndex.values()]) +
(0 if not includeMissing else list(srcIndex.values()).count([])))
centroidKey = sources.getCentroidKey()
isPrimary = sources.schema.find('detect_isPrimary').getKey()
nChild = sources.schema.find('force_deblend_nChild').getKey()
for ident, sindlist in srcIndex.items():
rMatched = fakeXY[ident][2]
if minRad is not None:
if rMatched < minRad:
rMatched = minRad
nMatched = len(sindlist)
nPrimary = np.sum(
[sources[int(obj)].get(isPrimary) for obj in sindlist])
nNoChild = np.sum([(sources[int(obj)].get(nChild) == 0)
for obj in sindlist])
if includeMissing and (nMatched == 0):
newRec = srcList.addNew()
newRec.set('fakeId', ident)
newRec.set('id', 0)
newRec.set('nMatched', 0)
newRec.set('rMatched', rMatched)
for ss in sindlist:
newRec = srcList.addNew()
newRec.assign(sources[int(ss)], mapper)
newRec.set('fakeId', ident)
newRec.set('nMatched', nMatched)
newRec.set('nPrimary', nPrimary)
newRec.set('nNoChild', nNoChild)
newRec.set('rMatched', rMatched)
offsetX = (sources[int(ss)].get(centroidKey).getX() -
fakeXY[ident][0])
newRec.set('fakeOffX', offsetX)
offsetY = (sources[int(ss)].get(centroidKey).getY() -
fakeXY[ident][1])
newRec.set('fakeOffY', offsetY)
newRec.set('fakeOffR', np.sqrt(offsetX**2.0 + offsetY**2.0))
if radecMatch:
if int(ss) == int(srcClose[ident]):
newRec.set('fakeClosest', True)
else:
newRec.set('fakeClosest', False)
if includeMissing:
srcList = srcList.copy(deep=True)
for extraName in set(extraCols).intersection(availExtras):
tempCol = srcList.get(extraName)
tempCol.fill(availExtras[extraName]['value'])
return srcList
def getFakeSources(butler,
dataId,
tol=1.0,
extraCols=('zeropoint', 'visit', 'ccd'),
includeMissing=False,
footprints=False,
radecMatch=None):
"""Get list of sources which agree in pixel position with fake ones with tol
this returns a sourceCatalog of all the matched fake objects,
note, there will be duplicates in this list, since I haven't checked deblend.nchild,
and I'm only doing a tolerance match, which could include extra sources
the outputs can include extraCols as long as they are one of:
zeropoint, visit, ccd, thetaNorth, pixelScale
if includeMissing is true, then the pipeline looks at the fake sources
added in the header and includes an entry in the table for sources without
any measurements, specifically the 'id' column will be 0
radecMatch is the fakes table. if it's not None(default), then do an ra/dec
match with the input catalog instead of looking in the header for where the
sources where added
"""
availExtras = {
'zeropoint': {
'type': float,
'doc': 'zeropoint'
},
'visit': {
'type': int,
'doc': 'visit id'
},
'ccd': {
'type': int,
'doc': 'ccd id'
},
'thetaNorth': {
'type': lsst.afw.geom.Angle,
'doc': 'angle to north'
},
'pixelScale': {
'type': float,
'doc': 'pixelscale in arcsec/pixel'
}
}
if not np.in1d(extraCols, availExtras.keys()).all():
print "extraCols must be in ", availExtras
try:
if not 'filter' in dataId:
sources = butler.get('src',
dataId,
flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
cal = butler.get('calexp', dataId, immediate=True)
cal_md = butler.get('calexp_md', dataId, immediate=True)
else:
sources = butler.get('deepCoadd_src',
dataId,
flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
cal = butler.get('deepCoadd', dataId, immediate=True)
cal_md = butler.get('deepCoadd_md', dataId, immediate=True)
except (lsst.pex.exceptions.LsstException, RuntimeError) as e:
print "skipping", dataId
return None
if ('pixelScale' in extraCols) or ('thetaNorth' in extraCols):
wcs = cal.getWcs()
availExtras['pixelScale']['value'] = wcs.pixelScale().asArcseconds()
availExtras['thetaNorth']['value'] = lsst.afw.geom.Angle(
np.arctan2(*tuple(wcs.getLinearTransform().invert()(
lsst.afw.geom.Point2D(1.0, 0.0)))))
if 'visit' in extraCols:
availExtras['visit']['value'] = dataId['visit']
if 'ccd' in extraCols:
availExtras['ccd']['value'] = dataId['ccd']
if 'zeropoint' in extraCols:
availExtras['zeropoint']['value'] = 2.5 * np.log10(
cal_md.get('FLUXMAG0'))
if radecMatch is None:
fakeXY, srcIndex = getFakeMatchesHeader(cal_md, sources, tol=tol)
else:
fakeXY, srcIndex = getFakeMatchesRaDec(
sources,
radecMatch,
lsst.afw.geom.Box2D(cal.getBBox(lsst.afw.image.PARENT)),
cal.getWcs(),
tol=tol)
mapper = SchemaMapper(sources.schema)
mapper.addMinimalSchema(sources.schema)
newSchema = mapper.getOutputSchema()
newSchema.addField('fakeId',
type=int,
doc='id of fake source matched to position')
newSchema.addField('fakeOffset',
type=lsst.afw.geom.Point2D,
doc='offset from input fake position (pixels)')
for extraName in set(extraCols).intersection(availExtras):
newSchema.addField(extraName,
type=availExtras[extraName]['type'],
doc=availExtras[extraName]['doc'])
srcList = SourceCatalog(newSchema)
srcList.reserve(
sum([len(s) for s in srcIndex.values()]) +
(0 if not includeMissing else srcIndex.values().count([])))
centroidKey = sources.schema.find('centroid.sdss').getKey()
for ident, sindlist in srcIndex.items():
if includeMissing and (len(sindlist) == 0):
newRec = srcList.addNew()
newRec.set('fakeId', ident)
newRec.set('id', 0)
for ss in sindlist:
newRec = srcList.addNew()
newRec.assign(sources[ss], mapper)
newRec.set('fakeId', ident)
newRec.set(
'fakeOffset',
lsst.afw.geom.Point2D(
sources[ss].get(centroidKey).getX() - fakeXY[ident][0],
sources[ss].get(centroidKey).getY() - fakeXY[ident][1]))
if includeMissing:
srcList = srcList.copy(deep=True)
for extraName in set(extraCols).intersection(availExtras):
tempCol = srcList.get(extraName)
tempCol.fill(availExtras[extraName]['value'])
return srcList
