def loadAndMatchData(repo, visitDataIds,
matchRadius=afwGeom.Angle(1, afwGeom.arcseconds),
verbose=False):
"""Load data from specific visit. Match with reference.
Parameters
----------
repo : string
The repository. This is generally the directory on disk
that contains the repository and mapper.
visitDataIds : list of dict
List of `butler` data IDs of Image catalogs to compare to reference.
The `calexp` cpixel image is needed for the photometric calibration.
matchRadius : afwGeom.Angle().
Radius for matching.
verbose : bool, optional
Output additional information on the analysis steps.
Returns
-------
afw.table.GroupView
An object of matched catalog.
"""
# Following
# https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
butler = dafPersist.Butler(repo)
dataset = 'src'
# 2016-02-08 MWV:
# I feel like I could be doing something more efficient with
# something along the lines of the following:
# dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in visitDataIds]
ccdKeyName = getCcdKeyName(visitDataIds[0])
schema = butler.get(dataset + "_schema", immediate=True).schema
mapper = SchemaMapper(schema)
mapper.addMinimalSchema(schema)
mapper.addOutputField(Field[float]('base_PsfFlux_snr', "PSF flux SNR"))
mapper.addOutputField(Field[float]('base_PsfFlux_mag', "PSF magnitude"))
mapper.addOutputField(Field[float]('base_PsfFlux_magerr', "PSF magnitude uncertainty"))
newSchema = mapper.getOutputSchema()
# Create an object that can match multiple catalogs with the same schema
mmatch = MultiMatch(newSchema,
dataIdFormat={'visit': int, ccdKeyName: int},
radius=matchRadius,
RecordClass=SimpleRecord)
# create the new extented source catalog
srcVis = SourceCatalog(newSchema)
for vId in visitDataIds:
try:
calexpMetadata = butler.get("calexp_md", vId, immediate=True)
except FitsError as fe:
print(fe)
print("Could not open calibrated image file for ", vId)
print("Skipping %s " % repr(vId))
continue
except TypeError as te:
# DECam images that haven't been properly reformatted
# can trigger a TypeError because of a residual FITS header
# LTV2 which is a float instead of the expected integer.
# This generates an error of the form:
#
# lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type'
#
# See, e.g., DM-2957 for details.
print(te)
print("Calibration image header information malformed.")
print("Skipping %s " % repr(vId))
continue
calib = afwImage.Calib(calexpMetadata)
oldSrc = butler.get('src', vId, immediate=True)
print(len(oldSrc), "sources in ccd %s visit %s" % (vId[ccdKeyName], vId["visit"]))
# create temporary catalog
tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
tmpCat.extend(oldSrc, mapper=mapper)
tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_flux'] / tmpCat['base_PsfFlux_fluxSigma']
with afwImageUtils.CalibNoThrow():
(tmpCat['base_PsfFlux_mag'][:], tmpCat['base_PsfFlux_magerr'][:]) = \
calib.getMagnitude(tmpCat['base_PsfFlux_flux'],
tmpCat['base_PsfFlux_fluxSigma'])
srcVis.extend(tmpCat, False)
mmatch.add(catalog=tmpCat, dataId=vId)
# Complete the match, returning a catalog that includes
# all matched sources with object IDs that can be used to group them.
matchCat = mmatch.finish()
# Create a mapping object that allows the matches to be manipulated
# as a mapping of object ID to catalog of sources.
allMatches = GroupView.build(matchCat)
return allMatches
def _loadAndMatchCatalogs(self, repo, dataIds, matchRadius):
"""Load data from specific visit. Match with reference.
Parameters
----------
repo : string
The repository. This is generally the directory on disk
that contains the repository and mapper.
dataIds : list of dict
List of `butler` data IDs of Image catalogs to compare to
reference. The `calexp` cpixel image is needed for the photometric
calibration.
matchRadius : afwGeom.Angle(), optional
Radius for matching. Default is 1 arcsecond.
Returns
-------
afw.table.GroupView
An object of matched catalog.
"""
# Following
# https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
butler = dafPersist.Butler(repo)
dataset = 'src'
# 2016-02-08 MWV:
# I feel like I could be doing something more efficient with
# something along the lines of the following:
# dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds]
ccdKeyName = getCcdKeyName(dataIds[0])
schema = butler.get(dataset + "_schema", immediate=True).schema
mapper = SchemaMapper(schema)
mapper.addMinimalSchema(schema)
mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
mapper.addOutputField(Field[float]('base_PsfFlux_mag',
'PSF magnitude'))
mapper.addOutputField(Field[float]('base_PsfFlux_magerr',
'PSF magnitude uncertainty'))
newSchema = mapper.getOutputSchema()
# Create an object that matches multiple catalogs with same schema
mmatch = MultiMatch(newSchema,
dataIdFormat={
'visit': np.int32,
ccdKeyName: np.int32
},
radius=matchRadius,
RecordClass=SimpleRecord)
# create the new extented source catalog
srcVis = SourceCatalog(newSchema)
for vId in dataIds:
try:
calexpMetadata = butler.get("calexp_md", vId, immediate=True)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open calibrated image file for ", vId)
print("Skipping %s " % repr(vId))
continue
except TypeError as te:
# DECam images that haven't been properly reformatted
# can trigger a TypeError because of a residual FITS header
# LTV2 which is a float instead of the expected integer.
# This generates an error of the form:
#
# lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type'
#
# See, e.g., DM-2957 for details.
print(te)
print("Calibration image header information malformed.")
print("Skipping %s " % repr(vId))
continue
calib = afwImage.Calib(calexpMetadata)
oldSrc = butler.get('src', vId, immediate=True)
print(
len(oldSrc), "sources in ccd %s visit %s" %
(vId[ccdKeyName], vId["visit"]))
# create temporary catalog
tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
tmpCat.extend(oldSrc, mapper=mapper)
tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_flux'] \
/ tmpCat['base_PsfFlux_fluxSigma']
with afwImageUtils.CalibNoThrow():
_ = calib.getMagnitude(tmpCat['base_PsfFlux_flux'],
tmpCat['base_PsfFlux_fluxSigma'])
tmpCat['base_PsfFlux_mag'][:] = _[0]
tmpCat['base_PsfFlux_magerr'][:] = _[1]
srcVis.extend(tmpCat, False)
mmatch.add(catalog=tmpCat, dataId=vId)
# Complete the match, returning a catalog that includes
# all matched sources with object IDs that can be used to group them.
matchCat = mmatch.finish()
# Create a mapping object that allows the matches to be manipulated
# as a mapping of object ID to catalog of sources.
allMatches = GroupView.build(matchCat)
return allMatches
def _loadAndMatchCatalogs(repo,
dataIds,
matchRadius,
useJointCal=False,
skipTEx=False):
"""Load data from specific visit. Match with reference.
Parameters
----------
repo : string or Butler
A Butler or a repository URL that can be used to construct one
dataIds : list of dict
List of `butler` data IDs of Image catalogs to compare to
reference. The `calexp` cpixel image is needed for the photometric
calibration.
matchRadius : afwGeom.Angle(), optional
Radius for matching. Default is 1 arcsecond.
useJointCal : `bool`, optional
Use jointcal/meas_mosaic outputs to calibrate positions and fluxes.
skipTEx : `bool`, optional
Skip TEx calculations (useful for older catalogs that don't have
PsfShape measurements).
Returns
-------
catalog_list : afw.table.SourceCatalog
List of all of the catalogs
matched_catalog : afw.table.GroupView
An object of matched catalog.
"""
# Following
# https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
if isinstance(repo, dafPersist.Butler):
butler = repo
else:
butler = dafPersist.Butler(repo)
dataset = 'src'
# 2016-02-08 MWV:
# I feel like I could be doing something more efficient with
# something along the lines of the following:
# dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds]
ccdKeyName = getCcdKeyName(dataIds[0])
# Hack to support raft and sensor 0,1 IDs as ints for multimatch
if ccdKeyName == 'sensor':
ccdKeyName = 'raft_sensor_int'
for vId in dataIds:
vId[ccdKeyName] = raftSensorToInt(vId)
schema = butler.get(dataset + "_schema").schema
mapper = SchemaMapper(schema)
mapper.addMinimalSchema(schema)
mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude'))
mapper.addOutputField(Field[float]('base_PsfFlux_magErr',
'PSF magnitude uncertainty'))
mapper.addOutputField(Field[float]('e1', 'Source Ellipticity 1'))
mapper.addOutputField(Field[float]('e2', 'Source Ellipticity 1'))
mapper.addOutputField(Field[float]('psf_e1', 'PSF Ellipticity 1'))
mapper.addOutputField(Field[float]('psf_e2', 'PSF Ellipticity 1'))
newSchema = mapper.getOutputSchema()
newSchema.setAliasMap(schema.getAliasMap())
# Create an object that matches multiple catalogs with same schema
mmatch = MultiMatch(newSchema,
dataIdFormat={
'visit': np.int32,
ccdKeyName: np.int32
},
radius=matchRadius,
RecordClass=SimpleRecord)
# create the new extented source catalog
srcVis = SourceCatalog(newSchema)
for vId in dataIds:
if useJointCal:
try:
photoCalib = butler.get("jointcal_photoCalib", vId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open photometric calibration for ", vId)
print("Skipping this dataId.")
continue
try:
wcs = butler.get("jointcal_wcs", vId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open updated WCS for ", vId)
print("Skipping this dataId.")
continue
else:
try:
photoCalib = butler.get("calexp_photoCalib", vId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open calibrated image file for ", vId)
print("Skipping this dataId.")
continue
except TypeError as te:
# DECam images that haven't been properly reformatted
# can trigger a TypeError because of a residual FITS header
# LTV2 which is a float instead of the expected integer.
# This generates an error of the form:
#
# lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type'
#
# See, e.g., DM-2957 for details.
print(te)
print("Calibration image header information malformed.")
print("Skipping this dataId.")
continue
# We don't want to put this above the first "if useJointCal block"
# because we need to use the first `butler.get` above to quickly
# catch data IDs with no usable outputs.
try:
# HSC supports these flags, which dramatically improve I/O
# performance; support for other cameras is DM-6927.
oldSrc = butler.get('src', vId, flags=SOURCE_IO_NO_FOOTPRINTS)
except (OperationalError, sqlite3.OperationalError):
oldSrc = butler.get('src', vId)
print(len(oldSrc),
"sources in ccd %s visit %s" % (vId[ccdKeyName], vId["visit"]))
# create temporary catalog
tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
tmpCat.extend(oldSrc, mapper=mapper)
tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_instFlux'] \
/ tmpCat['base_PsfFlux_instFluxErr']
if useJointCal:
for record in tmpCat:
record.updateCoord(wcs)
photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux")
if not skipTEx:
_, psf_e1, psf_e2 = ellipticity_from_cat(
oldSrc, slot_shape='slot_PsfShape')
_, star_e1, star_e2 = ellipticity_from_cat(oldSrc,
slot_shape='slot_Shape')
tmpCat['e1'][:] = star_e1
tmpCat['e2'][:] = star_e2
tmpCat['psf_e1'][:] = psf_e1
tmpCat['psf_e2'][:] = psf_e2
srcVis.extend(tmpCat, False)
mmatch.add(catalog=tmpCat, dataId=vId)
# Complete the match, returning a catalog that includes
# all matched sources with object IDs that can be used to group them.
matchCat = mmatch.finish()
# Create a mapping object that allows the matches to be manipulated
# as a mapping of object ID to catalog of sources.
allMatches = GroupView.build(matchCat)
return srcVis, allMatches
def _loadAndMatchCatalogs(self,
repo,
dataIds,
matchRadius,
useJointCal=False):
"""Load data from specific visit. Match with reference.
Parameters
----------
repo : string or Butler
A Butler or a repository URL that can be used to construct one
dataIds : list of dict
List of `butler` data IDs of Image catalogs to compare to
reference. The `calexp` cpixel image is needed for the photometric
calibration.
matchRadius : afwGeom.Angle(), optional
Radius for matching. Default is 1 arcsecond.
Returns
-------
afw.table.GroupView
An object of matched catalog.
"""
# Following
# https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
if isinstance(repo, dafPersist.Butler):
butler = repo
else:
butler = dafPersist.Butler(repo)
dataset = 'src'
# 2016-02-08 MWV:
# I feel like I could be doing something more efficient with
# something along the lines of the following:
# dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds]
ccdKeyName = getCcdKeyName(dataIds[0])
schema = butler.get(dataset + "_schema").schema
mapper = SchemaMapper(schema)
mapper.addMinimalSchema(schema)
mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
mapper.addOutputField(Field[float]('base_PsfFlux_mag',
'PSF magnitude'))
mapper.addOutputField(Field[float]('base_PsfFlux_magErr',
'PSF magnitude uncertainty'))
newSchema = mapper.getOutputSchema()
newSchema.setAliasMap(schema.getAliasMap())
# Create an object that matches multiple catalogs with same schema
mmatch = MultiMatch(newSchema,
dataIdFormat={
'visit': np.int32,
ccdKeyName: np.int32
},
radius=matchRadius,
RecordClass=SimpleRecord)
# create the new extented source catalog
srcVis = SourceCatalog(newSchema)
for vId in dataIds:
if useJointCal:
try:
photoCalib = butler.get("photoCalib", vId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open photometric calibration for ", vId)
print("Skipping %s " % repr(vId))
continue
try:
md = butler.get("wcs_md", vId)
wcs = afwImage.makeWcs(md)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open updated WCS for ", vId)
print("Skipping %s " % repr(vId))
continue
else:
try:
calexpMetadata = butler.get("calexp_md", vId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open calibrated image file for ", vId)
print("Skipping %s " % repr(vId))
continue
except TypeError as te:
# DECam images that haven't been properly reformatted
# can trigger a TypeError because of a residual FITS header
# LTV2 which is a float instead of the expected integer.
# This generates an error of the form:
#
# lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type'
#
# See, e.g., DM-2957 for details.
print(te)
print("Calibration image header information malformed.")
print("Skipping %s " % repr(vId))
continue
calib = afwImage.Calib(calexpMetadata)
# We don't want to put this above the first "if useJointCal block"
# because we need to use the first `butler.get` above to quickly
# catch data IDs with no usable outputs.
try:
# HSC supports these flags, which dramatically improve I/O
# performance; support for other cameras is DM-6927.
oldSrc = butler.get('src', vId, flags=SOURCE_IO_NO_FOOTPRINTS)
except:
oldSrc = butler.get('src', vId)
print(
len(oldSrc), "sources in ccd %s visit %s" %
(vId[ccdKeyName], vId["visit"]))
# create temporary catalog
tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
tmpCat.extend(oldSrc, mapper=mapper)
tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_flux'] \
/ tmpCat['base_PsfFlux_fluxSigma']
if useJointCal:
for record in tmpCat:
record.updateCoord(wcs)
photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux",
"base_PsfFlux")
else:
with afwImageUtils.CalibNoThrow():
_ = calib.getMagnitude(tmpCat['base_PsfFlux_flux'],
tmpCat['base_PsfFlux_fluxSigma'])
tmpCat['base_PsfFlux_mag'][:] = _[0]
tmpCat['base_PsfFlux_magErr'][:] = _[1]
srcVis.extend(tmpCat, False)
mmatch.add(catalog=tmpCat, dataId=vId)
# Complete the match, returning a catalog that includes
# all matched sources with object IDs that can be used to group them.
matchCat = mmatch.finish()
# Create a mapping object that allows the matches to be manipulated
# as a mapping of object ID to catalog of sources.
allMatches = GroupView.build(matchCat)
return allMatches
def _loadAndMatchCatalogs(repo,
dataIds,
matchRadius,
doApplyExternalPhotoCalib=False,
externalPhotoCalibName=None,
doApplyExternalSkyWcs=False,
externalSkyWcsName=None,
skipTEx=False,
skipNonSrd=False):
"""Load data from specific visits and returned a calibrated catalog matched
with a reference.
Parameters
----------
repo : `str` or `lsst.daf.persistence.Butler`
A Butler or a repository URL that can be used to construct one.
dataIds : list of dict
List of butler data IDs of Image catalogs to compare to
reference. The calexp cpixel image is needed for the photometric
calibration.
matchRadius : `lsst.geom.Angle`, optional
Radius for matching. Default is 1 arcsecond.
doApplyExternalPhotoCalib : bool, optional
Apply external photoCalib to calibrate fluxes.
externalPhotoCalibName : str, optional
Type of external `PhotoCalib` to apply. Currently supported are jointcal,
fgcm, and fgcm_tract. Must be set if doApplyExternalPhotoCalib is True.
doApplyExternalSkyWcs : bool, optional
Apply external wcs to calibrate positions.
externalSkyWcsName : str, optional
Type of external `wcs` to apply. Currently supported is jointcal.
Must be set if "doApplyExternalWcs" is True.
skipTEx : `bool`, optional
Skip TEx calculations (useful for older catalogs that don't have
PsfShape measurements).
skipNonSrd : `bool`, optional
Skip any metrics not defined in the LSST SRD; default False.
Returns
-------
catalog : `lsst.afw.table.SourceCatalog`
A new calibrated SourceCatalog.
matches : `lsst.afw.table.GroupView`
A GroupView of the matched sources.
Raises
------
RuntimeError:
Raised if "doApplyExternalPhotoCalib" is True and "externalPhotoCalibName"
is None, or if "doApplyExternalSkyWcs" is True and "externalSkyWcsName" is
None.
"""
if doApplyExternalPhotoCalib and externalPhotoCalibName is None:
raise RuntimeError(
"Must set externalPhotoCalibName if doApplyExternalPhotoCalib is True."
)
if doApplyExternalSkyWcs and externalSkyWcsName is None:
raise RuntimeError(
"Must set externalSkyWcsName if doApplyExternalSkyWcs is True.")
# Following
# https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
if isinstance(repo, dafPersist.Butler):
butler = repo
else:
butler = dafPersist.Butler(repo)
dataset = 'src'
# 2016-02-08 MWV:
# I feel like I could be doing something more efficient with
# something along the lines of the following:
# dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds]
ccdKeyName = getCcdKeyName(dataIds[0])
# Hack to support raft and sensor 0,1 IDs as ints for multimatch
if ccdKeyName == 'sensor':
ccdKeyName = 'raft_sensor_int'
for vId in dataIds:
vId[ccdKeyName] = raftSensorToInt(vId)
schema = butler.get(dataset + "_schema").schema
mapper = SchemaMapper(schema)
mapper.addMinimalSchema(schema)
mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude'))
mapper.addOutputField(Field[float]('base_PsfFlux_magErr',
'PSF magnitude uncertainty'))
if not skipNonSrd:
# Needed because addOutputField(... 'slot_ModelFlux_mag') will add a field with that literal name
aliasMap = schema.getAliasMap()
# Possibly not needed since base_GaussianFlux is the default, but this ought to be safe
modelName = aliasMap[
'slot_ModelFlux'] if 'slot_ModelFlux' in aliasMap.keys(
) else 'base_GaussianFlux'
mapper.addOutputField(Field[float](f'{modelName}_mag',
'Model magnitude'))
mapper.addOutputField(Field[float](f'{modelName}_magErr',
'Model magnitude uncertainty'))
mapper.addOutputField(Field[float](f'{modelName}_snr',
'Model flux snr'))
mapper.addOutputField(Field[float]('e1', 'Source Ellipticity 1'))
mapper.addOutputField(Field[float]('e2', 'Source Ellipticity 1'))
mapper.addOutputField(Field[float]('psf_e1', 'PSF Ellipticity 1'))
mapper.addOutputField(Field[float]('psf_e2', 'PSF Ellipticity 1'))
newSchema = mapper.getOutputSchema()
newSchema.setAliasMap(schema.getAliasMap())
# Create an object that matches multiple catalogs with same schema
mmatch = MultiMatch(newSchema,
dataIdFormat={
'visit': np.int32,
ccdKeyName: np.int32
},
radius=matchRadius,
RecordClass=SimpleRecord)
# create the new extented source catalog
srcVis = SourceCatalog(newSchema)
for vId in dataIds:
if not butler.datasetExists('src', vId):
print(f'Could not find source catalog for {vId}; skipping.')
continue
photoCalib = _loadPhotoCalib(butler, vId, doApplyExternalPhotoCalib,
externalPhotoCalibName)
if photoCalib is None:
continue
if doApplyExternalSkyWcs:
wcs = _loadExternalSkyWcs(butler, vId, externalSkyWcsName)
if wcs is None:
continue
# We don't want to put this above the first _loadPhotoCalib call
# because we need to use the first `butler.get` in there to quickly
# catch dataIDs with no usable outputs.
try:
# HSC supports these flags, which dramatically improve I/O
# performance; support for other cameras is DM-6927.
oldSrc = butler.get('src', vId, flags=SOURCE_IO_NO_FOOTPRINTS)
except (OperationalError, sqlite3.OperationalError):
oldSrc = butler.get('src', vId)
print(len(oldSrc),
"sources in ccd %s visit %s" % (vId[ccdKeyName], vId["visit"]))
# create temporary catalog
tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
tmpCat.extend(oldSrc, mapper=mapper)
tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_instFlux'] \
/ tmpCat['base_PsfFlux_instFluxErr']
if doApplyExternalSkyWcs:
afwTable.updateSourceCoords(wcs, tmpCat)
photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux")
if not skipNonSrd:
tmpCat['slot_ModelFlux_snr'][:] = (
tmpCat['slot_ModelFlux_instFlux'] /
tmpCat['slot_ModelFlux_instFluxErr'])
photoCalib.instFluxToMagnitude(tmpCat, "slot_ModelFlux",
"slot_ModelFlux")
if not skipTEx:
_, psf_e1, psf_e2 = ellipticity_from_cat(
oldSrc, slot_shape='slot_PsfShape')
_, star_e1, star_e2 = ellipticity_from_cat(oldSrc,
slot_shape='slot_Shape')
tmpCat['e1'][:] = star_e1
tmpCat['e2'][:] = star_e2
tmpCat['psf_e1'][:] = psf_e1
tmpCat['psf_e2'][:] = psf_e2
srcVis.extend(tmpCat, False)
mmatch.add(catalog=tmpCat, dataId=vId)
# Complete the match, returning a catalog that includes
# all matched sources with object IDs that can be used to group them.
matchCat = mmatch.finish()
# Create a mapping object that allows the matches to be manipulated
# as a mapping of object ID to catalog of sources.
allMatches = GroupView.build(matchCat)
return srcVis, allMatches
def match_catalogs(inputs,
photoCalibs,
astromCalibs,
vIds,
matchRadius,
apply_external_wcs=False,
logger=None):
schema = inputs[0].schema
mapper = SchemaMapper(schema)
mapper.addMinimalSchema(schema)
mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude'))
mapper.addOutputField(Field[float]('base_PsfFlux_magErr',
'PSF magnitude uncertainty'))
# Needed because addOutputField(... 'slot_ModelFlux_mag') will add a field with that literal name
aliasMap = schema.getAliasMap()
# Possibly not needed since base_GaussianFlux is the default, but this ought to be safe
modelName = aliasMap['slot_ModelFlux'] if 'slot_ModelFlux' in aliasMap.keys(
) else 'base_GaussianFlux'
mapper.addOutputField(Field[float](f'{modelName}_mag', 'Model magnitude'))
mapper.addOutputField(Field[float](f'{modelName}_magErr',
'Model magnitude uncertainty'))
mapper.addOutputField(Field[float](f'{modelName}_snr', 'Model flux snr'))
mapper.addOutputField(Field[float]('e1', 'Source Ellipticity 1'))
mapper.addOutputField(Field[float]('e2', 'Source Ellipticity 1'))
mapper.addOutputField(Field[float]('psf_e1', 'PSF Ellipticity 1'))
mapper.addOutputField(Field[float]('psf_e2', 'PSF Ellipticity 1'))
mapper.addOutputField(Field[np.int32]('filt', 'filter code'))
newSchema = mapper.getOutputSchema()
newSchema.setAliasMap(schema.getAliasMap())
# Create an object that matches multiple catalogs with same schema
mmatch = MultiMatch(newSchema,
dataIdFormat={
'visit': np.int32,
'detector': np.int32
},
radius=matchRadius,
RecordClass=SimpleRecord)
# create the new extended source catalog
srcVis = SourceCatalog(newSchema)
filter_dict = {
'u': 1,
'g': 2,
'r': 3,
'i': 4,
'z': 5,
'y': 6,
'HSC-U': 1,
'HSC-G': 2,
'HSC-R': 3,
'HSC-I': 4,
'HSC-Z': 5,
'HSC-Y': 6
}
# Sort by visit, detector, then filter
vislist = [v['visit'] for v in vIds]
ccdlist = [v['detector'] for v in vIds]
filtlist = [v['band'] for v in vIds]
tab_vids = Table([vislist, ccdlist, filtlist],
names=['vis', 'ccd', 'filt'])
sortinds = np.argsort(tab_vids, order=('vis', 'ccd', 'filt'))
for ind in sortinds:
oldSrc = inputs[ind]
photoCalib = photoCalibs[ind]
wcs = astromCalibs[ind]
vId = vIds[ind]
if logger:
logger.debug(
f"{len(oldSrc)} sources in ccd {vId['detector']} visit {vId['visit']}"
)
# create temporary catalog
tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
tmpCat.extend(oldSrc, mapper=mapper)
filtnum = filter_dict[vId['band']]
tmpCat['filt'] = np.repeat(filtnum, len(oldSrc))
tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_instFlux'] \
/ tmpCat['base_PsfFlux_instFluxErr']
if apply_external_wcs and wcs is not None:
updateSourceCoords(wcs, tmpCat)
photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux")
tmpCat['slot_ModelFlux_snr'][:] = (
tmpCat['slot_ModelFlux_instFlux'] /
tmpCat['slot_ModelFlux_instFluxErr'])
photoCalib.instFluxToMagnitude(tmpCat, "slot_ModelFlux",
"slot_ModelFlux")
_, psf_e1, psf_e2 = ellipticity_from_cat(oldSrc,
slot_shape='slot_PsfShape')
_, star_e1, star_e2 = ellipticity_from_cat(oldSrc,
slot_shape='slot_Shape')
tmpCat['e1'][:] = star_e1
tmpCat['e2'][:] = star_e2
tmpCat['psf_e1'][:] = psf_e1
tmpCat['psf_e2'][:] = psf_e2
srcVis.extend(tmpCat, False)
mmatch.add(catalog=tmpCat, dataId=vId)
# Complete the match, returning a catalog that includes
# all matched sources with object IDs that can be used to group them.
matchCat = mmatch.finish()
# Create a mapping object that allows the matches to be manipulated
# as a mapping of object ID to catalog of sources.
# I don't think I can persist a group view, so this may need to be called in a subsequent task
# allMatches = GroupView.build(matchCat)
return srcVis, matchCat
