def loadAndMatchData(repo, visitDataIds, matchRadius=afwGeom.Angle(1, afwGeom.arcseconds), verbose=False): """Load data from specific visit. Match with reference. Parameters ---------- repo : string The repository. This is generally the directory on disk that contains the repository and mapper. visitDataIds : list of dict List of `butler` data IDs of Image catalogs to compare to reference. The `calexp` cpixel image is needed for the photometric calibration. matchRadius : afwGeom.Angle(). Radius for matching. verbose : bool, optional Output additional information on the analysis steps. Returns ------- afw.table.GroupView An object of matched catalog. """ # Following # https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb butler = dafPersist.Butler(repo) dataset = 'src' # 2016-02-08 MWV: # I feel like I could be doing something more efficient with # something along the lines of the following: # dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in visitDataIds] ccdKeyName = getCcdKeyName(visitDataIds[0]) schema = butler.get(dataset + "_schema", immediate=True).schema mapper = SchemaMapper(schema) mapper.addMinimalSchema(schema) mapper.addOutputField(Field[float]('base_PsfFlux_snr', "PSF flux SNR")) mapper.addOutputField(Field[float]('base_PsfFlux_mag', "PSF magnitude")) mapper.addOutputField(Field[float]('base_PsfFlux_magerr', "PSF magnitude uncertainty")) newSchema = mapper.getOutputSchema() # Create an object that can match multiple catalogs with the same schema mmatch = MultiMatch(newSchema, dataIdFormat={'visit': int, ccdKeyName: int}, radius=matchRadius, RecordClass=SimpleRecord) # create the new extented source catalog srcVis = SourceCatalog(newSchema) for vId in visitDataIds: try: calexpMetadata = butler.get("calexp_md", vId, immediate=True) except FitsError as fe: print(fe) print("Could not open calibrated image file for ", vId) print("Skipping %s " % repr(vId)) continue except TypeError as te: # DECam images that haven't been properly reformatted # can trigger a TypeError because of a residual FITS header # LTV2 which is a float instead of the expected integer. # This generates an error of the form: # # lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type' # # See, e.g., DM-2957 for details. print(te) print("Calibration image header information malformed.") print("Skipping %s " % repr(vId)) continue calib = afwImage.Calib(calexpMetadata) oldSrc = butler.get('src', vId, immediate=True) print(len(oldSrc), "sources in ccd %s visit %s" % (vId[ccdKeyName], vId["visit"])) # create temporary catalog tmpCat = SourceCatalog(SourceCatalog(newSchema).table) tmpCat.extend(oldSrc, mapper=mapper) tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_flux'] / tmpCat['base_PsfFlux_fluxSigma'] with afwImageUtils.CalibNoThrow(): (tmpCat['base_PsfFlux_mag'][:], tmpCat['base_PsfFlux_magerr'][:]) = \ calib.getMagnitude(tmpCat['base_PsfFlux_flux'], tmpCat['base_PsfFlux_fluxSigma']) srcVis.extend(tmpCat, False) mmatch.add(catalog=tmpCat, dataId=vId) # Complete the match, returning a catalog that includes # all matched sources with object IDs that can be used to group them. matchCat = mmatch.finish() # Create a mapping object that allows the matches to be manipulated # as a mapping of object ID to catalog of sources. allMatches = GroupView.build(matchCat) return allMatches
def _loadAndMatchCatalogs(self, repo, dataIds, matchRadius): """Load data from specific visit. Match with reference. Parameters ---------- repo : string The repository. This is generally the directory on disk that contains the repository and mapper. dataIds : list of dict List of `butler` data IDs of Image catalogs to compare to reference. The `calexp` cpixel image is needed for the photometric calibration. matchRadius : afwGeom.Angle(), optional Radius for matching. Default is 1 arcsecond. Returns ------- afw.table.GroupView An object of matched catalog. """ # Following # https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb butler = dafPersist.Butler(repo) dataset = 'src' # 2016-02-08 MWV: # I feel like I could be doing something more efficient with # something along the lines of the following: # dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds] ccdKeyName = getCcdKeyName(dataIds[0]) schema = butler.get(dataset + "_schema", immediate=True).schema mapper = SchemaMapper(schema) mapper.addMinimalSchema(schema) mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR')) mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude')) mapper.addOutputField(Field[float]('base_PsfFlux_magerr', 'PSF magnitude uncertainty')) newSchema = mapper.getOutputSchema() # Create an object that matches multiple catalogs with same schema mmatch = MultiMatch(newSchema, dataIdFormat={ 'visit': np.int32, ccdKeyName: np.int32 }, radius=matchRadius, RecordClass=SimpleRecord) # create the new extented source catalog srcVis = SourceCatalog(newSchema) for vId in dataIds: try: calexpMetadata = butler.get("calexp_md", vId, immediate=True) except (FitsError, dafPersist.NoResults) as e: print(e) print("Could not open calibrated image file for ", vId) print("Skipping %s " % repr(vId)) continue except TypeError as te: # DECam images that haven't been properly reformatted # can trigger a TypeError because of a residual FITS header # LTV2 which is a float instead of the expected integer. # This generates an error of the form: # # lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type' # # See, e.g., DM-2957 for details. print(te) print("Calibration image header information malformed.") print("Skipping %s " % repr(vId)) continue calib = afwImage.Calib(calexpMetadata) oldSrc = butler.get('src', vId, immediate=True) print( len(oldSrc), "sources in ccd %s visit %s" % (vId[ccdKeyName], vId["visit"])) # create temporary catalog tmpCat = SourceCatalog(SourceCatalog(newSchema).table) tmpCat.extend(oldSrc, mapper=mapper) tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_flux'] \ / tmpCat['base_PsfFlux_fluxSigma'] with afwImageUtils.CalibNoThrow(): _ = calib.getMagnitude(tmpCat['base_PsfFlux_flux'], tmpCat['base_PsfFlux_fluxSigma']) tmpCat['base_PsfFlux_mag'][:] = _[0] tmpCat['base_PsfFlux_magerr'][:] = _[1] srcVis.extend(tmpCat, False) mmatch.add(catalog=tmpCat, dataId=vId) # Complete the match, returning a catalog that includes # all matched sources with object IDs that can be used to group them. matchCat = mmatch.finish() # Create a mapping object that allows the matches to be manipulated # as a mapping of object ID to catalog of sources. allMatches = GroupView.build(matchCat) return allMatches
def _loadAndMatchCatalogs(repo, dataIds, matchRadius, useJointCal=False, skipTEx=False): """Load data from specific visit. Match with reference. Parameters ---------- repo : string or Butler A Butler or a repository URL that can be used to construct one dataIds : list of dict List of `butler` data IDs of Image catalogs to compare to reference. The `calexp` cpixel image is needed for the photometric calibration. matchRadius : afwGeom.Angle(), optional Radius for matching. Default is 1 arcsecond. useJointCal : `bool`, optional Use jointcal/meas_mosaic outputs to calibrate positions and fluxes. skipTEx : `bool`, optional Skip TEx calculations (useful for older catalogs that don't have PsfShape measurements). Returns ------- catalog_list : afw.table.SourceCatalog List of all of the catalogs matched_catalog : afw.table.GroupView An object of matched catalog. """ # Following # https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb if isinstance(repo, dafPersist.Butler): butler = repo else: butler = dafPersist.Butler(repo) dataset = 'src' # 2016-02-08 MWV: # I feel like I could be doing something more efficient with # something along the lines of the following: # dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds] ccdKeyName = getCcdKeyName(dataIds[0]) # Hack to support raft and sensor 0,1 IDs as ints for multimatch if ccdKeyName == 'sensor': ccdKeyName = 'raft_sensor_int' for vId in dataIds: vId[ccdKeyName] = raftSensorToInt(vId) schema = butler.get(dataset + "_schema").schema mapper = SchemaMapper(schema) mapper.addMinimalSchema(schema) mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR')) mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude')) mapper.addOutputField(Field[float]('base_PsfFlux_magErr', 'PSF magnitude uncertainty')) mapper.addOutputField(Field[float]('e1', 'Source Ellipticity 1')) mapper.addOutputField(Field[float]('e2', 'Source Ellipticity 1')) mapper.addOutputField(Field[float]('psf_e1', 'PSF Ellipticity 1')) mapper.addOutputField(Field[float]('psf_e2', 'PSF Ellipticity 1')) newSchema = mapper.getOutputSchema() newSchema.setAliasMap(schema.getAliasMap()) # Create an object that matches multiple catalogs with same schema mmatch = MultiMatch(newSchema, dataIdFormat={ 'visit': np.int32, ccdKeyName: np.int32 }, radius=matchRadius, RecordClass=SimpleRecord) # create the new extented source catalog srcVis = SourceCatalog(newSchema) for vId in dataIds: if useJointCal: try: photoCalib = butler.get("jointcal_photoCalib", vId) except (FitsError, dafPersist.NoResults) as e: print(e) print("Could not open photometric calibration for ", vId) print("Skipping this dataId.") continue try: wcs = butler.get("jointcal_wcs", vId) except (FitsError, dafPersist.NoResults) as e: print(e) print("Could not open updated WCS for ", vId) print("Skipping this dataId.") continue else: try: photoCalib = butler.get("calexp_photoCalib", vId) except (FitsError, dafPersist.NoResults) as e: print(e) print("Could not open calibrated image file for ", vId) print("Skipping this dataId.") continue except TypeError as te: # DECam images that haven't been properly reformatted # can trigger a TypeError because of a residual FITS header # LTV2 which is a float instead of the expected integer. # This generates an error of the form: # # lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type' # # See, e.g., DM-2957 for details. print(te) print("Calibration image header information malformed.") print("Skipping this dataId.") continue # We don't want to put this above the first "if useJointCal block" # because we need to use the first `butler.get` above to quickly # catch data IDs with no usable outputs. try: # HSC supports these flags, which dramatically improve I/O # performance; support for other cameras is DM-6927. oldSrc = butler.get('src', vId, flags=SOURCE_IO_NO_FOOTPRINTS) except (OperationalError, sqlite3.OperationalError): oldSrc = butler.get('src', vId) print(len(oldSrc), "sources in ccd %s visit %s" % (vId[ccdKeyName], vId["visit"])) # create temporary catalog tmpCat = SourceCatalog(SourceCatalog(newSchema).table) tmpCat.extend(oldSrc, mapper=mapper) tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_instFlux'] \ / tmpCat['base_PsfFlux_instFluxErr'] if useJointCal: for record in tmpCat: record.updateCoord(wcs) photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux") if not skipTEx: _, psf_e1, psf_e2 = ellipticity_from_cat( oldSrc, slot_shape='slot_PsfShape') _, star_e1, star_e2 = ellipticity_from_cat(oldSrc, slot_shape='slot_Shape') tmpCat['e1'][:] = star_e1 tmpCat['e2'][:] = star_e2 tmpCat['psf_e1'][:] = psf_e1 tmpCat['psf_e2'][:] = psf_e2 srcVis.extend(tmpCat, False) mmatch.add(catalog=tmpCat, dataId=vId) # Complete the match, returning a catalog that includes # all matched sources with object IDs that can be used to group them. matchCat = mmatch.finish() # Create a mapping object that allows the matches to be manipulated # as a mapping of object ID to catalog of sources. allMatches = GroupView.build(matchCat) return srcVis, allMatches
def _loadAndMatchCatalogs(self, repo, dataIds, matchRadius, useJointCal=False): """Load data from specific visit. Match with reference. Parameters ---------- repo : string or Butler A Butler or a repository URL that can be used to construct one dataIds : list of dict List of `butler` data IDs of Image catalogs to compare to reference. The `calexp` cpixel image is needed for the photometric calibration. matchRadius : afwGeom.Angle(), optional Radius for matching. Default is 1 arcsecond. Returns ------- afw.table.GroupView An object of matched catalog. """ # Following # https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb if isinstance(repo, dafPersist.Butler): butler = repo else: butler = dafPersist.Butler(repo) dataset = 'src' # 2016-02-08 MWV: # I feel like I could be doing something more efficient with # something along the lines of the following: # dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds] ccdKeyName = getCcdKeyName(dataIds[0]) schema = butler.get(dataset + "_schema").schema mapper = SchemaMapper(schema) mapper.addMinimalSchema(schema) mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR')) mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude')) mapper.addOutputField(Field[float]('base_PsfFlux_magErr', 'PSF magnitude uncertainty')) newSchema = mapper.getOutputSchema() newSchema.setAliasMap(schema.getAliasMap()) # Create an object that matches multiple catalogs with same schema mmatch = MultiMatch(newSchema, dataIdFormat={ 'visit': np.int32, ccdKeyName: np.int32 }, radius=matchRadius, RecordClass=SimpleRecord) # create the new extented source catalog srcVis = SourceCatalog(newSchema) for vId in dataIds: if useJointCal: try: photoCalib = butler.get("photoCalib", vId) except (FitsError, dafPersist.NoResults) as e: print(e) print("Could not open photometric calibration for ", vId) print("Skipping %s " % repr(vId)) continue try: md = butler.get("wcs_md", vId) wcs = afwImage.makeWcs(md) except (FitsError, dafPersist.NoResults) as e: print(e) print("Could not open updated WCS for ", vId) print("Skipping %s " % repr(vId)) continue else: try: calexpMetadata = butler.get("calexp_md", vId) except (FitsError, dafPersist.NoResults) as e: print(e) print("Could not open calibrated image file for ", vId) print("Skipping %s " % repr(vId)) continue except TypeError as te: # DECam images that haven't been properly reformatted # can trigger a TypeError because of a residual FITS header # LTV2 which is a float instead of the expected integer. # This generates an error of the form: # # lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type' # # See, e.g., DM-2957 for details. print(te) print("Calibration image header information malformed.") print("Skipping %s " % repr(vId)) continue calib = afwImage.Calib(calexpMetadata) # We don't want to put this above the first "if useJointCal block" # because we need to use the first `butler.get` above to quickly # catch data IDs with no usable outputs. try: # HSC supports these flags, which dramatically improve I/O # performance; support for other cameras is DM-6927. oldSrc = butler.get('src', vId, flags=SOURCE_IO_NO_FOOTPRINTS) except: oldSrc = butler.get('src', vId) print( len(oldSrc), "sources in ccd %s visit %s" % (vId[ccdKeyName], vId["visit"])) # create temporary catalog tmpCat = SourceCatalog(SourceCatalog(newSchema).table) tmpCat.extend(oldSrc, mapper=mapper) tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_flux'] \ / tmpCat['base_PsfFlux_fluxSigma'] if useJointCal: for record in tmpCat: record.updateCoord(wcs) photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux") else: with afwImageUtils.CalibNoThrow(): _ = calib.getMagnitude(tmpCat['base_PsfFlux_flux'], tmpCat['base_PsfFlux_fluxSigma']) tmpCat['base_PsfFlux_mag'][:] = _[0] tmpCat['base_PsfFlux_magErr'][:] = _[1] srcVis.extend(tmpCat, False) mmatch.add(catalog=tmpCat, dataId=vId) # Complete the match, returning a catalog that includes # all matched sources with object IDs that can be used to group them. matchCat = mmatch.finish() # Create a mapping object that allows the matches to be manipulated # as a mapping of object ID to catalog of sources. allMatches = GroupView.build(matchCat) return allMatches
def _loadAndMatchCatalogs(repo, dataIds, matchRadius, doApplyExternalPhotoCalib=False, externalPhotoCalibName=None, doApplyExternalSkyWcs=False, externalSkyWcsName=None, skipTEx=False, skipNonSrd=False): """Load data from specific visits and returned a calibrated catalog matched with a reference. Parameters ---------- repo : `str` or `lsst.daf.persistence.Butler` A Butler or a repository URL that can be used to construct one. dataIds : list of dict List of butler data IDs of Image catalogs to compare to reference. The calexp cpixel image is needed for the photometric calibration. matchRadius : `lsst.geom.Angle`, optional Radius for matching. Default is 1 arcsecond. doApplyExternalPhotoCalib : bool, optional Apply external photoCalib to calibrate fluxes. externalPhotoCalibName : str, optional Type of external `PhotoCalib` to apply. Currently supported are jointcal, fgcm, and fgcm_tract. Must be set if doApplyExternalPhotoCalib is True. doApplyExternalSkyWcs : bool, optional Apply external wcs to calibrate positions. externalSkyWcsName : str, optional Type of external `wcs` to apply. Currently supported is jointcal. Must be set if "doApplyExternalWcs" is True. skipTEx : `bool`, optional Skip TEx calculations (useful for older catalogs that don't have PsfShape measurements). skipNonSrd : `bool`, optional Skip any metrics not defined in the LSST SRD; default False. Returns ------- catalog : `lsst.afw.table.SourceCatalog` A new calibrated SourceCatalog. matches : `lsst.afw.table.GroupView` A GroupView of the matched sources. Raises ------ RuntimeError: Raised if "doApplyExternalPhotoCalib" is True and "externalPhotoCalibName" is None, or if "doApplyExternalSkyWcs" is True and "externalSkyWcsName" is None. """ if doApplyExternalPhotoCalib and externalPhotoCalibName is None: raise RuntimeError( "Must set externalPhotoCalibName if doApplyExternalPhotoCalib is True." ) if doApplyExternalSkyWcs and externalSkyWcsName is None: raise RuntimeError( "Must set externalSkyWcsName if doApplyExternalSkyWcs is True.") # Following # https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb if isinstance(repo, dafPersist.Butler): butler = repo else: butler = dafPersist.Butler(repo) dataset = 'src' # 2016-02-08 MWV: # I feel like I could be doing something more efficient with # something along the lines of the following: # dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds] ccdKeyName = getCcdKeyName(dataIds[0]) # Hack to support raft and sensor 0,1 IDs as ints for multimatch if ccdKeyName == 'sensor': ccdKeyName = 'raft_sensor_int' for vId in dataIds: vId[ccdKeyName] = raftSensorToInt(vId) schema = butler.get(dataset + "_schema").schema mapper = SchemaMapper(schema) mapper.addMinimalSchema(schema) mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR')) mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude')) mapper.addOutputField(Field[float]('base_PsfFlux_magErr', 'PSF magnitude uncertainty')) if not skipNonSrd: # Needed because addOutputField(... 'slot_ModelFlux_mag') will add a field with that literal name aliasMap = schema.getAliasMap() # Possibly not needed since base_GaussianFlux is the default, but this ought to be safe modelName = aliasMap[ 'slot_ModelFlux'] if 'slot_ModelFlux' in aliasMap.keys( ) else 'base_GaussianFlux' mapper.addOutputField(Field[float](f'{modelName}_mag', 'Model magnitude')) mapper.addOutputField(Field[float](f'{modelName}_magErr', 'Model magnitude uncertainty')) mapper.addOutputField(Field[float](f'{modelName}_snr', 'Model flux snr')) mapper.addOutputField(Field[float]('e1', 'Source Ellipticity 1')) mapper.addOutputField(Field[float]('e2', 'Source Ellipticity 1')) mapper.addOutputField(Field[float]('psf_e1', 'PSF Ellipticity 1')) mapper.addOutputField(Field[float]('psf_e2', 'PSF Ellipticity 1')) newSchema = mapper.getOutputSchema() newSchema.setAliasMap(schema.getAliasMap()) # Create an object that matches multiple catalogs with same schema mmatch = MultiMatch(newSchema, dataIdFormat={ 'visit': np.int32, ccdKeyName: np.int32 }, radius=matchRadius, RecordClass=SimpleRecord) # create the new extented source catalog srcVis = SourceCatalog(newSchema) for vId in dataIds: if not butler.datasetExists('src', vId): print(f'Could not find source catalog for {vId}; skipping.') continue photoCalib = _loadPhotoCalib(butler, vId, doApplyExternalPhotoCalib, externalPhotoCalibName) if photoCalib is None: continue if doApplyExternalSkyWcs: wcs = _loadExternalSkyWcs(butler, vId, externalSkyWcsName) if wcs is None: continue # We don't want to put this above the first _loadPhotoCalib call # because we need to use the first `butler.get` in there to quickly # catch dataIDs with no usable outputs. try: # HSC supports these flags, which dramatically improve I/O # performance; support for other cameras is DM-6927. oldSrc = butler.get('src', vId, flags=SOURCE_IO_NO_FOOTPRINTS) except (OperationalError, sqlite3.OperationalError): oldSrc = butler.get('src', vId) print(len(oldSrc), "sources in ccd %s visit %s" % (vId[ccdKeyName], vId["visit"])) # create temporary catalog tmpCat = SourceCatalog(SourceCatalog(newSchema).table) tmpCat.extend(oldSrc, mapper=mapper) tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_instFlux'] \ / tmpCat['base_PsfFlux_instFluxErr'] if doApplyExternalSkyWcs: afwTable.updateSourceCoords(wcs, tmpCat) photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux") if not skipNonSrd: tmpCat['slot_ModelFlux_snr'][:] = ( tmpCat['slot_ModelFlux_instFlux'] / tmpCat['slot_ModelFlux_instFluxErr']) photoCalib.instFluxToMagnitude(tmpCat, "slot_ModelFlux", "slot_ModelFlux") if not skipTEx: _, psf_e1, psf_e2 = ellipticity_from_cat( oldSrc, slot_shape='slot_PsfShape') _, star_e1, star_e2 = ellipticity_from_cat(oldSrc, slot_shape='slot_Shape') tmpCat['e1'][:] = star_e1 tmpCat['e2'][:] = star_e2 tmpCat['psf_e1'][:] = psf_e1 tmpCat['psf_e2'][:] = psf_e2 srcVis.extend(tmpCat, False) mmatch.add(catalog=tmpCat, dataId=vId) # Complete the match, returning a catalog that includes # all matched sources with object IDs that can be used to group them. matchCat = mmatch.finish() # Create a mapping object that allows the matches to be manipulated # as a mapping of object ID to catalog of sources. allMatches = GroupView.build(matchCat) return srcVis, allMatches
def match_catalogs(inputs, photoCalibs, astromCalibs, vIds, matchRadius, apply_external_wcs=False, logger=None): schema = inputs[0].schema mapper = SchemaMapper(schema) mapper.addMinimalSchema(schema) mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR')) mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude')) mapper.addOutputField(Field[float]('base_PsfFlux_magErr', 'PSF magnitude uncertainty')) # Needed because addOutputField(... 'slot_ModelFlux_mag') will add a field with that literal name aliasMap = schema.getAliasMap() # Possibly not needed since base_GaussianFlux is the default, but this ought to be safe modelName = aliasMap['slot_ModelFlux'] if 'slot_ModelFlux' in aliasMap.keys( ) else 'base_GaussianFlux' mapper.addOutputField(Field[float](f'{modelName}_mag', 'Model magnitude')) mapper.addOutputField(Field[float](f'{modelName}_magErr', 'Model magnitude uncertainty')) mapper.addOutputField(Field[float](f'{modelName}_snr', 'Model flux snr')) mapper.addOutputField(Field[float]('e1', 'Source Ellipticity 1')) mapper.addOutputField(Field[float]('e2', 'Source Ellipticity 1')) mapper.addOutputField(Field[float]('psf_e1', 'PSF Ellipticity 1')) mapper.addOutputField(Field[float]('psf_e2', 'PSF Ellipticity 1')) mapper.addOutputField(Field[np.int32]('filt', 'filter code')) newSchema = mapper.getOutputSchema() newSchema.setAliasMap(schema.getAliasMap()) # Create an object that matches multiple catalogs with same schema mmatch = MultiMatch(newSchema, dataIdFormat={ 'visit': np.int32, 'detector': np.int32 }, radius=matchRadius, RecordClass=SimpleRecord) # create the new extended source catalog srcVis = SourceCatalog(newSchema) filter_dict = { 'u': 1, 'g': 2, 'r': 3, 'i': 4, 'z': 5, 'y': 6, 'HSC-U': 1, 'HSC-G': 2, 'HSC-R': 3, 'HSC-I': 4, 'HSC-Z': 5, 'HSC-Y': 6 } # Sort by visit, detector, then filter vislist = [v['visit'] for v in vIds] ccdlist = [v['detector'] for v in vIds] filtlist = [v['band'] for v in vIds] tab_vids = Table([vislist, ccdlist, filtlist], names=['vis', 'ccd', 'filt']) sortinds = np.argsort(tab_vids, order=('vis', 'ccd', 'filt')) for ind in sortinds: oldSrc = inputs[ind] photoCalib = photoCalibs[ind] wcs = astromCalibs[ind] vId = vIds[ind] if logger: logger.debug( f"{len(oldSrc)} sources in ccd {vId['detector']} visit {vId['visit']}" ) # create temporary catalog tmpCat = SourceCatalog(SourceCatalog(newSchema).table) tmpCat.extend(oldSrc, mapper=mapper) filtnum = filter_dict[vId['band']] tmpCat['filt'] = np.repeat(filtnum, len(oldSrc)) tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_instFlux'] \ / tmpCat['base_PsfFlux_instFluxErr'] if apply_external_wcs and wcs is not None: updateSourceCoords(wcs, tmpCat) photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux") tmpCat['slot_ModelFlux_snr'][:] = ( tmpCat['slot_ModelFlux_instFlux'] / tmpCat['slot_ModelFlux_instFluxErr']) photoCalib.instFluxToMagnitude(tmpCat, "slot_ModelFlux", "slot_ModelFlux") _, psf_e1, psf_e2 = ellipticity_from_cat(oldSrc, slot_shape='slot_PsfShape') _, star_e1, star_e2 = ellipticity_from_cat(oldSrc, slot_shape='slot_Shape') tmpCat['e1'][:] = star_e1 tmpCat['e2'][:] = star_e2 tmpCat['psf_e1'][:] = psf_e1 tmpCat['psf_e2'][:] = psf_e2 srcVis.extend(tmpCat, False) mmatch.add(catalog=tmpCat, dataId=vId) # Complete the match, returning a catalog that includes # all matched sources with object IDs that can be used to group them. matchCat = mmatch.finish() # Create a mapping object that allows the matches to be manipulated # as a mapping of object ID to catalog of sources. # I don't think I can persist a group view, so this may need to be called in a subsequent task # allMatches = GroupView.build(matchCat) return srcVis, matchCat