def getFakeSources(butler, dataId, tol=1.0, extraCols=('zeropoint', 'visit', 'ccd'), includeMissing=False, footprints=False, radecMatch=None, multiband=False, reffMatch=False, pix=0.168, minRad=None, raCol='RA', decCol='Dec'): """ Get list of sources which agree in pixel position with fake ones with tol. This returns a sourceCatalog of all the matched fake objects, note, there will be duplicates in this list, since I haven't checked deblend.nchild, and I'm only doing a tolerance match, which could include extra sources The outputs can include extraCols as long as they are one of: zeropoint, visit, ccd, thetaNorth, pixelScale If includeMissing is true, then the pipeline looks at the fake sources added in the header and includes an entry in the table for sources without any measurements, specifically the 'id' column will be 0 radecMatch is the fakes table. if it's not None(default), then do an ra/dec match with the input catalog instead of looking in the header for where the sources where added """ coaddData = "deepCoadd_calexp" coaddMeta = "deepCoadd_calexp_md" availExtras = { 'zeropoint': { 'type': float, 'doc': 'zeropoint' }, 'visit': { 'type': int, 'doc': 'visit id' }, 'ccd': { 'type': int, 'doc': 'ccd id' }, 'thetaNorth': { 'type': lsst.afw.geom.Angle, 'doc': 'angle to north' }, 'pixelScale': { 'type': float, 'doc': 'pixelscale in arcsec/pixel' } } if not np.in1d(extraCols, list(availExtras.keys())).all(): print("extraCols must be in ", availExtras) try: if 'filter' not in dataId: sources = butler.get('src', dataId, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True) cal = butler.get('calexp', dataId, immediate=True) cal_md = butler.get('calexp_md', dataId, immediate=True) else: meas = butler.get('deepCoadd_meas', dataId, flags=NO_FOOTPRINT, immediate=True) force = butler.get('deepCoadd_forced_src', dataId, flags=NO_FOOTPRINT, immediate=True) sources = combineWithForce(meas, force) cal = butler.get(coaddData, dataId, immediate=True) cal_md = butler.get(coaddMeta, dataId, immediate=True) except RuntimeError: print("skipping", dataId) return None if ('pixelScale' in extraCols) or ('thetaNorth' in extraCols): wcs = cal.getWcs() availExtras['pixelScale']['value'] = wcs.getPixelScale().asArcseconds() # The 8 lines of code below find the angle to north, first the mid pixel of the calexp is found, # then the pixel to sky matrix at this point, the coordinate this gives can then be used to find the # linearized sky to pixel matrix which can then be used to find the angle. xMid = cal.getWidth() // 2 yMid = cal.getHeight() // 2 midPoint = lsst.afw.geom.Point2D(xMid, yMid) midCoord = wcs.pixelToSky(midPoint) northSkyToPixelMatrix = wcs.linearizeSkyToPixel( midCoord, lsst.afw.geom.degrees) northSkyToPixelMatrix = northSkyToPixelMatrix.getLinear() availExtras['thetaNorth']['value'] = (np.arctan2( *tuple(northSkyToPixelMatrix(lsst.afw.geom.Point2D(1.0, 0.0)))) ) * lsst.afw.geom.radians if 'visit' in extraCols: availExtras['visit']['value'] = dataId['visit'] if 'ccd' in extraCols: availExtras['ccd']['value'] = dataId['ccd'] if 'zeropoint' in extraCols: zeropoint = 2.5 * np.log10(cal_md.getScalar('FLUXMAG0')) availExtras['zeropoint']['value'] = zeropoint if radecMatch is None: fakeXY, srcIndex = getFakeMatchesHeader(cal_md, sources, tol=tol) else: if minRad is not None: print("# The min matching radius is %4.1f pixel" % minRad) bbox = lsst.afw.geom.Box2D(cal.getBBox(lsst.afw.image.PARENT)) fakeXY, srcIndex, srcClose = getFakeMatchesRaDec(sources, radecMatch, bbox, cal.getWcs(), tol=tol, reffMatch=reffMatch, pix=pix, minRad=minRad, raCol=raCol, decCol=decCol) mapper = SchemaMapper(sources.schema) mapper.addMinimalSchema(sources.schema) newSchema = mapper.getOutputSchema() newSchema.addField('fakeId', type=np.int32, doc='id of fake source matched to position') newSchema.addField('nMatched', type=np.int32, doc='Number of matched objects') newSchema.addField('nPrimary', type=np.int32, doc='Number of unique matched objects') newSchema.addField('nNoChild', type=np.int32, doc='Number of matched objects with nchild==0') newSchema.addField('rMatched', type=float, doc='Radius used form atching obects, in pixel') newSchema.addField('fakeOffX', type=float, doc='offset from input fake position in X (pixels)') newSchema.addField('fakeOffY', type=float, doc='offset from input fake position in Y (pixels)') newSchema.addField('fakeOffR', type=float, doc='offset from input fake position in radius') newSchema.addField('fakeClosest', type="Flag", doc='Is this match the closest one?') for extraName in set(extraCols).intersection(availExtras): newSchema.addField(extraName, type=availExtras[extraName]['type'], doc=availExtras[extraName]['doc']) srcList = SourceCatalog(newSchema) srcList.reserve( sum([len(s) for s in srcIndex.values()]) + (0 if not includeMissing else list(srcIndex.values()).count([]))) centroidKey = sources.getCentroidKey() isPrimary = sources.schema.find('detect_isPrimary').getKey() nChild = sources.schema.find('force_deblend_nChild').getKey() for ident, sindlist in srcIndex.items(): rMatched = fakeXY[ident][2] if minRad is not None: if rMatched < minRad: rMatched = minRad nMatched = len(sindlist) nPrimary = np.sum( [sources[int(obj)].get(isPrimary) for obj in sindlist]) nNoChild = np.sum([(sources[int(obj)].get(nChild) == 0) for obj in sindlist]) if includeMissing and (nMatched == 0): newRec = srcList.addNew() newRec.set('fakeId', ident) newRec.set('id', 0) newRec.set('nMatched', 0) newRec.set('rMatched', rMatched) for ss in sindlist: newRec = srcList.addNew() newRec.assign(sources[int(ss)], mapper) newRec.set('fakeId', ident) newRec.set('nMatched', nMatched) newRec.set('nPrimary', nPrimary) newRec.set('nNoChild', nNoChild) newRec.set('rMatched', rMatched) offsetX = (sources[int(ss)].get(centroidKey).getX() - fakeXY[ident][0]) newRec.set('fakeOffX', offsetX) offsetY = (sources[int(ss)].get(centroidKey).getY() - fakeXY[ident][1]) newRec.set('fakeOffY', offsetY) newRec.set('fakeOffR', np.sqrt(offsetX**2.0 + offsetY**2.0)) if radecMatch: if int(ss) == int(srcClose[ident]): newRec.set('fakeClosest', True) else: newRec.set('fakeClosest', False) if includeMissing: srcList = srcList.copy(deep=True) for extraName in set(extraCols).intersection(availExtras): tempCol = srcList.get(extraName) tempCol.fill(availExtras[extraName]['value']) return srcList
def getFakeSources(butler, dataId, tol=1.0, extraCols=('zeropoint', 'visit', 'ccd'), includeMissing=False, footprints=False, radecMatch=None): """Get list of sources which agree in pixel position with fake ones with tol this returns a sourceCatalog of all the matched fake objects, note, there will be duplicates in this list, since I haven't checked deblend.nchild, and I'm only doing a tolerance match, which could include extra sources the outputs can include extraCols as long as they are one of: zeropoint, visit, ccd, thetaNorth, pixelScale if includeMissing is true, then the pipeline looks at the fake sources added in the header and includes an entry in the table for sources without any measurements, specifically the 'id' column will be 0 radecMatch is the fakes table. if it's not None(default), then do an ra/dec match with the input catalog instead of looking in the header for where the sources where added """ availExtras = {'zeropoint':{'type':float, 'doc':'zeropoint'}, 'visit':{'type':int, 'doc':'visit id'}, 'ccd':{'type':int, 'doc':'ccd id'}, 'thetaNorth':{'type':lsst.afw.geom.Angle, 'doc':'angle to north'}, 'pixelScale':{'type':float, 'doc':'pixelscale in arcsec/pixel'}} if not np.in1d(extraCols, availExtras.keys()).all(): print "extraCols must be in ",availExtras try: if not 'filter' in dataId: sources = butler.get('src', dataId, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True) cal = butler.get('calexp', dataId, immediate=True) cal_md = butler.get('calexp_md', dataId, immediate=True) else: sources = butler.get('deepCoadd_src', dataId, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True) cal = butler.get('deepCoadd', dataId, immediate=True) cal_md = butler.get('deepCoadd_md', dataId, immediate=True) except (lsst.pex.exceptions.LsstException, RuntimeError) as e: print "skipping", dataId return None if ('pixelScale' in extraCols) or ('thetaNorth' in extraCols): wcs = cal.getWcs() availExtras['pixelScale']['value'] = wcs.pixelScale().asArcseconds() availExtras['thetaNorth']['value'] = lsst.afw.geom.Angle( np.arctan2(*tuple(wcs.getLinearTransform().invert() (lsst.afw.geom.Point2D(1.0,0.0))))) if 'visit' in extraCols: availExtras['visit']['value'] = dataId['visit'] if 'ccd' in extraCols: availExtras['ccd']['value'] = dataId['ccd'] if 'zeropoint' in extraCols: availExtras['zeropoint']['value'] = 2.5*np.log10(cal_md.get('FLUXMAG0')) if radecMatch is None: fakeXY, srcIndex = getFakeMatchesHeader(cal_md, sources, tol=tol) else: fakeXY, srcIndex = getFakeMatchesRaDec(sources, radecMatch, lsst.afw.geom.Box2D(cal.getBBox(lsst.afw.image.PARENT)), cal.getWcs(), tol=tol) mapper = SchemaMapper(sources.schema) mapper.addMinimalSchema(sources.schema) newSchema = mapper.getOutputSchema() newSchema.addField('fakeId', type=int, doc='id of fake source matched to position') newSchema.addField('fakeOffset', type=lsst.afw.geom.Point2D, doc='offset from input fake position (pixels)') for extraName in set(extraCols).intersection(availExtras): newSchema.addField(extraName, type=availExtras[extraName]['type'], doc=availExtras[extraName]['doc']) srcList = SourceCatalog(newSchema) srcList.reserve(sum([len(s) for s in srcIndex.values()]) + (0 if not includeMissing else srcIndex.values().count([]))) centroidKey = sources.schema.find('centroid.sdss').getKey() for ident, sindlist in srcIndex.items(): if includeMissing and (len(sindlist)==0): newRec = srcList.addNew() newRec.set('fakeId', ident) newRec.set('id', 0) for ss in sindlist: newRec = srcList.addNew() newRec.assign(sources[ss], mapper) newRec.set('fakeId', ident) newRec.set('fakeOffset', lsst.afw.geom.Point2D(sources[ss].get(centroidKey).getX() - fakeXY[ident][0], sources[ss].get(centroidKey).getY() - fakeXY[ident][1])) if includeMissing: srcList = srcList.copy(deep=True) for extraName in set(extraCols).intersection(availExtras): tempCol = srcList.get(extraName) tempCol.fill(availExtras[extraName]['value']) return srcList
def getFakeSources( butler, dataId, tol=1.0, extraCols=("zeropoint", "visit", "ccd"), includeMissing=False, footprints=False, radecMatch=None, multiband=False, reffMatch=False, pix=0.168, minRad=None, raCol="RA", decCol="Dec", ): """ Get list of sources which agree in pixel position with fake ones with tol. This returns a sourceCatalog of all the matched fake objects, note, there will be duplicates in this list, since I haven't checked deblend.nchild, and I'm only doing a tolerance match, which could include extra sources The outputs can include extraCols as long as they are one of: zeropoint, visit, ccd, thetaNorth, pixelScale If includeMissing is true, then the pipeline looks at the fake sources added in the header and includes an entry in the table for sources without any measurements, specifically the 'id' column will be 0 radecMatch is the fakes table. if it's not None(default), then do an ra/dec match with the input catalog instead of looking in the header for where the sources where added """ pipeVersion = dafPersist.eupsVersions.EupsVersions().versions["hscPipe"] if StrictVersion(pipeVersion) >= StrictVersion("3.9.0"): coaddData = "deepCoadd_calexp" coaddMeta = "deepCoadd_calexp_md" else: coaddData = "deepCoadd" coaddMeta = "deepCoadd_md" availExtras = { "zeropoint": {"type": float, "doc": "zeropoint"}, "visit": {"type": int, "doc": "visit id"}, "ccd": {"type": int, "doc": "ccd id"}, "thetaNorth": {"type": lsst.afw.geom.Angle, "doc": "angle to north"}, "pixelScale": {"type": float, "doc": "pixelscale in arcsec/pixel"}, } if not np.in1d(extraCols, availExtras.keys()).all(): print "extraCols must be in ", availExtras try: if "filter" not in dataId: sources = butler.get("src", dataId, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True) cal = butler.get("calexp", dataId, immediate=True) cal_md = butler.get("calexp_md", dataId, immediate=True) else: meas = butler.get("deepCoadd_meas", dataId, flags=NO_FOOTPRINT, immediate=True) force = butler.get("deepCoadd_forced_src", dataId, flags=NO_FOOTPRINT, immediate=True) sources = combineWithForce(meas, force) cal = butler.get(coaddData, dataId, immediate=True) cal_md = butler.get(coaddMeta, dataId, immediate=True) except (lsst.pex.exceptions.LsstException, RuntimeError): print "skipping", dataId return None if ("pixelScale" in extraCols) or ("thetaNorth" in extraCols): wcs = cal.getWcs() availExtras["pixelScale"]["value"] = wcs.pixelScale().asArcseconds() availExtras["thetaNorth"]["value"] = lsst.afw.geom.Angle( np.arctan2(*tuple(wcs.getLinearTransform().invert()(lsst.afw.geom.Point2D(1.0, 0.0)))) ) if "visit" in extraCols: availExtras["visit"]["value"] = dataId["visit"] if "ccd" in extraCols: availExtras["ccd"]["value"] = dataId["ccd"] if "zeropoint" in extraCols: zeropoint = 2.5 * np.log10(cal_md.get("FLUXMAG0")) availExtras["zeropoint"]["value"] = zeropoint if radecMatch is None: fakeXY, srcIndex = getFakeMatchesHeader(cal_md, sources, tol=tol) else: if minRad is not None: print "# The min matching radius is %4.1f pixel" % minRad bbox = lsst.afw.geom.Box2D(cal.getBBox(lsst.afw.image.PARENT)) fakeXY, srcIndex, srcClose = getFakeMatchesRaDec( sources, radecMatch, bbox, cal.getWcs(), tol=tol, reffMatch=reffMatch, pix=pix, minRad=minRad, raCol=raCol, decCol=decCol, ) mapper = SchemaMapper(sources.schema) mapper.addMinimalSchema(sources.schema) newSchema = mapper.getOutputSchema() newSchema.addField("fakeId", type=int, doc="id of fake source matched to position") newSchema.addField("nMatched", type=int, doc="Number of matched objects") newSchema.addField("nPrimary", type=int, doc="Number of unique matched objects") newSchema.addField("nNoChild", type=int, doc="Number of matched objects with nchild==0") newSchema.addField("rMatched", type=float, doc="Radius used form atching obects, in pixel") newSchema.addField("fakeOffX", type=float, doc="offset from input fake position in X (pixels)") newSchema.addField("fakeOffY", type=float, doc="offset from input fake position in Y (pixels)") newSchema.addField("fakeOffR", type=float, doc="offset from input fake position in radius") newSchema.addField("fakeClosest", type="Flag", doc="Is this match the closest one?") for extraName in set(extraCols).intersection(availExtras): newSchema.addField(extraName, type=availExtras[extraName]["type"], doc=availExtras[extraName]["doc"]) srcList = SourceCatalog(newSchema) srcList.reserve( sum([len(s) for s in srcIndex.values()]) + (0 if not includeMissing else srcIndex.values().count([])) ) centroidKey = sources.schema.find("centroid.sdss").getKey() isPrimary = sources.schema.find("detect.is-primary").getKey() nChild = sources.schema.find("force.deblend.nchild").getKey() for ident, sindlist in srcIndex.items(): rMatched = fakeXY[ident][2] if minRad is not None: if rMatched < minRad: rMatched = minRad nMatched = len(sindlist) nPrimary = np.sum([sources[obj].get(isPrimary) for obj in sindlist]) nNoChild = np.sum([(sources[obj].get(nChild) == 0) for obj in sindlist]) if includeMissing and (nMatched == 0): newRec = srcList.addNew() newRec.set("fakeId", ident) newRec.set("id", 0) newRec.set("nMatched", 0) newRec.set("rMatched", rMatched) for ss in sindlist: newRec = srcList.addNew() newRec.assign(sources[ss], mapper) newRec.set("fakeId", ident) newRec.set("nMatched", nMatched) newRec.set("nPrimary", nPrimary) newRec.set("nNoChild", nNoChild) newRec.set("rMatched", rMatched) offsetX = sources[ss].get(centroidKey).getX() - fakeXY[ident][0] newRec.set("fakeOffX", offsetX) offsetY = sources[ss].get(centroidKey).getY() - fakeXY[ident][1] newRec.set("fakeOffY", offsetY) newRec.set("fakeOffR", np.sqrt(offsetX ** 2.0 + offsetY ** 2.0)) if radecMatch: if ss == srcClose[ident]: newRec.set("fakeClosest", True) else: newRec.set("fakeClosest", False) if includeMissing: srcList = srcList.copy(deep=True) for extraName in set(extraCols).intersection(availExtras): tempCol = srcList.get(extraName) tempCol.fill(availExtras[extraName]["value"]) return srcList
def getFakeSources(butler, dataId, tol=1.0, extraCols=('zeropoint', 'visit', 'ccd'), includeMissing=False, footprints=False, radecMatch=None): """Get list of sources which agree in pixel position with fake ones with tol this returns a sourceCatalog of all the matched fake objects, note, there will be duplicates in this list, since I haven't checked deblend.nchild, and I'm only doing a tolerance match, which could include extra sources the outputs can include extraCols as long as they are one of: zeropoint, visit, ccd, thetaNorth, pixelScale if includeMissing is true, then the pipeline looks at the fake sources added in the header and includes an entry in the table for sources without any measurements, specifically the 'id' column will be 0 radecMatch is the fakes table. if it's not None(default), then do an ra/dec match with the input catalog instead of looking in the header for where the sources where added """ availExtras = { 'zeropoint': { 'type': float, 'doc': 'zeropoint' }, 'visit': { 'type': int, 'doc': 'visit id' }, 'ccd': { 'type': int, 'doc': 'ccd id' }, 'thetaNorth': { 'type': lsst.afw.geom.Angle, 'doc': 'angle to north' }, 'pixelScale': { 'type': float, 'doc': 'pixelscale in arcsec/pixel' } } if not np.in1d(extraCols, availExtras.keys()).all(): print "extraCols must be in ", availExtras try: if not 'filter' in dataId: sources = butler.get('src', dataId, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True) cal = butler.get('calexp', dataId, immediate=True) cal_md = butler.get('calexp_md', dataId, immediate=True) else: sources = butler.get('deepCoadd_src', dataId, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True) cal = butler.get('deepCoadd', dataId, immediate=True) cal_md = butler.get('deepCoadd_md', dataId, immediate=True) except (lsst.pex.exceptions.LsstException, RuntimeError) as e: print "skipping", dataId return None if ('pixelScale' in extraCols) or ('thetaNorth' in extraCols): wcs = cal.getWcs() availExtras['pixelScale']['value'] = wcs.pixelScale().asArcseconds() availExtras['thetaNorth']['value'] = lsst.afw.geom.Angle( np.arctan2(*tuple(wcs.getLinearTransform().invert()( lsst.afw.geom.Point2D(1.0, 0.0))))) if 'visit' in extraCols: availExtras['visit']['value'] = dataId['visit'] if 'ccd' in extraCols: availExtras['ccd']['value'] = dataId['ccd'] if 'zeropoint' in extraCols: availExtras['zeropoint']['value'] = 2.5 * np.log10( cal_md.get('FLUXMAG0')) if radecMatch is None: fakeXY, srcIndex = getFakeMatchesHeader(cal_md, sources, tol=tol) else: fakeXY, srcIndex = getFakeMatchesRaDec( sources, radecMatch, lsst.afw.geom.Box2D(cal.getBBox(lsst.afw.image.PARENT)), cal.getWcs(), tol=tol) mapper = SchemaMapper(sources.schema) mapper.addMinimalSchema(sources.schema) newSchema = mapper.getOutputSchema() newSchema.addField('fakeId', type=int, doc='id of fake source matched to position') newSchema.addField('fakeOffset', type=lsst.afw.geom.Point2D, doc='offset from input fake position (pixels)') for extraName in set(extraCols).intersection(availExtras): newSchema.addField(extraName, type=availExtras[extraName]['type'], doc=availExtras[extraName]['doc']) srcList = SourceCatalog(newSchema) srcList.reserve( sum([len(s) for s in srcIndex.values()]) + (0 if not includeMissing else srcIndex.values().count([]))) centroidKey = sources.schema.find('centroid.sdss').getKey() for ident, sindlist in srcIndex.items(): if includeMissing and (len(sindlist) == 0): newRec = srcList.addNew() newRec.set('fakeId', ident) newRec.set('id', 0) for ss in sindlist: newRec = srcList.addNew() newRec.assign(sources[ss], mapper) newRec.set('fakeId', ident) newRec.set( 'fakeOffset', lsst.afw.geom.Point2D( sources[ss].get(centroidKey).getX() - fakeXY[ident][0], sources[ss].get(centroidKey).getY() - fakeXY[ident][1])) if includeMissing: srcList = srcList.copy(deep=True) for extraName in set(extraCols).intersection(availExtras): tempCol = srcList.get(extraName) tempCol.fill(availExtras[extraName]['value']) return srcList