Esempio n. 1
0
    def defineSchema(self, refSchema):

        self.mapper = SchemaMapper(refSchema)
        self.mapper.addMinimalSchema(SourceCatalog.Table.makeMinimalSchema(),
                                     True)
        schema = self.mapper.getOutputSchema()
        self.mKey = schema.addField("m",
                                    doc="template m",
                                    type="ArrayF",
                                    size=self.bfd.BFDConfig.MXYSIZE)
        self.dmKey = schema.addField("dm",
                                     doc="template m",
                                     type="ArrayF",
                                     size=self.bfd.BFDConfig.MSIZE *
                                     self.bfd.BFDConfig.DSIZE)
        self.dxyKey = schema.addField("dxy",
                                      doc="template m",
                                      type="ArrayF",
                                      size=self.bfd.BFDConfig.XYSIZE *
                                      self.bfd.BFDConfig.DSIZE)
        self.ndaKey = schema.addField("nda", doc="nda", type=np.float)
        self.idKey = schema.addField("bfd_id", doc="id", type=np.int64)
        if self.config.zFile:
            self.zKey = schema.addField("z", doc="redshift", type=np.float)
            # self.zIdKey = schema.addField("z_id", doc="redshift", type=np.int64)

        return schema
Esempio n. 2
0
def combineWithForce(meas, force):
    """Combine the meas and forced_src catalogs."""
    if len(meas) != len(force):
        raise Exception("# Meas and Forced_src catalogs should have " +
                        "the same size!")
    mapper = SchemaMapper(meas.schema)
    mapper.addMinimalSchema(meas.schema)
    newSchema = mapper.getOutputSchema()
    # Add new fields
    newSchema.addField('force.deblend.nchild', type=int)
    newSchema.addField('force.classification.extendedness', type=float)
    newSchema.addField('force.flux.kron', type=float)
    newSchema.addField('force.flux.kron.err', type=float)
    newSchema.addField('force.flux.psf', type=float)
    newSchema.addField('force.flux.psf.err', type=float)
    newSchema.addField('force.flux.kron.apcorr', type=float)
    newSchema.addField('force.flux.kron.apcorr.err', type=float)
    newSchema.addField('force.flux.psf.apcorr', type=float)
    newSchema.addField('force.flux.psf.apcorr.err', type=float)
    newSchema.addField('force.cmodel.flux', type=float)
    newSchema.addField('force.cmodel.flux.err', type=float)
    newSchema.addField('force.cmodel.fracDev', type=float)
    newSchema.addField('force.cmodel.exp.flux', type=float)
    newSchema.addField('force.cmodel.exp.flux.err', type=float)
    newSchema.addField('force.cmodel.dev.flux', type=float)
    newSchema.addField('force.cmodel.dev.flux.err', type=float)
    newSchema.addField('force.cmodel.flux.apcorr', type=float)
    newSchema.addField('force.cmodel.flux.apcorr.err', type=float)
    newSchema.addField('force.cmodel.exp.flux.apcorr', type=float)
    newSchema.addField('force.cmodel.exp.flux.apcorr.err', type=float)
    newSchema.addField('force.cmodel.dev.flux.apcorr', type=float)
    newSchema.addField('force.cmodel.dev.flux.apcorr.err', type=float)

    newCols = ['deblend.nchild', 'classification.extendedness',
               'flux.kron', 'flux.kron.err',
               'flux.psf', 'flux.psf.err',
               'flux.kron.apcorr', 'flux.kron.apcorr.err',
               'flux.psf.apcorr', 'flux.psf.apcorr.err',
               'cmodel.flux', 'cmodel.flux.err',
               'cmodel.flux', 'cmodel.flux.err',
               'cmodel.flux.apcorr', 'cmodel.flux.apcorr.err',
               'cmodel.exp.flux', 'cmodel.exp.flux.err',
               'cmodel.exp.flux.apcorr', 'cmodel.exp.flux.apcorr.err',
               'cmodel.dev.flux', 'cmodel.dev.flux.err',
               'cmodel.dev.flux.apcorr', 'cmodel.dev.flux.apcorr.err',
               'cmodel.fracDev']
    combSrc = SourceCatalog(newSchema)
    combSrc.extend(meas, mapper=mapper)

    for key in newCols:
        combSrc['force.' + key][:] = force[key][:]

    for name in ("Centroid", "Shape"):
        val = getattr(meas.table, "get" + name + "Key")()
        err = getattr(meas.table, "get" + name + "ErrKey")()
        flag = getattr(meas.table, "get" + name + "FlagKey")()
        getattr(combSrc.table, "define" + name)(val, err, flag)

    return combSrc
Esempio n. 3
0
def create_source_catalog_from_text_and_butler(repo_dir, info, dataset='src'):
    butler = dafPersistence.Butler(repo_dir)
    schema = butler.get(dataset + "_schema", immediate=True).schema
    mapper = SchemaMapper(schema)
    mapper.addMinimalSchema(schema)
    newSchema = mapper.getOutputSchema()

    src_cat = SourceCatalog(newSchema)
    for row in info:
        record = src_cat.addNew()
        record.set('coord_ra', Angle(row['RA']*degrees))
        record.set('coord_dec', Angle(row['Dec']*degrees))

    print(src_cat['coord_ra'], src_cat['coord_dec'])
    return(src_cat)
Esempio n. 4
0
    def defineSchema(self, refSchema):

        self.mapper = SchemaMapper(refSchema)
        self.mapper.addMinimalSchema(SourceCatalog.Table.makeMinimalSchema(), True)
        schema = self.mapper.getOutputSchema()

        self.even = schema.addField('bfd_even', type="ArrayF",
                                    size=self.n_even, doc="Even Bfd moments")
        self.odd = schema.addField('bfd_odd', type="ArrayF",
                                   size=self.n_odd, doc="odd moments")
        self.shift = schema.addField('bfd_shift', type="ArrayF",
                                     size=2, doc="amount shifted to null moments")
        self.cov_even = schema.addField('bfd_cov_even', type="ArrayF",
                                        size=self.n_even*(self.n_even+1)//2,
                                        doc="even moment covariance matrix")
        self.cov_odd = schema.addField('bfd_cov_odd', type="ArrayF",
                                       size=self.n_odd*(self.n_odd+1)//2,
                                       doc="odd moment covariance matrix")
        self.flag = schema.addField('bfd_flag', type="Flag", doc="Set to 1 for any fatal failure")
        self.centroid_flag = schema.addField('bfd_flag_centroid', type="Flag",
                                             doc="Set to 1 for any fatal failure of centroid")
        self.parent_flag = schema.addField('bfd_flag_parent', type="Flag",
                                            doc="Set to 1 for parents")
        if self.config.add_single_bands:
            self.filter_keys = defaultdict(dict)
            self.n_even_single = self.n_even - len(self.config.filters) + 1
            self.n_odd_single = self.n_odd
            for band in self.config.filters:
                self.filter_keys[band]['even'] = schema.addField(f'bfd_even_{band}', type="ArrayF",
                                                                 size=self.n_even_single,
                                                                 doc=f"Even Bfd moments for filter {band}")
                self.filter_keys[band]['odd'] = schema.addField(f'bfd_odd_{band}', type="ArrayF",
                                                                size=self.n_odd_single,
                                                                doc=f"Odd Bfd moments for filter {band}")
                self.filter_keys[band]['cov_even'] = schema.addField(f'bfd_cov_even_{band}', type="ArrayF",
                                                                     size=self.n_even_single*(self.n_even_single+1)//2,
                                                                     doc=f"even moment covariance matrix in filter {band}")
                self.filter_keys[band]['cov_odd'] = schema.addField(f'bfd_cov_odd_{band}', type="ArrayF",
                                                                    size=self.n_odd_single*(self.n_odd_single+1)//2,
                                                                    doc=f"odd moment covariance matrix in filter {band}")

        return schema
Esempio n. 5
0
    def defineSchema(self, refSchema):

        self.mapper = SchemaMapper(refSchema)
        self.mapper.addMinimalSchema(SourceCatalog.Table.makeMinimalSchema(),
                                     True)
        schema = self.mapper.getOutputSchema()

        self.even = schema.addField('bfd_even',
                                    type="ArrayF",
                                    size=self.n_even,
                                    doc="Even Bfd moments")
        self.odd = schema.addField('bfd_odd',
                                   type="ArrayF",
                                   size=self.n_odd,
                                   doc="odd moments")
        self.shift = schema.addField('bfd_shift',
                                     type="ArrayF",
                                     size=2,
                                     doc="amount shifted to null moments")
        self.cov_even = schema.addField('bfd_cov_even',
                                        type="ArrayF",
                                        size=self.n_even * (self.n_even + 1) //
                                        2,
                                        doc="even moment covariance matrix")
        self.cov_odd = schema.addField('bfd_cov_odd',
                                       type="ArrayF",
                                       size=self.n_odd * (self.n_odd + 1) // 2,
                                       doc="odd moment covariance matrix")
        self.flag = schema.addField('bfd_flag',
                                    type="Flag",
                                    doc="Set to 1 for any fatal failure")
        self.centroid_flag = schema.addField(
            'bfd_flag_centroid',
            type="Flag",
            doc="Set to 1 for any fatal failure of centroid")
        self.parent_flag = schema.addField('bfd_flag_parent',
                                           type="Flag",
                                           doc="Set to 1 for parents")
        return schema
Esempio n. 6
0
    def defineSchema(self, refSchema):

        self.mapper = SchemaMapper(refSchema)
        self.mapper.addMinimalSchema(SourceCatalog.Table.makeMinimalSchema(), True)
        schema = self.mapper.getOutputSchema()
        self.pqrKey = schema.addField("pqr", doc="pqr", type="ArrayF",
                                      size=self.bfd.BFDConfig.DSIZE)
        self.momKey = schema.addField("moment", doc="moment", type="ArrayF",
                                      size=self.n_even)
        self.momCovKey = schema.addField("moment_cov", doc="moment", type="ArrayF",
                                         size=self.n_even*(self.n_even+1)//2)
        self.numKey = schema.addField("n_templates", doc="number", type=np.int64)
        self.uniqKey = schema.addField("n_unique", doc="unique", type=np.int32)
        self.zKey = schema.addField("z", doc="redshift", type=np.float)
        self.g1Key = schema.addField("g1", doc="redshift", type=np.float)
        self.g2Key = schema.addField("g2", doc="redshift", type=np.float)
        self.kappaKey = schema.addField("kappa", doc="redshift", type=np.float)
        self.magKey = schema.addField("mag", doc="redshift", type=np.float)
        self.labelKey = schema.addField("label", doc="redshift", type=str, size=10)
            # self.zIdKey = schema.addField("z_id", doc="redshift", type=np.int64)

        return schema
Esempio n. 7
0
def getFakeSources(butler,
                   dataId,
                   tol=1.0,
                   extraCols=('zeropoint', 'visit', 'ccd'),
                   includeMissing=False,
                   footprints=False,
                   radecMatch=None,
                   multiband=False,
                   reffMatch=False,
                   pix=0.168,
                   minRad=None,
                   raCol='RA',
                   decCol='Dec'):
    """
    Get list of sources which agree in pixel position with fake ones with tol.

    This returns a sourceCatalog of all the matched fake objects,
    note, there will be duplicates in this list, since I haven't
    checked deblend.nchild, and I'm only doing a tolerance match,
    which could include extra sources

    The outputs can include extraCols as long as they are one of:
        zeropoint, visit, ccd, thetaNorth, pixelScale

    If includeMissing is true, then the pipeline looks at the fake sources
    added in the header and includes an entry in the table for sources without
    any measurements, specifically the 'id' column will be 0

    radecMatch is the fakes table. if it's not None(default), then do an ra/dec
    match with the input catalog instead of looking in the header for where the
    sources where added
    """
    coaddData = "deepCoadd_calexp"
    coaddMeta = "deepCoadd_calexp_md"

    availExtras = {
        'zeropoint': {
            'type': float,
            'doc': 'zeropoint'
        },
        'visit': {
            'type': int,
            'doc': 'visit id'
        },
        'ccd': {
            'type': int,
            'doc': 'ccd id'
        },
        'thetaNorth': {
            'type': lsst.afw.geom.Angle,
            'doc': 'angle to north'
        },
        'pixelScale': {
            'type': float,
            'doc': 'pixelscale in arcsec/pixel'
        }
    }

    if not np.in1d(extraCols, list(availExtras.keys())).all():
        print("extraCols must be in ", availExtras)

    try:
        if 'filter' not in dataId:
            sources = butler.get('src',
                                 dataId,
                                 flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
                                 immediate=True)
            cal = butler.get('calexp', dataId, immediate=True)
            cal_md = butler.get('calexp_md', dataId, immediate=True)
        else:
            meas = butler.get('deepCoadd_meas',
                              dataId,
                              flags=NO_FOOTPRINT,
                              immediate=True)
            force = butler.get('deepCoadd_forced_src',
                               dataId,
                               flags=NO_FOOTPRINT,
                               immediate=True)
            sources = combineWithForce(meas, force)
            cal = butler.get(coaddData, dataId, immediate=True)
            cal_md = butler.get(coaddMeta, dataId, immediate=True)
    except RuntimeError:
        print("skipping", dataId)
        return None

    if ('pixelScale' in extraCols) or ('thetaNorth' in extraCols):
        wcs = cal.getWcs()
        availExtras['pixelScale']['value'] = wcs.getPixelScale().asArcseconds()
        # The 8 lines of code below find the angle to north, first the mid pixel of the calexp is found,
        # then the pixel to sky matrix at this point, the coordinate this gives can then be used to find the
        # linearized sky to pixel matrix which can then be used to find the angle.
        xMid = cal.getWidth() // 2
        yMid = cal.getHeight() // 2
        midPoint = lsst.afw.geom.Point2D(xMid, yMid)
        midCoord = wcs.pixelToSky(midPoint)
        northSkyToPixelMatrix = wcs.linearizeSkyToPixel(
            midCoord, lsst.afw.geom.degrees)
        northSkyToPixelMatrix = northSkyToPixelMatrix.getLinear()
        availExtras['thetaNorth']['value'] = (np.arctan2(
            *tuple(northSkyToPixelMatrix(lsst.afw.geom.Point2D(1.0, 0.0))))
                                              ) * lsst.afw.geom.radians

    if 'visit' in extraCols:
        availExtras['visit']['value'] = dataId['visit']
    if 'ccd' in extraCols:
        availExtras['ccd']['value'] = dataId['ccd']
    if 'zeropoint' in extraCols:
        zeropoint = 2.5 * np.log10(cal_md.getScalar('FLUXMAG0'))
        availExtras['zeropoint']['value'] = zeropoint

    if radecMatch is None:
        fakeXY, srcIndex = getFakeMatchesHeader(cal_md, sources, tol=tol)
    else:
        if minRad is not None:
            print("# The min matching radius is %4.1f pixel" % minRad)
        bbox = lsst.afw.geom.Box2D(cal.getBBox(lsst.afw.image.PARENT))
        fakeXY, srcIndex, srcClose = getFakeMatchesRaDec(sources,
                                                         radecMatch,
                                                         bbox,
                                                         cal.getWcs(),
                                                         tol=tol,
                                                         reffMatch=reffMatch,
                                                         pix=pix,
                                                         minRad=minRad,
                                                         raCol=raCol,
                                                         decCol=decCol)

    mapper = SchemaMapper(sources.schema)
    mapper.addMinimalSchema(sources.schema)
    newSchema = mapper.getOutputSchema()
    newSchema.addField('fakeId',
                       type=np.int32,
                       doc='id of fake source matched to position')
    newSchema.addField('nMatched',
                       type=np.int32,
                       doc='Number of matched objects')
    newSchema.addField('nPrimary',
                       type=np.int32,
                       doc='Number of unique matched objects')
    newSchema.addField('nNoChild',
                       type=np.int32,
                       doc='Number of matched objects with nchild==0')
    newSchema.addField('rMatched',
                       type=float,
                       doc='Radius used form atching obects, in pixel')
    newSchema.addField('fakeOffX',
                       type=float,
                       doc='offset from input fake position in X (pixels)')
    newSchema.addField('fakeOffY',
                       type=float,
                       doc='offset from input fake position in Y (pixels)')
    newSchema.addField('fakeOffR',
                       type=float,
                       doc='offset from input fake position in radius')
    newSchema.addField('fakeClosest',
                       type="Flag",
                       doc='Is this match the closest one?')

    for extraName in set(extraCols).intersection(availExtras):
        newSchema.addField(extraName,
                           type=availExtras[extraName]['type'],
                           doc=availExtras[extraName]['doc'])

    srcList = SourceCatalog(newSchema)
    srcList.reserve(
        sum([len(s) for s in srcIndex.values()]) +
        (0 if not includeMissing else list(srcIndex.values()).count([])))

    centroidKey = sources.getCentroidKey()
    isPrimary = sources.schema.find('detect_isPrimary').getKey()
    nChild = sources.schema.find('force_deblend_nChild').getKey()
    for ident, sindlist in srcIndex.items():
        rMatched = fakeXY[ident][2]
        if minRad is not None:
            if rMatched < minRad:
                rMatched = minRad
        nMatched = len(sindlist)
        nPrimary = np.sum(
            [sources[int(obj)].get(isPrimary) for obj in sindlist])
        nNoChild = np.sum([(sources[int(obj)].get(nChild) == 0)
                           for obj in sindlist])
        if includeMissing and (nMatched == 0):
            newRec = srcList.addNew()
            newRec.set('fakeId', ident)
            newRec.set('id', 0)
            newRec.set('nMatched', 0)
            newRec.set('rMatched', rMatched)
        for ss in sindlist:
            newRec = srcList.addNew()
            newRec.assign(sources[int(ss)], mapper)
            newRec.set('fakeId', ident)
            newRec.set('nMatched', nMatched)
            newRec.set('nPrimary', nPrimary)
            newRec.set('nNoChild', nNoChild)
            newRec.set('rMatched', rMatched)
            offsetX = (sources[int(ss)].get(centroidKey).getX() -
                       fakeXY[ident][0])
            newRec.set('fakeOffX', offsetX)
            offsetY = (sources[int(ss)].get(centroidKey).getY() -
                       fakeXY[ident][1])
            newRec.set('fakeOffY', offsetY)
            newRec.set('fakeOffR', np.sqrt(offsetX**2.0 + offsetY**2.0))
            if radecMatch:
                if int(ss) == int(srcClose[ident]):
                    newRec.set('fakeClosest', True)
                else:
                    newRec.set('fakeClosest', False)

    if includeMissing:
        srcList = srcList.copy(deep=True)

    for extraName in set(extraCols).intersection(availExtras):
        tempCol = srcList.get(extraName)
        tempCol.fill(availExtras[extraName]['value'])

    return srcList
Esempio n. 8
0
def combineWithForce(meas, force):
    """Combine the meas and forced_src catalogs."""
    if len(meas) != len(force):
        raise Exception("# Meas and Forced_src catalogs should have "
                        "the same size!")
    mapper = SchemaMapper(meas.schema)
    mapper.addMinimalSchema(meas.schema)
    newSchema = mapper.getOutputSchema()
    # Add new fields
    newSchema.addField('force_deblend_nChild', type=np.int32)
    newSchema.addField('force_base_ClassificationExtendedness_value',
                       type=float)
    newSchema.addField('force_ext_photometryKron_KronFlux_instFlux',
                       type=float)
    newSchema.addField('force_ext_photometryKron_KronFlux_instFluxErr',
                       type=float)
    newSchema.addField('force_base_PsfFlux_instFlux', type=float)
    newSchema.addField('force_base_PsfFlux_instFluxErr', type=float)
    newSchema.addField('force_ext_photometryKron_KronFlux_apCorr', type=float)
    newSchema.addField('force_ext_photometryKron_KronFlux_apCorrErr',
                       type=float)
    newSchema.addField('force_base_PsfFlux_apCorr', type=float)
    newSchema.addField('force_base_PsfFlux_apCorrErr', type=float)
    newSchema.addField('force_modelfit_CModel_instFlux', type=float)
    newSchema.addField('force_modelfit_CModel_instFluxErr', type=float)
    newSchema.addField('force_modelfit_CModel_fracDev', type=float)
    newSchema.addField('force_modelfit_CModel_exp_instFlux', type=float)
    newSchema.addField('force_modelfit_CModel_exp_instFluxErr', type=float)
    newSchema.addField('force_modelfit_CModel_dev_instFlux', type=float)
    newSchema.addField('force_modelfit_CModel_dev_instFluxErr', type=float)
    newSchema.addField('force_modelfit_CModel_apCorr', type=float)
    newSchema.addField('force_modelfit_CModel_apCorrErr', type=float)
    newSchema.addField('force_modelfit_CModel_exp_apCorr', type=float)
    newSchema.addField('force_modelfit_CModel_exp_apCorrErr', type=float)
    newSchema.addField('force_modelfit_CModel_dev_apCorr', type=float)
    newSchema.addField('force_modelfit_CModel_dev_apCorrErr', type=float)

    newCols = [
        'deblend_nChild', 'base_ClassificationExtendedness_value',
        'ext_photometryKron_KronFlux_instFlux',
        'ext_photometryKron_KronFlux_instFluxErr', 'base_PsfFlux_instFlux',
        'base_PsfFlux_instFluxErr', 'ext_photometryKron_KronFlux_apCorr',
        'ext_photometryKron_KronFlux_apCorrErr', 'base_PsfFlux_apCorr',
        'base_PsfFlux_apCorrErr', 'modelfit_CModel_instFlux',
        'modelfit_CModel_instFluxErr', 'modelfit_CModel_exp_apCorr',
        'modelfit_CModel_exp_apCorrErr', 'modelfit_CModel_exp_instFlux',
        'modelfit_CModel_exp_instFlux', 'modelfit_CModel_exp_apCorr',
        'modelfit_CModel_exp_apCorrErr', 'modelfit_CModel_dev_instFlux',
        'modelfit_CModel_dev_instFluxErr', 'modelfit_CModel_dev_apCorr',
        'modelfit_CModel_dev_apCorrErr', 'modelfit_CModel_fracDev'
    ]
    measAlias = meas.schema.getAliasMap()
    newAlias = newSchema.getAliasMap()
    for aliasKey in measAlias.keys():
        newAlias.set(aliasKey, measAlias[aliasKey])
    combSrc = SourceCatalog(newSchema)
    combSrc.extend(meas, mapper=mapper)

    for key in newCols:
        combSrc['force_' + key][:] = force[key][:]

    return combSrc
Esempio n. 9
0
def getGalaxy(rootdir, visit, ccd, tol):
    """Get list of sources which agree in position with fake ones with tol
    """
    # Call the butler
    butler = dafPersist.Butler(rootdir)
    dataId = {'visit': visit, 'ccd': ccd}
    tol = float(tol)

    # Get the source catalog and metadata
    sources = butler.get('src', dataId)
    cal_md = butler.get('calexp_md', dataId)

    # Get the X, Y locations of objects on the CCD
    srcX, srcY = sources.getX(), sources.getY()
    # Get the zeropoint
    zeropoint = (2.5 * np.log10(cal_md.getScalar("FLUXMAG0")))
    # Get the parent ID
    parentID = sources.get('parent')
    # Check the star/galaxy separation
    extendClass = sources.get('classification.extendedness')
    # Get the nChild
    nChild = sources.get('deblend.nchild')

    # For Galaxies: Get these parameters
    # 1. Get the Kron flux and its error
    fluxKron, ferrKron = sources.get('flux.kron'), sources.get('flux.kron.err')
    magKron = (zeropoint - 2.5 * np.log10(fluxKron))
    merrKron = (2.5 / np.log(10) * (ferrKron / fluxKron))
    # X, Y locations of the fake galaxies
    fakeList = collections.defaultdict(tuple)
    # Regular Expression
    # Search for keywords like FAKE12
    fakename = re.compile('FAKE([0-9]+)')
    # Go through all the keywords
    counts = 0
    for card in cal_md.names():
        # To see if the card matches the pattern
        m = fakename.match(card)
        if m is not None:
            # Get the X,Y location for fake object
            x, y = list(map(float, (cal_md.getScalar(card)).split(',')))
            # Get the ID or index of the fake object
            fakeID = int(m.group(1))
            fakeList[counts] = [fakeID, x, y]
            counts += 1

    # Match the fake object to the source list
    srcIndex = collections.defaultdict(list)
    for fid, fcoord in fakeList.items():
        separation = np.sqrt(np.abs(srcX-fcoord[1])**2 +
                             np.abs(srcY-fcoord[2])**2)
        matched = (separation <= tol)
        matchId = np.where(matched)[0]
        matchSp = separation[matchId]
        sortId = [matchId for (matchSp, matchId) in
                  sorted(zip(matchSp, matchId))]
        # DEBUG:
        # print fid, fcoord, matchId
        # print sortId, sorted(matchSp), matchId
        # Select the index of all matched object
        srcIndex[fid] = sortId

    # Return the source list
    mapper = SchemaMapper(sources.schema)
    mapper.addMinimalSchema(sources.schema)
    newSchema = mapper.getOutputSchema()
    newSchema.addField('fakeId', type=int,
                       doc='id of fake source matched to position')
    srcList = SourceCatalog(newSchema)
    srcList.reserve(sum([len(s) for s in srcIndex.values()]))

    # Return a list of interesting parameters
    srcParam = []
    nFake = 0
    for matchIndex in srcIndex.values():
        # Check if there is a match
        if len(matchIndex) > 0:
            # Only select the one with the smallest separation
            # TODO: actually get the one with minimum separation
            ss = matchIndex[0]
            fakeObj = fakeList[nFake]
            diffX = srcX[ss] - fakeObj[1]
            diffY = srcY[ss] - fakeObj[2]
            paramList = (fakeObj[0], fakeObj[1], fakeObj[2],
                         magKron[ss], merrKron[ss], diffX, diffY,
                         parentID[ss], nChild[ss], extendClass[ss])
            srcParam.append(paramList)
        else:
            fakeObj = fakeList[nFake]
            paramList = (fakeObj[0], fakeObj[1], fakeObj[2],
                         0, 0, -1, -1, -1, -1, -1)
            srcParam.append(paramList)
        # Go to another fake object
        nFake += 1

    # Make a numpy record array
    srcParam = np.array(srcParam, dtype=[('fakeID', int),
                                         ('fakeX', float),
                                         ('fakeY', float),
                                         ('magKron', float),
                                         ('errKron', float),
                                         ('diffX', float),
                                         ('diffY', float),
                                         ('parentID', int),
                                         ('nChild', int),
                                         ('extendClass', float)])

    return srcIndex, srcParam, srcList, zeropoint
Esempio n. 10
0
    def _loadAndMatchCatalogs(self, repo, dataIds, matchRadius):
        """Load data from specific visit. Match with reference.

        Parameters
        ----------
        repo : string
            The repository.  This is generally the directory on disk
            that contains the repository and mapper.
        dataIds : list of dict
            List of `butler` data IDs of Image catalogs to compare to
            reference. The `calexp` cpixel image is needed for the photometric
            calibration.
        matchRadius :  afwGeom.Angle(), optional
            Radius for matching. Default is 1 arcsecond.

        Returns
        -------
        afw.table.GroupView
            An object of matched catalog.
        """
        # Following
        # https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
        butler = dafPersist.Butler(repo)
        dataset = 'src'

        # 2016-02-08 MWV:
        # I feel like I could be doing something more efficient with
        # something along the lines of the following:
        #    dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds]

        ccdKeyName = getCcdKeyName(dataIds[0])

        schema = butler.get(dataset + "_schema", immediate=True).schema
        mapper = SchemaMapper(schema)
        mapper.addMinimalSchema(schema)
        mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
        mapper.addOutputField(Field[float]('base_PsfFlux_mag',
                                           'PSF magnitude'))
        mapper.addOutputField(Field[float]('base_PsfFlux_magerr',
                                           'PSF magnitude uncertainty'))
        newSchema = mapper.getOutputSchema()

        # Create an object that matches multiple catalogs with same schema
        mmatch = MultiMatch(newSchema,
                            dataIdFormat={
                                'visit': np.int32,
                                ccdKeyName: np.int32
                            },
                            radius=matchRadius,
                            RecordClass=SimpleRecord)

        # create the new extented source catalog
        srcVis = SourceCatalog(newSchema)

        for vId in dataIds:
            try:
                calexpMetadata = butler.get("calexp_md", vId, immediate=True)
            except (FitsError, dafPersist.NoResults) as e:
                print(e)
                print("Could not open calibrated image file for ", vId)
                print("Skipping %s " % repr(vId))
                continue
            except TypeError as te:
                # DECam images that haven't been properly reformatted
                # can trigger a TypeError because of a residual FITS header
                # LTV2 which is a float instead of the expected integer.
                # This generates an error of the form:
                #
                # lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type'
                #
                # See, e.g., DM-2957 for details.
                print(te)
                print("Calibration image header information malformed.")
                print("Skipping %s " % repr(vId))
                continue

            calib = afwImage.Calib(calexpMetadata)

            oldSrc = butler.get('src', vId, immediate=True)
            print(
                len(oldSrc), "sources in ccd %s  visit %s" %
                (vId[ccdKeyName], vId["visit"]))

            # create temporary catalog
            tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
            tmpCat.extend(oldSrc, mapper=mapper)
            tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_flux'] \
                / tmpCat['base_PsfFlux_fluxSigma']
            with afwImageUtils.CalibNoThrow():
                _ = calib.getMagnitude(tmpCat['base_PsfFlux_flux'],
                                       tmpCat['base_PsfFlux_fluxSigma'])
                tmpCat['base_PsfFlux_mag'][:] = _[0]
                tmpCat['base_PsfFlux_magerr'][:] = _[1]

            srcVis.extend(tmpCat, False)
            mmatch.add(catalog=tmpCat, dataId=vId)

        # Complete the match, returning a catalog that includes
        # all matched sources with object IDs that can be used to group them.
        matchCat = mmatch.finish()

        # Create a mapping object that allows the matches to be manipulated
        # as a mapping of object ID to catalog of sources.
        allMatches = GroupView.build(matchCat)

        return allMatches
Esempio n. 11
0
def _loadAndMatchCatalogs(repo,
                          dataIds,
                          matchRadius,
                          useJointCal=False,
                          skipTEx=False):
    """Load data from specific visit. Match with reference.

    Parameters
    ----------
    repo : string or Butler
        A Butler or a repository URL that can be used to construct one
    dataIds : list of dict
        List of `butler` data IDs of Image catalogs to compare to
        reference. The `calexp` cpixel image is needed for the photometric
        calibration.
    matchRadius :  afwGeom.Angle(), optional
        Radius for matching. Default is 1 arcsecond.
    useJointCal : `bool`, optional
        Use jointcal/meas_mosaic outputs to calibrate positions and fluxes.
    skipTEx : `bool`, optional
        Skip TEx calculations (useful for older catalogs that don't have
        PsfShape measurements).

    Returns
    -------
    catalog_list : afw.table.SourceCatalog
        List of all of the catalogs
    matched_catalog : afw.table.GroupView
        An object of matched catalog.
    """
    # Following
    # https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
    if isinstance(repo, dafPersist.Butler):
        butler = repo
    else:
        butler = dafPersist.Butler(repo)
    dataset = 'src'

    # 2016-02-08 MWV:
    # I feel like I could be doing something more efficient with
    # something along the lines of the following:
    #    dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds]

    ccdKeyName = getCcdKeyName(dataIds[0])

    # Hack to support raft and sensor 0,1 IDs as ints for multimatch
    if ccdKeyName == 'sensor':
        ccdKeyName = 'raft_sensor_int'
        for vId in dataIds:
            vId[ccdKeyName] = raftSensorToInt(vId)

    schema = butler.get(dataset + "_schema").schema
    mapper = SchemaMapper(schema)
    mapper.addMinimalSchema(schema)
    mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
    mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude'))
    mapper.addOutputField(Field[float]('base_PsfFlux_magErr',
                                       'PSF magnitude uncertainty'))
    mapper.addOutputField(Field[float]('e1', 'Source Ellipticity 1'))
    mapper.addOutputField(Field[float]('e2', 'Source Ellipticity 1'))
    mapper.addOutputField(Field[float]('psf_e1', 'PSF Ellipticity 1'))
    mapper.addOutputField(Field[float]('psf_e2', 'PSF Ellipticity 1'))
    newSchema = mapper.getOutputSchema()
    newSchema.setAliasMap(schema.getAliasMap())

    # Create an object that matches multiple catalogs with same schema
    mmatch = MultiMatch(newSchema,
                        dataIdFormat={
                            'visit': np.int32,
                            ccdKeyName: np.int32
                        },
                        radius=matchRadius,
                        RecordClass=SimpleRecord)

    # create the new extented source catalog
    srcVis = SourceCatalog(newSchema)

    for vId in dataIds:

        if useJointCal:
            try:
                photoCalib = butler.get("jointcal_photoCalib", vId)
            except (FitsError, dafPersist.NoResults) as e:
                print(e)
                print("Could not open photometric calibration for ", vId)
                print("Skipping this dataId.")
                continue
            try:
                wcs = butler.get("jointcal_wcs", vId)
            except (FitsError, dafPersist.NoResults) as e:
                print(e)
                print("Could not open updated WCS for ", vId)
                print("Skipping this dataId.")
                continue
        else:
            try:
                photoCalib = butler.get("calexp_photoCalib", vId)
            except (FitsError, dafPersist.NoResults) as e:
                print(e)
                print("Could not open calibrated image file for ", vId)
                print("Skipping this dataId.")
                continue
            except TypeError as te:
                # DECam images that haven't been properly reformatted
                # can trigger a TypeError because of a residual FITS header
                # LTV2 which is a float instead of the expected integer.
                # This generates an error of the form:
                #
                # lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type'
                #
                # See, e.g., DM-2957 for details.
                print(te)
                print("Calibration image header information malformed.")
                print("Skipping this dataId.")
                continue

        # We don't want to put this above the first "if useJointCal block"
        # because we need to use the first `butler.get` above to quickly
        # catch data IDs with no usable outputs.
        try:
            # HSC supports these flags, which dramatically improve I/O
            # performance; support for other cameras is DM-6927.
            oldSrc = butler.get('src', vId, flags=SOURCE_IO_NO_FOOTPRINTS)
        except (OperationalError, sqlite3.OperationalError):
            oldSrc = butler.get('src', vId)

        print(len(oldSrc),
              "sources in ccd %s  visit %s" % (vId[ccdKeyName], vId["visit"]))

        # create temporary catalog
        tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
        tmpCat.extend(oldSrc, mapper=mapper)
        tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_instFlux'] \
            / tmpCat['base_PsfFlux_instFluxErr']

        if useJointCal:
            for record in tmpCat:
                record.updateCoord(wcs)
        photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux")

        if not skipTEx:
            _, psf_e1, psf_e2 = ellipticity_from_cat(
                oldSrc, slot_shape='slot_PsfShape')
            _, star_e1, star_e2 = ellipticity_from_cat(oldSrc,
                                                       slot_shape='slot_Shape')
            tmpCat['e1'][:] = star_e1
            tmpCat['e2'][:] = star_e2
            tmpCat['psf_e1'][:] = psf_e1
            tmpCat['psf_e2'][:] = psf_e2

        srcVis.extend(tmpCat, False)
        mmatch.add(catalog=tmpCat, dataId=vId)

    # Complete the match, returning a catalog that includes
    # all matched sources with object IDs that can be used to group them.
    matchCat = mmatch.finish()

    # Create a mapping object that allows the matches to be manipulated
    # as a mapping of object ID to catalog of sources.
    allMatches = GroupView.build(matchCat)

    return srcVis, allMatches
Esempio n. 12
0
    def _loadAndMatchCatalogs(self,
                              repo,
                              dataIds,
                              matchRadius,
                              useJointCal=False):
        """Load data from specific visit. Match with reference.

        Parameters
        ----------
        repo : string or Butler
            A Butler or a repository URL that can be used to construct one
        dataIds : list of dict
            List of `butler` data IDs of Image catalogs to compare to
            reference. The `calexp` cpixel image is needed for the photometric
            calibration.
        matchRadius :  afwGeom.Angle(), optional
            Radius for matching. Default is 1 arcsecond.

        Returns
        -------
        afw.table.GroupView
            An object of matched catalog.
        """
        # Following
        # https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
        if isinstance(repo, dafPersist.Butler):
            butler = repo
        else:
            butler = dafPersist.Butler(repo)
        dataset = 'src'

        # 2016-02-08 MWV:
        # I feel like I could be doing something more efficient with
        # something along the lines of the following:
        #    dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds]

        ccdKeyName = getCcdKeyName(dataIds[0])

        schema = butler.get(dataset + "_schema").schema
        mapper = SchemaMapper(schema)
        mapper.addMinimalSchema(schema)
        mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
        mapper.addOutputField(Field[float]('base_PsfFlux_mag',
                                           'PSF magnitude'))
        mapper.addOutputField(Field[float]('base_PsfFlux_magErr',
                                           'PSF magnitude uncertainty'))
        newSchema = mapper.getOutputSchema()
        newSchema.setAliasMap(schema.getAliasMap())

        # Create an object that matches multiple catalogs with same schema
        mmatch = MultiMatch(newSchema,
                            dataIdFormat={
                                'visit': np.int32,
                                ccdKeyName: np.int32
                            },
                            radius=matchRadius,
                            RecordClass=SimpleRecord)

        # create the new extented source catalog
        srcVis = SourceCatalog(newSchema)

        for vId in dataIds:

            if useJointCal:
                try:
                    photoCalib = butler.get("photoCalib", vId)
                except (FitsError, dafPersist.NoResults) as e:
                    print(e)
                    print("Could not open photometric calibration for ", vId)
                    print("Skipping %s " % repr(vId))
                    continue
                try:
                    md = butler.get("wcs_md", vId)
                    wcs = afwImage.makeWcs(md)
                except (FitsError, dafPersist.NoResults) as e:
                    print(e)
                    print("Could not open updated WCS for ", vId)
                    print("Skipping %s " % repr(vId))
                    continue
            else:
                try:
                    calexpMetadata = butler.get("calexp_md", vId)
                except (FitsError, dafPersist.NoResults) as e:
                    print(e)
                    print("Could not open calibrated image file for ", vId)
                    print("Skipping %s " % repr(vId))
                    continue
                except TypeError as te:
                    # DECam images that haven't been properly reformatted
                    # can trigger a TypeError because of a residual FITS header
                    # LTV2 which is a float instead of the expected integer.
                    # This generates an error of the form:
                    #
                    # lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type'
                    #
                    # See, e.g., DM-2957 for details.
                    print(te)
                    print("Calibration image header information malformed.")
                    print("Skipping %s " % repr(vId))
                    continue

                calib = afwImage.Calib(calexpMetadata)

            # We don't want to put this above the first "if useJointCal block"
            # because we need to use the first `butler.get` above to quickly
            # catch data IDs with no usable outputs.
            try:
                # HSC supports these flags, which dramatically improve I/O
                # performance; support for other cameras is DM-6927.
                oldSrc = butler.get('src', vId, flags=SOURCE_IO_NO_FOOTPRINTS)
            except:
                oldSrc = butler.get('src', vId)
            print(
                len(oldSrc), "sources in ccd %s  visit %s" %
                (vId[ccdKeyName], vId["visit"]))

            # create temporary catalog
            tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
            tmpCat.extend(oldSrc, mapper=mapper)
            tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_flux'] \
                / tmpCat['base_PsfFlux_fluxSigma']

            if useJointCal:
                for record in tmpCat:
                    record.updateCoord(wcs)
                photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux",
                                               "base_PsfFlux")
            else:
                with afwImageUtils.CalibNoThrow():
                    _ = calib.getMagnitude(tmpCat['base_PsfFlux_flux'],
                                           tmpCat['base_PsfFlux_fluxSigma'])
                    tmpCat['base_PsfFlux_mag'][:] = _[0]
                    tmpCat['base_PsfFlux_magErr'][:] = _[1]

            srcVis.extend(tmpCat, False)
            mmatch.add(catalog=tmpCat, dataId=vId)

        # Complete the match, returning a catalog that includes
        # all matched sources with object IDs that can be used to group them.
        matchCat = mmatch.finish()

        # Create a mapping object that allows the matches to be manipulated
        # as a mapping of object ID to catalog of sources.
        allMatches = GroupView.build(matchCat)

        return allMatches
Esempio n. 13
0
def _loadAndMatchCatalogs(repo,
                          dataIds,
                          matchRadius,
                          doApplyExternalPhotoCalib=False,
                          externalPhotoCalibName=None,
                          doApplyExternalSkyWcs=False,
                          externalSkyWcsName=None,
                          skipTEx=False,
                          skipNonSrd=False):
    """Load data from specific visits and returned a calibrated catalog matched
    with a reference.

    Parameters
    ----------
    repo : `str` or `lsst.daf.persistence.Butler`
        A Butler or a repository URL that can be used to construct one.
    dataIds : list of dict
        List of butler data IDs of Image catalogs to compare to
        reference. The calexp cpixel image is needed for the photometric
        calibration.
    matchRadius :  `lsst.geom.Angle`, optional
        Radius for matching. Default is 1 arcsecond.
    doApplyExternalPhotoCalib : bool, optional
        Apply external photoCalib to calibrate fluxes.
    externalPhotoCalibName : str, optional
        Type of external `PhotoCalib` to apply.  Currently supported are jointcal,
        fgcm, and fgcm_tract.  Must be set if doApplyExternalPhotoCalib is True.
    doApplyExternalSkyWcs : bool, optional
        Apply external wcs to calibrate positions.
    externalSkyWcsName : str, optional
        Type of external `wcs` to apply.  Currently supported is jointcal.
        Must be set if "doApplyExternalWcs" is True.
    skipTEx : `bool`, optional
        Skip TEx calculations (useful for older catalogs that don't have
        PsfShape measurements).
    skipNonSrd : `bool`, optional
        Skip any metrics not defined in the LSST SRD; default False.

    Returns
    -------
    catalog : `lsst.afw.table.SourceCatalog`
        A new calibrated SourceCatalog.
    matches : `lsst.afw.table.GroupView`
        A GroupView of the matched sources.

    Raises
    ------
    RuntimeError:
        Raised if "doApplyExternalPhotoCalib" is True and "externalPhotoCalibName"
        is None, or if "doApplyExternalSkyWcs" is True and "externalSkyWcsName" is
        None.
    """

    if doApplyExternalPhotoCalib and externalPhotoCalibName is None:
        raise RuntimeError(
            "Must set externalPhotoCalibName if doApplyExternalPhotoCalib is True."
        )
    if doApplyExternalSkyWcs and externalSkyWcsName is None:
        raise RuntimeError(
            "Must set externalSkyWcsName if doApplyExternalSkyWcs is True.")

    # Following
    # https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
    if isinstance(repo, dafPersist.Butler):
        butler = repo
    else:
        butler = dafPersist.Butler(repo)
    dataset = 'src'

    # 2016-02-08 MWV:
    # I feel like I could be doing something more efficient with
    # something along the lines of the following:
    #    dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds]

    ccdKeyName = getCcdKeyName(dataIds[0])

    # Hack to support raft and sensor 0,1 IDs as ints for multimatch
    if ccdKeyName == 'sensor':
        ccdKeyName = 'raft_sensor_int'
        for vId in dataIds:
            vId[ccdKeyName] = raftSensorToInt(vId)

    schema = butler.get(dataset + "_schema").schema
    mapper = SchemaMapper(schema)
    mapper.addMinimalSchema(schema)
    mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
    mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude'))
    mapper.addOutputField(Field[float]('base_PsfFlux_magErr',
                                       'PSF magnitude uncertainty'))
    if not skipNonSrd:
        # Needed because addOutputField(... 'slot_ModelFlux_mag') will add a field with that literal name
        aliasMap = schema.getAliasMap()
        # Possibly not needed since base_GaussianFlux is the default, but this ought to be safe
        modelName = aliasMap[
            'slot_ModelFlux'] if 'slot_ModelFlux' in aliasMap.keys(
            ) else 'base_GaussianFlux'
        mapper.addOutputField(Field[float](f'{modelName}_mag',
                                           'Model magnitude'))
        mapper.addOutputField(Field[float](f'{modelName}_magErr',
                                           'Model magnitude uncertainty'))
        mapper.addOutputField(Field[float](f'{modelName}_snr',
                                           'Model flux snr'))
    mapper.addOutputField(Field[float]('e1', 'Source Ellipticity 1'))
    mapper.addOutputField(Field[float]('e2', 'Source Ellipticity 1'))
    mapper.addOutputField(Field[float]('psf_e1', 'PSF Ellipticity 1'))
    mapper.addOutputField(Field[float]('psf_e2', 'PSF Ellipticity 1'))
    newSchema = mapper.getOutputSchema()
    newSchema.setAliasMap(schema.getAliasMap())

    # Create an object that matches multiple catalogs with same schema
    mmatch = MultiMatch(newSchema,
                        dataIdFormat={
                            'visit': np.int32,
                            ccdKeyName: np.int32
                        },
                        radius=matchRadius,
                        RecordClass=SimpleRecord)

    # create the new extented source catalog
    srcVis = SourceCatalog(newSchema)

    for vId in dataIds:
        if not butler.datasetExists('src', vId):
            print(f'Could not find source catalog for {vId}; skipping.')
            continue

        photoCalib = _loadPhotoCalib(butler, vId, doApplyExternalPhotoCalib,
                                     externalPhotoCalibName)
        if photoCalib is None:
            continue

        if doApplyExternalSkyWcs:
            wcs = _loadExternalSkyWcs(butler, vId, externalSkyWcsName)
            if wcs is None:
                continue

        # We don't want to put this above the first _loadPhotoCalib call
        # because we need to use the first `butler.get` in there to quickly
        # catch dataIDs with no usable outputs.
        try:
            # HSC supports these flags, which dramatically improve I/O
            # performance; support for other cameras is DM-6927.
            oldSrc = butler.get('src', vId, flags=SOURCE_IO_NO_FOOTPRINTS)
        except (OperationalError, sqlite3.OperationalError):
            oldSrc = butler.get('src', vId)

        print(len(oldSrc),
              "sources in ccd %s  visit %s" % (vId[ccdKeyName], vId["visit"]))

        # create temporary catalog
        tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
        tmpCat.extend(oldSrc, mapper=mapper)
        tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_instFlux'] \
            / tmpCat['base_PsfFlux_instFluxErr']

        if doApplyExternalSkyWcs:
            afwTable.updateSourceCoords(wcs, tmpCat)
        photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux")
        if not skipNonSrd:
            tmpCat['slot_ModelFlux_snr'][:] = (
                tmpCat['slot_ModelFlux_instFlux'] /
                tmpCat['slot_ModelFlux_instFluxErr'])
            photoCalib.instFluxToMagnitude(tmpCat, "slot_ModelFlux",
                                           "slot_ModelFlux")

        if not skipTEx:
            _, psf_e1, psf_e2 = ellipticity_from_cat(
                oldSrc, slot_shape='slot_PsfShape')
            _, star_e1, star_e2 = ellipticity_from_cat(oldSrc,
                                                       slot_shape='slot_Shape')
            tmpCat['e1'][:] = star_e1
            tmpCat['e2'][:] = star_e2
            tmpCat['psf_e1'][:] = psf_e1
            tmpCat['psf_e2'][:] = psf_e2

        srcVis.extend(tmpCat, False)
        mmatch.add(catalog=tmpCat, dataId=vId)

    # Complete the match, returning a catalog that includes
    # all matched sources with object IDs that can be used to group them.
    matchCat = mmatch.finish()

    # Create a mapping object that allows the matches to be manipulated
    # as a mapping of object ID to catalog of sources.
    allMatches = GroupView.build(matchCat)

    return srcVis, allMatches
Esempio n. 14
0
def match_catalogs(inputs,
                   photoCalibs,
                   astromCalibs,
                   vIds,
                   matchRadius,
                   apply_external_wcs=False,
                   logger=None):
    schema = inputs[0].schema
    mapper = SchemaMapper(schema)
    mapper.addMinimalSchema(schema)
    mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
    mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude'))
    mapper.addOutputField(Field[float]('base_PsfFlux_magErr',
                                       'PSF magnitude uncertainty'))
    # Needed because addOutputField(... 'slot_ModelFlux_mag') will add a field with that literal name
    aliasMap = schema.getAliasMap()
    # Possibly not needed since base_GaussianFlux is the default, but this ought to be safe
    modelName = aliasMap['slot_ModelFlux'] if 'slot_ModelFlux' in aliasMap.keys(
    ) else 'base_GaussianFlux'
    mapper.addOutputField(Field[float](f'{modelName}_mag', 'Model magnitude'))
    mapper.addOutputField(Field[float](f'{modelName}_magErr',
                                       'Model magnitude uncertainty'))
    mapper.addOutputField(Field[float](f'{modelName}_snr', 'Model flux snr'))
    mapper.addOutputField(Field[float]('e1', 'Source Ellipticity 1'))
    mapper.addOutputField(Field[float]('e2', 'Source Ellipticity 1'))
    mapper.addOutputField(Field[float]('psf_e1', 'PSF Ellipticity 1'))
    mapper.addOutputField(Field[float]('psf_e2', 'PSF Ellipticity 1'))
    mapper.addOutputField(Field[np.int32]('filt', 'filter code'))
    newSchema = mapper.getOutputSchema()
    newSchema.setAliasMap(schema.getAliasMap())

    # Create an object that matches multiple catalogs with same schema
    mmatch = MultiMatch(newSchema,
                        dataIdFormat={
                            'visit': np.int32,
                            'detector': np.int32
                        },
                        radius=matchRadius,
                        RecordClass=SimpleRecord)

    # create the new extended source catalog
    srcVis = SourceCatalog(newSchema)

    filter_dict = {
        'u': 1,
        'g': 2,
        'r': 3,
        'i': 4,
        'z': 5,
        'y': 6,
        'HSC-U': 1,
        'HSC-G': 2,
        'HSC-R': 3,
        'HSC-I': 4,
        'HSC-Z': 5,
        'HSC-Y': 6
    }

    # Sort by visit, detector, then filter
    vislist = [v['visit'] for v in vIds]
    ccdlist = [v['detector'] for v in vIds]
    filtlist = [v['band'] for v in vIds]
    tab_vids = Table([vislist, ccdlist, filtlist],
                     names=['vis', 'ccd', 'filt'])
    sortinds = np.argsort(tab_vids, order=('vis', 'ccd', 'filt'))

    for ind in sortinds:
        oldSrc = inputs[ind]
        photoCalib = photoCalibs[ind]
        wcs = astromCalibs[ind]
        vId = vIds[ind]

        if logger:
            logger.debug(
                f"{len(oldSrc)} sources in ccd {vId['detector']}  visit {vId['visit']}"
            )

        # create temporary catalog
        tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
        tmpCat.extend(oldSrc, mapper=mapper)

        filtnum = filter_dict[vId['band']]
        tmpCat['filt'] = np.repeat(filtnum, len(oldSrc))

        tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_instFlux'] \
            / tmpCat['base_PsfFlux_instFluxErr']

        if apply_external_wcs and wcs is not None:
            updateSourceCoords(wcs, tmpCat)

        photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux")
        tmpCat['slot_ModelFlux_snr'][:] = (
            tmpCat['slot_ModelFlux_instFlux'] /
            tmpCat['slot_ModelFlux_instFluxErr'])
        photoCalib.instFluxToMagnitude(tmpCat, "slot_ModelFlux",
                                       "slot_ModelFlux")

        _, psf_e1, psf_e2 = ellipticity_from_cat(oldSrc,
                                                 slot_shape='slot_PsfShape')
        _, star_e1, star_e2 = ellipticity_from_cat(oldSrc,
                                                   slot_shape='slot_Shape')
        tmpCat['e1'][:] = star_e1
        tmpCat['e2'][:] = star_e2
        tmpCat['psf_e1'][:] = psf_e1
        tmpCat['psf_e2'][:] = psf_e2

        srcVis.extend(tmpCat, False)
        mmatch.add(catalog=tmpCat, dataId=vId)

    # Complete the match, returning a catalog that includes
    # all matched sources with object IDs that can be used to group them.
    matchCat = mmatch.finish()

    # Create a mapping object that allows the matches to be manipulated
    # as a mapping of object ID to catalog of sources.

    # I don't think I can persist a group view, so this may need to be called in a subsequent task
    # allMatches = GroupView.build(matchCat)

    return srcVis, matchCat
Esempio n. 15
0
def getFakeSources(butler,
                   dataId,
                   tol=1.0,
                   extraCols=('zeropoint', 'visit', 'ccd'),
                   includeMissing=False,
                   footprints=False,
                   radecMatch=None):
    """Get list of sources which agree in pixel position with fake ones with tol
    
    this returns a sourceCatalog of all the matched fake objects,
    note, there will be duplicates in this list, since I haven't checked deblend.nchild,
    and I'm only doing a tolerance match, which could include extra sources
    
    the outputs can include extraCols as long as they are one of:
      zeropoint, visit, ccd, thetaNorth, pixelScale

    if includeMissing is true, then the pipeline looks at the fake sources
    added in the header and includes an entry in the table for sources without
    any measurements, specifically the 'id' column will be 0

    radecMatch is the fakes table. if it's not None(default), then do an ra/dec 
    match with the input catalog instead of looking in the header for where the 
    sources where added
    """

    availExtras = {
        'zeropoint': {
            'type': float,
            'doc': 'zeropoint'
        },
        'visit': {
            'type': int,
            'doc': 'visit id'
        },
        'ccd': {
            'type': int,
            'doc': 'ccd id'
        },
        'thetaNorth': {
            'type': lsst.afw.geom.Angle,
            'doc': 'angle to north'
        },
        'pixelScale': {
            'type': float,
            'doc': 'pixelscale in arcsec/pixel'
        }
    }

    if not np.in1d(extraCols, availExtras.keys()).all():
        print "extraCols must be in ", availExtras

    try:
        if not 'filter' in dataId:
            sources = butler.get('src',
                                 dataId,
                                 flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
                                 immediate=True)
            cal = butler.get('calexp', dataId, immediate=True)
            cal_md = butler.get('calexp_md', dataId, immediate=True)
        else:
            sources = butler.get('deepCoadd_src',
                                 dataId,
                                 flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
                                 immediate=True)
            cal = butler.get('deepCoadd', dataId, immediate=True)
            cal_md = butler.get('deepCoadd_md', dataId, immediate=True)
    except (lsst.pex.exceptions.LsstException, RuntimeError) as e:
        print "skipping", dataId
        return None

    if ('pixelScale' in extraCols) or ('thetaNorth' in extraCols):
        wcs = cal.getWcs()
        availExtras['pixelScale']['value'] = wcs.pixelScale().asArcseconds()
        availExtras['thetaNorth']['value'] = lsst.afw.geom.Angle(
            np.arctan2(*tuple(wcs.getLinearTransform().invert()(
                lsst.afw.geom.Point2D(1.0, 0.0)))))
    if 'visit' in extraCols:
        availExtras['visit']['value'] = dataId['visit']
    if 'ccd' in extraCols:
        availExtras['ccd']['value'] = dataId['ccd']
    if 'zeropoint' in extraCols:
        availExtras['zeropoint']['value'] = 2.5 * np.log10(
            cal_md.get('FLUXMAG0'))

    if radecMatch is None:
        fakeXY, srcIndex = getFakeMatchesHeader(cal_md, sources, tol=tol)
    else:
        fakeXY, srcIndex = getFakeMatchesRaDec(
            sources,
            radecMatch,
            lsst.afw.geom.Box2D(cal.getBBox(lsst.afw.image.PARENT)),
            cal.getWcs(),
            tol=tol)

    mapper = SchemaMapper(sources.schema)
    mapper.addMinimalSchema(sources.schema)
    newSchema = mapper.getOutputSchema()
    newSchema.addField('fakeId',
                       type=int,
                       doc='id of fake source matched to position')
    newSchema.addField('fakeOffset',
                       type=lsst.afw.geom.Point2D,
                       doc='offset from input fake position (pixels)')

    for extraName in set(extraCols).intersection(availExtras):
        newSchema.addField(extraName,
                           type=availExtras[extraName]['type'],
                           doc=availExtras[extraName]['doc'])

    srcList = SourceCatalog(newSchema)
    srcList.reserve(
        sum([len(s) for s in srcIndex.values()]) +
        (0 if not includeMissing else srcIndex.values().count([])))

    centroidKey = sources.schema.find('centroid.sdss').getKey()
    for ident, sindlist in srcIndex.items():
        if includeMissing and (len(sindlist) == 0):
            newRec = srcList.addNew()
            newRec.set('fakeId', ident)
            newRec.set('id', 0)
        for ss in sindlist:
            newRec = srcList.addNew()
            newRec.assign(sources[ss], mapper)
            newRec.set('fakeId', ident)
            newRec.set(
                'fakeOffset',
                lsst.afw.geom.Point2D(
                    sources[ss].get(centroidKey).getX() - fakeXY[ident][0],
                    sources[ss].get(centroidKey).getY() - fakeXY[ident][1]))

    if includeMissing:
        srcList = srcList.copy(deep=True)

    for extraName in set(extraCols).intersection(availExtras):
        tempCol = srcList.get(extraName)
        tempCol.fill(availExtras[extraName]['value'])

    return srcList