def defineSchema(self, refSchema):
self.mapper = SchemaMapper(refSchema)
self.mapper.addMinimalSchema(SourceCatalog.Table.makeMinimalSchema(),
True)
schema = self.mapper.getOutputSchema()
self.mKey = schema.addField("m",
doc="template m",
type="ArrayF",
size=self.bfd.BFDConfig.MXYSIZE)
self.dmKey = schema.addField("dm",
doc="template m",
type="ArrayF",
size=self.bfd.BFDConfig.MSIZE *
self.bfd.BFDConfig.DSIZE)
self.dxyKey = schema.addField("dxy",
doc="template m",
type="ArrayF",
size=self.bfd.BFDConfig.XYSIZE *
self.bfd.BFDConfig.DSIZE)
self.ndaKey = schema.addField("nda", doc="nda", type=np.float)
self.idKey = schema.addField("bfd_id", doc="id", type=np.int64)
if self.config.zFile:
self.zKey = schema.addField("z", doc="redshift", type=np.float)
# self.zIdKey = schema.addField("z_id", doc="redshift", type=np.int64)
return schema
def combineWithForce(meas, force):
"""Combine the meas and forced_src catalogs."""
if len(meas) != len(force):
raise Exception("# Meas and Forced_src catalogs should have " +
"the same size!")
mapper = SchemaMapper(meas.schema)
mapper.addMinimalSchema(meas.schema)
newSchema = mapper.getOutputSchema()
# Add new fields
newSchema.addField('force.deblend.nchild', type=int)
newSchema.addField('force.classification.extendedness', type=float)
newSchema.addField('force.flux.kron', type=float)
newSchema.addField('force.flux.kron.err', type=float)
newSchema.addField('force.flux.psf', type=float)
newSchema.addField('force.flux.psf.err', type=float)
newSchema.addField('force.flux.kron.apcorr', type=float)
newSchema.addField('force.flux.kron.apcorr.err', type=float)
newSchema.addField('force.flux.psf.apcorr', type=float)
newSchema.addField('force.flux.psf.apcorr.err', type=float)
newSchema.addField('force.cmodel.flux', type=float)
newSchema.addField('force.cmodel.flux.err', type=float)
newSchema.addField('force.cmodel.fracDev', type=float)
newSchema.addField('force.cmodel.exp.flux', type=float)
newSchema.addField('force.cmodel.exp.flux.err', type=float)
newSchema.addField('force.cmodel.dev.flux', type=float)
newSchema.addField('force.cmodel.dev.flux.err', type=float)
newSchema.addField('force.cmodel.flux.apcorr', type=float)
newSchema.addField('force.cmodel.flux.apcorr.err', type=float)
newSchema.addField('force.cmodel.exp.flux.apcorr', type=float)
newSchema.addField('force.cmodel.exp.flux.apcorr.err', type=float)
newSchema.addField('force.cmodel.dev.flux.apcorr', type=float)
newSchema.addField('force.cmodel.dev.flux.apcorr.err', type=float)
newCols = ['deblend.nchild', 'classification.extendedness',
'flux.kron', 'flux.kron.err',
'flux.psf', 'flux.psf.err',
'flux.kron.apcorr', 'flux.kron.apcorr.err',
'flux.psf.apcorr', 'flux.psf.apcorr.err',
'cmodel.flux', 'cmodel.flux.err',
'cmodel.flux', 'cmodel.flux.err',
'cmodel.flux.apcorr', 'cmodel.flux.apcorr.err',
'cmodel.exp.flux', 'cmodel.exp.flux.err',
'cmodel.exp.flux.apcorr', 'cmodel.exp.flux.apcorr.err',
'cmodel.dev.flux', 'cmodel.dev.flux.err',
'cmodel.dev.flux.apcorr', 'cmodel.dev.flux.apcorr.err',
'cmodel.fracDev']
combSrc = SourceCatalog(newSchema)
combSrc.extend(meas, mapper=mapper)
for key in newCols:
combSrc['force.' + key][:] = force[key][:]
for name in ("Centroid", "Shape"):
val = getattr(meas.table, "get" + name + "Key")()
err = getattr(meas.table, "get" + name + "ErrKey")()
flag = getattr(meas.table, "get" + name + "FlagKey")()
getattr(combSrc.table, "define" + name)(val, err, flag)
return combSrc
def combineWithForce(meas, force):
"""Combine the meas and forced_src catalogs."""
if len(meas) != len(force):
raise Exception("# Meas and Forced_src catalogs should have " +
"the same size!")
mapper = SchemaMapper(meas.schema)
mapper.addMinimalSchema(meas.schema)
newSchema = mapper.getOutputSchema()
# Add new fields
newSchema.addField('force.deblend.nchild', type=int)
newSchema.addField('force.classification.extendedness', type=float)
newSchema.addField('force.flux.kron', type=float)
newSchema.addField('force.flux.kron.err', type=float)
newSchema.addField('force.flux.psf', type=float)
newSchema.addField('force.flux.psf.err', type=float)
newSchema.addField('force.flux.kron.apcorr', type=float)
newSchema.addField('force.flux.kron.apcorr.err', type=float)
newSchema.addField('force.flux.psf.apcorr', type=float)
newSchema.addField('force.flux.psf.apcorr.err', type=float)
newSchema.addField('force.cmodel.flux', type=float)
newSchema.addField('force.cmodel.flux.err', type=float)
newSchema.addField('force.cmodel.fracDev', type=float)
newSchema.addField('force.cmodel.exp.flux', type=float)
newSchema.addField('force.cmodel.exp.flux.err', type=float)
newSchema.addField('force.cmodel.dev.flux', type=float)
newSchema.addField('force.cmodel.dev.flux.err', type=float)
newSchema.addField('force.cmodel.flux.apcorr', type=float)
newSchema.addField('force.cmodel.flux.apcorr.err', type=float)
newSchema.addField('force.cmodel.exp.flux.apcorr', type=float)
newSchema.addField('force.cmodel.exp.flux.apcorr.err', type=float)
newSchema.addField('force.cmodel.dev.flux.apcorr', type=float)
newSchema.addField('force.cmodel.dev.flux.apcorr.err', type=float)
newCols = ['deblend.nchild', 'classification.extendedness',
'flux.kron', 'flux.kron.err',
'flux.psf', 'flux.psf.err',
'flux.kron.apcorr', 'flux.kron.apcorr.err',
'flux.psf.apcorr', 'flux.psf.apcorr.err',
'cmodel.flux', 'cmodel.flux.err',
'cmodel.flux', 'cmodel.flux.err',
'cmodel.flux.apcorr', 'cmodel.flux.apcorr.err',
'cmodel.exp.flux', 'cmodel.exp.flux.err',
'cmodel.exp.flux.apcorr', 'cmodel.exp.flux.apcorr.err',
'cmodel.dev.flux', 'cmodel.dev.flux.err',
'cmodel.dev.flux.apcorr', 'cmodel.dev.flux.apcorr.err',
'cmodel.fracDev']
combSrc = SourceCatalog(newSchema)
combSrc.extend(meas, mapper=mapper)
for key in newCols:
combSrc['force.' + key][:] = force[key][:]
for name in ("Centroid", "Shape"):
val = getattr(meas.table, "get" + name + "Key")()
err = getattr(meas.table, "get" + name + "ErrKey")()
flag = getattr(meas.table, "get" + name + "FlagKey")()
getattr(combSrc.table, "define" + name)(val, err, flag)
return combSrc
def create_source_catalog_from_text_and_butler(repo_dir, info, dataset='src'):
butler = dafPersistence.Butler(repo_dir)
schema = butler.get(dataset + "_schema", immediate=True).schema
mapper = SchemaMapper(schema)
mapper.addMinimalSchema(schema)
newSchema = mapper.getOutputSchema()
src_cat = SourceCatalog(newSchema)
for row in info:
record = src_cat.addNew()
record.set('coord_ra', Angle(row['RA']*degrees))
record.set('coord_dec', Angle(row['Dec']*degrees))
print(src_cat['coord_ra'], src_cat['coord_dec'])
return(src_cat)
def defineSchema(self, refSchema):
self.mapper = SchemaMapper(refSchema)
self.mapper.addMinimalSchema(SourceCatalog.Table.makeMinimalSchema(), True)
schema = self.mapper.getOutputSchema()
self.even = schema.addField('bfd_even', type="ArrayF",
size=self.n_even, doc="Even Bfd moments")
self.odd = schema.addField('bfd_odd', type="ArrayF",
size=self.n_odd, doc="odd moments")
self.shift = schema.addField('bfd_shift', type="ArrayF",
size=2, doc="amount shifted to null moments")
self.cov_even = schema.addField('bfd_cov_even', type="ArrayF",
size=self.n_even*(self.n_even+1)//2,
doc="even moment covariance matrix")
self.cov_odd = schema.addField('bfd_cov_odd', type="ArrayF",
size=self.n_odd*(self.n_odd+1)//2,
doc="odd moment covariance matrix")
self.flag = schema.addField('bfd_flag', type="Flag", doc="Set to 1 for any fatal failure")
self.centroid_flag = schema.addField('bfd_flag_centroid', type="Flag",
doc="Set to 1 for any fatal failure of centroid")
self.parent_flag = schema.addField('bfd_flag_parent', type="Flag",
doc="Set to 1 for parents")
if self.config.add_single_bands:
self.filter_keys = defaultdict(dict)
self.n_even_single = self.n_even - len(self.config.filters) + 1
self.n_odd_single = self.n_odd
for band in self.config.filters:
self.filter_keys[band]['even'] = schema.addField(f'bfd_even_{band}', type="ArrayF",
size=self.n_even_single,
doc=f"Even Bfd moments for filter {band}")
self.filter_keys[band]['odd'] = schema.addField(f'bfd_odd_{band}', type="ArrayF",
size=self.n_odd_single,
doc=f"Odd Bfd moments for filter {band}")
self.filter_keys[band]['cov_even'] = schema.addField(f'bfd_cov_even_{band}', type="ArrayF",
size=self.n_even_single*(self.n_even_single+1)//2,
doc=f"even moment covariance matrix in filter {band}")
self.filter_keys[band]['cov_odd'] = schema.addField(f'bfd_cov_odd_{band}', type="ArrayF",
size=self.n_odd_single*(self.n_odd_single+1)//2,
doc=f"odd moment covariance matrix in filter {band}")
return schema
def defineSchema(self, refSchema):
self.mapper = SchemaMapper(refSchema)
self.mapper.addMinimalSchema(SourceCatalog.Table.makeMinimalSchema(),
True)
schema = self.mapper.getOutputSchema()
self.even = schema.addField('bfd_even',
type="ArrayF",
size=self.n_even,
doc="Even Bfd moments")
self.odd = schema.addField('bfd_odd',
type="ArrayF",
size=self.n_odd,
doc="odd moments")
self.shift = schema.addField('bfd_shift',
type="ArrayF",
size=2,
doc="amount shifted to null moments")
self.cov_even = schema.addField('bfd_cov_even',
type="ArrayF",
size=self.n_even * (self.n_even + 1) //
2,
doc="even moment covariance matrix")
self.cov_odd = schema.addField('bfd_cov_odd',
type="ArrayF",
size=self.n_odd * (self.n_odd + 1) // 2,
doc="odd moment covariance matrix")
self.flag = schema.addField('bfd_flag',
type="Flag",
doc="Set to 1 for any fatal failure")
self.centroid_flag = schema.addField(
'bfd_flag_centroid',
type="Flag",
doc="Set to 1 for any fatal failure of centroid")
self.parent_flag = schema.addField('bfd_flag_parent',
type="Flag",
doc="Set to 1 for parents")
return schema
def defineSchema(self, refSchema):
self.mapper = SchemaMapper(refSchema)
self.mapper.addMinimalSchema(SourceCatalog.Table.makeMinimalSchema(), True)
schema = self.mapper.getOutputSchema()
self.pqrKey = schema.addField("pqr", doc="pqr", type="ArrayF",
size=self.bfd.BFDConfig.DSIZE)
self.momKey = schema.addField("moment", doc="moment", type="ArrayF",
size=self.n_even)
self.momCovKey = schema.addField("moment_cov", doc="moment", type="ArrayF",
size=self.n_even*(self.n_even+1)//2)
self.numKey = schema.addField("n_templates", doc="number", type=np.int64)
self.uniqKey = schema.addField("n_unique", doc="unique", type=np.int32)
self.zKey = schema.addField("z", doc="redshift", type=np.float)
self.g1Key = schema.addField("g1", doc="redshift", type=np.float)
self.g2Key = schema.addField("g2", doc="redshift", type=np.float)
self.kappaKey = schema.addField("kappa", doc="redshift", type=np.float)
self.magKey = schema.addField("mag", doc="redshift", type=np.float)
self.labelKey = schema.addField("label", doc="redshift", type=str, size=10)
# self.zIdKey = schema.addField("z_id", doc="redshift", type=np.int64)
return schema
def combineWithForce(meas, force):
"""Combine the meas and forced_src catalogs."""
if len(meas) != len(force):
raise Exception("# Meas and Forced_src catalogs should have "
"the same size!")
mapper = SchemaMapper(meas.schema)
mapper.addMinimalSchema(meas.schema)
newSchema = mapper.getOutputSchema()
# Add new fields
newSchema.addField('force_deblend_nChild', type=np.int32)
newSchema.addField('force_base_ClassificationExtendedness_value',
type=float)
newSchema.addField('force_ext_photometryKron_KronFlux_instFlux',
type=float)
newSchema.addField('force_ext_photometryKron_KronFlux_instFluxErr',
type=float)
newSchema.addField('force_base_PsfFlux_instFlux', type=float)
newSchema.addField('force_base_PsfFlux_instFluxErr', type=float)
newSchema.addField('force_ext_photometryKron_KronFlux_apCorr', type=float)
newSchema.addField('force_ext_photometryKron_KronFlux_apCorrErr',
type=float)
newSchema.addField('force_base_PsfFlux_apCorr', type=float)
newSchema.addField('force_base_PsfFlux_apCorrErr', type=float)
newSchema.addField('force_modelfit_CModel_instFlux', type=float)
newSchema.addField('force_modelfit_CModel_instFluxErr', type=float)
newSchema.addField('force_modelfit_CModel_fracDev', type=float)
newSchema.addField('force_modelfit_CModel_exp_instFlux', type=float)
newSchema.addField('force_modelfit_CModel_exp_instFluxErr', type=float)
newSchema.addField('force_modelfit_CModel_dev_instFlux', type=float)
newSchema.addField('force_modelfit_CModel_dev_instFluxErr', type=float)
newSchema.addField('force_modelfit_CModel_apCorr', type=float)
newSchema.addField('force_modelfit_CModel_apCorrErr', type=float)
newSchema.addField('force_modelfit_CModel_exp_apCorr', type=float)
newSchema.addField('force_modelfit_CModel_exp_apCorrErr', type=float)
newSchema.addField('force_modelfit_CModel_dev_apCorr', type=float)
newSchema.addField('force_modelfit_CModel_dev_apCorrErr', type=float)
newCols = [
'deblend_nChild', 'base_ClassificationExtendedness_value',
'ext_photometryKron_KronFlux_instFlux',
'ext_photometryKron_KronFlux_instFluxErr', 'base_PsfFlux_instFlux',
'base_PsfFlux_instFluxErr', 'ext_photometryKron_KronFlux_apCorr',
'ext_photometryKron_KronFlux_apCorrErr', 'base_PsfFlux_apCorr',
'base_PsfFlux_apCorrErr', 'modelfit_CModel_instFlux',
'modelfit_CModel_instFluxErr', 'modelfit_CModel_exp_apCorr',
'modelfit_CModel_exp_apCorrErr', 'modelfit_CModel_exp_instFlux',
'modelfit_CModel_exp_instFlux', 'modelfit_CModel_exp_apCorr',
'modelfit_CModel_exp_apCorrErr', 'modelfit_CModel_dev_instFlux',
'modelfit_CModel_dev_instFluxErr', 'modelfit_CModel_dev_apCorr',
'modelfit_CModel_dev_apCorrErr', 'modelfit_CModel_fracDev'
]
measAlias = meas.schema.getAliasMap()
newAlias = newSchema.getAliasMap()
for aliasKey in measAlias.keys():
newAlias.set(aliasKey, measAlias[aliasKey])
combSrc = SourceCatalog(newSchema)
combSrc.extend(meas, mapper=mapper)
for key in newCols:
combSrc['force_' + key][:] = force[key][:]
return combSrc
def getGalaxy(rootdir, visit, ccd, tol):
"""Get list of sources which agree in position with fake ones with tol
"""
# Call the butler
butler = dafPersist.Butler(rootdir)
dataId = {'visit': visit, 'ccd': ccd}
tol = float(tol)
# Get the source catalog and metadata
sources = butler.get('src', dataId)
cal_md = butler.get('calexp_md', dataId)
# Get the X, Y locations of objects on the CCD
srcX, srcY = sources.getX(), sources.getY()
# Get the zeropoint
zeropoint = (2.5 * np.log10(cal_md.getScalar("FLUXMAG0")))
# Get the parent ID
parentID = sources.get('parent')
# Check the star/galaxy separation
extendClass = sources.get('classification.extendedness')
# Get the nChild
nChild = sources.get('deblend.nchild')
# For Galaxies: Get these parameters
# 1. Get the Kron flux and its error
fluxKron, ferrKron = sources.get('flux.kron'), sources.get('flux.kron.err')
magKron = (zeropoint - 2.5 * np.log10(fluxKron))
merrKron = (2.5 / np.log(10) * (ferrKron / fluxKron))
# X, Y locations of the fake galaxies
fakeList = collections.defaultdict(tuple)
# Regular Expression
# Search for keywords like FAKE12
fakename = re.compile('FAKE([0-9]+)')
# Go through all the keywords
counts = 0
for card in cal_md.names():
# To see if the card matches the pattern
m = fakename.match(card)
if m is not None:
# Get the X,Y location for fake object
x, y = list(map(float, (cal_md.getScalar(card)).split(',')))
# Get the ID or index of the fake object
fakeID = int(m.group(1))
fakeList[counts] = [fakeID, x, y]
counts += 1
# Match the fake object to the source list
srcIndex = collections.defaultdict(list)
for fid, fcoord in fakeList.items():
separation = np.sqrt(np.abs(srcX-fcoord[1])**2 +
np.abs(srcY-fcoord[2])**2)
matched = (separation <= tol)
matchId = np.where(matched)[0]
matchSp = separation[matchId]
sortId = [matchId for (matchSp, matchId) in
sorted(zip(matchSp, matchId))]
# DEBUG:
# print fid, fcoord, matchId
# print sortId, sorted(matchSp), matchId
# Select the index of all matched object
srcIndex[fid] = sortId
# Return the source list
mapper = SchemaMapper(sources.schema)
mapper.addMinimalSchema(sources.schema)
newSchema = mapper.getOutputSchema()
newSchema.addField('fakeId', type=int,
doc='id of fake source matched to position')
srcList = SourceCatalog(newSchema)
srcList.reserve(sum([len(s) for s in srcIndex.values()]))
# Return a list of interesting parameters
srcParam = []
nFake = 0
for matchIndex in srcIndex.values():
# Check if there is a match
if len(matchIndex) > 0:
# Only select the one with the smallest separation
# TODO: actually get the one with minimum separation
ss = matchIndex[0]
fakeObj = fakeList[nFake]
diffX = srcX[ss] - fakeObj[1]
diffY = srcY[ss] - fakeObj[2]
paramList = (fakeObj[0], fakeObj[1], fakeObj[2],
magKron[ss], merrKron[ss], diffX, diffY,
parentID[ss], nChild[ss], extendClass[ss])
srcParam.append(paramList)
else:
fakeObj = fakeList[nFake]
paramList = (fakeObj[0], fakeObj[1], fakeObj[2],
0, 0, -1, -1, -1, -1, -1)
srcParam.append(paramList)
# Go to another fake object
nFake += 1
# Make a numpy record array
srcParam = np.array(srcParam, dtype=[('fakeID', int),
('fakeX', float),
('fakeY', float),
('magKron', float),
('errKron', float),
('diffX', float),
('diffY', float),
('parentID', int),
('nChild', int),
('extendClass', float)])
return srcIndex, srcParam, srcList, zeropoint
def getGalaxy(rootdir, visit, ccd, tol):
"""Get list of sources which agree in position with fake ones with tol
"""
# Call the butler
butler = dafPersist.Butler(rootdir)
dataId = {'visit':visit, 'ccd':ccd}
tol = float(tol)
# Get the source catalog and metadata
sources = butler.get('src', dataId)
cal_md = butler.get('calexp_md', dataId)
# Get the X, Y locations of objects on the CCD
srcX, srcY = sources.getX(), sources.getY()
# Get the zeropoint
zeropoint = (2.5 * np.log10(cal_md.get("FLUXMAG0")))
# Get the parent ID
parentID = sources.get('parent')
# Check the star/galaxy separation
extendClass = sources.get('classification.extendedness')
# For Galaxies: Get these parameters
# 1. Get the Kron flux and its error
fluxKron, ferrKron = sources.get('flux.kron'), sources.get('flux.kron.err')
magKron, merrKron = (zeropoint - 2.5*np.log10(fluxKron)), (2.5/np.log(10)*
(ferrKron/fluxKron))
# 2. Get the CModel flux and its error
fluxCmod, ferrCmod = sources.get('cmodel.flux'), sources.get('cmodel.flux.err')
magCmod, merrCmod = (zeropoint - 2.5*np.log10(fluxCmod)), (2.5/np.log(10)*
(ferrCmod/fluxCmod))
# 3. Get the Exponential flux and its error
fluxExp, ferrExp = sources.get('cmodel.exp.flux'), sources.get('cmodel.exp.flux.err')
magExp, merrExp = (zeropoint - 2.5*np.log10(fluxExp)), (2.5/np.log(10)*
(ferrExp/fluxExp))
# 4. Get the de Vacouleurs flux and its error
fluxDev, ferrDev = sources.get('cmodel.dev.flux'), sources.get('cmodel.dev.flux.err')
magDev, merrDev = (zeropoint - 2.5*np.log10(fluxDev)), (2.5/np.log(10)*
(ferrDev/fluxDev))
# 5. Get the SDSS shapes (Re, b/a, PA)
sdssMoment = sources.get('shape.sdss')
sdssR, sdssBa, sdssPa = getSizeAndShape(sdssMoment)
# 6. Get the Exponential shapes (Re, b/a, PA)
expMoment = sources.get('cmodel.exp.ellipse')
expR, expBa, expPa = getSizeAndShape(expMoment)
# 7. Get the de Vaucouleurs shapes (Re, b/a, PA)
devMoment = sources.get('cmodel.dev.ellipse')
devR, devBa, devPa = getSizeAndShape(devMoment)
# 8. Get the fracDev
fracDev = sources.get('cmodel.fracDev')
# X, Y locations of the fake stars
fakeList = collections.defaultdict(tuple)
# Regular Expression
# Search for keywords like FAKE12
fakename = re.compile('FAKE([0-9]+)')
# Go through all the keywords
counts = 0
for card in cal_md.names():
# To see if the card matches the pattern
m = fakename.match(card)
if m is not None:
# Get the X,Y location for fake object
x,y = map(float, (cal_md.get(card)).split(','))
# Get the ID or index of the fake object
fakeID = int(m.group(1))
fakeList[counts] = [fakeID, x, y]
counts += 1
# Match the fake object to the source list
srcIndex = collections.defaultdict(list)
for fid, fcoord in fakeList.items():
separation = np.sqrt(np.abs(srcX-fcoord[1])**2 +
np.abs(srcY-fcoord[2])**2)
matched = (separation <= tol)
matchId = np.where(matched)[0]
matchSp = separation[matchId]
sortId = [matchId for (matchSp, matchId) in sorted(zip(matchSp,
matchId))]
# DEBUG:
# print fid, fcoord, matchId
print sortId, sorted(matchSp), matchId
# Select the index of all matched object
srcIndex[fid] = sortId
# Return the source list
mapper = SchemaMapper(sources.schema)
mapper.addMinimalSchema(sources.schema)
newSchema = mapper.getOutputSchema()
newSchema.addField('fakeId', type=int,
doc='id of fake source matched to position')
srcList = SourceCatalog(newSchema)
srcList.reserve(sum([len(s) for s in srcIndex.values()]))
# Return a list of interesting parameters
#srcParam = collections.defaultdict(list)
srcParam = []
nFake = 0
for matchIndex in srcIndex.values():
# Check if there is a match
if len(matchIndex) > 0:
# Only select the one with the smallest separation
# TODO: actually get the one with minimum separation
ss = matchIndex[0]
fakeObj = fakeList[nFake]
diffX = srcX[ss] - fakeObj[1]
diffY = srcY[ss] - fakeObj[2]
paramList = (fakeObj[0], fakeObj[1], fakeObj[2],
magKron[ss], merrKron[ss], magCmod[ss], merrCmod[ss],
magExp[ss], merrExp[ss], magDev[ss], merrDev[ss],
sdssR[ss], sdssBa[ss], sdssPa[ss],
expR[ss], expBa[ss], expPa[ss],
devR[ss], devBa[ss], devPa[ss],
diffX, diffY, fracDev[ss],
parentID[ss], extendClass[ss])
srcParam.append(paramList)
else:
paramList = (fakeObj[0], fakeObj[1], fakeObj[2],
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1,
0, -1, -1)
srcParam.append(paramList)
# Go to another fake object
nFake += 1
# Make a numpy record array
srcParam = np.array(srcParam, dtype=[('fakeID', int),
('fakeX', float),
('fakeY', float),
('magKron', float),
('errKron', float),
('magCmod', float),
('errCmod', float),
('magExp', float),
('errExp', float),
('magDev', float),
('errDev', float),
('sdssR', float),
('sdssBa', float),
('sdssPa', float),
('expR', float),
('expBa', float),
('expPa', float),
('devR', float),
('devBa', float),
('devPa', float),
('diffX', float),
('diffY', float),
('fracDev', float),
('parentID', int),
('extendClass', float)])
return srcIndex, srcParam, srcList, zeropoint
def getFakeSources(
butler,
dataId,
tol=1.0,
extraCols=("zeropoint", "visit", "ccd"),
includeMissing=False,
footprints=False,
radecMatch=None,
multiband=False,
reffMatch=False,
pix=0.168,
minRad=None,
raCol="RA",
decCol="Dec",
):
"""
Get list of sources which agree in pixel position with fake ones with tol.
This returns a sourceCatalog of all the matched fake objects,
note, there will be duplicates in this list, since I haven't
checked deblend.nchild, and I'm only doing a tolerance match,
which could include extra sources
The outputs can include extraCols as long as they are one of:
zeropoint, visit, ccd, thetaNorth, pixelScale
If includeMissing is true, then the pipeline looks at the fake sources
added in the header and includes an entry in the table for sources without
any measurements, specifically the 'id' column will be 0
radecMatch is the fakes table. if it's not None(default), then do an ra/dec
match with the input catalog instead of looking in the header for where the
sources where added
"""
pipeVersion = dafPersist.eupsVersions.EupsVersions().versions["hscPipe"]
if StrictVersion(pipeVersion) >= StrictVersion("3.9.0"):
coaddData = "deepCoadd_calexp"
coaddMeta = "deepCoadd_calexp_md"
else:
coaddData = "deepCoadd"
coaddMeta = "deepCoadd_md"
availExtras = {
"zeropoint": {"type": float, "doc": "zeropoint"},
"visit": {"type": int, "doc": "visit id"},
"ccd": {"type": int, "doc": "ccd id"},
"thetaNorth": {"type": lsst.afw.geom.Angle, "doc": "angle to north"},
"pixelScale": {"type": float, "doc": "pixelscale in arcsec/pixel"},
}
if not np.in1d(extraCols, availExtras.keys()).all():
print "extraCols must be in ", availExtras
try:
if "filter" not in dataId:
sources = butler.get("src", dataId, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
cal = butler.get("calexp", dataId, immediate=True)
cal_md = butler.get("calexp_md", dataId, immediate=True)
else:
meas = butler.get("deepCoadd_meas", dataId, flags=NO_FOOTPRINT, immediate=True)
force = butler.get("deepCoadd_forced_src", dataId, flags=NO_FOOTPRINT, immediate=True)
sources = combineWithForce(meas, force)
cal = butler.get(coaddData, dataId, immediate=True)
cal_md = butler.get(coaddMeta, dataId, immediate=True)
except (lsst.pex.exceptions.LsstException, RuntimeError):
print "skipping", dataId
return None
if ("pixelScale" in extraCols) or ("thetaNorth" in extraCols):
wcs = cal.getWcs()
availExtras["pixelScale"]["value"] = wcs.pixelScale().asArcseconds()
availExtras["thetaNorth"]["value"] = lsst.afw.geom.Angle(
np.arctan2(*tuple(wcs.getLinearTransform().invert()(lsst.afw.geom.Point2D(1.0, 0.0))))
)
if "visit" in extraCols:
availExtras["visit"]["value"] = dataId["visit"]
if "ccd" in extraCols:
availExtras["ccd"]["value"] = dataId["ccd"]
if "zeropoint" in extraCols:
zeropoint = 2.5 * np.log10(cal_md.get("FLUXMAG0"))
availExtras["zeropoint"]["value"] = zeropoint
if radecMatch is None:
fakeXY, srcIndex = getFakeMatchesHeader(cal_md, sources, tol=tol)
else:
if minRad is not None:
print "# The min matching radius is %4.1f pixel" % minRad
bbox = lsst.afw.geom.Box2D(cal.getBBox(lsst.afw.image.PARENT))
fakeXY, srcIndex, srcClose = getFakeMatchesRaDec(
sources,
radecMatch,
bbox,
cal.getWcs(),
tol=tol,
reffMatch=reffMatch,
pix=pix,
minRad=minRad,
raCol=raCol,
decCol=decCol,
)
mapper = SchemaMapper(sources.schema)
mapper.addMinimalSchema(sources.schema)
newSchema = mapper.getOutputSchema()
newSchema.addField("fakeId", type=int, doc="id of fake source matched to position")
newSchema.addField("nMatched", type=int, doc="Number of matched objects")
newSchema.addField("nPrimary", type=int, doc="Number of unique matched objects")
newSchema.addField("nNoChild", type=int, doc="Number of matched objects with nchild==0")
newSchema.addField("rMatched", type=float, doc="Radius used form atching obects, in pixel")
newSchema.addField("fakeOffX", type=float, doc="offset from input fake position in X (pixels)")
newSchema.addField("fakeOffY", type=float, doc="offset from input fake position in Y (pixels)")
newSchema.addField("fakeOffR", type=float, doc="offset from input fake position in radius")
newSchema.addField("fakeClosest", type="Flag", doc="Is this match the closest one?")
for extraName in set(extraCols).intersection(availExtras):
newSchema.addField(extraName, type=availExtras[extraName]["type"], doc=availExtras[extraName]["doc"])
srcList = SourceCatalog(newSchema)
srcList.reserve(
sum([len(s) for s in srcIndex.values()]) + (0 if not includeMissing else srcIndex.values().count([]))
)
centroidKey = sources.schema.find("centroid.sdss").getKey()
isPrimary = sources.schema.find("detect.is-primary").getKey()
nChild = sources.schema.find("force.deblend.nchild").getKey()
for ident, sindlist in srcIndex.items():
rMatched = fakeXY[ident][2]
if minRad is not None:
if rMatched < minRad:
rMatched = minRad
nMatched = len(sindlist)
nPrimary = np.sum([sources[obj].get(isPrimary) for obj in sindlist])
nNoChild = np.sum([(sources[obj].get(nChild) == 0) for obj in sindlist])
if includeMissing and (nMatched == 0):
newRec = srcList.addNew()
newRec.set("fakeId", ident)
newRec.set("id", 0)
newRec.set("nMatched", 0)
newRec.set("rMatched", rMatched)
for ss in sindlist:
newRec = srcList.addNew()
newRec.assign(sources[ss], mapper)
newRec.set("fakeId", ident)
newRec.set("nMatched", nMatched)
newRec.set("nPrimary", nPrimary)
newRec.set("nNoChild", nNoChild)
newRec.set("rMatched", rMatched)
offsetX = sources[ss].get(centroidKey).getX() - fakeXY[ident][0]
newRec.set("fakeOffX", offsetX)
offsetY = sources[ss].get(centroidKey).getY() - fakeXY[ident][1]
newRec.set("fakeOffY", offsetY)
newRec.set("fakeOffR", np.sqrt(offsetX ** 2.0 + offsetY ** 2.0))
if radecMatch:
if ss == srcClose[ident]:
newRec.set("fakeClosest", True)
else:
newRec.set("fakeClosest", False)
if includeMissing:
srcList = srcList.copy(deep=True)
for extraName in set(extraCols).intersection(availExtras):
tempCol = srcList.get(extraName)
tempCol.fill(availExtras[extraName]["value"])
return srcList
def getStars(rootdir, visit, ccd, tol):
"""Get list of sources which agree in position with fake ones with tol
"""
# Call the butler
butler = dafPersist.Butler(rootdir)
dataId = {'visit':visit, 'ccd':ccd}
tol = float(tol)
# Get the source catalog and metadata
sources = butler.get('src', dataId)
cal_md = butler.get('calexp_md', dataId)
# Get the X, Y locations of objects on the CCD
srcX, srcY = sources.getX(), sources.getY()
# Get the zeropoint
zeropoint = (2.5 * np.log10(cal_md.get("FLUXMAG0")))
# Get the parent ID
parentID = sources.get('parent')
# Check the star/galaxy separation
extendClass = sources.get('classification.extendedness')
# Get the nChild
nChild = sources.get('deblend.nchild')
# Get the aperture corrections
# apcorr = sources.get('correctfluxes.apcorr')
apcorr = sources.get('flux.sinc')
# For Stars: Get these parameters
# Get the PSF flux and its error
flux, ferr = sources.getPsfFlux(), sources.getPsfFluxErr()
# Convert them into magnitude and its error
mag, merr = 2.5*np.log10(flux), 2.5/np.log(10)*(ferr/flux)
mag = zeropoint - mag
apcorr = zeropoint - 2.5*np.log10(apcorr)
# X, Y locations of the fake stars
fakeList = collections.defaultdict(tuple)
# Regular Expression
# Search for keywords like FAKE12
fakename = re.compile('FAKE([0-9]+)')
# Go through all the keywords
counts = 0
for card in cal_md.names():
# To see if the card matches the pattern
m = fakename.match(card)
if m is not None:
# Get the X,Y location for fake object
x,y = map(float, (cal_md.get(card)).split(','))
# Get the ID or index of the fake object
fakeID = int(m.group(1))
fakeList[counts] = [fakeID, x, y]
counts += 1
# Match the fake object to the source list
srcIndex = collections.defaultdict(list)
for fid, fcoord in fakeList.items():
separation = np.sqrt(np.abs(srcX-fcoord[1])**2 +
np.abs(srcY-fcoord[2])**2)
matched = (separation <= tol)
matchId = np.where(matched)[0]
matchSp = separation[matchId]
sortId = [matchId for (matchSp, matchId) in sorted(zip(matchSp, matchId))]
# DEBUG:
# print fid, fcoord, matchId
# print sortId, sorted(matchSp), matchId
# Select the index of all matched object
srcIndex[fid] = sortId
# Return the source list
mapper = SchemaMapper(sources.schema)
mapper.addMinimalSchema(sources.schema)
newSchema = mapper.getOutputSchema()
newSchema.addField('fakeId', type=int,
doc='id of fake source matched to position')
srcList = SourceCatalog(newSchema)
srcList.reserve(sum([len(s) for s in srcIndex.values()]))
# Return a list of interesting parameters
#srcParam = collections.defaultdict(list)
srcParam = []
nFake = 0
for matchIndex in srcIndex.values():
# Check if there is a match
if len(matchIndex) > 0:
# Only select the one with the smallest separation
ss = matchIndex[0]
fakeObj = fakeList[nFake]
diffX = srcX[ss] - fakeObj[1]
diffY = srcY[ss] - fakeObj[2]
paramList = (fakeObj[0], fakeObj[1], fakeObj[2],
mag[ss], merr[ss], apcorr[ss], diffX, diffY,
parentID[ss], nChild[ss], extendClass[ss])
srcParam.append(paramList)
else:
fakeObj = fakeList[nFake]
paramList = (fakeObj[0], fakeObj[1], fakeObj[2],
0, 0, -1, -1, -1, -1, -1, -1)
srcParam.append(paramList)
# Go to another fake object
nFake += 1
# Make a numpy record array
srcParam = np.array(srcParam, dtype=[('fakeID', int),
('fakeX', float),
('fakeY', float),
('psfMag', float),
('psfMagErr', float),
('apCorr', float),
('diffX', float),
('diffY', float),
('parentID', int),
('nChild', int),
('extendClass', float)])
return srcIndex, srcParam, srcList, zeropoint
def _loadAndMatchCatalogs(self, repo, dataIds, matchRadius):
"""Load data from specific visit. Match with reference.
Parameters
----------
repo : string
The repository. This is generally the directory on disk
that contains the repository and mapper.
dataIds : list of dict
List of `butler` data IDs of Image catalogs to compare to
reference. The `calexp` cpixel image is needed for the photometric
calibration.
matchRadius : afwGeom.Angle(), optional
Radius for matching. Default is 1 arcsecond.
Returns
-------
afw.table.GroupView
An object of matched catalog.
"""
# Following
# https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
butler = dafPersist.Butler(repo)
dataset = 'src'
# 2016-02-08 MWV:
# I feel like I could be doing something more efficient with
# something along the lines of the following:
# dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds]
ccdKeyName = getCcdKeyName(dataIds[0])
schema = butler.get(dataset + "_schema", immediate=True).schema
mapper = SchemaMapper(schema)
mapper.addMinimalSchema(schema)
mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
mapper.addOutputField(Field[float]('base_PsfFlux_mag',
'PSF magnitude'))
mapper.addOutputField(Field[float]('base_PsfFlux_magerr',
'PSF magnitude uncertainty'))
newSchema = mapper.getOutputSchema()
# Create an object that matches multiple catalogs with same schema
mmatch = MultiMatch(newSchema,
dataIdFormat={
'visit': np.int32,
ccdKeyName: np.int32
},
radius=matchRadius,
RecordClass=SimpleRecord)
# create the new extented source catalog
srcVis = SourceCatalog(newSchema)
for vId in dataIds:
try:
calexpMetadata = butler.get("calexp_md", vId, immediate=True)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open calibrated image file for ", vId)
print("Skipping %s " % repr(vId))
continue
except TypeError as te:
# DECam images that haven't been properly reformatted
# can trigger a TypeError because of a residual FITS header
# LTV2 which is a float instead of the expected integer.
# This generates an error of the form:
#
# lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type'
#
# See, e.g., DM-2957 for details.
print(te)
print("Calibration image header information malformed.")
print("Skipping %s " % repr(vId))
continue
calib = afwImage.Calib(calexpMetadata)
oldSrc = butler.get('src', vId, immediate=True)
print(
len(oldSrc), "sources in ccd %s visit %s" %
(vId[ccdKeyName], vId["visit"]))
# create temporary catalog
tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
tmpCat.extend(oldSrc, mapper=mapper)
tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_flux'] \
/ tmpCat['base_PsfFlux_fluxSigma']
with afwImageUtils.CalibNoThrow():
_ = calib.getMagnitude(tmpCat['base_PsfFlux_flux'],
tmpCat['base_PsfFlux_fluxSigma'])
tmpCat['base_PsfFlux_mag'][:] = _[0]
tmpCat['base_PsfFlux_magerr'][:] = _[1]
srcVis.extend(tmpCat, False)
mmatch.add(catalog=tmpCat, dataId=vId)
# Complete the match, returning a catalog that includes
# all matched sources with object IDs that can be used to group them.
matchCat = mmatch.finish()
# Create a mapping object that allows the matches to be manipulated
# as a mapping of object ID to catalog of sources.
allMatches = GroupView.build(matchCat)
return allMatches
def _loadAndMatchCatalogs(repo,
dataIds,
matchRadius,
useJointCal=False,
skipTEx=False):
"""Load data from specific visit. Match with reference.
Parameters
----------
repo : string or Butler
A Butler or a repository URL that can be used to construct one
dataIds : list of dict
List of `butler` data IDs of Image catalogs to compare to
reference. The `calexp` cpixel image is needed for the photometric
calibration.
matchRadius : afwGeom.Angle(), optional
Radius for matching. Default is 1 arcsecond.
useJointCal : `bool`, optional
Use jointcal/meas_mosaic outputs to calibrate positions and fluxes.
skipTEx : `bool`, optional
Skip TEx calculations (useful for older catalogs that don't have
PsfShape measurements).
Returns
-------
catalog_list : afw.table.SourceCatalog
List of all of the catalogs
matched_catalog : afw.table.GroupView
An object of matched catalog.
"""
# Following
# https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
if isinstance(repo, dafPersist.Butler):
butler = repo
else:
butler = dafPersist.Butler(repo)
dataset = 'src'
# 2016-02-08 MWV:
# I feel like I could be doing something more efficient with
# something along the lines of the following:
# dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds]
ccdKeyName = getCcdKeyName(dataIds[0])
# Hack to support raft and sensor 0,1 IDs as ints for multimatch
if ccdKeyName == 'sensor':
ccdKeyName = 'raft_sensor_int'
for vId in dataIds:
vId[ccdKeyName] = raftSensorToInt(vId)
schema = butler.get(dataset + "_schema").schema
mapper = SchemaMapper(schema)
mapper.addMinimalSchema(schema)
mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude'))
mapper.addOutputField(Field[float]('base_PsfFlux_magErr',
'PSF magnitude uncertainty'))
mapper.addOutputField(Field[float]('e1', 'Source Ellipticity 1'))
mapper.addOutputField(Field[float]('e2', 'Source Ellipticity 1'))
mapper.addOutputField(Field[float]('psf_e1', 'PSF Ellipticity 1'))
mapper.addOutputField(Field[float]('psf_e2', 'PSF Ellipticity 1'))
newSchema = mapper.getOutputSchema()
newSchema.setAliasMap(schema.getAliasMap())
# Create an object that matches multiple catalogs with same schema
mmatch = MultiMatch(newSchema,
dataIdFormat={
'visit': np.int32,
ccdKeyName: np.int32
},
radius=matchRadius,
RecordClass=SimpleRecord)
# create the new extented source catalog
srcVis = SourceCatalog(newSchema)
for vId in dataIds:
if useJointCal:
try:
photoCalib = butler.get("jointcal_photoCalib", vId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open photometric calibration for ", vId)
print("Skipping this dataId.")
continue
try:
wcs = butler.get("jointcal_wcs", vId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open updated WCS for ", vId)
print("Skipping this dataId.")
continue
else:
try:
photoCalib = butler.get("calexp_photoCalib", vId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open calibrated image file for ", vId)
print("Skipping this dataId.")
continue
except TypeError as te:
# DECam images that haven't been properly reformatted
# can trigger a TypeError because of a residual FITS header
# LTV2 which is a float instead of the expected integer.
# This generates an error of the form:
#
# lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type'
#
# See, e.g., DM-2957 for details.
print(te)
print("Calibration image header information malformed.")
print("Skipping this dataId.")
continue
# We don't want to put this above the first "if useJointCal block"
# because we need to use the first `butler.get` above to quickly
# catch data IDs with no usable outputs.
try:
# HSC supports these flags, which dramatically improve I/O
# performance; support for other cameras is DM-6927.
oldSrc = butler.get('src', vId, flags=SOURCE_IO_NO_FOOTPRINTS)
except (OperationalError, sqlite3.OperationalError):
oldSrc = butler.get('src', vId)
print(len(oldSrc),
"sources in ccd %s visit %s" % (vId[ccdKeyName], vId["visit"]))
# create temporary catalog
tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
tmpCat.extend(oldSrc, mapper=mapper)
tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_instFlux'] \
/ tmpCat['base_PsfFlux_instFluxErr']
if useJointCal:
for record in tmpCat:
record.updateCoord(wcs)
photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux")
if not skipTEx:
_, psf_e1, psf_e2 = ellipticity_from_cat(
oldSrc, slot_shape='slot_PsfShape')
_, star_e1, star_e2 = ellipticity_from_cat(oldSrc,
slot_shape='slot_Shape')
tmpCat['e1'][:] = star_e1
tmpCat['e2'][:] = star_e2
tmpCat['psf_e1'][:] = psf_e1
tmpCat['psf_e2'][:] = psf_e2
srcVis.extend(tmpCat, False)
mmatch.add(catalog=tmpCat, dataId=vId)
# Complete the match, returning a catalog that includes
# all matched sources with object IDs that can be used to group them.
matchCat = mmatch.finish()
# Create a mapping object that allows the matches to be manipulated
# as a mapping of object ID to catalog of sources.
allMatches = GroupView.build(matchCat)
return srcVis, allMatches
def _loadAndMatchCatalogs(self,
repo,
dataIds,
matchRadius,
useJointCal=False):
"""Load data from specific visit. Match with reference.
Parameters
----------
repo : string or Butler
A Butler or a repository URL that can be used to construct one
dataIds : list of dict
List of `butler` data IDs of Image catalogs to compare to
reference. The `calexp` cpixel image is needed for the photometric
calibration.
matchRadius : afwGeom.Angle(), optional
Radius for matching. Default is 1 arcsecond.
Returns
-------
afw.table.GroupView
An object of matched catalog.
"""
# Following
# https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
if isinstance(repo, dafPersist.Butler):
butler = repo
else:
butler = dafPersist.Butler(repo)
dataset = 'src'
# 2016-02-08 MWV:
# I feel like I could be doing something more efficient with
# something along the lines of the following:
# dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds]
ccdKeyName = getCcdKeyName(dataIds[0])
schema = butler.get(dataset + "_schema").schema
mapper = SchemaMapper(schema)
mapper.addMinimalSchema(schema)
mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
mapper.addOutputField(Field[float]('base_PsfFlux_mag',
'PSF magnitude'))
mapper.addOutputField(Field[float]('base_PsfFlux_magErr',
'PSF magnitude uncertainty'))
newSchema = mapper.getOutputSchema()
newSchema.setAliasMap(schema.getAliasMap())
# Create an object that matches multiple catalogs with same schema
mmatch = MultiMatch(newSchema,
dataIdFormat={
'visit': np.int32,
ccdKeyName: np.int32
},
radius=matchRadius,
RecordClass=SimpleRecord)
# create the new extented source catalog
srcVis = SourceCatalog(newSchema)
for vId in dataIds:
if useJointCal:
try:
photoCalib = butler.get("photoCalib", vId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open photometric calibration for ", vId)
print("Skipping %s " % repr(vId))
continue
try:
md = butler.get("wcs_md", vId)
wcs = afwImage.makeWcs(md)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open updated WCS for ", vId)
print("Skipping %s " % repr(vId))
continue
else:
try:
calexpMetadata = butler.get("calexp_md", vId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open calibrated image file for ", vId)
print("Skipping %s " % repr(vId))
continue
except TypeError as te:
# DECam images that haven't been properly reformatted
# can trigger a TypeError because of a residual FITS header
# LTV2 which is a float instead of the expected integer.
# This generates an error of the form:
#
# lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type'
#
# See, e.g., DM-2957 for details.
print(te)
print("Calibration image header information malformed.")
print("Skipping %s " % repr(vId))
continue
calib = afwImage.Calib(calexpMetadata)
# We don't want to put this above the first "if useJointCal block"
# because we need to use the first `butler.get` above to quickly
# catch data IDs with no usable outputs.
try:
# HSC supports these flags, which dramatically improve I/O
# performance; support for other cameras is DM-6927.
oldSrc = butler.get('src', vId, flags=SOURCE_IO_NO_FOOTPRINTS)
except:
oldSrc = butler.get('src', vId)
print(
len(oldSrc), "sources in ccd %s visit %s" %
(vId[ccdKeyName], vId["visit"]))
# create temporary catalog
tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
tmpCat.extend(oldSrc, mapper=mapper)
tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_flux'] \
/ tmpCat['base_PsfFlux_fluxSigma']
if useJointCal:
for record in tmpCat:
record.updateCoord(wcs)
photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux",
"base_PsfFlux")
else:
with afwImageUtils.CalibNoThrow():
_ = calib.getMagnitude(tmpCat['base_PsfFlux_flux'],
tmpCat['base_PsfFlux_fluxSigma'])
tmpCat['base_PsfFlux_mag'][:] = _[0]
tmpCat['base_PsfFlux_magErr'][:] = _[1]
srcVis.extend(tmpCat, False)
mmatch.add(catalog=tmpCat, dataId=vId)
# Complete the match, returning a catalog that includes
# all matched sources with object IDs that can be used to group them.
matchCat = mmatch.finish()
# Create a mapping object that allows the matches to be manipulated
# as a mapping of object ID to catalog of sources.
allMatches = GroupView.build(matchCat)
return allMatches
class ProcessCoaddsMetacalMaxTask(ProcessCoaddsNGMixBaseTask):
_DefaultName = "processCoaddsMetacalMax"
ConfigClass = ProcessCoaddsMetacalMaxConfig
def defineSchema(self, refSchema):
"""Return the Schema for the output catalog.
This may add or modify self.
Parameters
----------
refSchema : `lsst.afw.table.Schema`
Schema of the input reference catalogs.
Returns
-------
outputSchema : `lsst.afw.table.Schema`
Schema of the output catalog. Will be added as ``self.schema``
by calling code.
"""
self.mapper = SchemaMapper(refSchema)
self.mapper.addMinimalSchema(SourceCatalog.Table.makeMinimalSchema(), True)
schema = self.mapper.getOutputSchema()
config=self.cdict
model=config['obj']['model']
n=self.get_namer()
pn=self.get_psf_namer()
# generic ngmix fields
mtypes=[
(n('flags'),'overall flags for the processing',np.int32,''),
(n('stamp_size'),'size of postage stamp',np.int32,''),
(n('maskfrac'),'mean masked fraction',np.float32,''),
]
for filt in config['filters']:
mtypes += [
(n('maskfrac_%s' % filt),'masked fraction in %s filter' % filt,np.float32,''),
]
# psf fitting related fields
mtypes += [
(pn('flags'),'overall flags for the PSF processing',np.int32,''),
# mean over filters
(pn('g2_mean'),'mean over filters of component 2 of the PSF ellipticity',np.float64,''),
(pn('g1_mean'),'mean over filters of component 2 of the PSF ellipticity',np.float64,''),
(pn('T_mean'),'mean over filters <x^2> + <y^2> for the gaussian mixture',np.float64,'arcsec^2'),
]
# PSF measurements by filter
for filt in config['filters']:
pfn=self.get_psf_namer(filt=filt)
mtypes += [
(pfn('flags'), 'overall flags for PSF processing in %s filter' % filt, np.int32, ''),
(pfn('row'),'offset from canonical row position',np.float64,'arcsec'),
(pfn('col'),'offset from canonical col position',np.float64,'arcsec'),
(pfn('g1'), 'component 1 of the PSF ellipticity in %s filter' % filt, np.float64, ''),
(pfn('g2'), 'component 2 of the PSF ellipticity in %s filter' % filt, np.float64, ''),
(pfn('T'), '<x^2> + <y^2> for the PSF in %s filter' % filt, np.float64, 'arcsec^2'),
]
# PSF flux measurements, on the object, by filter
#for filt in config['filters']:
# pfn=self.get_psf_flux_namer(filt)
# mtypes += [
# (pfn('flux_flags'),'flags for PSF template flux fitting in the %s filter' % filt,np.float64,''),
# (pfn('flux'),'PSF template flux in the %s filter' % filt,np.float64,''),
# (pfn('flux_err'),'error on PSF template flux in the %s filter' % filt,np.float64,''),
# ]
# object fitting related fields
for type in config['metacal']['types']:
mn=self.get_model_namer(type=type)
mtypes += [
(mn('flags'),'flags for model fit',np.int32,''),
(mn('nfev'),'number of function evaluations during fit',np.int32,''),
(mn('chi2per'),'chi^2 per degree of freedom',np.float64,''),
(mn('dof'),'number of degrees of freedom',np.int32,''),
(mn('s2n'),'S/N for the fit',np.float64,''),
(mn('row'),'offset from canonical row position',np.float64,'arcsec'),
(mn('row_err'),'error on offset from canonical row position',np.float64,'arcsec'),
(mn('col'),'offset from canonical col position',np.float64,'arcsec'),
(mn('col_err'),'error on offset from canonical col position',np.float64,'arcsec'),
(mn('g1'),'component 1 of the ellipticity',np.float64,''),
(mn('g1_err'),'error on component 1 of the ellipticity',np.float64,''),
(mn('g2'),'component 2 of the ellipticity',np.float64,''),
(mn('g2_err'),'error on component 2 of the ellipticity',np.float64,''),
(mn('T'),'<x^2> + <y^2> for the gaussian mixture',np.float64,'arcsec^2'),
(mn('T_err'),'error on <x^2> + <y^2> for the gaussian mixture',np.float64,'arcsec^2'),
]
if model in ['bd','bdf']:
mtypes += [
(mn('fracdev'),'fraction of light in the bulge',np.float64,''),
(mn('fracdev_err'),'error on fraction of light in the bulge',np.float64,''),
]
for filt in config['filters']:
mfn=self.get_model_flux_namer(filt, type=type)
mtypes += [
(mfn('flux'),'flux in the %s filter' % filt,np.float64,''),
(mfn('flux_err'),'error on flux in the %s filter' % filt,np.float64,''),
]
for name,doc,dtype,units in mtypes:
schema.addField(
name,
type=dtype,
doc=doc,
units=units,
)
return schema
def _process_observations(self, id, mbobs):
"""
process the input observations
Parameters
----------
mbobs: ngmix.MultiBandObsList
ngmix multi-band observation. we may loosen this to be alist
of them, for deblending
Returns
-------
id: int
ID of this observation
results : `dict`
Dictionary of outputs, with keys matching the fields added in
`defineSchema()`.
"""
if self.cdict['make_plots']:
self._make_plots(id, mbobs)
# start with a default result. may not use if we get to
# measurements
maskfrac, maskfrac_byband = self._get_masked_fraction(mbobs)
flags = self._check_obs(mbobs, maskfrac_byband)
if flags != 0:
res=self._get_default_result()
res['maskfrac'], res['maskfrac_byband'] = maskfrac, maskfrac_byband
res['flags'] = flags
return res
boot=self._get_bootstrapper(mbobs)
boot.go()
res=boot.result
res['maskfrac'], res['maskfrac_byband'] = maskfrac, maskfrac_byband
return res
def _get_bootstrapper(self, mbobs):
"""
get a bootstrapper to automate the processing
"""
return bootstrap.MetacalMaxBootstrapper(
mbobs,
self.cdict,
self.prior,
self.rng,
)
def _get_default_result(self):
"""
get the default result dict
"""
return bootstrap.get_default_mcal_result()
def _copy_result(self, mbobs, res, output):
"""
copy the result dict to the output record
"""
n=self.get_namer()
stamp_shape = mbobs[0][0].image.shape
stamp_size=stamp_shape[0]
output[n('flags')] = res['mcal_flags']
output[n('stamp_size')] = stamp_size
output[n('maskfrac')] = res['maskfrac']
for ifilt,filt in enumerate(self.cdict['filters']):
output[n('maskfrac_%s' % filt)] = res['maskfrac_byband'][ifilt]
self._copy_psf_fit_result(res['noshear']['psf'], output)
self._copy_psf_fit_results_byband(res['noshear']['psf'], output)
#self._copy_psf_flux_results_byband(res['psf_flux'], output)
self._copy_model_result(res, output)
def _copy_psf_fit_result(self, pres, output):
"""
copy the PSF result dict to the output record.
The statistics here are averaged over all bands
"""
n=self.get_psf_namer()
output[n('flags')] = pres['flags']
if pres['flags'] == 0:
output[n('g1_mean')] = pres['g1_mean']
output[n('g2_mean')] = pres['g2_mean']
output[n('T_mean')] = pres['T_mean']
def _copy_psf_fit_results_byband(self, pres, output):
"""
copy the PSF result from each band to the output record.
"""
if len(pres['byband'])==0:
return
config=self.cdict
for ifilt,filt in enumerate(config['filters']):
filt_res = pres['byband'][ifilt]
if filt_res is not None:
n=self.get_psf_namer(filt=filt)
output[n('flags')] = filt_res['flags']
if filt_res['flags'] == 0:
output[n('row')] = filt_res['pars'][0]
output[n('col')] = filt_res['pars'][1]
if filt_res['flags']==0:
for name in ['g1','g2','T']:
output[n(name)] = filt_res[name]
def _copy_psf_flux_results_byband(self, pres, output):
"""
copy the PSF flux fitting results from each band to the output record.
"""
if len(pres['byband'])==0:
return
config=self.cdict
for ifilt,filt in enumerate(config['filters']):
filt_res = pres['byband'][ifilt]
if filt_res is not None:
n=self.get_psf_flux_namer(filt=filt)
output[n('flux_flags')] = filt_res['flags']
if filt_res['flags']==0:
output[n('flux')] = filt_res['flux']
output[n('flux_err')] = filt_res['flux_err']
def _copy_model_result(self, res, output):
"""
copy the model fitting result dict to the output record.
"""
config=self.cdict
types=config['metacal']['types']
for type in types:
ores=res[type]['obj']
mn=self.get_model_namer(type=type)
output[mn('flags')] = ores['flags']
if 'nfev' in ores:
# can be there even if the fit failed, but won't be there
# if it wasn't attempted
output[mn('nfev')] = ores['nfev']
if ores['flags']==0:
for n in ['chi2per','dof','s2n']:
output[mn(n)] = ores[n]
ni=[('row',0),('col',1),('g1',2),('g2',3),('T',4)]
if self.cdict['obj']['model'] in ['bd','bdf']:
ni += [('fracdev',5)]
flux_start=6
else:
flux_start=5
pars=ores['pars']
perr=ores['pars_err']
for n,i in ni:
output[mn(n)] = pars[i]
output[mn(n+'_err')] = perr[i]
for ifilt, filt in enumerate(config['filters']):
ind=flux_start+ifilt
mfn=self.get_model_flux_namer(filt, type=type)
output[mfn('flux')] = pars[ind]
output[mfn('flux_err')] = perr[ind]
def get_namer(self, type=None):
"""
get a namer for this output type
"""
front='mcal'
back=None
if type is not None:
if type=='noshear':
back=None
else:
back=type
return Namer(front='mcal', back=back)
def get_psf_namer(self, filt=None):
"""
get a namer for this output type
"""
front='mcal_psf'
if filt is not None:
front='%s_%s' % (front,filt)
return Namer(front=front)
def get_model_namer(self, type=None):
"""
get a namer for this output type
"""
config=self.cdict
model=config['obj']['model']
front='mcal_%s' % model
if type is not None:
if type=='noshear':
back=None
else:
back=type
return Namer(front=front, back=back)
def get_model_flux_namer(self, filt, type=None):
"""
get a namer for this output type
"""
config=self.cdict
model=config['obj']['model']
front='mcal_%s' % model
back=filt
if type is not None:
if type!='noshear':
back='%s_%s' % (back, type)
return Namer(front=front, back=back)
def get_psf_flux_namer(self, filt):
"""
get a namer for this output type
"""
raise NotImplementedError('make work for metacal')
front='mcal_psf'
return Namer(front=front, back=filt)
@property
def prior(self):
"""
set the joint prior used for object fitting
"""
if not hasattr(self, '_prior'):
# this is temporary until I can figure out how to get
# an existing seeded rng
conf=self.cdict
nband=len(conf['filters'])
model=conf['obj']['model']
self._prior = priors.get_joint_prior(
conf['obj'],
nband,
self.rng,
)
return self._prior
def getFakeSources(butler, dataId, tol=1.0, extraCols=('zeropoint', 'visit', 'ccd'),
includeMissing=False, footprints=False, radecMatch=None):
"""Get list of sources which agree in pixel position with fake ones with tol
this returns a sourceCatalog of all the matched fake objects,
note, there will be duplicates in this list, since I haven't checked deblend.nchild,
and I'm only doing a tolerance match, which could include extra sources
the outputs can include extraCols as long as they are one of:
zeropoint, visit, ccd, thetaNorth, pixelScale
if includeMissing is true, then the pipeline looks at the fake sources
added in the header and includes an entry in the table for sources without
any measurements, specifically the 'id' column will be 0
radecMatch is the fakes table. if it's not None(default), then do an ra/dec
match with the input catalog instead of looking in the header for where the
sources where added
"""
availExtras = {'zeropoint':{'type':float, 'doc':'zeropoint'},
'visit':{'type':int, 'doc':'visit id'},
'ccd':{'type':int, 'doc':'ccd id'},
'thetaNorth':{'type':lsst.afw.geom.Angle, 'doc':'angle to north'},
'pixelScale':{'type':float, 'doc':'pixelscale in arcsec/pixel'}}
if not np.in1d(extraCols, availExtras.keys()).all():
print "extraCols must be in ",availExtras
try:
if not 'filter' in dataId:
sources = butler.get('src', dataId,
flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
cal = butler.get('calexp', dataId, immediate=True)
cal_md = butler.get('calexp_md', dataId, immediate=True)
else:
sources = butler.get('deepCoadd_src', dataId,
flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
cal = butler.get('deepCoadd', dataId, immediate=True)
cal_md = butler.get('deepCoadd_md', dataId, immediate=True)
except (lsst.pex.exceptions.LsstException, RuntimeError) as e:
print "skipping", dataId
return None
if ('pixelScale' in extraCols) or ('thetaNorth' in extraCols):
wcs = cal.getWcs()
availExtras['pixelScale']['value'] = wcs.pixelScale().asArcseconds()
availExtras['thetaNorth']['value'] = lsst.afw.geom.Angle(
np.arctan2(*tuple(wcs.getLinearTransform().invert()
(lsst.afw.geom.Point2D(1.0,0.0)))))
if 'visit' in extraCols:
availExtras['visit']['value'] = dataId['visit']
if 'ccd' in extraCols:
availExtras['ccd']['value'] = dataId['ccd']
if 'zeropoint' in extraCols:
availExtras['zeropoint']['value'] = 2.5*np.log10(cal_md.get('FLUXMAG0'))
if radecMatch is None:
fakeXY, srcIndex = getFakeMatchesHeader(cal_md, sources, tol=tol)
else:
fakeXY, srcIndex = getFakeMatchesRaDec(sources, radecMatch,
lsst.afw.geom.Box2D(cal.getBBox(lsst.afw.image.PARENT)),
cal.getWcs(),
tol=tol)
mapper = SchemaMapper(sources.schema)
mapper.addMinimalSchema(sources.schema)
newSchema = mapper.getOutputSchema()
newSchema.addField('fakeId', type=int, doc='id of fake source matched to position')
newSchema.addField('fakeOffset', type=lsst.afw.geom.Point2D,
doc='offset from input fake position (pixels)')
for extraName in set(extraCols).intersection(availExtras):
newSchema.addField(extraName, type=availExtras[extraName]['type'],
doc=availExtras[extraName]['doc'])
srcList = SourceCatalog(newSchema)
srcList.reserve(sum([len(s) for s in srcIndex.values()]) +
(0 if not includeMissing else srcIndex.values().count([])))
centroidKey = sources.schema.find('centroid.sdss').getKey()
for ident, sindlist in srcIndex.items():
if includeMissing and (len(sindlist)==0):
newRec = srcList.addNew()
newRec.set('fakeId', ident)
newRec.set('id', 0)
for ss in sindlist:
newRec = srcList.addNew()
newRec.assign(sources[ss], mapper)
newRec.set('fakeId', ident)
newRec.set('fakeOffset',
lsst.afw.geom.Point2D(sources[ss].get(centroidKey).getX() -
fakeXY[ident][0],
sources[ss].get(centroidKey).getY() -
fakeXY[ident][1]))
if includeMissing:
srcList = srcList.copy(deep=True)
for extraName in set(extraCols).intersection(availExtras):
tempCol = srcList.get(extraName)
tempCol.fill(availExtras[extraName]['value'])
return srcList
class MeasureCoaddsBfdSingleTask(ProcessCoaddsTogetherTask):
"""
Base class for ngmix tasks
"""
_DefaultName = "MeasureCoaddsBfdSingleTask"
ConfigClass = MeasureCoaddsBfdSingleConfig
def __init__(self,
*,
config=None,
refSchema=None,
butler=None,
initInputs=None,
**kwds):
ProcessCoaddsTogetherTask.__init__(self,
config=config,
refSchema=refSchema,
butler=butler,
initInputs=initInputs,
**kwds)
if refSchema is None:
if butler is None:
if initInputs is not None:
refSchema = initInputs.get("refSchema", None)
if refSchema is None:
refSchema = SourceCatalog.Table.makeMinimalSchema()
else:
refSchema = butler.get(self.config.ref.name + "_schema").schema
self.ncolors = 0
self.bfd = BFDConfig(use_conc=self.config.use_conc,
use_mag=self.config.use_mag,
ncolors=self.ncolors)
self.n_even = self.bfd.BFDConfig.MSIZE
self.n_odd = self.bfd.BFDConfig.XYSIZE
self.weight = KSigmaWeightF(self.config.weight_sigma,
self.config.weight_n)
self.schema = self.defineSchema(refSchema)
def defineSchema(self, refSchema):
self.mapper = SchemaMapper(refSchema)
self.mapper.addMinimalSchema(SourceCatalog.Table.makeMinimalSchema(),
True)
schema = self.mapper.getOutputSchema()
self.even = schema.addField('bfd_even',
type="ArrayF",
size=self.n_even,
doc="Even Bfd moments")
self.odd = schema.addField('bfd_odd',
type="ArrayF",
size=self.n_odd,
doc="odd moments")
self.shift = schema.addField('bfd_shift',
type="ArrayF",
size=2,
doc="amount shifted to null moments")
self.cov_even = schema.addField('bfd_cov_even',
type="ArrayF",
size=self.n_even * (self.n_even + 1) //
2,
doc="even moment covariance matrix")
self.cov_odd = schema.addField('bfd_cov_odd',
type="ArrayF",
size=self.n_odd * (self.n_odd + 1) // 2,
doc="odd moment covariance matrix")
self.flag = schema.addField('bfd_flag',
type="Flag",
doc="Set to 1 for any fatal failure")
self.centroid_flag = schema.addField(
'bfd_flag_centroid',
type="Flag",
doc="Set to 1 for any fatal failure of centroid")
self.parent_flag = schema.addField('bfd_flag_parent',
type="Flag",
doc="Set to 1 for parents")
return schema
def runDataRef(self, patchRefList):
"""Run this task via CmdLineTask and Gen2 Butler.
Parameters
----------
patchRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
A list of DataRefs for all filters in a single patch.
"""
#import pdb;pdb.set_trace()
images = {}
replacers = {}
mergedDataId = {
"tract": patchRefList[0].dataId["tract"],
"patch": patchRefList[0].dataId["patch"]
}
butler = patchRefList[0].butlerSubset.butler
ref = butler.get("deepCoadd_ref", dataId=mergedDataId)
imageId = butler.get("deepMergedCoaddId", dataId=mergedDataId)
for patchRef in patchRefList:
filt = getShortFilterName(patchRef.dataId["filter"])
images[filt] = patchRef.get(self.config.images.name)
fpCat = patchRef.get(self.config.deblendCatalog.name)
footprints = {
rec.getId(): (rec.getParent(), rec.getFootprint())
for rec in fpCat
}
replacers[filt] = NoiseReplacer(self.config.deblendReplacer,
exposure=images[filt],
footprints=footprints,
exposureId=imageId)
results = self.run(images, ref, imageId=imageId, replacers=replacers)
mergedDataId['filter'] = self.config.filters[0]
butler.put(results.output,
self.config.output.name,
dataId=mergedDataId)
def run(self, images, ref, replacers, imageId):
"""Process coadds from all bands for a single patch.
This method should not add or modify self.
So far all children are using this exact code so leaving
it here for now. If we specialize a lot, might make a
processor its own object
Parameters
----------
images : `dict` of `lsst.afw.image.ExposureF`
Coadd images and associated metadata, keyed by filter name.
ref : `lsst.afw.table.SourceCatalog`
A catalog with one record for each object, containing "best"
measurements across all bands.
replacers : `dict` of `lsst.meas.base.NoiseReplacer`, optional
A dictionary of `~lsst.meas.base.NoiseReplacer` objects that can
be used to insert and remove deblended pixels for each object.
When not `None`, all detected pixels in ``images`` will have
*already* been replaced with noise, and this *must* be used
to restore objects one at a time.
imageId : `int`
Unique ID for this unit of data. Should be used (possibly
indirectly) to seed random numbers.
Returns
-------
results : `lsst.pipe.base.Struct`
Struct with (at least) an `output` attribute that is a catalog
to be written as ``self.config.output``.
"""
if len(images) != len(self.config.filters):
self.log.info(
'Number of filters does not match the list of images given. Skipping'
)
return None
tm0 = time.time()
nproc = 0
#import pdb;pdb.set_trace()
# Make an empty catalog
output = SourceCatalog(self.schema)
# Add mostly-empty rows to it, copying IDs from the ref catalog.
output.extend(ref, mapper=self.mapper)
min_index = self.config.start_index
if self.config.num_to_process is None:
max_index = len(ref)
else:
max_index = self.config.start_index + self.config.num_to_process
#for n, (refRecord) in enumerate(zip(ref)):
for n, (refRecord, outRecord) in enumerate(zip(ref, output)):
if n < min_index or n >= max_index:
continue
if refRecord.get('deblend_nChild') != 0:
outRecord.set(self.flag, 1)
outRecord.set(self.parent_flag, 1)
continue
#outRecord = output.table.copyRecord(refRecord, self.mapper)
#output._append(outRecord)
self.log.info('index: %06d/%06d' % (n, max_index))
nproc += 1
outRecord.setFootprint(
None) # copied from ref; don't need to write these again
# Insert the deblended pixels for just this object into all images.
for r in replacers.values():
r.insertSource(refRecord.getId())
try:
kgals = self.buildKGalaxy(refRecord, images)
kc = KColorGalaxy(self.bfd, kgals)
except Exception as e:
kc = None
if kc is None:
outRecord.set(self.flag, 1)
continue
dx, badcentering, msg = kc.recenter(self.config.weight_sigma)
if badcentering:
self.log.info('Bad centering %s', msg)
outRecord.set(self.flag, 1)
outRecord.set(self.centroid_flag, 1)
dx = [0, 0]
mom, cov = kc.get_moment(dx[0], dx[1], True)
mom_even = mom.m
mom_odd = mom.xy
cov_even = cov.m
cov_odd = cov.xy
cov_even_save = []
cov_odd_save = []
for ii in range(cov_even.shape[0]):
cov_even_save.extend(cov_even[ii][ii:])
for ii in range(cov_odd.shape[0]):
cov_odd_save.extend(cov_odd[ii][ii:])
outRecord.set(self.even, np.array(mom_even, dtype=np.float32))
outRecord.set(self.odd, np.array(mom_odd, dtype=np.float32))
outRecord.set(self.cov_even,
np.array(cov_even_save, dtype=np.float32))
outRecord.set(self.cov_odd, np.array(cov_odd_save,
dtype=np.float32))
outRecord.set(self.shift, np.array([dx[0], dx[1]],
dtype=np.float32))
# Remove the deblended pixels for this object so we can process the next one.
for r in replacers.values():
r.removeSource(refRecord.getId())
del kgals
del kc
del mom
del cov
# Restore all original pixels in the images.
if replacers is not None:
for r in replacers.values():
r.end()
tm = time.time() - tm0
self.log.info('time: %g min' % (tm / 60.0))
self.log.info('time per: %g sec' % (tm / nproc))
return Struct(output=output[min_index:max_index])
def buildKGalaxy(self, record, exposures):
center = record.getCentroid()
band = self.config.filters[0]
local_lin_wcs = exposures[band].getWcs().linearizePixelToSky(
center, afwGeom.arcseconds)
jacobian = local_lin_wcs.getLinear().getMatrix()
sky_pos = exposures[band].getWcs().pixelToSky(center)
uvref = (sky_pos.getRa().asArcseconds(),
sky_pos.getDec().asArcseconds())
box = record.getFootprint().getBBox()
xy_pos = (center.getX() - box.getMinX(), center.getY() - box.getMinY())
bfd_wcs = bfd.WCS(jacobian, xyref=xy_pos, uvref=uvref)
kgals = []
for band in self.config.filters:
exposure = exposures[band]
factor = exposure.getMetadata().get('variance_scale')
noise = np.sqrt(np.median(exposure.variance[box].array) / factor)
image = exposure.image[box].array
psf_image = exposure.getPsf().computeKernelImage(center).array
kdata = bfd.generalImage(image,
uvref,
psf_image,
wcs=bfd_wcs,
pixel_noise=noise,
size=self.config.grid_size)
conjugate = set(np.where(kdata.conjugate.flatten() == False)[0])
kgal = self.bfd.KGalaxy(self.weight, kdata.kval.flatten(),
kdata.kx.flatten(), kdata.ky.flatten(),
kdata.kvar.flatten(), kdata.d2k, conjugate)
kgals.append(kgal)
return kgals
def selection(self, ref):
childName = 'deblend_nChild'
if ref.getParent() == 0 and ref.get(childName) > 0:
return False
return True
def _loadAndMatchCatalogs(repo,
dataIds,
matchRadius,
doApplyExternalPhotoCalib=False,
externalPhotoCalibName=None,
doApplyExternalSkyWcs=False,
externalSkyWcsName=None,
skipTEx=False,
skipNonSrd=False):
"""Load data from specific visits and returned a calibrated catalog matched
with a reference.
Parameters
----------
repo : `str` or `lsst.daf.persistence.Butler`
A Butler or a repository URL that can be used to construct one.
dataIds : list of dict
List of butler data IDs of Image catalogs to compare to
reference. The calexp cpixel image is needed for the photometric
calibration.
matchRadius : `lsst.geom.Angle`, optional
Radius for matching. Default is 1 arcsecond.
doApplyExternalPhotoCalib : bool, optional
Apply external photoCalib to calibrate fluxes.
externalPhotoCalibName : str, optional
Type of external `PhotoCalib` to apply. Currently supported are jointcal,
fgcm, and fgcm_tract. Must be set if doApplyExternalPhotoCalib is True.
doApplyExternalSkyWcs : bool, optional
Apply external wcs to calibrate positions.
externalSkyWcsName : str, optional
Type of external `wcs` to apply. Currently supported is jointcal.
Must be set if "doApplyExternalWcs" is True.
skipTEx : `bool`, optional
Skip TEx calculations (useful for older catalogs that don't have
PsfShape measurements).
skipNonSrd : `bool`, optional
Skip any metrics not defined in the LSST SRD; default False.
Returns
-------
catalog : `lsst.afw.table.SourceCatalog`
A new calibrated SourceCatalog.
matches : `lsst.afw.table.GroupView`
A GroupView of the matched sources.
Raises
------
RuntimeError:
Raised if "doApplyExternalPhotoCalib" is True and "externalPhotoCalibName"
is None, or if "doApplyExternalSkyWcs" is True and "externalSkyWcsName" is
None.
"""
if doApplyExternalPhotoCalib and externalPhotoCalibName is None:
raise RuntimeError(
"Must set externalPhotoCalibName if doApplyExternalPhotoCalib is True."
)
if doApplyExternalSkyWcs and externalSkyWcsName is None:
raise RuntimeError(
"Must set externalSkyWcsName if doApplyExternalSkyWcs is True.")
# Following
# https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
if isinstance(repo, dafPersist.Butler):
butler = repo
else:
butler = dafPersist.Butler(repo)
dataset = 'src'
# 2016-02-08 MWV:
# I feel like I could be doing something more efficient with
# something along the lines of the following:
# dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds]
ccdKeyName = getCcdKeyName(dataIds[0])
# Hack to support raft and sensor 0,1 IDs as ints for multimatch
if ccdKeyName == 'sensor':
ccdKeyName = 'raft_sensor_int'
for vId in dataIds:
vId[ccdKeyName] = raftSensorToInt(vId)
schema = butler.get(dataset + "_schema").schema
mapper = SchemaMapper(schema)
mapper.addMinimalSchema(schema)
mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude'))
mapper.addOutputField(Field[float]('base_PsfFlux_magErr',
'PSF magnitude uncertainty'))
if not skipNonSrd:
# Needed because addOutputField(... 'slot_ModelFlux_mag') will add a field with that literal name
aliasMap = schema.getAliasMap()
# Possibly not needed since base_GaussianFlux is the default, but this ought to be safe
modelName = aliasMap[
'slot_ModelFlux'] if 'slot_ModelFlux' in aliasMap.keys(
) else 'base_GaussianFlux'
mapper.addOutputField(Field[float](f'{modelName}_mag',
'Model magnitude'))
mapper.addOutputField(Field[float](f'{modelName}_magErr',
'Model magnitude uncertainty'))
mapper.addOutputField(Field[float](f'{modelName}_snr',
'Model flux snr'))
mapper.addOutputField(Field[float]('e1', 'Source Ellipticity 1'))
mapper.addOutputField(Field[float]('e2', 'Source Ellipticity 1'))
mapper.addOutputField(Field[float]('psf_e1', 'PSF Ellipticity 1'))
mapper.addOutputField(Field[float]('psf_e2', 'PSF Ellipticity 1'))
newSchema = mapper.getOutputSchema()
newSchema.setAliasMap(schema.getAliasMap())
# Create an object that matches multiple catalogs with same schema
mmatch = MultiMatch(newSchema,
dataIdFormat={
'visit': np.int32,
ccdKeyName: np.int32
},
radius=matchRadius,
RecordClass=SimpleRecord)
# create the new extented source catalog
srcVis = SourceCatalog(newSchema)
for vId in dataIds:
if not butler.datasetExists('src', vId):
print(f'Could not find source catalog for {vId}; skipping.')
continue
photoCalib = _loadPhotoCalib(butler, vId, doApplyExternalPhotoCalib,
externalPhotoCalibName)
if photoCalib is None:
continue
if doApplyExternalSkyWcs:
wcs = _loadExternalSkyWcs(butler, vId, externalSkyWcsName)
if wcs is None:
continue
# We don't want to put this above the first _loadPhotoCalib call
# because we need to use the first `butler.get` in there to quickly
# catch dataIDs with no usable outputs.
try:
# HSC supports these flags, which dramatically improve I/O
# performance; support for other cameras is DM-6927.
oldSrc = butler.get('src', vId, flags=SOURCE_IO_NO_FOOTPRINTS)
except (OperationalError, sqlite3.OperationalError):
oldSrc = butler.get('src', vId)
print(len(oldSrc),
"sources in ccd %s visit %s" % (vId[ccdKeyName], vId["visit"]))
# create temporary catalog
tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
tmpCat.extend(oldSrc, mapper=mapper)
tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_instFlux'] \
/ tmpCat['base_PsfFlux_instFluxErr']
if doApplyExternalSkyWcs:
afwTable.updateSourceCoords(wcs, tmpCat)
photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux")
if not skipNonSrd:
tmpCat['slot_ModelFlux_snr'][:] = (
tmpCat['slot_ModelFlux_instFlux'] /
tmpCat['slot_ModelFlux_instFluxErr'])
photoCalib.instFluxToMagnitude(tmpCat, "slot_ModelFlux",
"slot_ModelFlux")
if not skipTEx:
_, psf_e1, psf_e2 = ellipticity_from_cat(
oldSrc, slot_shape='slot_PsfShape')
_, star_e1, star_e2 = ellipticity_from_cat(oldSrc,
slot_shape='slot_Shape')
tmpCat['e1'][:] = star_e1
tmpCat['e2'][:] = star_e2
tmpCat['psf_e1'][:] = psf_e1
tmpCat['psf_e2'][:] = psf_e2
srcVis.extend(tmpCat, False)
mmatch.add(catalog=tmpCat, dataId=vId)
# Complete the match, returning a catalog that includes
# all matched sources with object IDs that can be used to group them.
matchCat = mmatch.finish()
# Create a mapping object that allows the matches to be manipulated
# as a mapping of object ID to catalog of sources.
allMatches = GroupView.build(matchCat)
return srcVis, allMatches
def match_catalogs(inputs,
photoCalibs,
astromCalibs,
vIds,
matchRadius,
apply_external_wcs=False,
logger=None):
schema = inputs[0].schema
mapper = SchemaMapper(schema)
mapper.addMinimalSchema(schema)
mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude'))
mapper.addOutputField(Field[float]('base_PsfFlux_magErr',
'PSF magnitude uncertainty'))
# Needed because addOutputField(... 'slot_ModelFlux_mag') will add a field with that literal name
aliasMap = schema.getAliasMap()
# Possibly not needed since base_GaussianFlux is the default, but this ought to be safe
modelName = aliasMap['slot_ModelFlux'] if 'slot_ModelFlux' in aliasMap.keys(
) else 'base_GaussianFlux'
mapper.addOutputField(Field[float](f'{modelName}_mag', 'Model magnitude'))
mapper.addOutputField(Field[float](f'{modelName}_magErr',
'Model magnitude uncertainty'))
mapper.addOutputField(Field[float](f'{modelName}_snr', 'Model flux snr'))
mapper.addOutputField(Field[float]('e1', 'Source Ellipticity 1'))
mapper.addOutputField(Field[float]('e2', 'Source Ellipticity 1'))
mapper.addOutputField(Field[float]('psf_e1', 'PSF Ellipticity 1'))
mapper.addOutputField(Field[float]('psf_e2', 'PSF Ellipticity 1'))
mapper.addOutputField(Field[np.int32]('filt', 'filter code'))
newSchema = mapper.getOutputSchema()
newSchema.setAliasMap(schema.getAliasMap())
# Create an object that matches multiple catalogs with same schema
mmatch = MultiMatch(newSchema,
dataIdFormat={
'visit': np.int32,
'detector': np.int32
},
radius=matchRadius,
RecordClass=SimpleRecord)
# create the new extended source catalog
srcVis = SourceCatalog(newSchema)
filter_dict = {
'u': 1,
'g': 2,
'r': 3,
'i': 4,
'z': 5,
'y': 6,
'HSC-U': 1,
'HSC-G': 2,
'HSC-R': 3,
'HSC-I': 4,
'HSC-Z': 5,
'HSC-Y': 6
}
# Sort by visit, detector, then filter
vislist = [v['visit'] for v in vIds]
ccdlist = [v['detector'] for v in vIds]
filtlist = [v['band'] for v in vIds]
tab_vids = Table([vislist, ccdlist, filtlist],
names=['vis', 'ccd', 'filt'])
sortinds = np.argsort(tab_vids, order=('vis', 'ccd', 'filt'))
for ind in sortinds:
oldSrc = inputs[ind]
photoCalib = photoCalibs[ind]
wcs = astromCalibs[ind]
vId = vIds[ind]
if logger:
logger.debug(
f"{len(oldSrc)} sources in ccd {vId['detector']} visit {vId['visit']}"
)
# create temporary catalog
tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
tmpCat.extend(oldSrc, mapper=mapper)
filtnum = filter_dict[vId['band']]
tmpCat['filt'] = np.repeat(filtnum, len(oldSrc))
tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_instFlux'] \
/ tmpCat['base_PsfFlux_instFluxErr']
if apply_external_wcs and wcs is not None:
updateSourceCoords(wcs, tmpCat)
photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux")
tmpCat['slot_ModelFlux_snr'][:] = (
tmpCat['slot_ModelFlux_instFlux'] /
tmpCat['slot_ModelFlux_instFluxErr'])
photoCalib.instFluxToMagnitude(tmpCat, "slot_ModelFlux",
"slot_ModelFlux")
_, psf_e1, psf_e2 = ellipticity_from_cat(oldSrc,
slot_shape='slot_PsfShape')
_, star_e1, star_e2 = ellipticity_from_cat(oldSrc,
slot_shape='slot_Shape')
tmpCat['e1'][:] = star_e1
tmpCat['e2'][:] = star_e2
tmpCat['psf_e1'][:] = psf_e1
tmpCat['psf_e2'][:] = psf_e2
srcVis.extend(tmpCat, False)
mmatch.add(catalog=tmpCat, dataId=vId)
# Complete the match, returning a catalog that includes
# all matched sources with object IDs that can be used to group them.
matchCat = mmatch.finish()
# Create a mapping object that allows the matches to be manipulated
# as a mapping of object ID to catalog of sources.
# I don't think I can persist a group view, so this may need to be called in a subsequent task
# allMatches = GroupView.build(matchCat)
return srcVis, matchCat
class MeasureCoaddsPqrTask(ProcessCoaddsTogetherTask):
"""
Base class for ngmix tasks
"""
_DefaultName = "MeasureCoaddsPqrTask"
ConfigClass = MeasureCoaddsPqrConfig
RunnerClass = MyTaskRunner
def __init__(self, *, config=None, refSchema=None, butler=None, initInputs=None, **kwds):
#import pdb;pdb.set_trace()
ProcessCoaddsTogetherTask.__init__(self, config=config, refSchema=refSchema, butler=butler,
initInputs=initInputs, **kwds)
if refSchema is None:
if butler is None:
if initInputs is not None:
refSchema = initInputs.get("refSchema", None)
if refSchema is None:
refSchema = SourceCatalog.Table.makeMinimalSchema()
else:
refSchema = butler.get(self.config.ref.name + "_schema").schema
self.ncolors = len(self.config.filters) - 1
self.bfd = BFDConfig(use_conc=self.config.use_conc, use_mag=self.config.use_mag,
ncolors=self.ncolors)
self.n_even = self.bfd.BFDConfig.MSIZE
self.n_odd = self.bfd.BFDConfig.XYSIZE
self.weight = KSigmaWeightF(self.config.weight_sigma, self.config.weight_n)
self.schema = self.defineSchema(refSchema)
self.ud = UniformDeviate()
print('Reinitialized')
self.initialized = False
@classmethod
def _makeArgumentParser(cls):
# Customize argument parsing for CmdLineTask.
parser = ArgumentParser(name=cls._DefaultName)
# This should be config.images.name, but there's no way to pass that
# information in here in Gen2.
parser.add_argument("--dir", dest="dir", default='./',
help="location of files")
return parser
def defineSchema(self, refSchema):
self.mapper = SchemaMapper(refSchema)
self.mapper.addMinimalSchema(SourceCatalog.Table.makeMinimalSchema(), True)
schema = self.mapper.getOutputSchema()
self.pqrKey = schema.addField("pqr", doc="pqr", type="ArrayF",
size=self.bfd.BFDConfig.DSIZE)
self.momKey = schema.addField("moment", doc="moment", type="ArrayF",
size=self.n_even)
self.momCovKey = schema.addField("moment_cov", doc="moment", type="ArrayF",
size=self.n_even*(self.n_even+1)//2)
self.numKey = schema.addField("n_templates", doc="number", type=np.int64)
self.uniqKey = schema.addField("n_unique", doc="unique", type=np.int32)
self.zKey = schema.addField("z", doc="redshift", type=np.float)
self.g1Key = schema.addField("g1", doc="redshift", type=np.float)
self.g2Key = schema.addField("g2", doc="redshift", type=np.float)
self.kappaKey = schema.addField("kappa", doc="redshift", type=np.float)
self.magKey = schema.addField("mag", doc="redshift", type=np.float)
self.labelKey = schema.addField("label", doc="redshift", type=str, size=10)
# self.zIdKey = schema.addField("z_id", doc="redshift", type=np.int64)
return schema
def runDataRef(self, files):
"""Run this task via CmdLineTask and Gen2 Butler.
Parameters
----------
patchRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
A list of DataRefs for all filters in a single patch.
"""
self.prep()
for file in files:
cat = afwTable.BaseCatalog.readFits(file)
mask = ((cat['moments_r_even'][:, 0] >= self.fluxMin) &
(cat['moments_r_even'][:, 0] < self.fluxMax) &
(cat['moments_r_cov_even'][:, 0] >= self.varMin) &
(cat['moments_r_cov_even'][:, 0] < self.varMax) &
(cat['z'] >= self.zMin) &
(cat['z'] < self.zMax))
tgs = []
for rec in cat[mask]:
cov_even = rec.get('moments_r_cov_even')
cov_odd = rec.get('moments_r_cov_odd')
full_cov_even = np.zeros((self.n_even, self.n_even), dtype=np.float32)
full_cov_odd = np.zeros((self.n_odd, self.n_odd), dtype=np.float32)
start = 0
for i in range(self.n_even):
full_cov_even[i][i:] = cov_even[start:start + self.n_even - i]
start += self.n_even - i
for i in range(self.n_even):
for j in range(i):
full_cov_even[i, j] = full_cov_even[j, i]
start = 0
for i in range(self.n_odd):
full_cov_odd[i][i:] = cov_odd[start:start + self.n_odd - i]
start += self.n_odd - i
for i in range(self.n_odd):
for j in range(i):
full_cov_odd[i, j] = full_cov_odd[j, i]
cov = self.bfd.MomentCov(full_cov_even, full_cov_odd)
#mom_odd = np.array([0, 0]
moment = self.bfd.Moment(rec.get('moments_r_even'), rec.get('moments_r_odd'))
pos = np.array([rec.get('ra'), rec.get('dec')])
tg = self.bfd.TargetGalaxy(moment, cov, pos, rec.get('id'))
tgs.append(tg)
results = self.prior.getPqrCatalog(tgs[:10], self.config.threads, self.config.chunk)
outcat = afwTable.BaseCatalog(self.schema)
raKey = self.schema['coord_ra'].asKey()
decKey = self.schema['coord_ra'].asKey()
idKey = self.schema['id'].asKey()
for r, rec in zip(results, cat[mask][:10]):
out = outcat.addNew()
out.set(self.pqrKey, r[0]._pqr)
out.set(self.numKey, r[1])
out.set(self.uniqKey, r[2])
out.set(self.momKey, rec.get('moments_r_even'))
out.set(self.momCovKey, rec.get('moments_r_cov_even'))
out.set(self.zKey, rec.get('z'))
out.set(self.g1Key, rec.get('g1'))
out.set(self.g1Key, rec.get('g2'))
out.set(self.kappaKey, rec.get('kappa'))
out.set(self.magKey, rec.get('mag'))
out.set(self.labelKey, self.config.label)
out.set(raKey, rec.get('ra')*afwGeom.degrees)
out.set(decKey, rec.get('dec')*afwGeom.degrees)
out.set(idKey, rec.get('id'))
outfile = file.replace('moment', 'pqr')
outcat.writeFits(outfile)
def prep(self):
if self.initialized:
return
self.prior = None
priorFiles = []
priorButler = Butler(self.config.priorRerun)
prior_skyMap = priorButler.get('deepCoadd_skyMap')
for tract in self.config.priorTracts:
for patchInfo in prior_skyMap[tract]:
patch = '%d,%d' % patchInfo.getIndex()
if self.config.priorPatches:
if patch not in self.config.priorPatches:
continue
if priorButler.datasetExists('deepCoadd_prior', tract=tract, patch=patch,
filter=self.config.priorFilter, label=self.config.priorLabel):
priorFiles.append(priorButler.getUri('deepCoadd_prior',
tract=tract, patch=patch,
filter=self.config.priorFilter,
label=self.config.priorLabel))
max_file = len(priorFiles)
if self.config.maxPriorFiles > 0:
max_file = self.config.maxPriorFiles
self.zBin = None
for file in priorFiles[:max_file]:
if file.find('_parent') > 0:
self.log.info("Skipping %s, from parent" % file)
continue
self.log.info("Adding prior %s" % file)
try:
cat = afwTable.BaseCatalog.readFits(file)
md = cat.getTable().getMetadata().toDict()
if self.prior is None:
self.fluxMin = md['FLUXMIN']
self.fluxMax = md['FLUXMAX']
self.varMin = md['VARMIN']
self.varMax = md['VARMAX']
cov_even = np.array(md['COV_EVEN'])
cov_odd = np.array(md['COV_ODD'])
self.zMax = md['ZMAXCUT']
self.zMin = md['ZMINCUT']
self.noiseFactor = md['noiseFactor']
self.priorSigmaCutoff = md['priorSigmaCutoff']
self.priorSigmaStep = md['priorSigmaStep']
self.priorSigmaBuffer = md['priorSigmaBuffer']
self.nSample = md['NSAMPLE']
self.selectionOnly = md['selectionOnly']
self.invariantCovariance = md['invariantCovariance']
covMat = self.bfd.MomentCov(cov_even.reshape(self.n_even, self.n_even),
cov_odd.reshape(self.n_odd, self.n_odd))
self.prior = self.bfd.KDTreePrior(self.fluxMin, self.fluxMax, covMat, self.ud,
self.nSample, self.selectionOnly,
self.noiseFactor, self.priorSigmaStep,
self.priorSigmaCutoff, self.priorSigmaBuffer,
self.invariantCovariance)
else:
fluxMin = md['FLUXMIN']
fluxMax = md['FLUXMAX']
varMin = md['VARMIN']
varMax = md['VARMAX']
cov_even = np.array(md['COV_EVEN'])
cov_odd = np.array(md['COV_ODD'])
zMax = md['ZMAXCUT']
zMin = md['ZMINCUT']
noiseFactor = md['noiseFactor']
priorSigmaCutoff = md['priorSigmaCutoff']
priorSigmaStep = md['priorSigmaStep']
priorSigmaBuffer = md['priorSigmaBuffer']
nSample = md['NSAMPLE']
selectionOnly = md['selectionOnly']
invariantCovariance = md['invariantCovariance']
mismatch = False
if fluxMin != self.fluxMin:
self.log.info('does not match fluxMin')
mismatch = True
if fluxMax != self.fluxMax:
self.log.info('does not match fluxMax')
mismatch = True
if varMin != self.varMin:
self.log.info('does not match varMin')
mismatch = True
if varMax != self.varMax:
self.log.info('does not match varMax')
mismatch = True
if zMin != self.zMin:
self.log.info('does not match zMin')
mismatch = True
if zMax != self.zMax:
self.log.info('does not match zMax')
mismatch = True
if noiseFactor != self.noiseFactor:
self.log.info('does not match fluxMin')
mismatch = True
if priorSigmaBuffer != self.priorSigmaBuffer:
self.log.info('does not match priorSigmaBuffer')
mismatch = True
if priorSigmaStep != self.priorSigmaStep:
self.log.info('does not match priorSigmaStep')
mismatch = True
if priorSigmaCutoff != self.priorSigmaCutoff:
self.log.info('does not match priorSigmaCutoff')
mismatch = True
if nSample != self.nSample:
self.log.info('does not match nSample')
mismatch = True
if selectionOnly != self.selectionOnly:
self.log.info('does not match selectionOnly')
mismatch = True
if invariantCovariance != self.invariantCovariance:
self.log.info('does not match invariantCovariance')
mismatch = True
if mismatch:
self.log.info('Skipping %s' % file)
continue
for s in cat:
ti = self.bfd.TemplateInfo()
ti.m = s.get('m')
ti.dm = s.get('dm').reshape(self.bfd.BFDConfig.MSIZE, self.bfd.BFDConfig.DSIZE)
ti.dxy = s.get('dxy').reshape(self.bfd.BFDConfig.XYSIZE, self.bfd.BFDConfig.DSIZE)
ti.nda = s.get('nda')
ti.id = s.get('bfd_id')
self.prior.addTemplateInfo(ti)
except Exception as e:
print('Failed to read', e)
continue
self.prior.prepare()
self.initialized = True
def defineSchema(self, refSchema):
"""Return the Schema for the output catalog.
This may add or modify self.
Parameters
----------
refSchema : `lsst.afw.table.Schema`
Schema of the input reference catalogs.
Returns
-------
outputSchema : `lsst.afw.table.Schema`
Schema of the output catalog. Will be added as ``self.schema``
by calling code.
"""
self.mapper = SchemaMapper(refSchema)
self.mapper.addMinimalSchema(SourceCatalog.Table.makeMinimalSchema(), True)
schema = self.mapper.getOutputSchema()
config=self.cdict
model=config['obj']['model']
n=self.get_namer()
pn=self.get_psf_namer()
# generic ngmix fields
mtypes=[
(n('flags'),'overall flags for the processing',np.int32,''),
(n('stamp_size'),'size of postage stamp',np.int32,''),
(n('maskfrac'),'mean masked fraction',np.float32,''),
]
for filt in config['filters']:
mtypes += [
(n('maskfrac_%s' % filt),'masked fraction in %s filter' % filt,np.float32,''),
]
# psf fitting related fields
mtypes += [
(pn('flags'),'overall flags for the PSF processing',np.int32,''),
# mean over filters
(pn('g2_mean'),'mean over filters of component 2 of the PSF ellipticity',np.float64,''),
(pn('g1_mean'),'mean over filters of component 2 of the PSF ellipticity',np.float64,''),
(pn('T_mean'),'mean over filters <x^2> + <y^2> for the gaussian mixture',np.float64,'arcsec^2'),
]
# PSF measurements by filter
for filt in config['filters']:
pfn=self.get_psf_namer(filt=filt)
mtypes += [
(pfn('flags'), 'overall flags for PSF processing in %s filter' % filt, np.int32, ''),
(pfn('row'),'offset from canonical row position',np.float64,'arcsec'),
(pfn('col'),'offset from canonical col position',np.float64,'arcsec'),
(pfn('g1'), 'component 1 of the PSF ellipticity in %s filter' % filt, np.float64, ''),
(pfn('g2'), 'component 2 of the PSF ellipticity in %s filter' % filt, np.float64, ''),
(pfn('T'), '<x^2> + <y^2> for the PSF in %s filter' % filt, np.float64, 'arcsec^2'),
]
# PSF flux measurements, on the object, by filter
#for filt in config['filters']:
# pfn=self.get_psf_flux_namer(filt)
# mtypes += [
# (pfn('flux_flags'),'flags for PSF template flux fitting in the %s filter' % filt,np.float64,''),
# (pfn('flux'),'PSF template flux in the %s filter' % filt,np.float64,''),
# (pfn('flux_err'),'error on PSF template flux in the %s filter' % filt,np.float64,''),
# ]
# object fitting related fields
for type in config['metacal']['types']:
mn=self.get_model_namer(type=type)
mtypes += [
(mn('flags'),'flags for model fit',np.int32,''),
(mn('nfev'),'number of function evaluations during fit',np.int32,''),
(mn('chi2per'),'chi^2 per degree of freedom',np.float64,''),
(mn('dof'),'number of degrees of freedom',np.int32,''),
(mn('s2n'),'S/N for the fit',np.float64,''),
(mn('row'),'offset from canonical row position',np.float64,'arcsec'),
(mn('row_err'),'error on offset from canonical row position',np.float64,'arcsec'),
(mn('col'),'offset from canonical col position',np.float64,'arcsec'),
(mn('col_err'),'error on offset from canonical col position',np.float64,'arcsec'),
(mn('g1'),'component 1 of the ellipticity',np.float64,''),
(mn('g1_err'),'error on component 1 of the ellipticity',np.float64,''),
(mn('g2'),'component 2 of the ellipticity',np.float64,''),
(mn('g2_err'),'error on component 2 of the ellipticity',np.float64,''),
(mn('T'),'<x^2> + <y^2> for the gaussian mixture',np.float64,'arcsec^2'),
(mn('T_err'),'error on <x^2> + <y^2> for the gaussian mixture',np.float64,'arcsec^2'),
]
if model in ['bd','bdf']:
mtypes += [
(mn('fracdev'),'fraction of light in the bulge',np.float64,''),
(mn('fracdev_err'),'error on fraction of light in the bulge',np.float64,''),
]
for filt in config['filters']:
mfn=self.get_model_flux_namer(filt, type=type)
mtypes += [
(mfn('flux'),'flux in the %s filter' % filt,np.float64,''),
(mfn('flux_err'),'error on flux in the %s filter' % filt,np.float64,''),
]
for name,doc,dtype,units in mtypes:
schema.addField(
name,
type=dtype,
doc=doc,
units=units,
)
return schema
def loadAndMatchData(repo, visitDataIds,
matchRadius=afwGeom.Angle(1, afwGeom.arcseconds),
verbose=False):
"""Load data from specific visit. Match with reference.
Parameters
----------
repo : string
The repository. This is generally the directory on disk
that contains the repository and mapper.
visitDataIds : list of dict
List of `butler` data IDs of Image catalogs to compare to reference.
The `calexp` cpixel image is needed for the photometric calibration.
matchRadius : afwGeom.Angle().
Radius for matching.
verbose : bool, optional
Output additional information on the analysis steps.
Returns
-------
afw.table.GroupView
An object of matched catalog.
"""
# Following
# https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
butler = dafPersist.Butler(repo)
dataset = 'src'
# 2016-02-08 MWV:
# I feel like I could be doing something more efficient with
# something along the lines of the following:
# dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in visitDataIds]
ccdKeyName = getCcdKeyName(visitDataIds[0])
schema = butler.get(dataset + "_schema", immediate=True).schema
mapper = SchemaMapper(schema)
mapper.addMinimalSchema(schema)
mapper.addOutputField(Field[float]('base_PsfFlux_snr', "PSF flux SNR"))
mapper.addOutputField(Field[float]('base_PsfFlux_mag', "PSF magnitude"))
mapper.addOutputField(Field[float]('base_PsfFlux_magerr', "PSF magnitude uncertainty"))
newSchema = mapper.getOutputSchema()
# Create an object that can match multiple catalogs with the same schema
mmatch = MultiMatch(newSchema,
dataIdFormat={'visit': int, ccdKeyName: int},
radius=matchRadius,
RecordClass=SimpleRecord)
# create the new extented source catalog
srcVis = SourceCatalog(newSchema)
for vId in visitDataIds:
try:
calexpMetadata = butler.get("calexp_md", vId, immediate=True)
except FitsError as fe:
print(fe)
print("Could not open calibrated image file for ", vId)
print("Skipping %s " % repr(vId))
continue
except TypeError as te:
# DECam images that haven't been properly reformatted
# can trigger a TypeError because of a residual FITS header
# LTV2 which is a float instead of the expected integer.
# This generates an error of the form:
#
# lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type'
#
# See, e.g., DM-2957 for details.
print(te)
print("Calibration image header information malformed.")
print("Skipping %s " % repr(vId))
continue
calib = afwImage.Calib(calexpMetadata)
oldSrc = butler.get('src', vId, immediate=True)
print(len(oldSrc), "sources in ccd %s visit %s" % (vId[ccdKeyName], vId["visit"]))
# create temporary catalog
tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
tmpCat.extend(oldSrc, mapper=mapper)
tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_flux'] / tmpCat['base_PsfFlux_fluxSigma']
with afwImageUtils.CalibNoThrow():
(tmpCat['base_PsfFlux_mag'][:], tmpCat['base_PsfFlux_magerr'][:]) = \
calib.getMagnitude(tmpCat['base_PsfFlux_flux'],
tmpCat['base_PsfFlux_fluxSigma'])
srcVis.extend(tmpCat, False)
mmatch.add(catalog=tmpCat, dataId=vId)
# Complete the match, returning a catalog that includes
# all matched sources with object IDs that can be used to group them.
matchCat = mmatch.finish()
# Create a mapping object that allows the matches to be manipulated
# as a mapping of object ID to catalog of sources.
allMatches = GroupView.build(matchCat)
return allMatches
class MeasureCoaddsPriorTask(ProcessCoaddsTogetherTask):
"""
Base class for ngmix tasks
"""
_DefaultName = "MeasureCoaddsPriorTask"
ConfigClass = MeasureCoaddsPriorConfig
def __init__(self,
*,
config=None,
refSchema=None,
butler=None,
initInputs=None,
**kwds):
#import pdb;pdb.set_trace()
ProcessCoaddsTogetherTask.__init__(self,
config=config,
refSchema=refSchema,
butler=butler,
initInputs=initInputs,
**kwds)
if refSchema is None:
if butler is None:
if initInputs is not None:
refSchema = initInputs.get("refSchema", None)
if refSchema is None:
refSchema = SourceCatalog.Table.makeMinimalSchema()
else:
refSchema = butler.get(self.config.ref.name + "_schema").schema
self.ncolors = len(self.config.filters) - 1
self.bfd = BFDConfig(use_conc=self.config.use_conc,
use_mag=self.config.use_mag,
ncolors=self.ncolors)
self.n_even = self.bfd.BFDConfig.MSIZE
self.n_odd = self.bfd.BFDConfig.XYSIZE
self.weight = KSigmaWeightF(self.config.weight_sigma,
self.config.weight_n)
self.schema = self.defineSchema(refSchema)
self.ud = UniformDeviate()
def getCovariances(self):
cat = afwTable.BaseCatalog.readFits(self.config.covFile)
covList = []
for rec in cat:
cov_even = rec.get('isoCovEven')
cov_odd = rec.get('isoCovOdd')
label = rec.get('label')
minVariance = rec.get('min')
maxVariance = rec.get('max')
full_cov_even = np.zeros((self.n_even, self.n_even),
dtype=np.float32)
full_cov_odd = np.zeros((self.n_odd, self.n_odd), dtype=np.float32)
start = 0
for i in range(self.n_even):
full_cov_even[i][i:] = cov_even[start:start + self.n_even - i]
start += self.n_even - i
for i in range(self.n_even):
for j in range(i):
full_cov_even[i, j] = full_cov_even[j, i]
start = 0
for i in range(self.n_odd):
full_cov_odd[i][i:] = cov_odd[start:start + self.n_odd - i]
start += self.n_odd - i
for i in range(self.n_odd):
for j in range(i):
full_cov_odd[i, j] = full_cov_odd[j, i]
covList.append(
(full_cov_even, full_cov_odd, label, minVariance, maxVariance))
return covList
def defineSchema(self, refSchema):
self.mapper = SchemaMapper(refSchema)
self.mapper.addMinimalSchema(SourceCatalog.Table.makeMinimalSchema(),
True)
schema = self.mapper.getOutputSchema()
self.mKey = schema.addField("m",
doc="template m",
type="ArrayF",
size=self.bfd.BFDConfig.MXYSIZE)
self.dmKey = schema.addField("dm",
doc="template m",
type="ArrayF",
size=self.bfd.BFDConfig.MSIZE *
self.bfd.BFDConfig.DSIZE)
self.dxyKey = schema.addField("dxy",
doc="template m",
type="ArrayF",
size=self.bfd.BFDConfig.XYSIZE *
self.bfd.BFDConfig.DSIZE)
self.ndaKey = schema.addField("nda", doc="nda", type=np.float)
self.idKey = schema.addField("bfd_id", doc="id", type=np.int64)
if self.config.zFile:
self.zKey = schema.addField("z", doc="redshift", type=np.float)
# self.zIdKey = schema.addField("z_id", doc="redshift", type=np.int64)
return schema
def runDataRef(self, patchRefList):
"""Run this task via CmdLineTask and Gen2 Butler.
Parameters
----------
patchRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
A list of DataRefs for all filters in a single patch.
"""
images = {}
replacers = {}
mergedDataId = {
"tract": patchRefList[0].dataId["tract"],
"patch": patchRefList[0].dataId["patch"]
}
butler = patchRefList[0].butlerSubset.butler
ref = butler.get("deepCoadd_ref", dataId=mergedDataId)
imageId = butler.get("deepMergedCoaddId", dataId=mergedDataId)
moments = butler.get('deepCoadd_moments', dataId=mergedDataId)
for patchRef in patchRefList:
filt = getShortFilterName(patchRef.dataId["filter"])
images[filt] = patchRef.get(self.config.images.name)
fpCat = patchRef.get(self.config.deblendCatalog.name)
footprints = {
rec.getId(): (rec.getParent(), rec.getFootprint())
for rec in fpCat
}
replacers[filt] = NoiseReplacer(self.config.deblendReplacer,
exposure=images[filt],
footprints=footprints,
exposureId=imageId)
results = self.run(butler,
mergedDataId,
images,
ref,
moments,
imageId=imageId,
replacers=replacers)
def run(self, butler, dataId, images, ref, moments, imageId, replacers):
"""Process coadds from all bands for a single patch.
This method should not add or modify self.
So far all children are using this exact code so leaving
it here for now. If we specialize a lot, might make a
processor its own object
Parameters
----------
images : `dict` of `lsst.afw.image.ExposureF`
Coadd images and associated metadata, keyed by filter name.
ref : `lsst.afw.table.SourceCatalog`
A catalog with one record for each object, containing "best"
measurements across all bands.
replacers : `dict` of `lsst.meas.base.NoiseReplacer`, optional
A dictionary of `~lsst.meas.base.NoiseReplacer` objects that can
be used to insert and remove deblended pixels for each object.
When not `None`, all detected pixels in ``images`` will have
*already* been replaced with noise, and this *must* be used
to restore objects one at a time.
imageId : `int`
Unique ID for this unit of data. Should be used (possibly
indirectly) to seed random numbers.
Returns
-------
results : `lsst.pipe.base.Struct`
Struct with (at least) an `output` attribute that is a catalog
to be written as ``self.config.output``.
"""
if len(images) != len(self.config.filters) != len(moments):
self.log.info(
'Number of filters does not match the list of images given. Skipping'
)
return None
covList = self.getCovariances()
if self.config.zFile:
zFile = Table.read(self.config.zFile)
z_redshift = zFile[self.config.zField]
# z_id = zFile[self.config.zId]
z_catalog = SkyCoord(ra=zFile[self.config.zRa] * u.deg,
dec=zFile[self.config.zDec] * u.deg)
self.z_list = {}
# self.z_id_list = {}
coord = SkyCoord(ra=ref['coord_ra'] * u.rad,
dec=ref['coord_dec'] * u.rad)
idx, d2d, d3d = match_coordinates_sky(coord, z_catalog)
d2d = d2d.arcsec
# we only need to compute the templates the first time through the loop
first = True
self.templates = []
for cov_even, cov_odd, label, varMin, varMax in covList:
tm0 = time.time()
nproc = 0
if (label not in self.config.useLabels) and len(
self.config.useLabels) > 0:
self.log.info("Label %s not in %s" %
(label, self.config.useLabels))
continue
# Build full label
full_label = label + self.config.label
if self.config.selectionOnly:
full_label += '_selection'
if self.config.noiseBin is not None:
full_label += '_n%d' % self.config.noiseBin
if self.config.zBin is not None:
full_label += '_z%d' % (self.config.zBin)
self.log.info('Processing label %s' % label)
sigmaFlux = np.sqrt(cov_even[0, 0])
minFlux = self.config.snMin * sigmaFlux
if self.config.fluxMin is not None:
minFlux = self.config.fluxMin
elif self.config.magMin is not None:
# Current assumption is that coadd zeropoint is 27, true for HSC
minFlux = 10**(-0.4 * (self.config.magMin - 27))
maxFlux = self.config.snMax * sigmaFlux
if self.config.fluxMax is not None:
maxFlux = self.config.fluxMax
elif self.config.magMax is not None:
maxFlux = 10**(-0.4 * (self.config.magMax - 27))
self.log.info('Min/Max %0.2f/%0.2f', minFlux, maxFlux)
covMat = self.bfd.MomentCov(cov_even, cov_odd)
momentPrior = self.bfd.KDTreePrior(
minFlux, maxFlux, covMat, self.ud, self.config.nSample,
self.config.selectionOnly, self.config.noiseFactor,
self.config.priorSigmaStep, self.config.priorSigmaCutoff,
self.config.priorSigmaBuffer, self.config.invariantCovariance)
if first == True:
last_index = self.config.num_to_process
if last_index is None:
last_index = len(ref)
for n, (refRecord, moment) in enumerate(
zip(ref[:last_index], moments[:last_index])):
self.log.info(f'Processing {n}/{last_index}')
if refRecord.get('deblend_nChild') != 0:
continue
if moment.get('bfd_flag') is True:
continue
if self.config.zFile:
if d2d[n] < self.config.zDistCut:
match_z = z_redshift[idx[n]]
# id_z = z_id[idx]
else:
self.log.info('No matching redshift')
continue
if match_z < self.config.zMinCut or match_z > self.config.zMaxCut:
self.log.info(
f'Does not match redshift range {match_z:0.2}')
continue
self.z_list[refRecord.getId()] = match_z
cov = moment.get(f'bfd_cov_even_{self.config.filters[0]}')
if cov[0] > self.config.maxVar and cov[
0] < self.config.minVar:
continue
# Insert the deblended pixels for just this object into all images.
for r in replacers.values():
r.insertSource(refRecord.getId())
try:
kgals = self.buildKGalaxy(refRecord, images)
kc = KColorGalaxy(self.bfd,
kgals,
id=refRecord.getId())
except Exception as e:
kc = None
continue
dx, badcentering, msg = kc.recenter(
self.config.weight_sigma)
if badcentering:
self.log.info('Bad centering %s', msg)
continue
template = kc.get_template(dx[0], dx[1])
self.templates.append(template)
nproc += 1
# Remove the deblended pixels for this object so we can process the next one.
for r in replacers.values():
r.removeSource(refRecord.getId())
first = False
for temp in self.templates:
momentPrior.addTemplate(temp)
tm = time.time() - tm0
if nproc > 0:
self.log.info(f'added {nproc} templates')
self.log.info(f'time: {tm/60.0} min')
self.log.info(f'time per: {tm/nproc} sec')
outputCat = self.buildCatalog(momentPrior)
dataId['label'] = full_label
dataId['filter'] = self.config.filters[0]
metadata = dafBase.PropertyList()
metadata.set('cov_even', np.array(cov_even.flatten(), dtype=float))
metadata.set('cov_odd', np.array(cov_odd.flatten(), dtype=float))
metadata.set('fluxMin', minFlux)
metadata.set('fluxMax', maxFlux)
metadata.set('varMin', varMin)
metadata.set('varMax', varMax)
if self.config.zFile is not None:
metadata.set('zFile', self.config.zFile)
metadata.set('zField', self.config.zField)
if self.config.zMaxCut is not None:
metadata.set('zMaxCut', self.config.zMaxCut)
if self.config.zMinCut is not None:
metadata.set('zMinCut', self.config.zMinCut)
if self.config.noiseBin is not None:
metadata.set('noiseBin', self.config.noiseBin)
metadata.set('noiseFactor', self.config.noiseFactor)
metadata.set('priorSigmaCutoff', self.config.priorSigmaCutoff)
metadata.set('priorSigmaStep', self.config.priorSigmaStep)
metadata.set('priorSigmaBuffer', self.config.priorSigmaBuffer)
metadata.set('nsample', self.config.nSample)
metadata.set('selectionOnly', self.config.selectionOnly)
metadata.set('invariantCovariance',
self.config.invariantCovariance)
metadata.set('maxXY', self.config.maxXY)
metadata.set('sigma', self.config.weight_sigma)
metadata.set('wIndex', self.config.weight_n)
metadata.set('covFile', self.config.covFile)
outputCat.getTable().setMetadata(metadata)
butler.put(outputCat, self.config.output.name, dataId)
# Restore all original pixels in the images.
if replacers is not None:
for r in replacers.values():
r.end()
def buildCatalog(self, momentPrior):
outCat = afwTable.BaseCatalog(self.schema)
for temp in momentPrior.templates:
rec = outCat.addNew()
rec.set(self.mKey, temp.m)
rec.set(self.dmKey, temp.dm.flatten())
rec.set(self.dxyKey, temp.dxy.flatten())
rec.set(self.ndaKey, temp.nda)
rec.set(self.idKey, temp.id)
if self.config.zFile:
rec.set(self.zKey, self.z_list[temp.id])
# rec.set(self.zIdKey, self.z_id_list[temp.id])
return outCat
def buildKGalaxy(self, record, exposures):
center = record.getCentroid()
band = self.config.filters[0]
local_lin_wcs = exposures[band].getWcs().linearizePixelToSky(
center, afwGeom.arcseconds)
jacobian = local_lin_wcs.getLinear().getMatrix()
sky_pos = exposures[band].getWcs().pixelToSky(center)
uvref = (sky_pos.getRa().asArcseconds(),
sky_pos.getDec().asArcseconds())
box = record.getFootprint().getBBox()
xy_pos = (center.getX() - box.getMinX(), center.getY() - box.getMinY())
bfd_wcs = bfd.WCS(jacobian, xyref=xy_pos, uvref=uvref)
kgals = []
for band in self.config.filters:
exposure = exposures[band]
factor = exposure.getMetadata().get('variance_scale')
noise = np.sqrt(np.median(exposure.variance[box].array) / factor)
image = exposure.image[box].array
psf_image = exposure.getPsf().computeKernelImage(center).array
kdata = bfd.generalImage(image,
uvref,
psf_image,
wcs=bfd_wcs,
pixel_noise=noise,
size=self.config.grid_size)
conjugate = set(np.where(kdata.conjugate.flatten() == False)[0])
kgal = self.bfd.KGalaxy(self.weight, kdata.kval.flatten(),
kdata.kx.flatten(), kdata.ky.flatten(),
kdata.kvar.flatten(), kdata.d2k, conjugate)
kgals.append(kgal)
return kgals
def selection(self, ref):
childName = 'deblend_nChild'
if ref.getParent() == 0 and ref.get(childName) > 0:
return False
return True
def getFakeSources(butler,
dataId,
tol=1.0,
extraCols=('zeropoint', 'visit', 'ccd'),
includeMissing=False,
footprints=False,
radecMatch=None,
multiband=False,
reffMatch=False,
pix=0.168,
minRad=None,
raCol='RA',
decCol='Dec'):
"""
Get list of sources which agree in pixel position with fake ones with tol.
This returns a sourceCatalog of all the matched fake objects,
note, there will be duplicates in this list, since I haven't
checked deblend.nchild, and I'm only doing a tolerance match,
which could include extra sources
The outputs can include extraCols as long as they are one of:
zeropoint, visit, ccd, thetaNorth, pixelScale
If includeMissing is true, then the pipeline looks at the fake sources
added in the header and includes an entry in the table for sources without
any measurements, specifically the 'id' column will be 0
radecMatch is the fakes table. if it's not None(default), then do an ra/dec
match with the input catalog instead of looking in the header for where the
sources where added
"""
coaddData = "deepCoadd_calexp"
coaddMeta = "deepCoadd_calexp_md"
availExtras = {
'zeropoint': {
'type': float,
'doc': 'zeropoint'
},
'visit': {
'type': int,
'doc': 'visit id'
},
'ccd': {
'type': int,
'doc': 'ccd id'
},
'thetaNorth': {
'type': lsst.afw.geom.Angle,
'doc': 'angle to north'
},
'pixelScale': {
'type': float,
'doc': 'pixelscale in arcsec/pixel'
}
}
if not np.in1d(extraCols, list(availExtras.keys())).all():
print("extraCols must be in ", availExtras)
try:
if 'filter' not in dataId:
sources = butler.get('src',
dataId,
flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
cal = butler.get('calexp', dataId, immediate=True)
cal_md = butler.get('calexp_md', dataId, immediate=True)
else:
meas = butler.get('deepCoadd_meas',
dataId,
flags=NO_FOOTPRINT,
immediate=True)
force = butler.get('deepCoadd_forced_src',
dataId,
flags=NO_FOOTPRINT,
immediate=True)
sources = combineWithForce(meas, force)
cal = butler.get(coaddData, dataId, immediate=True)
cal_md = butler.get(coaddMeta, dataId, immediate=True)
except RuntimeError:
print("skipping", dataId)
return None
if ('pixelScale' in extraCols) or ('thetaNorth' in extraCols):
wcs = cal.getWcs()
availExtras['pixelScale']['value'] = wcs.getPixelScale().asArcseconds()
# The 8 lines of code below find the angle to north, first the mid pixel of the calexp is found,
# then the pixel to sky matrix at this point, the coordinate this gives can then be used to find the
# linearized sky to pixel matrix which can then be used to find the angle.
xMid = cal.getWidth() // 2
yMid = cal.getHeight() // 2
midPoint = lsst.afw.geom.Point2D(xMid, yMid)
midCoord = wcs.pixelToSky(midPoint)
northSkyToPixelMatrix = wcs.linearizeSkyToPixel(
midCoord, lsst.afw.geom.degrees)
northSkyToPixelMatrix = northSkyToPixelMatrix.getLinear()
availExtras['thetaNorth']['value'] = (np.arctan2(
*tuple(northSkyToPixelMatrix(lsst.afw.geom.Point2D(1.0, 0.0))))
) * lsst.afw.geom.radians
if 'visit' in extraCols:
availExtras['visit']['value'] = dataId['visit']
if 'ccd' in extraCols:
availExtras['ccd']['value'] = dataId['ccd']
if 'zeropoint' in extraCols:
zeropoint = 2.5 * np.log10(cal_md.getScalar('FLUXMAG0'))
availExtras['zeropoint']['value'] = zeropoint
if radecMatch is None:
fakeXY, srcIndex = getFakeMatchesHeader(cal_md, sources, tol=tol)
else:
if minRad is not None:
print("# The min matching radius is %4.1f pixel" % minRad)
bbox = lsst.afw.geom.Box2D(cal.getBBox(lsst.afw.image.PARENT))
fakeXY, srcIndex, srcClose = getFakeMatchesRaDec(sources,
radecMatch,
bbox,
cal.getWcs(),
tol=tol,
reffMatch=reffMatch,
pix=pix,
minRad=minRad,
raCol=raCol,
decCol=decCol)
mapper = SchemaMapper(sources.schema)
mapper.addMinimalSchema(sources.schema)
newSchema = mapper.getOutputSchema()
newSchema.addField('fakeId',
type=np.int32,
doc='id of fake source matched to position')
newSchema.addField('nMatched',
type=np.int32,
doc='Number of matched objects')
newSchema.addField('nPrimary',
type=np.int32,
doc='Number of unique matched objects')
newSchema.addField('nNoChild',
type=np.int32,
doc='Number of matched objects with nchild==0')
newSchema.addField('rMatched',
type=float,
doc='Radius used form atching obects, in pixel')
newSchema.addField('fakeOffX',
type=float,
doc='offset from input fake position in X (pixels)')
newSchema.addField('fakeOffY',
type=float,
doc='offset from input fake position in Y (pixels)')
newSchema.addField('fakeOffR',
type=float,
doc='offset from input fake position in radius')
newSchema.addField('fakeClosest',
type="Flag",
doc='Is this match the closest one?')
for extraName in set(extraCols).intersection(availExtras):
newSchema.addField(extraName,
type=availExtras[extraName]['type'],
doc=availExtras[extraName]['doc'])
srcList = SourceCatalog(newSchema)
srcList.reserve(
sum([len(s) for s in srcIndex.values()]) +
(0 if not includeMissing else list(srcIndex.values()).count([])))
centroidKey = sources.getCentroidKey()
isPrimary = sources.schema.find('detect_isPrimary').getKey()
nChild = sources.schema.find('force_deblend_nChild').getKey()
for ident, sindlist in srcIndex.items():
rMatched = fakeXY[ident][2]
if minRad is not None:
if rMatched < minRad:
rMatched = minRad
nMatched = len(sindlist)
nPrimary = np.sum(
[sources[int(obj)].get(isPrimary) for obj in sindlist])
nNoChild = np.sum([(sources[int(obj)].get(nChild) == 0)
for obj in sindlist])
if includeMissing and (nMatched == 0):
newRec = srcList.addNew()
newRec.set('fakeId', ident)
newRec.set('id', 0)
newRec.set('nMatched', 0)
newRec.set('rMatched', rMatched)
for ss in sindlist:
newRec = srcList.addNew()
newRec.assign(sources[int(ss)], mapper)
newRec.set('fakeId', ident)
newRec.set('nMatched', nMatched)
newRec.set('nPrimary', nPrimary)
newRec.set('nNoChild', nNoChild)
newRec.set('rMatched', rMatched)
offsetX = (sources[int(ss)].get(centroidKey).getX() -
fakeXY[ident][0])
newRec.set('fakeOffX', offsetX)
offsetY = (sources[int(ss)].get(centroidKey).getY() -
fakeXY[ident][1])
newRec.set('fakeOffY', offsetY)
newRec.set('fakeOffR', np.sqrt(offsetX**2.0 + offsetY**2.0))
if radecMatch:
if int(ss) == int(srcClose[ident]):
newRec.set('fakeClosest', True)
else:
newRec.set('fakeClosest', False)
if includeMissing:
srcList = srcList.copy(deep=True)
for extraName in set(extraCols).intersection(availExtras):
tempCol = srcList.get(extraName)
tempCol.fill(availExtras[extraName]['value'])
return srcList
def getFakeSources(butler,
dataId,
tol=1.0,
extraCols=('zeropoint', 'visit', 'ccd'),
includeMissing=False,
footprints=False,
radecMatch=None):
"""Get list of sources which agree in pixel position with fake ones with tol
this returns a sourceCatalog of all the matched fake objects,
note, there will be duplicates in this list, since I haven't checked deblend.nchild,
and I'm only doing a tolerance match, which could include extra sources
the outputs can include extraCols as long as they are one of:
zeropoint, visit, ccd, thetaNorth, pixelScale
if includeMissing is true, then the pipeline looks at the fake sources
added in the header and includes an entry in the table for sources without
any measurements, specifically the 'id' column will be 0
radecMatch is the fakes table. if it's not None(default), then do an ra/dec
match with the input catalog instead of looking in the header for where the
sources where added
"""
availExtras = {
'zeropoint': {
'type': float,
'doc': 'zeropoint'
},
'visit': {
'type': int,
'doc': 'visit id'
},
'ccd': {
'type': int,
'doc': 'ccd id'
},
'thetaNorth': {
'type': lsst.afw.geom.Angle,
'doc': 'angle to north'
},
'pixelScale': {
'type': float,
'doc': 'pixelscale in arcsec/pixel'
}
}
if not np.in1d(extraCols, availExtras.keys()).all():
print "extraCols must be in ", availExtras
try:
if not 'filter' in dataId:
sources = butler.get('src',
dataId,
flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
cal = butler.get('calexp', dataId, immediate=True)
cal_md = butler.get('calexp_md', dataId, immediate=True)
else:
sources = butler.get('deepCoadd_src',
dataId,
flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
cal = butler.get('deepCoadd', dataId, immediate=True)
cal_md = butler.get('deepCoadd_md', dataId, immediate=True)
except (lsst.pex.exceptions.LsstException, RuntimeError) as e:
print "skipping", dataId
return None
if ('pixelScale' in extraCols) or ('thetaNorth' in extraCols):
wcs = cal.getWcs()
availExtras['pixelScale']['value'] = wcs.pixelScale().asArcseconds()
availExtras['thetaNorth']['value'] = lsst.afw.geom.Angle(
np.arctan2(*tuple(wcs.getLinearTransform().invert()(
lsst.afw.geom.Point2D(1.0, 0.0)))))
if 'visit' in extraCols:
availExtras['visit']['value'] = dataId['visit']
if 'ccd' in extraCols:
availExtras['ccd']['value'] = dataId['ccd']
if 'zeropoint' in extraCols:
availExtras['zeropoint']['value'] = 2.5 * np.log10(
cal_md.get('FLUXMAG0'))
if radecMatch is None:
fakeXY, srcIndex = getFakeMatchesHeader(cal_md, sources, tol=tol)
else:
fakeXY, srcIndex = getFakeMatchesRaDec(
sources,
radecMatch,
lsst.afw.geom.Box2D(cal.getBBox(lsst.afw.image.PARENT)),
cal.getWcs(),
tol=tol)
mapper = SchemaMapper(sources.schema)
mapper.addMinimalSchema(sources.schema)
newSchema = mapper.getOutputSchema()
newSchema.addField('fakeId',
type=int,
doc='id of fake source matched to position')
newSchema.addField('fakeOffset',
type=lsst.afw.geom.Point2D,
doc='offset from input fake position (pixels)')
for extraName in set(extraCols).intersection(availExtras):
newSchema.addField(extraName,
type=availExtras[extraName]['type'],
doc=availExtras[extraName]['doc'])
srcList = SourceCatalog(newSchema)
srcList.reserve(
sum([len(s) for s in srcIndex.values()]) +
(0 if not includeMissing else srcIndex.values().count([])))
centroidKey = sources.schema.find('centroid.sdss').getKey()
for ident, sindlist in srcIndex.items():
if includeMissing and (len(sindlist) == 0):
newRec = srcList.addNew()
newRec.set('fakeId', ident)
newRec.set('id', 0)
for ss in sindlist:
newRec = srcList.addNew()
newRec.assign(sources[ss], mapper)
newRec.set('fakeId', ident)
newRec.set(
'fakeOffset',
lsst.afw.geom.Point2D(
sources[ss].get(centroidKey).getX() - fakeXY[ident][0],
sources[ss].get(centroidKey).getY() - fakeXY[ident][1]))
if includeMissing:
srcList = srcList.copy(deep=True)
for extraName in set(extraCols).intersection(availExtras):
tempCol = srcList.get(extraName)
tempCol.fill(availExtras[extraName]['value'])
return srcList
