def testColumn(self):
self.columns.append("base_FootprintArea_value")
self.dataDict["base_FootprintArea_value"] = \
np.full(self.nRecords, 1)
parq = self.simulateMultiParquet(self.dataDict)
func = Column('base_FootprintArea_value', filt='g')
self._funcVal(func, parq)
parq = self.simulateParquet(self.dataDict)
func = Column('base_FootprintArea_value')
self._funcVal(func, parq)
def testCompositeFail(self):
"""Test a composite functor where one of the functors should be junk.
"""
self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000)
parq = self.simulateMultiParquet(self.dataDict)
funcDict = {
'good': Column("base_PsfFlux_instFlux"),
'bad': Column('not_a_column')
}
df = self._compositeFuncVal(CompositeFunctor(funcDict), parq) # noqa
def testNullFilter(self):
"""Test that columns for all filters are created despite they may not
exist in the input data.
"""
config = TransformObjectCatalogConfig()
config.camelCase = True
# Want y band columns despite the input data do not have them
# Exclude g band columns despite the input data have them
config.outputBands = ["r", "i", "y"]
# Arbitrarily choose a boolean flag column to be "good"
config.goodFlags = ['GoodFlagColumn']
task = TransformObjectCatalogTask(config=config)
# Add in a float column, an integer column, a good flag, and
# a bad flag. It does not matter which columns we choose, just
# that they have the appropriate type.
funcs = {
'FloatColumn': HsmFwhm(dataset='meas'),
'IntColumn': Column('base_InputCount_value', dataset='meas'),
'GoodFlagColumn': Column('slot_GaussianFlux_flag', dataset='meas'),
'BadFlagColumn': Column('slot_Centroid_flag', dataset='meas')
}
df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
self.assertIsInstance(df, pd.DataFrame)
for filt in config.outputBands:
self.assertIn(filt + 'FloatColumn', df.columns)
self.assertIn(filt + 'IntColumn', df.columns)
self.assertIn(filt + 'BadFlagColumn', df.columns)
self.assertIn(filt + 'GoodFlagColumn', df.columns)
# Check that the default filling has worked.
self.assertNotIn('gFloatColumn', df.columns)
self.assertTrue(df['yFloatColumn'].isnull().all())
self.assertTrue(df['iFloatColumn'].notnull().all())
self.assertTrue(np.all(df['iIntColumn'].values >= 0))
self.assertTrue(np.all(df['yIntColumn'].values < 0))
self.assertTrue(np.all(~df['yGoodFlagColumn'].values))
self.assertTrue(np.all(df['yBadFlagColumn'].values))
# Check that the datatypes are preserved.
self.assertEqual(df['iFloatColumn'].dtype, np.dtype(np.float64))
self.assertEqual(df['yFloatColumn'].dtype, np.dtype(np.float64))
self.assertEqual(df['iIntColumn'].dtype, np.dtype(np.int64))
self.assertEqual(df['yIntColumn'].dtype, np.dtype(np.int64))
self.assertEqual(df['iGoodFlagColumn'].dtype, np.dtype(np.bool_))
self.assertEqual(df['yGoodFlagColumn'].dtype, np.dtype(np.bool_))
self.assertEqual(df['iBadFlagColumn'].dtype, np.dtype(np.bool_))
self.assertEqual(df['yBadFlagColumn'].dtype, np.dtype(np.bool_))
def testCustom(self):
self.columns.append("base_FootprintArea_value")
self.dataDict["base_FootprintArea_value"] = \
np.random.rand(self.nRecords)
parq = self.simulateMultiParquet(self.dataDict)
func = CustomFunctor('2*base_FootprintArea_value', filt='g')
val = self._funcVal(func, parq)
func2 = Column('base_FootprintArea_value', filt='g')
np.allclose(val.values, 2 * func2(parq).values, atol=1e-13, rtol=0)
parq = self.simulateParquet(self.dataDict)
func = CustomFunctor('2 * base_FootprintArea_value')
val = self._funcVal(func, parq)
func2 = Column('base_FootprintArea_value')
np.allclose(val.values, 2 * func2(parq).values, atol=1e-13, rtol=0)
def addUnpackedFlagFunctors(self):
"""Add Column functor for each of the flags to the internal functor
dictionary.
"""
for flag in self.bit_pack_columns[0]['bitList']:
flagName = flag['name']
targetName = self.funcs.renameCol(
flagName, self.rename_rules[0]['flag_rename_rules'])
self.funcs.update({targetName: Column(flagName)})
def testCustom(self):
func = CustomFunctor('2*base_FootprintArea_value', filt='HSC-G')
val = self._funcVal(func)
func2 = Column('base_FootprintArea_value', filt='HSC-G')
np.allclose(val.values,
2 * func2(self.parq).values,
atol=1e-13,
rtol=0)
def _get_funcs(self, x_data, x_data_custom, y_data, y_data_custom,
labeller):
if self.x_data_custom:
xFunc = getFunc(self.x_data_custom)
else:
xFunc = getFunc(self.x_data)
if self.y_data_custom:
yFunc = getFunc(self.y_data_custom)
else:
yFunc = getFunc(self.y_data)
labeller = getLabeller(self.labeller)
return CompositeFunctor({
'x': xFunc,
'y': yFunc,
'label': labeller,
'id': Column('id'),
'ra': RAColumn(),
'dec': DecColumn()
})
import param
import parambokeh
from lsst.pipe.tasks.functors import (Mag, CustomFunctor, DeconvolvedMoments,
StarGalaxyLabeller, RAColumn, DecColumn,
Column, SdssTraceSize,
PsfSdssTraceSizeDiff, HsmTraceSize,
PsfHsmTraceSizeDiff, CompositeFunctor)
default_xFuncs = {
'base_PsfFlux': Mag('base_PsfFlux'),
'modelfit_CModel': Mag('modelfit_CModel')
}
default_yFuncs = {'modelfit_CModel - base_PsfFlux' : CustomFunctor('mag(modelfit_CModel) - mag(base_PsfFlux)'),
'Deconvolved Moments' : DeconvolvedMoments(),
'Footprint NPix' : Column('base_Footprint_nPix'),
'ext_photometryKron_KronFlux - base_PsfFlux' : \
CustomFunctor('mag(ext_photometryKron_KronFlux) - mag(base_PsfFlux)'),
'base_GaussianFlux - base_PsfFlux' : CustomFunctor('mag(base_GaussianFlux) - mag(base_PsfFlux)'),
'SDSS Trace Size' : SdssTraceSize(),
'PSF - SDSS Trace Size' : PsfSdssTraceSizeDiff(),
'HSM Trace Size' : HsmTraceSize(),
'PSF - HSM Trace Size': PsfHsmTraceSizeDiff()}
default_labellers = {'default': StarGalaxyLabeller()}
def getFunc(funcName):
if funcName in default_xFuncs:
return default_xFuncs[funcName]
elif funcName in default_yFuncs:
def testColumn(self):
func = Column('base_FootprintArea_value', filt='HSC-G')
self._funcVal(func)