def testUnderscoreColumnFormat(self):
"""Test the per-filter column format with an underscore"""
config = TransformObjectCatalogConfig()
config.outputBands = ["g", "r", "i"]
config.camelCase = False
task = TransformObjectCatalogTask(config=config)
funcs = {'Fwhm': HsmFwhm(dataset='meas')}
df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
self.assertIsInstance(df, pd.DataFrame)
for filt in config.outputBands:
self.assertIn(filt + '_Fwhm', df.columns)
def testUnderscoreColumnFormat(self):
"""Test the per-filter column format with an underscore"""
config = TransformObjectCatalogConfig()
filterMap = {"HSC-G": "g", "HSC-R": "r", "HSC-I": "i"}
config.filterMap = filterMap
config.camelCase = False
task = TransformObjectCatalogTask(config=config)
funcs = {'Fwhm': HsmFwhm(dataset='meas')}
df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
self.assertIsInstance(df, pd.DataFrame)
for filt in filterMap.values():
self.assertIn(filt + '_Fwhm', df.columns)
def testNoOutputBands(self):
"""All the input bands should go into the output, and nothing else.
"""
config = TransformObjectCatalogConfig()
config.multilevelOutput = True
task = TransformObjectCatalogTask(config=config)
funcs = {'Fwhm': HsmFwhm(dataset='meas')}
df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
self.assertIsInstance(df, pd.DataFrame)
self.assertNotIn('HSC-G', df)
for filt in ['g', 'r', 'i']:
self.assertIsInstance(df[filt], pd.DataFrame)
self.assertIn('Fwhm', df[filt].columns)
def testMultilevelOutput(self):
"""Test the non-flattened result dataframe with a multilevel column index"""
config = TransformObjectCatalogConfig()
config.outputBands = ["r", "i"]
config.multilevelOutput = True
task = TransformObjectCatalogTask(config=config)
funcs = {'Fwhm': HsmFwhm(dataset='meas')}
df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
self.assertIsInstance(df, pd.DataFrame)
self.assertNotIn('g', df)
for filt in config.outputBands:
self.assertIsInstance(df[filt], pd.DataFrame)
self.assertIn('Fwhm', df[filt].columns)
def testNullFilter(self):
"""Test that columns for all filters are created despite they may not
exist in the input data.
"""
config = TransformObjectCatalogConfig()
config.camelCase = True
# Want y band columns despite the input data do not have them
# Exclude g band columns despite the input data have them
config.outputBands = ["r", "i", "y"]
# Arbitrarily choose a boolean flag column to be "good"
config.goodFlags = ['GoodFlagColumn']
task = TransformObjectCatalogTask(config=config)
# Add in a float column, an integer column, a good flag, and
# a bad flag. It does not matter which columns we choose, just
# that they have the appropriate type.
funcs = {
'FloatColumn': HsmFwhm(dataset='meas'),
'IntColumn': Column('base_InputCount_value', dataset='meas'),
'GoodFlagColumn': Column('slot_GaussianFlux_flag', dataset='meas'),
'BadFlagColumn': Column('slot_Centroid_flag', dataset='meas')
}
df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
self.assertIsInstance(df, pd.DataFrame)
for filt in config.outputBands:
self.assertIn(filt + 'FloatColumn', df.columns)
self.assertIn(filt + 'IntColumn', df.columns)
self.assertIn(filt + 'BadFlagColumn', df.columns)
self.assertIn(filt + 'GoodFlagColumn', df.columns)
# Check that the default filling has worked.
self.assertNotIn('gFloatColumn', df.columns)
self.assertTrue(df['yFloatColumn'].isnull().all())
self.assertTrue(df['iFloatColumn'].notnull().all())
self.assertTrue(np.all(df['iIntColumn'].values >= 0))
self.assertTrue(np.all(df['yIntColumn'].values < 0))
self.assertTrue(np.all(~df['yGoodFlagColumn'].values))
self.assertTrue(np.all(df['yBadFlagColumn'].values))
# Check that the datatypes are preserved.
self.assertEqual(df['iFloatColumn'].dtype, np.dtype(np.float64))
self.assertEqual(df['yFloatColumn'].dtype, np.dtype(np.float64))
self.assertEqual(df['iIntColumn'].dtype, np.dtype(np.int64))
self.assertEqual(df['yIntColumn'].dtype, np.dtype(np.int64))
self.assertEqual(df['iGoodFlagColumn'].dtype, np.dtype(np.bool_))
self.assertEqual(df['yGoodFlagColumn'].dtype, np.dtype(np.bool_))
self.assertEqual(df['iBadFlagColumn'].dtype, np.dtype(np.bool_))
self.assertEqual(df['yBadFlagColumn'].dtype, np.dtype(np.bool_))
def testNullFilter(self):
"""Test that columns for all filters are created despite they may not
exist in the input data.
"""
config = TransformObjectCatalogConfig()
# Want y band columns despite the input data do not have them
# Exclude g band columns despite the input data have them
config.outputBands = ["r", "i", "y"]
task = TransformObjectCatalogTask(config=config)
funcs = {'Fwhm': HsmFwhm(dataset='meas')}
df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
self.assertIsInstance(df, pd.DataFrame)
for column in ('coord_ra', 'coord_dec'):
self.assertIn(column, df.columns)
for filt in config.outputBands:
self.assertIn(filt + 'Fwhm', df.columns)
self.assertNotIn('gFwhm', df.columns)
self.assertTrue(df['yFwhm'].isnull().all())
self.assertTrue(df['iFwhm'].notnull().all())