def testUnderscoreColumnFormat(self):
     """Test the per-filter column format with an underscore"""
     config = TransformObjectCatalogConfig()
     config.outputBands = ["g", "r", "i"]
     config.camelCase = False
     task = TransformObjectCatalogTask(config=config)
     funcs = {'Fwhm': HsmFwhm(dataset='meas')}
     df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
     self.assertIsInstance(df, pd.DataFrame)
     for filt in config.outputBands:
         self.assertIn(filt + '_Fwhm', df.columns)
 def testUnderscoreColumnFormat(self):
     """Test the per-filter column format with an underscore"""
     config = TransformObjectCatalogConfig()
     filterMap = {"HSC-G": "g", "HSC-R": "r", "HSC-I": "i"}
     config.filterMap = filterMap
     config.camelCase = False
     task = TransformObjectCatalogTask(config=config)
     funcs = {'Fwhm': HsmFwhm(dataset='meas')}
     df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
     self.assertIsInstance(df, pd.DataFrame)
     for filt in filterMap.values():
         self.assertIn(filt + '_Fwhm', df.columns)
 def testNoOutputBands(self):
     """All the input bands should go into the output, and nothing else.
     """
     config = TransformObjectCatalogConfig()
     config.multilevelOutput = True
     task = TransformObjectCatalogTask(config=config)
     funcs = {'Fwhm': HsmFwhm(dataset='meas')}
     df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
     self.assertIsInstance(df, pd.DataFrame)
     self.assertNotIn('HSC-G', df)
     for filt in ['g', 'r', 'i']:
         self.assertIsInstance(df[filt], pd.DataFrame)
         self.assertIn('Fwhm', df[filt].columns)
 def testMultilevelOutput(self):
     """Test the non-flattened result dataframe with a multilevel column index"""
     config = TransformObjectCatalogConfig()
     config.outputBands = ["r", "i"]
     config.multilevelOutput = True
     task = TransformObjectCatalogTask(config=config)
     funcs = {'Fwhm': HsmFwhm(dataset='meas')}
     df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
     self.assertIsInstance(df, pd.DataFrame)
     self.assertNotIn('g', df)
     for filt in config.outputBands:
         self.assertIsInstance(df[filt], pd.DataFrame)
         self.assertIn('Fwhm', df[filt].columns)
    def testNullFilter(self):
        """Test that columns for all filters are created despite they may not
        exist in the input data.
        """
        config = TransformObjectCatalogConfig()
        config.camelCase = True
        # Want y band columns despite the input data do not have them
        # Exclude g band columns despite the input data have them
        config.outputBands = ["r", "i", "y"]
        # Arbitrarily choose a boolean flag column to be "good"
        config.goodFlags = ['GoodFlagColumn']
        task = TransformObjectCatalogTask(config=config)
        # Add in a float column, an integer column, a good flag, and
        # a bad flag.  It does not matter which columns we choose, just
        # that they have the appropriate type.
        funcs = {
            'FloatColumn': HsmFwhm(dataset='meas'),
            'IntColumn': Column('base_InputCount_value', dataset='meas'),
            'GoodFlagColumn': Column('slot_GaussianFlux_flag', dataset='meas'),
            'BadFlagColumn': Column('slot_Centroid_flag', dataset='meas')
        }
        df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
        self.assertIsInstance(df, pd.DataFrame)

        for filt in config.outputBands:
            self.assertIn(filt + 'FloatColumn', df.columns)
            self.assertIn(filt + 'IntColumn', df.columns)
            self.assertIn(filt + 'BadFlagColumn', df.columns)
            self.assertIn(filt + 'GoodFlagColumn', df.columns)

        # Check that the default filling has worked.
        self.assertNotIn('gFloatColumn', df.columns)
        self.assertTrue(df['yFloatColumn'].isnull().all())
        self.assertTrue(df['iFloatColumn'].notnull().all())
        self.assertTrue(np.all(df['iIntColumn'].values >= 0))
        self.assertTrue(np.all(df['yIntColumn'].values < 0))
        self.assertTrue(np.all(~df['yGoodFlagColumn'].values))
        self.assertTrue(np.all(df['yBadFlagColumn'].values))

        # Check that the datatypes are preserved.
        self.assertEqual(df['iFloatColumn'].dtype, np.dtype(np.float64))
        self.assertEqual(df['yFloatColumn'].dtype, np.dtype(np.float64))
        self.assertEqual(df['iIntColumn'].dtype, np.dtype(np.int64))
        self.assertEqual(df['yIntColumn'].dtype, np.dtype(np.int64))
        self.assertEqual(df['iGoodFlagColumn'].dtype, np.dtype(np.bool_))
        self.assertEqual(df['yGoodFlagColumn'].dtype, np.dtype(np.bool_))
        self.assertEqual(df['iBadFlagColumn'].dtype, np.dtype(np.bool_))
        self.assertEqual(df['yBadFlagColumn'].dtype, np.dtype(np.bool_))
    def testNullFilter(self):
        """Test that columns for all filters are created despite they may not
        exist in the input data.
        """
        config = TransformObjectCatalogConfig()
        # Want y band columns despite the input data do not have them
        # Exclude g band columns despite the input data have them
        config.outputBands = ["r", "i", "y"]
        task = TransformObjectCatalogTask(config=config)
        funcs = {'Fwhm': HsmFwhm(dataset='meas')}
        df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
        self.assertIsInstance(df, pd.DataFrame)
        for column in ('coord_ra', 'coord_dec'):
            self.assertIn(column, df.columns)

        for filt in config.outputBands:
            self.assertIn(filt + 'Fwhm', df.columns)

        self.assertNotIn('gFwhm', df.columns)
        self.assertTrue(df['yFwhm'].isnull().all())
        self.assertTrue(df['iFwhm'].notnull().all())