def test_None_to_nan_conversion_all_none(self):
BaseCollection.serializer_class = InternalDtypeTestSerializer
b = BaseCollection()
b.load_data(self.dtyp_test_data_all_none)
df = b.to_dataframe() # would raise here
self.assertIsInstance(df, pd.DataFrame)
def test_base_collection_to_dataframe(self):
base = BaseCollection()
base.load_data(self.data)
test = base.to_dataframe()
assert_frame_equal(test, pd.DataFrame().from_dict(self.data))
def test_empty_collection_returns_empty_dataframe_in_to_dataframe(self):
BaseCollection.serializer_class = InternalSerializer
records = []
b = BaseCollection()
b.load_data(records)
df = b.to_dataframe()
self.assertIsInstance(df, pd.DataFrame)
self.assertEqual(len(df), 0)
def test_non_required_fields_not_present_do_not_raise_key_error_in_to_dataframe(
self):
BaseCollection.serializer_class = InternalSerializer # these fields are not required
records = [{'bdbid': 1}, {'bdbid': 2}]
b = BaseCollection()
b.load_data(records)
df = b.to_dataframe()
self.assertEqual(records, df.to_dict('records'))
def test_base_collection_dataframe_with_dtypes(self):
BaseCollection.serializer_class = InternalDtypeTestSerializer # NOTE patching a different serializer here
base = BaseCollection()
base.load_data(self.dtype_test_data)
base2 = BaseCollection()
base2.load_data(self.dtype_test_data_none)
df = base2.to_dataframe()
self.assertTrue(df.isnull().values.any())
BaseCollection.serializer_class = InternalSerializer #NOTE must patch this back here
def test_non_required_date_fields_do_not_raiseTypeError_in_to_dataframe(
self):
class TestDateSerializer(BaseSerializer):
test_date = fields.Date('%Y-%m-%d', allow_none=True)
BaseCollection.serializer_class = TestDateSerializer # these fields are not required
records = [{'test_date': None}, {'test_date': '2017-07-01'}]
b = BaseCollection()
b.load_data(records)
df = b.to_dataframe()
check = df.to_dict('records')
self.assertEqual(str(check[0]['test_date']), 'NaT')
self.assertEqual(pd.Timestamp('2017-07-01'), check[1]['test_date'])
def test_non_required_int_fields_do_not_raise_TypeError_in_to_dataframe(
self):
class TestIntSerializer(BaseSerializer):
test_id = fields.Integer(allow_none=True)
BaseCollection.serializer_class = TestIntSerializer # these fields are not required
records = [{'test_id': None}, {'test_id': 2}]
#expected_result = [{'test_id': np.nan}, {'test_id': 2.0}]
b = BaseCollection()
b.load_data(records)
df = b.to_dataframe()
check = df.to_dict('records')
self.assertTrue(np.isnan(
check[0]['test_id'])) #NOTE coerced to nan and float
self.assertEqual(check[1]['test_id'], 2.0)