def test_auto(self): binary_data = 'テスト'.encode('cp932') schema = GenericSchema({}, GenericSchema.AUTO) d = schema.transform({ 'k1': '123', 'k2': 456, 'k3': '789'.encode(), 'k4': 'xxx'.encode(), 'k5': binary_data, 'k6': True, 'k7': False, }) self.assertEqual({ 'k1': '123', 'k6': '1', 'k7': '0' }, dict(d.string_values)) self.assertEqual({'k2': 456}, dict(d.num_values)) self.assertEqual( { 'k3': '789'.encode(), 'k4': 'xxx'.encode(), 'k5': binary_data }, dict(d.binary_values))
def test_alias(self): schema = GenericSchema({ 'k1': GenericSchema.NUMBER, 'k2': (GenericSchema.NUMBER, 'alias_name'), }) d = schema.transform({ 'k1': '123', 'k2': 456, }) self.assertEqual({'k1': 123, 'alias_name': 456}, dict(d.num_values))
def test_alias(self): schema = GenericSchema({ 'k1': GenericSchema.NUMBER, 'k2': (GenericSchema.NUMBER, 'alias_name'), }) d = schema.transform({ 'k1': '123', 'k2': 456, }) self.assertEqual({'k1': 123, 'alias_name': 456}, dict(d.num_values))
def test_null(self): schema = GenericSchema({ 'k1': GenericSchema.STRING, 'k2': GenericSchema.NUMBER, }) d = schema.transform({ 'k1': None, 'k2': None, }) self.assertEqual({}, dict(d.string_values)) self.assertEqual({}, dict(d.num_values))
def test_numpy(self): schema = GenericSchema({ 'k1': GenericSchema.STRING, 'k2': GenericSchema.NUMBER, }) d = schema.transform({ 'k1': np.float64(1.0), 'k2': np.float64(1.0), }) self.assertEqual({'k1': '1.0'}, dict(d.string_values)) self.assertEqual({'k2': 1.0}, dict(d.num_values))
def test_numpy(self): schema = GenericSchema({ 'k1': GenericSchema.STRING, 'k2': GenericSchema.NUMBER, }) d = schema.transform({ 'k1': np.float64(1.0), 'k2': np.float64(1.0), }) self.assertEqual({'k1': '1.0'}, dict(d.string_values)) self.assertEqual({'k2': 1.0}, dict(d.num_values))
def test_ignore(self): schema = GenericSchema({ 'k1': GenericSchema.NUMBER, }, GenericSchema.IGNORE) d = schema.transform({ 'k1': '123', 'k2': 456, }) self.assertEqual({}, dict(d.string_values)) self.assertEqual({'k1': 123}, dict(d.num_values)) self.assertEqual({}, dict(d.binary_values))
def test_fallback(self): schema = GenericSchema({ 'k1': GenericSchema.STRING, }, GenericSchema.NUMBER) d = schema.transform({ 'k1': '123', 'k2': 456, 'k3': 789, }) self.assertEqual({'k1': '123'}, dict(d.string_values)) self.assertEqual({'k2': 456, 'k3': 789}, dict(d.num_values))
def test_fallback(self): schema = GenericSchema({ 'k1': GenericSchema.STRING, }, GenericSchema.NUMBER) d = schema.transform({ 'k1': '123', 'k2': 456, 'k3': 789, }) self.assertEqual({'k1': '123'}, dict(d.string_values)) self.assertEqual({'k2': 456, 'k3': 789}, dict(d.num_values))
def test_ignore(self): schema = GenericSchema({ 'k1': GenericSchema.NUMBER, }, GenericSchema.IGNORE) d = schema.transform({ 'k1': '123', 'k2': 456, }) self.assertEqual({}, dict(d.string_values)) self.assertEqual({'k1': 123}, dict(d.num_values)) self.assertEqual({}, dict(d.binary_values))
def test_null(self): schema = GenericSchema({ 'k1': GenericSchema.STRING, 'k2': GenericSchema.NUMBER, }) d = schema.transform({ 'k1': None, 'k2': None, }) self.assertEqual({}, dict(d.string_values)) self.assertEqual({}, dict(d.num_values))
def test_transform_as_datum(self): d = Datum() schema = GenericSchema({ 'k1': GenericSchema.NUMBER, }, GenericSchema.IGNORE) d2 = schema._transform_as_datum({ 'k1': '123', 'k2': 456, }, d) self.assertEqual(d2, d) self.assertEqual({'k1': 123}, dict(d.num_values))
def test_transform_as_datum(self): d = Datum() schema = GenericSchema({ 'k1': GenericSchema.NUMBER, }, GenericSchema.IGNORE) d2 = schema._transform_as_datum({ 'k1': '123', 'k2': 456, }, d) self.assertEqual(d2, d) self.assertEqual({'k1': 123}, dict(d.num_values))
def test_empty(self): schema = GenericSchema({ 'k1': GenericSchema.STRING, 'k2': GenericSchema.NUMBER, 'k3': GenericSchema.NUMBER, }) d = schema.transform({ 'k1': '', 'k2': '', 'k3': b'', }) self.assertEqual({'k1': ''}, dict(d.string_values)) self.assertEqual({}, dict(d.num_values))
def test_empty(self): schema = GenericSchema({ 'k1': GenericSchema.STRING, 'k2': GenericSchema.NUMBER, 'k3': GenericSchema.NUMBER, }) d = schema.transform({ 'k1': '', 'k2': '', 'k3': b'', }) self.assertEqual({'k1': ''}, dict(d.string_values)) self.assertEqual({}, dict(d.num_values))
def test_simple(self): schema = GenericSchema({ 'k1': GenericSchema.STRING, 'k2': GenericSchema.NUMBER, 'k3': GenericSchema.BINARY, }) d = schema.transform({ 'k1': '123', 'k2': 456, 'k3': 'xxx'.encode(), }) self.assertEqual({'k1': '123'}, dict(d.string_values)) self.assertEqual({'k2': 456}, dict(d.num_values)) self.assertEqual({'k3': 'xxx'.encode()}, dict(d.binary_values))
def test_simple(self): schema = GenericSchema({ 'k1': GenericSchema.STRING, 'k2': GenericSchema.NUMBER, 'k3': GenericSchema.BINARY, }) d = schema.transform({ 'k1': '123', 'k2': 456, 'k3': 'xxx'.encode(), }) self.assertEqual({'k1': '123'}, dict(d.string_values)) self.assertEqual({'k2': 456}, dict(d.num_values)) self.assertEqual({'k3': 'xxx'.encode()}, dict(d.binary_values))
def test_predict_typed(self): row = {'num1': 10, 'num2': 10.0, 'num3': 'inf', 'str1': 'abc', 'str2': '0.0.1'} schema = GenericSchema.predict(row, True) d = schema.transform(row) self.assertEqual({'str1': 'abc', 'str2': '0.0.1', 'num3': 'inf'}, dict(d.string_values)) self.assertEqual({'num1': 10, 'num2': 10.0}, dict(d.num_values))
def test_predict_typed(self): row = {'num1': 10, 'num2': 10.0, 'num3': 'inf', 'str1': 'abc', 'str2': '0.0.1'} schema = GenericSchema.predict(row, True) d = schema.transform(row) self.assertEqual({'str1': 'abc', 'str2': '0.0.1', 'num3': 'inf'}, dict(d.string_values)) self.assertEqual({'num1': 10, 'num2': 10.0}, dict(d.num_values))
def test_infer(self): binary_data = 'テスト'.encode('cp932') schema = GenericSchema({}, GenericSchema.INFER) d = schema.transform({ 'k1': '123', 'k2': 456, 'k3': '789'.encode(), 'k4': 'xxx'.encode(), 'k5': binary_data, 'k6': True, 'k7': False, }) self.assertEqual({'k4': 'xxx', 'k6': '1', 'k7': '0'}, dict(d.string_values)) self.assertEqual({'k1': 123, 'k2': 456, 'k3': 789}, dict(d.num_values)) self.assertEqual({'k5': binary_data}, dict(d.binary_values))
def test_unknown_key_name(self): schema = GenericSchema({ 'k1': GenericSchema.STRING, 'k2': GenericSchema.NUMBER, }) self.assertRaises(RuntimeError, schema.transform, { 'k1': '123', 'k2': 456, 'k3': 789, })
def test_number(self): schema = GenericSchema({}, GenericSchema.NUMBER) d = schema.transform({ 'k1': '123'.encode(), # bytes 'k2': '456', # unicode 'k3': 789, # int 'k4': 789.0, # float 'k5': True, # bool 'k6': False, # bool }) self.assertEqual({ 'k1': 123.0, 'k2': 456.0, 'k3': 789.0, 'k4': 789.0, 'k5': 1.0, 'k6': 0.0, }, dict(d.num_values)) self.assertEqual({}, dict(d.string_values)) self.assertEqual({}, dict(d.binary_values))
def test_string(self): schema = GenericSchema({}, GenericSchema.STRING) d = schema.transform({ 'k1': '123'.encode(), # bytes 'k2': '456', # unicode 'k3': 789, # int 'k4': 789.0, # float 'k5': True, # bool 'k6': False, # bool }) self.assertEqual({ 'k1': '123', 'k2': '456', 'k3': '789', 'k4': '789.0', 'k5': '1', 'k6': '0', }, dict(d.string_values)) self.assertEqual({}, dict(d.num_values)) self.assertEqual({}, dict(d.binary_values))
def test_infer(self): binary_data = 'テスト'.encode('cp932') schema = GenericSchema({ 'k1': GenericSchema.INFER, 'k2': GenericSchema.INFER, 'k3': GenericSchema.INFER, 'k4': GenericSchema.INFER, 'k5': GenericSchema.INFER, }) d = schema.transform({ 'k1': '123', 'k2': 456, 'k3': '789'.encode(), 'k4': 'xxx'.encode(), 'k5': binary_data, }) self.assertEqual({'k4': 'xxx'}, dict(d.string_values)) self.assertEqual({'k1': 123, 'k2': 456, 'k3': 789}, dict(d.num_values)) self.assertEqual({'k5': binary_data}, dict(d.binary_values))
def test_number(self): schema = GenericSchema({}, GenericSchema.NUMBER) d = schema.transform({ 'k1': '123'.encode(), # bytes 'k2': '456', # unicode 'k3': 789, # int 'k4': 789.0, # float 'k5': True, # bool 'k6': False, # bool }) self.assertEqual( { 'k1': 123.0, 'k2': 456.0, 'k3': 789.0, 'k4': 789.0, 'k5': 1.0, 'k6': 0.0, }, dict(d.num_values)) self.assertEqual({}, dict(d.string_values)) self.assertEqual({}, dict(d.binary_values))
def test_auto(self): binary_data = 'テスト'.encode('cp932') schema = GenericSchema({ 'k1': GenericSchema.AUTO, 'k2': GenericSchema.AUTO, 'k3': GenericSchema.AUTO, 'k4': GenericSchema.AUTO, 'k5': GenericSchema.AUTO, }) d = schema.transform({ 'k1': '123', 'k2': 456, 'k3': '789'.encode(), 'k4': 'xxx'.encode(), 'k5': binary_data, }) self.assertEqual({'k1': '123'}, dict(d.string_values)) self.assertEqual({'k2': 456}, dict(d.num_values)) self.assertEqual({'k3': '789'.encode(), 'k4': 'xxx'.encode(), 'k5': binary_data}, dict(d.binary_values))
def test_string(self): schema = GenericSchema({}, GenericSchema.STRING) d = schema.transform({ 'k1': '123'.encode(), # bytes 'k2': '456', # unicode 'k3': 789, # int 'k4': 789.0, # float 'k5': True, # bool 'k6': False, # bool }) self.assertEqual( { 'k1': '123', 'k2': '456', 'k3': '789', 'k4': '789.0', 'k5': '1', 'k6': '0', }, dict(d.string_values)) self.assertEqual({}, dict(d.num_values)) self.assertEqual({}, dict(d.binary_values))
def test_invalid_transform(self): schema = GenericSchema({'k1': 'unknown'}) self.assertRaises(RuntimeError, schema.transform, {'k1': '123'})
def test_invalid_predict(self): schema = GenericSchema({'k1': GenericSchema.STRING}) self.assertRaises(ValueError, schema.predict, {'k1': object()}, True)
class BaseDatasetTest(TestCase): SCHEMA = GenericSchema({ 'v': (GenericSchema.NUMBER, 'value'), }) def test_static(self): loader = StubLoader() ds = BaseDataset(loader, self.SCHEMA) self.assertTrue(ds.is_static()) self.assertEqual(3, len(ds)) self.assertEqual({'value': 1}, dict(ds[0].num_values)) self.assertEqual({'value': 2}, dict(ds[1].num_values)) self.assertEqual({'value': 3}, dict(ds[2].num_values)) self.assertEqual({'v': 1}, dict(ds.get(0))) self.assertEqual({'v': 2}, dict(ds.get(1))) self.assertEqual({'v': 3}, dict(ds.get(2))) ds2 = ds[(1, 2)] self.assertEqual(2, len(ds2)) self.assertEqual({'value': 2}, dict(ds2[0].num_values)) self.assertEqual({'value': 3}, dict(ds2[1].num_values)) self.assertEqual({'v': 2}, dict(ds2.get(0))) self.assertEqual({'v': 3}, dict(ds2.get(1))) expected_idx = 0 for (idx, row) in ds: self.assertEqual(expected_idx, idx) self.assertEqual({'value': idx + 1}, dict(row.num_values)) expected_idx += 1 def test_invalid_infinite_static(self): loader = StubInfiniteLoader() # infinite loader self.assertRaises(RuntimeError, BaseDataset, loader, None, True) # cannot be static def test_predict(self): loader = StubLoader() ds = BaseDataset(loader, self.SCHEMA) self.assertRaises(NotImplementedError, ds._predict, {}) def test_get_schema(self): loader = StubLoader() ds = BaseDataset(loader, self.SCHEMA) self.assertEqual(self.SCHEMA, ds.get_schema()) def test_shuffle(self): loader = StubLoader() ds = BaseDataset(loader, self.SCHEMA).shuffle(0) rows = [] for (_, row) in ds: rows.append(row) row_values = [x.num_values[0][1] for x in rows] self.assertEqual([1, 2, 3], sorted(row_values)) def test_convert(self): loader = StubLoader() ds1 = BaseDataset(loader, self.SCHEMA) def f(d): new_d = {} for (k, v) in d.items(): new_d[k] = d[k] + 1 return new_d ds2 = ds1.convert(lambda data: [f(d) for d in data]) self.assertEqual(1, ds1[0].num_values[0][1]) self.assertEqual(2, ds2[0].num_values[0][1]) def test_convert_empty(self): loader = StubLoader() ds1 = BaseDataset(loader, self.SCHEMA) ds2 = ds1.convert(lambda data: [None for d in data]) # ds2 should be empty for d in ds2: self.fail() def test_invalid_convert(self): loader = StubLoader() ds1 = BaseDataset(loader, self.SCHEMA) self.assertRaises(RuntimeError, ds1.convert, lambda x: None) def test_nonstatic(self): loader = StubLoader() ds = BaseDataset(loader, self.SCHEMA, False) self.assertFalse(ds.is_static()) expected_idx = 0 for (idx, row) in ds: self.assertEqual(expected_idx, idx) self.assertEqual({'value': idx + 1}, dict(row.num_values)) self.assertEqual(row.num_values, ds[idx].num_values) self.assertEqual({'v': idx + 1}, ds.get(idx)) expected_idx += 1 def test_nonstatic_ops(self): loader = StubLoader() ds = BaseDataset(loader, self.SCHEMA, False) self.assertRaises(RuntimeError, ds.shuffle, 0) self.assertRaises(RuntimeError, ds.convert, lambda x: x) self.assertRaises(RuntimeError, ds.get, 0) self.assertRaises(RuntimeError, len, ds) self.assertRaises(RuntimeError, lambda: ds[0]) def test_infinite(self): loader = StubInfiniteLoader() ds = BaseDataset(loader, self.SCHEMA) self.assertFalse(ds.is_static()) expected_idx = 0 for (idx, row) in ds: self.assertEqual(expected_idx, idx) self.assertEqual({'value': idx + 1}, dict(row.num_values)) expected_idx += 1 if 10 < expected_idx: break def test_index_access(self): loader = StubLoader() ds = BaseDataset(loader, self.SCHEMA) self.assertTrue(isinstance(ds[0], jubatus.common.Datum)) self.assertTrue(isinstance(ds[0:1], BaseDataset))