def test_auto(self):
binary_data = 'テスト'.encode('cp932')
schema = GenericSchema({}, GenericSchema.AUTO)
d = schema.transform({
'k1': '123',
'k2': 456,
'k3': '789'.encode(),
'k4': 'xxx'.encode(),
'k5': binary_data,
'k6': True,
'k7': False,
})
self.assertEqual({
'k1': '123',
'k6': '1',
'k7': '0'
}, dict(d.string_values))
self.assertEqual({'k2': 456}, dict(d.num_values))
self.assertEqual(
{
'k3': '789'.encode(),
'k4': 'xxx'.encode(),
'k5': binary_data
}, dict(d.binary_values))
def test_alias(self):
schema = GenericSchema({
'k1': GenericSchema.NUMBER,
'k2': (GenericSchema.NUMBER, 'alias_name'),
})
d = schema.transform({
'k1': '123',
'k2': 456,
})
self.assertEqual({'k1': 123, 'alias_name': 456}, dict(d.num_values))
def test_alias(self):
schema = GenericSchema({
'k1': GenericSchema.NUMBER,
'k2': (GenericSchema.NUMBER, 'alias_name'),
})
d = schema.transform({
'k1': '123',
'k2': 456,
})
self.assertEqual({'k1': 123, 'alias_name': 456}, dict(d.num_values))
def test_null(self):
schema = GenericSchema({
'k1': GenericSchema.STRING,
'k2': GenericSchema.NUMBER,
})
d = schema.transform({
'k1': None,
'k2': None,
})
self.assertEqual({}, dict(d.string_values))
self.assertEqual({}, dict(d.num_values))
def test_numpy(self):
schema = GenericSchema({
'k1': GenericSchema.STRING,
'k2': GenericSchema.NUMBER,
})
d = schema.transform({
'k1': np.float64(1.0),
'k2': np.float64(1.0),
})
self.assertEqual({'k1': '1.0'}, dict(d.string_values))
self.assertEqual({'k2': 1.0}, dict(d.num_values))
def test_numpy(self):
schema = GenericSchema({
'k1': GenericSchema.STRING,
'k2': GenericSchema.NUMBER,
})
d = schema.transform({
'k1': np.float64(1.0),
'k2': np.float64(1.0),
})
self.assertEqual({'k1': '1.0'}, dict(d.string_values))
self.assertEqual({'k2': 1.0}, dict(d.num_values))
def test_ignore(self):
schema = GenericSchema({
'k1': GenericSchema.NUMBER,
}, GenericSchema.IGNORE)
d = schema.transform({
'k1': '123',
'k2': 456,
})
self.assertEqual({}, dict(d.string_values))
self.assertEqual({'k1': 123}, dict(d.num_values))
self.assertEqual({}, dict(d.binary_values))
def test_fallback(self):
schema = GenericSchema({
'k1': GenericSchema.STRING,
}, GenericSchema.NUMBER)
d = schema.transform({
'k1': '123',
'k2': 456,
'k3': 789,
})
self.assertEqual({'k1': '123'}, dict(d.string_values))
self.assertEqual({'k2': 456, 'k3': 789}, dict(d.num_values))
def test_fallback(self):
schema = GenericSchema({
'k1': GenericSchema.STRING,
}, GenericSchema.NUMBER)
d = schema.transform({
'k1': '123',
'k2': 456,
'k3': 789,
})
self.assertEqual({'k1': '123'}, dict(d.string_values))
self.assertEqual({'k2': 456, 'k3': 789}, dict(d.num_values))
def test_ignore(self):
schema = GenericSchema({
'k1': GenericSchema.NUMBER,
}, GenericSchema.IGNORE)
d = schema.transform({
'k1': '123',
'k2': 456,
})
self.assertEqual({}, dict(d.string_values))
self.assertEqual({'k1': 123}, dict(d.num_values))
self.assertEqual({}, dict(d.binary_values))
def test_null(self):
schema = GenericSchema({
'k1': GenericSchema.STRING,
'k2': GenericSchema.NUMBER,
})
d = schema.transform({
'k1': None,
'k2': None,
})
self.assertEqual({}, dict(d.string_values))
self.assertEqual({}, dict(d.num_values))
def test_transform_as_datum(self):
d = Datum()
schema = GenericSchema({
'k1': GenericSchema.NUMBER,
}, GenericSchema.IGNORE)
d2 = schema._transform_as_datum({
'k1': '123',
'k2': 456,
}, d)
self.assertEqual(d2, d)
self.assertEqual({'k1': 123}, dict(d.num_values))
def test_transform_as_datum(self):
d = Datum()
schema = GenericSchema({
'k1': GenericSchema.NUMBER,
}, GenericSchema.IGNORE)
d2 = schema._transform_as_datum({
'k1': '123',
'k2': 456,
}, d)
self.assertEqual(d2, d)
self.assertEqual({'k1': 123}, dict(d.num_values))
def test_empty(self):
schema = GenericSchema({
'k1': GenericSchema.STRING,
'k2': GenericSchema.NUMBER,
'k3': GenericSchema.NUMBER,
})
d = schema.transform({
'k1': '',
'k2': '',
'k3': b'',
})
self.assertEqual({'k1': ''}, dict(d.string_values))
self.assertEqual({}, dict(d.num_values))
def test_empty(self):
schema = GenericSchema({
'k1': GenericSchema.STRING,
'k2': GenericSchema.NUMBER,
'k3': GenericSchema.NUMBER,
})
d = schema.transform({
'k1': '',
'k2': '',
'k3': b'',
})
self.assertEqual({'k1': ''}, dict(d.string_values))
self.assertEqual({}, dict(d.num_values))
def test_simple(self):
schema = GenericSchema({
'k1': GenericSchema.STRING,
'k2': GenericSchema.NUMBER,
'k3': GenericSchema.BINARY,
})
d = schema.transform({
'k1': '123',
'k2': 456,
'k3': 'xxx'.encode(),
})
self.assertEqual({'k1': '123'}, dict(d.string_values))
self.assertEqual({'k2': 456}, dict(d.num_values))
self.assertEqual({'k3': 'xxx'.encode()}, dict(d.binary_values))
def test_simple(self):
schema = GenericSchema({
'k1': GenericSchema.STRING,
'k2': GenericSchema.NUMBER,
'k3': GenericSchema.BINARY,
})
d = schema.transform({
'k1': '123',
'k2': 456,
'k3': 'xxx'.encode(),
})
self.assertEqual({'k1': '123'}, dict(d.string_values))
self.assertEqual({'k2': 456}, dict(d.num_values))
self.assertEqual({'k3': 'xxx'.encode()}, dict(d.binary_values))
def test_predict_typed(self):
row = {'num1': 10, 'num2': 10.0, 'num3': 'inf', 'str1': 'abc', 'str2': '0.0.1'}
schema = GenericSchema.predict(row, True)
d = schema.transform(row)
self.assertEqual({'str1': 'abc', 'str2': '0.0.1', 'num3': 'inf'}, dict(d.string_values))
self.assertEqual({'num1': 10, 'num2': 10.0}, dict(d.num_values))
def test_predict_typed(self):
row = {'num1': 10, 'num2': 10.0, 'num3': 'inf', 'str1': 'abc', 'str2': '0.0.1'}
schema = GenericSchema.predict(row, True)
d = schema.transform(row)
self.assertEqual({'str1': 'abc', 'str2': '0.0.1', 'num3': 'inf'}, dict(d.string_values))
self.assertEqual({'num1': 10, 'num2': 10.0}, dict(d.num_values))
def test_infer(self):
binary_data = 'テスト'.encode('cp932')
schema = GenericSchema({}, GenericSchema.INFER)
d = schema.transform({
'k1': '123',
'k2': 456,
'k3': '789'.encode(),
'k4': 'xxx'.encode(),
'k5': binary_data,
'k6': True,
'k7': False,
})
self.assertEqual({'k4': 'xxx', 'k6': '1', 'k7': '0'}, dict(d.string_values))
self.assertEqual({'k1': 123, 'k2': 456, 'k3': 789}, dict(d.num_values))
self.assertEqual({'k5': binary_data}, dict(d.binary_values))
def test_unknown_key_name(self):
schema = GenericSchema({
'k1': GenericSchema.STRING,
'k2': GenericSchema.NUMBER,
})
self.assertRaises(RuntimeError, schema.transform, {
'k1': '123',
'k2': 456,
'k3': 789,
})
def test_number(self):
schema = GenericSchema({}, GenericSchema.NUMBER)
d = schema.transform({
'k1': '123'.encode(), # bytes
'k2': '456', # unicode
'k3': 789, # int
'k4': 789.0, # float
'k5': True, # bool
'k6': False, # bool
})
self.assertEqual({
'k1': 123.0,
'k2': 456.0,
'k3': 789.0,
'k4': 789.0,
'k5': 1.0,
'k6': 0.0,
}, dict(d.num_values))
self.assertEqual({}, dict(d.string_values))
self.assertEqual({}, dict(d.binary_values))
def test_string(self):
schema = GenericSchema({}, GenericSchema.STRING)
d = schema.transform({
'k1': '123'.encode(), # bytes
'k2': '456', # unicode
'k3': 789, # int
'k4': 789.0, # float
'k5': True, # bool
'k6': False, # bool
})
self.assertEqual({
'k1': '123',
'k2': '456',
'k3': '789',
'k4': '789.0',
'k5': '1',
'k6': '0',
}, dict(d.string_values))
self.assertEqual({}, dict(d.num_values))
self.assertEqual({}, dict(d.binary_values))
def test_infer(self):
binary_data = 'テスト'.encode('cp932')
schema = GenericSchema({
'k1': GenericSchema.INFER,
'k2': GenericSchema.INFER,
'k3': GenericSchema.INFER,
'k4': GenericSchema.INFER,
'k5': GenericSchema.INFER,
})
d = schema.transform({
'k1': '123',
'k2': 456,
'k3': '789'.encode(),
'k4': 'xxx'.encode(),
'k5': binary_data,
})
self.assertEqual({'k4': 'xxx'}, dict(d.string_values))
self.assertEqual({'k1': 123, 'k2': 456, 'k3': 789}, dict(d.num_values))
self.assertEqual({'k5': binary_data}, dict(d.binary_values))
def test_number(self):
schema = GenericSchema({}, GenericSchema.NUMBER)
d = schema.transform({
'k1': '123'.encode(), # bytes
'k2': '456', # unicode
'k3': 789, # int
'k4': 789.0, # float
'k5': True, # bool
'k6': False, # bool
})
self.assertEqual(
{
'k1': 123.0,
'k2': 456.0,
'k3': 789.0,
'k4': 789.0,
'k5': 1.0,
'k6': 0.0,
}, dict(d.num_values))
self.assertEqual({}, dict(d.string_values))
self.assertEqual({}, dict(d.binary_values))
def test_auto(self):
binary_data = 'テスト'.encode('cp932')
schema = GenericSchema({
'k1': GenericSchema.AUTO,
'k2': GenericSchema.AUTO,
'k3': GenericSchema.AUTO,
'k4': GenericSchema.AUTO,
'k5': GenericSchema.AUTO,
})
d = schema.transform({
'k1': '123',
'k2': 456,
'k3': '789'.encode(),
'k4': 'xxx'.encode(),
'k5': binary_data,
})
self.assertEqual({'k1': '123'}, dict(d.string_values))
self.assertEqual({'k2': 456}, dict(d.num_values))
self.assertEqual({'k3': '789'.encode(), 'k4': 'xxx'.encode(), 'k5': binary_data}, dict(d.binary_values))
def test_string(self):
schema = GenericSchema({}, GenericSchema.STRING)
d = schema.transform({
'k1': '123'.encode(), # bytes
'k2': '456', # unicode
'k3': 789, # int
'k4': 789.0, # float
'k5': True, # bool
'k6': False, # bool
})
self.assertEqual(
{
'k1': '123',
'k2': '456',
'k3': '789',
'k4': '789.0',
'k5': '1',
'k6': '0',
}, dict(d.string_values))
self.assertEqual({}, dict(d.num_values))
self.assertEqual({}, dict(d.binary_values))
def test_invalid_transform(self):
schema = GenericSchema({'k1': 'unknown'})
self.assertRaises(RuntimeError, schema.transform, {'k1': '123'})
def test_invalid_predict(self):
schema = GenericSchema({'k1': GenericSchema.STRING})
self.assertRaises(ValueError, schema.predict, {'k1': object()}, True)
class BaseDatasetTest(TestCase):
SCHEMA = GenericSchema({
'v': (GenericSchema.NUMBER, 'value'),
})
def test_static(self):
loader = StubLoader()
ds = BaseDataset(loader, self.SCHEMA)
self.assertTrue(ds.is_static())
self.assertEqual(3, len(ds))
self.assertEqual({'value': 1}, dict(ds[0].num_values))
self.assertEqual({'value': 2}, dict(ds[1].num_values))
self.assertEqual({'value': 3}, dict(ds[2].num_values))
self.assertEqual({'v': 1}, dict(ds.get(0)))
self.assertEqual({'v': 2}, dict(ds.get(1)))
self.assertEqual({'v': 3}, dict(ds.get(2)))
ds2 = ds[(1, 2)]
self.assertEqual(2, len(ds2))
self.assertEqual({'value': 2}, dict(ds2[0].num_values))
self.assertEqual({'value': 3}, dict(ds2[1].num_values))
self.assertEqual({'v': 2}, dict(ds2.get(0)))
self.assertEqual({'v': 3}, dict(ds2.get(1)))
expected_idx = 0
for (idx, row) in ds:
self.assertEqual(expected_idx, idx)
self.assertEqual({'value': idx + 1}, dict(row.num_values))
expected_idx += 1
def test_invalid_infinite_static(self):
loader = StubInfiniteLoader() # infinite loader
self.assertRaises(RuntimeError, BaseDataset, loader, None,
True) # cannot be static
def test_predict(self):
loader = StubLoader()
ds = BaseDataset(loader, self.SCHEMA)
self.assertRaises(NotImplementedError, ds._predict, {})
def test_get_schema(self):
loader = StubLoader()
ds = BaseDataset(loader, self.SCHEMA)
self.assertEqual(self.SCHEMA, ds.get_schema())
def test_shuffle(self):
loader = StubLoader()
ds = BaseDataset(loader, self.SCHEMA).shuffle(0)
rows = []
for (_, row) in ds:
rows.append(row)
row_values = [x.num_values[0][1] for x in rows]
self.assertEqual([1, 2, 3], sorted(row_values))
def test_convert(self):
loader = StubLoader()
ds1 = BaseDataset(loader, self.SCHEMA)
def f(d):
new_d = {}
for (k, v) in d.items():
new_d[k] = d[k] + 1
return new_d
ds2 = ds1.convert(lambda data: [f(d) for d in data])
self.assertEqual(1, ds1[0].num_values[0][1])
self.assertEqual(2, ds2[0].num_values[0][1])
def test_convert_empty(self):
loader = StubLoader()
ds1 = BaseDataset(loader, self.SCHEMA)
ds2 = ds1.convert(lambda data: [None
for d in data]) # ds2 should be empty
for d in ds2:
self.fail()
def test_invalid_convert(self):
loader = StubLoader()
ds1 = BaseDataset(loader, self.SCHEMA)
self.assertRaises(RuntimeError, ds1.convert, lambda x: None)
def test_nonstatic(self):
loader = StubLoader()
ds = BaseDataset(loader, self.SCHEMA, False)
self.assertFalse(ds.is_static())
expected_idx = 0
for (idx, row) in ds:
self.assertEqual(expected_idx, idx)
self.assertEqual({'value': idx + 1}, dict(row.num_values))
self.assertEqual(row.num_values, ds[idx].num_values)
self.assertEqual({'v': idx + 1}, ds.get(idx))
expected_idx += 1
def test_nonstatic_ops(self):
loader = StubLoader()
ds = BaseDataset(loader, self.SCHEMA, False)
self.assertRaises(RuntimeError, ds.shuffle, 0)
self.assertRaises(RuntimeError, ds.convert, lambda x: x)
self.assertRaises(RuntimeError, ds.get, 0)
self.assertRaises(RuntimeError, len, ds)
self.assertRaises(RuntimeError, lambda: ds[0])
def test_infinite(self):
loader = StubInfiniteLoader()
ds = BaseDataset(loader, self.SCHEMA)
self.assertFalse(ds.is_static())
expected_idx = 0
for (idx, row) in ds:
self.assertEqual(expected_idx, idx)
self.assertEqual({'value': idx + 1}, dict(row.num_values))
expected_idx += 1
if 10 < expected_idx:
break
def test_index_access(self):
loader = StubLoader()
ds = BaseDataset(loader, self.SCHEMA)
self.assertTrue(isinstance(ds[0], jubatus.common.Datum))
self.assertTrue(isinstance(ds[0:1], BaseDataset))