def test_tuple_coder(self):
kv_coder = coders.TupleCoder(
(coders.VarIntCoder(), coders.BytesCoder()))
# Verify cloud object representation
self.assertEqual(
{
'@type':
'kind:pair',
'is_pair_like':
True,
'component_encodings': [
coders.VarIntCoder().as_cloud_object(),
coders.BytesCoder().as_cloud_object()
],
}, kv_coder.as_cloud_object())
# Test binary representation
self.assertEqual('\x04abc', kv_coder.encode((4, 'abc')))
# Test unnested
self.check_coder(kv_coder, (1, 'a'), (-2, 'a' * 100),
(300, 'abc\0' * 5))
# Test nested
self.check_coder(
coders.TupleCoder((coders.TupleCoder(
(coders.PickleCoder(), coders.VarIntCoder())),
coders.StrUtf8Coder())), ((1, 2), 'a'),
((-2, 5), u'a\u0101' * 100), ((300, 1), 'abc\0' * 5))
def test_windowed_value_coder(self):
coder = coders.WindowedValueCoder(coders.VarIntCoder(),
coders.GlobalWindowCoder())
# Verify cloud object representation
self.assertEqual(
{
'@type':
'kind:windowed_value',
'is_wrapper':
True,
'component_encodings': [
coders.VarIntCoder().as_cloud_object(),
coders.GlobalWindowCoder().as_cloud_object(),
],
}, coder.as_cloud_object())
# Test binary representation
self.assertEqual(
'\x01\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01',
coder.encode(window.GlobalWindows.windowed_value(1)))
# Test unnested
self.check_coder(coders.WindowedValueCoder(coders.VarIntCoder()),
windowed_value.WindowedValue(3, -100, ()),
windowed_value.WindowedValue(-1, 100, (1, 2, 3)))
# Test nested
self.check_coder(
coders.TupleCoder(
(coders.WindowedValueCoder(coders.FloatCoder()),
coders.WindowedValueCoder(coders.StrUtf8Coder()))),
(windowed_value.WindowedValue(1.5, 0, ()),
windowed_value.WindowedValue("abc", 10, ('window', ))))
def test_tuple_coder(self):
self.check_coder(
coders.TupleCoder((coders.VarIntCoder(), coders.BytesCoder())),
(1, 'a'), (-2, 'a' * 100), (300, 'abc\0' * 5))
self.check_coder(
coders.TupleCoder((coders.TupleCoder(
(coders.PickleCoder(), coders.VarIntCoder())),
coders.StrUtf8Coder())), ((1, 2), 'a'),
((-2, 5), u'a\u0101' * 100), ((300, 1), 'abc\0' * 5))
def test_varint_coder(self):
# Small ints.
self.check_coder(coders.VarIntCoder(), *range(-10, 10))
# Multi-byte encoding starts at 128
self.check_coder(coders.VarIntCoder(), *range(120, 140))
# Large values
self.check_coder(
coders.VarIntCoder(), *[
int(math.pow(-1, k) * math.exp(k))
for k in range(0, int(math.log(sys.maxint)))
])
def test_iterable_coder(self):
iterable_coder = coders.IterableCoder(coders.VarIntCoder())
# Verify cloud object representation
self.assertEqual(
{
'@type': 'kind:stream',
'is_stream_like': True,
'component_encodings':
[coders.VarIntCoder().as_cloud_object()]
}, iterable_coder.as_cloud_object())
# Test unnested
self.check_coder(iterable_coder, [1], [-1, 0, 100])
# Test nested
self.check_coder(
coders.TupleCoder((coders.VarIntCoder(),
coders.IterableCoder(coders.VarIntCoder()))),
(1, [1, 2, 3]))
def _test_iterable_coder_of_unknown_length(self, count):
def iter_generator(count):
for i in range(count):
yield i
iterable_coder = coders.IterableCoder(coders.VarIntCoder())
self.assertItemsEqual(
list(iter_generator(count)),
iterable_coder.decode(iterable_coder.encode(
iter_generator(count))))
def test_nested_observables(self):
class FakeObservableIterator(observable.ObservableMixin):
def __iter__(self):
return iter([1, 2, 3])
# Coder for elements from the observable iterator.
elem_coder = coders.VarIntCoder()
iter_coder = coders.TupleSequenceCoder(elem_coder)
# Test nested WindowedValue observable.
coder = coders.WindowedValueCoder(iter_coder)
observ = FakeObservableIterator()
value = windowed_value.WindowedValue(observ, 0, ())
self.assertEqual(
coder.get_impl().get_estimated_size_and_observables(value)[1],
[(observ, elem_coder.get_impl())])
# Test nested tuple observable.
coder = coders.TupleCoder((coders.StrUtf8Coder(), iter_coder))
value = (u'123', observ)
self.assertEqual(
coder.get_impl().get_estimated_size_and_observables(value)[1],
[(observ, elem_coder.get_impl())])
def test_dill_coder(self):
cell_value = (lambda x: lambda: x)(0).func_closure[0]
self.check_coder(coders.DillCoder(), 'a', 1, cell_value)
self.check_coder(
coders.TupleCoder((coders.VarIntCoder(), coders.DillCoder())),
(1, cell_value))
def test_tuple_sequence_coder(self):
int_tuple_coder = coders.TupleSequenceCoder(coders.VarIntCoder())
self.check_coder(int_tuple_coder, (1, -1, 0), (), tuple(range(1000)))
self.check_coder(
coders.TupleCoder((coders.VarIntCoder(), int_tuple_coder)),
(1, (1, 2, 3)))
