def test_tuple_coder(self): kv_coder = coders.TupleCoder( (coders.VarIntCoder(), coders.BytesCoder())) # Verify cloud object representation self.assertEqual( { '@type': 'kind:pair', 'is_pair_like': True, 'component_encodings': [ coders.VarIntCoder().as_cloud_object(), coders.BytesCoder().as_cloud_object() ], }, kv_coder.as_cloud_object()) # Test binary representation self.assertEqual('\x04abc', kv_coder.encode((4, 'abc'))) # Test unnested self.check_coder(kv_coder, (1, 'a'), (-2, 'a' * 100), (300, 'abc\0' * 5)) # Test nested self.check_coder( coders.TupleCoder((coders.TupleCoder( (coders.PickleCoder(), coders.VarIntCoder())), coders.StrUtf8Coder())), ((1, 2), 'a'), ((-2, 5), u'a\u0101' * 100), ((300, 1), 'abc\0' * 5))
def test_windowed_value_coder(self): coder = coders.WindowedValueCoder(coders.VarIntCoder(), coders.GlobalWindowCoder()) # Verify cloud object representation self.assertEqual( { '@type': 'kind:windowed_value', 'is_wrapper': True, 'component_encodings': [ coders.VarIntCoder().as_cloud_object(), coders.GlobalWindowCoder().as_cloud_object(), ], }, coder.as_cloud_object()) # Test binary representation self.assertEqual( '\x01\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01', coder.encode(window.GlobalWindows.windowed_value(1))) # Test unnested self.check_coder(coders.WindowedValueCoder(coders.VarIntCoder()), windowed_value.WindowedValue(3, -100, ()), windowed_value.WindowedValue(-1, 100, (1, 2, 3))) # Test nested self.check_coder( coders.TupleCoder( (coders.WindowedValueCoder(coders.FloatCoder()), coders.WindowedValueCoder(coders.StrUtf8Coder()))), (windowed_value.WindowedValue(1.5, 0, ()), windowed_value.WindowedValue("abc", 10, ('window', ))))
def test_tuple_coder(self): self.check_coder( coders.TupleCoder((coders.VarIntCoder(), coders.BytesCoder())), (1, 'a'), (-2, 'a' * 100), (300, 'abc\0' * 5)) self.check_coder( coders.TupleCoder((coders.TupleCoder( (coders.PickleCoder(), coders.VarIntCoder())), coders.StrUtf8Coder())), ((1, 2), 'a'), ((-2, 5), u'a\u0101' * 100), ((300, 1), 'abc\0' * 5))
def test_varint_coder(self): # Small ints. self.check_coder(coders.VarIntCoder(), *range(-10, 10)) # Multi-byte encoding starts at 128 self.check_coder(coders.VarIntCoder(), *range(120, 140)) # Large values self.check_coder( coders.VarIntCoder(), *[ int(math.pow(-1, k) * math.exp(k)) for k in range(0, int(math.log(sys.maxint))) ])
def test_iterable_coder(self): iterable_coder = coders.IterableCoder(coders.VarIntCoder()) # Verify cloud object representation self.assertEqual( { '@type': 'kind:stream', 'is_stream_like': True, 'component_encodings': [coders.VarIntCoder().as_cloud_object()] }, iterable_coder.as_cloud_object()) # Test unnested self.check_coder(iterable_coder, [1], [-1, 0, 100]) # Test nested self.check_coder( coders.TupleCoder((coders.VarIntCoder(), coders.IterableCoder(coders.VarIntCoder()))), (1, [1, 2, 3]))
def _test_iterable_coder_of_unknown_length(self, count): def iter_generator(count): for i in range(count): yield i iterable_coder = coders.IterableCoder(coders.VarIntCoder()) self.assertItemsEqual( list(iter_generator(count)), iterable_coder.decode(iterable_coder.encode( iter_generator(count))))
def test_nested_observables(self): class FakeObservableIterator(observable.ObservableMixin): def __iter__(self): return iter([1, 2, 3]) # Coder for elements from the observable iterator. elem_coder = coders.VarIntCoder() iter_coder = coders.TupleSequenceCoder(elem_coder) # Test nested WindowedValue observable. coder = coders.WindowedValueCoder(iter_coder) observ = FakeObservableIterator() value = windowed_value.WindowedValue(observ, 0, ()) self.assertEqual( coder.get_impl().get_estimated_size_and_observables(value)[1], [(observ, elem_coder.get_impl())]) # Test nested tuple observable. coder = coders.TupleCoder((coders.StrUtf8Coder(), iter_coder)) value = (u'123', observ) self.assertEqual( coder.get_impl().get_estimated_size_and_observables(value)[1], [(observ, elem_coder.get_impl())])
def test_dill_coder(self): cell_value = (lambda x: lambda: x)(0).func_closure[0] self.check_coder(coders.DillCoder(), 'a', 1, cell_value) self.check_coder( coders.TupleCoder((coders.VarIntCoder(), coders.DillCoder())), (1, cell_value))
def test_tuple_sequence_coder(self): int_tuple_coder = coders.TupleSequenceCoder(coders.VarIntCoder()) self.check_coder(int_tuple_coder, (1, -1, 0), (), tuple(range(1000))) self.check_coder( coders.TupleCoder((coders.VarIntCoder(), int_tuple_coder)), (1, (1, 2, 3)))