def test_tuple_coder(self): kv_coder = coders.TupleCoder( (coders.VarIntCoder(), coders.BytesCoder())) # Verify cloud object representation self.assertEqual( { '@type': 'kind:pair', 'is_pair_like': True, 'component_encodings': [ coders.VarIntCoder().as_cloud_object(), coders.BytesCoder().as_cloud_object() ], }, kv_coder.as_cloud_object()) # Test binary representation self.assertEqual('\x04abc', kv_coder.encode((4, 'abc'))) # Test unnested self.check_coder(kv_coder, (1, 'a'), (-2, 'a' * 100), (300, 'abc\0' * 5)) # Test nested self.check_coder( coders.TupleCoder((coders.TupleCoder( (coders.PickleCoder(), coders.VarIntCoder())), coders.StrUtf8Coder())), ((1, 2), 'a'), ((-2, 5), u'a\u0101' * 100), ((300, 1), 'abc\0' * 5))
def test_tuple_coder(self): self.check_coder( coders.TupleCoder((coders.VarIntCoder(), coders.BytesCoder())), (1, 'a'), (-2, 'a' * 100), (300, 'abc\0' * 5)) self.check_coder( coders.TupleCoder((coders.TupleCoder( (coders.PickleCoder(), coders.VarIntCoder())), coders.StrUtf8Coder())), ((1, 2), 'a'), ((-2, 5), u'a\u0101' * 100), ((300, 1), 'abc\0' * 5))
def test_windowed_value_coder(self): coder = coders.WindowedValueCoder(coders.VarIntCoder(), coders.GlobalWindowCoder()) # Verify cloud object representation self.assertEqual( { '@type': 'kind:windowed_value', 'is_wrapper': True, 'component_encodings': [ coders.VarIntCoder().as_cloud_object(), coders.GlobalWindowCoder().as_cloud_object(), ], }, coder.as_cloud_object()) # Test binary representation self.assertEqual( '\x01\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01', coder.encode(window.GlobalWindows.windowed_value(1))) # Test unnested self.check_coder(coders.WindowedValueCoder(coders.VarIntCoder()), windowed_value.WindowedValue(3, -100, ()), windowed_value.WindowedValue(-1, 100, (1, 2, 3))) # Test nested self.check_coder( coders.TupleCoder( (coders.WindowedValueCoder(coders.FloatCoder()), coders.WindowedValueCoder(coders.StrUtf8Coder()))), (windowed_value.WindowedValue(1.5, 0, ()), windowed_value.WindowedValue("abc", 10, ('window', ))))
def test_interval_window_coder(self): self.check_coder( coders.IntervalWindowCoder(), *[ window.IntervalWindow(x, y) for x in [-2**52, 0, 2**52] for y in range(-100, 100) ]) self.check_coder(coders.TupleCoder((coders.IntervalWindowCoder(), )), (window.IntervalWindow(0, 10), ))
def test_singleton_coder(self): a = 'anything' b = 'something else' self.check_coder(coders.SingletonCoder(a), a) self.check_coder(coders.SingletonCoder(b), b) self.check_coder( coders.TupleCoder( (coders.SingletonCoder(a), coders.SingletonCoder(b))), (a, b))
def test_float_coder(self): self.check_coder(coders.FloatCoder(), *[float(0.1 * x) for x in range(-100, 100)]) self.check_coder(coders.FloatCoder(), *[float(2**(0.1 * x)) for x in range(-100, 100)]) self.check_coder(coders.FloatCoder(), float('-Inf'), float('Inf')) self.check_coder( coders.TupleCoder((coders.FloatCoder(), coders.FloatCoder())), (0, 1), (-100, 100), (0.5, 0.25))
def test_deterministic_pickle_coder(self): coder = coders.DeterministicPickleCoder(coders.PickleCoder(), 'step') self.check_coder(coder, 'a', 1, 1.5, (1, 2, 3)) with self.assertRaises(TypeError): self.check_coder(coder, dict()) with self.assertRaises(TypeError): self.check_coder(coder, [1, dict()]) self.check_coder(coders.TupleCoder((coder, coders.PickleCoder())), (1, dict()), ('a', [dict()]))
def test_fast_primitives_coder(self): coder = coders.FastPrimitivesCoder(coders.SingletonCoder(len)) self.check_coder(coder, None, 1, -1, 1.5, 'str\0str', u'unicode\0\u0101') self.check_coder(coder, (), (1, 2, 3)) self.check_coder(coder, [], [1, 2, 3]) self.check_coder(coder, dict(), {'a': 'b'}, {0: dict(), 1: len}) self.check_coder(coder, set(), {'a', 'b'}) self.check_coder(coder, True, False) self.check_coder(coder, len) self.check_coder(coders.TupleCoder((coder, )), ('a', ), (1, ))
def test_deterministic_coder(self): coder = coders.FastPrimitivesCoder() deterministic_coder = coders.DeterministicFastPrimitivesCoder( coder, 'step') self.check_coder(deterministic_coder, 'a', 1, 1.5, (1, 2, 3)) with self.assertRaises(TypeError): self.check_coder(deterministic_coder, dict()) with self.assertRaises(TypeError): self.check_coder(deterministic_coder, [1, dict()]) self.check_coder(coders.TupleCoder((deterministic_coder, coder)), (1, dict()), ('a', [dict()]))
def test_timestamp_coder(self): self.check_coder( coders.TimestampCoder(), *[timestamp.Timestamp(micros=x) for x in range(-100, 100)]) self.check_coder(coders.TimestampCoder(), timestamp.Timestamp(micros=-1234567890), timestamp.Timestamp(micros=1234567890)) self.check_coder(coders.TimestampCoder(), timestamp.Timestamp(micros=-1234567890123456789), timestamp.Timestamp(micros=1234567890123456789)) self.check_coder( coders.TupleCoder((coders.TimestampCoder(), coders.BytesCoder())), (timestamp.Timestamp.of(27), 'abc'))
def test_global_window_coder(self): coder = coders.GlobalWindowCoder() value = window.GlobalWindow() # Verify cloud object representation self.assertEqual({'@type': 'kind:global_window'}, coder.as_cloud_object()) # Test binary representation self.assertEqual('', coder.encode(value)) self.assertEqual(value, coder.decode('')) # Test unnested self.check_coder(coder, value) # Test nested self.check_coder(coders.TupleCoder((coder, coder)), (value, value))
def test_proto_coder(self): # For instructions on how these test proto message were generated, # see coders_test.py ma = test_message.MessageA() mab = ma.field2.add() mab.field1 = True ma.field1 = u'hello world' mb = test_message.MessageA() mb.field1 = u'beam' proto_coder = coders.ProtoCoder(ma.__class__) self.check_coder(proto_coder, ma) self.check_coder(coders.TupleCoder((proto_coder, coders.BytesCoder())), (ma, 'a'), (mb, 'b'))
def test_iterable_coder(self): iterable_coder = coders.IterableCoder(coders.VarIntCoder()) # Verify cloud object representation self.assertEqual( { '@type': 'kind:stream', 'is_stream_like': True, 'component_encodings': [coders.VarIntCoder().as_cloud_object()] }, iterable_coder.as_cloud_object()) # Test unnested self.check_coder(iterable_coder, [1], [-1, 0, 100]) # Test nested self.check_coder( coders.TupleCoder((coders.VarIntCoder(), coders.IterableCoder(coders.VarIntCoder()))), (1, [1, 2, 3]))
def test_length_prefix_coder(self): coder = coders.LengthPrefixCoder(coders.BytesCoder()) # Verify cloud object representation self.assertEqual( { '@type': 'kind:length_prefix', 'component_encodings': [coders.BytesCoder().as_cloud_object()] }, coder.as_cloud_object()) # Test binary representation self.assertEqual('\x00', coder.encode('')) self.assertEqual('\x01a', coder.encode('a')) self.assertEqual('\x02bc', coder.encode('bc')) self.assertEqual('\xff\x7f' + 'z' * 16383, coder.encode('z' * 16383)) # Test unnested self.check_coder(coder, '', 'a', 'bc', 'def') # Test nested self.check_coder(coders.TupleCoder((coder, coder)), ('', 'a'), ('bc', 'def'))
def __setup_caches(self): self.bans = bans.BanStorage(self) self.online_status = OnlineStatus(self.redis, StoragePrefix.ONLINE_STATUS) for conf in Storage._get_cache_configs(): # Initialize Parameters prefix = _prefixize(conf.prefix.value) key_coder = coders.IntCoder().prefixed(prefix) value_coder = (conf.value_coder or protobuf(conf.proto_type))() value_coder = value_coder.compressed() timeout = conf.timeout or 0 if conf.subprefix is None: store = caches.RedisStore(self.redis, timeout=timeout) else: subprefix = _prefixize(conf.subprefix) subcoder = coders.ConstCoder(subprefix) if conf.subcoder is not None: subcoder = conf.subcoder.prefixed(subprefix) key_coder = coders.TupleCoder([key_coder, subcoder]) store = caches.RedisHashStore(self.redis, timeout=timeout) cache = caches.Cache(store, key_coder=key_coder, value_coder=value_coder) setattr(self, conf.attr, cache) # TODO(james7132): Uncomment the above once AggregateProtoHashCache # supports client side caching mapping = [] for conf in Storage._get_cache_configs(): if conf.prefix != StoragePrefix.GUILD_CONFIGS: continue attr = conf.attr if '_configs' in attr: attr = attr.replace('_configs', '') mapping.append((attr, getattr(self, conf.attr))) self.guild_configs = caches.AggregateProtoCache( proto.GuildConfig, mapping)
def test_nested_observables(self): class FakeObservableIterator(observable.ObservableMixin): def __iter__(self): return iter([1, 2, 3]) # Coder for elements from the observable iterator. elem_coder = coders.VarIntCoder() iter_coder = coders.TupleSequenceCoder(elem_coder) # Test nested WindowedValue observable. coder = coders.WindowedValueCoder(iter_coder) observ = FakeObservableIterator() value = windowed_value.WindowedValue(observ, 0, ()) self.assertEqual( coder.get_impl().get_estimated_size_and_observables(value)[1], [(observ, elem_coder.get_impl())]) # Test nested tuple observable. coder = coders.TupleCoder((coders.StrUtf8Coder(), iter_coder)) value = (u'123', observ) self.assertEqual( coder.get_impl().get_estimated_size_and_observables(value)[1], [(observ, elem_coder.get_impl())])
def test_dill_coder(self): cell_value = (lambda x: lambda: x)(0).func_closure[0] self.check_coder(coders.DillCoder(), 'a', 1, cell_value) self.check_coder( coders.TupleCoder((coders.VarIntCoder(), coders.DillCoder())), (1, cell_value))
def test_custom_coder(self): self.check_coder(CustomCoder(), 1, -10, 5) self.check_coder( coders.TupleCoder((CustomCoder(), coders.BytesCoder())), (1, 'a'), (-10, 'b'), (5, 'c'))
def test_tuple_sequence_coder(self): int_tuple_coder = coders.TupleSequenceCoder(coders.VarIntCoder()) self.check_coder(int_tuple_coder, (1, -1, 0), (), tuple(range(1000))) self.check_coder( coders.TupleCoder((coders.VarIntCoder(), int_tuple_coder)), (1, (1, 2, 3)))