def test_tuple_coder(self):
kv_coder = coders.TupleCoder(
(coders.VarIntCoder(), coders.BytesCoder()))
# Verify cloud object representation
self.assertEqual(
{
'@type':
'kind:pair',
'is_pair_like':
True,
'component_encodings': [
coders.VarIntCoder().as_cloud_object(),
coders.BytesCoder().as_cloud_object()
],
}, kv_coder.as_cloud_object())
# Test binary representation
self.assertEqual('\x04abc', kv_coder.encode((4, 'abc')))
# Test unnested
self.check_coder(kv_coder, (1, 'a'), (-2, 'a' * 100),
(300, 'abc\0' * 5))
# Test nested
self.check_coder(
coders.TupleCoder((coders.TupleCoder(
(coders.PickleCoder(), coders.VarIntCoder())),
coders.StrUtf8Coder())), ((1, 2), 'a'),
((-2, 5), u'a\u0101' * 100), ((300, 1), 'abc\0' * 5))
def test_tuple_coder(self):
self.check_coder(
coders.TupleCoder((coders.VarIntCoder(), coders.BytesCoder())),
(1, 'a'), (-2, 'a' * 100), (300, 'abc\0' * 5))
self.check_coder(
coders.TupleCoder((coders.TupleCoder(
(coders.PickleCoder(), coders.VarIntCoder())),
coders.StrUtf8Coder())), ((1, 2), 'a'),
((-2, 5), u'a\u0101' * 100), ((300, 1), 'abc\0' * 5))
def test_windowed_value_coder(self):
coder = coders.WindowedValueCoder(coders.VarIntCoder(),
coders.GlobalWindowCoder())
# Verify cloud object representation
self.assertEqual(
{
'@type':
'kind:windowed_value',
'is_wrapper':
True,
'component_encodings': [
coders.VarIntCoder().as_cloud_object(),
coders.GlobalWindowCoder().as_cloud_object(),
],
}, coder.as_cloud_object())
# Test binary representation
self.assertEqual(
'\x01\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01',
coder.encode(window.GlobalWindows.windowed_value(1)))
# Test unnested
self.check_coder(coders.WindowedValueCoder(coders.VarIntCoder()),
windowed_value.WindowedValue(3, -100, ()),
windowed_value.WindowedValue(-1, 100, (1, 2, 3)))
# Test nested
self.check_coder(
coders.TupleCoder(
(coders.WindowedValueCoder(coders.FloatCoder()),
coders.WindowedValueCoder(coders.StrUtf8Coder()))),
(windowed_value.WindowedValue(1.5, 0, ()),
windowed_value.WindowedValue("abc", 10, ('window', ))))
def test_interval_window_coder(self):
self.check_coder(
coders.IntervalWindowCoder(), *[
window.IntervalWindow(x, y) for x in [-2**52, 0, 2**52]
for y in range(-100, 100)
])
self.check_coder(coders.TupleCoder((coders.IntervalWindowCoder(), )),
(window.IntervalWindow(0, 10), ))
def test_singleton_coder(self):
a = 'anything'
b = 'something else'
self.check_coder(coders.SingletonCoder(a), a)
self.check_coder(coders.SingletonCoder(b), b)
self.check_coder(
coders.TupleCoder(
(coders.SingletonCoder(a), coders.SingletonCoder(b))), (a, b))
def test_float_coder(self):
self.check_coder(coders.FloatCoder(),
*[float(0.1 * x) for x in range(-100, 100)])
self.check_coder(coders.FloatCoder(),
*[float(2**(0.1 * x)) for x in range(-100, 100)])
self.check_coder(coders.FloatCoder(), float('-Inf'), float('Inf'))
self.check_coder(
coders.TupleCoder((coders.FloatCoder(), coders.FloatCoder())),
(0, 1), (-100, 100), (0.5, 0.25))
def test_deterministic_pickle_coder(self):
coder = coders.DeterministicPickleCoder(coders.PickleCoder(), 'step')
self.check_coder(coder, 'a', 1, 1.5, (1, 2, 3))
with self.assertRaises(TypeError):
self.check_coder(coder, dict())
with self.assertRaises(TypeError):
self.check_coder(coder, [1, dict()])
self.check_coder(coders.TupleCoder((coder, coders.PickleCoder())),
(1, dict()), ('a', [dict()]))
def test_fast_primitives_coder(self):
coder = coders.FastPrimitivesCoder(coders.SingletonCoder(len))
self.check_coder(coder, None, 1, -1, 1.5, 'str\0str',
u'unicode\0\u0101')
self.check_coder(coder, (), (1, 2, 3))
self.check_coder(coder, [], [1, 2, 3])
self.check_coder(coder, dict(), {'a': 'b'}, {0: dict(), 1: len})
self.check_coder(coder, set(), {'a', 'b'})
self.check_coder(coder, True, False)
self.check_coder(coder, len)
self.check_coder(coders.TupleCoder((coder, )), ('a', ), (1, ))
def test_deterministic_coder(self):
coder = coders.FastPrimitivesCoder()
deterministic_coder = coders.DeterministicFastPrimitivesCoder(
coder, 'step')
self.check_coder(deterministic_coder, 'a', 1, 1.5, (1, 2, 3))
with self.assertRaises(TypeError):
self.check_coder(deterministic_coder, dict())
with self.assertRaises(TypeError):
self.check_coder(deterministic_coder, [1, dict()])
self.check_coder(coders.TupleCoder((deterministic_coder, coder)),
(1, dict()), ('a', [dict()]))
def test_timestamp_coder(self):
self.check_coder(
coders.TimestampCoder(),
*[timestamp.Timestamp(micros=x) for x in range(-100, 100)])
self.check_coder(coders.TimestampCoder(),
timestamp.Timestamp(micros=-1234567890),
timestamp.Timestamp(micros=1234567890))
self.check_coder(coders.TimestampCoder(),
timestamp.Timestamp(micros=-1234567890123456789),
timestamp.Timestamp(micros=1234567890123456789))
self.check_coder(
coders.TupleCoder((coders.TimestampCoder(), coders.BytesCoder())),
(timestamp.Timestamp.of(27), 'abc'))
def test_global_window_coder(self):
coder = coders.GlobalWindowCoder()
value = window.GlobalWindow()
# Verify cloud object representation
self.assertEqual({'@type': 'kind:global_window'},
coder.as_cloud_object())
# Test binary representation
self.assertEqual('', coder.encode(value))
self.assertEqual(value, coder.decode(''))
# Test unnested
self.check_coder(coder, value)
# Test nested
self.check_coder(coders.TupleCoder((coder, coder)), (value, value))
def test_proto_coder(self):
# For instructions on how these test proto message were generated,
# see coders_test.py
ma = test_message.MessageA()
mab = ma.field2.add()
mab.field1 = True
ma.field1 = u'hello world'
mb = test_message.MessageA()
mb.field1 = u'beam'
proto_coder = coders.ProtoCoder(ma.__class__)
self.check_coder(proto_coder, ma)
self.check_coder(coders.TupleCoder((proto_coder, coders.BytesCoder())),
(ma, 'a'), (mb, 'b'))
def test_iterable_coder(self):
iterable_coder = coders.IterableCoder(coders.VarIntCoder())
# Verify cloud object representation
self.assertEqual(
{
'@type': 'kind:stream',
'is_stream_like': True,
'component_encodings':
[coders.VarIntCoder().as_cloud_object()]
}, iterable_coder.as_cloud_object())
# Test unnested
self.check_coder(iterable_coder, [1], [-1, 0, 100])
# Test nested
self.check_coder(
coders.TupleCoder((coders.VarIntCoder(),
coders.IterableCoder(coders.VarIntCoder()))),
(1, [1, 2, 3]))
def test_length_prefix_coder(self):
coder = coders.LengthPrefixCoder(coders.BytesCoder())
# Verify cloud object representation
self.assertEqual(
{
'@type': 'kind:length_prefix',
'component_encodings': [coders.BytesCoder().as_cloud_object()]
}, coder.as_cloud_object())
# Test binary representation
self.assertEqual('\x00', coder.encode(''))
self.assertEqual('\x01a', coder.encode('a'))
self.assertEqual('\x02bc', coder.encode('bc'))
self.assertEqual('\xff\x7f' + 'z' * 16383, coder.encode('z' * 16383))
# Test unnested
self.check_coder(coder, '', 'a', 'bc', 'def')
# Test nested
self.check_coder(coders.TupleCoder((coder, coder)), ('', 'a'),
('bc', 'def'))
def __setup_caches(self):
self.bans = bans.BanStorage(self)
self.online_status = OnlineStatus(self.redis,
StoragePrefix.ONLINE_STATUS)
for conf in Storage._get_cache_configs():
# Initialize Parameters
prefix = _prefixize(conf.prefix.value)
key_coder = coders.IntCoder().prefixed(prefix)
value_coder = (conf.value_coder or protobuf(conf.proto_type))()
value_coder = value_coder.compressed()
timeout = conf.timeout or 0
if conf.subprefix is None:
store = caches.RedisStore(self.redis, timeout=timeout)
else:
subprefix = _prefixize(conf.subprefix)
subcoder = coders.ConstCoder(subprefix)
if conf.subcoder is not None:
subcoder = conf.subcoder.prefixed(subprefix)
key_coder = coders.TupleCoder([key_coder, subcoder])
store = caches.RedisHashStore(self.redis, timeout=timeout)
cache = caches.Cache(store,
key_coder=key_coder,
value_coder=value_coder)
setattr(self, conf.attr, cache)
# TODO(james7132): Uncomment the above once AggregateProtoHashCache
# supports client side caching
mapping = []
for conf in Storage._get_cache_configs():
if conf.prefix != StoragePrefix.GUILD_CONFIGS:
continue
attr = conf.attr
if '_configs' in attr:
attr = attr.replace('_configs', '')
mapping.append((attr, getattr(self, conf.attr)))
self.guild_configs = caches.AggregateProtoCache(
proto.GuildConfig, mapping)
def test_nested_observables(self):
class FakeObservableIterator(observable.ObservableMixin):
def __iter__(self):
return iter([1, 2, 3])
# Coder for elements from the observable iterator.
elem_coder = coders.VarIntCoder()
iter_coder = coders.TupleSequenceCoder(elem_coder)
# Test nested WindowedValue observable.
coder = coders.WindowedValueCoder(iter_coder)
observ = FakeObservableIterator()
value = windowed_value.WindowedValue(observ, 0, ())
self.assertEqual(
coder.get_impl().get_estimated_size_and_observables(value)[1],
[(observ, elem_coder.get_impl())])
# Test nested tuple observable.
coder = coders.TupleCoder((coders.StrUtf8Coder(), iter_coder))
value = (u'123', observ)
self.assertEqual(
coder.get_impl().get_estimated_size_and_observables(value)[1],
[(observ, elem_coder.get_impl())])
def test_dill_coder(self):
cell_value = (lambda x: lambda: x)(0).func_closure[0]
self.check_coder(coders.DillCoder(), 'a', 1, cell_value)
self.check_coder(
coders.TupleCoder((coders.VarIntCoder(), coders.DillCoder())),
(1, cell_value))
def test_custom_coder(self):
self.check_coder(CustomCoder(), 1, -10, 5)
self.check_coder(
coders.TupleCoder((CustomCoder(), coders.BytesCoder())), (1, 'a'),
(-10, 'b'), (5, 'c'))
def test_tuple_sequence_coder(self):
int_tuple_coder = coders.TupleSequenceCoder(coders.VarIntCoder())
self.check_coder(int_tuple_coder, (1, -1, 0), (), tuple(range(1000)))
self.check_coder(
coders.TupleCoder((coders.VarIntCoder(), int_tuple_coder)),
(1, (1, 2, 3)))
