def test_basics(self):
v = ('a' * 10, 'b' * 90)
pickler = coders.PickleCoder()
self.assertEqual(v, pickler.decode(pickler.encode(v)))
pickler = coders.Base64PickleCoder()
self.assertEqual(v, pickler.decode(pickler.encode(v)))
self.assertEqual(coders.Base64PickleCoder().encode(v),
base64.b64encode(coders.PickleCoder().encode(v)))
def test_tuple_coder(self):
kv_coder = coders.TupleCoder(
(coders.VarIntCoder(), coders.BytesCoder()))
# Verify cloud object representation
self.assertEqual(
{
'@type':
'kind:pair',
'is_pair_like':
True,
'component_encodings': [
coders.VarIntCoder().as_cloud_object(),
coders.BytesCoder().as_cloud_object()
],
}, kv_coder.as_cloud_object())
# Test binary representation
self.assertEqual('\x04abc', kv_coder.encode((4, 'abc')))
# Test unnested
self.check_coder(kv_coder, (1, 'a'), (-2, 'a' * 100),
(300, 'abc\0' * 5))
# Test nested
self.check_coder(
coders.TupleCoder((coders.TupleCoder(
(coders.PickleCoder(), coders.VarIntCoder())),
coders.StrUtf8Coder())), ((1, 2), 'a'),
((-2, 5), u'a\u0101' * 100), ((300, 1), 'abc\0' * 5))
def __init__(self,
file_path_prefix,
header,
file_name_suffix='',
num_shards=0,
shard_name_template=None,
coder=coders.PickleCoder(),
compression_type=filesystem.CompressionTypes.AUTO):
super(CsvSink, self).__init__(file_path_prefix,
file_name_suffix=file_name_suffix,
num_shards=num_shards,
shard_name_template=shard_name_template,
coder=coder,
mime_type='text/csv',
compression_type=compression_type)
self._header = header
def test_pickle_coder(self):
self.check_coder(coders.PickleCoder(), 'a', 1, 1.5, (1, 2, 3))
def test_pickle_coder(self):
coder = coders.PickleCoder()
self.check_coder(coder, *self.test_values)
def test_fake_deterministic_fast_primitives_coder(self):
coder = coders.FakeDeterministicFastPrimitivesCoder(
coders.PickleCoder())
self.check_coder(coder, *self.test_values)
for v in self.test_values:
self.check_coder(coders.TupleCoder((coder, )), (v, ))
def test_equality(self):
self.assertEqual(coders.PickleCoder(), coders.PickleCoder())
self.assertEqual(coders.Base64PickleCoder(),
coders.Base64PickleCoder())
self.assertNotEqual(coders.Base64PickleCoder(), coders.PickleCoder())
self.assertNotEqual(coders.Base64PickleCoder(), object())