def get_value_stream(self, stream):
private_encoding = self.private_encoding
for cmd, args in stream:
if cmd == 'close':
stream.push_back((cmd, args))
raise StopIteration
elif cmd == 'reduceValue':
yield private_decode(args[0]) if private_encoding else args[0]
else:
stream.push_back((cmd, args))
raise StopIteration
raise StopIteration
def next(self): # FIXME: only for timing comparison purposes
for cmd, args in self.stream:
if cmd == 'close':
raise StopIteration
elif cmd == 'reduceKey':
values_stream = self.get_value_stream(self.stream)
key = private_decode(
args[0]) if self.private_encoding else args[0]
return key, values_stream
elif cmd == 'reduceValue':
continue
else:
raise ProtocolError('out of order command: {}'.format(cmd))
raise StopIteration
def fast_iterator(self):
stream = self.stream
private_encoding = self.private_encoding
for cmd, args in stream:
if cmd == 'close':
raise StopIteration
elif cmd == 'reduceKey':
values_stream = self.get_value_stream(self.stream)
key = private_decode(args[0]) if private_encoding else args[0]
yield key, values_stream
elif cmd == 'reduceValue':
continue
else:
raise ProtocolError('out of order command: {}'.format(cmd))
raise StopIteration
def __init__(self, context):
super(Partitioner, self).__init__(context)
self.logger = LOGGER.getChild("Partitioner")
with io.open(self.BREAK_POINTS_CACHE_FILE, 'rb') as f:
self.selector = srl.private_decode(f.read())
def test_private_serialize(self):
for obj in [1, 0.4, "Hello", [1, 2, 3], {"key": "value"}]:
self.assertEqual(obj, srl.private_decode(srl.private_encode(obj)))
def decoded_key_values_stream(stream):
for k, vstream in raw_key_values_stream(stream):
yield private_decode(k), (private_decode(_) for _ in vstream)
def __init__(self, context):
super(Partitioner, self).__init__(context)
self.logger = LOGGER.getChild("Partitioner")
with io.open(self.BREAK_POINTS_CACHE_FILE, 'rb') as f:
self.selector = srl.private_decode(f.read())
def decoded_key_values_stream(stream):
for k, vstream in raw_key_values_stream(stream):
yield private_decode(k), (private_decode(_) for _ in vstream)
def test_private_serialize(self):
for obj in [1, 0.4, "Hello", [1, 2, 3], {"key": "value"}]:
self.assertEqual(obj, srl.private_decode(srl.private_encode(obj)))
