def test_02_collectorha(self,
mock_SimpleConsumer,
mock_get_uve, mock_get_part,
mock_send_agg_uve, mock_clear_agg_uve, mock_reconnect_agg_uve):
m_get_part = Mock_get_part()
m_get_part[(1,("127.0.0.1",0,0))] = "127.0.0.1:0", \
{ "gen1" :
{ "ObjectXX:uve1" : { "type1":{} } }}
m_get_part[(1,("127.0.0.5",0,0))] = "127.0.0.5:0", \
{ "gen1" :
{ "ObjectZZ:uve3" : { "type3":{}} }}
mock_get_part.side_effect = m_get_part
m_get_uve = Mock_get_uve()
m_get_uve["ObjectXX:uve1"] = {"type1": {"xx": 0}}
m_get_uve["ObjectYY:uve2"] = {"type2": {"yy": 1}}
m_get_uve["ObjectZZ:uve3"] = {"type3": {"zz": 2}}
mock_get_uve.side_effect = m_get_uve
# When this message is read, 127.0.0.5 will not be present
m_get_messages = Mock_get_messages()
m_get_messages["ObjectYY:uve2"] = OffsetAndMessage(offset=0,
message=Message(magic=0, attributes=0,
key='ObjectYY:uve2|type2|gen1|127.0.0.5:0',
value='{}'))
mock_SimpleConsumer.return_value.get_messages.side_effect = \
m_get_messages
self._ag.disc_cb_coll([{"ip-address":"127.0.0.1","pid":0}])
self._ag.libpart_cb([1])
# Now bringup collector 127.0.0.5
self.assertTrue(self.checker_dict([1, "ObjectZZ", "uve3"], self._ag.ptab_info, False))
self._ag.disc_cb_coll([{"ip-address":"127.0.0.1","pid":0}, {"ip-address":"127.0.0.5","pid":0}])
self.assertTrue(self.checker_dict([1, "ObjectZZ", "uve3"], self._ag.ptab_info))
self.assertTrue(self.checker_dict([1, "ObjectYY", "uve2"], self._ag.ptab_info, False))
# Feed the message in again
m_get_messages["ObjectYY:uve2"] = OffsetAndMessage(offset=0,
message=Message(magic=0, attributes=0,
key='ObjectYY:uve2|type2|gen1|127.0.0.5:0',
value='{}'))
self.assertTrue(self.checker_dict([1, "ObjectYY", "uve2"], self._ag.ptab_info))
# Withdraw collector 127.0.0.1
self.assertTrue(self.checker_dict([1, "ObjectXX", "uve1"], self._ag.ptab_info))
del m_get_uve["ObjectXX:uve1"]
self._ag.disc_cb_coll([{"ip-address":"127.0.0.5","pid":0}])
self.assertTrue(self.checker_dict([1, "ObjectXX", "uve1"], self._ag.ptab_info, False))
def _decode_message(cls, data, offset):
"""
Decode a single Message
The only caller of this method is decode_message_set_iter.
They are decoupled to support nested messages (compressed MessageSets).
The offset is actually read from decode_message_set_iter (it is part
of the MessageSet payload).
"""
((crc, magic, att), cur) = relative_unpack('>iBB', data, 0)
if crc != zlib.crc32(data[4:]):
raise ChecksumError("Message checksum failed")
(key, cur) = read_int_string(data, cur)
(value, cur) = read_int_string(data, cur)
codec = att & KafkaProtocol.ATTRIBUTE_CODEC_MASK
if codec == KafkaProtocol.CODEC_NONE:
yield (offset, Message(magic, att, key, value))
elif codec == KafkaProtocol.CODEC_GZIP:
gz = gzip_decode(value)
for (offset, msg) in KafkaProtocol._decode_message_set_iter(gz):
yield (offset, msg)
elif codec == KafkaProtocol.CODEC_SNAPPY:
snp = snappy_decode(value)
for (offset, msg) in KafkaProtocol._decode_message_set_iter(snp):
yield (offset, msg)
def test_01_rxmsg(self, mock_SimpleConsumer, mock_get_uve, mock_get_part,
mock_send_agg_uve, mock_clear_agg_uve,
mock_reconnect_agg_uve):
m_get_part = Mock_get_part()
m_get_part[(1,("127.0.0.1",0,0))] = "127.0.0.1:0", \
{ "gen1" :
{ "ObjectXX:uve1" : {"type1":{}} }}
mock_get_part.side_effect = m_get_part
# Boostraped UVE ObjectXX:uve1 is not present!
m_get_uve = Mock_get_uve()
m_get_uve["ObjectYY:uve2"] = {"type2": {"yy": 1}}
mock_get_uve.side_effect = m_get_uve
m_get_messages = Mock_get_messages()
m_get_messages["ObjectYY:uve2"] = OffsetAndMessage(
offset=0,
message=Message(magic=0,
attributes=0,
key='ObjectYY:uve2|type2|gen1|127.0.0.1:0',
value='{}'))
mock_SimpleConsumer.return_value.get_messages.side_effect = \
m_get_messages
self._ag.disc_cb_coll([{"ip-address": "127.0.0.1", "pid": 0}])
self._ag.libpart_cb([1])
self.assertTrue(
self.checker_dict([1, "ObjectXX", "uve1"], self._ag.ptab_info,
False))
self.assertTrue(
self.checker_dict([1, "ObjectYY", "uve2"], self._ag.ptab_info))
self.assertTrue(self.checker_exact(\
self._ag.ptab_info[1]["ObjectYY"]["uve2"].values(), {"type2" : {"yy": 1}}))
def create_message(payload, key=None):
"""
Construct a Message
Params
======
payload: bytes, the payload to send to Kafka
key: bytes, a key used for partition routing (optional)
"""
return Message(0, 0, key, payload)
def create_snappy_message(payloads, key=None):
"""
Construct a Snappy Message containing multiple Messages
The given payloads will be encoded, compressed, and sent as a single atomic
message to Kafka.
Params
======
payloads: list(bytes), a list of payload to send be sent to Kafka
key: bytes, a key used for partition routing (optional)
"""
message_set = KafkaProtocol._encode_message_set(
[create_message(payload) for payload in payloads])
snapped = snappy_encode(message_set)
codec = KafkaProtocol.ATTRIBUTE_CODEC_MASK & KafkaProtocol.CODEC_SNAPPY
return Message(0, 0x00 | codec, key, snapped)
def create_gzip_message(payloads, key=None):
"""
Construct a Gzipped Message containing multiple Messages
The given payloads will be encoded, compressed, and sent as a single atomic
message to Kafka.
Arguments:
payloads: list(bytes), a list of payload to send be sent to Kafka
key: bytes, a key used for partition routing (optional)
"""
message_set = KafkaProtocol._encode_message_set(
[create_message(payload, key) for payload in payloads])
gzipped = gzip_encode(message_set)
codec = ATTRIBUTE_CODEC_MASK & CODEC_GZIP
return Message(0, 0x00 | codec, key, gzipped)
def test_01_rxmsg(self, mock_SimpleConsumer, mock_get_uve, mock_get_part):
m_get_part = Mock_get_part()
m_get_part[(1, ("127.0.0.1", 0, 0))] = {
"127.0.0.1:0": {
"gen1": {
"ObjectXX:uve1": set(["type1"])
}
}
}
mock_get_part.side_effect = m_get_part
# Boostraped UVE ObjectXX:uve1 is not present!
m_get_uve = Mock_get_uve()
m_get_uve["ObjectYY:uve2"] = {"type2": {"yy": 1}}
mock_get_uve.side_effect = m_get_uve
m_get_messages = Mock_get_messages()
m_get_messages["ObjectYY:uve2"] = OffsetAndMessage(
offset=0,
message=Message(
magic=0,
attributes=0,
key='',
value=('{"message":"UVEUpdate","key":"ObjectYY:uve2",'
'"type":"type2","gen":"gen1","coll":'
'"127.0.0.1:0","deleted":false}')))
mock_SimpleConsumer.return_value.get_messages.side_effect = \
m_get_messages
self._ag.disc_cb_coll([{"ip-address": "127.0.0.1", "pid": 0}])
self._ag.libpart_cb([1])
self.assertTrue(
self.checker_dict([1, "ObjectXX", "uve1"], self._ag.ptab_info,
False))
self.assertTrue(
self.checker_dict([1, "ObjectYY", "uve2"], self._ag.ptab_info))
self.assertTrue(self.checker_exact(\
self._ag.ptab_info[1]["ObjectYY"]["uve2"].values(), {"type2" : {"yy": 1}}))
def test_encode_message_failure(self):
with self.assertRaises(ProtocolError):
KafkaProtocol._encode_message(Message(1, 0, "key", "test"))
def test_00_init(self, mock_SimpleConsumer, mock_KafkaClient,
mock_UVEServer):
self.test_spec = [
TestStage(i=PartHandlerInput(
redis_instances=set([("127.0.0.1", 44444, 0)]),
get_part={
"127.0.0.1:44444": {
"gen1": {
"ObjectXX:uve1": set(["type1"])
}
}
},
get_messages=[
OffsetAndMessage(
offset=0,
message=Message(
magic=0,
attributes=0,
key='',
value=(
'{"message":"UVEUpdate","key":"ObjectYY:uve2",'
'"type":"type2","gen":"gen1","coll":'
'"127.0.0.1:44444","deleted":false}')))
]),
o=PartHandlerOutput(callbacks=[
{
"ObjectXX:uve1": None
},
{
"ObjectYY:uve2": set(["type2"])
},
],
uvedb=None)),
TestStage(
i=PartHandlerInput(redis_instances=gevent.GreenletExit(),
get_part=None,
get_messages=None),
o=PartHandlerOutput(callbacks=[
{
"ObjectXX:uve1": None,
"ObjectYY:uve2": None
},
],
uvedb={
"127.0.0.1:44444": {
"gen1": {
"ObjectXX:uve1":
set(["type1"]),
"ObjectYY:uve2": set(["type2"])
}
}
}),
)
]
mock_UVEServer.return_value.redis_instances.side_effect = \
[x.i.redis_instances for x in self.test_spec]
mock_UVEServer.return_value.get_part.side_effect = \
[x.i.get_part for x in self.test_spec if x.i.get_part is not None]
mock_SimpleConsumer.return_value.get_messages.side_effect = \
[x.i.get_messages for x in self.test_spec]
self.ph = UveStreamProc('no-brokers', 1, "uve-1", logging,
self.callback_proc, "127.0.0.1",
mock_UVEServer.return_value)
self.ph.start()
res, db = self.ph.get(timeout=10)
if (isinstance(res, AssertionError)):
raise res
self.assertEqual(db, self.test_spec[-1].o.uvedb)
def test_encode_message_failure(self):
self.assertRaises(Exception, KafkaProtocol._encode_message,
Message(1, 0, "key", "test"))
def sample_messages(self, payload, count):
return count * [Message(0, 0, None, payload)], count * [payload]
def sample_messages(self, payload, count):
raw_message = namedtuple('RawMessage', 'message')
return count * [raw_message(message=Message(0, 0, None, payload))
], count * [payload]