def test_02_collectorha(self, mock_SimpleConsumer, mock_get_uve, mock_get_part, mock_send_agg_uve, mock_clear_agg_uve, mock_reconnect_agg_uve): m_get_part = Mock_get_part() m_get_part[(1,("127.0.0.1",0,0))] = "127.0.0.1:0", \ { "gen1" : { "ObjectXX:uve1" : { "type1":{} } }} m_get_part[(1,("127.0.0.5",0,0))] = "127.0.0.5:0", \ { "gen1" : { "ObjectZZ:uve3" : { "type3":{}} }} mock_get_part.side_effect = m_get_part m_get_uve = Mock_get_uve() m_get_uve["ObjectXX:uve1"] = {"type1": {"xx": 0}} m_get_uve["ObjectYY:uve2"] = {"type2": {"yy": 1}} m_get_uve["ObjectZZ:uve3"] = {"type3": {"zz": 2}} mock_get_uve.side_effect = m_get_uve # When this message is read, 127.0.0.5 will not be present m_get_messages = Mock_get_messages() m_get_messages["ObjectYY:uve2"] = OffsetAndMessage(offset=0, message=Message(magic=0, attributes=0, key='ObjectYY:uve2|type2|gen1|127.0.0.5:0', value='{}')) mock_SimpleConsumer.return_value.get_messages.side_effect = \ m_get_messages self._ag.disc_cb_coll([{"ip-address":"127.0.0.1","pid":0}]) self._ag.libpart_cb([1]) # Now bringup collector 127.0.0.5 self.assertTrue(self.checker_dict([1, "ObjectZZ", "uve3"], self._ag.ptab_info, False)) self._ag.disc_cb_coll([{"ip-address":"127.0.0.1","pid":0}, {"ip-address":"127.0.0.5","pid":0}]) self.assertTrue(self.checker_dict([1, "ObjectZZ", "uve3"], self._ag.ptab_info)) self.assertTrue(self.checker_dict([1, "ObjectYY", "uve2"], self._ag.ptab_info, False)) # Feed the message in again m_get_messages["ObjectYY:uve2"] = OffsetAndMessage(offset=0, message=Message(magic=0, attributes=0, key='ObjectYY:uve2|type2|gen1|127.0.0.5:0', value='{}')) self.assertTrue(self.checker_dict([1, "ObjectYY", "uve2"], self._ag.ptab_info)) # Withdraw collector 127.0.0.1 self.assertTrue(self.checker_dict([1, "ObjectXX", "uve1"], self._ag.ptab_info)) del m_get_uve["ObjectXX:uve1"] self._ag.disc_cb_coll([{"ip-address":"127.0.0.5","pid":0}]) self.assertTrue(self.checker_dict([1, "ObjectXX", "uve1"], self._ag.ptab_info, False))
def _decode_message(cls, data, offset): """ Decode a single Message The only caller of this method is decode_message_set_iter. They are decoupled to support nested messages (compressed MessageSets). The offset is actually read from decode_message_set_iter (it is part of the MessageSet payload). """ ((crc, magic, att), cur) = relative_unpack('>iBB', data, 0) if crc != zlib.crc32(data[4:]): raise ChecksumError("Message checksum failed") (key, cur) = read_int_string(data, cur) (value, cur) = read_int_string(data, cur) codec = att & KafkaProtocol.ATTRIBUTE_CODEC_MASK if codec == KafkaProtocol.CODEC_NONE: yield (offset, Message(magic, att, key, value)) elif codec == KafkaProtocol.CODEC_GZIP: gz = gzip_decode(value) for (offset, msg) in KafkaProtocol._decode_message_set_iter(gz): yield (offset, msg) elif codec == KafkaProtocol.CODEC_SNAPPY: snp = snappy_decode(value) for (offset, msg) in KafkaProtocol._decode_message_set_iter(snp): yield (offset, msg)
def test_01_rxmsg(self, mock_SimpleConsumer, mock_get_uve, mock_get_part, mock_send_agg_uve, mock_clear_agg_uve, mock_reconnect_agg_uve): m_get_part = Mock_get_part() m_get_part[(1,("127.0.0.1",0,0))] = "127.0.0.1:0", \ { "gen1" : { "ObjectXX:uve1" : {"type1":{}} }} mock_get_part.side_effect = m_get_part # Boostraped UVE ObjectXX:uve1 is not present! m_get_uve = Mock_get_uve() m_get_uve["ObjectYY:uve2"] = {"type2": {"yy": 1}} mock_get_uve.side_effect = m_get_uve m_get_messages = Mock_get_messages() m_get_messages["ObjectYY:uve2"] = OffsetAndMessage( offset=0, message=Message(magic=0, attributes=0, key='ObjectYY:uve2|type2|gen1|127.0.0.1:0', value='{}')) mock_SimpleConsumer.return_value.get_messages.side_effect = \ m_get_messages self._ag.disc_cb_coll([{"ip-address": "127.0.0.1", "pid": 0}]) self._ag.libpart_cb([1]) self.assertTrue( self.checker_dict([1, "ObjectXX", "uve1"], self._ag.ptab_info, False)) self.assertTrue( self.checker_dict([1, "ObjectYY", "uve2"], self._ag.ptab_info)) self.assertTrue(self.checker_exact(\ self._ag.ptab_info[1]["ObjectYY"]["uve2"].values(), {"type2" : {"yy": 1}}))
def create_message(payload, key=None): """ Construct a Message Params ====== payload: bytes, the payload to send to Kafka key: bytes, a key used for partition routing (optional) """ return Message(0, 0, key, payload)
def create_snappy_message(payloads, key=None): """ Construct a Snappy Message containing multiple Messages The given payloads will be encoded, compressed, and sent as a single atomic message to Kafka. Params ====== payloads: list(bytes), a list of payload to send be sent to Kafka key: bytes, a key used for partition routing (optional) """ message_set = KafkaProtocol._encode_message_set( [create_message(payload) for payload in payloads]) snapped = snappy_encode(message_set) codec = KafkaProtocol.ATTRIBUTE_CODEC_MASK & KafkaProtocol.CODEC_SNAPPY return Message(0, 0x00 | codec, key, snapped)
def create_gzip_message(payloads, key=None): """ Construct a Gzipped Message containing multiple Messages The given payloads will be encoded, compressed, and sent as a single atomic message to Kafka. Arguments: payloads: list(bytes), a list of payload to send be sent to Kafka key: bytes, a key used for partition routing (optional) """ message_set = KafkaProtocol._encode_message_set( [create_message(payload, key) for payload in payloads]) gzipped = gzip_encode(message_set) codec = ATTRIBUTE_CODEC_MASK & CODEC_GZIP return Message(0, 0x00 | codec, key, gzipped)
def test_01_rxmsg(self, mock_SimpleConsumer, mock_get_uve, mock_get_part): m_get_part = Mock_get_part() m_get_part[(1, ("127.0.0.1", 0, 0))] = { "127.0.0.1:0": { "gen1": { "ObjectXX:uve1": set(["type1"]) } } } mock_get_part.side_effect = m_get_part # Boostraped UVE ObjectXX:uve1 is not present! m_get_uve = Mock_get_uve() m_get_uve["ObjectYY:uve2"] = {"type2": {"yy": 1}} mock_get_uve.side_effect = m_get_uve m_get_messages = Mock_get_messages() m_get_messages["ObjectYY:uve2"] = OffsetAndMessage( offset=0, message=Message( magic=0, attributes=0, key='', value=('{"message":"UVEUpdate","key":"ObjectYY:uve2",' '"type":"type2","gen":"gen1","coll":' '"127.0.0.1:0","deleted":false}'))) mock_SimpleConsumer.return_value.get_messages.side_effect = \ m_get_messages self._ag.disc_cb_coll([{"ip-address": "127.0.0.1", "pid": 0}]) self._ag.libpart_cb([1]) self.assertTrue( self.checker_dict([1, "ObjectXX", "uve1"], self._ag.ptab_info, False)) self.assertTrue( self.checker_dict([1, "ObjectYY", "uve2"], self._ag.ptab_info)) self.assertTrue(self.checker_exact(\ self._ag.ptab_info[1]["ObjectYY"]["uve2"].values(), {"type2" : {"yy": 1}}))
def test_encode_message_failure(self): with self.assertRaises(ProtocolError): KafkaProtocol._encode_message(Message(1, 0, "key", "test"))
def test_00_init(self, mock_SimpleConsumer, mock_KafkaClient, mock_UVEServer): self.test_spec = [ TestStage(i=PartHandlerInput( redis_instances=set([("127.0.0.1", 44444, 0)]), get_part={ "127.0.0.1:44444": { "gen1": { "ObjectXX:uve1": set(["type1"]) } } }, get_messages=[ OffsetAndMessage( offset=0, message=Message( magic=0, attributes=0, key='', value=( '{"message":"UVEUpdate","key":"ObjectYY:uve2",' '"type":"type2","gen":"gen1","coll":' '"127.0.0.1:44444","deleted":false}'))) ]), o=PartHandlerOutput(callbacks=[ { "ObjectXX:uve1": None }, { "ObjectYY:uve2": set(["type2"]) }, ], uvedb=None)), TestStage( i=PartHandlerInput(redis_instances=gevent.GreenletExit(), get_part=None, get_messages=None), o=PartHandlerOutput(callbacks=[ { "ObjectXX:uve1": None, "ObjectYY:uve2": None }, ], uvedb={ "127.0.0.1:44444": { "gen1": { "ObjectXX:uve1": set(["type1"]), "ObjectYY:uve2": set(["type2"]) } } }), ) ] mock_UVEServer.return_value.redis_instances.side_effect = \ [x.i.redis_instances for x in self.test_spec] mock_UVEServer.return_value.get_part.side_effect = \ [x.i.get_part for x in self.test_spec if x.i.get_part is not None] mock_SimpleConsumer.return_value.get_messages.side_effect = \ [x.i.get_messages for x in self.test_spec] self.ph = UveStreamProc('no-brokers', 1, "uve-1", logging, self.callback_proc, "127.0.0.1", mock_UVEServer.return_value) self.ph.start() res, db = self.ph.get(timeout=10) if (isinstance(res, AssertionError)): raise res self.assertEqual(db, self.test_spec[-1].o.uvedb)
def test_encode_message_failure(self): self.assertRaises(Exception, KafkaProtocol._encode_message, Message(1, 0, "key", "test"))
def sample_messages(self, payload, count): return count * [Message(0, 0, None, payload)], count * [payload]
def sample_messages(self, payload, count): raw_message = namedtuple('RawMessage', 'message') return count * [raw_message(message=Message(0, 0, None, payload)) ], count * [payload]