def test_object_send(self, KafkaProducer):
future = MagicMock()
future.get.return_value = self._get_record_metadata()
def test_send_call(topic, key, value):
self.assertEqual(topic, 'us-states')
self.assertEqual(key, b'NY')
self.assertIn(b'json:', value)
self.assertIn(b'"message":{"', value)
self.assertIn(b'"code":"NY"', value)
self.assertIn(b'"name":"New York"', value)
self.assertIn(b'"version":1', value)
return future
client = MagicMock()
client.send.side_effect = test_send_call
KafkaProducer.return_value = client
producer = Producer(TOPIC_STATES, StateSerializer)
obj = StateModel()
obj.code = 'NY'
obj.name = 'New York'
ret = producer.send(obj)
self.assertEqual(ret.topic, TOPIC_STATES)
self.assertEqual(ret.partition, 0)
self.assertEqual(ret.offset, 42)
self.assertEqual(KafkaProducer.call_count, 1)
self.assertEqual(client.send.call_count, 1)
self.assertEqual(future.get.call_count, 1)
KafkaProducer.assert_called_with(bootstrap_servers=['kafka:9092'],
retries=5)
future.get.assert_called_with(timeout=5)
def test_send_serializer_method_field(self):
fake_client = mock.MagicMock()
fake_client.send = mock.MagicMock()
def check_args(topic, key, value):
self.assertEqual(topic, TOPIC_STATES)
self.assertEqual(key, 'NY')
self.assertJSONEqual(
value.decode().replace('json:', ''), {
'type': 'us-state',
'version': 1,
'message': {
'code': 'NY',
'name': 'New York',
'my_ser_method_field': 'value-NY',
},
})
fake_client.send.side_effect = check_args
get_producer_backend = mock.MagicMock()
get_producer_backend.return_value = fake_client
with mock.patch('logpipe.producer.get_producer_backend',
get_producer_backend):
producer = Producer(TOPIC_STATES, CustomStateSerializer)
ny = StateModel()
ny.id = 5
ny.code = 'NY'
ny.name = 'New York'
producer.send(ny)
self.assertEqual(fake_client.send.call_count, 1)
def test_object_send(self):
client = boto3.client("kinesis", region_name="us-east-1")
client.create_stream(StreamName=TOPIC_STATES, ShardCount=1)
producer = Producer(TOPIC_STATES, StateSerializer)
obj = StateModel()
obj.code = "NY"
obj.name = "New York"
ret = producer.send(obj)
self.assertEqual(ret.topic, TOPIC_STATES)
self.assertEqual(ret.partition, "shardId-000000000000")
self.assertEqual(ret.offset, "1")
obj = StateModel()
obj.code = "PA"
obj.name = "Pennsylvania"
ret = producer.send(obj)
self.assertEqual(ret.topic, TOPIC_STATES)
self.assertEqual(ret.partition, "shardId-000000000000")
self.assertEqual(ret.offset, "2")
shard_iter = client.get_shard_iterator(
StreamName=TOPIC_STATES,
ShardId="shardId-000000000000",
ShardIteratorType="TRIM_HORIZON",
)["ShardIterator"]
response = client.get_records(ShardIterator=shard_iter, Limit=100)
self.assertEqual(response["Records"][0]["SequenceNumber"], "1")
self.assertJSONEqual(
response["Records"][0]["Data"].decode().replace("json:", ""),
{
"type": "us-state",
"version": 1,
"message": {
"code": "NY",
"name": "New York",
},
},
)
self.assertEqual(response["Records"][0]["PartitionKey"], "NY")
self.assertEqual(response["Records"][1]["SequenceNumber"], "2")
self.assertJSONEqual(
response["Records"][1]["Data"].decode().replace("json:", ""),
{
"type": "us-state",
"version": 1,
"message": {
"code": "PA",
"name": "Pennsylvania",
},
},
)
self.assertEqual(response["Records"][1]["PartitionKey"], "PA")
def test_object_send(self):
client = boto3.client('kinesis', region_name='us-east-1')
client.create_stream(StreamName=TOPIC_STATES, ShardCount=1)
producer = Producer(TOPIC_STATES, StateSerializer)
obj = StateModel()
obj.code = 'NY'
obj.name = 'New York'
ret = producer.send(obj)
self.assertEqual(ret.topic, TOPIC_STATES)
self.assertEqual(ret.partition, 'shardId-000000000000')
self.assertEqual(ret.offset, '1')
obj = StateModel()
obj.code = 'PA'
obj.name = 'Pennsylvania'
ret = producer.send(obj)
self.assertEqual(ret.topic, TOPIC_STATES)
self.assertEqual(ret.partition, 'shardId-000000000000')
self.assertEqual(ret.offset, '2')
shard_iter = client.get_shard_iterator(
StreamName=TOPIC_STATES,
ShardId='shardId-000000000000',
ShardIteratorType='TRIM_HORIZON')['ShardIterator']
response = client.get_records(ShardIterator=shard_iter, Limit=100)
self.assertEqual(response['Records'][0]['SequenceNumber'], '1')
self.assertJSONEqual(
response['Records'][0]['Data'].decode().replace('json:', ''), {
'type': 'us-state',
'version': 1,
'message': {
'code': 'NY',
'name': 'New York',
},
})
self.assertEqual(response['Records'][0]['PartitionKey'], 'NY')
self.assertEqual(response['Records'][1]['SequenceNumber'], '2')
self.assertJSONEqual(
response['Records'][1]['Data'].decode().replace('json:', ''), {
'type': 'us-state',
'version': 1,
'message': {
'code': 'PA',
'name': 'Pennsylvania',
},
})
self.assertEqual(response['Records'][1]['PartitionKey'], 'PA')
def test_send_serializer_method_field(self):
fake_client = mock.MagicMock()
fake_client.send = mock.MagicMock()
def check_args(topic, key, value):
self.assertEqual(topic, TOPIC_STATES)
self.assertEqual(key, "NY")
self.assertJSONEqual(
value.decode().replace("json:", ""),
{
"type": "us-state",
"version": 1,
"message": {
"code": "NY",
"name": "New York",
"my_ser_method_field": "value-NY",
},
},
)
fake_client.send.side_effect = check_args
get_producer_backend = mock.MagicMock()
get_producer_backend.return_value = fake_client
with mock.patch("logpipe.producer.get_producer_backend",
get_producer_backend):
producer = Producer(TOPIC_STATES, CustomStateSerializer)
ny = StateModel()
ny.id = 5
ny.code = "NY"
ny.name = "New York"
producer.send(ny)
self.assertEqual(fake_client.send.call_count, 1)