def test_object_send(self, KafkaProducer):
future = MagicMock()
future.get.return_value = self._get_record_metadata()
def test_send_call(topic, key, value):
self.assertEqual(topic, 'us-states')
self.assertEqual(key, b'NY')
self.assertIn(b'json:', value)
self.assertIn(b'"message":{"', value)
self.assertIn(b'"code":"NY"', value)
self.assertIn(b'"name":"New York"', value)
self.assertIn(b'"version":1', value)
return future
client = MagicMock()
client.send.side_effect = test_send_call
KafkaProducer.return_value = client
producer = Producer(TOPIC_STATES, StateSerializer)
obj = StateModel()
obj.code = 'NY'
obj.name = 'New York'
ret = producer.send(obj)
self.assertEqual(ret.topic, TOPIC_STATES)
self.assertEqual(ret.partition, 0)
self.assertEqual(ret.offset, 42)
self.assertEqual(KafkaProducer.call_count, 1)
self.assertEqual(client.send.call_count, 1)
self.assertEqual(future.get.call_count, 1)
KafkaProducer.assert_called_with(bootstrap_servers=['kafka:9092'],
retries=5)
future.get.assert_called_with(timeout=5)
def test_normal_send(self, KafkaProducer):
future = MagicMock()
future.get.return_value = self._get_record_metadata()
def test_send_call(topic, key, value):
self.assertEqual(topic, "us-states")
self.assertEqual(key, b"NY")
self.assertIn(b"json:", value)
self.assertIn(b'"message":{"', value)
self.assertIn(b'"code":"NY"', value)
self.assertIn(b'"name":"New York"', value)
self.assertIn(b'"version":1', value)
return future
client = MagicMock()
client.send.side_effect = test_send_call
KafkaProducer.return_value = client
producer = Producer(TOPIC_STATES, StateSerializer)
ret = producer.send({"code": "NY", "name": "New York"})
self.assertEqual(ret.topic, TOPIC_STATES)
self.assertEqual(ret.partition, 0)
self.assertEqual(ret.offset, 42)
self.assertEqual(KafkaProducer.call_count, 1)
self.assertEqual(client.send.call_count, 1)
self.assertEqual(future.get.call_count, 1)
KafkaProducer.assert_called_with(bootstrap_servers=["kafka:9092"],
retries=5)
future.get.assert_called_with(timeout=5)
def test_send_serializer_method_field(self):
fake_client = mock.MagicMock()
fake_client.send = mock.MagicMock()
def check_args(topic, key, value):
self.assertEqual(topic, TOPIC_STATES)
self.assertEqual(key, 'NY')
self.assertJSONEqual(
value.decode().replace('json:', ''), {
'type': 'us-state',
'version': 1,
'message': {
'code': 'NY',
'name': 'New York',
'my_ser_method_field': 'value-NY',
},
})
fake_client.send.side_effect = check_args
get_producer_backend = mock.MagicMock()
get_producer_backend.return_value = fake_client
with mock.patch('logpipe.producer.get_producer_backend',
get_producer_backend):
producer = Producer(TOPIC_STATES, CustomStateSerializer)
ny = StateModel()
ny.id = 5
ny.code = 'NY'
ny.name = 'New York'
producer.send(ny)
self.assertEqual(fake_client.send.call_count, 1)
def test_object_send(self):
client = boto3.client("kinesis", region_name="us-east-1")
client.create_stream(StreamName=TOPIC_STATES, ShardCount=1)
producer = Producer(TOPIC_STATES, StateSerializer)
obj = StateModel()
obj.code = "NY"
obj.name = "New York"
ret = producer.send(obj)
self.assertEqual(ret.topic, TOPIC_STATES)
self.assertEqual(ret.partition, "shardId-000000000000")
self.assertEqual(ret.offset, "1")
obj = StateModel()
obj.code = "PA"
obj.name = "Pennsylvania"
ret = producer.send(obj)
self.assertEqual(ret.topic, TOPIC_STATES)
self.assertEqual(ret.partition, "shardId-000000000000")
self.assertEqual(ret.offset, "2")
shard_iter = client.get_shard_iterator(
StreamName=TOPIC_STATES,
ShardId="shardId-000000000000",
ShardIteratorType="TRIM_HORIZON",
)["ShardIterator"]
response = client.get_records(ShardIterator=shard_iter, Limit=100)
self.assertEqual(response["Records"][0]["SequenceNumber"], "1")
self.assertJSONEqual(
response["Records"][0]["Data"].decode().replace("json:", ""),
{
"type": "us-state",
"version": 1,
"message": {
"code": "NY",
"name": "New York",
},
},
)
self.assertEqual(response["Records"][0]["PartitionKey"], "NY")
self.assertEqual(response["Records"][1]["SequenceNumber"], "2")
self.assertJSONEqual(
response["Records"][1]["Data"].decode().replace("json:", ""),
{
"type": "us-state",
"version": 1,
"message": {
"code": "PA",
"name": "Pennsylvania",
},
},
)
self.assertEqual(response["Records"][1]["PartitionKey"], "PA")
def addbookproc(request):
bookname = request.POST['title']
# b1 = Book(title = bookname)
# b1.save()
b1 = Book.objects.create(title=bookname)
prod1 = Producer('quickstart-events', bookSerializer)
# sendbook1 = Book.objects.get(title = bookname)
# prod1.send(sendbook1)
prod1.send(b1)
return HttpResponseRedirect(reverse('app1:bookadded'))
def test_object_send(self):
client = boto3.client('kinesis', region_name='us-east-1')
client.create_stream(StreamName=TOPIC_STATES, ShardCount=1)
producer = Producer(TOPIC_STATES, StateSerializer)
obj = StateModel()
obj.code = 'NY'
obj.name = 'New York'
ret = producer.send(obj)
self.assertEqual(ret.topic, TOPIC_STATES)
self.assertEqual(ret.partition, 'shardId-000000000000')
self.assertEqual(ret.offset, '1')
obj = StateModel()
obj.code = 'PA'
obj.name = 'Pennsylvania'
ret = producer.send(obj)
self.assertEqual(ret.topic, TOPIC_STATES)
self.assertEqual(ret.partition, 'shardId-000000000000')
self.assertEqual(ret.offset, '2')
shard_iter = client.get_shard_iterator(
StreamName=TOPIC_STATES,
ShardId='shardId-000000000000',
ShardIteratorType='TRIM_HORIZON')['ShardIterator']
response = client.get_records(ShardIterator=shard_iter, Limit=100)
self.assertEqual(response['Records'][0]['SequenceNumber'], '1')
self.assertJSONEqual(
response['Records'][0]['Data'].decode().replace('json:', ''), {
'type': 'us-state',
'version': 1,
'message': {
'code': 'NY',
'name': 'New York',
},
})
self.assertEqual(response['Records'][0]['PartitionKey'], 'NY')
self.assertEqual(response['Records'][1]['SequenceNumber'], '2')
self.assertJSONEqual(
response['Records'][1]['Data'].decode().replace('json:', ''), {
'type': 'us-state',
'version': 1,
'message': {
'code': 'PA',
'name': 'Pennsylvania',
},
})
self.assertEqual(response['Records'][1]['PartitionKey'], 'PA')
def test_roundtrip_state(self):
def save(ser):
self.assertEqual(ser.validated_data["code"], "NY")
self.assertEqual(ser.validated_data["name"], "New York")
FakeStateSerializer = self.mock_state_serializer(save)
producer = Producer(TOPIC_STATES, StateSerializer)
record = producer.send({"code": "NY", "name": "New York"})
self.assertEqual(record.topic, "us-states")
self.assertEqual(record.partition, 0)
self.assertTrue(record.offset >= 0)
# producer.client.flush()
consumer = Consumer(TOPIC_STATES, consumer_timeout_ms=1000)
consumer.register(FakeStateSerializer)
consumer.run(iter_limit=1)
self.assertEqual(FakeStateSerializer.call_count, 1)
self.assertEqual(self.serializers["state"].save.call_count, 1)
def test_roundtrip_state(self):
def save(ser):
self.assertEqual(ser.validated_data['code'], 'NY')
self.assertEqual(ser.validated_data['name'], 'New York')
FakeStateSerializer = self.mock_state_serializer(save)
producer = Producer(TOPIC_STATES, StateSerializer)
record = producer.send({'code': 'NY', 'name': 'New York'})
self.assertEqual(record.topic, 'us-states')
self.assertEqual(record.partition, 0)
self.assertTrue(record.offset >= 0)
# producer.client.flush()
consumer = Consumer(TOPIC_STATES, consumer_timeout_ms=1000)
consumer.register(FakeStateSerializer)
consumer.run(iter_limit=1)
self.assertEqual(FakeStateSerializer.call_count, 1)
self.assertEqual(self.serializers['state'].save.call_count, 1)
def RideProducer():
"""
Create a ride
"""
ride_obj = Ride.objects.create(vendorid=18,
pickup_datetime=datetime.datetime.now(),
dropoff_datetime=datetime.datetime.now() +
datetime.timedelta(minutes=15),
rate_code=1,
pickup_longitude=-73.95162200927734,
pickup_latitude=40.71432876586913,
dropoff_longitude=-73.95046997070312,
dropoff_latitude=40.71106338500977,
passenger_count=2,
trip_distance=2.4,
fare_amount=10,
tip_amount=1,
total_amount=11,
payment_type=1)
producer = Producer('ride', RideSerializer)
producer.send(ride_obj)
def test_send_serializer_method_field(self):
fake_client = mock.MagicMock()
fake_client.send = mock.MagicMock()
def check_args(topic, key, value):
self.assertEqual(topic, TOPIC_STATES)
self.assertEqual(key, "NY")
self.assertJSONEqual(
value.decode().replace("json:", ""),
{
"type": "us-state",
"version": 1,
"message": {
"code": "NY",
"name": "New York",
"my_ser_method_field": "value-NY",
},
},
)
fake_client.send.side_effect = check_args
get_producer_backend = mock.MagicMock()
get_producer_backend.return_value = fake_client
with mock.patch("logpipe.producer.get_producer_backend",
get_producer_backend):
producer = Producer(TOPIC_STATES, CustomStateSerializer)
ny = StateModel()
ny.id = 5
ny.code = "NY"
ny.name = "New York"
producer.send(ny)
self.assertEqual(fake_client.send.call_count, 1)
def send_person_altered_message(sender, person, **kwargs):
producer = Producer(constants.TOPIC_PEOPLE, serializers.PersonSerializer)
producer.send(person)
def send_kafka_token(sender, user, request, **kwargs):
token = str(Token.objects.get(user_id=user.id))
test = TokenUser.objects.create(tokenid=token)
producer = Producer('login2', TokenUserSerializer)
producer.send(test)
def send_to_acc(instance):
from .serializers import AccountingSaleSerializer
producer = Producer('acc_sales', AccountingSaleSerializer)
producer.send(instance)