def test_murmur2_java_compatibility(bytes_payload, partition_number):
partitioner = DefaultPartitioner()
all_partitions = available = list(range(1000))
# compare with output from Kafka's org.apache.kafka.clients.producer.Partitioner
assert partitioner(bytes_payload, all_partitions,
available) == partition_number
# print(str(key) + ": " + str(metrics[key]) )
##################################### MAIN FUNCTION #####################################
if __name__ == "__main__":
# Check the arguments passed to the CL (excluding the script name)
if(len(sys.argv) - 1 != 2):
print ('Error: "data_source.py" requires 2 argument\n')
print ('Usage: data_source.py <kafka_bootstrap_server> <initial_data_size>')
sys.exit(1)
else:
kafka_server = sys.argv[1]
initial_size = int(sys.argv[2])
# Create a producer and a connection to the Kafka Broker
partitioner = DefaultPartitioner()
producer = KafkaProducer(bootstrap_servers=[kafka_server],
# linger_ms = 500,
# batch_size = 0,
retries = 3,
key_serializer=str.encode,
value_serializer=lambda x: dumps(x).encode('utf-8'),
# partitioner=partitioner,
metrics_num_samples=2,
metrics_sample_window_ms=30000)
print()
print("...Connecting to bootstrap_server on " + kafka_server)
print()
if(producer.bootstrap_connected()):
print("Initial connection established")
import pickle
from socket import IPPROTO_TCP, TCP_NODELAY
from kafka.partitioner import DefaultPartitioner
PRODUCER_CONFIG = {
'bootstrap_servers': ["localhost:9092"],
'client_id': None,
'key_serializer': lambda k: pickle.dumps(k),
'value_serializer': lambda k: pickle.dumps(k),
'acks': 1,
'bootstrap_topics_filter': set(),
'compression_type': None,
'retries': 0,
'batch_size': 16384,
'linger_ms': 0,
'partitioner': DefaultPartitioner(),
'buffer_memory': 33554432,
'connections_max_idle_ms': 9 * 60 * 1000,
'max_block_ms': 60000,
'max_request_size': 1048576,
'metadata_max_age_ms': 300000,
'retry_backoff_ms': 100,
'request_timeout_ms': 30000,
'receive_buffer_bytes': None,
'send_buffer_bytes': None,
'socket_options': [(IPPROTO_TCP, TCP_NODELAY, 1)],
'sock_chunk_bytes': 4096, # undocumented experimental option
'sock_chunk_buffer_count': 1000, # undocumented experimental option
'reconnect_backoff_ms': 50,
'reconnect_backoff_max_ms': 1000,
'max_in_flight_requests_per_connection': 5,