def _load_data():
"""
Sends 50 messages (1 .. 50) to samza-test-topic.
"""
logger.info("Running test_samza_job")
kafka = util.get_kafka_client()
kafka.ensure_topic_exists(TEST_INPUT_TOPIC)
producer = SimpleProducer(kafka, async=False, req_acks=SimpleProducer.ACK_AFTER_CLUSTER_COMMIT, ack_timeout=30000)
for i in range(1, NUM_MESSAGES + 1):
producer.send_messages(TEST_INPUT_TOPIC, str(i))
kafka.close()
def validate_samza_job():
"""
Validates that negate-number negated all messages, and sent the output to
samza-test-topic-output.
"""
logger.info("Running validate_samza_job")
kafka = util.get_kafka_client()
kafka.ensure_topic_exists(TEST_OUTPUT_TOPIC)
consumer = SimpleConsumer(kafka, "samza-test-group", TEST_OUTPUT_TOPIC)
messages = consumer.get_messages(count=NUM_MESSAGES, block=True, timeout=300)
message_count = len(messages)
assert NUM_MESSAGES == message_count, "Expected {0} lines, but found {1}".format(NUM_MESSAGES, message_count)
for message in map(lambda m: m.message.value, messages):
assert int(message) < 0, "Expected negative integer but received {0}".format(message)
kafka.close()
def _load_data():
"""
Sends 50 messages (1 .. 50) to samza-test-topic.
"""
logger.info('Running test_samza_job')
kafka = util.get_kafka_client()
kafka.ensure_topic_exists(TEST_INPUT_TOPIC)
producer = SimpleProducer(
kafka,
async=False,
req_acks=SimpleProducer.ACK_AFTER_CLUSTER_COMMIT,
ack_timeout=30000)
for i in range(1, NUM_MESSAGES + 1):
producer.send_messages(TEST_INPUT_TOPIC, str(i))
kafka.close()
def validate_samza_job():
"""
Validates that negate-number negated all messages, and sent the output to
samza-test-topic-output.
"""
logger.info('Running validate_samza_job')
kafka = util.get_kafka_client()
kafka.ensure_topic_exists(TEST_OUTPUT_TOPIC)
consumer = SimpleConsumer(kafka, 'samza-test-group', TEST_OUTPUT_TOPIC)
messages = consumer.get_messages(count=NUM_MESSAGES, block=True, timeout=300)
message_count = len(messages)
assert NUM_MESSAGES == message_count, 'Expected {0} lines, but found {1}'.format(NUM_MESSAGES, message_count)
for message in map(lambda m: m.message.value, messages):
assert int(message) < 0 , 'Expected negative integer but received {0}'.format(message)
kafka.close()
def _load_data():
"""
Sends 10 million messages to kafka-read-write-performance-input.
"""
logger.info('Running test_kafka_read_write_performance')
kafka = util.get_kafka_client()
kafka.ensure_topic_exists(TEST_INPUT_TOPIC)
producer = SimpleProducer(kafka,
req_acks=SimpleProducer.ACK_AFTER_CLUSTER_COMMIT,
ack_timeout=30000,
batch_send=True,
batch_send_every_n=200)
logger.info('Loading {0} test messages.'.format(NUM_MESSAGES))
for i in range(0, NUM_MESSAGES):
if i % 100000 == 0:
logger.info('Loaded {0} messages.'.format(i))
producer.send_messages(TEST_INPUT_TOPIC, MESSAGE)
kafka.close()
def _load_data():
"""
Sends 10 million messages to kafka-read-write-performance-input.
"""
logger.info('Running test_kafka_read_write_performance')
kafka = util.get_kafka_client()
kafka.ensure_topic_exists(TEST_INPUT_TOPIC)
producer = SimpleProducer(
kafka,
req_acks=SimpleProducer.ACK_AFTER_CLUSTER_COMMIT,
ack_timeout=30000,
batch_send=True,
batch_send_every_n=200)
logger.info('Loading {0} test messages.'.format(NUM_MESSAGES))
for i in range(0, NUM_MESSAGES):
if i % 100000 == 0:
logger.info('Loaded {0} messages.'.format(i))
producer.send_messages(TEST_INPUT_TOPIC, MESSAGE)
kafka.close()
def validate_kafka_read_write_performance():
"""
Validates that all messages were sent to the output topic.
"""
logger.info('Running validate_kafka_read_write_performance')
kafka = util.get_kafka_client()
kafka.ensure_topic_exists(TEST_OUTPUT_TOPIC)
consumer = SimpleConsumer(
kafka,
'samza-test-group',
TEST_OUTPUT_TOPIC,
fetch_size_bytes=1000000,
buffer_size=32768,
max_buffer_size=None)
# wait 5 minutes to get all million messages
messages = consumer.get_messages(count=NUM_MESSAGES, block=True, timeout=300)
message_count = len(messages)
assert NUM_MESSAGES == message_count, 'Expected {0} lines, but found {1}'.format(NUM_MESSAGES, message_count)
kafka.close()
def validate_kafka_read_write_performance():
"""
Validates that all messages were sent to the output topic.
"""
logger.info('Running validate_kafka_read_write_performance')
kafka = util.get_kafka_client()
kafka.ensure_topic_exists(TEST_OUTPUT_TOPIC)
consumer = SimpleConsumer(kafka,
'samza-test-group',
TEST_OUTPUT_TOPIC,
fetch_size_bytes=1000000,
buffer_size=32768,
max_buffer_size=None)
# wait 5 minutes to get all million messages
messages = consumer.get_messages(count=NUM_MESSAGES,
block=True,
timeout=300)
message_count = len(messages)
assert NUM_MESSAGES == message_count, 'Expected {0} lines, but found {1}'.format(
NUM_MESSAGES, message_count)
kafka.close()
def __pump_messages_into_input_topic():
"""
Produce 50 messages into input topic: standalone_integration_test_kafka_input_topic.
"""
kafka_client = None
input_topic = 'standalone_integration_test_kafka_input_topic'
try:
kafka_client = util.get_kafka_client()
kafka_client.ensure_topic_exists(input_topic)
producer = SimpleProducer(
kafka_client,
async=False,
req_acks=SimpleProducer.ACK_AFTER_CLUSTER_COMMIT,
ack_timeout=30000)
logger.info('Producing {0} messages to topic: {1}'.format(
NUM_MESSAGES, input_topic))
for message_index in range(1, NUM_MESSAGES + 1):
producer.send_messages(input_topic, str(message_index))
except:
logger.error(traceback.format_exc(sys.exc_info()))
finally:
if kafka_client is not None:
kafka_client.close()
