def test_when_error_in_output_then_pipeline_fails():
"""
This function tests the integration of input and output.
It assumes connectivity to streaming client is correct, therefore the steaming client is mocked.
We look to find that once a message is received, it is send out by the output writer.
This is the maximal integration test we can have in an isolated environment.
:return:
"""
# setup input
input_client_mock = StreamingClientMock()
ir_config = {
'APP_PORT': os.getenv("APP_PORT"),
'APP_HOST': os.getenv("APP_HOST"),
}
print('Creating reader')
ir = InputReaderFactory.create(ir_config, input_client_mock)
print('start_receiving_messages')
ir.start_receiving_messages()
# setup app
app = TestApp(os.getenv("APP_PORT"), os.getenv("APP_HOST"),
os.getenv("OUTPUT_WRITER_PORT"),
os.getenv("OUTPUT_WRITER_HOST"))
app.start()
# setup output
ow_config = {
'OUTPUT_WRITER_PORT': os.getenv("OUTPUT_WRITER_PORT"),
'OUTPUT_WRITER_HOST': os.getenv("OUTPUT_WRITER_HOST"),
}
print('Creating writer')
output_client_mock = StreamingClientMock()
ow = OutputWriterFactory.create(ow_config, output_client_mock, None)
print('start_incoming_messages')
# Set to fail on Output Send:
output_client_mock.set_fail_send(True)
print('sending test message to reader')
test_msg = str(time.clock())
# send a message from INPUT reader, and expect it to flow in the pipeline,
# and eventually be picked up by the output writer
result = input_client_mock.fake_incoming_message_from_streaming(test_msg)
last_msg = output_client_mock.get_last_msg()
assert last_msg is None
assert result is False
ir.stop_incoming_messages()
ow.stop_incoming_messages()
def test_output_writer_factory_kafka():
config = {
'client': {
'type': 'kafka',
'config': {
'KAFKA_TOPIC': os.getenv("KAFKA_TOPIC_OUTPUT"),
'KAFKA_ADDRESS': os.getenv("KAFKA_ADDRESS"),
'OUTPUT_WRITER_PORT': os.getenv("OUTPUT_WRITER_PORT"),
}
}
}
ow = OutputWriterFactory.create(config)
assert ow is not None
assert isinstance(ow.messaging_client, KafkaStreamingClient)
ow.messaging_client.stop()
def test_output_writer_factory_event_hub():
config = {
'client': {
'type': 'eventhub',
'config': {
'EVENT_HUB_NAMESPACE': os.getenv("EVENT_HUB_NAMESPACE"),
'EVENT_HUB_NAME': os.getenv("EVENT_HUB_NAME"),
'EVENT_HUB_SAS_POLICY': os.getenv("EVENT_HUB_SAS_POLICY"),
'EVENT_HUB_SAS_KEY': os.getenv("EVENT_HUB_SAS_KEY"),
'OUTPUT_WRITER_PORT': os.getenv("OUTPUT_WRITER_PORT"),
}
}
}
ow = OutputWriterFactory.create(config)
assert ow is not None
assert isinstance(ow.messaging_client, EventHubStreamingClient)
ow.messaging_client.stop()
def test_output_writer_flask():
config = {
'client': {
'type': 'kafka',
'config': {
'KAFKA_TOPIC': os.getenv("KAFKA_TOPIC_OUTPUT"),
'KAFKA_ADDRESS': os.getenv("KAFKA_ADDRESS"),
'OUTPUT_WRITER_PORT': os.getenv("OUTPUT_WRITER_PORT"),
'OUTPUT_WRITER_HOST': os.getenv("OUTPUT_WRITER_HOST"),
}
}
}
ow = OutputWriterFactory.create(config)
assert ow is not None
assert isinstance(ow.listener, FlaskHttpListenerClient)
received_messages = []
def receive_callback(message):
received_messages.append(message)
ow.listener.start(receive_callback)
def test_when_unknown_client_throw():
config = {'client': {'type': 'aaa'}}
with pytest.raises(Exception):
OutputWriterFactory.create(config)
def test_when_messages_sent_to_kafka_then_all_messages_are_sent_via_output():
"""
This function tests the integration of input and output.
It assumes connectivity to streaming client is correct, therefore the steaming client is mocked.
We look to find that once a message is received, it is send out by the output writer.
This is the maximal integration test we can have in an isolated environment.
Please allow KAFKA_TIMEOUT of 30 seconds or more
:return:
"""
# setup input
ir_config = {
'client': {
'type': 'kafka',
'config': {
"KAFKA_TOPIC":
os.getenv("KAFKA_TOPIC_INPUT"),
"KAFKA_CONSUMER_GROUP":
os.getenv("KAFKA_CONSUMER_GROUP"),
"KAFKA_ADDRESS":
os.getenv("KAFKA_ADDRESS"),
"TIMEOUT":
os.getenv("KAFKA_TIMEOUT"),
"EVENT_HUB_KAFKA_CONNECTION_STRING":
os.getenv('EVENT_HUB_KAFKA_CONNECTION_STRING')
}
},
'APP_PORT': os.getenv("APP_PORT"),
'APP_HOST': os.getenv("APP_HOST"),
}
print('Creating reader')
ir = InputReaderFactory.create(ir_config)
# setup app
app = TestApp(os.getenv("APP_PORT"), os.getenv("APP_HOST"),
os.getenv("OUTPUT_WRITER_PORT"),
os.getenv("OUTPUT_WRITER_HOST"))
app.start()
# setup output
ow_config = {
'client': {
'type': 'kafka',
'config': {
"KAFKA_TOPIC":
os.getenv("KAFKA_TOPIC_OUTPUT"),
"KAFKA_ADDRESS":
os.getenv("KAFKA_ADDRESS"),
"TIMEOUT":
os.getenv("KAFKA_TIMEOUT"),
'EVENT_HUB_KAFKA_CONNECTION_STRING':
os.getenv('EVENT_HUB_KAFKA_CONNECTION_STRING')
}
},
'OUTPUT_WRITER_PORT': os.getenv("OUTPUT_WRITER_PORT"),
'OUTPUT_WRITER_HOST': os.getenv("OUTPUT_WRITER_HOST"),
}
print('Creating writer')
ow = OutputWriterFactory.create(ow_config, None, None)
print('start_incoming_messages')
ow.start_incoming_messages()
print('start_receiving_messages')
# ir.start_receiving_messages()
t_ir = threading.Thread(name='testir', target=ir.start_receiving_messages)
t_ir.setDaemon(True)
t_ir.start()
print('sending test message to reader')
test_msg = str(time.clock())
print("sending %s to input topic" % test_msg)
# send a message from INPUT reader, and expect it to flow in the pipeline,
# and eventually be picked up by the output writer
send_message_to_kafka(test_msg)
last_msg = read_message_from_kafka()
print("received %s from output topic" % last_msg)
assert last_msg == test_msg
ir.stop_incoming_messages()
ow.stop_incoming_messages()
if msg_type == 'eventhub':
# FOR NOW, LOAD THE CONFIG VARS IN FROM .ENV
CFG = {
'client': {
'type': 'eventhub',
'config': {
'EVENT_HUB_NAMESPACE': os.getenv("EVENT_HUB_NAMESPACE"),
'EVENT_HUB_NAME': os.getenv("EVENT_HUB_NAME"),
'EVENT_HUB_SAS_POLICY': os.getenv("EVENT_HUB_SAS_POLICY"),
'EVENT_HUB_SAS_KEY': os.getenv("EVENT_HUB_SAS_KEY"),
'OUTPUT_WRITER_PORT': os.getenv("OUTPUT_WRITER_PORT"),
'OUTPUT_WRITER_HOST': os.getenv("OUTPUT_WRITER_HOST"),
}
}
}
elif msg_type == 'kafka':
CFG = {
'client': {
'type': 'kafka',
'config': {
'KAFKA_TOPIC': os.getenv("KAFKA_TOPIC"),
'KAFKA_ADDRESS': os.getenv("KAFKA_ADDRESS"),
'OUTPUT_WRITER_PORT': os.getenv("OUTPUT_WRITER_PORT"),
'OUTPUT_WRITER_HOST': os.getenv("OUTPUT_WRITER_HOST"),
}
}
}
OUTPUT = OutputWriterFactory.create(CFG)
OUTPUT.start_incoming_messages()