def test_when_error_in_output_then_pipeline_fails(): """ This function tests the integration of input and output. It assumes connectivity to streaming client is correct, therefore the steaming client is mocked. We look to find that once a message is received, it is send out by the output writer. This is the maximal integration test we can have in an isolated environment. :return: """ # setup input input_client_mock = StreamingClientMock() ir_config = { 'APP_PORT': os.getenv("APP_PORT"), 'APP_HOST': os.getenv("APP_HOST"), } print('Creating reader') ir = InputReaderFactory.create(ir_config, input_client_mock) print('start_receiving_messages') ir.start_receiving_messages() # setup app app = TestApp(os.getenv("APP_PORT"), os.getenv("APP_HOST"), os.getenv("OUTPUT_WRITER_PORT"), os.getenv("OUTPUT_WRITER_HOST")) app.start() # setup output ow_config = { 'OUTPUT_WRITER_PORT': os.getenv("OUTPUT_WRITER_PORT"), 'OUTPUT_WRITER_HOST': os.getenv("OUTPUT_WRITER_HOST"), } print('Creating writer') output_client_mock = StreamingClientMock() ow = OutputWriterFactory.create(ow_config, output_client_mock, None) print('start_incoming_messages') # Set to fail on Output Send: output_client_mock.set_fail_send(True) print('sending test message to reader') test_msg = str(time.clock()) # send a message from INPUT reader, and expect it to flow in the pipeline, # and eventually be picked up by the output writer result = input_client_mock.fake_incoming_message_from_streaming(test_msg) last_msg = output_client_mock.get_last_msg() assert last_msg is None assert result is False ir.stop_incoming_messages() ow.stop_incoming_messages()
def test_output_writer_factory_kafka(): config = { 'client': { 'type': 'kafka', 'config': { 'KAFKA_TOPIC': os.getenv("KAFKA_TOPIC_OUTPUT"), 'KAFKA_ADDRESS': os.getenv("KAFKA_ADDRESS"), 'OUTPUT_WRITER_PORT': os.getenv("OUTPUT_WRITER_PORT"), } } } ow = OutputWriterFactory.create(config) assert ow is not None assert isinstance(ow.messaging_client, KafkaStreamingClient) ow.messaging_client.stop()
def test_output_writer_factory_event_hub(): config = { 'client': { 'type': 'eventhub', 'config': { 'EVENT_HUB_NAMESPACE': os.getenv("EVENT_HUB_NAMESPACE"), 'EVENT_HUB_NAME': os.getenv("EVENT_HUB_NAME"), 'EVENT_HUB_SAS_POLICY': os.getenv("EVENT_HUB_SAS_POLICY"), 'EVENT_HUB_SAS_KEY': os.getenv("EVENT_HUB_SAS_KEY"), 'OUTPUT_WRITER_PORT': os.getenv("OUTPUT_WRITER_PORT"), } } } ow = OutputWriterFactory.create(config) assert ow is not None assert isinstance(ow.messaging_client, EventHubStreamingClient) ow.messaging_client.stop()
def test_output_writer_flask(): config = { 'client': { 'type': 'kafka', 'config': { 'KAFKA_TOPIC': os.getenv("KAFKA_TOPIC_OUTPUT"), 'KAFKA_ADDRESS': os.getenv("KAFKA_ADDRESS"), 'OUTPUT_WRITER_PORT': os.getenv("OUTPUT_WRITER_PORT"), 'OUTPUT_WRITER_HOST': os.getenv("OUTPUT_WRITER_HOST"), } } } ow = OutputWriterFactory.create(config) assert ow is not None assert isinstance(ow.listener, FlaskHttpListenerClient) received_messages = [] def receive_callback(message): received_messages.append(message) ow.listener.start(receive_callback)
def test_when_unknown_client_throw(): config = {'client': {'type': 'aaa'}} with pytest.raises(Exception): OutputWriterFactory.create(config)
def test_when_messages_sent_to_kafka_then_all_messages_are_sent_via_output(): """ This function tests the integration of input and output. It assumes connectivity to streaming client is correct, therefore the steaming client is mocked. We look to find that once a message is received, it is send out by the output writer. This is the maximal integration test we can have in an isolated environment. Please allow KAFKA_TIMEOUT of 30 seconds or more :return: """ # setup input ir_config = { 'client': { 'type': 'kafka', 'config': { "KAFKA_TOPIC": os.getenv("KAFKA_TOPIC_INPUT"), "KAFKA_CONSUMER_GROUP": os.getenv("KAFKA_CONSUMER_GROUP"), "KAFKA_ADDRESS": os.getenv("KAFKA_ADDRESS"), "TIMEOUT": os.getenv("KAFKA_TIMEOUT"), "EVENT_HUB_KAFKA_CONNECTION_STRING": os.getenv('EVENT_HUB_KAFKA_CONNECTION_STRING') } }, 'APP_PORT': os.getenv("APP_PORT"), 'APP_HOST': os.getenv("APP_HOST"), } print('Creating reader') ir = InputReaderFactory.create(ir_config) # setup app app = TestApp(os.getenv("APP_PORT"), os.getenv("APP_HOST"), os.getenv("OUTPUT_WRITER_PORT"), os.getenv("OUTPUT_WRITER_HOST")) app.start() # setup output ow_config = { 'client': { 'type': 'kafka', 'config': { "KAFKA_TOPIC": os.getenv("KAFKA_TOPIC_OUTPUT"), "KAFKA_ADDRESS": os.getenv("KAFKA_ADDRESS"), "TIMEOUT": os.getenv("KAFKA_TIMEOUT"), 'EVENT_HUB_KAFKA_CONNECTION_STRING': os.getenv('EVENT_HUB_KAFKA_CONNECTION_STRING') } }, 'OUTPUT_WRITER_PORT': os.getenv("OUTPUT_WRITER_PORT"), 'OUTPUT_WRITER_HOST': os.getenv("OUTPUT_WRITER_HOST"), } print('Creating writer') ow = OutputWriterFactory.create(ow_config, None, None) print('start_incoming_messages') ow.start_incoming_messages() print('start_receiving_messages') # ir.start_receiving_messages() t_ir = threading.Thread(name='testir', target=ir.start_receiving_messages) t_ir.setDaemon(True) t_ir.start() print('sending test message to reader') test_msg = str(time.clock()) print("sending %s to input topic" % test_msg) # send a message from INPUT reader, and expect it to flow in the pipeline, # and eventually be picked up by the output writer send_message_to_kafka(test_msg) last_msg = read_message_from_kafka() print("received %s from output topic" % last_msg) assert last_msg == test_msg ir.stop_incoming_messages() ow.stop_incoming_messages()
if msg_type == 'eventhub': # FOR NOW, LOAD THE CONFIG VARS IN FROM .ENV CFG = { 'client': { 'type': 'eventhub', 'config': { 'EVENT_HUB_NAMESPACE': os.getenv("EVENT_HUB_NAMESPACE"), 'EVENT_HUB_NAME': os.getenv("EVENT_HUB_NAME"), 'EVENT_HUB_SAS_POLICY': os.getenv("EVENT_HUB_SAS_POLICY"), 'EVENT_HUB_SAS_KEY': os.getenv("EVENT_HUB_SAS_KEY"), 'OUTPUT_WRITER_PORT': os.getenv("OUTPUT_WRITER_PORT"), 'OUTPUT_WRITER_HOST': os.getenv("OUTPUT_WRITER_HOST"), } } } elif msg_type == 'kafka': CFG = { 'client': { 'type': 'kafka', 'config': { 'KAFKA_TOPIC': os.getenv("KAFKA_TOPIC"), 'KAFKA_ADDRESS': os.getenv("KAFKA_ADDRESS"), 'OUTPUT_WRITER_PORT': os.getenv("OUTPUT_WRITER_PORT"), 'OUTPUT_WRITER_HOST': os.getenv("OUTPUT_WRITER_HOST"), } } } OUTPUT = OutputWriterFactory.create(CFG) OUTPUT.start_incoming_messages()