def test_azure_iot_hub_producer(sdc_builder, sdc_executor, azure):
"""Test for Azure IoT Hub producer stage. Assumption is we setup Azure IoT Hub to route data
to a Azure Service Bus topic and then have the IoT Hub producer stage publish data which can be read at the
Service Bus topic to assert. We use a record deduplicator processor in between dev raw data source origin
and IoT Hub producer destination in order to determine exactly what has been ingested. The pipeline looks like:
dev_raw_data_source >> record_deduplicator >> azure_iot_hub_producer
record_deduplicator >> trash
"""
# Note: Test will fail till SDC-7638 is addressed/fixed
device_id = get_random_string(string.ascii_letters, 10)
raw_records = [{'Body': f'Hello {msg}'} for msg in range(10)]
try:
iot_hub = azure.iot_hub
logger.info('Creating %s IoT Hub device on %s IoT Hub', device_id, iot_hub.namespace)
device_info = iot_hub.create_device_id(device_id)
device_access_key = device_info['authentication']['symmetricKey']['primaryKey']
# Build the pipeline
builder = sdc_builder.get_pipeline_builder()
dev_raw_data_source = builder.add_stage('Dev Raw Data Source').set_attributes(data_format='JSON',
json_content='ARRAY_OBJECTS',
raw_data=json.dumps(raw_records))
record_deduplicator = builder.add_stage('Record Deduplicator')
trash = builder.add_stage('Trash')
azure_iot_hub_producer = builder.add_stage('Azure IoT Hub Producer')
azure_iot_hub_producer.set_attributes(data_format='JSON', device_id=device_id, json_content='MULTIPLE_OBJECTS',
shared_access_key=device_access_key)
dev_raw_data_source >> record_deduplicator >> azure_iot_hub_producer
record_deduplicator >> trash
producer_dest_pipeline = builder.build(title='IoT Producer pipeline').configure_for_environment(azure)
# Note: set topic subscriber first, else data will not be read.
topic_name = azure.iot_hub_topic
subscriber_id = get_random_string(string.ascii_letters, 10)
sb_service = azure.service_bus.service
logger.info('Creating %s Service Bus subscriber on topic %s', subscriber_id, topic_name)
sb_service.create_subscription(topic_name, subscriber_id)
# Use a Azure Rule filter to read topic for our specific Device ID.
rule = servicebus.Rule()
rule.filter_type = 'SqlFilter'
rule.filter_expression = f'iothub-connection-device-id = "{device_id}"'
sb_service.create_rule(topic_name, subscriber_id, f'{subscriber_id}Filter', rule)
sdc_executor.add_pipeline(producer_dest_pipeline)
sdc_executor.start_pipeline(producer_dest_pipeline).wait_for_pipeline_output_records_count(len(raw_records))
sdc_executor.stop_pipeline(producer_dest_pipeline)
# read data from Azure Service Bus Topic and assert
logger.info('Reading data from Azure Service Bus topic %s ...', topic_name)
messages = [json.loads(sb_service.receive_subscription_message(topic_name=topic_name,
subscription_name=subscriber_id, peek_lock=False,
timeout=10).body.decode().strip())
for _ in range(len(raw_records))]
assert raw_records == messages
finally:
logger.info('Deleting %s Service Bus subscriber on topic %s', subscriber_id, topic_name)
sb_service.delete_rule(topic_name, subscriber_id, servicebus.DEFAULT_RULE_NAME)
sb_service.delete_subscription(topic_name, subscriber_id)
logger.info('Deleting %s IoT Hub device on %s IoT Hub', device_id, iot_hub.namespace)
iot_hub.delete_device_id(device_id)
def test_push_pull(sdc_builder, sdc_executor, azure):
"""
We plan to verify that the connector works fine with Dev Raw Data Source and Dev Data Generator, an example of pull
and push strategies, so as we already verified Dev Raw Data Source, we will use Dev Data Generator here to complete
the coverage.
"""
device_id = get_random_string(string.ascii_letters, 10)
try:
iot_hub = azure.iot_hub
logger.info('Creating %s IoT Hub device on %s IoT Hub', device_id, iot_hub.namespace)
device_info = iot_hub.create_device_id(device_id)
device_access_key = device_info['authentication']['symmetricKey']['primaryKey']
pipeline_builder = sdc_builder.get_pipeline_builder()
pipeline_builder.add_error_stage('Discard')
dev_data_generator = pipeline_builder.add_stage('Dev Data Generator')
dev_data_generator.set_attributes(batch_size=1,
fields_to_generate=[
{'field': 'stringField', 'type': 'STRING', 'precision': 10, 'scale': 2}])
record_deduplicator = pipeline_builder.add_stage('Record Deduplicator')
trash = pipeline_builder.add_stage('Trash')
azure_iot_hub_producer = pipeline_builder.add_stage('Azure IoT Hub Producer')
azure_iot_hub_producer.set_attributes(data_format='JSON',
device_id=device_id,
json_content='MULTIPLE_OBJECTS',
shared_access_key=device_access_key)
dev_data_generator >> record_deduplicator >> azure_iot_hub_producer
record_deduplicator >> trash
producer_dest_pipeline = pipeline_builder.build().configure_for_environment(azure)
# Note: set topic subscriber first, else data will not be read.
topic_name = azure.iot_hub_topic
subscriber_id = get_random_string(string.ascii_letters, 10)
sb_service = azure.service_bus.service
logger.info('Creating %s Service Bus subscriber on topic %s', subscriber_id, topic_name)
sb_service.create_subscription(topic_name, subscriber_id)
# Use a Azure Rule filter to read topic for our specific Device ID.
rule = servicebus.Rule()
rule.filter_type = 'SqlFilter'
rule.filter_expression = f'iothub-connection-device-id = "{device_id}"'
sb_service.create_rule(topic_name, subscriber_id, f'{subscriber_id}Filter', rule)
sdc_executor.add_pipeline(producer_dest_pipeline)
sdc_executor.start_pipeline(producer_dest_pipeline).wait_for_pipeline_output_records_count(25, timeout_sec=120)
sdc_executor.stop_pipeline(producer_dest_pipeline)
# read data from Azure Service Bus Topic and assert
logger.info('Reading data from Azure Service Bus topic %s ...', topic_name)
messages = json.loads(sb_service.receive_subscription_message(topic_name=topic_name,
subscription_name=subscriber_id,
peek_lock=False,
timeout=10).body.decode().strip())
assert len(messages) == 1
finally:
logger.info('Deleting %s Service Bus subscriber on topic %s', subscriber_id, topic_name)
sb_service.delete_rule(topic_name, subscriber_id, servicebus.DEFAULT_RULE_NAME)
sb_service.delete_subscription(topic_name, subscriber_id)
logger.info('Deleting %s IoT Hub device on %s IoT Hub', device_id, iot_hub.namespace)
iot_hub.delete_device_id(device_id)
def test_data_types(sdc_builder, sdc_executor, azure, input, converter_type, expected):
device_id = get_random_string(string.ascii_letters, 10)
try:
iot_hub = azure.iot_hub
logger.info('Creating %s IoT Hub device on %s IoT Hub', device_id, iot_hub.namespace)
device_info = iot_hub.create_device_id(device_id)
device_access_key = device_info['authentication']['symmetricKey']['primaryKey']
pipeline_builder = sdc_builder.get_pipeline_builder()
pipeline_builder.add_error_stage('Discard')
origin = pipeline_builder.add_stage('Dev Raw Data Source')
origin.set_attributes(data_format='JSON',
raw_data=json.dumps({"value": input}))
record_deduplicator = pipeline_builder.add_stage('Record Deduplicator')
trash = pipeline_builder.add_stage('Trash')
converter = pipeline_builder.add_stage('Field Type Converter')
converter.set_attributes(conversion_method='BY_FIELD',
field_type_converter_configs=[{
'fields': ['/value'],
'targetType': converter_type,
'dataLocale': 'en,US',
'dateFormat': 'YYYY_MM_DD_HH_MM_SS',
'zonedDateTimeFormat': 'ISO_OFFSET_DATE_TIME',
'scale': 2
}])
azure_iot_hub_producer = pipeline_builder.add_stage('Azure IoT Hub Producer')
azure_iot_hub_producer.set_attributes(data_format='JSON',
device_id=device_id,
json_content='MULTIPLE_OBJECTS',
shared_access_key=device_access_key)
origin >> converter >> record_deduplicator >> azure_iot_hub_producer
record_deduplicator >> trash
producer_dest_pipeline = pipeline_builder.build().configure_for_environment(azure)
# Note: set topic subscriber first, else data will not be read.
topic_name = azure.iot_hub_topic
subscriber_id = get_random_string(string.ascii_letters, 10)
sb_service = azure.service_bus.service
logger.info('Creating %s Service Bus subscriber on topic %s', subscriber_id, topic_name)
sb_service.create_subscription(topic_name, subscriber_id)
# Use a Azure Rule filter to read topic for our specific Device ID.
rule = servicebus.Rule()
rule.filter_type = 'SqlFilter'
rule.filter_expression = f'iothub-connection-device-id = "{device_id}"'
sb_service.create_rule(topic_name, subscriber_id, f'{subscriber_id}Filter', rule)
sdc_executor.add_pipeline(producer_dest_pipeline)
sdc_executor.start_pipeline(producer_dest_pipeline).wait_for_pipeline_output_records_count(1, timeout_sec=120)
sdc_executor.stop_pipeline(producer_dest_pipeline)
# read data from Azure Service Bus Topic and assert
logger.info('Reading data from Azure Service Bus topic %s ...', topic_name)
messages = json.loads(sb_service.receive_subscription_message(topic_name=topic_name,
subscription_name=subscriber_id,
peek_lock=False,
timeout=10).body.decode().strip())
assert messages['value'] == expected
finally:
logger.info('Deleting %s Service Bus subscriber on topic %s', subscriber_id, topic_name)
sb_service.delete_rule(topic_name, subscriber_id, servicebus.DEFAULT_RULE_NAME)
sb_service.delete_subscription(topic_name, subscriber_id)
logger.info('Deleting %s IoT Hub device on %s IoT Hub', device_id, iot_hub.namespace)
iot_hub.delete_device_id(device_id)
def test_data_format_xml(sdc_builder, sdc_executor, azure):
"""
Test Azure EventHub origin parses XML in a variety of data types.
"""
device_id = get_random_string(string.ascii_letters, 10)
raw_data = '{"key":"value"}'
EXPECTED_XML_OUTPUT = '<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n<key>value</key>'
try:
iot_hub = azure.iot_hub
logger.info('Creating %s IoT Hub device on %s IoT Hub', device_id, iot_hub.namespace)
device_info = iot_hub.create_device_id(device_id)
device_access_key = device_info['authentication']['symmetricKey']['primaryKey']
pipeline_builder = sdc_builder.get_pipeline_builder()
pipeline_builder.add_error_stage('Discard')
origin = pipeline_builder.add_stage('Dev Raw Data Source')
origin.set_attributes(data_format='JSON',
raw_data=raw_data)
record_deduplicator = pipeline_builder.add_stage('Record Deduplicator')
trash = pipeline_builder.add_stage('Trash')
azure_iot_hub_producer = pipeline_builder.add_stage('Azure IoT Hub Producer')
azure_iot_hub_producer.set_attributes(data_format='XML',
device_id=device_id,
shared_access_key=device_access_key)
origin >> record_deduplicator >> azure_iot_hub_producer
record_deduplicator >> trash
producer_dest_pipeline = pipeline_builder.build().configure_for_environment(azure)
# Note: set topic subscriber first, else data will not be read.
topic_name = azure.iot_hub_topic
subscriber_id = get_random_string(string.ascii_letters, 10)
sb_service = azure.service_bus.service
logger.info('Creating %s Service Bus subscriber on topic %s', subscriber_id, topic_name)
sb_service.create_subscription(topic_name, subscriber_id)
# Use a Azure Rule filter to read topic for our specific Device ID.
rule = servicebus.Rule()
rule.filter_type = 'SqlFilter'
rule.filter_expression = f'iothub-connection-device-id = "{device_id}"'
sb_service.create_rule(topic_name, subscriber_id, f'{subscriber_id}Filter', rule)
sdc_executor.add_pipeline(producer_dest_pipeline)
sdc_executor.start_pipeline(producer_dest_pipeline).wait_for_pipeline_output_records_count(1, timeout_sec=120)
sdc_executor.stop_pipeline(producer_dest_pipeline)
# read data from Azure Service Bus Topic and assert
logger.info('Reading data from Azure Service Bus topic %s ...', topic_name)
messages = sb_service.receive_subscription_message(topic_name=topic_name,
subscription_name=subscriber_id,
peek_lock=False,
timeout=10).body.decode().rstrip()
assert messages == EXPECTED_XML_OUTPUT
finally:
logger.info('Deleting %s Service Bus subscriber on topic %s', subscriber_id, topic_name)
sb_service.delete_rule(topic_name, subscriber_id, servicebus.DEFAULT_RULE_NAME)
sb_service.delete_subscription(topic_name, subscriber_id)
logger.info('Deleting %s IoT Hub device on %s IoT Hub', device_id, iot_hub.namespace)
iot_hub.delete_device_id(device_id)
def test_multiple_batches(sdc_builder, sdc_executor, azure, batch_size):
"""
Test read multiple batches.
"""
device_id = get_random_string(string.ascii_letters, 10)
batches = 3
no_of_records = batches * batch_size
try:
iot_hub = azure.iot_hub
logger.info('Creating %s IoT Hub device on %s IoT Hub', device_id, iot_hub.namespace)
device_info = iot_hub.create_device_id(device_id)
device_access_key = device_info['authentication']['symmetricKey']['primaryKey']
pipeline_builder = sdc_builder.get_pipeline_builder()
pipeline_builder.add_error_stage('Discard')
origin = pipeline_builder.add_stage('Dev Data Generator')
origin.batch_size = batch_size
origin.fields_to_generate = [{
"type": "LONG_SEQUENCE",
"field": "seq"
}]
record_deduplicator = pipeline_builder.add_stage('Record Deduplicator')
trash = pipeline_builder.add_stage('Trash')
azure_iot_hub_producer = pipeline_builder.add_stage('Azure IoT Hub Producer')
azure_iot_hub_producer.set_attributes(data_format='JSON',
device_id=device_id,
json_content='MULTIPLE_OBJECTS',
shared_access_key=device_access_key)
origin >> record_deduplicator >> azure_iot_hub_producer
record_deduplicator >> trash
producer_dest_pipeline = pipeline_builder.build().configure_for_environment(azure)
# Note: set topic subscriber first, else data will not be read.
topic_name = azure.iot_hub_topic
subscriber_id = get_random_string(string.ascii_letters, 10)
sb_service = azure.service_bus.service
logger.info('Creating %s Service Bus subscriber on topic %s', subscriber_id, topic_name)
sb_service.create_subscription(topic_name, subscriber_id)
# Use a Azure Rule filter to read topic for our specific Device ID.
rule = servicebus.Rule()
rule.filter_type = 'SqlFilter'
rule.filter_expression = f'iothub-connection-device-id = "{device_id}"'
sb_service.create_rule(topic_name, subscriber_id, f'{subscriber_id}Filter', rule)
sdc_executor.add_pipeline(producer_dest_pipeline)
sdc_executor.start_pipeline(producer_dest_pipeline).wait_for_pipeline_batch_count(batches)
sdc_executor.stop_pipeline(producer_dest_pipeline)
# read data from Azure Service Bus Topic and assert
logger.info('Reading data from Azure Service Bus topic %s ...', topic_name)
messages = [json.loads(sb_service.receive_subscription_message(topic_name=topic_name,
subscription_name=subscriber_id,
peek_lock=False,
timeout=10).body.decode().strip())
for _ in range(no_of_records)]
assert len(messages) == no_of_records
finally:
logger.info('Deleting %s Service Bus subscriber on topic %s', subscriber_id, topic_name)
sb_service.delete_rule(topic_name, subscriber_id, servicebus.DEFAULT_RULE_NAME)
sb_service.delete_subscription(topic_name, subscriber_id)
logger.info('Deleting %s IoT Hub device on %s IoT Hub', device_id, iot_hub.namespace)
iot_hub.delete_device_id(device_id)
def test_object_names_device(sdc_builder, sdc_executor, azure, device_name_category, device_name):
"""
Verify that we can respect all the documented containers names possible
"""
device_id = device_name
raw_records = [{'Body': f'Hello {msg}'} for msg in range(10)]
try:
iot_hub = azure.iot_hub
logger.info('Creating %s IoT Hub device on %s IoT Hub', device_id, iot_hub.namespace)
device_info = iot_hub.create_device_id(device_id)
device_access_key = device_info['authentication']['symmetricKey']['primaryKey']
pipeline_builder = sdc_builder.get_pipeline_builder()
pipeline_builder.add_error_stage('Discard')
origin = pipeline_builder.add_stage('Dev Raw Data Source')
origin.set_attributes(data_format='JSON',
raw_data=json.dumps(raw_records))
record_deduplicator = pipeline_builder.add_stage('Record Deduplicator')
trash = pipeline_builder.add_stage('Trash')
azure_iot_hub_producer = pipeline_builder.add_stage('Azure IoT Hub Producer')
azure_iot_hub_producer.set_attributes(data_format='JSON',
device_id=device_id,
json_content='MULTIPLE_OBJECTS',
shared_access_key=device_access_key)
origin >> record_deduplicator >> azure_iot_hub_producer
record_deduplicator >> trash
producer_dest_pipeline = pipeline_builder.build().configure_for_environment(azure)
# Note: set topic subscriber first, else data will not be read.
topic_name = azure.iot_hub_topic
subscriber_id = get_random_string(string.ascii_letters, 10)
sb_service = azure.service_bus.service
logger.info('Creating %s Service Bus subscriber on topic %s', subscriber_id, topic_name)
sb_service.create_subscription(topic_name, subscriber_id)
# Use a Azure Rule filter to read topic for our specific Device ID.
rule = servicebus.Rule()
rule.filter_type = 'SqlFilter'
rule.filter_expression = f'iothub-connection-device-id = "{device_id}"'
sb_service.create_rule(topic_name, subscriber_id, f'{subscriber_id}Filter', rule)
sdc_executor.add_pipeline(producer_dest_pipeline)
sdc_executor.start_pipeline(producer_dest_pipeline).wait_for_pipeline_output_records_count(1, timeout_sec=120)
sdc_executor.stop_pipeline(producer_dest_pipeline)
# read data from Azure Service Bus Topic and assert
logger.info('Reading data from Azure Service Bus topic %s ...', topic_name)
messages = json.loads(sb_service.receive_subscription_message(topic_name=topic_name,
subscription_name=subscriber_id,
peek_lock=False,
timeout=10).body.decode().strip())
assert messages == raw_records
finally:
logger.info('Deleting %s Service Bus subscriber on topic %s', subscriber_id, topic_name)
sb_service.delete_rule(topic_name, subscriber_id, servicebus.DEFAULT_RULE_NAME)
sb_service.delete_subscription(topic_name, subscriber_id)
logger.info('Deleting %s IoT Hub device on %s IoT Hub', device_id, iot_hub.namespace)
iot_hub.delete_device_id(device_id)