def _run_op(self, op):
if isinstance(op, pipeline_ops_base.InitializePipelineOperation):
if self.pipeline_root.pipeline_configuration.module_id:
# Module Format
client_id = "{}/{}".format(
self.pipeline_root.pipeline_configuration.device_id,
self.pipeline_root.pipeline_configuration.module_id,
)
else:
# Device Format
client_id = self.pipeline_root.pipeline_configuration.device_id
query_param_seq = []
# Apply query parameters (i.e. key1=value1&key2=value2...&keyN=valueN format)
custom_product_info = str(
self.pipeline_root.pipeline_configuration.product_info)
if custom_product_info.startswith(
pkg_constant.DIGITAL_TWIN_PREFIX): # Digital Twin Stuff
query_param_seq.append(
("api-version", pkg_constant.DIGITAL_TWIN_API_VERSION))
query_param_seq.append(
("DeviceClientType", user_agent.get_iothub_user_agent()))
query_param_seq.append((pkg_constant.DIGITAL_TWIN_QUERY_HEADER,
custom_product_info))
else:
query_param_seq.append(
("api-version", pkg_constant.IOTHUB_API_VERSION))
query_param_seq.append(
("DeviceClientType",
user_agent.get_iothub_user_agent() + custom_product_info))
# NOTE: Client ID (including the device and/or module ids that are in it)
# is NOT url encoded as part of the username. Neither is the hostname.
# The sequence of key/value property pairs (query_param_seq) however, MUST have all
# keys and values URL encoded.
# See the repo wiki article for details:
# https://github.com/Azure/azure-iot-sdk-python/wiki/URL-Encoding-(MQTT)
username = "******".format(
hostname=self.pipeline_root.pipeline_configuration.hostname,
client_id=client_id,
query_params=version_compat.urlencode(
query_param_seq, quote_via=urllib.parse.quote),
)
# Dynamically attach the derived MQTT values to the InitalizePipelineOperation
# to be used later down the pipeline
op.username = username
op.client_id = client_id
self.send_op_down(op)
elif isinstance(
op, pipeline_ops_iothub.SendD2CMessageOperation) or isinstance(
op, pipeline_ops_iothub.SendOutputMessageOperation):
# Convert SendTelementry and SendOutputMessageOperation operations into MQTT Publish operations
telemetry_topic = mqtt_topic_iothub.get_telemetry_topic_for_publish(
device_id=self.pipeline_root.pipeline_configuration.device_id,
module_id=self.pipeline_root.pipeline_configuration.module_id,
)
topic = mqtt_topic_iothub.encode_message_properties_in_topic(
op.message, telemetry_topic)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=op.message.data,
)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_iothub.SendMethodResponseOperation):
# Sending a Method Response gets translated into an MQTT Publish operation
topic = mqtt_topic_iothub.get_method_topic_for_publish(
op.method_response.request_id, op.method_response.status)
payload = json.dumps(op.method_response.payload)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=payload)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.EnableFeatureOperation):
# Enabling a feature gets translated into an MQTT subscribe operation
topic = self._get_feature_subscription_topic(op.feature_name)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTSubscribeOperation,
topic=topic)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.DisableFeatureOperation):
# Disabling a feature gets turned into an MQTT unsubscribe operation
topic = self._get_feature_subscription_topic(op.feature_name)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTUnsubscribeOperation,
topic=topic)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.RequestOperation):
if op.request_type == pipeline_constant.TWIN:
topic = mqtt_topic_iothub.get_twin_topic_for_publish(
method=op.method,
resource_location=op.resource_location,
request_id=op.request_id,
)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=op.request_body,
)
self.send_op_down(worker_op)
else:
raise pipeline_exceptions.OperationError(
"RequestOperation request_type {} not supported".format(
op.request_type))
else:
# All other operations get passed down
super(IoTHubMQTTTranslationStage, self)._run_op(op)
def _run_op(self, op):
if isinstance(op, pipeline_ops_base.InitializePipelineOperation):
if self.pipeline_root.pipeline_configuration.module_id:
# Module Format
client_id = "{}/{}".format(
self.pipeline_root.pipeline_configuration.device_id,
self.pipeline_root.pipeline_configuration.module_id,
)
else:
# Device Format
client_id = self.pipeline_root.pipeline_configuration.device_id
# Apply query parameters (i.e. key1=value1&key2=value2...&keyN=valueN format)
query_param_seq = [
("api-version", pkg_constant.IOTHUB_API_VERSION),
(
"DeviceClientType",
user_agent.get_iothub_user_agent() +
self.pipeline_root.pipeline_configuration.product_info,
),
]
username = "******".format(
hostname=self.pipeline_root.pipeline_configuration.hostname,
client_id=client_id,
query_params=version_compat.urlencode(
query_param_seq, quote_via=urllib.parse.quote),
)
# Dynamically attach the derived MQTT values to the InitalizePipelineOperation
# to be used later down the pipeline
op.username = username
op.client_id = client_id
self.send_op_down(op)
elif isinstance(
op, pipeline_ops_iothub.SendD2CMessageOperation) or isinstance(
op, pipeline_ops_iothub.SendOutputMessageOperation):
# Convert SendTelementry and SendOutputMessageOperation operations into MQTT Publish operations
telemetry_topic = mqtt_topic_iothub.get_telemetry_topic_for_publish(
device_id=self.pipeline_root.pipeline_configuration.device_id,
module_id=self.pipeline_root.pipeline_configuration.module_id,
)
topic = mqtt_topic_iothub.encode_message_properties_in_topic(
op.message, telemetry_topic)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=op.message.data,
)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_iothub.SendMethodResponseOperation):
# Sending a Method Response gets translated into an MQTT Publish operation
topic = mqtt_topic_iothub.get_method_topic_for_publish(
op.method_response.request_id, op.method_response.status)
payload = json.dumps(op.method_response.payload)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=payload)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.EnableFeatureOperation):
# Enabling a feature gets translated into an MQTT subscribe operation
topic = self._get_feature_subscription_topic(op.feature_name)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTSubscribeOperation,
topic=topic)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.DisableFeatureOperation):
# Disabling a feature gets turned into an MQTT unsubscribe operation
topic = self._get_feature_subscription_topic(op.feature_name)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTUnsubscribeOperation,
topic=topic)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.RequestOperation):
if op.request_type == pipeline_constant.TWIN:
topic = mqtt_topic_iothub.get_twin_topic_for_publish(
method=op.method,
resource_location=op.resource_location,
request_id=op.request_id,
)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=op.request_body,
)
self.send_op_down(worker_op)
else:
raise pipeline_exceptions.OperationError(
"RequestOperation request_type {} not supported".format(
op.request_type))
else:
# All other operations get passed down
super(IoTHubMQTTTranslationStage, self)._run_op(op)
def encode_message_properties_in_topic(message_to_send, topic):
"""
uri-encode the system properties of a message as key-value pairs on the topic with defined keys.
Additionally if the message has user defined properties, the property keys and values shall be
uri-encoded and appended at the end of the above topic with the following convention:
'<key>=<value>&<key2>=<value2>&<key3>=<value3>(...)'
:param message_to_send: The message to send
:param topic: The topic which has not been encoded yet. For a device it looks like
"devices/<deviceId>/messages/events/" and for a module it looks like
"devices/<deviceId>/modules/<moduleId>/messages/events/
:return: The topic which has been uri-encoded
"""
system_properties = []
if message_to_send.output_name:
system_properties.append(("$.on", str(message_to_send.output_name)))
if message_to_send.message_id:
system_properties.append(("$.mid", str(message_to_send.message_id)))
if message_to_send.correlation_id:
system_properties.append(
("$.cid", str(message_to_send.correlation_id)))
if message_to_send.user_id:
system_properties.append(("$.uid", str(message_to_send.user_id)))
if message_to_send.content_type:
system_properties.append(("$.ct", str(message_to_send.content_type)))
if message_to_send.content_encoding:
system_properties.append(
("$.ce", str(message_to_send.content_encoding)))
if message_to_send.iothub_interface_id:
system_properties.append(
("$.ifid", str(message_to_send.iothub_interface_id)))
if message_to_send.expiry_time_utc:
system_properties.append((
"$.exp",
message_to_send.expiry_time_utc.isoformat() # returns string
if isinstance(message_to_send.expiry_time_utc, date) else
message_to_send.expiry_time_utc,
))
system_properties_encoded = version_compat.urlencode(
system_properties, quote_via=urllib.parse.quote)
topic += system_properties_encoded
if message_to_send.custom_properties and len(
message_to_send.custom_properties) > 0:
if system_properties and len(system_properties) > 0:
topic += "&"
# Convert the custom properties to a sorted list in order to ensure the
# resulting ordering in the topic string is consistent across versions of Python.
# Convert to the properties to strings for safety.
custom_prop_seq = [
(str(i[0]), str(i[1]))
for i in list(message_to_send.custom_properties.items())
]
custom_prop_seq.sort()
# Validate that string conversion has not created duplicate keys
keys = [i[0] for i in custom_prop_seq]
if len(keys) != len(set(keys)):
raise ValueError("Duplicate keys in custom properties!")
user_properties_encoded = version_compat.urlencode(
custom_prop_seq, quote_via=urllib.parse.quote)
topic += user_properties_encoded
return topic
def _run_op(self, op):
if isinstance(op, pipeline_ops_base.InitializePipelineOperation):
client_id = self.pipeline_root.pipeline_configuration.registration_id
query_param_seq = [
("api-version", pkg_constant.PROVISIONING_API_VERSION),
("ClientVersion", user_agent.get_provisioning_user_agent()),
]
username = "******".format(
id_scope=self.pipeline_root.pipeline_configuration.id_scope,
registration_id=self.pipeline_root.pipeline_configuration.
registration_id,
query_params=version_compat.urlencode(
query_param_seq, quote_via=urllib.parse.quote),
)
# Dynamically attach the derived MQTT values to the InitalizePipelineOperation
# to be used later down the pipeline
op.username = username
op.client_id = client_id
self.send_op_down(op)
elif isinstance(op, pipeline_ops_base.RequestOperation):
if op.request_type == pipeline_constant.REGISTER:
topic = mqtt_topic_provisioning.get_register_topic_for_publish(
request_id=op.request_id)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=op.request_body,
)
self.send_op_down(worker_op)
elif op.request_type == pipeline_constant.QUERY:
topic = mqtt_topic_provisioning.get_query_topic_for_publish(
request_id=op.request_id,
operation_id=op.query_params["operation_id"])
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=op.request_body,
)
self.send_op_down(worker_op)
else:
raise pipeline_exceptions.OperationError(
"RequestOperation request_type {} not supported".format(
op.request_type))
elif isinstance(op, pipeline_ops_base.EnableFeatureOperation):
# The only supported feature is REGISTER
if not op.feature_name == pipeline_constant.REGISTER:
raise pipeline_exceptions.OperationError(
"Trying to enable/disable invalid feature - {}".format(
op.feature_name))
# Enabling for register gets translated into an MQTT subscribe operation
topic = mqtt_topic_provisioning.get_register_topic_for_subscribe()
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTSubscribeOperation,
topic=topic)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.DisableFeatureOperation):
# The only supported feature is REGISTER
if not op.feature_name == pipeline_constant.REGISTER:
raise pipeline_exceptions.OperationError(
"Trying to enable/disable invalid feature - {}".format(
op.feature_name))
# Disabling a register response gets turned into an MQTT unsubscribe operation
topic = mqtt_topic_provisioning.get_register_topic_for_subscribe()
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTUnsubscribeOperation,
topic=topic)
self.send_op_down(worker_op)
else:
# All other operations get passed down
super(ProvisioningMQTTTranslationStage, self)._run_op(op)
def _run_op(self, op):
if isinstance(op, pipeline_ops_iothub.SetIoTHubConnectionArgsOperation):
self.device_id = op.device_id
self.module_id = op.module_id
# if we get auth provider args from above, we save some, use some to build topic names,
# and always pass it down because we know that the MQTT protocol stage will also want
# to receive these args.
self._set_topic_names(device_id=op.device_id, module_id=op.module_id)
if op.module_id:
client_id = "{}/{}".format(op.device_id, op.module_id)
else:
client_id = op.device_id
# For MQTT, the entire user agent string should be appended to the username field in the connect packet
# For example, the username may look like this without custom parameters:
# yosephsandboxhub.azure-devices.net/alpha/?api-version=2018-06-30&DeviceClientType=py-azure-iot-device%2F2.0.0-preview.12
# The customer user agent string would simply be appended to the end of this username, in URL Encoded format.
query_param_seq = [
("api-version", pkg_constant.IOTHUB_API_VERSION),
(
"DeviceClientType",
ProductInfo.get_iothub_user_agent()
+ str(self.pipeline_root.pipeline_configuration.product_info),
),
]
username = "******".format(
hostname=op.hostname,
client_id=client_id,
query_params=version_compat.urlencode(
query_param_seq, quote_via=urllib.parse.quote
),
)
if op.gateway_hostname:
hostname = op.gateway_hostname
else:
hostname = op.hostname
# TODO: test to make sure client_cert and sas_token travel down correctly
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.SetMQTTConnectionArgsOperation,
client_id=client_id,
hostname=hostname,
username=username,
server_verification_cert=op.server_verification_cert,
client_cert=op.client_cert,
sas_token=op.sas_token,
)
self.send_op_down(worker_op)
elif (
isinstance(op, pipeline_ops_base.UpdateSasTokenOperation)
and self.pipeline_root.connected
):
logger.debug(
"{}({}): Connected. Passing op down and reauthorizing after token is updated.".format(
self.name, op.name
)
)
# make a callback that either fails the UpdateSasTokenOperation (if the lower level failed it),
# or issues a ReauthorizeConnectionOperation (if the lower level returned success for the UpdateSasTokenOperation)
def on_token_update_complete(op, error):
if error:
logger.error(
"{}({}) token update failed. returning failure {}".format(
self.name, op.name, error
)
)
else:
logger.debug(
"{}({}) token update succeeded. reauthorizing".format(self.name, op.name)
)
# Stop completion of Token Update op, and only continue upon completion of ReauthorizeConnectionOperation
op.halt_completion()
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_base.ReauthorizeConnectionOperation
)
self.send_op_down(worker_op)
# now, pass the UpdateSasTokenOperation down with our new callback.
op.add_callback(on_token_update_complete)
self.send_op_down(op)
elif isinstance(op, pipeline_ops_iothub.SendD2CMessageOperation) or isinstance(
op, pipeline_ops_iothub.SendOutputEventOperation
):
# Convert SendTelementry and SendOutputEventOperation operations into MQTT Publish operations
topic = mqtt_topic_iothub.encode_message_properties_in_topic(
op.message, self.telemetry_topic
)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=op.message.data,
)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_iothub.SendMethodResponseOperation):
# Sending a Method Response gets translated into an MQTT Publish operation
topic = mqtt_topic_iothub.get_method_topic_for_publish(
op.method_response.request_id, str(op.method_response.status)
)
payload = json.dumps(op.method_response.payload)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation, topic=topic, payload=payload
)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.EnableFeatureOperation):
# Enabling a feature gets translated into an MQTT subscribe operation
topic = self.feature_to_topic[op.feature_name]
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTSubscribeOperation, topic=topic
)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.DisableFeatureOperation):
# Disabling a feature gets turned into an MQTT unsubscribe operation
topic = self.feature_to_topic[op.feature_name]
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTUnsubscribeOperation, topic=topic
)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.RequestOperation):
if op.request_type == pipeline_constant.TWIN:
topic = mqtt_topic_iothub.get_twin_topic_for_publish(
method=op.method,
resource_location=op.resource_location,
request_id=op.request_id,
)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=op.request_body,
)
self.send_op_down(worker_op)
else:
raise pipeline_exceptions.OperationError(
"RequestOperation request_type {} not supported".format(op.request_type)
)
else:
# All other operations get passed down
super(IoTHubMQTTTranslationStage, self)._run_op(op)
def _run_op(self, op):
if isinstance(op, pipeline_ops_provisioning.SetProvisioningClientConnectionArgsOperation):
# get security client args from above, save some, use some to build topic names,
# always pass it down because MQTT protocol stage will also want to receive these args.
client_id = op.registration_id
query_param_seq = [
("api-version", pkg_constant.PROVISIONING_API_VERSION),
("ClientVersion", ProductInfo.get_provisioning_user_agent()),
]
username = "******".format(
id_scope=op.id_scope,
registration_id=op.registration_id,
query_params=version_compat.urlencode(
query_param_seq, quote_via=urllib.parse.quote
),
)
hostname = op.provisioning_host
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.SetMQTTConnectionArgsOperation,
client_id=client_id,
hostname=hostname,
username=username,
client_cert=op.client_cert,
sas_token=op.sas_token,
)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.RequestOperation):
if op.request_type == pipeline_constant.REGISTER:
topic = mqtt_topic_provisioning.get_register_topic_for_publish(
request_id=op.request_id
)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=op.request_body,
)
self.send_op_down(worker_op)
else:
topic = mqtt_topic_provisioning.get_query_topic_for_publish(
request_id=op.request_id, operation_id=op.query_params["operation_id"]
)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=op.request_body,
)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.EnableFeatureOperation):
# Enabling for register gets translated into an MQTT subscribe operation
topic = mqtt_topic_provisioning.get_register_topic_for_subscribe()
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTSubscribeOperation, topic=topic
)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.DisableFeatureOperation):
# Disabling a register response gets turned into an MQTT unsubscribe operation
topic = mqtt_topic_provisioning.get_register_topic_for_subscribe()
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTUnsubscribeOperation, topic=topic
)
self.send_op_down(worker_op)
else:
# All other operations get passed down
super(ProvisioningMQTTTranslationStage, self)._run_op(op)