def get_certificate(self):
"""
Return the server verification certificate from the trust bundle that can be used to
validate the server-side SSL TLS connection that we use to talk to Edge
:return: The server verification certificate to use for connections to the Azure IoT Edge
instance, as a PEM certificate in string form.
:raises: IoTEdgeError if unable to retrieve the certificate.
"""
r = requests.get(
self.workload_uri + "trust-bundle",
params={"api-version": self.api_version},
headers={
"User-Agent":
urllib.parse.quote_plus(user_agent.get_iothub_user_agent())
},
)
# Validate that the request was successful
try:
r.raise_for_status()
except requests.exceptions.HTTPError as e:
raise IoTEdgeError("Unable to get trust bundle from Edge") from e
# Decode the trust bundle
try:
bundle = r.json()
except ValueError as e:
raise IoTEdgeError("Unable to decode trust bundle") from e
# Retrieve the certificate
try:
cert = bundle["certificate"]
except KeyError as e:
raise IoTEdgeError("No certificate in trust bundle") from e
return cert
def test_requests_data_signing(self, mocker, edge_hsm):
data_str = "somedata"
data_str_b64 = "c29tZWRhdGE="
mock_request_post = mocker.patch.object(requests, "post")
mock_request_post.return_value.json.return_value = {
"digest": "somedigest"
}
expected_url = "{workload_uri}modules/{module_id}/genid/{generation_id}/sign".format(
workload_uri=edge_hsm.workload_uri,
module_id=edge_hsm.module_id,
generation_id=edge_hsm.generation_id,
)
expected_params = {"api-version": edge_hsm.api_version}
expected_headers = {
"User-Agent":
urllib.parse.quote(user_agent.get_iothub_user_agent(), safe="")
}
expected_json = json.dumps({
"keyId": "primary",
"algo": "HMACSHA256",
"data": data_str_b64
})
edge_hsm.sign(data_str)
assert mock_request_post.call_count == 1
assert mock_request_post.call_args == mocker.call(
url=expected_url,
params=expected_params,
headers=expected_headers,
data=expected_json)
def test_username(self, stage, op, pipeline_config, cust_product_info):
pipeline_config.product_info = cust_product_info
assert not hasattr(op, "username")
stage.run_op(op)
expected_username = "******".format(
hostname=pipeline_config.hostname,
client_id=pipeline_config.device_id,
api_version=pkg_constant.IOTHUB_API_VERSION,
user_agent=urllib.parse.quote(user_agent.get_iothub_user_agent(), safe=""),
custom_product_info=urllib.parse.quote(pipeline_config.product_info, safe=""),
)
assert op.username == expected_username
def test_username_for_digital_twin(self, stage, op, pipeline_config, digital_twin_product_info):
pipeline_config.product_info = digital_twin_product_info
assert not hasattr(op, "username")
stage.run_op(op)
expected_username = "******".format(
hostname=pipeline_config.hostname,
client_id=pipeline_config.device_id,
api_version=pkg_constant.DIGITAL_TWIN_API_VERSION,
user_agent=urllib.parse.quote(user_agent.get_iothub_user_agent(), safe=""),
digital_twin_prefix=pkg_constant.DIGITAL_TWIN_QUERY_HEADER,
custom_product_info=urllib.parse.quote(pipeline_config.product_info, safe=""),
)
assert op.username == expected_username
def test_requests_trust_bundle(self, mocker, edge_hsm):
mock_request_get = mocker.patch.object(requests, "get")
expected_url = edge_hsm.workload_uri + "trust-bundle"
expected_params = {"api-version": edge_hsm.api_version}
expected_headers = {
"User-Agent":
urllib.parse.quote_plus(user_agent.get_iothub_user_agent())
}
edge_hsm.get_certificate()
assert mock_request_get.call_count == 1
assert mock_request_get.call_args == mocker.call(
expected_url, params=expected_params, headers=expected_headers)
def sign(self, data_str):
"""
Use the IoTEdge HSM to sign a piece of string data. The caller should then insert the
returned value (the signature) into the 'sig' field of a SharedAccessSignature string.
:param str data_str: The data string to sign
:return: The signature, as a URI-encoded and base64-encoded value that is ready to
directly insert into the SharedAccessSignature string.
:raises: IoTEdgeError if unable to sign the data.
"""
encoded_data_str = base64.b64encode(data_str.encode("utf-8")).decode()
path = "{workload_uri}modules/{module_id}/genid/{gen_id}/sign".format(
workload_uri=self.workload_uri,
module_id=self.module_id,
gen_id=self.generation_id)
sign_request = {
"keyId": "primary",
"algo": "HMACSHA256",
"data": encoded_data_str
}
r = requests.post( # can we use json field instead of data?
url=path,
params={"api-version": self.api_version},
headers={
"User-Agent":
urllib.parse.quote(user_agent.get_iothub_user_agent(), safe="")
},
data=json.dumps(sign_request),
)
try:
r.raise_for_status()
except requests.exceptions.HTTPError as e:
raise IoTEdgeError("Unable to sign data") from e
try:
sign_response = r.json()
except ValueError as e:
raise IoTEdgeError("Unable to decode signed data") from e
try:
signed_data_str = sign_response["digest"]
except KeyError as e:
raise IoTEdgeError("No signed data received") from e
return signed_data_str # what format is this? string? bytes?
def test_get_iothub_user_agent(self):
user_agent_str = user_agent.get_iothub_user_agent()
assert IOTHUB_IDENTIFIER in user_agent_str
assert VERSION in user_agent_str
assert platform.python_version() in user_agent_str
assert platform.system() in user_agent_str
assert platform.version() in user_agent_str
assert platform.machine() in user_agent_str
expected_part_agent = check_agent_format.format(
identifier=IOTHUB_IDENTIFIER,
version=VERSION,
python_runtime=platform.python_version(),
os_type=platform.system(),
os_release=platform.version(),
architecture=platform.machine(),
)
assert expected_part_agent == user_agent_str
def test_new_op_headers(self, mocker, stage, op, custom_user_agent,
pipeline_config):
stage.pipeline_root.pipeline_configuration.product_info = custom_user_agent
stage.run_op(op)
# Op was sent down
assert stage.send_op_down.call_count == 1
new_op = stage.send_op_down.call_args[0][0]
assert isinstance(new_op,
pipeline_ops_http.HTTPRequestAndResponseOperation)
# Validate headers
expected_user_agent = urllib.parse.quote_plus(
user_agent.get_iothub_user_agent() + str(custom_user_agent))
assert new_op.headers["Host"] == pipeline_config.hostname
assert new_op.headers[
"Content-Type"] == "application/json; charset=utf-8"
assert new_op.headers["Content-Length"] == len(new_op.body)
assert new_op.headers["User-Agent"] == expected_user_agent
def test_new_op_headers(self, mocker, stage, op, custom_user_agent,
pipeline_config):
stage.pipeline_root.pipeline_configuration.product_info = custom_user_agent
stage.run_op(op)
# Op was sent down
assert stage.send_op_down.call_count == 1
new_op = stage.send_op_down.call_args[0][0]
assert isinstance(new_op,
pipeline_ops_http.HTTPRequestAndResponseOperation)
# Validate headers
expected_user_agent = urllib.parse.quote_plus(
user_agent.get_iothub_user_agent() + str(custom_user_agent))
expected_edge_string = "{}/{}".format(pipeline_config.device_id,
pipeline_config.module_id)
assert new_op.headers["Host"] == pipeline_config.gateway_hostname
assert new_op.headers["Content-Type"] == "application/json"
assert new_op.headers["Content-Length"] == str(len(new_op.body))
assert new_op.headers["x-ms-edge-moduleId"] == expected_edge_string
assert new_op.headers["User-Agent"] == expected_user_agent
def _run_op(self, op):
if isinstance(op, pipeline_ops_base.InitializePipelineOperation):
if self.pipeline_root.pipeline_configuration.module_id:
# Module Format
client_id = "{}/{}".format(
self.pipeline_root.pipeline_configuration.device_id,
self.pipeline_root.pipeline_configuration.module_id,
)
else:
# Device Format
client_id = self.pipeline_root.pipeline_configuration.device_id
query_param_seq = []
# Apply query parameters (i.e. key1=value1&key2=value2...&keyN=valueN format)
custom_product_info = str(
self.pipeline_root.pipeline_configuration.product_info)
if custom_product_info.startswith(
pkg_constant.DIGITAL_TWIN_PREFIX): # Digital Twin Stuff
query_param_seq.append(
("api-version", pkg_constant.DIGITAL_TWIN_API_VERSION))
query_param_seq.append(
("DeviceClientType", user_agent.get_iothub_user_agent()))
query_param_seq.append((pkg_constant.DIGITAL_TWIN_QUERY_HEADER,
custom_product_info))
else:
query_param_seq.append(
("api-version", pkg_constant.IOTHUB_API_VERSION))
query_param_seq.append(
("DeviceClientType",
user_agent.get_iothub_user_agent() + custom_product_info))
# NOTE: Client ID (including the device and/or module ids that are in it)
# is NOT url encoded as part of the username. Neither is the hostname.
# The sequence of key/value property pairs (query_param_seq) however, MUST have all
# keys and values URL encoded.
# See the repo wiki article for details:
# https://github.com/Azure/azure-iot-sdk-python/wiki/URL-Encoding-(MQTT)
username = "******".format(
hostname=self.pipeline_root.pipeline_configuration.hostname,
client_id=client_id,
query_params=version_compat.urlencode(
query_param_seq, quote_via=urllib.parse.quote),
)
# Dynamically attach the derived MQTT values to the InitalizePipelineOperation
# to be used later down the pipeline
op.username = username
op.client_id = client_id
self.send_op_down(op)
elif isinstance(
op, pipeline_ops_iothub.SendD2CMessageOperation) or isinstance(
op, pipeline_ops_iothub.SendOutputMessageOperation):
# Convert SendTelementry and SendOutputMessageOperation operations into MQTT Publish operations
telemetry_topic = mqtt_topic_iothub.get_telemetry_topic_for_publish(
device_id=self.pipeline_root.pipeline_configuration.device_id,
module_id=self.pipeline_root.pipeline_configuration.module_id,
)
topic = mqtt_topic_iothub.encode_message_properties_in_topic(
op.message, telemetry_topic)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=op.message.data,
)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_iothub.SendMethodResponseOperation):
# Sending a Method Response gets translated into an MQTT Publish operation
topic = mqtt_topic_iothub.get_method_topic_for_publish(
op.method_response.request_id, op.method_response.status)
payload = json.dumps(op.method_response.payload)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=payload)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.EnableFeatureOperation):
# Enabling a feature gets translated into an MQTT subscribe operation
topic = self._get_feature_subscription_topic(op.feature_name)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTSubscribeOperation,
topic=topic)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.DisableFeatureOperation):
# Disabling a feature gets turned into an MQTT unsubscribe operation
topic = self._get_feature_subscription_topic(op.feature_name)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTUnsubscribeOperation,
topic=topic)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.RequestOperation):
if op.request_type == pipeline_constant.TWIN:
topic = mqtt_topic_iothub.get_twin_topic_for_publish(
method=op.method,
resource_location=op.resource_location,
request_id=op.request_id,
)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=op.request_body,
)
self.send_op_down(worker_op)
else:
raise pipeline_exceptions.OperationError(
"RequestOperation request_type {} not supported".format(
op.request_type))
else:
# All other operations get passed down
super(IoTHubMQTTTranslationStage, self)._run_op(op)
def _run_op(self, op):
if isinstance(op, pipeline_ops_base.InitializePipelineOperation):
if self.pipeline_root.pipeline_configuration.module_id:
# Module Format
client_id = "{}/{}".format(
self.pipeline_root.pipeline_configuration.device_id,
self.pipeline_root.pipeline_configuration.module_id,
)
else:
# Device Format
client_id = self.pipeline_root.pipeline_configuration.device_id
# Apply query parameters (i.e. key1=value1&key2=value2...&keyN=valueN format)
query_param_seq = [
("api-version", pkg_constant.IOTHUB_API_VERSION),
(
"DeviceClientType",
user_agent.get_iothub_user_agent() +
self.pipeline_root.pipeline_configuration.product_info,
),
]
username = "******".format(
hostname=self.pipeline_root.pipeline_configuration.hostname,
client_id=client_id,
query_params=version_compat.urlencode(
query_param_seq, quote_via=urllib.parse.quote),
)
# Dynamically attach the derived MQTT values to the InitalizePipelineOperation
# to be used later down the pipeline
op.username = username
op.client_id = client_id
self.send_op_down(op)
elif isinstance(
op, pipeline_ops_iothub.SendD2CMessageOperation) or isinstance(
op, pipeline_ops_iothub.SendOutputMessageOperation):
# Convert SendTelementry and SendOutputMessageOperation operations into MQTT Publish operations
telemetry_topic = mqtt_topic_iothub.get_telemetry_topic_for_publish(
device_id=self.pipeline_root.pipeline_configuration.device_id,
module_id=self.pipeline_root.pipeline_configuration.module_id,
)
topic = mqtt_topic_iothub.encode_message_properties_in_topic(
op.message, telemetry_topic)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=op.message.data,
)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_iothub.SendMethodResponseOperation):
# Sending a Method Response gets translated into an MQTT Publish operation
topic = mqtt_topic_iothub.get_method_topic_for_publish(
op.method_response.request_id, op.method_response.status)
payload = json.dumps(op.method_response.payload)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=payload)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.EnableFeatureOperation):
# Enabling a feature gets translated into an MQTT subscribe operation
topic = self._get_feature_subscription_topic(op.feature_name)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTSubscribeOperation,
topic=topic)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.DisableFeatureOperation):
# Disabling a feature gets turned into an MQTT unsubscribe operation
topic = self._get_feature_subscription_topic(op.feature_name)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTUnsubscribeOperation,
topic=topic)
self.send_op_down(worker_op)
elif isinstance(op, pipeline_ops_base.RequestOperation):
if op.request_type == pipeline_constant.TWIN:
topic = mqtt_topic_iothub.get_twin_topic_for_publish(
method=op.method,
resource_location=op.resource_location,
request_id=op.request_id,
)
worker_op = op.spawn_worker_op(
worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation,
topic=topic,
payload=op.request_body,
)
self.send_op_down(worker_op)
else:
raise pipeline_exceptions.OperationError(
"RequestOperation request_type {} not supported".format(
op.request_type))
else:
# All other operations get passed down
super(IoTHubMQTTTranslationStage, self)._run_op(op)
def _run_op(self, op):
if isinstance(op, pipeline_ops_iothub_http.MethodInvokeOperation):
logger.debug(
"{}({}): Translating Method Invoke Operation for HTTP.".format(
self.name, op.name))
query_params = "api-version={apiVersion}".format(
apiVersion=pkg_constant.IOTHUB_API_VERSION)
# if the target is a module.
body = json.dumps(op.method_params)
path = http_path_iothub.get_method_invoke_path(
op.target_device_id, op.target_module_id)
# NOTE: we do not add the sas Authorization header here. Instead we add it later on in
# the HTTPTransportStage
x_ms_edge_string = "{deviceId}/{moduleId}".format(
deviceId=self.pipeline_root.pipeline_configuration.device_id,
moduleId=self.pipeline_root.pipeline_configuration.module_id,
) # these are the identifiers of the current module
user_agent_string = urllib.parse.quote_plus(
user_agent.get_iothub_user_agent() +
str(self.pipeline_root.pipeline_configuration.product_info))
# Method Invoke must be addressed to the gateway hostname because it is an Edge op
headers = {
"Host":
self.pipeline_root.pipeline_configuration.gateway_hostname,
"Content-Type": "application/json",
"Content-Length": str(len(str(body))),
"x-ms-edge-moduleId": x_ms_edge_string,
"User-Agent": user_agent_string,
}
op_waiting_for_response = op
def on_request_response(op, error):
logger.debug(
"{}({}): Got response for MethodInvokeOperation".format(
self.name, op.name))
error = map_http_error(error=error, http_op=op)
if not error:
op_waiting_for_response.method_response = json.loads(
op.response_body)
op_waiting_for_response.complete(error=error)
self.send_op_down(
pipeline_ops_http.HTTPRequestAndResponseOperation(
method="POST",
path=path,
headers=headers,
body=body,
query_params=query_params,
callback=on_request_response,
))
elif isinstance(op, pipeline_ops_iothub_http.GetStorageInfoOperation):
logger.debug(
"{}({}): Translating Get Storage Info Operation to HTTP.".
format(self.name, op.name))
query_params = "api-version={apiVersion}".format(
apiVersion=pkg_constant.IOTHUB_API_VERSION)
path = http_path_iothub.get_storage_info_for_blob_path(
self.pipeline_root.pipeline_configuration.device_id)
body = json.dumps({"blobName": op.blob_name})
user_agent_string = urllib.parse.quote_plus(
user_agent.get_iothub_user_agent() +
str(self.pipeline_root.pipeline_configuration.product_info))
headers = {
"Host": self.pipeline_root.pipeline_configuration.hostname,
"Accept": "application/json",
"Content-Type": "application/json",
"Content-Length": str(len(str(body))),
"User-Agent": user_agent_string,
}
op_waiting_for_response = op
def on_request_response(op, error):
logger.debug(
"{}({}): Got response for GetStorageInfoOperation".format(
self.name, op.name))
error = map_http_error(error=error, http_op=op)
if not error:
op_waiting_for_response.storage_info = json.loads(
op.response_body)
op_waiting_for_response.complete(error=error)
self.send_op_down(
pipeline_ops_http.HTTPRequestAndResponseOperation(
method="POST",
path=path,
headers=headers,
body=body,
query_params=query_params,
callback=on_request_response,
))
elif isinstance(
op, pipeline_ops_iothub_http.NotifyBlobUploadStatusOperation):
logger.debug(
"{}({}): Translating Get Storage Info Operation to HTTP.".
format(self.name, op.name))
query_params = "api-version={apiVersion}".format(
apiVersion=pkg_constant.IOTHUB_API_VERSION)
path = http_path_iothub.get_notify_blob_upload_status_path(
self.pipeline_root.pipeline_configuration.device_id)
body = json.dumps({
"correlationId": op.correlation_id,
"isSuccess": op.is_success,
"statusCode": op.request_status_code,
"statusDescription": op.status_description,
})
user_agent_string = urllib.parse.quote_plus(
user_agent.get_iothub_user_agent() +
str(self.pipeline_root.pipeline_configuration.product_info))
# NOTE we do not add the sas Authorization header here. Instead we add it later on in
# the HTTPTransportStage
headers = {
"Host": self.pipeline_root.pipeline_configuration.hostname,
"Content-Type": "application/json; charset=utf-8",
"Content-Length": str(len(str(body))),
"User-Agent": user_agent_string,
}
op_waiting_for_response = op
def on_request_response(op, error):
logger.debug(
"{}({}): Got response for GetStorageInfoOperation".format(
self.name, op.name))
error = map_http_error(error=error, http_op=op)
op_waiting_for_response.complete(error=error)
self.send_op_down(
pipeline_ops_http.HTTPRequestAndResponseOperation(
method="POST",
path=path,
headers=headers,
body=body,
query_params=query_params,
callback=on_request_response,
))
else:
# All other operations get passed down
self.send_op_down(op)
