async def send_operation(self, op_id):
telemetry = make_message_payload()
telemetry["op_id"] = op_id
with self.op_id_list_lock:
logger("setting op_id {} to not_received".format(op_id))
self.op_id_list[op_id] = not_received
await self.client.send_event(telemetry)
async def test_dropped_c2d_2nd_call(self, client, service, before_api_call,
after_api_call):
# 1st call
payload = sample_content.make_message_payload()
await client.enable_c2d()
test_input_future = asyncio.ensure_future(
client.wait_for_c2d_message())
await service.send_c2d(client.device_id, payload)
received_message = await test_input_future
assert received_message.body == payload
# 2nd call
payload = sample_content.make_message_payload()
await before_api_call()
test_input_future = asyncio.ensure_future(
client.wait_for_c2d_message())
await after_api_call()
await service.send_c2d(client.device_id, payload)
logger("Awaiting input")
received_message = await test_input_future
assert received_message.body == payload
def fin():
adapter_config.default_api_timeout = previous_timeout
logger(
"Finishing test class: Replacing old REST timeout of {} seconds".format(
previous_timeout
)
)
async def set_platform_properties(*, client, longhaul_control_device,
system_control):
wrapper_stats = await client.settings.wrapper_api.get_wrapper_stats()
system_stats = await system_control.get_system_stats(0)
properties = PlatformProperties()
properties.language = wrapper_stats.get("language", "")
properties.language_version = wrapper_stats.get("languageVersion", "")
properties.os = system_stats.get("osType", "")
properties.os_release = system_stats.get("osRelease", "")
properties.system_architecture = system_stats.get("systemArchitecture", "")
properties.sdk_repo = system_stats.get("sdkRepo", "")
properties.sdk_commit = system_stats.get("sdkCommit", "")
properties.sdk_sha = system_stats.get("sdkSha", "")
properties.test_hub_name = client.settings.iothub_host_name
properties.test_device_id = client.device_id
properties.test_module_id = getattr(client, "module_id", "")
properties.system_memory_size_in_kb = int(
system_stats.get("system_MemTotal", 0))
patch = {"reported": properties.to_dict()}
logger("reporting: {}".format(patch))
await longhaul_control_device.patch_twin(patch)
return wrapper_stats["wrapperPid"]
def print_progress(self):
now = datetime.datetime.now()
logger("Start time: {}".format(pretty_time(self.test_start_time)))
logger("Duration: {}".format(pretty_time(self.test_run_time)))
logger("End time: {}".format(pretty_time(self.test_end_time)))
logger("now: {}".format(pretty_time(now)))
logger("Time left: {}".format(pretty_time(self.test_end_time - now)))
async def wait_for_desired_properties_patch(*,
client,
expected_twin,
mistakes=1):
mistakes_left = mistakes
while True:
patch_received = await client.wait_for_desired_property_patch()
logger("desired properties sent: " +
str(expected_twin["desired"]["foo"]))
logger("desired properties received: " +
str(patch_received["desired"]["foo"]))
if expected_twin["desired"]["foo"] == patch_received["desired"]["foo"]:
logger("success")
return
else:
if mistakes_left:
# We sometimes get the old value before we get the new value, and that's
# perfectly valid (especially with QOS 1 on MQTT). If we got the wrong
# value, we just try again.
mistakes_left = mistakes_left - 1
logger("trying again. We still have {} mistakes left".format(
mistakes_left))
else:
logger("too many mistakes. Failing")
assert False
async def run_one_op(self):
try:
measure_send_latency = MeasureLatency(
tracker=self.average_send_latency)
measure_verify_latency = MeasureLatency(
tracker=self.average_verify_latency)
op_id = self.next_op_id.increment()
with self.uncompleted_ops_lock:
self.uncompleted_ops.add(op_id)
with self.count_sending, measure_send_latency:
await self.send_operation(op_id)
with self.count_verifying, measure_verify_latency:
await self.verify_operation(op_id)
with self.uncompleted_ops_lock:
self.uncompleted_ops.remove(op_id)
self.total_count_completed.increment()
except Exception as e:
logger("OP FAILED: Exception running op: {}".format(type(e)))
logger(traceback.format_exc())
self.total_count_failed.increment()
async def test_perf_simple_throughput(self, client):
"""
Send a large number of messsages all at once and measure how many messages
can be sent per second. This can be used to establish a theoretical maximum for
our library, though the number of messages per second reported by this function
is higher than typical because of burst effects.
"""
count = 2500
payloads = [sample_content.make_message_payload() for x in range(0, count)]
start_time = datetime.datetime.now()
await asyncio.gather(*[client.send_event(payload) for payload in payloads])
duration = (datetime.datetime.now() - start_time).total_seconds()
mps = math.floor(count / duration)
logger(
"{} messages were sent in {} seconds for {} messages/second".format(
count, duration, mps
)
)
# Arbitrary goal based on experimental evidence. The goal of this assert is to
# flag gigantic performance drops. Experimentally, 46 is typical. For this assert,
# even 25 would be acceptable
assert mps > 25
async def wait_for_disconnection_event(self, *, client):
status = await client.get_connection_status()
assert status == "connected"
logger("waiting for client disconnection event")
await client.wait_for_connection_status_change("disconnected")
logger("client disconnection event received")
async def test_perf_send_event_longhaul(self, client):
"""
Run a linghaul test to validate that send_event can keep up with a consistent
and regular cadence of events. This tests an arbitrary time period, with an arbitrary
number of messages sent every second and it verifies two things:
1. The send_event latency doesn't go too high
2. The maximum number of running threads doesn't go too high.
Both of these factors can indicae a slowdown in sending.
The choice for events_per_second is arbitray baased on the results from
test_perf_measure_send_event_capacity. That test indicates that we can maintain
21 messages-per-second for 30 seconds, so this test choses 15 messages per second
but runs it for 2 hours.
"""
duration = 7200
events_per = 15
max_threads = 5
max_latency = 11
threads = await self.do_test_perf_send_event(
client, events_per, duration, max_threads, max_latency
)
logger("FINAL RESULT:")
logger(
"Sent {} events per second for {} seconds with max {} threads".format(
events_per, duration, threads
)
)
async def test_send_5_telemetry_events_to_iothub(self, client, eventhub):
if not limitations.can_always_overlap_telemetry_messages(client):
pytest.skip("client's can't reliably overlap telemetry messages")
payloads = []
send_futures = []
# start listening before we send
await eventhub.connect()
received_message_future = asyncio.ensure_future(
eventhub.wait_for_next_event(client.device_id))
for _ in range(0, 5):
ensure_send_telemetry_message(client=client,
payloads=payloads,
send_futures=send_futures)
# wait for the sends to complete, and verify that they arrive
await asyncio.gather(*send_futures)
logger("All messages sent. Awaiting reception")
await wait_for_all_telemetry_messages_to_arrive(
received_message_future=received_message_future,
payloads=payloads,
eventhub=eventhub,
client=client,
)
def restart_edgehub(hard=False):
logger("restarting edgehub")
sleep(5)
client = docker.from_env()
edgeHub = client.containers.get(EDGEHUB_NAME)
try:
if hard:
client = docker.from_env()
containerList = list(
map(lambda x: x.name, client.containers.list()))
for containerName in containerList:
if "Mod" or "edgeHub" in containerName:
currentContainer = client.containers.get(containerName)
currentContainer.restart()
while EDGEHUB_NAME not in list(
map(lambda x: x.name, client.containers.list())):
print("waiting for edge daemon to revive edgehub...")
sleep(1)
print("updating pointer to edgehub container")
edgeHub.reload()
else:
edgeHub.restart()
sleep(5)
except Exception as e:
logger("Error: {}".format(sys.exc_info()[0]))
raise e
async def net_control():
api = getattr(settings.net_control, "api", None)
try:
yield api
finally:
if api:
logger(separator("net_control finalizer"))
await settings.net_control.api.reconnect()
async def reconnect_after_each_test(self, system_control):
# if this test is going to drop packets, add a finalizer to make sure we always stop
# stop dropping it when we're done. Calling reconnect twice in a row is allowed.
try:
yield
finally:
logger("in finalizer: no longer dropping packets")
await system_control.reconnect_network()
async def system_control():
adapter = getattr(settings.system_control, "adapter", None)
try:
yield adapter
finally:
if adapter:
logger("system_control finalizer".center(132, "-"))
await adapter.reconnect_network()
async def service():
service = await connections.connect_service_client()
try:
yield service
finally:
logger(separator("service finalizer"))
try:
await service.disconnect()
except Exception as e:
logger("exception disconnecting service: {}".format(e))
async def move_blob_status_into_eventhub(service, client):
"""
get_blob_update_status() returns the status once, so we might receive status from
a different instance of this test that's running in parallel. we copy the update
status into eventhub where it's available to any instance of this test.
"""
while True:
status = await service.get_blob_upload_status()
logger("got upload status = {}".format(status))
await client.send_event(json.loads(str(status)))
async def registry():
registry = await connections.connect_registry_client()
try:
yield registry
finally:
logger(separator("registry finalizer"))
try:
await registry.disconnect()
except Exception as e:
logger("exception disconnecting registry: {}".format(e))
async def _log_exception(aw):
"""
Log any exceptions that happen while running this awaitable
"""
try:
await aw
except Exception:
logger("Exception raised")
logger(traceback.format_exc())
raise
def is_test_done(self):
if self.error:
raise self.error
logger("{}: Change in {}, Remaining Time: {}".format(
self.prefix,
pretty_time(self.next_change - datetime.datetime.now()),
pretty_time(self.test_end_time - datetime.datetime.now()),
))
return datetime.datetime.now() >= self.test_end_time
async def do_test_handle_method_to_friend(self, *, client, friend, count,
time_limit):
for i in range(0, count):
if time_limit.is_test_done():
return
logger("method_to_friend {}/{}".format(i + 1, count))
await run_method_call_test(source=client, destination=friend)
async def do_test_handle_method_from_service(self, *, client, service,
count, time_limit):
for i in range(0, count):
if time_limit.is_test_done():
return
logger("method_from_service {}/{}".format(i + 1, count))
# BKTODO: pull enable_methods out of run_method_call_test
await run_method_call_test(source=service, destination=client)
async def eventhub(event_loop):
eventhub = adapters.create_adapter(settings.eventhub.adapter_address,
"eventhub")
await eventhub.create_from_connection_string(
settings.eventhub.connection_string)
try:
yield eventhub
finally:
logger(separator("eventhub finalizer"))
try:
await eventhub.disconnect()
except Exception as e:
logger("exception disconnecting eventhub: {}".format(e))
def pytest_pyfunc_call(pyfuncitem):
# this hook wraps test runs. this yield runs the actual test
outcome = yield
try:
# this will raise if the outcome was an exception
outcome.get_result()
logger(separator("TEST PASSED (before cleanup)"))
except Exception as e:
logger(separator("TEST FAILED BACAUSE OF {}".format(e)))
async def test_module():
test_module = await connections.get_module_client(settings.test_module)
try:
yield test_module
finally:
logger(separator("module finalizer"))
try:
if test_module.capabilities.v2_connect_group:
await test_module.destroy()
else:
await test_module.disconnect()
except Exception as e:
logger("exception disconnecting test module: {}".format(e))
async def patch_desired(self, *, client, registry, mistakes=1):
twin_sent = sample_content.make_desired_props()
logger("Patching desired properties to {}".format(twin_sent))
patch_future = asyncio.ensure_future(
wait_for_desired_properties_patch(client=client,
expected_twin=twin_sent,
mistakes=mistakes))
await asyncio.sleep(1) # wait for async call to take effect
await patch_desired_props(registry, client, twin_sent)
await patch_future # raises if patch not received
async def report_loop():
stop_after_sending = False
while True:
if stop_reporter:
stop_after_sending = True
patch = {"reported": test_report.to_dict()}
logger("reporting: {}".format(patch))
await client.patch_twin(patch)
if stop_after_sending:
return
await asyncio.sleep(5) # todo: make configurable.
def extend_rest_timeout(self, request):
previous_timeout = adapter_config.default_api_timeout
adapter_config.default_api_timeout = max(300, previous_timeout)
logger(
"Starting test class: Adjusting REST timeout to {} seconds".format(
adapter_config.default_api_timeout))
def fin():
adapter_config.default_api_timeout = previous_timeout
logger(
"Finishing test class: Replacing old REST timeout of {} seconds"
.format(previous_timeout))
request.addfinalizer(fin)
async def do_test_get_twin(self, *, client, registry, count, time_limit):
await client.enable_twin()
for i in range(0, count):
if time_limit.is_test_done():
return
logger("get_twin {}/{}".format(i + 1, count))
twin_sent = sample_content.make_desired_props()
await patch_desired_props(registry, client, twin_sent)
while True:
twin_received = await client.get_twin()
logger("twin sent: " + str(twin_sent))
logger("twin received:" + str(twin_received))
if twin_sent["desired"]["foo"] == twin_received["desired"][
"foo"]:
break
else:
logger(
"Twin does not match. Sleeping for 2 seconds and retrying."
)
await asyncio.sleep(2)
async def test_send_telemetry_to_iothub(self, client, eventhub,
telemetry_payload):
if len(str(telemetry_payload)
) > limitations.get_maximum_telemetry_message_size(client):
pytest.skip("message is too big")
await eventhub.connect()
logger('sending "{}"'.format(telemetry_payload))
await client.send_event(telemetry_payload)
received_message = await eventhub.wait_for_next_event(
client.device_id, expected=telemetry_payload)
assert received_message is not None, "Message not received"
