def test_send_and_receive_large_body_size(connstr_receivers):
if sys.platform.startswith('darwin'):
pytest.skip("Skipping on OSX - open issue regarding message size")
connection_str, receivers = connstr_receivers
client = EventHubProducerClient.from_connection_string(connection_str)
with client:
payload = 250 * 1024
client.send(EventData("A" * payload))
received = []
for r in receivers:
received.extend([EventData._from_message(x) for x in r.receive_message_batch(timeout=10000)])
assert len(received) == 1
assert len(list(received[0].body)[0]) == payload
def test_send_list(connstr_receivers):
connection_str, receivers = connstr_receivers
client = EventHubProducerClient.from_connection_string(connection_str)
payload = "A1"
with client:
client.send_batch([EventData(payload)])
received = []
for r in receivers:
received.extend([
EventData._from_message(x)
for x in r.receive_message_batch(timeout=10000)
])
assert len(received) == 1
assert received[0].body_as_str() == payload
async def test_send_non_ascii_async(connstr_receivers):
connection_str, receivers = connstr_receivers
client = EventHubProducerClient.from_connection_string(connection_str)
async with client:
await client.send(EventData(u"é,è,à,ù,â,ê,î,ô,û"), partition_id="0")
await client.send(EventData(json.dumps({"foo": u"漢字"})),
partition_id="0")
await asyncio.sleep(1)
partition_0 = [
EventData._from_message(x)
for x in receivers[0].receive_message_batch(timeout=5000)
]
assert len(partition_0) == 2
assert partition_0[0].body_as_str() == u"é,è,à,ù,â,ê,î,ô,û"
assert partition_0[1].body_as_json() == {"foo": u"漢字"}
async def test_send_and_receive_small_body_async(connstr_receivers, payload):
connection_str, receivers = connstr_receivers
client = EventHubProducerClient.from_connection_string(connection_str)
async with client:
batch = await client.create_batch()
batch.add(EventData(payload))
await client.send_batch(batch)
received = []
for r in receivers:
received.extend([
EventData._from_message(x)
for x in r.receive_message_batch(timeout=5000)
])
assert len(received) == 1
assert list(received[0].body)[0] == payload
def test_send_non_ascii(connstr_receivers):
connection_str, receivers = connstr_receivers
client = EventHubProducerClient.from_connection_string(connection_str)
with client:
batch = client.create_batch(partition_id="0")
batch.add(EventData(u"é,è,à,ù,â,ê,î,ô,û"))
batch.add(EventData(json.dumps({"foo": u"漢字"})))
client.send_batch(batch)
time.sleep(1)
partition_0 = [
EventData._from_message(x)
for x in receivers[0].receive_message_batch(timeout=5000)
]
assert len(partition_0) == 2
assert partition_0[0].body_as_str() == u"é,è,à,ù,â,ê,î,ô,û"
assert partition_0[1].body_as_json() == {"foo": u"漢字"}
async def test_send_with_long_interval_async(live_eventhub, sleep):
test_partition = "0"
sender = EventHubProducerClient(
live_eventhub['hostname'], live_eventhub['event_hub'],
EventHubSharedKeyCredential(live_eventhub['key_name'],
live_eventhub['access_key']))
async with sender:
batch = await sender.create_batch(partition_id=test_partition)
batch.add(EventData(b"A single event"))
await sender.send_batch(batch)
if sleep:
await asyncio.sleep(250
) # EH server side idle timeout is 240 second
else:
await sender._producers[test_partition
]._handler._connection._conn.destroy()
batch = await sender.create_batch(partition_id=test_partition)
batch.add(EventData(b"A single event"))
await sender.send_batch(batch)
received = []
uri = "sb://{}/{}".format(live_eventhub['hostname'],
live_eventhub['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub['key_name'], live_eventhub['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub['hostname'], live_eventhub['event_hub'],
live_eventhub['consumer_group'], test_partition)
receiver = uamqp.ReceiveClient(source,
auth=sas_auth,
debug=False,
timeout=10000,
prefetch=10)
try:
receiver.open()
received.extend([
EventData._from_message(x)
for x in receiver.receive_message_batch(max_batch_size=2,
timeout=10000)
])
finally:
receiver.close()
assert len(received) == 2
assert list(received[0].body)[0] == b"A single event"
def test_send_with_long_interval_sync(live_eventhub, sleep):
sender = EventHubProducerClient(
live_eventhub['hostname'], live_eventhub['event_hub'],
EventHubSharedKeyCredential(live_eventhub['key_name'],
live_eventhub['access_key']))
with sender:
batch = sender.create_batch()
batch.add(EventData(b"A single event"))
sender.send_batch(batch)
for _ in range(1):
if sleep:
time.sleep(300)
else:
sender._producers[-1]._handler._connection._conn.destroy()
batch = sender.create_batch()
batch.add(EventData(b"A single event"))
sender.send_batch(batch)
partition_ids = sender.get_partition_ids()
received = []
for p in partition_ids:
uri = "sb://{}/{}".format(live_eventhub['hostname'],
live_eventhub['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub['key_name'], live_eventhub['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub['hostname'], live_eventhub['event_hub'],
live_eventhub['consumer_group'], p)
receiver = uamqp.ReceiveClient(source,
auth=sas_auth,
debug=False,
timeout=5000,
prefetch=500)
try:
receiver.open()
received.extend([
EventData._from_message(x)
for x in receiver.receive_message_batch(timeout=5000)
])
finally:
receiver.close()
assert len(received) == 2
assert list(received[0].body)[0] == b"A single event"
async def __anext__(self):
max_retries = self.client.config.max_retries
retry_count = 0
while True:
try:
await self._open()
if not self.messages_iter:
self.messages_iter = self._handler.receive_messages_iter_async(
)
message = await self.messages_iter.__anext__()
event_data = EventData._from_message(message)
self.offset = EventPosition(event_data.offset, inclusive=False)
retry_count = 0
return event_data
except Exception as exception:
await self._handle_exception(exception, retry_count,
max_retries)
retry_count += 1
async def __anext__(self):
max_retries = self.client.config.max_retries
retry_count = 0
while True:
try:
await self._open()
if not self.messages_iter:
self.messages_iter = self._handler.receive_messages_iter_async(
)
message = await self.messages_iter.__anext__()
event_data = EventData._from_message(message) # pylint:disable=protected-access
self.offset = EventPosition(event_data.offset, inclusive=False)
retry_count = 0
return event_data
except Exception as exception: # pylint:disable=broad-except
await self._handle_exception(exception,
retry_count,
max_retries,
timeout_time=None)
retry_count += 1
def test_send_connection_idle_timeout_and_reconnect_sync(connstr_receivers):
connection_str, receivers = connstr_receivers
client = EventHubProducerClient.from_connection_string(conn_str=connection_str, idle_timeout=10)
with client:
ed = EventData('data')
sender = client._create_producer(partition_id='0')
with sender:
sender._open_with_retry()
time.sleep(11)
sender._unsent_events = [ed.message]
ed.message.on_send_complete = sender._on_outcome
with pytest.raises((uamqp.errors.ConnectionClose,
uamqp.errors.MessageHandlerError, OperationTimeoutError)):
# Mac may raise OperationTimeoutError or MessageHandlerError
sender._send_event_data()
sender._send_event_data_with_retry()
messages = receivers[0].receive_message_batch(max_batch_size=10, timeout=10000)
received_ed1 = EventData._from_message(messages[0])
assert received_ed1.body_as_str() == 'data'
async def __anext__(self):
retried_times = 0
last_exception = None
while retried_times < self._client._config.max_retries: # pylint:disable=protected-access
try:
await self._open()
if not self._messages_iter:
self._messages_iter = self._handler.receive_messages_iter_async()
message = await self._messages_iter.__anext__()
event_data = EventData._from_message(message) # pylint:disable=protected-access
self._offset = EventPosition(event_data.offset, inclusive=False)
retried_times = 0
return event_data
except Exception as exception: # pylint:disable=broad-except
last_exception = await self._handle_exception(exception)
await self._client._try_delay(retried_times=retried_times, last_exception=last_exception, # pylint:disable=protected-access
entity_name=self._name)
retried_times += 1
log.info("%r operation has exhausted retry. Last exception: %r.", self._name, last_exception)
raise last_exception
def test_sys_properties():
properties = uamqp.message.MessageProperties()
properties.message_id = "message_id"
properties.user_id = "user_id"
properties.to = "to"
properties.subject = "subject"
properties.reply_to = "reply_to"
properties.correlation_id = "correlation_id"
properties.content_type = "content_type"
properties.content_encoding = "content_encoding"
properties.absolute_expiry_time = 1
properties.creation_time = 1
properties.group_id = "group_id"
properties.group_sequence = 1
properties.reply_to_group_id = "reply_to_group_id"
message = uamqp.Message(properties=properties)
message.annotations = {_common.PROP_OFFSET: "@latest"}
ed = EventData._from_message(message) # type: EventData
assert ed.system_properties[_common.PROP_OFFSET] == "@latest"
assert ed.system_properties[
_common.PROP_CORRELATION_ID] == properties.correlation_id
assert ed.system_properties[
_common.PROP_MESSAGE_ID] == properties.message_id
assert ed.system_properties[
_common.PROP_CONTENT_ENCODING] == properties.content_encoding
assert ed.system_properties[
_common.PROP_CONTENT_TYPE] == properties.content_type
assert ed.system_properties[_common.PROP_USER_ID] == properties.user_id
assert ed.system_properties[_common.PROP_TO] == properties.to
assert ed.system_properties[_common.PROP_SUBJECT] == properties.subject
assert ed.system_properties[_common.PROP_REPLY_TO] == properties.reply_to
assert ed.system_properties[
_common.PROP_ABSOLUTE_EXPIRY_TIME] == properties.absolute_expiry_time
assert ed.system_properties[
_common.PROP_CREATION_TIME] == properties.creation_time
assert ed.system_properties[_common.PROP_GROUP_ID] == properties.group_id
assert ed.system_properties[
_common.PROP_GROUP_SEQUENCE] == properties.group_sequence
assert ed.system_properties[
_common.PROP_REPLY_TO_GROUP_ID] == properties.reply_to_group_id
def test_send_with_create_event_batch_with_app_prop_sync(connstr_receivers):
connection_str, receivers = connstr_receivers
app_prop_key = "raw_prop"
app_prop_value = "raw_value"
app_prop = {app_prop_key: app_prop_value}
client = EventHubProducerClient.from_connection_string(connection_str, transport_type=TransportType.AmqpOverWebsocket)
with client:
event_data_batch = client.create_batch(max_size_in_bytes=100000)
while True:
try:
ed = EventData('A single event data')
ed.properties = app_prop
event_data_batch.add(ed)
except ValueError:
break
client.send_batch(event_data_batch)
received = []
for r in receivers:
received.extend(r.receive_message_batch(timeout=5000))
assert len(received) >= 1
assert EventData._from_message(received[0]).properties[b"raw_prop"] == b"raw_value"
async def _receive(self, timeout_time=None, max_batch_size=None, **kwargs):
last_exception = kwargs.get("last_exception")
data_batch = kwargs.get("data_batch")
await self._open(timeout_time)
remaining_time = timeout_time - time.time()
if remaining_time <= 0.0:
if last_exception:
log.info("%r receive operation timed out. (%r)", self.name,
last_exception)
raise last_exception
return data_batch
remaining_time_ms = 1000 * remaining_time
message_batch = await self._handler.receive_message_batch_async(
max_batch_size=max_batch_size, timeout=remaining_time_ms)
for message in message_batch:
event_data = EventData._from_message(message)
self.offset = EventPosition(event_data.offset)
data_batch.append(event_data)
return data_batch
def test_send_with_partition_key(connstr_receivers):
connection_str, receivers = connstr_receivers
client = EventHubProducerClient.from_connection_string(connection_str)
with client:
data_val = 0
for partition in [b"a", b"b", b"c", b"d", b"e", b"f"]:
partition_key = b"test_partition_" + partition
for i in range(50):
data = EventData(str(data_val))
data_val += 1
client.send(data, partition_key=partition_key)
found_partition_keys = {}
for index, partition in enumerate(receivers):
received = partition.receive_message_batch(timeout=5000)
for message in received:
try:
event_data = EventData._from_message(message)
existing = found_partition_keys[event_data.partition_key]
assert existing == index
except KeyError:
found_partition_keys[event_data.partition_key] = index