def test_datum_collections_python_value(self):
class DatumCollectionString:
def __init__(self, *args: List[str]):
self.string = args
datum = meta.Datum(value=DatumCollectionString("string 1", "string 2"),
type="collection_string")
self.assertListEqual(datum.python_value, ["string 1", "string 2"])
self.assertEqual(datum.python_type, list)
class DatumCollectionBytes:
def __init__(self, *args: List[bytes]):
self.bytes = args
datum = meta.Datum(value=DatumCollectionBytes(b"bytes 1", b"bytes 2"),
type="collection_bytes")
self.assertListEqual(datum.python_value, [b"bytes 1", b"bytes 2"])
self.assertEqual(datum.python_type, list)
class DatumCollectionSint64:
def __init__(self, *args: List[int]):
self.sint64 = args
datum = meta.Datum(value=DatumCollectionSint64(1234567, 8901234),
type="collection_sint64")
self.assertListEqual(datum.python_value, [1234567, 8901234])
self.assertEqual(datum.python_type, list)
def test_multiple_servicebus_trigger_non_existing_properties(self):
servicebus_msgs = azf_sb.ServiceBusMessageInConverter.decode(
data=self._generate_multiple_service_bus_data(),
trigger_metadata={
'MessageIdArray':
meta.Datum(type='collection_string',
value=CollectionString([
self.MOCKED_MESSAGE_ID, self.MOCKED_MESSAGE_ID,
self.MOCKED_MESSAGE_ID
])),
'UserPropertiesArray':
meta.Datum(type='json',
value='''[{ "UserId": 1 },
{ "UserId": 2 },
{ "UserId": 3 }]
''')
})
# Non existing properties should return None
self.assertIsNone(servicebus_msgs[0].content_type)
self.assertIsNone(servicebus_msgs[1].content_type)
self.assertIsNone(servicebus_msgs[2].content_type)
# Message Id should always be available
self.assertEqual(servicebus_msgs[0].message_id, self.MOCKED_MESSAGE_ID)
self.assertEqual(servicebus_msgs[1].message_id, self.MOCKED_MESSAGE_ID)
self.assertEqual(servicebus_msgs[2].message_id, self.MOCKED_MESSAGE_ID)
# User properties should always be available
self.assertEqual(servicebus_msgs[0].user_properties['UserId'], 1)
self.assertEqual(servicebus_msgs[1].user_properties['UserId'], 2)
self.assertEqual(servicebus_msgs[2].user_properties['UserId'], 3)
def test_scalar_typed_data_decoder_not_ok(self):
metadata = {
'unsupported_type': bind_meta.Datum(type='bytes', value=b'aaa'),
'unexpected_json': bind_meta.Datum(type='json', value='[1, 2, 3]'),
'unexpected_data': bind_meta.Datum(type='json', value='"foo"'),
}
cases = [
('unsupported_type', int, ValueError,
"unsupported type of field 'unsupported_type' in "
"trigger metadata: bytes"),
('unexpected_json', int, ValueError,
"cannot convert value of field 'unexpected_json' in "
"trigger metadata into int"),
('unexpected_data', int, ValueError,
"cannot convert value of field "
"'unexpected_data' in trigger metadata into int: "
"invalid literal for int"),
('unexpected_data', (int, float), ValueError,
"unexpected value type in field "
"'unexpected_data' in trigger metadata: str, "
"expected one of: int, float"),
]
for field, pytype, exc, msg in cases:
with self.subTest(field=field):
with self.assertRaisesRegex(exc, msg):
Converter._decode_trigger_metadata_field(
metadata, field, python_type=pytype)
def _zip(self, key: str, expected_type: str,
*args: List[Dict[str, meta.Datum]]) -> meta.Datum:
"""Combining multiple metadata into one:
string -> collection_string
bytes -> collection_bytes
int -> collection_sint64
sint64 -> collection_sint64
json -> json (with array in it)
"""
convertible = {
'collection_string': CollectionString,
'collection_bytes': CollectionBytes,
'collection_sint64': CollectionSint64
}
datum_type = args[0][key].type
if expected_type in convertible.keys():
return meta.Datum(
value=convertible[expected_type]([d[key].value for d in args]),
type=expected_type)
elif expected_type == 'json':
if datum_type == 'json':
value = json.dumps([json.loads(d[key].value) for d in args])
else:
value = json.dumps([d[key].value for d in args])
return meta.Datum(value=value, type='json')
else:
raise NotImplementedError(f'Unknown convertion {key}: '
f'{datum_type} -> {expected_type}')
def _generate_single_trigger_metadatum(self):
return {
'EnqueuedTime':
meta.Datum(f'"{self.MOCKED_ENQUEUE_TIME.isoformat()}"', 'json'),
'SystemProperties':
meta.Datum('{"iothub-connection-device-id": "MyTestDevice"}',
'json')
}
def _generate_servicebus_metadata(self):
mocked_metadata: Mapping[str, meta.Datum] = {}
mocked_metadata['DeliveryCount'] = meta.Datum(1, 'int')
mocked_metadata['LockToken'] = meta.Datum(
'87931fd2-39f4-415a-9fdc-adfdcbed3148', 'string')
mocked_metadata['ExpiresAtUtc'] = meta.Datum('2020-07-02T05:39:12.17Z',
'string')
mocked_metadata['EnqueuedTimeUtc'] = meta.Datum(
self.MOCKED_ENQUEUE_TIME.isoformat(), 'string')
mocked_metadata['MessageId'] = meta.Datum(
'87c66eaf88e84119b66a26278a7b4149', 'string')
mocked_metadata['ContentType'] = meta.Datum('application/json',
'string')
mocked_metadata['SequenceNumber'] = meta.Datum(3, 'int')
mocked_metadata['Label'] = meta.Datum('Microsoft.Azure.ServiceBus',
'string')
mocked_metadata['sys'] = meta.Datum(type='json',
value='''
{
"MethodName": "ServiceBusSMany",
"UtcNow": "2020-06-18T05:39:12.2860411Z",
"RandGuid": "bb38deae-cc75-49f2-89f5-96ec6eb857db"
}
''')
return mocked_metadata
def _generate_multiple_trigger_metadata(self) -> Dict[str, meta.Datum]:
"""Generate a metadatum containing 3 service bus messages which can be
distingushed by enqueued_sequence_number
"""
sb_a = self._generate_single_trigger_metadata()
sb_b = self._generate_single_trigger_metadata()
sb_c = self._generate_single_trigger_metadata()
sb_a['MessageId'] = meta.Datum(self.MOCKED_MESSAGE_ID_A, 'string')
sb_b['MessageId'] = meta.Datum(self.MOCKED_MESSAGE_ID_B, 'string')
sb_c['MessageId'] = meta.Datum(self.MOCKED_MESSAGE_ID_C, 'string')
combine_from = lambda key, et: self._zip(key, et, sb_a, sb_b, sb_c)
mocked_metadata = {
'ContentTypeArray':
combine_from('ContentType', 'collection_string'),
'CorrelationIdArray':
combine_from('CorrelationId', 'collection_string'),
'DeadLetterSourceArray':
combine_from('DeadLetterSource', 'collection_string'),
'EnqueuedTimeUtcArray':
combine_from('EnqueuedTimeUtc', 'json'),
'ExpiresAtUtcArray':
combine_from('ExpiresAtUtc', 'json'),
'LabelArray':
combine_from('Label', 'collection_string'),
'LockTokenArray':
combine_from('LockToken', 'collection_string'),
'MessageIdArray':
combine_from('MessageId', 'collection_string'),
'PartitionKeyArray':
combine_from('PartitionKey', 'collection_string'),
'ReplyToArray':
combine_from('ReplyTo', 'collection_string'),
'ReplyToSessionIdArray':
combine_from('ReplyToSessionId', 'collection_string'),
'ScheduledEnqueueTimeUtcArray':
combine_from('ScheduledEnqueueTimeUtc', 'collection_string'),
'SessionIdArray':
combine_from('SessionId', 'collection_string'),
'SequenceNumberArray':
combine_from('SequenceNumber', 'collection_sint64'),
'TimeToLiveArray':
combine_from('TimeToLive', 'collection_string'),
'ToArray':
combine_from('To', 'collection_string'),
'UserPropertiesArray':
combine_from('UserProperties', 'json')
}
return mocked_metadata
def test_servicebus_properties(self):
# SystemProperties in metadata should propagate to class properties
msg = azf_sb.ServiceBusMessageInConverter.decode(
data=meta.Datum(b'body_bytes', 'bytes'),
trigger_metadata=self._generate_single_trigger_metadata())
self.assertEqual(msg.get_body(), b'body_bytes')
# Test individual ServiceBus properties respectively
self.assertEqual(msg.content_type, self.MOCKED_CONTENT_TYPE)
self.assertEqual(msg.correlation_id, self.MOCKED_CORROLATION_ID)
self.assertEqual(msg.dead_letter_source, self.MOCKED_DEADLETTER_SOURCE)
self.assertEqual(msg.enqueued_time_utc, self.MOCKED_ENQUEUE_TIME_UTC)
self.assertEqual(msg.expires_at_utc, self.MOCKED_EXPIRY_AT_UTC)
self.assertEqual(msg.expiration_time, self.MOCKED_EXPIRY_AT_UTC)
self.assertEqual(msg.label, self.MOCKED_LABEL)
self.assertEqual(msg.message_id, self.MOCKED_MESSAGE_ID)
self.assertEqual(msg.partition_key, self.MOCKED_PARTITION_KEY)
self.assertEqual(msg.reply_to, self.MOCKED_REPLY_TO)
self.assertEqual(msg.reply_to_session_id,
self.MOCKED_REPLY_TO_SESSION_ID)
self.assertEqual(msg.scheduled_enqueue_time,
self.MOCKED_SCHEDULED_ENQUEUE_TIME_UTC)
self.assertEqual(msg.scheduled_enqueue_time_utc,
self.MOCKED_SCHEDULED_ENQUEUE_TIME_UTC)
self.assertEqual(msg.session_id, self.MOCKED_SESSION_ID)
self.assertEqual(msg.time_to_live, self.MOCKED_TIME_TO_LIVE_TIMEDELTA)
self.assertEqual(msg.to, self.MOCKED_TO)
self.assertDictEqual(
msg.user_properties, {
'$AzureWebJobsParentId': self.MOCKED_AZURE_PARTNER_ID,
'x-opt-enqueue-sequence-number': 0
})
def test_servicebus_non_existing_property(self):
# The function should not fail even when property does not work
msg = azf_sb.ServiceBusMessageInConverter.decode(
data=self._generate_single_servicebus_data(),
trigger_metadata={
'MessageId': meta.Datum(self.MOCKED_MESSAGE_ID, 'string'),
'UserProperties': meta.Datum('{ "UserId": 1 }', 'json')
})
# Property that are not passed from extension should be None
self.assertIsNone(msg.content_type)
# Message id should always be available
self.assertEqual(msg.message_id, self.MOCKED_MESSAGE_ID)
# User property should always be available
self.assertEqual(msg.user_properties['UserId'], 1)
def _generate_multiple_trigger_metadata(self):
key_array = [None, None]
partition_array = [1, 2]
timestamp_array = [
"2020-06-20T05:06:25.139Z", "2020-06-20T05:06:25.945Z"
]
return {
'KeyArray':
meta.Datum(json.dumps(key_array), 'json'),
'OffsetArray':
meta.Datum(CollectionSint64([62, 63]), 'collection_sint64'),
'PartitionArray':
meta.Datum(json.dumps(partition_array), 'json'),
'TimestampArray':
meta.Datum(json.dumps(timestamp_array), 'json'),
'TopicArray':
meta.Datum(CollectionString(['message', 'message']),
"collection_string"),
'HeadersArray':
meta.Datum(
json.dumps([['{"Key":"test","Value":"1"}'],
['{"Key":"test2","Value":"2"}']]), 'json'),
'sys':
meta.Datum(
'{"MethodName":"KafkaTriggerMany",'
'"UtcNow":"2020-06-20T05:06:26.5550868Z",'
'"RandGuid":"57d5eeb7-c86c-4924-a14a-160092154093"}', 'json')
}
def _generate_multiple_service_bus_data(self) -> meta.Datum:
return meta.Datum(value=json.dumps([{
'lucky_number': 23
}, {
'lucky_number': 34
}, {
'lucky_number': 45
}]),
type='json')
def test_double_input_unsupported(self):
"""
Verify that the given input is unsupported by SharedMemoryManager.
This input is double.
"""
manager = SharedMemoryManager()
datum = bind_meta.Datum(type='double', value=1.0)
is_supported = manager.is_supported(datum)
self.assertFalse(is_supported)
def test_json_input_unsupported(self):
"""
Verify that the given input is unsupported by SharedMemoryManager.
This input is json.
"""
manager = SharedMemoryManager()
content = {'name': 'foo', 'val': 'bar'}
datum = bind_meta.Datum(type='json', value=json.dumps(content))
is_supported = manager.is_supported(datum)
self.assertFalse(is_supported)
def test_collection_sint64_unsupported(self):
"""
Verify that the given input is unsupported by SharedMemoryManager.
This input is collection_sint64.
"""
manager = SharedMemoryManager()
content = [1, 2]
datum = bind_meta.Datum(type='collection_sint64', value=content)
is_supported = manager.is_supported(datum)
self.assertFalse(is_supported)
def _generate_multiple_iothub_data(self, data_type='json'):
data = '[{"device-status": "good1"}, {"device-status": "good2"}]'
if data_type == 'collection_bytes':
data = list(
map(lambda x: json.dumps(x).encode('utf-8'), json.loads(data)))
data = CollectionBytes(data)
elif data_type == 'collection_string':
data = list(map(lambda x: json.dumps(x), json.loads(data)))
data = CollectionString(data)
return meta.Datum(data, data_type)
def test_bytes_input_support(self):
"""
Verify that the given input is supported by SharedMemoryManager to be
transfered over shared memory.
The input is bytes.
"""
manager = SharedMemoryManager()
content_size = consts.MIN_BYTES_FOR_SHARED_MEM_TRANSFER + 10
content = self.get_random_bytes(content_size)
bytes_datum = bind_meta.Datum(type='bytes', value=content)
is_supported = manager.is_supported(bytes_datum)
self.assertTrue(is_supported)
def test_scalar_typed_data_decoder_ok(self):
metadata = {
'int_as_json': bind_meta.Datum(type='json', value='1'),
'int_as_string': bind_meta.Datum(type='string', value='1'),
'int_as_int': bind_meta.Datum(type='int', value=1),
'string_as_json': bind_meta.Datum(type='json', value='"aaa"'),
'string_as_string': bind_meta.Datum(type='string', value='aaa'),
'dict_as_json': bind_meta.Datum(type='json', value='{"foo":"bar"}')
}
cases = [
('int_as_json', int, 1),
('int_as_string', int, 1),
('int_as_int', int, 1),
('string_as_json', str, 'aaa'),
('string_as_string', str, 'aaa'),
('dict_as_json', dict, {'foo': 'bar'}),
]
for field, pytype, expected in cases:
with self.subTest(field=field):
value = Converter._decode_trigger_metadata_field(
metadata, field, python_type=pytype)
self.assertIsInstance(value, pytype)
self.assertEqual(value, expected)
def test_string_input_support(self):
"""
Verify that the given input is supported by SharedMemoryManager to be
transfered over shared memory.
The input is string.
"""
manager = SharedMemoryManager()
content_size = consts.MIN_BYTES_FOR_SHARED_MEM_TRANSFER + 10
num_chars = math.floor(content_size / consts.SIZE_OF_CHAR_BYTES)
content = self.get_random_string(num_chars)
bytes_datum = bind_meta.Datum(type='string', value=content)
is_supported = manager.is_supported(bytes_datum)
self.assertTrue(is_supported)
def _generate_full_metadata(self):
mocked_metadata: Mapping[str, meta.Datum] = {}
mocked_metadata['Offset'] = meta.Datum(type='string', value='3696')
mocked_metadata['EnqueuedTimeUtc'] = meta.Datum(
type='string', value='2020-07-14T01:27:55.627Z')
mocked_metadata['SequenceNumber'] = meta.Datum(type='int', value=47)
mocked_metadata['Properties'] = meta.Datum(type='json', value='{}')
mocked_metadata['sys'] = meta.Datum(type='json', value='''
{
"MethodName":"metadata_trigger",
"UtcNow":"2020-07-14T01:27:55.8940305Z",
"RandGuid":"db413fd6-8411-4e51-844c-c9b5345e537d"
}''')
mocked_metadata['SystemProperties'] = meta.Datum(type='json', value='''
{
"x-opt-sequence-number":47,
"x-opt-offset":"3696",
"x-opt-enqueued-time":"2020-07-14T01:27:55.627Z",
"SequenceNumber":47,
"Offset":"3696",
"PartitionKey":null,
"EnqueuedTimeUtc":"2020-07-14T01:27:55.627Z",
"iothub-connection-device-id":"awesome-device-id"
}''')
mocked_metadata['PartitionContext'] = meta.Datum(type='json', value='''
{
"CancellationToken":{
"IsCancellationRequested":false,
"CanBeCanceled":true,
"WaitHandle":{
"Handle":{
"value":2472
},
"SafeWaitHandle":{
"IsInvalid":false,
"IsClosed":false
}
}
},
"ConsumerGroupName":"$Default",
"EventHubPath":"python-worker-ci-eventhub-one-metadata",
"PartitionId":"0",
"Owner":"88cec2e2-94c9-4e08-acb6-4f2b97cd888e",
"RuntimeInformation":{
"PartitionId":"0",
"LastSequenceNumber":0,
"LastEnqueuedTimeUtc":"0001-01-01T00:00:00",
"LastEnqueuedOffset":null,
"RetrievalTime":"0001-01-01T00:00:00"
}
}''')
return mocked_metadata
def _generate_multiple_kafka_data(self, data_type='json'):
data = '[{"Offset":62,"Partition":1,"Topic":"message",'\
'"Timestamp":"2020-06-20T05:06:25.139Z","Value":"a"},'\
' {"Offset":63,"Partition":1,"Topic":"message",'\
'"Timestamp":"2020-06-20T05:06:25.945Z","Value":"a"}]'
if data_type == 'collection_bytes':
data = list(
map(lambda x: json.dumps(x).encode('utf-8'), json.loads(data)))
data = CollectionBytes(data)
elif data_type == 'collection_string':
data = list(map(lambda x: json.dumps(x), json.loads(data)))
data = CollectionString(data)
return meta.Datum(data, data_type)
def test_large_invalid_bytes_input_support(self):
"""
Verify that the given input is NOT supported by SharedMemoryManager to
be transfered over shared memory.
The input is bytes of larger than the allowed size.
"""
manager = SharedMemoryManager()
content_size = consts.MAX_BYTES_FOR_SHARED_MEM_TRANSFER + 10
# Not using get_random_bytes to avoid slowing down for creating a large
# random input
content = b'x01' * content_size
bytes_datum = bind_meta.Datum(type='bytes', value=content)
is_supported = manager.is_supported(bytes_datum)
self.assertFalse(is_supported)
def test_large_invalid_string_input_support(self):
"""
Verify that the given input is NOT supported by SharedMemoryManager to
be transfered over shared memory.
The input is string of larger than the allowed size.
"""
manager = SharedMemoryManager()
content_size = consts.MAX_BYTES_FOR_SHARED_MEM_TRANSFER + 10
num_chars = math.floor(content_size / consts.SIZE_OF_CHAR_BYTES)
# Not using get_random_string to avoid slowing down for creating a large
# random input
content = 'a' * num_chars
string_datum = bind_meta.Datum(type='string', value=content)
is_supported = manager.is_supported(string_datum)
self.assertFalse(is_supported)
def test_eventhub_properties(self):
"""Test if properties from public interface _eventhub.py returns
the correct values from metadata"""
result = azf_eh.EventHubTriggerConverter.decode(
data=meta.Datum(b'body_bytes', 'bytes'),
trigger_metadata=self._generate_full_metadata()
)
self.assertEqual(result.get_body(), b'body_bytes')
self.assertIsNone(result.partition_key)
self.assertDictEqual(result.iothub_metadata,
{'connection-device-id': 'awesome-device-id'})
self.assertEqual(result.sequence_number, 47)
self.assertEqual(result.enqueued_time.isoformat(),
'2020-07-14T01:27:55.627000+00:00')
self.assertEqual(result.offset, '3696')
def _generate_multiple_trigger_metadata(self):
system_props_array = [{
'EnqueuedTimeUtc':
self.MOCKED_ENQUEUE_TIME.isoformat(),
'iothub-connection-device-id':
'MyTestDevice1',
}, {
'EnqueuedTimeUtc':
self.MOCKED_ENQUEUE_TIME.isoformat(),
'iothub-connection-device-id':
'MyTestDevice2',
}]
return {
'SystemPropertiesArray':
meta.Datum(json.dumps(system_props_array), 'json')
}
def test_datum_json_python_value(self):
# None
datum = meta.Datum(value='null', type="json")
self.assertEqual(datum.python_value, None)
self.assertEqual(datum.python_type, type(None))
# Int
datum = meta.Datum(value='123', type="json")
self.assertEqual(datum.python_value, 123)
self.assertEqual(datum.python_type, int)
# Float
datum = meta.Datum(value='456.789', type="json")
self.assertEqual(datum.python_value, 456.789)
self.assertEqual(datum.python_type, float)
# String
datum = meta.Datum(value='"string in json"', type="json")
self.assertEqual(datum.python_value, "string in json")
self.assertEqual(datum.python_type, str)
# List
datum = meta.Datum(value='["a", "b", "c"]', type="json")
self.assertListEqual(datum.python_value, ["a", "b", "c"])
self.assertEqual(datum.python_type, list)
# Object
datum = meta.Datum(value='{"name": "awesome", "value": "cool"}',
type="json")
self.assertDictEqual(datum.python_value, {
"name": "awesome",
"value": "cool"
})
self.assertEqual(datum.python_type, dict)
# Should ignore Newlines and Spaces
datum = meta.Datum(value='{ "name" : "awesome",\n "value": "cool"\n}',
type="json")
self.assertDictEqual(datum.python_value, {
"name": "awesome",
"value": "cool"
})
self.assertEqual(datum.python_type, dict)
def _generate_single_trigger_metadatum(self):
return {
'Offset':
meta.Datum('1', 'string'),
'Partition':
meta.Datum('0', 'string'),
'Timestamp':
meta.Datum(self.SINGLE_KAFKA_TIMESTAMP, 'string'),
'Topic':
meta.Datum('users', 'string'),
'Value':
meta.Datum('hello', 'string'),
'sys':
meta.Datum('{"MethodName":"KafkaTrigger",'
'"UtcNow":"2020-06-20T04:43:30.6756278Z",'
'"RandGuid":"b0870c0c-2b7a-40dc-b4be-45224c91a49c"}',
'json') # __len__: 6
}
def test_datum_single_level_python_value(self):
datum: Mapping[str, meta.Datum] = meta.Datum(value=None, type="int")
self.assertEqual(datum.python_value, None)
self.assertEqual(datum.python_type, type(None))
datum = meta.Datum(value=1, type=None)
self.assertEqual(datum.python_value, None)
self.assertEqual(datum.python_type, type(None))
datum = meta.Datum(value=b"awesome bytes", type="bytes")
self.assertEqual(datum.python_value, b"awesome bytes")
self.assertEqual(datum.python_type, bytes)
datum = meta.Datum(value="awesome string", type="string")
self.assertEqual(datum.python_value, 'awesome string')
self.assertEqual(datum.python_type, str)
datum = meta.Datum(value=42, type="int")
self.assertEqual(datum.python_value, 42)
self.assertEqual(datum.python_type, int)
datum = meta.Datum(value=43.2103, type="double")
self.assertEqual(datum.python_value, 43.2103)
self.assertEqual(datum.python_type, float)
def _generate_single_kafka_datum(self, datum_type='string'):
datum = self.SINGLE_KAFKA_DATAUM
if datum_type == 'bytes':
datum = datum.encode('utf-8')
return meta.Datum(datum, datum_type)
def encode(
cls, obj: typing.Any, *, expected_type: typing.Optional[type]
) -> meta.Datum:
print(f"Encoding {obj}")
return meta.Datum(obj, object)
def _generate_single_iothub_datum(self, datum_type='json'):
datum = '{"device-status": "good"}'
if datum_type == 'bytes':
datum = datum.encode('utf-8')
return meta.Datum(datum, datum_type)