def test_datum_collections_python_value(self): class DatumCollectionString: def __init__(self, *args: List[str]): self.string = args datum = meta.Datum(value=DatumCollectionString("string 1", "string 2"), type="collection_string") self.assertListEqual(datum.python_value, ["string 1", "string 2"]) self.assertEqual(datum.python_type, list) class DatumCollectionBytes: def __init__(self, *args: List[bytes]): self.bytes = args datum = meta.Datum(value=DatumCollectionBytes(b"bytes 1", b"bytes 2"), type="collection_bytes") self.assertListEqual(datum.python_value, [b"bytes 1", b"bytes 2"]) self.assertEqual(datum.python_type, list) class DatumCollectionSint64: def __init__(self, *args: List[int]): self.sint64 = args datum = meta.Datum(value=DatumCollectionSint64(1234567, 8901234), type="collection_sint64") self.assertListEqual(datum.python_value, [1234567, 8901234]) self.assertEqual(datum.python_type, list)
def test_multiple_servicebus_trigger_non_existing_properties(self): servicebus_msgs = azf_sb.ServiceBusMessageInConverter.decode( data=self._generate_multiple_service_bus_data(), trigger_metadata={ 'MessageIdArray': meta.Datum(type='collection_string', value=CollectionString([ self.MOCKED_MESSAGE_ID, self.MOCKED_MESSAGE_ID, self.MOCKED_MESSAGE_ID ])), 'UserPropertiesArray': meta.Datum(type='json', value='''[{ "UserId": 1 }, { "UserId": 2 }, { "UserId": 3 }] ''') }) # Non existing properties should return None self.assertIsNone(servicebus_msgs[0].content_type) self.assertIsNone(servicebus_msgs[1].content_type) self.assertIsNone(servicebus_msgs[2].content_type) # Message Id should always be available self.assertEqual(servicebus_msgs[0].message_id, self.MOCKED_MESSAGE_ID) self.assertEqual(servicebus_msgs[1].message_id, self.MOCKED_MESSAGE_ID) self.assertEqual(servicebus_msgs[2].message_id, self.MOCKED_MESSAGE_ID) # User properties should always be available self.assertEqual(servicebus_msgs[0].user_properties['UserId'], 1) self.assertEqual(servicebus_msgs[1].user_properties['UserId'], 2) self.assertEqual(servicebus_msgs[2].user_properties['UserId'], 3)
def test_scalar_typed_data_decoder_not_ok(self): metadata = { 'unsupported_type': bind_meta.Datum(type='bytes', value=b'aaa'), 'unexpected_json': bind_meta.Datum(type='json', value='[1, 2, 3]'), 'unexpected_data': bind_meta.Datum(type='json', value='"foo"'), } cases = [ ('unsupported_type', int, ValueError, "unsupported type of field 'unsupported_type' in " "trigger metadata: bytes"), ('unexpected_json', int, ValueError, "cannot convert value of field 'unexpected_json' in " "trigger metadata into int"), ('unexpected_data', int, ValueError, "cannot convert value of field " "'unexpected_data' in trigger metadata into int: " "invalid literal for int"), ('unexpected_data', (int, float), ValueError, "unexpected value type in field " "'unexpected_data' in trigger metadata: str, " "expected one of: int, float"), ] for field, pytype, exc, msg in cases: with self.subTest(field=field): with self.assertRaisesRegex(exc, msg): Converter._decode_trigger_metadata_field( metadata, field, python_type=pytype)
def _zip(self, key: str, expected_type: str, *args: List[Dict[str, meta.Datum]]) -> meta.Datum: """Combining multiple metadata into one: string -> collection_string bytes -> collection_bytes int -> collection_sint64 sint64 -> collection_sint64 json -> json (with array in it) """ convertible = { 'collection_string': CollectionString, 'collection_bytes': CollectionBytes, 'collection_sint64': CollectionSint64 } datum_type = args[0][key].type if expected_type in convertible.keys(): return meta.Datum( value=convertible[expected_type]([d[key].value for d in args]), type=expected_type) elif expected_type == 'json': if datum_type == 'json': value = json.dumps([json.loads(d[key].value) for d in args]) else: value = json.dumps([d[key].value for d in args]) return meta.Datum(value=value, type='json') else: raise NotImplementedError(f'Unknown convertion {key}: ' f'{datum_type} -> {expected_type}')
def _generate_single_trigger_metadatum(self): return { 'EnqueuedTime': meta.Datum(f'"{self.MOCKED_ENQUEUE_TIME.isoformat()}"', 'json'), 'SystemProperties': meta.Datum('{"iothub-connection-device-id": "MyTestDevice"}', 'json') }
def _generate_servicebus_metadata(self): mocked_metadata: Mapping[str, meta.Datum] = {} mocked_metadata['DeliveryCount'] = meta.Datum(1, 'int') mocked_metadata['LockToken'] = meta.Datum( '87931fd2-39f4-415a-9fdc-adfdcbed3148', 'string') mocked_metadata['ExpiresAtUtc'] = meta.Datum('2020-07-02T05:39:12.17Z', 'string') mocked_metadata['EnqueuedTimeUtc'] = meta.Datum( self.MOCKED_ENQUEUE_TIME.isoformat(), 'string') mocked_metadata['MessageId'] = meta.Datum( '87c66eaf88e84119b66a26278a7b4149', 'string') mocked_metadata['ContentType'] = meta.Datum('application/json', 'string') mocked_metadata['SequenceNumber'] = meta.Datum(3, 'int') mocked_metadata['Label'] = meta.Datum('Microsoft.Azure.ServiceBus', 'string') mocked_metadata['sys'] = meta.Datum(type='json', value=''' { "MethodName": "ServiceBusSMany", "UtcNow": "2020-06-18T05:39:12.2860411Z", "RandGuid": "bb38deae-cc75-49f2-89f5-96ec6eb857db" } ''') return mocked_metadata
def _generate_multiple_trigger_metadata(self) -> Dict[str, meta.Datum]: """Generate a metadatum containing 3 service bus messages which can be distingushed by enqueued_sequence_number """ sb_a = self._generate_single_trigger_metadata() sb_b = self._generate_single_trigger_metadata() sb_c = self._generate_single_trigger_metadata() sb_a['MessageId'] = meta.Datum(self.MOCKED_MESSAGE_ID_A, 'string') sb_b['MessageId'] = meta.Datum(self.MOCKED_MESSAGE_ID_B, 'string') sb_c['MessageId'] = meta.Datum(self.MOCKED_MESSAGE_ID_C, 'string') combine_from = lambda key, et: self._zip(key, et, sb_a, sb_b, sb_c) mocked_metadata = { 'ContentTypeArray': combine_from('ContentType', 'collection_string'), 'CorrelationIdArray': combine_from('CorrelationId', 'collection_string'), 'DeadLetterSourceArray': combine_from('DeadLetterSource', 'collection_string'), 'EnqueuedTimeUtcArray': combine_from('EnqueuedTimeUtc', 'json'), 'ExpiresAtUtcArray': combine_from('ExpiresAtUtc', 'json'), 'LabelArray': combine_from('Label', 'collection_string'), 'LockTokenArray': combine_from('LockToken', 'collection_string'), 'MessageIdArray': combine_from('MessageId', 'collection_string'), 'PartitionKeyArray': combine_from('PartitionKey', 'collection_string'), 'ReplyToArray': combine_from('ReplyTo', 'collection_string'), 'ReplyToSessionIdArray': combine_from('ReplyToSessionId', 'collection_string'), 'ScheduledEnqueueTimeUtcArray': combine_from('ScheduledEnqueueTimeUtc', 'collection_string'), 'SessionIdArray': combine_from('SessionId', 'collection_string'), 'SequenceNumberArray': combine_from('SequenceNumber', 'collection_sint64'), 'TimeToLiveArray': combine_from('TimeToLive', 'collection_string'), 'ToArray': combine_from('To', 'collection_string'), 'UserPropertiesArray': combine_from('UserProperties', 'json') } return mocked_metadata
def test_servicebus_properties(self): # SystemProperties in metadata should propagate to class properties msg = azf_sb.ServiceBusMessageInConverter.decode( data=meta.Datum(b'body_bytes', 'bytes'), trigger_metadata=self._generate_single_trigger_metadata()) self.assertEqual(msg.get_body(), b'body_bytes') # Test individual ServiceBus properties respectively self.assertEqual(msg.content_type, self.MOCKED_CONTENT_TYPE) self.assertEqual(msg.correlation_id, self.MOCKED_CORROLATION_ID) self.assertEqual(msg.dead_letter_source, self.MOCKED_DEADLETTER_SOURCE) self.assertEqual(msg.enqueued_time_utc, self.MOCKED_ENQUEUE_TIME_UTC) self.assertEqual(msg.expires_at_utc, self.MOCKED_EXPIRY_AT_UTC) self.assertEqual(msg.expiration_time, self.MOCKED_EXPIRY_AT_UTC) self.assertEqual(msg.label, self.MOCKED_LABEL) self.assertEqual(msg.message_id, self.MOCKED_MESSAGE_ID) self.assertEqual(msg.partition_key, self.MOCKED_PARTITION_KEY) self.assertEqual(msg.reply_to, self.MOCKED_REPLY_TO) self.assertEqual(msg.reply_to_session_id, self.MOCKED_REPLY_TO_SESSION_ID) self.assertEqual(msg.scheduled_enqueue_time, self.MOCKED_SCHEDULED_ENQUEUE_TIME_UTC) self.assertEqual(msg.scheduled_enqueue_time_utc, self.MOCKED_SCHEDULED_ENQUEUE_TIME_UTC) self.assertEqual(msg.session_id, self.MOCKED_SESSION_ID) self.assertEqual(msg.time_to_live, self.MOCKED_TIME_TO_LIVE_TIMEDELTA) self.assertEqual(msg.to, self.MOCKED_TO) self.assertDictEqual( msg.user_properties, { '$AzureWebJobsParentId': self.MOCKED_AZURE_PARTNER_ID, 'x-opt-enqueue-sequence-number': 0 })
def test_servicebus_non_existing_property(self): # The function should not fail even when property does not work msg = azf_sb.ServiceBusMessageInConverter.decode( data=self._generate_single_servicebus_data(), trigger_metadata={ 'MessageId': meta.Datum(self.MOCKED_MESSAGE_ID, 'string'), 'UserProperties': meta.Datum('{ "UserId": 1 }', 'json') }) # Property that are not passed from extension should be None self.assertIsNone(msg.content_type) # Message id should always be available self.assertEqual(msg.message_id, self.MOCKED_MESSAGE_ID) # User property should always be available self.assertEqual(msg.user_properties['UserId'], 1)
def _generate_multiple_trigger_metadata(self): key_array = [None, None] partition_array = [1, 2] timestamp_array = [ "2020-06-20T05:06:25.139Z", "2020-06-20T05:06:25.945Z" ] return { 'KeyArray': meta.Datum(json.dumps(key_array), 'json'), 'OffsetArray': meta.Datum(CollectionSint64([62, 63]), 'collection_sint64'), 'PartitionArray': meta.Datum(json.dumps(partition_array), 'json'), 'TimestampArray': meta.Datum(json.dumps(timestamp_array), 'json'), 'TopicArray': meta.Datum(CollectionString(['message', 'message']), "collection_string"), 'HeadersArray': meta.Datum( json.dumps([['{"Key":"test","Value":"1"}'], ['{"Key":"test2","Value":"2"}']]), 'json'), 'sys': meta.Datum( '{"MethodName":"KafkaTriggerMany",' '"UtcNow":"2020-06-20T05:06:26.5550868Z",' '"RandGuid":"57d5eeb7-c86c-4924-a14a-160092154093"}', 'json') }
def _generate_multiple_service_bus_data(self) -> meta.Datum: return meta.Datum(value=json.dumps([{ 'lucky_number': 23 }, { 'lucky_number': 34 }, { 'lucky_number': 45 }]), type='json')
def test_double_input_unsupported(self): """ Verify that the given input is unsupported by SharedMemoryManager. This input is double. """ manager = SharedMemoryManager() datum = bind_meta.Datum(type='double', value=1.0) is_supported = manager.is_supported(datum) self.assertFalse(is_supported)
def test_json_input_unsupported(self): """ Verify that the given input is unsupported by SharedMemoryManager. This input is json. """ manager = SharedMemoryManager() content = {'name': 'foo', 'val': 'bar'} datum = bind_meta.Datum(type='json', value=json.dumps(content)) is_supported = manager.is_supported(datum) self.assertFalse(is_supported)
def test_collection_sint64_unsupported(self): """ Verify that the given input is unsupported by SharedMemoryManager. This input is collection_sint64. """ manager = SharedMemoryManager() content = [1, 2] datum = bind_meta.Datum(type='collection_sint64', value=content) is_supported = manager.is_supported(datum) self.assertFalse(is_supported)
def _generate_multiple_iothub_data(self, data_type='json'): data = '[{"device-status": "good1"}, {"device-status": "good2"}]' if data_type == 'collection_bytes': data = list( map(lambda x: json.dumps(x).encode('utf-8'), json.loads(data))) data = CollectionBytes(data) elif data_type == 'collection_string': data = list(map(lambda x: json.dumps(x), json.loads(data))) data = CollectionString(data) return meta.Datum(data, data_type)
def test_bytes_input_support(self): """ Verify that the given input is supported by SharedMemoryManager to be transfered over shared memory. The input is bytes. """ manager = SharedMemoryManager() content_size = consts.MIN_BYTES_FOR_SHARED_MEM_TRANSFER + 10 content = self.get_random_bytes(content_size) bytes_datum = bind_meta.Datum(type='bytes', value=content) is_supported = manager.is_supported(bytes_datum) self.assertTrue(is_supported)
def test_scalar_typed_data_decoder_ok(self): metadata = { 'int_as_json': bind_meta.Datum(type='json', value='1'), 'int_as_string': bind_meta.Datum(type='string', value='1'), 'int_as_int': bind_meta.Datum(type='int', value=1), 'string_as_json': bind_meta.Datum(type='json', value='"aaa"'), 'string_as_string': bind_meta.Datum(type='string', value='aaa'), 'dict_as_json': bind_meta.Datum(type='json', value='{"foo":"bar"}') } cases = [ ('int_as_json', int, 1), ('int_as_string', int, 1), ('int_as_int', int, 1), ('string_as_json', str, 'aaa'), ('string_as_string', str, 'aaa'), ('dict_as_json', dict, {'foo': 'bar'}), ] for field, pytype, expected in cases: with self.subTest(field=field): value = Converter._decode_trigger_metadata_field( metadata, field, python_type=pytype) self.assertIsInstance(value, pytype) self.assertEqual(value, expected)
def test_string_input_support(self): """ Verify that the given input is supported by SharedMemoryManager to be transfered over shared memory. The input is string. """ manager = SharedMemoryManager() content_size = consts.MIN_BYTES_FOR_SHARED_MEM_TRANSFER + 10 num_chars = math.floor(content_size / consts.SIZE_OF_CHAR_BYTES) content = self.get_random_string(num_chars) bytes_datum = bind_meta.Datum(type='string', value=content) is_supported = manager.is_supported(bytes_datum) self.assertTrue(is_supported)
def _generate_full_metadata(self): mocked_metadata: Mapping[str, meta.Datum] = {} mocked_metadata['Offset'] = meta.Datum(type='string', value='3696') mocked_metadata['EnqueuedTimeUtc'] = meta.Datum( type='string', value='2020-07-14T01:27:55.627Z') mocked_metadata['SequenceNumber'] = meta.Datum(type='int', value=47) mocked_metadata['Properties'] = meta.Datum(type='json', value='{}') mocked_metadata['sys'] = meta.Datum(type='json', value=''' { "MethodName":"metadata_trigger", "UtcNow":"2020-07-14T01:27:55.8940305Z", "RandGuid":"db413fd6-8411-4e51-844c-c9b5345e537d" }''') mocked_metadata['SystemProperties'] = meta.Datum(type='json', value=''' { "x-opt-sequence-number":47, "x-opt-offset":"3696", "x-opt-enqueued-time":"2020-07-14T01:27:55.627Z", "SequenceNumber":47, "Offset":"3696", "PartitionKey":null, "EnqueuedTimeUtc":"2020-07-14T01:27:55.627Z", "iothub-connection-device-id":"awesome-device-id" }''') mocked_metadata['PartitionContext'] = meta.Datum(type='json', value=''' { "CancellationToken":{ "IsCancellationRequested":false, "CanBeCanceled":true, "WaitHandle":{ "Handle":{ "value":2472 }, "SafeWaitHandle":{ "IsInvalid":false, "IsClosed":false } } }, "ConsumerGroupName":"$Default", "EventHubPath":"python-worker-ci-eventhub-one-metadata", "PartitionId":"0", "Owner":"88cec2e2-94c9-4e08-acb6-4f2b97cd888e", "RuntimeInformation":{ "PartitionId":"0", "LastSequenceNumber":0, "LastEnqueuedTimeUtc":"0001-01-01T00:00:00", "LastEnqueuedOffset":null, "RetrievalTime":"0001-01-01T00:00:00" } }''') return mocked_metadata
def _generate_multiple_kafka_data(self, data_type='json'): data = '[{"Offset":62,"Partition":1,"Topic":"message",'\ '"Timestamp":"2020-06-20T05:06:25.139Z","Value":"a"},'\ ' {"Offset":63,"Partition":1,"Topic":"message",'\ '"Timestamp":"2020-06-20T05:06:25.945Z","Value":"a"}]' if data_type == 'collection_bytes': data = list( map(lambda x: json.dumps(x).encode('utf-8'), json.loads(data))) data = CollectionBytes(data) elif data_type == 'collection_string': data = list(map(lambda x: json.dumps(x), json.loads(data))) data = CollectionString(data) return meta.Datum(data, data_type)
def test_large_invalid_bytes_input_support(self): """ Verify that the given input is NOT supported by SharedMemoryManager to be transfered over shared memory. The input is bytes of larger than the allowed size. """ manager = SharedMemoryManager() content_size = consts.MAX_BYTES_FOR_SHARED_MEM_TRANSFER + 10 # Not using get_random_bytes to avoid slowing down for creating a large # random input content = b'x01' * content_size bytes_datum = bind_meta.Datum(type='bytes', value=content) is_supported = manager.is_supported(bytes_datum) self.assertFalse(is_supported)
def test_large_invalid_string_input_support(self): """ Verify that the given input is NOT supported by SharedMemoryManager to be transfered over shared memory. The input is string of larger than the allowed size. """ manager = SharedMemoryManager() content_size = consts.MAX_BYTES_FOR_SHARED_MEM_TRANSFER + 10 num_chars = math.floor(content_size / consts.SIZE_OF_CHAR_BYTES) # Not using get_random_string to avoid slowing down for creating a large # random input content = 'a' * num_chars string_datum = bind_meta.Datum(type='string', value=content) is_supported = manager.is_supported(string_datum) self.assertFalse(is_supported)
def test_eventhub_properties(self): """Test if properties from public interface _eventhub.py returns the correct values from metadata""" result = azf_eh.EventHubTriggerConverter.decode( data=meta.Datum(b'body_bytes', 'bytes'), trigger_metadata=self._generate_full_metadata() ) self.assertEqual(result.get_body(), b'body_bytes') self.assertIsNone(result.partition_key) self.assertDictEqual(result.iothub_metadata, {'connection-device-id': 'awesome-device-id'}) self.assertEqual(result.sequence_number, 47) self.assertEqual(result.enqueued_time.isoformat(), '2020-07-14T01:27:55.627000+00:00') self.assertEqual(result.offset, '3696')
def _generate_multiple_trigger_metadata(self): system_props_array = [{ 'EnqueuedTimeUtc': self.MOCKED_ENQUEUE_TIME.isoformat(), 'iothub-connection-device-id': 'MyTestDevice1', }, { 'EnqueuedTimeUtc': self.MOCKED_ENQUEUE_TIME.isoformat(), 'iothub-connection-device-id': 'MyTestDevice2', }] return { 'SystemPropertiesArray': meta.Datum(json.dumps(system_props_array), 'json') }
def test_datum_json_python_value(self): # None datum = meta.Datum(value='null', type="json") self.assertEqual(datum.python_value, None) self.assertEqual(datum.python_type, type(None)) # Int datum = meta.Datum(value='123', type="json") self.assertEqual(datum.python_value, 123) self.assertEqual(datum.python_type, int) # Float datum = meta.Datum(value='456.789', type="json") self.assertEqual(datum.python_value, 456.789) self.assertEqual(datum.python_type, float) # String datum = meta.Datum(value='"string in json"', type="json") self.assertEqual(datum.python_value, "string in json") self.assertEqual(datum.python_type, str) # List datum = meta.Datum(value='["a", "b", "c"]', type="json") self.assertListEqual(datum.python_value, ["a", "b", "c"]) self.assertEqual(datum.python_type, list) # Object datum = meta.Datum(value='{"name": "awesome", "value": "cool"}', type="json") self.assertDictEqual(datum.python_value, { "name": "awesome", "value": "cool" }) self.assertEqual(datum.python_type, dict) # Should ignore Newlines and Spaces datum = meta.Datum(value='{ "name" : "awesome",\n "value": "cool"\n}', type="json") self.assertDictEqual(datum.python_value, { "name": "awesome", "value": "cool" }) self.assertEqual(datum.python_type, dict)
def _generate_single_trigger_metadatum(self): return { 'Offset': meta.Datum('1', 'string'), 'Partition': meta.Datum('0', 'string'), 'Timestamp': meta.Datum(self.SINGLE_KAFKA_TIMESTAMP, 'string'), 'Topic': meta.Datum('users', 'string'), 'Value': meta.Datum('hello', 'string'), 'sys': meta.Datum('{"MethodName":"KafkaTrigger",' '"UtcNow":"2020-06-20T04:43:30.6756278Z",' '"RandGuid":"b0870c0c-2b7a-40dc-b4be-45224c91a49c"}', 'json') # __len__: 6 }
def test_datum_single_level_python_value(self): datum: Mapping[str, meta.Datum] = meta.Datum(value=None, type="int") self.assertEqual(datum.python_value, None) self.assertEqual(datum.python_type, type(None)) datum = meta.Datum(value=1, type=None) self.assertEqual(datum.python_value, None) self.assertEqual(datum.python_type, type(None)) datum = meta.Datum(value=b"awesome bytes", type="bytes") self.assertEqual(datum.python_value, b"awesome bytes") self.assertEqual(datum.python_type, bytes) datum = meta.Datum(value="awesome string", type="string") self.assertEqual(datum.python_value, 'awesome string') self.assertEqual(datum.python_type, str) datum = meta.Datum(value=42, type="int") self.assertEqual(datum.python_value, 42) self.assertEqual(datum.python_type, int) datum = meta.Datum(value=43.2103, type="double") self.assertEqual(datum.python_value, 43.2103) self.assertEqual(datum.python_type, float)
def _generate_single_kafka_datum(self, datum_type='string'): datum = self.SINGLE_KAFKA_DATAUM if datum_type == 'bytes': datum = datum.encode('utf-8') return meta.Datum(datum, datum_type)
def encode( cls, obj: typing.Any, *, expected_type: typing.Optional[type] ) -> meta.Datum: print(f"Encoding {obj}") return meta.Datum(obj, object)
def _generate_single_iothub_datum(self, datum_type='json'): datum = '{"device-status": "good"}' if datum_type == 'bytes': datum = datum.encode('utf-8') return meta.Datum(datum, datum_type)