Beispiel #1
0
    async def _test_binary_blob_read_function(self, func_name):
        """
        Verify that the function executed successfully when the worker received
        inputs for the function over shared memory.
        """
        async with testutils.start_mockhost(script_root=self.blob_funcs_dir) \
                as host:
            await host.load_function(func_name)

            # Write binary content into shared memory
            mem_map_name = self.get_new_mem_map_name()
            content_size = consts.MIN_BYTES_FOR_SHARED_MEM_TRANSFER + 10
            content = self.get_random_bytes(content_size)
            content_md5 = hashlib.md5(content).hexdigest()
            mem_map_size = consts.CONTENT_HEADER_TOTAL_BYTES + content_size
            mem_map = self.file_accessor.create_mem_map(
                mem_map_name, mem_map_size)
            shared_mem_map = SharedMemoryMap(self.file_accessor, mem_map_name,
                                             mem_map)
            num_bytes_written = shared_mem_map.put_bytes(content)

            # Create a message to send to the worker containing info about the
            # shared memory region to read input from
            value = protos.RpcSharedMemory(name=mem_map_name,
                                           offset=0,
                                           count=num_bytes_written,
                                           type=protos.RpcDataType.bytes)

            # Invoke the function; it should read the input blob from shared
            # memory and respond back in the HTTP body with the number of bytes
            # it read in the input
            _, response_msg = await host.invoke_function(
                func_name, [
                    protos.ParameterBinding(
                        name='req',
                        data=protos.TypedData(http=protos.RpcHttp(
                            method='GET'))),
                    protos.ParameterBinding(name='file',
                                            rpc_shared_memory=value)
                ])

            # Dispose the shared memory map since the function is done using it
            shared_mem_map.dispose()

            # Verify if the function executed successfully
            self.assertEqual(protos.StatusResult.Success,
                             response_msg.response.result.status)

            response_bytes = response_msg.response.return_value.http.body.bytes
            json_response = json.loads(response_bytes)
            func_received_content_size = json_response['content_size']
            func_received_content_md5 = json_response['content_md5']

            # Check the function response to ensure that it read the complete
            # input that we provided and the md5 matches
            self.assertEqual(content_size, func_received_content_size)
            self.assertEqual(content_md5, func_received_content_md5)
Beispiel #2
0
 async def call_and_check(due: bool):
     _, r = await host.invoke_function('return_pastdue', [
         protos.ParameterBinding(
             name='timer',
             data=protos.TypedData(
                 json=json.dumps({'IsPastDue': due})))
     ])
     self.assertEqual(r.response.result.status,
                      protos.StatusResult.Success)
     self.assertEqual(list(r.response.output_data), [
         protos.ParameterBinding(
             name='pastdue', data=protos.TypedData(string=str(due)))
     ])
    async def test_call_async_function_check_logs(self):
        async with testutils.start_mockhost() as host:
            await host.load_function('async_logging')

            invoke_id, r = await host.invoke_function(
                'async_logging', [
                    protos.ParameterBinding(
                        name='req',
                        data=protos.TypedData(
                            http=protos.RpcHttp(
                                method='GET')))
                ])

            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Success)

            user_logs = [line for line in r.logs if
                         line.category == 'my function']
            self.assertEqual(len(user_logs), 2)

            first_msg = user_logs[0]
            self.assertEqual(first_msg.invocation_id, invoke_id)
            self.assertEqual(first_msg.message, 'hello info')
            self.assertEqual(first_msg.level, protos.RpcLog.Information)

            second_msg = user_logs[1]
            self.assertEqual(second_msg.invocation_id, invoke_id)
            self.assertTrue(second_msg.message.startswith('and another error'))
            self.assertEqual(second_msg.level, protos.RpcLog.Error)

            self.assertEqual(r.response.return_value.string, 'OK-async')
Beispiel #4
0
    async def test_mock_eventhub_cardinality_many(self):
        async with testutils.start_mockhost(
                script_root=self.mock_funcs_dir) as host:

            func_id, r = await host.load_function('eventhub_cardinality_many')
            self.assertEqual(r.response.function_id, func_id)
            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Success)

            _, r = await host.invoke_function(
                'eventhub_cardinality_many',
                [
                    protos.ParameterBinding(
                        name='events',
                        data=protos.TypedData(
                            json=json.dumps({
                                'id': 'cardinality_many'
                            })
                        ),
                    ),
                ],
                metadata={}
            )

            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Success)
            self.assertEqual(r.response.return_value.string, 'OK_MANY')
Beispiel #5
0
    async def test_mock_eventhub_cardinality_many(self):
        async with testutils.start_mockhost(
                script_root=self.mock_funcs_dir) as host:

            func_id, r = await host.load_function('eventhub_cardinality_many')
            self.assertEqual(r.response.function_id, func_id)
            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Success)

            _, r = await host.invoke_function(
                'eventhub_cardinality_many',
                [
                    protos.ParameterBinding(
                        name='events',
                        data=protos.TypedData(
                            json=json.dumps([{
                                'id': 'cardinality_many'
                            }])
                        ),
                    ),
                ],
                metadata={
                    'SystemPropertiesArray': protos.TypedData(json=json.dumps([
                        {
                            'iothub-device-id': 'mock-iothub-device-id',
                            'iothub-auth-data': 'mock-iothub-auth-data',
                            'EnqueuedTimeUtc': '2020-02-18T21:28:42.5888539Z'
                        }
                    ]))
                }
            )

            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Success)
            self.assertEqual(r.response.return_value.string, 'OK_MANY')
Beispiel #6
0
            async def call_and_check():
                _, r = await host.invoke_function(
                    'eventhub_trigger_iot',
                    [
                        protos.ParameterBinding(
                            name='event',
                            data=protos.TypedData(
                                json=json.dumps({
                                    'id': 'foo'
                                })
                            ),
                        ),
                    ],
                    metadata={
                        'iothub-device-id': protos.TypedData(
                            string='mock-iothub-device-id'
                        ),
                        'iothub-auth-data': protos.TypedData(
                            string='mock-iothub-auth-data'
                        )
                    }
                )

                self.assertEqual(r.response.result.status,
                                 protos.StatusResult.Success)
                self.assertIn(
                    'mock-iothub-device-id',
                    r.response.return_value.string
                )
    async def test_call_sync_function_check_logs(self):
        async with testutils.start_mockhost() as host:
            await host.load_function('sync_logging')

            invoke_id, r = await host.invoke_function(
                'sync_logging', [
                    protos.ParameterBinding(
                        name='req',
                        data=protos.TypedData(
                            http=protos.RpcHttp(
                                method='GET')))
                ])

            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Success)

            user_logs = [line for line in r.logs
                         if line.category == 'my function']
            self.assertEqual(len(user_logs), 1)

            log = user_logs[0]
            self.assertEqual(log.invocation_id, invoke_id)
            self.assertTrue(log.message.startswith(
                'a gracefully handled error'))

            self.assertEqual(r.response.return_value.string, 'OK-sync')
Beispiel #8
0
            async def call_and_check():
                _, r = await host.invoke_function(
                    'eventhub_trigger_iot',
                    [
                        protos.ParameterBinding(
                            name='event',
                            data=protos.TypedData(
                                json=json.dumps({
                                    'id': 'foo'
                                })
                            ),
                        ),
                    ],
                    metadata={
                        'SystemProperties': protos.TypedData(json=json.dumps({
                            'iothub-device-id': 'mock-iothub-device-id',
                            'iothub-auth-data': 'mock-iothub-auth-data',
                            'EnqueuedTimeUtc': '2020-02-18T21:28:42.5888539Z'
                        }))
                    }
                )

                self.assertEqual(r.response.result.status,
                                 protos.StatusResult.Success)

                res_json_string = r.response.return_value.string
                self.assertIn('device-id', res_json_string)
                self.assertIn('mock-iothub-device-id', res_json_string)
                self.assertIn('auth-data', res_json_string)
                self.assertIn('mock-iothub-auth-data', res_json_string)
Beispiel #9
0
    async def test_mock_generic_as_bytes_no_anno(self):
        async with testutils.start_mockhost(
                script_root=self.generic_funcs_dir) as host:

            func_id, r = await host.load_function('foobar_as_bytes_no_anno')

            self.assertEqual(r.response.function_id, func_id)
            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Success)

            _, r = await host.invoke_function(
                'foobar_as_bytes_no_anno', [
                    protos.ParameterBinding(
                        name='input',
                        data=protos.TypedData(
                            bytes=b'\x00\x01'
                        )
                    )
                ]
            )
            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Success)
            self.assertEqual(
                r.response.return_value,
                protos.TypedData(bytes=b'\x00\x01')
            )
Beispiel #10
0
    async def _invoke_function(self, host: testutils._MockWebHost,
                               function_name: str):
        _, r = await host.invoke_function(function_name, [
            protos.ParameterBinding(
                name='req',
                data=protos.TypedData(http=protos.RpcHttp(method='GET')))
        ])

        self.assertEqual(r.response.result.status, protos.StatusResult.Success)
Beispiel #11
0
 async def call_and_check():
     _, r = await host.invoke_function('user_event_loop_timer', [
         protos.ParameterBinding(
             name='timer',
             data=protos.TypedData(
                 json=json.dumps({'IsPastDue': False})))
     ])
     self.assertEqual(r.response.result.status,
                      protos.StatusResult.Success)
Beispiel #12
0
    async def test_close_shared_memory_maps(self):
        """
        Close the shared memory maps created by the worker to transfer output
        blob to the host after the host is done processing the response.
        """
        func_name = 'put_blob_as_bytes_return_http_response'
        async with testutils.start_mockhost(script_root=self.blob_funcs_dir) \
                as host:
            await host.load_function(func_name)

            content_size = consts.MIN_BYTES_FOR_SHARED_MEM_TRANSFER + 10
            http_params = {'content_size': str(content_size)}

            # Invoke the function; it should read the input blob from shared
            # memory and respond back in the HTTP body with the number of bytes
            # it read in the input
            _, response_msg = await host.invoke_function(
                func_name, [
                    protos.ParameterBinding(
                        name='req',
                        data=protos.TypedData(http=protos.RpcHttp(
                            method='GET', query=http_params))),
                ])

            # Verify if the function executed successfully
            self.assertEqual(protos.StatusResult.Success,
                             response_msg.response.result.status)

            # Verify if the worker produced an output blob which was written
            # in shared memory
            output_data = response_msg.response.output_data
            output_binding = output_data[0]

            # Get the information about the shared memory region in which the
            # worker wrote the function's output blob
            shmem = output_binding.rpc_shared_memory
            mem_map_name = shmem.name

            # Request the worker to close the memory maps
            mem_map_names = [mem_map_name]
            response_msg = \
                await host.close_shared_memory_resources(mem_map_names)

            # Verify that the worker responds with a successful status after
            # closing the requested memory map
            mem_map_statuses = response_msg.response.close_map_results
            self.assertEqual(len(mem_map_names), len(mem_map_statuses.keys()))
            for mem_map_name in mem_map_names:
                self.assertTrue(mem_map_name in mem_map_statuses)
                status = mem_map_statuses[mem_map_name]
                self.assertTrue(status)
Beispiel #13
0
    async def _check_if_function_is_ok(self, host):
        # Ensure the function can be properly loaded
        func_id, load_r = await host.load_function('show_context')
        self.assertEqual(load_r.response.function_id, func_id)
        self.assertEqual(load_r.response.result.status,
                         protos.StatusResult.Success)

        # Ensure the function can be properly invoked
        invoke_id, call_r = await host.invoke_function('show_context', [
            protos.ParameterBinding(
                name='req',
                data=protos.TypedData(http=protos.RpcHttp(method='GET')))
        ])
        self.assertIsNotNone(invoke_id)
        self.assertEqual(call_r.response.result.status,
                         protos.StatusResult.Success)
    async def test_mock_generic_should_not_support_implicit_output(self):
        async with testutils.start_mockhost(
                script_root=self.generic_funcs_dir) as host:

            func_id, r = await host.load_function('foobar_implicit_output')

            self.assertEqual(r.response.function_id, func_id)
            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Success)

            _, r = await host.invoke_function('foobar_as_bytes_no_anno', [
                protos.ParameterBinding(
                    name='input', data=protos.TypedData(bytes=b'\x00\x01'))
            ])
            # It should fail here, since generic binding requires
            # $return statement in function.json to pass output
            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Failure)
    async def test_mock_generic_should_support_without_datatype(self):
        async with testutils.start_mockhost(
                script_root=self.generic_funcs_dir) as host:

            func_id, r = await host.load_function('foobar_with_no_datatype')

            self.assertEqual(r.response.function_id, func_id)
            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Success)

            _, r = await host.invoke_function('foobar_with_no_datatype', [
                protos.ParameterBinding(
                    name='input', data=protos.TypedData(bytes=b'\x00\x01'))
            ])
            # It should fail here, since the generic binding requires datatype
            # to be defined in function.json
            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Failure)
Beispiel #16
0
    async def test_mock_orchestration_trigger(self):
        async with testutils.start_mockhost(
                script_root=self.durable_functions_dir) as host:

            func_id, r = await host.load_function('orchestration_trigger')

            self.assertEqual(r.response.function_id, func_id)
            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Success)

            _, r = await host.invoke_function('orchestration_trigger', [
                protos.ParameterBinding(
                    name='context',
                    data=protos.TypedData(
                        string='Durable functions coming soon'))
            ])
            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Success)
            self.assertEqual(
                r.response.return_value,
                protos.TypedData(json='Durable functions coming soon :)'))
Beispiel #17
0
    async def test_mock_activity_trigger(self):
        async with testutils.start_mockhost(
                script_root=self.durable_functions_dir) as host:

            func_id, r = await host.load_function('activity_trigger')

            self.assertEqual(r.response.function_id, func_id)
            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Success)

            _, r = await host.invoke_function(
                'activity_trigger',
                [
                    # According to Durable Python
                    # Activity Trigger's input must be json serializable
                    protos.ParameterBinding(
                        name='input',
                        data=protos.TypedData(string='test single_word'))
                ])
            self.assertEqual(r.response.result.status,
                             protos.StatusResult.Success)
            self.assertEqual(r.response.return_value,
                             protos.TypedData(json='"test single_word"'))
Beispiel #18
0
    async def test_shared_memory_not_used_with_small_output(self):
        """
        Even though shared memory is enabled, small inputs will not be
        transferred over shared memory (in this case from the worker to the
        host.)
        """
        func_name = 'put_blob_as_bytes_return_http_response'
        async with testutils.start_mockhost(script_root=self.blob_funcs_dir) \
                as host:
            await host.load_function(func_name)

            content_size = consts.MIN_BYTES_FOR_SHARED_MEM_TRANSFER - 10
            http_params = {'content_size': str(content_size)}

            # Invoke the function; it should read the input blob from shared
            # memory and respond back in the HTTP body with the number of bytes
            # it read in the input
            _, response_msg = await host.invoke_function(
                func_name, [
                    protos.ParameterBinding(
                        name='req',
                        data=protos.TypedData(http=protos.RpcHttp(
                            method='GET', query=http_params))),
                ])

            # Verify if the function executed successfully
            self.assertEqual(protos.StatusResult.Success,
                             response_msg.response.result.status)

            # Verify if the worker produced an output blob which was sent over
            # RPC instead of shared memory
            output_data = response_msg.response.output_data
            self.assertEqual(1, len(output_data))

            output_binding = output_data[0]
            binding_type = output_binding.WhichOneof('rpc_data')
            self.assertEqual('data', binding_type)
Beispiel #19
0
    async def test_binary_blob_write_function(self):
        """
        Write a blob with binary output that was transferred between the worker
        and host over shared memory.
        """
        func_name = 'put_blob_as_bytes_return_http_response'
        async with testutils.start_mockhost(script_root=self.blob_funcs_dir) \
                as host:
            await host.load_function(func_name)

            content_size = consts.MIN_BYTES_FOR_SHARED_MEM_TRANSFER + 10
            http_params = {'content_size': str(content_size)}

            # Invoke the function; it should read the input blob from shared
            # memory and respond back in the HTTP body with the number of bytes
            # it read in the input
            _, response_msg = await host.invoke_function(
                func_name, [
                    protos.ParameterBinding(
                        name='req',
                        data=protos.TypedData(http=protos.RpcHttp(
                            method='GET', query=http_params))),
                ])

            # Verify if the function executed successfully
            self.assertEqual(protos.StatusResult.Success,
                             response_msg.response.result.status)

            # The function responds back in the HTTP body with the md5 digest of
            # the output it created along with its size
            response_bytes = response_msg.response.return_value.http.body.bytes
            json_response = json.loads(response_bytes)
            func_created_content_size = json_response['content_size']
            func_created_content_md5 = json_response['content_md5']

            # Verify if the worker produced an output blob which was written
            # in shared memory
            output_data = response_msg.response.output_data
            self.assertEqual(1, len(output_data))

            output_binding = output_data[0]
            binding_type = output_binding.WhichOneof('rpc_data')
            self.assertEqual('rpc_shared_memory', binding_type)

            # Get the information about the shared memory region in which the
            # worker wrote the function's output blob
            shmem = output_binding.rpc_shared_memory
            mem_map_name = shmem.name
            offset = shmem.offset
            count = shmem.count
            data_type = shmem.type

            # Verify if the shared memory region's information is valid
            self.assertTrue(self.is_valid_uuid(mem_map_name))
            self.assertEqual(0, offset)
            self.assertEqual(func_created_content_size, count)
            self.assertEqual(protos.RpcDataType.bytes, data_type)

            # Read data from the shared memory region
            mem_map_size = consts.CONTENT_HEADER_TOTAL_BYTES + count
            mem_map = self.file_accessor.open_mem_map(mem_map_name,
                                                      mem_map_size)
            shared_mem_map = SharedMemoryMap(self.file_accessor, mem_map_name,
                                             mem_map)
            read_content = shared_mem_map.get_bytes()

            # Dispose the shared memory map since we have read the function's
            # output now
            shared_mem_map.dispose()

            # Verify if we were able to read the correct output that the
            # function has produced
            read_content_md5 = hashlib.md5(read_content).hexdigest()
            self.assertEqual(func_created_content_md5, read_content_md5)
            self.assertEqual(len(read_content), func_created_content_size)
Beispiel #20
0
    async def test_multiple_input_output_blobs(self):
        """
        Read two blobs and write two blobs, all over shared memory.
        """
        func_name = 'put_get_multiple_blobs_as_bytes_return_http_response'
        async with testutils.start_mockhost(script_root=self.blob_funcs_dir) \
                as host:
            await host.load_function(func_name)

            # Input 1
            # Write binary content into shared memory
            mem_map_name_1 = self.get_new_mem_map_name()
            input_content_size_1 = consts.MIN_BYTES_FOR_SHARED_MEM_TRANSFER + 10
            input_content_1 = self.get_random_bytes(input_content_size_1)
            input_content_md5_1 = hashlib.md5(input_content_1).hexdigest()
            input_mem_map_size_1 = \
                consts.CONTENT_HEADER_TOTAL_BYTES + input_content_size_1
            input_mem_map_1 = \
                self.file_accessor.create_mem_map(mem_map_name_1,
                                                  input_mem_map_size_1)
            input_shared_mem_map_1 = \
                SharedMemoryMap(self.file_accessor, mem_map_name_1,
                                input_mem_map_1)
            input_num_bytes_written_1 = \
                input_shared_mem_map_1.put_bytes(input_content_1)

            # Create a message to send to the worker containing info about the
            # shared memory region to read input from
            input_value_1 = protos.RpcSharedMemory(
                name=mem_map_name_1,
                offset=0,
                count=input_num_bytes_written_1,
                type=protos.RpcDataType.bytes)

            # Input 2
            # Write binary content into shared memory
            mem_map_name_2 = self.get_new_mem_map_name()
            input_content_size_2 = consts.MIN_BYTES_FOR_SHARED_MEM_TRANSFER + 20
            input_content_2 = self.get_random_bytes(input_content_size_2)
            input_content_md5_2 = hashlib.md5(input_content_2).hexdigest()
            input_mem_map_size_2 = \
                consts.CONTENT_HEADER_TOTAL_BYTES + input_content_size_2
            input_mem_map_2 = \
                self.file_accessor.create_mem_map(mem_map_name_2,
                                                  input_mem_map_size_2)
            input_shared_mem_map_2 = \
                SharedMemoryMap(self.file_accessor, mem_map_name_2,
                                input_mem_map_2)
            input_num_bytes_written_2 = \
                input_shared_mem_map_2.put_bytes(input_content_2)

            # Outputs
            output_content_size_1 = \
                consts.MIN_BYTES_FOR_SHARED_MEM_TRANSFER + 11
            output_content_size_2 = \
                consts.MIN_BYTES_FOR_SHARED_MEM_TRANSFER + 22
            http_params = {
                'output_content_size_1': str(output_content_size_1),
                'output_content_size_2': str(output_content_size_2)
            }

            # Create a message to send to the worker containing info about the
            # shared memory region to read input from
            input_value_2 = protos.RpcSharedMemory(
                name=mem_map_name_2,
                offset=0,
                count=input_num_bytes_written_2,
                type=protos.RpcDataType.bytes)

            # Invoke the function; it should read the input blob from shared
            # memory and respond back in the HTTP body with the number of bytes
            # it read in the input
            _, response_msg = await host.invoke_function(
                func_name, [
                    protos.ParameterBinding(
                        name='req',
                        data=protos.TypedData(http=protos.RpcHttp(
                            method='GET', query=http_params))),
                    protos.ParameterBinding(name='inputfile1',
                                            rpc_shared_memory=input_value_1),
                    protos.ParameterBinding(name='inputfile2',
                                            rpc_shared_memory=input_value_2)
                ])
            time.sleep(1)

            # Dispose the shared memory map since the function is done using it
            input_shared_mem_map_1.dispose()
            input_shared_mem_map_2.dispose()

            # Verify if the function executed successfully
            self.assertEqual(protos.StatusResult.Success,
                             response_msg.response.result.status)

            response_bytes = response_msg.response.return_value.http.body.bytes
            json_response = json.loads(response_bytes)

            func_received_content_size_1 = json_response[
                'input_content_size_1']
            func_received_content_md5_1 = json_response['input_content_md5_1']
            func_received_content_size_2 = json_response[
                'input_content_size_2']
            func_received_content_md5_2 = json_response['input_content_md5_2']
            func_created_content_size_1 = json_response[
                'output_content_size_1']
            func_created_content_size_2 = json_response[
                'output_content_size_2']
            func_created_content_md5_1 = json_response['output_content_md5_1']
            func_created_content_md5_2 = json_response['output_content_md5_2']

            # Check the function response to ensure that it read the complete
            # input that we provided and the md5 matches
            self.assertEqual(input_content_size_1,
                             func_received_content_size_1)
            self.assertEqual(input_content_md5_1, func_received_content_md5_1)
            self.assertEqual(input_content_size_2,
                             func_received_content_size_2)
            self.assertEqual(input_content_md5_2, func_received_content_md5_2)

            # Verify if the worker produced two output blobs which were written
            # in shared memory
            output_data = response_msg.response.output_data
            self.assertEqual(2, len(output_data))

            # Output 1
            output_binding_1 = output_data[0]
            binding_type = output_binding_1.WhichOneof('rpc_data')
            self.assertEqual('rpc_shared_memory', binding_type)

            shmem_1 = output_binding_1.rpc_shared_memory
            self._verify_function_output(shmem_1, func_created_content_size_1,
                                         func_created_content_md5_1)

            # Output 2
            output_binding_2 = output_data[1]
            binding_type = output_binding_2.WhichOneof('rpc_data')
            self.assertEqual('rpc_shared_memory', binding_type)

            shmem_2 = output_binding_2.rpc_shared_memory
            self._verify_function_output(shmem_2, func_created_content_size_2,
                                         func_created_content_md5_2)