def serialise_args_and_kwargs(self, args, kwargs) -> Any:
"""
Serialises Python args and kwargs into protobuf
If there is a single arg and no kwargs and the arg is already a protobuf it returns
that instance packed inside an Any. Otherwise it returns an ArgsAndKwargs packed in an Any.
:param args: the Python args
:param kwargs: the Python kwargs
:return: protobuf Any
"""
# if kwargs are empty and this is a single protobuf arguments then
# send in simple format i.e. fn(protobuf) -> protobuf as opposed to fn(*args, **kwargs) -> (*results,)
# so as to aid calling flink functions written in other frameworks that might not understand
# how to deal with the concept of keyword arguments or tuples of arguments
if isinstance(args, Message) and not kwargs:
request = args
else:
args = args if _is_tuple(args) else (args, )
request = ArgsAndKwargs()
request.args.CopyFrom(
_convert_to_proto(args, self._protobuf_converters))
request.kwargs.CopyFrom(
_convert_to_proto(kwargs, self._protobuf_converters))
return pack_any(request)
def test_converting_without_suitable_converter(self):
obj = self.MyType('my_val')
try:
_convert_to_proto(obj, [])
except ValueError as e:
self.assertIn('Cannot convert value of type', str(e))
else:
self.fail('Expected an exception')
def serialise_result(self, task_result: TaskResult, result, state):
"""
Serialises the result of a task invocation into a TaskResult
:param task_result: the TaskResult
:param result: task result
:param state: task state
"""
task_result.result.CopyFrom(
pack_any(_convert_to_proto(result, self._protobuf_converters)))
task_result.state.CopyFrom(
pack_any(_convert_to_proto(state, self._protobuf_converters)))
def test_convert_dict_to_protofbuf(self):
data = {
'int': 123,
'float': 1.23,
'str': '123',
'list': [1, 2, 3],
'dict': {
'a': 1,
'b': 2
},
'dict_in_list': [1, {
'a': 1
}],
'proto': Address(namespace="tests", type="test", id="id")
}
proto = _convert_to_proto(data, self.default_converters)
reconsituted_data = _convert_from_proto(proto, [Address],
self.default_converters)
self.assertEqual(reconsituted_data['int'], 123)
self.assertEqual(reconsituted_data['float'], 1.23)
self.assertEqual(reconsituted_data['str'], '123')
self.assertEqual(reconsituted_data['list'], [1, 2, 3])
self.assertEqual(reconsituted_data['dict']['a'], 1)
self.assertEqual(reconsituted_data['dict']['b'], 2)
self.assertEqual(reconsituted_data['dict_in_list'][0], 1)
self.assertEqual(reconsituted_data['dict_in_list'][1]['a'], 1)
self.assertTrue(isinstance(reconsituted_data['proto'], Address))
self.assertEqual(reconsituted_data['proto'].namespace, 'tests')
self.assertEqual(reconsituted_data['proto'].type, 'test')
self.assertEqual(reconsituted_data['proto'].id, 'id')
def to_proto(self, item) -> Message:
"""
Converts from Python to protobuf
:param item: the item to convert
:return: the protobuf message possibly packed in an Any
"""
return _convert_to_proto(item, self._protobuf_converters)
def test_to_and_from_protobuf(self):
obj = self.MyType('my_val')
converters = [self.MyConverter()]
proto_message = _convert_to_proto(obj, converters)
python_val = _convert_from_proto(proto_message, [], converters)
self.assertIsInstance(proto_message, TestMyType)
self.assertIsInstance(python_val, self.MyType)
self.assertEqual(python_val.string_field, 'my_val')
def test_convert_list_to_protofbuf(self):
data = [Address(namespace="tests", type="test", id="id"), 1, '123']
proto = _convert_to_proto(data, _generate_default_converters())
reconsituted_data = _convert_from_proto(proto, [Address],
self.default_converters)
self.assertTrue(isinstance(reconsituted_data[0], Address))
self.assertEqual(reconsituted_data[0].namespace, 'tests')
self.assertEqual(reconsituted_data[0].type, 'test')
self.assertEqual(reconsituted_data[0].id, 'id')
self.assertEqual(reconsituted_data[1], 1)
self.assertEqual(reconsituted_data[2], '123')
def serialise_request(self,
task_request: TaskRequest,
request: Any,
state=None,
retry_policy=None):
"""
Serialises args, kwargs and optional state into a TaskRequest
:param task_request: the TaskRequest
:param request: request (proto format)
:param optional state: task state
:param optional retry_policy: task retry policy
"""
task_request.request.CopyFrom(request)
task_request.state.CopyFrom(
pack_any(_convert_to_proto(state, self._protobuf_converters)))
if retry_policy:
task_request.retry_policy.CopyFrom(retry_policy)