def test_protobuf_any_converter():
unpacker = py_converters.ProtobufAnyUnpacker(
{"compiler_gym.FloatTensor": FloatTensor}
)
type_based_converter = py_converters.TypeBasedConverter(
conversion_map={FloatTensor: py_converters.convert_tensor_message_to_numpy}
)
converter = py_converters.ProtobufAnyConverter(
unpacker=unpacker, message_converter=type_based_converter
)
any_msg = any_pb2.Any()
tensor_message = FloatTensor(shape=[1], value=[1])
any_msg.Pack(tensor_message)
tensor = converter(any_msg)
assert isinstance(tensor, np.ndarray)
def testConstructWithCustomConfig(self):
custom_config = example_gen_pb2.CustomConfig(
custom_config=any_pb2.Any())
example_gen = component.FileBasedExampleGen(
input_base='path',
custom_config=custom_config,
custom_executor_spec=executor_spec.BeamExecutorSpec(
TestExampleGenExecutor))
stored_custom_config = example_gen_pb2.CustomConfig()
proto_utils.json_to_proto(
example_gen.exec_properties[
standard_component_specs.CUSTOM_CONFIG_KEY],
stored_custom_config)
self.assertEqual(custom_config, stored_custom_config)
async def wrapper(request):
try:
content = await request.content.read()
try:
request_message = ExpectedMessage.FromString(content)
except Exception as e:
raise exceptions.ApiError(
code='api.wrong_request_message',
message='request message has unexpected format')
response_message = await handler(request_message,
config=request.app['config'])
data = any_pb2.Any()
data.Pack(response_message)
body = base_pb2.ApiResponse(
server_time=datetime.datetime.now().timestamp(),
status=base_pb2.ApiResponse.SUCCESS,
data=data)
except exceptions.ApiError as e:
error = base_pb2.ApiError(code=e.code,
message=e.message,
details=e.details)
body = base_pb2.ApiResponse(
server_time=datetime.datetime.now().timestamp(),
status=base_pb2.ApiResponse.ERROR,
error=error)
except Exception as e:
import traceback
traceback.print_exc()
error = base_pb2.ApiError(
code='api.unknown_error',
message='unkown error occured during request processing',
details={})
body = base_pb2.ApiResponse(
server_time=datetime.datetime.now().timestamp(),
status=base_pb2.ApiResponse.ERROR,
error=error)
finally:
return web.Response(content_type='application/protobuf',
body=body.SerializeToString())
def encode_tensor_node(node):
"""Encode a "reference" to a Tensor/SparseTensor as a TensorInfo in an Any.
We put the Tensor / SparseTensor in a TensorInfo, which we then wrap in an
Any so that it can be added to the CollectionDef.
Args:
node: Tensor node.
Returns:
Any proto wrapping a TensorInfo.
"""
any_buf = any_pb2.Any()
tensor_info = tf.saved_model.utils.build_tensor_info(node)
any_buf.Pack(tensor_info)
return any_buf
def pack_xla_computation(xla_computation):
"""Pack a `XlaComputation` into `Any` proto with a HLO module proto payload.
Args:
xla_computation: An instance of `xla_client.XlaComputation` to pack.
Returns:
A `google.protobuf.Any` protocol buffer message containing this
computation's `HloModuleProto` in a binary-serialized form.
Raises:
TypeError: if `xla_computation` is not an `xla_client.XlaComputation`.
"""
py_typecheck.check_type(xla_computation, xla_client.XlaComputation)
return any_pb2.Any(type_url=_HLO_MODULE_PROTO_URI,
value=xla_computation.as_serialized_hlo_module_proto())
def exception_handler(self, exc, context):
log.exception(exc)
detail = any_pb2.Any()
if self.debug:
detail.Pack(
error_details_pb2.DebugInfo(
stack_entries=traceback.format_stack(),
detail=repr(exc)
)
)
status = status_pb2.Status(
code=code_pb2.INTERNAL,
message=str(exc),
details=[detail]
)
context.abort_with_status(rpc_status.to_status(status))
def pack_graph_def(graph_def):
"""Pack a `tf.GraphDef` into a proto3 `Any` message.
Args:
graph_def: the `tf.GraphDef` to pack into a protocol buffer message.
Returns:
A `google.protobuf.Any` protocol buffer message.
Raises:
TypeError: if `graph_def` is not a `tf.GraphDef`.
"""
py_typecheck.check_type(graph_def, tf.GraphDef)
any_pb = any_pb2.Any()
any_pb.Pack(graph_def)
return any_pb
def pack_to_any_message(msg: message.Message) -> any_pb2.Any:
"""Packs a protobuf Message type to protobuf Any type.
Args:
msg (message.Message): protobuf Message to be packed
Returns:
any_pb2.Any: to be used for `google.protobuf.Any` type
"""
assert isinstance(msg, message.Message), "Wrong type"
packed_any = any_pb2.Any()
packed_any.Pack(msg, type_url_prefix="/")
return packed_any
def SendRawTransactions(self, request, context):
with futures.ProcessPoolExecutor(max_workers=1) as executor:
task = executor.submit(self.web3Task, request,
'eth_sendRawTransaction')
try:
tx = task.result(timeout=30)
print(tx)
return ethereum_pb2.TransactionInfo(
transaction=Web3.toJSON(tx))
except Exception as exc:
print("Exception: ", exc)
detail = any_pb2.Any()
rich_status = rpc_status.status_pb2.Status(
code=code_pb2.NOT_FOUND,
message=str(exc),
details=[detail])
context.abort_with_status(rpc_status.to_status(rich_status))
def testEnableCache(self):
input_base = standard_artifacts.ExternalArtifact()
custom_config = example_gen_pb2.CustomConfig(
custom_config=any_pb2.Any())
example_gen_1 = component.FileBasedExampleGen(
input=channel_utils.as_channel([input_base]),
custom_config=custom_config,
custom_executor_spec=executor_spec.ExecutorClassSpec(
TestExampleGenExecutor))
self.assertEqual(None, example_gen_1.enable_cache)
example_gen_2 = component.FileBasedExampleGen(
input=channel_utils.as_channel([input_base]),
custom_config=custom_config,
custom_executor_spec=executor_spec.ExecutorClassSpec(
TestExampleGenExecutor),
enable_cache=True)
self.assertEqual(True, example_gen_2.enable_cache)
def callRpc(self, token, jStr):
j = json.loads(jStr)
channel = grpc.insecure_channel(self.host + ':' + str(self.rpc_port))
stub = seldon_pb2.SeldonStub(channel)
data = iris_pb2.IrisPredictRequest(f1=j["f1"],
f2=j["f2"],
f3=j["f3"],
f4=j["f4"])
dataAny = any_pb2.Any()
dataAny.Pack(data)
meta = seldon_pb2.ClassificationRequestMeta(puid="12345")
request = seldon_pb2.ClassificationRequest(meta=meta, data=dataAny)
metadata = [(b'oauth_token', token)]
reply = stub.Classify(request, 999, metadata=metadata)
print reply
def test_load_mconfig(self, get_service_config_value_mock):
# Fixture mconfig has 1 missing service, 1 unregistered type
magmad_fixture = mconfigs_pb2.MagmaD(
checkin_interval=10,
checkin_timeout=5,
autoupgrade_enabled=True,
autoupgrade_poll_interval=300,
package_version='1.0.0-0',
images=[],
tier_id='default',
feature_flags={'flag1': False},
)
magmad_fixture_any = any_pb2.Any()
magmad_fixture_any.Pack(magmad_fixture)
magmad_fixture_serialized = MessageToJson(magmad_fixture_any)
fixture = '''
{
"offset": 42,
"configs": {
"configs_by_key": {
"magmad": %s,
"mme": {
"@type": "type.googleapis.com/magma.mconfig.NotAType",
"value": "test1"
},
"not_a_service": {
"@type": "type.googleapis.com/magma.mconfig.MagmaD",
"value": "test2"
}
}
}
}
''' % magmad_fixture_serialized
get_service_config_value_mock.return_value = ['mme']
with mock.patch('builtins.open', mock.mock_open(read_data=fixture)):
manager = mconfig_managers.StreamedMconfigManager()
actual = manager.load_mconfig()
expected_configs_by_key = {'magmad': magmad_fixture_any}
expected = mconfig_pb2.OffsetGatewayConfigs(
offset=42,
configs=mconfig_pb2.GatewayConfigs(
configs_by_key=expected_configs_by_key, ),
)
self.assertEqual(expected, actual)
def setup_proto_payload(lines: list, log_status: LogCompletionStatusEnum,
**kwargs):
"""
Build the log protoPayload portion of the log entry. Thread safe.
:param lines: List of LogMessage lines to add.
:param log_status: Logging completion status value.
:return: RequestLog pb2 object.
"""
# Base set of values for proto_payload object.
req_dict = {
"@type": REQUEST_LOG_TYPE,
"startTime": datetime.now(timezone.utc).isoformat(),
"ip": "0.0.0.0",
"first": True,
"finished": True,
"endTime": datetime.now(timezone.utc).isoformat(),
"responseSize": 355,
"line": [],
# If we see these lines in the logs, we know something isn't working correctly.
"userAgent": "Bad Mojo",
"resource": "/Bad-Mojo-Teacups/Logging",
}
# Override any key values.
for k, v in kwargs.items():
req_dict[k] = v
# Set completion statuses
if log_status == LogCompletionStatusEnum.PARTIAL_BEGIN:
req_dict['finished'] = False
elif log_status == LogCompletionStatusEnum.PARTIAL_MORE:
req_dict['first'] = False
req_dict['finished'] = False
elif log_status == LogCompletionStatusEnum.PARTIAL_FINISHED:
req_dict['first'] = False
if lines and len(lines) > 0:
for x in range(len(lines)):
req_dict["line"].append(lines[x])
# Convert dict to Generic pb2 message object, requires gcp_request_log_pb2 import.
request_log_pb2 = gcp_json_format.ParseDict(req_dict, gcp_any_pb2.Any())
return request_log_pb2
def _pack_any_proto(value):
"""Helper function to pack Any proto, iff it's not already packed."""
if isinstance(value, any_pb2.Any):
return value
elif isinstance(value, message.Message):
any_proto = any_pb2.Any()
any_proto.Pack(value)
return any_proto
else:
# If we reach this exception, it is normally because the type being packed
# is not supported. Raise exception with some typical examples.
raise ValueError(
"Trying to pack an Any proto with a type that's not "
f"recognized! Type: {type(value)}, value: '{value}'. "
'Is the value a jagged iterable? Is the data type not a '
'supported primitive type like strings, floats, integers '
'or protobuf messages? Are all elements in the array the '
'same type?')
def test_compute_returns_result(self, mock_stub):
tensor_proto = tf.make_tensor_proto(1)
any_pb = any_pb2.Any()
any_pb.Pack(tensor_proto)
value = executor_pb2.Value(tensor=any_pb)
mock_stub.compute.return_value = executor_pb2.ComputeResponse(
value=value)
executor = remote_executor.RemoteExecutor(mock_stub)
_set_cardinalities_with_mock(executor, mock_stub)
executor.set_cardinalities({placements.CLIENTS: 3})
type_signature = computation_types.FunctionType(None, tf.int32)
comp = remote_executor.RemoteValue(executor_pb2.ValueRef(),
type_signature, executor)
result = asyncio.run(comp.compute())
mock_stub.compute.assert_called_once()
self.assertEqual(result, 1)
def Subscribe(self, request_iterator, context):
q = Queue.Queue()
queues.append(q)
while True:
batch = q.get()
any_msg = any_pb2.Any()
any_msg.Pack(batch)
t = gnmi_pb2.TypedValue(any_val=any_msg)
update_msg = [gnmi_pb2.Update(val=t)]
tm = int(time.time() * 1000)
notif = gnmi_pb2.Notification(timestamp=tm, update=update_msg)
response = gnmi_pb2.SubscribeResponse(update=notif)
print datetime.now(), "yielding batch of size:", len(batch.ip)
yield response
print "Streaming done!"
queues.remove(q)
def test_translation_of_search_hit_with_primitives(self):
"""Verify that well-known types get translated into their values rather than into dicts."""
values_and_serializations = (
(wrappers_pb2.BoolValue(value=True), True),
(wrappers_pb2.Int32Value(value=1), 1),
(wrappers_pb2.UInt32Value(value=2), 2),
# int64 is expected to be converted to a string because of JSON!!
(wrappers_pb2.Int64Value(value=3), "3"),
(wrappers_pb2.StringValue(value="abc"), "abc"),
)
for value, expected in values_and_serializations:
any_proto = any_pb2.Any()
any_proto.Pack(value)
self.assertEqual(
expected,
response_translation.translate_message_to_dict(any_proto))
def test_compute_returns_result(self, mock_stub):
tensor_proto = tf.make_tensor_proto(1)
any_pb = any_pb2.Any()
any_pb.Pack(tensor_proto)
value = executor_pb2.Value(tensor=any_pb)
response = executor_pb2.ComputeResponse(value=value)
instance = mock_stub.return_value
instance.Compute = mock.Mock(side_effect=[response])
loop = asyncio.get_event_loop()
executor = create_remote_executor()
type_signature = computation_types.FunctionType(None, tf.int32)
comp = remote_executor.RemoteValue(executor_pb2.ValueRef(),
type_signature, executor)
result = loop.run_until_complete(comp.compute())
instance.Compute.assert_called_once()
self.assertEqual(result, 1)
def build_query_seed():
"""
Do the elaborate proto packing necessary to feed
a QueryExampleGen custom_config.
"""
seed_runtime = data_types.RuntimeParameter(
name='seed_pattern',
default="'%meni%','%avw3%'",
ptype=str
)
bigquery_seed_proto = bigquery_example_gen_pb2.BigQuerySeed()
bigquery_seed_proto.seed = json_utils.dumps(seed_runtime)
any_proto = any_pb2.Any()
any_proto.Pack(bigquery_seed_proto, 'bigqueryseed.dstillery.com')
return example_gen_pb2.CustomConfig(custom_config=any_proto)
def read(self, key: str):
"""Reads the value of a property.
Args:
key: A string key that represents the property to read.
Returns:
The value of the property, either as a scalar (float, int, string, etc.)
or, if the response tensor has a non-empty `shape` attribute, a NumPy
array of the payload with the correct type and shape. See
tensor_utils.unpack for more details.
"""
response = properties_pb2.PropertyResponse()
packed_request = any_pb2.Any()
packed_request.Pack(
properties_pb2.PropertyRequest(
read_property=properties_pb2.ReadPropertyRequest(key=key)))
self._connection.send(packed_request).Unpack(response)
return tensor_utils.unpack_tensor(response.read_property.value)
def testInvalidAny(self):
message = any_pb2.Any()
text = '{"@type": "type.googleapis.com/google.protobuf.Int32Value"}'
self.assertRaisesRegexp(KeyError, 'value', json_format.Parse, text,
message)
text = '{"value": 1234}'
self.assertRaisesRegexp(json_format.ParseError,
'@type is missing when parsing any message.',
json_format.Parse, text, message)
text = '{"@type": "type.googleapis.com/MessageNotExist", "value": 1234}'
self.assertRaisesRegexp(
TypeError, 'Can not find message descriptor by type_url: '
'type.googleapis.com/MessageNotExist.', json_format.Parse, text,
message)
# Only last part is to be used: b/25630112
text = (
r'{"@type": "incorrect.googleapis.com/google.protobuf.Int32Value",'
r'"value": 1234}')
json_format.Parse(text, message)
def testPackWithCustomTypeUrl(self):
submessage = any_test_pb2.TestAny()
submessage.int_value = 12345
msg = any_pb2.Any()
# Pack with a custom type URL prefix.
msg.Pack(submessage, 'type.myservice.com')
self.assertEqual(msg.type_url,
'type.myservice.com/%s' % submessage.DESCRIPTOR.full_name)
# Pack with a custom type URL prefix ending with '/'.
msg.Pack(submessage, 'type.myservice.com/')
self.assertEqual(msg.type_url,
'type.myservice.com/%s' % submessage.DESCRIPTOR.full_name)
# Pack with an empty type URL prefix.
msg.Pack(submessage, '')
self.assertEqual(msg.type_url,
'/%s' % submessage.DESCRIPTOR.full_name)
# Test unpacking the type.
unpacked_message = any_test_pb2.TestAny()
self.assertTrue(msg.Unpack(unpacked_message))
self.assertEqual(submessage, unpacked_message)
def _create_expected_feature_name_statistics(self,
feature_coverage=None,
avg_token_length=None):
custom_stats = []
nls = statistics_pb2.NaturalLanguageStatistics()
if feature_coverage is not None:
nls.feature_coverage = feature_coverage
custom_stats.append(
statistics_pb2.CustomStatistic(name='nl_feature_coverage',
num=feature_coverage))
if avg_token_length is not None:
nls.avg_token_length = avg_token_length
custom_stats.append(
statistics_pb2.CustomStatistic(name='nl_avg_token_length',
num=avg_token_length))
my_proto = any_pb2.Any()
custom_stats.append(
statistics_pb2.CustomStatistic(name='nl_statistics',
any=my_proto.Pack(nls)))
return statistics_pb2.FeatureNameStatistics(custom_stats=custom_stats)
def test_compute_returns_result(self, mock_executor_grpc_stub):
tensor_proto = tf.make_tensor_proto(1)
any_pb = any_pb2.Any()
any_pb.Pack(tensor_proto)
value = executor_pb2.Value(tensor=any_pb)
response = executor_pb2.ComputeResponse(value=value)
instance = mock_executor_grpc_stub.return_value
instance.Compute = mock.Mock(side_effect=[response])
request = executor_pb2.ComputeRequest(
executor=executor_pb2.ExecutorId(),
value_ref=executor_pb2.ValueRef())
stub = create_stub()
result = stub.compute(request)
instance.Compute.assert_called_once()
value, _ = value_serialization.deserialize_value(result.value)
self.assertEqual(value, 1)
def list(self, key: str = '') -> Sequence[PropertySpec]:
"""Lists properties residing under the provided key.
Args:
key: A string key to list properties at this location. If empty, returns
properties registered at the root level.
Returns:
A sequence of PropertySpecs.
"""
response = properties_pb2.PropertyResponse()
packed_request = any_pb2.Any()
packed_request.Pack(
properties_pb2.PropertyRequest(
list_property=properties_pb2.ListPropertyRequest(key=key)))
self._connection.send(packed_request).Unpack(response)
return tuple(
PropertySpec(sub_property)
for sub_property in response.list_property.values)
def Subscribe(self, request_iterator, context):
#create a channel connecting to the southbound device
channel = grpc.insecure_channel(device_ip + ":" + str(device_port))
stub = gnmi_pb2.gNMIStub(channel)
counter = 0
global interval
eventQ = Queue.Queue()
ptime = time.time()
#start streaming
for response in stub.Subscribe(request_iterator):
#logger.debug(response)
path =""
for update in response.update.update:
event = p4_pb2.P4_int()
update.val.any_val.Unpack(event)
eventQ.put(event)
path = update.path
else:
pass
ctime = time.time()
if(ctime - ptime >= interval):
eventlist = []
while not eventQ.empty():
eventlist.append(eventQ.get())
if len(eventlist) > 0:
for e in eventlist:
jsonObj = MessageToJson(e)
#logger.debug(jsonObj)
aggregated = self.processQ(eventlist)
logger.debug("aggregatedTo: %s" %(MessageToJson(aggregated)))
any_msg = any_pb2.Any()
any_msg.Pack(aggregated)
typedValue = gnmi_pb2.TypedValue(any_val=any_msg)
a_update = gnmi_pb2.Update(path=path, val=typedValue)
a_updates = []
a_updates.append(a_update)
a_noti = gnmi_pb2.Notification(timestamp=response.update.timestamp, update=a_updates)
yield gnmi_pb2.SubscribeResponse(update=a_noti)
ptime = time.time()
print "Streaming done!"
def test_with_undelete_cluster_metadata(self):
from google.protobuf import any_pb2
from google.protobuf.timestamp_pb2 import Timestamp
from gcloud.bigtable._generated import (bigtable_cluster_data_pb2 as
data_pb2)
from gcloud.bigtable._generated import (
bigtable_cluster_service_messages_pb2 as messages_pb2)
type_url = ('type.googleapis.com/' +
messages_pb2._UNDELETECLUSTERMETADATA.full_name)
metadata = messages_pb2.UndeleteClusterMetadata(
request_time=Timestamp(seconds=1, nanos=1234),
finish_time=Timestamp(seconds=10, nanos=891011),
)
any_val = any_pb2.Any(
type_url=type_url,
value=metadata.SerializeToString(),
)
result = self._callFUT(any_val)
self.assertEqual(result, metadata)
def test_with_known_type_url(self):
from google.protobuf import any_pb2
from gcloud._testing import _Monkey
from gcloud.bigtable import cluster as MUT
cell = _CellPB(
timestamp_micros=0,
value=b'foobar',
)
type_url = 'type.googleapis.com/' + cell.DESCRIPTOR.full_name
fake_type_url_map = {type_url: cell.__class__}
any_val = any_pb2.Any(
type_url=type_url,
value=cell.SerializeToString(),
)
with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map):
result = self._callFUT(any_val)
self.assertEqual(result, cell)
def test_from_any_pb_wrapped_success():
# Declare a message class conforming to wrapped messages.
class WrappedDate(object):
def __init__(self, **kwargs):
self._pb = date_pb2.Date(**kwargs)
def __eq__(self, other):
return self._pb == other
@classmethod
def pb(cls, msg):
return msg._pb
# Run the same test as `test_from_any_pb_success`, but using the
# wrapped class.
in_message = date_pb2.Date(year=1990)
in_message_any = any_pb2.Any()
in_message_any.Pack(in_message)
out_message = protobuf_helpers.from_any_pb(WrappedDate, in_message_any)
assert out_message == in_message
def test_with_known_type_url(self):
from google.protobuf import any_pb2
from gcloud._testing import _Monkey
from gcloud.bigtable._generated import (data_pb2 as data_v2_pb2)
from gcloud.bigtable import instance as MUT
TYPE_URL = 'type.googleapis.com/' + data_v2_pb2._CELL.full_name
fake_type_url_map = {TYPE_URL: data_v2_pb2.Cell}
cell = data_v2_pb2.Cell(
timestamp_micros=0,
value=b'foobar',
)
any_val = any_pb2.Any(
type_url=TYPE_URL,
value=cell.SerializeToString(),
)
with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map):
result = self._callFUT(any_val)
self.assertEqual(result, cell)
