def _pb_attr_value(val):
"""Given a value, return the protobuf attribute name and proper value.
The Protobuf API uses different attribute names based on value types
rather than inferring the type. This function simply determines the
proper attribute name based on the type of the value provided and
returns the attribute name as well as a properly formatted value.
Certain value types need to be coerced into a different type (such
as a `datetime.datetime` into an integer timestamp, or a
`gcloud.datastore.key.Key` into a Protobuf representation. This
function handles that for you.
.. note::
Values which are "text" ('unicode' in Python2, 'str' in Python3) map
to 'string_value' in the datastore; values which are "bytes"
('str' in Python2, 'bytes' in Python3) map to 'blob_value'.
For example:
>>> _pb_attr_value(1234)
('integer_value', 1234)
>>> _pb_attr_value('my_string')
('string_value', 'my_string')
:type val: `datetime.datetime`, :class:`gcloud.datastore.key.Key`,
bool, float, integer, string
:param val: The value to be scrutinized.
:rtype: tuple
:returns: A tuple of the attribute name and proper value type.
"""
if isinstance(val, datetime.datetime):
name = 'timestamp'
value = _datetime_to_pb_timestamp(val)
elif isinstance(val, Key):
name, value = 'key', val.to_protobuf()
elif isinstance(val, bool):
name, value = 'boolean', val
elif isinstance(val, float):
name, value = 'double', val
elif isinstance(val, six.integer_types):
name, value = 'integer', val
elif isinstance(val, six.text_type):
name, value = 'string', val
elif isinstance(val, (bytes, str)):
name, value = 'blob', val
elif isinstance(val, Entity):
name, value = 'entity', val
elif isinstance(val, list):
name, value = 'array', val
elif isinstance(val, GeoPoint):
name, value = 'geo_point', val.to_protobuf()
elif val is None:
name, value = 'null', struct_pb2.NULL_VALUE
else:
raise ValueError("Unknown protobuf attr type %s" % type(val))
return name + '_value', value
def _pb_attr_value(val):
"""Given a value, return the protobuf attribute name and proper value.
The Protobuf API uses different attribute names based on value types
rather than inferring the type. This function simply determines the
proper attribute name based on the type of the value provided and
returns the attribute name as well as a properly formatted value.
Certain value types need to be coerced into a different type (such
as a `datetime.datetime` into an integer timestamp, or a
`gcloud.datastore.key.Key` into a Protobuf representation. This
function handles that for you.
.. note::
Values which are "text" ('unicode' in Python2, 'str' in Python3) map
to 'string_value' in the datastore; values which are "bytes"
('str' in Python2, 'bytes' in Python3) map to 'blob_value'.
For example:
>>> _pb_attr_value(1234)
('integer_value', 1234)
>>> _pb_attr_value('my_string')
('string_value', 'my_string')
:type val: `datetime.datetime`, :class:`gcloud.datastore.key.Key`,
bool, float, integer, string
:param val: The value to be scrutinized.
:rtype: tuple
:returns: A tuple of the attribute name and proper value type.
"""
if isinstance(val, datetime.datetime):
name = 'timestamp'
value = _datetime_to_pb_timestamp(val)
elif isinstance(val, Key):
name, value = 'key', val.to_protobuf()
elif isinstance(val, bool):
name, value = 'boolean', val
elif isinstance(val, float):
name, value = 'double', val
elif isinstance(val, six.integer_types):
name, value = 'integer', val
elif isinstance(val, six.text_type):
name, value = 'string', val
elif isinstance(val, (bytes, str)):
name, value = 'blob', val
elif isinstance(val, Entity):
name, value = 'entity', val
elif isinstance(val, list):
name, value = 'array', val
elif isinstance(val, GeoPoint):
name, value = 'geo_point', val.to_protobuf()
elif val is None:
name, value = 'null', struct_pb2.NULL_VALUE
else:
raise ValueError("Unknown protobuf attr type %s" % type(val))
return name + '_value', value
def test_create(self):
import datetime
from google.longrunning import operations_pb2
from google.protobuf.any_pb2 import Any
from gcloud.bigtable._generated import (
bigtable_instance_admin_pb2 as messages_v2_pb2)
from gcloud._helpers import _datetime_to_pb_timestamp
from gcloud.bigtable._testing import _FakeStub
from gcloud.operation import Operation
from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES
from gcloud.bigtable.instance import _CREATE_INSTANCE_METADATA_URL
NOW = datetime.datetime.utcnow()
NOW_PB = _datetime_to_pb_timestamp(NOW)
client = _Client(self.PROJECT)
instance = self._makeOne(self.INSTANCE_ID, client, self.LOCATION_ID,
display_name=self.DISPLAY_NAME)
# Create response_pb
metadata = messages_v2_pb2.CreateInstanceMetadata(request_time=NOW_PB)
response_pb = operations_pb2.Operation(
name=self.OP_NAME,
metadata=Any(
type_url=_CREATE_INSTANCE_METADATA_URL,
value=metadata.SerializeToString(),
)
)
# Patch the stub used by the API method.
client._instance_stub = stub = _FakeStub(response_pb)
# Perform the method and check the result.
result = instance.create()
self.assertTrue(isinstance(result, Operation))
self.assertEqual(result.name, self.OP_NAME)
self.assertTrue(result.target is instance)
self.assertTrue(result.client is client)
self.assertIsInstance(result.pb_metadata,
messages_v2_pb2.CreateInstanceMetadata)
self.assertEqual(result.pb_metadata.request_time, NOW_PB)
self.assertEqual(result.metadata, {'request_type': 'CreateInstance'})
self.assertEqual(len(stub.method_calls), 1)
api_name, args, kwargs = stub.method_calls[0]
self.assertEqual(api_name, 'CreateInstance')
request_pb, = args
self.assertTrue(
isinstance(request_pb, messages_v2_pb2.CreateInstanceRequest))
self.assertEqual(request_pb.parent, 'projects/%s' % (self.PROJECT,))
self.assertEqual(request_pb.instance_id, self.INSTANCE_ID)
self.assertEqual(request_pb.instance.display_name, self.DISPLAY_NAME)
cluster = request_pb.clusters[self.INSTANCE_ID]
self.assertEqual(cluster.serve_nodes, DEFAULT_SERVE_NODES)
self.assertEqual(kwargs, {})
def _log_entry_mapping_to_pb(mapping):
"""Helper for :meth:`write_entries`, et aliae
Ideally, would use a function from :mod:`protobuf.json_format`, but
the right one isn't public. See:
https://github.com/google/protobuf/issues/1351
"""
# pylint: disable=too-many-branches
entry_pb = LogEntry()
optional_scalar_keys = {
'logName': 'log_name',
'insertId': 'insert_id',
'textPayload': 'text_payload',
}
for key, pb_name in optional_scalar_keys.items():
if key in mapping:
setattr(entry_pb, pb_name, mapping[key])
if 'resource' in mapping:
entry_pb.resource.type = mapping['resource']['type']
if 'severity' in mapping:
severity = mapping['severity']
if isinstance(severity, str):
severity = LogSeverity.Value(severity)
entry_pb.severity = severity
if 'timestamp' in mapping:
timestamp = _datetime_to_pb_timestamp(mapping['timestamp'])
entry_pb.timestamp.CopyFrom(timestamp)
if 'labels' in mapping:
for key, value in mapping['labels'].items():
entry_pb.labels[key] = value
if 'jsonPayload' in mapping:
for key, value in mapping['jsonPayload'].items():
entry_pb.json_payload[key] = value
if 'protoPayload' in mapping:
Parse(json.dumps(mapping['protoPayload']), entry_pb.proto_payload)
if 'httpRequest' in mapping:
_http_request_mapping_to_pb(
mapping['httpRequest'], entry_pb.http_request)
if 'operation' in mapping:
_log_operation_mapping_to_pb(
mapping['operation'], entry_pb.operation)
return entry_pb
def _log_entry_mapping_to_pb(mapping):
"""Helper for :meth:`write_entries`, et aliae
Performs "impedance matching" between the protobuf attrs and the keys
expected in the JSON API.
"""
# pylint: disable=too-many-branches
entry_pb = LogEntry()
optional_scalar_keys = {
'logName': 'log_name',
'insertId': 'insert_id',
'textPayload': 'text_payload',
}
for key, pb_name in optional_scalar_keys.items():
if key in mapping:
setattr(entry_pb, pb_name, mapping[key])
if 'resource' in mapping:
entry_pb.resource.type = mapping['resource']['type']
if 'severity' in mapping:
severity = mapping['severity']
if isinstance(severity, str):
severity = LogSeverity.Value(severity)
entry_pb.severity = severity
if 'timestamp' in mapping:
timestamp = _datetime_to_pb_timestamp(mapping['timestamp'])
entry_pb.timestamp.CopyFrom(timestamp)
if 'labels' in mapping:
for key, value in mapping['labels'].items():
entry_pb.labels[key] = value
if 'jsonPayload' in mapping:
for key, value in mapping['jsonPayload'].items():
entry_pb.json_payload[key] = value
if 'protoPayload' in mapping:
Parse(json.dumps(mapping['protoPayload']), entry_pb.proto_payload)
if 'httpRequest' in mapping:
_http_request_mapping_to_pb(mapping['httpRequest'],
entry_pb.http_request)
if 'operation' in mapping:
_log_operation_mapping_to_pb(mapping['operation'], entry_pb.operation)
return entry_pb
def _callFUT(self, when):
from gcloud._helpers import _datetime_to_pb_timestamp
return _datetime_to_pb_timestamp(when)
def test_list_entries_with_extra_properties(self):
from datetime import datetime
from google.logging.type.log_severity_pb2 import WARNING
from gcloud._testing import _GAXPageIterator
from gcloud._helpers import UTC
from gcloud._helpers import _datetime_to_rfc3339
from gcloud._helpers import _datetime_to_pb_timestamp
NOW = datetime.utcnow().replace(tzinfo=UTC)
SIZE = 23
TOKEN = 'TOKEN'
NEW_TOKEN = 'NEW_TOKEN'
PAYLOAD = {'message': 'MESSAGE', 'weather': 'sunny'}
SEVERITY = 'WARNING'
LABELS = {
'foo': 'bar',
}
IID = 'IID'
request = _HTTPRequestPB()
operation = _LogEntryOperationPB()
EXTRAS = {
'severity': WARNING,
'labels': LABELS,
'insert_id': IID,
'http_request': request,
'operation': operation,
}
ENTRY = _LogEntryPB(self.LOG_NAME, proto_payload=PAYLOAD, **EXTRAS)
ENTRY.resource.labels['foo'] = 'bar'
ENTRY.timestamp = _datetime_to_pb_timestamp(NOW)
response = _GAXPageIterator([ENTRY], NEW_TOKEN)
gax_api = _GAXLoggingAPI(_list_log_entries_response=response)
api = self._makeOne(gax_api)
entries, next_token = api.list_entries(
[self.PROJECT], page_size=SIZE, page_token=TOKEN)
self.assertEqual(len(entries), 1)
entry = entries[0]
self.assertIsInstance(entry, dict)
self.assertEqual(entry['logName'], self.LOG_NAME)
self.assertEqual(entry['resource'],
{'type': 'global', 'labels': {'foo': 'bar'}})
self.assertEqual(entry['protoPayload'], PAYLOAD)
self.assertEqual(entry['severity'], SEVERITY)
self.assertEqual(entry['labels'], LABELS)
self.assertEqual(entry['insertId'], IID)
self.assertEqual(entry['timestamp'], _datetime_to_rfc3339(NOW))
EXPECTED_REQUEST = {
'requestMethod': request.request_method,
'requestUrl': request.request_url,
'status': request.status,
'requestSize': request.request_size,
'responseSize': request.response_size,
'referer': request.referer,
'userAgent': request.user_agent,
'remoteIp': request.remote_ip,
'cacheHit': request.cache_hit,
}
self.assertEqual(entry['httpRequest'], EXPECTED_REQUEST)
EXPECTED_OPERATION = {
'producer': operation.producer,
'id': operation.id,
'first': operation.first,
'last': operation.last,
}
self.assertEqual(entry['operation'], EXPECTED_OPERATION)
self.assertEqual(next_token, NEW_TOKEN)
projects, filter_, order_by, page_size, options = (
gax_api._list_log_entries_called_with)
self.assertEqual(projects, [self.PROJECT])
self.assertEqual(filter_, '')
self.assertEqual(order_by, '')
self.assertEqual(page_size, SIZE)
self.assertEqual(options.page_token, TOKEN)
def _make_timestamp():
from datetime import datetime
from gcloud._helpers import UTC
from gcloud._helpers import _datetime_to_pb_timestamp
NOW = datetime.utcnow().replace(tzinfo=UTC)
return _datetime_to_pb_timestamp(NOW)
def _callFUT(self, when):
from gcloud._helpers import _datetime_to_pb_timestamp
return _datetime_to_pb_timestamp(when)
def test_update(self):
import datetime
from google.longrunning import operations_pb2
from gcloud.operation import Operation
from google.protobuf.any_pb2 import Any
from gcloud._helpers import _datetime_to_pb_timestamp
from gcloud.bigtable._generated import (instance_pb2 as data_v2_pb2)
from gcloud.bigtable._generated import (bigtable_instance_admin_pb2 as
messages_v2_pb2)
from gcloud.bigtable._testing import _FakeStub
from gcloud.bigtable.cluster import _UPDATE_CLUSTER_METADATA_URL
NOW = datetime.datetime.utcnow()
NOW_PB = _datetime_to_pb_timestamp(NOW)
SERVE_NODES = 81
client = _Client(self.PROJECT)
instance = _Instance(self.INSTANCE_ID, client)
cluster = self._makeOne(self.CLUSTER_ID,
instance,
serve_nodes=SERVE_NODES)
# Create request_pb
request_pb = _ClusterPB(
name=self.CLUSTER_NAME,
serve_nodes=SERVE_NODES,
)
# Create response_pb
OP_ID = 5678
OP_NAME = (
'operations/projects/%s/instances/%s/clusters/%s/operations/%d' %
(self.PROJECT, self.INSTANCE_ID, self.CLUSTER_ID, OP_ID))
metadata = messages_v2_pb2.UpdateClusterMetadata(request_time=NOW_PB)
response_pb = operations_pb2.Operation(
name=OP_NAME,
metadata=Any(type_url=_UPDATE_CLUSTER_METADATA_URL,
value=metadata.SerializeToString()))
# Patch the stub used by the API method.
client._instance_stub = stub = _FakeStub(response_pb)
result = cluster.update()
self.assertTrue(isinstance(result, Operation))
self.assertEqual(result.name, OP_NAME)
self.assertTrue(result.target is cluster)
self.assertTrue(result.client is client)
self.assertIsInstance(result.pb_metadata,
messages_v2_pb2.UpdateClusterMetadata)
self.assertEqual(result.pb_metadata.request_time, NOW_PB)
self.assertEqual(result.metadata, {'request_type': 'UpdateCluster'})
self.assertEqual(len(stub.method_calls), 1)
api_name, args, kwargs = stub.method_calls[0]
self.assertEqual(api_name, 'UpdateCluster')
request_pb, = args
self.assertTrue(isinstance(request_pb, data_v2_pb2.Cluster))
self.assertEqual(request_pb.name, self.CLUSTER_NAME)
self.assertEqual(request_pb.serve_nodes, SERVE_NODES)
self.assertEqual(kwargs, {})
