def _pb_attr_value(val): """Given a value, return the protobuf attribute name and proper value. The Protobuf API uses different attribute names based on value types rather than inferring the type. This function simply determines the proper attribute name based on the type of the value provided and returns the attribute name as well as a properly formatted value. Certain value types need to be coerced into a different type (such as a `datetime.datetime` into an integer timestamp, or a `gcloud.datastore.key.Key` into a Protobuf representation. This function handles that for you. .. note:: Values which are "text" ('unicode' in Python2, 'str' in Python3) map to 'string_value' in the datastore; values which are "bytes" ('str' in Python2, 'bytes' in Python3) map to 'blob_value'. For example: >>> _pb_attr_value(1234) ('integer_value', 1234) >>> _pb_attr_value('my_string') ('string_value', 'my_string') :type val: `datetime.datetime`, :class:`gcloud.datastore.key.Key`, bool, float, integer, string :param val: The value to be scrutinized. :rtype: tuple :returns: A tuple of the attribute name and proper value type. """ if isinstance(val, datetime.datetime): name = 'timestamp' value = _datetime_to_pb_timestamp(val) elif isinstance(val, Key): name, value = 'key', val.to_protobuf() elif isinstance(val, bool): name, value = 'boolean', val elif isinstance(val, float): name, value = 'double', val elif isinstance(val, six.integer_types): name, value = 'integer', val elif isinstance(val, six.text_type): name, value = 'string', val elif isinstance(val, (bytes, str)): name, value = 'blob', val elif isinstance(val, Entity): name, value = 'entity', val elif isinstance(val, list): name, value = 'array', val elif isinstance(val, GeoPoint): name, value = 'geo_point', val.to_protobuf() elif val is None: name, value = 'null', struct_pb2.NULL_VALUE else: raise ValueError("Unknown protobuf attr type %s" % type(val)) return name + '_value', value
def _pb_attr_value(val): """Given a value, return the protobuf attribute name and proper value. The Protobuf API uses different attribute names based on value types rather than inferring the type. This function simply determines the proper attribute name based on the type of the value provided and returns the attribute name as well as a properly formatted value. Certain value types need to be coerced into a different type (such as a `datetime.datetime` into an integer timestamp, or a `gcloud.datastore.key.Key` into a Protobuf representation. This function handles that for you. .. note:: Values which are "text" ('unicode' in Python2, 'str' in Python3) map to 'string_value' in the datastore; values which are "bytes" ('str' in Python2, 'bytes' in Python3) map to 'blob_value'. For example: >>> _pb_attr_value(1234) ('integer_value', 1234) >>> _pb_attr_value('my_string') ('string_value', 'my_string') :type val: `datetime.datetime`, :class:`gcloud.datastore.key.Key`, bool, float, integer, string :param val: The value to be scrutinized. :rtype: tuple :returns: A tuple of the attribute name and proper value type. """ if isinstance(val, datetime.datetime): name = 'timestamp' value = _datetime_to_pb_timestamp(val) elif isinstance(val, Key): name, value = 'key', val.to_protobuf() elif isinstance(val, bool): name, value = 'boolean', val elif isinstance(val, float): name, value = 'double', val elif isinstance(val, six.integer_types): name, value = 'integer', val elif isinstance(val, six.text_type): name, value = 'string', val elif isinstance(val, (bytes, str)): name, value = 'blob', val elif isinstance(val, Entity): name, value = 'entity', val elif isinstance(val, list): name, value = 'array', val elif isinstance(val, GeoPoint): name, value = 'geo_point', val.to_protobuf() elif val is None: name, value = 'null', struct_pb2.NULL_VALUE else: raise ValueError("Unknown protobuf attr type %s" % type(val)) return name + '_value', value
def test_create(self): import datetime from google.longrunning import operations_pb2 from google.protobuf.any_pb2 import Any from gcloud.bigtable._generated import ( bigtable_instance_admin_pb2 as messages_v2_pb2) from gcloud._helpers import _datetime_to_pb_timestamp from gcloud.bigtable._testing import _FakeStub from gcloud.operation import Operation from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES from gcloud.bigtable.instance import _CREATE_INSTANCE_METADATA_URL NOW = datetime.datetime.utcnow() NOW_PB = _datetime_to_pb_timestamp(NOW) client = _Client(self.PROJECT) instance = self._makeOne(self.INSTANCE_ID, client, self.LOCATION_ID, display_name=self.DISPLAY_NAME) # Create response_pb metadata = messages_v2_pb2.CreateInstanceMetadata(request_time=NOW_PB) response_pb = operations_pb2.Operation( name=self.OP_NAME, metadata=Any( type_url=_CREATE_INSTANCE_METADATA_URL, value=metadata.SerializeToString(), ) ) # Patch the stub used by the API method. client._instance_stub = stub = _FakeStub(response_pb) # Perform the method and check the result. result = instance.create() self.assertTrue(isinstance(result, Operation)) self.assertEqual(result.name, self.OP_NAME) self.assertTrue(result.target is instance) self.assertTrue(result.client is client) self.assertIsInstance(result.pb_metadata, messages_v2_pb2.CreateInstanceMetadata) self.assertEqual(result.pb_metadata.request_time, NOW_PB) self.assertEqual(result.metadata, {'request_type': 'CreateInstance'}) self.assertEqual(len(stub.method_calls), 1) api_name, args, kwargs = stub.method_calls[0] self.assertEqual(api_name, 'CreateInstance') request_pb, = args self.assertTrue( isinstance(request_pb, messages_v2_pb2.CreateInstanceRequest)) self.assertEqual(request_pb.parent, 'projects/%s' % (self.PROJECT,)) self.assertEqual(request_pb.instance_id, self.INSTANCE_ID) self.assertEqual(request_pb.instance.display_name, self.DISPLAY_NAME) cluster = request_pb.clusters[self.INSTANCE_ID] self.assertEqual(cluster.serve_nodes, DEFAULT_SERVE_NODES) self.assertEqual(kwargs, {})
def _log_entry_mapping_to_pb(mapping): """Helper for :meth:`write_entries`, et aliae Ideally, would use a function from :mod:`protobuf.json_format`, but the right one isn't public. See: https://github.com/google/protobuf/issues/1351 """ # pylint: disable=too-many-branches entry_pb = LogEntry() optional_scalar_keys = { 'logName': 'log_name', 'insertId': 'insert_id', 'textPayload': 'text_payload', } for key, pb_name in optional_scalar_keys.items(): if key in mapping: setattr(entry_pb, pb_name, mapping[key]) if 'resource' in mapping: entry_pb.resource.type = mapping['resource']['type'] if 'severity' in mapping: severity = mapping['severity'] if isinstance(severity, str): severity = LogSeverity.Value(severity) entry_pb.severity = severity if 'timestamp' in mapping: timestamp = _datetime_to_pb_timestamp(mapping['timestamp']) entry_pb.timestamp.CopyFrom(timestamp) if 'labels' in mapping: for key, value in mapping['labels'].items(): entry_pb.labels[key] = value if 'jsonPayload' in mapping: for key, value in mapping['jsonPayload'].items(): entry_pb.json_payload[key] = value if 'protoPayload' in mapping: Parse(json.dumps(mapping['protoPayload']), entry_pb.proto_payload) if 'httpRequest' in mapping: _http_request_mapping_to_pb( mapping['httpRequest'], entry_pb.http_request) if 'operation' in mapping: _log_operation_mapping_to_pb( mapping['operation'], entry_pb.operation) return entry_pb
def _log_entry_mapping_to_pb(mapping): """Helper for :meth:`write_entries`, et aliae Performs "impedance matching" between the protobuf attrs and the keys expected in the JSON API. """ # pylint: disable=too-many-branches entry_pb = LogEntry() optional_scalar_keys = { 'logName': 'log_name', 'insertId': 'insert_id', 'textPayload': 'text_payload', } for key, pb_name in optional_scalar_keys.items(): if key in mapping: setattr(entry_pb, pb_name, mapping[key]) if 'resource' in mapping: entry_pb.resource.type = mapping['resource']['type'] if 'severity' in mapping: severity = mapping['severity'] if isinstance(severity, str): severity = LogSeverity.Value(severity) entry_pb.severity = severity if 'timestamp' in mapping: timestamp = _datetime_to_pb_timestamp(mapping['timestamp']) entry_pb.timestamp.CopyFrom(timestamp) if 'labels' in mapping: for key, value in mapping['labels'].items(): entry_pb.labels[key] = value if 'jsonPayload' in mapping: for key, value in mapping['jsonPayload'].items(): entry_pb.json_payload[key] = value if 'protoPayload' in mapping: Parse(json.dumps(mapping['protoPayload']), entry_pb.proto_payload) if 'httpRequest' in mapping: _http_request_mapping_to_pb(mapping['httpRequest'], entry_pb.http_request) if 'operation' in mapping: _log_operation_mapping_to_pb(mapping['operation'], entry_pb.operation) return entry_pb
def _callFUT(self, when): from gcloud._helpers import _datetime_to_pb_timestamp return _datetime_to_pb_timestamp(when)
def test_list_entries_with_extra_properties(self): from datetime import datetime from google.logging.type.log_severity_pb2 import WARNING from gcloud._testing import _GAXPageIterator from gcloud._helpers import UTC from gcloud._helpers import _datetime_to_rfc3339 from gcloud._helpers import _datetime_to_pb_timestamp NOW = datetime.utcnow().replace(tzinfo=UTC) SIZE = 23 TOKEN = 'TOKEN' NEW_TOKEN = 'NEW_TOKEN' PAYLOAD = {'message': 'MESSAGE', 'weather': 'sunny'} SEVERITY = 'WARNING' LABELS = { 'foo': 'bar', } IID = 'IID' request = _HTTPRequestPB() operation = _LogEntryOperationPB() EXTRAS = { 'severity': WARNING, 'labels': LABELS, 'insert_id': IID, 'http_request': request, 'operation': operation, } ENTRY = _LogEntryPB(self.LOG_NAME, proto_payload=PAYLOAD, **EXTRAS) ENTRY.resource.labels['foo'] = 'bar' ENTRY.timestamp = _datetime_to_pb_timestamp(NOW) response = _GAXPageIterator([ENTRY], NEW_TOKEN) gax_api = _GAXLoggingAPI(_list_log_entries_response=response) api = self._makeOne(gax_api) entries, next_token = api.list_entries( [self.PROJECT], page_size=SIZE, page_token=TOKEN) self.assertEqual(len(entries), 1) entry = entries[0] self.assertIsInstance(entry, dict) self.assertEqual(entry['logName'], self.LOG_NAME) self.assertEqual(entry['resource'], {'type': 'global', 'labels': {'foo': 'bar'}}) self.assertEqual(entry['protoPayload'], PAYLOAD) self.assertEqual(entry['severity'], SEVERITY) self.assertEqual(entry['labels'], LABELS) self.assertEqual(entry['insertId'], IID) self.assertEqual(entry['timestamp'], _datetime_to_rfc3339(NOW)) EXPECTED_REQUEST = { 'requestMethod': request.request_method, 'requestUrl': request.request_url, 'status': request.status, 'requestSize': request.request_size, 'responseSize': request.response_size, 'referer': request.referer, 'userAgent': request.user_agent, 'remoteIp': request.remote_ip, 'cacheHit': request.cache_hit, } self.assertEqual(entry['httpRequest'], EXPECTED_REQUEST) EXPECTED_OPERATION = { 'producer': operation.producer, 'id': operation.id, 'first': operation.first, 'last': operation.last, } self.assertEqual(entry['operation'], EXPECTED_OPERATION) self.assertEqual(next_token, NEW_TOKEN) projects, filter_, order_by, page_size, options = ( gax_api._list_log_entries_called_with) self.assertEqual(projects, [self.PROJECT]) self.assertEqual(filter_, '') self.assertEqual(order_by, '') self.assertEqual(page_size, SIZE) self.assertEqual(options.page_token, TOKEN)
def _make_timestamp(): from datetime import datetime from gcloud._helpers import UTC from gcloud._helpers import _datetime_to_pb_timestamp NOW = datetime.utcnow().replace(tzinfo=UTC) return _datetime_to_pb_timestamp(NOW)
def _callFUT(self, when): from gcloud._helpers import _datetime_to_pb_timestamp return _datetime_to_pb_timestamp(when)
def test_update(self): import datetime from google.longrunning import operations_pb2 from gcloud.operation import Operation from google.protobuf.any_pb2 import Any from gcloud._helpers import _datetime_to_pb_timestamp from gcloud.bigtable._generated import (instance_pb2 as data_v2_pb2) from gcloud.bigtable._generated import (bigtable_instance_admin_pb2 as messages_v2_pb2) from gcloud.bigtable._testing import _FakeStub from gcloud.bigtable.cluster import _UPDATE_CLUSTER_METADATA_URL NOW = datetime.datetime.utcnow() NOW_PB = _datetime_to_pb_timestamp(NOW) SERVE_NODES = 81 client = _Client(self.PROJECT) instance = _Instance(self.INSTANCE_ID, client) cluster = self._makeOne(self.CLUSTER_ID, instance, serve_nodes=SERVE_NODES) # Create request_pb request_pb = _ClusterPB( name=self.CLUSTER_NAME, serve_nodes=SERVE_NODES, ) # Create response_pb OP_ID = 5678 OP_NAME = ( 'operations/projects/%s/instances/%s/clusters/%s/operations/%d' % (self.PROJECT, self.INSTANCE_ID, self.CLUSTER_ID, OP_ID)) metadata = messages_v2_pb2.UpdateClusterMetadata(request_time=NOW_PB) response_pb = operations_pb2.Operation( name=OP_NAME, metadata=Any(type_url=_UPDATE_CLUSTER_METADATA_URL, value=metadata.SerializeToString())) # Patch the stub used by the API method. client._instance_stub = stub = _FakeStub(response_pb) result = cluster.update() self.assertTrue(isinstance(result, Operation)) self.assertEqual(result.name, OP_NAME) self.assertTrue(result.target is cluster) self.assertTrue(result.client is client) self.assertIsInstance(result.pb_metadata, messages_v2_pb2.UpdateClusterMetadata) self.assertEqual(result.pb_metadata.request_time, NOW_PB) self.assertEqual(result.metadata, {'request_type': 'UpdateCluster'}) self.assertEqual(len(stub.method_calls), 1) api_name, args, kwargs = stub.method_calls[0] self.assertEqual(api_name, 'UpdateCluster') request_pb, = args self.assertTrue(isinstance(request_pb, data_v2_pb2.Cluster)) self.assertEqual(request_pb.name, self.CLUSTER_NAME) self.assertEqual(request_pb.serve_nodes, SERVE_NODES) self.assertEqual(kwargs, {})