def test_w_struct_value(self): from google.protobuf.struct_pb2 import Value STRING = 'STRING' PI = 3.1415926 value_pb = Value() value_pb.struct_value.fields['string'].string_value = STRING value_pb.struct_value.fields['bool'].bool_value = True value_pb.struct_value.fields['number'].number_value = PI self.assertEqual(self._callFUT(value_pb), {'string': STRING, 'bool': True, 'number': PI})
def test_w_empty_value(self): from google.protobuf.struct_pb2 import Value from google.cloud.spanner_v1 import Type from google.cloud.spanner_v1 import TypeCode field_type = Type(code=TypeCode.STRING) value_pb = Value() with self.assertRaises(ValueError): self._callFUT(value_pb, field_type)
def test_create_grpc_response_jsondata(): user_model = UserObject() request_data = np.array([[5, 6, 7]]) datadef = scu.array_to_grpc_datadef("ndarray", request_data) request = prediction_pb2.SeldonMessage(data=datadef) raw_response = {"output": "data"} sm = scu.construct_response(user_model, True, request, raw_response) assert sm.data.WhichOneof("data_oneof") == None emptyValue = Value() assert sm.jsonData != emptyValue
def test_w_bool(self): from google.protobuf.struct_pb2 import Value from google.cloud.spanner_v1 import Type from google.cloud.spanner_v1 import TypeCode VALUE = True field_type = Type(code=TypeCode.BOOL) value_pb = Value(bool_value=VALUE) self.assertEqual(self._callFUT(value_pb, field_type), VALUE)
def test__update_state_metadata(self): from google.longrunning import operations_pb2 from google.protobuf.any_pb2 import Any from google.protobuf.struct_pb2 import Value from google.cloud._testing import _Monkey from google.cloud import operation as MUT operation = self._make_one(None, None) self.assertIsNone(operation.metadata) val_pb = Value(number_value=1337) type_url = 'type.googleapis.com/%s' % (Value.DESCRIPTOR.full_name, ) val_any = Any(type_url=type_url, value=val_pb.SerializeToString()) operation_pb = operations_pb2.Operation(metadata=val_any) with _Monkey(MUT, _TYPE_URL_MAP={type_url: Value}): operation._update_state(operation_pb) self.assertEqual(operation.metadata, val_pb)
def test_w_array_empty(self): from google.protobuf.struct_pb2 import Value, ListValue from google.cloud.spanner_v1 import Type from google.cloud.spanner_v1 import TypeCode field_type = Type(code=TypeCode.ARRAY, array_element_type=Type(code=TypeCode.INT64)) value_pb = Value(list_value=ListValue(values=[])) self.assertEqual(self._callFUT(value_pb, field_type), [])
def test_create_rest_response_jsondata(): user_model = UserObject() request_data = np.array([[5, 6, 7]]) datadef = scu.array_to_rest_datadef("ndarray", request_data) json_request = { "jsonData": datadef } raw_response = {"output": "data"} json_response = scu.construct_response_json(user_model, True, json_request, raw_response) assert "data" not in json_response emptyValue = Value() assert json_response["jsonData"] != emptyValue
def test_create_grpc_response_customdata(): user_model = UserObject() request_data = np.array([[5, 6, 7]]) datadef = scu.array_to_grpc_datadef("ndarray", request_data) request = prediction_pb2.SeldonMessage(data=datadef) raw_response = any_pb2.Any(value=b"testdata") sm = scu.construct_response(user_model, True, request, raw_response) assert sm.data.WhichOneof("data_oneof") is None emptyValue = Value() assert sm.customData != emptyValue
def test_w_unknown_type(self): from google.protobuf.struct_pb2 import Value from google.cloud.spanner_v1 import Type from google.cloud.spanner_v1 import TypeCode field_type = Type(code=TypeCode.TYPE_CODE_UNSPECIFIED) value_pb = Value(string_value="Borked") with self.assertRaises(ValueError): self._callFUT(value_pb, field_type)
def test_w_float(self): from google.protobuf.struct_pb2 import Value from google.cloud.spanner_v1 import Type from google.cloud.spanner_v1 import TypeCode VALUE = 3.14159 field_type = Type(code=TypeCode.FLOAT64) value_pb = Value(number_value=VALUE) self.assertEqual(self._callFUT(value_pb, field_type), VALUE)
def get_prediction(instance): logging.info('Sending prediction request to AI Platform ...') try: pb_instance = json_format.ParseDict(instance, Value()) response = aip_client.predict(endpoint=aip_endpoint_name, instances=[pb_instance]) return list(response.predictions[0]) except Exception as err: logging.error(f'Prediction request failed: {type(err)}: {err}') return None
def test_w_numeric(self): import decimal from google.protobuf.struct_pb2 import Value from google.cloud.spanner_v1.proto.type_pb2 import Type, NUMERIC VALUE = decimal.Decimal("99999999999999999999999999999.999999999") field_type = Type(code=NUMERIC) value_pb = Value(string_value=str(VALUE)) self.assertEqual(self._callFUT(value_pb, field_type), VALUE)
def _merge_struct(lhs, rhs, type_): """Helper for '_merge_by_type'.""" fields = type_.struct_type.fields lhs, rhs = list(lhs.list_value.values), list(rhs.list_value.values) # Sanity check: If either list is empty, short-circuit. # This is effectively a no-op. if not len(lhs) or not len(rhs): return Value(list_value=ListValue(values=(lhs + rhs))) candidate_type = fields[len(lhs) - 1].type first = rhs.pop(0) if first.HasField( "null_value") or candidate_type.code in _UNMERGEABLE_TYPES: lhs.append(first) else: last = lhs.pop() lhs.append(_merge_by_type(last, first, candidate_type)) return Value(list_value=ListValue(values=lhs + rhs))
def test_w_bytes(self): from google.protobuf.struct_pb2 import Value from google.cloud.spanner_v1 import Type from google.cloud.spanner_v1 import TypeCode VALUE = b"Value" field_type = Type(code=TypeCode.BYTES) value_pb = Value(string_value=VALUE) self.assertEqual(self._callFUT(value_pb, field_type), VALUE)
def test_w_int(self): from google.protobuf.struct_pb2 import Value from google.cloud.spanner_v1 import Type from google.cloud.spanner_v1 import TypeCode VALUE = 12345 field_type = Type(code=TypeCode.INT64) value_pb = Value(string_value=str(VALUE)) self.assertEqual(self._callFUT(value_pb, field_type), VALUE)
def test_write_entries_multiple(self): from google.protobuf.struct_pb2 import Struct from google.protobuf.struct_pb2 import Value json_payload = {'payload': 'PAYLOAD', 'type': 'json'} json_struct_pb = Struct(fields={ key: Value(string_value=value) for key, value in json_payload.items() }) self._write_entries_multiple_helper(json_payload, json_struct_pb)
def test_list_entries_with_paging(self): from google.protobuf.struct_pb2 import Struct from google.protobuf.struct_pb2 import Value payload = {'message': 'MESSAGE', 'weather': 'sunny'} struct_pb = Struct(fields={ key: Value(string_value=value) for key, value in payload.items() }) self._list_entries_with_paging_helper(payload, struct_pb)
def test_w_date(self): import datetime from google.protobuf.struct_pb2 import Value from google.cloud.proto.spanner.v1.type_pb2 import Type, DATE VALUE = datetime.date.today() field_type = Type(code=DATE) value_pb = Value(string_value=VALUE.isoformat()) self.assertEqual(self._callFUT(value_pb, field_type), VALUE)
def test_log_proto_explicit(self): import datetime from google.cloud.logging.resource import Resource from google.cloud.logging.entries import ProtobufEntry from google.protobuf.struct_pb2 import Struct from google.protobuf.struct_pb2 import Value message = Struct(fields={"foo": Value(bool_value=True)}) LABELS = {"foo": "bar", "baz": "qux"} IID = "IID" SEVERITY = "CRITICAL" METHOD = "POST" URI = "https://api.example.com/endpoint" STATUS = "500" TRACE = "12345678-1234-5678-1234-567812345678" SPANID = "000000000000004a" REQUEST = { "requestMethod": METHOD, "requestUrl": URI, "status": STATUS } TIMESTAMP = datetime.datetime(2016, 12, 31, 0, 1, 2, 999999) RESOURCE = Resource(type="gae_app", labels={ "module_id": "default", "version_id": "test" }) ENTRY = ProtobufEntry( payload=message, labels=LABELS, insert_id=IID, severity=SEVERITY, http_request=REQUEST, timestamp=TIMESTAMP, resource=RESOURCE, trace=TRACE, span_id=SPANID, trace_sampled=True, ) client = _Client(project=self.PROJECT, connection=_make_credentials()) logger = _Logger() batch = self._make_one(logger, client=client) batch.log_proto( message, labels=LABELS, insert_id=IID, severity=SEVERITY, http_request=REQUEST, timestamp=TIMESTAMP, resource=RESOURCE, trace=TRACE, span_id=SPANID, trace_sampled=True, ) self.assertEqual(batch.entries, [ENTRY])
def test_commit_w_alternate_client(self): import json from google.protobuf.json_format import MessageToJson from google.protobuf.struct_pb2 import Struct from google.protobuf.struct_pb2 import Value from google.cloud.logging.logger import Logger from google.cloud.logging.entries import _GLOBAL_RESOURCE TEXT = "This is the entry text" STRUCT = {"message": TEXT, "weather": "partly cloudy"} message = Struct(fields={"foo": Value(bool_value=True)}) DEFAULT_LABELS = {"foo": "spam"} LABELS = {"foo": "bar", "baz": "qux"} SEVERITY = "CRITICAL" METHOD = "POST" URI = "https://api.example.com/endpoint" STATUS = "500" REQUEST = { "requestMethod": METHOD, "requestUrl": URI, "status": STATUS } client1 = _Client(project=self.PROJECT) client2 = _Client(project=self.PROJECT) api = client2.logging_api = _DummyLoggingAPI() logger = Logger("logger_name", client1, labels=DEFAULT_LABELS) ENTRIES = [ { "textPayload": TEXT, "labels": LABELS, "resource": _GLOBAL_RESOURCE._to_dict(), }, { "jsonPayload": STRUCT, "severity": SEVERITY, "resource": _GLOBAL_RESOURCE._to_dict(), }, { "protoPayload": json.loads(MessageToJson(message)), "httpRequest": REQUEST, "resource": _GLOBAL_RESOURCE._to_dict(), }, ] batch = self._make_one(logger, client=client1) batch.log_text(TEXT, labels=LABELS) batch.log_struct(STRUCT, severity=SEVERITY) batch.log_proto(message, http_request=REQUEST) batch.commit(client=client2) self.assertEqual(list(batch.entries), []) self.assertEqual( api._write_entries_called_with, (ENTRIES, logger.full_name, None, DEFAULT_LABELS), )
def to_value(self: Message) -> Value: """Converts a message type to a :class:`~google.protobuf.struct_pb2.Value` object. Args: message: the message to convert Returns: the message as a :class:`~google.protobuf.struct_pb2.Value` object """ tmp_dict = json_format.MessageToDict(self._pb) return json_format.ParseDict(tmp_dict, Value())
def test_log_proto_defaults(self): from google.protobuf.struct_pb2 import Struct from google.protobuf.struct_pb2 import Value message = Struct(fields={'foo': Value(bool_value=True)}) client = _Client(project=self.PROJECT, connection=_make_credentials()) logger = _Logger() batch = self._make_one(logger, client=client) batch.log_proto(message) self.assertEqual(batch.entries, [('proto', message, None, None, None, None, None)])
def test_log_proto_defaults(self): from google.protobuf.struct_pb2 import Struct, Value message = Struct(fields={'foo': Value(bool_value=True)}) connection = _Connection() CLIENT = _Client(project=self.PROJECT, connection=connection) logger = _Logger() batch = self._makeOne(logger, client=CLIENT) batch.log_proto(message) self.assertEqual(len(connection._requested), 0) self.assertEqual(batch.entries, [('proto', message, None, None, None, None)])
def test_w_float_str(self): from google.protobuf.struct_pb2 import Value from google.cloud.spanner_v1 import Type from google.cloud.spanner_v1 import TypeCode VALUE = "3.14159" field_type = Type(code=TypeCode.FLOAT64) value_pb = Value(string_value=VALUE) expected_value = 3.14159 self.assertEqual(self._callFUT(value_pb, field_type), expected_value)
def update_finding(source_name): # [START update_finding] from google.cloud import securitycenter from google.protobuf.struct_pb2 import Value from google.protobuf import field_mask_pb2 from google.protobuf.timestamp_pb2 import Timestamp client = securitycenter.SecurityCenterClient() # Only update the specific source property and event_time. event_time # is required for updates. field_mask = field_mask_pb2.FieldMask( paths=["source_properties.s_value", "event_time"]) value = Value() value.string_value = "new_string" # Set the update time to Now. This must be some time greater then the # event_time on the original finding. now_proto = Timestamp() now_proto.GetCurrentTime() # source_name is the resource path for a source that has been # created previously (you can use list_sources to find a specific one). # Its format is: # source_name = "organizations/{organization_id}/sources/{source_id}" # e.g.: # source_name = "organizations/111122222444/sources/1234" finding_name = "{}/findings/samplefindingid2".format(source_name) updated_finding = client.update_finding( { "name": finding_name, "source_properties": { "s_value": value }, "event_time": now_proto, }, update_mask=field_mask, ) print("New Source properties: {}, Event Time {}".format( updated_finding.source_properties, updated_finding.event_time.ToDatetime()))
def _merge_float64(lhs, rhs, type_): # pylint: disable=unused-argument """Helper for '_merge_by_type'.""" lhs_kind = lhs.WhichOneof("kind") if lhs_kind == "string_value": return Value(string_value=lhs.string_value + rhs.string_value) rhs_kind = rhs.WhichOneof("kind") array_continuation = (lhs_kind == "number_value" and rhs_kind == "string_value" and rhs.string_value == "") if array_continuation: return lhs raise Unmergeable(lhs, rhs, type_)
def create_training_pipeline_custom_job_sample( project: str, display_name: str, model_display_name: str, container_image_uri: str, base_output_directory_prefix: str, location: str = "us-central1", api_endpoint: str = "us-central1-aiplatform.googleapis.com", ): client_options = {"api_endpoint": api_endpoint} # Initialize client that will be used to create and send requests. # This client only needs to be created once, and can be reused for multiple requests. client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) training_task_inputs_dict = { "workerPoolSpecs": [ { "replicaCount": 1, "machineSpec": {"machineType": "n1-standard-4"}, "containerSpec": { # A working docker image can be found at gs://cloud-samples-data/ai-platform/mnist_tfrecord/custom_job "imageUri": container_image_uri, "args": [ # AIP_MODEL_DIR is set by the service according to baseOutputDirectory. "--model_dir=$(AIP_MODEL_DIR)", ], }, } ], "baseOutputDirectory": { # The GCS location for outputs must be accessible by the project's AI Platform service account. "output_uri_prefix": base_output_directory_prefix }, } training_task_inputs = json_format.ParseDict(training_task_inputs_dict, Value()) training_task_definition = "gs://google-cloud-aiplatform/schema/trainingjob/definition/custom_task_1.0.0.yaml" image_uri = "gcr.io/cloud-aiplatform/prediction/tf-cpu.1-15:latest" training_pipeline = { "display_name": display_name, "training_task_definition": training_task_definition, "training_task_inputs": training_task_inputs, "model_to_upload": { "display_name": model_display_name, "container_spec": {"image_uri": image_uri}, }, } parent = f"projects/{project}/locations/{location}" response = client.create_training_pipeline( parent=parent, training_pipeline=training_pipeline ) print("response:", response)
def test_log_proto_w_explicit_client_labels_severity_httpreq(self): import json from google.protobuf.json_format import MessageToJson from google.protobuf.struct_pb2 import Struct from google.protobuf.struct_pb2 import Value message = Struct(fields={'foo': Value(bool_value=True)}) DEFAULT_LABELS = {'foo': 'spam'} LABELS = {'foo': 'bar', 'baz': 'qux'} IID = 'IID' SEVERITY = 'CRITICAL' METHOD = 'POST' URI = 'https://api.example.com/endpoint' STATUS = '500' REQUEST = { 'requestMethod': METHOD, 'requestUrl': URI, 'status': STATUS, } ENTRIES = [{ 'logName': 'projects/%s/logs/%s' % (self.PROJECT, self.LOGGER_NAME), 'protoPayload': json.loads(MessageToJson(message)), 'resource': { 'type': 'global', 'labels': {}, }, 'labels': LABELS, 'insertId': IID, 'severity': SEVERITY, 'httpRequest': REQUEST, }] client1 = _Client(self.PROJECT) client2 = _Client(self.PROJECT) api = client2.logging_api = _DummyLoggingAPI() logger = self._make_one(self.LOGGER_NAME, client=client1, labels=DEFAULT_LABELS) logger.log_proto(message, client=client2, labels=LABELS, insert_id=IID, severity=SEVERITY, http_request=REQUEST) self.assertEqual(api._write_entries_called_with, (ENTRIES, None, None, None))
def test_w_json(self): import json from google.protobuf.struct_pb2 import Value from google.cloud.spanner_v1 import Type from google.cloud.spanner_v1 import TypeCode VALUE = {"id": 27863, "Name": "Anamika"} str_repr = json.dumps(VALUE, sort_keys=True, separators=(",", ":")) field_type = Type(code=TypeCode.JSON) value_pb = Value(string_value=str_repr) self.assertEqual(self._callFUT(value_pb, field_type), VALUE) VALUE = None str_repr = json.dumps(VALUE, sort_keys=True, separators=(",", ":")) field_type = Type(code=TypeCode.JSON) value_pb = Value(string_value=str_repr) self.assertEqual(self._callFUT(value_pb, field_type), {})
def test_commit_w_alternate_client(self): import json from google.protobuf.json_format import MessageToJson from google.protobuf.struct_pb2 import Struct from google.protobuf.struct_pb2 import Value from google.cloud.logging.logger import Logger TEXT = 'This is the entry text' STRUCT = {'message': TEXT, 'weather': 'partly cloudy'} message = Struct(fields={'foo': Value(bool_value=True)}) DEFAULT_LABELS = {'foo': 'spam'} LABELS = { 'foo': 'bar', 'baz': 'qux', } SEVERITY = 'CRITICAL' METHOD = 'POST' URI = 'https://api.example.com/endpoint' STATUS = '500' REQUEST = { 'requestMethod': METHOD, 'requestUrl': URI, 'status': STATUS, } client1 = _Client(project=self.PROJECT) client2 = _Client(project=self.PROJECT) api = client2.logging_api = _DummyLoggingAPI() logger = Logger('logger_name', client1, labels=DEFAULT_LABELS) RESOURCE = {'type': 'global'} ENTRIES = [ { 'textPayload': TEXT, 'labels': LABELS }, { 'jsonPayload': STRUCT, 'severity': SEVERITY }, { 'protoPayload': json.loads(MessageToJson(message)), 'httpRequest': REQUEST }, ] batch = self._make_one(logger, client=client1) batch.log_text(TEXT, labels=LABELS) batch.log_struct(STRUCT, severity=SEVERITY) batch.log_proto(message, http_request=REQUEST) batch.commit(client=client2) self.assertEqual(list(batch.entries), []) self.assertEqual(api._write_entries_called_with, (ENTRIES, logger.full_name, RESOURCE, DEFAULT_LABELS))