def build_messages( self, payloads: List[Union[str, dict]]) -> List[Intent.Message]: messages = [] for payload in payloads: if isinstance(payload, str): message = Intent.Message( text=Intent.Message.Text(text=[payload.strip()]), platform=self.platform, ) elif isinstance(payload, list): message = Intent.Message( text=Intent.Message.Text( text=[text.strip() for text in payload]), platform=self.platform, ) elif isinstance(payload, dict): payload_struct = Struct() if 'payload' in payload: payload_struct.update({ self.PLATFORMS[self.platform].lower(): payload['payload'] }) elif 'choice_buttons' in payload: _payload = self.build_choice_buttons( payload['choice_buttons']) payload_struct.update( {self.PLATFORMS[self.platform].lower(): _payload}) else: payload_struct.update(payload) message = Intent.Message( payload=payload_struct, platform=self.platform, ) else: raise AttributeError(f'Invalid payload "{payload}"') messages.append(message) return messages
def _execute_update_helper(self, count=0): from google.protobuf.struct_pb2 import Struct from google.cloud.spanner_v1.proto.result_set_pb2 import ( ResultSet, ResultSetStats, ) from google.cloud.spanner_v1.proto.transaction_pb2 import TransactionSelector from google.cloud.spanner_v1._helpers import _make_value_pb MODE = 2 # PROFILE stats_pb = ResultSetStats(row_count_exact=1) database = _Database() api = database.spanner_api = self._make_spanner_api() api.execute_sql.return_value = ResultSet(stats=stats_pb) session = _Session(database) transaction = self._make_one(session) transaction._transaction_id = self.TRANSACTION_ID transaction._execute_sql_count = count row_count = transaction.execute_update( DML_QUERY_WITH_PARAM, PARAMS, PARAM_TYPES, query_mode=MODE ) self.assertEqual(row_count, 1) expected_transaction = TransactionSelector(id=self.TRANSACTION_ID) expected_params = Struct( fields={key: _make_value_pb(value) for (key, value) in PARAMS.items()} ) api.execute_sql.assert_called_once_with( self.SESSION_NAME, DML_QUERY_WITH_PARAM, transaction=expected_transaction, params=expected_params, param_types=PARAM_TYPES, query_mode=MODE, seqno=count, metadata=[("google-cloud-resource-prefix", database.name)], ) self.assertEqual(transaction._execute_sql_count, count + 1)
def deserialize_properties(props_struct: struct_pb2.Struct) -> Any: """ Deserializes a protobuf `struct_pb2.Struct` into a Python dictionary containing normal Python types. """ # Check out this link for details on what sort of types Protobuf is going to generate: # https://developers.google.com/protocol-buffers/docs/reference/python-generated # # We assume that we are deserializing properties that we got from a Resource RPC endpoint, # which has type `Struct` in our gRPC proto definition. if _special_sig_key in props_struct: if props_struct[_special_sig_key] == _special_asset_sig: # This is an asset. Re-hydrate this object into an Asset. if "path" in props_struct: return known_types.new_file_asset(props_struct["path"]) if "text" in props_struct: return known_types.new_string_asset(props_struct["text"]) if "uri" in props_struct: return known_types.new_remote_asset(props_struct["uri"]) raise AssertionError("Invalid asset encountered when unmarshaling resource property") elif props_struct[_special_sig_key] == _special_archive_sig: # This is an archive. Re-hydrate this object into an Archive. if "assets" in props_struct: return known_types.new_asset_archive(deserialize_property(props_struct["assets"])) if "path" in props_struct: return known_types.new_file_archive(props_struct["path"]) if "uri" in props_struct: return known_types.new_remote_archive(props_struct["uri"]) elif props_struct[_special_sig_key] == _special_secret_sig: raise AssertionError("this version of the Pulumi SDK does not support first-class secrets") raise AssertionError("Unrecognized signature when unmarshaling resource property") # Struct is duck-typed like a dictionary, so we can iterate over it in the normal ways. output = {} for k, v in list(props_struct.items()): value = deserialize_property(v) # We treat values that deserialize to "None" as if they don't exist. if value is not None: output[k] = value return output
def test_log_proto_defaults(self): import json from google.protobuf.json_format import MessageToJson from google.protobuf.struct_pb2 import Struct, Value message = Struct(fields={"foo": Value(bool_value=True)}) ENTRIES = [ { "logName": "projects/%s/logs/%s" % (self.PROJECT, self.LOGGER_NAME), "protoPayload": json.loads(MessageToJson(message)), "resource": {"type": "global", "labels": {}}, } ] client = _Client(self.PROJECT) api = client.logging_api = _DummyLoggingAPI() logger = self._make_one(self.LOGGER_NAME, client=client) logger.log_proto(message) self.assertEqual(api._write_entries_called_with, (ENTRIES, None, None, None))
def test_log_proto_w_explicit_client_labels_severity_httpreq(self): import json from google.protobuf.json_format import MessageToJson from google.protobuf.struct_pb2 import Struct, Value message = Struct(fields={'foo': Value(bool_value=True)}) DEFAULT_LABELS = {'foo': 'spam'} LABELS = {'foo': 'bar', 'baz': 'qux'} IID = 'IID' SEVERITY = 'CRITICAL' METHOD = 'POST' URI = 'https://api.example.com/endpoint' STATUS = '500' REQUEST = { 'requestMethod': METHOD, 'requestUrl': URI, 'status': STATUS, } ENTRIES = [{ 'logName': 'projects/%s/logs/%s' % ( self.PROJECT, self.LOGGER_NAME), 'protoPayload': json.loads(MessageToJson(message)), 'resource': { 'type': 'global', }, 'labels': LABELS, 'insertId': IID, 'severity': SEVERITY, 'httpRequest': REQUEST, }] client1 = _Client(self.PROJECT) client2 = _Client(self.PROJECT) api = client2.logging_api = _DummyLoggingAPI() logger = self._make_one(self.LOGGER_NAME, client=client1, labels=DEFAULT_LABELS) logger.log_proto(message, client=client2, labels=LABELS, insert_id=IID, severity=SEVERITY, http_request=REQUEST) self.assertEqual(api._write_entries_called_with, (ENTRIES, None, None, None))
def test_log_proto_explicit(self): import datetime from google.cloud.logging.resource import Resource from google.protobuf.struct_pb2 import Struct from google.protobuf.struct_pb2 import Value message = Struct(fields={'foo': Value(bool_value=True)}) LABELS = {'foo': 'bar', 'baz': 'qux'} IID = 'IID' SEVERITY = 'CRITICAL' METHOD = 'POST' URI = 'https://api.example.com/endpoint' STATUS = '500' TRACE = '12345678-1234-5678-1234-567812345678' SPANID = '000000000000004a' REQUEST = { 'requestMethod': METHOD, 'requestUrl': URI, 'status': STATUS, } TIMESTAMP = datetime.datetime(2016, 12, 31, 0, 1, 2, 999999) RESOURCE = Resource(type='gae_app', labels={ 'module_id': 'default', 'version_id': 'test', }) client = _Client(project=self.PROJECT, connection=_make_credentials()) logger = _Logger() batch = self._make_one(logger, client=client) batch.log_proto(message, labels=LABELS, insert_id=IID, severity=SEVERITY, http_request=REQUEST, timestamp=TIMESTAMP, resource=RESOURCE, trace=TRACE, span_id=SPANID) self.assertEqual(batch.entries, [('proto', message, LABELS, IID, SEVERITY, REQUEST, TIMESTAMP, RESOURCE, TRACE, SPANID)])
def test_context_mgr_success(self): import json from google.protobuf.json_format import MessageToJson from google.protobuf.struct_pb2 import Struct, Value from google.cloud.logging.logger import Logger TEXT = 'This is the entry text' STRUCT = {'message': TEXT, 'weather': 'partly cloudy'} message = Struct(fields={'foo': Value(bool_value=True)}) DEFAULT_LABELS = {'foo': 'spam'} LABELS = {'foo': 'bar', 'baz': 'qux'} SEVERITY = 'CRITICAL' METHOD = 'POST' URI = 'https://api.example.com/endpoint' STATUS = '500' REQUEST = { 'requestMethod': METHOD, 'requestUrl': URI, 'status': STATUS, } client = _Client(project=self.PROJECT) api = client.logging_api = _DummyLoggingAPI() logger = Logger('logger_name', client, labels=DEFAULT_LABELS) RESOURCE = { 'type': 'global', } ENTRIES = [ {'textPayload': TEXT, 'httpRequest': REQUEST}, {'jsonPayload': STRUCT, 'labels': LABELS}, {'protoPayload': json.loads(MessageToJson(message)), 'severity': SEVERITY}, ] batch = self._make_one(logger, client=client) with batch as other: other.log_text(TEXT, http_request=REQUEST) other.log_struct(STRUCT, labels=LABELS) other.log_proto(message, severity=SEVERITY) self.assertEqual(list(batch.entries), []) self.assertEqual(api._write_entries_called_with, (ENTRIES, logger.full_name, RESOURCE, DEFAULT_LABELS))
def test_context_mgr_failure(self): import datetime from google.protobuf.struct_pb2 import Struct from google.protobuf.struct_pb2 import Value TEXT = 'This is the entry text' STRUCT = {'message': TEXT, 'weather': 'partly cloudy'} LABELS = {'foo': 'bar', 'baz': 'qux'} IID = 'IID' SEVERITY = 'CRITICAL' METHOD = 'POST' URI = 'https://api.example.com/endpoint' STATUS = '500' REQUEST = { 'requestMethod': METHOD, 'requestUrl': URI, 'status': STATUS, } TIMESTAMP = datetime.datetime(2016, 12, 31, 0, 1, 2, 999999) message = Struct(fields={'foo': Value(bool_value=True)}) client = _Client(project=self.PROJECT) api = client.logging_api = _DummyLoggingAPI() logger = _Logger() UNSENT = [ ('text', TEXT, None, IID, None, None, TIMESTAMP), ('struct', STRUCT, None, None, SEVERITY, None, None), ('proto', message, LABELS, None, None, REQUEST, None), ] batch = self._make_one(logger, client=client) try: with batch as other: other.log_text(TEXT, insert_id=IID, timestamp=TIMESTAMP) other.log_struct(STRUCT, severity=SEVERITY) other.log_proto(message, labels=LABELS, http_request=REQUEST) raise _Bugout() except _Bugout: pass self.assertEqual(list(batch.entries), UNSENT) self.assertIsNone(api._write_entries_called_with)
def test_log_proto_w_implicit_client(self): import json from google.protobuf.json_format import MessageToJson from google.protobuf.struct_pb2 import Struct, Value message = Struct(fields={'foo': Value(bool_value=True)}) ENTRIES = [{ 'logName': 'projects/%s/logs/%s' % ( self.PROJECT, self.LOGGER_NAME), 'protoPayload': json.loads(MessageToJson(message)), 'resource': { 'type': 'global', }, }] client = _Client(self.PROJECT) api = client.logging_api = _DummyLoggingAPI() logger = self._make_one(self.LOGGER_NAME, client=client) logger.log_proto(message) self.assertEqual(api._write_entries_called_with, (ENTRIES, None, None, None))
def test_struct_converts_primitive_fields(self): converted = dict_to_protobuf( Struct, json.loads(""" { "field1": "value1", "field2": 3, "field3": 1.0, "field4": true, "field5": null } """)) s = Struct() s['field1'] = 'value1' s['field2'] = 3 s['field3'] = 1.0 s['field4'] = True s['field5'] = None assert converted == s
def test_log_proto_explicit(self): from google.protobuf.struct_pb2 import Struct, Value message = Struct(fields={'foo': Value(bool_value=True)}) LABELS = {'foo': 'bar', 'baz': 'qux'} IID = 'IID' SEVERITY = 'CRITICAL' METHOD = 'POST' URI = 'https://api.example.com/endpoint' STATUS = '500' REQUEST = { 'requestMethod': METHOD, 'requestUrl': URI, 'status': STATUS, } client = _Client(project=self.PROJECT, connection=object()) logger = _Logger() batch = self._make_one(logger, client=client) batch.log_proto(message, labels=LABELS, insert_id=IID, severity=SEVERITY, http_request=REQUEST) self.assertEqual(batch.entries, [('proto', message, LABELS, IID, SEVERITY, REQUEST)])
def to_value(value): if value is None: return Value(null_value=NULL_VALUE) elif isinstance(value, bool): # This check needs to happen before isinstance(value, int), # isinstance(value, int) returns True when value is bool. return Value(bool_value=value) elif isinstance(value, six.integer_types) or isinstance(value, float): return Value(number_value=value) elif isinstance(value, six.string_types) or isinstance( value, six.text_type): return Value(string_value=value) elif isinstance(value, dict): return Value(struct_value=Struct( fields={k: to_value(v) for k, v in value.items()})) elif isinstance(value, list): return Value(list_value=ListValue( values=[to_value(item) for item in value])) else: raise ValueError('Unsupported data type: {}'.format(type(value)))
def test_context_mgr_failure(self): import datetime from google.protobuf.struct_pb2 import Struct from google.protobuf.struct_pb2 import Value from google.cloud.logging import TextEntry from google.cloud.logging import StructEntry from google.cloud.logging import ProtobufEntry TEXT = "This is the entry text" STRUCT = {"message": TEXT, "weather": "partly cloudy"} LABELS = {"foo": "bar", "baz": "qux"} IID = "IID" SEVERITY = "CRITICAL" METHOD = "POST" URI = "https://api.example.com/endpoint" STATUS = "500" REQUEST = {"requestMethod": METHOD, "requestUrl": URI, "status": STATUS} TIMESTAMP = datetime.datetime(2016, 12, 31, 0, 1, 2, 999999) message = Struct(fields={"foo": Value(bool_value=True)}) client = _Client(project=self.PROJECT) api = client.logging_api = _DummyLoggingAPI() logger = _Logger() UNSENT = [ TextEntry(payload=TEXT, insert_id=IID, timestamp=TIMESTAMP), StructEntry(payload=STRUCT, severity=SEVERITY), ProtobufEntry(payload=message, labels=LABELS, http_request=REQUEST), ] batch = self._make_one(logger, client=client) try: with batch as other: other.log_text(TEXT, insert_id=IID, timestamp=TIMESTAMP) other.log_struct(STRUCT, severity=SEVERITY) other.log_proto(message, labels=LABELS, http_request=REQUEST) raise _Bugout() except _Bugout: pass self.assertEqual(list(batch.entries), UNSENT) self.assertIsNone(api._write_entries_called_with)
def test_commit_w_bound_client(self): import json from google.protobuf.json_format import MessageToJson from google.protobuf.struct_pb2 import Struct, Value TEXT = 'This is the entry text' STRUCT = {'message': TEXT, 'weather': 'partly cloudy'} message = Struct(fields={'foo': Value(bool_value=True)}) IID1 = 'IID1' IID2 = 'IID2' IID3 = 'IID3' RESOURCE = { 'type': 'global', } ENTRIES = [ { 'textPayload': TEXT, 'insertId': IID1 }, { 'jsonPayload': STRUCT, 'insertId': IID2 }, { 'protoPayload': json.loads(MessageToJson(message)), 'insertId': IID3 }, ] client = _Client(project=self.PROJECT) api = client.logging_api = _DummyLoggingAPI() logger = _Logger() batch = self._makeOne(logger, client=client) batch.log_text(TEXT, insert_id=IID1) batch.log_struct(STRUCT, insert_id=IID2) batch.log_proto(message, insert_id=IID3) batch.commit() self.assertEqual(list(batch.entries), []) self.assertEqual(api._write_entries_called_with, (ENTRIES, logger.path, RESOURCE, None))
def detect_intent(request): response = { 'status_code': 500, 'message': 'Failed to fetch data from dialogflow.', 'data': [] } try: request_json = request.get_json(silent=True, force=True) project_id = "build-agent-local" session_client = dialogflow.SessionsClient.from_service_account_json('key.json') payload = {'project_id': project_id, 'session_id': request_json.get("session_id")} struct_pb = Struct(fields={ key: Value(string_value=value) for key, value in payload.items() }) params = dialogflow.types.QueryParameters( time_zone="PST", payload=struct_pb) session = session_client.session_path(project_id, request_json.get("session_id")) text_input = dialogflow.types.TextInput( text=request_json.get("text"), language_code='en-US') query_input = dialogflow.types.QueryInput(text=text_input) df_response = session_client.detect_intent( session=session, query_input=query_input, query_params=params) # Convert proto object and serialize it to a json format string. json_obj = MessageToJson(df_response) result = json.loads(json_obj) response['status_code'] = 200 response['data'] = result response['message'] = "Successfully fetched response from dialogflow." except Exception as e: print(e) return jsonify(response)
def test_from_pb_w_unknown_metadata(self): from google.longrunning import operations_pb2 from google.protobuf.any_pb2 import Any from google.protobuf.json_format import ParseDict from google.protobuf.struct_pb2 import Struct from google.cloud._testing import _Monkey from google.cloud import operation as MUT type_url = 'type.googleapis.com/%s' % (Struct.DESCRIPTOR.full_name, ) client = _Client() meta = ParseDict({'foo': 'Bar'}, Struct()) metadata_pb = Any(type_url=type_url, value=meta.SerializeToString()) operation_pb = operations_pb2.Operation(name=self.OPERATION_NAME, metadata=metadata_pb) klass = self._get_target_class() with _Monkey(MUT, _TYPE_URL_MAP={type_url: Struct}): operation = klass.from_pb(operation_pb, client) self.assertEqual(operation.name, self.OPERATION_NAME) self.assertIs(operation.client, client) self.assertEqual(operation.metadata, meta) self.assertEqual(operation.caller_metadata, {})
def test_context_mgr_failure(self): from google.protobuf.struct_pb2 import Struct, Value TEXT = 'This is the entry text' STRUCT = {'message': TEXT, 'weather': 'partly cloudy'} LABELS = {'foo': 'bar', 'baz': 'qux'} IID = 'IID' SEVERITY = 'CRITICAL' METHOD = 'POST' URI = 'https://api.example.com/endpoint' STATUS = '500' REQUEST = { 'requestMethod': METHOD, 'requestUrl': URI, 'status': STATUS, } message = Struct(fields={'foo': Value(bool_value=True)}) conn = _Connection({}) CLIENT = _Client(project=self.PROJECT, connection=conn) logger = _Logger() UNSENT = [ ('text', TEXT, None, IID, None, None), ('struct', STRUCT, None, None, SEVERITY, None), ('proto', message, LABELS, None, None, REQUEST), ] batch = self._makeOne(logger, client=CLIENT) try: with batch as other: other.log_text(TEXT, insert_id=IID) other.log_struct(STRUCT, severity=SEVERITY) other.log_proto(message, labels=LABELS, http_request=REQUEST) raise _Bugout() except _Bugout: pass self.assertEqual(list(batch.entries), UNSENT) self.assertEqual(len(conn._requested), 0)
def test_log_proto_w_implicit_client(self): import json from google.protobuf.json_format import MessageToJson from google.protobuf.struct_pb2 import Struct, Value message = Struct(fields={'foo': Value(bool_value=True)}) conn = _Connection({}) client = _Client(self.PROJECT, conn) logger = self._makeOne(self.LOGGER_NAME, client=client) logger.log_proto(message) self.assertEqual(len(conn._requested), 1) req = conn._requested[0] SENT = { 'entries': [{ 'logName': 'projects/%s/logs/%s' % ( self.PROJECT, self.LOGGER_NAME), 'protoPayload': json.loads(MessageToJson(message)), 'resource': { 'type': 'global', }, }], } self.assertEqual(req['method'], 'POST') self.assertEqual(req['path'], '/entries:write') self.assertEqual(req['data'], SENT)
def test_commit_w_bound_client(self): import json from google.protobuf.json_format import MessageToJson from google.protobuf.struct_pb2 import Struct, Value TEXT = 'This is the entry text' STRUCT = {'message': TEXT, 'weather': 'partly cloudy'} message = Struct(fields={'foo': Value(bool_value=True)}) IID1 = 'IID1' IID2 = 'IID2' IID3 = 'IID3' conn = _Connection({}) CLIENT = _Client(project=self.PROJECT, connection=conn) logger = _Logger() SENT = { 'logName': logger.path, 'resource': { 'type': 'global', }, 'entries': [ {'textPayload': TEXT, 'insertId': IID1}, {'jsonPayload': STRUCT, 'insertId': IID2}, {'protoPayload': json.loads(MessageToJson(message)), 'insertId': IID3}, ], } batch = self._makeOne(logger, client=CLIENT) batch.log_text(TEXT, insert_id=IID1) batch.log_struct(STRUCT, insert_id=IID2) batch.log_proto(message, insert_id=IID3) batch.commit() self.assertEqual(list(batch.entries), []) self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'POST') self.assertEqual(req['path'], '/entries:write') self.assertEqual(req['data'], SENT)
def test_log_proto_w_timestamp(self): import json import datetime from google.protobuf.json_format import MessageToJson from google.protobuf.struct_pb2 import Struct, Value message = Struct(fields={'foo': Value(bool_value=True)}) TIMESTAMP = datetime.datetime(2016, 12, 31, 0, 1, 2, 999999) ENTRIES = [{ 'logName': 'projects/%s/logs/%s' % ( self.PROJECT, self.LOGGER_NAME), 'protoPayload': json.loads(MessageToJson(message)), 'timestamp': '2016-12-31T00:01:02.999999Z', 'resource': { 'type': 'global', }, }] client = _Client(self.PROJECT) api = client.logging_api = _DummyLoggingAPI() logger = self._make_one(self.LOGGER_NAME, client=client) logger.log_proto(message, timestamp=TIMESTAMP) self.assertEqual(api._write_entries_called_with, (ENTRIES, None, None, None))
def detectEvent(self, session_id, event_name, payload): session_client = dialogflow.SessionsClient() session = session_client.session_path(self._project_id, session_id) parameters = Struct(fields={'value': Value(string_value=payload)}) js = self.safeParseJson(payload) if hasattr(js, 'items'): for key, value in js.items(): nKey = self.normilizeKeyDialogflow(key) parameters[nKey] = value event_input = dialogflow.EventInput(name=event_name, language_code='ru-RU', parameters=parameters) query_input = dialogflow.QueryInput(event=event_input) try: response = session_client.detect_intent(session=session, query_input=query_input) self.handleDialogflowResponse(response) except: print("error detect event") self._isEndConversation = True if not self._isEndConversation: self.playSound(self._wake_sound_file) print('event conversation continue') self.stopDetectHotword() self.runDetectIntent(session_id) else: print('event conversation finished') self.stopDetectIntent() self.runDetectHotword()
def test_execute_sql_explicit(self): from google.protobuf.struct_pb2 import Struct, Value from google.cloud.spanner_v1.proto.type_pb2 import STRING SQL = 'SELECT first_name, age FROM citizens' database = self._make_database() session = self._make_one(database) session._session_id = 'DEADBEEF' params = Struct(fields={'foo': Value(string_value='bar')}) param_types = {'foo': STRING} with mock.patch( 'google.cloud.spanner_v1.session.Snapshot') as snapshot: found = session.execute_sql(SQL, params, param_types, 'PLAN') self.assertIs(found, snapshot().execute_sql.return_value) snapshot().execute_sql.assert_called_once_with( SQL, params, param_types, 'PLAN', )
def test_json_converts_single_object(self): s = Struct() s['field1'] = { 'key1': 'value1', 'key2': 3.0, 'key3': 1.0, 'key4': True, 'key5': None, } s['field2'] = 'key2' converted = protobuf_to_dict(s) assert (converted == json.loads(""" { "field1": { "key1": "value1", "key2": 3.0, "key3": 1.0, "key4": true, "key5": null }, "field2": "key2" } """))
def partition_query( self, sql, params=None, param_types=None, partition_size_bytes=None, max_partitions=None, *, retry=gapic_v1.method.DEFAULT, timeout=gapic_v1.method.DEFAULT, ): """Perform a ``PartitionQuery`` API request. :type sql: str :param sql: SQL query statement :type params: dict, {str -> column value} :param params: values for parameter replacement. Keys must match the names used in ``sql``. :type param_types: dict[str -> Union[dict, .types.Type]] :param param_types: (Optional) maps explicit types for one or more param values; required if parameters are passed. :type partition_size_bytes: int :param partition_size_bytes: (Optional) desired size for each partition generated. The service uses this as a hint, the actual partition size may differ. :type max_partitions: int :param max_partitions: (Optional) desired maximum number of partitions generated. The service uses this as a hint, the actual number of partitions may differ. :type retry: :class:`~google.api_core.retry.Retry` :param retry: (Optional) The retry settings for this request. :type timeout: float :param timeout: (Optional) The timeout for this request. :rtype: iterable of bytes :returns: a sequence of partition tokens :raises ValueError: for single-use snapshots, or if a transaction ID is already associated with the snapshot. """ if not self._multi_use: raise ValueError("Cannot use single-use snapshot.") if self._transaction_id is None: raise ValueError("Transaction not started.") if params is not None: if param_types is None: raise ValueError( "Specify 'param_types' when passing 'params'.") params_pb = Struct(fields={ key: _make_value_pb(value) for (key, value) in params.items() }) else: params_pb = Struct() database = self._session._database api = database.spanner_api metadata = _metadata_with_prefix(database.name) transaction = self._make_txn_selector() partition_options = PartitionOptions( partition_size_bytes=partition_size_bytes, max_partitions=max_partitions) request = PartitionQueryRequest( session=self._session.name, sql=sql, transaction=transaction, params=params_pb, param_types=param_types, partition_options=partition_options, ) trace_attributes = {"db.statement": sql} with trace_call( "CloudSpanner.PartitionReadWriteTransaction", self._session, trace_attributes, ): response = api.partition_query( request=request, metadata=metadata, retry=retry, timeout=timeout, ) return [partition.partition_token for partition in response.partitions]
def execute_sql( self, sql, params=None, param_types=None, query_mode=None, query_options=None, request_options=None, partition=None, retry=gapic_v1.method.DEFAULT, timeout=gapic_v1.method.DEFAULT, ): """Perform an ``ExecuteStreamingSql`` API request. :type sql: str :param sql: SQL query statement :type params: dict, {str -> column value} :param params: values for parameter replacement. Keys must match the names used in ``sql``. :type param_types: dict[str -> Union[dict, .types.Type]] :param param_types: (Optional) maps explicit types for one or more param values; required if parameters are passed. :type query_mode: :class:`~google.cloud.spanner_v1.types.ExecuteSqlRequest.QueryMode` :param query_mode: Mode governing return of results / query plan. See: `QueryMode <https://cloud.google.com/spanner/reference/rpc/google.spanner.v1#google.spanner.v1.ExecuteSqlRequest.QueryMode>`_. :type query_options: :class:`~google.cloud.spanner_v1.types.ExecuteSqlRequest.QueryOptions` or :class:`dict` :param query_options: (Optional) Query optimizer configuration to use for the given query. If a dict is provided, it must be of the same form as the protobuf message :class:`~google.cloud.spanner_v1.types.QueryOptions` :type request_options: :class:`google.cloud.spanner_v1.types.RequestOptions` :param request_options: (Optional) Common options for this request. If a dict is provided, it must be of the same form as the protobuf message :class:`~google.cloud.spanner_v1.types.RequestOptions`. :type partition: bytes :param partition: (Optional) one of the partition tokens returned from :meth:`partition_query`. :rtype: :class:`~google.cloud.spanner_v1.streamed.StreamedResultSet` :returns: a result set instance which can be used to consume rows. :type retry: :class:`~google.api_core.retry.Retry` :param retry: (Optional) The retry settings for this request. :type timeout: float :param timeout: (Optional) The timeout for this request. :raises ValueError: for reuse of single-use snapshots, or if a transaction ID is already pending for multiple-use snapshots. """ if self._read_request_count > 0: if not self._multi_use: raise ValueError("Cannot re-use single-use snapshot.") if self._transaction_id is None: raise ValueError("Transaction ID pending.") if params is not None: if param_types is None: raise ValueError( "Specify 'param_types' when passing 'params'.") params_pb = Struct(fields={ key: _make_value_pb(value) for key, value in params.items() }) else: params_pb = {} database = self._session._database metadata = _metadata_with_prefix(database.name) transaction = self._make_txn_selector() api = database.spanner_api # Query-level options have higher precedence than client-level and # environment-level options default_query_options = database._instance._client._query_options query_options = _merge_query_options(default_query_options, query_options) if type(request_options) == dict: request_options = RequestOptions(request_options) request = ExecuteSqlRequest( session=self._session.name, sql=sql, transaction=transaction, params=params_pb, param_types=param_types, query_mode=query_mode, partition_token=partition, seqno=self._execute_sql_count, query_options=query_options, request_options=request_options, ) restart = functools.partial( api.execute_streaming_sql, request=request, metadata=metadata, retry=retry, timeout=timeout, ) trace_attributes = {"db.statement": sql} iterator = _restart_on_unavailable( restart, request, "CloudSpanner.ReadWriteTransaction", self._session, trace_attributes, ) self._read_request_count += 1 self._execute_sql_count += 1 if self._multi_use: return StreamedResultSet(iterator, source=self) else: return StreamedResultSet(iterator)
def generic_command(client, args): params = json_format.Parse(args.params, Struct()) response = client.GenericCommand( magmad_pb2.GenericCommandParams(command=args.command, params=params), ) print(response)
def deserialize_properties(props_struct: struct_pb2.Struct, keep_unknowns: Optional[bool] = None) -> Any: """ Deserializes a protobuf `struct_pb2.Struct` into a Python dictionary containing normal Python types. """ # Check out this link for details on what sort of types Protobuf is going to generate: # https://developers.google.com/protocol-buffers/docs/reference/python-generated # # We assume that we are deserializing properties that we got from a Resource RPC endpoint, # which has type `Struct` in our gRPC proto definition. if _special_sig_key in props_struct: from .. import FileAsset, StringAsset, RemoteAsset, AssetArchive, FileArchive, RemoteArchive # pylint: disable=import-outside-toplevel if props_struct[_special_sig_key] == _special_asset_sig: # This is an asset. Re-hydrate this object into an Asset. if "path" in props_struct: return FileAsset(props_struct["path"]) if "text" in props_struct: return StringAsset(props_struct["text"]) if "uri" in props_struct: return RemoteAsset(props_struct["uri"]) raise AssertionError( "Invalid asset encountered when unmarshalling resource property" ) if props_struct[_special_sig_key] == _special_archive_sig: # This is an archive. Re-hydrate this object into an Archive. if "assets" in props_struct: return AssetArchive( deserialize_property(props_struct["assets"])) if "path" in props_struct: return FileArchive(props_struct["path"]) if "uri" in props_struct: return RemoteArchive(props_struct["uri"]) raise AssertionError( "Invalid archive encountered when unmarshalling resource property" ) if props_struct[_special_sig_key] == _special_secret_sig: return wrap_rpc_secret(deserialize_property(props_struct["value"])) if props_struct[_special_sig_key] == _special_resource_sig: return deserialize_resource(props_struct, keep_unknowns) raise AssertionError( "Unrecognized signature when unmarshalling resource property") # Struct is duck-typed like a dictionary, so we can iterate over it in the normal ways. Note # that if the struct had any secret properties, we push the secretness of the object up to us # since we can only set secret outputs on top level properties. output = {} for k, v in list(props_struct.items()): # Unilaterally skip properties considered internal by the Pulumi engine. # These don't actually contribute to the exposed shape of the object, do # not need to be passed back to the engine, and often will not match the # expected type we are deserializing into. # Keep "__provider" as it's the property name used by Python dynamic providers. if k.startswith("__") and k != "__provider": continue value = deserialize_property(v, keep_unknowns) # We treat values that deserialize to "None" as if they don't exist. if value is not None: output[k] = value return output
def _dict_to_struct(data): return Struct( fields={k: TestLogging._to_value(v) for k, v in data.items()})
def get_bento_service_metadata_pb(self): from bentoml.yatai.proto.repository_pb2 import BentoServiceMetadata bento_service_metadata = BentoServiceMetadata() bento_service_metadata.name = self.config["metadata"]["service_name"] bento_service_metadata.version = self.config["metadata"][ "service_version"] bento_service_metadata.created_at.FromDatetime( self.config["metadata"]["created_at"]) if "env" in self.config: if "setup_sh" in self.config["env"]: bento_service_metadata.env.setup_sh = self.config["env"][ "setup_sh"] if "conda_env" in self.config["env"]: bento_service_metadata.env.conda_env = dump_to_yaml_str( self.config["env"]["conda_env"]) if "pip_packages" in self.config["env"]: for pip_package in self.config["env"]["pip_packages"]: bento_service_metadata.env.pip_packages.append(pip_package) if "python_version" in self.config["env"]: bento_service_metadata.env.python_version = self.config["env"][ "python_version"] if "docker_base_image" in self.config["env"]: bento_service_metadata.env.docker_base_image = self.config[ "env"]["docker_base_image"] if "apis" in self.config: for api_config in self.config["apis"]: if 'handler_type' in api_config: # Convert handler type to input type for saved bundle created # before version 0.8.0 input_type = api_config.get('handler_type') elif 'input_type' in api_config: input_type = api_config.get('input_type') else: input_type = "unknown" if 'output_type' in api_config: output_type = api_config.get('output_type') else: output_type = "DefaultOutput" api_metadata = BentoServiceMetadata.BentoServiceApi( name=api_config["name"], docs=api_config["docs"], input_type=input_type, output_type=output_type, ) if "handler_config" in api_config: # Supports viewing API input config info for saved bundle created # before version 0.8.0 for k, v in api_config["handler_config"].items(): if k in {'mb_max_latency', 'mb_max_batch_size'}: setattr(api_metadata, k, v) else: api_metadata.input_config[k] = v else: if 'mb_max_latency' in api_config: api_metadata.mb_max_latency = api_config[ "mb_max_latency"] else: api_metadata.mb_max_latency = DEFAULT_MAX_LATENCY if 'mb_max_batch_size' in api_config: api_metadata.mb_max_batch_size = api_config[ "mb_max_batch_size"] else: api_metadata.mb_max_batch_size = DEFAULT_MAX_BATCH_SIZE if "input_config" in api_config: for k, v in api_config["input_config"].items(): api_metadata.input_config[k] = v if "output_config" in api_config: for k, v in api_config["output_config"].items(): api_metadata.output_config[k] = v api_metadata.batch = api_config.get("batch", False) bento_service_metadata.apis.extend([api_metadata]) if "artifacts" in self.config: for artifact_config in self.config["artifacts"]: artifact_metadata = BentoServiceMetadata.BentoArtifact() if "name" in artifact_config: artifact_metadata.name = artifact_config["name"] if "artifact_type" in artifact_config: artifact_metadata.artifact_type = artifact_config[ "artifact_type"] if "metadata" in artifact_config: if isinstance(artifact_config["metadata"], dict): s = Struct() s.update(artifact_config["metadata"]) artifact_metadata.metadata.CopyFrom(s) else: logger.warning( "Tried to get non-dictionary metadata for artifact " f"{artifact_metadata.name}. Ignoring metadata...") bento_service_metadata.artifacts.extend([artifact_metadata]) return bento_service_metadata
def access(self, resp, req, environ, request_time): super().access(resp, req, environ, request_time) # Ignore health check if environ['PATH_INFO'] == '/healthz': return # See gunicorn/glogging.py status = resp.status if isinstance(status, str): status = status.split(None, 1)[0] now = datetime.datetime.utcnow() level = logging.NOTSET message = { '@type': 'type.googleapis.com/google.appengine.logging.v1.RequestLog', 'ip': environ.get('REMOTE_ADDR'), 'startTime': format_time(now - request_time), 'endTime': format_time(now), 'latency': '%d.%06ds' % (request_time.seconds, request_time.microseconds), 'method': environ['REQUEST_METHOD'], 'resource': environ['PATH_INFO'], 'httpVersion': environ['SERVER_PROTOCOL'], 'status': status, 'responseSize': getattr(resp, 'sent', None), 'userAgent': environ.get('HTTP_USER_AGENT'), } request_log = environ.get(REQUEST_LOG_VARIABLE) if request_log: message['urlMapEntry'] = request_log.endpoint message['line'] = [ { 'time': format_time( datetime.datetime.utcfromtimestamp(record.created)), 'severity': record.levelname, 'logMessage': access_formatter.format(record), # The log viewer only wants real App Engine files, so we # can't put the actual file here. 'sourceLocation': None, } for record in request_log.lines ] level = max( (record.levelno for record in request_log.lines), default=logging.NOTSET, ) if level > logging.NOTSET: severity = logging.getLevelName(level) else: severity = None struct_pb = ParseDict(message, Struct()) self.cloud_logger.log_proto(struct_pb, severity=severity)