def _get_request_message(request_message, flask_request=request): if flask_request.method == 'GET' and len(flask_request.query_string) > 0: # This is a hack to make arrays of length 1 work with the parser. # for example experiment_ids%5B%5D=0 should be parsed to {experiment_ids: [0]} # but it gets parsed to {experiment_ids: 0} # but it doesn't. However, experiment_ids%5B0%5D=0 will get parsed to the right # result. query_string = re.sub('%5B%5D', '%5B0%5D', flask_request.query_string.decode("utf-8")) request_dict = parser.parse(query_string, normalized=True) parse_dict(request_dict, request_message) return request_message request_json = _get_request_json(flask_request) # Older clients may post their JSON double-encoded as strings, so the get_json # above actually converts it to a string. Therefore, we check this condition # (which we can tell for sure because any proper request should be a dictionary), # and decode it a second time. if isinstance(request_json, six.string_types): request_json = json.loads(request_json) # If request doesn't have json body then assume it's empty. if request_json is None: request_json = {} parse_dict(request_json, request_message) return request_message
def test_parse_legacy_experiment(): in_json = {"experiment_id": 123, "name": "name", "unknown": "field"} message = ProtoExperiment() parse_dict(in_json, message) experiment = Experiment.from_proto(message) assert experiment.experiment_id == "123" assert experiment.name == 'name' assert experiment.artifact_location == ''
def call_endpoint(host_creds, endpoint, method, json_body, response_proto): # Convert json string to json dictionary, to pass to requests if json_body: json_body = json.loads(json_body) if method == 'GET': response = http_request( host_creds=host_creds, endpoint=endpoint, method=method, params=json_body) else: response = http_request( host_creds=host_creds, endpoint=endpoint, method=method, json=json_body) response = verify_rest_response(response, endpoint) js_dict = json.loads(response.text) parse_dict(js_dict=js_dict, message=response_proto) return response_proto
def _call_endpoint(self, api, json_body): endpoint, method = _METHOD_TO_INFO[api] response_proto = api.Response() # Convert json string to json dictionary, to pass to requests if json_body: json_body = json.loads(json_body) host_creds = self.get_host_creds() response = http_request_safe(host_creds=host_creds, endpoint=endpoint, method=method, json=json_body) js_dict = json.loads(response.text) parse_dict(js_dict=js_dict, message=response_proto) return response_proto
def _call_endpoint(self, api, json_body): endpoint, method = _METHOD_TO_INFO[api] response_proto = api.Response() # Convert json string to json dictionary, to pass to requests if json_body: json_body = json.loads(json_body) response = http_request(endpoint=endpoint, method=method, json=json_body, **self.http_request_kwargs) js_dict = json.loads(response.text) if 'error_code' in js_dict: raise RestException(js_dict) parse_dict(js_dict=js_dict, message=response_proto) return response_proto
def _get_request_message(request_message, flask_request=request): from querystring_parser import parser if flask_request.method == "GET" and len(flask_request.query_string) > 0: # This is a hack to make arrays of length 1 work with the parser. # for example experiment_ids%5B%5D=0 should be parsed to {experiment_ids: [0]} # but it gets parsed to {experiment_ids: 0} # but it doesn't. However, experiment_ids%5B0%5D=0 will get parsed to the right # result. query_string = re.sub("%5B%5D", "%5B0%5D", flask_request.query_string.decode("utf-8")) request_dict = parser.parse(query_string, normalized=True) # Convert atomic values of repeated fields to lists before calling protobuf deserialization. # Context: We parse the parameter string into a dictionary outside of protobuf since # protobuf does not know how to read the query parameters directly. The query parser above # has no type information and hence any parameter that occurs exactly once is parsed as an # atomic value. Since protobuf requires that the values of repeated fields are lists, # deserialization will fail unless we do the fix below. for field in request_message.DESCRIPTOR.fields: if ( field.label == descriptor.FieldDescriptor.LABEL_REPEATED and field.name in request_dict ): if not isinstance(request_dict[field.name], list): request_dict[field.name] = [request_dict[field.name]] parse_dict(request_dict, request_message) return request_message request_json = _get_request_json(flask_request) # Older clients may post their JSON double-encoded as strings, so the get_json # above actually converts it to a string. Therefore, we check this condition # (which we can tell for sure because any proper request should be a dictionary), # and decode it a second time. if is_string_type(request_json): request_json = json.loads(request_json) # If request doesn't have json body then assume it's empty. if request_json is None: request_json = {} parse_dict(request_json, request_message) return request_message
def test_parse_dict_int_as_string_backcompat(): in_json = {"timestamp": "123"} message = ProtoMetric() parse_dict(in_json, message) experiment = Metric.from_proto(message) assert experiment.timestamp == 123
def test_message_to_json(): json_out = message_to_json( Experiment("123", "name", "arty", "active").to_proto()) assert json.loads(json_out) == { "experiment_id": "123", "name": "name", "artifact_location": "arty", "lifecycle_stage": "active", } original_proto_message = RegisteredModel( name="model_1", creation_timestamp=111, last_updated_timestamp=222, description="Test model", latest_versions=[ ModelVersion( name="mv-1", version="1", creation_timestamp=333, last_updated_timestamp=444, description="v 1", user_id="u1", current_stage="Production", source="A/B", run_id="9245c6ce1e2d475b82af84b0d36b52f4", status="READY", status_message=None, ), ModelVersion( name="mv-2", version="2", creation_timestamp=555, last_updated_timestamp=666, description="v 2", user_id="u2", current_stage="Staging", source="A/C", run_id="123", status="READY", status_message=None, ), ], ).to_proto() json_out = message_to_json(original_proto_message) json_dict = json.loads(json_out) assert json_dict == { "name": "model_1", "creation_timestamp": 111, "last_updated_timestamp": 222, "description": "Test model", "latest_versions": [ { "name": "mv-1", "version": "1", "creation_timestamp": 333, "last_updated_timestamp": 444, "current_stage": "Production", "description": "v 1", "user_id": "u1", "source": "A/B", "run_id": "9245c6ce1e2d475b82af84b0d36b52f4", "status": "READY", }, { "name": "mv-2", "version": "2", "creation_timestamp": 555, "last_updated_timestamp": 666, "current_stage": "Staging", "description": "v 2", "user_id": "u2", "source": "A/C", "run_id": "123", "status": "READY", }, ], } new_proto_message = ProtoRegisteredModel() parse_dict(json_dict, new_proto_message) assert original_proto_message == new_proto_message test_message = ParseTextIntoProto( """ field_int32: 11 field_int64: 12 field_uint32: 13 field_uint64: 14 field_sint32: 15 field_sint64: 16 field_fixed32: 17 field_fixed64: 18 field_sfixed32: 19 field_sfixed64: 20 field_bool: true field_string: "Im a string" field_with_default1: 111 field_repeated_int64: [1, 2, 3] field_enum: ENUM_VALUE1 field_inner_message { field_inner_int64: 101 field_inner_repeated_int64: [102, 103] } field_inner_message { field_inner_int64: 104 field_inner_repeated_int64: [105, 106] } oneof1: 207 [mlflow.ExtensionMessage.field_extended_int64]: 100 field_map1: [{key: 51 value: "52"}, {key: 53 value: "54"}] field_map2: [{key: "61" value: 62}, {key: "63" value: 64}] field_map3: [{key: 561 value: 562}, {key: 563 value: 564}] field_map4: [{key: 71 value: {field_inner_int64: 72 field_inner_repeated_int64: [81, 82] field_inner_string: "str1"}}, {key: 73 value: {field_inner_int64: 74 field_inner_repeated_int64: 83 field_inner_string: "str2"}}] """, TestMessage(), ) json_out = message_to_json(test_message) json_dict = json.loads(json_out) assert json_dict == { "field_int32": 11, "field_int64": 12, "field_uint32": 13, "field_uint64": 14, "field_sint32": 15, "field_sint64": 16, "field_fixed32": 17, "field_fixed64": 18, "field_sfixed32": 19, "field_sfixed64": 20, "field_bool": True, "field_string": "Im a string", "field_with_default1": 111, "field_repeated_int64": [1, 2, 3], "field_enum": "ENUM_VALUE1", "field_inner_message": [ { "field_inner_int64": 101, "field_inner_repeated_int64": [102, 103] }, { "field_inner_int64": 104, "field_inner_repeated_int64": [105, 106] }, ], "oneof1": 207, # JSON doesn't support non-string keys, so the int keys will be converted to strings. "field_map1": { "51": "52", "53": "54" }, "field_map2": { "63": 64, "61": 62 }, "field_map3": { "561": 562, "563": 564 }, "field_map4": { "73": { "field_inner_int64": 74, "field_inner_repeated_int64": [83], "field_inner_string": "str2", }, "71": { "field_inner_int64": 72, "field_inner_repeated_int64": [81, 82], "field_inner_string": "str1", }, }, "[mlflow.ExtensionMessage.field_extended_int64]": "100", } new_test_message = TestMessage() parse_dict(json_dict, new_test_message) assert new_test_message == test_message