def _get_request_message(request_message, flask_request=request):
if flask_request.method == 'GET' and len(flask_request.query_string) > 0:
# This is a hack to make arrays of length 1 work with the parser.
# for example experiment_ids%5B%5D=0 should be parsed to {experiment_ids: [0]}
# but it gets parsed to {experiment_ids: 0}
# but it doesn't. However, experiment_ids%5B0%5D=0 will get parsed to the right
# result.
query_string = re.sub('%5B%5D', '%5B0%5D',
flask_request.query_string.decode("utf-8"))
request_dict = parser.parse(query_string, normalized=True)
parse_dict(request_dict, request_message)
return request_message
request_json = _get_request_json(flask_request)
# Older clients may post their JSON double-encoded as strings, so the get_json
# above actually converts it to a string. Therefore, we check this condition
# (which we can tell for sure because any proper request should be a dictionary),
# and decode it a second time.
if isinstance(request_json, six.string_types):
request_json = json.loads(request_json)
# If request doesn't have json body then assume it's empty.
if request_json is None:
request_json = {}
parse_dict(request_json, request_message)
return request_message
def test_parse_legacy_experiment():
in_json = {"experiment_id": 123, "name": "name", "unknown": "field"}
message = ProtoExperiment()
parse_dict(in_json, message)
experiment = Experiment.from_proto(message)
assert experiment.experiment_id == "123"
assert experiment.name == 'name'
assert experiment.artifact_location == ''
def call_endpoint(host_creds, endpoint, method, json_body, response_proto):
# Convert json string to json dictionary, to pass to requests
if json_body:
json_body = json.loads(json_body)
if method == 'GET':
response = http_request(
host_creds=host_creds, endpoint=endpoint, method=method, params=json_body)
else:
response = http_request(
host_creds=host_creds, endpoint=endpoint, method=method, json=json_body)
response = verify_rest_response(response, endpoint)
js_dict = json.loads(response.text)
parse_dict(js_dict=js_dict, message=response_proto)
return response_proto
def _call_endpoint(self, api, json_body):
endpoint, method = _METHOD_TO_INFO[api]
response_proto = api.Response()
# Convert json string to json dictionary, to pass to requests
if json_body:
json_body = json.loads(json_body)
host_creds = self.get_host_creds()
response = http_request_safe(host_creds=host_creds,
endpoint=endpoint,
method=method,
json=json_body)
js_dict = json.loads(response.text)
parse_dict(js_dict=js_dict, message=response_proto)
return response_proto
def _call_endpoint(self, api, json_body):
endpoint, method = _METHOD_TO_INFO[api]
response_proto = api.Response()
# Convert json string to json dictionary, to pass to requests
if json_body:
json_body = json.loads(json_body)
response = http_request(endpoint=endpoint,
method=method,
json=json_body,
**self.http_request_kwargs)
js_dict = json.loads(response.text)
if 'error_code' in js_dict:
raise RestException(js_dict)
parse_dict(js_dict=js_dict, message=response_proto)
return response_proto
def _get_request_message(request_message, flask_request=request):
from querystring_parser import parser
if flask_request.method == "GET" and len(flask_request.query_string) > 0:
# This is a hack to make arrays of length 1 work with the parser.
# for example experiment_ids%5B%5D=0 should be parsed to {experiment_ids: [0]}
# but it gets parsed to {experiment_ids: 0}
# but it doesn't. However, experiment_ids%5B0%5D=0 will get parsed to the right
# result.
query_string = re.sub("%5B%5D", "%5B0%5D", flask_request.query_string.decode("utf-8"))
request_dict = parser.parse(query_string, normalized=True)
# Convert atomic values of repeated fields to lists before calling protobuf deserialization.
# Context: We parse the parameter string into a dictionary outside of protobuf since
# protobuf does not know how to read the query parameters directly. The query parser above
# has no type information and hence any parameter that occurs exactly once is parsed as an
# atomic value. Since protobuf requires that the values of repeated fields are lists,
# deserialization will fail unless we do the fix below.
for field in request_message.DESCRIPTOR.fields:
if (
field.label == descriptor.FieldDescriptor.LABEL_REPEATED
and field.name in request_dict
):
if not isinstance(request_dict[field.name], list):
request_dict[field.name] = [request_dict[field.name]]
parse_dict(request_dict, request_message)
return request_message
request_json = _get_request_json(flask_request)
# Older clients may post their JSON double-encoded as strings, so the get_json
# above actually converts it to a string. Therefore, we check this condition
# (which we can tell for sure because any proper request should be a dictionary),
# and decode it a second time.
if is_string_type(request_json):
request_json = json.loads(request_json)
# If request doesn't have json body then assume it's empty.
if request_json is None:
request_json = {}
parse_dict(request_json, request_message)
return request_message
def test_parse_dict_int_as_string_backcompat():
in_json = {"timestamp": "123"}
message = ProtoMetric()
parse_dict(in_json, message)
experiment = Metric.from_proto(message)
assert experiment.timestamp == 123
def test_message_to_json():
json_out = message_to_json(
Experiment("123", "name", "arty", "active").to_proto())
assert json.loads(json_out) == {
"experiment_id": "123",
"name": "name",
"artifact_location": "arty",
"lifecycle_stage": "active",
}
original_proto_message = RegisteredModel(
name="model_1",
creation_timestamp=111,
last_updated_timestamp=222,
description="Test model",
latest_versions=[
ModelVersion(
name="mv-1",
version="1",
creation_timestamp=333,
last_updated_timestamp=444,
description="v 1",
user_id="u1",
current_stage="Production",
source="A/B",
run_id="9245c6ce1e2d475b82af84b0d36b52f4",
status="READY",
status_message=None,
),
ModelVersion(
name="mv-2",
version="2",
creation_timestamp=555,
last_updated_timestamp=666,
description="v 2",
user_id="u2",
current_stage="Staging",
source="A/C",
run_id="123",
status="READY",
status_message=None,
),
],
).to_proto()
json_out = message_to_json(original_proto_message)
json_dict = json.loads(json_out)
assert json_dict == {
"name":
"model_1",
"creation_timestamp":
111,
"last_updated_timestamp":
222,
"description":
"Test model",
"latest_versions": [
{
"name": "mv-1",
"version": "1",
"creation_timestamp": 333,
"last_updated_timestamp": 444,
"current_stage": "Production",
"description": "v 1",
"user_id": "u1",
"source": "A/B",
"run_id": "9245c6ce1e2d475b82af84b0d36b52f4",
"status": "READY",
},
{
"name": "mv-2",
"version": "2",
"creation_timestamp": 555,
"last_updated_timestamp": 666,
"current_stage": "Staging",
"description": "v 2",
"user_id": "u2",
"source": "A/C",
"run_id": "123",
"status": "READY",
},
],
}
new_proto_message = ProtoRegisteredModel()
parse_dict(json_dict, new_proto_message)
assert original_proto_message == new_proto_message
test_message = ParseTextIntoProto(
"""
field_int32: 11
field_int64: 12
field_uint32: 13
field_uint64: 14
field_sint32: 15
field_sint64: 16
field_fixed32: 17
field_fixed64: 18
field_sfixed32: 19
field_sfixed64: 20
field_bool: true
field_string: "Im a string"
field_with_default1: 111
field_repeated_int64: [1, 2, 3]
field_enum: ENUM_VALUE1
field_inner_message {
field_inner_int64: 101
field_inner_repeated_int64: [102, 103]
}
field_inner_message {
field_inner_int64: 104
field_inner_repeated_int64: [105, 106]
}
oneof1: 207
[mlflow.ExtensionMessage.field_extended_int64]: 100
field_map1: [{key: 51 value: "52"}, {key: 53 value: "54"}]
field_map2: [{key: "61" value: 62}, {key: "63" value: 64}]
field_map3: [{key: 561 value: 562}, {key: 563 value: 564}]
field_map4: [{key: 71
value: {field_inner_int64: 72
field_inner_repeated_int64: [81, 82]
field_inner_string: "str1"}},
{key: 73
value: {field_inner_int64: 74
field_inner_repeated_int64: 83
field_inner_string: "str2"}}]
""",
TestMessage(),
)
json_out = message_to_json(test_message)
json_dict = json.loads(json_out)
assert json_dict == {
"field_int32":
11,
"field_int64":
12,
"field_uint32":
13,
"field_uint64":
14,
"field_sint32":
15,
"field_sint64":
16,
"field_fixed32":
17,
"field_fixed64":
18,
"field_sfixed32":
19,
"field_sfixed64":
20,
"field_bool":
True,
"field_string":
"Im a string",
"field_with_default1":
111,
"field_repeated_int64": [1, 2, 3],
"field_enum":
"ENUM_VALUE1",
"field_inner_message": [
{
"field_inner_int64": 101,
"field_inner_repeated_int64": [102, 103]
},
{
"field_inner_int64": 104,
"field_inner_repeated_int64": [105, 106]
},
],
"oneof1":
207,
# JSON doesn't support non-string keys, so the int keys will be converted to strings.
"field_map1": {
"51": "52",
"53": "54"
},
"field_map2": {
"63": 64,
"61": 62
},
"field_map3": {
"561": 562,
"563": 564
},
"field_map4": {
"73": {
"field_inner_int64": 74,
"field_inner_repeated_int64": [83],
"field_inner_string": "str2",
},
"71": {
"field_inner_int64": 72,
"field_inner_repeated_int64": [81, 82],
"field_inner_string": "str1",
},
},
"[mlflow.ExtensionMessage.field_extended_int64]":
"100",
}
new_test_message = TestMessage()
parse_dict(json_dict, new_test_message)
assert new_test_message == test_message
