def test_default_values_with_colon_in_env():
with patch("seldon_core.metadata.MODEL_IMAGE",
"localhost:32000/sklearn-iris:0.1"):
meta = validate_model_metadata({})
assert meta == {
"name": "localhost:32000/sklearn-iris",
"versions": ["0.1"],
"platform": "",
"inputs": [],
"outputs": [],
"custom": {},
}
def test_default_values_with_env():
with patch("seldon_core.metadata.MODEL_IMAGE",
"seldonio/sklearn-iris:0.1"):
meta = validate_model_metadata({})
assert meta == {
"apiVersion": "v2",
"name": "seldonio/sklearn-iris",
"versions": ["0.1"],
"platform": "",
"inputs": [],
"outputs": [],
}
def test_v1_mixed_multiple_inputs():
data = """
name: my-model-name
versions: [ my-model-version-01 ]
platform: seldon
inputs:
- messagetype: "tensor"
schema:
names: [a, b, c, d]
shape: [ 2, 2 ]
- messagetype: jsonData
outputs:
- messagetype: "binData"
custom:
tag-key: tag-value
"""
meta_json = validate_model_metadata(yaml.safe_load(data))
meta_proto = json_to_seldon_model_metadata(meta_json)
assert meta_json == {
"name": "my-model-name",
"versions": ["my-model-version-01"],
"platform": "seldon",
"inputs": [
{
"messagetype": "tensor",
"schema": {"names": ["a", "b", "c", "d"], "shape": [2, 2]},
},
{"messagetype": "jsonData"},
],
"outputs": [{"messagetype": "binData"}],
"custom": {"tag-key": "tag-value"},
}
assert json.loads(json_format.MessageToJson(meta_proto)) == {
"name": "my-model-name",
"versions": ["my-model-version-01"],
"platform": "seldon",
"inputs": [
{
"messagetype": "tensor",
"schema": {"names": ["a", "b", "c", "d"], "shape": [2, 2]},
},
{"messagetype": "jsonData"},
],
"outputs": [{"messagetype": "binData"}],
"custom": {"tag-key": "tag-value"},
}
def test_validate_model_metadata():
meta = {
"name": "my-model-name",
"versions": ["model-version"],
"platform": "model-platform",
"inputs": [{
"name": "input",
"datatype": "BYTES",
"shape": [1]
}],
"outputs": [{
"name": "output",
"datatype": "BYTES",
"shape": [1]
}],
}
assert meta == validate_model_metadata(meta)
def test_v1_array(messagetype):
data = f"""
name: my-model-name
versions: [ my-model-version-01 ]
platform: seldon
inputs:
- messagetype: {messagetype}
schema:
names: [a, b, c, d]
shape: [ 2, 2 ]
outputs:
- messagetype: {messagetype}
schema:
shape: [ 1 ]
custom:
tag-key: tag-value
"""
meta_json = validate_model_metadata(yaml.safe_load(data))
meta_proto = json_to_seldon_model_metadata(meta_json)
assert meta_json == {
"name": "my-model-name",
"versions": ["my-model-version-01"],
"platform": "seldon",
"inputs": [
{
"messagetype": f"{messagetype}",
"schema": {"names": ["a", "b", "c", "d"], "shape": [2, 2]},
}
],
"outputs": [{"messagetype": f"{messagetype}", "schema": {"shape": [1]}}],
"custom": {"tag-key": "tag-value"},
}
assert json.loads(json_format.MessageToJson(meta_proto)) == {
"name": "my-model-name",
"versions": ["my-model-version-01"],
"platform": "seldon",
"inputs": [
{
"messagetype": f"{messagetype}",
"schema": {"names": ["a", "b", "c", "d"], "shape": [2.0, 2.0]},
}
],
"outputs": [{"messagetype": f"{messagetype}", "schema": {"shape": [1.0]}}],
"custom": {"tag-key": "tag-value"},
}
def test_validate_model_metadata():
meta = {
"name": "my-model-name",
"versions": ["model-version"],
"platform": "model-platform",
"inputs": [{
"name": "input",
"datatype": "BYTES",
"shape": [1]
}],
"outputs": [{
"name": "output",
"datatype": "BYTES",
"shape": [1]
}],
}
with patch("seldon_core.metadata.MODEL_IMAGE", None):
assert {"apiVersion": "v2", **meta} == validate_model_metadata(meta)
def init_metadata(user_model: Any) -> Dict:
"""
Call the user model to get the model init_metadata
Parameters
----------
user_model
User defined class instance
Returns
-------
Validated model metadata
"""
# meta_user: load metadata defined in the user_model instance
if hasattr(user_model, "init_metadata"):
try:
meta_user = user_model.init_metadata()
except SeldonNotImplementedError:
meta_user = {}
pass
else:
meta_user = {}
if not isinstance(meta_user, dict):
logger.error("init_metadata must return dict")
meta_user = {}
# meta_env: load metadata from environmental variable
try:
meta_env = yaml.safe_load(os.environ.get("MODEL_METADATA", "{}"))
except yaml.YAMLError as e:
logger.error(
f"Reading metadata from MODEL_METADATA env variable failed: {e}")
meta_env = {}
meta = {**meta_user, **meta_env}
try:
return validate_model_metadata(meta)
except SeldonInvalidMetadataError as e:
logger.error(f"Metadata validation error\n{e}")
logger.error(f"Failed to validate metadata {meta}")
return None
def test_validate_model_metadata_with_colon_in_env():
meta = {
"name": "my-model-name",
"versions": ["model-version"],
"platform": "model-platform",
"inputs": [{
"name": "input",
"datatype": "BYTES",
"shape": [1]
}],
"outputs": [{
"name": "output",
"datatype": "BYTES",
"shape": [1]
}],
}
with patch("seldon_core.metadata.MODEL_IMAGE",
"localhost:32000/sklearn-iris:0.1"):
assert {"apiVersion": "v2", **meta} == validate_model_metadata(meta)
def test_validate_model_metadata_with_env():
meta = {
"name": "my-model-name",
"versions": ["model-version"],
"platform": "model-platform",
"inputs": [{
"name": "input",
"datatype": "BYTES",
"shape": [1]
}],
"outputs": [{
"name": "output",
"datatype": "BYTES",
"shape": [1]
}],
}
with patch("seldon_core.metadata.MODEL_IMAGE",
"seldonio/sklearn-iris:0.1"):
assert meta == validate_model_metadata(meta)
def test_v1_only_messagetype(messagetype):
data = f"""
name: my-model-name
versions: [ my-model-version-01 ]
platform: seldon
inputs:
- messagetype: {messagetype}
outputs:
- messagetype: {messagetype}
custom:
tag-key: tag-value
"""
meta_json = validate_model_metadata(yaml.safe_load(data))
meta_proto = json_to_seldon_model_metadata(meta_json)
assert meta_json == {
"name": "my-model-name",
"versions": ["my-model-version-01"],
"platform": "seldon",
"inputs": [{
"messagetype": f"{messagetype}"
}],
"outputs": [{
"messagetype": f"{messagetype}"
}],
"custom": {
"tag-key": "tag-value"
},
}
assert json.loads(json_format.MessageToJson(meta_proto)) == {
"name": "my-model-name",
"versions": ["my-model-version-01"],
"platform": "seldon",
"inputs": [{
"messagetype": f"{messagetype}"
}],
"outputs": [{
"messagetype": f"{messagetype}"
}],
"custom": {
"tag-key": "tag-value"
},
}
def test_v1_v2_mix():
data = """
name: my-model-name
versions: [ my-model-version-01 ]
platform: seldon
inputs:
- datatype: BYTES
name: input
shape: [ 1, 4 ]
- messagetype: jsonData
outputs:
- datatype: BYTES
name: output
shape: [ 3 ]
custom:
tag-key: tag-value
"""
meta_json = validate_model_metadata(yaml.safe_load(data))
meta_proto = json_to_seldon_model_metadata(meta_json)
assert meta_json == {
"name": "my-model-name",
"versions": ["my-model-version-01"],
"platform": "seldon",
"inputs": [
{"datatype": "BYTES", "name": "input", "shape": [1, 4]},
{"messagetype": "jsonData"},
],
"outputs": [{"datatype": "BYTES", "name": "output", "shape": [3]}],
"custom": {"tag-key": "tag-value"},
}
assert json.loads(json_format.MessageToJson(meta_proto)) == {
"name": "my-model-name",
"versions": ["my-model-version-01"],
"platform": "seldon",
"inputs": [
{"name": "input", "datatype": "BYTES", "shape": ["1", "4"]},
{"messagetype": "jsonData"},
],
"outputs": [{"name": "output", "datatype": "BYTES", "shape": ["3"]}],
"custom": {"tag-key": "tag-value"},
}
def test_validate_model_metadata():
meta = {
"name": "my-model-name",
"versions": ["model-version"],
"platform": "model-platform",
"inputs": [{
"name": "input",
"datatype": "BYTES",
"shape": [1]
}],
"outputs": [{
"name": "output",
"datatype": "BYTES",
"shape": [1]
}],
"custom": {
"tag-key": "tag-value"
},
}
with patch("seldon_core.metadata.MODEL_IMAGE", None):
assert meta == validate_model_metadata(meta)
def test_validate_model_metadata_wrong_versions(invalid_versions):
with pytest.raises(SeldonInvalidMetadataError):
validate_model_metadata({"versions": invalid_versions})
def test_v1_json_with_schema():
data = """
name: my-model-name
versions: [ my-model-version-01 ]
platform: seldon
inputs:
- messagetype: jsonData
schema:
type: object
properties:
names:
type: array
items:
type: string
data:
type: array
items:
type: number
format: double
outputs:
- messagetype: jsonData
"""
meta_json = validate_model_metadata(yaml.safe_load(data))
meta_proto = json_to_seldon_model_metadata(meta_json)
assert meta_json == {
"name": "my-model-name",
"versions": ["my-model-version-01"],
"platform": "seldon",
"inputs": [
{
"messagetype": "jsonData",
"schema": {
"type": "object",
"properties": {
"names": {"type": "array", "items": {"type": "string"}},
"data": {
"type": "array",
"items": {"type": "number", "format": "double"},
},
},
},
}
],
"outputs": [{"messagetype": "jsonData"}],
}
assert json.loads(json_format.MessageToJson(meta_proto)) == {
"name": "my-model-name",
"versions": ["my-model-version-01"],
"platform": "seldon",
"inputs": [
{
"messagetype": "jsonData",
"schema": {
"properties": {
"names": {"items": {"type": "string"}, "type": "array"},
"data": {
"type": "array",
"items": {"type": "number", "format": "double"},
},
},
"type": "object",
},
}
],
"outputs": [{"messagetype": "jsonData"}],
}
