def test_parse_submit_job_imported_function_project_assignment(
db: Session, client: TestClient, monkeypatch):
task_name = "task_name"
task_project = "task-project"
_mock_import_function(monkeypatch)
submit_job_body = {
"task": {
"spec": {
"function": "hub://gen_class_data"
},
"metadata": {
"name": task_name,
"project": task_project
},
},
"function": {
"spec": {
"resources": {
"limits": {},
"requests": {}
}
}
},
}
parsed_function_object, task = _parse_submit_run_body(
db, mlrun.api.schemas.AuthInfo(), submit_job_body)
assert parsed_function_object.metadata.project == task_project
def test_parse_submit_job_body_keep_resources(db: Session, client: TestClient):
task_name = "task_name"
task_project = "task-project"
project, function_name, function_tag, original_function = _mock_original_function(
client
)
submit_job_body = {
"task": {
"spec": {"function": f"{project}/{function_name}:{function_tag}"},
"metadata": {"name": task_name, "project": task_project},
},
"function": {"spec": {"resources": {"limits": {}, "requests": {}}}},
}
parsed_function_object, task = _parse_submit_run_body(db, submit_job_body)
assert parsed_function_object.metadata.name == function_name
assert parsed_function_object.metadata.project == project
assert parsed_function_object.metadata.tag == function_tag
assert (
DeepDiff(
parsed_function_object.spec.resources,
submit_job_body["function"]["spec"]["resources"],
ignore_order=True,
)
!= {}
)
assert (
DeepDiff(
parsed_function_object.spec.resources,
original_function["spec"]["resources"],
ignore_order=True,
)
== {}
)
def test_parse_submit_job_body_override_values(db: Session,
client: TestClient):
task_name = "task_name"
task_project = "task_project"
project, function_name, function_tag, original_function = _mock_original_function(
client)
submit_job_body = {
"task": {
"spec": {
"function": f"{project}/{function_name}:{function_tag}"
},
"metadata": {
"name": task_name,
"project": task_project
},
},
"function": {
"spec": {
"volumes": [
{
"name": "override-volume-name",
"flexVolume": {
"driver": "v3io/fuse",
"options": {
"container": "users",
"accessKey":
"4dbc1521-f6f2-4b28-aeac-29073413b9ae",
"subPath": "/pipelines/.mlrun",
},
},
},
{
"name": "new-volume-name",
"secret": {
"secretName": "secret-name"
},
},
],
"volume_mounts": [
{
"name": "override-volume-name",
"mountPath": "/v3io/volume/mount/path",
},
{
"name": "new-volume-name",
"mountPath": "/secret/volume/mount/path",
},
],
"env": [
{
"name": "OVERRIDE_ENV_VAR_KEY",
"value": "new-env-var-value"
},
{
"name": "NEW_ENV_VAR_KEY",
"value": "env-var-value"
},
],
"resources": {
"limits": {
"cpu": "250m",
"memory": "64Mi",
"nvidia.com/gpu": "2"
},
"requests": {
"cpu": "200m",
"memory": "32Mi"
},
},
"image_pull_policy":
"Always",
"replicas":
"3",
}
},
}
parsed_function_object, task = _parse_submit_run_body(db, submit_job_body)
assert parsed_function_object.metadata.name == function_name
assert parsed_function_object.metadata.project == project
assert parsed_function_object.metadata.tag == function_tag
assert (DeepDiff(
parsed_function_object.spec.resources,
submit_job_body["function"]["spec"]["resources"],
ignore_order=True,
) == {})
assert (parsed_function_object.spec.image_pull_policy ==
submit_job_body["function"]["spec"]["image_pull_policy"])
assert (parsed_function_object.spec.replicas == submit_job_body["function"]
["spec"]["replicas"])
_assert_volumes_and_volume_mounts(parsed_function_object, submit_job_body,
original_function)
_assert_env_vars(parsed_function_object, submit_job_body,
original_function)
def test_parse_submit_job_body_override_values(db: Session,
client: TestClient):
task_name = "task_name"
task_project = "task-project"
project, function_name, function_tag, original_function = _mock_original_function(
client)
submit_job_body = {
"task": {
"spec": {
"function": f"{project}/{function_name}:{function_tag}"
},
"metadata": {
"name": task_name,
"project": task_project
},
},
"function": {
"spec": {
"volumes": [
{
"name": "override-volume-name",
"flexVolume": {
"driver": "v3io/fuse",
"options": {
"container": "users",
"accessKey":
"4dbc1521-f6f2-4b28-aeac-29073413b9ae",
"subPath": "/pipelines/.mlrun",
},
},
},
{
"name": "new-volume-name",
"secret": {
"secretName": "secret-name"
},
},
],
"volume_mounts": [
{
"name": "old-volume-name",
"mountPath": "/v3io/old/volume/mount/path",
},
{
"name": "override-volume-name",
"mountPath": "/v3io/volume/mount/path",
},
{
"name": "new-volume-name",
"mountPath": "/secret/volume/mount/path",
},
],
"env": [
{
"name": "OVERRIDE_ENV_VAR_KEY",
"value": "new-env-var-value"
},
{
"name": "NEW_ENV_VAR_KEY",
"value": "env-var-value"
},
],
"resources": {
"limits": {
"cpu": "250m",
"memory": "64Mi",
"nvidia.com/gpu": "2"
},
"requests": {
"cpu": "200m",
"memory": "32Mi"
},
},
"image_pull_policy":
"Always",
"replicas":
"3",
"node_name":
"k8s-node1",
"node_selector": {
"kubernetes.io/hostname": "k8s-node1"
},
"affinity": {
"nodeAffinity": {
"preferredDuringSchedulingIgnoredDuringExecution": [{
"preference": {
"matchExpressions": [{
"key":
"some_node_label",
"operator":
"In",
"values": [
"possible-label-value-1",
"possible-label-value-2",
],
}]
},
"weight":
1,
}],
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key":
"some_node_label",
"operator":
"In",
"values": [
"required-label-value-1",
"required-label-value-2",
],
}]
}]
},
},
"podAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": [{
"labelSelector": {
"matchLabels": {
"some-pod-label-key":
"some-pod-label-value"
}
},
"namespaces": ["namespace-a", "namespace-b"],
"topologyKey":
"key-1",
}]
},
"podAntiAffinity": {
"preferredDuringSchedulingIgnoredDuringExecution": [{
"podAffinityTerm": {
"labelSelector": {
"matchExpressions": [{
"key":
"some_pod_label",
"operator":
"NotIn",
"values": [
"forbidden-label-value-1",
"forbidden-label-value-2",
],
}]
},
"namespaces": ["namespace-c"],
"topologyKey": "key-2",
},
"weight":
1,
}]
},
},
}
},
}
parsed_function_object, task = _parse_submit_run_body(
db, mlrun.api.schemas.AuthInfo(), submit_job_body)
assert parsed_function_object.metadata.name == function_name
assert parsed_function_object.metadata.project == project
assert parsed_function_object.metadata.tag == function_tag
assert (DeepDiff(
parsed_function_object.spec.resources,
submit_job_body["function"]["spec"]["resources"],
ignore_order=True,
) == {})
assert (parsed_function_object.spec.image_pull_policy ==
submit_job_body["function"]["spec"]["image_pull_policy"])
assert (parsed_function_object.spec.replicas == submit_job_body["function"]
["spec"]["replicas"])
_assert_volumes_and_volume_mounts(parsed_function_object, submit_job_body,
original_function)
_assert_env_vars(parsed_function_object, submit_job_body,
original_function)
assert (parsed_function_object.spec.node_name ==
submit_job_body["function"]["spec"]["node_name"])
assert (DeepDiff(
parsed_function_object.spec.node_selector,
submit_job_body["function"]["spec"]["node_selector"],
ignore_order=True,
) == {})
assert (DeepDiff(
parsed_function_object.spec._get_sanitized_affinity(),
submit_job_body["function"]["spec"]["affinity"],
ignore_order=True,
) == {})