def get_empty_preset(self):
return OperationSpecification.read(
{
"version": pkg.SCHEMA_VERSION,
"name": None,
"isPreset": True,
"description": "",
"tags": [],
"presets": [],
"queue": "",
"cache": {},
"termination": {},
"plugins": {},
"hooks": [],
"params": {},
"runPatch": {
"init": [],
"connections": [],
"container": {},
"environment": {
"nodeSelector": {},
"serviceAccountName": "",
"imagePullSecrets": [],
},
},
"schedule": None,
"events": [],
"joins": [],
"matrix": None,
"dependencies": [],
"trigger": None,
"conditions": None,
"skipOnUpstreamSkip": None,
}
)
def get_op_spec_by_name(self, name):
from polyaxon.polyaxonfile import OperationSpecification
op_dict = self.dag[name].op.to_dict()
op_dict[OperationSpecification.VERSION] = op_dict.get(
OperationSpecification.VERSION, SCHEMA_VERSION)
return OperationSpecification.read(op_dict)
def resume(ctx, project, uid, polyaxonfile):
"""Resume run.
Uses /docs/core/cli/#caching
Examples:
\b
$ polyaxon ops resume --uid=8aac02e3a62a4f0aaa257c59da5eab80
"""
content = None
if polyaxonfile:
content = OperationSpecification.read(
polyaxonfile, is_preset=True).to_dict(dump=True)
owner, project_name, run_uuid = get_project_run_or_local(
project or ctx.obj.get("project"),
uid or ctx.obj.get("run_uuid"),
is_cli=True,
)
try:
polyaxon_client = RunClient(owner=owner,
project=project_name,
run_uuid=run_uuid)
response = polyaxon_client.resume(override_config=content)
Printer.print_success("Run was resumed with uid {}".format(
response.uuid))
except (ApiException, HTTPError) as e:
handle_cli_error(e,
message="Could not resume run `{}`.".format(run_uuid))
sys.exit(1)
def create(self, validated_data):
is_managed = validated_data["is_managed"]
content = validated_data.get("content")
if content:
is_managed = True if is_managed is None else is_managed
if is_managed and not content:
raise ValidationError(
"Managed runs require a content with valid specification")
user = validated_data.get("user")
if is_managed:
try:
op_spec = OperationSpecification.read(content)
except Exception as e:
raise ValidationError(e)
return compile_operation_run(
project_id=validated_data["project"].id,
user_id=user.id if user else None,
op_spec=op_spec,
name=validated_data.get("name"),
description=validated_data.get("description"),
tags=validated_data.get("tags"),
)
else:
return create_run(
project_id=validated_data["project"].id,
user_id=user.id if user else None,
name=validated_data.get("name"),
description=validated_data.get("description"),
tags=validated_data.get("tags"),
)
def test_create_run_with_templated_service_spec(self):
config_dict = get_fxt_service_with_inputs()
spec = OperationSpecification.read(values=config_dict)
run = compile_operation_run(
project_id=self.project.id, user_id=self.user.id, op_spec=spec
)
assert run.kind == V1RunKind.SERVICE
assert run.name == "foo"
assert run.description == "a description"
assert set(run.tags) == {"backend", "lab"}
job_spec = CompiledOperationSpecification.read(run.content)
assert job_spec.run.container.image == "{{ image }}"
compiled_operation = CompiledOperationSpecification.read(run.content)
compiled_operation = CompiledOperationSpecification.apply_params(
compiled_operation, params=spec.params
)
compiled_operation = CompiledOperationSpecification.apply_operation_contexts(
compiled_operation
)
CompiledOperationSpecification.apply_runtime_contexts(compiled_operation)
run.content = compiled_operation.to_dict(dump=True)
run.save(update_fields=["content"])
job_spec = CompiledOperationSpecification.read(run.content)
job_spec = CompiledOperationSpecification.apply_runtime_contexts(job_spec)
assert job_spec.run.container.image == "foo/bar"
def setUp(self):
super().setUp()
op_spec = OperationSpecification.read({
"version": 1.1,
"kind": "operation",
"name": "foo",
"description": "a description",
"tags": ["tag1", "tag2"],
"trigger": "all_succeeded",
"component": {
"name": "build-template",
"tags": ["tag1", "tag2"],
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
},
"init": [{
"connection": "foo",
"git": {
"revision": "dev"
}
}],
},
},
})
self.compiled_operation = OperationSpecification.compile_operation(
op_spec)
self.preset = {
"runPatch": {},
"patchStrategy": V1PatchStrategy.POST_MERGE
}
def setUp(self):
super().setUp()
self.object.delete()
values = {
"version": 1.1,
"kind": "operation",
"name": "foo",
"description": "a description",
"params": {"image": {"value": "foo/bar"}},
"component": {
"name": "build-template",
"inputs": [{"name": "image", "type": "str"}],
"tags": ["tag1", "tag2"],
"run": {
"kind": V1RunKind.JOB,
"container": {"image": "{{ image }}"},
"init": [{"connection": "foo", "git": {"revision": "dev"}}],
},
},
}
op_spec = OperationSpecification.read(values=values)
self.object = compile_operation_run(
project_id=self.project.id, op_spec=op_spec, user_id=self.user.id
)
self.url = "/{}/{}/{}/runs/{}/".format(
API_V1, self.user.username, self.project.name, self.object.uuid.hex
)
def get_op_spec_by_index(self, idx):
from polyaxon.polyaxonfile import OperationSpecification
op_dict = self.operations[idx].to_dict()
op_dict[OperationSpecification.VERSION] = op_dict.get(
OperationSpecification.VERSION, SCHEMA_VERSION)
return OperationSpecification.read(op_dict)
def create(self, validated_data):
is_managed = validated_data["is_managed"]
content = validated_data.get("content")
meta_info = validated_data.get("meta_info") or {}
if content:
is_managed = True if is_managed is None else is_managed
if is_managed and not content:
raise ValidationError(
"Managed runs require a content with valid specification"
)
project_id = validated_data["project"].id
user = validated_data.get("user")
name = validated_data.get("name")
description = validated_data.get("description")
tags = validated_data.get("tags")
pending = validated_data.get("pending")
# Check the deprecated `is_approved` flag
if pending is None:
is_approved = validated_data.get("is_approved")
if is_approved is False:
pending = V1RunPending.UPLOAD
if is_managed or content:
try:
op_spec = OperationSpecification.read(content)
except Exception as e:
raise ValidationError(e)
if op_spec.is_template():
raise ValidationError(
"Received a template polyaxonfile, "
"Please customize the specification or disable the template."
)
try:
return compile_operation_run(
project_id=project_id,
user_id=user.id if user else None,
op_spec=op_spec,
name=name,
description=description,
tags=tags,
meta_info=meta_info,
is_managed=is_managed,
pending=pending,
supported_kinds=validated_data.get("supported_kinds"),
supported_owners=validated_data.get("supported_owners"),
)
except (MarshmallowValidationError, PolyaxonException, ValueError) as e:
raise ValidationError(e)
else:
return create_run(
project_id=project_id,
user_id=user.id if user else None,
name=name,
description=description,
tags=tags,
meta_info=meta_info,
)
def setUp(self):
super().setUp()
self.user = UserFactory()
self.user2 = UserFactory()
self.project = ProjectFactory()
op_spec = OperationSpecification.read(values=get_fxt_job_with_inputs())
self.run = compile_operation_run(project_id=self.project.id,
op_spec=op_spec,
user_id=self.user.id)
def test_create_run_with_service_spec(self):
config_dict = get_fxt_service()
spec = OperationSpecification.read(values=config_dict)
run = compile_operation_run(
project_id=self.project.id, user_id=self.user.id, op_spec=spec
)
assert run.kind == V1RunKind.SERVICE
assert run.name == "foo"
assert run.description == "a description"
assert set(run.tags) == {"backend", "lab", "tag1", "tag2"}
service_spec = CompiledOperationSpecification.read(run.content)
assert service_spec.run.container.image == "jupyter"
def create(self, name=None, tags=None, description=None, content=None):
operation = polyaxon_sdk.V1OperationBody()
if name:
operation.name = name
if tags:
operation.tags = tags
if description:
operation.description = description
if content:
try:
specification = OperationSpecification.read(content)
except Exception as e:
raise PolyaxonClientException("Client error: %s" % e) from e
operation.content = specification.to_dict(dump=True)
else:
operation.is_managed = False
if self.client:
try:
run = self.client.runs_v1.create_run(owner=self.owner,
project=self.project,
body=operation)
except (ApiException, HTTPError) as e:
raise PolyaxonClientException("Client error: %s" % e) from e
if not run:
raise PolyaxonClientException("Could not create a run.")
else:
run = polyaxon_sdk.V1Run(
name=operation.name,
tags=operation.tags,
description=operation.description,
content=operation.content,
is_managed=operation.is_managed,
)
self._run = run
self._run_uuid = run.uuid
if self.artifacts_path:
self.set_run_event_logger()
if self.track_code:
self.log_code_ref()
if self.track_env:
self.log_run_env()
if not settings.CLIENT_CONFIG.is_managed:
self._start()
else:
self._register_wait()
return self
def restart(
ctx,
project,
uid,
name,
tags,
description,
copy,
copy_dirs,
copy_files,
polyaxonfile,
):
"""Restart run.
Uses /docs/core/cli/#caching
Examples:
\b
$ polyaxon run --uid=8aac02e3a62a4f0aaa257c59da5eab80 restart
"""
content = None
if polyaxonfile:
content = OperationSpecification.read(
polyaxonfile, is_preset=True).to_dict(dump=True)
owner, project_name, run_uuid = get_project_run_or_local(
project or ctx.obj.get("project"),
uid or ctx.obj.get("run_uuid"),
is_cli=True,
)
try:
polyaxon_client = RunClient(owner=owner,
project=project_name,
run_uuid=run_uuid)
response = polyaxon_client.restart(
name=name,
description=description,
tags=tags,
override_config=content,
copy=copy,
copy_dirs=copy_dirs,
copy_files=copy_files,
)
Printer.print_success("Run was {} with uid {}".format(
"copied" if copy else "restarted", response.uuid))
except (ApiException, HTTPError) as e:
handle_cli_error(
e, message="Could not restart run `{}`.".format(run_uuid))
sys.exit(1)
def create(self, validated_data):
is_managed = validated_data["is_managed"]
content = validated_data.get("content")
if content:
is_managed = True if is_managed is None else is_managed
if is_managed and not content:
raise ValidationError(
"Managed runs require a content with valid specification"
)
project_id = validated_data["project"].id
user = validated_data.get("user")
name = validated_data.get("name")
description = validated_data.get("description")
tags = validated_data.get("tags")
if is_managed or content:
try:
op_spec = OperationSpecification.read(content)
except Exception as e:
raise ValidationError(e)
if op_spec.is_template():
raise ValidationError(
"Received a template polyaxonfile, "
"Please customize the specification or disable the template."
)
try:
return compile_operation_run(
project_id=project_id,
user_id=user.id if user else None,
op_spec=op_spec,
name=name,
description=description,
tags=tags,
is_managed=is_managed,
supported_kinds=validated_data.get("supported_kinds"),
)
except (PolyaxonException, ValueError) as e:
raise ValidationError(e)
else:
return create_run(
project_id=project_id,
user_id=user.id if user else None,
name=name,
description=description,
tags=tags,
)
def make_and_convert(
owner_name: str,
project_name: str,
run_uuid: str,
run_name: str,
content: str,
):
operation = OperationSpecification.read(content)
compiled_operation = OperationSpecification.compile_operation(operation)
return make(
owner_name=owner_name,
project_name=project_name,
project_uuid=project_name,
run_uuid=run_uuid,
run_name=run_name,
run_path=run_uuid,
compiled_operation=compiled_operation,
params=None,
)
def test_create_run_with_job_spec(self):
config_dict = get_fxt_job()
spec = OperationSpecification.read(values=config_dict)
run = compile_operation_run(project_id=self.project.id,
user_id=self.user.id,
op_spec=spec)
assert run.kind == V1RunKind.JOB
assert run.name == "foo"
assert run.description == "a description"
assert set(run.tags) == {"tag1", "tag2"}
# Check compiled operation passes
compiled_operation = CompiledOperationSpecification.read(run.content)
compiled_operation = CompiledOperationSpecification.apply_params(
compiled_operation)
CompiledOperationSpecification.apply_run_contexts(compiled_operation, )
# Check job
job_spec = CompiledOperationSpecification.read(run.content)
assert job_spec.run.container.image == "test"
job_spec = CompiledOperationSpecification.apply_context(job_spec)
assert job_spec.run.container.image == "test"
def make_and_convert(
owner_name: str,
project_name: str,
run_uuid: str,
run_name: str,
content: str,
default_auth: bool = False,
):
operation = OperationSpecification.read(content)
compiled_operation = OperationSpecification.compile_operation(operation)
return make(
owner_name=owner_name,
project_name=project_name,
project_uuid=project_name,
run_uuid=run_uuid,
run_name=run_name,
run_path=run_uuid,
compiled_operation=compiled_operation,
params=operation.params,
converters=PLATFORM_CONVERTERS,
default_auth=default_auth,
)
def get_empty_operation(self):
return OperationSpecification.read(
{"version": pkg.SCHEMA_VERSION, "hubRef": "test"}
)
def get_full_preset(self):
return OperationSpecification.read({
"version":
pkg.SCHEMA_VERSION,
"name":
self.PATCH_STR_VALUE,
"isPreset":
True,
"description":
self.PATCH_STR_VALUE,
"tags": [
"{}1".format(self.PATCH_STR_VALUE),
"{}2".format(self.PATCH_STR_VALUE),
],
"presets": [self.PATCH_STR_VALUE],
"queue":
"{}/{}".format(self.PATCH_STR_VALUE, self.PATCH_STR_VALUE),
"cache": {
"disable": True,
"ttl": self.PATCH_INT_VALUE,
},
"termination": {
"maxRetries": self.PATCH_INT_VALUE,
"ttl": self.PATCH_INT_VALUE,
"timeout": self.PATCH_INT_VALUE,
},
"plugins": {
"auth": True,
"shm": True,
"collectLogs": True,
"collectArtifacts": True,
"collectResources": True,
},
"actions": [
{
"hubRef": "{}1".format(self.PATCH_STR_VALUE)
},
{
"hubRef": "{}2".format(self.PATCH_STR_VALUE),
"label": "customLabel",
"many": True,
},
],
"hooks": [
{
"trigger": "succeeded",
"connection": "{}1".format(self.PATCH_STR_VALUE),
},
{
"connection": "{}1".format(self.PATCH_STR_VALUE),
"hubRef": "{}2".format(self.PATCH_STR_VALUE),
},
],
"params": {
"patch-key1": {
"value": "{}2".format(self.PATCH_STR_VALUE)
},
"patch-key2": {
"value": "{}1".format(self.PATCH_STR_VALUE)
},
},
"runPatch": {
"init": [{
"connection": self.PATCH_STR_VALUE,
"git": {
"revision": "dev"
}
}],
"connections": [
"{}1".format(self.PATCH_STR_VALUE),
"{}2".format(self.PATCH_STR_VALUE),
],
"container": {
"resources": {
"requests": {
"cpu": self.PATCH_INT_VALUE,
"memory": self.PATCH_INT_VALUE,
}
}
},
"environment": {
"nodeSelector": {
"polyaxon-patch": "core"
},
"serviceAccountName":
self.PATCH_STR_VALUE,
"imagePullSecrets": [
"{}1".format(self.PATCH_STR_VALUE),
"{}2".format(self.PATCH_STR_VALUE),
],
},
},
"schedule": {
"kind": "exact_time",
"startAt": self.PATCH_DT_VALUE
},
"events":
None,
"matrix": {
"concurrency":
self.PATCH_INT_VALUE,
"kind":
"mapping",
"values": [
{
"a": self.PATCH_INT_VALUE
},
{
"c": self.PATCH_INT_VALUE
},
],
},
"dependencies": [
"{}1".format(self.PATCH_STR_VALUE),
"{}2".format(self.PATCH_STR_VALUE),
],
"trigger":
"all_succeeded",
"conditions":
"",
"skipOnUpstreamSkip":
True,
})
def get_full_operation(self):
return OperationSpecification.read(
{
"version": pkg.SCHEMA_VERSION,
"name": self.DEFAULT_STR_VALUE,
"description": self.DEFAULT_STR_VALUE,
"tags": [
"{}1".format(self.DEFAULT_STR_VALUE),
"{}2".format(self.DEFAULT_STR_VALUE),
],
"presets": [self.DEFAULT_STR_VALUE],
"queue": "{}/{}".format(self.DEFAULT_STR_VALUE, self.DEFAULT_STR_VALUE),
"cache": {
"disable": False,
"ttl": self.DEFAULT_INT_VALUE,
},
"termination": {
"maxRetries": self.DEFAULT_INT_VALUE,
"ttl": self.DEFAULT_INT_VALUE,
"timeout": self.DEFAULT_INT_VALUE,
},
"plugins": {
"auth": False,
"shm": False,
"collectLogs": False,
"collectArtifacts": False,
"collectResources": False,
},
"hooks": [
{
"hubRef": "{}1".format(self.DEFAULT_STR_VALUE),
"trigger": "succeeded",
"connection": "{}1".format(self.DEFAULT_STR_VALUE),
},
{
"connection": "{}2".format(self.DEFAULT_STR_VALUE),
"hubRef": "{}2".format(self.DEFAULT_STR_VALUE),
},
],
"params": {
"patch-key1": {"value": "{}2".format(self.DEFAULT_STR_VALUE)},
"patch-key2": {"value": "{}1".format(self.DEFAULT_STR_VALUE)},
},
"runPatch": {
"init": [
{
"connection": self.DEFAULT_STR_VALUE,
"git": {"revision": self.DEFAULT_STR_VALUE},
}
],
"connections": [
"{}1".format(self.DEFAULT_STR_VALUE),
"{}2".format(self.DEFAULT_STR_VALUE),
],
"container": {
"resources": {"requests": {"cpu": self.DEFAULT_INT_VALUE}}
},
"environment": {
"nodeSelector": {"polyaxon": "core"},
"serviceAccountName": self.DEFAULT_STR_VALUE,
"imagePullSecrets": [
"{}1".format(self.DEFAULT_STR_VALUE),
"{}2".format(self.DEFAULT_STR_VALUE),
],
},
},
"schedule": {
"kind": "cron",
"cron": "0 0 * * *",
"startAt": self.DEFAULT_DT_VALUE,
"endAt": self.DEFAULT_DT_VALUE,
},
"events": [
{
"kinds": [V1EventKind.RUN_STATUS_SCHEDULED],
"ref": "{}1".format(self.DEFAULT_STR_VALUE),
},
{
"kinds": [V1EventKind.RUN_STATUS_SCHEDULED],
"ref": "{}2".format(self.DEFAULT_STR_VALUE),
},
],
"joins": [
{
"query": "{}1".format(self.DEFAULT_STR_VALUE),
"sort": "{}1".format(self.DEFAULT_STR_VALUE),
"params": {
"u": {"value": "{}1".format(self.DEFAULT_STR_VALUE)},
},
},
{
"query": "{}2".format(self.DEFAULT_STR_VALUE),
"sort": "{}2".format(self.DEFAULT_STR_VALUE),
"params": {
"v": {
"value": "{}2".format(self.DEFAULT_STR_VALUE),
"contextOnly": True,
},
},
},
],
"matrix": {
"concurrency": self.DEFAULT_INT_VALUE,
"kind": "mapping",
"values": [
{"a": self.DEFAULT_INT_VALUE},
{"b": self.DEFAULT_INT_VALUE},
],
},
"dependencies": [
"{}1".format(self.DEFAULT_STR_VALUE),
"{}2".format(self.DEFAULT_STR_VALUE),
],
"trigger": "all_succeeded",
"conditions": self.DEFAULT_STR_VALUE,
"skipOnUpstreamSkip": True,
"hubRef": self.DEFAULT_STR_VALUE,
}
)
