def get_empty_preset(self): return OperationSpecification.read( { "version": pkg.SCHEMA_VERSION, "name": None, "isPreset": True, "description": "", "tags": [], "presets": [], "queue": "", "cache": {}, "termination": {}, "plugins": {}, "hooks": [], "params": {}, "runPatch": { "init": [], "connections": [], "container": {}, "environment": { "nodeSelector": {}, "serviceAccountName": "", "imagePullSecrets": [], }, }, "schedule": None, "events": [], "joins": [], "matrix": None, "dependencies": [], "trigger": None, "conditions": None, "skipOnUpstreamSkip": None, } )
def get_op_spec_by_name(self, name): from polyaxon.polyaxonfile import OperationSpecification op_dict = self.dag[name].op.to_dict() op_dict[OperationSpecification.VERSION] = op_dict.get( OperationSpecification.VERSION, SCHEMA_VERSION) return OperationSpecification.read(op_dict)
def resume(ctx, project, uid, polyaxonfile): """Resume run. Uses /docs/core/cli/#caching Examples: \b $ polyaxon ops resume --uid=8aac02e3a62a4f0aaa257c59da5eab80 """ content = None if polyaxonfile: content = OperationSpecification.read( polyaxonfile, is_preset=True).to_dict(dump=True) owner, project_name, run_uuid = get_project_run_or_local( project or ctx.obj.get("project"), uid or ctx.obj.get("run_uuid"), is_cli=True, ) try: polyaxon_client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) response = polyaxon_client.resume(override_config=content) Printer.print_success("Run was resumed with uid {}".format( response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not resume run `{}`.".format(run_uuid)) sys.exit(1)
def create(self, validated_data): is_managed = validated_data["is_managed"] content = validated_data.get("content") if content: is_managed = True if is_managed is None else is_managed if is_managed and not content: raise ValidationError( "Managed runs require a content with valid specification") user = validated_data.get("user") if is_managed: try: op_spec = OperationSpecification.read(content) except Exception as e: raise ValidationError(e) return compile_operation_run( project_id=validated_data["project"].id, user_id=user.id if user else None, op_spec=op_spec, name=validated_data.get("name"), description=validated_data.get("description"), tags=validated_data.get("tags"), ) else: return create_run( project_id=validated_data["project"].id, user_id=user.id if user else None, name=validated_data.get("name"), description=validated_data.get("description"), tags=validated_data.get("tags"), )
def test_create_run_with_templated_service_spec(self): config_dict = get_fxt_service_with_inputs() spec = OperationSpecification.read(values=config_dict) run = compile_operation_run( project_id=self.project.id, user_id=self.user.id, op_spec=spec ) assert run.kind == V1RunKind.SERVICE assert run.name == "foo" assert run.description == "a description" assert set(run.tags) == {"backend", "lab"} job_spec = CompiledOperationSpecification.read(run.content) assert job_spec.run.container.image == "{{ image }}" compiled_operation = CompiledOperationSpecification.read(run.content) compiled_operation = CompiledOperationSpecification.apply_params( compiled_operation, params=spec.params ) compiled_operation = CompiledOperationSpecification.apply_operation_contexts( compiled_operation ) CompiledOperationSpecification.apply_runtime_contexts(compiled_operation) run.content = compiled_operation.to_dict(dump=True) run.save(update_fields=["content"]) job_spec = CompiledOperationSpecification.read(run.content) job_spec = CompiledOperationSpecification.apply_runtime_contexts(job_spec) assert job_spec.run.container.image == "foo/bar"
def setUp(self): super().setUp() op_spec = OperationSpecification.read({ "version": 1.1, "kind": "operation", "name": "foo", "description": "a description", "tags": ["tag1", "tag2"], "trigger": "all_succeeded", "component": { "name": "build-template", "tags": ["tag1", "tag2"], "run": { "kind": V1RunKind.JOB, "container": { "image": "test" }, "init": [{ "connection": "foo", "git": { "revision": "dev" } }], }, }, }) self.compiled_operation = OperationSpecification.compile_operation( op_spec) self.preset = { "runPatch": {}, "patchStrategy": V1PatchStrategy.POST_MERGE }
def setUp(self): super().setUp() self.object.delete() values = { "version": 1.1, "kind": "operation", "name": "foo", "description": "a description", "params": {"image": {"value": "foo/bar"}}, "component": { "name": "build-template", "inputs": [{"name": "image", "type": "str"}], "tags": ["tag1", "tag2"], "run": { "kind": V1RunKind.JOB, "container": {"image": "{{ image }}"}, "init": [{"connection": "foo", "git": {"revision": "dev"}}], }, }, } op_spec = OperationSpecification.read(values=values) self.object = compile_operation_run( project_id=self.project.id, op_spec=op_spec, user_id=self.user.id ) self.url = "/{}/{}/{}/runs/{}/".format( API_V1, self.user.username, self.project.name, self.object.uuid.hex )
def get_op_spec_by_index(self, idx): from polyaxon.polyaxonfile import OperationSpecification op_dict = self.operations[idx].to_dict() op_dict[OperationSpecification.VERSION] = op_dict.get( OperationSpecification.VERSION, SCHEMA_VERSION) return OperationSpecification.read(op_dict)
def create(self, validated_data): is_managed = validated_data["is_managed"] content = validated_data.get("content") meta_info = validated_data.get("meta_info") or {} if content: is_managed = True if is_managed is None else is_managed if is_managed and not content: raise ValidationError( "Managed runs require a content with valid specification" ) project_id = validated_data["project"].id user = validated_data.get("user") name = validated_data.get("name") description = validated_data.get("description") tags = validated_data.get("tags") pending = validated_data.get("pending") # Check the deprecated `is_approved` flag if pending is None: is_approved = validated_data.get("is_approved") if is_approved is False: pending = V1RunPending.UPLOAD if is_managed or content: try: op_spec = OperationSpecification.read(content) except Exception as e: raise ValidationError(e) if op_spec.is_template(): raise ValidationError( "Received a template polyaxonfile, " "Please customize the specification or disable the template." ) try: return compile_operation_run( project_id=project_id, user_id=user.id if user else None, op_spec=op_spec, name=name, description=description, tags=tags, meta_info=meta_info, is_managed=is_managed, pending=pending, supported_kinds=validated_data.get("supported_kinds"), supported_owners=validated_data.get("supported_owners"), ) except (MarshmallowValidationError, PolyaxonException, ValueError) as e: raise ValidationError(e) else: return create_run( project_id=project_id, user_id=user.id if user else None, name=name, description=description, tags=tags, meta_info=meta_info, )
def setUp(self): super().setUp() self.user = UserFactory() self.user2 = UserFactory() self.project = ProjectFactory() op_spec = OperationSpecification.read(values=get_fxt_job_with_inputs()) self.run = compile_operation_run(project_id=self.project.id, op_spec=op_spec, user_id=self.user.id)
def test_create_run_with_service_spec(self): config_dict = get_fxt_service() spec = OperationSpecification.read(values=config_dict) run = compile_operation_run( project_id=self.project.id, user_id=self.user.id, op_spec=spec ) assert run.kind == V1RunKind.SERVICE assert run.name == "foo" assert run.description == "a description" assert set(run.tags) == {"backend", "lab", "tag1", "tag2"} service_spec = CompiledOperationSpecification.read(run.content) assert service_spec.run.container.image == "jupyter"
def create(self, name=None, tags=None, description=None, content=None): operation = polyaxon_sdk.V1OperationBody() if name: operation.name = name if tags: operation.tags = tags if description: operation.description = description if content: try: specification = OperationSpecification.read(content) except Exception as e: raise PolyaxonClientException("Client error: %s" % e) from e operation.content = specification.to_dict(dump=True) else: operation.is_managed = False if self.client: try: run = self.client.runs_v1.create_run(owner=self.owner, project=self.project, body=operation) except (ApiException, HTTPError) as e: raise PolyaxonClientException("Client error: %s" % e) from e if not run: raise PolyaxonClientException("Could not create a run.") else: run = polyaxon_sdk.V1Run( name=operation.name, tags=operation.tags, description=operation.description, content=operation.content, is_managed=operation.is_managed, ) self._run = run self._run_uuid = run.uuid if self.artifacts_path: self.set_run_event_logger() if self.track_code: self.log_code_ref() if self.track_env: self.log_run_env() if not settings.CLIENT_CONFIG.is_managed: self._start() else: self._register_wait() return self
def restart( ctx, project, uid, name, tags, description, copy, copy_dirs, copy_files, polyaxonfile, ): """Restart run. Uses /docs/core/cli/#caching Examples: \b $ polyaxon run --uid=8aac02e3a62a4f0aaa257c59da5eab80 restart """ content = None if polyaxonfile: content = OperationSpecification.read( polyaxonfile, is_preset=True).to_dict(dump=True) owner, project_name, run_uuid = get_project_run_or_local( project or ctx.obj.get("project"), uid or ctx.obj.get("run_uuid"), is_cli=True, ) try: polyaxon_client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) response = polyaxon_client.restart( name=name, description=description, tags=tags, override_config=content, copy=copy, copy_dirs=copy_dirs, copy_files=copy_files, ) Printer.print_success("Run was {} with uid {}".format( "copied" if copy else "restarted", response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not restart run `{}`.".format(run_uuid)) sys.exit(1)
def create(self, validated_data): is_managed = validated_data["is_managed"] content = validated_data.get("content") if content: is_managed = True if is_managed is None else is_managed if is_managed and not content: raise ValidationError( "Managed runs require a content with valid specification" ) project_id = validated_data["project"].id user = validated_data.get("user") name = validated_data.get("name") description = validated_data.get("description") tags = validated_data.get("tags") if is_managed or content: try: op_spec = OperationSpecification.read(content) except Exception as e: raise ValidationError(e) if op_spec.is_template(): raise ValidationError( "Received a template polyaxonfile, " "Please customize the specification or disable the template." ) try: return compile_operation_run( project_id=project_id, user_id=user.id if user else None, op_spec=op_spec, name=name, description=description, tags=tags, is_managed=is_managed, supported_kinds=validated_data.get("supported_kinds"), ) except (PolyaxonException, ValueError) as e: raise ValidationError(e) else: return create_run( project_id=project_id, user_id=user.id if user else None, name=name, description=description, tags=tags, )
def make_and_convert( owner_name: str, project_name: str, run_uuid: str, run_name: str, content: str, ): operation = OperationSpecification.read(content) compiled_operation = OperationSpecification.compile_operation(operation) return make( owner_name=owner_name, project_name=project_name, project_uuid=project_name, run_uuid=run_uuid, run_name=run_name, run_path=run_uuid, compiled_operation=compiled_operation, params=None, )
def test_create_run_with_job_spec(self): config_dict = get_fxt_job() spec = OperationSpecification.read(values=config_dict) run = compile_operation_run(project_id=self.project.id, user_id=self.user.id, op_spec=spec) assert run.kind == V1RunKind.JOB assert run.name == "foo" assert run.description == "a description" assert set(run.tags) == {"tag1", "tag2"} # Check compiled operation passes compiled_operation = CompiledOperationSpecification.read(run.content) compiled_operation = CompiledOperationSpecification.apply_params( compiled_operation) CompiledOperationSpecification.apply_run_contexts(compiled_operation, ) # Check job job_spec = CompiledOperationSpecification.read(run.content) assert job_spec.run.container.image == "test" job_spec = CompiledOperationSpecification.apply_context(job_spec) assert job_spec.run.container.image == "test"
def make_and_convert( owner_name: str, project_name: str, run_uuid: str, run_name: str, content: str, default_auth: bool = False, ): operation = OperationSpecification.read(content) compiled_operation = OperationSpecification.compile_operation(operation) return make( owner_name=owner_name, project_name=project_name, project_uuid=project_name, run_uuid=run_uuid, run_name=run_name, run_path=run_uuid, compiled_operation=compiled_operation, params=operation.params, converters=PLATFORM_CONVERTERS, default_auth=default_auth, )
def get_empty_operation(self): return OperationSpecification.read( {"version": pkg.SCHEMA_VERSION, "hubRef": "test"} )
def get_full_preset(self): return OperationSpecification.read({ "version": pkg.SCHEMA_VERSION, "name": self.PATCH_STR_VALUE, "isPreset": True, "description": self.PATCH_STR_VALUE, "tags": [ "{}1".format(self.PATCH_STR_VALUE), "{}2".format(self.PATCH_STR_VALUE), ], "presets": [self.PATCH_STR_VALUE], "queue": "{}/{}".format(self.PATCH_STR_VALUE, self.PATCH_STR_VALUE), "cache": { "disable": True, "ttl": self.PATCH_INT_VALUE, }, "termination": { "maxRetries": self.PATCH_INT_VALUE, "ttl": self.PATCH_INT_VALUE, "timeout": self.PATCH_INT_VALUE, }, "plugins": { "auth": True, "shm": True, "collectLogs": True, "collectArtifacts": True, "collectResources": True, }, "actions": [ { "hubRef": "{}1".format(self.PATCH_STR_VALUE) }, { "hubRef": "{}2".format(self.PATCH_STR_VALUE), "label": "customLabel", "many": True, }, ], "hooks": [ { "trigger": "succeeded", "connection": "{}1".format(self.PATCH_STR_VALUE), }, { "connection": "{}1".format(self.PATCH_STR_VALUE), "hubRef": "{}2".format(self.PATCH_STR_VALUE), }, ], "params": { "patch-key1": { "value": "{}2".format(self.PATCH_STR_VALUE) }, "patch-key2": { "value": "{}1".format(self.PATCH_STR_VALUE) }, }, "runPatch": { "init": [{ "connection": self.PATCH_STR_VALUE, "git": { "revision": "dev" } }], "connections": [ "{}1".format(self.PATCH_STR_VALUE), "{}2".format(self.PATCH_STR_VALUE), ], "container": { "resources": { "requests": { "cpu": self.PATCH_INT_VALUE, "memory": self.PATCH_INT_VALUE, } } }, "environment": { "nodeSelector": { "polyaxon-patch": "core" }, "serviceAccountName": self.PATCH_STR_VALUE, "imagePullSecrets": [ "{}1".format(self.PATCH_STR_VALUE), "{}2".format(self.PATCH_STR_VALUE), ], }, }, "schedule": { "kind": "exact_time", "startAt": self.PATCH_DT_VALUE }, "events": None, "matrix": { "concurrency": self.PATCH_INT_VALUE, "kind": "mapping", "values": [ { "a": self.PATCH_INT_VALUE }, { "c": self.PATCH_INT_VALUE }, ], }, "dependencies": [ "{}1".format(self.PATCH_STR_VALUE), "{}2".format(self.PATCH_STR_VALUE), ], "trigger": "all_succeeded", "conditions": "", "skipOnUpstreamSkip": True, })
def get_full_operation(self): return OperationSpecification.read( { "version": pkg.SCHEMA_VERSION, "name": self.DEFAULT_STR_VALUE, "description": self.DEFAULT_STR_VALUE, "tags": [ "{}1".format(self.DEFAULT_STR_VALUE), "{}2".format(self.DEFAULT_STR_VALUE), ], "presets": [self.DEFAULT_STR_VALUE], "queue": "{}/{}".format(self.DEFAULT_STR_VALUE, self.DEFAULT_STR_VALUE), "cache": { "disable": False, "ttl": self.DEFAULT_INT_VALUE, }, "termination": { "maxRetries": self.DEFAULT_INT_VALUE, "ttl": self.DEFAULT_INT_VALUE, "timeout": self.DEFAULT_INT_VALUE, }, "plugins": { "auth": False, "shm": False, "collectLogs": False, "collectArtifacts": False, "collectResources": False, }, "hooks": [ { "hubRef": "{}1".format(self.DEFAULT_STR_VALUE), "trigger": "succeeded", "connection": "{}1".format(self.DEFAULT_STR_VALUE), }, { "connection": "{}2".format(self.DEFAULT_STR_VALUE), "hubRef": "{}2".format(self.DEFAULT_STR_VALUE), }, ], "params": { "patch-key1": {"value": "{}2".format(self.DEFAULT_STR_VALUE)}, "patch-key2": {"value": "{}1".format(self.DEFAULT_STR_VALUE)}, }, "runPatch": { "init": [ { "connection": self.DEFAULT_STR_VALUE, "git": {"revision": self.DEFAULT_STR_VALUE}, } ], "connections": [ "{}1".format(self.DEFAULT_STR_VALUE), "{}2".format(self.DEFAULT_STR_VALUE), ], "container": { "resources": {"requests": {"cpu": self.DEFAULT_INT_VALUE}} }, "environment": { "nodeSelector": {"polyaxon": "core"}, "serviceAccountName": self.DEFAULT_STR_VALUE, "imagePullSecrets": [ "{}1".format(self.DEFAULT_STR_VALUE), "{}2".format(self.DEFAULT_STR_VALUE), ], }, }, "schedule": { "kind": "cron", "cron": "0 0 * * *", "startAt": self.DEFAULT_DT_VALUE, "endAt": self.DEFAULT_DT_VALUE, }, "events": [ { "kinds": [V1EventKind.RUN_STATUS_SCHEDULED], "ref": "{}1".format(self.DEFAULT_STR_VALUE), }, { "kinds": [V1EventKind.RUN_STATUS_SCHEDULED], "ref": "{}2".format(self.DEFAULT_STR_VALUE), }, ], "joins": [ { "query": "{}1".format(self.DEFAULT_STR_VALUE), "sort": "{}1".format(self.DEFAULT_STR_VALUE), "params": { "u": {"value": "{}1".format(self.DEFAULT_STR_VALUE)}, }, }, { "query": "{}2".format(self.DEFAULT_STR_VALUE), "sort": "{}2".format(self.DEFAULT_STR_VALUE), "params": { "v": { "value": "{}2".format(self.DEFAULT_STR_VALUE), "contextOnly": True, }, }, }, ], "matrix": { "concurrency": self.DEFAULT_INT_VALUE, "kind": "mapping", "values": [ {"a": self.DEFAULT_INT_VALUE}, {"b": self.DEFAULT_INT_VALUE}, ], }, "dependencies": [ "{}1".format(self.DEFAULT_STR_VALUE), "{}2".format(self.DEFAULT_STR_VALUE), ], "trigger": "all_succeeded", "conditions": self.DEFAULT_STR_VALUE, "skipOnUpstreamSkip": True, "hubRef": self.DEFAULT_STR_VALUE, } )