def test_get_artifacts_path_container_with_managed_mount_store(self): store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/claim/path", volume_claim="claim"), ) container = get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent"), artifacts_store=store, run_path="run_uid", auto_resume=True, ) init_args = init_artifact_context_args("run_uid") init_args.append( get_volume_args( store=store, mount_path=CONTEXT_MOUNT_ARTIFACTS, artifacts=V1ArtifactsType(dirs=["run_uid"]), )) assert container == get_base_store_container( container=k8s_schemas.V1Container(name="default"), container_name=INIT_ARTIFACTS_CONTAINER.format("default"), polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent"), store=store, env=[], env_from=[], volume_mounts=[get_artifacts_context_mount()], args=[" ".join(init_args)], )
def get_artifacts_path_container( polyaxon_init: V1PolyaxonInitContainer, artifacts_store: V1ConnectionType, run_path: str, auto_resume: bool, ) -> Optional[k8s_schemas.V1Container]: if not artifacts_store: raise PolypodException("Init artifacts container requires a store.") init_args = init_artifact_context_args(run_path=run_path) if auto_resume: init_args.append( get_volume_args( store=artifacts_store, mount_path=CONTEXT_MOUNT_ARTIFACTS, artifacts=V1ArtifactsType(dirs=[run_path]), )) container_name = generate_container_name(INIT_ARTIFACTS_CONTAINER_PREFIX, DEFAULT) container = k8s_schemas.V1Container(name=container_name) return get_base_store_container( container_name=container_name, container=container, polyaxon_init=polyaxon_init, store=artifacts_store, env=[], env_from=[], volume_mounts=[get_artifacts_context_mount()], # If we are dealing with a volume we need to make sure the path exists for the user # We also clean the path if this is not a resume run args=[" ".join(init_args)], )
def test_get_volume_args_s3(self): s3_store = V1ConnectionType( name="test_s3", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3//:foo"), ) path_to = "/path-to/" path_from = os.path.join(s3_store.store_path, "") assert get_volume_args(s3_store, path_to, None) == "; ".join( [ get_or_create_args(path=path_to), cp_s3_args(path_from=path_from, path_to=path_to, is_file=False), ] ) s3_store = V1ConnectionType( name="test_s3", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3//:foo"), ) base_path = "/path/to/" path_to1 = "/path/to/path1" path_to2 = "/path/to/path2" path_from1 = os.path.join(s3_store.store_path, "path1") path_from2 = os.path.join(s3_store.store_path, "path2") assert get_volume_args( s3_store, "/path/to", artifacts=V1ArtifactsType(files=["path1", "path2"]) ) == "; ".join( [ get_or_create_args(path=base_path), cp_s3_args(path_from=path_from1, path_to=path_to1, is_file=True), get_or_create_args(path=base_path), cp_s3_args(path_from=path_from2, path_to=path_to2, is_file=True), ] )
def test_get_artifacts_path_container_with_bucket_store(self): store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), ) container = get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent"), artifacts_store=store, run_path="run_uid", auto_resume=True, ) init_args = init_artifact_context_args("run_uid") init_args.append( get_volume_args( store=store, mount_path=CONTEXT_MOUNT_ARTIFACTS, artifacts=V1ArtifactsType(dirs=["run_uid"]), )) assert container == get_base_store_container( container=k8s_schemas.V1Container(name="default"), container_name=generate_container_name( INIT_ARTIFACTS_CONTAINER_PREFIX, "default", False), polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent"), store=store, env=[], env_from=[], volume_mounts=[get_artifacts_context_mount()], args=[" ".join(init_args)], )
def clone_run( run: BaseRun, cloning_kind: str, user_id: int = None, name: str = None, description: str = None, content: str = None, readme: str = None, tags: List[int] = None, supported_kinds: Set[str] = None, **kwargs, ) -> BaseRun: op_spec = V1Operation.read(run.raw_content) meta_info = kwargs.pop("meta_info", {}) or {} original_meta_info = run.meta_info or {} original_uuid = run.uuid.hex upload_artifacts = original_meta_info.get(META_UPLOAD_ARTIFACTS) if upload_artifacts: meta_info[META_UPLOAD_ARTIFACTS] = upload_artifacts if cloning_kind == V1CloningKind.COPY and META_COPY_ARTIFACTS not in meta_info: # Handle default copy mode meta_info[META_COPY_ARTIFACTS] = V1ArtifactsType( dirs=[original_uuid]).to_dict() if META_COPY_ARTIFACTS not in meta_info and upload_artifacts: # Handle default copy mode meta_info[META_COPY_ARTIFACTS] = V1ArtifactsType( dirs=["{}/{}".format(original_uuid, upload_artifacts)]).to_dict() compiled_operation, instance = operations.init_run( project_id=run.project_id, user_id=user_id or run.user_id, name=name or run.name, description=description or run.description, readme=readme or run.readme, op_spec=op_spec, original_id=run.id, original_uuid=original_uuid, cloning_kind=cloning_kind, tags=tags or run.tags, override=content, supported_kinds=supported_kinds, meta_info=meta_info, **kwargs, ) instance.save() return instance
def test_get_volume_args_host(self): host_path_store = V1ConnectionType( name="test_path", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection( mount_path="/tmp", host_path="/tmp", read_only=True ), ) path_to = "/path/to/" path_from = os.path.join(host_path_store.store_path, "") assert get_volume_args(host_path_store, path_to, None) == "; ".join( [ get_or_create_args(path=path_to), cp_copy_args(path_from=path_from, path_to=path_to, is_file=False), ] ) host_path_store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection( mount_path="/tmp", host_path="/tmp", read_only=True ), ) base_path = "/path/to/" path_to1 = "/path/to/path1" path_to2 = "/path/to/path2" path_from1 = os.path.join(host_path_store.store_path, "path1") path_from2 = os.path.join(host_path_store.store_path, "path2") assert get_volume_args( host_path_store, "/path/to", artifacts=V1ArtifactsType(dirs=["path1", "path2"]), ) == "; ".join( [ get_or_create_args(path=base_path), cp_copy_args(path_from=path_from1, path_to=path_to1, is_file=False), get_or_create_args(path=base_path), cp_copy_args(path_from=path_from2, path_to=path_to2, is_file=False), ] )
def _get_meta_artifacts_presets(self) -> List: if not self.run.meta_info or META_COPY_ARTIFACTS not in self.run.meta_info: return [] artifacts = self.run.meta_info.pop(META_COPY_ARTIFACTS) artifacts = V1ArtifactsType.read(artifacts) def get_relative_to_run_artifacts(v: str): paths = v.split("/")[1:] paths = ["{{ globals.run_artifacts_path }}"] + paths return "/".join(paths) # Populate all paths if artifacts.dirs: artifacts.dirs = [[d, get_relative_to_run_artifacts(d)] for d in artifacts.dirs] if artifacts.files: artifacts.files = [[d, get_relative_to_run_artifacts(d)] for d in artifacts.files] init = V1Init(artifacts=artifacts) return [{"runPatch": {"init": [init.to_dict()]}}]
def test_get_volume_args_claim(self): claim_store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/tmp", volume_claim="test", read_only=True), ) path_to = "/path/to/" path_from = os.path.join(claim_store.store_path, "") assert get_volume_args(claim_store, path_to, None) == "; ".join([ get_or_create_args(path=path_to), cp_copy_args(path_from=path_from, path_to=path_to, is_file=False), ]) claim_store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/tmp", volume_claim="test", read_only=True), ) base_path = "/path/to/" path_to1 = "/path/to/path1" path_to2 = "/path/to/path2" path_from1 = os.path.join(claim_store.store_path, "path1") path_from2 = os.path.join(claim_store.store_path, "path2") assert get_volume_args( claim_store, "/path/to", artifacts=V1ArtifactsType(files=["path1", "path2"])) == "; ".join([ get_or_create_args(path=base_path), cp_copy_args(path_from=path_from1, path_to=path_to1, is_file=True), get_or_create_args(path=base_path), cp_copy_args(path_from=path_from2, path_to=path_to2, is_file=True), ])
def test_get_volume_args_gcs(self): gcs_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), ) path_to = "/path/to/" path_from = os.path.join(gcs_store.store_path, "") assert get_volume_args(gcs_store, path_to, None) == " ".join([ get_or_create_args(path=path_to), cp_gcs_args(path_from=path_from, path_to=path_to, is_file=False), ]) gcs_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="Congs//:foo"), ) base_path = "/path/to/" path_to1 = "/path/to/path1" path_to2 = "/path/to/path2" path_from1 = os.path.join(gcs_store.store_path, "path1") path_from2 = os.path.join(gcs_store.store_path, "path2") assert get_volume_args( gcs_store, "/path/to", artifacts=V1ArtifactsType(dirs=["path1", "path2"])) == " ".join([ get_or_create_args(path=base_path), cp_gcs_args(path_from=path_from1, path_to=path_to1, is_file=False), get_or_create_args(path=base_path), cp_gcs_args(path_from=path_from2, path_to=path_to2, is_file=False), ])
def test_get_volume_args_az(self): az_store = V1ConnectionType( name="test_az", kind=V1ConnectionKind.WASB, schema=V1BucketConnection(bucket="Conwasb://[email protected]"), ) path_to = "/path/to/" path_from = os.path.join(az_store.store_path, "") assert get_volume_args(az_store, path_to, None) == "; ".join( [ get_or_create_args(path=path_to), cp_wasb_args(path_from=path_from, path_to=path_to, is_file=False), ] ) az_store = V1ConnectionType( name="test_az", kind=V1ConnectionKind.WASB, schema=V1BucketConnection(bucket="Conwasb://[email protected]"), ) base_path = "/path/to/" path_to1 = "/path/to/path1" path_to2 = "/path/to/path2" path_from1 = os.path.join(az_store.store_path, "path1") path_from2 = os.path.join(az_store.store_path, "path2") assert get_volume_args( az_store, "/path/to", artifacts=V1ArtifactsType(files=["path1"], dirs=["path2"]), ) == "; ".join( [ get_or_create_args(path=base_path), cp_wasb_args(path_from=path_from1, path_to=path_to1, is_file=True), get_or_create_args(path=base_path), cp_wasb_args(path_from=path_from2, path_to=path_to2, is_file=False), ] )
def init_run( self, project_id: int, user_id: int, op_spec: V1Operation = None, compiled_operation: V1CompiledOperation = None, name: str = None, description: str = None, tags: str = None, override: Union[str, Dict] = None, params: Dict = None, readme: str = None, original_id: int = None, original_uuid: int = None, cloning_kind: str = None, is_managed: bool = True, is_approved: bool = True, meta_info: Dict = None, supported_kinds: Set[str] = None, init: Optional[List[V1Init]] = None, **kwargs, ) -> Tuple[V1CompiledOperation, BaseRun]: if op_spec: op_spec, kwargs = self.set_spec(op_spec, **kwargs) if op_spec: if not compiled_operation or override: compiled_operation = OperationSpecification.compile_operation( op_spec, override=override ) params = op_spec.params params = params or {} inputs = {p: pv.value for p, pv in params.items() if pv.is_literal} params = {p: pv.to_dict() for p, pv in params.items()} kind = None meta_info = meta_info or {} if compiled_operation: if is_approved and compiled_operation.is_approved is not None: is_approved = compiled_operation.is_approved name = name or compiled_operation.name description = description or compiled_operation.description tags = tags or compiled_operation.tags kind, runtime = self.get_kind(compiled_operation) kind, runtime, meta_info = self.get_meta_info( compiled_operation, kind, runtime, meta_info, **kwargs ) self.supports_kind(kind, runtime, supported_kinds, is_managed) if cloning_kind == V1CloningKind.COPY: if runtime not in {V1RunKind.JOB, V1RunKind.SERVICE}: raise ValueError( "Operation with kind `{}` does not support restart with copy mode.".format( runtime ) ) compiled_operation.run.add_init( V1Init( artifacts=V1ArtifactsType( dirs=[[original_uuid, "{{ globals.run_artifacts_path }}"]] ) ) ) if init: if runtime not in {V1RunKind.JOB, V1RunKind.SERVICE}: raise ValueError( "Operation with kind `{}` does not support " "additional init containers.".format(runtime) ) compiled_operation.run.add_init(init) kwargs["content"] = compiled_operation.to_dict(dump=True) instance = get_run_model()( project_id=project_id, user_id=user_id, name=name, description=description, tags=tags, readme=readme, params=params, inputs=inputs, kind=kind, runtime=runtime, meta_info=meta_info, original_id=original_id, cloning_kind=cloning_kind, is_managed=is_managed, is_approved=is_approved, status_conditions=[ V1StatusCondition.get_condition( type=V1Statuses.CREATED, status="True", reason=kwargs.pop("reason", "OperationServiceInit"), message=kwargs.pop("message", "Run is created"), ).to_dict() ], **self.sanitize_kwargs(**kwargs), ) return compiled_operation, instance
def run( ctx, name: str, owner: str, project_name: str, description: str, tags: List[str], op_spec: V1Operation, log: bool, upload: bool, upload_to: str, upload_from: str, watch: bool, eager: bool, ): polyaxon_client = RunClient(owner=owner, project=project_name) def cache_run(data): config = polyaxon_client.client.sanitize_for_serialization(data) cache.cache( config_manager=RunConfigManager, config=config, owner=owner, project=project_name, ) def create_run( is_managed: bool = True, meta_info: Dict = None, pending: str = None ): try: response = polyaxon_client.create( name=name, description=description, tags=tags, content=op_spec, is_managed=is_managed, meta_info=meta_info, pending=pending, ) Printer.print_success("A new run `{}` was created".format(response.uuid)) if not eager: cache_run(response) click.echo( "You can view this run on Polyaxon UI: {}".format( get_dashboard_url( subpath="{}/{}/runs/{}".format( owner, project_name, response.uuid ) ) ) ) return response.uuid except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not create a run.", http_messages_mapping={ 404: "Make sure you have a project initialized in your current workdir, " "otherwise you need to pass a project with `-p/--project`. " "The project {}/{} does not exist.".format(owner, project_name) }, ) sys.exit(1) def refresh_run(): try: polyaxon_client.refresh_data() except (ApiException, HTTPError) as e: handle_cli_error( e, message="The current eager operation does not exist anymore." ) sys.exit(1) def delete_run(): try: polyaxon_client.delete() except (ApiException, HTTPError) as e: handle_cli_error( e, message="The current eager operation does not exist anymore." ) sys.exit(1) click.echo("Creating a new run...") run_meta_info = None if eager: run_meta_info = {META_EAGER_MODE: True} if upload: run_meta_info = run_meta_info or {} run_meta_info[META_UPLOAD_ARTIFACTS] = upload_to or DEFAULT_UPLOADS_PATH run_uuid = create_run( not eager, run_meta_info, pending=V1RunPending.UPLOAD if upload else None ) ctx.obj = {"project": "{}/{}".format(owner, project_name), "run_uuid": run_uuid} if upload: ctx.invoke( run_upload, path_to=upload_to, path_from=upload_from, sync_failure=True ) ctx.invoke(approve) if eager: from polyaxon.polyaxonfile.manager import get_eager_matrix_operations refresh_run() # Prepare artifacts if upload: run_meta_info = { META_UPLOAD_ARTIFACTS: upload_to or DEFAULT_UPLOADS_PATH, META_COPY_ARTIFACTS: V1ArtifactsType(dirs=[run_uuid]).to_dict(), } compiled_operation = V1CompiledOperation.read(polyaxon_client.run_data.content) matrix_content = polyaxon_client.run_data.raw_content # Delete matrix placeholder click.echo("Cleaning matrix run placeholder...") delete_run() # Suggestions click.echo("Starting eager mode...") for op_spec in get_eager_matrix_operations( content=matrix_content, compiled_operation=compiled_operation, is_cli=True, ): create_run(meta_info=run_meta_info) return # Check if we need to invoke logs if watch and not eager: ctx.invoke(statuses, watch=True) # Check if we need to invoke logs if log and not eager: ctx.invoke(run_logs)
def convert_to_artifacts_init(x): if not isinstance(x, Mapping): x = convert_to_dict(x, key) return V1ArtifactsType.from_dict(x)
def test_get_init_containers_with_claim_outputs(self): store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/claim/path", volume_claim="claim", read_only=True), ) # No context to enable the outputs containers = self.converter.get_init_containers( contexts=None, artifacts_store=store.name, init_connections=None, connection_by_names={}, init_containers=[], polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [] # Enable outputs containers = self.converter.get_init_containers( contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=True, collect_logs=False)), artifacts_store=store, connection_by_names={}, init_connections=None, init_containers=[], polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [ get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), artifacts_store=store, run_path=self.converter.run_path, auto_resume=True, ), ] # Use store for init containers = self.converter.get_init_containers( contexts=None, artifacts_store=None, connection_by_names={store.name: store}, init_connections=[V1Init(connection=store.name)], init_containers=[], polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [ get_store_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), connection=store, artifacts=None, env=self.converter.get_init_service_env_vars(), ) ] # Use store for init and outputs containers = self.converter.get_init_containers( contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=True, collect_logs=False)), artifacts_store=store, init_connections=[V1Init(connection=store.name)], connection_by_names={store.name: store}, init_containers=[], polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [ get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), artifacts_store=store, run_path=self.converter.run_path, auto_resume=True, ), get_store_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), connection=store, artifacts=None, env=self.converter.get_init_service_env_vars(), ), ] # Add Store store1 = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3://foo"), secret=None, ) containers = self.converter.get_init_containers( contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=True, collect_logs=False, auth=True)), artifacts_store=store, init_connections=[ V1Init( connection=store.name, artifacts=V1ArtifactsType(files=["/foo", "/bar"]), ), V1Init( connection=store1.name, artifacts=V1ArtifactsType(files=["/foo", "/bar"]), ), ], connection_by_names={ store.name: store, store1.name: store1 }, init_containers=[], polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [ get_auth_context_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), env=self.converter.get_auth_service_env_vars(), ), get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), artifacts_store=store, run_path=self.converter.run_path, auto_resume=True, ), get_store_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), connection=store, artifacts=V1ArtifactsType(files=["/foo", "/bar"]), env=self.converter.get_init_service_env_vars(), ), get_store_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), connection=store1, artifacts=V1ArtifactsType(files=["/foo", "/bar"]), env=self.converter.get_init_service_env_vars(), ), ]
def init_run( self, project_id: int, user_id: int, op_spec: V1Operation = None, compiled_operation: V1CompiledOperation = None, name: str = None, description: str = None, tags: str = None, override: Union[str, Dict] = None, override_post: bool = True, params: Dict = None, readme: str = None, original_id: int = None, original_uuid: int = None, cloning_kind: str = None, is_managed: bool = True, supported_kinds: Set[str] = None, **kwargs, ) -> Tuple[V1CompiledOperation, BaseRun]: content = None raw_content = None if op_spec: op_spec = self.set_spec(op_spec) raw_content = op_spec.to_dict(dump=True) if op_spec: if not compiled_operation or override: compiled_operation = OperationSpecification.compile_operation( op_spec, override=override, override_post=override_post) params = op_spec.params params = params or {} inputs = {p: pv.value for p, pv in params.items() if pv.is_literal} params = {p: pv.to_dict() for p, pv in params.items()} kind = None meta_info = {} if compiled_operation: name = name or compiled_operation.name description = description or compiled_operation.description tags = tags or compiled_operation.tags kind, meta_kind = self.get_kind(compiled_operation) kind, meta_info = self.get_meta_info(compiled_operation, kind, meta_kind) self.supports_kind(kind, meta_kind, supported_kinds, is_managed) if cloning_kind == V1CloningKind.COPY: if meta_kind not in {V1RunKind.JOB, V1RunKind.SERVICE}: raise ValueError( "Operation with kind `{}` does not support restart with copy mode." ) compiled_operation.run.add_init( V1Init(artifacts=V1ArtifactsType(dirs=[original_uuid]))) content = compiled_operation.to_dict(dump=True) instance = get_run_model()( project_id=project_id, user_id=user_id, name=name, description=description, tags=tags, readme=readme, raw_content=raw_content, content=content, params=params, inputs=inputs, kind=kind, meta_info=meta_info, original_id=original_id, cloning_kind=cloning_kind, is_managed=is_managed, status_conditions=[ V1StatusCondition.get_condition( type=V1Statuses.CREATED, status="True", reason="PolyaxonRunCreated", message="Run is created", ).to_dict() ], **self.sanitize_kwargs(**kwargs), ) return compiled_operation, instance