def init_artifact_context_args(run_path: str) -> List[str]: return [ 'if [ ! -d "{dir}" ]; then mkdir -p {dir}; fi;'.format( dir=CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_path)), 'if [ ! -d "{dir}" ]; then mkdir -p {dir}; fi;'.format( dir=CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_path) + "/outputs"), ]
def test_get_store_container_mount_stores(self): # Managed store store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection( mount_path="/tmp", volume_claim="test", read_only=True ), ) container = get_store_container( polyaxon_init=V1PolyaxonInitContainer( image="foo/foo", image_tag="foo", image_pull_policy="IfNotPresent" ), connection=store, artifacts=None, ) mount_path = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name) assert container.name == INIT_ARTIFACTS_CONTAINER.format(store.name) assert container.image == "foo/foo:foo" assert container.image_pull_policy == "IfNotPresent" assert container.command == ["/bin/sh", "-c"] assert container.args == [ get_volume_args(store=store, mount_path=mount_path, artifacts=None) ] assert container.env == get_connection_env_var(connection=store, secret=None) assert container.env_from == [] assert container.resources is not None assert container.volume_mounts == [ get_connections_context_mount( name=constants.CONTEXT_VOLUME_ARTIFACTS, mount_path=CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name), ), get_mount_from_store(store=store), ]
def test_event_logger_from_non_manged_run(self): settings.CLIENT_CONFIG.is_managed = False run = Run(project="owner-test.test") assert run.get_artifacts_path() == TEMP_RUN_ARTIFACTS assert run.get_outputs_path() == TEMP_RUN_ARTIFACTS assert run._event_logger is None # Add run id run = Run(project="owner-test.test", run_uuid="uuid") assert run.get_artifacts_path() == TEMP_RUN_ARTIFACTS assert run.get_outputs_path() == TEMP_RUN_ARTIFACTS assert run._event_logger is None run.set_artifacts_path() assert run.get_artifacts_path() == CONTEXT_MOUNT_ARTIFACTS_FORMAT.format("uuid") assert run.get_outputs_path() == CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("uuid") with patch("polyaxon.tracking.run.EventFileWriter") as mock_call: run.set_run_event_logger() assert mock_call.call_count == 1 with patch("polyaxon.tracking.run.ResourceFileWriter") as mock_call: run.set_run_resource_logger() assert mock_call.call_count == 1 settings.CLIENT_CONFIG.is_managed = True with patch("polyaxon.tracking.run.EventFileWriter") as event_call: with patch("polyaxon.tracking.run.ResourceFileWriter") as resource_call: run = Run(project="owner-test.test", run_uuid="uuid") assert event_call.call_count == 1 assert resource_call.call_count == 1 assert run.get_artifacts_path() == CONTEXT_MOUNT_ARTIFACTS_FORMAT.format("uuid") assert run.get_outputs_path() == CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("uuid")
def get_store_container( polyaxon_init: V1PolyaxonInitContainer, connection: V1ConnectionType, artifacts: V1ArtifactsType, container: Optional[k8s_schemas.V1Container] = None, env: List[k8s_schemas.V1EnvVar] = None, mount_path: str = None, ) -> Optional[k8s_schemas.V1Container]: container_name = INIT_ARTIFACTS_CONTAINER.format(connection.name) if not container: container = k8s_schemas.V1Container(name=container_name) volume_name = (get_volume_name(mount_path) if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS) mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS_FORMAT.format( connection.name) volume_mounts = [ get_connections_context_mount(name=volume_name, mount_path=mount_path) ] return get_base_store_container( container=container, container_name=container_name, polyaxon_init=polyaxon_init, store=connection, env=env, env_from=[], volume_mounts=volume_mounts, args=[ get_volume_args(store=connection, mount_path=mount_path, artifacts=artifacts) ], )
async def sync_logs( run_uuid: str, k8s_manager: AsyncK8SManager, pod: V1Pod, last_time: Optional[AwareDT], stream: bool = False, is_running: bool = True, ): path_from = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid) path_from = "{}/.tmpplxlogs".format(path_from) if not is_running: delete_path(path_from) return logs, _ = await query_k8s_pod_logs( k8s_manager=k8s_manager, pod=pod, last_time=last_time, stream=stream, ) if not logs: return path_from = "{}/{}".format(path_from, pod.metadata.name) check_or_create_path(path_from, is_dir=False) async with aiofiles.open(path_from, "w") as filepath: await filepath.write(V1Logs(logs=logs).to_dict(dump=True))
def get_dockerfile_init_container( polyaxon_init: V1PolyaxonInitContainer, dockerfile_args: V1DockerfileType, contexts: PluginsContextsSpec, run_path: str, env: List[k8s_schemas.V1EnvVar] = None, mount_path: Optional[str] = None, ) -> k8s_schemas.V1Container: env = to_list(env, check_none=True) env = env + [get_run_instance_env_var()] volume_name = (get_volume_name(mount_path) if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS) mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS volume_mounts = [ get_connections_context_mount(name=volume_name, mount_path=mount_path) ] if contexts and contexts.auth: volume_mounts.append(get_auth_context_mount(read_only=True)) return k8s_schemas.V1Container( name=generate_container_name(INIT_DOCKERFILE_CONTAINER_PREFIX), image=polyaxon_init.get_image(), image_pull_policy=polyaxon_init.image_pull_policy, command=["polyaxon", "docker", "generate"], args=[ "--build-context={}".format(dockerfile_args.to_dict(dump=True)), "--destination={}".format(mount_path), "--copy-path={}".format( CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_path) + "/outputs"), ], env=env, resources=polyaxon_init.get_resources(), volume_mounts=volume_mounts, )
def test_event_logger_from_a_managed_run(self): # Set managed flag settings.CLIENT_CONFIG.is_managed = True os.environ[POLYAXON_KEYS_RUN_INSTANCE] = "user.project_bar.runs.uid" run = Run() assert run.get_artifacts_path() == CONTEXT_MOUNT_ARTIFACTS_FORMAT.format("uid") assert run.get_outputs_path() == CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("uid") assert run._event_logger is None # Set collect flag os.environ[POLYAXON_KEYS_COLLECT_ARTIFACTS] = "true" os.environ[POLYAXON_KEYS_COLLECT_RESOURCES] = "true" # Add run id with patch("polyaxon.tracking.run.Run.set_run_event_logger") as event_call: with patch( "polyaxon.tracking.run.Run.set_run_resource_logger" ) as resource_call: Run(project="test.test", run_uuid="uuid") assert event_call.call_count == 1 assert resource_call.call_count == 1 # Set run info os.environ[POLYAXON_KEYS_RUN_INSTANCE] = "user.project_bar.runs.uid" # Add run id with patch("polyaxon.tracking.run.Run.set_run_event_logger") as event_call: with patch( "polyaxon.tracking.run.Run.set_run_resource_logger" ) as resource_call: Run() assert event_call.call_count == 1 assert resource_call.call_count == 1
def set_artifacts_path(self, artifacts_path: str = None): _artifacts_path = artifacts_path or CONTEXT_MOUNT_ARTIFACTS_FORMAT.format( self.run_uuid ) if artifacts_path: _outputs_path = "{}/outputs".format(artifacts_path) else: _outputs_path = CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format(self.run_uuid) self._artifacts_path = _artifacts_path self._outputs_path = _outputs_path
def set_artifacts_path(self, artifacts_path: str = None): """Sets an artifacts_path. Be careful, this method is called automatically when a job is running in-cluster and follows some flags that Polyaxon sets. Polyaxon has some processes to automatically sync your run's artifacts and outputs. Args: artifacts_path: str, optional """ _artifacts_path = artifacts_path or CONTEXT_MOUNT_ARTIFACTS_FORMAT.format( self.run_uuid) _outputs_path = "{}/outputs".format(_artifacts_path) self._artifacts_path = _artifacts_path self._outputs_path = _outputs_path
def sync_artifacts(last_check: Optional[datetime], run_uuid: str): new_check = now() connection_type = get_artifacts_connection() path_from = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid) # check if there's a path to sync if os.path.exists(path_from): path_to = os.path.join(connection_type.store_path, run_uuid) upload_file_or_dir( path_from=path_from, path_to=path_to, is_file=False, workers=5, last_time=last_check, connection_type=connection_type, ) return new_check
def sync_artifacts(last_check: Optional[datetime], run_uuid: str): connection_type = get_artifacts_connection() path_from = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid) new_check = path_last_modified(path_from) # check if there's a path to sync if os.path.exists(path_from): path_to = os.path.join(connection_type.store_path, run_uuid) upload_file_or_dir( path_from=path_from, path_to=path_to, is_file=False, workers=5, last_time=last_check, connection_type=connection_type, exclude=["plxlogs"], ) # Check if this run has trigger some related run paths if os.path.exists(CONTEXT_MOUNT_ARTIFACTS_RELATED): for sub_path in os.listdir(CONTEXT_MOUNT_ARTIFACTS_RELATED): # check if there's a path to sync path_from = CONTEXT_MOUNT_ARTIFACTS_RELATED_FORMAT.format(sub_path) if os.path.exists(path_from): path_to = os.path.join(connection_type.store_path, sub_path) upload_file_or_dir( path_from=path_from, path_to=path_to, is_file=False, workers=5, last_time=last_check, connection_type=connection_type, ) return new_check
def set_run_event_path(self): self._artifacts_path = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(self.run_uuid) self._outputs_path = CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format(self.run_uuid)