def init_artifact_context_args(run_path: str) -> List[str]:
return [
'if [ ! -d "{dir}" ]; then mkdir -p {dir}; fi;'.format(
dir=CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_path)),
'if [ ! -d "{dir}" ]; then mkdir -p {dir}; fi;'.format(
dir=CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_path) + "/outputs"),
]
def test_get_store_container_mount_stores(self):
# Managed store
store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(
mount_path="/tmp", volume_claim="test", read_only=True
),
)
container = get_store_container(
polyaxon_init=V1PolyaxonInitContainer(
image="foo/foo", image_tag="foo", image_pull_policy="IfNotPresent"
),
connection=store,
artifacts=None,
)
mount_path = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name)
assert container.name == INIT_ARTIFACTS_CONTAINER.format(store.name)
assert container.image == "foo/foo:foo"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["/bin/sh", "-c"]
assert container.args == [
get_volume_args(store=store, mount_path=mount_path, artifacts=None)
]
assert container.env == get_connection_env_var(connection=store, secret=None)
assert container.env_from == []
assert container.resources is not None
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name),
),
get_mount_from_store(store=store),
]
def test_event_logger_from_non_manged_run(self):
settings.CLIENT_CONFIG.is_managed = False
run = Run(project="owner-test.test")
assert run.get_artifacts_path() == TEMP_RUN_ARTIFACTS
assert run.get_outputs_path() == TEMP_RUN_ARTIFACTS
assert run._event_logger is None
# Add run id
run = Run(project="owner-test.test", run_uuid="uuid")
assert run.get_artifacts_path() == TEMP_RUN_ARTIFACTS
assert run.get_outputs_path() == TEMP_RUN_ARTIFACTS
assert run._event_logger is None
run.set_artifacts_path()
assert run.get_artifacts_path() == CONTEXT_MOUNT_ARTIFACTS_FORMAT.format("uuid")
assert run.get_outputs_path() == CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("uuid")
with patch("polyaxon.tracking.run.EventFileWriter") as mock_call:
run.set_run_event_logger()
assert mock_call.call_count == 1
with patch("polyaxon.tracking.run.ResourceFileWriter") as mock_call:
run.set_run_resource_logger()
assert mock_call.call_count == 1
settings.CLIENT_CONFIG.is_managed = True
with patch("polyaxon.tracking.run.EventFileWriter") as event_call:
with patch("polyaxon.tracking.run.ResourceFileWriter") as resource_call:
run = Run(project="owner-test.test", run_uuid="uuid")
assert event_call.call_count == 1
assert resource_call.call_count == 1
assert run.get_artifacts_path() == CONTEXT_MOUNT_ARTIFACTS_FORMAT.format("uuid")
assert run.get_outputs_path() == CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("uuid")
def get_store_container(
polyaxon_init: V1PolyaxonInitContainer,
connection: V1ConnectionType,
artifacts: V1ArtifactsType,
container: Optional[k8s_schemas.V1Container] = None,
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: str = None,
) -> Optional[k8s_schemas.V1Container]:
container_name = INIT_ARTIFACTS_CONTAINER.format(connection.name)
if not container:
container = k8s_schemas.V1Container(name=container_name)
volume_name = (get_volume_name(mount_path)
if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(
connection.name)
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
return get_base_store_container(
container=container,
container_name=container_name,
polyaxon_init=polyaxon_init,
store=connection,
env=env,
env_from=[],
volume_mounts=volume_mounts,
args=[
get_volume_args(store=connection,
mount_path=mount_path,
artifacts=artifacts)
],
)
async def sync_logs(
run_uuid: str,
k8s_manager: AsyncK8SManager,
pod: V1Pod,
last_time: Optional[AwareDT],
stream: bool = False,
is_running: bool = True,
):
path_from = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid)
path_from = "{}/.tmpplxlogs".format(path_from)
if not is_running:
delete_path(path_from)
return
logs, _ = await query_k8s_pod_logs(
k8s_manager=k8s_manager,
pod=pod,
last_time=last_time,
stream=stream,
)
if not logs:
return
path_from = "{}/{}".format(path_from, pod.metadata.name)
check_or_create_path(path_from, is_dir=False)
async with aiofiles.open(path_from, "w") as filepath:
await filepath.write(V1Logs(logs=logs).to_dict(dump=True))
def get_dockerfile_init_container(
polyaxon_init: V1PolyaxonInitContainer,
dockerfile_args: V1DockerfileType,
contexts: PluginsContextsSpec,
run_path: str,
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: Optional[str] = None,
) -> k8s_schemas.V1Container:
env = to_list(env, check_none=True)
env = env + [get_run_instance_env_var()]
volume_name = (get_volume_name(mount_path)
if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
if contexts and contexts.auth:
volume_mounts.append(get_auth_context_mount(read_only=True))
return k8s_schemas.V1Container(
name=generate_container_name(INIT_DOCKERFILE_CONTAINER_PREFIX),
image=polyaxon_init.get_image(),
image_pull_policy=polyaxon_init.image_pull_policy,
command=["polyaxon", "docker", "generate"],
args=[
"--build-context={}".format(dockerfile_args.to_dict(dump=True)),
"--destination={}".format(mount_path),
"--copy-path={}".format(
CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_path) + "/outputs"),
],
env=env,
resources=polyaxon_init.get_resources(),
volume_mounts=volume_mounts,
)
def test_event_logger_from_a_managed_run(self):
# Set managed flag
settings.CLIENT_CONFIG.is_managed = True
os.environ[POLYAXON_KEYS_RUN_INSTANCE] = "user.project_bar.runs.uid"
run = Run()
assert run.get_artifacts_path() == CONTEXT_MOUNT_ARTIFACTS_FORMAT.format("uid")
assert run.get_outputs_path() == CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("uid")
assert run._event_logger is None
# Set collect flag
os.environ[POLYAXON_KEYS_COLLECT_ARTIFACTS] = "true"
os.environ[POLYAXON_KEYS_COLLECT_RESOURCES] = "true"
# Add run id
with patch("polyaxon.tracking.run.Run.set_run_event_logger") as event_call:
with patch(
"polyaxon.tracking.run.Run.set_run_resource_logger"
) as resource_call:
Run(project="test.test", run_uuid="uuid")
assert event_call.call_count == 1
assert resource_call.call_count == 1
# Set run info
os.environ[POLYAXON_KEYS_RUN_INSTANCE] = "user.project_bar.runs.uid"
# Add run id
with patch("polyaxon.tracking.run.Run.set_run_event_logger") as event_call:
with patch(
"polyaxon.tracking.run.Run.set_run_resource_logger"
) as resource_call:
Run()
assert event_call.call_count == 1
assert resource_call.call_count == 1
def set_artifacts_path(self, artifacts_path: str = None):
_artifacts_path = artifacts_path or CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(
self.run_uuid
)
if artifacts_path:
_outputs_path = "{}/outputs".format(artifacts_path)
else:
_outputs_path = CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format(self.run_uuid)
self._artifacts_path = _artifacts_path
self._outputs_path = _outputs_path
def set_artifacts_path(self, artifacts_path: str = None):
"""Sets an artifacts_path.
Be careful, this method is called automatically
when a job is running in-cluster and follows some flags that Polyaxon sets. Polyaxon
has some processes to automatically sync your run's artifacts and outputs.
Args:
artifacts_path: str, optional
"""
_artifacts_path = artifacts_path or CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(
self.run_uuid)
_outputs_path = "{}/outputs".format(_artifacts_path)
self._artifacts_path = _artifacts_path
self._outputs_path = _outputs_path
def sync_artifacts(last_check: Optional[datetime], run_uuid: str):
new_check = now()
connection_type = get_artifacts_connection()
path_from = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid)
# check if there's a path to sync
if os.path.exists(path_from):
path_to = os.path.join(connection_type.store_path, run_uuid)
upload_file_or_dir(
path_from=path_from,
path_to=path_to,
is_file=False,
workers=5,
last_time=last_check,
connection_type=connection_type,
)
return new_check
def sync_artifacts(last_check: Optional[datetime], run_uuid: str):
connection_type = get_artifacts_connection()
path_from = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid)
new_check = path_last_modified(path_from)
# check if there's a path to sync
if os.path.exists(path_from):
path_to = os.path.join(connection_type.store_path, run_uuid)
upload_file_or_dir(
path_from=path_from,
path_to=path_to,
is_file=False,
workers=5,
last_time=last_check,
connection_type=connection_type,
exclude=["plxlogs"],
)
# Check if this run has trigger some related run paths
if os.path.exists(CONTEXT_MOUNT_ARTIFACTS_RELATED):
for sub_path in os.listdir(CONTEXT_MOUNT_ARTIFACTS_RELATED):
# check if there's a path to sync
path_from = CONTEXT_MOUNT_ARTIFACTS_RELATED_FORMAT.format(sub_path)
if os.path.exists(path_from):
path_to = os.path.join(connection_type.store_path, sub_path)
upload_file_or_dir(
path_from=path_from,
path_to=path_to,
is_file=False,
workers=5,
last_time=last_check,
connection_type=connection_type,
)
return new_check
def set_run_event_path(self):
self._artifacts_path = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(self.run_uuid)
self._outputs_path = CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format(self.run_uuid)
