def test_get_interval_counter(self):
assert get_sync_interval(0, 0) == -1
assert get_sync_interval(1, 2) == 0
assert get_sync_interval(2, 2) == 2
assert get_sync_interval(3, 2) == 2
assert get_sync_interval(4, 2) == 3
assert get_sync_interval(4, 2) == 3
assert get_sync_interval(6, 2) == 4
async def start_sidecar(
container_id: str,
sleep_interval: int,
sync_interval: int,
monitor_outputs: bool,
monitor_logs: bool,
):
sync_interval = get_sync_interval(
interval=sync_interval, sleep_interval=sleep_interval
)
try:
pod_id = os.environ[POLYAXON_KEYS_K8S_POD_ID]
except KeyError as e:
raise PolyaxonContainerException(
"Please make sure that this job has been "
"started by Polyaxon with all required context."
) from e
try:
owner, project, run_uuid = get_run_info()
except PolyaxonClientException as e:
raise PolyaxonContainerException(e)
client = RunClient(owner=owner, project=project, run_uuid=run_uuid)
k8s_manager = AsyncK8SManager(namespace=CLIENT_CONFIG.namespace, in_cluster=True)
await k8s_manager.setup()
pod = await k8s_manager.get_pod(pod_id, reraise=True)
retry = 1
is_running = True
counter = 0
state = {
"last_artifacts_check": None,
"last_logs_check": None,
}
async def monitor():
if monitor_logs:
await sync_logs(
run_uuid=run_uuid,
k8s_manager=k8s_manager,
pod=pod,
last_time=None,
stream=True,
is_running=is_running,
)
if monitor_outputs:
last_check = state["last_artifacts_check"]
state["last_artifacts_check"] = sync_artifacts(
last_check=last_check,
run_uuid=run_uuid,
)
sync_summaries(
last_check=last_check,
run_uuid=run_uuid,
client=client,
)
while is_running and retry <= 3:
await asyncio.sleep(sleep_interval)
try:
is_running = await k8s_manager.is_pod_running(pod_id, container_id)
except ApiException as e:
retry += 1
logger.info("Exception %s" % repr(e))
logger.info("Sleeping ...")
await asyncio.sleep(retry)
continue
logger.debug("Syncing ...")
if is_running:
retry = 1
counter += 1
if counter == sync_interval:
counter = 0
try:
await monitor()
except Exception as e:
logger.warning("Polyaxon sidecar error: %s" % repr(e))
await monitor()
logger.info("Cleaning non main containers")
if k8s_manager:
await k8s_manager.close()
def start_sidecar(
container_id: str,
sleep_interval: int,
sync_interval: int,
monitor_outputs: bool,
monitor_logs: bool,
):
sync_interval = get_sync_interval(interval=sync_interval,
sleep_interval=sleep_interval)
try:
owner, project, run_uuid = get_run_info()
except PolyaxonClientException as e:
raise PolyaxonContainerException(e)
client = RunClient(owner=owner, project=project, run_uuid=run_uuid)
pod_id = CLIENT_CONFIG.pod_id
if not pod_id:
raise PolyaxonContainerException(
"Please make sure that this job has been "
"started by Polyaxon with all required context.")
k8s_manager = K8SManager(namespace=CLIENT_CONFIG.namespace,
in_cluster=True)
retry = 1
is_running = True
counter = 0
state = {
"last_artifacts_check": None,
"last_logs_check": None,
}
def monitor():
if monitor_outputs:
last_check = state["last_artifacts_check"]
state["last_artifacts_check"] = sync_artifacts(
last_check=last_check,
run_uuid=run_uuid,
)
sync_summaries(
last_check=last_check,
run_uuid=run_uuid,
client=client,
)
if monitor_logs:
state["last_logs_check"] = sync_logs(
k8s_manager=k8s_manager,
client=client,
last_check=state["last_logs_check"],
run_uuid=run_uuid,
pod_id=pod_id,
container_id=container_id,
owner=owner,
project=project,
)
while is_running and retry <= 3:
time.sleep(sleep_interval)
try:
is_running = is_pod_running(k8s_manager, pod_id, container_id)
except ApiException as e:
retry += 1
time.sleep(1 * retry)
logger.info("Exception %s" % repr(e))
logger.info("Sleeping ...")
logger.debug("Syncing ...")
if is_running:
retry = 1
counter += 1
if counter == sync_interval:
counter = 0
try:
monitor()
except Exception as e:
logger.warning("Polyaxon sidecar error: %e", e)
monitor()
logger.info("Cleaning non main containers")