def test_get_interval_counter(self): assert get_sync_interval(0, 0) == -1 assert get_sync_interval(1, 2) == 0 assert get_sync_interval(2, 2) == 2 assert get_sync_interval(3, 2) == 2 assert get_sync_interval(4, 2) == 3 assert get_sync_interval(4, 2) == 3 assert get_sync_interval(6, 2) == 4
async def start_sidecar( container_id: str, sleep_interval: int, sync_interval: int, monitor_outputs: bool, monitor_logs: bool, ): sync_interval = get_sync_interval( interval=sync_interval, sleep_interval=sleep_interval ) try: pod_id = os.environ[POLYAXON_KEYS_K8S_POD_ID] except KeyError as e: raise PolyaxonContainerException( "Please make sure that this job has been " "started by Polyaxon with all required context." ) from e try: owner, project, run_uuid = get_run_info() except PolyaxonClientException as e: raise PolyaxonContainerException(e) client = RunClient(owner=owner, project=project, run_uuid=run_uuid) k8s_manager = AsyncK8SManager(namespace=CLIENT_CONFIG.namespace, in_cluster=True) await k8s_manager.setup() pod = await k8s_manager.get_pod(pod_id, reraise=True) retry = 1 is_running = True counter = 0 state = { "last_artifacts_check": None, "last_logs_check": None, } async def monitor(): if monitor_logs: await sync_logs( run_uuid=run_uuid, k8s_manager=k8s_manager, pod=pod, last_time=None, stream=True, is_running=is_running, ) if monitor_outputs: last_check = state["last_artifacts_check"] state["last_artifacts_check"] = sync_artifacts( last_check=last_check, run_uuid=run_uuid, ) sync_summaries( last_check=last_check, run_uuid=run_uuid, client=client, ) while is_running and retry <= 3: await asyncio.sleep(sleep_interval) try: is_running = await k8s_manager.is_pod_running(pod_id, container_id) except ApiException as e: retry += 1 logger.info("Exception %s" % repr(e)) logger.info("Sleeping ...") await asyncio.sleep(retry) continue logger.debug("Syncing ...") if is_running: retry = 1 counter += 1 if counter == sync_interval: counter = 0 try: await monitor() except Exception as e: logger.warning("Polyaxon sidecar error: %s" % repr(e)) await monitor() logger.info("Cleaning non main containers") if k8s_manager: await k8s_manager.close()
def start_sidecar( container_id: str, sleep_interval: int, sync_interval: int, monitor_outputs: bool, monitor_logs: bool, ): sync_interval = get_sync_interval(interval=sync_interval, sleep_interval=sleep_interval) try: owner, project, run_uuid = get_run_info() except PolyaxonClientException as e: raise PolyaxonContainerException(e) client = RunClient(owner=owner, project=project, run_uuid=run_uuid) pod_id = CLIENT_CONFIG.pod_id if not pod_id: raise PolyaxonContainerException( "Please make sure that this job has been " "started by Polyaxon with all required context.") k8s_manager = K8SManager(namespace=CLIENT_CONFIG.namespace, in_cluster=True) retry = 1 is_running = True counter = 0 state = { "last_artifacts_check": None, "last_logs_check": None, } def monitor(): if monitor_outputs: last_check = state["last_artifacts_check"] state["last_artifacts_check"] = sync_artifacts( last_check=last_check, run_uuid=run_uuid, ) sync_summaries( last_check=last_check, run_uuid=run_uuid, client=client, ) if monitor_logs: state["last_logs_check"] = sync_logs( k8s_manager=k8s_manager, client=client, last_check=state["last_logs_check"], run_uuid=run_uuid, pod_id=pod_id, container_id=container_id, owner=owner, project=project, ) while is_running and retry <= 3: time.sleep(sleep_interval) try: is_running = is_pod_running(k8s_manager, pod_id, container_id) except ApiException as e: retry += 1 time.sleep(1 * retry) logger.info("Exception %s" % repr(e)) logger.info("Sleeping ...") logger.debug("Syncing ...") if is_running: retry = 1 counter += 1 if counter == sync_interval: counter = 0 try: monitor() except Exception as e: logger.warning("Polyaxon sidecar error: %e", e) monitor() logger.info("Cleaning non main containers")