def test_is_pending(self): assert LifeCycle.is_pending(None) is False for status in LifeCycle.VALUES: if status in LifeCycle.PENDING_VALUES: assert LifeCycle.is_pending(status) is True else: assert LifeCycle.is_pending(status) is False
def test_is_starting(self): assert LifeCycle.is_starting(None) is False for status in LifeCycle.VALUES: if status == V1Statuses.STARTING: assert LifeCycle.is_starting(status) is True else: assert LifeCycle.is_starting(status) is False
def test_can_check_heartbeat(self): assert LifeCycle.can_check_heartbeat(None) is False for status in LifeCycle.VALUES: if LifeCycle.is_running(status): assert LifeCycle.can_check_heartbeat(status) is True else: assert LifeCycle.can_check_heartbeat(status) is False
def test_is_unschedulable(self): assert LifeCycle.is_unschedulable(None) is False for status in LifeCycle.VALUES: if status == V1Statuses.UNSCHEDULABLE: assert LifeCycle.is_unschedulable(status) is True else: assert LifeCycle.is_unschedulable(status) is False
def test_failed(self): assert LifeCycle.failed(None) is False for status in LifeCycle.VALUES: if status in {V1Statuses.FAILED, V1Statuses.UPSTREAM_FAILED}: assert LifeCycle.failed(status) is True else: assert LifeCycle.failed(status) is False
def test_skipped(self): assert LifeCycle.skipped(None) is False for status in LifeCycle.VALUES: if status == V1Statuses.SKIPPED: assert LifeCycle.skipped(status) is True else: assert LifeCycle.skipped(status) is False
def test_is_done(self): assert LifeCycle.is_done(None) is False for status in LifeCycle.VALUES: if status in LifeCycle.DONE_VALUES: assert LifeCycle.is_done(status) is True else: assert LifeCycle.is_done(status) is False
def test_succeeded(self): assert LifeCycle.succeeded(None) is False for status in LifeCycle.VALUES: if status == V1Statuses.SUCCEEDED: assert LifeCycle.succeeded(status) is True else: assert LifeCycle.succeeded(status) is False
def test_is_stoppable(self): assert LifeCycle.is_stoppable(None) is True for status in LifeCycle.VALUES: if not LifeCycle.is_done(status): assert LifeCycle.is_stoppable(status) is True else: assert LifeCycle.is_stoppable(status) is False
def is_stopping(self): assert LifeCycle.is_stopping(None) is False for status in LifeCycle.VALUES: if status == V1Statuses.STOPPING: assert LifeCycle.is_stopping(status) is True else: assert LifeCycle.is_stopping(status) is False
def test_is_unknown(self): assert LifeCycle.is_unknown(None) is False for status in LifeCycle.VALUES: if status == V1Statuses.UNKNOWN: assert LifeCycle.is_unknown(status) is True else: assert LifeCycle.is_unknown(status) is False
def test_is_compilable(self): assert LifeCycle.is_unschedulable(None) is False for status in V1Statuses.allowable_values: if status in LifeCycle.COMPILABLE_VALUES: assert LifeCycle.is_compilable(status) is True else: assert LifeCycle.is_compilable(status) is False
def test_is_running(self): assert LifeCycle.is_running(None) is False for status in LifeCycle.VALUES: if status in LifeCycle.RUNNING_VALUES: assert LifeCycle.is_running(status) is True else: assert LifeCycle.is_running(status) is False
def test_is_unschedulable(self): assert LifeCycle.is_unschedulable(None) is False for status in V1Statuses.allowable_values: if status == V1Statuses.UNSCHEDULABLE: assert LifeCycle.is_unschedulable(status) is True else: assert LifeCycle.is_unschedulable(status) is False
def test_can_check_heartbeat(self): assert LifeCycle.can_check_heartbeat(None) is False for status in V1Statuses.allowable_values: if LifeCycle.is_running(status): assert LifeCycle.can_check_heartbeat(status) is True else: assert LifeCycle.can_check_heartbeat(status) is False
def test_is_starting(self): assert LifeCycle.is_starting(None) is False for status in V1Statuses.allowable_values: if status == V1Statuses.STARTING: assert LifeCycle.is_starting(status) is True else: assert LifeCycle.is_starting(status) is False
def test_failed(self): assert LifeCycle.failed(None) is False for status in V1Statuses.allowable_values: if status in {V1Statuses.FAILED, V1Statuses.UPSTREAM_FAILED}: assert LifeCycle.failed(status) is True else: assert LifeCycle.failed(status) is False
def test_skipped(self): assert LifeCycle.skipped(None) is False for status in V1Statuses.allowable_values: if status == V1Statuses.SKIPPED: assert LifeCycle.skipped(status) is True else: assert LifeCycle.skipped(status) is False
def test_is_done(self): assert LifeCycle.is_done(None) is False for status in V1Statuses.allowable_values: if status in LifeCycle.DONE_VALUES: assert LifeCycle.is_done(status) is True else: assert LifeCycle.is_done(status) is False
def test_succeeded(self): assert LifeCycle.succeeded(None) is False for status in V1Statuses.allowable_values: if status == V1Statuses.SUCCEEDED: assert LifeCycle.succeeded(status) is True else: assert LifeCycle.succeeded(status) is False
def is_stopping(self): assert LifeCycle.is_stopping(None) is False for status in V1Statuses.allowable_values: if status == V1Statuses.STOPPING: assert LifeCycle.is_stopping(status) is True else: assert LifeCycle.is_stopping(status) is False
def test_is_stoppable(self): assert LifeCycle.is_stoppable(None) is True for status in V1Statuses.allowable_values: if not LifeCycle.is_done(status): assert LifeCycle.is_stoppable(status) is True else: assert LifeCycle.is_stoppable(status) is False
def test_is_unknown(self): assert LifeCycle.is_unknown(None) is False for status in V1Statuses.allowable_values: if status == V1Statuses.UNKNOWN: assert LifeCycle.is_unknown(status) is True else: assert LifeCycle.is_unknown(status) is False
def test_is_pending(self): assert LifeCycle.is_pending(None) is False for status in V1Statuses.allowable_values: if status in LifeCycle.PENDING_VALUES: assert LifeCycle.is_pending(status) is True else: assert LifeCycle.is_pending(status) is False
def new_status( entity, condition: V1StatusCondition, additional_fields: List[str] = None ): previous_status = entity.status if condition.type == V1Statuses.CREATED: return previous_status if previous_status == V1Statuses.STOPPING and not LifeCycle.is_done(condition.type): return previous_status entity = set_entity_status(entity=entity, condition=condition) LifeCycle.set_started_at(entity=entity) LifeCycle.set_finished_at(entity=entity) additional_fields = additional_fields or [] entity.save( update_fields=additional_fields + [ "status_conditions", "status", "started_at", "updated_at", "finished_at", "wait_time", "duration", ] ) return previous_status
def runs_stop(run_id: int, run: Optional[BaseRun], update_status=False, message=None) -> bool: run = get_run(run_id=run_id, run=run) if not run: return True stopped = True should_stop = (LifeCycle.is_k8s_stoppable(run.status) or run.status == V1Statuses.STOPPING) if run.is_managed and should_stop: in_cluster = conf.get(K8S_IN_CLUSTER) if in_cluster and (run.is_service or run.is_job): stopped = manager.stop( run_uuid=run.uuid.hex, run_kind=run.kind, namespace=conf.get(K8S_NAMESPACE), in_cluster=in_cluster, ) if not stopped: return False if not update_status: return True new_run_stop_status(run=run, message=message) return True
def new_run_status(run: BaseRun, condition: V1StatusCondition): previous_status = new_status(entity=run, condition=condition) # Do not audit the new status since it's the same as the previous one if (condition.type in {V1Statuses.CREATED, V1Statuses.STOPPING} or previous_status == run.status): return auditor.record(event_type=RUN_NEW_STATUS, instance=run, previous_status=previous_status) if run.status == V1Statuses.STOPPED: auditor.record(event_type=RUN_STOPPED, instance=run, previous_status=previous_status) elif run.status == V1Statuses.FAILED: auditor.record(event_type=RUN_FAILED, instance=run, previous_status=previous_status) elif run.status == V1Statuses.SUCCEEDED: auditor.record(event_type=RUN_SUCCEEDED, instance=run, previous_status=previous_status) elif run.status == V1Statuses.SKIPPED: auditor.record(event_type=RUN_SKIPPED, instance=run, previous_status=previous_status) elif run.status == V1Statuses.RESUMING: auditor.record(event_type=RUN_RESUMED, instance=run) # handle done status if LifeCycle.is_done(run.status): auditor.record(event_type=RUN_DONE, instance=run, previous_status=previous_status)
def new_run_stopping_status(run, message) -> bool: if LifeCycle.is_done(run.status, progressing=True): return False if LifeCycle.is_safe_stoppable(run.status): new_run_stop_status(run, message) return True message = f"Run is stopping; {message}" if message else "Run is stopping" condition = V1StatusCondition.get_condition( type=V1Statuses.STOPPING, status="True", reason="StateManager", message=message, ) new_run_status(run=run, condition=condition) return True
def runs_start(run_id: int, run: Optional[BaseRun]): run = get_run(run_id=run_id, run=run) if not run: return if not run.is_managed: return if not LifeCycle.is_compiled(run.status): _logger.info( "Run `%s` cannot transition from `%s` to `%s`.", run_id, run.status, V1Statuses.QUEUED, ) return condition = V1StatusCondition.get_condition( type=V1Statuses.QUEUED, status="True", reason="PolyaxonRunQueued", message="Run is queued", ) new_run_status(run=run, condition=condition) def _log_error(exc: Exception, message: str = None): message = message or "Could not start the operation.\n" message += "error: {}\n{}".format(repr(exc), traceback.format_exc()) cond = V1StatusCondition.get_condition( type=V1Statuses.FAILED, status="True", reason="PolyaxonRunFailed", message=message, ) new_run_status(run=run, condition=cond) try: in_cluster = conf.get(K8S_IN_CLUSTER) if in_cluster and (run.is_service or run.is_job): manager.start( content=run.content, owner_name=run.project.owner.name, project_name=run.project.name, run_name=run.name, run_uuid=run.uuid.hex, run_kind=run.kind, namespace=conf.get(K8S_NAMESPACE), in_cluster=in_cluster, default_auth=False, ) return except (PolyaxonK8SError, ApiException) as e: _log_error( exc=e, message="Kubernetes manager could not start the operation.\n") except PolypodException as e: _log_error(exc=e, message="Failed converting the run manifest.\n") except Exception as e: _log_error(exc=e, message="Failed with unknown exception.\n")
def runs_stop( run_id: int, run: Optional[BaseRun], update_status=False, message=None, clean=False, ) -> bool: run = get_run(run_id=run_id, run=run) if not run: return True stopped = True should_stop = (LifeCycle.is_k8s_stoppable(run.status) or run.status == V1Statuses.STOPPING) def _clean(): try: manager.clean( run_uuid=run.uuid.hex, run_kind=run.kind, namespace=conf.get(K8S_NAMESPACE), in_cluster=in_cluster, ) except (PolyaxonK8SError, ApiException) as e: _logger.warning( "Something went wrong, the run `%s` could not be stopped, error %s", run.uuid, e, ) return False if run.is_managed and should_stop: in_cluster = conf.get(K8S_IN_CLUSTER) if in_cluster and (run.is_service or run.is_job): if clean: _clean() try: stopped = manager.stop( run_uuid=run.uuid.hex, run_kind=run.kind, namespace=conf.get(K8S_NAMESPACE), in_cluster=in_cluster, ) except (PolyaxonK8SError, ApiException) as e: _logger.warning( "Something went wrong, the run `%s` could not be stopped, error %s", run.uuid, e, ) return False if not stopped: return False if not update_status: return True new_run_stop_status(run=run, message=message) return True