def test_is_done(self): assert LifeCycle.is_done(None) is False for status in V1Statuses.allowable_values: if status in LifeCycle.DONE_VALUES: assert LifeCycle.is_done(status) is True else: assert LifeCycle.is_done(status) is False
def test_is_done(self): assert LifeCycle.is_done(None) is False for status in LifeCycle.VALUES: if status in LifeCycle.DONE_VALUES: assert LifeCycle.is_done(status) is True else: assert LifeCycle.is_done(status) is False
def test_is_stoppable(self): assert LifeCycle.is_stoppable(None) is True for status in V1Statuses.allowable_values: if not LifeCycle.is_done(status): assert LifeCycle.is_stoppable(status) is True else: assert LifeCycle.is_stoppable(status) is False
def new_run_status(run: BaseRun, condition: V1StatusCondition): previous_status = new_status(entity=run, condition=condition) # Do not audit the new status since it's the same as the previous one if (condition.type in {V1Statuses.CREATED, V1Statuses.STOPPING} or previous_status == run.status): return auditor.record(event_type=RUN_NEW_STATUS, instance=run, previous_status=previous_status) if run.status == V1Statuses.STOPPED: auditor.record(event_type=RUN_STOPPED, instance=run, previous_status=previous_status) elif run.status == V1Statuses.FAILED: auditor.record(event_type=RUN_FAILED, instance=run, previous_status=previous_status) elif run.status == V1Statuses.SUCCEEDED: auditor.record(event_type=RUN_SUCCEEDED, instance=run, previous_status=previous_status) elif run.status == V1Statuses.SKIPPED: auditor.record(event_type=RUN_SKIPPED, instance=run, previous_status=previous_status) elif run.status == V1Statuses.RESUMING: auditor.record(event_type=RUN_RESUMED, instance=run) # handle done status if LifeCycle.is_done(run.status): auditor.record(event_type=RUN_DONE, instance=run, previous_status=previous_status)
def test_is_stoppable(self): assert LifeCycle.is_stoppable(None) is True for status in LifeCycle.VALUES: if not LifeCycle.is_done(status): assert LifeCycle.is_stoppable(status) is True else: assert LifeCycle.is_stoppable(status) is False
def new_status( entity, condition: V1StatusCondition, additional_fields: List[str] = None ): previous_status = entity.status if condition.type == V1Statuses.CREATED: return previous_status if previous_status == V1Statuses.STOPPING and not LifeCycle.is_done(condition.type): return previous_status entity = set_entity_status(entity=entity, condition=condition) LifeCycle.set_started_at(entity=entity) LifeCycle.set_finished_at(entity=entity) additional_fields = additional_fields or [] entity.save( update_fields=additional_fields + [ "status_conditions", "status", "started_at", "updated_at", "finished_at", "wait_time", "duration", ] ) return previous_status
def set_finished_at(run: BaseRun) -> bool: if LifeCycle.is_done(run.status) and run.finished_at is None: run.finished_at = now() if run.started_at is None: # We should not have this case run.started_at = run.created_at # Update run time if run.run_time is None: run.run_time = (run.finished_at - run.started_at).seconds return True return False
def handle_logs(): is_done = False last_time = None last_file = None _status = None last_status, _ = client.get_statuses() while not LifeCycle.is_done(last_status) and not LifeCycle.is_running( last_status ): time.sleep(1) last_status, _ = client.get_statuses() if _status != last_status: _status = handle_status(last_status) while not is_done: response = get_logs(last_time=last_time, last_file=last_file) if response: last_time = response.last_time last_file = response.last_file else: last_time = None last_file = None # Follow logic if not any([last_file, last_time]): if follow: last_status, _ = client.get_statuses() if _status != last_status: _status = handle_status(last_status) is_done = LifeCycle.is_done(last_status) else: is_done = True if last_time and not follow: is_done = True if not is_done: if last_file: time.sleep(1) else: time.sleep(settings.CLIENT_CONFIG.watch_interval)
def stop_run(view, request, *args, **kwargs): if LifeCycle.is_done(view.run.status): return Response(status=status.HTTP_200_OK, data={}) condition = V1StatusCondition.get_condition( type=V1Statuses.STOPPING, status="True", reason="PolyaxonRunStopping", message="User requested to stop the run.", ) new_run_status(run=view.run, condition=condition) view.audit(request, *args, **kwargs) return Response(status=status.HTTP_200_OK, data={})
def new_run_stopping_status(run, message) -> bool: if LifeCycle.is_done(run.status, progressing=True): return False message = f"Run is stopping; {message}" if message else "Run is stopping" condition = V1StatusCondition.get_condition( type=V1Statuses.STOPPING, status="True", reason="PolyaxonRunStopping", message=message, ) new_run_status(run=run, condition=condition) return True
def new_status(entity, condition: V1StatusCondition): previous_status = entity.status if condition.type == V1Statuses.CREATED: return previous_status if previous_status == V1Statuses.STOPPING and not LifeCycle.is_done( condition.type): return previous_status entity = set_entity_status(entity=entity, condition=condition) set_started_at(run=entity) set_finished_at(run=entity) entity.save(update_fields=[ "status_conditions", "status", "started_at", "updated_at", "finished_at", "run_time", ]) return previous_status
def condition(): if statuses: return last_status in statuses return LifeCycle.is_done(last_status)
def handle_logs(): is_done = False last_file = None _status = None files = [] last_transition_time = now() last_status, conditions = client.get_statuses() if conditions: last_transition_time = conditions[0].last_transition_time while not LifeCycle.is_done(last_status) and not LifeCycle.is_running( last_status): time.sleep(settings.CLIENT_CONFIG.watch_interval) last_status, conditions = client.get_statuses() if conditions: last_transition_time = conditions[0].last_transition_time if _status != last_status: _status = handle_status(last_status) if LifeCycle.is_done(last_status): last_time = None else: last_time = last_transition_time checks = 0 while not is_done: response = get_logs(last_time=last_time, last_file=last_file) if response: last_time = response.last_time last_file = response.last_file files = response.files else: last_time = None last_file = None # Follow logic if not any([last_file, last_time]) or checks > 3: if follow: last_status, _ = client.get_statuses() if _status != last_status: _status = handle_status(last_status) is_done = LifeCycle.is_done(last_status) if not is_done: checks = 0 if not last_time: last_time = last_transition_time else: is_done = True if last_time and not follow: is_done = True if not is_done: if last_file: if len(files) > 1 and last_file != files[-1]: time.sleep(1) else: is_done = True else: time.sleep(settings.CLIENT_CONFIG.watch_interval) checks += 1
def watch_run_statuses() -> Tuple[str, Iterator]: last_status = None while not LifeCycle.is_done(last_status): last_status, _conditions = self.get_statuses(last_status) yield last_status, _conditions time.sleep(settings.CLIENT_CONFIG.watch_interval)
def pre_validate(self, obj): if not LifeCycle.is_done(obj.status): raise ValidationError( "Cannot resume this run, the run must reach a final state first, " "current status error: {}".format(obj.status))
def service(ctx, yes, external, url): """Open the operation service in browser. N.B. The operation must have a run kind service, otherwise it will raise an error. You can open the service embedded in Polyaxon UI or using the real service URL, please use the `--external` flag. """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) client.refresh_data() if client.run_data.kind != V1RunKind.SERVICE: Printer.print_warning( "Command expected an operation of " "kind `service` received kind: `{}`!".format(client.run_data.kind) ) sys.exit(1) Printer.print_header("Waiting for running condition ...") client.wait_for_condition( statuses={V1Statuses.RUNNING} | LifeCycle.DONE_VALUES, print_status=True ) client.refresh_data() if LifeCycle.is_done(client.run_data.status): Printer.print_header("The operations reached a done statuses.") latest_status = Printer.add_status_color( {"status": client.run_data.status}, status_key="status" ) click.echo("{}\n".format(latest_status["status"])) run_url = get_dashboard_url( subpath="{}/{}/runs/{}/service".format(owner, project_name, run_uuid) ) namespace = client.run_data.settings.namespace service_endpoint = SERVICES_V1 if client.run_data.meta_info.get("rewrite_path", False): service_endpoint = REWRITE_SERVICES_V1 external_run_url = get_dashboard_url( base=service_endpoint, subpath="{}/{}/{}/runs/{}/".format(namespace, owner, project_name, run_uuid), ) if url: Printer.print_header("The service will be available at: {}".format(run_url)) Printer.print_header( "You can also view it in an external link at: {}".format(external_run_url) ) sys.exit(0) if not yes: click.confirm( "Dashboard page will now open in your browser. Continue?", abort=True, default=True, ) if external: click.launch(external_run_url) sys.exit(0) click.launch(run_url)