def test_is_done(self):
assert LifeCycle.is_done(None) is False
for status in V1Statuses.allowable_values:
if status in LifeCycle.DONE_VALUES:
assert LifeCycle.is_done(status) is True
else:
assert LifeCycle.is_done(status) is False
def test_is_done(self):
assert LifeCycle.is_done(None) is False
for status in LifeCycle.VALUES:
if status in LifeCycle.DONE_VALUES:
assert LifeCycle.is_done(status) is True
else:
assert LifeCycle.is_done(status) is False
def test_is_stoppable(self):
assert LifeCycle.is_stoppable(None) is True
for status in V1Statuses.allowable_values:
if not LifeCycle.is_done(status):
assert LifeCycle.is_stoppable(status) is True
else:
assert LifeCycle.is_stoppable(status) is False
def new_run_status(run: BaseRun, condition: V1StatusCondition):
previous_status = new_status(entity=run, condition=condition)
# Do not audit the new status since it's the same as the previous one
if (condition.type in {V1Statuses.CREATED, V1Statuses.STOPPING}
or previous_status == run.status):
return
auditor.record(event_type=RUN_NEW_STATUS,
instance=run,
previous_status=previous_status)
if run.status == V1Statuses.STOPPED:
auditor.record(event_type=RUN_STOPPED,
instance=run,
previous_status=previous_status)
elif run.status == V1Statuses.FAILED:
auditor.record(event_type=RUN_FAILED,
instance=run,
previous_status=previous_status)
elif run.status == V1Statuses.SUCCEEDED:
auditor.record(event_type=RUN_SUCCEEDED,
instance=run,
previous_status=previous_status)
elif run.status == V1Statuses.SKIPPED:
auditor.record(event_type=RUN_SKIPPED,
instance=run,
previous_status=previous_status)
elif run.status == V1Statuses.RESUMING:
auditor.record(event_type=RUN_RESUMED, instance=run)
# handle done status
if LifeCycle.is_done(run.status):
auditor.record(event_type=RUN_DONE,
instance=run,
previous_status=previous_status)
def test_is_stoppable(self):
assert LifeCycle.is_stoppable(None) is True
for status in LifeCycle.VALUES:
if not LifeCycle.is_done(status):
assert LifeCycle.is_stoppable(status) is True
else:
assert LifeCycle.is_stoppable(status) is False
def new_status(
entity, condition: V1StatusCondition, additional_fields: List[str] = None
):
previous_status = entity.status
if condition.type == V1Statuses.CREATED:
return previous_status
if previous_status == V1Statuses.STOPPING and not LifeCycle.is_done(condition.type):
return previous_status
entity = set_entity_status(entity=entity, condition=condition)
LifeCycle.set_started_at(entity=entity)
LifeCycle.set_finished_at(entity=entity)
additional_fields = additional_fields or []
entity.save(
update_fields=additional_fields
+ [
"status_conditions",
"status",
"started_at",
"updated_at",
"finished_at",
"wait_time",
"duration",
]
)
return previous_status
def set_finished_at(run: BaseRun) -> bool:
if LifeCycle.is_done(run.status) and run.finished_at is None:
run.finished_at = now()
if run.started_at is None: # We should not have this case
run.started_at = run.created_at
# Update run time
if run.run_time is None:
run.run_time = (run.finished_at - run.started_at).seconds
return True
return False
def handle_logs():
is_done = False
last_time = None
last_file = None
_status = None
last_status, _ = client.get_statuses()
while not LifeCycle.is_done(last_status) and not LifeCycle.is_running(
last_status
):
time.sleep(1)
last_status, _ = client.get_statuses()
if _status != last_status:
_status = handle_status(last_status)
while not is_done:
response = get_logs(last_time=last_time, last_file=last_file)
if response:
last_time = response.last_time
last_file = response.last_file
else:
last_time = None
last_file = None
# Follow logic
if not any([last_file, last_time]):
if follow:
last_status, _ = client.get_statuses()
if _status != last_status:
_status = handle_status(last_status)
is_done = LifeCycle.is_done(last_status)
else:
is_done = True
if last_time and not follow:
is_done = True
if not is_done:
if last_file:
time.sleep(1)
else:
time.sleep(settings.CLIENT_CONFIG.watch_interval)
def stop_run(view, request, *args, **kwargs):
if LifeCycle.is_done(view.run.status):
return Response(status=status.HTTP_200_OK, data={})
condition = V1StatusCondition.get_condition(
type=V1Statuses.STOPPING,
status="True",
reason="PolyaxonRunStopping",
message="User requested to stop the run.",
)
new_run_status(run=view.run, condition=condition)
view.audit(request, *args, **kwargs)
return Response(status=status.HTTP_200_OK, data={})
def new_run_stopping_status(run, message) -> bool:
if LifeCycle.is_done(run.status, progressing=True):
return False
message = f"Run is stopping; {message}" if message else "Run is stopping"
condition = V1StatusCondition.get_condition(
type=V1Statuses.STOPPING,
status="True",
reason="PolyaxonRunStopping",
message=message,
)
new_run_status(run=run, condition=condition)
return True
def new_status(entity, condition: V1StatusCondition):
previous_status = entity.status
if condition.type == V1Statuses.CREATED:
return previous_status
if previous_status == V1Statuses.STOPPING and not LifeCycle.is_done(
condition.type):
return previous_status
entity = set_entity_status(entity=entity, condition=condition)
set_started_at(run=entity)
set_finished_at(run=entity)
entity.save(update_fields=[
"status_conditions",
"status",
"started_at",
"updated_at",
"finished_at",
"run_time",
])
return previous_status
def condition():
if statuses:
return last_status in statuses
return LifeCycle.is_done(last_status)
def handle_logs():
is_done = False
last_file = None
_status = None
files = []
last_transition_time = now()
last_status, conditions = client.get_statuses()
if conditions:
last_transition_time = conditions[0].last_transition_time
while not LifeCycle.is_done(last_status) and not LifeCycle.is_running(
last_status):
time.sleep(settings.CLIENT_CONFIG.watch_interval)
last_status, conditions = client.get_statuses()
if conditions:
last_transition_time = conditions[0].last_transition_time
if _status != last_status:
_status = handle_status(last_status)
if LifeCycle.is_done(last_status):
last_time = None
else:
last_time = last_transition_time
checks = 0
while not is_done:
response = get_logs(last_time=last_time, last_file=last_file)
if response:
last_time = response.last_time
last_file = response.last_file
files = response.files
else:
last_time = None
last_file = None
# Follow logic
if not any([last_file, last_time]) or checks > 3:
if follow:
last_status, _ = client.get_statuses()
if _status != last_status:
_status = handle_status(last_status)
is_done = LifeCycle.is_done(last_status)
if not is_done:
checks = 0
if not last_time:
last_time = last_transition_time
else:
is_done = True
if last_time and not follow:
is_done = True
if not is_done:
if last_file:
if len(files) > 1 and last_file != files[-1]:
time.sleep(1)
else:
is_done = True
else:
time.sleep(settings.CLIENT_CONFIG.watch_interval)
checks += 1
def watch_run_statuses() -> Tuple[str, Iterator]:
last_status = None
while not LifeCycle.is_done(last_status):
last_status, _conditions = self.get_statuses(last_status)
yield last_status, _conditions
time.sleep(settings.CLIENT_CONFIG.watch_interval)
def pre_validate(self, obj):
if not LifeCycle.is_done(obj.status):
raise ValidationError(
"Cannot resume this run, the run must reach a final state first, "
"current status error: {}".format(obj.status))
def service(ctx, yes, external, url):
"""Open the operation service in browser.
N.B. The operation must have a run kind service, otherwise it will raise an error.
You can open the service embedded in Polyaxon UI or using the real service URL,
please use the `--external` flag.
"""
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True,
)
client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid)
client.refresh_data()
if client.run_data.kind != V1RunKind.SERVICE:
Printer.print_warning(
"Command expected an operation of "
"kind `service` received kind: `{}`!".format(client.run_data.kind)
)
sys.exit(1)
Printer.print_header("Waiting for running condition ...")
client.wait_for_condition(
statuses={V1Statuses.RUNNING} | LifeCycle.DONE_VALUES, print_status=True
)
client.refresh_data()
if LifeCycle.is_done(client.run_data.status):
Printer.print_header("The operations reached a done statuses.")
latest_status = Printer.add_status_color(
{"status": client.run_data.status}, status_key="status"
)
click.echo("{}\n".format(latest_status["status"]))
run_url = get_dashboard_url(
subpath="{}/{}/runs/{}/service".format(owner, project_name, run_uuid)
)
namespace = client.run_data.settings.namespace
service_endpoint = SERVICES_V1
if client.run_data.meta_info.get("rewrite_path", False):
service_endpoint = REWRITE_SERVICES_V1
external_run_url = get_dashboard_url(
base=service_endpoint,
subpath="{}/{}/{}/runs/{}/".format(namespace, owner, project_name, run_uuid),
)
if url:
Printer.print_header("The service will be available at: {}".format(run_url))
Printer.print_header(
"You can also view it in an external link at: {}".format(external_run_url)
)
sys.exit(0)
if not yes:
click.confirm(
"Dashboard page will now open in your browser. Continue?",
abort=True,
default=True,
)
if external:
click.launch(external_run_url)
sys.exit(0)
click.launch(run_url)
