def poll(self):
"""
Wait until the job finishes.
This method will continuously query the server about the status of the job, until the job reaches a
completion. During this time we will display (in stdout) a progress bar with % completion status.
"""
try:
hidden = not H2OJob.__PROGRESS_BAR__
pb = ProgressBar(title=self._job_type + " progress", hidden=hidden)
pb.execute(self._refresh_job_status)
except StopIteration as e:
if str(e) == "cancelled":
h2o.api("POST /3/Jobs/%s/cancel" % self.job_key)
self.status = "CANCELLED"
# Potentially we may want to re-raise the exception here
assert self.status in {"DONE", "CANCELLED", "FAILED"} or self._poll_count <= 0, \
"Polling finished while the job has status %s" % self.status
if self.warnings:
for w in self.warnings:
warnings.warn(w)
# check if failed... and politely print relevant message
if self.status == "CANCELLED":
raise H2OJobCancelled("Job<%s> was cancelled by the user." % self.job_key)
if self.status == "FAILED":
if (isinstance(self.job, dict)) and ("stacktrace" in list(self.job)):
raise EnvironmentError("Job with key {} failed with an exception: {}\nstacktrace: "
"\n{}".format(self.job_key, self.exception, self.job["stacktrace"]))
else:
raise EnvironmentError("Job with key %s failed with an exception: %s" % (self.job_key, self.exception))
return self
def poll(self):
"""
Wait until the job finishes.
This method will continuously query the server about the status of the job, until the job reaches a
completion. During this time we will display (in stdout) a progress bar with % completion status.
"""
try:
pb = ProgressBar(self._job_type + " progress")
pb.execute(self._refresh_job_status)
except StopIteration as e:
if str(e) == "cancelled":
self.status = "CANCELLED"
h2o.api("POST /3/Jobs/%s/cancel" % self.job_key)
print("Job {} was cancelled.".format(self.job_key))
# Potentially we may want to re-raise the exception here
if self.warnings:
for w in self.warnings:
warnings.warn(w)
# TODO: this needs to br thought through more carefully
# check if failed... and politely print relevant message
if self.status == "CANCELLED":
raise EnvironmentError("Job with key {} was cancelled by the user.".format(self.job_key))
if self.status == "FAILED":
if (isinstance(self.job, dict)) and ("stacktrace" in list(self.job)):
raise EnvironmentError("Job with key {} failed with an exception: {}\nstacktrace: "
"\n{}".format(self.job_key, self.exception, self.job["stacktrace"]))
else:
raise EnvironmentError("Job with key %s failed with an exception: %s" % (self.job_key, self.exception))
return self
def poll(self, verbose_model_scoring_history=False):
"""
Wait until the job finishes.
This method will continuously query the server about the status of the job, until the job reaches a
completion. During this time we will display (in stdout) a progress bar with % completion status.
"""
try:
hidden = not H2OJob.__PROGRESS_BAR__
pb = ProgressBar(title=self._job_type + " progress", hidden=hidden)
if verbose_model_scoring_history:
pb.execute(
self._refresh_job_status,
print_verbose_info=lambda x: self._print_verbose_info()
if int(x * 10) % 5 == 0 else " ")
else:
pb.execute(self._refresh_job_status)
except StopIteration as e:
if str(e) == "cancelled":
h2o.api("POST /3/Jobs/%s/cancel" % self.job_key)
self.status = "CANCELLED"
# Potentially we may want to re-raise the exception here
assert self.status in {"DONE", "CANCELLED", "FAILED"} or self._poll_count <= 0, \
"Polling finished while the job has status %s" % self.status
if self.warnings:
for w in self.warnings:
warnings.warn(w)
# check if failed... and politely print relevant message
if self.status == "CANCELLED":
raise H2OJobCancelled("Job<%s> was cancelled by the user." %
self.job_key)
if self.status == "FAILED":
if (isinstance(self.job, dict)) and ("stacktrace" in list(
self.job)):
raise EnvironmentError(
"Job with key {} failed with an exception: {}\nstacktrace: "
"\n{}".format(self.job_key, self.exception,
self.job["stacktrace"]))
else:
raise EnvironmentError(
"Job with key %s failed with an exception: %s" %
(self.job_key, self.exception))
return self
def poll(self, poll_updates=None):
"""
Wait until the job finishes.
This method will continuously query the server about the status of the job, until the job reaches a
completion. During this time we will display (in stdout) a progress bar with % completion status.
:param poll_updates: a callback function called a each polling iteration with 2 arguments:
(current_job: H2OJob, bar_progression: float)
"""
try:
hidden = not H2OJob.__PROGRESS_BAR__
pb = ProgressBar(title=self._job_type + " progress", hidden=hidden)
if poll_updates:
pb.execute(self._refresh_job_status,
print_verbose_info=ft.partial(poll_updates, self))
else:
pb.execute(self._refresh_job_status)
except StopIteration as e:
if str(e) == "cancelled":
self.cancel()
# Potentially we may want to re-raise the exception here
assert self.status in {"DONE", "CANCELLED", "FAILED"} or self._poll_count <= 0, \
"Polling finished while the job has status %s" % self.status
if self.warnings:
for w in self.warnings:
warnings.warn(w)
# check if failed... and politely print relevant message
if self.status == "CANCELLED":
raise H2OJobCancelled("Job<%s> was cancelled by the user." %
self.job_key)
if self.status == "FAILED":
if (isinstance(self.job, dict)) and ("stacktrace" in list(
self.job)):
raise EnvironmentError(
"Job with key {} failed with an exception: {}\nstacktrace: "
"\n{}".format(self.job_key, self.exception,
self.job["stacktrace"]))
else:
raise EnvironmentError(
"Job with key %s failed with an exception: %s" %
(self.job_key, self.exception))
return self
def poll(self):
"""
Wait until the job finishes.
This method will continuously query the server about the status of the job, until the job reaches a
completion. During this time we will display (in stdout) a progress bar with % completion status.
"""
try:
pb = ProgressBar(self._job_type + " progress")
pb.execute(self._refresh_job_status)
except StopIteration as e:
if str(e) == "cancelled":
h2o.api("POST /3/Jobs/%s/cancel" % self.job_key)
self.status = "CANCELLED"
# Potentially we may want to re-raise the exception here
assert self.status in {"DONE", "CANCELLED", "FAILED"} or self._poll_count <= 0, \
"Polling finished while the job has status %s" % self.status
if self.warnings:
for w in self.warnings:
warnings.warn(w)
# TODO: this needs to br thought through more carefully
# Right now if the user presses Ctrl+C the progress bar handles this gracefully and passes the
# exception up, but these calls create ugly stacktrace dumps...
# check if failed... and politely print relevant message
if self.status == "CANCELLED":
raise EnvironmentError(
"Job with key {} was cancelled by the user.".format(
self.job_key))
if self.status == "FAILED":
if (isinstance(self.job, dict)) and ("stacktrace" in list(
self.job)):
raise EnvironmentError(
"Job with key {} failed with an exception: {}\nstacktrace: "
"\n{}".format(self.job_key, self.exception,
self.job["stacktrace"]))
else:
raise EnvironmentError(
"Job with key %s failed with an exception: %s" %
(self.job_key, self.exception))
return self