def update(self, new_value, text=None):
self.value = new_value
self.text = text if text is not None else '{} / {}'.format(
self.value, self.max)
if self.display_id is not None:
update_display(self, display_id=self.display_id)
return self
def _display_plot(self, model_id, refresh=False):
try:
self._active_plot = model_id
# only update the plot if there is more data since the last viewing, or if a new model is running
if refresh or self.model_data[model_id].num_data_rows > len(
self.model_plots[model_id].lines[0].y):
plot_data = self.model_data[model_id].get_plot_data()
self.model_plots[model_id].update(plot_data)
with self.plot_output:
clear_output(wait=True)
if self.model_displays[model_id] is None:
self.model_displays[model_id] = display(
self.model_plots[model_id], display_id=True)
else:
update_display(self.model_plots[model_id],
display_id=self.model_displays[model_id])
except Exception as e:
self.debug.append_stdout(
"Exception while switching to plot {}: {}\n".format(
model_id,
traceback.format_exception(etype=e.__class__,
value=e,
tb=e.__traceback__)))
def viewCameraStreamJupyterWindows(self):
# Live view in a Jupyter Notebook
try:
start = self.getImage(saveImage=False, saveNumpy=False)
g = BytesIO()
PIL.Image.fromarray(start).save(g, 'jpeg')
obj = Image(data=g.getvalue())
dis = display(obj, display_id=True)
while True:
img = self.getImage(saveImage=False, saveNumpy=False)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
f = BytesIO()
PIL.Image.fromarray(img).save(f, 'jpeg')
obj = Image(data=f.getvalue())
update_display(obj, display_id=dis.display_id)
except KeyboardInterrupt:
self.quit_and_open()
def progress(n, width=50):
"""Display progress bar for long running operations.
:param n: total number of steps to completion
:param width: width of the progress bar (only for the text version)
>>> import arlpy
>>> progress = arlpy.utils.progress(100)
>>> for j in range(100):
next(progress)
"""
if _notebook:
import IPython.display as _ipyd
p = _ipyd.ProgressBar(total=n)
did = str(_uuid.uuid4())
_ipyd.display(p, display_id=did)
for j in range(1, n):
p.progress = j
_ipyd.update_display(p, display_id=did)
yield j
_ipyd.update_display(_ipyd.HTML(''), display_id=did)
yield None
else:
_sys.stdout.write('%s|\n' % ('-' * width))
_sys.stdout.flush()
c = 0
for j in range(n):
c1 = int(width * (j + 1) / n)
if c1 > c:
_sys.stdout.write('>' * (c1 - c))
c = c1
if c == width:
_sys.stdout.write('\n')
_sys.stdout.flush()
yield j
def display_visualization(self, event, widget_instance):
try:
self.debug.append_stdout("Event received: {}\n".format(event))
# this means that all rows have been deselected
if len(event['new']) == 0:
return
row_id = event['new'][0]
model_id = self.param_table.get_changed_df().index[row_id]
self._active_plot = model_id
# only update the plot if there is more data since the last viewing
if self.model_data[model_id].num_data_rows > len(
self.model_plots[model_id].lines[0].y):
plot_data = self.model_data[model_id].get_plot_data()
self.model_plots[model_id].update(plot_data)
with self.output:
clear_output(wait=True)
if self.model_displays[model_id] is None:
self.model_displays[model_id] = display(
self.model_plots[model_id], display_id=True)
else:
update_display(self.model_plots[model_id],
display_id=self.model_displays[model_id])
except Exception as e:
self.debug.append_stdout(
"Exception while switching to plot {}: {}\n".format(
event, e.args))
def jupyter_play(env, agent, steps=1000):
from IPython import display
policy = agent.get_policy()
vdisplay = Xvfb(width=1280, height=740)
vdisplay.start()
observation = env.reset()
def showarray(a, fmt='png'):
a = np.uint8(a)
f = io.BytesIO()
ima = PIL.Image.fromarray(a).save(f, fmt)
return f.getvalue()
imagehandle = display.display(display.Image(data=showarray(env.render(mode='rgb_array')), width=450), display_id='gymscr')
action = 0
reward = 0
for _ in range(steps):
time.sleep(0.001)
action = agent.compute_action(observation, prev_reward=reward, prev_action=action)
observation, reward, done, info = env.step(action)
display.update_display(display.Image(data=showarray(env.render(mode='rgb_array')), width=450), display_id='gymscr')
if done: break
vdisplay.stop()
def _update_display(self, df, *args, **kwargs):
if self.enviroment == "google.colab":
try:
return update_display(df.data, *args, **kwargs)
except:
return update_display(df, *args, **kwargs)
else:
return update_display(df, *args, **kwargs)
def display(self, content):
if not self.is_ipython:
print(content, end='')
print('\033[' + str(nlines(content)) + 'A')
else:
update_display({'text/plain': content},
display_id=self.display_id,
raw=True)
def on_trial_end(self, hyper_model, space, trial_no, reward, improved,
elapsed):
self.last_trial_no = trial_no
self.last_reward = reward
best_trial = hyper_model.get_best_trial()
if best_trial is not None and self.best_trial_display_id is not None:
update_display(best_trial.space_sample,
display_id=self.best_trial_display_id)
def draw_graph(self):
while self.running:
f, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, sharex='col')
f.set_size_inches(10, 5)
ax1.grid(alpha=0.5)
ax1.set_title("Network Performance")
ax1.set_xlabel('Sec')
ax1.set_ylabel('Mb')
ax2.grid(alpha=0.5)
ax2.set_title("DISK IO")
ax2.set_xlabel('Sec')
ax2.set_ylabel('MB')
ax3.grid(alpha=0.5)
ax3.set_title("CPU Usage")
ax3.set_xlabel('Sec')
ax3.set_ylabel('Percentage')
ax4.grid(alpha=0.5)
ax4.set_title("Memory Usage")
ax4.set_xlabel('Sec')
ax4.set_ylabel('Percentage')
# print(self.network_monitor_list)
netarr = numpy.array(self.network_monitor_list[-25:])
netrec = numpy.array(self.network_rec_list[-25:])
netsnt = numpy.array(self.network_snt_list[-25:])
diskarr = numpy.array(self.diskio_monitor_list[-25:])
diskw = numpy.array(self.diskio_write_list[-25:])
diskr = numpy.array(self.diskio_read_list[-25:])
cpuarr = numpy.array(self.cup_monitor_list[-25:])
memarr = numpy.array(self.mem_monitor_list[-25:])
ax1.plot(((netarr)), color="blue", label="total")
ax1.plot(((netrec)), color="red", label="receive")
ax1.plot(((netsnt)), color="green", label="sent")
ax1.legend(loc='upper right',
shadow=False,
framealpha=0.2,
fancybox=True)
ax2.plot(((diskarr)), color="blue", label="total")
ax2.plot(((diskw)), color="red", label="write")
ax2.plot(((diskr)), color="green", label="read")
ax2.legend(loc='upper right',
shadow=False,
framealpha=0.2,
fancybox=True)
ax3.plot(((cpuarr)))
ax4.plot(((memarr)))
display.update_display(plt.show(), display_id="DISPLAY_ID_1")
time.sleep(1)
display.clear_output(wait=True)
def display_state(self):
# Set squares from variables and display the state.
for row in self.rows:
#if len(self.variables[row]) == 1:
for i, j in enumerate(self.variables[row][0]):
self.squares[len(self.rows)-row-1, i] = int(j)
for col in self.cols:
#if len(self.variables[col]) == 1:
for i, j in enumerate(self.variables[col][0]):
self.squares[i, (col-len(self.rows))] = int(j)
update_display(HTML(self.board_html()), display_id=str(id(self)))
def _update_display(self, df, *args, **kwargs):
if not self.html_param:
try:
print(df.data)
except:
print(df)
return
elif self.enviroment == "google.colab":
try:
return update_display(df.data, *args, **kwargs)
except:
return update_display(df, *args, **kwargs)
else:
return update_display(df, *args, **kwargs)
def response(self, change):
if self.validate():
traceType = self.choose_trace(self.textbox1.value, self.textbox2.value)
with self.g.batch_update():
if traceType == 'table':
self.g.update_layout({'height':10, 'width':10})
self.g.layout.xaxis.title = ""
self.g.layout.yaxis.title = ""
self.g.layout.title = ""
self.button.layout.display = 'flex'
else:
self.updateTraces()
update_display(Nothing(), display_id='1')
self.button.layout.display = 'none'
def play(filename,
player_id,
autoplay=True,
normalize=True,
start=None,
stop=None):
if start is not None and stop is not None:
raw_wav = raw_wav_segment(filename, start, stop)
update_display(Audio(data=raw_wav,
autoplay=autoplay,
normalize=normalize),
display_id=player_id)
else:
update_display(Audio(filename, autoplay=autoplay, normalize=normalize),
display_id=player_id)
def _do_display(self):
"""
Do the display, only to be called internally on update.
"""
from IPython.display import display, update_display
# TODO(do something where we have more control besides pandas directly)
with pd.option_context('display.html.table_schema', True):
metadata={"application/vnd.dataresource+json": { "sampled": self.sampled} }
if not self.display:
self.display = display(
self.pdf,
metadata=metadata,
display_id=True)
else:
update_display(self.pdf, display_id=self.display, metadata=metadata)
def on_trial_begin(self, hyper_model, space, trial_no):
df_summary = pd.DataFrame(
[(trial_no, self.last_reward, hyper_model.best_trial_no,
hyper_model.best_reward, time.time() - self.start_time,
len([t for t in hyper_model.history.trials if t.succeeded
]), self.max_trials)],
columns=[
'Trial No.', 'Previous reward', 'Best trial', 'Best reward',
'Total elapsed', 'Valid trials', 'Max trials'
])
if self.search_summary_display_id is not None:
update_display(df_summary,
display_id=self.search_summary_display_id)
if self.current_trial_display_id is not None:
update_display(space, display_id=self.current_trial_display_id)
def _display_resources(self, model_id):
try:
with self.resources_output:
clear_output(wait=True)
if self.model_resource_displays[model_id] is None:
self.model_resource_displays[model_id] = display(
self.model_resource_plots[model_id], display_id=True)
else:
update_display(
self.model_resource_plots[model_id],
display_id=self.model_resource_displays[model_id])
except Exception as e:
self.debug.append_stdout(
"Exception while displaying resources for {}: {}\n".format(
model_id,
traceback.format_exception(etype=e.__class__,
value=e,
tb=e.__traceback__)))
def test_update_display():
ip = get_ipython()
with mock.patch.object(ip.display_pub, 'publish') as pub:
with nt.assert_raises(TypeError):
display.update_display('x')
display.update_display('x', display_id='1')
display.update_display('y', display_id='2')
args, kwargs = pub.call_args_list[0]
nt.assert_equal(args, ())
nt.assert_equal(
kwargs, {
'data': {
'text/plain': repr('x')
},
'metadata': {},
'transient': {
'display_id': '1',
},
'update': True,
})
args, kwargs = pub.call_args_list[1]
nt.assert_equal(args, ())
nt.assert_equal(
kwargs, {
'data': {
'text/plain': repr('y')
},
'metadata': {},
'transient': {
'display_id': '2',
},
'update': True,
})
def on_search_end(self, hyper_model):
df_summary = pd.DataFrame(
[(self.last_trial_no, self.last_reward, hyper_model.best_trial_no,
hyper_model.best_reward, time.time() - self.start_time,
len([t for t in hyper_model.history.trials if t.succeeded
]), self.max_trials)],
columns=[
'Trial No.', 'Previous reward', 'Best trial', 'Best reward',
'Total elapsed', 'Valid trials', 'Max trials'
])
if self.search_summary_display_id is not None:
update_display(df_summary,
display_id=self.search_summary_display_id)
if self.title_display_id is not None:
update_display({'text/markdown': '#### Top trials:'},
raw=True,
include=['text/markdown'],
display_id=self.title_display_id)
df_best_trials = pd.DataFrame(
[(t.trial_no, t.reward, t.elapsed, t.space_sample.vectors)
for t in hyper_model.get_top_trials(5)],
columns=['Trial No.', 'Reward', 'Elapsed', 'Space Vector'])
if self.current_trial_display_id is not None:
update_display(df_best_trials,
display_id=self.current_trial_display_id)
def dispatch(self, hyper_model, X, y, X_val, y_val, max_trails, dataset_id, trail_store,
**fit_kwargs):
retry_limit = int(config('search_retry', '1000'))
trail_no = 1
retry_counter = 0
current_trail_display_id = None
search_summary_display_id = None
title_display_id = None
start_time = time.time()
last_reward = 0
while trail_no <= max_trails:
space_sample = hyper_model.searcher.sample()
if hyper_model.history.is_existed(space_sample):
if retry_counter >= retry_limit:
logger.info(f'Unable to take valid sample and exceed the retry limit {retry_limit}.')
break
trail = hyper_model.history.get_trail(space_sample)
for callback in hyper_model.callbacks:
callback.on_skip_trail(hyper_model, space_sample, trail_no, 'trail_existed', trail.reward, False,
trail.elapsed)
retry_counter += 1
continue
try:
if trail_store is not None:
trail = trail_store.get(dataset_id, space_sample)
if trail is not None:
reward = trail.reward
elapsed = trail.elapsed
trail = Trail(space_sample, trail_no, reward, elapsed)
improved = hyper_model.history.append(trail)
hyper_model.searcher.update_result(space_sample, reward)
for callback in hyper_model.callbacks:
callback.on_skip_trail(hyper_model, space_sample, trail_no, 'hit_trail_store', reward,
improved,
elapsed)
trail_no += 1
continue
for callback in hyper_model.callbacks:
# callback.on_build_estimator(hyper_model, space_sample, estimator, trail_no) #fixme
callback.on_trail_begin(hyper_model, space_sample, trail_no)
model_file = '%s/%05d_%s.pkl' % (self.models_dir, trail_no, space_sample.space_id)
df_summary = pd.DataFrame([(trail_no, last_reward, hyper_model.best_trail_no, hyper_model.best_reward,
time.time() - start_time, max_trails)],
columns=['trail No.', 'Previous reward', 'Best trail', 'Best reward',
'Total elapsed',
'Max trails'])
if search_summary_display_id is None:
handle = display(df_summary, display_id=True)
if handle is not None:
search_summary_display_id = handle.display_id
else:
update_display(df_summary, display_id=search_summary_display_id)
if current_trail_display_id is None:
handle = display({'text/markdown': '#### Current Trail:'}, raw=True, include=['text/markdown'],
display_id=True)
if handle is not None:
title_display_id = handle.display_id
handle = display(space_sample, display_id=True)
if handle is not None:
current_trail_display_id = handle.display_id
else:
update_display(space_sample, display_id=current_trail_display_id)
trail = hyper_model._run_trial(space_sample, trail_no, X, y, X_val, y_val, model_file, **fit_kwargs)
last_reward = trail.reward
if trail.reward != 0: # success
improved = hyper_model.history.append(trail)
for callback in hyper_model.callbacks:
callback.on_trail_end(hyper_model, space_sample, trail_no, trail.reward,
improved, trail.elapsed)
else:
for callback in hyper_model.callbacks:
callback.on_trail_error(hyper_model, space_sample, trail_no)
if logger.is_info_enabled():
msg = f'Trail {trail_no} done, reward: {trail.reward}, best_trail_no:{hyper_model.best_trail_no}, best_reward:{hyper_model.best_reward}\n'
logger.info(msg)
if trail_store is not None:
trail_store.put(dataset_id, trail)
except EarlyStoppingError:
break
# TODO: early stopping
except Exception as e:
import sys
import traceback
msg = f'{">" * 20} Trail {trail_no} failed! {"<" * 20}\n' \
+ f'{e.__class__.__name__}: {e}\n' \
+ traceback.format_exc() \
+ '*' * 50
logger.error(msg)
finally:
trail_no += 1
retry_counter = 0
update_display({'text/markdown': '#### Top trails:'}, raw=True, include=['text/markdown'],
display_id=title_display_id)
df_best_trails = pd.DataFrame([
(t.trail_no, t.reward, t.elapsed) for t in hyper_model.get_top_trails(5)],
columns=['Trail No.', 'Reward', 'Elapsed'])
if current_trail_display_id is None:
display(df_best_trails, display_id=True)
else:
update_display(df_best_trails, display_id=current_trail_display_id)
return trail_no
def update(self, samples):
# If this is a module-level import, readthedocs fails because
# this triggers an import of _tkinter, which isn't built in to
# the python that they use.
import seaborn as sns
with Timer() as timer:
timings = analyze.timings(samples)
bucketed_timings = analyze.bucket_resample_timings(samples)
for name, sources in self._sources.items():
array = timings[name].values
hist, edges = np.histogram(array, density=True, bins="auto")
x, y = sns.distributions._statsmodels_univariate_kde(
array,
"gau",
"scott",
200,
3,
(-np.inf, np.inf),
cumulative=False,
)
whisker_height = np.max(y) / 2
lower, median, upper = np.percentile(array, [25., 50., 75.])
sources["hist"].data = {
"top": hist, "left": edges[:-1], "right": edges[1:]
}
sources["pdf"].data = {"x": x, "y": y}
sources["stddev"].data = {
"base": [whisker_height],
"lower": [lower],
"upper": [upper],
}
sources["median"].data = {"x": [median], "y": [whisker_height]}
describe_html = (
timings.describe().style.set_precision(3).set_caption(
"Descriptive Timing Statistics"
).render()
)
if len(self._sources) > 1:
ks_frame = analyze.ks_test(timings)
ks_bk_frame = analyze.ks_test(bucketed_timings)
ks_html = (
ks_frame.style.applymap(self._ks_style).set_precision(
3
).set_caption(
"K-S test"
).render()
)
ks_bk_html = (
ks_bk_frame.style.applymap(self._ks_style).set_precision(
2
).set_caption(
"Bucketed K-S test"
).render()
)
html = describe_html + ks_html + ks_bk_html
self._describe_widget.data = html.replace(
"table", 'table style="display:inline"'
)
else:
self._describe_widget.data = describe_html
total_bench_time = timings[self._initial_size:].sum().sum() / 1000.
elapsed = time.perf_counter() - self._start
num_samples = len(timings.index)
title = (
"{} samples, {:.2f} sec elapsed, {:.2f} samples/sec, "
"{:.2f}% efficiency"
).format(
num_samples,
elapsed,
(num_samples - self._initial_size) / elapsed,
100. * total_bench_time / elapsed
)
if self._plot is None:
self._plot = self.initialize_plot(title)
bi.show(self._plot, notebook_handle=True)
ipdisplay.display(
self._describe_widget, display_id=self._display_id
)
else:
self._plot.title.text = title
bi.push_notebook()
ipdisplay.update_display(
self._describe_widget, display_id=self._display_id
)
self._elapsed_rendering_seconds += timer.elapsed_seconds()
def UpdateDisplay(self):
update_display(self.AsGraphViz(), display_id=self.display_id)
def display_state(self):
update_display(HTML(self.board_html()), display_id=str(id(self)))
def update_table(self, change):
update_display(self.pivot_table(), display_id='1');
self.button.layout.display = 'flex'
