def display_html(vals):
'''
Convert objects into three AXON forms for output in IPython notebook.
:param vals:
List of objects or ordered dict.
:returns:
IPython TabbedView widget.
'''
from IPython.html.widgets import Tab, HTML
# Compact form
p1 = dumps(vals, crossref=1)
# Formatted form without braces
p2 = dumps(vals, pretty=1, crossref=1)
# Formatted form with braces
p3 = dumps(vals, pretty=1, braces=1, crossref=1)
wg = Tab((HTML("<pre>%s</pre>" % p1), HTML("<pre>%s</pre>" % p2),
HTML("<pre>%s</pre>" % p3)))
wg.set_title(0, "Compact")
wg.set_title(1, "Formatted")
wg.set_title(2, "Formatted with {}")
wg.selected_index = 1
return wg
def status_printer(_, total=None, desc=None, ncols=None, img=None):
"""
Manage the printing of an IPython/Jupyter Notebook progress bar widget.
"""
# Fallback to text bar if there's no total
# DEPRECATED: replaced with an 'info' style bar
# if not total:
# return super(mytqdm, mytqdm).status_printer(file)
# fp = file
# Prepare IPython progress bar
try:
if total:
pbar = IntProgress(min=0, max=total)
else: # No total? Show info style bar with no progress tqdm status
pbar = IntProgress(min=0, max=1)
pbar.value = 1
pbar.bar_style = 'info'
except NameError:
# #187 #451 #558
raise ImportError(
"IntProgress not found. Please update jupyter and ipywidgets."
" See https://ipywidgets.readthedocs.io/en/stable"
"/user_install.html")
if desc:
pbar.description = desc
if IPYW >= 7:
pbar.style.description_width = 'initial'
# Prepare status text
ptext = HTML()
timg = HTML()
if img:
timg.value = "<br>%s<br>" % img
# Only way to place text to the right of the bar is to use a container
container = VBox([HBox(children=[pbar, ptext]), timg])
# Prepare layout
if ncols is not None: # use default style of ipywidgets
# ncols could be 100, "100px", "100%"
ncols = str(ncols) # ipywidgets only accepts string
try:
if int(ncols) > 0: # isnumeric and positive
ncols += 'px'
except ValueError:
pass
pbar.layout.flex = '2'
container.layout.width = ncols
container.layout.display = 'inline-flex'
container.layout.flex_flow = 'row wrap'
display(container)
return container
def __init__(self,start,end):
self.pbar = IntProgress(orientation='horizontal',min=start, max=end)
self.ptext = HTML()
# Only way to place text to the right of the bar is to use a container
container = HBox(children=[self.pbar, self.ptext])
display(container)
def status_printer(_, total=None, desc=None, ncols=None):
"""
Manage the printing of an IPython/Jupyter Notebook progress bar widget.
"""
# Fallback to text bar if there's no total
# DEPRECATED: replaced with an 'info' style bar
# if not total:
# return super(tqdm_notebook, tqdm_notebook).status_printer(file)
# fp = file
# Prepare IPython progress bar
if IProgress is None: # #187 #451 #558 #872
raise ImportError(
"IProgress not found. Please update jupyter and ipywidgets."
" See https://ipywidgets.readthedocs.io/en/stable"
"/user_install.html")
if total:
pbar = IProgress(min=0, max=total)
else: # No total? Show info style bar with no progress tqdm status
pbar = IProgress(min=0, max=1)
pbar.value = 1
pbar.bar_style = 'info'
if ncols is None:
pbar.layout.width = "20px"
ltext = HTML()
rtext = HTML()
if desc:
ltext.value = desc
container = HBox(children=[ltext, pbar, rtext])
# Prepare layout
if ncols is not None: # use default style of ipywidgets
# ncols could be 100, "100px", "100%"
ncols = str(ncols) # ipywidgets only accepts string
try:
if int(ncols) > 0: # isnumeric and positive
ncols += 'px'
except ValueError:
pass
pbar.layout.flex = '2'
container.layout.width = ncols
container.layout.display = 'inline-flex'
container.layout.flex_flow = 'row wrap'
display(container)
return container
def status_printer(_, total=None, desc=None, ncols=None):
"""
Manage the printing of an IPython/Jupyter Notebook progress bar widget.
"""
# Fallback to text bar if there's no total
# DEPRECATED: replaced with an 'info' style bar
# if not total:
# return super(tqdm_notebook, tqdm_notebook).status_printer(file)
# fp = file
# Prepare IPython progress bar
try:
if total:
pbar = IntProgress(min=0, max=total)
else: # No total? Show info style bar with no progress tqdm status
pbar = IntProgress(min=0, max=1)
pbar.value = 1
pbar.bar_style = 'info'
except NameError:
# #187 #451 #558
raise ImportError(
"IntProgress not found. Please update jupyter and ipywidgets."
" See https://ipywidgets.readthedocs.io/en/stable"
"/user_install.html")
if desc:
pbar.description = desc
if IPYW >= 7:
pbar.style.description_width = 'initial'
# Prepare status text
ptext = HTML()
# Only way to place text to the right of the bar is to use a container
container = HBox(children=[pbar, ptext])
# Prepare layout
if ncols is not None: # use default style of ipywidgets
# ncols could be 100, "100px", "100%"
ncols = str(ncols) # ipywidgets only accepts string
try:
if int(ncols) > 0: # isnumeric and positive
ncols += 'px'
except ValueError:
pass
pbar.layout.flex = '2'
container.layout.width = ncols
container.layout.display = 'inline-flex'
container.layout.flex_flow = 'row wrap'
display(container)
def print_status(s='', close=False, bar_style=None, desc=None):
# Note: contrary to native tqdm, s='' does NOT clear bar
# goal is to keep all infos if error happens so user knows
# at which iteration the loop failed.
# Clear previous output (really necessary?)
# clear_output(wait=1)
# Get current iteration value from format_meter string
if total:
# n = None
if s:
npos = s.find(r'/|/') # cause we use bar_format=r'{n}|...'
# Check that n can be found in s (else n > total)
if npos >= 0:
n = float(s[:npos]) # get n from string
s = s[npos + 3:] # remove from string
# Update bar with current n value
if n is not None:
pbar.value = n
# Print stats
if s: # never clear the bar (signal: s='')
s = s.replace('||', '') # remove inesthetical pipes
s = escape(s) # html escape special characters (like '?')
ptext.value = s
# Change bar style
if bar_style:
# Hack-ish way to avoid the danger bar_style being overriden by
# success because the bar gets closed after the error...
if not (pbar.bar_style == 'danger' and bar_style == 'success'):
pbar.bar_style = bar_style
# Special signal to close the bar
if close and pbar.bar_style != 'danger': # hide only if no error
try:
container.close()
except AttributeError:
container.visible = False
# Update description
if desc:
pbar.description = desc
if IPYW >= 7:
pbar.style.description_width = 'initial'
return print_status
def status_printer(_, total=None, desc=None):
"""
Manage the printing of an IPython/Jupyter Notebook progress bar widget.
"""
# Fallback to text bar if there's no total
# DEPRECATED: replaced with an 'info' style bar
# if not total:
# return super(tqdm_notebook, tqdm_notebook).status_printer(file)
# fp = file
# Prepare IPython progress bar
if total:
pbar = IntProgress(min=0, max=total)
else: # No total? Show info style bar with no progress tqdm status
pbar = IntProgress(min=0, max=1)
pbar.value = 1
pbar.bar_style = 'info'
if desc:
pbar.description = desc
# Prepare status text
ptext = HTML()
# Only way to place text to the right of the bar is to use a container
container = HBox(children=[pbar, ptext])
display(container)
def print_status(s='', close=False, bar_style=None):
# Note: contrary to native tqdm, s='' does NOT clear bar
# goal is to keep all infos if error happens so user knows
# at which iteration the loop failed.
# Clear previous output (really necessary?)
# clear_output(wait=1)
# Get current iteration value from format_meter string
if total:
n = None
if s:
npos = s.find(r'/|/') # cause we use bar_format=r'{n}|...'
# Check that n can be found in s (else n > total)
if npos >= 0:
n = int(s[:npos]) # get n from string
s = s[npos + 3:] # remove from string
# Update bar with current n value
if n is not None:
pbar.value = n
# Print stats
if s: # never clear the bar (signal: s='')
s = s.replace('||', '') # remove inesthetical pipes
s = escape(s) # html escape special characters (like '?')
ptext.value = s
# Change bar style
if bar_style:
# Hack-ish way to avoid the danger bar_style being overriden by
# success because the bar gets closed after the error...
if not (pbar.bar_style == 'danger' and bar_style == 'success'):
pbar.bar_style = bar_style
# Special signal to close the bar
if close and pbar.bar_style != 'danger': # hide only if no error
container.visible = False
return print_status
def status_printer(self, _, total=None, desc=None, ncols=None):
"""
Manage the printing of an IPython/Jupyter Notebook progress bar widget.
"""
# Fallback to text bar if there's no total
# DEPRECATED: replaced with an 'info' style bar
# if not total:
# return super(tqdm_notebook_EX, tqdm_notebook).status_printer(file)
# fp = file
# Prepare IPython progress bar
if total:
pbar = IntProgress(min=0, max=total)
else: # No total? Show info style bar with no progress tqdm status
pbar = IntProgress(min=0, max=1)
pbar.value = 1
pbar.bar_style = 'info'
if desc:
pbar.description = desc
# Prepare status text
ptext = HTML()
# Only way to place text to the right of the bar is to use a container
container = HBox(children=[pbar, ptext])
if self.vbox:
self.vbox.children = self.vbox.children + (container, )
else:
display(container)
# Prepare layout
if ncols is not None: # use default style of ipywidgets
# ncols could be 100, "100px", "100%"
ncols = str(ncols) # ipywidgets only accepts string
if ncols[-1].isnumeric():
# if last value is digit, assume the value is digit
ncols += 'px'
pbar.layout.flex = '2'
container.layout.width = ncols
container.layout.display = 'inline-flex'
container.layout.flex_flow = 'row wrap'
def print_status(s='', close=False, bar_style=None, desc=None):
# Note: contrary to native tqdm, s='' does NOT clear bar
# goal is to keep all infos if error happens so user knows
# at which iteration the loop failed.
# Clear previous output (really necessary?)
# clear_output(wait=1)
# Get current iteration value from format_meter string
if total:
# n = None
if s:
npos = s.find(r'/|/') # cause we use bar_format=r'{n}|…'
# Check that n can be found in s (else n > total)
if npos >= 0:
n = int(s[:npos]) # get n from string
s = s[npos + 3:] # remove from string
# Update bar with current n value
if n is not None:
pbar.value = n
# Print stats
if s: # never clear the bar (signal: s='')
s = s.replace('||', '') # remove inesthetical pipes
s = escape(s) # html escape special characters (like '?')
ptext.value = s
# Change bar style
if bar_style:
# Hack-ish way to avoid the danger bar_style being overriden by
# success because the bar gets closed after the error…
if not (pbar.bar_style == 'danger' and bar_style == 'success'):
pbar.bar_style = bar_style
# Special signal to close the bar
if close and pbar.bar_style != 'danger': # hide only if no error
try:
container.close()
except AttributeError:
container.visible = False
if self.vbox:
self.vbox.children = tuple(ch for ch in self.vbox.children
if ch is not container)
# Update description
if desc:
pbar.description = desc
return print_status
def __init__(self,
model,
opt,
maxiters,
verbose=False,
current_iteration=0,
ipython_notebook=True,
clear_after_finish=False):
self.verbose = verbose
if self.verbose:
self.model = model
self.iteration = current_iteration
self.p_iter = self.iteration
self.maxiters = maxiters
self.len_maxiters = len(str(maxiters))
self.opt_name = opt.opt_name
self.model.add_observer(self, self.print_status)
self.status = 'running'
self.clear = clear_after_finish
self.update()
try:
from IPython.display import display
from IPython.html.widgets import IntProgress, HTML, Box, VBox, HBox, FlexBox
self.text = HTML(width='100%')
self.progress = IntProgress(min=0, max=maxiters)
#self.progresstext = Text(width='100%', disabled=True, value='0/{}'.format(maxiters))
self.model_show = HTML()
self.ipython_notebook = ipython_notebook
except:
# Not in Ipython notebook
self.ipython_notebook = False
if self.ipython_notebook:
left_col = VBox(children=[self.progress, self.text],
padding=2,
width='40%')
right_col = Box(children=[self.model_show],
padding=2,
width='60%')
self.hor_align = FlexBox(children=[left_col, right_col],
width='100%',
orientation='horizontal')
display(self.hor_align)
try:
self.text.set_css('width', '100%')
left_col.set_css({
'padding': '2px',
'width': "100%",
})
right_col.set_css({
'padding': '2px',
})
self.hor_align.set_css({
'width': "100%",
})
self.hor_align.remove_class('vbox')
self.hor_align.add_class('hbox')
left_col.add_class("box-flex1")
right_col.add_class('box-flex0')
except:
pass
#self.text.add_class('box-flex2')
#self.progress.add_class('box-flex1')
else:
self.exps = exponents(self.fnow, self.current_gradient)
print('Running {} Code:'.format(self.opt_name))
print(' {3:7s} {0:{mi}s} {1:11s} {2:11s}'.format(
"i", "f", "|g|", "runtime", mi=self.len_maxiters))
class VerboseOptimization(object):
def __init__(self,
model,
opt,
maxiters,
verbose=False,
current_iteration=0,
ipython_notebook=True,
clear_after_finish=False):
self.verbose = verbose
if self.verbose:
self.model = model
self.iteration = current_iteration
self.p_iter = self.iteration
self.maxiters = maxiters
self.len_maxiters = len(str(maxiters))
self.opt_name = opt.opt_name
self.model.add_observer(self, self.print_status)
self.status = 'running'
self.clear = clear_after_finish
self.update()
try:
from IPython.display import display
from IPython.html.widgets import IntProgress, HTML, Box, VBox, HBox, FlexBox
self.text = HTML(width='100%')
self.progress = IntProgress(min=0, max=maxiters)
#self.progresstext = Text(width='100%', disabled=True, value='0/{}'.format(maxiters))
self.model_show = HTML()
self.ipython_notebook = ipython_notebook
except:
# Not in Ipython notebook
self.ipython_notebook = False
if self.ipython_notebook:
left_col = VBox(children=[self.progress, self.text],
padding=2,
width='40%')
right_col = Box(children=[self.model_show],
padding=2,
width='60%')
self.hor_align = FlexBox(children=[left_col, right_col],
width='100%',
orientation='horizontal')
display(self.hor_align)
try:
self.text.set_css('width', '100%')
left_col.set_css({
'padding': '2px',
'width': "100%",
})
right_col.set_css({
'padding': '2px',
})
self.hor_align.set_css({
'width': "100%",
})
self.hor_align.remove_class('vbox')
self.hor_align.add_class('hbox')
left_col.add_class("box-flex1")
right_col.add_class('box-flex0')
except:
pass
#self.text.add_class('box-flex2')
#self.progress.add_class('box-flex1')
else:
self.exps = exponents(self.fnow, self.current_gradient)
print('Running {} Code:'.format(self.opt_name))
print(' {3:7s} {0:{mi}s} {1:11s} {2:11s}'.format(
"i", "f", "|g|", "runtime", mi=self.len_maxiters))
def __enter__(self):
self.start = time.time()
self._time = self.start
return self
def print_out(self, seconds):
if seconds < 60:
ms = (seconds % 1) * 100
self.timestring = "{s:0>2d}s{ms:0>2d}".format(s=int(seconds),
ms=int(ms))
else:
m, s = divmod(seconds, 60)
if m > 59:
h, m = divmod(m, 60)
if h > 23:
d, h = divmod(h, 24)
self.timestring = '{d:0>2d}d{h:0>2d}h{m:0>2d}'.format(
m=int(m), h=int(h), d=int(d))
else:
self.timestring = '{h:0>2d}h{m:0>2d}m{s:0>2d}'.format(
m=int(m), s=int(s), h=int(h))
else:
ms = (seconds % 1) * 100
self.timestring = '{m:0>2d}m{s:0>2d}s{ms:0>2d}'.format(
m=int(m), s=int(s), ms=int(ms))
if self.ipython_notebook:
names_vals = [
['optimizer', "{:s}".format(self.opt_name)],
['runtime', "{:>s}".format(self.timestring)],
[
'evaluation', "{:>0{l}}".format(self.iteration,
l=self.len_maxiters)
],
['objective', "{: > 12.3E}".format(self.fnow)],
[
'||gradient||',
"{: >+12.3E}".format(float(self.current_gradient))
],
['status', "{:s}".format(self.status)],
]
#message = "Lik:{:5.3E} Grad:{:5.3E} Lik:{:5.3E} Len:{!s}".format(float(m.log_likelihood()), np.einsum('i,i->', grads, grads), float(m.likelihood.variance), " ".join(["{:3.2E}".format(l) for l in m.kern.lengthscale.values]))
html_begin = """<style type="text/css">
.tg-opt {font-family:"Courier New", Courier, monospace !important;padding:2px 3px;word-break:normal;border-collapse:collapse;border-spacing:0;border-color:#DCDCDC;margin:0px auto;width:100%;}
.tg-opt td{font-family:"Courier New", Courier, monospace !important;font-weight:bold;color:#444;background-color:#F7FDFA;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;border-color:#DCDCDC;}
.tg-opt th{font-family:"Courier New", Courier, monospace !important;font-weight:normal;color:#fff;background-color:#26ADE4;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;border-color:#DCDCDC;}
.tg-opt .tg-left{font-family:"Courier New", Courier, monospace !important;font-weight:normal;text-align:left;}
.tg-opt .tg-right{font-family:"Courier New", Courier, monospace !important;font-weight:normal;text-align:right;}
</style>
<table class="tg-opt">"""
html_end = "</table>"
html_body = ""
for name, val in names_vals:
html_body += "<tr>"
html_body += "<td class='tg-left'>{}</td>".format(name)
html_body += "<td class='tg-right'>{}</td>".format(val)
html_body += "</tr>"
self.text.value = html_begin + html_body + html_end
self.progress.value = (self.iteration + 1)
#self.progresstext.value = '0/{}'.format((self.iteration+1))
self.model_show.value = self.model._repr_html_()
else:
n_exps = exponents(self.fnow, self.current_gradient)
if self.iteration - self.p_iter >= 20 * np.random.rand():
a = self.iteration >= self.p_iter * 2.78
b = np.any(n_exps < self.exps)
if a or b:
self.p_iter = self.iteration
print('')
if b:
self.exps = n_exps
print('\r', end=' ')
print(
'{3:} {0:>0{mi}g} {1:> 12e} {2:> 12e}'.format(
self.iteration,
float(self.fnow),
float(self.current_gradient),
"{:>8s}".format(self.timestring),
mi=self.len_maxiters),
end=' '
) # print 'Iteration:', iteration, ' Objective:', fnow, ' Scale:', beta, '\r',
sys.stdout.flush()
def print_status(self, me, which=None):
self.update()
t = time.time()
seconds = t - self.start
#sys.stdout.write(" "*len(self.message))
if t - self._time > .3 or seconds < .3:
self.print_out(seconds)
self._time = t
self.iteration += 1
def update(self):
self.fnow = self.model.objective_function()
if self.model.obj_grads is not None:
grad = self.model.obj_grads
self.current_gradient = np.dot(grad, grad)
else:
self.current_gradient = np.nan
def finish(self, opt):
self.status = opt.status
if self.verbose and self.ipython_notebook:
if 'conv' in self.status.lower():
self.progress.bar_style = 'success'
elif self.iteration >= self.maxiters:
self.progress.bar_style = 'warning'
else:
self.progress.bar_style = 'danger'
def __exit__(self, type, value, traceback):
if self.verbose:
self.stop = time.time()
self.model.remove_observer(self)
self.print_out(self.stop - self.start)
if not self.ipython_notebook:
print()
print('Runtime: {}'.format("{:>9s}".format(self.timestring)))
print('Optimization status: {0}'.format(self.status))
print()
elif self.clear:
self.hor_align.close()
def __init__(self, model, opt, maxiters, verbose=False, current_iteration=0, ipython_notebook=True, clear_after_finish=False):
self.verbose = verbose
if self.verbose:
self.model = model
self.iteration = current_iteration
self.p_iter = self.iteration
self.maxiters = maxiters
self.len_maxiters = len(str(maxiters))
self.opt_name = opt.opt_name
self.model.add_observer(self, self.print_status)
self.status = 'running'
self.clear = clear_after_finish
self.update()
try:
from IPython.display import display
from IPython.html.widgets import IntProgress, HTML, Box, VBox, HBox, FlexBox
self.text = HTML(width='100%')
self.progress = IntProgress(min=0, max=maxiters)
#self.progresstext = Text(width='100%', disabled=True, value='0/{}'.format(maxiters))
self.model_show = HTML()
self.ipython_notebook = ipython_notebook
except:
# Not in Ipython notebook
self.ipython_notebook = False
if self.ipython_notebook:
left_col = VBox(children=[self.progress, self.text], padding=2, width='40%')
right_col = Box(children=[self.model_show], padding=2, width='60%')
self.hor_align = FlexBox(children = [left_col, right_col], width='100%', orientation='horizontal')
display(self.hor_align)
try:
self.text.set_css('width', '100%')
left_col.set_css({
'padding': '2px',
'width': "100%",
})
right_col.set_css({
'padding': '2px',
})
self.hor_align.set_css({
'width': "100%",
})
self.hor_align.remove_class('vbox')
self.hor_align.add_class('hbox')
left_col.add_class("box-flex1")
right_col.add_class('box-flex0')
except:
pass
#self.text.add_class('box-flex2')
#self.progress.add_class('box-flex1')
else:
self.exps = exponents(self.fnow, self.current_gradient)
print('Running {} Code:'.format(self.opt_name))
print(' {3:7s} {0:{mi}s} {1:11s} {2:11s}'.format("i", "f", "|g|", "runtime", mi=self.len_maxiters))
class VerboseOptimization(object):
def __init__(self, model, opt, maxiters, verbose=False, current_iteration=0, ipython_notebook=True, clear_after_finish=False):
self.verbose = verbose
if self.verbose:
self.model = model
self.iteration = current_iteration
self.p_iter = self.iteration
self.maxiters = maxiters
self.len_maxiters = len(str(maxiters))
self.opt_name = opt.opt_name
self.model.add_observer(self, self.print_status)
self.status = 'running'
self.clear = clear_after_finish
self.update()
try:
from IPython.display import display
from IPython.html.widgets import IntProgress, HTML, Box, VBox, HBox, FlexBox
self.text = HTML(width='100%')
self.progress = IntProgress(min=0, max=maxiters)
#self.progresstext = Text(width='100%', disabled=True, value='0/{}'.format(maxiters))
self.model_show = HTML()
self.ipython_notebook = ipython_notebook
except:
# Not in Ipython notebook
self.ipython_notebook = False
if self.ipython_notebook:
left_col = VBox(children=[self.progress, self.text], padding=2, width='40%')
right_col = Box(children=[self.model_show], padding=2, width='60%')
self.hor_align = FlexBox(children = [left_col, right_col], width='100%', orientation='horizontal')
display(self.hor_align)
try:
self.text.set_css('width', '100%')
left_col.set_css({
'padding': '2px',
'width': "100%",
})
right_col.set_css({
'padding': '2px',
})
self.hor_align.set_css({
'width': "100%",
})
self.hor_align.remove_class('vbox')
self.hor_align.add_class('hbox')
left_col.add_class("box-flex1")
right_col.add_class('box-flex0')
except:
pass
#self.text.add_class('box-flex2')
#self.progress.add_class('box-flex1')
else:
self.exps = exponents(self.fnow, self.current_gradient)
print('Running {} Code:'.format(self.opt_name))
print(' {3:7s} {0:{mi}s} {1:11s} {2:11s}'.format("i", "f", "|g|", "runtime", mi=self.len_maxiters))
def __enter__(self):
self.start = time.time()
self._time = self.start
return self
def print_out(self, seconds):
if seconds<60:
ms = (seconds%1)*100
self.timestring = "{s:0>2d}s{ms:0>2d}".format(s=int(seconds), ms=int(ms))
else:
m, s = divmod(seconds, 60)
if m>59:
h, m = divmod(m, 60)
if h>23:
d, h = divmod(h, 24)
self.timestring = '{d:0>2d}d{h:0>2d}h{m:0>2d}'.format(m=int(m), h=int(h), d=int(d))
else:
self.timestring = '{h:0>2d}h{m:0>2d}m{s:0>2d}'.format(m=int(m), s=int(s), h=int(h))
else:
ms = (seconds%1)*100
self.timestring = '{m:0>2d}m{s:0>2d}s{ms:0>2d}'.format(m=int(m), s=int(s), ms=int(ms))
if self.ipython_notebook:
names_vals = [['optimizer', "{:s}".format(self.opt_name)],
['runtime', "{:>s}".format(self.timestring)],
['evaluation', "{:>0{l}}".format(self.iteration, l=self.len_maxiters)],
['objective', "{: > 12.3E}".format(self.fnow)],
['||gradient||', "{: >+12.3E}".format(float(self.current_gradient))],
['status', "{:s}".format(self.status)],
]
#message = "Lik:{:5.3E} Grad:{:5.3E} Lik:{:5.3E} Len:{!s}".format(float(m.log_likelihood()), np.einsum('i,i->', grads, grads), float(m.likelihood.variance), " ".join(["{:3.2E}".format(l) for l in m.kern.lengthscale.values]))
html_begin = """<style type="text/css">
.tg-opt {font-family:"Courier New", Courier, monospace !important;padding:2px 3px;word-break:normal;border-collapse:collapse;border-spacing:0;border-color:#DCDCDC;margin:0px auto;width:100%;}
.tg-opt td{font-family:"Courier New", Courier, monospace !important;font-weight:bold;color:#444;background-color:#F7FDFA;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;border-color:#DCDCDC;}
.tg-opt th{font-family:"Courier New", Courier, monospace !important;font-weight:normal;color:#fff;background-color:#26ADE4;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;border-color:#DCDCDC;}
.tg-opt .tg-left{font-family:"Courier New", Courier, monospace !important;font-weight:normal;text-align:left;}
.tg-opt .tg-right{font-family:"Courier New", Courier, monospace !important;font-weight:normal;text-align:right;}
</style>
<table class="tg-opt">"""
html_end = "</table>"
html_body = ""
for name, val in names_vals:
html_body += "<tr>"
html_body += "<td class='tg-left'>{}</td>".format(name)
html_body += "<td class='tg-right'>{}</td>".format(val)
html_body += "</tr>"
self.text.value = html_begin + html_body + html_end
self.progress.value = (self.iteration+1)
#self.progresstext.value = '0/{}'.format((self.iteration+1))
self.model_show.value = self.model._repr_html_()
else:
n_exps = exponents(self.fnow, self.current_gradient)
if self.iteration - self.p_iter >= 20 * np.random.rand():
a = self.iteration >= self.p_iter * 2.78
b = np.any(n_exps < self.exps)
if a or b:
self.p_iter = self.iteration
print('')
if b:
self.exps = n_exps
print('\r', end=' ')
print('{3:} {0:>0{mi}g} {1:> 12e} {2:> 12e}'.format(self.iteration, float(self.fnow), float(self.current_gradient), "{:>8s}".format(self.timestring), mi=self.len_maxiters), end=' ') # print 'Iteration:', iteration, ' Objective:', fnow, ' Scale:', beta, '\r',
sys.stdout.flush()
def print_status(self, me, which=None):
self.update()
t = time.time()
seconds = t-self.start
#sys.stdout.write(" "*len(self.message))
if t-self._time > .3 or seconds < .3:
self.print_out(seconds)
self._time = t
self.iteration += 1
def update(self):
self.fnow = self.model.objective_function()
if self.model.obj_grads is not None:
grad = self.model.obj_grads
self.current_gradient = np.dot(grad, grad)
else:
self.current_gradient = np.nan
def finish(self, opt):
self.status = opt.status
if self.verbose and self.ipython_notebook:
if 'conv' in self.status.lower():
self.progress.bar_style = 'success'
elif self.iteration >= self.maxiters:
self.progress.bar_style = 'warning'
else:
self.progress.bar_style = 'danger'
def __exit__(self, type, value, traceback):
if self.verbose:
self.stop = time.time()
self.model.remove_observer(self)
self.print_out(self.stop - self.start)
if not self.ipython_notebook:
print()
print('Runtime: {}'.format("{:>9s}".format(self.timestring)))
print('Optimization status: {0}'.format(self.status))
print()
elif self.clear:
self.hor_align.close()
def optimize(self, method='HS', maxiter=500, ftol=1e-6, gtol=1e-6, step_length=1., callback=None, verbose=True):
"""
Optimize the model.
The strategy is to run conjugate natural gradients on the variational
parameters, interleaved with gradient based optimization of any
non-variational parameters. self.hyperparam_interval dictates how
often this happens.
Arguments
---------
:method: ['FR', 'PR','HS','steepest'] -- conjugate gradient method
:maxiter: int
:ftol: float
:gtol: float
:step_length: float
"""
assert method in ['FR', 'PR','HS','steepest'], 'invalid conjugate gradient method specified.'
## For GPy style notebook verbosity
if verbose:
try:
from IPython.display import display
from IPython.html.widgets import IntProgress, HTML, Box, VBox, HBox, FlexBox
self.text = HTML(width='100%')
self.progress = IntProgress(min=0, max=maxiter)
self.progress.bar_style = 'info'
self.status = 'Running'
html_begin = """<style type="text/css">
.tg-opt {font-family:"Courier New", Courier, monospace !important;padding:2px 3px;word-break:normal;border-collapse:collapse;border-spacing:0;border-color:#DCDCDC;margin:0px auto;width:100%;}
.tg-opt td{font-family:"Courier New", Courier, monospace !important;font-weight:bold;color:#444;background-color:#F7FDFA;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;border-color:#DCDCDC;}
.tg-opt th{font-family:"Courier New", Courier, monospace !important;font-weight:normal;color:#fff;background-color:#26ADE4;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;border-color:#DCDCDC;}
.tg-opt .tg-left{font-family:"Courier New", Courier, monospace !important;font-weight:normal;text-align:left;}
.tg-opt .tg-right{font-family:"Courier New", Courier, monospace !important;font-weight:normal;text-align:right;}
</style>
<table class="tg-opt">"""
html_end = "</table>"
self.ipython_notebook = True
except:
# Not in Ipython notebook
self.ipython_notebook = False
else:
self.ipython_notebook = False
if self.ipython_notebook:
left_col = VBox(children=[self.progress, self.text], padding=2, width='100%')
self.hor_align = FlexBox(children = [left_col], width='100%', orientation='horizontal')
display(self.hor_align)
try:
self.text.set_css('width', '100%')
left_col.set_css({
'padding': '2px',
'width': "100%",
})
self.hor_align.set_css({
'width': "100%",
})
self.hor_align.remove_class('vbox')
self.hor_align.add_class('hbox')
left_col.add_class("box-flex1")
except:
pass
self.start = time.time()
self._time = self.start
## ---
iteration = 0
bound_old = self.bound()
searchDir_old = 0.
iteration_failed = False
while True:
if callback is not None:
callback()
grad,natgrad = self.vb_grad_natgrad()
grad,natgrad = -grad,-natgrad
squareNorm = np.dot(natgrad,grad) # used to monitor convergence
#find search direction
if (method=='steepest') or not iteration:
beta = 0
elif (method=='PR'):
beta = np.dot((natgrad-natgrad_old),grad)/squareNorm_old
elif (method=='FR'):
beta = squareNorm/squareNorm_old
elif (method=='HS'):
beta = np.dot((natgrad-natgrad_old),grad)/np.dot((natgrad-natgrad_old),grad_old)
if np.isnan(beta) or (beta < 0.):
beta = 0.
searchDir = -natgrad + beta*searchDir_old
# Try a conjugate step
phi_old = self.get_vb_param().copy()
try:
self.set_vb_param(phi_old + step_length*searchDir)
bound = self.bound()
except LinAlgError:
self.set_vb_param(phi_old)
bound = bound_old-1
iteration += 1
# Make sure there's an increase in the bound, else revert to steepest, which is guaranteed to increase the bound.
# (It's the same as VBEM.)
if bound < bound_old:
searchDir = -natgrad
try:
self.set_vb_param(phi_old + step_length*searchDir)
bound = self.bound()
except LinAlgError:
import warnings
warnings.warn("Caught LinalgError in setting variational parameters, trying to continue with old parameter settings", LinAlgWarning)
self.set_vb_param(phi_old)
bound = self.bound()
iteration_failed = False
iteration += 1
if verbose:
if self.ipython_notebook:
t = time.time()
seconds = t-self.start
self.status = 'Running'
self.progress.bar_style = 'info'
names_vals = [['conjugate gradient method', "{:s}".format(method)],
['runtime', "{:.1f}s".format(seconds)],
['evaluation', "{}".format(iteration)],
['objective', "{:12.5f}".format(-bound)],
['||gradient||', "{:12.5f}".format(float(squareNorm))],
['beta', "{:12.5f}".format(beta)],
['status', "{:s}".format(self.status)],
]
html_body = ""
for name, val in names_vals:
html_body += "<tr>"
html_body += "<td class='tg-left'>{}</td>".format(name)
html_body += "<td class='tg-right'>{}</td>".format(val)
html_body += "</tr>"
self.progress.value = iteration
self.text.value = html_begin + html_body + html_end
else:
print('\riteration '+str(iteration)+' bound='+str(bound) + ' grad='+str(squareNorm) + ', beta='+str(beta))
sys.stdout.flush()
# Converged yet? try the parameters if so
if np.abs(bound-bound_old) <= ftol:
if verbose:
if self.ipython_notebook:
self.status = 'vb converged (ftol)'
names_vals[-1] = ['status', "{:s}".format(self.status)]
html_body = ""
for name, val in names_vals:
html_body += "<tr>"
html_body += "<td class='tg-left'>{}</td>".format(name)
html_body += "<td class='tg-right'>{}</td>".format(val)
html_body += "</tr>"
self.text.value = html_begin + html_body + html_end
self.progress.bar_style = 'success'
else:
print('vb converged (ftol)')
if self.optimize_parameters() < 1e-1:
break
if squareNorm <= gtol:
if verbose:
if self.ipython_notebook:
self.status = 'vb converged (gtol)'
names_vals[-1] = ['status', "{:s}".format(self.status)]
html_body = ""
for name, val in names_vals:
html_body += "<tr>"
html_body += "<td class='tg-left'>{}</td>".format(name)
html_body += "<td class='tg-right'>{}</td>".format(val)
html_body += "</tr>"
self.text.value = html_begin + html_body + html_end
self.progress.bar_style = 'success'
else:
print('vb converged (gtol)')
if self.optimize_parameters() < 1e-1:
break
if iteration >= maxiter:
if verbose:
if self.ipython_notebook:
self.status = 'maxiter exceeded'
names_vals[-1] = ['status', "{:s}".format(self.status)]
html_body = ""
for name, val in names_vals:
html_body += "<tr>"
html_body += "<td class='tg-left'>{}</td>".format(name)
html_body += "<td class='tg-right'>{}</td>".format(val)
html_body += "</tr>"
self.text.value = html_begin + html_body + html_end
self.progress.bar_style = 'warning'
else:
print('maxiter exceeded')
break
#store essentials of previous iteration
natgrad_old = natgrad.copy() # copy: better safe than sorry.
grad_old = grad.copy()
searchDir_old = searchDir.copy()
squareNorm_old = squareNorm
# hyper param_optimisation
if ((iteration >1) and not (iteration%self.hyperparam_interval)) or iteration_failed:
self.optimize_parameters()
bound_old = bound
# Clean up temporary fields after optimization
if self.ipython_notebook:
del self.text
del self.progress
del self.hor_align
def optimize(self,
method='HS',
maxiter=500,
ftol=1e-6,
gtol=1e-6,
step_length=1.,
callback=None,
verbose=True):
"""
Optimize the model.
The strategy is to run conjugate natural gradients on the variational
parameters, interleaved with gradient based optimization of any
non-variational parameters. self.hyperparam_interval dictates how
often this happens.
Arguments
---------
:method: ['FR', 'PR','HS','steepest'] -- conjugate gradient method
:maxiter: int
:ftol: float
:gtol: float
:step_length: float
"""
assert method in ['FR', 'PR', 'HS', 'steepest'
], 'invalid conjugate gradient method specified.'
## For GPy style notebook verbosity
if verbose:
try:
from IPython.display import display
from IPython.html.widgets import IntProgress, HTML, Box, VBox, HBox, FlexBox
self.text = HTML(width='100%')
self.progress = IntProgress(min=0, max=maxiter)
self.progress.bar_style = 'info'
self.status = 'Running'
html_begin = """<style type="text/css">
.tg-opt {font-family:"Courier New", Courier, monospace !important;padding:2px 3px;word-break:normal;border-collapse:collapse;border-spacing:0;border-color:#DCDCDC;margin:0px auto;width:100%;}
.tg-opt td{font-family:"Courier New", Courier, monospace !important;font-weight:bold;color:#444;background-color:#F7FDFA;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;border-color:#DCDCDC;}
.tg-opt th{font-family:"Courier New", Courier, monospace !important;font-weight:normal;color:#fff;background-color:#26ADE4;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;border-color:#DCDCDC;}
.tg-opt .tg-left{font-family:"Courier New", Courier, monospace !important;font-weight:normal;text-align:left;}
.tg-opt .tg-right{font-family:"Courier New", Courier, monospace !important;font-weight:normal;text-align:right;}
</style>
<table class="tg-opt">"""
html_end = "</table>"
self.ipython_notebook = True
except:
# Not in Ipython notebook
self.ipython_notebook = False
else:
self.ipython_notebook = False
if self.ipython_notebook:
left_col = VBox(children=[self.progress, self.text],
padding=2,
width='100%')
self.hor_align = FlexBox(children=[left_col],
width='100%',
orientation='horizontal')
display(self.hor_align)
try:
self.text.set_css('width', '100%')
left_col.set_css({
'padding': '2px',
'width': "100%",
})
self.hor_align.set_css({
'width': "100%",
})
self.hor_align.remove_class('vbox')
self.hor_align.add_class('hbox')
left_col.add_class("box-flex1")
except:
pass
self.start = time.time()
self._time = self.start
## ---
iteration = 0
bound_old = self.bound()
searchDir_old = 0.
iteration_failed = False
while True:
if callback is not None:
callback()
grad, natgrad = self.vb_grad_natgrad()
grad, natgrad = -grad, -natgrad
squareNorm = np.dot(natgrad, grad) # used to monitor convergence
#find search direction
if (method == 'steepest') or not iteration:
beta = 0
elif (method == 'PR'):
beta = np.dot((natgrad - natgrad_old), grad) / squareNorm_old
elif (method == 'FR'):
beta = squareNorm / squareNorm_old
elif (method == 'HS'):
beta = np.dot((natgrad - natgrad_old), grad) / np.dot(
(natgrad - natgrad_old), grad_old)
if np.isnan(beta) or (beta < 0.):
beta = 0.
searchDir = -natgrad + beta * searchDir_old
# Try a conjugate step
phi_old = self.get_vb_param().copy()
try:
self.set_vb_param(phi_old + step_length * searchDir)
bound = self.bound()
except LinAlgError:
self.set_vb_param(phi_old)
bound = bound_old - 1
iteration += 1
# Make sure there's an increase in the bound, else revert to steepest, which is guaranteed to increase the bound.
# (It's the same as VBEM.)
if bound < bound_old:
searchDir = -natgrad
try:
self.set_vb_param(phi_old + step_length * searchDir)
bound = self.bound()
except LinAlgError:
import warnings
warnings.warn(
"Caught LinalgError in setting variational parameters, trying to continue with old parameter settings",
LinAlgWarning)
self.set_vb_param(phi_old)
bound = self.bound()
iteration_failed = False
iteration += 1
if verbose:
if self.ipython_notebook:
t = time.time()
seconds = t - self.start
self.status = 'Running'
self.progress.bar_style = 'info'
names_vals = [
['conjugate gradient method', "{:s}".format(method)],
['runtime', "{:.1f}s".format(seconds)],
['evaluation', "{}".format(iteration)],
['objective', "{:12.5f}".format(-bound)],
['||gradient||', "{:12.5f}".format(float(squareNorm))],
['beta', "{:12.5f}".format(beta)],
['status', "{:s}".format(self.status)],
]
html_body = ""
for name, val in names_vals:
html_body += "<tr>"
html_body += "<td class='tg-left'>{}</td>".format(name)
html_body += "<td class='tg-right'>{}</td>".format(val)
html_body += "</tr>"
self.progress.value = iteration
self.text.value = html_begin + html_body + html_end
else:
print('\riteration ' + str(iteration) + ' bound=' +
str(bound) + ' grad=' + str(squareNorm) + ', beta=' +
str(beta))
sys.stdout.flush()
# Converged yet? try the parameters if so
if np.abs(bound - bound_old) <= ftol:
if verbose:
if self.ipython_notebook:
self.status = 'vb converged (ftol)'
names_vals[-1] = ['status', "{:s}".format(self.status)]
html_body = ""
for name, val in names_vals:
html_body += "<tr>"
html_body += "<td class='tg-left'>{}</td>".format(
name)
html_body += "<td class='tg-right'>{}</td>".format(
val)
html_body += "</tr>"
self.text.value = html_begin + html_body + html_end
self.progress.bar_style = 'success'
else:
print('vb converged (ftol)')
if self.optimize_parameters() < 1e-1:
break
if squareNorm <= gtol:
if verbose:
if self.ipython_notebook:
self.status = 'vb converged (gtol)'
names_vals[-1] = ['status', "{:s}".format(self.status)]
html_body = ""
for name, val in names_vals:
html_body += "<tr>"
html_body += "<td class='tg-left'>{}</td>".format(
name)
html_body += "<td class='tg-right'>{}</td>".format(
val)
html_body += "</tr>"
self.text.value = html_begin + html_body + html_end
self.progress.bar_style = 'success'
else:
print('vb converged (gtol)')
if self.optimize_parameters() < 1e-1:
break
if iteration >= maxiter:
if verbose:
if self.ipython_notebook:
self.status = 'maxiter exceeded'
names_vals[-1] = ['status', "{:s}".format(self.status)]
html_body = ""
for name, val in names_vals:
html_body += "<tr>"
html_body += "<td class='tg-left'>{}</td>".format(
name)
html_body += "<td class='tg-right'>{}</td>".format(
val)
html_body += "</tr>"
self.text.value = html_begin + html_body + html_end
self.progress.bar_style = 'warning'
else:
print('maxiter exceeded')
break
#store essentials of previous iteration
natgrad_old = natgrad.copy() # copy: better safe than sorry.
grad_old = grad.copy()
searchDir_old = searchDir.copy()
squareNorm_old = squareNorm
# hyper param_optimisation
if ((iteration > 1) and not (iteration % self.hyperparam_interval)
) or iteration_failed:
self.optimize_parameters()
bound_old = bound
# Clean up temporary fields after optimization
if self.ipython_notebook:
del self.text
del self.progress
del self.hor_align
