def __init__(self, metric, get_hyp_config_fn, run_fn, top_k_fn,
get_best_fn):
self.get_hyperparameter_configuration = get_hyp_config_fn
self.run_then_return_val_performance = run_fn
self.top_k = top_k_fn
self.get_best_performance = get_best_fn
self.history = History()
def __init__(self,
searcher,
evaluator,
metric='val_loss',
resource_type='epoch'):
assert (metric == 'val_loss' or metric == 'val_accuracy')
assert (resource_type == 'epoch' or resource_type == 'training_time')
self.searcher = searcher
self.evaluator = evaluator
self.metric = metric
self.resource_type = resource_type
self.history = History()
def test_description_deletion(self):
ht = HistoryTracker(gen_proj(), History())
ht.stage(POp_DelDesc, 0)
p2 = gen_proj()
self.assertTrue(same(ht.ctx, p2))
ht.commit()
ht.undo()
self.assertTrue(same(ht.ctx, p2))
def main():
root = VarTk()
root.title(_("Edit history test"))
root.geometry("500x500")
menubar = VarMenu(root)
global hotkeys
hotkeys = HotKey(root)
editmenu = VarMenu(menubar, tearoff = False)
editmenu.add_command(
label = _("Undo"),
command = undo,
accelerator = hotkeys.get_keycode_string(undo)
)
editmenu.add_command(
label = _("Redo"),
command = redo,
accelerator = hotkeys.get_keycode_string(redo)
)
menubar.add_cascade(label = _("Edit"), menu = editmenu)
root.config(menu = menubar)
root.after(5000, reg_hotkeys)
root.grid()
root.columnconfigure(0, weight = 1)
root.rowconfigure(0, weight = 1)
history = History()
global tracker
tracker = HistoryTracker(history)
cnv = HistCanvasDnD(root, history_tracker = tracker)
cnv.grid(row = 0, column = 0, sticky="NEWS")
cnv.canvas.create_rectangle(
10, 10, 100, 100,
tags = "DnD",
fill = "red"
)
root.mainloop()
def __init__(self, layouts = [], build_path = None, **kw):
# Any description in GUI project has a serial number.
self.__next_id = 0
QProject.__init__(self, **kw)
self.build_path = build_path
self.layouts = {}
for l in layouts:
# backward compatibility
if isinstance(l, tuple):
self.add_layout(*l)
continue
dn = l.desc_name
try:
l_dict = self.layouts[dn]
except KeyError:
self.layouts[dn] = l_dict = {}
l_dict[l.lid] = l
self.history = History()
class SimpleArchitectureSearchHyperBand(HyperBandExtended):
"""
Hyperband algorithm for simple architecture search experiments. It is designed
to maximize the hyperband algorithm, thus some arguments need to conform certain
specifications. For more flexibility, one can use HyperBandExtended and
define the required functions.
Args:
searcher: DeepArchitect Searcher (RandomSearch is most common)
evaluator: DeepArchitect Evaluator. The evaluator is similar to the normal
evaluator defined in DeepArchitect (see mnist tutorial) with one
difference: The evaluate function takes in (DeepArchitect inputs,
DeepArchitect outputs, max resource) and returns a dictionary:
{"val_accuracy": ...,
"history": {"resource": [], "val_acc": [], etc.}}
where "history" is the history of evaluating that architecture
metric: must be either "val_loss" or "val_accuracy"
resource_type must be either "epoch" or "training_time"
The Configurations T and Performance P will have the following representations
Configurations T = (arch_id, (inputs, outputs)). Note that inputs,
outputs, and hs sampled from the searcher are enough to define
an architecture
Performance P = (final_val_perf, dict("resource":[], "val_loss":[]...))
TODO: perhaps have an enabled mode of "multiworker" or not?
"""
def __init__(self,
searcher,
evaluator,
metric='val_loss',
resource_type='epoch'):
assert (metric == 'val_loss' or metric == 'val_accuracy')
assert (resource_type == 'epoch' or resource_type == 'training_time')
self.searcher = searcher
self.evaluator = evaluator
self.metric = metric
self.resource_type = resource_type
self.history = History()
def get_hyperparameter_configuration(self, n):
configs = []
for i in range(n):
(inputs, outputs, h_value_his,
searcher_token) = self.searcher.sample()
configs.append((i, (inputs, outputs)))
return configs
def run_then_return_val_performance(self, T, r):
P = []
for (arch_id, (inputs, outputs)) in T:
results = self.evaluator.evaluate(inputs, outputs, r)
P.append(results['val_accuracy'], results['history'])
return P
def top_k(self, T, P, k):
reverse = False if self.metric == 'val_loss' else True
# combine T, P and sort according to validation performance
sorted_tp = sorted(list(zip(T, P)),
key=lambda tp: tp[1][0],
reverse=reverse)
# map back to only T and get the top k
return list(map(lambda tp: tp[0], sorted_tp[:k]))
def get_best_performance(self, T, P, best_config, best_perf):
assert (len(T) == len(P) == 0)
get_max = False if self.metric == 'val_loss' else True
cur_config, cur_perf = T[0], P[0]
if (self.metric == 'val_accuracy'):
return cur_config if best_config == None or (
cur_perf[0] > best_perf[0]) else best_config
elif (self.metric == 'val_loss'):
return cur_config if best_config == None or (
cur_perf[0] < best_perf[0]) else best_config
def graph_hyperband(self, save_dir='./tmp/hyperband.jpg'):
"""More customized graphing functionality
"""
import pandas as pd
from plotnine import *
data = self.history.convert_to_graph_hyperband_ready(
self.metric, self.resource_type)
df = pd.DataFrame(data)
plot = ggplot(
df,
aes(x=self.resource_type,
y=self.metric,
color='factor(configuration)')) + geom_line()
plot.save(save_dir)
def graph_sucessiveHalving(self, bracket_id, save_dir='./tmp/sh.jpg'):
import pandas as pd
import plotnine as plot
data = self.history.convert_a_bracket_to_graph_ready(
bracket_id, self.metric, self.resource_type)
df = pd.DataFrame(data)
plot = ggplot(
df,
aes(x=self.resource_type,
y=self.metric,
color='factor(configuration)')) + geom_line()
plot.save(save_dir)
class HyperBandExtended(object):
"""An Extended Version of HyperBand Algorithm (Li et. al 2017)
The main difference is that the user is not restricted to using the validation
loss as the metric. The user will need to define the following functions:
get_hyp_config_fn(n): returns a list of n sampled configurations
run_fn(T, r): takes list of configurations T and resource r for each
config, and returns the results (often a validation metric) from
evaluating that configuration with resource r
top_k_fn(T, P, k): takes in list of config T, list of validation performance
P, integer k and returns the (sorted) top k list of configuration
get_best_fn (T, P, cur_config, cur_perf): takes in list of
validation performance T, list of corresponding configs P, best config
and best validation performance so far. Returns the updated best
config and best validation performance.
NOTE: T, P, and r need to be consistent across functions. We recommend the
following representation:
- resource r should be a float or an int, and serializable.
- T should take the form list of (config_id, config_representation). Each
element should be serializable (for storage in History object)
- For validation performance P, there are 2 possible representations:
(1) the most basic form is a list of float indicating the validation
result at each resource r.
(2) Another common representation is P = list[(val_perf, history)],
where val_perf is the validation performance of a particular configuration,
and history is a dictionary of results (progression of val_loss and val_perf
vs intervals of resources) from evaluating configuration. This is particularly
useful when graphing so that we have a more complete picture of the evaluation.
For an example see ArchitectureSearchHyperBand class.
This class also stores all history in a History object, for reference and
graphing purposes.
"""
def __init__(self, metric, get_hyp_config_fn, run_fn, top_k_fn,
get_best_fn):
self.get_hyperparameter_configuration = get_hyp_config_fn
self.run_then_return_val_performance = run_fn
self.top_k = top_k_fn
self.get_best_performance = get_best_fn
self.history = History()
def evaluate(self, R, eta=3):
"""HyperBand algorithm for hyperparameter optimization
Args:
R (int): maximum amount of resource that can be allocated to a single
configuration
eta (int): the proportion of configurations discarded in each round of
SucessiveHalving
Return: configuration with the best performance, and the corresponding
performance
"""
s_max = int(math.floor(math.log(
R, eta))) # number of grid search, or "bracket"
B = (s_max + 1) * R # total resources
best_config, best_perf = None, None
for s in xrange(s_max, -1, -1):
n = int(math.ceil(float(B) * (eta**s) / (float(R) *
(s + 1)))) # num configs
r = R * (eta**(-s)) # resource for each config
# begin SuccessiveHalving
T = self.get_hyperparameter_configuration(n)
self.history.total_configs += n
for i in xrange(s + 1):
if len(T) == 0: # no more configurations
return max_perf
n_i = int(math.floor(n * (eta**(-i))))
r_i = int(math.floor(r * (eta**i)))
P = self.run_then_return_val_performance(T, r_i)
self.history.record_succesiveHalving(s, T, P, r_i)
k = int(math.floor(float(n_i) / eta))
if (k == 0): break
T = self.top_k(T, P, k)
self.history.total_resources += len(T) * r_i
self.history.record_best_of_bracket(s, T, P)
(best_config,
best_perf) = self.get_best_performance(T, P, best_config,
best_perf)
self.history.record_best_config(best_config, best_perf)
return (best_config, best_perf)
def graph_hyperband(self, save_dir='./tmp/hyperband.jpg'):
"""Extract from history and show Hyperband graph (best performance for
each successive halving bracket)
"""
pass
def graph_sucessiveHalving(self, bracket_id, save_dir='./tmp/sh.jpg'):
pass