def fit_generator(self,
x,
y,
generator,
loss_func='categorical_crossentropy',
optimizer=SGD(lr=0.01, momentum=0.9),
clip=None,
callbacks=[],
transformer=None,
verbose=1):
# Train
for batch_x, batch_y in generator.generate(x, y):
batch_x = to_list(batch_x)
batch_y = to_list(batch_y)
t1 = time.time()
# Compile optimization function
if not self._f_optimize_:
# Train memory usage
batch_size = len(batch_x[0])
print "Training",
self._show_memory_usage(self._effective_layers_, batch_size)
timer = Timer()
target_dim_list = self._get_target_dim_list(
batch_x, batch_y, transformer)
self._f_optimize_ = self.get_optimization_func(
target_dim_list, loss_func, optimizer, clip)
timer.show("Compiling f_optimize time:")
# Compile for callback
timer = Timer()
if callbacks is not None:
callbacks = to_list(callbacks)
for callback in callbacks:
callback.compile(self)
timer.show("Compiling callbacks time:")
if transformer:
(batch_x, batch_y) = transformer.transform(batch_x, batch_y)
batch_x = format_data_list(batch_x)
batch_y = format_data_list(batch_y)
# Callback
for callback in callbacks:
if (self.iter_ % callback.call_freq_ == 0):
print
callback.call()
in_list = batch_x + batch_y + [1.] # training phase
loss = self._f_optimize_(*in_list)[0]
self._iter_ += 1
t2 = time.time()
self._tr_time_ += (t2 - t1)
sys.stdout.write(
"iteration: %d loss: %f time per batch: %.2f \r" %
(self._iter_, loss, t2 - t1))
sys.stdout.flush()
def train_on_batch(self, batch_x, batch_y, loss_func='categorical_crossentropy',
optimizer=SGD(lr=0.01, momentum=0.9), clip=None, callbacks=[],
transformer=None, recompile=False):
"""Train model on single batch data.
Args:
batch_x: ndarray | list of ndarray.
batch_y: ndarray | list of ndarray.
loss_func: str | function.
optimizer: optimization object.
clip: real value.
callbacks: list of Callback object.
transformer: Transformation object.
recompile: bool. Recompile the optimize function or not.
Returns: None. (All trained models, results should be saved using callbacks.)
"""
batch_x = to_list(batch_x)
batch_y = to_list(batch_y)
# Format data
batch_x = format_data_list(batch_x)
batch_y = format_data_list(batch_y)
# Compile optimization function
if (not self._f_optimize_) or (recompile):
timer = Timer()
target_dim_list = self._get_target_dim_list(batch_x, batch_y, transformer)
self._f_optimize_ = self.get_optimization_func(target_dim_list, loss_func, optimizer, clip)
timer.show("Compiling f_optimize time:")
batch_size = len(batch_x[0])
self._show_memory_usage(self._effective_layers_, batch_size)
# Compile for callback
timer = Timer()
if callbacks is not None:
callbacks = to_list(callbacks)
for callback in callbacks:
callback.compile(self)
timer.show("Compiling callbacks time:")
for callback in callbacks:
if (self.iter_ % callback.call_freq_ == 0):
print self.iter_, 'th iteration:'
callback.call()
# Train
t1 = time.time()
if transformer:
(batch_x, batch_y) = transformer.transform(batch_x, batch_y)
batch_x = format_data_list(batch_x)
batch_y = format_data_list(batch_y)
in_list = batch_x + batch_y + [1.] # training phase
loss = self._f_optimize_(*in_list)[0]
self._iter_ += 1
t2 = time.time()
self._tr_time_ += (t2 - t1)
return loss
def _evaluate(self, x, y, eval_type):
# get metric losses node
loss_nodes = []
for metric in self._metrics_:
# if use default objective
if type(metric) is str:
assert len(self._md_.out_nodes_)==len(self._md_.gt_nodes_), "If you are using default objectives, " \
+ "out_node of out_layers must match ground truth!"
loss_node = sum([obj.get(metric)(pred_node, gt_node)
for pred_node, gt_node in zip(self._md_.out_nodes_, self._md_.gt_nodes_)])
# if user define their objective function
elif isfunction(metric):
loss_node = metric(self._md_)
else:
loss_node = metric
loss_nodes.append(loss_node)
# compile evaluation function
if not hasattr(self, '_f_evaluate'):
print 'compiling evaluation function ..'
inputs = self._md_.in_nodes_ + self._md_.gt_nodes_ + [self._md_.tr_phase_node_]
self._f_evaluate = K.function_no_given(inputs, loss_nodes)
print 'compile finished. '
# calculate metric values
t1 = time.time()
if self._generator_:
generator = self._generator_
else:
generator = self._DefaultGenerator(self._batch_size_)
n_all = 0.
cnt= 0.
metric_vals = np.zeros(len(self._metrics_))
batch_num = sum(1 for it in generator.generate(x, y))
for batch_x, batch_y in generator.generate(x, y):
batch_x = to_list(batch_x)
batch_y = to_list(batch_y)
curr_batch_size = batch_x[0].shape[0]
n_all += curr_batch_size
cnt += 1.
if self._transformer_:
(batch_x, batch_y) = self._transformer_.transform(batch_x, batch_y)
batch_x = format_data_list(batch_x)
batch_y = format_data_list(batch_y)
in_list = batch_x + batch_y + [0.]
batch_metric_vals = np.array(self._f_evaluate(*in_list))
metric_vals += batch_metric_vals * curr_batch_size
if self._verbose_==1: self._print_progress(batch_num, cnt)
metric_vals /= n_all
# timer
t2 = time.time()
# print results
self._print_time_results(eval_type, self._metrics_, metric_vals, t2-t1)
def generate(self, x, y):
x = to_list(x)
y = to_list(y)
N = len(x[0])
batch_num = int(np.ceil(float(N) / self._batch_size_))
# evaluate for each batch
for i1 in xrange(batch_num):
curr_batch_size = min((i1+1)*self._batch_size_, N) - i1*self._batch_size_
batch_x = [e[i1*self._batch_size_ : min((i1+1)*self._batch_size_, N)] for e in x]
batch_y = [e[i1*self._batch_size_ : min((i1+1)*self._batch_size_, N)] for e in y]
yield batch_x, batch_y
def __init__(self, in_layers, out_layers, any_layers=[]):
super(Model, self).__init__(in_layers)
# out layers
out_layers = to_list(out_layers)
self._out_layers_ = out_layers
self._out_nodes_ = [layer.output_ for layer in self._out_layers_]
# inter layers
any_layers = to_list(any_layers)
self._any_layers_ = any_layers
self._any_nodes_ = [layer.output_ for layer in self._any_layers_]
def fit_generator(self, x, y, generator, loss_func='categorical_crossentropy',
optimizer=SGD(lr=0.01, momentum=0.9), clip=None, callbacks=[], transformer=None, verbose=1):
# Train
for batch_x, batch_y in generator.generate(x, y):
batch_x = to_list(batch_x)
batch_y = to_list(batch_y)
t1 = time.time()
# Compile optimization function
if not self._f_optimize_:
# Train memory usage
batch_size = len(batch_x[0])
print "Training",
self._show_memory_usage(self._effective_layers_, batch_size)
timer = Timer()
target_dim_list = self._get_target_dim_list(batch_x, batch_y, transformer)
self._f_optimize_ = self.get_optimization_func(target_dim_list, loss_func, optimizer, clip)
timer.show("Compiling f_optimize time:")
# Compile for callback
timer = Timer()
if callbacks is not None:
callbacks = to_list(callbacks)
for callback in callbacks:
callback.compile(self)
timer.show("Compiling callbacks time:")
if transformer:
(batch_x, batch_y) = transformer.transform(batch_x, batch_y)
batch_x = format_data_list(batch_x)
batch_y = format_data_list(batch_y)
# Callback
for callback in callbacks:
if (self.iter_ % callback.call_freq_ == 0):
print
callback.call()
in_list = batch_x + batch_y + [1.] # training phase
loss = self._f_optimize_(*in_list)[0]
self._iter_ += 1
t2 = time.time()
self._tr_time_ += (t2 - t1)
sys.stdout.write("iteration: %d loss: %f time per batch: %.2f \r" % (self._iter_, loss, t2-t1))
sys.stdout.flush()
def get_observe_forward_func(self, observe_nodes):
observe_nodes = to_list(observe_nodes)
inputs = self.in_nodes_ + [self.tr_phase_node_]
timer = Timer()
f_observe_forward = K.function_no_given(inputs, observe_nodes)
timer.show("Compiling f_observe_forward time:")
return f_observe_forward
def __init__(self, in_layers):
# reset global id
reset_id_to_zero()
# inlayers
in_layers = to_list(in_layers)
self._in_layers_ = in_layers
self._in_nodes_ = [layer.output_ for layer in self._in_layers_]
# find all nodes using BFT
self._id_list_, self._layer_list_ = BFT(self._in_layers_)
# get params by traveling all layers
self._params_ = []
for layer in self._layer_list_:
self._params_ += layer.params_
# get inner updates by traveling all layers
self._inner_updates_ = []
for layer in self._layer_list_:
if hasattr(layer, 'inner_updates_'):
self._inner_updates_ += layer.inner_updates_
# sum regs by traveling all layers
self._reg_value_ = 0.
for layer in self._layer_list_:
self._reg_value_ += layer.reg_value_
# tr_phase_node
self._tr_phase_node_ = K.common_tr_phase_node
# time
self._tr_time_ = 0.
self._epoch_ = 0
def __init__(self, tr_x=None, tr_y=None, va_x=None, va_y=None,
te_x=None, te_y=None, batch_size=100, call_freq=3,
metrics=['categorical_error'], generator=None, transformer=None,
dump_path=None, type='epoch', verbose=1, is_logging=False):
# init values
super(Validation, self).__init__(call_freq, type)
self._batch_size_ = batch_size
self._metrics_ = to_list(metrics)
self._generator_ = generator
self._transformer_ = transformer
self._dump_path_ = dump_path
self._type_ = type
self._verbose_ = verbose
self._r_ = self._init_result()
self._is_logging_ = is_logging
# judge if train or valid or test data exists
self._is_tr_ = self._is_data(tr_x, tr_y)
self._is_va_ = self._is_data(va_x, va_y)
self._is_te_ = self._is_data(te_x, te_y)
self._tr_x_ = tr_x
self._tr_y_ = tr_y
self._va_x_ = va_x
self._va_y_ = va_y
self._te_x_ = te_x
self._te_y_ = te_y
def get_observe_forward_func(self, observe_nodes):
observe_nodes = to_list(observe_nodes)
inputs = self.in_nodes_ + [self.tr_phase_node_]
timer = Timer()
f_observe_forward = K.function_no_given(inputs, observe_nodes)
timer.show("Compiling f_observe_forward time:")
return f_observe_forward
def __init__(self,
tr_x=None,
tr_y=None,
va_x=None,
va_y=None,
te_x=None,
te_y=None,
batch_size=100,
call_freq=3,
metrics=['categorical_error'],
dump_path='validation.p',
verbose=1):
# init values
self._batch_size_ = batch_size
self._call_freq_ = call_freq
self._metrics_ = to_list(metrics)
self._dump_path_ = dump_path
self._verbose_ = verbose
self._r_ = self._init_result()
# judge if train or valid or test data exists
self._is_tr_ = self._is_data(tr_x, tr_y)
self._is_va_ = self._is_data(va_x, va_y)
self._is_te_ = self._is_data(te_x, te_y)
# format data, to list
if self._is_tr_:
self._tr_x_ = [K.format_data(e) for e in to_list(tr_x)]
self._tr_y_ = [K.format_data(e) for e in to_list(tr_y)]
if self._is_va_:
self._va_x_ = [K.format_data(e) for e in to_list(va_x)]
self._va_y_ = [K.format_data(e) for e in to_list(va_y)]
if self._is_te_:
self._te_x_ = [K.format_data(e) for e in to_list(te_x)]
self._te_y_ = [K.format_data(e) for e in to_list(te_y)]
def get_observe_backward_func(self, observe_nodes):
if self.gt_nodes_ is None:
raise Exception("You must call set_gt_nodes method before call observe_backward method!")
observe_nodes = to_list(observe_nodes)
inputs = self.in_nodes_ + self.gt_nodes_ + [self.tr_phase_node_]
timer = Timer()
f_observe_backward = K.function_no_given(inputs, observe_nodes)
timer.show("Compiling f_observe_backward time:")
return f_observe_backward
def __init__(self, in_layers=[], out_layers=[], any_layers=[]):
self._in_layers_ = to_list(in_layers)
self._out_layers_ = to_list(out_layers)
self._any_layers_ = to_list(any_layers)
self._gt_nodes_ = None
self._f_predict = None
self._epoch_ = 0
self._iter_= 0
self._tr_time_ = 0.
self._tr_phase_node_ = K.common_tr_phase_node
self._effective_layers_ = self.get_effective_layers(self._in_layers_, self._out_layers_)
self._check_duplicate_name(self._effective_layers_)
self._trainable_table_ = self._init_trainable_table(self._effective_layers_)
self._f_optimize_ = None
def train_on_batch_with_func(self, func, batch_x, batch_y):
"""Train model on batch data.
Args:
func: function.
batch_x: ndarray | list of ndarray.
batch_y: ndarray | list of ndarray.
"""
batch_x = to_list(batch_x)
batch_y = to_list(batch_y)
# Format data
batch_x = format_data_list(batch_x)
batch_y = format_data_list(batch_y)
in_list = batch_x + batch_y + [1.] # training phase
loss = func(*in_list)[0]
self._set_iter(self.iter_ + 1)
return loss
def train_on_batch_with_func(self, func, batch_x, batch_y):
"""Train model on batch data.
Args:
func: function.
batch_x: ndarray | list of ndarray.
batch_y: ndarray | list of ndarray.
"""
batch_x = to_list(batch_x)
batch_y = to_list(batch_y)
# Format data
batch_x = format_data_list(batch_x)
batch_y = format_data_list(batch_y)
in_list = batch_x + batch_y + [1.] # training phase
loss = func(*in_list)[0]
self._set_iter(self.iter_ + 1)
return loss
def generate(self, x, y):
x = to_list(x)
y = to_list(y)
N = len(x[0])
batch_num = int(np.ceil(float(N) / self._batch_size_))
# evaluate for each batch
for i1 in xrange(batch_num):
curr_batch_size = min(
(i1 + 1) * self._batch_size_, N) - i1 * self._batch_size_
batch_x = [
e[i1 * self._batch_size_:min((i1 + 1) *
self._batch_size_, N)]
for e in x
]
batch_y = [
e[i1 * self._batch_size_:min((i1 + 1) *
self._batch_size_, N)]
for e in y
]
yield batch_x, batch_y
def set_layers_trainability(self, layer_names, bool):
"""Set list of layers' trainablility.
"""
layer_names = to_list(layer_names)
for layer_name in layer_names:
row = self._find_tt_row_by_name(layer_name, self._trainable_table_)
if row is None:
raise Exception("Can not find layer_name!")
else:
index = row[0]
self._trainable_table_[index][-1] = bool
def set_layers_trainability(self, layer_names, bool):
"""Set list of layers' trainablility.
"""
layer_names = to_list(layer_names)
for layer_name in layer_names:
row = self._find_tt_row_by_name(layer_name, self._trainable_table_)
if row is None:
raise Exception("Can not find layer_name!")
else:
index = row[0]
self._trainable_table_[index][-1] = bool
def get_observe_backward_func(self, observe_nodes):
if self.gt_nodes_ is None:
raise Exception(
"You must call set_gt_nodes method before call observe_backward method!"
)
observe_nodes = to_list(observe_nodes)
inputs = self.in_nodes_ + self.gt_nodes_ + [self.tr_phase_node_]
timer = Timer()
f_observe_backward = K.function_no_given(inputs, observe_nodes)
timer.show("Compiling f_observe_backward time:")
return f_observe_backward
def __init__(self, in_layers=[], out_layers=[], any_layers=[]):
self._in_layers_ = to_list(in_layers)
self._out_layers_ = to_list(out_layers)
self._any_layers_ = to_list(any_layers)
self._gt_nodes_ = None
self._f_predict = None
self._epoch_ = 0
self._iter_ = 0
self._tr_time_ = 0.
self._tr_phase_node_ = K.common_tr_phase_node
self._effective_layers_ = self.get_effective_layers(
self._in_layers_, self._out_layers_)
self._check_duplicate_name(self._effective_layers_)
self._trainable_table_ = self._init_trainable_table(
self._effective_layers_)
self._f_optimize_ = None
def run_function(self, func, z, batch_size, tr_phase):
"""Return output of a function given value.
Args:
func: function
z: ndarray | list of ndarray. Can be [inputs] for computing forward,
or [inputs]+[outputs] for computing backward
Returns:
list of ndarray
"""
# Format data
z = to_list(z)
z = format_data_list(z)
# Calculating all in same time
if batch_size is None:
in_list = z + [tr_phase]
y_out = func(*in_list)
# Calculating in batch
else:
N = len(z[0])
batch_num = int(np.ceil(float(N) / batch_size))
n_out_nodes = len(self.out_nodes_)
y_out = [] # list of batch_y_out
for i1 in xrange(batch_num):
in_list = [
e[i1 * batch_size:min((i1 + 1) * batch_size, N)] for e in z
] + [tr_phase]
batch_y_out = func(*in_list) # list of ndarray
y_out.append(batch_y_out)
def _reform(y_out):
outs = []
for i1 in xrange(len(y_out[0])):
tmp_list = []
for j1 in xrange(len(y_out)):
tmp_list.append(y_out[j1][i1])
out = np.concatenate(tmp_list, axis=0)
outs.append(out)
return outs
y_out = _reform(y_out)
return y_out
def run_function(self, func, z, batch_size, tr_phase):
"""Return output of a function given value.
Args:
func: function
z: ndarray | list of ndarray. Can be [inputs] for computing forward,
or [inputs]+[outputs] for computing backward
Returns:
list of ndarray
"""
# Format data
z = to_list(z)
z = format_data_list(z)
# Calculating all in same time
if batch_size is None:
in_list = z + [tr_phase]
y_out = func(*in_list)
# Calculating in batch
else:
N = len(z[0])
batch_num = int(np.ceil(float(N) / batch_size))
n_out_nodes = len(self.out_nodes_)
y_out = [] # list of batch_y_out
for i1 in xrange(batch_num):
in_list = [e[i1*batch_size : min((i1+1)*batch_size, N)] for e in z] + [tr_phase]
batch_y_out = func(*in_list) # list of ndarray
y_out.append(batch_y_out)
def _reform(y_out):
outs = []
for i1 in xrange(len(y_out[0])):
tmp_list = []
for j1 in xrange(len(y_out)):
tmp_list.append(y_out[j1][i1])
out = np.concatenate(tmp_list, axis=0)
outs.append(out)
return outs
y_out = _reform(y_out)
return y_out
def __init__(self,
tr_x=None,
tr_y=None,
va_x=None,
va_y=None,
te_x=None,
te_y=None,
batch_size=100,
call_freq=3,
metrics=['categorical_error'],
generator=None,
transformer=None,
dump_path=None,
type='epoch',
verbose=1,
is_logging=False):
# init values
super(Validation, self).__init__(call_freq, type)
self._batch_size_ = batch_size
self._metrics_ = to_list(metrics)
self._generator_ = generator
self._transformer_ = transformer
self._dump_path_ = dump_path
self._type_ = type
self._verbose_ = verbose
self._r_ = self._init_result()
self._is_logging_ = is_logging
# judge if train or valid or test data exists
self._is_tr_ = self._is_data(tr_x, tr_y)
self._is_va_ = self._is_data(va_x, va_y)
self._is_te_ = self._is_data(te_x, te_y)
self._tr_x_ = tr_x
self._tr_y_ = tr_y
self._va_x_ = va_x
self._va_y_ = va_y
self._te_x_ = te_x
self._te_y_ = te_y
def add_models(self, mds):
"""Add list of model to existing model, update in_layers, out_layers, any_layers and trainable_table.
"""
mds = to_list(mds)
new_in_layers = self.in_layers_
new_out_layers = self.out_layers_
new_any_layers = self.any_layers_
init_trainable_table = self.trainable_table_
for md in mds:
new_in_layers += md.in_layers_
new_out_layers += md.out_layers_
new_any_layers += md.any_layers_
init_trainable_table += md.trainable_table_
self.__init__(new_in_layers, new_out_layers, new_any_layers)
# New trainable_table is inherited from old concatenated trainable_table
for row1 in init_trainable_table:
row2 = self._find_tt_row_by_name(row1[1].name_, self._trainable_table_)
if row2:
index = row2[0]
self._trainable_table_[index][-1] = row1[-1]
def predict(self, x, batch_size=100):
# format data
x = to_list(x)
x = [K.format_data(e) for e in x]
# compile predict model
if not hasattr(self, '_f_predict'):
self._f_predict = K.function_no_given(self._in_nodes_,
self._tr_phase_node_,
self._out_nodes_)
# do predict
# put all data in GPU
if batch_size is None:
in_list = x + [0.]
y_out = self._f_predict(*in_list)
# put batch data in GPU
else:
N = len(x[0])
batch_num = int(np.ceil(float(N) / batch_size))
n_out_nodes = len(self._out_nodes_)
y_out = [[] for e in self._out_nodes_]
for i1 in xrange(batch_num):
in_list = [
e[i1 * batch_size:min((i1 + 1) * batch_size, N)] for e in x
] + [0.]
batch_y_out = self._f_predict(*in_list)
for j1 in xrange(n_out_nodes):
y_out[j1].append(batch_y_out[j1])
# get y_out
y_out = [np.concatenate(e, axis=0) for e in y_out]
if len(y_out) == 1:
return y_out[0]
else:
return y_out
def add_models(self, mds):
"""Add list of model to existing model, update in_layers, out_layers, any_layers and trainable_table.
"""
mds = to_list(mds)
new_in_layers = self.in_layers_
new_out_layers = self.out_layers_
new_any_layers = self.any_layers_
init_trainable_table = self.trainable_table_
for md in mds:
new_in_layers += md.in_layers_
new_out_layers += md.out_layers_
new_any_layers += md.any_layers_
init_trainable_table += md.trainable_table_
self.__init__(new_in_layers, new_out_layers, new_any_layers)
# New trainable_table is inherited from old concatenated trainable_table
for row1 in init_trainable_table:
row2 = self._find_tt_row_by_name(row1[1].name_,
self._trainable_table_)
if row2:
index = row2[0]
self._trainable_table_[index][-1] = row1[-1]
def _evaluate(self, x, y, eval_type):
# get metric losses node
loss_nodes = []
for metric in self._metrics_:
# if use default objective
if type(metric) is str:
assert len(self._md_.out_nodes_)==len(self._md_.gt_nodes_), "If you are using default objectives, " \
+ "out_node of out_layers must match ground truth!"
loss_node = sum([
obj.get(metric)(pred_node, gt_node) for pred_node, gt_node
in zip(self._md_.out_nodes_, self._md_.gt_nodes_)
])
# if user define their objective function
elif isfunction(metric):
loss_node = metric(self._md_)
else:
loss_node = metric
loss_nodes.append(loss_node)
# compile evaluation function
if not hasattr(self, '_f_evaluate'):
print 'compiling evaluation function ..'
inputs = self._md_.in_nodes_ + self._md_.gt_nodes_ + [
self._md_.tr_phase_node_
]
self._f_evaluate = K.function_no_given(inputs, loss_nodes)
print 'compile finished. '
# calculate metric values
t1 = time.time()
if self._generator_:
generator = self._generator_
else:
generator = self._DefaultGenerator(self._batch_size_)
n_all = 0.
cnt = 0.
metric_vals = np.zeros(len(self._metrics_))
batch_num = sum(1 for it in generator.generate(x, y))
for batch_x, batch_y in generator.generate(x, y):
batch_x = to_list(batch_x)
batch_y = to_list(batch_y)
curr_batch_size = batch_x[0].shape[0]
n_all += curr_batch_size
cnt += 1.
if self._transformer_:
(batch_x,
batch_y) = self._transformer_.transform(batch_x, batch_y)
batch_x = format_data_list(batch_x)
batch_y = format_data_list(batch_y)
in_list = batch_x + batch_y + [0.]
batch_metric_vals = np.array(self._f_evaluate(*in_list))
metric_vals += batch_metric_vals * curr_batch_size
if self._verbose_ == 1: self._print_progress(batch_num, cnt)
metric_vals /= n_all
# timer
t2 = time.time()
# print results
self._print_time_results(eval_type, self._metrics_, metric_vals,
t2 - t1)
def fit(self, x, y, batch_size=100, n_epochs=10, loss_func='categorical_crossentropy',
optimizer=SGD(lr=0.01, momentum=0.9), clip=None, callbacks=[], shuffle=True, transformer=None, verbose=1):
"""Fit data and train model. The data is reshuffled after every epoch.
Args:
x: ndarray | list of numpy ndarray.
y: ndarray | list of numpy ndarray.
batch_size: integar.
n_epoch: integar. Number of training epochs.
loss_func: str | function.
optimizer: optimization object.
clip: real value.
callbacks: list of Callback object.
shuffle: bool.
transformer: Transformation object.
verbose: 0 | 1 | 2
Returns: None. (All trained models, results should be saved using callbacks.)
"""
x = to_list(x)
y = to_list(y)
# Format data
x = format_data_list(x)
y = format_data_list(y)
# Train memory usage
print "Training",
self._show_memory_usage(self._effective_layers_, batch_size)
# Compile optimization function
timer = Timer()
target_dim_list = self._get_target_dim_list(x, y, transformer)
f_optimize = self.get_optimization_func(target_dim_list, loss_func, optimizer, clip)
timer.show("Compiling f_optimize time:")
# Compile for callback
timer = Timer()
if callbacks is not None:
callbacks = to_list(callbacks)
for callback in callbacks:
callback.compile(self)
timer.show("Compiling callbacks time:")
# Train
N = len(x[0])
batch_num = int(np.ceil(float(N) / batch_size))
max_epoch = n_epochs + self.epoch_
# Callback
print '\n', self.epoch_, 'th epoch:'
for callback in callbacks:
if (self.epoch_ % callback.call_freq_ == 0):
callback.call()
while self.epoch_ < max_epoch:
# Shuffle data
if shuffle: x, y = supports.shuffle(x, y)
# Train
t1 = time.time()
loss_list = []
for i2 in xrange(batch_num):
batch_x = [e[i2*batch_size : min((i2+1)*batch_size, N)] for e in x]
batch_y = [e[i2*batch_size : min((i2+1)*batch_size, N)] for e in y]
if transformer:
(batch_x, batch_y) = transformer.transform(batch_x, batch_y)
batch_x = format_data_list(batch_x)
batch_y = format_data_list(batch_y)
in_list = batch_x + batch_y + [1.] # training phase
loss = f_optimize(*in_list)[0]
loss_list.append(loss)
# self._iter_ += 1
if verbose==1: self.print_progress(self.epoch_, batch_num, i2)
if verbose==2: self.print_progress_loss(self.epoch_, batch_num, i2, loss)
t2 = time.time()
self._tr_time_ += (t2 - t1)
if verbose!=0: print '\n', ' tr_time: ', "%.2f" % (t2-t1), 's' # print an empty line
self._epoch_ += 1
# Callback
for callback in callbacks:
if (self.epoch_ % callback.call_freq_ == 0):
callback.call()
def fit(self,
x,
y,
batch_size=100,
n_epochs=10,
loss_func='categorical_crossentropy',
optimizer=SGD(lr=0.01, rho=0.9),
clip=None,
callbacks=[],
shuffle=True,
verbose=1):
x = to_list(x)
y = to_list(y)
# format
x = [K.format_data(e) for e in x]
y = [K.format_data(e) for e in y]
# shuffle data
if shuffle:
x, y = supports.shuffle(x, y)
# check data
self._check_data(y, loss_func)
# init gt_nodes
self._gt_nodes_ = [K.placeholder(e.ndim) for e in y]
# memory usage
print "Train",
self._show_memory_usage(self._layer_list_, batch_size)
# default objective
if type(loss_func) is str:
assert len(self._out_nodes_)==len(self._gt_nodes_), "If you are using default objectives, " \
+ "out_node of out_layers must match ground truth!"
loss_node = sum([
obj.get(loss_func)(pred_node,
gt_node) for pred_node, gt_node in zip(
self._out_nodes_, self._gt_nodes_)
])
# user defined objective
else:
loss_node = loss_func(self._out_nodes_, self._any_nodes_,
self._gt_nodes_)
#loss_node = loss_func( self )
# gradient
gparams = K.grad(loss_node + self._reg_value_, self._params_)
# todo clip gradient
if clip is not None:
gparams = [K.clip(gparam, -clip, clip) for gparam in gparams]
# gradient based opt
param_updates = optimizer.get_updates(self._params_, gparams)
# get all updates
updates = param_updates + self._inner_updates_
# compile for callback
if callbacks is not None:
callbacks = to_list(callbacks)
for callback in callbacks:
callback.compile(self)
# compile model
input_nodes = self._in_nodes_ + self._gt_nodes_
output_nodes = [loss_node]
f = K.function_no_given(input_nodes, self._tr_phase_node_,
output_nodes, updates)
# train
N = len(x[0])
batch_num = int(np.ceil(float(N) / batch_size))
n_abs_epoch = n_epochs + self._epoch_
# callback
print '\n0th epoch:'
for callback in callbacks:
if (self._epoch_ % callback.call_freq == 0):
callback.call()
while self._epoch_ < n_abs_epoch:
self._epoch_ += 1
# train
t1 = time.time()
loss_list = []
for i2 in xrange(batch_num):
batch_x = [
e[i2 * batch_size:min((i2 + 1) * batch_size, N)] for e in x
]
batch_y = [
e[i2 * batch_size:min((i2 + 1) * batch_size, N)] for e in y
]
in_list = batch_x + batch_y + [1.]
loss = f(*in_list)[0] # training phase
loss_list.append(loss)
if verbose == 1:
self._print_progress(self._epoch_, batch_num, i2)
if verbose == 2:
self._print_progress_loss(self._epoch_, batch_num, i2,
loss)
t2 = time.time()
self._tr_time_ += (t2 - t1)
if verbose != 0:
print '\n', ' tr_time: ', "%.2f" % (
t2 - t1), 's' # print an empty line
# callback
for callback in callbacks:
if (self._epoch_ % callback.call_freq == 0):
callback.call()
def fit(self,
x,
y,
batch_size=100,
n_epochs=10,
loss_func='categorical_crossentropy',
optimizer=SGD(lr=0.01, momentum=0.9),
clip=None,
callbacks=[],
shuffle=True,
transformer=None,
verbose=1):
"""Fit data and train model. The data is reshuffled after every epoch.
Args:
x: ndarray | list of numpy ndarray.
y: ndarray | list of numpy ndarray.
batch_size: integar.
n_epoch: integar. Number of training epochs.
loss_func: str | function.
optimizer: optimization object.
clip: real value.
callbacks: list of Callback object.
shuffle: bool.
transformer: Transformation object.
verbose: 0 | 1 | 2
Returns: None. (All trained models, results should be saved using callbacks.)
"""
x = to_list(x)
y = to_list(y)
# Format data
x = format_data_list(x)
y = format_data_list(y)
# Train memory usage
print "Training",
self._show_memory_usage(self._effective_layers_, batch_size)
# Compile optimization function
timer = Timer()
target_dim_list = self._get_target_dim_list(x, y, transformer)
f_optimize = self.get_optimization_func(target_dim_list, loss_func,
optimizer, clip)
timer.show("Compiling f_optimize time:")
# Compile for callback
timer = Timer()
if callbacks is not None:
callbacks = to_list(callbacks)
for callback in callbacks:
callback.compile(self)
timer.show("Compiling callbacks time:")
# Train
N = len(x[0])
batch_num = int(np.ceil(float(N) / batch_size))
max_epoch = n_epochs + self.epoch_
# Callback
print '\n', self.epoch_, 'th epoch:'
for callback in callbacks:
if (self.epoch_ % callback.call_freq_ == 0):
callback.call()
while self.epoch_ < max_epoch:
# Shuffle data
if shuffle: x, y = supports.shuffle(x, y)
# Train
t1 = time.time()
loss_list = []
for i2 in xrange(batch_num):
batch_x = [
e[i2 * batch_size:min((i2 + 1) * batch_size, N)] for e in x
]
batch_y = [
e[i2 * batch_size:min((i2 + 1) * batch_size, N)] for e in y
]
if transformer:
(batch_x,
batch_y) = transformer.transform(batch_x, batch_y)
batch_x = format_data_list(batch_x)
batch_y = format_data_list(batch_y)
in_list = batch_x + batch_y + [1.] # training phase
loss = f_optimize(*in_list)[0]
loss_list.append(loss)
# self._iter_ += 1
if verbose == 1:
self.print_progress(self.epoch_, batch_num, i2)
if verbose == 2:
self.print_progress_loss(self.epoch_, batch_num, i2, loss)
t2 = time.time()
self._tr_time_ += (t2 - t1)
if verbose != 0:
print '\n', ' tr_time: ', "%.2f" % (
t2 - t1), 's' # print an empty line
self._epoch_ += 1
# Callback
for callback in callbacks:
if (self.epoch_ % callback.call_freq_ == 0):
callback.call()
def train_on_batch(self,
batch_x,
batch_y,
loss_func='categorical_crossentropy',
optimizer=SGD(lr=0.01, momentum=0.9),
clip=None,
callbacks=[],
transformer=None,
recompile=False):
"""Train model on single batch data.
Args:
batch_x: ndarray | list of ndarray.
batch_y: ndarray | list of ndarray.
loss_func: str | function.
optimizer: optimization object.
clip: real value.
callbacks: list of Callback object.
transformer: Transformation object.
recompile: bool. Recompile the optimize function or not.
Returns: None. (All trained models, results should be saved using callbacks.)
"""
batch_x = to_list(batch_x)
batch_y = to_list(batch_y)
# Format data
batch_x = format_data_list(batch_x)
batch_y = format_data_list(batch_y)
# Compile optimization function
if (not self._f_optimize_) or (recompile):
timer = Timer()
target_dim_list = self._get_target_dim_list(
batch_x, batch_y, transformer)
self._f_optimize_ = self.get_optimization_func(
target_dim_list, loss_func, optimizer, clip)
timer.show("Compiling f_optimize time:")
batch_size = len(batch_x[0])
self._show_memory_usage(self._effective_layers_, batch_size)
# Compile for callback
timer = Timer()
if callbacks is not None:
callbacks = to_list(callbacks)
for callback in callbacks:
callback.compile(self)
timer.show("Compiling callbacks time:")
for callback in callbacks:
if (self.iter_ % callback.call_freq_ == 0):
print self.iter_, 'th iteration:'
callback.call()
# Train
t1 = time.time()
if transformer:
(batch_x, batch_y) = transformer.transform(batch_x, batch_y)
batch_x = format_data_list(batch_x)
batch_y = format_data_list(batch_y)
in_list = batch_x + batch_y + [1.] # training phase
loss = self._f_optimize_(*in_list)[0]
self._iter_ += 1
t2 = time.time()
self._tr_time_ += (t2 - t1)
return loss
