def addLayer(self, n_units, type):
"""Adds a new layer to the network,
:param n_units: number of units in the layer
:param type: a string specification of the activation function
"""
# check number of units
assert util.isposint(n_units), 'Number of units must be a positive integer.'
# choose activation function
actfun = util.select_theano_act_function(type, dtype)
n_prev_units = self.n_outputs
self.n_outputs = n_units
self.n_units.append(n_units)
self.n_layers += 1
self.n_params += (n_prev_units + 1) * n_units
W = theano.shared((rng.randn(n_prev_units, n_units) / np.sqrt(n_prev_units + 1)).astype(dtype), name='W' + str(self.n_layers), borrow=True)
b = theano.shared(np.zeros(n_units, dtype=dtype), name='b' + str(self.n_layers), borrow=True)
h = actfun(tt.dot(self.hs[-1], W) + b)
h.name = 'h' + str(self.n_layers)
self.Ws.append(W)
self.bs.append(b)
self.hs.append(h)
self.parms = self.Ws + self.bs
self.output = self.hs[-1]
self.eval_f = None
def __init__(self, n_inputs, input=None):
"""Constructs a net with a given number of inputs and no layers."""
assert util.isposint(n_inputs), 'Number of inputs must be a positive integer.'
self.n_inputs = n_inputs
self.n_outputs = n_inputs
self.n_units = [n_inputs]
self.n_layers = 0
self.n_params = 0
self.Ws = []
self.bs = []
self.hs = [tt.matrix('x') if input is None else input]
self.parms = self.Ws + self.bs
self.input = self.hs[0]
self.output = self.hs[-1]
self.eval_f = None
def train(self,
minibatch=None,
tol=None,
maxepochs=None,
monitor_every=1,
patience=None,
verbose=True,
show_progress=False,
val_in_same_plot=True):
"""
Trains the model.
:param minibatch: minibatch size
:param tol: tolerance
:param maxepochs: maximum number of epochs
:param monitor_every: monitoring frequency
:param patience: maximum number of validation steps to wait for improvement before early stopping
:param verbose: if True, print progress during training
:param show_progress: if True, plot training and validation progress
:param val_in_same_plot: if True, plot validation progress in same plot as training progress
:return: None
"""
# parse input
assert minibatch is None or util.isposint(
minibatch), 'Minibatch size must be a positive integer or None.'
assert tol is None or tol > 0.0, 'Tolerance must be positive or None.'
tol = -1. if tol is None else tol
assert maxepochs is None or maxepochs > 0.0, 'Maximum number of epochs must be positive or None.'
assert monitor_every is None or monitor_every > 0.0, 'Monitoring frequency must be positive or None.'
assert patience is None or util.isposint(
patience), 'Patience must be a positive integer or None.'
assert isinstance(verbose, bool), 'verbose must be boolean.'
assert isinstance(show_progress,
bool), 'store_progress must be boolean.'
assert isinstance(val_in_same_plot,
bool), 'val_in_same_plot must be boolean.'
# initialize some variables
iter = 0
progress_epc = []
progress_trn = []
progress_val = []
minibatch = self.n_trn_data if minibatch is None else minibatch
maxiter = float('inf') if maxepochs is None else np.ceil(
maxepochs * self.n_trn_data / float(minibatch))
monitor_every = float('inf') if monitor_every is None else np.ceil(
monitor_every * self.n_trn_data / float(minibatch))
patience = float('inf') if patience is None else patience
patience_left = patience
best_epoch = None
# main training loop
while True:
# make update to parameters
trn_loss = self.make_update(self.idx_stream.gen(minibatch))
diff = self.trn_loss - trn_loss
iter += 1
self.trn_loss = trn_loss
if iter % monitor_every == 0:
epoch = iter * float(minibatch) / self.n_trn_data
# do validation
if self.do_validation:
if self.set_batch_norm_stats is not None:
self.set_batch_norm_stats()
val_loss = self.validate()
patience_left -= 1
if val_loss < self.best_val_loss:
self.best_val_loss = val_loss
self.checkpointer.checkpoint()
best_epoch = epoch
patience_left = patience
# monitor progress
if show_progress:
progress_epc.append(epoch)
progress_trn.append(trn_loss)
if self.do_validation: progress_val.append(val_loss)
# print info
if verbose:
if self.do_validation:
print(
'Epoch = {0:.2f}, train loss = {1}, validation loss = {2}'
.format(epoch, trn_loss, val_loss))
else:
print('Epoch = {0:.2f}, train loss = {1}'.format(
epoch, trn_loss))
# check for convergence
if abs(diff) < tol or iter >= maxiter or patience_left <= 0:
if self.do_validation: self.checkpointer.restore()
if self.set_batch_norm_stats is not None:
self.set_batch_norm_stats()
break
# plot progress
if show_progress:
if self.do_validation:
if val_in_same_plot:
fig, ax = plt.subplots(1, 1)
ax.semilogx(progress_epc,
progress_trn,
'b',
label='training')
ax.semilogx(progress_epc,
progress_val,
'r',
label='validation')
ax.vlines(best_epoch,
ax.get_ylim()[0],
ax.get_ylim()[1],
color='g',
linestyles='dashed',
label='best')
ax.set_xlabel('epochs')
ax.set_ylabel('loss')
ax.legend()
else:
fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True)
ax1.semilogx(progress_epc, progress_trn, 'b')
ax2.semilogx(progress_epc, progress_val, 'r')
ax1.vlines(best_epoch,
ax1.get_ylim()[0],
ax1.get_ylim()[1],
color='g',
linestyles='dashed',
label='best')
ax2.vlines(best_epoch,
ax2.get_ylim()[0],
ax2.get_ylim()[1],
color='g',
linestyles='dashed',
label='best')
ax2.set_xlabel('epochs')
ax1.set_ylabel('training loss')
ax2.set_ylabel('validation loss')
else:
fig, ax = plt.subplots(1, 1)
ax.semilogx(progress_epc, progress_trn, 'b')
ax.set_xlabel('epochs')
ax.set_ylabel('training loss')
ax.legend()
plt.show(block=False)