def train_model():
m = build_model()
stop = climin.stops.any_([
climin.stops.after_n_iterations(max_iter),
])
pause = climin.stops.modulo_n_iterations(n_report)
weight_decay = ((m.parameters.hidden_to_out ** 2).sum())
# + (m.parameters.hidden_to_hidden_0**2).sum()
# + (m.parameters.hidden_to_out**2).sum())
weight_decay /= m.exprs['inpt'].shape[0]
m.exprs['true_loss'] = m.exprs['loss']
m.exprs['loss'] = m.exprs['loss'] + c_wd * weight_decay
f_wd = m.function(['inpt'], c_wd * weight_decay)
n_wrong = 1 - T.eq(T.argmax(m.exprs['output'], axis=1), T.argmax(m.exprs['target'], axis=1)).mean()
f_n_wrong = m.function(['inpt', 'target'], n_wrong)
losses = []
v_losses = []
print 'max iter', max_iter
start = time.time()
# Set up a nice printout.
keys = '#', 'loss', 'val loss', 'seconds', 'wd', 'train emp', 'test emp'
max_len = max(len(i) for i in keys)
header = '\t'.join(i for i in keys)
print header
print '-' * len(header)
f_loss = m.function(['inpt', 'target'], ['true_loss', 'loss'])
for i, info in enumerate(m.powerfit((X, Z), (TX, TZ), stop, pause)):
if info['n_iter'] % n_report != 0:
continue
passed = time.time() - start
losses.append(info['loss'])
v_losses.append(info['val_loss'])
#img = tile_raster_images(fe.parameters['in_to_hidden'].T, image_dims, feature_dims, (1, 1))
#save_and_display(img, 'filters-%i.png' % i)
info.update({
'time': passed,
'l2-loss': scalar(f_wd(X)),
'train_emp': scalar(f_n_wrong(X, Z)),
'test_emp': scalar(f_n_wrong(TX, TZ)),
})
row = '%(n_iter)i\t%(loss)g\t%(val_loss)g\t%(time)g\t%(l2-loss)g\t%(train_emp)g\t%(test_emp)g' % info
print row
np.savez_compressed(savepath, parameters=gp.as_numpy_array(m.parameters.data[...]))
def iter_fit(self, *fit_data):
start = time.time()
for info in self.model.iter_fit(*fit_data):
if self.pause(info):
# Take care of batch norm
# Things done here shouldn't affect running metrics since no learning is supposed to happen.
if self.using_bn:
self.model.phase_select(phase_id='valid')
if 'loss' not in info:
info['loss'] = ma.scalar(
self.score_fun(self.model.score, *self.data['train'])
)
if self.evaluate:
info['val_loss'] = ma.scalar(
self.score_fun(self.model.score, *self.data['val'])
)
if info['val_loss'] < self.best_loss:
self.best_loss = info['val_loss']
self.best_pars = self.model.parameters.data.copy()
self.losses.append((info['loss'], info['val_loss']))
else:
self.losses.append(info['loss'])
if self.test:
info['test_avg'] = ma.scalar(
self.score_fun(self.model.score, *self.data['test'])
)
self.test_performance.append(info['test_avg'])
self.runtime = time.time() - start
info.update({
'best_loss': self.best_loss,
'best_pars': self.best_pars,
'runtime': self.runtime
})
self.n_epochs_done = info['n_iter'] / self.n_report
self.n_iters_done = info['n_iter']
# Return to training mode, keep learning running metrics.
if self.using_bn:
self.model.phase_select(phase_id='train')
yield info
if self.stop(info):
break
def iter_fit(self, *fit_data):
start = time.time()
for info in self.model.iter_fit(*fit_data):
if self.pause(info):
# Take care of batch norm
# Things done here shouldn't affect running metrics since no learning is supposed to happen.
if self.using_bn:
self.model.phase_select(phase_id='valid')
if 'loss' not in info:
info['loss'] = ma.scalar(
self.score_fun(self.model.score, *self.data['train']))
if self.evaluate:
info['val_loss'] = ma.scalar(
self.score_fun(self.model.score, *self.data['val']))
if info['val_loss'] < self.best_loss:
self.best_loss = info['val_loss']
self.best_pars = self.model.parameters.data.copy()
self.losses.append((info['loss'], info['val_loss']))
else:
self.losses.append(info['loss'])
if self.test:
info['test_avg'] = ma.scalar(
self.score_fun(self.model.score, *self.data['test']))
self.test_performance.append(info['test_avg'])
self.runtime = time.time() - start
info.update({
'best_loss': self.best_loss,
'best_pars': self.best_pars,
'runtime': self.runtime
})
self.n_epochs_done = info['n_iter'] / self.n_report
self.n_iters_done = info['n_iter']
# Return to training mode, keep learning running metrics.
if self.using_bn:
self.model.phase_select(phase_id='train')
yield info
if self.stop(info):
break
def iter_fit(self, *fit_data):
"""Iteratively fit the given training data.
The values yielded from this function will be climin info dictionaries
stripped from any numpy or gnumpy arrays.
"""
start = time.time()
for info in self.model.iter_fit(*fit_data, info_opt=self.current_info):
interrupt = self.interrupt(info)
if self.pause(info) or interrupt:
info['val_loss'] = ma.scalar(
self.model._f_loss(self.model.parameters.data, *self.data[self.val_key]))
for i in self.info_keys:
info['{}_loss'.format(i)] = ma.scalar(
self.model._f_loss(
self.model.parameters.data,
*self.data[i]))
cur_val_loss = info['%s_loss' % self.val_key]
if cur_val_loss < self.best_loss:
self.best_loss = cur_val_loss
self.best_pars = self.model.parameters.data.copy()
self.runtime += time.time() - start
info.update({
'best_loss': self.best_loss,
'best_pars': self.best_pars,
'datetime': datetime.datetime.now(),
'runtime': self.runtime
})
filtered_info = clear_info(info)
self.infos.append(filtered_info)
self.current_info = info
yield filtered_info
start = time.time()
if self.stop(info):
self.stopped = True
break
if interrupt:
break
def __call__(self, f_score, *data):
""""Return the score of the data."""
batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1)
score = 0.
seen_samples = 0.
for batch in batches:
this_samples = int(batch[0].shape[self.sample_dims[0]])
score += f_score(*batch) * this_samples
seen_samples += this_samples
return ma.scalar(score / seen_samples)
def __call__(self, f_score, *data):
""""Return the score of the data."""
batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1)
score = 0.
seen_samples = 0.
for batch in batches:
this_samples = batch[0].shape[self.sample_dims[0]]
score += f_score(*batch) * this_samples
seen_samples += this_samples
return ma.scalar(score / seen_samples)
def __call__(self, f_score, *data):
batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1)
score = 0.
seen_samples = 0.
for batch in batches:
x = batch[0]
z = batch[1]
this_samples = int(x.shape[0])
score += f_score(x, z) * this_samples
seen_samples += this_samples
return ma.scalar(score / seen_samples)
def __call__(self, f_score, *data):
batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1)
score = 0.
seen_samples = 0.
for batch in batches:
x = batch[0]
z = batch[1]
this_samples = int(x.shape[0])
score += f_score(x, z) * this_samples
seen_samples += this_samples
return ma.scalar(score / seen_samples)
def iter_fit(self, *fit_data):
"""Iteratively fit the given training data.
Generator function containing the main logic of the Trainer object.
The arguments are of variable length and have to match that of the
``model.iter_fit()`` and ultimately the used loss function of that
model.
Each iteration of the fitting constitutes of running the optimizer of
the model until either interrupt or pause returns True.
In both cases, the generator will yield to the user. Additionally:
- If interrupt returns True, the generator will stop yielding
values afterwards.
- stop will be tested. If it is true it will stop yielding
afterwards and additionally ``.stopped`` will be set to True
afterwards.
- ``best_pars`` and ``best_loss`` will be updated.
The values yielded from this function will be climin info dictionaries
stripped from any numpy or gnumpy arrays.
"""
for info in self.model.iter_fit(*fit_data, info_opt=self.current_info):
interrupt = self.interrupt(info)
if self.pause(info) or interrupt:
info['val_loss'] = ma.scalar(
self.score(*self.eval_data[self.val_key]))
cur_val_loss = info['%s_loss' % self.val_key]
if cur_val_loss < self.best_loss:
self.best_loss = cur_val_loss
self.best_pars = self.model.parameters.data.copy()
info['best_loss'] = self.best_loss
info['best_pars'] = self.best_pars
info.update({
'datetime': datetime.datetime.now(),
})
filtered_info = clear_info(info)
self.infos.append(filtered_info)
self.current_info = info
yield info
if self.stop(info):
self.stopped = True
break
if interrupt:
break
def iter_fit(self, *fit_data):
"""Iteratively fit the given training data.
Generator function containing the main logic of the Trainer object.
The arguments are of variable length and have to match that of the
``model.iter_fit()`` and ultimately the used loss function of that
model.
Each iteration of the fitting constitutes of running the optimizer of
the model until either interrupt or pause returns True.
In both cases, the generator will yield to the user. Additionally:
- If interrupt returns True, the generator will stop yielding
values afterwards.
- stop will be tested. If it is true it will stop yielding
afterwards and additionally ``.stopped`` will be set to True
afterwards.
- ``best_pars`` and ``best_loss`` will be updated.
The values yielded from this function will be climin info dictionaries
stripped from any numpy or gnumpy arrays.
"""
for info in self.model.iter_fit(*fit_data, info_opt=self.current_info):
interrupt = self.interrupt(info)
if self.pause(info) or interrupt:
info['val_loss'] = ma.scalar(self.score(*self.data[self.val_key]))
cur_val_loss = info['%s_loss' % self.val_key]
if cur_val_loss < self.best_loss:
self.best_loss = cur_val_loss
self.best_pars = self.model.parameters.data.copy()
info['best_loss'] = self.best_loss
info['best_pars'] = self.best_pars
info.update({
'datetime': datetime.datetime.now(),
})
filtered_info = clear_info(info)
self.infos.append(filtered_info)
self.current_info = info
yield info
if self.stop(info):
self.stopped = True
break
if interrupt:
break
def __call__(self, predict_f, *data):
batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1)
seen_samples = 0.
score = 0.
for batch in batches:
x, z = batch
y = predict_f(x)
this_samples = int(y.shape[1])
errs = (y.argmax(axis=2) != z.argmax(axis=2)).sum()
score += errs
seen_samples += this_samples
return ma.scalar(score / seen_samples)
def __call__(self, predict_f, *data):
batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1)
seen_samples = 0.
score = 0.
for batch in batches:
x, z = batch
y = predict_f(x)
this_samples = int(y.shape[1])
errs = (y.argmax(axis=2) != z.argmax(axis=2)).sum()
score += errs
seen_samples += this_samples
return ma.scalar(score / seen_samples)
def handle_update(self, fit_data):
update_losses = {}
for key, data in self.eval_data.items():
update_losses['%s_loss' % key] = ma.scalar(self.score(*data))
return update_losses
def powerfit(self,
fit_data,
eval_data,
stop,
report,
eval_train_loss=True):
"""Iteratively fit the model.
This is a convenience function which combines iteratively fitting a
model with stopping criterions and keeping track of the best parameters
found so far.
An iterator of dictionaries is returned; values are only yielded in the
case that the call `report(info)` returns True. The iterator
stops as soon as the call `stop(info)` returns True.
Each dictionary yielded is directly obtained from the optimizer used to
optimize the loss. It is augmented with the keys `loss`, `best_pars`
and `best_loss`. The best loss is obtained by evaluating the loss of
the model (given by model.exprs['loss']) on `eval_data`, while training
is done on `fit_data`.
This method respects a ``true_loss`` entry in the ``exprs``
dictionary: if it is present, it will be used for reporting the loss
and for comparing models throughout optimization instead of ``loss``,
which will be used for the optimization itself. This makes it possible
to add regularization terms to the loss and use other losses (such as
the zero-one loss) for comparison of parameters.
:param fit_data: A tuple containing arrays representing the data the
model should be fitted on.
:param eval_data: A tuple containing arrays representing the data the
model should be evaluated on, and which gives which model is
"best".
:param stop: A function receiving an info dictionary which returns True
if the iterator should stop.
:param report: A function receiving an info dictionary which should
return True if the iterator should yield a value.
:returns: An iterator over info dictionaries.
"""
# TODO document eval train loss
self.CTRL_C_FLAG = False
signal.signal(signal.SIGINT, self._ctrl_c_handler)
best_pars = None
best_loss = float('inf')
for info in self.iter_fit(*fit_data):
if report(info):
if 'loss' not in info:
# Not all optimizers, e.g. ilne and gd, do actually
# calculate the loss.
if eval_train_loss:
info['loss'] = ma.scalar(self.score(*fit_data))
else:
info['loss'] = 0.
info['val_loss'] = ma.scalar(self.score(*eval_data))
if info['val_loss'] < best_loss:
best_loss = info['val_loss']
best_pars = self.parameters.data.copy()
info['best_loss'] = best_loss
info['best_pars'] = best_pars
yield info
if stop(info) or self.CTRL_C_FLAG:
break
def powerfit(self, fit_data, eval_data, stop, report, eval_train_loss=True):
"""Iteratively fit the model.
This is a convenience function which combines iteratively fitting a
model with stopping criterions and keeping track of the best parameters
found so far.
An iterator of dictionaries is returned; values are only yielded in the
case that the call `report(info)` returns True. The iterator
stops as soon as the call `stop(info)` returns True.
Each dictionary yielded is directly obtained from the optimizer used to
optimize the loss. It is augmented with the keys `loss`, `best_pars`
and `best_loss`. The best loss is obtained by evaluating the loss of
the model (given by model.exprs['loss']) on `eval_data`, while training
is done on `fit_data`.
This method respects a ``true_loss`` entry in the ``exprs``
dictionary: if it is present, it will be used for reporting the loss
and for comparing models throughout optimization instead of ``loss``,
which will be used for the optimization itself. This makes it possible
to add regularization terms to the loss and use other losses (such as
the zero-one loss) for comparison of parameters.
:param fit_data: A tuple containing arrays representing the data the
model should be fitted on.
:param eval_data: A tuple containing arrays representing the data the
model should be evaluated on, and which gives which model is
"best".
:param stop: A function receiving an info dictionary which returns True
if the iterator should stop.
:param report: A function receiving an info dictionary which should
return True if the iterator should yield a value.
:returns: An iterator over info dictionaries.
"""
# TODO document eval train loss
self.CTRL_C_FLAG = False
signal.signal(signal.SIGINT, self._ctrl_c_handler)
best_pars = None
best_loss = float('inf')
for info in self.iter_fit(*fit_data):
if report(info):
if 'loss' not in info:
# Not all optimizers, e.g. ilne and gd, do actually
# calculate the loss.
if eval_train_loss:
info['loss'] = ma.scalar(self.score(*fit_data))
else:
info['loss'] = 0.
info['val_loss'] = ma.scalar(self.score(*eval_data))
if info['val_loss'] < best_loss:
best_loss = info['val_loss']
best_pars = self.parameters.data.copy()
info['best_loss'] = best_loss
info['best_pars'] = best_pars
yield info
if stop(info) or self.CTRL_C_FLAG:
break
report_fun = report.OneLinePrinter(
['n_iter', 'runtime', 'loss', 'val_loss', 'test_avg'],
spaces=['4', '7.4f', '5.4f', '7.4f', '7.4f']
)
score_fun = ash.MinibatchScoreFCN(max_samples=batch_size, sample_dims=[0, 0])
data = {
'train':(train_x, train_y),
'val':(valid_x, valid_y),
'test':(test_x, test_y)
}
test_fun = ash.MinibatchTestFCN(max_samples=batch_size, sample_dims=[0, 0])
initial_err = ma.scalar(score_fun(pkchu.score, *data['train']))
print 'Initial train loss: %.4f' % initial_err
coach = PocketTrainer(
model=pkchu, data=data, stop=stop,
pause=pause, score_fun=score_fun,
report_fun=report_fun, test_fun=test_fun,
evaluate=True, test=True
)
coach.fit()
pkchu.parameters.data[...] = coach.best_pars
plt.plot(coach.losses)
def RNN_EEG(n_neurons=100,
batch_size=50,
participant=[1],
series=[1, 2, 3, 4, 5, 6, 7, 8, 9],
subsample=0,
imp_weights_skip=150,
n_layers=1):
format_string = '%d_%d_['
for p in participant:
format_string += '%d,' % p
format_string = format_string[:-1]
format_string += ']_['
for s in series:
format_string += '%d,' % s
format_string = format_string[:-1]
format_string += ']_%d_%d_%d_%d'
net_info = format_string % (n_neurons, batch_size, subsample,
imp_weights_skip, n_layers, time.time())
logging.info('--------------------------------')
logging.info('EEG - DATA')
logging.info(
'Rnn: n_neurons: %i, batch_size: %i, participant: %s, series: %s, subsample: %i, imp_weights_skip: %i'
% (n_neurons, batch_size, participant, series, subsample,
imp_weights_skip))
#optimizer = 'rmsprop', {'step_rate': 0.0001, 'momentum': 0.9, 'decay': 0.9}
decay = 0.9
offset = 1e-6
mom = .9
step_rate = .1
optimizer = 'adadelta', {
'decay': decay,
'offset': offset,
'momentum': mom,
'step_rate': step_rate
}
# optimizer = 'adam'
n_hiddens = [n_neurons] * n_layers
logging.info('optimizer: %s' % str(optimizer))
m = SupervisedRnn(st.N_EEG_SENSORS,
n_hiddens,
st.N_EEG_TARGETS,
out_transfer='sigmoid',
loss='bern_ces',
hidden_transfers=['tanh'] * n_layers,
batch_size=batch_size,
imp_weight=True,
optimizer=optimizer)
sX, sZ, sVX, sVZ, TX, TZ, seqlength, eventNames = get_shaped_input(
participant, series, subsample)
W = np.ones_like(sZ)
WV = np.ones_like(sVZ)
WT = np.ones_like(TX)
W[:imp_weights_skip, :, :] = 0
WV[:imp_weights_skip, :, :] = 0
WT[:imp_weights_skip, :, :] = 0
m.exprs['true_loss'] = m.exprs['loss']
# TODO: Test Loss: Doesn't work, don't know why, trying a hacky workaround in test_loss() - don't know if right this way
# f_loss = m.function(['inpt', 'target', 'imp_weight'], 'true_loss')
# print(f_loss([TX[:, :, :], TZ[:, :, :]]))
# print(m.score(TX[:,0:1,:], TZ[:,0:1,:], WT[:,0:1,:])) # similar error...
# bern_ces changes the prediction!!! *facepalm*
# ... just be careful to call it with a copy of the array
def test_loss():
return bern_ces(
m.predict(TX)[:imp_weights_skip, :seqlength, :],
np.copy(TZ[:imp_weights_skip, :seqlength, :])).eval().mean()
'''
def test_nll():
nll = 0
n_time_steps = 0
for x, z in zip(tx, tz):
nll += f_loss(x[:, np.newaxis], z[:, np.newaxis]) * x.shape[0]
n_time_steps += x.shape[0]
return nll / n_time_steps
'''
climin.initialize.randomize_normal(m.parameters.data, 0, 0.1)
#climin.initialize.bound_spectral_radius(m.parameters.data)
def plot(test_sample=0,
save_name='images/%s_EEG.png' % net_info,
test_loss=None):
colors = ['blue', 'red', 'green', 'cyan', 'magenta']
figure, (axes) = plt.subplots(3, 1)
#input_for_plot = sVX.transpose(1,0,2).reshape((-1, seqlength, st.N_EEG_SENSORS)).transpose(1,0,2)[:, 0:1, :]
#target_for_plot = sVZ.transpose(1,0,2).reshape((-1, seqlength, st.N_EEG_TARGETS)).transpose(1,0,2)[:, 0:1, :]
input_for_plot = TX[:, test_sample:test_sample + 1, :]
# to be able to plot the test samples in correct length we have to determine where the '0' padding starts
sample_length_arr = np.where(
input_for_plot == np.zeros((st.N_EEG_SENSORS)))[0]
if len(sample_length_arr != 0):
sample_length = min(
np.where(
np.bincount(sample_length_arr) == st.N_EEG_SENSORS)[0])
else:
sample_length = input_for_plot.shape[0]
input_for_plot = input_for_plot[:sample_length]
target_for_plot = TZ[:sample_length, test_sample:test_sample + 1, :]
result = m.predict(input_for_plot)
x_axis = np.arange(input_for_plot.shape[0])
for i in range(st.N_EEG_TARGETS):
axes[0].set_title('TARGETS')
axes[0].fill_between(x_axis,
0,
target_for_plot[:, 0, i],
facecolor=colors[i],
alpha=0.8,
label=eventNames[st.SEQ_EEG_TARGETS.index(i)])
#axes[0].plot(x_axis, target_for_plot[:, 0, i])
if test_loss:
axes[1].set_title('RNN (overall test loss: %f)' % (test_loss))
else:
axes[1].set_title('RNN')
axes[1].plot(x_axis, result[:, 0, i], color=colors[i])
train_loss = []
val_loss = []
test_loss = []
for i in infos:
train_loss.append(i['loss'])
val_loss.append(i['val_loss'])
test_loss.append(i['test_loss'])
axes[2].set_title('LOSSES')
axes[2].plot(np.arange(len(infos)), train_loss, label='train loss')
axes[2].plot(np.arange(len(infos)), val_loss, label='validation loss')
axes[2].plot(np.arange(len(infos)), test_loss, label='test loss')
axes[0].legend(
loc=0, shadow=True,
fontsize='x-small') # loc: 0=best, 1=upper right, 2=upper left
axes[2].legend(loc=0, shadow=True, fontsize='x-small')
figure.subplots_adjust(hspace=0.5)
figure.savefig(save_name)
plt.close(figure)
max_passes = 30
max_minutes = 200
max_iter = max_passes * sX.shape[1] / m.batch_size
batches_per_pass = int(math.ceil(float(sX.shape[1]) / m.batch_size))
pause = climin.stops.ModuloNIterations(
batches_per_pass * 1) # after each pass through all data
stop = climin.stops.Any([
# climin.stops.TimeElapsed(max_minutes * 60), # maximal time in seconds
climin.stops.AfterNIterations(max_iter), # maximal iterations
# climin.stops.Patience('val_loss', batches_per_pass*10, grow_factor=1.5, threshold=0.0001), # kind of early stopping
# climin.stops.NotBetterThanAfter(30, 100), # error under 30 after 100 iterations?
])
start = time.time()
header = '#', 'seconds', 'loss', 'val loss', 'test loss'
print '\t'.join(header)
logging.info('\t'.join(header))
infos = []
try:
os.makedirs('losses')
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir('losses'):
pass
else:
raise
f = open('losses/%s_EEG.csv' % net_info, 'wt')
try:
writer = csv.writer(f)
writer.writerow(('Train loss', 'Validation loss', 'Test Loss'))
for i, info in enumerate(
m.powerfit((sX, sZ, W), (sVX, sVZ, WV),
stop=stop,
report=pause,
eval_train_loss=True)):
info['loss'] = float(info['loss'])
info['val_loss'] = float(info['val_loss'])
info['test_loss'] = float(ma.scalar(test_loss()))
writer.writerow(
(info['loss'], info['val_loss'], info['test_loss']))
info.update({
'time': time.time() - start,
# 'spectral_radius': get_spectral_radius(m.parameters['recurrent_0']),
})
template = '\t'.join([
'%(n_iter)i', '%(time)g', '%(loss)g', '%(val_loss)g',
'%(test_loss)g'
])
row = template % info
print row
logging.info(row)
filtered_info = dict(
(k, v) for k, v in info.items()
# if (not isinstance(v, (np.ndarray, gp.garray)) or v.size <= 1) and k not in ('args', 'kwargs'))
if (not isinstance(v, (np.ndarray, )) or v.size <= 1)
and k not in ('args', 'kwargs'))
for key in filtered_info:
if isinstance(filtered_info[key], np.float32):
filtered_info[key] = float(filtered_info[key])
infos.append(filtered_info)
finally:
f.close()
m.parameters.data[...] = info['best_pars']
save_timestmp = toUTCtimestamp(datetime.utcnow())
dot_idx = str(save_timestmp).index('.')
save_timestmp = str(save_timestmp)[:dot_idx]
logging.info('saved at: %s' % save_timestmp)
plot(0, 'images/%s_eeg_test0.png' % net_info, test_loss())
plot(1, 'images/%s_eeg_test1.png' % net_info, test_loss())
plot(2, 'images/%s_eeg_test2.png' % net_info, test_loss())
plot(3, 'images/%s_eeg_test3.png' % net_info, test_loss())
def handle_update(self, fit_data):
update_losses = {}
for key, data in self.eval_data.items():
update_losses['%s_loss' % key] = ma.scalar(self.score(*data))
return update_losses
def test_RNN(n_neurons=100, batch_size=50, participant=[1], series=[1, 2], subsample=10,
imp_weights_skip=150, n_layers=1):
format_string = '%d_%d_['
for p in participant:
format_string += '%d,' % p
format_string = format_string[:-1]
format_string += ']_['
for s in series:
format_string += '%d,' % s
format_string = format_string[:-1]
format_string += ']_%d_%d_%d_%d'
net_info = format_string % (n_neurons, batch_size, subsample, imp_weights_skip, n_layers, time.time())
logging.info('--------------------------------')
logging.info('Rnn: n_neurons: %i, batch_size: %i, participant: %s, series: %s, subsample: %i, imp_weights_skip: %i'
% (n_neurons, batch_size, participant, series, subsample, imp_weights_skip))
#optimizer = 'rmsprop', {'step_rate': 0.0001, 'momentum': 0.9, 'decay': 0.9}
decay = 0.9
offset = 1e-6
mom = .9
step_rate = .1
optimizer = 'adadelta', {'decay': decay, 'offset': offset, 'momentum': mom, 'step_rate': step_rate}
# optimizer = 'adam'
n_hiddens = [n_neurons] * n_layers
logging.info('optimizer: %s' % str(optimizer))
m = SupervisedRnn(
st.N_EMG_SENSORS, n_hiddens, st.N_EMG_TARGETS, out_transfer='sigmoid', loss='bern_ces',
hidden_transfers=['tanh'] * n_layers,
batch_size=batch_size,
imp_weight=True,
optimizer=optimizer)
sX, sZ, sVX, sVZ, sTX, sTZ, eventNames = get_shaped_input(participant, series, subsample)
W = np.ones_like(sZ)
WV = np.ones_like(sVZ)
#WT = np.ones_like(TX)
W[:imp_weights_skip, :, :] = 0
WV[:imp_weights_skip, :, :] = 0
#WT[:imp_weights_skip, :, :] = 0
m.exprs['true_loss'] = m.exprs['loss']
# TODO: Test Loss: Doesn't work, don't know why, trying a hacky workaround in test_loss() - don't know if right this way
# f_loss = m.function(['inpt', 'target', 'imp_weight'], 'true_loss')
# print(f_loss([TX[:, :, :], TZ[:, :, :]]))
# print(m.score(TX[:,0:1,:], TZ[:,0:1,:], WT[:,0:1,:])) # similar error...
# bern_ces changes the prediction!!! *facepalm*
# ... just be careful to call it with a copy of the array
def test_loss():
pred = m.predict(sTX)
return bern_ces(np.copy(sTZ), pred).eval().mean()
climin.initialize.randomize_normal(m.parameters.data, 0, 0.1)
def plot(test_sample=0, save_name='images/%s_test.png' % net_info, test_loss=None):
colors = ['blue', 'red', 'green', 'cyan', 'magenta']
plt.figure(figsize=(40, 10))
figure, (axes) = plt.subplots(3, 1,figsize=(20,5))
input_for_plot = sTX
target_for_plot = sTZ
result = m.predict(input_for_plot)
x_axis = np.arange(input_for_plot.shape[0])
for i in range(st.N_EMG_TARGETS):
axes[0].set_title('TARGETS')
axes[0].fill_between(x_axis, 0, target_for_plot[:, 0, i], facecolor=colors[i], alpha=0.8,
label=eventNames[st.SEQ_EMG_TARGETS.index(i)])
if test_loss:
axes[1].set_title('RNN (overall test loss: %f)' %(test_loss))
else:
axes[1].set_title('RNN')
axes[1].plot(x_axis, result[:, 0, i], color=colors[i])
train_loss = []
val_loss = []
test_loss = []
for i in infos:
train_loss.append(i['loss'])
val_loss.append(i['val_loss'])
test_loss.append(i['test_loss'])
axes[2].set_title('LOSSES')
axes[2].plot(np.arange(len(infos)), train_loss, label='train loss')
axes[2].plot(np.arange(len(infos)), val_loss, label='validation loss')
axes[2].plot(np.arange(len(infos)), test_loss, label='test loss')
axes[0].legend(loc=0, shadow=True, fontsize='x-small') # loc: 0=best, 1=upper right, 2=upper left
axes[2].legend(loc=0, shadow=True, fontsize='x-small')
figure.subplots_adjust(hspace=0.5)
figure.savefig(save_name)
plt.close(figure)
max_passes = 400
max_minutes = 60
max_iter = max_passes * sX.shape[1] / m.batch_size
batches_per_pass = int(math.ceil(float(sX.shape[1]) / m.batch_size))
pause = climin.stops.ModuloNIterations(batches_per_pass * 1) # after each pass through all data
stop = climin.stops.Any([
climin.stops.TimeElapsed(max_minutes * 60), # maximal time in seconds
climin.stops.AfterNIterations(max_iter) # maximal iterations
#climin.stops.Patience('val_loss', batches_per_pass*50, grow_factor=2.0, threshold=0.00001), # kind of early stopping
# climin.stops.NotBetterThanAfter(30, 100), # error under 30 after 100 iterations?
])
start = time.time()
header = '#', 'seconds', 'loss', 'val loss', 'test loss'
print '\t'.join(header)
logging.info('\t'.join(header))
infos = []
try:
os.makedirs('losses')
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir('losses'):
pass
else:
raise
f = open('losses/%s.csv' % net_info, 'wt')
try:
writer = csv.writer(f)
writer.writerow(('Train loss', 'Validation loss', 'Test Loss'))
for i, info in enumerate(m.powerfit((sX, sZ, W), (sVX, sVZ, WV), stop=stop,
report=pause, eval_train_loss=True)):
info['loss'] = float(info['loss'])
info['val_loss'] = float(info['val_loss'])
info['test_loss'] = float(ma.scalar(test_loss()))
writer.writerow((info['loss'], info['val_loss'], info['test_loss']))
info.update({
'time': time.time() - start,
})
template = '\t'.join(
['%(n_iter)i', '%(time)g', '%(loss)g', '%(val_loss)g', '%(test_loss)g'])
row = template % info
print row
logging.info(row)
filtered_info = dict(
(k, v) for k, v in info.items()
# if (not isinstance(v, (np.ndarray, gp.garray)) or v.size <= 1) and k not in ('args', 'kwargs'))
if (not isinstance(v, (np.ndarray,)) or v.size <= 1) and k not in ('args', 'kwargs'))
for key in filtered_info:
if isinstance(filtered_info[key], np.float32):
filtered_info[key] = float(filtered_info[key])
infos.append(filtered_info)
finally:
f.close()
m.parameters.data[...] = info['best_pars']
save_timestmp = toUTCtimestamp(datetime.utcnow())
dot_idx = str(save_timestmp).index('.')
save_timestmp = str(save_timestmp)[:dot_idx]
logging.info('saved at: %s' % save_timestmp)
plot(0, 'images/%s_emg_test0.png' % net_info, test_loss())
pkchu.parameters.data[...] = pickle.load(f)
report_fun = report.OneLinePrinter(
['n_iter', 'runtime', 'loss', 'val_loss', 'test_avg'],
spaces=['4', '7.4f', '5.4f', '7.4f', '7.4f'])
score_fun = ash.MinibatchScoreFCN(max_samples=batch_size, sample_dims=[0, 0])
data = {
'train': (train_x, train_y),
'val': (valid_x, valid_y),
'test': (test_x, test_y)
}
test_fun = ash.MinibatchTestFCN(max_samples=batch_size, sample_dims=[0, 0])
initial_err = ma.scalar(score_fun(pkchu.score, *data['train']))
print 'Initial train loss: %.4f' % initial_err
coach = PocketTrainer(model=pkchu,
data=data,
stop=stop,
pause=pause,
score_fun=score_fun,
report_fun=report_fun,
test_fun=test_fun,
evaluate=True,
test=True)
coach.fit()
