def train(self, nmultilayer, output, config, dataset):
ml = self.MultiLayer[nmultilayer]
ml.next_run()
pprint('%d.%d.2.1 - Visualize Weights' % (
self._run+1, nmultilayer+1), end='')
output.visualize_all_weights(self,nmultilayer,config)
pprint(' (%0.2f MB)' % mem_usage())
# if (len(self.MultiLayer) == 1):
# TODO: for greedy model: output Likelihood of up to the
# current MultiLayer
pprint('%d.%d.2.2 - LogLikelihood' % (
self._run+1, nmultilayer+1), end='')
output.write_loglikelihood(self, nmultilayer)
pprint(' (%0.2f MB)' % mem_usage())
# variables for stopping criterion
# TODO: generalize for all MultiLayers
try:
STOPPING_CRITERION = config.get()\
['model']['MultiLayer1']['StoppingCriterion']
except:
STOPPING_CRITERION = False
if STOPPING_CRITERION:
try:
mvngwidth = int(config.get()\
['model']['MultiLayer1']['MovingWidth'])
except:
pprint('WARNING (model.Model::Train): No width for \
moving average was given. It will be set to %d'%20)
mvngwidth = 20
loglikelihood = np.asarray([], dtype=np.float32)
mvng_avg, mvng_std = 0., 0.
max_mvng_avg = float('-inf')
last_weights = []
else:
loglikelihood = np.asarray([None])
STOP = False
MPI.COMM_WORLD.Barrier()
pprint('%d.%d.2.3 - Training Iterations' % (self._run+1,nmultilayer+1))
for niteration in xrange(ml.get_iterations()):
pprint('Iteration: %*d' % (
int(math.log10(ml.get_iterations()))+1,
ml.get_iteration()+1), end='')
# pprint('2.2.3.1 - Convergence', end='')
output.conv_pre(ml)
# pprint(' - Memory usage: %s (Mb)' % mem_usage())
MPI.COMM_WORLD.Barrier()
# pprint('2.2.3.2 - Learning Iteration', end='')
ml.learning_iteration(self._theanoscan)
# pprint(' - Memory usage: %s (Mb)\n' % mem_usage())
MPI.COMM_WORLD.Barrier()
# experimental: variing learning rates
# if ((niteration % 1 == 0) and
# (ml.Layer[1]._epsilon >= 0.0008)):
# ml.Layer[1]._epsilon *= 0.98
# MPI.COMM_WORLD.Barrier()
# to save learned weights every iteration
# output.save_weights(self.MultiLayer[nmultilayer])
# to save posterior distribution of training data every iteration
# if nmultilayer == self.number_of_multilayers()-1:
# output.save_posterior(self, config, dataset)
# Stopping criterion
if (len(self.MultiLayer) == 1):
# TODO: for greedy model: output Likelihood of up to the
# current MultiLayer
if STOPPING_CRITERION:
t0 = time.time()
loglikelihood = np.append(
loglikelihood,
self.loglikelihood()
)
if (niteration >= mvngwidth):
# save only the last #mvngwidth values
loglikelihood = loglikelihood[1:]
pprint(' | Log-Likelihood: %f (%f s)'%(
loglikelihood[-1], time.time()-t0), end='')
if STOPPING_CRITERION:
# save only the last #mvngwidht/2-1 weights
# for centered moving average
last_weights.append([ml.Layer[nlayer].get_weights()
for nlayer in xrange(1,len(ml.Layer))])
if (niteration > mvngwidth/2):
last_weights = last_weights[1:]
# calculate moving average over last #mvngwidth values
if (niteration >= mvngwidth-1):
mvng_avg = np.mean(loglikelihood)
mvng_std = np.std(loglikelihood)
pprint(' | Moving Average (%d): %f +- %f'%
((niteration+1 - mvngwidth/2),mvng_avg,mvng_std),
end='')
if (mvng_avg > max_mvng_avg):
max_mvng_avg = mvng_avg
elif (mvng_avg < max_mvng_avg - mvng_std):
# if the moving average drops below the maximum
# moving average by more than the moving standard
# deviation, stop the learning iteration and revert
# back to the point of the centered moving average
for nlayer in xrange(1,len(ml.Layer)):
ml.Layer[nlayer].set_weights(last_weights[0][nlayer-1])
stopping_iteration = niteration+1 - mvngwidth/2
pprint('\nStopping criterion met at iteration %d.'%
stopping_iteration)
STOP = True
ml.set_iteration(stopping_iteration)
ml.set_iterations(stopping_iteration)
# abort on numerical error (nan in any of the weights)
if any([np.any(np.isnan(ml.Layer[nlayer].get_weights()))
for nlayer in xrange(1,len(ml.Layer))]):
pprint('\nNumerical error: try to decrease learning ' +
'rate and/or mini-batch size.')
STOP = True
try:
ml.set_iteration(
stopping_iteration)
ml.set_iterations(
stopping_iteration)
except:
pass
this_loglikelihood = output.write_loglikelihood(self, nmultilayer, loglikelihood[-1])
# pprint(' - Memory usage: %s (Mb)' % mem_usage())
if (len(self.MultiLayer) == 1):
# pprint('2.2.3.7 - Test Error', end='')
output.write_online_results(self, config, dataset, this_loglikelihood)
# pprint(' - Memory usage: %s (Mb)' % mem_usage())
# pprint('2.2.3.4 - Visualize Weights', end='')
output.visualize_all_weights(self,nmultilayer,config)
# pprint(' - Memory usage: %s (Mb)' % mem_usage())
# pprint('2.2.3.5 - Convergence 2', end='')
output.conv_post(self.MultiLayer[nmultilayer])
# pprint(' - Memory usage: %s (Mb)' % mem_usage())
# pprint('2.2.3.6 - Visualize Convergence', end='')
output.visualize_convergence(self,nmultilayer)
# pprint(' - Memory usage: %s (Mb)' % mem_usage())
if STOP:
break
pprint('') #linebreak
# Copyright (C) 2015, Dennis Forster <*****@*****.**>
#
# LICENSE: THE SOFTWARE IS PROVIDED "AS IS" UNDER THE
# ACADEMIC FREE LICENSE (AFL) v3.0.
#
"""Module docstring.
"""
from utils.sysfunctions import mem_usage
from utils.parallel import pprint
pprint('Start (RAM Usage: %0.2f MB)' % mem_usage())
# system imports
pprint('0.0.0 - System Imports', end='')
import sys
from mpi4py import MPI
pprint(' (%0.2f MB)' % mem_usage())
# custom imports
pprint('0.0.1 - Custom Imports')
from classes.model.Model import Model
from classes.input.DataSet import DataSet
from classes.config.Config import Config
from classes.output.Output import Output
# MPI definitions
pprint('0.1 - MPI Definitions', end='')
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
# Copyright (C) 2015, Dennis Forster <*****@*****.**>
#
# LICENSE: THE SOFTWARE IS PROVIDED "AS IS" UNDER THE
# ACADEMIC FREE LICENSE (AFL) v3.0.
#
"""Module docstring.
"""
from utils.sysfunctions import mem_usage
from utils.parallel import pprint
pprint('Start (RAM Usage: %0.2f MB)' % mem_usage())
# system imports
pprint('0.0.0 - System Imports', end='')
import sys
from mpi4py import MPI
pprint(' (%0.2f MB)' % mem_usage())
# custom imports
pprint('0.0.1 - Custom Imports')
from classes.model.Model import Model
from classes.input.DataSet import DataSet
from classes.config.Config import Config
from classes.output.Output import Output
# MPI definitions
pprint('0.1 - MPI Definitions', end='')
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
def train(self, nmultilayer, output, config, dataset):
ml = self.MultiLayer[nmultilayer]
ml.next_run()
pprint('%d.%d.2.1 - Visualize Weights' %
(self._run + 1, nmultilayer + 1),
end='')
output.visualize_all_weights(self, nmultilayer, config)
pprint(' (%0.2f MB)' % mem_usage())
# if (len(self.MultiLayer) == 1):
# TODO: for greedy model: output Likelihood of up to the
# current MultiLayer
pprint('%d.%d.2.2 - LogLikelihood' % (self._run + 1, nmultilayer + 1),
end='')
output.write_loglikelihood(self, nmultilayer)
pprint(' (%0.2f MB)' % mem_usage())
# variables for stopping criterion
# TODO: generalize for all MultiLayers
try:
STOPPING_CRITERION = config.get()\
['model']['MultiLayer1']['StoppingCriterion']
except:
STOPPING_CRITERION = False
if STOPPING_CRITERION:
try:
mvngwidth = int(config.get()\
['model']['MultiLayer1']['MovingWidth'])
except:
pprint('WARNING (model.Model::Train): No width for \
moving average was given. It will be set to %d' % 20)
mvngwidth = 20
loglikelihood = np.asarray([], dtype=np.float32)
mvng_avg, mvng_std = 0., 0.
max_mvng_avg = float('-inf')
last_weights = []
else:
loglikelihood = np.asarray([None])
STOP = False
MPI.COMM_WORLD.Barrier()
pprint('%d.%d.2.3 - Training Iterations' %
(self._run + 1, nmultilayer + 1))
for niteration in xrange(ml.get_iterations()):
pprint('Iteration: %*d' % (int(math.log10(ml.get_iterations())) +
1, ml.get_iteration() + 1),
end='')
# pprint('2.2.3.1 - Convergence', end='')
output.conv_pre(ml)
# pprint(' - Memory usage: %s (Mb)' % mem_usage())
MPI.COMM_WORLD.Barrier()
# pprint('2.2.3.2 - Learning Iteration', end='')
ml.learning_iteration(self._theanoscan)
# pprint(' - Memory usage: %s (Mb)\n' % mem_usage())
MPI.COMM_WORLD.Barrier()
# experimental: variing learning rates
# if ((niteration % 1 == 0) and
# (ml.Layer[1]._epsilon >= 0.0008)):
# ml.Layer[1]._epsilon *= 0.98
# MPI.COMM_WORLD.Barrier()
# to save learned weights every iteration
# output.save_weights(self.MultiLayer[nmultilayer])
# to save posterior distribution of training data every iteration
# if nmultilayer == self.number_of_multilayers()-1:
# output.save_posterior(self, config, dataset)
# Stopping criterion
if (len(self.MultiLayer) == 1):
# TODO: for greedy model: output Likelihood of up to the
# current MultiLayer
if STOPPING_CRITERION:
t0 = time.time()
loglikelihood = np.append(loglikelihood,
self.loglikelihood())
if (niteration >= mvngwidth):
# save only the last #mvngwidth values
loglikelihood = loglikelihood[1:]
pprint(' | Log-Likelihood: %f (%f s)' %
(loglikelihood[-1], time.time() - t0),
end='')
if STOPPING_CRITERION:
# save only the last #mvngwidht/2-1 weights
# for centered moving average
last_weights.append([
ml.Layer[nlayer].get_weights()
for nlayer in xrange(1, len(ml.Layer))
])
if (niteration > mvngwidth / 2):
last_weights = last_weights[1:]
# calculate moving average over last #mvngwidth values
if (niteration >= mvngwidth - 1):
mvng_avg = np.mean(loglikelihood)
mvng_std = np.std(loglikelihood)
pprint(' | Moving Average (%d): %f +- %f' %
((niteration + 1 - mvngwidth / 2), mvng_avg,
mvng_std),
end='')
if (mvng_avg > max_mvng_avg):
max_mvng_avg = mvng_avg
elif (mvng_avg < max_mvng_avg - mvng_std):
# if the moving average drops below the maximum
# moving average by more than the moving standard
# deviation, stop the learning iteration and revert
# back to the point of the centered moving average
for nlayer in xrange(1, len(ml.Layer)):
ml.Layer[nlayer].set_weights(
last_weights[0][nlayer - 1])
stopping_iteration = niteration + 1 - mvngwidth / 2
pprint(
'\nStopping criterion met at iteration %d.' %
stopping_iteration)
STOP = True
ml.set_iteration(stopping_iteration)
ml.set_iterations(stopping_iteration)
# abort on numerical error (nan in any of the weights)
if any([
np.any(np.isnan(ml.Layer[nlayer].get_weights()))
for nlayer in xrange(1, len(ml.Layer))
]):
pprint('\nNumerical error: try to decrease learning ' +
'rate and/or mini-batch size.')
STOP = True
try:
ml.set_iteration(stopping_iteration)
ml.set_iterations(stopping_iteration)
except:
pass
this_loglikelihood = output.write_loglikelihood(
self, nmultilayer, loglikelihood[-1])
# pprint(' - Memory usage: %s (Mb)' % mem_usage())
if (len(self.MultiLayer) == 1):
# pprint('2.2.3.7 - Test Error', end='')
output.write_online_results(self, config, dataset,
this_loglikelihood)
# pprint(' - Memory usage: %s (Mb)' % mem_usage())
# pprint('2.2.3.4 - Visualize Weights', end='')
output.visualize_all_weights(self, nmultilayer, config)
# pprint(' - Memory usage: %s (Mb)' % mem_usage())
# pprint('2.2.3.5 - Convergence 2', end='')
output.conv_post(self.MultiLayer[nmultilayer])
# pprint(' - Memory usage: %s (Mb)' % mem_usage())
# pprint('2.2.3.6 - Visualize Convergence', end='')
output.visualize_convergence(self, nmultilayer)
# pprint(' - Memory usage: %s (Mb)' % mem_usage())
if STOP:
break
pprint('') #linebreak
