def train(self, func):
for layer_index in xrange(self.layer_index,
self.last_layer_to_train + 1):
self.loss = sys.float_info.max
for epoch in xrange(self.iter, self.max_iters):
filename, loss = func(layer_index, epoch)
filename = os.path.join(self.wdir, filename)
save(self.model, filename)
if loss < self.loss:
self.loss = loss
self.mlp_best = filename
self.mlp_crrnt = filename
#self.logger.info("L%d ITER %02d: %s" % (layer_index, epoch, str(info)))
self.logger.info("L%d ITER %02d: best-of-this-layer: %s" %
(layer_index, epoch, self.mlp_best))
if epoch != self.max_iters - 1:
self.layer_index = layer_index
self.iter = epoch + 1
self._save_state()
self.iter = 0
self.layer_index = layer_index + 1
self._save_state()
# select the best network
if self.mlp_best == self.mlp_init:
self.logger.critical("Error training neural network...")
#sys.exit(1)
output_file = os.path.join(self.wdir, self.output_file)
shutil.copy2(self.mlp_best, output_file)
self.logger.info("Succeeded training: " + output_file)
def train(self, func):
for layer_index in xrange(self.layer_index, self.last_layer_to_train+1):
self.loss = sys.float_info.max
for epoch in xrange(self.iter, self.max_iters):
filename, loss = func(layer_index, epoch)
filename = os.path.join(self.wdir, filename)
save(self.model, filename)
if loss < self.loss:
self.loss = loss
self.mlp_best = filename
self.mlp_crrnt = filename
#self.logger.info("L%d ITER %02d: %s" % (layer_index, epoch, str(info)))
self.logger.info("L%d ITER %02d: best-of-this-layer: %s" % (layer_index, epoch, self.mlp_best))
if epoch != self.max_iters - 1:
self.layer_index = layer_index
self.iter = epoch + 1
self._save_state()
self.iter = 0
self.layer_index = layer_index + 1
self._save_state()
# select the best network
if self.mlp_best == self.mlp_init:
self.logger.critical("Error training neural network...")
#sys.exit(1)
output_file = os.path.join(self.wdir, self.output_file)
shutil.copy2(self.mlp_best, output_file)
self.logger.info("Succeeded training: " + output_file)
def train(self):
if not self.resume:
# cross-validation on original network
if True:
valid_error, loss = self._validate()
assert(_isnum(loss))
else:
valid_error, loss = (9999999,9999999)
self.logger.info("ITER %02d: CROSSVAL PRERUN ERROR %.4f AVG.LOSS %.4f" % (self.iter, 100*valid_error, loss))
self.loss = loss
while not self.finished():
load(self.model, filename = self.mlp_best, gradients=self.wasAccepted)
if self.iter >= self.momentum_start:
moment = self.momentum
else:
moment = 0
if False:
tr_error, tr_loss = 0, 0
else:
tr_error, tr_loss = self._train(learning_rate=self.rate, momentum=moment)
assert(_isnum(tr_loss))
self.logger.info("ITER %02d: TRAIN ERROR %03.4f AVG.LOSS %.4f lrate %.6g" % (self.iter, 100*tr_error, tr_loss, self.rate))
# cross-validation
valid_error, valid_loss = self._validate()
self.logger.info("ITER %02d: CROSSVAL ERROR %.4f AVG.LOSS %.4f lrate %.6f" % (self.iter, 100*valid_error, valid_loss, self.rate))
mlp_next = "%s/it%02drate%4f_Terr%.4f_Tloss%4f_CVerr%4f_CVloss%4f" % (self.wdir, self.iter, self.rate, 100*tr_error, tr_loss, 100*valid_error, valid_loss)
save(self.model, filename = mlp_next+".tmp")
if self.halving_criteria == "loss":
loss_new = valid_loss
elif self.halving_criteria == "frame_err":
loss_new = valid_error
else:
raise Exception("bad halving_criteria")
if self.accepted(loss_new, mlp_next):
os.rename(mlp_next+".tmp", mlp_next)
self.logger.info("accepted")
else:
os.rename(mlp_next+".tmp", mlp_next+"_rejected")
self.logger.info("rejected")
self.finalize(loss_new)
# select the best network
if self.mlp_best == self.mlp_init:
self.logger.critical("Error training neural network...")
#sys.exit(1)
output_file = os.path.join(self.wdir, self.output_file)
shutil.copy2(self.mlp_best, output_file)
self.logger.info("Succeeded training: " + output_file)
def __init__(self, arguments, model, train_fn, valid_fn,
train_sets, valid_sets):
self.logger = logging.getLogger(__name__)
self.model = model
self.train_fn = train_fn
self.valid_fn = valid_fn
self.train_sets = train_sets
self.valid_sets = valid_sets
#################### parse configs ####################
self.resume = False
if "resume" in arguments:
self.resume = utils.to_bool(arguments["resume"])
self.wdir = arguments["wdir"]
self.output_file = arguments["output_file"]
self.learn_rate = 0.1
if "learn_rate" in arguments:
self.learn_rate = float(arguments["learn_rate"])
self.halving_factor = 0.5
if "halving_factor" in arguments:
self.halving_factor = float(arguments["halving_factor"])
self.max_iters = 20
if "max_iters" in arguments:
self.max_iters = int(arguments["max_iters"])
self.min_iters = 0
if "min_iters" in arguments:
self.min_iters = int(arguments["min_iters"])
self.keep_lr_iters = 15
if "keep_lr_iters" in arguments:
self.keep_lr_iters = int(arguments["keep_lr_iters"])
self.start_halving_impr=0.01
if "start_halving_impr" in arguments:
self.start_halving_impr = float(arguments["start_halving_impr"])
self.end_halving_impr=0.001
if "end_halving_impr" in arguments:
self.end_halving_impr = float(arguments["end_halving_impr"])
self.continue_with_rate = False
if "continue_with_rate" in arguments:
self.continue_with_rate = utils.to_bool(arguments["continue_with_rate"])
self.halving_criteria = "loss"
if "halving_criteria" in arguments:
self.halving_criteria = arguments["halving_criteria"]
criteria_list = ["loss", "frame_err"]
if self.halving_criteria not in criteria_list:
raise Exception("invalid halving criteria. must be one of " + str(criteria_list))
# batch_size and momentum
self.batch_size=256
if arguments.has_key('batch_size'):
self.batch_size = int(arguments['batch_size'])
self.momentum=0.5
self.momentum_start = 1
if arguments.has_key('momentum'):
self.momentum = float(arguments['momentum'])
if 'momentum_start' in arguments:
self.momentum_start = int(arguments['momentum_start'])
# other stuff
if self.resume:
if not os.path.exists(self.wdir):
raise Exception("wdir must exist if resume=True")
else:
if not os.path.exists(self.wdir):
os.makedirs(self.wdir)
else:
self.logger.info("Directory already exists...")
out = StringIO.StringIO()
print >>out, "\n********** Trainer **********"
print >>out, "resume", self.resume
print >>out, "wdir", self.wdir
print >>out, "output_file", self.output_file
print >>out, "learn_rate", self.learn_rate
print >>out, "halving_factor", self.halving_factor
print >>out, "max_iters", self.max_iters
print >>out, "min_iters", self.min_iters
print >>out, "keep_lr_iters", self.keep_lr_iters
print >>out, "start_halving_impr", self.start_halving_impr
print >>out, "end_halving_impr", self.end_halving_impr
print >>out, "continue_with_rate", self.continue_with_rate
print >>out, "halving_criteria", self.halving_criteria
print >>out, "batch_size", self.batch_size
print >>out, "momentum", self.momentum
print >>out, "momentum_start", self.momentum_start
self.logger.info(out.getvalue())
self.mlp_init = self.wdir + "/mlp_init"
if not self.resume: # brand new
save(self.model, self.mlp_init)
# runtime state
self.iter = 0
self.done = False
self.loss = sys.float_info.max
self.rate = self.learn_rate
self.mlp_best = self.mlp_init
self.halving = False
self.wasAccepted = True
if self.resume:
if os.path.isfile(self.wdir+"/trainer_state"):
self._load_state()
def __init__(self, arguments, model, train_sets):
self.logger = logging.getLogger(__name__)
self.model = model
self.train_sets = train_sets
#################### parse configs ####################
self.resume = False
if "resume" in arguments:
self.resume = utils.to_bool(arguments["resume"])
self.wdir = arguments["wdir"]
self.output_file = arguments["output_file"]
self.max_iters = 20
if "max_iters" in arguments:
self.max_iters = int(arguments["max_iters"])
#self.max_iters_without_impr = 3
#if "max_iters_without_impr" in arguments:
# self.max_iters_without_impr = int(arguments["max_iters_without_impr"])
self.first_layer_to_train = 0
if "first_layer_to_train" in arguments:
self.first_layer_to_train = int(arguments["first_layer_to_train"])
self.last_layer_to_train = model.n_layers - 1 # number hidden layers - 1
if "last_layer_to_train" in arguments:
self.last_layer_to_train = int(arguments["last_layer_to_train"])
# other stuff
if self.resume:
if not os.path.exists(self.wdir):
raise Exception("wdir must exist if resume=True")
else:
if not os.path.exists(self.wdir):
os.makedirs(self.wdir)
else:
self.logger.info("Directory already exists...")
out = StringIO.StringIO()
print >> out, "\n********** LayerwiseTrainer **********"
print >> out, "resume", self.resume
print >> out, "wdir", self.wdir
print >> out, "output_file", self.output_file
print >> out, "max_iters", self.max_iters
print >> out, "first_layer_to_train", self.first_layer_to_train
print >> out, "last_layer_to_train", self.last_layer_to_train
self.logger.info(out.getvalue())
self.mlp_init = self.wdir + "/mlp_init"
if not self.resume: # brand new
save(self.model, self.mlp_init)
# runtime state
self.layer_index = self.first_layer_to_train
self.iter = 0
self.loss = sys.float_info.max
self.mlp_best = self.mlp_init
self.mlp_crrnt = self.mlp_init
self.iters_without_impr = 0
if self.resume:
if os.path.isfile(self.wdir + "/layerwisetrainer_state"):
self._load_state()
def __init__(self, arguments, model, train_fn, valid_fn, train_sets,
valid_sets):
self.logger = logging.getLogger(__name__)
self.model = model
self.train_fn = train_fn
self.valid_fn = valid_fn
self.train_sets = train_sets
self.valid_sets = valid_sets
#################### parse configs ####################
self.resume = False
if "resume" in arguments:
self.resume = utils.to_bool(arguments["resume"])
self.wdir = arguments["wdir"]
self.output_file = arguments["output_file"]
self.learn_rate = 0.1
if "learn_rate" in arguments:
self.learn_rate = float(arguments["learn_rate"])
self.halving_factor = 0.5
if "halving_factor" in arguments:
self.halving_factor = float(arguments["halving_factor"])
self.max_iters = 20
if "max_iters" in arguments:
self.max_iters = int(arguments["max_iters"])
self.min_iters = 0
if "min_iters" in arguments:
self.min_iters = int(arguments["min_iters"])
self.keep_lr_iters = 15
if "keep_lr_iters" in arguments:
self.keep_lr_iters = int(arguments["keep_lr_iters"])
self.start_halving_impr = 0.01
if "start_halving_impr" in arguments:
self.start_halving_impr = float(arguments["start_halving_impr"])
self.end_halving_impr = 0.001
if "end_halving_impr" in arguments:
self.end_halving_impr = float(arguments["end_halving_impr"])
self.continue_with_rate = False
if "continue_with_rate" in arguments:
self.continue_with_rate = utils.to_bool(
arguments["continue_with_rate"])
self.halving_criteria = "loss"
if "halving_criteria" in arguments:
self.halving_criteria = arguments["halving_criteria"]
criteria_list = ["loss", "frame_err"]
if self.halving_criteria not in criteria_list:
raise Exception("invalid halving criteria. must be one of " +
str(criteria_list))
# batch_size and momentum
self.batch_size = 256
if arguments.has_key('batch_size'):
self.batch_size = int(arguments['batch_size'])
self.momentum = 0.5
self.momentum_start = 1
if arguments.has_key('momentum'):
self.momentum = float(arguments['momentum'])
if 'momentum_start' in arguments:
self.momentum_start = int(arguments['momentum_start'])
# other stuff
if self.resume:
if not os.path.exists(self.wdir):
raise Exception("wdir must exist if resume=True")
else:
if not os.path.exists(self.wdir):
os.makedirs(self.wdir)
else:
self.logger.info("Directory already exists...")
out = StringIO.StringIO()
print >> out, "\n********** Trainer **********"
print >> out, "resume", self.resume
print >> out, "wdir", self.wdir
print >> out, "output_file", self.output_file
print >> out, "learn_rate", self.learn_rate
print >> out, "halving_factor", self.halving_factor
print >> out, "max_iters", self.max_iters
print >> out, "min_iters", self.min_iters
print >> out, "keep_lr_iters", self.keep_lr_iters
print >> out, "start_halving_impr", self.start_halving_impr
print >> out, "end_halving_impr", self.end_halving_impr
print >> out, "continue_with_rate", self.continue_with_rate
print >> out, "halving_criteria", self.halving_criteria
print >> out, "batch_size", self.batch_size
print >> out, "momentum", self.momentum
print >> out, "momentum_start", self.momentum_start
self.logger.info(out.getvalue())
self.mlp_init = self.wdir + "/mlp_init"
if not self.resume: # brand new
save(self.model, self.mlp_init)
# runtime state
self.iter = 0
self.done = False
self.loss = sys.float_info.max
self.rate = self.learn_rate
self.mlp_best = self.mlp_init
self.halving = False
self.wasAccepted = True
if self.resume:
if os.path.isfile(self.wdir + "/trainer_state"):
self._load_state()
def train(self):
if not self.resume:
# cross-validation on original network
if True:
valid_error, loss = self._validate()
assert (_isnum(loss))
else:
valid_error, loss = (9999999, 9999999)
self.logger.info(
"ITER %02d: CROSSVAL PRERUN ERROR %.4f AVG.LOSS %.4f" %
(self.iter, 100 * valid_error, loss))
self.loss = loss
while not self.finished():
load(self.model,
filename=self.mlp_best,
gradients=self.wasAccepted)
if self.iter >= self.momentum_start:
moment = self.momentum
else:
moment = 0
if False:
tr_error, tr_loss = 0, 0
else:
tr_error, tr_loss = self._train(learning_rate=self.rate,
momentum=moment)
assert (_isnum(tr_loss))
self.logger.info(
"ITER %02d: TRAIN ERROR %03.4f AVG.LOSS %.4f lrate %.6g" %
(self.iter, 100 * tr_error, tr_loss, self.rate))
# cross-validation
valid_error, valid_loss = self._validate()
self.logger.info(
"ITER %02d: CROSSVAL ERROR %.4f AVG.LOSS %.4f lrate %.6f" %
(self.iter, 100 * valid_error, valid_loss, self.rate))
mlp_next = "%s/it%02drate%4f_Terr%.4f_Tloss%4f_CVerr%4f_CVloss%4f" % (
self.wdir, self.iter, self.rate, 100 * tr_error, tr_loss,
100 * valid_error, valid_loss)
save(self.model, filename=mlp_next + ".tmp")
if self.halving_criteria == "loss":
loss_new = valid_loss
elif self.halving_criteria == "frame_err":
loss_new = valid_error
else:
raise Exception("bad halving_criteria")
if self.accepted(loss_new, mlp_next):
os.rename(mlp_next + ".tmp", mlp_next)
self.logger.info("accepted")
else:
os.rename(mlp_next + ".tmp", mlp_next + "_rejected")
self.logger.info("rejected")
self.finalize(loss_new)
# select the best network
if self.mlp_best == self.mlp_init:
self.logger.critical("Error training neural network...")
#sys.exit(1)
output_file = os.path.join(self.wdir, self.output_file)
shutil.copy2(self.mlp_best, output_file)
self.logger.info("Succeeded training: " + output_file)
def __init__(self, arguments, model, train_sets):
self.logger = logging.getLogger(__name__)
self.model = model
self.train_sets = train_sets
#################### parse configs ####################
self.resume = False
if "resume" in arguments:
self.resume = utils.to_bool(arguments["resume"])
self.wdir = arguments["wdir"]
self.output_file = arguments["output_file"]
self.max_iters = 20
if "max_iters" in arguments:
self.max_iters = int(arguments["max_iters"])
#self.max_iters_without_impr = 3
#if "max_iters_without_impr" in arguments:
# self.max_iters_without_impr = int(arguments["max_iters_without_impr"])
self.first_layer_to_train = 0
if "first_layer_to_train" in arguments:
self.first_layer_to_train = int(arguments["first_layer_to_train"])
self.last_layer_to_train = model.n_layers - 1 # number hidden layers - 1
if "last_layer_to_train" in arguments:
self.last_layer_to_train = int(arguments["last_layer_to_train"])
# other stuff
if self.resume:
if not os.path.exists(self.wdir):
raise Exception("wdir must exist if resume=True")
else:
if not os.path.exists(self.wdir):
os.makedirs(self.wdir)
else:
self.logger.info("Directory already exists...")
out = StringIO.StringIO()
print >>out, "\n********** LayerwiseTrainer **********"
print >>out, "resume", self.resume
print >>out, "wdir", self.wdir
print >>out, "output_file", self.output_file
print >>out, "max_iters", self.max_iters
print >>out, "first_layer_to_train", self.first_layer_to_train
print >>out, "last_layer_to_train", self.last_layer_to_train
self.logger.info(out.getvalue())
self.mlp_init = self.wdir + "/mlp_init"
if not self.resume: # brand new
save(self.model, self.mlp_init)
# runtime state
self.layer_index = self.first_layer_to_train
self.iter = 0
self.loss = sys.float_info.max
self.mlp_best = self.mlp_init
self.mlp_crrnt = self.mlp_init
self.iters_without_impr = 0
if self.resume:
if os.path.isfile(self.wdir+"/layerwisetrainer_state"):
self._load_state()