def train(hyper):
apollo.set_random_seed(hyper['random_seed'])
if hyper['gpu'] is None:
apollo.set_mode_cpu()
logging.info('Using cpu device (pass --gpu X to train on the gpu)')
else:
apollo.set_mode_gpu()
apollo.set_device(hyper['gpu'])
logging.info('Using gpu device %d' % hyper['gpu'])
apollo.set_logging_verbosity(hyper['loglevel'])
net = apollo.Net()
forward(net, hyper)
network_path = '%s/network.jpg' % hyper['schematic_prefix']
net.draw_to_file(network_path)
logging.info('Drawing network to %s' % network_path)
net.reset_forward()
if 'weights' in hyper:
logging.info('Loading weights from %s' % hyper['weights'])
net.load(hyper['weights'])
train_loss_hist = []
for i in xrange(hyper['start_iter'], hyper['max_iter']):
train_loss_hist.append(forward(net, hyper))
net.backward()
lr = (hyper['base_lr'] * hyper['gamma']**(i // hyper['stepsize']))
net.update(lr=lr,
momentum=hyper['momentum'],
clip_gradients=hyper['clip_gradients'])
if i % hyper['display_interval'] == 0:
logging.info(
'Iteration %d: %s' %
(i, np.mean(train_loss_hist[-hyper['display_interval']:])))
if i % hyper['snapshot_interval'] == 0 and i > hyper['start_iter']:
filename = '%s/%d.h5' % (hyper['snapshot_prefix'], i)
logging.info('Saving net to: %s' % filename)
net.save(filename)
if i % hyper['graph_interval'] == 0 and i > hyper['start_iter']:
sub = hyper.get('sub', 100)
plt.plot(
np.convolve(train_loss_hist,
np.ones(sub) / sub)[sub:-sub])
filename = '%s/train_loss.jpg' % hyper['graph_prefix']
logging.info('Saving figure to: %s' % filename)
plt.savefig(filename)
def train(hyper):
apollo.set_random_seed(hyper['random_seed'])
if hyper['gpu'] is None:
apollo.set_mode_cpu()
logging.info('Using cpu device (pass --gpu X to train on the gpu)')
else:
apollo.set_mode_gpu()
apollo.set_device(hyper['gpu'])
logging.info('Using gpu device %d' % hyper['gpu'])
apollo.set_logging_verbosity(hyper['loglevel'])
net = apollo.Net()
forward(net, hyper)
network_path = '%s/network.jpg' % hyper['schematic_prefix']
net.draw_to_file(network_path)
logging.info('Drawing network to %s' % network_path)
net.reset_forward()
if 'weights' in hyper:
logging.info('Loading weights from %s' % hyper['weights'])
net.load(hyper['weights'])
train_loss_hist = []
for i in xrange(hyper['start_iter'], hyper['max_iter']):
train_loss_hist.append(forward(net, hyper))
net.backward()
lr = (hyper['base_lr'] * hyper['gamma']**(i // hyper['stepsize']))
net.update(lr=lr, momentum=hyper['momentum'],
clip_gradients=hyper['clip_gradients'])
if i % hyper['display_interval'] == 0:
logging.info('Iteration %d: %s' % (i, np.mean(train_loss_hist[-hyper['display_interval']:])))
if i % hyper['snapshot_interval'] == 0 and i > hyper['start_iter']:
filename = '%s/%d.h5' % (hyper['snapshot_prefix'], i)
logging.info('Saving net to: %s' % filename)
net.save(filename)
if i % hyper['graph_interval'] == 0 and i > hyper['start_iter']:
sub = hyper.get('sub', 100)
plt.plot(np.convolve(train_loss_hist, np.ones(sub)/sub)[sub:-sub])
filename = '%s/train_loss.jpg' % hyper['graph_prefix']
logging.info('Saving figure to: %s' % filename)
plt.savefig(filename)
def default_train(hyper, forward, test_forward=None):
if test_forward is None:
test_forward = forward
import matplotlib
matplotlib.use("Agg", warn=False)
import matplotlib.pyplot as plt
d = default_hyper()
d.update(hyper)
hyper = d
validate_hyper(hyper)
apollo.set_random_seed(hyper["random_seed"])
if hyper["gpu"] is None:
apollo.set_mode_cpu()
logging.info("Using cpu device (pass --gpu X to train on the gpu)")
else:
apollo.set_mode_gpu()
apollo.set_device(hyper["gpu"])
logging.info("Using gpu device %d" % hyper["gpu"])
apollo.set_logging_verbosity(hyper["loglevel"])
net = apollo.Net()
forward(net, hyper)
network_path = "%s/network.jpg" % hyper["schematic_prefix"]
net.draw_to_file(network_path)
logging.info("Drawing network to %s" % network_path)
net.reset_forward()
if hyper.get("separate_test_net", True) == True:
test_net = apollo.Net()
test_forward(test_net, hyper)
test_net.reset_forward()
else:
test_net = net
if "weights" in hyper:
logging.info("Loading weights from %s" % weights)
net.load(hyper["weights"])
train_loss_hist = []
for i in xrange(hyper["start_iter"], hyper["max_iter"]):
train_loss_hist.append(forward(net, hyper))
net.backward()
lr = hyper["base_lr"] * hyper.get("gamma", 1.0) ** (i // hyper.get("stepsize", sys.maxint))
net.update(
lr=lr,
momentum=hyper.get("momentum", 0.0),
clip_gradients=hyper.get("clip_gradients", -1),
weight_decay=hyper.get("weight_decay", 0.0),
)
if i % hyper["display_interval"] == 0:
logging.info("Iteration %d: %s" % (i, np.mean(train_loss_hist[-hyper["display_interval"] :])))
if i % hyper["snapshot_interval"] == 0 and i > hyper["start_iter"]:
filename = "%s/%d.h5" % (hyper["snapshot_prefix"], i)
logging.info("Saving net to: %s" % filename)
net.save(filename)
if i % hyper["graph_interval"] == 0 and i > hyper["start_iter"]:
sub = hyper.get("sub", 100)
plt.plot(np.convolve(train_loss_hist, np.ones(sub) / sub)[sub:-sub])
filename = "%s/train_loss.jpg" % hyper["graph_prefix"]
logging.info("Saving figure to: %s" % filename)
plt.savefig(filename)
if hyper["test_interval"] is not None and i % hyper["test_interval"] == 0:
test_loss = []
accuracy = []
test_net.phase = "test"
test_net.copy_params_from(net)
for j in xrange(hyper["test_iter"]):
test_loss.append(test_forward(test_net, hyper))
test_net.reset_forward()
if "accuracy" in test_net.tops:
accuracy.append(test_net.tops["accuracy"].data.flatten()[0])
if len(accuracy) > 0:
logging.info("Accuracy: %.5f" % np.mean(accuracy))
logging.info("Test loss: %f" % np.mean(test_loss))
test_net.phase = "train"
def default_train(hyper, forward, test_forward=None):
if test_forward is None:
test_forward=forward
import matplotlib; matplotlib.use('Agg', warn=False); import matplotlib.pyplot as plt
d = default_hyper()
d.update(hyper)
hyper = d
validate_hyper(hyper)
apollo.set_random_seed(hyper['random_seed'])
if hyper['gpu'] is None:
apollo.set_mode_cpu()
logging.info('Using cpu device (pass --gpu X to train on the gpu)')
else:
apollo.set_mode_gpu()
apollo.set_device(hyper['gpu'])
logging.info('Using gpu device %d' % hyper['gpu'])
apollo.set_logging_verbosity(hyper['loglevel'])
net = apollo.Net()
forward(net, hyper)
network_path = '%s/network.jpg' % hyper['schematic_prefix']
net.draw_to_file(network_path)
logging.info('Drawing network to %s' % network_path)
net.reset_forward()
if hyper.get('separate_test_net', True) == True:
test_net = apollo.Net()
test_forward(test_net, hyper)
test_net.reset_forward()
else:
test_net = net
if 'weights' in hyper:
logging.info('Loading weights from %s' % hyper['weights'])
net.load(hyper['weights'])
train_loss_hist = []
for i in xrange(hyper['start_iter'], hyper['max_iter']):
train_loss_hist.append(forward(net, hyper))
net.backward()
lr = (hyper['base_lr'] * hyper.get('gamma', 1.)**(i // hyper.get('stepsize', sys.maxint)))
net.update(lr=lr, momentum=hyper.get('momentum', 0.0),
clip_gradients=hyper.get('clip_gradients', -1), weight_decay=hyper.get('weight_decay', 0.0))
if i % hyper['display_interval'] == 0:
logging.info('Iteration %d: %s' % (i, np.mean(train_loss_hist[-hyper['display_interval']:])))
if i % hyper['snapshot_interval'] == 0 and i > hyper['start_iter']:
filename = '%s/%d.h5' % (hyper['snapshot_prefix'], i)
logging.info('Saving net to: %s' % filename)
net.save(filename)
if i % hyper['graph_interval'] == 0 and i > hyper['start_iter']:
sub = hyper.get('sub', 100)
plt.plot(np.convolve(train_loss_hist, np.ones(sub)/sub)[sub:-sub])
filename = '%s/train_loss.jpg' % hyper['graph_prefix']
logging.info('Saving figure to: %s' % filename)
plt.savefig(filename)
if hyper['test_interval'] is not None and i % hyper['test_interval'] == 0:
test_loss = []
accuracy = []
test_net.phase = 'test'
test_net.copy_params_from(net)
for j in xrange(hyper['test_iter']):
test_loss.append(test_forward(test_net, hyper))
test_net.reset_forward()
if 'accuracy' in test_net.tops:
accuracy.append(test_net.tops['accuracy'].data.flatten()[0])
if len(accuracy) > 0:
logging.info('Accuracy: %.5f' % np.mean(accuracy))
logging.info('Test loss: %f' % np.mean(test_loss))
test_net.phase = 'train'