def main():
config_file = open('./config.json')
config = json.load(config_file,
object_hook=lambda d: namedtuple('x', d.keys())
(*d.values()))
num_unrolls = config.num_steps // config.unroll_length
with tf.Session() as sess:
model = util.load_model(sess, config, logger)
all_y = []
for i in range(10):
print(i)
_, loss, reset, fx_array, x_array = model.step()
cost, others = util.run_epoch(sess, loss, [fx_array, x_array],
reset, num_unrolls)
Y, X = others
all_y.append(Y)
all_y = np.hstack(all_y)
np.save('srnn.npy', all_y)
plt.figure(1)
y_mean = np.mean(all_y, axis=1)
plt.plot(y_mean)
print(min(y_mean))
plt.show()
def main(_):
# Configuration.
num_unrolls = FLAGS.num_steps
if FLAGS.seed:
tf.set_random_seed(FLAGS.seed)
# Problem.
problem, net_config, net_assignments = util.get_config(
FLAGS.problem, FLAGS.path)
optimizer = meta.MetaOptimizer(**net_config)
meta_loss = optimizer.meta_loss(problem,
1,
net_assignments=net_assignments)
_, update, reset, cost_op, _ = meta_loss
with ms.MonitoredSession() as sess:
# Prevent accidental changes to the graph.
tf.get_default_graph().finalize()
total_time = 0
total_cost = 0
for _ in xrange(FLAGS.num_epochs):
# Training.
time, cost = util.run_epoch(sess, cost_op, [update], reset,
num_unrolls)
total_time += time
total_cost += cost
# Results.
util.print_stats("Epoch {}".format(FLAGS.num_epochs), total_cost,
total_time, FLAGS.num_epochs)
def main(_):
# Configuration.
num_unrolls = FLAGS.num_steps
if FLAGS.seed:
tf.set_random_seed(FLAGS.seed)
# Problem.
problem, net_config, net_assignments = util.get_config(
FLAGS.problem,
main_parade_path,
first_batch_parade_path,
path=FLAGS.path)
# Optimizer setup.
if FLAGS.optimizer == "Adam":
cost_op = problem()
problem_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
problem_reset = tf.variables_initializer(problem_vars)
optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
# optimizer = tf.train.GradientDescentOptimizer(FLAGS.learning_rate)
optimizer_reset = tf.variables_initializer(optimizer.get_slot_names())
grads_and_vars = optimizer.compute_gradients(cost_op)
grads, v = zip(*grads_and_vars)
grads, _ = tf.clip_by_global_norm(grads, 1.)
update = optimizer.apply_gradients(zip(grads, v))
# update = optimizer.minimize(cost_op)
reset = [problem_reset, optimizer_reset]
elif FLAGS.optimizer == "L2L":
if FLAGS.path is None:
logging.warning("Evaluating untrained L2L optimizer")
optimizer = meta.MetaOptimizer(**net_config)
meta_loss = optimizer.meta_loss(problem,
1,
net_assignments=net_assignments)
_, update, reset, cost_op, _ = meta_loss
else:
raise ValueError("{} is not a valid optimizer".format(FLAGS.optimizer))
with ms.MonitoredSession() as sess:
# Prevent accidental changes to the graph.
tf.get_default_graph().finalize()
total_time = 0
total_cost = 0
for i in xrange(FLAGS.num_epochs):
# Training.
time, cost = util.run_epoch(sess, cost_op, [update], reset,
num_unrolls)
total_time += time
total_cost += cost
# Results.
util.print_stats("Epoch {}".format(FLAGS.num_epochs), total_cost,
total_time, FLAGS.num_epochs)
def main():
config_file = open('./config.json')
config = json.load(config_file,
object_hook=lambda d: namedtuple('x', d.keys())
(*d.values()))
config_file.seek(0)
logger.info(str(json.load(config_file)))
config_file.close()
num_unrolls = config.num_steps // config.unroll_length
with tf.Session() as sess:
# tf.get_default_graph().finalize()
model = util.create_model(sess, config, logger)
step, loss, reset, fx_array, x_array = model.step()
best_evaluation = float('inf')
total_cost = 0
for e in range(config.num_epochs):
cost, _ = util.run_epoch(sess, loss, [step], reset, num_unrolls)
total_cost += cost
if (e + 1) % config.log_period == 0:
lm_e = total_cost / config.log_period
logger.info('Epoch {}, Mean Error: {:.3f}'.format(e, lm_e))
total_cost = 0
if (e + 1) % config.evaluation_period == 0:
eval_cost = 0
for _ in range(config.evaluation_epochs):
cost, _ = util.run_epoch(sess, loss, [
step,
], reset, num_unrolls)
eval_cost += cost
elm_e = eval_cost / config.evaluation_epochs
logger.info('EVALUATION, Mean Error: {:.3f}'.format(elm_e))
if config.save_path is not None and eval_cost < best_evaluation:
logger.info('Save current model ...')
model.saver.save(sess, config.save_path, global_step=e)
best_evaluation = eval_cost
def main(**kwargs):
config_name = kwargs.get("config_name", "config.json")
path = osp.join(osp.dirname(osp.realpath(__file__)), config_name)
config_file = open(path)
config = json.load(config_file,
object_hook=lambda d: namedtuple('x', d.keys())
(*d.values()))
config_file.seek(0)
logger.info(str(json.load(config_file)))
config_file.close()
num_unrolls = config.num_steps // config.unroll_length
with tf.Session() as sess:
# tf.get_default_graph().finalize()
model = util.create_model(sess, config, logger)
step, loss, reset, fx_array, x_array = model.step()
best_cost = [float('inf')] * 3
epoch_cost = 0
total_cost = 0
for e in range(config.num_epochs):
cost, _ = util.run_epoch(sess, loss, [step], reset, num_unrolls)
epoch_cost += cost
total_cost += cost
if (e + 1) % config.log_period == 0:
lm_e = epoch_cost / config.log_period
logger.info('Epoch {}, Mean Error: {:.3f}'.format(e, lm_e))
epoch_cost = 0
if (e + 1) % config.evaluation_period == 0:
elm_e = total_cost / config.evaluation_period
logger.info('Current {} epochs, Mean Error: {:.3f}'.format(
config.evaluation_period, elm_e))
mbc = max(best_cost)
if config.save_path is not None and total_cost < mbc:
save_path = osp.join(osp.dirname(osp.realpath(__file__)),
config.save_path)
best_cost.remove(mbc)
best_cost.append(total_cost)
logger.info('Save current model ...')
model.saver.save(sess, save_path, global_step=e)
total_cost = 0
def main():
config_file = open('./config.json')
config = json.load(config_file,
object_hook=lambda d:namedtuple('x', d.keys())(*d.values()))
num_unrolls = config.num_steps // config.unroll_length
with tf.Session() as sess:
model = util.load_model(sess, config, logger)
_, loss, reset, fx_array, x_array = model.step()
cost, others = util.run_epoch(sess, loss, [fx_array, x_array],
reset, num_unrolls)
Y, X = others
Yp = []
for i in range(config.batch_size):
arr = np.squeeze(Y[:, i])
if arr[-1] <= 0.3:
Yp.append(arr)
np.save('./scratch/gmm0.npy', np.array(Yp))
plt.figure(1)
plt.plot(np.squeeze(Y))
plt.show()
def main(_):
# Configuration.
num_unrolls = FLAGS.num_steps // FLAGS.unroll_length
# if FLAGS.save_path is not None:
# if os.path.exists(FLAGS.save_path):
# raise ValueError("Folder {} already exists".format(FLAGS.save_path))
# else:
# os.mkdir(FLAGS.save_path)
# Problem.
problem, net_config, net_assignments = util.get_config(
FLAGS.problem, main_parade_path, first_batch_parade_path)
# Optimizer setup.
optimizer = meta.MetaOptimizer(**net_config)
minimize = optimizer.meta_minimize(
problem, FLAGS.unroll_length,
learning_rate=FLAGS.learning_rate,
net_assignments=net_assignments,
second_derivatives=FLAGS.second_derivatives)
step, update, reset, cost_op, _ = minimize
with ms.MonitoredSession() as sess:
# Prevent accidental changes to the graph.
tf.get_default_graph().finalize()
writer = tf.summary.FileWriter('summary')
writer.add_graph(tf.get_default_graph())
best_evaluation = float("inf")
total_time = 0
total_cost = 0
for e in xrange(FLAGS.num_epochs):
# Training.
time, cost = util.run_epoch(sess, cost_op, [update, step], reset,
num_unrolls)
total_time += time
total_cost += cost
# Logging.
if (e + 1) % FLAGS.log_period == 0:
util.print_stats("Epoch {}".format(e + 1), total_cost, total_time,
FLAGS.log_period)
total_time = 0
total_cost = 0
# Evaluation.
if (e + 1) % FLAGS.evaluation_period == 0:
eval_cost = 0
eval_time = 0
for _ in xrange(FLAGS.evaluation_epochs):
time, cost = util.run_epoch(sess, cost_op, [update], reset,
num_unrolls)
eval_time += time
eval_cost += cost
util.print_stats("EVALUATION", eval_cost, eval_time,
FLAGS.evaluation_epochs)
if FLAGS.save_path is not None and eval_cost < best_evaluation:
print("Removing previously saved meta-optimizer")
for f in os.listdir(FLAGS.save_path):
os.remove(os.path.join(FLAGS.save_path, f))
print("Saving meta-optimizer to {}".format(FLAGS.save_path))
optimizer.save(sess, FLAGS.save_path)
best_evaluation = eval_cost
# use time batch len of 1 so that every target is covered
test_data = util.batch_data(pickle['test'], time_batch_len = 1,
max_time_batches = -1, softmax = True)
else:
raise Exception("Other datasets not yet implemented")
print config
with tf.Graph().as_default(), tf.Session() as session:
with tf.variable_scope("model", reuse=None):
test_model = model_class(config, training=False)
saver = tf.train.Saver(tf.all_variables())
model_path = os.path.join(os.path.dirname(args.config_file),
config.model_name)
saver.restore(session, model_path)
test_loss, test_probs = util.run_epoch(session, test_model, test_data,
training=False, testing=True)
print 'Testing Loss: {}'.format(test_loss)
if config.dataset == 'softmax':
if args.seperate:
nottingham_util.seperate_accuracy(test_probs, test_data, num_samples=args.num_samples)
else:
nottingham_util.accuracy(test_probs, test_data, num_samples=args.num_samples)
else:
util.accuracy(test_probs, test_data, num_samples=50)
sys.exit(1)
def main(_):
# Configuration.
num_unrolls = FLAGS.num_steps // FLAGS.unroll_length
# Problem.
problem, net_config, net_assignments = util.get_config(FLAGS.problem)
# Optimizer setup.
optimizer = meta.MetaOptimizer(**net_config)
minimize = optimizer.meta_minimize(
problem, FLAGS.unroll_length,
learning_rate=FLAGS.learning_rate,
net_assignments=net_assignments,
second_derivatives=FLAGS.second_derivatives)
step, loss, update, reset, cost_op, farray, lropt, _ = minimize
with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess:
# Prevent accidental changes to the graph.
graph_writer = tf.summary.FileWriter(logs_path, sess.graph)
sess.run(tf.global_variables_initializer())
best_evaluation = float("inf")
start = timer()
losstrain = []
lrtrain = []
losseval = []
plotlosstrain = []
plotlrtrain = []
plotlosseval = []
for e in range(FLAGS.num_epochs):
cost, trainloss, lropttrain = util.run_epoch(sess, cost_op, farray, lropt, [step, update], reset, num_unrolls)
print(cost)
losstrain.append(cost)
lrtrain.append(lropttrain)
util.print_stats("Training Epoch {}".format(e), trainloss, timer() - start)
saver = tf.train.Saver()
if (e + 1) % FLAGS.logging_period == 0:
plotlosstrain.append(cost)
plotlrtrain.append(lropttrain)
if (e + 1) % FLAGS.evaluation_period == 0:
for _ in range(FLAGS.evaluation_epochs):
evalcost, evaloss, _ = util.run_epoch(sess, cost_op, farray, lropt, [update], reset, num_unrolls)
losseval.append(evalcost)
if save_path is not None and evaloss < best_evaluation:
print("Saving meta-optimizer to {}".format(save_path))
saver.save(sess, save_path + '/model.ckpt', global_step=e + 1)
best_evaluation = evaloss
plotlosseval.append(evalcost)
slengths = np.arange(FLAGS.num_steps)
slengthlr = np.arange(FLAGS.num_steps - num_unrolls)
np.savetxt(save_path + '/plotlosstrain.out', plotlosstrain, delimiter=',')
np.savetxt(save_path + '/plotlrtrain.out', plotlrtrain, delimiter=',')
np.savetxt(save_path + '/plotlosseval.out', plotlosseval, delimiter=',')
np.savetxt(save_path + '/losstrain.out', losstrain, delimiter=',')
np.savetxt(save_path + '/lrtrain.out', plotlosstrain, delimiter=',')
np.savetxt(save_path + '/losseval.out', losseval, delimiter=',')
plt.figure(figsize=(8, 5))
plt.plot(slengths, np.mean(plotlosstrain, 0), 'r-', label='Training Loss')
plt.xlabel('Epoch')
plt.ylabel('Training Loss')
plt.legend()
savefig(save_path + '/Training.png')
plt.close()
plt.figure(figsize=(8, 5))
plt.plot(slengths, np.mean(plotlosseval, 0), 'b-', label='Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Validation Loss')
plt.legend()
savefig(save_path + '/Validation.png')
plt.close()
plt.figure(figsize=(8, 5))
plt.plot(slengthlr, np.mean(plotlrtrain, 0), 'r-', label='Learning Rate')
plt.xlabel('Epoch')
plt.ylabel('Average Learning Rate')
plt.legend()
savefig(save_path + '/LearningRate.png')
plt.close()
graph_writer.close()
def main(_):
# Configuration.
num_unrolls = FLAGS.num_steps
if FLAGS.seed:
tf.set_random_seed(FLAGS.seed)
# Problem.
problem, net_config, net_assignments = util.get_config(
FLAGS.problem, FLAGS.path, FLAGS.problem_path)
# Optimizer setup.
if FLAGS.optimizer == "Adam":
cost_op = problem()
problem_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
problem_reset = tf.variables_initializer(problem_vars)
x_op = problem_vars
optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
optimizer_reset = tf.variables_initializer(optimizer.get_slot_names())
update = optimizer.minimize(cost_op)
reset = [problem_reset, optimizer_reset]
elif FLAGS.optimizer == "SGD":
cost_op = problem()
problem_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
problem_reset = tf.variables_initializer(problem_vars)
x_op = problem_vars
optimizer = tf.train.GradientDescentOptimizer(FLAGS.learning_rate)
optimizer_reset = tf.variables_initializer(optimizer.get_slot_names())
update = optimizer.minimize(cost_op)
reset = [problem_reset, optimizer_reset]
elif FLAGS.optimizer == "RMSProp":
cost_op = problem()
problem_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
problem_reset = tf.variables_initializer(problem_vars)
x_op = problem_vars
optimizer = tf.train.RMSPropOptimizer(FLAGS.learning_rate)
optimizer_reset = tf.variables_initializer(optimizer.get_slot_names())
update = optimizer.minimize(cost_op)
reset = [problem_reset, optimizer_reset]
elif FLAGS.optimizer == "Momentum":
cost_op = problem()
problem_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
problem_reset = tf.variables_initializer(problem_vars)
x_op = problem_vars
optimizer = tf.train.MomentumOptimizer(FLAGS.learning_rate,
momentum=0.9)
optimizer_reset = tf.variables_initializer(optimizer.get_slot_names())
update = optimizer.minimize(cost_op)
reset = [problem_reset, optimizer_reset]
elif FLAGS.optimizer == "NAG":
cost_op = problem()
problem_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
problem_reset = tf.variables_initializer(problem_vars)
x_op = problem_vars
optimizer = tf.train.MomentumOptimizer(FLAGS.learning_rate,
momentum=0.1,
use_nesterov=True)
optimizer_reset = tf.variables_initializer(optimizer.get_slot_names())
update = optimizer.minimize(cost_op)
reset = [problem_reset, optimizer_reset]
elif FLAGS.optimizer == "L2L":
if FLAGS.path is None:
logging.warning("Evaluating untrained L2L optimizer")
optimizer = meta.MetaOptimizer(**net_config)
meta_loss = optimizer.meta_loss(problem,
1,
net_assignments=net_assignments)
_, update, reset, cost_op, x_op = meta_loss
else:
raise ValueError("{} is not a valid optimizer".format(FLAGS.optimizer))
with ms.MonitoredSession() as sess:
# Prevent accidental changes to the graph.
tf.get_default_graph().finalize()
total_time = 0
total_cost = 0
for _ in xrange(FLAGS.num_epochs):
# Training.
time, cost, x_values = util.run_epoch(sess, cost_op, x_op,
[update], reset, num_unrolls)
total_time += time
total_cost += cost
x_values = np.swapaxes(np.squeeze(x_values), 0, 1)
if FLAGS.problem.find('wav') != -1:
np.save(os.path.join('results', '{}_wav'.format(FLAGS.optimizer)),
x_values)
else:
np.save(os.path.join('results', '{}'.format(FLAGS.optimizer)),
x_values)
# print("x_values shape: {}".format(x_values.shape))
# print("x_values: {}".format(x_values))
# np.savetxt(os.path.join('results', '{}.txt'.format(FLAGS.optimizer)), x_values, fmt='%f')
# Results.
util.print_stats(
"Epoch {}, Optimizer {}".format(FLAGS.num_epochs, FLAGS.optimizer),
total_cost, total_time, FLAGS.num_epochs)
best_evaluation = float("inf")
if restore_network:
print 'Resotring Optimizer'
saver.restore(sess, load_path)
best_evaluation = np.load('best_eval.npy')
print 'Best Eval loaded', best_evaluation
mean_mats_values_list = list()
print 'Starting Training...'
for epoch in range(epochs):
mean_mats_values = sess.run(mean_mats)
# print mean_mats_values
mean_mats_values_list.append(mean_mats_values)
time, loss = util.run_epoch(sess, loss_final, [step, update],
reset, num_unrolls_per_epoch)
total_loss += loss
total_time += time
if (epoch + 1) % epoch_interval == 0:
log10loss = np.log10(total_loss / epoch_interval)
util.print_update(epoch, epochs, log10loss, epoch_interval,
total_time)
util.write_update(log10loss, total_time, mean_mats_values_list)
total_loss = 0
total_time = 0
mean_mats_values_list = list()
if (epoch + 1) % eval_interval == 0:
print 'EVALUATION'
loss_eval_total = 0
for eval_epoch in range(eval_epochs):
with tf.variable_scope("model", reuse=None):
train_model = model_class(config, training=True)
with tf.variable_scope("model", reuse=True):
valid_model = model_class(config, training=False)
saver = tf.train.Saver(tf.all_variables(), max_to_keep=40)
tf.initialize_all_variables().run()
# training
early_stop_best_loss = None
start_saving = False
saved_flag = False
train_losses, valid_losses = [], []
start_time = time.time()
for i in range(config.num_epochs):
loss = util.run_epoch(session, train_model,
data["train"]["data"], training=True, testing=False)
train_losses.append((i, loss))
if i == 0:
continue
logger.info('Epoch: {}, Train Loss: {}, Time Per Epoch: {}'.format(\
i, loss, (time.time() - start_time)/i))
valid_loss = util.run_epoch(session, valid_model, data["valid"]["data"], training=False, testing=False)
valid_losses.append((i, valid_loss))
logger.info('Valid Loss: {}'.format(valid_loss))
if early_stop_best_loss == None:
early_stop_best_loss = valid_loss
elif valid_loss < early_stop_best_loss:
early_stop_best_loss = valid_loss
if start_saving:
def main(_):
# Configuration.
num_iter = FLAGS.num_steps // FLAGS.unroll_length
# Problem.
config = {
'hidden_size': FLAGS.hidden_size,
'num_layer': FLAGS.num_layer,
'unroll_nn': FLAGS.unroll_length,
'lr': FLAGS.lr
}
problem = util.get_problem_config(FLAGS.problem)
optimizer = MetaLearner.MetaOpt(**config)
step, loss_opt, update, reset, cost_tot, cost_op, arraycost, _ = optimizer.metaoptimizer(
problem)
with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess:
graph_writer = tf.summary.FileWriter(
os.path.join(logs_path, FLAGS.problem, '/MetaLog/'), sess.graph)
sess.run(tf.global_variables_initializer())
best_evaluation = float("inf")
start = timer()
losstrain = []
losseval = []
plotlosstrain = []
plotlosseval = []
# Training
for e in range(FLAGS.num_epochs):
cost, trainloss = util.run_epoch(sess, num_iter, arraycost,
cost_op, [step, update], reset)
print(cost)
losstrain.append(cost)
util.print_stats("Training Epoch {}".format(e), trainloss,
timer() - start)
saver = tf.train.Saver()
if (e + 1) % FLAGS.logging_period == 0:
plotlosstrain.append(cost)
if (e + 1) % FLAGS.evaluation_period == 0:
for _ in range(FLAGS.evaluation_epochs):
evalcost, evaloss = util.run_epoch(sess, num_iter,
arraycost, cost_op,
[update], reset)
losseval.append(evalcost)
if save_path is not None and evaloss < best_evaluation:
print("Saving meta-optimizer to {}".format(save_path))
saver.save(sess,
os.path.join(save_path, FLAGS.problem,
'/MetaSave/model.ckpt'),
global_step=e + 1)
best_evaluation = evaloss
plotlosseval.append(evalcost)
slengths = np.arange(FLAGS.num_steps)
np.savetxt(os.path.join(save_path, FLAGS.problem,
'/MetaSave/plotlosstrain.out'),
plotlosstrain,
delimiter=',')
np.savetxt(os.path.join(save_path, FLAGS.problem,
'/MetaSave/plotlosseval.out'),
plotlosseval,
delimiter=',')
np.savetxt(os.path.join(save_path, FLAGS.problem,
'/MetaSave/losstrain.out'),
losstrain,
delimiter=',')
np.savetxt(os.path.join(save_path, FLAGS.problem,
'/MetaSave/losseval.out'),
losseval,
delimiter=',')
plt.figure(figsize=(8, 5))
plt.plot(slengths,
np.mean(plotlosstrain, 0),
'r-',
label='Training Loss')
plt.xlabel('Epoch')
plt.ylabel('Training Loss')
plt.legend()
savefig(
os.path.join(save_path, FLAGS.problem, '/MetaSave/Training.png'))
plt.close()
plt.figure(figsize=(8, 5))
plt.plot(slengths,
np.mean(plotlosseval, 0),
'b-',
label='Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Validation Loss')
plt.legend()
savefig(
os.path.join(save_path, FLAGS.problem, '/MetaSave/Validation.png'))
plt.close()
graph_writer.close()
def train_model():
np.random.seed()
parser = argparse.ArgumentParser(description='Script to train and save a model.')
parser.add_argument('--dataset', type=str, default='softmax',
# choices = ['bach', 'nottingham', 'softmax'],
choices = ['softmax'])
parser.add_argument('--model_dir', type=str, default='models')
parser.add_argument('--run_name', type=str, default=time.strftime("%m%d_%H%M"))
args = parser.parse_args()
if args.dataset == 'softmax':
resolution = 480
time_step = 120
model_class = NottinghamModel
with open(nottingham_util.PICKLE_LOC, 'r') as f:
pickle = cPickle.load(f)
chord_to_idx = pickle['chord_to_idx']
input_dim = pickle["train"][0].shape[1]
print 'Finished loading data, input dim: {}'.format(input_dim)
else:
raise Exception("Other datasets not yet implemented")
initializer = tf.random_uniform_initializer(-0.1, 0.1)
best_config = None
best_valid_loss = None
# set up run dir
run_folder = os.path.join(args.model_dir, args.run_name)
if os.path.exists(run_folder):
raise Exception("Run name {} already exists, choose a different one", format(run_folder))
os.makedirs(run_folder)
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())
logger.addHandler(logging.FileHandler(os.path.join(run_folder, "training.log")))
grid = {
"dropout_prob": [0.5],
"input_dropout_prob": [0.8],
"melody_coeff": [0.5],
"num_layers": [2],
"hidden_size": [200],
"num_epochs": [250],
"learning_rate": [5e-3],
"learning_rate_decay": [0.9],
"time_batch_len": [128],
}
# Generate product of hyperparams
runs = list(list(itertools.izip(grid, x)) for x in itertools.product(*grid.itervalues()))
logger.info("{} runs detected".format(len(runs)))
for combination in runs:
config = DefaultConfig()
config.dataset = args.dataset
config.model_name = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(12)) + '.model'
for attr, value in combination:
setattr(config, attr, value)
if config.dataset == 'softmax':
data = util.load_data('', time_step, config.time_batch_len, config.max_time_batches, nottingham=pickle)
config.input_dim = data["input_dim"]
else:
raise Exception("Other datasets not yet implemented")
logger.info(config)
config_file_path = os.path.join(run_folder, get_config_name(config) + '.config')
with open(config_file_path, 'w') as f:
cPickle.dump(config, f)
with tf.Graph().as_default(), tf.Session() as session:
with tf.variable_scope("model", reuse=None):
train_model = model_class(config, training=True)
with tf.variable_scope("model", reuse=True):
valid_model = model_class(config, training=False)
saver = tf.train.Saver(tf.all_variables(), max_to_keep=40)
tf.initialize_all_variables().run()
# training
early_stop_best_loss = None
start_saving = False
saved_flag = False
train_losses, valid_losses = [], []
start_time = time.time()
for i in range(config.num_epochs):
loss = util.run_epoch(session, train_model,
data["train"]["data"], training=True, testing=False)
train_losses.append((i, loss))
if i == 0:
continue
logger.info('Epoch: {}, Train Loss: {}, Time Per Epoch: {}'.format(\
i, loss, (time.time() - start_time)/i))
valid_loss = util.run_epoch(session, valid_model, data["valid"]["data"], training=False, testing=False)
valid_losses.append((i, valid_loss))
logger.info('Valid Loss: {}'.format(valid_loss))
if early_stop_best_loss == None:
early_stop_best_loss = valid_loss
elif valid_loss < early_stop_best_loss:
early_stop_best_loss = valid_loss
if start_saving:
logger.info('Best loss so far encountered, saving model.')
saver.save(session, os.path.join(run_folder, config.model_name))
saved_flag = True
elif not start_saving:
start_saving = True
logger.info('Valid loss increased for the first time, will start saving models')
saver.save(session, os.path.join(run_folder, config.model_name))
saved_flag = True
if not saved_flag:
saver.save(session, os.path.join(run_folder, config.model_name))
# set loss axis max to 20
axes = plt.gca()
if config.dataset == 'softmax':
axes.set_ylim([0, 2])
else:
axes.set_ylim([0, 100])
plt.plot([t[0] for t in train_losses], [t[1] for t in train_losses])
plt.plot([t[0] for t in valid_losses], [t[1] for t in valid_losses])
plt.legend(['Train Loss', 'Validation Loss'])
chart_file_path = os.path.join(run_folder, get_config_name(config) + '.png')
plt.savefig(chart_file_path)
plt.clf()
logger.info("Config {}, Loss: {}".format(config, early_stop_best_loss))
if best_valid_loss == None or early_stop_best_loss < best_valid_loss:
logger.info("Found best new model!")
best_valid_loss = early_stop_best_loss
best_config = config
logger.info("Best Config: {}, Loss: {}".format(best_config, best_valid_loss))
def main(_):
# Configuration.
if FLAGS.seed:
tf.set_random_seed(FLAGS.seed)
num_unrolls = FLAGS.num_steps // FLAGS.unroll_length
if FLAGS.seed:
tf.set_random_seed(FLAGS.seed)
if FLAGS.save_path is not None:
if os.path.exists(FLAGS.save_path):
# raise ValueError("Folder {} already exists".format(FLAGS.save_path))
pass
else:
os.mkdir(FLAGS.save_path)
# Problem.
problem, net_config, net_assignments = util.get_config(FLAGS.problem)
loss_op = problem()
# Optimizer setup.
optimizer = meta.MetaOptimizer(**net_config)
minimize = optimizer.meta_minimize(
problem,
FLAGS.unroll_length,
learning_rate=FLAGS.learning_rate,
net_assignments=net_assignments,
second_derivatives=FLAGS.second_derivatives)
step, update, reset, cost_op, _ = minimize
if FLAGS.problem == "mnist":
var_name_mlp = [
"mlp/linear_0/w:0", "mlp/linear_0/b:0", "mlp/linear_1/w:0",
"mlp/linear_1/b:0"
]
else:
var_name_mlp = []
problem_vars = tf.get_collection(tf.GraphKeys.VARIABLES)
if var_name_mlp:
saver_vars = [vv for vv in problem_vars if vv.name in var_name_mlp]
else:
saver_vars = problem_vars
saver = tf.train.Saver(saver_vars)
process_id = os.getpid()
exp_folder = os.path.join(FLAGS.save_path, str(process_id))
writer = tf.summary.FileWriter(exp_folder)
if not os.path.isdir(exp_folder):
os.mkdir(exp_folder)
with ms.MonitoredSession() as sess:
# a quick hack!
regular_sess = sess._sess._sess._sess._sess
# Prevent accidental changes to the graph.
tf.get_default_graph().finalize()
# print("Initial loss = {}".format(sess.run(loss_op)))
# raw_input("wait")
if FLAGS.load_trained_model == True:
print("We are loading trained model here!")
saver.restore(regular_sess, FLAGS.model_path)
# init_state = regular_sess.run(optimizer.init_state)
# pickle.dump(init_state, open("init_state.p", "wb"))
best_evaluation = float("inf")
total_time = 0
total_cost = 0
for e in xrange(FLAGS.num_epochs):
# Training.
time, cost = util.run_epoch(sess, cost_op, [update, step], reset,
num_unrolls, e, writer)
total_time += time
total_cost += cost
writer.flush()
# Logging.
if (e + 1) % FLAGS.log_period == 0:
util.print_stats("Epoch {}".format(e + 1), total_cost,
total_time, FLAGS.log_period)
total_time = 0
total_cost = 0
# Evaluation.
if (e + 1) % FLAGS.evaluation_period == 0 or e == 0:
eval_cost = 0
eval_time = 0
for _ in xrange(FLAGS.evaluation_epochs):
time, cost = util.run_epoch_val(sess, cost_op, [update],
reset, num_unrolls, e,
writer)
eval_time += time
eval_cost += cost
util.print_stats("EVALUATION", eval_cost, eval_time,
FLAGS.evaluation_epochs)
if FLAGS.save_path is not None and eval_cost < best_evaluation:
# print("Removing previously saved meta-optimizer")
# for f in os.listdir(FLAGS.save_path):
# os.remove(os.path.join(FLAGS.save_path, f))
# print("Saving meta-optimizer to {}".format(FLAGS.save_path))
# optimizer.save(sess, FLAGS.save_path)
optimizer.save(sess, exp_folder, e + 1)
best_evaluation = eval_cost
def train_model():
np.random.seed()
parser = argparse.ArgumentParser(
description='Script to train and save a model.')
parser.add_argument(
'--dataset',
type=str,
default='softmax',
# choices = ['bach', 'nottingham', 'softmax'],
choices=['softmax'])
parser.add_argument('--model_dir', type=str, default='models')
parser.add_argument('--run_name',
type=str,
default=time.strftime("%m%d_%H%M"))
args = parser.parse_args()
if args.dataset == 'softmax':
resolution = 480
time_step = 120
model_class = NottinghamModel
with open(nottingham_util.PICKLE_LOC, 'r') as f:
pickle = cPickle.load(f)
chord_to_idx = pickle['chord_to_idx']
input_dim = pickle["train"][0].shape[1]
print 'Finished loading data, input dim: {}'.format(input_dim)
else:
raise Exception("Other datasets not yet implemented")
initializer = tf.random_uniform_initializer(-0.1, 0.1)
best_config = None
best_valid_loss = None
# set up run dir
run_folder = os.path.join(args.model_dir, args.run_name)
if os.path.exists(run_folder):
raise Exception("Run name {} already exists, choose a different one",
format(run_folder))
os.makedirs(run_folder)
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())
logger.addHandler(
logging.FileHandler(os.path.join(run_folder, "training.log")))
grid = {
"dropout_prob": [0.5],
"input_dropout_prob": [0.8],
"melody_coeff": [0.5],
"num_layers": [2],
"hidden_size": [200],
"num_epochs": [250],
"learning_rate": [5e-3],
"learning_rate_decay": [0.9],
"time_batch_len": [128],
}
# Generate product of hyperparams
runs = list(
list(itertools.izip(grid, x))
for x in itertools.product(*grid.itervalues()))
logger.info("{} runs detected".format(len(runs)))
for combination in runs:
config = DefaultConfig()
config.dataset = args.dataset
config.model_name = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(12)) + '.model'
for attr, value in combination:
setattr(config, attr, value)
if config.dataset == 'softmax':
data = util.load_data('',
time_step,
config.time_batch_len,
config.max_time_batches,
nottingham=pickle)
config.input_dim = data["input_dim"]
else:
raise Exception("Other datasets not yet implemented")
logger.info(config)
config_file_path = os.path.join(run_folder,
get_config_name(config) + '.config')
with open(config_file_path, 'w') as f:
cPickle.dump(config, f)
with tf.Graph().as_default(), tf.Session() as session:
with tf.variable_scope("model", reuse=None):
train_model = model_class(config, training=True)
with tf.variable_scope("model", reuse=True):
valid_model = model_class(config, training=False)
saver = tf.train.Saver(tf.all_variables(), max_to_keep=40)
tf.initialize_all_variables().run()
# training
early_stop_best_loss = None
start_saving = False
saved_flag = False
train_losses, valid_losses = [], []
start_time = time.time()
for i in range(config.num_epochs):
loss = util.run_epoch(session,
train_model,
data["train"]["data"],
training=True,
testing=False)
train_losses.append((i, loss))
if i == 0:
continue
logger.info('Epoch: {}, Train Loss: {}, Time Per Epoch: {}'.format(\
i, loss, (time.time() - start_time)/i))
valid_loss = util.run_epoch(session,
valid_model,
data["valid"]["data"],
training=False,
testing=False)
valid_losses.append((i, valid_loss))
logger.info('Valid Loss: {}'.format(valid_loss))
if early_stop_best_loss == None:
early_stop_best_loss = valid_loss
elif valid_loss < early_stop_best_loss:
early_stop_best_loss = valid_loss
if start_saving:
logger.info(
'Best loss so far encountered, saving model.')
saver.save(session,
os.path.join(run_folder, config.model_name))
saved_flag = True
elif not start_saving:
start_saving = True
logger.info(
'Valid loss increased for the first time, will start saving models'
)
saver.save(session,
os.path.join(run_folder, config.model_name))
saved_flag = True
if not saved_flag:
saver.save(session, os.path.join(run_folder,
config.model_name))
# set loss axis max to 20
axes = plt.gca()
if config.dataset == 'softmax':
axes.set_ylim([0, 2])
else:
axes.set_ylim([0, 100])
plt.plot([t[0] for t in train_losses],
[t[1] for t in train_losses])
plt.plot([t[0] for t in valid_losses],
[t[1] for t in valid_losses])
plt.legend(['Train Loss', 'Validation Loss'])
chart_file_path = os.path.join(run_folder,
get_config_name(config) + '.png')
plt.savefig(chart_file_path)
plt.clf()
logger.info("Config {}, Loss: {}".format(config,
early_stop_best_loss))
if best_valid_loss == None or early_stop_best_loss < best_valid_loss:
logger.info("Found best new model!")
best_valid_loss = early_stop_best_loss
best_config = config
logger.info("Best Config: {}, Loss: {}".format(best_config,
best_valid_loss))
def main(_):
# Configuration.
if FLAGS.if_cl:
num_steps = [100, 200, 500, 1000, 1500, 2000, 2500, 3000]
num_unrolls = [int(ns / FLAGS.unroll_length) for ns in num_steps]
num_unrolls_eval = num_unrolls[1:]
min_num_eval = 5
curriculum_idx = 0
else:
num_unrolls = FLAGS.num_steps // FLAGS.unroll_length
if FLAGS.save_path is not None:
if not os.path.exists(FLAGS.save_path):
os.mkdir(FLAGS.save_path)
# Problem.
problem, net_config, net_assignments = util.get_config(
FLAGS.problem,
mode=FLAGS.mode, #加入mode
num_linear_heads=1,
init=FLAGS.init,
path=FLAGS.path,
param=param_dict)
# Optimizer setup.
optimizer = meta.MetaOptimizer(FLAGS.num_mt, **net_config)
minimize, scale, var_x, constants, subsets,\
loss_mt, steps_mt, update_mt, reset_mt, mt_labels, mt_inputs, hess_norm_approx = optimizer.meta_minimize(
problem, FLAGS.unroll_length,
learning_rate=FLAGS.learning_rate,
net_assignments=net_assignments,
second_derivatives=FLAGS.second_derivatives)
step, update, reset, cost_op, _ = minimize
if FLAGS.if_mt:
data_mt = data_loader(problem, var_x, constants, subsets, scale,
FLAGS.optimizers, FLAGS.unroll_length,
hess_norm_approx)
if FLAGS.if_cl:
mt_ratios = [float(r) for r in FLAGS.mt_ratios.split()]
p_val_x = []
for k in var_x:
p_val_x.append(tf.placeholder(tf.float32, shape=k.shape))
assign_ops = [
tf.assign(var_x[k_id], p_val_x[k_id]) for k_id in range(len(p_val_x))
]
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
start_time = timer()
with ms.MonitoredSession() as sess:
def assign_func(val_x):
sess.run(assign_ops,
feed_dict={p: v
for p, v in zip(p_val_x, val_x)})
tf.get_default_graph().finalize()
best_evaluation = float("inf")
train_loss_record = []
eval_loss_record = []
num_steps_train = []
num_eval = 0
improved = False
mti = -1
task_id_record = []
for e in xrange(FLAGS.num_epochs):
# choose task
if FLAGS.if_mt:
if FLAGS.if_cl:
mt_ratio = mt_ratios[curriculum_idx]
else:
mt_ratio = FLAGS.mt_ratio
if random.random() < mt_ratio:
mti = (mti + 1) % FLAGS.num_mt
task_i = mti
else:
task_i = -1
task_id_record.append(task_i)
else:
task_i = -1
# Training.
if FLAGS.if_cl:
num_unrolls_cur = num_unrolls[curriculum_idx]
else:
num_unrolls_cur = num_unrolls
if task_i == -1:
time, cost = util.run_epoch(
sess,
cost_op, [update, step],
reset,
num_unrolls_cur,
scale=scale,
rd_scale=FLAGS.if_scale,
rd_scale_bound=FLAGS.rd_scale_bound,
assign_func=assign_func,
var_x=var_x,
if_hess_init=FLAGS.init == "hessian",
hess_norm=hess_norm_approx)
else:
data_e = data_mt.get_data(task_i,
sess,
num_unrolls_cur,
assign_func,
FLAGS.rd_scale_bound,
if_scale=FLAGS.if_scale,
mt_k=FLAGS.k,
if_hess_init=FLAGS.init == "hessian")
time, cost = util.run_epoch(
sess,
loss_mt[task_i], [update_mt[task_i], steps_mt[task_i]],
reset_mt[task_i],
num_unrolls_cur,
scale=scale,
rd_scale=FLAGS.if_scale,
rd_scale_bound=FLAGS.rd_scale_bound,
assign_func=assign_func,
var_x=var_x,
task_i=task_i,
data=data_e,
label_pl=mt_labels[task_i],
input_pl=mt_inputs[task_i])
train_loss_record.append(cost)
# Evaluation.
if (e + 1) % FLAGS.evaluation_period == 0:
if FLAGS.if_cl:
num_unrolls_eval_cur = num_unrolls_eval[curriculum_idx]
else:
num_unrolls_eval_cur = num_unrolls
num_eval += 1
eval_cost = 0
for _ in xrange(FLAGS.evaluation_epochs):
time, cost = util.run_epoch(sess, cost_op, [update], reset,
num_unrolls_eval_cur)
eval_cost += cost
if FLAGS.if_cl:
num_steps_cur = num_steps[curriculum_idx]
else:
num_steps_cur = FLAGS.num_steps
print("epoch={}, num_steps={}, eval loss={}".format(
e, num_steps_cur, eval_cost / FLAGS.evaluation_epochs),
flush=True)
eval_loss_record.append(eval_cost / FLAGS.evaluation_epochs)
num_steps_train.append(num_steps_cur)
if not FLAGS.if_cl:
if eval_cost < best_evaluation:
best_evaluation = eval_cost
optimizer.save(sess, FLAGS.save_path, e + 1)
optimizer.save(sess, FLAGS.save_path, 0)
print("Saving optimizer...")
continue
# update curriculum
if eval_cost < best_evaluation:
best_evaluation = eval_cost
improved = True
# save model
optimizer.save(sess, FLAGS.save_path, curriculum_idx)
optimizer.save(sess, FLAGS.save_path, 0)
elif num_eval >= min_num_eval and improved:
# restore model
optimizer.restore(sess, FLAGS.save_path, curriculum_idx)
num_eval = 0
improved = False
curriculum_idx += 1
if curriculum_idx >= len(num_unrolls):
curriculum_idx = -1
# new evaluation
eval_cost = 0
for _ in xrange(FLAGS.evaluation_epochs):
time, cost = util.run_epoch(
sess, cost_op, [update], reset,
num_unrolls_eval[curriculum_idx])
eval_cost += cost
best_evaluation = eval_cost
print("epoch={}, num_steps={}, eval loss={}".format(
e, num_steps[curriculum_idx],
eval_cost / FLAGS.evaluation_epochs),
flush=True)
eval_loss_record.append(eval_cost /
FLAGS.evaluation_epochs)
num_steps_train.append(num_steps[curriculum_idx])
elif num_eval >= min_num_eval and not improved:
print("no improve during curriculum {} --> stop".format(
curriculum_idx))
break
print("total time = {}s...".format(timer() - start_time))
# output
with open('{}/log.pickle'.format(FLAGS.save_path), 'wb') as l_record:
records = {
"eval_loss": eval_loss_record,
"train_loss": train_loss_record,
"task_id": task_id_record,
"num_steps": num_steps_train
}
pickle.dump(records, l_record)
l_record.close()
def main(_):
# Configuration.
if FLAGS.if_cl:
num_steps = [100, 200, 500, 1000, 1500, 2000, 2500, 3000]
num_unrolls = [int(ns / FLAGS.unroll_length) for ns in num_steps]
num_unrolls_eval = num_unrolls[1:]
curriculum_idx = 0
else:
num_unrolls = FLAGS.num_steps // FLAGS.unroll_length
# Output path.
if FLAGS.save_path is not None:
if not os.path.exists(FLAGS.save_path):
os.mkdir(FLAGS.save_path)
# Problem.
problem, net_config, net_assignments = util.get_config(FLAGS.problem,
net_name="RNNprop")
# Optimizer setup.
optimizer = meta.MetaOptimizer(FLAGS.num_mt, FLAGS.beta1, FLAGS.beta2,
**net_config)
minimize, scale, var_x, constants, subsets, seq_step, \
loss_mt, steps_mt, update_mt, reset_mt, mt_labels, mt_inputs = optimizer.meta_minimize(
problem, FLAGS.unroll_length,
learning_rate=FLAGS.learning_rate,
net_assignments=net_assignments,
second_derivatives=FLAGS.second_derivatives)
step, update, reset, cost_op, _ = minimize
# Data generator for multi-task learning.
if FLAGS.if_mt:
data_mt = data_loader(problem, var_x, constants, subsets, scale,
FLAGS.optimizers, FLAGS.unroll_length)
if FLAGS.if_cl:
mt_ratios = [float(r) for r in FLAGS.mt_ratios.split()]
# Assign func.
p_val_x = []
for k in var_x:
p_val_x.append(tf.placeholder(tf.float32, shape=k.shape))
assign_ops = [
tf.assign(var_x[k_id], p_val_x[k_id]) for k_id in range(len(p_val_x))
]
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with ms.MonitoredSession() as sess:
def assign_func(val_x):
sess.run(assign_ops,
feed_dict={p: v
for p, v in zip(p_val_x, val_x)})
# tf1.14
for rst in [reset] + reset_mt:
sess.run(rst)
# Start.
start_time = timer()
tf.get_default_graph().finalize()
best_evaluation = float("inf")
num_eval = 0
improved = False
mti = -1
for e in xrange(FLAGS.num_epochs):
# Pick a task if it's multi-task learning.
if FLAGS.if_mt:
if FLAGS.if_cl:
if curriculum_idx >= len(mt_ratios):
mt_ratio = mt_ratios[-1]
else:
mt_ratio = mt_ratios[curriculum_idx]
else:
mt_ratio = FLAGS.mt_ratio
if random.random() < mt_ratio:
mti = (mti + 1) % FLAGS.num_mt
task_i = mti
else:
task_i = -1
else:
task_i = -1
# Training.
if FLAGS.if_cl:
num_unrolls_cur = num_unrolls[curriculum_idx]
else:
num_unrolls_cur = num_unrolls
if task_i == -1:
time, cost = util.run_epoch(
sess,
cost_op, [update, step],
reset,
num_unrolls_cur,
scale=scale,
rd_scale=FLAGS.if_scale,
rd_scale_bound=FLAGS.rd_scale_bound,
assign_func=assign_func,
var_x=var_x,
step=seq_step,
unroll_len=FLAGS.unroll_length)
else:
data_e = data_mt.get_data(task_i,
sess,
num_unrolls_cur,
assign_func,
FLAGS.rd_scale_bound,
if_scale=FLAGS.if_scale,
mt_k=FLAGS.k)
time, cost = util.run_epoch(
sess,
loss_mt[task_i], [update_mt[task_i], steps_mt[task_i]],
reset_mt[task_i],
num_unrolls_cur,
scale=scale,
rd_scale=FLAGS.if_scale,
rd_scale_bound=FLAGS.rd_scale_bound,
assign_func=assign_func,
var_x=var_x,
step=seq_step,
unroll_len=FLAGS.unroll_length,
task_i=task_i,
data=data_e,
label_pl=mt_labels[task_i],
input_pl=mt_inputs[task_i])
print("training_loss={}".format(cost))
# Evaluation.
if (e + 1) % FLAGS.evaluation_period == 0:
if FLAGS.if_cl:
num_unrolls_eval_cur = num_unrolls_eval[curriculum_idx]
else:
num_unrolls_eval_cur = num_unrolls
num_eval += 1
eval_cost = 0
for _ in xrange(FLAGS.evaluation_epochs):
time, cost = util.run_epoch(sess,
cost_op, [update],
reset,
num_unrolls_eval_cur,
step=seq_step,
unroll_len=FLAGS.unroll_length)
eval_cost += cost
if FLAGS.if_cl:
num_steps_cur = num_steps[curriculum_idx]
else:
num_steps_cur = FLAGS.num_steps
print("epoch={}, num_steps={}, eval_loss={}".format(
e, num_steps_cur, eval_cost / FLAGS.evaluation_epochs),
flush=True)
if not FLAGS.if_cl:
if eval_cost < best_evaluation:
best_evaluation = eval_cost
optimizer.save(sess, FLAGS.save_path, e + 1)
optimizer.save(sess, FLAGS.save_path, 0)
print("Saving optimizer of epoch {}...".format(e + 1))
continue
# Curriculum learning.
# update curriculum
if eval_cost < best_evaluation:
best_evaluation = eval_cost
improved = True
# save model
optimizer.save(sess, FLAGS.save_path, curriculum_idx)
optimizer.save(sess, FLAGS.save_path, 0)
elif num_eval >= FLAGS.min_num_eval and improved:
# restore model
optimizer.restore(sess, FLAGS.save_path, curriculum_idx)
num_eval = 0
improved = False
curriculum_idx += 1
if curriculum_idx >= len(num_unrolls):
curriculum_idx = -1
# initial evaluation for next curriculum
eval_cost = 0
for _ in xrange(FLAGS.evaluation_epochs):
time, cost = util.run_epoch(
sess,
cost_op, [update],
reset,
num_unrolls_eval[curriculum_idx],
step=seq_step,
unroll_len=FLAGS.unroll_length)
eval_cost += cost
best_evaluation = eval_cost
print("epoch={}, num_steps={}, eval loss={}".format(
e, num_steps[curriculum_idx],
eval_cost / FLAGS.evaluation_epochs),
flush=True)
elif num_eval >= FLAGS.min_num_eval and not improved:
print("no improve during curriculum {} --> stop".format(
curriculum_idx))
break
print("total time = {}s...".format(timer() - start_time))
if meta:
print('Resotring Optimizer')
optimizer.load(sess, load_path)
print('Init Problem Vars: ', sess.run(mean_problem_variables))
if mean_optim_variables is not None:
print('Init Optim Vars: ', sess.run(mean_optim_variables))
total_loss_final = 0
total_loss = 0
total_time = 0
time = 0
flat_grads_list, pre_pro_grads_list, deltas_list = list(), list(
), list()
print('---------------------------------\n')
for epoch in range(epochs):
time, loss_value = util.run_epoch(sess, loss, final_step, None,
num_unrolls_per_epoch)
total_time += time
total_loss += loss_value
total_loss_final += loss_value
if (epoch + 1) % epoch_interval == 0:
print('Problem Vars: ', sess.run(mean_problem_variables))
if mean_optim_variables is not None:
print('Optim Vars: ', sess.run(mean_optim_variables))
log10loss = np.log10(total_loss / epoch_interval)
util.print_update(epoch, epochs, log10loss, epoch_interval,
total_time)
total_loss = 0
total_time = 0
mean_mats_values_list = list()
total_loss_final = np.log10(total_loss_final / epochs)
print('Final Loss: ', total_loss_final)
with tf.variable_scope("model", reuse=True):
valid_model = model_class(config, training=False)
saver = tf.train.Saver(tf.all_variables())
tf.initialize_all_variables().run()
# training
early_stop_best_loss = None
start_saving = False
saved_flag = False
train_losses, valid_losses = [], []
start_time = time.time()
for i in range(config.num_epochs):
loss = util.run_epoch(session,
train_model,
data["train"]["data"],
training=True,
testing=False)
train_losses.append((i, loss))
if i == 0:
continue
valid_loss = util.run_epoch(session,
valid_model,
data["valid"]["data"],
training=False,
testing=False)
valid_losses.append((i, valid_loss))
logger.info('Epoch: {}, Train Loss: {}, Valid Loss: {}, Time Per Epoch: {}'.format(\
i, loss, valid_loss, (time.time() - start_time)/i))
unroll_length,
learning_rate=learning_rate,
net_assignments=net_assignments,
second_derivatives=second_derivatives)
step, update, reset, cost_op, _ = minimize
with ms.MonitoredSession() as sess:
# Prevent accidental changes to the graph.
tf.get_default_graph().finalize()
best_evaluation = float("inf")
total_time = 0
total_cost = 0
for e in xrange(num_epochs):
# Training.
time, cost = util.run_epoch(sess, cost_op, [update, step], reset,
num_unrolls)
total_time += time
total_cost += cost
# Logging.
if (e + 1) % log_period == 0:
util.print_stats("Epoch {}".format(e + 1), total_cost, total_time,
log_period)
total_time = 0
total_cost = 0
# Evaluation.
if (e + 1) % evaluation_period == 0:
eval_cost = 0
eval_time = 0
for _ in xrange(evaluation_epochs):
def main(_):
# Configuration.
num_unrolls = FLAGS.num_steps // FLAGS.unroll_length
problem, net_config, net_assignments = util.get_config(FLAGS.problem)
optimizer = meta.MetaOptimizer(**net_config)
if FLAGS.save_path is not None:
if not os.path.exists(FLAGS.save_path):
os.mkdir(FLAGS.save_path)
path = None
# raise ValueError("Folder {} already exists".format(FLAGS.save_path))
else:
if os.path.exists('{}/loss-record.pickle'.format(FLAGS.save_path)):
path = FLAGS.save_path
else:
path = None
# Problem.
# Optimizer setup.
minimize = optimizer.meta_minimize(
problem,
FLAGS.unroll_length,
learning_rate=FLAGS.learning_rate,
net_assignments=net_assignments,
model_path=path,
second_derivatives=FLAGS.second_derivatives)
step, update, reset, cost_op, x_final, test, fc_weights, fc_bias, fc_va = minimize
# saver=tf.train.Saver()
with ms.MonitoredSession() as sess:
# Prevent accidental changes to the graph.
tf.get_default_graph().finalize()
# Step=[step for i in range(len(cost_op))]
best_evaluation = float("inf")
total_time = 0
total_cost = 0
loss_record = []
constants = []
for e in xrange(FLAGS.num_epochs):
# Training.
time, cost, constant, Weights = util.run_epoch(
sess, cost_op, [update, step], reset, num_unrolls, test,
[fc_weights, fc_bias, fc_va])
cost = sum(cost) / len(cost_op)
total_time += time
total_cost += cost
loss_record.append(cost)
constants.append(constant)
# Logging.
if (e + 1) % FLAGS.log_period == 0:
util.print_stats("Epoch {}".format(e + 1), total_cost,
total_time, FLAGS.log_period)
total_time = 0
total_cost = 0
# Evaluation.
if (e + 1) % FLAGS.evaluation_period == 0:
eval_cost = 0
eval_time = 0
for _ in xrange(FLAGS.evaluation_epochs):
time, cost, constant, weights = util.run_epoch(
sess, cost_op, [update, step], reset, num_unrolls,
test, [fc_weights, fc_bias, fc_va])
# cost/=len(cost_op)
eval_time += time
eval_cost += sum(cost) / len(cost_op)
util.print_stats("EVALUATION", eval_cost, eval_time,
FLAGS.evaluation_epochs)
if FLAGS.save_path is not None and eval_cost < best_evaluation:
print("Removing previously saved meta-optimizer")
for f in os.listdir(FLAGS.save_path):
os.remove(os.path.join(FLAGS.save_path, f))
print("Saving meta-optimizer to {}".format(
FLAGS.save_path))
# saver.save(sess,'./quadratic/quadratic.ckpt',global_step = e)
optimizer.save(sess, FLAGS.save_path)
with open(FLAGS.save_path + '/loss_record.pickle',
'wb') as l_record:
record = {'loss_record':loss_record, 'fc_weights':sess.run(weights[0]), \
'fc_bias':sess.run(weights[1]), 'fc_va':sess.run(weights[2]), 'constant':sess.run(constant)}
pickle.dump(record, l_record)
best_evaluation = eval_cost
test_data = util.batch_data(pickle['test'], time_batch_len = 1,
max_time_batches = -1, softmax = True)
else:
raise Exception("Other datasets not yet implemented")
print config
with tf.Graph().as_default(), tf.Session() as session:
with tf.variable_scope("model", reuse=None):
test_model = model_class(config, training=False)
saver = tf.train.Saver(tf.all_variables())
model_path = os.path.join(os.path.dirname(args.config_file),
config.model_name)
saver.restore(session, model_path)
test_loss, test_probs = util.run_epoch(session, test_model, test_data,
training=False, testing=True)
print 'Testing Loss: {}'.format(test_loss)
if config.dataset == 'softmax':
if args.seperate:
nottingham_util.seperate_accuracy(test_probs, test_data, num_samples=args.num_samples)
else:
nottingham_util.accuracy(test_probs, test_data, num_samples=args.num_samples)
else:
util.accuracy(test_probs, test_data, num_samples=50)
sys.exit(1)