def train(train_data, valid_data, test_data, emb_matrix):
"""Train the model"""
start_time = time.time()
paras = ModelParas()
tf.reset_default_graph()
sess = tf.Session()
# Init initialzer
uniform_initializer = tf.random_uniform_initializer(
minval=-paras.uniform_init_scale, maxval=paras.uniform_init_scale)
# Define model for train and evaluate
with tf.name_scope('train'):
with tf.variable_scope('Model',
reuse=None,
initializer=uniform_initializer):
model_train = Model(paras, sess, tf.contrib.learn.ModeKeys.TRAIN,
emb_matrix)
with tf.name_scope('valid'):
with tf.variable_scope('Model',
reuse=True,
initializer=uniform_initializer):
model_eval = Model(paras, sess, tf.contrib.learn.ModeKeys.EVAL,
emb_matrix)
# Model Train
init_op = tf.global_variables_initializer()
sess.run(init_op)
best_score = -np.inf
saver = tf.train.Saver()
save_path = os.path.join(config.model_path, 'model/model.ckpt')
for epoch in xrange(paras.epochs):
logger.debug('>>> Epoch %d, learning_rate=%.4f', epoch,
paras.learning_rate)
run_epoch(model_train, train_data)
logger.debug('>>> Running Valid')
score, global_step = run_epoch(model_eval, valid_data)
if score > best_score:
best_score = score
saver.save(sess, save_path)
logger.debug('Score improved, save model to %s', save_path)
else:
saver.restore(sess, save_path)
logger.debug('Score not improved, load previous best model')
logger.debug('Epoch %d done, time=%.4f minutes', epoch,
(time.time() - start_time) / 60)
logger.debug('>>> Running Test')
run_epoch(model_eval, test_data)
del model_train
del model_eval
logger.debug('Predict result')
predict(save_path=os.path.join(config.model_path, 'result_%f' %
best_score))
def sample(args):
with open(os.path.join(args.save_dir, 'config.pkl'), 'rb') as f:
saved_args = cPickle.load(f)
with open(os.path.join(args.save_dir, 'chars_vocab.pkl'), 'rb') as f:
chars, vocab = cPickle.load(f)
model = Model(saved_args, training=False)
with tf.Session() as sess:
tf.global_variables_initializer().run()
saver = tf.train.Saver(tf.global_variables())
ckpt = tf.train.get_checkpoint_state(args.save_dir)
with open("./data/words_dictionary.json", 'r') as f:
valid_words = json.load(f)
for model_path in ckpt.all_model_checkpoint_paths:
print(model_path)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, model_path)
samples = []
sample_size = 200
for i in range(sample_size):
prime = random.choice(
string.letters).lower().decode('utf-8')
samp = model.sample(sess, chars, vocab, args.n, prime,
args.sample).encode('utf-8')
samples.append(samp)
print("Generating sample: " + str(i))
batch_new_pos, batch_new_neg, batch_pos_size, batch_neg_size = split_pos_neg_words(
samples, valid_words)
file_name = model_path.replace("/", "-")
with gfile.GFile('./rnn_result_10/pos_samples-' + file_name +
'.txt',
mode="w") as f:
f.write("Pos size: " + str(batch_pos_size) + "\n")
f.write("Pos rate: " +
str(float(batch_pos_size) / sample_size) + "\n")
for s in batch_new_pos:
s = "".join(s)
f.write(s + "\n")
print("pos sample created.")
with gfile.GFile('./rnn_result_10/neg_samples-' + file_name +
'.txt',
mode="w") as f:
f.write("Neg size: " + str(batch_neg_size) + "\n")
for s in batch_new_neg:
s = "".join(s)
f.write(s + "\n")
print("neg sample created.")
def load_model(mode):
tf.reset_default_graph()
paras = ModelParas()
sess = tf.Session()
save_path = os.path.join(config.model_path, 'model/model.ckpt')
emb_matrix = Helper.get_emb_matrix()
with tf.variable_scope('Model'):
model = Model(paras, sess, mode, emb_matrix)
saver = tf.train.Saver()
saver.restore(model.sess, save_path)
return model
with open(os.path.join(args.save_dir, "config.pkl"), "rb") as f:
saved_config = pickle.load(f)
with open(os.path.join(args.save_dir, "events_vocab.pkl"), "rb") as f:
events, vocab = pickle.load(f)
print("Loading data...")
data_mode = saved_config.mode
data_dir = args.data_dir
batch_size = saved_config.batch_size
seq_length = saved_config.seq_length
loader = DataLoader(data_mode, data_dir, batch_size, seq_length)
print("Loading model...\n")
model = Model(saved_config, op_mode=RNNMode.SAMPLE)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
ckpt = tf.train.get_checkpoint_state(args.save_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
x, y = loader.test_batch
src_events = x.flatten()
target_evs = y.flatten()
clk = loader.test_clock.flatten()
net_losses = []
dist_losses = []
num_events = 0
with open(os.path.join(args.save_dir, "config.pkl"), "rb") as f:
saved_config = pickle.load(f)
with open(os.path.join(args.save_dir, "events_vocab.pkl"), "rb") as f:
events, vocab = pickle.load(f)
print("Loading data...")
data_mode = saved_config.mode
data_dir = args.data_dir
batch_size = saved_config.batch_size
seq_length = saved_config.seq_length
loader = DataLoader(data_mode, data_dir, batch_size, seq_length)
print("Loading model...\n")
model = Model(saved_config, op_mode=RNNMode.TEST)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
ckpt = tf.train.get_checkpoint_state(args.save_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
x, y = loader.test_batch
test_feed = {model.input_data: x, model.target_data: y}
if data_mode == DataLoader.Mode.MUSIC:
test_feed[model.clock_input] = loader.test_clock
t = sess.run(model.loss_vector, feed_dict=test_feed)
print("shape of t:", t.shape)
print("value of t:", t)
print("mean of t:", np.mean(t))
def train(args):
data_loader = TextLoader(args.data_dir, args.batch_size, args.seq_length, args.min_length)
args.vocab_size = data_loader.vocab_size
# check compatibility if training is continued from previously saved model
if args.init_from is not None:
# check if all necessary files exist
assert os.path.isdir(args.init_from)," %s must be a a path" % args.init_from
assert os.path.isfile(os.path.join(args.init_from,"config.pkl")),"config.pkl file does not exist in path %s"%args.init_from
assert os.path.isfile(os.path.join(args.init_from,"chars_vocab.pkl")),"chars_vocab.pkl.pkl file does not exist in path %s" % args.init_from
ckpt = tf.train.get_checkpoint_state(args.init_from)
assert ckpt, "No checkpoint found"
assert ckpt.model_checkpoint_path, "No model path found in checkpoint"
# open old config and check if models are compatible
with open(os.path.join(args.init_from, 'config.pkl'), 'rb') as f:
saved_model_args = cPickle.load(f)
need_be_same = ["model", "rnn_size", "num_layers", "seq_length"]
for checkme in need_be_same:
assert vars(saved_model_args)[checkme]==vars(args)[checkme],"Command line argument and saved model disagree on '%s' "%checkme
# open saved vocab/dict and check if vocabs/dicts are compatible
with open(os.path.join(args.init_from, 'chars_vocab.pkl'), 'rb') as f:
saved_chars, saved_vocab = cPickle.load(f)
assert saved_chars==data_loader.chars, "Data and loaded model disagree on character set!"
assert saved_vocab==data_loader.vocab, "Data and loaded model disagree on dictionary mappings!"
if not os.path.isdir(args.save_dir):
os.makedirs(args.save_dir)
with open(os.path.join(args.save_dir, 'config.pkl'), 'wb') as f:
cPickle.dump(args, f)
with open(os.path.join(args.save_dir, 'chars_vocab.pkl'), 'wb') as f:
cPickle.dump((data_loader.chars, data_loader.vocab), f)
model = Model(args)
with tf.Session() as sess:
# instrument for tensorboard
summaries = tf.summary.merge_all()
writer = tf.summary.FileWriter(
os.path.join(args.log_dir, time.strftime("%Y-%m-%d-%H-%M-%S")))
writer.add_graph(sess.graph)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables(), max_to_keep=1000)
# restore model
if args.init_from is not None:
saver.restore(sess, ckpt.model_checkpoint_path)
for e in range(args.num_epochs):
sess.run(tf.assign(model.lr,
args.learning_rate * (args.decay_rate ** e)))
data_loader.reset_batch_pointer()
state = sess.run(model.initial_state)
for b in range(data_loader.num_batches):
start = time.time()
x, y = data_loader.next_batch()
feed = {model.input_data: x, model.targets: y}
if args.model == "lstm":
for i, (c, h) in enumerate(model.initial_state):
feed[c] = state[i].c
feed[h] = state[i].h
elif args.model == "rnn":
for i, tensor in enumerate(model.initial_state):
feed[tensor] = state[i]
# instrument for tensorboard
summ, train_loss, state, _ = sess.run([summaries, model.cost, model.final_state, model.train_op], feed)
writer.add_summary(summ, e * data_loader.num_batches + b)
end = time.time()
print("{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}"
.format(e * data_loader.num_batches + b,
args.num_epochs * data_loader.num_batches,
e, train_loss, end - start))
if e % args.save_every == 0 \
or (e == args.num_epochs - 1 and
b == data_loader.num_batches - 1):
# save for the last result
checkpoint_path = os.path.join(args.save_dir, 'model.ckpt')
saver.save(sess, checkpoint_path,
global_step=e)
print("model saved to {}".format(checkpoint_path))
events, vocab = pickle.load(f)
# load target piece to predict
quant = 1
with open(args.corpus_path, "r") as f:
corpus = json.load(f)["corpus"]
test_corpus = corpus["validate"]
targets = map(
lambda p: encode_json_notes(p["notes"], quant, p["time_sig_amt"]),
test_corpus)
# load model
print("Loading model...")
model = Model(saved_config, op_mode=RNNMode.SAMPLE)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
ckpt = tf.train.get_checkpoint_state(args.save_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
else:
exit()
print("Starting prediction...")
state = None
seq_xents = []
for target in targets:
xentropies: List[float] = []
# Creating a mapping from unique characters to indices
char2idx = {u: i for i, u in enumerate(vocab)}
idx2char = np.array(vocab)
text_as_int = np.array([char2idx[c] for c in input_corpus])
# Create training examples / targets
if TRAIN:
chunks = tf.data.Dataset.from_tensor_slices(text_as_int).batch(
SEQ_LENGTH + 1, drop_remainder=True)
dataset = chunks.map(split_input_target)
dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE,
drop_remainder=True)
# create model
model = Model(vocab_size, EMBEDDING_DIM, UNITS)
model.build(tf.TensorShape([BATCH_SIZE, SEQ_LENGTH]))
print('\nCreate model summary:')
model.summary()
optimizer = tf.train.AdamOptimizer() # define optimizer
checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=model)
# Directory where the checkpoints will be saved
if not os.path.exists('./checkpoints'):
os.makedirs('./checkpoints')
checkpoint_dir = './checkpoints/tweets_' + TWITTER_SCREEN_NAME
checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
###########################################
# 3. train the model or load a previous one
if TRAIN:
# load saved state
with open(os.path.join(args.save_dir, "config.pkl"), "rb") as f:
saved_config = pickle.load(f)
with open(os.path.join(args.save_dir, "events_vocab.pkl"), "rb") as f:
events, vocab = pickle.load(f)
# the second of these two imports introduces a dependency on music21
# so we lazy load this so as to not require this for a char-RNN
if saved_config.mode == DataLoader.Mode.MUSIC:
from rnn_music_rep import decode_events, divtoken, str_to_duration
from score_utils import render_music21
print("Loading model...\n")
model = Model(saved_config, op_mode=RNNMode.SAMPLE)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
ckpt = tf.train.get_checkpoint_state(args.save_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
if saved_config.mode == DataLoader.Mode.CHAR:
prime = list(args.prime)
sampled = model.sample(sess, events, vocab, args.n, prime)
print("".join(sampled))
else:
offset = 0
clock = 0
prev_dur = 0
plt.legend()
plt.ylabel("Loss")
plt.xlabel("Epoch")
plt.show()
if __name__ == "__main__":
print(8)
with open("testtrain8.p", "rb") as f:
train_x, train_y, test_x8, test_y8, mean_y8 = pickle.load(f)
model8 = Model(training_data=(train_x, train_y),
validation_data=(test_x8, test_y8),
epochs=400,
input_feature_amount=train_x.shape[2],
output_feature_amount=train_y.shape[2])
model8.build_model()
model8.model.load_weights("models/8nodesmodel.h5")
print(5)
with open("testtrain5.p", "rb") as f:
train_x, train_y, test_x5, test_y5, mean_y5 = pickle.load(f)
model5 = Model(training_data=(train_x, train_y),
validation_data=(test_x5, test_y5),
epochs=400,
input_feature_amount=train_x.shape[2],
output_feature_amount=train_y.shape[2])
assert saved_events == loader.events,\
"Data and loaded model disagree on event symbols!"
assert saved_vocab == loader.vocab,\
"Data and loaded model disagree on vocab!"
# dump model config and the text loader's events/vocab
with open(os.path.join(args.save_dir, "config.pkl"), "wb") as f:
if args.init_from is not None:
pickle.dump(saved_config, f)
else:
pickle.dump(config, f)
with open(os.path.join(args.save_dir, "events_vocab.pkl"), "wb") as f:
pickle.dump((loader.events, loader.vocab), f)
print("Initialising model, constructing graph...")
model = Model(config)
print("Starting session.")
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
summary_op = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(args.log_dir, graph=sess.graph)
saver = tf.train.Saver(tf.global_variables())
# set up RNN input embedding for visualisation in Tensorboard
proj_config = projector.ProjectorConfig()
emb = proj_config.embeddings.add()
emb.tensor_name = "embedding"
emb.metadata_path = os.path.join(args.data_dir, "event_metadata.tsv")
projector.visualize_embeddings(train_writer, proj_config)
def initialize_session():
config = tf.ConfigProto()
#config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.4
return tf.Session(config=config)
##################################################
BATCH_SIZE = 20 # 배치 사이즈
num_k = 7 # 앞에 볼 단어 개수
emb_dim = 64 # 단어 embedding dimension
hidden_dim = 128 # RNN hidden dim
learning_rate = 0.001 # Learning rate
use_clip = True # Gradient clipping 쓸지 여부
##################################################
model = Model(num_k=num_k, emb_dim=emb_dim, hidden_dim=hidden_dim, vocab_size=data.vocab_size,
use_clip=True, learning_rate=learning_rate)
sess = initialize_session()
sess.run(tf.global_variables_initializer())
def sample_test(test_input=""):
# test_input = raw_input("test text: ") # input("test text: ") for python 2, 3
test_x = np.zeros((1, num_k), dtype=np.int32)
words = test_input.split()
for i in range(min(num_k, len(words))):
test_x[0][-i-1] = data.w2idx[words[-i-1]]
out_x = sess.run(model.out_y, feed_dict={model.x: test_x})
print(out_x[0], data.idx2w[out_x[0]])
def test_model():
""" just loads the RNN model, loads some data, and configures the model accordingly. this is just to check that all the bits fit together and load properly. """ from data_loader import DataLoader from rnn_model import Model, ModelConfig dl = DataLoader(DataLoader.Mode.MUSIC, "data/chorales", 10, 20) cfg = ModelConfig(dl) model = Model(cfg)
def train(args):
# load data
iomodel = models.ModelRNNRecentGames(args.seq_length)
# the data model to use
data_model = load_data.DATA_MODEL_DETAILED
teamstats = team_stats.TeamStatsRecentGameSequence(args.seq_length)
tpl = load_data.generate_sequenced_data(iomodel, data_model, teamstats, range(0,2020))
data, labels = tpl[0], tpl[1]
filenames = load_data.split_data_to_files(data, labels, 10)
data_partition_fractions = [0.8, 0.1, 0.1]
datasets = DataSetsFiles(filenames, data_partition_fractions, load_data.read_file, randomize=False, feature_dim=2)
args.num_features = datasets.num_features
args.num_classes = 2
# load model
model = Model(args)
with tf.Session() as sess:
# instrument for tensorboard
summaries = tf.summary.merge_all()
writer = tf.summary.FileWriter(
os.path.join(args.log_dir, time.strftime("%Y-%m-%d-%H-%M-%S")))
writer.add_graph(sess.graph)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
# restore model
if args.init_from is not None:
saver.restore(sess, ckpt.model_checkpoint_path)
total_epochs = args.num_epochs * args.batch_size
for epoch in range(total_epochs):
sess.run(tf.assign(model.lr,
args.learning_rate * (args.decay_rate ** epoch)))
state = sess.run(model.initial_state)
x, y = datasets.train.next_batch(args.batch_size)
print("x, y shapes", x.shape, y.shape)
feed = {model.input_data: x, model.targets: y}
for i, (c, h) in enumerate(model.initial_state):
feed[c] = state[i].c
feed[h] = state[i].h
train_loss, state, _ = sess.run([model.cost, model.final_state, model.train_op], feed)
# instrument for tensorboard
summ, train_loss, state, _ = sess.run([summaries, model.cost, model.final_state, model.train_op], feed)
writer.add_summary(summ, epoch)
if epoch % 1000 == 0:
print("epoch {}, train_loss = {:.3f}".format(epoch, train_loss))
if epoch % args.save_every == 0\
or (epoch == total_epochs-1):
# save for the last result
checkpoint_path = os.path.join(args.save_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=epoch)
print("model saved to {}".format(checkpoint_path))
valid_x, valid_y = datasets.valid.next_batch(args.batch_size)
feed[model.input_data] = valid_x
feed[model.targets] = valid_y
print("valid shapes", valid_x.shape, valid_y.shape)
summ, valid_loss, valid_acc = sess.run([summaries, model.cost, model.accuracy], feed)
print("valid loss/acc ", valid_loss, valid_acc)