def validate(self, dev_word_lists, dev_tag_lists, word2id, tag2id):
self.model.eval()
with torch.no_grad():
val_losses = 0.
val_step = 0
for ind in range(0, len(dev_word_lists), self.batch_size):
val_step += 1
# 准备batch数据
batch_sents = dev_word_lists[ind:ind + self.batch_size]
batch_tags = dev_tag_lists[ind:ind + self.batch_size]
tensorized_sents, lengths = tensorized(batch_sents, word2id)
tensorized_sents = tensorized_sents.to(self.device)
targets, lengths = tensorized(batch_tags, tag2id)
targets = targets.to(self.device)
# forward
scores = self.model(tensorized_sents, lengths)
# 计算损失
loss = self.cal_loss_func(scores, targets,
tag2id).to(self.device)
val_losses += loss.item()
val_loss = val_losses / val_step
if val_loss < self._best_val_loss:
print("保存模型...")
save_model(self.model,
self.args.save_model + '/bilstm_crf.ckpt')
self._best_val_loss = val_loss
return val_loss
def train(sess):
if not IS_ON_NSML:
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
steps_per_epoch = train_manager.len_data
early_stop = 0
logger.info("start training")
loss = []
for epoch in range(FLAGS.max_epoch):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"loss:{:>9.6f}".format(iteration, step%steps_per_epoch, \
steps_per_epoch, np.mean(loss)))
best, dev_f1 = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
# early stopping
if best: early_stop = 0
else:
early_stop += 1
if early_stop > FLAGS.patience: break
# save model
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
if IS_ON_NSML:
nsml.report(summary=True, scope=locals(), epoch=epoch, epoch_total=FLAGS.max_epoch, train__loss=float(np.mean(loss)), valid__f1score=dev_f1)
nsml.save(epoch)
loss = []
def tournament(self, root_state):
_, scoreboard = self.self_play(root_state, best_only=False,
num_episodes=config.NUM_TOURNAMENT_EPISODES, deterministic=True)
if scoreboard["new"] > scoreboard["best"]*config.SCORING_THRESHOLD:
model_utils.save_model(self.new)
self.best = model_utils.load_model()
elif scoreboard["new"]*config.SCORING_THRESHOLD < scoreboard["best"]:
self.new = model_utils.load_model()
def train(sess, context_embeddings_op, elmo_context, elmo_ids, config):
if not IS_ON_NSML:
with open(FLAGS.necessary, "wb") as f:
pickle.dump([
word_to_id, id_to_word, pumsa_to_id, id_to_pumsa, char_to_id,
id_to_char, tag_to_id, id_to_tag, pre_to_id, id_to_pre,
suf_to_id, id_to_suf
], f)
steps_per_epoch = train_manager.len_data
early_stop = 0
logger.info("start training")
loss = []
for epoch in range(FLAGS.max_epoch):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, context_embeddings_op,
elmo_context, elmo_ids, True,
batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("epoch:{} iteration:{} step:{}/{}, "
"loss:{:>9.6f}".format(epoch, iteration, step%steps_per_epoch, \
steps_per_epoch, np.mean(loss)))
# break
best, dev_f1 = evaluate(sess, context_embeddings_op, elmo_context,
elmo_ids, model, "dev", dev_manager, id_to_tag,
logger)
# early stopping
if best:
print("best dev_f1 : %s, epoch : %s" % (dev_f1, epoch))
early_stop = 0
else:
print("dev_f1 : %s" % dev_f1)
early_stop += 1
if early_stop > FLAGS.patience:
print("Final best score : %s" % best)
break
# save model
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
if IS_ON_NSML:
nsml.report(summary=True,
scope=locals(),
epoch=epoch,
epoch_total=FLAGS.max_epoch,
train__loss=float(np.mean(loss)),
valid__f1score=dev_f1)
nsml.save(epoch)
loss = []
def train():
dataset = build_DataLoader()
vocabulary_list = sorted(dataset.vocabulary.word2index.items(),\
key=lambda x: x[1])
save_vocabulary(vocabulary_list)
vocab_size = dataset.get_vocabulary_size()
model = build_model(vocab_size, load_ckpt = True)
print(model)
optimizer = optim.SGD(model.parameters(), lr = config.learning_rate)
criterion = nn.CrossEntropyLoss()
start = time.time()
total_batch = len(dataset)
ckpts = get_ckpts()
iter_idx = 0 if len(ckpts) == 0 else max(ckpts)
print_loss_total = 0.0
milestones = init_milestone(total_batch)
n_iters = milestones[-1]
scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
milestones = milestones, gamma=0.5)
scheduler.step(iter_idx)
print('Start Training. total: %s iterations'%n_iters)
while iter_idx < n_iters:
iter_idx += 1
scheduler.step()
input_group, target_group = dataset.random_batch()
# zero gradients
optimizer.zero_grad()
# run seq2seq
_, loss = model(criterion, input_group, target_group,
teacher_forcing_ratio=1)
print_loss_total += loss.data[0]
loss.backward()
clip_grad_norm(model.parameters(), config.clip)
# update parameters
optimizer.step()
if iter_idx % config.print_every == 0:
test_loss = model_evaluate(model, criterion, dataset)
print_summary(start, iter_idx, n_iters,\
math.exp(print_loss_total / config.print_every),\
optimizer.param_groups[0]['lr'])
print('Test loss: %.4f ' % (math.exp(test_loss)))
print_loss_total = 0.0
# hot_update_lr(optimizer)
if iter_idx % config.save_every == 0:
save_model(model, iter_idx)
# break
save_model(model, iter_idx)
def main():
in_args = utils.get_model_args()
# load the model from the --arch and --hidden_units parameters
print("Loading the model...")
model = model_utils.load_model(in_args.arch, in_args.hidden_units)
# train the model
print("Now training the model...")
model = model_utils.train_model(in_args.arch, model, in_args.learn_rate,in_args.epochs,device='cuda' if in_args.gpu else 'cpu')
#save the model
print("Saving the model to {}".format(in_args.save_dir))
model_utils.save_model(model,in_args.arch,in_args.epochs,in_args.gpu,in_args.learn_rate,in_args.save_dir,in_args.output_size)
def main():
parser = argparse.ArgumentParser()
add_train_args(parser)
add_common_args(parser)
args = parser.parse_args()
add_experiment(args)
device = model_utils.get_device()
# Load dataset from disk
train_ds, dev_ds = model_utils.load_training_data(args)
# Initialize a model
model = models.get_model(args.model)(size=args.size)
# load from checkpoint if path specified
if args.load_path is not None:
model = model_utils.load_model(model, args.load_path)
# Move model to GPU if necessary
model.to(device)
# Initialize optimizer
optimizer = optim.Adam(
model.parameters(),
lr=args.learning_rate,
weight_decay=args.weight_decay,
)
# Scheduler
scheduler = optim.lr_scheduler.StepLR(optimizer, 5, 0.1, verbose=True)
os.makedirs(f'{args.save_path}/{args.experiment}')
print(f'Created new experiment: {args.experiment}')
save_arguments(args, f'{args.save_path}/{args.experiment}/args.txt')
# Train!
trained_model = train_model(
train_ds,
dev_ds,
model,
optimizer,
scheduler,
args,
)
# Save trained model
filename = f'{args.save_path}/{args.experiment}/{model.__class__.__name__}_trained.checkpoint'
model_utils.save_model(trained_model, filename)
def train_fn(model, data_loader, optimizer, save_model=False):
model.train()
fin_loss = 0
tk = tqdm(data_loader, total=len(data_loader))
for data in tk:
for key, value in data.items():
data[key] = value.to(config.DEVICE)
optimizer.zero_grad()
_, loss = model(**data)
loss.backward()
optimizer.step()
fin_loss += loss.item()
if save_model:
model_utils.save_model(model, optimizer, loss, config.CHECKPOINT_PATH)
return fin_loss / len(data_loader)
def train(sess):
logger.info("start training")
loss = []
best_dev = ""
steps_per_epoch = train_manager.len_data
early_stop = 0
writer = tf.summary.FileWriter(config.summary_path, sess.graph)
for epoch in range(config.max_epoch):
for _, batch in enumerate(train_manager.iter_batch(shuffle= True)):
step, batch_loss, summary = SRL_Model.run_step(sess, ELMo_context, ELMo_ids, True, batch)
loss.append(batch_loss)
if step % config.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("epoch:{} iteration:{} step:{}/{}, "hghg
"loss:{:>9.6f}".format(epoch, iteration, step % steps_per_epoch, \
steps_per_epoch, np.mean(loss)))
writer.add_summary(summary, step)
best, dev_acc, final_report = evaluate(sess, SRL_Model, "dev", test_manager, idx2label, logger)
if final_report != None:
best_dev = final_report
if best:
print("best dev_f1 : %s, epoch : %s" %(dev_acc, epoch))
early_stop = 0
save_model(sess, SRL_Model, logger)
else:
print("dev_f1 : %s" % dev_acc)
early_stop += 1
if early_stop > config.patience:
print("Final best test score : %s" % best)
print("============DEV=============")
for line in best_dev:
logger.info(line.strip())
break
loss = []
def main():
midi_dir = '../../../data/query_symlinks'
model_num = 12
model_dir = '../../../models/keras/01_vanilla_rnn/{}'.format(model_num)
window_size = 20
# only creates if doesn't already exist
model_utils.create_model_dir(model_dir)
files = [os.path.join(midi_dir, path) for path in os.listdir(midi_dir)]
train_generator = data_utils.get_data_generator(files[0:1000],
form=FORMAT,
window_size=20)
val_generator = data_utils.get_data_generator(files[1000:1100],
form=FORMAT,
window_size=20)
model, epoch = get_model(window_size=window_size)
print(model.summary())
model_utils.save_model(model, model_dir)
callbacks = model_utils.get_callbacks(model_dir)
print('fitting model...')
model.fit_generator(train_generator,
steps_per_epoch=10000,
epochs=10,
validation_data=val_generator,
validation_steps=2000,
verbose=1,
callbacks=callbacks,
initial_epoch=epoch)
# generate 10 tracks using random seeds
X, y = val_generator.next()
generated = model_utils.generate(model, X)
for i, midi in enumerate(generated):
file = os.path.join(model_dir, 'generated', '{}.mid'.format(i + 1))
midi.write(file.format(i + 1))
print('wrote midi file to {}'.format(file))
def train():
data_set = build_data_loader(batch_size=BATCH_SIZE)
vocabulary_list = sorted(data_set.vocabulary.word2index.items(),
key=lambda x: x[1])
save_vocabulary(vocabulary_list)
vocab_size = data_set.get_vocabulary_size()
model = build_model(vocab_size)
model_optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)
start = time.time()
data_set_len = len(data_set)
epoch = 0
print_loss_total = 0.0
print('Start Training.')
while epoch < N_EPOCHS:
epoch += 1
input_group, target_group = data_set.random_batch()
# zero gradients
model_optimizer.zero_grad()
# run seq2seq
all_decoder_outputs = model(input_group,
target_group,
teacher_forcing_ratio=1)
target_var, target_lens = target_group
# loss calculation and back-propagation
loss = masked_cross_entropy(
all_decoder_outputs.transpose(0, 1).contiguous(),
target_var.transpose(0, 1).contiguous(), target_lens)
print_loss_total += loss.data
loss.backward()
clip_grad_norm_(model.parameters(), CLIP)
# update parameters
model_optimizer.step()
if epoch % PRINT_EVERY == 0:
test_loss = model_evaluate(model, data_set)
print_summary(start, epoch,
math.exp(print_loss_total / PRINT_EVERY))
print('Test PPL: %.4f' % math.exp(test_loss))
print_loss_total = 0.0
if epoch % SAVE_EVERY == 0:
save_model(model, epoch)
# break
save_model(model, epoch)
def train_loop(model, criterion, optimizer, scheduler, trainloader,
epochs=100, epoch_start=0, T_print=200, T_save=10):
T_start = time.time()
for epoch in range(epoch_start, epochs):
running_loss = 0.0
running_loss_mini = 0.0
for i, (inputs, targets) in enumerate(trainloader):
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
outputs = model(inputs)
# remove the channel dimension
targets = torch.argmax(targets, dim=1) # convert to (N, H, W)
loss = criterion(outputs, targets)
loss.backward()
optimizer.step()
running_loss += loss.item()
running_loss_mini += loss.item()
if i % T_print == T_print-1:
T_end = time.time()
print('%d-th minibatch\tloss: %f' % (i+1, running_loss/T_print), end='\t')
print(T_end-T_start, 'secs elapsed')
running_loss_mini = 0.0
# call scheduler every epoch
scheduler.step()
# print statistics
T_end = time.time()
print('epoch %3d\tloss: %f' % (epoch + 1, running_loss))
print(T_end-T_start, 'secs elapsed')
# save
if epoch % T_save == T_save-1:
save_model(model, weights_dir, weight_fname)
print("Done!")
def new_models(mt):
print("Creating new models")
w1 = np.random.normal(0.0, 0.1, (784, FLAGS.hidden))
b1 = np.random.normal(0.0, 0.1, (FLAGS.hidden))
w2 = np.random.normal(0.0, 0.1, (FLAGS.hidden, NUM_OUTPUT))
b2 = np.random.normal(0.0, 0.1, (NUM_OUTPUT))
architecture = [FLAGS.hidden, NUM_OUTPUT]
gold = AgentModel(architecture)
gold.set_vars([(w1, b1), (w2, b2)])
print("Training gold...")
gold.train_model(mt.all_train, mt.all_test)
model_utils.save_model(gold, "gold")
digits1 = AgentModel(architecture)
digits1.set_vars([(w1, b1), (w2, b2)])
print("Training mnist1 digits...")
digits1.train_model(mt.mnist1_train, mt.mnist1_test)
model_utils.save_model(digits1, "digit1")
digits2 = AgentModel(architecture)
digits2.set_vars([(w1, b1), (w2, b2)])
print("Training mnist2 digits...")
digits2.train_model(mt.mnist2_train, mt.mnist2_test)
model_utils.save_model(digits2, "digit2")
return digits1, digits2, gold
def s2s_fit(
s2s_model,
dataloader,
save_epoch_freq,
model_save_path=None,
epochs=1,
teacher_forcing=0.5,
tb_expt=None,
print_epoch_results=True,
):
"""
s2s_model: :class:`Seq2SeqAutoEncoder`
dataloader: :class:`SentDataloader` yields train_sents, dev_sents
train_sents, dev_sents: :class:`preprocessing.BatchSentIxs`
save_epoch_freq: If 0 or None, we never save the model.
save_path: If save_epoch_freq is truthy, this must be a valid path.
"""
if save_epoch_freq:
assert model_save_path is not None
epoch_stats_list = []
for epoch_num in tqdm.tqdm_notebook(range(epochs), desc='Epochs'):
epoch_stats = run_epoch(
s2s_model=s2s_model,
dataloader=dataloader,
teacher_forcing=teacher_forcing,
)
log_losses(
epoch_stats=epoch_stats,
tb_expt=tb_expt,
epoch_num=epoch_num,
print_epoch_results=print_epoch_results,
)
epoch_stats_list.append(epoch_stats)
# TODO: Dont do any lr decay
# s2s_model.lr_scheduler.step(e_loss)
if save_epoch_freq and epoch_num % save_epoch_freq == 0:
log.info('Saving model to `{}`...'.format(model_save_path))
mu.save_model(model=s2s_model, path=model_save_path)
log.info('Saved model to `{}`.'.format(model_save_path))
return epoch_stats_list
def new_models(mt):
print("Creating new models")
w1 = np.random.normal(0.0, 0.1, (784, FLAGS.hidden))
b1 = np.random.normal(0.0, 0.1, (FLAGS.hidden))
w2 = np.random.normal(0.0, 0.1, (FLAGS.hidden, NUM_OUTPUT))
b2 = np.random.normal(0.0, 0.1, (NUM_OUTPUT))
gold = FlexibleModel(FLAGS.hidden)
gold.set_vars(w1, b1, w2, b2)
print("Training gold...")
gold.train_model(mt.all_train, mt.all_test)
model_utils.save_model(gold, "gold")
digits1 = FlexibleModel(FLAGS.hidden)
digits1.set_vars(w1, b1, w2, b2)
print("Training mnist1 digits...")
digits1.train_model(mt.mnist1_train, mt.mnist1_test)
model_utils.save_model(digits1, "digit1")
digits2 = FlexibleModel(FLAGS.hidden)
digits2.set_vars(w1, b1, w2, b2)
print("Training mnist2 digits...")
digits2.train_model(mt.mnist2_train, mt.mnist2_test)
model_utils.save_model(digits2, "digit2")
return digits1, digits2, gold
input_units = model.classifier[0].in_features
model.name = 'vgg16'
elif pre_tr_model == 'vgg19':
input_units = model.classifier[0].in_features
model.name = 'vgg19'
elif pre_tr_model == 'densenet':
input_units = model.classifier.in_features
model.name = 'densenet'
elif pre_tr_model == 'alexnet':
input_units = model.classifier[1].in_features
model.name = 'alexnet'
#building classifier of model
model = build_classifier(model, input_units, hidden_units, dropout)
print(model)
#Set criterion and optimizer
criterion = nn.NLLLoss()
optimizer = optim.Adam(model.classifier.parameters(), learning_rate)
model.to(device)
# Training model
model = train_model(model, epochs, trainloader, validloader, criterion,
optimizer, device)
# Testing model
test_model(model, testloader, device)
# Saving model
save_model(model, train_data, save_dir)
def train_model(config):
if config.start_date is not None:
print("Training start date: ", config.start_date)
if config.start_date is not None:
print("Training end date: ", config.end_date)
print("Loading training data from %s ..." % config.datafile)
train_data = None
valid_data = None
data_path = os.path.join(config.data_dir, config.datafile)
batches = BatchGenerator(data_path, config, is_training_only=True)
train_data = batches.train_batches(verbose=True)
valid_data = batches.valid_batches(verbose=True)
tf_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
with tf.Graph().as_default(), tf.Session(config=tf_config) as session:
if config.seed is not None:
tf.set_random_seed(config.seed)
print("Constructing model ...")
model = model_utils.get_model(session, config, verbose=True)
params = model_utils.get_scaling_params(config,
train_data,
verbose=True)
model.set_scaling_params(session, **params)
noise_model = None
if config.early_stop is not None:
print("Training will early stop without "
"improvement after %d epochs." % config.early_stop)
sys.stdout.flush()
train_history = list()
valid_history = list()
lr = model.set_learning_rate(session, config.learning_rate)
train_data.cache(verbose=True)
valid_data.cache(verbose=True)
for i in range(config.max_epoch):
(train_mse, valid_mse) = run_epoch(session,
model,
train_data,
valid_data,
keep_prob=config.keep_prob,
passes=config.passes,
noise_model=noise_model,
verbose=True)
print((
'Epoch: %d Train MSE: %.6f Valid MSE: %.6f Learning rate: %.4f'
) % (i + 1, train_mse, valid_mse, lr))
sys.stdout.flush()
train_history.append(train_mse)
valid_history.append(valid_mse)
if re.match("Gradient|Momentum", config.optimizer):
lr = model_utils.adjust_learning_rate(session, model, lr,
config.lr_decay,
train_history)
if not os.path.exists(config.model_dir):
print("Creating directory %s" % config.model_dir)
os.mkdir(config.model_dir)
if math.isnan(valid_mse):
print("Training failed due to nan.")
quit()
elif stop_training(config, valid_history):
print("Training stopped.")
quit()
else:
if ((config.early_stop is None)
or (valid_history[-1] <= min(valid_history))):
model_utils.save_model(session, config, i)
kernel_initializer='random_normal',
bias_initializer='random_normal'))
model.add(
keras.layers.Dense(8,
activation='relu',
kernel_initializer='random_normal',
bias_initializer='random_normal'))
model.add(
keras.layers.Dense(1,
kernel_initializer='random_normal',
bias_initializer='random_normal'))
# construct signals
N = 100
x = 500 * np.sin(np.arange(N) * np.pi * 0.1)
y = model(x.reshape(-1, 1))
y = y.numpy()
# plot signals
plt.figure()
plt.plot(x)
plt.plot(y, '--')
plt.ylim(-1.1, 1.1)
plt.savefig('python/dense.png')
# save signals
np.savetxt('test_data/dense_x_python.csv', x, delimiter=',')
np.savetxt('test_data/dense_y_python.csv', y, delimiter=',')
save_model(model, 'models/dense.json')
def train(train_data, dev_data, model, args):
timestamp = str(int(time.time() * 1e7))
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "tmp-runs", timestamp))
print("Saving the model to {}\n".format(out_dir))
if not os.path.exists(out_dir):
os.makedirs(out_dir)
best = 100
best_path = ""
sub_cycle = 0
optimizer, scheduler = _init_optimizer(model, args)
tar_train_batches = None if (args.mode == 'train_clf' or args.mode == 'test_clf') else \
data_utils.data_loader(train_data[args.tar_dataset], args.batch_size, oneEpoch=False)
src_unlbl_train_batches = None if (args.mode == 'train_clf' or args.mode == 'test_clf') else \
data_utils.data_loader(train_data[args.src_dataset[0]], args.batch_size, oneEpoch=False)
src_train_batches = data_utils.data_dict_loader(train_data,
args.src_dataset,
args.batch_size)
tar_dev_data = None if args.tar_dataset == '' else dev_data[
args.tar_dataset]
ep = 1
while True:
start = time.time()
train_res = []
if args.dispatcher:
for i in range(args.epoch_size):
cur_res = train_batch(model, next(src_train_batches),
src_unlbl_train_batches,
tar_train_batches, optimizer, args)
train_res.append(cur_res)
else:
for batch in tqdm(range(args.epoch_size), dynamic_ncols=True):
cur_res = train_batch(model, next(src_train_batches),
src_unlbl_train_batches,
tar_train_batches, optimizer, args)
train_res.append(cur_res)
end = time.time()
print("\n{}, Updates {:5d}, Time Cost: {} seconds".format(
datetime.datetime.now().strftime('%02y/%02m/%02d %H:%M:%S'),
ep * args.epoch_size, end - start))
_print_train_res(train_res, args)
# evaluate on dev set
print('=== DEV ===')
dev_res = []
for task in args.src_dataset:
writer = open(
os.path.join(out_dir, str(ep)) + '.' + task + '.out', 'w')
cur_res = evaluate_task(dev_data[task],
task,
tar_dev_data,
model,
optimizer,
args,
writer=writer)
writer.close()
dev_res.append(cur_res)
scheduler[task].step(cur_res['loss_lbl'])
dev_res = print_dev_res(dev_res, args)
# adjust the encoder loss based on avg. loss lbl plus avg. loss wd
scheduler['encoder'].step(dev_res['loss_encoder'])
# adjust the encoder loss based on avg. r2a loss
scheduler['r2a'].step(dev_res['loss_r2a'])
if (args.mode != 'train_clf' and dev_res['loss_lbl_r2a'] < best) or\
(args.mode == 'train_clf' and dev_res['loss_lbl'] < best):
best = dev_res[
'loss_lbl_r2a'] if args.mode != 'train_clf' else dev_res[
'loss_lbl']
best_path = os.path.join(out_dir, str(ep))
model_utils.save_model(model, best_path)
sub_cycle = 0
else:
sub_cycle += 1
if sub_cycle == args.patience * 2:
break
ep += 1
print("End of training. Restore the best weights")
model = model_utils.load_saved_model(best_path, args)
print('=== BEST DEV ===')
dev_res = []
for task in args.src_dataset:
cur_res = evaluate_task(dev_data[task], task, tar_dev_data, model,
None, args)
dev_res.append(cur_res)
dev_res = print_dev_res(dev_res, args)
print("Deleting model snapshot")
os.system("rm -rf {}/*".format(out_dir)) # delete model snapshot for space
if args.save:
print("Save the best model to director saved-runs")
best_dir = os.path.abspath(os.path.join(os.path.curdir, "saved-runs", args.mode, \
"-".join(args.src_dataset) + '_' + args.tar_dataset + '_' + timestamp))
if not os.path.exists(best_dir):
os.makedirs(best_dir)
best_dir = os.path.join(best_dir, 'best')
model_utils.save_model(model, best_dir)
with open(best_dir + '_args.txt', 'w') as f:
for attr, value in sorted(args.__dict__.items()):
f.write("{}={}\n".format(attr.upper(), value))
return dev_res, best_dir, model
return dev_res, out_dir, model
iterstion = step // steps_per_epoch + 1
logger.info("iteration:{} step{}/{},NER loss:{:>9.6f}".format(iterstion, step%steps_per_epoch, steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess,model,"dev", dev_manager, id_to_tag, logger)
if best:
model_utils.save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def main():
midi_dir = '../../../data/t-sne_selections/1/0/'
model_num = 7 # same as 3
model_dir = '../../../models/keras/' \
'03_vanilla_rnn_intervals/{}'.format(model_num)
window_size = 20
# only creates if doesn't already exist
model_utils.create_model_dir(model_dir)
files = [os.path.join(midi_dir, path) for path in os.listdir(midi_dir)]
val_split = 0.2
print(int(float(len(files)) * val_split))
train_generator = data_utils.get_data_generator(
files[0:int(float(len(files)) * val_split)],
form=FORMAT,
window_size=20)
val_generator = data_utils.get_data_generator(
files[int(float(len(files)) * val_split):],
form=FORMAT,
window_size=20)
# intervals = []
# for i in range(1000):
# data = train_generator.next()
# for window in data[0]:
# for interval in window:
# if interval not in intervals:
# intervals.append(interval)
# pdb.set_trace()
# for i in range(1000):
# data = train_generator.next()
# pdb.set_trace()
model, epoch = get_model(window_size=window_size)
print(model.summary())
model_utils.save_model(model, model_dir)
callbacks = model_utils.get_callbacks(model_dir)
# print('fitting model...')
model.fit_generator(train_generator,
steps_per_epoch=1000,
epochs=100,
validation_data=val_generator,
validation_steps=200,
verbose=1,
callbacks=callbacks,
initial_epoch=epoch)
# generate 10 tracks using random seeds
X, y = val_generator.next()
generated = model_utils.generate_notes_from_intervals(model, X, length=200)
for i, midi in enumerate(generated):
file = os.path.join(model_dir, 'generated', '{}.mid'.format(i + 1))
midi.write(file.format(i + 1))
print('wrote midi file to {}'.format(file))
def ppo(args):
env = environ.GymEnvironment(args.dir_path,
args.env_name,
'ppo_log',
args.model_name,
skip_frames=6)
env_render = environ.GymEnvironment(args.dir_path,
args.env_name,
'ppo_log_render',
args.model_name,
record=True,
render=args.render)
fps = env_render.env.metadata.get('video.frames_per_second', 30)
logging.info(str(env))
brain_name = env.external_brain_names[0]
tf.reset_default_graph()
ppo_model = model_utils.create_agent_model(env,
args.beta,
args.epsilon,
args.hidden_units,
args.learning_rate,
args.max_steps,
args.normalize_steps,
args.num_layers)
is_continuous = env.brains[brain_name].action_space_type == 'continuous'
use_observations = False
use_states = True
model_path = os.path.join(args.dir_path, 'models', args.model_name)
if not os.path.exists(model_path):
os.makedirs(model_path)
tf.set_random_seed(np.random.randint(1024))
init = tf.global_variables_initializer()
saver = tf.train.Saver(max_to_keep=args.keep_checkpoints)
with tf.Session() as sess:
# Instantiate model parameters
if args.load_model:
logging.info('model:loading')
ckpt = tf.train.get_checkpoint_state(model_path)
if ckpt is None:
logging.error('model could not be found:{}'.format(model_path))
saver.restore(sess, ckpt.model_checkpoint_path) # pylint:disable=E1101
else:
sess.run(init)
steps, _ = sess.run([ppo_model.global_step, ppo_model.last_reward])
summary_writer = tf.summary.FileWriter(model_path)
info = env.reset()[brain_name]
trainer = ppo_tnr.Trainer(ppo_model,
sess,
info,
is_continuous,
use_observations,
use_states,
args.train_model)
trainer_monitor = ppo_tnr.Trainer(ppo_model,
sess,
info,
is_continuous,
use_observations,
use_states,
training=False)
render_started = False
while steps <= args.max_steps or not args.train_model:
if env.global_done:
info = env.reset()[brain_name]
trainer.reset_buffers(info, total=True)
if not render_started and args.render:
renderthread = ppo_thread.RenderThread(sess,
trainer_monitor,
env_render,
brain_name,
args.normalize_steps,
fps)
renderthread.start()
render_started = True
info = trainer.take_action(info, env, brain_name, steps, args.normalize_steps)
trainer.process_experiences(info, args.time_horizon, args.gamma, args.lambd)
if len(trainer.training_buffer['actions']) > args.buffer_size and args.train_model:
if args.render:
renderthread.pause()
logging.info('optimization:started')
t = time.time()
# perform gradient descent with experience buffer
trainer.update_model(args.batch_size, args.num_epoch)
logging.info('optimization:finished:duration:{:.1f} seconds'.format(float(time.time() - t)))
if args.render:
renderthread.resume()
if steps % args.summary_freq == 0 and steps != 0 and args.train_model:
trainer.write_summary(summary_writer, steps)
if steps % args.save_freq == 0 and steps != 0 and args.train_model:
model_utils.save_model(sess, saver, model_path, steps)
if args.train_model:
steps += 1
sess.run(ppo_model.increment_step)
if len(trainer.stats['cumulative_reward']) > 0:
mean_reward = np.mean(trainer.stats['cumulative_reward'])
sess.run(ppo_model.update_reward, {ppo_model.new_reward: mean_reward})
_ = sess.run(ppo_model.last_reward)
# final save Tensorflow model
if steps != 0 and args.train_model:
model_utils.save_model(sess, saver, model_path, steps)
env.close()
model_utils.export_graph(model_path, args.env_name)
os.system('shutdown')
def train():
# 加载数据集
train_sentences = data_loader.load_sentences(FLAGS.train_file)
dev_sentences = data_loader.load_sentences(FLAGS.dev_file)
test_sentences = data_loader.load_sentences(FLAGS.test_file)
# 转换编码
data_loader.update_tag_scheme(train_sentences, FLAGS.tag_schema)
data_loader.update_tag_scheme(dev_sentences, FLAGS.tag_schema)
data_loader.update_tag_scheme(test_sentences, FLAGS.tag_schema)
# 创建单词和词典映射
if not os.path.isfile(FLAGS.map_file):
if FLAGS.pre_emb:
dico_words_train = data_loader.word_mapping(train_sentences)[0]
dico_word, word_to_id, id_to_word = data_utils.augment_with_pretrained(
dico_words_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_, word_to_id, id_to_word = data_loader.word_mapping(
train_sentences)
_, tag_to_id, id_to_tag = data_loader.tag_mapping(train_sentences)
with open(FLAGS.map_file, 'wb') as f:
pickle.dump([word_to_id, id_to_word, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, 'rb') as f:
word_to_id, id_to_word, tag_to_id, id_to_tag = pickle.load(f)
# 准备数据
train_data = data_loader.prepare_dataset(train_sentences, word_to_id,
tag_to_id)
dev_data = data_loader.prepare_dataset(dev_sentences, word_to_id,
tag_to_id)
test_data = data_loader.prepare_dataset(test_sentences, word_to_id,
tag_to_id)
# 将数据分批处理
train_manager = data_utils.BatchManager(train_data, FLAGS.batch_size)
dev_manager = data_utils.BatchManager(dev_data, FLAGS.batch_size)
test_manager = data_utils.BatchManager(test_data, FLAGS.batch_size)
# 创建不存在的文件夹
model_utils.make_path(FLAGS)
# 判断配置文件
if os.path.isfile(FLAGS.config_file):
config = model_utils.load_config(FLAGS.config_file)
else:
config = model_utils.config_model(FLAGS, word_to_id, tag_to_id)
model_utils.save_config(config, FLAGS.config_file)
# 配置印logger
log_path = os.path.join('log', FLAGS.log_file)
logger = model_utils.get_logger(log_path)
model_utils.print_config(config, logger)
tf_config = tf.ConfigProto(allow_soft_placement=True)
tf_config.gpu_options.allow_growth = True
step_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = model_utils.create(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_word, logger)
logger.info('开始训练')
loss = []
start = time.time()
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.setps_chech == 0:
iteration = step // step_per_epoch + 1
logger.info(
"iteration{}: step{}/{}, NER loss:{:>9.6f}".format(
iteration, step % step_per_epoch, step_per_epoch,
np.mean(loss)))
loss = []
best = evaluate(sess, model, 'dev', dev_manager, id_to_tag, logger)
if best:
model_utils.save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, 'test', test_manager, id_to_tag, logger)
t = time.time() - start
logger.info('cost time: %f' % t)
def train(inqueues, outqueues, net, mid_net, loss_fn, opt, loaders, args,
_logger):
def _update_grad(net, mid_net, loss_fn, grad_list):
if not args.fix_net:
for name, param in net.named_parameters():
grad_step = sum([
data['total'] * data['net'][name] for data in grad_list
]) / sum([data['total'] for data in grad_list])
param.grad = Variable(grad_step)
if mid_net is not None:
for name, param in mid_net.named_parameters():
grad_step = sum([
data['total'] * data['mid_net'][name]
for data in grad_list
]) / sum([data['total'] for data in grad_list])
param.grad = Variable(grad_step)
for i in range(args.loss_step):
for name, param in loss_fn[i].named_parameters():
grad_step = sum([
data['total'] * data['loss_fn'][i][name]
for data in grad_list
]) / sum([data['total'] for data in grad_list])
param.grad = Variable(grad_step)
for name in loss_fn[i].buffers.keys():
loss_fn[i].buffers[name].copy_(
grad_list[0]['buffer']['loss_fn'][i][name])
def _update_correct(grad_list, corrects):
for i in range(len(corrects)):
corrects[i] += sum([data['correct'][i] for data in grad_list])
def _step_stat(grad_list):
step_sample_total = sum([data['total'] for data in grad_list])
step_loss_total = sum([data['loss_total'] for data in grad_list])
return step_sample_total, step_loss_total
# Variables for training
t = time.time()
cur_epoch_training_total = 0
training_total = 0
valid_total = 0
sample_total = 0
loss_total = 0
correct_total = [0] * (args.loss_step)
recorder = None
if args.record is not None:
recorder = HistoryRecorder(args.record)
epoch = 0
epoch_end = False
while epoch < args.epoch:
data = {}
if not args.fix_net or training_total == 0:
data['fix_net'] = False
else:
data['fix_net'] = True
data['args'] = args
if not data['fix_net']:
data['net'] = net.state_dict()
if mid_net is not None:
data['mid_net'] = mid_net.state_dict()
data['loss_fn'] = []
for loss in loss_fn:
data['loss_fn'].append(loss.state_dict())
data['test'] = False
for i in range(args.worker):
inqueues[i].send(data)
grad_list = []
for i in range(args.worker):
data = outqueues[i].recv()
grad_list.append(data)
_update_grad(net, mid_net, loss_fn, grad_list)
_update_correct(grad_list, correct_total)
step_sample_total, step_loss_total = _step_stat(grad_list)
cur_epoch_training_total += step_sample_total
training_total += step_sample_total
valid_total += step_sample_total
sample_total += step_sample_total
loss_total += step_loss_total
opt.step()
if (epoch + 1) * args.epoch_len <= training_total:
_logger.info('Epoch END!!!')
epoch_end = True
if sample_total > args.observe or epoch_end:
end_str = ' END!' if epoch_end else ''
_logger.info('Epoch: %d%s Iteration: %d Loss: %.5f perTime: %.3f',
epoch, end_str, cur_epoch_training_total,
loss_total / sample_total,
(time.time() - t) / sample_total)
accs = []
for k in range(len(loss_fn)):
accs.append('acc %d: %.5f' %
(k, correct_total[k] / sample_total))
if recorder is not None:
recorder.train_acc(training_total,
correct_total[-1] / sample_total)
recorder.save_record()
_logger.info(' '.join(accs))
sample_total = 0
loss_total = 0
correct_total = [0] * (args.loss_step)
t = time.time()
if valid_total > args.check_num or (epoch_end
and epoch == args.epoch - 1):
aux = {}
aux['epoch'] = epoch
aux['cur_iter'] = cur_epoch_training_total
aux['total_iter'] = training_total
save_model(
aux, args, net, mid_net, loss_fn,
args.output + '_%d_%d' % (epoch, cur_epoch_training_total))
_logger.warning(
'Model saved to %s',
args.output + '_%d_%d' % (epoch, cur_epoch_training_total))
_logger.warning('Start validation!')
valid_start = time.time()
data = {}
data['fix_net'] = False
data['args'] = args
data['net'] = net.state_dict()
if mid_net is not None:
data['mid_net'] = mid_net.state_dict()
data['loss_fn'] = []
for loss in loss_fn:
data['loss_fn'].append(loss.state_dict())
data['test'] = True
for i in range(args.worker):
inqueues[i].send(data)
result_correct = 0
result_total = 0
for i in range(args.worker):
data = outqueues[i].recv()
result_correct += data['correct']
result_total += data['total']
result_ = result_correct / result_total
_logger.warning(
'Validation complete! Time lapse: %.3f, Test acc: %.5f' %
(time.time() - valid_start, result_))
if recorder is not None:
recorder.test_acc(training_total, result_)
recorder.save_record()
valid_total = 0
if args.fix_net:
_logger.warning('learning rate decreases from %.6f to %.6f',
args.learning_rate, args.learning_rate / 3)
args.learning_rate /= 2
opt = get_opt(net, mid_net, loss_fn, args)
if args.unfix_net_after is not None and training_total > args.unfix_net_after:
args.fix_net = False
if epoch_end and args.learning_rate > args.min_lr:
_logger.warning('learning rate decreases from %.6f to %.6f',
args.learning_rate, args.learning_rate / 3)
args.learning_rate /= 3
opt = get_opt(net, mid_net, loss_fn, args)
if epoch_end:
cur_epoch_training_total = 0
epoch_end = False
epoch += 1
def train(train_features_tensor_dataset,
dev_features_tensor_dataset,
test_features_tensor_dataset=None,
data_mode='mongo',
demo=False,
use_w2v=False):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
args.n_gpu = torch.cuda.device_count()
args.batch_size = int(args.batch_size)
if 'set random seed':
seed_everything(args.seed)
if args.n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
logger.info("train examples {}".format(len(train_features_tensor_dataset)))
num_train_steps = int(
len(train_features_tensor_dataset) / args.batch_size *
args.num_train_epochs)
steps_a_epoch = int(num_train_steps // args.num_train_epochs)
if 'create train TensorDataset':
train_sampler = RandomSampler(train_features_tensor_dataset)
train_dataloader = DataLoader(train_features_tensor_dataset,
sampler=train_sampler,
batch_size=args.batch_size,
drop_last=True)
if 'create dev TensorDataset':
dev_sampler = SequentialSampler(dev_features_tensor_dataset)
dev_dataloader = DataLoader(dev_features_tensor_dataset,
sampler=dev_sampler,
batch_size=args.batch_size,
drop_last=False)
if 'create test TensorDataset':
test_sampler = SequentialSampler(test_features_tensor_dataset)
test_dataloader = DataLoader(test_features_tensor_dataset,
sampler=test_sampler,
batch_size=args.batch_size,
drop_last=False)
log = Log(args, running_time, baseline, benchmark)
if 'create model':
bert_config = BertConfig.from_json_file(args.bert_config_file)
weight = ''
if use_w2v:
weight = load_pkl_data(args.w2v_weight)
if baseline == 'baseline1':
model = Baseline1(args, use_w2v=use_w2v, weight=weight)
elif baseline == 'baseline2':
model = Baseline2(args, use_w2v=use_w2v, weight=weight)
elif baseline == 'baseline3':
model = Baseline3(args, use_w2v=use_w2v, weight=weight)
elif baseline == 'baseline4':
model = Baseline4(args, use_w2v=use_w2v, weight=weight)
elif baseline == 'baseline5':
model = Baseline5(args, use_w2v=use_w2v, weight=weight)
elif baseline == 'baseline6':
model = Baseline6(args, use_w2v=use_w2v, weight=weight)
elif baseline == 'MSCNN':
model = MSCNN(args, use_w2v=use_w2v, weight=weight)
elif baseline == 'MSAIN':
model = MSAIN(args, use_w2v=use_w2v, weight=weight)
elif baseline == 'MLCCNN':
model = MLCCNN(args, use_w2v=use_w2v, weight=weight)
else:
raise Exception('model not exsit')
model.to(device)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
optimizer_grouped_parameters = [{
'params': [p for n, p in param_optimizer],
'weight_decay':
0.0
}]
optimizer = torch.optim.Adagrad(optimizer_grouped_parameters,
lr=args.learning_rate)
# DataParallel training
if args.n_gpu > 1:
model = torch.nn.DataParallel(model)
global_step = 0
best_top1 = 0
for curEpoch in range(int(args.num_train_epochs)):
model.train()
with tqdm(total=steps_a_epoch) as pbar:
for step, batch in enumerate(train_dataloader):
if data_mode == 'mongo':
indexes = batch[0].sort().values.tolist()
batch_data = [
item for item in collection.find(
{'index': {
'$in': indexes
}})
]
if use_w2v:
q_input_ids_w2v = torch.tensor(
[item['q_w2v_ids'] for item in batch_data],
dtype=torch.long).to(device)
p_a_input_ids_w2v = torch.tensor(
[item['p_a_w2v_ids'] for item in batch_data],
dtype=torch.long).to(device)
n_a_input_ids_w2v = torch.tensor(
[item['n_a_w2v_ids'] for item in batch_data],
dtype=torch.long).to(device)
q_input_mask_tensor = torch.tensor(
[item['q_input_mask'] for item in batch_data],
dtype=torch.uint8).to(device)
p_a_input_mask_tensor = torch.tensor(
[item['p_a_input_mask'] for item in batch_data],
dtype=torch.uint8).to(device)
n_a_input_mask_tensor = torch.tensor(
[item['n_a_input_mask'] for item in batch_data],
dtype=torch.uint8).to(device)
else:
q_input_ids_tensor = torch.tensor(
[item['q_input_ids'] for item in batch_data],
dtype=torch.long).to(device)
q_input_mask_tensor = torch.tensor(
[item['q_input_mask'] for item in batch_data],
dtype=torch.long).to(device)
q_segment_ids_tensor = torch.tensor(
[item['q_segment_ids'] for item in batch_data],
dtype=torch.long).to(device)
p_a_input_ids_tensor = torch.tensor(
[item['p_a_input_ids'] for item in batch_data],
dtype=torch.long).to(device)
p_a_input_mask_tensor = torch.tensor(
[item['p_a_input_mask'] for item in batch_data],
dtype=torch.long).to(device)
p_a_segment_ids_tensor = torch.tensor(
[item['p_a_segment_ids'] for item in batch_data],
dtype=torch.long).to(device)
n_a_input_ids_tensor = torch.tensor(
[item['n_a_input_ids'] for item in batch_data],
dtype=torch.long).to(device)
n_a_input_mask_tensor = torch.tensor(
[item['n_a_input_mask'] for item in batch_data],
dtype=torch.long).to(device)
n_a_segment_ids_tensor = torch.tensor(
[item['n_a_segment_ids'] for item in batch_data],
dtype=torch.long).to(device)
elif data_mode == 'local':
batch = tuple(input.to(device) for input in batch)
question_id, pos_ans_id, neg_ans_id, \
q_input_ids, q_input_mask, q_segment_ids, \
p_a_input_ids, p_a_input_mask, p_a_segment_ids, \
n_a_input_ids, n_a_input_mask, n_a_segment_ids = batch
if use_w2v:
loss, pre_pos_sim = model(
q_input_ids_w2v=q_input_ids_w2v,
p_a_input_ids_w2v=p_a_input_ids_w2v,
n_a_input_ids_w2v=n_a_input_ids_w2v,
q_input_mask=q_input_mask_tensor,
p_a_input_mask=p_a_input_mask_tensor,
n_a_input_mask=n_a_input_mask_tensor,
)
else:
loss, pre_pos_sim = model(
q_input_ids=q_input_ids_tensor,
q_input_mask=q_input_mask_tensor,
q_segment_ids=q_segment_ids_tensor,
p_a_input_ids=p_a_input_ids_tensor,
p_a_input_mask=p_a_input_mask_tensor,
p_a_segment_ids=p_a_segment_ids_tensor,
n_a_input_ids=n_a_input_ids_tensor,
n_a_input_mask=n_a_input_mask_tensor,
n_a_segment_ids=n_a_segment_ids_tensor)
if args.n_gpu > 1:
loss = loss.mean()
loss.backward()
if 'clip grad norm':
torch.nn.utils.clip_grad_norm_(model.parameters(),
args.max_grad_norm)
#lr_scheduler.step() # initial_lr = 0
optimizer.step()
optimizer.zero_grad()
pbar.set_description(
f'{baseline}, benchmark-{benchmark}, train loss: {loss:.6f}'
)
log.print(
f'{baseline}, benchmark-{benchmark}, step: {global_step}, train loss: {loss:.6f}'
)
pbar.update(1)
global_step += 1
if step % args.dev_step == 0 and step != 0:
top1 = dev(args,
model,
dev_dataloader,
device,
demo=False,
use_w2v=use_w2v)
if top1 > best_top1:
best_top1 = top1
logging.info(
f'{baseline}, benchmark-{benchmark}, epoch:{curEpoch}, global_step:{global_step}, dev top1:{top1:.2%}, best top1: {best_top1:.2%}'
)
log.print(
f'{baseline}, benchmark-{benchmark}, epoch:{curEpoch}, global_step:{global_step}, dev top1:{top1:.2%}, best top1: {best_top1:.2%}'
)
save_model(model, global_step, args, logging, log,
running_time, benchmark, baseline)
else:
logging.info(
f'{baseline}, benchmark-{benchmark}, epoch:{curEpoch}, global_step:{global_step}, dev top1:{top1:.2%}, best top1: {best_top1:.2%}'
)
log.print(
f'{baseline}, benchmark-{benchmark}, epoch:{curEpoch}, global_step:{global_step}, dev top1:{top1:.2%}, best top1: {best_top1:.2%}'
)
model.train()
top1 = dev(args,
model,
dev_dataloader,
device,
demo=False,
use_w2v=use_w2v)
if top1 > best_top1:
best_top1 = top1
save_model(model, global_step, args, logging, log, running_time,
benchmark, baseline)
logging.info(
f'{baseline}, benchmark-{benchmark}, epoch:{curEpoch}, global_step:{global_step}, dev top1:{top1:.2%}, best top1: {best_top1:.2%}'
)
log.print(
f'{baseline}, benchmark-{benchmark}, epoch:{curEpoch}, global_step:{global_step}, dev top1:{top1:.2%}, best top1: {best_top1:.2%}'
)
test_top1 = test(args,
model,
test_dataloader,
device,
running_time,
baseline,
benchmark,
demo=False,
use_w2v=use_w2v)
logging.info(
f'{baseline}, benchmark-{benchmark}, best test top1:{test_top1:.2%}')
log.print(
f'{baseline}, benchmark-{benchmark}, best test top1:{test_top1:.2%}')
def train_model(
train_graph: pyg.torch_geometric.data.Data,
valid_graph: pyg.torch_geometric.data.Data,
train_dl: data.DataLoader,
dev_dl: data.DataLoader,
evaluator: Evaluator,
model: nn.Module,
optimizer: optim.Optimizer,
lr_scheduler: optim.lr_scheduler._LRScheduler,
args: argparse.Namespace,
) -> nn.Module:
device = model_utils.get_device()
loss_fn = nn.functional.binary_cross_entropy
val_loss_fn = nn.functional.binary_cross_entropy
best_val_loss = torch.tensor(float('inf'))
best_val_hits = torch.tensor(0.0)
saved_checkpoints = []
writer = SummaryWriter(log_dir=f'{args.log_dir}/{args.experiment}')
for e in range(1, args.train_epochs + 1):
print(f'Training epoch {e}...')
# Training portion
torch.cuda.empty_cache()
torch.set_grad_enabled(True)
with tqdm(total=args.train_batch_size * len(train_dl)) as progress_bar:
model.train()
# Load graph into GPU
adj_t = train_graph.adj_t.to(device)
edge_index = train_graph.edge_index.to(device)
x = train_graph.x.to(device)
pos_pred = []
neg_pred = []
for i, (y_pos_edges,) in enumerate(train_dl):
y_pos_edges = y_pos_edges.to(device).T
y_neg_edges = negative_sampling(
edge_index,
num_nodes=train_graph.num_nodes,
num_neg_samples=y_pos_edges.shape[1]
).to(device)
y_batch = torch.cat([torch.ones(y_pos_edges.shape[1]), torch.zeros(
y_neg_edges.shape[1])], dim=0).to(device) # Ground truth edge labels (1 or 0)
# Forward pass on model
optimizer.zero_grad()
y_pred = model(adj_t, torch.cat(
[y_pos_edges, y_neg_edges], dim=1))
loss = loss_fn(y_pred, y_batch)
# Backward pass and optimization
loss.backward()
optimizer.step()
if args.use_scheduler:
lr_scheduler.step(loss)
batch_acc = torch.mean(
1 - torch.abs(y_batch.detach() - torch.round(y_pred.detach()))).item()
pos_pred += [y_pred[y_batch == 1].detach()]
neg_pred += [y_pred[y_batch == 0].detach()]
progress_bar.update(y_pos_edges.shape[1])
progress_bar.set_postfix(loss=loss.item(), acc=batch_acc)
writer.add_scalar(
"train/Loss", loss, ((e - 1) * len(train_dl) + i) * args.train_batch_size)
writer.add_scalar("train/Accuracy", batch_acc,
((e - 1) * len(train_dl) + i) * args.train_batch_size)
del y_pos_edges
del y_neg_edges
del y_pred
del loss
del adj_t
del edge_index
del x
# Training set evaluation Hits@K Metrics
pos_pred = torch.cat(pos_pred, dim=0)
neg_pred = torch.cat(neg_pred, dim=0)
results = {}
for K in [10, 20, 30]:
evaluator.K = K
hits = evaluator.eval({
'y_pred_pos': pos_pred,
'y_pred_neg': neg_pred,
})[f'hits@{K}']
results[f'Hits@{K}'] = hits
print()
print(f'Train Statistics')
print('*' * 30)
for k, v in results.items():
print(f'{k}: {v}')
writer.add_scalar(
f"train/{k}", v, (pos_pred.shape[0] + neg_pred.shape[0]) * e)
print('*' * 30)
del pos_pred
del neg_pred
# Validation portion
torch.cuda.empty_cache()
torch.set_grad_enabled(False)
with tqdm(total=args.val_batch_size * len(dev_dl)) as progress_bar:
model.eval()
adj_t = valid_graph.adj_t.to(device)
edge_index = valid_graph.edge_index.to(device)
x = valid_graph.x.to(device)
val_loss = 0.0
accuracy = 0
num_samples_processed = 0
pos_pred = []
neg_pred = []
for i, (edges_batch, y_batch) in enumerate(dev_dl):
edges_batch = edges_batch.T.to(device)
y_batch = y_batch.to(device)
# Forward pass on model in validation environment
y_pred = model(adj_t, edges_batch)
loss = val_loss_fn(y_pred, y_batch)
num_samples_processed += edges_batch.shape[1]
batch_acc = torch.mean(
1 - torch.abs(y_batch - torch.round(y_pred))).item()
accuracy += batch_acc * edges_batch.shape[1]
val_loss += loss.item() * edges_batch.shape[1]
pos_pred += [y_pred[y_batch == 1].detach()]
neg_pred += [y_pred[y_batch == 0].detach()]
progress_bar.update(edges_batch.shape[1])
progress_bar.set_postfix(
val_loss=val_loss / num_samples_processed,
acc=accuracy/num_samples_processed)
writer.add_scalar(
"Val/Loss", loss, ((e - 1) * len(dev_dl) + i) * args.val_batch_size)
writer.add_scalar(
"Val/Accuracy", batch_acc, ((e - 1) * len(dev_dl) + i) * args.val_batch_size)
del edges_batch
del y_batch
del y_pred
del loss
del adj_t
del edge_index
del x
# Validation evaluation Hits@K Metrics
pos_pred = torch.cat(pos_pred, dim=0)
neg_pred = torch.cat(neg_pred, dim=0)
results = {}
for K in [10, 20, 30]:
evaluator.K = K
hits = evaluator.eval({
'y_pred_pos': pos_pred,
'y_pred_neg': neg_pred,
})[f'hits@{K}']
results[f'Hits@{K}'] = hits
print()
print(f'Validation Statistics')
print('*' * 30)
for k, v in results.items():
print(f'{k}: {v}')
writer.add_scalar(
f"Val/{k}", v, (pos_pred.shape[0] + neg_pred.shape[0]) * e)
print('*' * 30)
del pos_pred
del neg_pred
# Save model if it's the best one yet.
if results['Hits@20'] > best_val_hits:
best_val_hits = results['Hits@20']
filename = f'{args.save_path}/{args.experiment}/{model.__class__.__name__}_best_val.checkpoint'
model_utils.save_model(model, filename)
print(f'Model saved!')
print(f'Best validation Hits@20 yet: {best_val_hits}')
# Save model on checkpoints.
if e % args.checkpoint_freq == 0:
filename = f'{args.save_path}/{args.experiment}/{model.__class__.__name__}_epoch_{e}.checkpoint'
model_utils.save_model(model, filename)
print(f'Model checkpoint reached!')
saved_checkpoints.append(filename)
# Delete checkpoints if there are too many
while len(saved_checkpoints) > args.num_checkpoints:
os.remove(saved_checkpoints.pop(0))
return model
def main():
parser = argparse.ArgumentParser()
add_train_args(parser)
add_common_args(parser)
args = parser.parse_args()
add_experiment(args)
device = model_utils.get_device()
# Load dataset from disk
print('Loading train data...')
train_graph, valid_graph, train_edges, eval_edges, valid_edges = model_utils.load_training_data()
if args.train_partial_graph:
train_edges['edge'] = eval_edges['edge']
train_dl = data.DataLoader(
data.TensorDataset(train_edges['edge']),
batch_size=args.train_batch_size,
shuffle=True,
)
dev_dl = data.DataLoader(
data.TensorDataset(
torch.cat([valid_edges['edge'], valid_edges['edge_neg']], dim=0),
torch.cat([torch.ones(valid_edges['edge'].shape[0]),
torch.zeros(valid_edges['edge_neg'].shape[0])], dim=0),
),
batch_size=args.val_batch_size,
shuffle=True,
)
# Initialize node embeddings
print('Computing initial embeddings')
train_graph = model_utils.initialize_embeddings(
train_graph, 'train_embeddings.pt', args.refresh_embeddings)
valid_graph = model_utils.initialize_embeddings(
valid_graph, 'valid_embeddings.pt', args.refresh_embeddings)
if not args.train_partial_graph:
train_graph = valid_graph
# Stats evaluator
evaluator = Evaluator(name='ogbl-ddi')
# Initialize a model
model = models.get_model(args.model)(
# train_graph.x.shape, train_graph.adj_t.to(device)
num_nodes=train_graph.num_nodes, adj_t=train_graph.adj_t.to(device)
)
# load from checkpoint if path specified
if args.load_path is not None:
model = model_utils.load_model(model, args.load_path)
print(f"Parameters: {model_utils.count_parameters(model)}")
# Move model to GPU if necessary
model.to(device)
# Initialize optimizer
optimizer = optim.Adam(
model.parameters(),
lr=args.learning_rate,
weight_decay=args.weight_decay,
)
# Scheduler
scheduler = optim.lr_scheduler.ReduceLROnPlateau(
optimizer,
mode='min',
factor=0.5,
patience=30,
verbose=True,
)
os.makedirs(f'{args.save_path}/{args.experiment}')
print(f'Created new experiment: {args.experiment}')
save_arguments(args, f'{args.save_path}/{args.experiment}/args.txt')
# Train!
trained_model = train_model(
train_graph,
valid_graph,
train_dl,
dev_dl,
evaluator,
model,
optimizer,
scheduler,
args,
)
# Save trained model
filename = f'{args.save_path}/{args.experiment}/{model.__class__.__name__}_trained.checkpoint'
model_utils.save_model(trained_model, filename)
bias_initializer='random_normal'))
model.add(
keras.layers.Dense(8,
activation='sigmoid',
kernel_initializer=tf.keras.initializers.Orthogonal(),
bias_initializer='random_normal'))
model.add(
keras.layers.Dense(1,
kernel_initializer='orthogonal',
bias_initializer='random_normal'))
# construct signals
N = 100
x = 10 * np.sin(np.arange(N) * np.pi * 0.1)
y = model.predict((x.reshape((1, -1, 1))))
print(y.shape)
y = y.flatten()
# plot signals
plt.figure()
plt.plot(x)
plt.plot(y, '--')
plt.ylim(-0.5, 0.5)
plt.savefig('python/conv.png')
# save signals
np.savetxt('test_data/conv_x_python.csv', x, delimiter=',')
np.savetxt('test_data/conv_y_python.csv', y, delimiter=',')
save_model(model, 'models/conv.json')
def save(self):
return save_model('model', self)
def train():
# 加载数据集
train_sentences = data_loader.load_sentences(FLAGS.train_file)
dev_sentences = data_loader.load_sentences(FLAGS.dev_file)
test_sentences = data_loader.load_sentences(FLAGS.test_file)
# 转换编码 bio转bioes
data_loader.update_tag_scheme(train_sentences, FLAGS.tag_schema)
data_loader.update_tag_scheme(test_sentences, FLAGS.tag_schema)
data_loader.update_tag_scheme(dev_sentences, FLAGS.tag_schema)
# 创建单词映射及标签映射
if not os.path.isfile(FLAGS.map_file):
if FLAGS.pre_emb:
dico_words_train = data_loader.word_mapping(train_sentences)[0]
dico_word, word_to_id, id_to_word = data_utils.augment_with_pretrained(
dico_words_train.copy(),
FLAGS.emb_file,
list(
itertools.chain.from_iterable(
[[w[0] for w in s] for s in test_sentences]
)
)
)
else:
_, word_to_id, id_to_word = data_loader.word_mapping(train_sentences)
_, tag_to_id, id_to_tag = data_loader.tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([word_to_id, id_to_word, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, 'rb') as f:
word_to_id, id_to_word, tag_to_id, id_to_tag = pickle.load(f)
train_data = data_loader.prepare_dataset(
train_sentences, word_to_id, tag_to_id
)
dev_data = data_loader.prepare_dataset(
dev_sentences, word_to_id, tag_to_id
)
test_data = data_loader.prepare_dataset(
test_sentences, word_to_id, tag_to_id
)
train_manager = data_utils.BatchManager(train_data, FLAGS.batch_size)
dev_manager = data_utils.BatchManager(dev_data, FLAGS.batch_size)
test_manager = data_utils.BatchManager(test_data, FLAGS.batch_size)
print('train_data_num %i, dev_data_num %i, test_data_num %i' % (len(train_data), len(dev_data), len(test_data)))
model_utils.make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = model_utils.load_config(FLAGS.config_file)
else:
config = model_utils.config_model(FLAGS, word_to_id, tag_to_id)
model_utils.save_config(config, FLAGS.config_file)
log_path = os.path.join("log", FLAGS.log_file)
logger = model_utils.get_logger(log_path)
model_utils.print_config(config, logger)
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch =train_manager.len_data
with tf.Session(config = tf_config) as sess:
model = model_utils.create(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_word, logger)
logger.info("开始训练")
loss = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.setps_chech== 0:
iterstion = step // steps_per_epoch + 1
logger.info("iteration:{} step{}/{},NER loss:{:>9.6f}".format(iterstion, step%steps_per_epoch, steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess,model,"dev", dev_manager, id_to_tag, logger)
if best:
model_utils.save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def main():
midi_dir = '../../../data/query_symlinks'
model_num = 3
model_dir = '../../../models/keras/02_event_rnn/{}'.format(model_num)
rythm_test = False
window_size = 20
# only creates if doesn't already exist
model_utils.create_model_dir(model_dir)
files = [os.path.join(midi_dir, path) for path in os.listdir(midi_dir)]
train_files = files[0:1000]
val_files = files[1000:1250]
# get the train/val d
train_generator = data_utils.get_data_generator(train_files,
form=data_utils.F_EVENT_WINDOW_TIMES,
window_size=20)
val_generator = data_utils.get_data_generator(val_files,
form=data_utils.F_EVENT_WINDOW_TIMES,
window_size=20)
# train_generator = scale_data(train_generator)
# val_generator = scale_data(val_generator)
X_timing_seed, _ = val_generator.next()
# pdb.set_trace()
timing_model, epoch = get_timing_model(model_dir,
window_size=window_size,
model_index=1)
note_model, _ = get_note_model(model_dir,
window_size=window_size,
model_index=0)
model_utils.save_model(note_model, model_dir, model_index=0)
model_utils.save_model(timing_model, model_dir, model_index=1)
callbacks = model_utils.get_callbacks(model_dir,
checkpoint_monitor='val_mae',
model_index=1)
# print('fitting timing model...')
# timing_model.fit_generator(train_generator,
# steps_per_epoch=data_utils.WINDOWS_PER_FILE * len(train_files),
# epochs=30,
# validation_data=val_generator,
# validation_steps=data_utils.WINDOWS_PER_FILE * len(val_files),
# verbose=1,
# callbacks=callbacks,
# initial_epoch=epoch)
train_generator = data_utils.get_data_generator(train_files,
form=data_utils.F_EVENT_WINDOW_NOTES,
window_size=20)
callbacks = model_utils.get_callbacks(model_dir,
checkpoint_monitor='val_mae',
model_index=0)
val_generator = data_utils.get_data_generator(val_files,
form=data_utils.F_EVENT_WINDOW_NOTES,
window_size=20)
# print('fitting note model...')
# note_model.fit_generator(train_generator,
# steps_per_epoch=data_utils.WINDOWS_PER_FILE * 10,
# epochs=10,
# validation_data=val_generator,
# validation_steps=data_utils.WINDOWS_PER_FILE * 2,
# verbose=1,
# callbacks=callbacks,
# initial_epoch=epoch)
# generate 10 tracks using random seeds
X_note_seed, _ = val_generator.next()
print('generating notes...')
generated_notes = model_utils.generate_notes(note_model, X_note_seed)
print('generating timings...')
# replace start/end note events with values generated from timing models
generated_timings = model_utils.generate_timings(timing_model, X_timing_seed)
for i, midi in enumerate(generated_notes):
for instrument in midi.instruments:
wall_time = 0
if rythm_test:
instrument.program = 125
for j, note in enumerate(instrument.notes):
if rythm_test:
note.pitch = 60
offset = generated_timings[i][j][0]
duration = generated_timings[i][j][1]
note.start = wall_time + offset
note.end = wall_time + offset + duration
wall_time = wall_time + offset + duration
for i, midi in enumerate(generated_notes):
file = os.path.join(model_dir, 'generated', '{}.mid'.format(i + 1))
midi.write(file.format(i + 1))
print('wrote midi file to {}'.format(file))
