def build_classifier(args, n_class, t_vocab, n_vocab):
model = nets.uniLSTM_iVAT(n_vocab=n_vocab,
emb_dim=args.emb_dim,
hidden_dim=args.hidden_dim,
use_dropout=args.dropout,
n_layers=args.n_layers,
hidden_classifier=args.hidden_cls_dim,
use_adv=args.use_adv,
xi_var=args.xi_var,
n_class=n_class,
args=args)
model.train_vocab_size = t_vocab
model.vocab_size = n_vocab
model.logging = logging
if args.pretrained_model != '':
# load pretrained LM model
pretrain_model = lm_nets.RNNForLM(
n_vocab,
1024,
args.n_layers,
0.50,
share_embedding=False,
adaptive_softmax=args.adaptive_softmax)
serializers.load_npz(args.pretrained_model, pretrain_model)
pretrain_model.lstm = pretrain_model.rnn
model.set_pretrained_lstm(pretrain_model, word_only=args.word_only)
return model
def main():
logging.basicConfig(
format='%(asctime)s : %(threadName)s : %(levelname)s : %(message)s',
level=logging.INFO)
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--gpu', '-g', default=-1, type=int,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--batchsize', dest='batchsize', type=int,
default=32, help='learning minibatch size')
parser.add_argument('--batchsize_semi', dest='batchsize_semi', type=int,
default=64, help='learning minibatch size')
parser.add_argument('--n_epoch', dest='n_epoch', type=int, default=30,
help='n_epoch')
parser.add_argument('--pretrained_model', dest='pretrained_model',
type=str, default='', help='pretrained_model')
parser.add_argument('--use_unlabled_to_vocab', dest='use_unlabled_to_vocab',
type=int, default=1, help='use_unlabled_to_vocab')
parser.add_argument('--use_rational', dest='use_rational',
type=int, default=0, help='use_rational')
parser.add_argument('--save_name', dest='save_name', type=str,
default='sentiment_model', help='save_name')
parser.add_argument('--n_layers', dest='n_layers', type=int,
default=1, help='n_layers')
parser.add_argument('--alpha', dest='alpha',
type=float, default=0.001, help='alpha')
parser.add_argument('--alpha_decay', dest='alpha_decay',
type=float, default=0.0, help='alpha_decay')
parser.add_argument('--clip', dest='clip',
type=float, default=5.0, help='clip')
parser.add_argument('--debug_mode', dest='debug_mode',
type=int, default=0, help='debug_mode')
parser.add_argument('--use_exp_decay', dest='use_exp_decay',
type=int, default=1, help='use_exp_decay')
parser.add_argument('--load_trained_lstm', dest='load_trained_lstm',
type=str, default='', help='load_trained_lstm')
parser.add_argument('--freeze_word_emb', dest='freeze_word_emb',
type=int, default=0, help='freeze_word_emb')
parser.add_argument('--dropout', dest='dropout',
type=float, default=0.50, help='dropout')
parser.add_argument('--use_adv', dest='use_adv',
type=int, default=0, help='use_adv')
parser.add_argument('--xi_var', dest='xi_var',
type=float, default=1.0, help='xi_var')
parser.add_argument('--xi_var_first', dest='xi_var_first',
type=float, default=1.0, help='xi_var_first')
parser.add_argument('--lower', dest='lower',
type=int, default=1, help='lower')
parser.add_argument('--nl_factor', dest='nl_factor', type=float,
default=1.0, help='nl_factor')
parser.add_argument('--min_count', dest='min_count', type=int,
default=1, help='min_count')
parser.add_argument('--ignore_unk', dest='ignore_unk', type=int,
default=0, help='ignore_unk')
parser.add_argument('--use_semi_data', dest='use_semi_data',
type=int, default=0, help='use_semi_data')
parser.add_argument('--add_labeld_to_unlabel', dest='add_labeld_to_unlabel',
type=int, default=1, help='add_labeld_to_unlabel')
parser.add_argument('--norm_sentence_level', dest='norm_sentence_level',
type=int, default=1, help='norm_sentence_level')
parser.add_argument('--dataset', default='imdb',
choices=['imdb', 'elec', 'rotten', 'dbpedia', 'rcv1', 'fce'])
parser.add_argument('--eval', dest='eval', type=int, default=0, help='eval')
parser.add_argument('--emb_dim', dest='emb_dim', type=int,
default=256, help='emb_dim')
parser.add_argument('--hidden_dim', dest='hidden_dim', type=int,
default=1024, help='hidden_dim')
parser.add_argument('--hidden_cls_dim', dest='hidden_cls_dim', type=int,
default=30, help='hidden_cls_dim')
parser.add_argument('--adaptive_softmax', dest='adaptive_softmax',
type=int, default=1, help='adaptive_softmax')
parser.add_argument('--random_seed', dest='random_seed', type=int,
default=1234, help='random_seed')
parser.add_argument('--n_class', dest='n_class', type=int,
default=2, help='n_class')
parser.add_argument('--word_only', dest='word_only', type=int,
default=0, help='word_only')
# iVAT
parser.add_argument('--use_attn_d', dest='use_attn_d',
type=int, default=0, help='use_attn_d')
parser.add_argument('--nn_k', dest='nn_k', type=int, default=10, help='nn_k')
parser.add_argument('--nn_k_offset', dest='nn_k_offset',
type=int, default=1, help='nn_k_offset')
parser.add_argument('--online_nn', dest='online_nn',
type=int, default=0, help='online_nn')
parser.add_argument('--use_limit_vocab', dest='use_limit_vocab', type=int,
default=1, help='use_limit_vocab')
parser.add_argument('--batchsize_nn', dest='batchsize_nn',
type=int, default=10, help='batchsize_nn')
parser.add_argument('--update_nearest_epoch', dest='update_nearest_epoch',
type=int, default=1, help='update_nearest_epoch')
parser.add_argument('--use_seq_labeling', dest='use_seq_labeling',
type=int, default=0, help='use_seq_labeling')
parser.add_argument('--use_bilstm', dest='use_bilstm',
type=int, default=0, help='use_bilstm')
args = parser.parse_args()
batchsize = args.batchsize
batchsize_semi = args.batchsize_semi
print(args)
random.seed(args.random_seed)
np.random.seed(args.random_seed)
os.environ["CHAINER_SEED"] = str(args.random_seed)
os.makedirs("models", exist_ok=True)
if args.debug_mode:
chainer.set_debug(True)
use_unlabled_to_vocab = args.use_unlabled_to_vocab
lower = args.lower == 1
n_char_vocab = 1
n_class = 2
if args.dataset == 'imdb':
vocab_obj, dataset, lm_data, t_vocab = utils.load_dataset_imdb(
include_pretrain=use_unlabled_to_vocab, lower=lower,
min_count=args.min_count, ignore_unk=args.ignore_unk,
use_semi_data=args.use_semi_data,
add_labeld_to_unlabel=args.add_labeld_to_unlabel)
(train_x, train_x_len, train_y,
dev_x, dev_x_len, dev_y,
test_x, test_x_len, test_y) = dataset
vocab, vocab_count = vocab_obj
n_class = 2
# TODO: add other dataset code
elif args.dataset == 'fce':
vocab, doc_counts, dataset, lm_dataset, w2v = utils.load_fce(lower=args.lower, min_count=args.min_count, ignore_unk=False, use_w2v_flag=0, use_semi_data=args.use_semi_data)
(train_x, train_x_len, train_y,
dev_x, dev_x_len, dev_y,
test_x, test_x_len, test_y) = dataset
vocab_count = doc_counts
train_vocab_size = len(vocab)
lm_dataset = (train_x, train_x_len)
if args.use_semi_data:
semi_train_x, semi_train_x_len = lm_data
print('train_vocab_size:', t_vocab)
vocab_inv = dict([(widx, w) for w, widx in vocab.items()])
print('vocab_inv:', len(vocab_inv))
xp = cuda.cupy if args.gpu >= 0 else np
if args.gpu >= 0:
cuda.get_device(args.gpu).use()
xp.random.seed(args.random_seed)
n_vocab = len(vocab)
model = nets.uniLSTM_iVAT(n_vocab=n_vocab, emb_dim=args.emb_dim,
hidden_dim=args.hidden_dim,
use_dropout=args.dropout, n_layers=args.n_layers,
hidden_classifier=args.hidden_cls_dim,
use_adv=args.use_adv, xi_var=args.xi_var,
n_class=n_class, args=args)
model.train_vocab_size = t_vocab
model.vocab_size = n_vocab
model.logging = logging
if args.pretrained_model != '':
# load pretrained LM model
pretrain_model = lm_nets.RNNForLM(n_vocab, 1024, args.n_layers, 0.50,
share_embedding=False,
adaptive_softmax=args.adaptive_softmax)
serializers.load_npz(args.pretrained_model, pretrain_model)
pretrain_model.lstm = pretrain_model.rnn
model.set_pretrained_lstm(pretrain_model, word_only=args.word_only)
all_nn_flag = args.use_attn_d
if all_nn_flag and args.online_nn == 0:
word_embs = model.word_embed.W.data
model.norm_word_embs = word_embs / np.linalg.norm(word_embs, axis=1).reshape(-1, 1)
model.norm_word_embs = np.array(model.norm_word_embs, dtype=np.float32)
if args.load_trained_lstm != '':
serializers.load_hdf5(args.load_trained_lstm, model)
if args.gpu >= 0:
model.to_gpu()
if all_nn_flag and args.online_nn == 0:
model.compute_all_nearest_words(top_k=args.nn_k)
# check nearest words
def most_sims(word):
if word not in vocab:
logging.info('[not found]:{}'.format(word))
return False
idx = vocab[word]
idx_gpu = xp.array([idx], dtype=xp.int32)
top_idx = model.get_nearest_words(idx_gpu)
sim_ids = top_idx[0]
words = [vocab_inv[int(i)] for i in sim_ids]
word_line = ','.join(words)
logging.info('{}\t\t{}'.format(word, word_line))
most_sims(u'good')
most_sims(u'this')
most_sims(u'that')
most_sims(u'awesome')
most_sims(u'bad')
most_sims(u'wrong')
def evaluate(x_set, x_length_set, y_set):
chainer.config.train = False
chainer.config.enable_backprop = False
iteration_list = range(0, len(x_set), batchsize)
correct_cnt = 0
total_cnt = 0.0
predicted_np = []
for i_index, index in enumerate(iteration_list):
x = [to_gpu(_x) for _x in x_set[index:index + batchsize]]
x_length = x_length_set[index:index + batchsize]
if args.use_seq_labeling:
# for sequence labeling (Grammaly error detection)
y_flat = np.concatenate([train_y[_i] for _i in sample_idx], axis=0).astype(np.int32)
y = to_gpu(y_flat)
else:
y = to_gpu(y_set[index:index + batchsize])
output = model(x, x_length)
predict = xp.argmax(output.data, axis=1)
correct_cnt += xp.sum(predict == y)
total_cnt += len(y)
accuracy = (correct_cnt / total_cnt) * 100.0
chainer.config.enable_backprop = True
return accuracy
def get_unlabled(perm_semi, i_index):
index = i_index * batchsize_semi
sample_idx = perm_semi[index:index + batchsize_semi]
x = [to_gpu(semi_train_x[_i]) for _i in sample_idx]
x_length = [semi_train_x_len[_i] for _i in sample_idx]
return x, x_length
base_alpha = args.alpha
opt = optimizers.Adam(alpha=base_alpha)
opt.setup(model)
opt.add_hook(chainer.optimizer.GradientClipping(args.clip))
if args.freeze_word_emb:
model.freeze_word_emb()
prev_dev_accuracy = 0.0
global_step = 0.0
adv_rep_num_statics = {}
adv_rep_pos_statics = {}
if args.eval:
dev_accuracy = evaluate(dev_x, dev_x_len, dev_y)
log_str = ' [dev] accuracy:{}, length:{}'.format(str(dev_accuracy))
logging.info(log_str)
# test
test_accuracy = evaluate(test_x, test_x_len, test_y)
log_str = ' [test] accuracy:{}, length:{}'.format(str(test_accuracy))
logging.info(log_str)
for epoch in range(args.n_epoch):
logging.info('epoch:' + str(epoch))
# train
model.cleargrads()
chainer.config.train = True
iteration_list = range(0, len(train_x), batchsize)
perm = np.random.permutation(len(train_x))
if args.use_semi_data:
perm_semi = [np.random.permutation(len(semi_train_x)) for _ in range(2)]
perm_semi = np.concatenate(perm_semi, axis=0)
def idx_func(shape):
return xp.arange(shape).astype(xp.int32)
sum_loss = 0.0
sum_loss_z = 0.0
sum_loss_z_sparse = 0.0
sum_loss_label = 0.0
avg_rate = 0.0
avg_rate_num = 0.0
correct_cnt = 0
total_cnt = 0.0
N = len(iteration_list)
is_adv_example_list = []
is_adv_example_disc_list = []
is_adv_example_disc_craft_list = []
y_np = []
predicted_np = []
save_items = []
for i_index, index in enumerate(iteration_list):
global_step += 1.0
model.set_train(True)
sample_idx = perm[index:index + batchsize]
x = [to_gpu(train_x[_i]) for _i in sample_idx]
x_length = [train_x_len[_i] for _i in sample_idx]
if args.use_seq_labeling:
# for sequence labeling (Grammaly error detection)
y_flat = np.concatenate([train_y[_i] for _i in sample_idx], axis=0).astype(np.int32)
y = to_gpu(y_flat)
else:
# for sentiment classification
y = to_gpu(train_y[sample_idx])
d = None
# Classification loss
output = model(x, x_length)
output_original = output
loss = F.softmax_cross_entropy(output, y, normalize=True)
if args.use_adv or args.use_semi_data:
# Adversarial Training
if args.use_adv:
output = model(x, x_length, first_step=True, d=None)
# Adversarial loss (First step)
loss_adv_first = F.softmax_cross_entropy(output, y, normalize=True)
model.cleargrads()
loss_adv_first.backward()
if args.use_attn_d:
# iAdv
attn_d_grad = model.attention_d_var.grad
attn_d_grad = F.normalize(attn_d_grad, axis=1)
# Get directional vector
dir_normed = model.dir_normed.data
attn_d = F.broadcast_to(attn_d_grad, dir_normed.shape).data
d = xp.sum(attn_d * dir_normed, axis=1)
else:
# Adv
d = model.d_var.grad
output = model(x, x_length, d=d)
# Adversarial loss
loss_adv = F.softmax_cross_entropy(output, y, normalize=True)
loss += loss_adv * args.nl_factor
# Virtual Adversarial Training
if args.use_semi_data:
x, length = get_unlabled(perm_semi, i_index)
output_original = model(x, length)
output_vat = model(x, length, first_step=True, d=None)
loss_vat_first = nets.kl_loss(xp, output_original.data, output_vat)
model.cleargrads()
loss_vat_first.backward()
if args.use_attn_d:
# iVAT (ours)
attn_d_grad = model.attention_d_var.grad
attn_d_grad = F.normalize(attn_d_grad, axis=1)
# Get directional vector
dir_normed = model.dir_normed.data
attn_d = F.broadcast_to(attn_d_grad, dir_normed.shape).data
d_vat = xp.sum(attn_d * dir_normed, axis=1)
else:
# VAT
d_vat = model.d_var.grad
output_vat = model(x, length, d=d_vat)
loss_vat = nets.kl_loss(xp, output_original.data, output_vat)
loss += loss_vat
predict = xp.argmax(output.data, axis=1)
correct_cnt += xp.sum(predict == y)
total_cnt += len(y)
# update
model.cleargrads()
loss.backward()
opt.update()
if args.alpha_decay > 0.0:
if args.use_exp_decay:
opt.hyperparam.alpha = (base_alpha) * (args.alpha_decay**global_step)
else:
opt.hyperparam.alpha *= args.alpha_decay # 0.9999
sum_loss += loss.data
accuracy = (correct_cnt / total_cnt) * 100.0
logging.info(' [train] sum_loss: {}'.format(sum_loss / N))
logging.info(' [train] apha:{}, global_step:{}'.format(opt.hyperparam.alpha, global_step))
logging.info(' [train] accuracy:{}'.format(accuracy))
model.set_train(False)
# dev
dev_accuracy = evaluate(dev_x, dev_x_len, dev_y)
log_str = ' [dev] accuracy:{}'.format(str(dev_accuracy))
logging.info(log_str)
# test
test_accuracy = evaluate(test_x, test_x_len, test_y)
log_str = ' [test] accuracy:{}'.format(str(test_accuracy))
logging.info(log_str)
last_epoch_flag = args.n_epoch - 1 == epoch
if prev_dev_accuracy < dev_accuracy:
logging.info(' => '.join([str(prev_dev_accuracy), str(dev_accuracy)]))
result_str = 'dev_acc_' + str(dev_accuracy)
result_str += '_test_acc_' + str(test_accuracy)
model_filename = './models/' + '_'.join([args.save_name,
str(epoch), result_str])
# if len(sentences_train_list) == 1:
serializers.save_hdf5(model_filename + '.model', model)
prev_dev_accuracy = dev_accuracy
nn_update_flag = args.update_nearest_epoch > 0 and (epoch % args.update_nearest_epoch == 0)
if all_nn_flag and nn_update_flag and args.online_nn == 0:
model.cleargrads()
x = None
x_length = None
y = None
model.compute_all_nearest_words(top_k=args.nn_k)
def main():
logging.basicConfig(
format='%(asctime)s : %(threadName)s : %(levelname)s : %(message)s',
level=logging.INFO)
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--gpu',
'-g',
default=-1,
type=int,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--batchsize',
dest='batchsize',
type=int,
default=32,
help='learning minibatch size')
parser.add_argument('--batchsize_semi',
dest='batchsize_semi',
type=int,
default=64,
help='learning minibatch size')
parser.add_argument('--n_epoch',
dest='n_epoch',
type=int,
default=30,
help='n_epoch')
parser.add_argument('--pretrained_model',
dest='pretrained_model',
type=str,
default='',
help='pretrained_model')
parser.add_argument('--use_unlabled_to_vocab',
dest='use_unlabled_to_vocab',
type=int,
default=1,
help='use_unlabled_to_vocab')
parser.add_argument('--use_rational',
dest='use_rational',
type=int,
default=0,
help='use_rational')
parser.add_argument('--save_name',
dest='save_name',
type=str,
default='sentiment_model',
help='save_name')
parser.add_argument('--n_layers',
dest='n_layers',
type=int,
default=1,
help='n_layers')
parser.add_argument('--alpha',
dest='alpha',
type=float,
default=0.001,
help='alpha')
parser.add_argument('--alpha_decay',
dest='alpha_decay',
type=float,
default=0.0,
help='alpha_decay')
parser.add_argument('--clip',
dest='clip',
type=float,
default=5.0,
help='clip')
parser.add_argument('--debug_mode',
dest='debug_mode',
type=int,
default=0,
help='debug_mode')
parser.add_argument('--use_exp_decay',
dest='use_exp_decay',
type=int,
default=1,
help='use_exp_decay')
parser.add_argument('--load_trained_lstm',
dest='load_trained_lstm',
type=str,
default='',
help='load_trained_lstm')
parser.add_argument('--freeze_word_emb',
dest='freeze_word_emb',
type=int,
default=0,
help='freeze_word_emb')
parser.add_argument('--dropout',
dest='dropout',
type=float,
default=0.50,
help='dropout')
parser.add_argument('--use_adv',
dest='use_adv',
type=int,
default=0,
help='use_adv')
parser.add_argument('--xi_var',
dest='xi_var',
type=float,
default=1.0,
help='xi_var')
parser.add_argument('--xi_var_first',
dest='xi_var_first',
type=float,
default=1.0,
help='xi_var_first')
parser.add_argument('--lower',
dest='lower',
type=int,
default=0,
help='lower')
parser.add_argument('--nl_factor',
dest='nl_factor',
type=float,
default=1.0,
help='nl_factor')
parser.add_argument('--min_count',
dest='min_count',
type=int,
default=1,
help='min_count')
parser.add_argument('--ignore_unk',
dest='ignore_unk',
type=int,
default=0,
help='ignore_unk')
parser.add_argument('--use_semi_data',
dest='use_semi_data',
type=int,
default=0,
help='use_semi_data')
parser.add_argument('--add_labeld_to_unlabel',
dest='add_labeld_to_unlabel',
type=int,
default=1,
help='add_labeld_to_unlabel')
parser.add_argument('--norm_sentence_level',
dest='norm_sentence_level',
type=int,
default=1,
help='norm_sentence_level')
parser.add_argument('--dataset',
default='imdb',
choices=['imdb', 'elec', 'rotten', 'dbpedia', 'rcv1'])
parser.add_argument('--eval',
dest='eval',
type=int,
default=0,
help='eval')
parser.add_argument('--emb_dim',
dest='emb_dim',
type=int,
default=256,
help='emb_dim')
parser.add_argument('--hidden_dim',
dest='hidden_dim',
type=int,
default=1024,
help='hidden_dim')
parser.add_argument('--hidden_cls_dim',
dest='hidden_cls_dim',
type=int,
default=30,
help='hidden_cls_dim')
parser.add_argument('--adaptive_softmax',
dest='adaptive_softmax',
type=int,
default=1,
help='adaptive_softmax')
parser.add_argument('--random_seed',
dest='random_seed',
type=int,
default=1234,
help='random_seed')
parser.add_argument('--n_class',
dest='n_class',
type=int,
default=2,
help='n_class')
parser.add_argument('--word_only',
dest='word_only',
type=int,
default=0,
help='word_only')
# iVAT
parser.add_argument('--use_attn_d',
dest='use_attn_d',
type=int,
default=0,
help='use_attn_d')
parser.add_argument('--nn_k',
dest='nn_k',
type=int,
default=10,
help='nn_k')
parser.add_argument('--nn_k_offset',
dest='nn_k_offset',
type=int,
default=1,
help='nn_k_offset')
parser.add_argument('--online_nn',
dest='online_nn',
type=int,
default=0,
help='online_nn')
parser.add_argument('--use_limit_vocab',
dest='use_limit_vocab',
type=int,
default=1,
help='use_limit_vocab')
parser.add_argument('--batchsize_nn',
dest='batchsize_nn',
type=int,
default=10,
help='batchsize_nn')
# Visualize
parser.add_argument('--analysis_mode',
dest='analysis_mode',
type=int,
default=0,
help='analysis_mode')
parser.add_argument('--analysis_limit',
dest='analysis_limit',
type=int,
default=100,
help='analysis_limit')
args = parser.parse_args()
batchsize = args.batchsize
batchsize_semi = args.batchsize_semi
print(args)
random.seed(args.random_seed)
np.random.seed(args.random_seed)
os.environ["CHAINER_SEED"] = str(args.random_seed)
os.makedirs("models", exist_ok=True)
if args.debug_mode:
chainer.set_debug(True)
use_unlabled_to_vocab = args.use_unlabled_to_vocab
lower = args.lower == 1
n_char_vocab = 1
n_class = 2
if args.dataset == 'imdb':
vocab_obj, dataset, lm_data, t_vocab = utils.load_dataset_imdb(
include_pretrain=use_unlabled_to_vocab,
lower=lower,
min_count=args.min_count,
ignore_unk=args.ignore_unk,
use_semi_data=args.use_semi_data,
add_labeld_to_unlabel=args.add_labeld_to_unlabel)
(train_x, train_x_len, train_y, dev_x, dev_x_len, dev_y, test_x,
test_x_len, test_y) = dataset
vocab, vocab_count = vocab_obj
n_class = 2
# TODO: add other dataset code
if args.use_semi_data:
semi_train_x, semi_train_x_len = lm_data
print('train_vocab_size:', t_vocab)
vocab_inv = dict([(widx, w) for w, widx in vocab.items()])
print('vocab_inv:', len(vocab_inv))
xp = cuda.cupy if args.gpu >= 0 else np
if args.gpu >= 0:
cuda.get_device(args.gpu).use()
xp.random.seed(args.random_seed)
n_vocab = len(vocab)
model = nets.uniLSTM_iVAT(n_vocab=n_vocab,
emb_dim=args.emb_dim,
hidden_dim=args.hidden_dim,
use_dropout=args.dropout,
n_layers=args.n_layers,
hidden_classifier=args.hidden_cls_dim,
use_adv=args.use_adv,
xi_var=args.xi_var,
n_class=n_class,
args=args)
model.train_vocab_size = t_vocab
model.vocab_size = n_vocab
model.logging = logging
if args.pretrained_model != '':
# load pretrained LM model
pretrain_model = lm_nets.RNNForLM(
n_vocab,
1024,
args.n_layers,
0.50,
share_embedding=False,
adaptive_softmax=args.adaptive_softmax)
serializers.load_npz(args.pretrained_model, pretrain_model)
pretrain_model.lstm = pretrain_model.rnn
model.set_pretrained_lstm(pretrain_model, word_only=args.word_only)
all_nn_flag = args.use_attn_d
if all_nn_flag and args.online_nn == 0:
word_embs = model.word_embed.W.data
model.norm_word_embs = word_embs / np.linalg.norm(
word_embs, axis=1).reshape(-1, 1)
model.norm_word_embs = np.array(model.norm_word_embs, dtype=np.float32)
if args.load_trained_lstm != '':
serializers.load_hdf5(args.load_trained_lstm, model)
if args.gpu >= 0:
model.to_gpu()
# Visualize mode
if args.analysis_mode:
def sort_statics(_x_len, name=''):
sorted_len = sorted([(x_len, idx)
for idx, x_len in enumerate(_x_len)],
key=lambda x: x[0])
return [idx for _len, idx in sorted_len]
test_sorted = sort_statics(test_x_len, 'test')
if args.analysis_limit > 0:
test_sorted = test_sorted[:args.analysis_limit]
if all_nn_flag and args.online_nn == 0:
model.compute_all_nearest_words(top_k=args.nn_k)
# check nearest words
def most_sims(word):
if word not in vocab:
logging.info('[not found]:{}'.format(word))
return False
idx = vocab[word]
idx_gpu = xp.array([idx], dtype=xp.int32)
top_idx = model.get_nearest_words(idx_gpu)
sim_ids = top_idx[0]
words = [vocab_inv[int(i)] for i in sim_ids]
word_line = ','.join(words)
logging.info('{}\t\t{}'.format(word, word_line))
most_sims(u'good')
most_sims(u'this')
most_sims(u'that')
most_sims(u'awesome')
most_sims(u'bad')
most_sims(u'wrong')
def evaluate(x_set, x_length_set, y_set):
chainer.config.train = False
chainer.config.enable_backprop = False
iteration_list = range(0, len(x_set), batchsize)
correct_cnt = 0
total_cnt = 0.0
predicted_np = []
for i_index, index in enumerate(iteration_list):
x = [to_gpu(_x) for _x in x_set[index:index + batchsize]]
x_length = x_length_set[index:index + batchsize]
y = to_gpu(y_set[index:index + batchsize])
output = model(x, x_length)
predict = xp.argmax(output.data, axis=1)
correct_cnt += xp.sum(predict == y)
total_cnt += len(y)
accuracy = (correct_cnt / total_cnt) * 100.0
chainer.config.enable_backprop = True
return accuracy
def get_unlabled(perm_semi, i_index):
index = i_index * batchsize_semi
sample_idx = perm_semi[index:index + batchsize_semi]
x = [to_gpu(semi_train_x[_i]) for _i in sample_idx]
x_length = [semi_train_x_len[_i] for _i in sample_idx]
return x, x_length
base_alpha = args.alpha
opt = optimizers.Adam(alpha=base_alpha)
opt.setup(model)
opt.add_hook(chainer.optimizer.GradientClipping(args.clip))
if args.freeze_word_emb:
model.freeze_word_emb()
prev_dev_accuracy = 0.0
global_step = 0.0
adv_rep_num_statics = {}
adv_rep_pos_statics = {}
if args.eval:
dev_accuracy = evaluate(dev_x, dev_x_len, dev_y)
log_str = ' [dev] accuracy:{}, length:{}'.format(str(dev_accuracy))
logging.info(log_str)
# test
test_accuracy = evaluate(test_x, test_x_len, test_y)
log_str = ' [test] accuracy:{}, length:{}'.format(str(test_accuracy))
logging.info(log_str)
for epoch in range(args.n_epoch):
logging.info('epoch:' + str(epoch))
# train
model.cleargrads()
chainer.config.train = True
iteration_list = range(0, len(train_x), batchsize)
if args.analysis_mode:
# Visualize mode
iteration_list = range(0, len(test_sorted), batchsize)
chainer.config.train = False
chainer.config.enable_backprop = True
chainer.config.cudnn_deterministic = True
chainer.config.use_cudnn = 'never'
perm = np.random.permutation(len(train_x))
if args.use_semi_data:
perm_semi = [
np.random.permutation(len(semi_train_x)) for _ in range(2)
]
perm_semi = np.concatenate(perm_semi, axis=0)
# print 'perm_semi:', perm_semi.shape
def idx_func(shape):
return xp.arange(shape).astype(xp.int32)
sum_loss = 0.0
sum_loss_z = 0.0
sum_loss_z_sparse = 0.0
sum_loss_label = 0.0
avg_rate = 0.0
avg_rate_num = 0.0
correct_cnt = 0
total_cnt = 0.0
N = len(iteration_list)
is_adv_example_list = []
is_adv_example_disc_list = []
is_adv_example_disc_craft_list = []
y_np = []
predicted_np = []
save_items = []
vis_lists = []
for i_index, index in enumerate(iteration_list):
global_step += 1.0
model.set_train(True)
sample_idx = [test_sorted[i_index]]
x = [to_gpu(test_x[_i]) for _i in sample_idx]
x_length = [test_x_len[_i] for _i in sample_idx]
y = to_gpu(test_y[sample_idx])
d = None
d_hidden = None
# Classification loss
output = model(x, x_length)
output_original = output
loss = F.softmax_cross_entropy(output, y, normalize=True)
# Adversarial Training
output = model(x, x_length, first_step=True, d=None)
# Adversarial loss (First step)
loss_adv_first = F.softmax_cross_entropy(output, y, normalize=True)
model.cleargrads()
loss_adv_first.backward()
if args.use_attn_d:
# iAdv
attn_d_grad = model.attention_d_var.grad
attn_d_grad = F.normalize(attn_d_grad, axis=1)
# Get directional vector
dir_normed = model.dir_normed.data
attn_d = F.broadcast_to(attn_d_grad, dir_normed.shape).data
d = xp.sum(attn_d * dir_normed, axis=1)
else:
# Adv
d = model.d_var.grad
attn_d_grad = chainer.Variable(d)
d_data = d.data if isinstance(d, chainer.Variable) else d
# sentence-normalize
d_data = d_data / xp.linalg.norm(d_data)
# Analysis mode
predict_adv = xp.argmax(output.data, axis=1)
predict = xp.argmax(output_original.data, axis=1)
logging.info('predict:{}, gold:{}'.format(predict, y))
x_concat = xp.concatenate(x, axis=0)
is_wrong_predict = predict != y
if is_wrong_predict:
continue
is_adv_example = predict_adv != y
logging.info('is_adv_example:{}'.format(is_adv_example))
is_adv_example = to_cpu(is_adv_example)
idx = xp.arange(x_concat.shape[0]).astype(xp.int32)
# compute Nearest Neighbor
nearest_ids = model.get_nearest_words(x_concat)
nn_words = model.word_embed(nearest_ids)
nn_words = F.dropout(nn_words, ratio=args.dropout)
xs = model.word_embed(x_concat)
xs = F.dropout(xs, ratio=args.dropout)
xs_broad = F.reshape(xs, (xs.shape[0], 1, -1))
xs_broad = F.broadcast_to(xs_broad, nn_words.shape)
diff = nn_words - xs_broad
# compute similarity
dir_normed = nets.get_normalized_vector(diff, None, (2)).data
d_norm = nets.get_normalized_vector(d, xp)
d_norm = xp.reshape(d_norm, (d_norm.shape[0], 1, -1))
sims = F.matmul(dir_normed, d_norm, False, True)
sims = xp.reshape(sims.data, (sims.shape[0], -1))
most_sims_idx_top = xp.argsort(-sims,
axis=1)[idx_func(sims.shape[0]),
0].reshape(-1)
vis_items = []
r_len = x[0].shape[0]
for r_i in range(r_len):
idx = r_i
# most similar words in nearest neighbors
max_sim_idx = most_sims_idx_top[idx]
replace_word_idx = nearest_ids[idx, max_sim_idx]
max_sim_scalar = xp.max(sims, axis=1)[idx].reshape(-1)
attn_d_value = d_data[idx].reshape(-1)
# grad_scale = xp.linalg.norm(d_data[idx]) / xp.max(xp.linalg.norm(d_data))
grad_scale = xp.linalg.norm(d_data[idx]) / xp.max(
xp.linalg.norm(d_data, axis=1))
nn_words_list = [
vocab_inv[int(n_i)] for n_i in nearest_ids[idx]
]
nn_words = ','.join(nn_words_list)
sims_nn = sims[idx]
diff_norm_scala = xp.linalg.norm(diff.data[idx, max_sim_idx])
d_data_scala = xp.linalg.norm(d_data[idx])
vis_item = [
r_i, vocab_inv[int(x_concat[idx])],
vocab_inv[int(replace_word_idx)],
to_cpu(max_sim_scalar),
to_cpu(attn_d_value), nn_words,
to_cpu(grad_scale), is_adv_example,
to_cpu(sims_nn),
to_cpu(diff_norm_scala),
to_cpu(d_data_scala)
]
vis_items.append(vis_item)
save_items.append([vis_items, to_cpu(x[0]), to_cpu(y)])
with open(args.save_name, mode='wb') as f:
# Save as pickle file
pickle.dump(save_items, f, protocol=2)