def evaluate(model, model_name, sents, ivocab):
train = False
loss = 0.0
acc = 0.0
count = 0
vocab_size = model.vocab_size
for data_i in pyprind.prog_bar(xrange(len(sents))):
words = sents[data_i:data_i + 1]
if model_name == "bd_lstm":
xs, ms = utils.make_batch(words,
train=train,
tail=False,
mask=True)
ys = model.forward(xs=xs, ms=ms, train=train)
else:
xs = utils.make_batch(words, train=train, tail=False)
ys = model.forward(ts=xs, train=train)
ys = F.concat(ys, axis=0)
ts = F.concat(xs, axis=0)
ys = F.reshape(ys, (-1, vocab_size))
ts = F.reshape(ts, (-1, ))
loss += F.softmax_cross_entropy(ys, ts) * len(words[0])
acc += F.accuracy(ys, ts, ignore_label=-1) * len(words[0])
count += len(words[0])
loss_data = float(cuda.to_cpu(loss.data)) / count
acc_data = float(cuda.to_cpu(acc.data)) / count
return loss_data, acc_data
def repeated_test(self,
test_data,
extra_data,
ratio,
batchsize,
alpha,
nrepeats,
windowsize=0):
n_extra = np.int(np.floor(ratio * batchsize)) # can be 0, no problem
n_test = batchsize - n_extra # can be 0, no problem
RES = [] # result of out-of-specs tests
ACC = [
] # result of filtering by density or score (only for positive tests)
for i in range(nrepeats):
batch_test = make_batch(test_data, n_test)
if n_extra == 0:
batch = batch_test
else:
batch_extra = make_batch(extra_data, n_extra)
batch = np.concatenate((batch_test, batch_extra))
perturb_inplace(batch, self.eps)
res = self.threshold_tests(batch)
RES.append(res)
return np.asarray(RES)
def repeated_test(self, test_data, extra_data, ratio, batchsize, alpha, nrepeats, windowsize=0):
n_extra = np.int(np.floor(ratio*batchsize)) # can be 0, no problem
n_test = batchsize-n_extra # can be 0, no problem
RES = [] # result of out-of-specs tests
ACC = [] # result of filtering by density or score (only for positive tests)
for i in range(nrepeats):
batch_test = make_batch(test_data, n_test)
if n_extra == 0:
batch = batch_test
else:
batch_extra = make_batch(extra_data, n_extra)
batch = np.concatenate((batch_test, batch_extra))
perturb_inplace(batch, self.eps)
res = self.flat_test(batch, alpha=alpha)
RES.append(res)
if windowsize>0: # apply filtering
suspicious_examples = self.filter_suspicious_examples(batch, windowsize)
labels = np.ones_like(batch_test)
if n_extra > 0:
label_extra = np.zeros_like(batch_extra)
labels = np.concatenate((labels, label_extra))
assert( suspicious_examples.any() )
acc_suspicious = labels[suspicious_examples].mean()
extreme_examples = filter_extreme_examples(batch, windowsize)
assert( extreme_examples.any() )
acc_extreme = labels[extreme_examples].mean()
ACC.append([acc_suspicious,acc_extreme])
return np.asarray(RES),np.asarray(ACC)
def evaluate(
model_path,
corpus_path,
pairs_path,
batch_size=100,
):
model = torch.load(model_path)
model = model.cuda()
model.eval()
corpus = Corpus([tuple([corpus_path, os.path.dirname(corpus_path)])])
pairs_batch_loader = FileLoader(
[tuple([pairs_path, os.path.dirname(pairs_path)])], batch_size)
code = []
nl = []
for data in tqdm.tqdm(pairs_batch_loader):
data = map(corpus.get, data)
batch = (make_batch(model.embedding_layer, data[0][0]),
make_batch(model.embedding_layer, data[1][0]))
batch = [x.cuda() for x in batch]
batch = (Variable(batch[0],
volatile=True), Variable(batch[1], volatile=True))
# embed code and NL
repr_left = model(batch[0])
repr_right = model(batch[1])
# accumulate for evaluation
code.extend(repr_left.cpu().data.numpy())
nl.extend(repr_right.cpu().data.numpy())
code = np.array(code)
nl = np.array(nl)
sim_mat = cosine_similarity(nl, code)
ans_locs = location_of_correct(sim_mat)
summary = {}
mr = np.mean(ans_locs)
mrr = get_mrr(ans_locs)
summary["mrr"] = mrr
cutoffs = [1, 5, 10]
fracs = []
for c in cutoffs:
frac = get_fraction_correct_at(ans_locs, c)
fracs.append(frac)
print("Num obs: {}".format(code.shape[0]))
print("Mean Rank: {}".format(mr))
print("MRR: {}".format(mrr))
for c, f in zip(cutoffs, fracs):
print("Fraction Correct@{}: {}".format(c, f))
summary["success@{}".format(c)] = f
return summary
def test(self, data, task):
print('test {} {} {}'.format(self.dataset_name, task, self.model_name))
res = []
if task in ['topk', 'all']:
K = 10
n = 10000
batch_size = 64
n_users = data.n_users
n_items = data.n_items
ind = np.random.choice(data.n_test, size=n, replace=False)
data.init_test()
edgelist = data.test_edgelist[ind]
user_score = {}
hrs, ndcgs = [], []
for num, total_num, edges in make_batch(n, batch_size, True,
edgelist):
score = self.eval(edges,
user_shift=data.n_entities,
n_users=n_users)
users = edges[:, 0]
item_lists = data.test_negative[users]
score_neg = self.eval2(users, item_lists, n_users=n_users)
for pos, neg in zip(score, score_neg):
hit_ratio, ndcg = test_one_user(pos, neg, K)
hrs.append(hit_ratio)
ndcgs.append(ndcg)
res += [np.mean(hrs), np.mean(ndcgs)]
if task in ['ctr', 'all']:
n = 10000
batch_size = 1000
n_test = data.n_test
ind = np.random.choice(data.n_test, size=n, replace=False)
edgelist = data.test_edgelist[ind]
aucs, f1s = [], []
for num, total_num, edges, in make_batch(n, batch_size, True,
edgelist):
neg_items = data.negative_sample(edges)
edges = np.vstack((edges, np.vstack((edges[:,
0], neg_items)).T))
y = np.hstack((np.ones(neg_items.shape[0]),
np.zeros(neg_items.shape[0])))
score = softmax(
self.eval(edges,
user_shift=data.n_entities,
n_users=data.n_users))
auc = roc_auc_score(y_true=y, y_score=score)
#score[score >= 0.5] = 1
#score[score < 0.5] = 0
#f1 = f1_score(y_true=y, y_pred=score)
aucs.append(auc)
#f1s.append(f1)
#res = [np.mean(aucs), np.mean(f1s)]
res += [np.mean(aucs)]
return res
def classify(sess, label_list, softmax_output, coder, images, image_file):
print('Running file %s' % image_file)
image_batch = make_batch(image_file, coder, not FLAGS.single_look)
batch_results = sess.run(softmax_output,
feed_dict={images: image_batch.eval()})
output = batch_results[0]
batch_sz = batch_results.shape[0]
for i in range(1, batch_sz):
output = output + batch_results[i]
output /= batch_sz
best = np.argmax(output)
best_choice = (label_list[best], output[best])
print('Guess @ 1 %s, prob = %.2f' % best_choice)
#calculate face score
score = scoreAge(output)
nlabels = len(label_list)
if nlabels > 2:
output[best] = 0
second_best = np.argmax(output)
print('Guess @ 2 %s, prob = %.2f' %
(label_list[second_best], output[second_best]))
return label_list[best], score
def adjust_thresholds(self, val_data, alpha, batchsize, nrepeats=10000, maxrepeats=1e6):
self.thr_Z = normal_dist.ppf(alpha, loc=self.valmean, scale=self.valstd/np.sqrt(batchsize)), np.inf # no upper bound, only lower
self.thr_logZ = normal_dist.ppf(alpha, loc=self.vallogmean, scale=self.vallogstd/np.sqrt(batchsize)), np.inf # no upper bound, only lower
self.thr_symZ = normal_dist.ppf(0.5*alpha, loc=self.valmean, scale=self.valstd/np.sqrt(batchsize)), normal_dist.ppf(1.-0.5*alpha, loc=self.valmean, scale=self.valstd/np.sqrt(batchsize))
self.thr_symlogZ = normal_dist.ppf(0.5*alpha, loc=self.vallogmean, scale=self.vallogstd/np.sqrt(batchsize)), normal_dist.ppf(1.-0.5*alpha, loc=self.vallogmean, scale=self.vallogstd/np.sqrt(batchsize))
nrepeats = max( nrepeats, np.int(np.ceil(2./alpha)) )
if nrepeats <= maxrepeats:
mean_stat,logmean_stat = [],[]
for i in range(nrepeats):
batch = make_batch(val_data, batchsize)
mean_stat.append( np.mean(batch) )
logmean_stat.append( np.mean(np.log(batch) ) )
mean_stat = np.sort(mean_stat)
logmean_stat = np.sort(logmean_stat)
index = np.int(np.floor(alpha*nrepeats)) # number of permitted outliers
self.thr_mean = mean_stat[index], np.inf
self.thr_logmean = logmean_stat[index], np.inf
self.thr_symmean = mean_stat[(index-1)//2], mean_stat[-index//2]
self.thr_symlogmean = logmean_stat[(index-1)//2], mean_stat[-index//2]
else: # disable tests
self.thr_mean = -np.inf, np.inf
self.thr_logmean = -np.inf, np.inf
self.thr_symmean = -np.inf, np.inf
self.thr_symlogmean = -np.inf, np.inf
def test_batch(encdec, src_vocab, trg_vocab, lines):
g = Graph()
Graph.set_default(g)
src_batch = make_batch(lines, list(range(len(lines))), src_vocab)
encdec.encode(src_batch, False)
# Generates target words one-by-one.
trg_ids = [np.array([trg_vocab["<bos>"]] * len(lines))]
eos_id = trg_vocab["<eos>"]
eos_ids = np.array([eos_id] * len(lines))
while (trg_ids[-1] != eos_ids).any():
if len(trg_ids) > GENERATION_LIMIT + 1:
print("Warning: Sentence generation did not finish in",
GENERATION_LIMIT,
"iterations.",
file=sys.stderr)
trg_ids.append(eos_ids)
break
y = encdec.decode_step(trg_ids[-1], False)
trg_ids.append(np.array(y.argmax(0)).T)
return [
hyp[:np.where(hyp == eos_id)[0][0]] for hyp in np.array(trg_ids[1:]).T
]
def test(model, config):
max_sentences = config.get("max_sentences", 1e9)
max_tokens = config.get("max_tokens", 1e9)
corpus_prefix = Path(config['corpus_prefix']) / "subword"
model_path = corpus_prefix / "spm.model"
tokenizer = spm.SentencePieceProcessor()
tokenizer.Load(str(model_path))
test_src = load_corpus(corpus_prefix / Path(config["test_source"]).name,
tokenizer)
num_test_sents = len(test_src)
eos_id = tokenizer.eos_id()
test_ids = list(range(num_test_sents))
test_itr = create_batch_itr(test_src,
max_tokens=max_tokens,
max_sentences=max_sentences,
shuffle=False)
test_itr = tqdm(test_itr, desc='test')
for batch_ids in test_itr:
src_batch = make_batch(test_src, batch_ids, eos_id)
src_mask = padding_mask(src_batch, eos_id)
src_encode = model.encode(src_batch, src_mask, train=False)
trg_ids = [np.array([tokenizer.PieceToId('<s>')] * len(batch_ids))]
eos_ids = np.array([eos_id] * len(batch_ids))
while (trg_ids[-1] != eos_ids).any():
if len(trg_ids) > config['generation_limit']:
print("Warning: Sentence generation did not finish in",
config['generation_limit'],
"iterations.",
file=sys.stderr)
trg_ids.append(eos_ids)
break
trg_mask = [
subsequent_mask(len(trg_ids))
for _ in padding_mask(trg_ids, eos_id)
]
out = model.decode(src_encode,
trg_ids,
src_mask,
trg_mask,
train=False)
y = TF.pick(out, [out.shape()[0] - 1], 0)
y = np.array(y.argmax(1))
trg_ids.append(y)
hyp = [
hyp_sent[:np.where(hyp_sent == eos_id)[0][0]]
for hyp_sent in np.array(trg_ids).T
]
for ids in hyp:
sent = tokenizer.DecodeIds(ids.tolist())
print(sent)
def repeated_test(self, test_data, extra_data, ratio, batchsize, alpha, nrepeats, windowsize=0):
n_extra = np.int(np.floor(ratio*batchsize)) # can be 0, no problem
n_test = batchsize-n_extra # can be 0, no problem
RES = [] # result of out-of-specs tests
ACC = [] # result of filtering by density or score (only for positive tests)
for i in range(nrepeats):
batch_test = make_batch(test_data, n_test)
if n_extra == 0:
batch = batch_test
else:
batch_extra = make_batch(extra_data, n_extra)
batch = np.concatenate((batch_test, batch_extra))
perturb_inplace(batch, self.eps)
res = self.threshold_tests(batch)
RES.append(res)
return np.asarray(RES)
def adjust_thresholds(self,
val_data,
alpha,
batchsize,
nrepeats=10000,
maxrepeats=1e6):
self.thr_Z = normal_dist.ppf(
alpha, loc=self.valmean, scale=self.valstd /
np.sqrt(batchsize)), np.inf # no upper bound, only lower
self.thr_logZ = normal_dist.ppf(
alpha,
loc=self.vallogmean,
scale=self.vallogstd /
np.sqrt(batchsize)), np.inf # no upper bound, only lower
self.thr_symZ = normal_dist.ppf(
0.5 * alpha,
loc=self.valmean,
scale=self.valstd / np.sqrt(batchsize)), normal_dist.ppf(
1. - 0.5 * alpha,
loc=self.valmean,
scale=self.valstd / np.sqrt(batchsize))
self.thr_symlogZ = normal_dist.ppf(
0.5 * alpha,
loc=self.vallogmean,
scale=self.vallogstd / np.sqrt(batchsize)), normal_dist.ppf(
1. - 0.5 * alpha,
loc=self.vallogmean,
scale=self.vallogstd / np.sqrt(batchsize))
nrepeats = max(nrepeats, np.int(np.ceil(2. / alpha)))
if nrepeats <= maxrepeats:
mean_stat, logmean_stat = [], []
for i in range(nrepeats):
batch = make_batch(val_data, batchsize)
mean_stat.append(np.mean(batch))
logmean_stat.append(np.mean(np.log(batch)))
mean_stat = np.sort(mean_stat)
logmean_stat = np.sort(logmean_stat)
index = np.int(np.floor(alpha *
nrepeats)) # number of permitted outliers
self.thr_mean = mean_stat[index], np.inf
self.thr_logmean = logmean_stat[index], np.inf
self.thr_symmean = mean_stat[(index - 1) // 2], mean_stat[-index //
2]
self.thr_symlogmean = logmean_stat[(index - 1) //
2], mean_stat[-index // 2]
else: # disable tests
self.thr_mean = -np.inf, np.inf
self.thr_logmean = -np.inf, np.inf
self.thr_symmean = -np.inf, np.inf
self.thr_symlogmean = -np.inf, np.inf
def evaluate(model, corpus):
train = False
loss = 0.0
acc = 0.0
count = 0
vocab_size = model.vocab_size
for data_i in pyprind.prog_bar(xrange(len(corpus))):
# batch_sents = sents[data_i:data_i+1]
batch_sents = corpus.next_sample()
batch_sents = [batch_sents]
xs = utils.make_batch(batch_sents, train=train, tail=False)
ys = model.forward(ts=xs, train=train)
ys = F.concat(ys, axis=0)
ts = F.concat(xs, axis=0)
ys = F.reshape(ys, (-1, vocab_size))
ts = F.reshape(ts, (-1,))
loss += F.softmax_cross_entropy(ys, ts) * len(batch_sents[0])
acc += F.accuracy(ys, ts, ignore_label=-1) * len(batch_sents[0])
count += len(batch_sents[0])
loss_data = float(cuda.to_cpu(loss.data)) / count
acc_data = float(cuda.to_cpu(acc.data)) / count
for i in xrange(5):
s = corpus.random_sample()
xs = utils.make_batch([s], train=train, tail=False)
ys = model.forward(x_init=xs[0], train=train)
words_ref = [corpus.ivocab[w] for w in s]
words_gen = [words_ref[0]] + [corpus.ivocab[w[0]] for w in ys]
print "[check] <Ref.> %s" % " ".join(words_ref)
print "[check] <Gen.> %s" % " ".join(words_gen)
return loss_data, acc_data
def evaluate(model, sents, ivocab, word_dim):
train = False
loss = 0.0
acc = 0.0
count = 0
vocab_size = model.vocab_size
for data_i in pyprind.prog_bar(xrange(len(sents))):
words = sents[data_i:data_i+1]
T = len(words) - 2 # BOSとEOSを取り除く
xs, ms = utils.make_batch(words, train=train, tail=False, mask=True)
batch_size = len(xs[0])
words_without_edge = [w[1:-1] for w in words]
xs_without_edge, ms_without_edge = utils.make_batch(words_without_edge, train=train, tail=False, mask=True)
ys = model.forward(xs=xs, ms=ms, train=train)
masked_ys = []
for y, m in zip(ys, ms_without_edge):
masked_ys.append(y*F.broadcast_to(F.reshape(m, (batch_size, 1)), (batch_size, vocab_size)))
ys = F.concat(masked_ys, axis=0)
ts = F.concat(xs_without_edge, axis=0)
ys = F.reshape(ys, (-1, vocab_size))
ts = F.reshape(ts, (-1,))
loss += F.softmax_cross_entropy(ys, ts) * T
acc += F.accuracy(ys, ts, ignore_label=-1) * T
count += T
loss_data = float(cuda.to_cpu(loss.data)) /count
acc_data = float(cuda.to_cpu(acc.data)) / count
return loss_data, acc_data
def inspect(model, model_name, sents):
train = False
probs = []
for index in pyprind.prog_bar(xrange(len(sents))):
words = sents[index:index + 1]
if model_name == "bd_lstm":
xs, ms = utils.make_batch(words,
train=train,
tail=False,
mask=True)
ys = model.forward(xs=xs, ms=ms, train=train)
else:
xs = utils.make_batch(words, train=train, tail=False)
ys = model.forward(ts=xs, train=train)
ys = F.concat(ys, axis=0)
ys = F.softmax(ys)
ys = cuda.to_cpu(ys.data)
probs.append(ys[np.arange(len(ys)), words[0]])
return probs
def build_dataset(self):
#从excel文件中读取数据集并构建tf.data.Dataset
self.train_data = pd.read_excel(hps.DATA_PATH +
'/id_cut_train_data.xlsx')
self.test_data = pd.read_excel(hps.DATA_PATH +
'/id_cut_test_data.xlsx')
fw_inputs, bw_inputs, lengths, stances = utils.make_batch(
id_cut_data=self.train_data, batch_size=len(model.train_data))
self.Train_data = tf.data.Dataset.from_tensor_slices({
'fw_inputs': fw_inputs,
'bw_inputs': bw_inputs,
'lengths': lengths,
'stances': stances
})
fw_inputs, bw_inputs, lengths, stances = utils.get_inputs_for_testing(
self.test_data)
self.Test_data = tf.data.Dataset.from_tensor_slices({
'fw_inputs': fw_inputs,
'bw_inputs': bw_inputs,
'lengths': lengths,
'stances': stances
})
def model_suite(self, mdl):
data_iter = self.data_iter
args = self.args
initial_embeddings = self.initial_embeddings
args.style = mdl
# Model Class.
model_cls = getattr(models, mdl)
# Test model.
model = model_cls(
model_dim=args.model_dim,
mlp_dim=args.mlp_dim,
num_classes=args.num_classes,
word_embedding_dim=args.word_embedding_dim,
initial_embeddings=initial_embeddings,
)
data, target, lengths = utils.make_batch(next(data_iter),
args.style == "dynamic")
y = model(data, lengths)
def classify(sess, label_list, softmax_output, coder, images, image_file):
print('Running file %s' % image_file)
image_batch = make_batch(image_file, coder, not FLAGS.single_look)
batch_results = sess.run(softmax_output, feed_dict={images:image_batch.eval()})
output = batch_results[0]
batch_sz = batch_results.shape[0]
for i in range(1, batch_sz):
output = output + batch_results[i]
output /= batch_sz
best = np.argmax(output)
best_choice = (label_list[best], output[best])
print('Guess @ 1 %s, prob = %.2f' % best_choice)
nlabels = len(label_list)
if nlabels > 2:
output[best] = 0
second_best = np.argmax(output)
print('Guess @ 2 %s, prob = %.2f' % (label_list[second_best], output[second_best]))
return best_choice
def classify(sess, label_list, softmax_output, coder, images, image_file):
#print('Running file %s' % image_file)
image_batch = make_batch(image_file, coder, True)
batch_results = sess.run(softmax_output,
feed_dict={images: image_batch.eval()})
output = batch_results[0]
batch_sz = batch_results.shape[0]
for i in range(1, batch_sz):
output = output + batch_results[i]
output /= batch_sz
best = np.argmax(output)
best_choice = (label_list[best], output[best])
#print('Guess @ 1 %s, prob = %.2f' % best_choice)
# if label_list[best] =='(15, 20)':
# final_list[1][0]+=1
# final_list[1][1]+=output[best]
# if label_list[best]=='(25, 32)':
# final_list[0][0]+=1
# final_list[0][1]+=output[best]
#print('Guess @ 2 %s, prob = %.2f' % (label_list[second_best], output[second_best]))
return best_choice
def evaluate(iter_cnt, filepath, model, corpus, args, logging=True):
if logging:
valid_writer = FileWriter(args.run_path + "/valid", flush_secs=5)
pos_file_path = "{}.pos.txt".format(filepath)
neg_file_path = "{}.neg.txt".format(filepath)
pos_batch_loader = FileLoader(
[tuple([pos_file_path, os.path.dirname(args.eval)])], args.batch_size)
neg_batch_loader = FileLoader(
[tuple([neg_file_path, os.path.dirname(args.eval)])], args.batch_size)
batchify = lambda bch: make_batch(model.embedding_layer, bch)
model.eval()
criterion = model.compute_loss
auc_meter = AUCMeter()
scores = [np.asarray([], dtype='float32') for i in range(2)]
for loader_id, loader in tqdm(
enumerate((neg_batch_loader, pos_batch_loader))):
for data in tqdm(loader):
data = map(corpus.get, data)
batch = None
if not args.eval_use_content:
batch = (batchify(data[0][0]), batchify(data[1][0]))
else:
batch = (map(batchify, data[0]), map(batchify, data[1]))
new_batch = []
for x in batch:
for y in x:
new_batch.append(y)
batch = new_batch
labels = torch.ones(batch[0].size(1)).type(
torch.LongTensor) * loader_id
if args.cuda:
batch = [x.cuda() for x in batch]
labels = labels.cuda()
if not args.eval_use_content:
batch = (Variable(batch[0], volatile=True),
Variable(batch[1], volatile=True))
else:
batch = (Variable(batch[0], volatile=True),
Variable(batch[1], volatile=True),
Variable(batch[2], volatile=True),
Variable(batch[3], volatile=True))
labels = Variable(labels)
if not args.eval_use_content:
repr_left = model(batch[0])
repr_right = model(batch[1])
else:
repr_left = model(batch[0]) + model(batch[1])
repr_right = model(batch[2]) + model(batch[3])
output = model.compute_similarity(repr_left, repr_right)
if model.criterion.startswith('classification'):
assert output.size(1) == 2
output = nn.functional.log_softmax(output)
current_scores = -output[:, loader_id].data.cpu().squeeze(
).numpy()
output = output[:, 1]
else:
assert output.size(1) == 1
current_scores = output.data.cpu().squeeze().numpy()
auc_meter.add(output.data, labels.data)
scores[loader_id] = np.append(scores[loader_id], current_scores)
auc_score = auc_meter.value()
auc10_score = auc_meter.value(0.1)
auc05_score = auc_meter.value(0.05)
auc02_score = auc_meter.value(0.02)
auc01_score = auc_meter.value(0.01)
if model.criterion.startswith('classification'):
avg_score = (scores[1].mean() + scores[0].mean()) * 0.5
else:
avg_score = scores[1].mean() - scores[0].mean()
outputManager.say(
"\r[{}] auc(.01): {:.3f} auc(.02): {:.3f} auc(.05): {:.3f}"
" auc(.1): {:.3f} auc: {:.3f}"
" scores: {:.2f} ({:.2f} {:.2f})\n".format(
os.path.basename(filepath).split('.')[0], auc01_score, auc02_score,
auc05_score, auc10_score, auc_score, avg_score, scores[1].mean(),
scores[0].mean()))
if logging:
s = summary.scalar('auc', auc_score)
valid_writer.add_summary(s, iter_cnt)
s = summary.scalar('auc (fpr<0.1)', auc10_score)
valid_writer.add_summary(s, iter_cnt)
s = summary.scalar('auc (fpr<0.05)', auc05_score)
valid_writer.add_summary(s, iter_cnt)
s = summary.scalar('auc (fpr<0.02)', auc02_score)
valid_writer.add_summary(s, iter_cnt)
s = summary.scalar('auc (fpr<0.01)', auc01_score)
valid_writer.add_summary(s, iter_cnt)
valid_writer.close()
return auc05_score
def train(args):
# load data
num_nodes, num_rels = utils.get_total_number('./data/' + args.dataset,
'stat.txt')
train_data, train_times = utils.load_quadruples('./data/' + args.dataset,
'train.txt')
valid_data, valid_times = utils.load_quadruples('./data/' + args.dataset,
'valid.txt')
total_data, total_times = utils.load_quadruples('./data/' + args.dataset,
'train.txt', 'valid.txt',
'test.txt')
# check cuda
use_cuda = args.gpu >= 0 and torch.cuda.is_available()
if use_cuda:
torch.cuda.set_device(args.gpu)
torch.cuda.manual_seed_all(999)
os.makedirs('models', exist_ok=True)
if args.model == 0:
model_state_file = 'models/' + args.dataset + 'attn.pth'
elif args.model == 1:
model_state_file = 'models/' + args.dataset + 'mean.pth'
elif args.model == 2:
model_state_file = 'models/' + args.dataset + 'gcn.pth'
print("start training...")
model = RENet(num_nodes,
args.n_hidden,
num_rels,
dropout=args.dropout,
model=args.model,
seq_len=args.seq_len)
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=0.00001)
if use_cuda:
model.cuda()
with open('./data/' + args.dataset + '/train_history_sub.txt', 'rb') as f:
s_history = pickle.load(f)
with open('./data/' + args.dataset + '/train_history_ob.txt', 'rb') as f:
o_history = pickle.load(f)
with open('./data/' + args.dataset + '/dev_history_sub.txt', 'rb') as f:
s_history_valid = pickle.load(f)
with open('./data/' + args.dataset + '/dev_history_ob.txt', 'rb') as f:
o_history_valid = pickle.load(f)
valid_data = torch.from_numpy(valid_data)
epoch = 0
best_mrr = 0
while True:
model.train()
if epoch == args.max_epochs:
break
epoch += 1
loss_epoch = 0
t0 = time.time()
train_data, s_history, o_history = shuffle(train_data, s_history,
o_history)
i = 0
for batch_data, s_hist, o_hist in utils.make_batch(
train_data, s_history, o_history, args.batch_size):
batch_data = torch.from_numpy(batch_data)
if use_cuda:
batch_data = batch_data.cuda()
loss = model.get_loss(batch_data, s_hist, o_hist)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(),
args.grad_norm) # clip gradients
optimizer.step()
optimizer.zero_grad()
loss_epoch += loss.item()
i += 1
t3 = time.time()
print("Epoch {:04d} | Loss {:.4f} | time {:.4f}".format(
epoch, loss_epoch / (len(train_data) / args.batch_size), t3 - t0))
if epoch % 1 == 0:
model.eval()
total_loss = 0
total_ranks = np.array([])
model.init_history()
model.latest_time = valid_data[0][3]
for i in range(len(valid_data)):
batch_data = valid_data[i]
s_hist = s_history_valid[i]
o_hist = o_history_valid[i]
if use_cuda:
batch_data = batch_data.cuda()
with torch.no_grad():
ranks, loss = model.evaluate(batch_data, s_hist, o_hist)
total_ranks = np.concatenate((total_ranks, ranks))
total_loss += loss.item()
mrr = np.mean(1.0 / total_ranks)
mr = np.mean(total_ranks)
hits = []
for hit in [1, 3, 10]:
avg_count = np.mean((total_ranks <= hit))
hits.append(avg_count)
print("valid Hits (raw) @ {}: {:.6f}".format(hit, avg_count))
print("valid MRR (raw): {:.6f}".format(mrr))
print("valid MR (raw): {:.6f}".format(mr))
print("valid Loss: {:.6f}".format(total_loss / (len(valid_data))))
if mrr > best_mrr:
best_mrr = mrr
torch.save(
{
'state_dict': model.state_dict(),
'epoch': epoch,
's_hist': model.s_hist_test,
's_cache': model.s_his_cache,
'o_hist': model.o_hist_test,
'o_cache': model.o_his_cache,
'latest_time': model.latest_time
}, model_state_file)
print("training done")
t_start = time.time()
for e in range(num_epochs):
loss_sum = 0
acc_sum = 0
iou_sum = 0
num_batches = 0
model.train()
t_train_start = time.time()
for i, (images, labels) in enumerate(dl_train):
print('[{}/{}] '.format(e + 1, num_epochs), end='')
print('{}/{} '.format((i + 1) * len(images), len(dset_train)), end='')
num_batches += 1
x = make_batch(images).to(device)
y_class = make_batch(labels, 'class').to(device)
y_bbox = make_batch(labels, 'bbox').to(device)
predict_class, predict_bbox = model(x)
optimizer.zero_grad()
loss = loss_func(predict_class, predict_bbox, y_class, y_bbox)
loss.backward()
optimizer.step()
loss = loss.item()
acc_class = (predict_class.argmax(dim=1) == y_class.argmax(dim=1)).sum().item() / len(images)
iou = calculate_iou_batch(predict_bbox, y_bbox)
loss_sum += loss
n_batch = 0
train_loss = 0
train_acc = 0
model.train()
t_train_start = time.time()
for i, (images, labels) in enumerate(train_loader):
mini_batch = len(images)
n_images += mini_batch
n_batch += 1
print('[{}/{}] {}/{} '.format(e + 1, num_epoch,
min(n_images, n_train_data),
n_train_data),
end='')
x = make_batch(images).to(device)
y = make_batch(labels, 'class').to(device)
output = model(x)
optimizer.zero_grad()
loss = loss_func(output, y)
loss.backward()
optimizer.step()
acc = torch.true_divide(
(output.argmax(dim=1) == y.argmax(dim=1)).sum(), mini_batch)
train_loss += loss.item()
train_acc += acc.item()
def train(model, optimizer, config, best_valid):
max_epoch = config.get("max_epoch", int(1e9))
max_iteration = config.get("max_iteration", int(1e9))
max_sentences = config.get("max_sentences", 1e9)
max_tokens = config.get("max_tokens", 1e9)
update_freq = config.get('update_freq', 1)
optimizer.add(model)
corpus_prefix = Path(config['corpus_prefix']) / "subword"
model_path = corpus_prefix / "spm.model"
tokenizer = spm.SentencePieceProcessor()
tokenizer.Load(str(model_path))
train_src = load_corpus(corpus_prefix / Path(config["train_source"]).name,
tokenizer)
train_trg = load_corpus(corpus_prefix / Path(config["train_target"]).name,
tokenizer)
train_src, train_trg = clean_corpus(train_src, train_trg, config)
dev_src = load_corpus(corpus_prefix / Path(config["dev_source"]).name,
tokenizer)
dev_trg = load_corpus(corpus_prefix / Path(config["dev_target"]).name,
tokenizer)
dev_src, dev_trg = clean_corpus(dev_src, dev_trg, config)
num_train_sents = len(train_src)
num_dev_sents = len(dev_src)
eos_id = tokenizer.eos_id()
epoch = 0
iteration = 0
while epoch < max_epoch and iteration < max_iteration:
epoch += 1
g = Graph()
Graph.set_default(g)
train_itr = create_batch_itr(train_src,
train_trg,
max_tokens,
max_sentences,
shuffle=True)
train_itr = tqdm(train_itr, desc='train epoch {}'.format(epoch))
train_loss = 0.
itr_loss = 0.
itr_tokens = 0
itr_sentences = 0
optimizer.reset_gradients()
for step, batch_ids in enumerate(train_itr):
src_batch = make_batch(train_src, batch_ids, eos_id)
trg_batch = make_batch(train_trg, batch_ids, eos_id)
src_mask = padding_mask(src_batch, eos_id)
trg_mask = [
x | subsequent_mask(len(trg_batch) - 1)
for x in padding_mask(trg_batch[:-1], eos_id)
]
itr_tokens += len(src_batch) * len(src_batch[0])
itr_sentences += len(batch_ids)
g.clear()
loss = model.loss(src_batch, trg_batch, src_mask, trg_mask)
loss /= update_freq
loss.backward()
loss_val = loss.to_float()
train_loss += loss_val * update_freq * len(batch_ids)
itr_loss += loss_val
# with open('graph.dot', 'w') as f:
# print(g.dump("dot"), end="", file=f)
if (step + 1) % update_freq == 0:
step_num = optimizer.get_epoch() + 1
new_scale = config['d_model'] ** (-0.5) * \
min(step_num ** (-0.5), step_num * config['warmup_steps'] ** (-1.5))
optimizer.set_learning_rate_scaling(new_scale)
optimizer.update()
optimizer.reset_gradients()
iteration += 1
train_itr.set_postfix(itr=("%d" % (iteration)),
loss=("%.3lf" % (itr_loss)),
wpb=("%d" % (itr_tokens)),
spb=("%d" % (itr_sentences)),
lr=optimizer.get_learning_rate_scaling())
itr_loss = 0.
itr_tokens = 0
itr_sentences = 0
if iteration >= max_iteration:
break
print("\ttrain loss = %.4f" % (train_loss / num_train_sents))
g.clear()
valid_loss = 0.
valid_itr = create_batch_itr(dev_src,
dev_trg,
max_tokens,
max_sentences,
shuffle=False)
valid_itr = tqdm(valid_itr, desc='valid epoch {}'.format(epoch))
for batch_ids in valid_itr:
src_batch = make_batch(dev_src, batch_ids, eos_id)
trg_batch = make_batch(dev_trg, batch_ids, eos_id)
src_mask = padding_mask(src_batch, eos_id)
trg_mask = [
x | subsequent_mask(len(trg_batch) - 1)
for x in padding_mask(trg_batch[:-1], eos_id)
]
loss = model.loss(src_batch,
trg_batch,
src_mask,
trg_mask,
train=False)
valid_loss += loss.to_float() * len(batch_ids)
valid_itr.set_postfix(loss=loss.to_float())
print("\tvalid loss = %.4f" % (valid_loss / num_dev_sents))
if valid_loss < best_valid:
best_valid = valid_loss
print('\tsaving model/optimizer ... ', end="", flush=True)
prefix = config['model_prefix']
model.save(prefix + '.model')
optimizer.save(prefix + '.optimizer')
with Path(prefix).with_suffix('.valid').open('w') as f:
f.write(str(best_valid))
print('done.')
# Init optimizer.
optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(0.9, 0.999))
print(model)
print("Total Params: {}".format(
sum(torch.numel(p.data) for p in model.parameters())))
A = Accumulator()
# Train loop.
for step in range(args.max_training_steps):
start = time.time()
data, target, lengths = make_batch(next(training_iter),
args.style == "dynamic")
model.train()
optimizer.zero_grad()
y = model(data, lengths)
loss = F.nll_loss(y, Variable(target, volatile=False))
loss.backward()
optimizer.step()
pred = y.data.max(1)[1]
acc = pred.eq(target).sum() / float(args.batch_size)
end = time.time()
avg_time = (end - start) / float(args.batch_size)
def train(args):
# load data
num_nodes, num_rels = utils.get_total_number('./data/' + args.dataset,
'stat.txt')
train_data, train_times_origin = utils.load_quadruples(
'./data/' + args.dataset, 'train.txt')
# check cuda
use_cuda = args.gpu >= 0 and torch.cuda.is_available()
seed = 999
np.random.seed(seed)
torch.manual_seed(seed)
if use_cuda:
torch.cuda.set_device(args.gpu)
os.makedirs('models', exist_ok=True)
os.makedirs('models/' + args.dataset, exist_ok=True)
if args.model == 0:
model_state_file = 'models/' + args.dataset + 'attn.pth'
elif args.model == 1:
model_state_file = 'models/' + args.dataset + 'mean.pth'
elif args.model == 2:
model_state_file = 'models/' + args.dataset + 'gcn.pth'
elif args.model == 3:
model_state_file = 'models/' + args.dataset + '/max' + str(
args.maxpool) + 'rgcn_global.pth'
# model_graph_file = 'models/' + args.dataset + 'rgcn_graph.pth'
model_state_file_backup = 'models/' + args.dataset + '/max' + str(
args.maxpool) + 'rgcn__global_backup.pth'
# model_graph_file_backup = 'models/' + args.dataset + 'rgcn_graph_backup.pth'
print("start training...")
model = RENet_global(num_nodes,
args.n_hidden,
num_rels,
dropout=args.dropout,
model=args.model,
seq_len=args.seq_len,
num_k=args.num_k,
maxpool=args.maxpool)
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=0.00001)
if use_cuda:
model.cuda()
# train_times = torch.from_numpy(train_times)
with open('./data/' + args.dataset + '/train_graphs.txt', 'rb') as f:
graph_dict = pickle.load(f)
true_prob_s, true_prob_o = utils.get_true_distribution(
train_data, num_nodes)
epoch = 0
loss_small = 10000
while True:
model.train()
if epoch == args.max_epochs:
break
epoch += 1
loss_epoch = 0
t0 = time.time()
# print(graph_dict.keys())
# print(train_times_origin)
train_times, true_prob_s, true_prob_o = shuffle(
train_times_origin, true_prob_s, true_prob_o)
for batch_data, true_s, true_o in utils.make_batch(
train_times, true_prob_s, true_prob_o, args.batch_size):
batch_data = torch.from_numpy(batch_data)
true_s = torch.from_numpy(true_s)
true_o = torch.from_numpy(true_o)
if use_cuda:
batch_data = batch_data.cuda()
true_s = true_s.cuda()
true_o = true_o.cuda()
loss = model(batch_data, true_s, true_o, graph_dict)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(),
args.grad_norm) # clip gradients
optimizer.step()
optimizer.zero_grad()
loss_epoch += loss.item()
t3 = time.time()
model.global_emb = model.get_global_emb(train_times_origin, graph_dict)
print("Epoch {:04d} | Loss {:.4f} | time {:.4f}".format(
epoch, loss_epoch / (len(train_times) / args.batch_size), t3 - t0))
if loss_epoch < loss_small:
loss_small = loss_epoch
if args.model == 3:
torch.save(
{
'state_dict': model.state_dict(),
'global_emb': model.global_emb
}, model_state_file)
# with open(model_graph_file, 'wb') as fp:
# pickle.dump(model.graph_dict, fp)
else:
torch.save(
{
'state_dict': model.state_dict(),
'epoch': epoch,
's_hist': model.s_hist_test,
's_cache': model.s_his_cache,
'o_hist': model.o_hist_test,
'o_cache': model.o_his_cache,
'latest_time': model.latest_time
}, model_state_file)
print("training done")
input_shape=(FLAGS['sample_length'] - 1,
len(unique_tokens))),
tf.keras.layers.Dense(len(unique_tokens)),
tf.keras.layers.Activation('softmax')
])
optimizer = tf.keras.optimizers.Nadam()
model.compile(optimizer=optimizer, loss='categorical_crossentropy')
t1 = dt.now()
sz = samples.shape
for i in range(0, FLAGS['epochs']):
widgets = [
f'Epoch {i+1} batch ',
pb.Counter(format='%(value)d/%(max_value)d'), ' ',
pb.Bar(marker='.', left='[', right=']'), ' ',
pb.ETA()
]
with pb.ProgressBar(widgets=widgets,
max_value=m.ceil(sz[0] / FLAGS['batch_size'])) as bar:
batch_i = 0
while batch_i < sz[0]:
bar.update(batch_i / FLAGS['batch_size'])
batch = u.make_batch(samples, batch_i, FLAGS['batch_size'])
one_hot = u.one_hot_batch(batch, unique_tokens)
loss = model.train_on_batch(one_hot.x, one_hot.y)
batch_i = batch_i + FLAGS['batch_size']
print(f'Epoch: {i+1}, Loss: {loss}')
t2 = dt.now()
print(t2 - t1)
from time import time
from utils import make_batch
from models import WaveNet, Generator
from IPython.display import Audio
inputs, targets = make_batch('./voice.wav')
num_time_samples = inputs.shape[1]
num_channels = 1
gpu_fraction = 1
model = WaveNet(num_time_samples=num_time_samples,
num_channels=num_channels,
gpu_fraction=gpu_fraction)
Audio(inputs.reshape(inputs.shape[1]), rate=44100)
tic = time()
model.train(inputs, targets)
toc = time()
print('Training time = {} seconds'.format(toc - tic))
# generator = Generator(model)
# input_ = inputs[:, 0:1, 0]
# tic = time()
# predictions = generator.run(input_, 32000)
# toc = time()
# print('Generating time = {} seconds'.format(toc-tic))
def main():
# Loads vocab.
vocab = make_vocab("data/ptb.train.txt")
print("#vocab:", len(vocab)) # maybe 10000
eos_id = vocab["<s>"]
# Loads all corpus.
train_corpus = load_corpus("data/ptb.train.txt", vocab)
valid_corpus = load_corpus("data/ptb.valid.txt", vocab)
num_train_sents = len(train_corpus)
num_valid_sents = len(valid_corpus)
num_train_labels = count_labels(train_corpus)
num_valid_labels = count_labels(valid_corpus)
print("train:", num_train_sents, "sentences,", num_train_labels, "labels")
print("valid:", num_valid_sents, "sentences,", num_valid_labels, "labels")
# Device and computation graph.
dev = D.CUDA(0)
Device.set_default(dev)
g = Graph()
Graph.set_default(g)
# Our LM.
lm = RNNLM(len(vocab), eos_id)
# Optimizer.
optimizer = O.SGD(1)
#optimizer.set_weight_decay(1e-6)
optimizer.set_gradient_clipping(5)
optimizer.add(lm)
# Sentence IDs.
train_ids = list(range(num_train_sents))
valid_ids = list(range(num_valid_sents))
best_valid_ppl = 1e10
# Train/valid loop.
for epoch in range(MAX_EPOCH):
print("epoch", epoch + 1, "/", MAX_EPOCH, ":")
# Shuffles train sentence IDs.
random.shuffle(train_ids)
# Training.
train_loss = 0
for ofs in range(0, num_train_sents, BATCH_SIZE):
batch_ids = train_ids[ofs:min(ofs + BATCH_SIZE, num_train_sents)]
batch = make_batch(train_corpus, batch_ids, eos_id)
g.clear()
outputs = lm.forward(batch, True)
loss = lm.loss(outputs, batch)
train_loss += loss.to_float() * len(batch_ids)
optimizer.reset_gradients()
loss.backward()
optimizer.update()
print("%d" % ofs, end="\r")
sys.stdout.flush()
train_ppl = math.exp(train_loss / num_train_labels)
print(" train ppl =", train_ppl)
# Validation.
valid_loss = 0
for ofs in range(0, num_valid_sents, BATCH_SIZE):
batch_ids = valid_ids[ofs:min(ofs + BATCH_SIZE, num_valid_sents)]
batch = make_batch(valid_corpus, batch_ids, eos_id)
g.clear()
outputs = lm.forward(batch, False)
loss = lm.loss(outputs, batch)
valid_loss += loss.to_float() * len(batch_ids)
print("%d" % ofs, end="\r")
sys.stdout.flush()
valid_ppl = math.exp(valid_loss / num_valid_labels)
print(" valid ppl =", valid_ppl)
if valid_ppl < best_valid_ppl:
best_valid_ppl = valid_ppl
print(" BEST")
else:
old_lr = optimizer.get_learning_rate_scaling()
new_lr = 0.5 * old_lr
optimizer.set_learning_rate_scaling(new_lr)
print(" learning rate scaled:", old_lr, "->", new_lr)
# load data
data = utils.load_data("./data/Political-media-DFE.csv")
data = utils.remove_urls(data)
train_data = data[:100]
test_data = data[100:120]
test_data_list = utils.get_test_token(test_data)
word2idx, idx2word, length, train_data_list = utils.get_unique_word(train_data)
utils.mkdir_folder(config.save_model)
# train epoch
for ep in range(epoch):
data_generator = utils.make_batch(train_data_list, word2idx,
window_size=config.window_size,
batch_size=config.batch_size,
if_gpu=config.use_gpu)
test_data_generator = utils.make_batch(test_data_list, word2idx,
window_size = config.window_size,
batch_size = 8,
if_gpu = config.use_gpu)
batch_loss = train_epoch(data_generator)
test_loss = predict(test_data_generator)
if ep % 100 == 0:
print(f'Epoch: {ep}, Train Loss: {batch_loss: .5f}')
print(f'Epoch: {ep}, Test Loss: {test_loss: .5f}')
model.train()
def train_step(train_data_gen, num_epochs, num_stock_size, num_iters):
if hp._gpu:
physical_devices = tf.config.experimental.list_physical_devices('GPU')
if len(physical_devices) > 0:
for k in range(len(physical_devices)):
tf.config.experimental.set_memory_growth(
physical_devices[k], True)
print(
'memory growth:',
tf.config.experimental.get_memory_growth(
physical_devices[k]))
else:
print("Not enough GPU hardware devices available")
M = hp.M
optimizer = tf.keras.optimizers.Adam(learning_rate=hp.lr)
if hp.model_type == "dnn":
model = models.dnn()
elif hp.model_type == "cnn":
model = models.cnn()
elif hp.model_type == "rnn":
model = models.rnn()
elif hp.model_type == "cond_dnn":
model = models.cond_dnn()
else:
print("model type error")
if hp._task == "class":
loss_fn = tf.keras.losses.BinaryCrossentropy()
else:
loss_fn = tf.keras.losses.MeanSquaredError()
#model.summary()
train_losses = []
val_losses = []
for epoch in range(num_epochs):
epoch_loss_avg = tf.keras.metrics.Mean()
val_loss_avg = tf.keras.metrics.Mean()
for i in range(num_stock_size):
data_ = next(train_data_gen) #data_[M+N]
train_data = data_[:M, :]
test_data = data_[M:, :]
for j in range(num_iters):
x_batch, y_batch = make_batch(train_data, hp.n_features)
with tf.GradientTape() as tape:
y_ = model(x_batch)
loss_value = loss_fn(y_batch, y_)
grads = tape.gradient(loss_value,
model.trainable_variables)
optimizer.apply_gradients(zip(grads,
model.trainable_variables))
epoch_loss_avg(loss_value)
num_test_iters = num_iters // 4
for j in range(num_test_iters):
x_batch, y_batch = make_batch(test_data,
hp.n_features,
_test=True)
y_ = model(x_batch)
val_loss_value = loss_fn(y_batch, y_)
val_loss_avg(val_loss_value)
train_losses.append(loss_value)
val_losses.append(val_loss_value)
print(
"Epoch {:03d}: , Number of stock time {:03d}, Loss: {:.5f}".format(
epoch, i, epoch_loss_avg.result()))
print("Val_Loss: {:.3f}".format(val_loss_avg.result()))
if hp.b_loss_plot:
losses = [train_losses, val_losses]
utils.plot_loss(losses)
return model
def main(gpu, path_corpus, path_config, path_word2vec):
MAX_EPOCH = 50
EVAL = 200
MAX_LENGTH = 70
config = utils.Config(path_config)
model_name = config.getstr("model")
word_dim = config.getint("word_dim")
state_dim = config.getint("state_dim")
grad_clip = config.getfloat("grad_clip")
weight_decay = config.getfloat("weight_decay")
batch_size = config.getint("batch_size")
print "[info] CORPUS: %s" % path_corpus
print "[info] CONFIG: %s" % path_config
print "[info] PRE-TRAINED WORD EMBEDDINGS: %s" % path_word2vec
print "[info] MODEL: %s" % model_name
print "[info] WORD DIM: %d" % word_dim
print "[info] STATE DIM: %d" % state_dim
print "[info] GRADIENT CLIPPING: %f" % grad_clip
print "[info] WEIGHT DECAY: %f" % weight_decay
print "[info] BATCH SIZE: %d" % batch_size
path_save_head = os.path.join(
config.getpath("snapshot"),
"rnnlm.%s.%s" % (os.path.basename(path_corpus),
os.path.splitext(os.path.basename(path_config))[0]))
print "[info] SNAPSHOT: %s" % path_save_head
sents_train, sents_val, vocab, ivocab = \
utils.load_corpus(path_corpus=path_corpus, max_length=MAX_LENGTH)
if path_word2vec is not None:
word2vec = utils.load_word2vec(path_word2vec, word_dim)
initialW = utils.create_word_embeddings(vocab,
word2vec,
dim=word_dim,
scale=0.001)
else:
initialW = None
cuda.get_device(gpu).use()
if model_name == "rnn":
model = models.RNN(vocab_size=len(vocab),
word_dim=word_dim,
state_dim=state_dim,
initialW=initialW,
EOS_ID=vocab["<EOS>"])
elif model_name == "lstm":
model = models.LSTM(vocab_size=len(vocab),
word_dim=word_dim,
state_dim=state_dim,
initialW=initialW,
EOS_ID=vocab["<EOS>"])
elif model_name == "gru":
model = models.GRU(vocab_size=len(vocab),
word_dim=word_dim,
state_dim=state_dim,
initialW=initialW,
EOS_ID=vocab["<EOS>"])
elif model_name == "bd_lstm":
model = models.BD_LSTM(vocab_size=len(vocab),
word_dim=word_dim,
state_dim=state_dim,
initialW=initialW,
EOS_ID=vocab["<EOS>"])
else:
print "[error] Unknown model name: %s" % model_name
sys.exit(-1)
model.to_gpu(gpu)
opt = optimizers.SMORMS3()
opt.setup(model)
opt.add_hook(chainer.optimizer.GradientClipping(grad_clip))
opt.add_hook(chainer.optimizer.WeightDecay(weight_decay))
print "[info] Evaluating on the validation sentences ..."
loss_data, acc_data = evaluate(model, model_name, sents_val, ivocab)
perp = math.exp(loss_data)
print "[validation] iter=0, epoch=0, perplexity=%f, accuracy=%.2f%%" \
% (perp, acc_data*100)
it = 0
n_train = len(sents_train)
vocab_size = model.vocab_size
for epoch in xrange(1, MAX_EPOCH + 1):
perm = np.random.permutation(n_train)
for data_i in xrange(0, n_train, batch_size):
if data_i + batch_size > n_train:
break
words = sents_train[perm[data_i:data_i + batch_size]]
if model_name == "bd_lstm":
xs, ms = utils.make_batch(words,
train=True,
tail=False,
mask=True)
ys = model.forward(xs=xs, ms=ms, train=True)
else:
xs = utils.make_batch(words, train=True, tail=False)
ys = model.forward(ts=xs, train=True)
ys = F.concat(ys, axis=0)
ts = F.concat(xs, axis=0)
ys = F.reshape(ys, (-1, vocab_size)) # (TN, |V|)
ts = F.reshape(ts, (-1, )) # (TN,)
loss = F.softmax_cross_entropy(ys, ts)
acc = F.accuracy(ys, ts, ignore_label=-1)
model.zerograds()
loss.backward()
loss.unchain_backward()
opt.update()
it += 1
loss_data = float(cuda.to_cpu(loss.data))
perp = math.exp(loss_data)
acc_data = float(cuda.to_cpu(acc.data))
print "[training] iter=%d, epoch=%d (%d/%d=%.03f%%), perplexity=%f, accuracy=%.2f%%" \
% (it, epoch, data_i+batch_size, n_train,
float(data_i+batch_size)/n_train*100,
perp, acc_data*100)
if it % EVAL == 0:
print "[info] Evaluating on the validation sentences ..."
loss_data, acc_data = evaluate(model, model_name, sents_val,
ivocab)
perp = math.exp(loss_data)
print "[validation] iter=%d, epoch=%d, perplexity=%f, accuracy=%.2f%%" \
% (it, epoch, perp, acc_data*100)
serializers.save_npz(
path_save_head + ".iter_%d.epoch_%d.model" % (it, epoch),
model)
utils.save_word2vec(
path_save_head + ".iter_%d.epoch_%d.vectors.txt" %
(it, epoch), utils.extract_word2vec(model, vocab))
print "[info] Saved."
print "[info] Done."
from time import time
from IPython.display import Audio
from utils import make_batch
from models import Model, Generator
import tensorflow as tf
from scipy.io import wavfile
import numpy as np
tf.compat.v1.disable_eager_execution()
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if __name__ == '__main__':
# inputs, targets = make_batch('voice.wav')
inputs, targets = make_batch('jaekoo.wav')
num_time_samples = inputs.shape[1]
num_channels = 1
gpu_fraction = 1.0
print(inputs.shape, targets.shape)
model = Model(num_time_samples=num_time_samples,
num_channels=num_channels,
gpu_fraction=gpu_fraction)
try:
tic = time()
model.train(inputs, targets)
except:
toc = time()
args.num_targets, args.n_folds, args.seed)
mlflow.set_tracking_uri('../exp_logs/mlflow')
client = MlflowClient()
try:
EXP_ID = client.create_experiment(args.exp_name)
except:
experiments = client.get_experiment_by_name(args.exp_name)
EXP_ID = experiments.experiment_id
with mlflow.start_run(experiment_id=EXP_ID):
mlflow.log_params(vars(args))
test_loss, test_precision, test_recall, test_accuracy, test_auc = [], [], [], [], []
for fold in range(args.n_folds):
train_loaders, num_train = make_batch(train_ids[fold],
args.batch_size, args.seed)
print('\nFOLD {}, train {}, valid {}, test {}'.format(
fold, num_train, len(val_ids[fold]), len(test_ids[fold])))
print('\n------------- Initialize Model -------------')
model, optimizer = init_model(args)
print('\n------------- Training -------------')
train(dataset, train_loaders, val_ids[fold])
print('\n------------- Testing -------------')
model.load_state_dict(
torch.load('../model/model_{}_{}'.format(
args.exp_name, args.model_name_suffix)))
loss, precision, recall, accuracy, auc = test(
