def iterX(self, X):
for x_chunk, chunk_idxs in iter_data(X, np.arange(len(X)), size=self.size*20):
sort = np.argsort([len(x) for x in x_chunk])
x_chunk = [x_chunk[idx] for idx in sort]
chunk_idxs = [chunk_idxs[idx] for idx in sort]
for xmb, idxmb in iter_data(x_chunk, chunk_idxs, size=self.size):
xmb, padsize = _padded(xmb, return_sizes=True, final=self.y_lag)
yield self.x_dtype(xmb), (idxmb, padsize.T)
def iterX(self, X):
for x_chunk, chunk_idxs in iter_data(X, np.arange(len(X)), size=self.size*20):
sort = np.argsort([len(x) for x in x_chunk])
x_chunk = [x_chunk[idx] for idx in sort]
chunk_idxs = [chunk_idxs[idx] for idx in sort]
for xmb, idxmb in iter_data(x_chunk, chunk_idxs, size=self.size):
xmb = self.teXt(xmb)
yield xmb, idxmb
def iterX(self, X):
for x_chunk, chunk_idxs in iter_data(X, np.arange(len(X)), size=self.size*20):
sort = np.argsort([len(x) for x in x_chunk])
x_chunk = [x_chunk[idx] for idx in sort]
chunk_idxs = [chunk_idxs[idx] for idx in sort]
for xmb, idxmb in iter_data(x_chunk, chunk_idxs, size=self.size):
xmb = padded(xmb)
yield self.x_dtype(xmb), idxmb
def iterX(self, X):
"""
DOCSTRING
"""
for x_chunk, chunk_idxs in utils.iter_data(X,
numpy.arange(len(X)),
size=self.size * 20):
sort = numpy.argsort([len(x) for x in x_chunk])
x_chunk = [x_chunk[idx] for idx in sort]
chunk_idxs = [chunk_idxs[idx] for idx in sort]
for xmb, idxmb in utils.iter_data(x_chunk,
chunk_idxs,
size=self.size):
xmb = self.teXt(xmb)
yield xmb, idxmb
def iter_apply_msmarco(Xs, Ms, Ys):
# fns = [lambda x: np.concatenate(x, 0), lambda x: float(np.sum(x))]
logits = []
cost = 0
with torch.no_grad():
dh_model.eval()
for xmb, mmb, ymb in iter_data(Xs,
Ms,
Ys,
n_batch=n_batch_train,
truncate=True,
verbose=True):
n = len(xmb)
XMB = torch.tensor(xmb, dtype=torch.long).to(device)
YMB = torch.tensor(ymb, dtype=torch.float).to(device)
MMB = torch.tensor(mmb).to(device)
ret = dh_model(XMB)
clf_logits = []
for item in ret:
clf_logits.append(item[1])
clf_logits = torch.cat(clf_logits, dim=1)
clf_logits = clf_logits.view(n, -1)
logits.append(clf_logits.to("cpu").numpy())
logits = np.concatenate(logits, 0)
return logits #, cost
def iterX(self, X):
self.loader = Loader(X, self.test_load, self.test_transform, self.size)
self.proc = Process(target=self.loader.load)
self.proc.start()
for xmb in iter_data(X, size=self.size):
xmb = self.loader.get()
yield xmb
def evaluate(sess,
evX,
evY,
X,
Y,
gen_loss,
clf_loss,
accuracy,
n_batch,
desc,
permute=False):
metrics = []
#import ipdb; ipdb.set_trace(context=5)
for xmb, ymb in iter_data(evX,
evY,
n_batch=n_batch,
truncate=True,
verbose=True):
metrics.append(
sess.run([gen_loss[0], clf_loss[0], accuracy[0]], {
X: xmb,
Y: ymb
}))
np_metrics = np.array(metrics[:, 0])
np.save('LossResults.npy', np_metrics)
def iter_apply(Xs, Ms, Ys):
logits = []
# Compute BCE Loss
bce = torch.nn.BCEWithLogitsLoss()
bce_losses = []
with torch.no_grad():
dh_model.eval()
for xmb, mmb, ymb in iter_data(Xs,
Ms,
Ys,
n_batch=n_batch_train,
truncate=False,
verbose=False):
XMB = torch.tensor(xmb, dtype=torch.long).to(device)
YMB = torch.tensor(ymb, dtype=torch.long).to(device)
MMB = torch.tensor(mmb).to(device)
_, clf_logits = dh_model(XMB)
bce_loss = bce(clf_logits.softmax(1)[:, 1], YMB.float()).item()
bce_losses.append(bce_loss)
logits.append(clf_logits.cpu().detach().numpy())
logits = np.concatenate(logits, 0)
bce_losses = sum(bce_losses) / len(bce_losses)
return logits, bce_losses
def evaluate(sess,
evX,
evY,
X,
Y,
gen_loss,
clf_loss,
accuracy,
n_batch,
desc,
permute=False):
metrics = []
for xmb, ymb in iter_data(evX,
evY,
n_batch=n_batch,
truncate=True,
verbose=True):
metrics.append(
sess.run([gen_loss[0], clf_loss[0], accuracy[0]], {
X: xmb,
Y: ymb
}))
eval_gen_loss, eval_clf_loss, eval_accuracy = [
np.mean(m) for m in zip(*metrics)
]
print(
f"{desc} gen: {eval_gen_loss:.4f} clf: {eval_clf_loss:.4f} acc: {eval_accuracy:.2f}"
)
def run_epoch(iter):
losses = []
i = 0
for xmb, mmb in iter_data(*shuffle(trX, trM, random_state=np.random),
n_batch=n_batch_train,
truncate=True,
verbose=True):
global n_updates
dh_model.train()
XMB = torch.tensor(xmb, dtype=torch.long).to(device)
MMB = torch.tensor(mmb).to(device)
XMB = torch.tensor(xmb, dtype=torch.long).to(device)
MMB = torch.tensor(mmb).to(device)
lm_logits, past = dh_model(XMB)
loss = compute_loss_fct(lm_logits=lm_logits,
lm_labels=XMB,
encoder=text_encoder,
batch_num=n_updates,
accum_steps=int(16 / args.n_batch))
print(loss)
losses.append(loss)
n_updates += 1
if (n_updates + 1) % 10000 == 0: # and n_epochs == 0:
log(save_dir, desc, iter, save='_' + str(n_updates))
log_value('batch_train_loss', loss, n_updates)
log_value('mean_train_loss', np.mean(losses), n_updates)
log_value('total_train_loss', np.sum(losses), n_updates)
def iter_apply(Xs, Ms, Ys):
logits = []
losses = []
pred = []
gold = []
with torch.no_grad():
dh_model.eval()
for xmb, mmb, ymb in iter_data(Xs, Ms, Ys, n_batch=n_batch_train, truncate=False, verbose=True):
n = len(xmb)
XMB = torch.tensor(xmb, dtype=torch.long).to(device)
YMB = torch.tensor(ymb, dtype=torch.long).to(device)
MMB = torch.tensor(mmb, dtype=torch.long).to(device)
outputs = dh_model(XMB,attention_mask=MMB,next_sentence_label=YMB)
loss, scores = outputs[:2]
scores = torch.argmax(torch.nn.functional.softmax(scores),dim=1).tolist()
YMB = YMB.tolist()
pred.extend(scores)
gold.extend(YMB)
losses.append(float(loss))
precision = precision_score(gold, pred)
recall = recall_score(gold, pred)
f1 = f1_score(gold,pred)
acc = np.mean([gold[i] == pred[i] for i in range(len(pred))])
print(precision)
print(recall)
print(f1)
print(acc)
return np.sum(losses), np.mean(losses),precision,recall,f1,acc
def iter_apply(Xs, Ms, Ys):
# fns = [lambda x: np.concatenate(x, 0), lambda x: float(np.sum(x))]
logits = []
cost = 0
with chainer.using_config('train', False), \
chainer.using_config('enable_backprop', False):
for xmb, mmb, ymb in iter_data(Xs,
Ms,
Ys,
n_batch=n_batch_train,
truncate=False,
verbose=True):
n = len(xmb)
XMB = model.xp.asarray(xmb)
YMB = model.xp.asarray(ymb)
MMB = model.xp.asarray(mmb)
h = model(XMB)
clf_logits = clf_head(h, XMB)
clf_logits *= n
clf_losses = compute_loss_fct(XMB,
YMB,
MMB,
clf_logits,
only_return_losses=True)
clf_losses *= n
logits.append(cuda.to_cpu(clf_logits.array))
cost += cuda.to_cpu(F.sum(clf_losses).array)
logits = np.concatenate(logits, 0)
return logits, cost
def iterX(self, X):
self.loader = Loader(X, self.test_load, self.test_transform, self.size)
self.proc = Process(target=self.loader.load)
self.proc.start()
for xmb in iter_data(X, size=self.size):
xmb = self.loader.get()
yield xmb
def run_epoch(iter):
losses = []
i = 0
for xmb, ymb in iter_data(*shuffle(trX, trY, random_state=np.random),
n_batch=n_batch_train,
truncate=True,
verbose=True):
global n_updates
dh_model.train()
XMB = torch.tensor(xmb, dtype=torch.long).to(device)
YMB = torch.tensor(ymb, dtype=torch.long).to(device)
scores = dh_model(XMB)
scores = scores[0]
loss = loss_function(scores.view(-1, 2), YMB.view(-1))
losses.append(float(loss))
loss.backward()
model_opt.step()
model_opt.zero_grad()
n_updates += 1
if (n_updates + 1) % 10000 == 0:
print(acc)
if (n_updates + 1) % 10000 == 0:
log(save_dir, desc, iter, save='_' + str(n_updates))
log_value('batch_train_loss', loss, n_updates)
log_value('mean_train_loss', np.mean(losses), n_updates)
log_value('total_train_loss', np.sum(losses), n_updates)
def iter_apply_lm(Xs, Ms, denom):
logits = []
cost = 0
total_loss = 0
total_preds = 0
with torch.no_grad():
dh_model.eval()
for xmb, mmb in iter_data(Xs,
Ms,
n_batch=n_batch_train,
truncate=False,
verbose=True):
n = len(xmb)
XMB = torch.tensor(xmb, dtype=torch.long).to(device)
MMB = torch.tensor(mmb).to(device)
lm_logits, clf_logits = dh_model(XMB)
lm_losses = compute_loss_fct(XMB,
None,
MMB,
None,
lm_logits,
only_return_losses=True)
total_loss += lm_losses.item()
total_preds += torch.sum(MMB[:, 1:]).item()
return total_loss / total_preds
def iter_apply(Xs, Ms, Ys):
fns = [lambda x: np.concatenate(x, 0), lambda x: float(np.sum(x))]
results = []
for xmb, mmb, ymb in iter_data(Xs,
Ms,
Ys,
n_batch=n_batch_train,
truncate=False,
verbose=True):
n = len(xmb)
if n == n_batch_train:
res = sess.run([eval_mgpu_logits, eval_mgpu_clf_loss], {
X_train: xmb,
M_train: mmb,
Y_train: ymb
})
else:
res = sess.run([eval_logits, eval_clf_loss], {
X: xmb,
M: mmb,
Y: ymb
})
res = [r * n for r in res]
results.append(res)
results = zip(*results)
return [fn(res) for res, fn in zip(results, fns)]
def iter_apply(self, Xs, Ms, Ys):
# fns = [lambda x: np.concatenate(x, 0), lambda x: float(np.sum(x))]
logits = []
cost = 0
with torch.no_grad():
dh_model.eval()
for xmb, mmb, ymb in iter_data(Xs,
Ms,
Ys,
n_batch=n_batch_train,
truncate=False,
verbose=True):
n = len(xmb)
XMB = torch.tensor(xmb, dtype=torch.long).to(device)
YMB = torch.tensor(ymb, dtype=torch.long).to(device)
MMB = torch.tensor(mmb).to(device)
_, clf_logits = dh_model(XMB)
clf_logits *= n
clf_losses = compute_loss_fct(XMB,
YMB,
MMB,
clf_logits,
only_return_losses=True)
clf_losses *= n
logits.append(clf_logits.to("cpu").numpy())
cost += clf_losses.sum().item()
logits = np.concatenate(logits, 0)
return logits, cost
def predict(self, X):
"""
DOCSTRING
"""
for xmb in utils.iter_data(X, size=self.size):
xmb = self.teXt(xmb)
yield xmb
def iter_apply(Xs, Ms, Ys):
"""Return the cost and the logits out of the inputs, the word-embedded inputs and the outputs.
Parameters:
-----------
Xs Inputs (encoded tweets).
Ms Word-embedded inputs.
Ys Outputs (stance).
"""
logits = []
cost = 0
with torch.no_grad():
dh_model.eval()
for xmb, mmb, ymb in iter_data(Xs,
Ms,
Ys,
n_batch=n_batch_train,
truncate=False,
verbose=True):
n = len(xmb)
XMB = torch.tensor(xmb, dtype=torch.long).to(device)
YMB = torch.tensor(ymb, dtype=torch.long).to(device)
MMB = torch.tensor(mmb).to(device)
_, clf_logits = dh_model(XMB)
clf_logits *= n
clf_losses = compute_loss_fct(XMB,
YMB,
MMB,
clf_logits,
only_return_losses=True)
clf_losses *= n
logits.append(clf_logits.to("cpu").numpy())
cost += clf_losses.sum().item()
logits = np.concatenate(logits, 0)
return logits, cost
def iterXY(self, X, Y):
if self.shuffle:
X, Y = shuffle(X, Y)
for xmb, ymb in iter_data(X, Y, size=self.size):
xmb = padded(xmb)
yield self.x_dtype(xmb), self.y_dtype(ymb)
def iterXY(self, X, Y):
if self.shuffle:
X, Y = shuffle(X, Y)
for xmb, ymb in iter_data(X, Y, size=self.size):
xmb = padded(xmb)
yield self.x_dtype(xmb), self.y_dtype(ymb)
def iterXY(self, X, Y):
if self.shuffle:
X, Y = shuffle(X, Y)
for xmb, ymb in iter_data(X, Y, size=self.size):
xmb = self.trXt(xmb)
ymb = self.trYt(ymb)
yield xmb, ymb
def load(self):
"""
DOCSTRING
"""
pool = multiprocessing.Pool(8)
for xmb in utils.iter_data(self.X, size=self.size):
xmb = pool.map(self.load_fn, xmb)
self.batches.put(xmb)
self.batches.put(StopIteration)
def run_epoch(iter):
losses = []
i = 0
for xmb_kg, xmb_st, mem in iter_data(*shuffle(trX_kg,
trX_st,
trMem,
random_state=np.random),
n_batch=n_batch_train,
truncate=True,
verbose=True):
global n_updates
global past_loss
if n_updates == 0:
past_loss = math.inf
model.train()
XMB_KG = torch.tensor(xmb_kg, dtype=torch.long).to(device)
XMB_ST = torch.tensor(xmb_st, dtype=torch.long).to(device)
if args.use_mem:
mem = [
handle_empty(mem[i][0][:max_mem_size]) for i in range(len(mem))
]
mem = [torch.LongTensor([pad_rels(r) for r in m]) for m in mem]
mem = torch.stack(mem)
lm_logits, past = model(XMB_KG,
update_mem=mem,
clear_mem=True,
use_pointer=args.use_pointer,
use_scores=args.use_scores,
mem_k=args.mem_k)
loss = compute_loss_fct(lm_logits=lm_logits,
lm_labels=XMB_KG,
encoder=text_encoder,
batch_num=n_updates,
accum_steps=int(16 / args.n_batch))
else:
lm_logits, past = model(XMB_KG)
loss = compute_loss_fct(lm_logits=lm_logits,
lm_labels=XMB_KG,
encoder=text_encoder,
batch_num=n_updates,
accum_steps=int(16 / args.n_batch))
loss = float(loss)
losses.append(loss)
if abs(past_loss - loss) < args.early_stop:
break
print(loss)
n_updates += 1
if (n_updates + 1) % 20000 == 0:
va_loss = log(save_dir,
desc,
iter,
save='_' + str(n_updates),
save_model=False)
log_value('batch_train_loss', loss, n_updates)
log_value('mean_train_loss', np.mean(losses), n_updates)
log_value('total_train_loss', np.sum(losses), n_updates)
def iterXY(self, X, Y):
"""
DOCSTRING
"""
if self.shuffle:
X, Y = utils.shuffle(X, Y)
for xmb, ymb in utils.iter_data(X, Y, size=self.size):
xmb = self.trXt(xmb)
ymb = self.trYt(ymb)
yield xmb, ymb
def iterX(self, X):
"""
DOCSTRING
"""
self.loader = Loader(X, self.test_load, self.test_transform, self.size)
self.proc = multiprocessing.Process(target=self.loader.load)
self.proc.start()
for xmb in utils.iter_data(X, size=self.size):
xmb = self.loader.get()
yield xmb
def iterX(self, X):
for xmb in iter_data(X, size=self.size):
xmb = self.x_dtype(xmb)
shape = range(len(xmb.shape))
shape[0] = 1
shape[1] = 0
shape = tuple(shape)
xmb = xmb.transpose(*shape)
yield xmb
def iter_predict(Xs, Ms):
logits = []
for xmb, mmb in iter_data(n_batch_train, False, True, float("inf"), *(Xs, Ms)):
n = len(xmb)
if n == n_batch_train:
logits.append(sess.run(eval_mgpu_logits, {X_train:xmb, M_train:mmb}))
else:
logits.append(sess.run(eval_logits, {X:xmb, M:mmb}))
logits = np.concatenate(logits, 0)
return logits
def iter_predict(Xs, Ms):
logits = []
for xmb, mmb in iter_data(Xs, Ms, n_batch=n_batch_train, truncate=False, verbose=True):
n = len(xmb)
if n == n_batch_train:
logits.append(sess.run(eval_mgpu_logits, {X_train:xmb, M_train:mmb}))
else:
logits.append(sess.run(eval_logits, {X:xmb, M:mmb}))
logits = np.concatenate(logits, 0)
return logits
def iterX(self, X):
for xmb in iter_data(X, size=self.size):
xmb = self.x_dtype(xmb)
shape = range(len(xmb.shape))
shape[0] = 1
shape[1] = 0
shape = tuple(shape)
xmb = xmb.transpose(*shape)
yield xmb
def run_epoch():
for xmb, mmb, ymb in iter_data(*shuffle(trlmX, trlmM, trlmM, random_state=np.random),
n_batch=n_batch_train, truncate=True, verbose=True):
global n_updates
dh_model.train()
XMB = torch.tensor(xmb, dtype=torch.long).to(device)
YMB = torch.tensor(flatten_list(ymb), dtype=torch.long).to(device)
MMB = torch.tensor(mmb).to(device)
lm_logits, clf_logits = dh_model(XMB)
compute_loss_fct(XMB, YMB, MMB, None,lm_logits)
n_updates += 1
def iterXY(self, X, Y):
if self.shuffle:
X, Y = shuffle(X, Y)
self.loader = Loader(X, self.train_load, self.train_transform, self.size)
self.proc = Process(target=self.loader.load)
self.proc.start()
for ymb in iter_data(Y, size=self.size):
xmb = self.loader.get()
yield xmb, floatX(ymb)
def iter_apply(Xs, Ms):
logits = []
cost = 0
losses = []
with torch.no_grad():
dh_model.eval()
for xmb, mmb in iter_data(Xs, Ms, n_batch=n_batch_train, truncate=False, verbose=True):
n = len(xmb)
XMB = torch.tensor(xmb, dtype=torch.long).to(device)
loss = compute_loss_fct(dh_model(XMB)[0],XMB,text_encoder,None,True,0)
losses.append(float(loss.sum()))
return np.sum(losses), np.mean(losses)
def iter_predict(Xs, Ms):
logits = []
with torch.no_grad():
dh_model.eval()
for xmb, mmb in iter_data(Xs, Ms, n_batch=n_batch_train, truncate=False, verbose=True):
n = len(xmb)
XMB = torch.tensor(xmb, dtype=torch.long).to(device)
MMB = torch.tensor(mmb).to(device)
_, clf_logits = dh_model(XMB)
logits.append(clf_logits.to("cpu").numpy())
logits = np.concatenate(logits, 0)
return logits
def iter_predict(Xs, dh_model, n_batch_train, device, enable_dropout=False):
logits = []
with torch.no_grad():
dh_model.eval()
if enable_dropout:
dh_model.apply(enable_dropout_module)
for xmb in iter_data(Xs, n_batch=n_batch_train, truncate=False, verbose=not enable_dropout):
XMB = torch.tensor(xmb, dtype=torch.long).to(device)
_, clf_logits = dh_model(XMB)
logits.append(clf_logits.to("cpu").numpy())
logits = np.concatenate(logits, 0)
return logits
def iterXY(self, X, Y):
if self.shuffle:
X, Y = shuffle(X, Y)
for xmb, ymb in iter_data(X, Y, size=self.size):
xmb = self.x_dtype(xmb)
shape = range(len(xmb.shape))
shape[0] = 1
shape[1] = 0
shape = tuple(shape)
xmb = xmb.transpose(*shape)
yield xmb, self.y_dtype(ymb)
def iterXY(self, X, Y):
if self.shuffle:
X, Y = shuffle(X, Y)
for xmb, ymb in iter_data(X, Y, size=self.size):
xmb = _padded(xmb, final=self.y_lag)
if ymb[0].ndim == 2:
# sequence prediction
ymb, padsize = _padded(ymb, return_sizes=True, initial=self.y_lag)
yield self.x_dtype(xmb), (self.y_dtype(ymb), padsize.T)
else:
yield self.x_dtype(xmb), self.y_dtype(ymb)
def run_epoch():
for xmb, mmb, ymb in iter_data(*shuffle(trX, trM, trYt, random_state=np.random),
n_batch=n_batch_train, truncate=True, verbose=True):
global n_updates
dh_model.train()
XMB = torch.tensor(xmb, dtype=torch.long).to(device)
YMB = torch.tensor(ymb, dtype=torch.long).to(device)
MMB = torch.tensor(mmb).to(device)
lm_logits, clf_logits = dh_model(XMB)
compute_loss_fct(XMB, YMB, MMB, clf_logits, lm_logits)
n_updates += 1
if n_updates in [1000, 2000, 4000, 8000, 16000, 32000] and n_epochs == 0:
log(save_dir, desc)
def iterXY(self, X, Y):
if self.shuffle:
X, Y = shuffle(X, Y)
for xmb, ymb in iter_data(X, Y, size=self.size):
xmb = self.x_dtype(xmb)
shape = range(len(xmb.shape))
shape[0] = 1
shape[1] = 0
shape = tuple(shape)
xmb = xmb.transpose(*shape)
yield xmb, self.y_dtype(ymb)
def iter_predict(Xs, Ms):
logits = []
with chainer.using_config('train', False), \
chainer.using_config('enable_backprop', False):
for xmb, mmb in iter_data(
Xs, Ms, n_batch=n_batch_train, truncate=False, verbose=True):
n = len(xmb)
XMB = model.xp.asarray(xmb)
MMB = model.xp.asarray(mmb)
h = model(XMB)
clf_logits = clf_head(h, XMB)
logits.append(cuda.to_cpu(clf_logits.array))
logits = np.concatenate(logits, 0)
return logits
def iterXY(self, X, Y):
if self.shuffle:
X, Y = shuffle(X, Y)
for x_chunk, y_chunk in iter_data(X, Y, size=self.size*20):
sort = np.argsort([len(x) for x in x_chunk])
x_chunk = [x_chunk[idx] for idx in sort]
y_chunk = [y_chunk[idx] for idx in sort]
mb_chunks = [[x_chunk[idx:idx+self.size], y_chunk[idx:idx+self.size]] for idx in range(len(x_chunk))[::self.size]]
mb_chunks = shuffle(mb_chunks)
for xmb, ymb in mb_chunks:
xmb = padded(xmb)
yield self.x_dtype(xmb), self.y_dtype(ymb)
def iterXY(self, X, Y):
if self.shuffle:
X, Y = shuffle(X, Y)
for x_chunk, y_chunk in iter_data(X, Y, size=self.size*20):
sort = np.argsort([len(x) for x in x_chunk])
x_chunk = [x_chunk[idx] for idx in sort]
y_chunk = [y_chunk[idx] for idx in sort]
mb_chunks = [[x_chunk[idx:idx+self.size], y_chunk[idx:idx+self.size]] for idx in range(len(x_chunk))[::self.size]]
py_rng.shuffle(mb_chunks)
for xmb, ymb in mb_chunks:
xmb = self.trXt(xmb)
ymb = self.trYt(ymb)
yield xmb, ymb
def iterXY(self, X, Y):
if self.shuffle:
X, Y = shuffle(X, Y)
for xmb, ymb in iter_data(X, Y, size=self.size):
xmb = self.x_dtype(xmb)
shape = range(len(xmb.shape))
shape[0] = 1
shape[1] = 0
shape = tuple(shape)
xmb = xmb.transpose(*shape)
ymb = self.y_dtype(ymb)
if ymb.ndim == 3:
# sequence prediction! also reorder ymb.
ymb = ymb.transpose(*shape)
yield xmb, ymb
def iterXY(self, X, Y):
if self.shuffle:
X, Y = shuffle(X, Y)
for x_chunk, y_chunk in iter_data(X, Y, size=self.size*20):
sort = np.argsort([len(x) for x in x_chunk])
x_chunk = [x_chunk[idx] for idx in sort]
y_chunk = [y_chunk[idx] for idx in sort]
mb_chunks = [[x_chunk[idx:idx+self.size], y_chunk[idx:idx+self.size]] for idx in range(len(x_chunk))[::self.size]]
mb_chunks = shuffle(mb_chunks)
for xmb, ymb in mb_chunks:
xmb = _padded(xmb, final=self.y_lag)
if ymb[0].ndim == 2:
ymb, padsize = _padded(ymb, return_sizes=True, initial=self.y_lag)
yield self.x_dtype(xmb), (self.y_dtype(ymb), padsize.T)
else:
yield self.x_dtype(xmb), self.y_dtype(ymb)
def run_epoch():
for xmb, mmb, ymb in iter_data(*shuffle(trX, trM, trYt, random_state=np.random),
n_batch=n_batch_train, truncate=True, verbose=True):
global n_updates
XMB = model.xp.asarray(xmb)
YMB = model.xp.asarray(ymb)
MMB = model.xp.asarray(mmb)
h = model(XMB)
lm_logits = lm_head(h)
clf_logits = clf_head(h, XMB)
compute_loss_fct(XMB, YMB, MMB, clf_logits, lm_logits)
n_updates += 1
if n_updates in [
1000,
2000,
4000,
8000,
16000,
32000] and n_epochs == 0:
log()
def iter_apply(Xs, Ms, Ys):
# fns = [lambda x: np.concatenate(x, 0), lambda x: float(np.sum(x))]
logits = []
cost = 0
with chainer.using_config('train', False), \
chainer.using_config('enable_backprop', False):
for xmb, mmb, ymb in iter_data(
Xs, Ms, Ys, n_batch=n_batch_train, truncate=False, verbose=True):
n = len(xmb)
XMB = model.xp.asarray(xmb)
YMB = model.xp.asarray(ymb)
MMB = model.xp.asarray(mmb)
h = model(XMB)
clf_logits = clf_head(h, XMB)
clf_logits *= n
clf_losses = compute_loss_fct(
XMB, YMB, MMB, clf_logits, only_return_losses=True)
clf_losses *= n
logits.append(cuda.to_cpu(clf_logits.array))
cost += cuda.to_cpu(F.sum(clf_losses).array)
logits = np.concatenate(logits, 0)
return logits, cost
def load(self):
pool = Pool(8)
for xmb in iter_data(self.X, size=self.size):
xmb = pool.map(self.load_fn, xmb)
self.batches.put(xmb)
self.batches.put(StopIteration)
def predict(self, X):
for xmb in iter_data(X, size=self.size):
xmb = self.teXt(xmb)
yield xmb
def iterX(self, X):
for xmb in iter_data(X, size=self.size):
xmb, padsizes = _padded(xmb, return_sizes=True, final=self.y_lag)
yield self.x_dtype(xmb), padsizes.T
def iterX(self, X):
for xmb in iter_data(X, size=self.size):
xmb = padded(xmb)
yield self.x_dtype(xmb)
