def iterX(self, X): for x_chunk, chunk_idxs in iter_data(X, np.arange(len(X)), size=self.size*20): sort = np.argsort([len(x) for x in x_chunk]) x_chunk = [x_chunk[idx] for idx in sort] chunk_idxs = [chunk_idxs[idx] for idx in sort] for xmb, idxmb in iter_data(x_chunk, chunk_idxs, size=self.size): xmb, padsize = _padded(xmb, return_sizes=True, final=self.y_lag) yield self.x_dtype(xmb), (idxmb, padsize.T)
def iterX(self, X): for x_chunk, chunk_idxs in iter_data(X, np.arange(len(X)), size=self.size*20): sort = np.argsort([len(x) for x in x_chunk]) x_chunk = [x_chunk[idx] for idx in sort] chunk_idxs = [chunk_idxs[idx] for idx in sort] for xmb, idxmb in iter_data(x_chunk, chunk_idxs, size=self.size): xmb = self.teXt(xmb) yield xmb, idxmb
def iterX(self, X): for x_chunk, chunk_idxs in iter_data(X, np.arange(len(X)), size=self.size*20): sort = np.argsort([len(x) for x in x_chunk]) x_chunk = [x_chunk[idx] for idx in sort] chunk_idxs = [chunk_idxs[idx] for idx in sort] for xmb, idxmb in iter_data(x_chunk, chunk_idxs, size=self.size): xmb = padded(xmb) yield self.x_dtype(xmb), idxmb
def iterX(self, X): """ DOCSTRING """ for x_chunk, chunk_idxs in utils.iter_data(X, numpy.arange(len(X)), size=self.size * 20): sort = numpy.argsort([len(x) for x in x_chunk]) x_chunk = [x_chunk[idx] for idx in sort] chunk_idxs = [chunk_idxs[idx] for idx in sort] for xmb, idxmb in utils.iter_data(x_chunk, chunk_idxs, size=self.size): xmb = self.teXt(xmb) yield xmb, idxmb
def iter_apply_msmarco(Xs, Ms, Ys): # fns = [lambda x: np.concatenate(x, 0), lambda x: float(np.sum(x))] logits = [] cost = 0 with torch.no_grad(): dh_model.eval() for xmb, mmb, ymb in iter_data(Xs, Ms, Ys, n_batch=n_batch_train, truncate=True, verbose=True): n = len(xmb) XMB = torch.tensor(xmb, dtype=torch.long).to(device) YMB = torch.tensor(ymb, dtype=torch.float).to(device) MMB = torch.tensor(mmb).to(device) ret = dh_model(XMB) clf_logits = [] for item in ret: clf_logits.append(item[1]) clf_logits = torch.cat(clf_logits, dim=1) clf_logits = clf_logits.view(n, -1) logits.append(clf_logits.to("cpu").numpy()) logits = np.concatenate(logits, 0) return logits #, cost
def iterX(self, X): self.loader = Loader(X, self.test_load, self.test_transform, self.size) self.proc = Process(target=self.loader.load) self.proc.start() for xmb in iter_data(X, size=self.size): xmb = self.loader.get() yield xmb
def evaluate(sess, evX, evY, X, Y, gen_loss, clf_loss, accuracy, n_batch, desc, permute=False): metrics = [] #import ipdb; ipdb.set_trace(context=5) for xmb, ymb in iter_data(evX, evY, n_batch=n_batch, truncate=True, verbose=True): metrics.append( sess.run([gen_loss[0], clf_loss[0], accuracy[0]], { X: xmb, Y: ymb })) np_metrics = np.array(metrics[:, 0]) np.save('LossResults.npy', np_metrics)
def iter_apply(Xs, Ms, Ys): logits = [] # Compute BCE Loss bce = torch.nn.BCEWithLogitsLoss() bce_losses = [] with torch.no_grad(): dh_model.eval() for xmb, mmb, ymb in iter_data(Xs, Ms, Ys, n_batch=n_batch_train, truncate=False, verbose=False): XMB = torch.tensor(xmb, dtype=torch.long).to(device) YMB = torch.tensor(ymb, dtype=torch.long).to(device) MMB = torch.tensor(mmb).to(device) _, clf_logits = dh_model(XMB) bce_loss = bce(clf_logits.softmax(1)[:, 1], YMB.float()).item() bce_losses.append(bce_loss) logits.append(clf_logits.cpu().detach().numpy()) logits = np.concatenate(logits, 0) bce_losses = sum(bce_losses) / len(bce_losses) return logits, bce_losses
def evaluate(sess, evX, evY, X, Y, gen_loss, clf_loss, accuracy, n_batch, desc, permute=False): metrics = [] for xmb, ymb in iter_data(evX, evY, n_batch=n_batch, truncate=True, verbose=True): metrics.append( sess.run([gen_loss[0], clf_loss[0], accuracy[0]], { X: xmb, Y: ymb })) eval_gen_loss, eval_clf_loss, eval_accuracy = [ np.mean(m) for m in zip(*metrics) ] print( f"{desc} gen: {eval_gen_loss:.4f} clf: {eval_clf_loss:.4f} acc: {eval_accuracy:.2f}" )
def run_epoch(iter): losses = [] i = 0 for xmb, mmb in iter_data(*shuffle(trX, trM, random_state=np.random), n_batch=n_batch_train, truncate=True, verbose=True): global n_updates dh_model.train() XMB = torch.tensor(xmb, dtype=torch.long).to(device) MMB = torch.tensor(mmb).to(device) XMB = torch.tensor(xmb, dtype=torch.long).to(device) MMB = torch.tensor(mmb).to(device) lm_logits, past = dh_model(XMB) loss = compute_loss_fct(lm_logits=lm_logits, lm_labels=XMB, encoder=text_encoder, batch_num=n_updates, accum_steps=int(16 / args.n_batch)) print(loss) losses.append(loss) n_updates += 1 if (n_updates + 1) % 10000 == 0: # and n_epochs == 0: log(save_dir, desc, iter, save='_' + str(n_updates)) log_value('batch_train_loss', loss, n_updates) log_value('mean_train_loss', np.mean(losses), n_updates) log_value('total_train_loss', np.sum(losses), n_updates)
def iter_apply(Xs, Ms, Ys): logits = [] losses = [] pred = [] gold = [] with torch.no_grad(): dh_model.eval() for xmb, mmb, ymb in iter_data(Xs, Ms, Ys, n_batch=n_batch_train, truncate=False, verbose=True): n = len(xmb) XMB = torch.tensor(xmb, dtype=torch.long).to(device) YMB = torch.tensor(ymb, dtype=torch.long).to(device) MMB = torch.tensor(mmb, dtype=torch.long).to(device) outputs = dh_model(XMB,attention_mask=MMB,next_sentence_label=YMB) loss, scores = outputs[:2] scores = torch.argmax(torch.nn.functional.softmax(scores),dim=1).tolist() YMB = YMB.tolist() pred.extend(scores) gold.extend(YMB) losses.append(float(loss)) precision = precision_score(gold, pred) recall = recall_score(gold, pred) f1 = f1_score(gold,pred) acc = np.mean([gold[i] == pred[i] for i in range(len(pred))]) print(precision) print(recall) print(f1) print(acc) return np.sum(losses), np.mean(losses),precision,recall,f1,acc
def iter_apply(Xs, Ms, Ys): # fns = [lambda x: np.concatenate(x, 0), lambda x: float(np.sum(x))] logits = [] cost = 0 with chainer.using_config('train', False), \ chainer.using_config('enable_backprop', False): for xmb, mmb, ymb in iter_data(Xs, Ms, Ys, n_batch=n_batch_train, truncate=False, verbose=True): n = len(xmb) XMB = model.xp.asarray(xmb) YMB = model.xp.asarray(ymb) MMB = model.xp.asarray(mmb) h = model(XMB) clf_logits = clf_head(h, XMB) clf_logits *= n clf_losses = compute_loss_fct(XMB, YMB, MMB, clf_logits, only_return_losses=True) clf_losses *= n logits.append(cuda.to_cpu(clf_logits.array)) cost += cuda.to_cpu(F.sum(clf_losses).array) logits = np.concatenate(logits, 0) return logits, cost
def run_epoch(iter): losses = [] i = 0 for xmb, ymb in iter_data(*shuffle(trX, trY, random_state=np.random), n_batch=n_batch_train, truncate=True, verbose=True): global n_updates dh_model.train() XMB = torch.tensor(xmb, dtype=torch.long).to(device) YMB = torch.tensor(ymb, dtype=torch.long).to(device) scores = dh_model(XMB) scores = scores[0] loss = loss_function(scores.view(-1, 2), YMB.view(-1)) losses.append(float(loss)) loss.backward() model_opt.step() model_opt.zero_grad() n_updates += 1 if (n_updates + 1) % 10000 == 0: print(acc) if (n_updates + 1) % 10000 == 0: log(save_dir, desc, iter, save='_' + str(n_updates)) log_value('batch_train_loss', loss, n_updates) log_value('mean_train_loss', np.mean(losses), n_updates) log_value('total_train_loss', np.sum(losses), n_updates)
def iter_apply_lm(Xs, Ms, denom): logits = [] cost = 0 total_loss = 0 total_preds = 0 with torch.no_grad(): dh_model.eval() for xmb, mmb in iter_data(Xs, Ms, n_batch=n_batch_train, truncate=False, verbose=True): n = len(xmb) XMB = torch.tensor(xmb, dtype=torch.long).to(device) MMB = torch.tensor(mmb).to(device) lm_logits, clf_logits = dh_model(XMB) lm_losses = compute_loss_fct(XMB, None, MMB, None, lm_logits, only_return_losses=True) total_loss += lm_losses.item() total_preds += torch.sum(MMB[:, 1:]).item() return total_loss / total_preds
def iter_apply(Xs, Ms, Ys): fns = [lambda x: np.concatenate(x, 0), lambda x: float(np.sum(x))] results = [] for xmb, mmb, ymb in iter_data(Xs, Ms, Ys, n_batch=n_batch_train, truncate=False, verbose=True): n = len(xmb) if n == n_batch_train: res = sess.run([eval_mgpu_logits, eval_mgpu_clf_loss], { X_train: xmb, M_train: mmb, Y_train: ymb }) else: res = sess.run([eval_logits, eval_clf_loss], { X: xmb, M: mmb, Y: ymb }) res = [r * n for r in res] results.append(res) results = zip(*results) return [fn(res) for res, fn in zip(results, fns)]
def iter_apply(self, Xs, Ms, Ys): # fns = [lambda x: np.concatenate(x, 0), lambda x: float(np.sum(x))] logits = [] cost = 0 with torch.no_grad(): dh_model.eval() for xmb, mmb, ymb in iter_data(Xs, Ms, Ys, n_batch=n_batch_train, truncate=False, verbose=True): n = len(xmb) XMB = torch.tensor(xmb, dtype=torch.long).to(device) YMB = torch.tensor(ymb, dtype=torch.long).to(device) MMB = torch.tensor(mmb).to(device) _, clf_logits = dh_model(XMB) clf_logits *= n clf_losses = compute_loss_fct(XMB, YMB, MMB, clf_logits, only_return_losses=True) clf_losses *= n logits.append(clf_logits.to("cpu").numpy()) cost += clf_losses.sum().item() logits = np.concatenate(logits, 0) return logits, cost
def predict(self, X): """ DOCSTRING """ for xmb in utils.iter_data(X, size=self.size): xmb = self.teXt(xmb) yield xmb
def iter_apply(Xs, Ms, Ys): """Return the cost and the logits out of the inputs, the word-embedded inputs and the outputs. Parameters: ----------- Xs Inputs (encoded tweets). Ms Word-embedded inputs. Ys Outputs (stance). """ logits = [] cost = 0 with torch.no_grad(): dh_model.eval() for xmb, mmb, ymb in iter_data(Xs, Ms, Ys, n_batch=n_batch_train, truncate=False, verbose=True): n = len(xmb) XMB = torch.tensor(xmb, dtype=torch.long).to(device) YMB = torch.tensor(ymb, dtype=torch.long).to(device) MMB = torch.tensor(mmb).to(device) _, clf_logits = dh_model(XMB) clf_logits *= n clf_losses = compute_loss_fct(XMB, YMB, MMB, clf_logits, only_return_losses=True) clf_losses *= n logits.append(clf_logits.to("cpu").numpy()) cost += clf_losses.sum().item() logits = np.concatenate(logits, 0) return logits, cost
def iterXY(self, X, Y): if self.shuffle: X, Y = shuffle(X, Y) for xmb, ymb in iter_data(X, Y, size=self.size): xmb = padded(xmb) yield self.x_dtype(xmb), self.y_dtype(ymb)
def iterXY(self, X, Y): if self.shuffle: X, Y = shuffle(X, Y) for xmb, ymb in iter_data(X, Y, size=self.size): xmb = self.trXt(xmb) ymb = self.trYt(ymb) yield xmb, ymb
def load(self): """ DOCSTRING """ pool = multiprocessing.Pool(8) for xmb in utils.iter_data(self.X, size=self.size): xmb = pool.map(self.load_fn, xmb) self.batches.put(xmb) self.batches.put(StopIteration)
def run_epoch(iter): losses = [] i = 0 for xmb_kg, xmb_st, mem in iter_data(*shuffle(trX_kg, trX_st, trMem, random_state=np.random), n_batch=n_batch_train, truncate=True, verbose=True): global n_updates global past_loss if n_updates == 0: past_loss = math.inf model.train() XMB_KG = torch.tensor(xmb_kg, dtype=torch.long).to(device) XMB_ST = torch.tensor(xmb_st, dtype=torch.long).to(device) if args.use_mem: mem = [ handle_empty(mem[i][0][:max_mem_size]) for i in range(len(mem)) ] mem = [torch.LongTensor([pad_rels(r) for r in m]) for m in mem] mem = torch.stack(mem) lm_logits, past = model(XMB_KG, update_mem=mem, clear_mem=True, use_pointer=args.use_pointer, use_scores=args.use_scores, mem_k=args.mem_k) loss = compute_loss_fct(lm_logits=lm_logits, lm_labels=XMB_KG, encoder=text_encoder, batch_num=n_updates, accum_steps=int(16 / args.n_batch)) else: lm_logits, past = model(XMB_KG) loss = compute_loss_fct(lm_logits=lm_logits, lm_labels=XMB_KG, encoder=text_encoder, batch_num=n_updates, accum_steps=int(16 / args.n_batch)) loss = float(loss) losses.append(loss) if abs(past_loss - loss) < args.early_stop: break print(loss) n_updates += 1 if (n_updates + 1) % 20000 == 0: va_loss = log(save_dir, desc, iter, save='_' + str(n_updates), save_model=False) log_value('batch_train_loss', loss, n_updates) log_value('mean_train_loss', np.mean(losses), n_updates) log_value('total_train_loss', np.sum(losses), n_updates)
def iterXY(self, X, Y): """ DOCSTRING """ if self.shuffle: X, Y = utils.shuffle(X, Y) for xmb, ymb in utils.iter_data(X, Y, size=self.size): xmb = self.trXt(xmb) ymb = self.trYt(ymb) yield xmb, ymb
def iterX(self, X): """ DOCSTRING """ self.loader = Loader(X, self.test_load, self.test_transform, self.size) self.proc = multiprocessing.Process(target=self.loader.load) self.proc.start() for xmb in utils.iter_data(X, size=self.size): xmb = self.loader.get() yield xmb
def iterX(self, X): for xmb in iter_data(X, size=self.size): xmb = self.x_dtype(xmb) shape = range(len(xmb.shape)) shape[0] = 1 shape[1] = 0 shape = tuple(shape) xmb = xmb.transpose(*shape) yield xmb
def iter_predict(Xs, Ms): logits = [] for xmb, mmb in iter_data(n_batch_train, False, True, float("inf"), *(Xs, Ms)): n = len(xmb) if n == n_batch_train: logits.append(sess.run(eval_mgpu_logits, {X_train:xmb, M_train:mmb})) else: logits.append(sess.run(eval_logits, {X:xmb, M:mmb})) logits = np.concatenate(logits, 0) return logits
def iter_predict(Xs, Ms): logits = [] for xmb, mmb in iter_data(Xs, Ms, n_batch=n_batch_train, truncate=False, verbose=True): n = len(xmb) if n == n_batch_train: logits.append(sess.run(eval_mgpu_logits, {X_train:xmb, M_train:mmb})) else: logits.append(sess.run(eval_logits, {X:xmb, M:mmb})) logits = np.concatenate(logits, 0) return logits
def run_epoch(): for xmb, mmb, ymb in iter_data(*shuffle(trlmX, trlmM, trlmM, random_state=np.random), n_batch=n_batch_train, truncate=True, verbose=True): global n_updates dh_model.train() XMB = torch.tensor(xmb, dtype=torch.long).to(device) YMB = torch.tensor(flatten_list(ymb), dtype=torch.long).to(device) MMB = torch.tensor(mmb).to(device) lm_logits, clf_logits = dh_model(XMB) compute_loss_fct(XMB, YMB, MMB, None,lm_logits) n_updates += 1
def iterXY(self, X, Y): if self.shuffle: X, Y = shuffle(X, Y) self.loader = Loader(X, self.train_load, self.train_transform, self.size) self.proc = Process(target=self.loader.load) self.proc.start() for ymb in iter_data(Y, size=self.size): xmb = self.loader.get() yield xmb, floatX(ymb)
def iter_apply(Xs, Ms): logits = [] cost = 0 losses = [] with torch.no_grad(): dh_model.eval() for xmb, mmb in iter_data(Xs, Ms, n_batch=n_batch_train, truncate=False, verbose=True): n = len(xmb) XMB = torch.tensor(xmb, dtype=torch.long).to(device) loss = compute_loss_fct(dh_model(XMB)[0],XMB,text_encoder,None,True,0) losses.append(float(loss.sum())) return np.sum(losses), np.mean(losses)
def iter_predict(Xs, Ms): logits = [] with torch.no_grad(): dh_model.eval() for xmb, mmb in iter_data(Xs, Ms, n_batch=n_batch_train, truncate=False, verbose=True): n = len(xmb) XMB = torch.tensor(xmb, dtype=torch.long).to(device) MMB = torch.tensor(mmb).to(device) _, clf_logits = dh_model(XMB) logits.append(clf_logits.to("cpu").numpy()) logits = np.concatenate(logits, 0) return logits
def iter_predict(Xs, dh_model, n_batch_train, device, enable_dropout=False): logits = [] with torch.no_grad(): dh_model.eval() if enable_dropout: dh_model.apply(enable_dropout_module) for xmb in iter_data(Xs, n_batch=n_batch_train, truncate=False, verbose=not enable_dropout): XMB = torch.tensor(xmb, dtype=torch.long).to(device) _, clf_logits = dh_model(XMB) logits.append(clf_logits.to("cpu").numpy()) logits = np.concatenate(logits, 0) return logits
def iterXY(self, X, Y): if self.shuffle: X, Y = shuffle(X, Y) for xmb, ymb in iter_data(X, Y, size=self.size): xmb = self.x_dtype(xmb) shape = range(len(xmb.shape)) shape[0] = 1 shape[1] = 0 shape = tuple(shape) xmb = xmb.transpose(*shape) yield xmb, self.y_dtype(ymb)
def iterXY(self, X, Y): if self.shuffle: X, Y = shuffle(X, Y) for xmb, ymb in iter_data(X, Y, size=self.size): xmb = _padded(xmb, final=self.y_lag) if ymb[0].ndim == 2: # sequence prediction ymb, padsize = _padded(ymb, return_sizes=True, initial=self.y_lag) yield self.x_dtype(xmb), (self.y_dtype(ymb), padsize.T) else: yield self.x_dtype(xmb), self.y_dtype(ymb)
def run_epoch(): for xmb, mmb, ymb in iter_data(*shuffle(trX, trM, trYt, random_state=np.random), n_batch=n_batch_train, truncate=True, verbose=True): global n_updates dh_model.train() XMB = torch.tensor(xmb, dtype=torch.long).to(device) YMB = torch.tensor(ymb, dtype=torch.long).to(device) MMB = torch.tensor(mmb).to(device) lm_logits, clf_logits = dh_model(XMB) compute_loss_fct(XMB, YMB, MMB, clf_logits, lm_logits) n_updates += 1 if n_updates in [1000, 2000, 4000, 8000, 16000, 32000] and n_epochs == 0: log(save_dir, desc)
def iter_predict(Xs, Ms): logits = [] with chainer.using_config('train', False), \ chainer.using_config('enable_backprop', False): for xmb, mmb in iter_data( Xs, Ms, n_batch=n_batch_train, truncate=False, verbose=True): n = len(xmb) XMB = model.xp.asarray(xmb) MMB = model.xp.asarray(mmb) h = model(XMB) clf_logits = clf_head(h, XMB) logits.append(cuda.to_cpu(clf_logits.array)) logits = np.concatenate(logits, 0) return logits
def iterXY(self, X, Y): if self.shuffle: X, Y = shuffle(X, Y) for x_chunk, y_chunk in iter_data(X, Y, size=self.size*20): sort = np.argsort([len(x) for x in x_chunk]) x_chunk = [x_chunk[idx] for idx in sort] y_chunk = [y_chunk[idx] for idx in sort] mb_chunks = [[x_chunk[idx:idx+self.size], y_chunk[idx:idx+self.size]] for idx in range(len(x_chunk))[::self.size]] mb_chunks = shuffle(mb_chunks) for xmb, ymb in mb_chunks: xmb = padded(xmb) yield self.x_dtype(xmb), self.y_dtype(ymb)
def iterXY(self, X, Y): if self.shuffle: X, Y = shuffle(X, Y) for x_chunk, y_chunk in iter_data(X, Y, size=self.size*20): sort = np.argsort([len(x) for x in x_chunk]) x_chunk = [x_chunk[idx] for idx in sort] y_chunk = [y_chunk[idx] for idx in sort] mb_chunks = [[x_chunk[idx:idx+self.size], y_chunk[idx:idx+self.size]] for idx in range(len(x_chunk))[::self.size]] py_rng.shuffle(mb_chunks) for xmb, ymb in mb_chunks: xmb = self.trXt(xmb) ymb = self.trYt(ymb) yield xmb, ymb
def iterXY(self, X, Y): if self.shuffle: X, Y = shuffle(X, Y) for xmb, ymb in iter_data(X, Y, size=self.size): xmb = self.x_dtype(xmb) shape = range(len(xmb.shape)) shape[0] = 1 shape[1] = 0 shape = tuple(shape) xmb = xmb.transpose(*shape) ymb = self.y_dtype(ymb) if ymb.ndim == 3: # sequence prediction! also reorder ymb. ymb = ymb.transpose(*shape) yield xmb, ymb
def iterXY(self, X, Y): if self.shuffle: X, Y = shuffle(X, Y) for x_chunk, y_chunk in iter_data(X, Y, size=self.size*20): sort = np.argsort([len(x) for x in x_chunk]) x_chunk = [x_chunk[idx] for idx in sort] y_chunk = [y_chunk[idx] for idx in sort] mb_chunks = [[x_chunk[idx:idx+self.size], y_chunk[idx:idx+self.size]] for idx in range(len(x_chunk))[::self.size]] mb_chunks = shuffle(mb_chunks) for xmb, ymb in mb_chunks: xmb = _padded(xmb, final=self.y_lag) if ymb[0].ndim == 2: ymb, padsize = _padded(ymb, return_sizes=True, initial=self.y_lag) yield self.x_dtype(xmb), (self.y_dtype(ymb), padsize.T) else: yield self.x_dtype(xmb), self.y_dtype(ymb)
def run_epoch(): for xmb, mmb, ymb in iter_data(*shuffle(trX, trM, trYt, random_state=np.random), n_batch=n_batch_train, truncate=True, verbose=True): global n_updates XMB = model.xp.asarray(xmb) YMB = model.xp.asarray(ymb) MMB = model.xp.asarray(mmb) h = model(XMB) lm_logits = lm_head(h) clf_logits = clf_head(h, XMB) compute_loss_fct(XMB, YMB, MMB, clf_logits, lm_logits) n_updates += 1 if n_updates in [ 1000, 2000, 4000, 8000, 16000, 32000] and n_epochs == 0: log()
def iter_apply(Xs, Ms, Ys): # fns = [lambda x: np.concatenate(x, 0), lambda x: float(np.sum(x))] logits = [] cost = 0 with chainer.using_config('train', False), \ chainer.using_config('enable_backprop', False): for xmb, mmb, ymb in iter_data( Xs, Ms, Ys, n_batch=n_batch_train, truncate=False, verbose=True): n = len(xmb) XMB = model.xp.asarray(xmb) YMB = model.xp.asarray(ymb) MMB = model.xp.asarray(mmb) h = model(XMB) clf_logits = clf_head(h, XMB) clf_logits *= n clf_losses = compute_loss_fct( XMB, YMB, MMB, clf_logits, only_return_losses=True) clf_losses *= n logits.append(cuda.to_cpu(clf_logits.array)) cost += cuda.to_cpu(F.sum(clf_losses).array) logits = np.concatenate(logits, 0) return logits, cost
def load(self): pool = Pool(8) for xmb in iter_data(self.X, size=self.size): xmb = pool.map(self.load_fn, xmb) self.batches.put(xmb) self.batches.put(StopIteration)
def predict(self, X): for xmb in iter_data(X, size=self.size): xmb = self.teXt(xmb) yield xmb
def iterX(self, X): for xmb in iter_data(X, size=self.size): xmb, padsizes = _padded(xmb, return_sizes=True, final=self.y_lag) yield self.x_dtype(xmb), padsizes.T
def iterX(self, X): for xmb in iter_data(X, size=self.size): xmb = padded(xmb) yield self.x_dtype(xmb)