def train_one_epoch(epoch, model, train_loader, optimizer, tokenizer, params):
device = params.device
avg_loss = AverageMeter()
avg_acc = Accuracy(ignore_index=-1)
model.train()
for i, batch in enumerate(train_loader):
optimizer.zero_grad()
batch = batch.to(device)
# segment = create_dummy_segment(batch)
inputs, labels = mask_tokens(batch, tokenizer, params)
inputs = inputs.to(device)
labels = labels.to(device)
outputs = model(inputs, masked_lm_labels=labels)
loss, prediction_scores = outputs[:2] # model outputs are always tuple in transformers (see doc)
loss.backward()
optimizer.step()
avg_acc.update(prediction_scores.view(-1, params.vocab_size), labels.view(-1))
avg_loss.update(loss.item())
logging.info('Train-E-{}: loss: {:.4f}'.format(epoch, avg_loss()))
def _eval_epoch(self, epoch):
batch_time = AverageMeter()
data_time = AverageMeter()
ave_total_loss = AverageMeter()
ave_acc = AverageMeter()
ave_iou = AverageMeter()
# set model mode
self.model.eval()
with torch.no_grad():
tic = time.time()
for steps, (data, target) in enumerate(self.valid_data_loder,
start=1):
# processing no blocking
# non_blocking tries to convert asynchronously with respect to the host if possible
# converting CPU tensor with pinned memory to CUDA tensor
# overlap transfer if pinned memory
data = data.to(self.device, non_blocking=True)
target = target.to(self.device, non_blocking=True)
data_time.update(time.time() - tic)
logits = self.model(data)
loss = self.loss(logits, target)
# calculate metrics
acc = Accuracy(logits, target)
miou = MIoU(logits, target, self.config.nb_classes)
#print("===========acc, miou==========", acc, miou)
# update ave metrics
batch_time.update(time.time() - tic)
ave_total_loss.update(loss.data.item())
ave_acc.update(acc.item())
ave_iou.update(miou.item())
tic = time.time()
# display validation at the end
print('Epoch {} validation done !'.format(epoch))
print('Time: {:.4f}, Data: {:.4f},\n'
'MIoU: {:6.4f}, Accuracy: {:6.4f}, Loss: {:.6f}'.
format(batch_time.average(), data_time.average(),
ave_iou.average(), ave_acc.average(),
ave_total_loss.average()))
self.history['valid']['epoch'].append(epoch)
self.history['valid']['loss'].append(ave_total_loss.average())
self.history['valid']['acc'].append(ave_acc.average())
self.history['valid']['miou'].append(ave_iou.average())
# validation log and return
return {
'epoch': epoch,
'val_Loss': ave_total_loss.average(),
'val_Accuracy': ave_acc.average(),
'val_MIoU': ave_iou.average(),
}
def fit(self, x, y, epochs=10, learning_rate=0.001, showfig=False):
self.X = x
self.Y = y
self.Weights = np.random.randn(self.X.shape[1], self.Y.shape[1])
self.bias = np.random.randn(self.Y.shape[1])
loss = []
metric = []
for i in range(epochs):
Y_pred = softmax(self.X, self.Weights, self.bias)
self.Weights -= learning_rate * (
(self.X.T.dot(Y_pred - self.Y)) / len(self.X))
self.bias -= learning_rate * (
(Y_pred.T.dot(1 - Y_pred).sum(axis=1)) / len(self.X))
epoch_loss = Cross_Entropy(self.Y, Y_pred)
epoch_metric = Accuracy(self.Y, Y_pred) * 100
loss.append(epoch_loss)
metric.append(epoch_metric)
print("epoch : ", i + 1, "loss : ", epoch_loss, "Accuracy : ",
epoch_metric)
if (showfig):
plt.plot(range(1, epochs + 1), loss, label='Training loss')
plt.plot(range(1, epochs + 1), metric, label='Training metric')
plt.legend()
plt.show()
def fit(self,
X,
Y,
C=0.001,
epochs=10,
learning_rate=0.001,
show_fig=False):
self.X = X
self.Y = Y.astype(np.int32)
self.Weights = np.random.randn(len(set(self.Y)), self.X.shape[1])
self.bias = np.zeros((len(set(self.Y))))
self.C = C
loss = []
metric = []
for i in range(epochs):
cost = 0
for j in set(self.Y):
grad_w = np.zeros(self.X.shape[1])
grad_b = 0
for k in [h for h in set(self.Y) if h != j]:
posidx = np.where(self.Y == j)
negidx = np.where(self.Y == k)
tmp_x = np.concatenate((self.X[posidx], self.X[negidx]),
axis=0)
tmp_y = np.concatenate((self.Y[posidx], self.Y[negidx]),
axis=0)
tmp_y[tmp_y == j] = 1
tmp_y[tmp_y == k] = -1
tmp_pred = tmp_x.dot(self.Weights[j]) + self.bias[j]
marg = self.margins(tmp_y, tmp_pred)
cost += self.loss(self.Weights[j], marg)
grad_w += self.Weights[j] - self.C * (
tmp_x[marg > 0].T.dot(tmp_y[marg > 0]))
grad_b += self.C * tmp_y[marg > 0].sum()
self.Weights[j] -= learning_rate * grad_w
self.bias[j] -= learning_rate * grad_b
Y_pred = self.predict(self.X)
epoch_acc = Accuracy(self.Y, Y_pred)
print("epoch : ", i + 1, "Training Accuracy : ", epoch_acc,
'loss : ', cost)
metric.append(epoch_acc * 100)
loss.append(cost)
if (show_fig):
plt.plot(range(1, epochs + 1), metric, label='training accuracy')
plt.legend()
plt.show()
plt.plot(range(1, epochs + 1), loss, label='training loss')
plt.legend()
plt.show()
def lg_k_folds(X_train, y_train, lr, b, epochs, lamda, bias, k=5, verbose=False):
results = {
'accuracy': [],
'recall': [],
'precision': []
}
metric_means = {}
accuracy = Accuracy()
recall = Recall()
precision = Precision()
chunk_size = int(len(X_train) / k)
logistic_regression = LogisticRegression(bias)
for i in range(0, len(X_train), chunk_size):
end = i + chunk_size if i + chunk_size <= len(X_train) else len(X_train)
new_X_valid = X_train[i: end]
new_y_valid = y_train[i: end]
new_X_train = np.concatenate([X_train[: i], X_train[end:]])
new_y_train = np.concatenate([y_train[: i], y_train[end:]])
logistic_regression.fit(new_X_train, new_y_train, lr, b, epochs, lamda, verbose=verbose)
predictions = logistic_regression.predict(new_X_valid)
results['accuracy'].append(accuracy(new_y_valid, predictions))
results['recall'].append(recall(new_y_valid, predictions))
results['precision'].append(precision(new_y_valid, predictions))
metric_means['accuracy'] = np.mean(results['accuracy'])
metric_means['recall'] = np.mean(results['recall'])
metric_means['precision'] = np.mean(results['precision'])
return metric_means
def metrics(self):
tpr = TruePositiveRate(self.cm).get()
fpr = FalsePositiveRate(self.cm).get()
acc = Accuracy(self.cm).get()
f1 = F1(self.cm).get()
f1 = round(f1, 2)
return tpr, fpr, acc, f1
def main():
# c=Collection()
# c.load(Path("./2021/ref/training/medline.1200.es.txt"))
# pickle_postag(c)
file = './2021/ref/training/medline.1200.es.txt'
data = SentenceDataset(
file,
transform=sentence_to_tensor,
target_transform=lambda l: torch.stack(tuple(map(label_to_tensor, l))))
data_loader = DataLoader(data,
batch_size=4,
collate_fn=my_collate_fn,
shuffle=True)
n = MyLSTM(50, 50, len(TAGS), 113, 50)
n.to(DEVICE)
optimizer = torch.optim.SGD(n.parameters(), lr=learning_rate)
metrics = {
'acc':
lambda pred, true: Accuracy()(pred, true),
'f1':
lambda pred, true: F1Score()
(torch.tensor(pred.argmax(dim=1), dtype=torch.float32),
torch.tensor(true, dtype=torch.float32))
}
train(data_loader,
n,
criterion,
optimizer,
5,
filename='test_lstm.pth',
metrics=metrics)
def main():
device = set_device(FLAGS.device)
fluid.enable_dygraph(device) if FLAGS.dynamic else None
train_dataset = MnistDataset(mode='train')
val_dataset = MnistDataset(mode='test')
inputs = [Input([None, 784], 'float32', name='image')]
labels = [Input([None, 1], 'int64', name='label')]
model = MNIST()
optim = Momentum(learning_rate=FLAGS.lr,
momentum=.9,
parameter_list=model.parameters())
model.prepare(optim,
CrossEntropy(),
Accuracy(topk=(1, 2)),
inputs,
labels,
device=FLAGS.device)
if FLAGS.resume is not None:
model.load(FLAGS.resume)
model.fit(train_dataset,
val_dataset,
epochs=FLAGS.epoch,
batch_size=FLAGS.batch_size,
save_dir='mnist_checkpoint')
def run_epoch(self, data, training):
print('Model will be saved to %s' % self.ckpt_path)
total_loss = 0
accuracy = Accuracy()
self.encoder.train(training)
self.decoder.train(training)
if training:
description = 'Train'
random.shuffle(data)
else:
description = 'Valid'
lines = [self.tensorFromSentence(line) for line in data]
bar = tqdm(range(len(lines)), desc=description)
tqdm.write('[-] Start training!')
for iter in bar:
input_tensor = lines[iter][1:-1] # do not input SOS and EOS
target_tensor = lines[iter][1:] # not include SOS in target
loss, predict_tensor = self.run_iter(input_tensor, target_tensor)
if training:
loss.backward()
self.encoder_optim.step()
self.decoder_optim.step()
accuracy(predict_tensor, target_tensor)
loss = loss.item() / target_tensor.size(0)
total_loss += loss
bar.set_postfix(avg_loss=total_loss / (iter + 1),
score='%d/%d' % (accuracy.correct, accuracy.total),
accuracy="%.2f" % accuracy.value())
if training:
self.history['train'].append({
'accuracy': accuracy.value(),
'loss': total_loss / len(bar)
})
else:
self.history['valid'].append({
'accuracy': accuracy.value(),
'loss': total_loss / len(bar)
})
self.encoder_scheduler.step()
self.decoder_scheduler.step()
def evaluate(self, x, y):
Y_pred = self.predict(x)
return Accuracy(y, Y_pred)
################################################################################
def run_epoch(self, epoch, training):
self.model.train(training)
if training:
description = '[Train]'
dataset = self.trainData
shuffle = True
else:
description = '[Valid]'
dataset = self.validData
shuffle = False
# dataloader for train and valid
dataloader = DataLoader(
dataset,
batch_size=self.batch_size,
shuffle=shuffle,
num_workers=8,
collate_fn=dataset.collate_fn,
)
trange = tqdm(enumerate(dataloader),
total=len(dataloader),
desc=description)
loss = 0
accuracy = Accuracy()
for i, (
x,
y,
) in trange: # (x,y) = 128*128
o_labels, batch_loss = self.run_iter(x, y)
if training:
self.opt.zero_grad() # reset gradient to 0
batch_loss.backward() # calculate gradient
self.opt.step() # update parameter by gradient
loss += batch_loss.item() # .item() to get python number in Tensor
accuracy.update(o_labels.cpu(), y)
trange.set_postfix(accuracy=accuracy.print_score(),
loss=loss / (i + 1))
if training:
self.history['train'].append({
'accuracy': accuracy.get_score(),
'loss': loss / len(trange)
})
self.scheduler.step()
else:
self.history['valid'].append({
'accuracy': accuracy.get_score(),
'loss': loss / len(trange)
})
if loss < self.min_loss:
self.save_best_model(epoch)
self.min_loss = loss
self.save_hist()
def _construct_loss(self):
# neutral = self.neutral_input_idx
self.metrics = {
'accuracy': Accuracy(ignore_index=None)
}
self.criterion = CrossEntropyLoss()
self.val_metrics = {
'loss': self.criterion,
**self.metrics
}
logger.info('Selected metrics.', extra={'metrics': list(self.metrics.keys())})
def run_epoch(self, epoch, training):
self.model.train(training)
if training:
description = 'Train'
dataset = self.trainData
shuffle = True
else:
description = 'Valid'
dataset = self.validData
shuffle = False
dataloader = DataLoader(dataset=dataset,
batch_size=self.batch_size,
shuffle=shuffle,
collate_fn=dataset.collate_fn,
num_workers=4)
trange = tqdm(enumerate(dataloader),
total=len(dataloader),
desc=description,
ascii=True)
f_loss = 0
l_loss = 0
accuracy = Accuracy()
for i, (x, missing, y) in trange:
o_labels, batch_f_loss, batch_l_loss = self.run_iter(x, missing, y)
batch_loss = batch_f_loss + batch_l_loss
if training:
self.opt.zero_grad()
batch_loss.backward()
self.opt.step()
f_loss += batch_f_loss.item()
l_loss += batch_l_loss.item()
accuracy.update(o_labels.cpu(), y)
trange.set_postfix(accuracy=accuracy.print_score(),
f_loss=f_loss / (i + 1),
l_loss=l_loss / (i + 1))
if training:
self.history['train'].append({
'accuracy': accuracy.get_score(),
'loss': f_loss / len(trange)
})
self.scheduler.step()
else:
self.history['valid'].append({
'accuracy': accuracy.get_score(),
'loss': f_loss / len(trange)
})
def main(args):
# load data
data, meta = data_utils.load_data(args.dataset_root,
args.dataset,
is_training=False)
# build val dataloader
dataset = data_utils.ImageDataset(*data, is_training=False)
dataloader = torch.utils.data.DataLoader(dataset,
args.batch_size,
shuffle=False,
num_workers=2,
pin_memory=True)
# remove temp dataset variables to reduce memory usage
del data
device = torch.device(args.device)
# build model
if args.model == 'resnet_20':
model = models.Resnet20
else:
model = models.SimpleCNN
net = model(dataset.shape, meta['n_class']).to(device=device)
criterion = torch.nn.CrossEntropyLoss()
state = torch.load(args.cpt)
net.load_state_dict(state['net'])
net.eval()
mean_loss, acc = MeanValue(), Accuracy()
for x, y in dataloader:
if device.type == 'cuda':
x = x.cuda(device, non_blocking=True)
y = y.cuda(device, non_blocking=True)
logits = net(x)
loss = criterion(logits, y)
loss = loss.detach().cpu().numpy()
predicts = torch.argmax(logits, dim=1).detach().cpu().numpy()
y = y.detach().cpu().numpy()
mean_loss.add(loss)
acc.add(predicts, y)
print('loss: {:.4f}, acc: {:.2%}'.format(mean_loss.get(), acc.get()))
def run_epoch(self, epoch, source_dataloader, target_dataloader, lamb):
trange = tqdm(zip(source_dataloader, target_dataloader),
total=len(source_dataloader),
desc=f'[epoch {epoch}]')
total_D_loss, total_F_loss = 0.0, 0.0
acc = Accuracy()
for i, ((source_data, source_label), (target_data,
_)) in enumerate(trange):
source_data = source_data.to(self.device)
source_label = source_label.to(self.device)
target_data = target_data.to(self.device)
# =========== Preprocess =================
# mean/var of source and target datas are different, so we put them together for properly batch_norm
mixed_data = torch.cat([source_data, target_data], dim=0)
domain_label = torch.zeros(
[source_data.shape[0] + target_data.shape[0],
1]).to(self.device)
domain_label[:source_data.shape[
0]] = 1 # source data label=1, target data lebel=0
feature = self.feature_extractor(mixed_data)
# =========== Step 1 : Train Domain Classifier (fix feature extractor by feature.detach()) =================
domain_logits = self.domain_classifier(feature.detach())
loss = self.domain_criterion(domain_logits, domain_label)
total_D_loss += loss.item()
loss.backward()
self.optimizer_D.step()
# =========== Step 2: Train Feature Extractor and Label Predictor =================
class_logits = self.label_predictor(feature[:source_data.shape[0]])
domain_logits = self.domain_classifier(feature)
loss = self.class_criterion(
class_logits, source_label) - lamb * self.domain_criterion(
domain_logits, domain_label)
total_F_loss += loss.item()
loss.backward()
self.optimizer_F.step()
self.optimizer_C.step()
self.optimizer_D.zero_grad()
self.optimizer_F.zero_grad()
self.optimizer_C.zero_grad()
acc.update(class_logits, source_label)
trange.set_postfix(D_loss=total_D_loss / (i + 1),
F_loss=total_F_loss / (i + 1),
acc=acc.print_score())
self.history['d_loss'].append(total_D_loss / len(trange))
self.history['f_loss'].append(total_F_loss / len(trange))
self.history['acc'].append(acc.get_score())
self.save_hist()
self.save_model()
def metrics_dict(self, prefix="train"):
if self.tusk == "classification":
return {
f"{prefix}_top1": Top(n=1),
f"{prefix}_top5": Top(n=5),
f"{prefix}_MRR": MRR()
}
elif self.tusk == "generation":
ignore_idxs = (self.dm.target_eos_idx, self.dm.target_pad_idx)
return {
f"{prefix}_accuracy": Accuracy(),
f"{prefix}_precision": Precision(ignore_idxs),
f"{prefix}_recall": Recall(ignore_idxs),
f"{prefix}_F1": F1(ignore_idxs)
}
else:
return ValueError(f"{self.tusk} tusk is not supported")
def main(args):
config_path = os.path.join(args.model_dir, 'config.json')
with open(config_path) as f:
config = json.load(f)
logging.info('loading word dictionary...')
with open(config['words_dict'], 'rb') as f:
words_dict = pickle.load(f)
logging.info('loading train data...')
with open(config['train'], 'rb') as f:
train = pickle.load(f)
logging.info('loading validation data...')
with open(config['model_parameters']['valid'], 'rb') as f:
valid = pickle.load(f)
config['model_parameters']['valid'] = valid
if args.lr_finder:
pass
else:
if config['arch'] == 'Predictor':
from predictor import Predictor
PredictorClass = Predictor
predictor = PredictorClass(metrics=[Accuracy()],
word_dict=words_dict,
**config['model_parameters'])
metrics_logger = MetricsLogger(os.path.join(args.model_dir,
'log.json'))
if args.load is not None:
predictor.load(args.load)
try:
metrics_logger.load(int(args.load.split('.')[-1]))
except:
metrics_logger.load(448)
model_checkpoint = ModelCheckpoint(
os.path.join(args.model_dir, 'model.pkl'), 'Accuracy', 1, 'max')
logging.info('start training!')
predictor.fit_dataset(train, train.collate_fn,
[model_checkpoint, metrics_logger])
def main():
device = set_device(FLAGS.device)
fluid.enable_dygraph(device) if FLAGS.dynamic else None
model = models.__dict__[FLAGS.arch](pretrained=FLAGS.eval_only and
not FLAGS.resume)
if FLAGS.resume is not None:
model.load(FLAGS.resume)
inputs = [Input([None, 3, 224, 224], 'float32', name='image')]
labels = [Input([None, 1], 'int64', name='label')]
train_dataset = ImageNetDataset(
os.path.join(FLAGS.data, 'train'), mode='train')
val_dataset = ImageNetDataset(os.path.join(FLAGS.data, 'val'), mode='val')
optim = make_optimizer(
np.ceil(
len(train_dataset) * 1. / FLAGS.batch_size / ParallelEnv().nranks),
parameter_list=model.parameters())
model.prepare(optim, CrossEntropy(), Accuracy(topk=(1, 5)), inputs, labels)
if FLAGS.eval_only:
model.evaluate(
val_dataset,
batch_size=FLAGS.batch_size,
num_workers=FLAGS.num_workers)
return
output_dir = os.path.join(FLAGS.output_dir, FLAGS.arch,
time.strftime('%Y-%m-%d-%H-%M',
time.localtime()))
if ParallelEnv().local_rank == 0 and not os.path.exists(output_dir):
os.makedirs(output_dir)
model.fit(train_dataset,
val_dataset,
batch_size=FLAGS.batch_size,
epochs=FLAGS.epoch,
save_dir=output_dir,
num_workers=FLAGS.num_workers)
def training(args, train_loader, valid_loader, model, optimizer, device):
train_metrics = Accuracy()
best_valid_acc = 0
total_iter = 0
criterion = torch.nn.CrossEntropyLoss()
for epoch in range(args.epochs):
train_trange = tqdm(enumerate(train_loader),
total=len(train_loader),
desc='training')
train_loss = 0
train_metrics.reset()
for i, batch in train_trange:
model.train()
prob = run_iter(batch, model, device, training=True)
answer = batch['label'].to(device)
loss = criterion(prob, answer)
optimizer.zero_grad()
loss.backward()
optimizer.step()
total_iter += 1
train_loss += loss.item()
train_metrics.update(prob, answer)
train_trange.set_postfix(
loss=train_loss / (i + 1),
**{train_metrics.name: train_metrics.print_score()})
if total_iter % args.eval_steps == 0:
valid_acc = testing(valid_loader, model, device, valid=True)
if valid_acc > best_valid_acc:
best_valid_acc = valid_acc
torch.save(
model,
os.path.join(
args.model_dir,
'fine-tuned_bert_{}.pkl'.format(args.seed)))
# Final validation
valid_acc = testing(valid_loader, model, device, valid=True)
if valid_acc > best_valid_acc:
best_valid_acc = valid_acc
torch.save(
model,
os.path.join(args.model_dir,
'fine-tuned_bert_{}.pkl'.format(args.seed)))
print('Best Valid Accuracy:{}'.format(best_valid_acc))
def _construct_criterion_and_metrics(self):
neutral = self.neutral_input_idx
self.metrics = {'accuracy': Accuracy(ignore_index=neutral)}
if len(self.out_classes) == 2:
logger.info(
'Binary segmentation, will use both BCE & Dice loss components.'
)
self.metrics.update({'dice': Dice(ignore_index=neutral)})
l_weights = self.config['loss_weights']
self.criterion = BCEDiceLoss(ignore_index=neutral,
w_bce=l_weights['bce'],
w_dice=l_weights['dice'])
else:
logger.info('Multiclass segmentation, will use NLLLoss only.')
self.criterion = NLLLoss(ignore_index=neutral)
self.val_metrics = {'loss': self.criterion, **self.metrics}
logger.info('Selected metrics.',
extra={'metrics': list(self.metrics.keys())})
def fit(self,
X,
Y,
C=0.001,
epochs=10,
learning_rate=0.001,
show_fig=False):
self.X = X
self.Y = Y
self.Y[self.Y == 0] = -1
self.Weights = np.random.randn(self.X.shape[1])
self.bias = 0
self.C = C
loss = []
metric = []
for i in range(epochs):
Y_pred = self.X.dot(self.Weights) + self.bias
marg = self.margins(Y_pred)
cost = self.loss(marg)
grad_w = self.Weights - self.C * (self.X[marg > 0].T.dot(
self.Y[marg > 0]))
grad_b = self.C * self.Y[marg > 0].sum()
self.Weights -= learning_rate * grad_w
self.bias -= learning_rate * grad_b
epoch_acc = Accuracy(self.Y, np.sign(Y_pred))
print("epoch : ", i + 1, "Training Accuracy : ", epoch_acc)
metric.append(epoch_acc * 100)
loss.append(cost)
if (show_fig):
plt.plot(range(1, epochs + 1), metric, label='training accuracy')
plt.legend()
plt.show()
plt.plot(range(1, epochs + 1), loss, label='training loss')
plt.legend()
plt.show()
def k_folds(X_train, y_train, lr, b, epochs, k=5):
l_regression = LogisticRegression()
error = Accuracy()
chunk_size = int(len(X_train) / k)
mse_list = []
for i in range(0, len(X_train), chunk_size):
end = i + chunk_size if i + chunk_size <= len(X_train) else len(
X_train)
new_X_valid = X_train[i:end]
new_y_valid = y_train[i:end]
new_X_train = np.concatenate([X_train[:i], X_train[end:]])
new_y_train = np.concatenate([y_train[:i], y_train[end:]])
l_regression.fit(new_X_train, new_y_train, lr, b, epochs)
prediction = l_regression.predict(new_X_valid)
mse_list.append(error(new_y_valid, prediction))
mean_MSE = np.mean(mse_list)
return mean_MSE
def testing(dataloader, model, device, valid):
metrics = Accuracy()
criterion = torch.nn.CrossEntropyLoss()
trange = tqdm(enumerate(dataloader),
total=len(dataloader),
desc='validation' if valid else 'testing')
model.eval()
total_loss = 0
metrics.reset()
for k, batch in trange:
model.eval()
prob = run_iter(batch, model, device, training=False)
answer = batch['label'].to(device)
loss = criterion(prob, batch['label'].to(device))
total_loss += loss.item()
metrics.update(prob, answer)
trange.set_postfix(loss=total_loss / (k + 1),
**{metrics.name: metrics.print_score()})
acc = metrics.match / metrics.n
return acc
def run_epoch(self, epoch, dataset, training, desc=''):
self.model.train(training)
shuffle = training
dataloader = DataLoader(dataset, self.batch_size, shuffle=shuffle)
trange = tqdm(enumerate(dataloader), total=len(dataloader), desc=desc)
loss = 0
acc = Accuracy()
for i, (imgs, labels) in trange: # (b, 3, 128, 128), (b, 1)
labels = labels.view(-1) # (b,)
o_labels, batch_loss = self.run_iters(imgs, labels)
if training:
batch_loss /= self.accum_steps
batch_loss.backward()
if (i + 1) % self.accum_steps == 0:
self.opt.step()
self.opt.zero_grad()
batch_loss *= self.accum_steps
loss += batch_loss.item()
acc.update(o_labels.cpu(), labels)
trange.set_postfix(loss=loss / (i + 1), acc=acc.print_score())
if training:
self.history['train'].append({
'loss': loss / len(trange),
'acc': acc.get_score()
})
self.scheduler.step()
else:
self.history['valid'].append({
'loss': loss / len(trange),
'acc': acc.get_score()
})
if loss < self.best_score:
self.save_best()
self.save_hist()
def fit(self, dynamic, is_mlp=False):
device = set_device('gpu')
fluid.enable_dygraph(device) if dynamic else None
im_shape = (-1, 784)
batch_size = 128
inputs = [Input(im_shape, 'float32', name='image')]
labels = [Input([None, 1], 'int64', name='label')]
train_dataset = MnistDataset(mode='train')
val_dataset = MnistDataset(mode='test')
test_dataset = TestMnistDataset()
model = MNIST() if not is_mlp else MLP()
optim = fluid.optimizer.Momentum(
learning_rate=0.01, momentum=.9, parameter_list=model.parameters())
loss = CrossEntropy() if not is_mlp else MyCrossEntropy()
model.prepare(optim, loss, Accuracy(), inputs, labels, device=device)
cbk = ProgBarLogger(50)
model.fit(train_dataset,
val_dataset,
epochs=2,
batch_size=batch_size,
callbacks=cbk)
eval_result = model.evaluate(val_dataset, batch_size=batch_size)
output = model.predict(test_dataset, batch_size=batch_size)
np.testing.assert_equal(output[0].shape[0], len(test_dataset))
acc = get_predict_accuracy(output[0], val_dataset.labels)
np.testing.assert_allclose(acc, eval_result['acc'])
def run_epoch(self, epoch, dataloader):
self.feature_extractor.train(True)
self.label_predictor.train(True)
trange = tqdm(dataloader,
total=len(dataloader),
desc=f'[epoch {epoch}]')
total_loss = 0
acc = Accuracy()
for i, (target_data, target_label) in enumerate(trange): # (b,1,32,32)
target_data = target_data.to(self.device)
target_label = target_label.view(-1).to(self.device) # (b)
feature = self.feature_extractor(target_data) # (b, 512)
class_logits = self.label_predictor(feature) # (b, 10)
loss = self.class_criterion(class_logits, target_label)
total_loss += loss.item()
loss.backward()
self.optimizer_F.step()
self.optimizer_C.step()
self.optimizer_F.zero_grad()
self.optimizer_C.zero_grad()
acc.update(class_logits, target_label)
trange.set_postfix(loss=total_loss / (i + 1),
acc=acc.print_score())
self.history['loss'].append(total_loss / len(trange))
self.history['acc'].append(acc.get_score())
self.save_hist()
self.save_model()
def run_epoch(self, epoch, training, stage1):
if stage1:
self.model1.train(training)
else:
self.model1.train(False)
self.model2.train(training)
if training:
description = '[Stage1 Train]' if stage1 else '[Stage2 Train]'
dataset = self.trainData
shuffle = True
else:
description = '[Stage1 Valid]' if stage1 else '[Stage2 Valid]'
dataset = self.validData
shuffle = False
# dataloader for train and valid
dataloader = DataLoader(
dataset,
batch_size=self.batch_size,
shuffle=shuffle,
num_workers=8,
collate_fn=dataset.collate_fn,
)
trange = tqdm(enumerate(dataloader),
total=len(dataloader),
desc=description)
loss = 0
loss2 = 0
accuracy = Accuracy()
if stage1:
for i, (x, y, miss) in trange: # (x,y) = b*b
pdb()
o_f1, batch_loss = self.run_iter_stage1(x, miss)
if training:
self.opt.zero_grad() # reset gradient to 0
batch_loss.backward() # calculate gradient
self.opt.step() # update parameter by gradient
loss += batch_loss.item(
) # .item() to get python number in Tensor
trange.set_postfix(loss=loss / (i + 1))
else:
for i, (x, y, miss) in trange: # (x,y) = b*b
o_labels, batch_loss, missing_loss = self.run_iter_stage2(
x, miss, y) # x=(256, 8), y=(256)
loss2 += missing_loss.item()
if training:
self.opt.zero_grad() # reset gradient to 0
batch_loss.backward() # calculate gradient
self.opt.step() # update parameter by gradient
loss += batch_loss.item(
) #.item() to get python number in Tensor
accuracy.update(o_labels.cpu(), y)
trange.set_postfix(accuracy=accuracy.print_score(),
loss=loss / (i + 1),
missing_loss=loss2 / (i + 1))
batch_list = np.linspace(1, 30, 30)
kfolds_b = np.zeros(batch_list.shape)
for i, b in enumerate(batch_list):
kfolds_b[i] = k_folds(X_train, y_train.reshape(-1, 1), best_lr, b, 100)
best_b = batch_list[np.argmax(kfolds_b)]
# Fit model and predict with optimized parameters
logistic_regression = LogisticRegression()
logistic_regression.fit(X_train, y_train.reshape(-1, 1), best_lr, best_b, 50000)
print(logistic_regression.model)
predictions = logistic_regression.predict(X_test)
slope = -(logistic_regression.model[1] / logistic_regression.model[2])
intercept = -(logistic_regression.model[0] / logistic_regression.model[2])
# Metrics
metrics = [Precision(), Accuracy(), Recall()]
for metric in metrics:
print('{metric}: {value}'.format(metric=metric.__class__.__name__, value=metric(y_test, predictions[:, 0])))
# Graphics
plt.figure(1)
plt.scatter(dataset.dataset['x_1'], dataset.dataset['x_2'], c=dataset.dataset['y'])
plt.xlabel('X1')
plt.ylabel('X2')
ax = plt.gca()
y_vals = intercept + (slope * dataset.dataset['x_1'])
ax.autoscale(False)
plt.plot(dataset.dataset['x_1'], y_vals, c="k")
plt.show()
f, (ax, bx) = plt.subplots(2, 1, sharey='col')
shuffle=True)
val_dataloader = DataLoader(CombinedData(val_stories, embedding_val, device),
batch_size=BATCH_SIZE,
shuffle=False)
test_dataloader = DataLoader(CombinedData(stories_test, embedding_test,
device),
batch_size=BATCH_SIZE,
shuffle=False)
net = CombinedNet(device)
net.to(device)
ce_loss = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(net.parameters(), lr=LR)
metric_acc = Accuracy()
n_iteration = len(train_dataloader)
v_iteration = len(val_dataloader)
val_accuracy_prev = 0
for epoch in range(NUM_EPOCHS):
running_loss_train = 0.0
running_loss_val = 0.0
for i, train_batch in enumerate(train_dataloader):
optimizer.zero_grad()
logits = net(train_batch)
train_loss = ce_loss(logits, train_batch['labels'])
# output, train_loss = utils.run_step(train_batch, net, tokenizer, ce_loss, device)
train_loss.backward()
def _train_epoch(self, epoch):
# lr update
if self.lr_scheduler is not None:
self.lr_scheduler.step(epoch)
for param_group in self.optimizer.param_groups:
self.current_lr = param_group['lr']
batch_time = AverageMeter()
data_time = AverageMeter()
ave_total_loss = AverageMeter()
ave_acc = AverageMeter()
ave_iou = AverageMeter()
# set model mode
self.model.train()
tic = time.time()
for steps, (data, target) in enumerate(self.train_data_loader,
start=1):
data = data.to(self.device, non_blocking=True)
target = target.to(self.device, non_blocking=True)
# 加载数据所用的时间
data_time.update(time.time() - tic)
# forward calculate
logits = self.model(data)
loss = self.loss(logits, target)
acc = Accuracy(logits, target)
miou = MIoU(logits, target, self.config.nb_classes)
# compute gradient and do SGD step
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# update average metrics
batch_time.update(time.time() - tic)
ave_total_loss.update(loss.data.item())
ave_acc.update(acc.item())
ave_iou.update(miou.item())
# display on the screen per display_steps
if steps % self.dis_period == 0:
print(
'Epoch: [{}][{}/{}],\n'
'Learning_Rate: {:.6f},\n'
'Time: {:.4f}, Data: {:.4f},\n'
'MIoU: {:6.4f}, Accuracy: {:6.4f}, Loss: {:.6f}'.
format(epoch, steps, len(self.train_data_loader),
self.current_lr, batch_time.average(),
data_time.average(), ave_iou.average(),
ave_acc.average(), ave_total_loss.average()))
tic = time.time()
# train log and return
self.history['train']['epoch'].append(epoch)
self.history['train']['loss'].append(ave_total_loss.average())
self.history['train']['acc'].append(ave_acc.average())
self.history['train']['miou'].append(ave_iou.average())
return {
'epoch': epoch,
'loss': ave_total_loss.average(),
'acc': ave_acc.average(),
'miou': ave_iou.average(),
}
def annotate(model, inpath, outfile, show_ratio, lowmethod, unkmethod, informat, annotated,
ignore_entities, n_folds, quiet=False):
"""Annotate tokens in the file."""
assert(informat == FORMAT_LOG or model in SUPPORTED_MODELS)
verbose = False
# Fire up a lidder if we're not just reading from log
if informat != FORMAT_LOG:
lidder = default_lidder(model, RATIOLIST_DEFAULT_CONFIG)
# Force model1 to use MLE for the low method and UNK_METHOD to left, so we're sure no randomness
# gets applied
if model == MODEL1:
lowmethod = LOW_METHOD_MLE
unkmethod = UNK_METHOD_LEFT
# Set the flag for whether we're evaluating only message LID, not token-by-token
token_eval = not(informat == FORMAT_LOG or model == MODEL0)
# Evaluators
token_acc = Accuracy()
# For when unk is allowed
all_lid_acc = Accuracy()
twoway_lid_acc = Accuracy()
cs_perf = SDMetrics()
# When unk is excluded
nounk_all_lid_acc = Accuracy()
nounk_twoway_lid_acc = Accuracy()
nounk_cs_perf = SDMetrics()
# Code switch points
cs_boundaries = SDMetrics()
if n_folds:
train_paths, test_paths = kfolds(inpath, n_folds)
infiles = [_open_infile(inpath, informat) for inpath in test_paths]
else:
infiles =[_open_infile(inpath, informat)]
fold_accuracies = []
for infile in infiles:
# To match the SVM_HMM evaluation, we have a special token accuracy that's reset every fold
fold_token_acc = Accuracy()
for tokens, tags, gold_langs, lid, gold_lid in _tokens_tags_langs(infile, informat, annotated):
# Put in dummy tags if needed
if not tags:
tags = [JERBOA_NOTAG] * len(tokens)
tokens_lower = [token.lower() for token in tokens]
# We label all tokens only if it's annotated, as the annotations will later
# wipe out anything we shouldn't have labeled
# TODO: This is a little wonky as labeled bad tokens can affect the lid/cs
# decision, but for model 1.0 this doesn't actually matter as they aren't
# in the wordlist
if informat == FORMAT_LOG:
# Skip lines with no gold annotation
if not gold_lid:
continue
# Don't label anything, just use what we got from the log file
verdict = lid == MULTIPLE_LANGS
elif model == MODEL0:
lid, langspresent, hits, verdict = lidder.idlangs(tokens_lower)
ratios = out_langs = unk_rate = None
else:
lid, langspresent, hits, ratios, out_langs, unk_rate, verdict = \
(lidder.idlangs(tokens_lower, lowmethod, unkmethod, tags) if model == MODEL1_5 else
lidder.idlangs(tokens_lower))
output_lang = lid if not verdict else MULTIPLE_LANGS
# Token labeling
if token_eval:
# For model 1.0, apply MLE
if model == MODEL1:
out_langs = [choose_lang(token, lang, lidder.langs, tag, ratio, lowmethod,
unkmethod, False)
for token, tag, lang, ratio in zip(tokens_lower, tags, out_langs, ratios)]
# Carry over NO_LANG labels from the gold standard
if gold_langs:
out_langs = [out_lang if gold_lang != NO_LANG else NO_LANG
for out_lang, gold_lang in zip(out_langs, gold_langs)]
# Truncate to one char
out_langs = [lang[0] if lang else UNKNOWN_LANG[0] for lang in out_langs]
# Output tokens
if not quiet:
out_tokens = ([(token, "{0:1.3f}".format(ratio)) for token, ratio in zip(tokens, ratios)]
if show_ratio else
zip(tokens, out_langs))
print >> outfile, " ".join(["/".join(token_pair) for token_pair in out_tokens])
# If it isn't annotated, skip over scoring and go to the next tokens
if not annotated:
continue
# Scoring!
# First, tokens
if token_eval:
# Individual tokens
for pred_lang, gold_lang, token in zip(out_langs, gold_langs, tokens_lower):
# Clean gold_lang of entities
gold_lang_clean = clean_entities(gold_lang)
if (gold_lang_clean not in (NO_LANG, 'o') and pred_lang != NO_LANG and
(not ignore_entities or not contains_entity(gold_lang))):
token_acc.score(pred_lang, gold_lang_clean, token.lower())
fold_token_acc.score(pred_lang, gold_lang_clean, token.lower())
# Codeswitch points
last_pred_lang = None
last_gold_lang = None
last_token = None
for pred_lang, gold_lang, token in zip(out_langs, gold_langs, tokens_lower):
# Skip non-linguistic tokens
if gold_lang not in VALID_CS_LANGS:
continue
# Score if we have a valid last token
if last_gold_lang is not None:
# True label is whenever the language changes, but don't predict codeswitching
# if one of the langs was unknown. Since the label's been truncated, we take
# the first char of UNKNOWN_LANG.
pred_cs = (pred_lang != UNKNOWN_LANG[0] and last_pred_lang != UNKNOWN_LANG[0] and
pred_lang != last_pred_lang)
gold_cs = gold_lang != last_gold_lang
cs_boundaries.score(pred_cs, gold_cs, (last_token, token))
# Update last langs/token
last_pred_lang = pred_lang
last_gold_lang = gold_lang
last_token = token
# Next, messages
# Compute a gold_lid if we don't know it already
if not gold_lid:
gold_valid_langs = _valid_langs_set(gold_langs)
gold_lid = list(gold_valid_langs)[0] if len(gold_valid_langs) == 1 else MULTIPLE_LANGS
if gold_lid != MULTIPLE_LANGS:
# One lang means we should check lid accuracy
twoway_lid_acc.score(output_lang, gold_lid)
cs_perf.score(verdict, False)
else:
# Multiple langs means we should check for codeswitching
cs_perf.score(verdict, True)
# Always record all-way LID
all_lid_acc.score(output_lang, gold_lid)
# Repeat not unk
if gold_lid != UNKNOWN_LANG:
if gold_lid != MULTIPLE_LANGS:
# One lang means we should check lid accuracy
nounk_twoway_lid_acc.score(output_lang, gold_lid)
nounk_cs_perf.score(verdict, False)
else:
# Multiple langs means we should check for codeswitching
nounk_cs_perf.score(verdict, True)
# Always record all-way LID
nounk_all_lid_acc.score(output_lang, gold_lid)
# Track fold accuracy
fold_accuracies.append(fold_token_acc.accuracy)
if annotated:
output = sys.stderr
print >> output, '*' * 10 + "All data evaluation" + '*' * 10
print >> output, "All message LID:"
print >> output, all_lid_acc
print >> output, all_lid_acc.confusion_matrix()
print >> output
print >> output, "Non-codeswitched message LID:"
print >> output, twoway_lid_acc
print >> output, twoway_lid_acc.confusion_matrix()
print >> output
print >> output, "Message CS:"
print >> output, cs_perf
print >> output, cs_perf.confusion_matrix()
print >> output
print >> output, '*' * 10 + "No unknown lang data evaluation" + '*' * 10
print >> output, "All message LID:"
print >> output, nounk_all_lid_acc
print >> output, nounk_all_lid_acc.confusion_matrix()
print >> output
print >> output, "Non-codeswitched message LID:"
print >> output, nounk_twoway_lid_acc
print >> output, nounk_twoway_lid_acc.confusion_matrix()
print >> output
print >> output, "Message CS:"
print >> output, nounk_cs_perf
print >> output, nounk_cs_perf.confusion_matrix()
print >> output
if token_eval:
print >> output, '*' * 10 + "Token by token evaluation" + '*' * 10
print >> output, "Token-by-token LID:"
print >> output, "Low method:", lowmethod
if model != MODEL1: # Model 1 doesn't actually do unk attachment
print >> output, "Unk method:", unkmethod
print >> output, token_acc
print >> output, token_acc.confusion_matrix()
print >> output
print >> output, "Codeswitching boundaries:"
print >> output, cs_boundaries
print >> output, cs_boundaries.confusion_matrix()
print >> output
if not quiet and verbose:
for gold, subdict in token_acc.confusion.items():
for pred, errors in subdict.items():
if gold == pred or not errors:
continue
print >> output, '*' * 40
print >> output, "Gold:", gold, "Pred:", pred
for error in sorted(set(errors)):
print >> output, error
print >> output
# Report average fold accuracy if needed
if len(fold_accuracies) > 1:
mean_accuracy = sum(fold_accuracies) / len(fold_accuracies)
print >> output, "Fold token accuracies: " + ", ".join("%.4f" % acc for acc in fold_accuracies)
print >> output, "Mean token accuracy across folds: %.4f" % mean_accuracy
return (all_lid_acc, twoway_lid_acc, cs_perf, nounk_all_lid_acc, nounk_twoway_lid_acc,
nounk_cs_perf, token_acc)
