def main(img1, img2, chg_map, args=None):
img_shape = np.shape(img1)
im1 = np.reshape(img1, newshape=[-1,img_shape[-1]])
im2 = np.reshape(img2, newshape=[-1,img_shape[-1]])
im1 = utils.normlize(im1)
im2 = utils.normlize(im2)
chg_ref = np.reshape(chg_map, newshape=[-1])
imm = None
all_magnitude = None
differ = np.zeros(shape=[np.shape(chg_ref)[0],net_shape[-1], args.iter])
# load cva pre-detection result
ind = sio.loadmat(args.area+'/cva_ref.mat')
cva_ind = ind['cva_ref']
cva_ind = np.reshape(cva_ind, newshape=[-1])
for k1 in range(args.iter):
logging.info('In %2d-th iteration········' % (k1))
i1, i2 = utils.getTrainSamples(cva_ind, im1, im2, args.trn)
loss_log, vpro, fcx, fcy, bval = dsfa(
xtrain=i1, ytrain=i2, xtest=im1, ytest=im2, net_shape=net_shape, args=args)
imm, magnitude, differ_map = utils.linear_sfa(fcx, fcy, vpro, shape=img_shape)
magnitude = np.reshape(magnitude, img_shape[0:-1])
differ[:, :, k1] = differ_map
if all_magnitude is None:
all_magnitude = magnitude / np.max(magnitude)
else:
all_magnitude = all_magnitude + magnitude / np.max(magnitude)
change_map = np.reshape(utils.kmeans(np.reshape(all_magnitude, [-1])), img_shape[0:-1])
logging.info('Max value of change magnitude: %.4f'%(np.max(all_magnitude)))
logging.info('Min value of change magnitude: %.4f'%(np.min(all_magnitude)))
# magnitude
acc_un, acc_chg, acc_all2, acc_tp = utils.metric(1-change_map, chg_map)
acc_un, acc_chg, acc_all3, acc_tp = utils.metric(change_map, chg_map)
plt.imsave('results.png',all_magnitude, cmap='gray')
#plt.show()
return None
def train(train_queue, model, device, optimizer, criterion, args):
objs = utils.AverageMeter()
top1 = utils.AverageMeter()
model.train()
confusion_matrix = None
for step, (input_train, target_train) in enumerate(train_queue):
n = input_train.size(0)
input_train = input_train.to(device)
target_train = target_train.squeeze().long().to(device)
logits = model(input_train)
loss = criterion(logits, target_train)
optimizer.zero_grad()
loss.backward()
optimizer.step()
acc, confusion = utils.metric(logits, target_train)
objs.update(loss.item(), n)
top1.update(acc, n)
if step % args.report_freq == 0:
logging.info('Train | Step: %d | Loss: %e | Accuracy: %.3f', step,
objs.avg, top1.avg)
if confusion_matrix is None:
confusion_matrix = confusion
else:
confusion_matrix += confusion
return top1.avg, objs.avg, confusion_matrix
def __init__(self,
target_img,
source_tiles,
tile_size,
reuse_count=float('inf'),
metric_type='color_distance',
randomize=False,
bw=False,
color_scale=False):
self.target_img = target_img
self.reuse_count = reuse_count
self.source_tiles = source_tiles
self.tile_size = tile_size
self.randomize = randomize
self.target_tiles = []
self.bw = bw
self.color_scale = color_scale
self._get_target_tiles()
self._process_source_tiles()
shuffle(self.target_tiles)
shuffle(self.source_tiles)
self.tile_use_count = Counter(range(len(self.source_tiles)))
if bw:
self.target_tiles = convert_tile_to_bw(self.target_tiles)
self.source_tiles = convert_tile_to_bw(self.source_tiles)
self.metric = lambda img1, img2: metric(img1, img2, metric_type)
print(len(self.target_tiles), len(self.source_tiles))
def train_by_seed(self, seed, module):
seed_everything(seed)
self.param["seed"] = seed
self.param["model_param"]["random_state"] = seed
kf_manager = KFManager(self.param, module)
X = module["X"]
y = module["y"]
print(X.columns)
oof_preds = np.zeros((X.shape[0], ))
if self.param["metric"] == "RMSLE":
y = np.log1p(y)
for fold, (train_index, valid_index) in enumerate(kf_manager.split()):
train_X = X.loc[train_index].drop("Publisher", axis=1).values
valid_X = X.loc[valid_index].drop("Publisher", axis=1).values
train_y = y.loc[train_index].values
valid_y = y.loc[valid_index].values
print(train_X.shape)
print(valid_X.shape)
oof = self.train_by_fold(train_X, train_y, valid_X, valid_y,
self.param, fold)
oof_preds[valid_index] += oof
if self.param["metric"] == "RMSLE":
oof_preds = np.expm1(oof_preds)
y = np.expm1(y)
err = metric(y, oof_preds)
print(f"seed {seed} : RMSLE is {err}")
return oof_preds, err
def evaluate(model, dataloader, device="cuda"):
with torch.no_grad():
model.eval()
final_loss = 0
counter = 0
final_outputs = []
final_targets = []
print(f"{'='*15}Evaluating{'='*15}")
for _, data in tqdm(enumerate(dataloader), total=int(len(dataloader))):
counter = counter + 1
image = data["image"]
target = data["target"]
image = image.to(device, dtype=torch.float)
target = target.to(device, dtype=torch.float)
outputs = model(image)
loss = nn.BCEWithLogitsLoss()(outputs, target.view(-1, 1))
final_loss += loss
final_outputs.append(outputs)
final_targets.append(target)
final_outputs = torch.cat(final_outputs).cpu().numpy()
final_targets = torch.cat(final_targets).cpu().numpy()
auc_metric = metric(final_targets, final_outputs)
return final_loss / counter, auc_metric
def train(train_queue, valid_queue, model, device):
objs = utils.AverageMeter()
top1 = utils.AverageMeter()
model.train()
for step, (input_train, target_train) in enumerate(train_queue):
n = input_train.size(0)
# get a random minibatch from the search queue with replacement
input_val, target_val = next(iter(valid_queue))
input_train = input_train.to(device)
target_train = target_train.squeeze().long().to(device)
input_val = input_val.to(device)
target_val = target_val.squeeze().long().to(device)
model.module.alphas_step(input_train, target_train, input_val, target_val, args.order)
logits, loss = model.module.weights_step(input_train, target_train, args.grad_clip)
acc = utils.metric(logits, target_train)
objs.update(loss.item(), n)
top1.update(acc, n)
if step % args.report_freq == 0:
logging.info('Train | Step: %d | Loss: %e | Accuracy: %.3f', step, objs.avg, top1.avg)
return top1.avg, objs.avg
def model_predict():
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
if K.backend() == 'tensorflow':
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
tokenizer = get_tokenizer(args.bert_vocab_path)
# read data and relation
test_data, id2rel, rel2id, num_rels = load_data(args.test_path,
args.rel_dict_path)
# load model
subject_model, object_model, hbt_model = E2EModel(
args.bert_config_path, args.bert_checkpoint_path, args.LR, num_rels)
hbt_model.load_weights(args.save_weights_path)
isExactMatch = True if args.dataset == 'Wiki-KBP' else False
if isExactMatch:
print("Exact Match")
else:
print("Partial Match")
precision, recall, f1_score = metric(subject_model, object_model,
test_data, id2rel, tokenizer,
isExactMatch, args.test_result_path)
print(f'{precision}\t{recall}\t{f1_score}')
def dev(config, bert_config, dev_path, id2rel, tokenizer, output_path=None):
dev_data = json.load(open(dev_path))
for sent in dev_data:
data.to_tuple(sent)
with torch.no_grad():
Bert_model = BertModel(bert_config).to(device).eval()
submodel = sub_model(config).to(device).eval()
objmodel = obj_model(config).to(device).eval()
state = torch.load(
os.path.join(config.output_dir, config.load_model_name))
Bert_model.load_state_dict(state['bert_state_dict'])
submodel.load_state_dict(state['subject_state_dict'])
objmodel.load_state_dict(state['object_state_dict'])
precision, recall, f1 = utils.metric(Bert_model,
submodel,
objmodel,
dev_data,
id2rel,
tokenizer,
output_path=output_path)
logger.info('precision: %.4f' % precision)
logger.info('recall: %.4f' % recall)
logger.info('F1: %.4f' % f1)
def test(self, epoch, iteration):
# performance on the entire holdout set
yhat_test = self.predict(self.holdout.X)
corr_test = metric(yhat_test.ravel(), self.holdout.y)
# performance on a subset of the training data
training_sample_size = yhat_test.shape[0]
inds = choice(self.training.y.shape[0], training_sample_size, replace=False)
yhat_train = self.predict(self.training.X[inds, ...])
corr_train = metric(yhat_train.ravel(), self.training.y[inds])
# save the results to a CSV file
results = [epoch, iteration, corr_train, corr_test]
self.save_csv(results)
return results
def eval(model, dataLoader_valid):
with torch.no_grad():
model.eval()
model.mode = 'valid'
valid_loss, index_valid = 0, 0
all_results = []
all_masks = []
all_fcs = []
all_fc_labels = []
for valid_data in dataLoader_valid:
images, masks = valid_data
images = images.cuda()
masks = masks.cuda()
b = images.size(0)
outs, fc = data_parallel(model, images)
model.loss = model.get_loss(outs, fc, masks)
outs = torch.sigmoid(outs).cpu()
fc = torch.sigmoid(fc).cpu()
masks = masks.cpu()
# all_results.append(outs)
# all_masks.append(masks)
all_results.append(F.upsample_bilinear(outs, (256, 256)))
all_masks.append((F.upsample_bilinear(masks,
(256, 256)) > 0.5).float())
all_fcs.append(fc)
all_fc_labels.append(
(masks.view(b, -1).sum(-1) > 0).float().view(b, 1))
if index_valid == 0:
drawing(images, outs, masks, mode='valid')
b = len(masks)
valid_loss += model.loss.item() * b
index_valid += b
valid_loss = valid_loss / index_valid
all_results = torch.cat(all_results, 0)
all_masks = torch.cat(all_masks, 0)
ts = np.linspace(0.3, 0.7, 5)
mIoUs = []
mnegIoUs = []
mposIoUs = []
for t in ts:
s, s_neg, s_pos, _, _ = metric(all_results, all_masks, t)
mIoUs.append(s)
mnegIoUs.append(s_neg)
mposIoUs.append(s_pos)
max_iou = max(mIoUs)
max_index = mIoUs.index(max_iou)
best_t = ts[max_index]
valid_iou = max_iou
return valid_loss, valid_iou, best_t, mnegIoUs[max_index], mposIoUs[
max_index],
def evaluate_test_results(cur_predict, cur_target, class_predict, class_traget,
saved_thresholds, epoch, start_time, eval_loss,
stats_path):
total_precision = []
total_recall = []
cur_precision, cur_recall = metric(class_predict, class_traget,
saved_thresholds)
total_precision.append(np.array(cur_precision))
total_recall.append(np.array(cur_recall))
TPVFs, dices, PPVs, FPVFs = segmentation_metrics(cur_predict, cur_target)
logging.info(
'***************************************************************************'
)
logging.info(
'Esophagus --> Global dice is [%.5f], TPR is [%.5f], Precision is [%.5f] '
% (dices[0], TPVFs[0], PPVs[0]))
logging.info(
'heart --> Global dice is [%.5f], TPR is [%.5f], Precision is [%.5f] '
% (dices[1], TPVFs[1], PPVs[1]))
logging.info(
'trachea --> Global dice is [%.5f], TPR is [%.5f], Precision is [%.5f] '
% (dices[2], TPVFs[2], PPVs[2]))
logging.info(
'aorta --> Global dice is [%.5f], TPR is [%.5f], Precision is [%.5f] '
% (dices[3], TPVFs[3], PPVs[3]))
total_precision = np.stack(total_precision, 1)
total_recall = np.stack(total_recall, 1)
logging.info(
'Epoch[%d], [precision=%.4f, -->%.3f, -->%.3f, -->%.3f, -->%.3f], using %.1f s!'
% (epoch, np.mean(total_precision), np.mean(total_precision[0]),
np.mean(total_precision[1]), np.mean(total_precision[2]),
np.mean(total_precision[3]), time.time() - start_time))
logging.info(
'Epoch[%d], [recall=%.4f, -->%.3f, -->%.3f, -->%.3f, -->%.3f], using %.1f s!'
% (epoch, np.mean(total_recall), np.mean(total_recall[0]),
np.mean(total_recall[1]), np.mean(total_recall[2]),
np.mean(total_recall[3]), time.time() - start_time))
with open(stats_path, 'a') as f:
writer = csv.writer(f)
writer.writerow([
epoch, "NA",
np.mean(eval_loss),
np.mean(total_precision),
np.mean(total_precision[0]),
np.mean(total_precision[1]),
np.mean(total_precision[2]),
np.mean(total_precision[3]),
np.mean(dices), dices[0], dices[1], dices[2], dices[3]
])
logging.info(
'Epoch[%d], [total loss=%.6f], mean_dice=%.4f, using %.1f s!' %
(epoch, np.mean(eval_loss), np.mean(dices), time.time() - start_time))
logging.info(
'***************************************************************************'
)
return np.mean(dices), np.mean(total_precision)
def update(self, targets, outputs):
probs = torch.sigmoid(outputs)
dice, dice_neg, dice_pos, _, _ = metric(probs, targets,
self.base_threshold)
self.base_dice_scores.append(dice)
self.dice_pos_scores.append(dice_pos)
self.dice_neg_scores.append(dice_neg)
preds = predict(probs, self.base_threshold)
iou = compute_iou_batch(preds, targets, classes=[1])
self.iou_scores.append(iou)
def test(self, epoch, iteration):
# performance on the entire holdout set
yhat_test = self.predict(self.holdout.X)
corr_test = metric(yhat_test.ravel(), self.holdout.y)
# performance on a subset of the training data
training_sample_size = yhat_test.shape[0]
inds = choice(self.training.y.shape[0],
training_sample_size,
replace=False)
yhat_train = self.predict(self.training.X[inds, ...])
corr_train = metric(yhat_train.ravel(), self.training.y[inds])
# save the results to a CSV file
results = [epoch, iteration, corr_train, corr_test]
self.save_csv(results)
return results
def main(img1, img2, chg_map, args=None):
img_shape = np.shape(img1)
im1 = np.reshape(img1, newshape=[-1,img_shape[-1]])
im2 = np.reshape(img2, newshape=[-1,img_shape[-1]])
im1 = utils.normlize(im1)
im2 = utils.normlize(im2)
chg_ref = np.reshape(chg_map, newshape=[-1])
imm = None
all_magnitude = None
differ = np.zeros(shape=[np.shape(chg_ref)[0],net_shape[-1]])
# load cva pre-detection result
ind = sio.loadmat(args.area+'/cva_ref.mat')
cva_ind = ind['cva_ref']
cva_ind = np.reshape(cva_ind, newshape=[-1])
i1, i2 = utils.getTrainSamples(cva_ind, im1, im2, args.trn)
loss_log, vpro, fcx, fcy, bval = dsfa(
xtrain=i1, ytrain=i2, xtest=im1, ytest=im2, net_shape=net_shape, args=args)
imm, magnitude, differ_map = utils.linear_sfa(fcx, fcy, vpro, shape=img_shape)
magnitude = np.reshape(magnitude, img_shape[0:-1])
differ = differ_map
change_map = np.reshape(utils.kmeans(np.reshape(magnitude, [-1])), img_shape[0:-1])
# magnitude
acc_un, acc_chg, acc_all2, acc_tp = utils.metric(1-change_map, chg_map)
acc_un, acc_chg, acc_all3, acc_tp = utils.metric(change_map, chg_map)
plt.imsave('results.png',change_map, cmap='gray')
#plt.show()
return None
def __call__(self, module):
train_preds = []
errs = []
# label_encode:
for seed in self.param["seeds"]:
oof_preds, err = self.train_by_seed(seed, module)
train_preds.append(oof_preds)
errs.append(err)
train_preds = np.mean(np.array(train_preds), axis=0)
err = metric(module["y"], train_preds)
print(f"final RMSLE is {err}")
module["train_preds"] = train_preds
module["errs"] = errs
module["err"] = err
return module
def on_epoch_end(self, epoch, logs=None):
precision, recall, f1 = metric(self.subject_model, self.object_model,
self.eval_data, self.id2rel,
self.tokenizer)
if self.monitor_op(f1 - self.min_delta, self.best) or self.monitor_op(
self.min_delta, f1):
self.best = f1
self.wait = 0
self.model.save_weights(self.save_weights_path)
else:
self.wait += 1
if self.wait >= self.patience:
self.stopped_epoch = epoch
self.model.stop_training = True
print('f1: %.4f, precision: %.4f, recall: %.4f, best f1: %.4f\n' %
(f1, precision, recall, self.best))
def evaluate(test_queue, model, device): objs = utils.AverageMeter() top1 = utils.AverageMeter() for step, (input_test, target_test) in enumerate(test_queue): model.eval() input_test = input_test.to(device) target_test = target_test.squeeze().long().to(device) with torch.no_grad(): logits, loss = model.module.loss(input_test, target_test) acc = utils.metric(logits, target_test) n = input_test.size(0) objs.update(loss.item(), n) top1.update(acc, n) return top1.avg, objs.avg
def do_valid(net, valid_loader, out_dir=None):
valid_loss = np.zeros(6, np.float32)
valid_num = np.zeros_like(valid_loss)
valid_probability = [[],[],[],]
valid_truth = [[],[],[],]
for t, (input, truth, infor) in enumerate(valid_loader):
batch_size = len(infor)
net.eval()
input = input.cuda()
truth = [t.cuda() for t in truth]
with torch.no_grad():
logit = net(input)
probability = logit_to_probability(logit)
loss = criterion(logit, truth)
correct = metric(probability, truth)
loss = [l.item() for l in loss]
l = np.array([ *loss, *correct, ])*batch_size
n = np.array([ 1, 1, 1, 1, 1, 1 ])*batch_size
valid_loss += l
valid_num += n
for i in range(NUM_TASK):
valid_probability[i].append(probability[i].data.cpu().numpy())
valid_truth[i].append(truth[i].data.cpu().numpy())
print('\r %8d /%d'%(valid_num[0], len(valid_loader.dataset)),end='',flush=True)
pass
assert(valid_num[0] == len(valid_loader.dataset))
valid_loss = valid_loss/(valid_num+1e-8)
for i in range(NUM_TASK):
valid_probability[i] = np.concatenate(valid_probability[i])
valid_truth[i] = np.concatenate(valid_truth[i])
recall, avgerage_recall = compute_kaggle_metric(valid_probability, valid_truth)
return valid_loss, (recall, avgerage_recall)
def train(data_loader, net, loss, epoch, optimizer, get_lr, save_dir):
start_time = time.time()
net.train()
lr = get_lr(epoch)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
total_train_loss = []
class_predict = []
class_target = []
for i, sample in enumerate(data_loader):
data = sample['image']
target_c = sample['label_c']
target_s = sample['label_s']
data = data.to(DEVICE)
target_c = target_c.to(DEVICE)
target_s = target_s.to(DEVICE)
output_s, output_c = net(data)
optimizer.zero_grad()
cur_loss, _, _, c_p = loss(output_s, output_c, target_s, target_c)
total_train_loss.append(cur_loss.item())
class_target.append(target_c.detach().cpu().numpy())
class_predict.append(c_p.detach().cpu().numpy())
cur_loss.backward()
optimizer.step()
logging.info(
'Epoch[%d], Batch [%d], total loss is %.6f, using %.1f s!' %
(epoch, i, np.mean(total_train_loss), time.time() - start_time))
total_train_class_predict = np.concatenate(class_predict, 0)
total_train_class_target = np.concatenate(class_target, 0)
adaptive_thresholds = get_threshold(total_train_class_predict,
total_train_class_target, 0.995)
cur_precision, _ = metric(total_train_class_predict,
total_train_class_target, adaptive_thresholds)
logging.info(
'Epoch[%d], [precision=%.4f, -->%.3f, -->%.3f, -->%.3f, -->%.3f]' %
(epoch, np.mean(cur_precision), np.mean(cur_precision[0]),
np.mean(cur_precision[1]), np.mean(
cur_precision[2]), np.mean(cur_precision[3])))
logging.info('the adaptive thresholds is [%.4f, %.4f, %.4f, %.4f]' %
(adaptive_thresholds[0], adaptive_thresholds[1],
adaptive_thresholds[2], adaptive_thresholds[3]))
return np.mean(total_train_loss), adaptive_thresholds
def train(model,
dataloader,
optimizer,
scaler,
device="cuda"):
model.train()
final_loss = 0
counter = 0
final_outputs = []
final_targets = []
print(f"{'='*15}Training{'='*15}")
for _, data in tqdm(enumerate(dataloader), total=int(len(dataloader))):
counter = counter + 1
image = data["image"]
targets = data["target"]
image = image.to(device, dtype=torch.float)
targets = targets.to(device, dtype=torch.float)
optimizer.zero_grad()
with amp.autocast():
outputs = model(image)
loss = nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))
scaler.scale(loss).backward()
scaler.step(optimizer)
scaler.update()
final_loss += loss
final_outputs.append(outputs)
final_targets.append(targets)
final_outputs = torch.cat(final_outputs).detach().cpu().numpy()
final_targets = torch.cat(final_targets).detach().cpu().numpy()
auc_metric = metric(
final_targets, final_outputs)
return final_loss / counter, auc_metric
def test(test_dataloader, test_entity_list, id2tag, model):
"""
测试模型
"""
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.eval()
output_tag_ids = []
with torch.no_grad():
for _, (batch_tensor, batch_sent_len,
batch_tags) in enumerate(test_dataloader):
batch_tensor = batch_tensor.to(device)
batch_sent_len = batch_sent_len.to(device)
batch_tags = batch_tags.to(device)
tag_seq = model.sequence_decode(batch_tensor, batch_sent_len)
output_tag_ids = output_tag_ids + tag_seq
output_tag = entity_recover(output_tag_ids, id2tag)
P, R, F1 = metric(output_tag, test_entity_list)
return output_tag_ids, P, R, F1
def train(train_queue, model, device):
objs = utils.AverageMeter()
top1 = utils.AverageMeter()
for step, (input_train, target_train) in enumerate(train_queue):
model.train()
n = input_train.size(0)
input_train = input_train.to(device)
target_train = target_train.squeeze().long().to(device)
logits, loss = model.module.weights_step(input_train, target_train, args.grad_clip)
acc = utils.metric(logits, target_train)
objs.update(loss.item(), n)
top1.update(acc, n)
if step % args.report_freq == 0:
logging.info('Train | Step: %d | Loss: %e | Accuracy: %.3f', step, objs.avg, top1.avg)
return top1.avg, objs.avg
def evaluate(test_queue, model, device, criterion):
objs = utils.AverageMeter()
top1 = utils.AverageMeter()
model.eval()
confusion_matrix = None
for step, (input_test, target_test) in enumerate(test_queue):
input_test = input_test.to(device)
target_test = target_test.squeeze().long().to(device)
with torch.no_grad():
logits = model(input_test)
loss = criterion(logits, target_test)
acc, confusion = utils.metric(logits, target_test)
n = input_test.size(0)
objs.update(loss.item(), n)
top1.update(acc, n)
if confusion_matrix is None:
confusion_matrix = confusion
else:
confusion_matrix += confusion
return top1.avg, objs.avg, confusion_matrix
results = []
# Try to understand what the criterion means, why is it mse?
# https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html
neigh = KNeighborsClassifier(n_neighbors=10)
neigh.fit(x_train, y_train_class)
result = neigh.predict(x_test)
import numpy as np
# Write a function to describe the error
from utils import metric
error = metric(y_test, result)
plt.close()
print(error)
i1, i2 = 0, 0
error = np.zeros(10)
N = 100 #dauert lang !!
while i1 < 10:
#print(i1)
while i2 < N:
np.random.seed(i2 + 13 * i1 + 5)
neigh = KNeighborsClassifier(n_neighbors=i1 + 1)
neigh.fit(x_train, y_train_class)
result = neigh.predict(x_test)
error[i1] = error[i1] + metric(y_test, result)
i2 = i2 + 1
def training(split_method, weight_grapheme_root, weight_vowel_diacritic,
weight_consonant_diacritic, weight_grapheme, n_splits, fold,
train_data_loader, val_data_loader, model_type, optimizer_name,
lr_scheduler_name, lr, warmup_proportion, batch_size,
valid_batch_size, num_epoch, start_epoch, accumulation_steps,
checkpoint_folder, load_pretrain, seed, loss_type, early_stopping,
beta, alpha, cutmix_prob, mixup_prob, apex, freeze_first,
freeze_epoches):
torch.cuda.empty_cache()
COMMON_STRING = '@%s: \n' % os.path.basename(__file__)
COMMON_STRING += '\tset random seed\n'
COMMON_STRING += '\t\tseed = %d\n' % seed
COMMON_STRING += '\tset cuda environment\n'
COMMON_STRING += '\t\ttorch.__version__ = %s\n' % torch.__version__
COMMON_STRING += '\t\ttorch.version.cuda = %s\n' % torch.version.cuda
COMMON_STRING += '\t\ttorch.backends.cudnn.version() = %s\n' % torch.backends.cudnn.version(
)
try:
COMMON_STRING += '\t\tos[\'CUDA_VISIBLE_DEVICES\'] = %s\n' % os.environ[
'CUDA_VISIBLE_DEVICES']
NUM_CUDA_DEVICES = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
except Exception:
COMMON_STRING += '\t\tos[\'CUDA_VISIBLE_DEVICES\'] = None\n'
NUM_CUDA_DEVICES = 1
COMMON_STRING += '\t\ttorch.cuda.device_count() = %d\n' % torch.cuda.device_count(
)
COMMON_STRING += '\n'
checkpoint_folder = os.path.join(checkpoint_folder, model_type + '/' + loss_type + '-' + \
optimizer_name + '-' + lr_scheduler_name + '-' + split_method + '-' + str(seed))
os.makedirs(checkpoint_folder, exist_ok=True)
checkpoint_filename = 'fold_' + str(fold) + "_checkpoint.pth"
checkpoint_filepath = os.path.join(checkpoint_folder, checkpoint_filename)
log = Logger()
log.open(os.path.join(checkpoint_folder,
'fold_' + str(fold) + '_log_train.txt'),
mode='a+')
log.write('\t%s\n' % COMMON_STRING)
log.write('\n')
log.write('\tseed = %u\n' % seed)
log.write('\tFOLD = %s\n' % fold)
log.write('\t__file__ = %s\n' % __file__)
log.write('\tout_dir = %s\n' % checkpoint_folder)
log.write('weight_grapheme_root\n %s\n' % (str(weight_grapheme_root)))
log.write('weight_vowel_diacritic\n %s\n' % (str(weight_vowel_diacritic)))
log.write('weight_consonant_diacritic\n %s\n' %
(str(weight_consonant_diacritic)))
log.write('weight_grapheme\n %s\n' % (str(weight_grapheme)))
log.write('\n')
############################################################################## define unet model with backbone
def load(model, pretrain_file, skip=[]):
pretrain_state_dict = torch.load(pretrain_file)
state_dict = model.state_dict()
keys = list(state_dict.keys())
for key in keys:
if any(s in key for s in skip): continue
try:
state_dict[key] = pretrain_state_dict[key]
except:
print(key)
model.load_state_dict(state_dict)
return model
############################################################################### model
model = BengaliaiNet(model_type=model_type, \
n_classes=NUM_CLASSES)
model = model.cuda()
if load_pretrain:
print("Load pretrain model")
model = load(model,
checkpoint_filepath,
skip=[
'avg_poolings.0.p', 'avg_poolings.1.p',
'avg_poolings.2.p', 'avg_poolings.3.p'
])
# model = load(model, checkpoint_filepath, skip=[])
############################################################################### optimizer
# param_optimizer = list(model.parameters())
# param_optimizer = list(model.named_parameters())
if freeze_first:
param_optimizer = list(model.head.named_parameters()) + list(
model.tail.named_parameters()) + list(
model.logits.named_parameters())
else:
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \
'lr': lr, \
'weight_decay': WEIGHT_DECAY}, \
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \
'lr': lr, \
'weight_decay': 0.0}
]
if optimizer_name == "Adam":
optimizer = torch.optim.Adam(optimizer_grouped_parameters)
min_lr = 1e-4
elif optimizer_name == "SGD":
optimizer = torch.optim.SGD(optimizer_grouped_parameters, momentum=0.5)
min_lr = 1e-2
elif optimizer_name == "Ranger":
optimizer = Ranger(optimizer_grouped_parameters)
min_lr = 1e-4
elif optimizer_name == "RAdam":
optimizer = RAdam(optimizer_grouped_parameters)
min_lr = 1e-4
elif optimizer_name == "AdamW":
optimizer = AdamW(optimizer_grouped_parameters, eps=4e-5)
min_lr = 1e-4
elif optimizer_name == "FusedAdam":
optimizer = FusedAdam(optimizer_grouped_parameters,
bias_correction=False)
min_lr = 1e-4
else:
raise NotImplementedError
# add swa
# optimizer = torchcontrib.optim.SWA(optimizer, swa_start=int(len(train_data_loader) / 10), \
# swa_freq=int(len(train_data_loader) / 10), swa_lr=0.05)
# optimizer = torchcontrib.optim.SWA(optimizer)
############################################################################### lr_scheduler
if lr_scheduler_name == "WarmupCosineAnealing":
num_train_optimization_steps = num_epoch * len(
train_data_loader) // accumulation_steps
scheduler = get_cosine_schedule_with_warmup(optimizer, \
num_warmup_steps=int(num_train_optimization_steps*warmup_proportion), \
num_training_steps=num_train_optimization_steps)
lr_scheduler_each_iter = True
elif lr_scheduler_name == "WarmupLinearSchedule":
num_train_optimization_steps = num_epoch * len(
train_data_loader) // accumulation_steps
scheduler = get_linear_schedule_with_warmup(optimizer, \
num_warmup_steps=int(num_train_optimization_steps*warmup_proportion), \
num_training_steps=num_train_optimization_steps)
lr_scheduler_each_iter = True
elif lr_scheduler_name == "WarmupCosineAnealingWithHardRestart":
num_train_optimization_steps = num_epoch * len(
train_data_loader) // accumulation_steps
scheduler = get_cosine_with_hard_restarts_schedule_with_warmup(optimizer, \
num_warmup_steps=int(num_train_optimization_steps*warmup_proportion), \
num_training_steps=num_train_optimization_steps)
lr_scheduler_each_iter = True
elif lr_scheduler_name == "ReduceLROnPlateau":
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='max',
factor=0.6,
patience=3,
min_lr=min_lr)
lr_scheduler_each_iter = False
elif lr_scheduler_name == "OneCycleLR":
scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, lr, total_steps=None, epochs=num_epoch, \
steps_per_epoch=int(len(train_data_loader)/accumulation_steps), \
pct_start=0.3, anneal_strategy='cos', cycle_momentum=True, \
base_momentum=0.85, max_momentum=0.95, div_factor=25.0, final_div_factor=10000.0, last_epoch=-1)
lr_scheduler_each_iter = True
elif lr_scheduler_name == "CycleLR":
step_size = len(train_data_loader) * 5 / (accumulation_steps)
base_lr, max_lr = lr / 6, lr
scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer,
base_lr=base_lr,
max_lr=max_lr,
step_size_up=step_size,
mode='exp_range',
gamma=0.99994)
lr_scheduler_each_iter = True
else:
raise NotImplementedError
log.write('net\n %s\n' % (model_type))
log.write('optimizer\n %s\n' % (optimizer_name))
log.write('schduler\n %s\n' % (lr_scheduler_name))
log.write('\n')
############################################################################### mix precision
if apex:
model, optimizer = amp.initialize(model, optimizer, opt_level="O1")
# model = nn.DataParallel(model)
############################################################################### eval setting
eval_step = len(train_data_loader) # or len(train_data_loader)
log_step = int(len(train_data_loader) / 20)
eval_count = 0
count = 0
############################################################################### training
log.write('** start training here! **\n')
log.write(' batch_size=%d, accumulation_steps=%d\n' %
(batch_size, accumulation_steps))
log.write(' experiment = %s\n' % str(__file__.split('/')[-2:]))
valid_loss = np.zeros(1, np.float32)
train_loss = np.zeros(1, np.float32)
valid_metric_optimal = -np.inf
# define tensorboard writer and timer
writer = SummaryWriter()
# define criterion
if loss_type == 'mse':
criterion = nn.MSELoss()
elif loss_type == 'mae':
criterion = nn.SmoothL1Loss()
elif loss_type == 'bce':
criterion = nn.BCEWithLogitsLoss()
elif loss_type == 'focal':
criterion = FocalLoss()
elif loss_type == 'focalonehot':
criterion = FocalOnehotLoss()
elif loss_type == 'ceonehot':
criterion = CrossEntropyOnehotLoss()
elif loss_type == "ceonehotohem":
criterion = CrossEntropyOnehotLossOHEM(top_k=0.8)
elif loss_type == "focalonehotohem":
criterion = FocalOnehotLossOHEM(top_k=0.8)
else:
raise NotImplementedError
for epoch in range(1, num_epoch + 1):
# init cache
torch.cuda.empty_cache()
if freeze_first:
# if epoch < freeze_epoches:
# for param_group in optimizer.param_groups:
# param_group['lr'] = 1e-4
if epoch == freeze_epoches:
optimizer.add_param_group(
{'params': model.basemodel.parameters()})
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# if (epoch + 1) % 10 == 0:
# for param_group in optimizer.param_groups:
# param_group['lr'] = lr / 1.5
# init in-epoch statistics
grapheme_root_train = []
vowel_diacritic_train = []
consonant_diacritic_train = []
grapheme_train = []
grapheme_root_prediction_train = []
vowel_diacritic_prediction_train = []
consonant_diacritic_prediction_train = []
grapheme_prediction_train = []
# update lr and start from start_epoch
if lr_scheduler_each_iter:
if (epoch < start_epoch):
for _ in range(len(train_data_loader)):
scheduler.step()
continue
else:
if lr_scheduler_name != "ReduceLROnPlateau":
scheduler.step()
if (epoch < start_epoch):
continue
log.write("Epoch%s\n" % epoch)
log.write('\n')
sum_train_loss = np.zeros_like(train_loss)
sum_train = np.zeros_like(train_loss)
# init optimizer
torch.cuda.empty_cache()
model.zero_grad()
for tr_batch_i, (image, grapheme_root, vowel_diacritic,
consonant_diacritic,
grapheme) in enumerate(train_data_loader):
rate = 0
for param_group in optimizer.param_groups:
rate += param_group['lr'] / len(optimizer.param_groups)
# set model training mode
model.train()
# set input to cuda mode
image = image.cuda()
image_copy = image.clone()
grapheme_root = grapheme_root.cuda().float()
vowel_diacritic = vowel_diacritic.cuda().float()
consonant_diacritic = consonant_diacritic.cuda().float()
grapheme = grapheme.cuda().float()
if ((loss_type == 'mae') or (loss_type == 'mse')):
_, grapheme_root = torch.max(grapheme_root, 1)
grapheme_root = torch.squeeze(grapheme_root.float())
_, vowel_diacritic = torch.max(vowel_diacritic, 1)
vowel_diacritic = torch.squeeze(vowel_diacritic.float())
_, consonant_diacritic = torch.max(consonant_diacritic, 1)
consonant_diacritic = torch.squeeze(
consonant_diacritic.float())
_, grapheme = torch.max(grapheme, 1)
grapheme = torch.squeeze(grapheme.float())
# predict and calculate loss (cutmix added)
r = np.random.rand(1)
if r < cutmix_prob:
def rand_bbox(size, lam):
W = size[2]
H = size[3]
cut_rat = np.sqrt(1. - lam)
cut_w = np.int(W * cut_rat)
cut_h = np.int(H * cut_rat)
# uniform
cx = np.random.randint(W)
cy = np.random.randint(H)
bbx1 = np.clip(cx - cut_w // 2, 0, W)
bby1 = np.clip(cy - cut_h // 2, 0, H)
bbx2 = np.clip(cx + cut_w // 2, 0, W)
bby2 = np.clip(cy + cut_h // 2, 0, H)
return bbx1, bby1, bbx2, bby2
# generate mixed sample
lam = np.random.beta(beta, beta)
rand_index = torch.randperm(image.shape[0]).cuda()
# shuffled_image = image[rand_index]
grapheme_root_a = grapheme_root
grapheme_root_b = grapheme_root[rand_index]
vowel_diacritic_a = vowel_diacritic
vowel_diacritic_b = vowel_diacritic[rand_index]
consonant_diacritic_a = consonant_diacritic
consonant_diacritic_b = consonant_diacritic[rand_index]
grapheme_a = grapheme
grapheme_b = grapheme[rand_index]
bbx1, bby1, bbx2, bby2 = rand_bbox(image.shape, lam)
image[:, :, bbx1:bbx2, bby1:bby2] = image[rand_index, :,
bbx1:bbx2, bby1:bby2]
# adjust lambda to exactly match pixel ratio
lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) /
(image.size()[-1] * image.size()[-2]))
# lam = 1 - (torch.mean(image[:, :, bbx1:bbx2, bby1:bby2]) / torch.mean(image))
# compute output
predictions = model(image)
grapheme_root_prediction = torch.squeeze(predictions[0])
vowel_diacritic_prediction = torch.squeeze(predictions[1])
consonant_diacritic_prediction = torch.squeeze(predictions[2])
grapheme_prediction = torch.squeeze(predictions[3])
loss = weight_grapheme_root * criterion(grapheme_root_prediction, grapheme_root_a) * lam + \
weight_grapheme_root * criterion(grapheme_root_prediction, grapheme_root_b) * (1 - lam) + \
weight_vowel_diacritic * criterion(vowel_diacritic_prediction, vowel_diacritic_a) * lam + \
weight_vowel_diacritic * criterion(vowel_diacritic_prediction, vowel_diacritic_b) * (1- lam) + \
weight_consonant_diacritic * criterion(consonant_diacritic_prediction, consonant_diacritic_a) * lam + \
weight_consonant_diacritic * criterion(consonant_diacritic_prediction, consonant_diacritic_b) * (1 - lam) + \
weight_grapheme * criterion(grapheme_prediction, grapheme_a) * lam + \
weight_grapheme * criterion(grapheme_prediction, grapheme_b) * (1 - lam)
elif r < (mixup_prob + cutmix_prob):
indices = torch.randperm(image.size(0))
shuffled_image = image[indices]
shuffled_grapheme_root = grapheme_root[indices]
shuffled_vowel_diacritic = vowel_diacritic[indices]
shuffled_consonant_diacritic = consonant_diacritic[indices]
shuffled_grapheme = grapheme[indices]
lam = np.random.beta(alpha, alpha)
image = image * lam + shuffled_image * (1 - lam)
predictions = model(image)
grapheme_root_prediction = torch.squeeze(predictions[0])
vowel_diacritic_prediction = torch.squeeze(predictions[1])
consonant_diacritic_prediction = torch.squeeze(predictions[2])
grapheme_prediction = torch.squeeze(predictions[3])
loss = weight_grapheme_root * criterion(grapheme_root_prediction, grapheme_root) * lam + \
weight_grapheme_root * criterion(grapheme_root_prediction, shuffled_grapheme_root) * (1 - lam) + \
weight_vowel_diacritic * criterion(vowel_diacritic_prediction, vowel_diacritic) * lam + \
weight_vowel_diacritic * criterion(vowel_diacritic_prediction, shuffled_vowel_diacritic) * (1- lam) + \
weight_consonant_diacritic * criterion(consonant_diacritic_prediction, consonant_diacritic) * lam + \
weight_consonant_diacritic * criterion(consonant_diacritic_prediction, shuffled_consonant_diacritic) * (1 - lam) + \
weight_grapheme * criterion(grapheme_prediction, grapheme) * lam + \
weight_grapheme * criterion(grapheme_prediction, shuffled_grapheme) * (1 - lam)
else:
predictions = model(image)
grapheme_root_prediction = torch.squeeze(predictions[0])
vowel_diacritic_prediction = torch.squeeze(predictions[1])
consonant_diacritic_prediction = torch.squeeze(predictions[2])
grapheme_prediction = torch.squeeze(predictions[3])
loss = weight_grapheme_root * criterion(grapheme_root_prediction, grapheme_root) + \
weight_vowel_diacritic * criterion(vowel_diacritic_prediction, vowel_diacritic) + \
weight_consonant_diacritic * criterion(consonant_diacritic_prediction, consonant_diacritic) + \
weight_grapheme * criterion(grapheme_prediction, grapheme)
# use apex
if apex:
with amp.scale_loss(loss / accumulation_steps,
optimizer) as scaled_loss:
scaled_loss.backward()
else:
# don't use apex
loss.backward()
if ((tr_batch_i + 1) % accumulation_steps == 0):
torch.nn.utils.clip_grad_norm_(model.parameters(),
max_norm=5.0,
norm_type=2)
optimizer.step()
# if (tr_batch_i+1) > int(len(train_data_loader) / 10) and (tr_batch_i+1) % int(len(train_data_loader) / 5) == 0:
# optimizer.update_swa()
model.zero_grad()
# adjust lr
if (lr_scheduler_each_iter):
scheduler.step()
writer.add_scalar(
'train_loss_' + str(fold), loss.item(),
(epoch - 1) * len(train_data_loader) * batch_size +
tr_batch_i * batch_size)
# calculate statistics
if not ((loss_type == "mae") or (loss_type == "mse")):
grapheme_root = torch.argmax(grapheme_root, 1)
grapheme_root = torch.squeeze(grapheme_root.float())
vowel_diacritic = torch.argmax(vowel_diacritic, 1)
vowel_diacritic = torch.squeeze(vowel_diacritic.float())
consonant_diacritic = torch.argmax(consonant_diacritic, 1)
consonant_diacritic = torch.squeeze(
consonant_diacritic.float())
grapheme = torch.argmax(grapheme, 1)
grapheme = torch.squeeze(grapheme.float())
# calculate traing result without cutmix, otherwise prediction is not related to image
with torch.no_grad():
predictions_no_cutmix = model(image_copy)
grapheme_root_prediction_no_cutmix = torch.squeeze(
predictions_no_cutmix[0])
vowel_diacritic_prediction_no_cutmix = torch.squeeze(
predictions_no_cutmix[1])
consonant_diacritic_prediction_no_cutmix = torch.squeeze(
predictions_no_cutmix[2])
grapheme_prediction_no_cutmix = torch.squeeze(
predictions_no_cutmix[3])
# calculate statistics
if not ((loss_type == "mae") or (loss_type == "mse")):
grapheme_root_prediction_no_cutmix = F.softmax(
grapheme_root_prediction_no_cutmix, 1)
grapheme_root_prediction_no_cutmix = torch.argmax(
grapheme_root_prediction_no_cutmix, 1)
grapheme_root_prediction_no_cutmix = torch.squeeze(
grapheme_root_prediction_no_cutmix.float())
vowel_diacritic_prediction_no_cutmix = F.softmax(
vowel_diacritic_prediction_no_cutmix, 1)
vowel_diacritic_prediction_no_cutmix = torch.argmax(
vowel_diacritic_prediction_no_cutmix, 1)
vowel_diacritic_prediction_no_cutmix = torch.squeeze(
vowel_diacritic_prediction_no_cutmix.float())
consonant_diacritic_prediction_no_cutmix = F.softmax(
consonant_diacritic_prediction_no_cutmix, 1)
consonant_diacritic_prediction_no_cutmix = torch.argmax(
consonant_diacritic_prediction_no_cutmix, 1)
consonant_diacritic_prediction_no_cutmix = torch.squeeze(
consonant_diacritic_prediction_no_cutmix.float())
grapheme_prediction_no_cutmix = F.softmax(
grapheme_prediction_no_cutmix, 1)
grapheme_prediction_no_cutmix = torch.argmax(
grapheme_prediction_no_cutmix, 1)
grapheme_prediction_no_cutmix = torch.squeeze(
grapheme_prediction_no_cutmix.float())
grapheme_root = grapheme_root.cpu().detach().numpy()
vowel_diacritic = vowel_diacritic.cpu().detach().numpy()
consonant_diacritic = consonant_diacritic.cpu().detach().numpy()
grapheme = grapheme.cpu().detach().numpy()
grapheme_root_prediction = grapheme_root_prediction_no_cutmix.cpu(
).detach().numpy()
vowel_diacritic_prediction = vowel_diacritic_prediction_no_cutmix.cpu(
).detach().numpy()
consonant_diacritic_prediction = consonant_diacritic_prediction_no_cutmix.cpu(
).detach().numpy()
grapheme_prediction = grapheme_prediction_no_cutmix.cpu().detach(
).numpy()
l = np.array([loss.item() * batch_size])
n = np.array([batch_size])
sum_train_loss = sum_train_loss + l
sum_train = sum_train + n
grapheme_root_train.append(grapheme_root)
vowel_diacritic_train.append(vowel_diacritic)
consonant_diacritic_train.append(consonant_diacritic)
grapheme_train.append(grapheme)
grapheme_root_prediction_train.append(grapheme_root_prediction)
vowel_diacritic_prediction_train.append(vowel_diacritic_prediction)
consonant_diacritic_prediction_train.append(
consonant_diacritic_prediction)
grapheme_prediction_train.append(grapheme_prediction)
grapheme_root_recall_train = metric(np.concatenate(grapheme_root_prediction_train, axis=0), \
np.concatenate(grapheme_root_train, axis=0))
vowel_diacritic_recall_train = metric(np.concatenate(vowel_diacritic_prediction_train, axis=0), \
np.concatenate(vowel_diacritic_train, axis=0))
consonant_diacritic_recall_train = metric(np.concatenate(consonant_diacritic_prediction_train, axis=0), \
np.concatenate(consonant_diacritic_train, axis=0))
grapheme_recall_train = metric(np.concatenate(grapheme_prediction_train, axis=0), \
np.concatenate(grapheme_train, axis=0))
average_recall_train = np.average([grapheme_root_recall_train, vowel_diacritic_recall_train, consonant_diacritic_recall_train], \
weights=[2,1,1])
# log for training
if (tr_batch_i + 1) % log_step == 0:
train_loss = sum_train_loss / (sum_train + 1e-12)
sum_train_loss[...] = 0
sum_train[...] = 0
log.write('lr: %f train loss: %f average_recall: %f grapheme_root_recall: %f vowel_diacritic_recall: %f consonant_diacritic_recall: %f grapheme_recall: %f\n' % \
(rate, train_loss[0], average_recall_train, \
grapheme_root_recall_train, \
vowel_diacritic_recall_train, \
consonant_diacritic_recall_train, \
grapheme_recall_train))
if (tr_batch_i + 1) % eval_step == 0:
# finish training then update swa
# optimizer.bn_update(train_data_loader, model)
# optimizer.swap_swa_sgd()
eval_count += 1
valid_loss = np.zeros(1, np.float32)
valid_num = np.zeros_like(valid_loss)
grapheme_root_val = []
vowel_diacritic_val = []
consonant_diacritic_val = []
grapheme_val = []
grapheme_root_prediction_val = []
vowel_diacritic_prediction_val = []
consonant_diacritic_prediction_val = []
grapheme_prediction_val = []
grapheme_root_prediction_decode_val = []
vowel_diacritic_prediction_decode_val = []
consonant_diacritic_prediction_decode_val = []
grapheme_prediction_decode_val = []
with torch.no_grad():
# init cache
torch.cuda.empty_cache()
for val_batch_i, (image, grapheme_root, vowel_diacritic,
consonant_diacritic,
grapheme) in enumerate(val_data_loader):
# set model to eval mode
model.eval()
# set input to cuda mode
image = image.cuda()
grapheme_root = grapheme_root.cuda().float()
vowel_diacritic = vowel_diacritic.cuda().float()
consonant_diacritic = consonant_diacritic.cuda().float(
)
grapheme = grapheme.cuda().float()
if ((loss_type == 'mae') or (loss_type == 'mse')):
_, grapheme_root = torch.max(grapheme_root, 1)
grapheme_root = torch.squeeze(
grapheme_root.float())
_, vowel_diacritic = torch.max(vowel_diacritic, 1)
vowel_diacritic = torch.squeeze(
vowel_diacritic.float())
_, consonant_diacritic = torch.max(
consonant_diacritic, 1)
consonant_diacritic = torch.squeeze(
consonant_diacritic.float())
_, grapheme = torch.max(grapheme, 1)
grapheme = torch.squeeze(grapheme.float())
# predict and calculate loss (only need torch.sigmoid when inference)
predictions = model(image)
grapheme_root_prediction = torch.squeeze(
predictions[0])
vowel_diacritic_prediction = torch.squeeze(
predictions[1])
consonant_diacritic_prediction = torch.squeeze(
predictions[2])
grapheme_prediction = torch.squeeze(predictions[3])
loss = weight_grapheme_root * criterion(grapheme_root_prediction, grapheme_root) + \
weight_vowel_diacritic * criterion(vowel_diacritic_prediction, vowel_diacritic) + \
weight_consonant_diacritic * criterion(consonant_diacritic_prediction, consonant_diacritic) + \
weight_grapheme * criterion(grapheme_prediction, grapheme)
writer.add_scalar(
'val_loss_' + str(fold), loss.item(),
(eval_count - 1) * len(val_data_loader) *
valid_batch_size + val_batch_i * valid_batch_size)
# calculate statistics
if not ((loss_type == "mae") or (loss_type == "mse")):
grapheme_root = torch.argmax(grapheme_root, 1)
grapheme_root = torch.squeeze(
grapheme_root.float())
vowel_diacritic = torch.argmax(vowel_diacritic, 1)
vowel_diacritic = torch.squeeze(
vowel_diacritic.float())
consonant_diacritic = torch.argmax(
consonant_diacritic, 1)
consonant_diacritic = torch.squeeze(
consonant_diacritic.float())
grapheme = torch.argmax(grapheme, 1)
grapheme = torch.squeeze(grapheme.float())
grapheme_root_prediction = F.softmax(
grapheme_root_prediction, 1)
grapheme_root_prediction = torch.argmax(
grapheme_root_prediction, 1)
grapheme_root_prediction = torch.squeeze(
grapheme_root_prediction.float())
vowel_diacritic_prediction = F.softmax(
vowel_diacritic_prediction, 1)
vowel_diacritic_prediction = torch.argmax(
vowel_diacritic_prediction, 1)
vowel_diacritic_prediction = torch.squeeze(
vowel_diacritic_prediction.float())
consonant_diacritic_prediction = F.softmax(
consonant_diacritic_prediction, 1)
consonant_diacritic_prediction = torch.argmax(
consonant_diacritic_prediction, 1)
consonant_diacritic_prediction = torch.squeeze(
consonant_diacritic_prediction.float())
grapheme_prediction = F.softmax(
grapheme_prediction, 1)
grapheme_prediction = torch.argmax(
grapheme_prediction, 1)
grapheme_prediction = torch.squeeze(
grapheme_prediction.float())
grapheme_root = grapheme_root.cpu().detach().numpy()
vowel_diacritic = vowel_diacritic.cpu().detach().numpy(
)
consonant_diacritic = consonant_diacritic.cpu().detach(
).numpy()
grapheme = grapheme.cpu().detach().numpy()
grapheme_root_prediction = grapheme_root_prediction.cpu(
).detach().numpy()
vowel_diacritic_prediction = vowel_diacritic_prediction.cpu(
).detach().numpy()
consonant_diacritic_prediction = consonant_diacritic_prediction.cpu(
).detach().numpy()
grapheme_prediction = grapheme_prediction.cpu().detach(
).numpy()
grapheme_root_prediction_decode, \
vowel_diacritic_prediction_decode, \
consonant_diacritic_prediction_decode, \
grapheme_prediction_decode = decode([grapheme_root_prediction.copy(), vowel_diacritic_prediction.copy(), \
consonant_diacritic_prediction.copy(), grapheme_prediction.copy()], CLASS_MAP)
l = np.array([loss.item() * valid_batch_size])
n = np.array([valid_batch_size])
valid_loss = valid_loss + l
valid_num = valid_num + n
grapheme_root_val.append(grapheme_root)
vowel_diacritic_val.append(vowel_diacritic)
consonant_diacritic_val.append(consonant_diacritic)
grapheme_val.append(grapheme)
grapheme_root_prediction_val.append(
grapheme_root_prediction)
vowel_diacritic_prediction_val.append(
vowel_diacritic_prediction)
consonant_diacritic_prediction_val.append(
consonant_diacritic_prediction)
grapheme_prediction_val.append(grapheme_prediction)
grapheme_root_prediction_decode_val.append(
grapheme_root_prediction_decode)
vowel_diacritic_prediction_decode_val.append(
vowel_diacritic_prediction_decode)
consonant_diacritic_prediction_decode_val.append(
consonant_diacritic_prediction_decode)
grapheme_prediction_decode_val.append(
grapheme_prediction_decode)
grapheme_root_recall_val = metric(np.concatenate(grapheme_root_prediction_val, axis=0), \
np.concatenate(grapheme_root_val, axis=0))
vowel_diacritic_recall_val = metric(np.concatenate(vowel_diacritic_prediction_val, axis=0), \
np.concatenate(vowel_diacritic_val, axis=0))
consonant_diacritic_recall_val = metric(np.concatenate(consonant_diacritic_prediction_val, axis=0), \
np.concatenate(consonant_diacritic_val, axis=0))
grapheme_recall_val = metric(np.concatenate(grapheme_prediction_val, axis=0), \
np.concatenate(grapheme_val, axis=0))
grapheme_root_recall_decode_val = metric(np.concatenate(grapheme_root_prediction_decode_val, axis=0), \
np.concatenate(grapheme_root_val, axis=0))
vowel_diacritic_recall_decode_val = metric(np.concatenate(vowel_diacritic_prediction_decode_val, axis=0), \
np.concatenate(vowel_diacritic_val, axis=0))
consonant_diacritic_recall_decode_val = metric(np.concatenate(consonant_diacritic_prediction_decode_val, axis=0), \
np.concatenate(consonant_diacritic_val, axis=0))
average_recall_val = np.average([grapheme_root_recall_val, vowel_diacritic_recall_val, consonant_diacritic_recall_val], \
weights=[2,1,1])
average_recall_decode_val = np.average([grapheme_root_recall_decode_val, vowel_diacritic_recall_decode_val, \
consonant_diacritic_recall_decode_val], \
weights=[2,1,1])
valid_loss = valid_loss / valid_num
log.write('valid loss: %f average_recall: %f grapheme_root_recall: %f vowel_diacritic_recall: %f consonant_diacritic_recall: %f grapheme_recall: %f average_recall_decode: %f grapheme_root_recall_decode: %f vowel_diacritic_recall_decode: %f consonant_diacritic_recall_decode: %f\n' % \
(valid_loss[0], average_recall_val, \
grapheme_root_recall_val, \
vowel_diacritic_recall_val, \
consonant_diacritic_recall_val, \
grapheme_recall_val, \
average_recall_decode_val, \
grapheme_root_recall_decode_val, \
vowel_diacritic_recall_decode_val, \
consonant_diacritic_recall_decode_val))
val_metric_epoch = average_recall_val
# scheduler lr by metric
if lr_scheduler_name == "ReduceLROnPlateau":
scheduler.step(val_metric_epoch)
if (val_metric_epoch >= valid_metric_optimal):
log.write('Validation metric improved ({:.6f} --> {:.6f}). Saving model ...'.format(\
valid_metric_optimal, val_metric_epoch))
valid_metric_optimal = val_metric_epoch
torch.save(model.state_dict(), checkpoint_filepath)
count = 0
else:
count += 1
if (count == early_stopping):
break
def tune_mahalanobis_hyperparams():
def print_tuning_results(results, stypes):
mtypes = ['FPR', 'DTERR', 'AUROC', 'AUIN', 'AUOUT']
for stype in stypes:
print(' OOD detection method: ' + stype)
for mtype in mtypes:
print(' {mtype:6s}'.format(mtype=mtype), end='')
print('\n{val:6.2f}'.format(val=100. * results[stype]['FPR']),
end='')
print(' {val:6.2f}'.format(val=100. * results[stype]['DTERR']),
end='')
print(' {val:6.2f}'.format(val=100. * results[stype]['AUROC']),
end='')
print(' {val:6.2f}'.format(val=100. * results[stype]['AUIN']),
end='')
print(' {val:6.2f}\n'.format(val=100. * results[stype]['AUOUT']),
end='')
print('')
print('Tuning hyper-parameters...')
stypes = ['mahalanobis']
save_dir = os.path.join('output/hyperparams/', args.name, 'tmp')
if not os.path.exists(save_dir):
os.makedirs(save_dir)
normalizer = transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0))
transform = transforms.Compose([
transforms.ToTensor(),
])
if args.in_dataset == "CIFAR-10":
trainset = torchvision.datasets.CIFAR10('./datasets/cifar10',
train=True,
download=True,
transform=transform)
trainloaderIn = torch.utils.data.DataLoader(trainset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
testset = torchvision.datasets.CIFAR10(root='./datasets/cifar10',
train=False,
download=True,
transform=transform)
testloaderIn = torch.utils.data.DataLoader(testset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
num_classes = 10
elif args.in_dataset == "CIFAR-100":
trainset = torchvision.datasets.CIFAR100('./datasets/cifar10',
train=True,
download=True,
transform=transform)
trainloaderIn = torch.utils.data.DataLoader(trainset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
testset = torchvision.datasets.CIFAR100(root='./datasets/cifar100',
train=False,
download=True,
transform=transform)
testloaderIn = torch.utils.data.DataLoader(testset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
num_classes = 100
valloaderOut = torch.utils.data.DataLoader(
TinyImages(transform=transforms.Compose([
transforms.ToTensor(),
transforms.ToPILImage(),
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor()
])),
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
model = dn.DenseNet3(args.layers, num_classes, normalizer=normalizer)
checkpoint = torch.load(
"./checkpoints/{name}/checkpoint_{epochs}.pth.tar".format(
name=args.name, epochs=args.epochs))
model.load_state_dict(checkpoint['state_dict'])
model.eval()
model.cuda()
# set information about feature extaction
temp_x = torch.rand(2, 3, 32, 32)
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print('get sample mean and covariance')
sample_mean, precision = sample_estimator(model, num_classes, feature_list,
trainloaderIn)
print('train logistic regression model')
m = 1000
val_in = []
val_out = []
cnt = 0
for data, target in trainloaderIn:
for x in data:
val_in.append(x.numpy())
cnt += 1
if cnt == m:
break
if cnt == m:
break
cnt = 0
for data, target in valloaderOut:
for x in data:
val_out.append(data[0].numpy())
cnt += 1
if cnt == m:
break
if cnt == m:
break
train_lr_data = []
train_lr_label = []
train_lr_data.extend(val_in)
train_lr_label.extend(np.zeros(m))
train_lr_data.extend(val_out)
train_lr_label.extend(np.ones(m))
train_lr_data = torch.tensor(train_lr_data)
train_lr_label = torch.tensor(train_lr_label)
best_fpr = 1.1
best_magnitude = 0.0
for magnitude in np.arange(0, 0.0041, 0.004 / 20):
train_lr_Mahalanobis = []
total = 0
for data_index in range(
int(np.floor(train_lr_data.size(0) / args.batch_size))):
data = train_lr_data[total:total + args.batch_size]
total += args.batch_size
Mahalanobis_scores = get_Mahalanobis_score(model, data,
num_classes,
sample_mean, precision,
num_output, magnitude)
train_lr_Mahalanobis.extend(Mahalanobis_scores)
train_lr_Mahalanobis = np.asarray(train_lr_Mahalanobis,
dtype=np.float32)
regressor = LogisticRegressionCV().fit(train_lr_Mahalanobis,
train_lr_label)
print('Logistic Regressor params:', regressor.coef_,
regressor.intercept_)
t0 = time.time()
f1 = open(os.path.join(save_dir, "confidence_mahalanobis_In.txt"), 'w')
f2 = open(os.path.join(save_dir, "confidence_mahalanobis_Out.txt"),
'w')
########################################In-distribution###########################################
print("Processing in-distribution images")
count = 0
for i in range(int(m / args.batch_size) + 1):
if i * args.batch_size >= m:
break
images = torch.tensor(
val_in[i * args.batch_size:min((i + 1) * args.batch_size, m)])
# if j<1000: continue
batch_size = images.shape[0]
Mahalanobis_scores = get_Mahalanobis_score(model, images,
num_classes,
sample_mean, precision,
num_output, magnitude)
confidence_scores = regressor.predict_proba(Mahalanobis_scores)[:,
1]
for k in range(batch_size):
f1.write("{}\n".format(-confidence_scores[k]))
count += batch_size
print("{:4}/{:4} images processed, {:.1f} seconds used.".format(
count, m,
time.time() - t0))
t0 = time.time()
###################################Out-of-Distributions#####################################
t0 = time.time()
print("Processing out-of-distribution images")
count = 0
for i in range(int(m / args.batch_size) + 1):
if i * args.batch_size >= m:
break
images = torch.tensor(
val_out[i * args.batch_size:min((i + 1) * args.batch_size, m)])
# if j<1000: continue
batch_size = images.shape[0]
Mahalanobis_scores = get_Mahalanobis_score(model, images,
num_classes,
sample_mean, precision,
num_output, magnitude)
confidence_scores = regressor.predict_proba(Mahalanobis_scores)[:,
1]
for k in range(batch_size):
f2.write("{}\n".format(-confidence_scores[k]))
count += batch_size
print("{:4}/{:4} images processed, {:.1f} seconds used.".format(
count, m,
time.time() - t0))
t0 = time.time()
f1.close()
f2.close()
results = metric(save_dir, stypes)
print_tuning_results(results, stypes)
fpr = results['mahalanobis']['FPR']
if fpr < best_fpr:
best_fpr = fpr
best_magnitude = magnitude
best_regressor = regressor
print('Best Logistic Regressor params:', best_regressor.coef_,
best_regressor.intercept_)
print('Best magnitude', best_magnitude)
return sample_mean, precision, best_regressor, best_magnitude
def eval_msp_and_odin():
stypes = ['MSP', 'ODIN']
save_dir = os.path.join('output/ood_scores/', args.out_dataset, args.name,
'adv' if args.adv else 'nat')
if not os.path.exists(save_dir):
os.makedirs(save_dir)
start = time.time()
#loading data sets
normalizer = transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0))
transform = transforms.Compose([
transforms.ToTensor(),
])
if args.in_dataset == "CIFAR-10":
testset = torchvision.datasets.CIFAR10(root='./datasets/cifar10',
train=False,
download=True,
transform=transform)
testloaderIn = torch.utils.data.DataLoader(testset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
num_classes = 10
elif args.in_dataset == "CIFAR-100":
testset = torchvision.datasets.CIFAR100(root='./datasets/cifar100',
train=False,
download=True,
transform=transform)
testloaderIn = torch.utils.data.DataLoader(testset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
num_classes = 100
model = dn.DenseNet3(args.layers, num_classes, normalizer=normalizer)
checkpoint = torch.load(
"./checkpoints/{name}/checkpoint_{epochs}.pth.tar".format(
name=args.name, epochs=args.epochs))
model.load_state_dict(checkpoint['state_dict'])
model.eval()
model.cuda()
if args.out_dataset == 'SVHN':
testsetout = svhn.SVHN('datasets/ood_datasets/svhn/',
split='test',
transform=transforms.ToTensor(),
download=False)
testloaderOut = torch.utils.data.DataLoader(testsetout,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
elif args.out_dataset == 'dtd':
testsetout = torchvision.datasets.ImageFolder(
root="datasets/ood_datasets/dtd/images",
transform=transforms.Compose([
transforms.Resize(32),
transforms.CenterCrop(32),
transforms.ToTensor()
]))
testloaderOut = torch.utils.data.DataLoader(testsetout,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
elif args.out_dataset == 'places365':
testsetout = torchvision.datasets.ImageFolder(
root="datasets/ood_datasets/places365/test_subset",
transform=transforms.Compose([
transforms.Resize(32),
transforms.CenterCrop(32),
transforms.ToTensor()
]))
testloaderOut = torch.utils.data.DataLoader(testsetout,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
else:
testsetout = torchvision.datasets.ImageFolder(
"./datasets/ood_datasets/{}".format(args.out_dataset),
transform=transform)
testloaderOut = torch.utils.data.DataLoader(testsetout,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
t0 = time.time()
f1 = open(os.path.join(save_dir, "confidence_MSP_In.txt"), 'w')
f2 = open(os.path.join(save_dir, "confidence_MSP_Out.txt"), 'w')
g1 = open(os.path.join(save_dir, "confidence_ODIN_In.txt"), 'w')
g2 = open(os.path.join(save_dir, "confidence_ODIN_Out.txt"), 'w')
N = 10000
if args.out_dataset == "iSUN": N = 8925
if args.out_dataset == "dtd": N = 5640
########################################In-distribution###########################################
print("Processing in-distribution images")
if args.adv:
attack = ConfidenceLinfPGDAttack(model,
eps=args.epsilon,
nb_iter=args.iters,
eps_iter=args.iter_size,
rand_init=True,
clip_min=0.,
clip_max=1.,
in_distribution=True,
num_classes=num_classes)
count = 0
for j, data in enumerate(testloaderIn):
images, _ = data
batch_size = images.shape[0]
if count + batch_size > N:
images = images[:N - count]
batch_size = images.shape[0]
if args.adv:
adv_images = attack.perturb(images)
inputs = Variable(adv_images, requires_grad=True)
else:
inputs = Variable(images, requires_grad=True)
outputs = model(inputs)
nnOutputs = MSP(outputs, model)
for k in range(batch_size):
f1.write("{}\n".format(np.max(nnOutputs[k])))
nnOutputs = ODIN(inputs,
outputs,
model,
temper=args.temperature,
noiseMagnitude1=args.magnitude)
for k in range(batch_size):
g1.write("{}\n".format(np.max(nnOutputs[k])))
count += batch_size
print("{:4}/{:4} images processed, {:.1f} seconds used.".format(
count, N,
time.time() - t0))
t0 = time.time()
if count == N: break
###################################Out-of-Distributions#####################################
t0 = time.time()
print("Processing out-of-distribution images")
if args.adv:
attack = ConfidenceLinfPGDAttack(model,
eps=args.epsilon,
nb_iter=args.iters,
eps_iter=args.iter_size,
rand_init=True,
clip_min=0.,
clip_max=1.,
in_distribution=False,
num_classes=num_classes)
count = 0
for j, data in enumerate(testloaderOut):
images, labels = data
batch_size = images.shape[0]
if args.adv:
adv_images = attack.perturb(images)
inputs = Variable(adv_images, requires_grad=True)
else:
inputs = Variable(images, requires_grad=True)
outputs = model(inputs)
nnOutputs = MSP(outputs, model)
for k in range(batch_size):
f2.write("{}\n".format(np.max(nnOutputs[k])))
nnOutputs = ODIN(inputs,
outputs,
model,
temper=args.temperature,
noiseMagnitude1=args.magnitude)
for k in range(batch_size):
g2.write("{}\n".format(np.max(nnOutputs[k])))
count += batch_size
print("{:4}/{:4} images processed, {:.1f} seconds used.".format(
count, N,
time.time() - t0))
t0 = time.time()
if count == N: break
f1.close()
f2.close()
g1.close()
g2.close()
results = metric(save_dir, stypes)
print_results(results, stypes)
def eval_mahalanobis(sample_mean, precision, regressor, magnitude):
stypes = ['mahalanobis']
save_dir = os.path.join('output/ood_scores/', args.out_dataset, args.name,
'adv' if args.adv else 'nat')
if not os.path.exists(save_dir):
os.makedirs(save_dir)
start = time.time()
#loading data sets
normalizer = transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0))
transform = transforms.Compose([
transforms.ToTensor(),
])
if args.in_dataset == "CIFAR-10":
trainset = torchvision.datasets.CIFAR10('./datasets/cifar10',
train=True,
download=True,
transform=transform)
trainloaderIn = torch.utils.data.DataLoader(trainset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
testset = torchvision.datasets.CIFAR10(root='./datasets/cifar10',
train=False,
download=True,
transform=transform)
testloaderIn = torch.utils.data.DataLoader(testset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
num_classes = 10
elif args.in_dataset == "CIFAR-100":
trainset = torchvision.datasets.CIFAR100('./datasets/cifar10',
train=True,
download=True,
transform=transform)
trainloaderIn = torch.utils.data.DataLoader(trainset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
testset = torchvision.datasets.CIFAR100(root='./datasets/cifar100',
train=False,
download=True,
transform=transform)
testloaderIn = torch.utils.data.DataLoader(testset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
num_classes = 100
model = dn.DenseNet3(args.layers, num_classes, normalizer=normalizer)
checkpoint = torch.load(
"./checkpoints/{name}/checkpoint_{epochs}.pth.tar".format(
name=args.name, epochs=args.epochs))
model.load_state_dict(checkpoint['state_dict'])
model.eval()
model.cuda()
if args.out_dataset == 'SVHN':
testsetout = svhn.SVHN('datasets/ood_datasets/svhn/',
split='test',
transform=transforms.ToTensor(),
download=False)
testloaderOut = torch.utils.data.DataLoader(testsetout,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
elif args.out_dataset == 'dtd':
testsetout = torchvision.datasets.ImageFolder(
root="datasets/ood_datasets/dtd/images",
transform=transforms.Compose([
transforms.Resize(32),
transforms.CenterCrop(32),
transforms.ToTensor()
]))
testloaderOut = torch.utils.data.DataLoader(testsetout,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
elif args.out_dataset == 'places365':
testsetout = torchvision.datasets.ImageFolder(
root="datasets/ood_datasets/places365/test_subset",
transform=transforms.Compose([
transforms.Resize(32),
transforms.CenterCrop(32),
transforms.ToTensor()
]))
testloaderOut = torch.utils.data.DataLoader(testsetout,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
else:
testsetout = torchvision.datasets.ImageFolder(
"./datasets/ood_datasets/{}".format(args.out_dataset),
transform=transform)
testloaderOut = torch.utils.data.DataLoader(testsetout,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
# set information about feature extaction
temp_x = torch.rand(2, 3, 32, 32)
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
t0 = time.time()
f1 = open(os.path.join(save_dir, "confidence_mahalanobis_In.txt"), 'w')
f2 = open(os.path.join(save_dir, "confidence_mahalanobis_Out.txt"), 'w')
N = 10000
if args.out_dataset == "iSUN": N = 8925
if args.out_dataset == "dtd": N = 5640
########################################In-distribution###########################################
print("Processing in-distribution images")
if args.adv:
attack = MahalanobisLinfPGDAttack(model,
eps=args.epsilon,
nb_iter=args.iters,
eps_iter=args.iter_size,
rand_init=True,
clip_min=0.,
clip_max=1.,
in_distribution=True,
num_classes=num_classes,
sample_mean=sample_mean,
precision=precision,
num_output=num_output,
regressor=regressor)
count = 0
for j, data in enumerate(testloaderIn):
images, _ = data
batch_size = images.shape[0]
if count + batch_size > N:
images = images[:N - count]
batch_size = images.shape[0]
if args.adv:
inputs = attack.perturb(images)
else:
inputs = images
Mahalanobis_scores = get_Mahalanobis_score(model, inputs, num_classes,
sample_mean, precision,
num_output, magnitude)
confidence_scores = regressor.predict_proba(Mahalanobis_scores)[:, 1]
for k in range(batch_size):
f1.write("{}\n".format(-confidence_scores[k]))
count += batch_size
print("{:4}/{:4} images processed, {:.1f} seconds used.".format(
count, N,
time.time() - t0))
t0 = time.time()
if count == N: break
###################################Out-of-Distributions#####################################
t0 = time.time()
print("Processing out-of-distribution images")
if args.adv:
attack = MahalanobisLinfPGDAttack(model,
eps=args.epsilon,
nb_iter=args.iters,
eps_iter=args.iter_size,
rand_init=True,
clip_min=0.,
clip_max=1.,
in_distribution=False,
num_classes=num_classes,
sample_mean=sample_mean,
precision=precision,
num_output=num_output,
regressor=regressor)
count = 0
for j, data in enumerate(testloaderOut):
images, labels = data
batch_size = images.shape[0]
if args.adv:
inputs = attack.perturb(images)
else:
inputs = images
Mahalanobis_scores = get_Mahalanobis_score(model, inputs, num_classes,
sample_mean, precision,
num_output, magnitude)
confidence_scores = regressor.predict_proba(Mahalanobis_scores)[:, 1]
for k in range(batch_size):
f2.write("{}\n".format(-confidence_scores[k]))
count += batch_size
print("{:4}/{:4} images processed, {:.1f} seconds used.".format(
count, N,
time.time() - t0))
t0 = time.time()
if count == N: break
f1.close()
f2.close()
results = metric(save_dir, stypes)
print_results(results, stypes)
return
def item(report_id, want_object=False):
result = (db.session.query(Report, OpSysComponent)
.join(OpSysComponent)
.filter(Report.id == report_id)
.first())
if result is None:
abort(404)
report, component = result
solutions = None
if report.max_certainty is not None:
osr = get_report_opsysrelease(db=db, report_id=report.id)
solutions = [find_solution(report, db=db, osr=osr)]
releases = (db.session.query(ReportOpSysRelease, ReportOpSysRelease.count)
.filter(ReportOpSysRelease.report_id == report_id)
.order_by(desc(ReportOpSysRelease.count))
.all())
arches = (db.session.query(ReportArch, ReportArch.count)
.filter(ReportArch.report_id == report_id)
.order_by(desc(ReportArch.count))
.all())
modes = (db.session.query(ReportSelinuxMode, ReportSelinuxMode.count)
.filter(ReportSelinuxMode.report_id == report_id)
.order_by(desc(ReportSelinuxMode.count))
.all())
history_select = lambda table, date, date_range: (db.session.query(table).
filter(table.report_id == report_id)
.filter(date >= date_range)
# Flot is confused if not ordered
.order_by(date)
.all())
MAX_DAYS = 20 # Default set on 20
MAX_WEEK = 20 # Default set on 20
MAX_MONTH = 20 # Default set on 20
today = datetime.date.today()
# Show only 20 days
daily_history = history_select(ReportHistoryDaily, ReportHistoryDaily.day,
(today - timedelta(days=MAX_DAYS)))
if len(daily_history) == 0:
for x in range(0, MAX_DAYS):
daily_history.append({'day': today - timedelta(x),
'count': 0,
'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id})
elif len(daily_history) < MAX_DAYS:
if daily_history[-1].day < (today):
daily_history.append({'day': today,
'count': 0,
'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id
})
if daily_history[0].day > (today - timedelta(MAX_DAYS)):
daily_history.append({'day': today - timedelta(MAX_DAYS),
'count': 0,
'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id
})
# Show only 20 weeks
last_monday = datetime.datetime.today() - timedelta(datetime.datetime.today().weekday())
weekly_history = history_select(ReportHistoryWeekly, ReportHistoryWeekly.week,
(last_monday - timedelta(days=MAX_WEEK*7)))
if len(weekly_history) == 0:
for x in range(0, MAX_WEEK):
weekly_history.append({'week': last_monday - timedelta(x*7),
'count': 0,
'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id})
elif len(weekly_history) < MAX_WEEK:
if weekly_history[-1].week < (last_monday.date()):
weekly_history.append({'week': last_monday,
'count': 0,
'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id})
if weekly_history[0].week > ((last_monday - timedelta(7*MAX_WEEK)).date()):
weekly_history.append({'week': last_monday - timedelta(7*MAX_WEEK),
'count': 0,
'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id})
# Show only 20 months
monthly_history = history_select(ReportHistoryMonthly, ReportHistoryMonthly.month,
(today - relativedelta(months=MAX_MONTH)))
first_day_of_month = lambda t: (datetime.date(t.year, t.month, 1))
fdom = first_day_of_month(datetime.datetime.today())
if len(monthly_history) == 0:
for x in range(0, MAX_MONTH):
monthly_history.append({'month': fdom - relativedelta(months=x),
'count': 0,
'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id})
elif len(monthly_history) < MAX_MONTH:
if monthly_history[-1].month < (fdom):
monthly_history.append({'month': fdom,
'count': 0,
'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id})
if monthly_history[0].month > (fdom - relativedelta(months=MAX_MONTH)):
monthly_history.append({'month': fdom - relativedelta(months=MAX_MONTH),
'count': 0,
'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id})
complete_history = history_select(ReportHistoryMonthly, ReportHistoryMonthly.month,
(datetime.datetime.strptime('1970-01-01', '%Y-%m-%d')))
unique_ocurrence_os = {}
if len(complete_history) > 0:
for ch in complete_history:
os_name = "{0} {1}".format(ch.opsysrelease.opsys.name, ch.opsysrelease.version)
if ch.count is None:
ch.count = 0
if ch.unique is None:
ch.count = 0
if os_name not in unique_ocurrence_os:
unique_ocurrence_os[os_name] = {'count': ch.count, 'unique': ch.unique}
else:
unique_ocurrence_os[os_name]['count'] += ch.count
unique_ocurrence_os[os_name]['unique'] += ch.unique
sorted(unique_ocurrence_os)
packages = load_packages(db, report_id)
# creates a package_counts list with this structure:
# [(package name, count, [(package version, count in the version)])]
names = defaultdict(lambda: {"count": 0, "versions": defaultdict(int)})
for pkg in packages:
names[pkg.iname]["name"] = pkg.iname
names[pkg.iname]["count"] += pkg.count
names[pkg.iname]["versions"]["{0}:{1}-{2}"
.format(pkg.iepoch, pkg.iversion, pkg.irelease)] += pkg.count
package_counts = []
for pkg in sorted(names.values(), key=itemgetter("count"), reverse=True):
package_counts.append((
pkg["name"],
pkg["count"],
sorted(pkg["versions"].items(), key=itemgetter(1), reverse=True)))
try:
backtrace = report.backtraces[0].frames
except:
backtrace = []
fid = 0
for frame in backtrace:
fid += 1
frame.nice_order = fid
is_maintainer = is_component_maintainer(db, g.user, component)
contact_emails = []
if is_maintainer:
contact_emails = [email_address for (email_address, ) in
(db.session.query(ContactEmail.email_address)
.join(ReportContactEmail)
.filter(ReportContactEmail.report == report))]
maintainer = (db.session.query(AssociatePeople)
.join(OpSysReleaseComponentAssociate)
.join(OpSysReleaseComponent)
.join(OpSysComponent)
.filter(OpSysComponent.name == component.name)).first()
maintainer_contact = ""
if maintainer:
maintainer_contact = maintainer.name
probably_fixed = (db.session.query(ProblemOpSysRelease, Build)
.join(Problem)
.join(Report)
.join(Build)
.filter(Report.id == report_id)
.first())
forward = dict(report=report,
probably_fixed=probably_fixed,
component=component,
releases=metric(releases),
arches=metric(arches),
modes=metric(modes),
daily_history=daily_history,
weekly_history=weekly_history,
monthly_history=monthly_history,
complete_history=complete_history,
unique_ocurrence_os=unique_ocurrence_os,
crashed_packages=packages,
package_counts=package_counts,
backtrace=backtrace,
contact_emails=contact_emails,
solutions=solutions,
maintainer_contact=maintainer_contact)
forward['error_name'] = report.error_name
forward['oops'] = report.oops
if want_object:
try:
cf = component.name
if len(report.backtraces[0].crash_function) > 0:
cf += " in {0}".format(report.backtraces[0].crash_function)
forward['crash_function'] = cf
except:
forward['crash_function'] = ""
if probably_fixed:
tmp_dict = probably_fixed.ProblemOpSysRelease.serialize
tmp_dict['probable_fix_build'] = probably_fixed.Build.serialize
forward['probably_fixed'] = tmp_dict
# Avg count occurrence from first to last occurence
forward['avg_count_per_month'] = get_avg_count(report.first_occurrence,
report.last_occurrence,
report.count)
if len(forward['report'].bugs) > 0:
forward['bugs'] = []
for bug in forward['report'].bugs:
try:
forward['bugs'].append(bug.serialize)
except:
print "Bug serialize failed"
return forward
if request_wants_json():
return jsonify(forward)
forward["is_maintainer"] = is_maintainer
forward["extfafs"] = get_external_faf_instances(db)
return render_template("reports/item.html", **forward)
def item(report_id):
result = (db.session.query(Report, OpSysComponent)
.join(OpSysComponent)
.filter(Report.id == report_id)
.first())
if result is None:
abort(404)
report, component = result
releases = (db.session.query(ReportOpSysRelease, ReportOpSysRelease.count)
.filter(ReportOpSysRelease.report_id == report_id)
.order_by(desc(ReportOpSysRelease.count))
.all())
arches = (db.session.query(ReportArch, ReportArch.count)
.filter(ReportArch.report_id == report_id)
.order_by(desc(ReportArch.count))
.all())
modes = (db.session.query(ReportSelinuxMode, ReportSelinuxMode.count)
.filter(ReportSelinuxMode.report_id == report_id)
.order_by(desc(ReportSelinuxMode.count))
.all())
history_select = lambda table: (db.session.query(table).
filter(table.report_id == report_id)
.all())
daily_history = history_select(ReportHistoryDaily)
weekly_history = history_select(ReportHistoryWeekly)
monthly_history = history_select(ReportHistoryMonthly)
packages = load_packages(db, report_id, "CRASHED")
related_packages = load_packages(db, report_id, "RELATED")
related_packages_nevr = sorted(
[metric_tuple(name="{0}-{1}:{2}-{3}".format(
pkg.iname, pkg.iepoch, pkg.iversion, pkg.irelease),
count=pkg.count) for pkg in related_packages],
key=itemgetter(0))
merged_name = dict()
for package in related_packages:
if package.iname in merged_name:
merged_name[package.iname] += package.count
else:
merged_name[package.iname] = package.count
related_packages_name = sorted([metric_tuple(name=item[0], count=item[1])
for item in merged_name.items()],
key=itemgetter(0),
reverse=True)
try:
backtrace = report.backtraces[0].frames
except:
backtrace = []
fid = 0
for frame in backtrace:
fid += 1
frame.nice_order = fid
forward = dict(report=report,
component=component,
releases=metric(releases),
arches=metric(arches),
modes=metric(modes),
daily_history=daily_history,
weekly_history=weekly_history,
monthly_history=monthly_history,
crashed_packages=packages,
related_packages_nevr=related_packages_nevr,
related_packages_name=related_packages_name,
backtrace=backtrace)
if request_wants_json():
return jsonify(forward)
return render_template("reports/item.html", **forward)
def item(report_id):
result = (db.session.query(Report, OpSysComponent)
.join(OpSysComponent)
.filter(Report.id == report_id)
.first())
if result is None:
abort(404)
report, component = result
releases = (db.session.query(ReportOpSysRelease, ReportOpSysRelease.count)
.filter(ReportOpSysRelease.report_id == report_id)
.order_by(desc(ReportOpSysRelease.count))
.all())
arches = (db.session.query(ReportArch, ReportArch.count)
.filter(ReportArch.report_id == report_id)
.order_by(desc(ReportArch.count))
.all())
modes = (db.session.query(ReportSelinuxMode, ReportSelinuxMode.count)
.filter(ReportSelinuxMode.report_id == report_id)
.order_by(desc(ReportSelinuxMode.count))
.all())
history_select = lambda table, date: (db.session.query(table).
filter(table.report_id == report_id)
# Flot is confused if not ordered
.order_by(date)
.all())
daily_history = history_select(ReportHistoryDaily, ReportHistoryDaily.day)
weekly_history = history_select(ReportHistoryWeekly, ReportHistoryWeekly.week)
monthly_history = history_select(ReportHistoryMonthly, ReportHistoryMonthly.month)
packages = load_packages(db, report_id)
# creates a package_counts list with this structure:
# [(package name, count, [(package version, count in the version)])]
names = defaultdict(lambda: {"count": 0, "versions": defaultdict(int)})
for pkg in packages:
names[pkg.iname]["name"] = pkg.iname
names[pkg.iname]["count"] += pkg.count
names[pkg.iname]["versions"]["{0}:{1}-{2}"
.format(pkg.iepoch, pkg.iversion, pkg.irelease)] += pkg.count
package_counts = []
for pkg in sorted(names.values(), key=itemgetter("count"), reverse=True):
package_counts.append((
pkg["name"],
pkg["count"],
sorted(pkg["versions"].items(), key=itemgetter(1), reverse=True)))
try:
backtrace = report.backtraces[0].frames
except:
backtrace = []
fid = 0
for frame in backtrace:
fid += 1
frame.nice_order = fid
is_maintainer = is_component_maintainer(db, g.user, component)
contact_emails = []
if is_maintainer:
contact_emails = [email_address for (email_address, ) in
(db.session.query(ContactEmail.email_address)
.join(ReportContactEmail)
.filter(ReportContactEmail.report == report))]
forward = dict(report=report,
component=component,
releases=metric(releases),
arches=metric(arches),
modes=metric(modes),
daily_history=daily_history,
weekly_history=weekly_history,
monthly_history=monthly_history,
crashed_packages=packages,
package_counts=package_counts,
backtrace=backtrace,
contact_emails=contact_emails)
if request_wants_json():
return jsonify(forward)
forward["is_maintainer"] = is_maintainer
forward["extfafs"] = get_external_faf_instances(db)
return render_template("reports/item.html", **forward)
testPred = []
num_batch = math.ceil(num_test / args.batch_size)
start_test = time.time()
for batch_idx in range(num_batch):
start_idx = batch_idx * args.batch_size
end_idx = min(num_test, (batch_idx + 1) * args.batch_size)
feed_dict = {
X: testX[start_idx:end_idx],
TE: testTE[start_idx:end_idx],
is_training: False
}
pred_batch = sess.run(pred, feed_dict=feed_dict)
testPred.append(pred_batch)
end_test = time.time()
testPred = np.concatenate(testPred, axis=0)
train_mae, train_rmse, train_mape = utils.metric(trainPred, trainY)
val_mae, val_rmse, val_mape = utils.metric(valPred, valY)
test_mae, test_rmse, test_mape = utils.metric(testPred, testY)
utils.log_string(log, 'testing time: %.1fs' % (end_test - start_test))
utils.log_string(log, ' MAE\t\tRMSE\t\tMAPE')
utils.log_string(
log, 'train %.2f\t\t%.2f\t\t%.2f%%' %
(train_mae, train_rmse, train_mape * 100))
utils.log_string(
log, 'val %.2f\t\t%.2f\t\t%.2f%%' %
(val_mae, val_rmse, val_mape * 100))
utils.log_string(
log, 'test %.2f\t\t%.2f\t\t%.2f%%' %
(test_mae, test_rmse, test_mape * 100))
utils.log_string(log, 'performance in each prediction step')
MAE, RMSE, MAPE = [], [], []
def item(problem_id):
problem = db.session.query(Problem).filter(
Problem.id == problem_id).first()
if problem is None:
raise abort(404)
report_ids = [report.id for report in problem.reports]
sub = (db.session.query(ReportOpSysRelease.opsysrelease_id,
func.sum(ReportOpSysRelease.count).label("cnt"))
.join(Report)
.filter(Report.id.in_(report_ids))
.group_by(ReportOpSysRelease.opsysrelease_id)
.subquery())
osreleases = (db.session.query(OpSysRelease, sub.c.cnt)
.join(sub)
.order_by(desc("cnt"))
.all())
sub = (db.session.query(ReportArch.arch_id,
func.sum(ReportArch.count).label("cnt"))
.join(Report)
.filter(Report.id.in_(report_ids))
.group_by(ReportArch.arch_id)
.subquery())
arches = (db.session.query(Arch, sub.c.cnt).join(sub)
.order_by(desc("cnt"))
.all())
exes = (db.session.query(ReportExecutable.path,
func.sum(ReportExecutable.count).label("cnt"))
.join(Report)
.filter(Report.id.in_(report_ids))
.group_by(ReportExecutable.path)
.order_by(desc("cnt"))
.all())
sub = (db.session.query(ReportPackage.installed_package_id,
func.sum(ReportPackage.count).label("cnt"))
.join(Report)
.filter(Report.id.in_(report_ids))
.group_by(ReportPackage.installed_package_id)
.subquery())
packages_known = db.session.query(Package, sub.c.cnt).join(sub).all()
packages_unknown = (db.session.query(ReportUnknownPackage,
ReportUnknownPackage.count)
.join(Report)
.filter(Report.id.in_(report_ids))).all()
packages = packages_known + packages_unknown
# creates a package_counts list with this structure:
# [(package name, count, [(package version, count in the version)])]
names = defaultdict(lambda: {"count": 0, "versions": defaultdict(int)})
for (pkg, cnt) in packages:
names[pkg.name]["name"] = pkg.name
names[pkg.name]["count"] += cnt
names[pkg.name]["versions"][pkg.evr()] += cnt
package_counts = []
for pkg in sorted(names.values(), key=itemgetter("count"), reverse=True):
package_counts.append((
pkg["name"],
pkg["count"],
sorted(pkg["versions"].items(), key=itemgetter(1), reverse=True)))
for report in problem.reports:
for backtrace in report.backtraces:
fid = 0
for frame in backtrace.frames:
fid += 1
frame.nice_order = fid
bt_hashes = (db.session.query(ReportHash.hash)
.join(Report)
.join(Problem)
.filter(Problem.id == problem_id)
.distinct(ReportHash.hash).all())
# Limit to 10 bt_hashes (otherwise the URL can get too long)
# Select the 10 hashes uniformly from the entire list to make sure it is a
# good representation. (Slicing the 10 first could mean the 10 oldest
# are selected which is not a good representation.)
k = min(len(bt_hashes), 10)
a = 0
d = len(bt_hashes)/float(k)
bt_hashes_limited = []
for i in range(k):
bt_hashes_limited.append("bth=" + bt_hashes[int(a)][0])
a += d
bt_hash_qs = "&".join(bt_hashes_limited)
forward = {"problem": problem,
"osreleases": metric(osreleases),
"arches": metric(arches),
"exes": metric(exes),
"package_counts": package_counts,
"bt_hash_qs": bt_hash_qs
}
if request_wants_json():
return jsonify(forward)
is_maintainer = is_problem_maintainer(db, g.user, problem)
forward["is_maintainer"] = is_maintainer
forward["extfafs"] = get_external_faf_instances(db)
if report_ids:
bt_diff_form = BacktraceDiffForm()
bt_diff_form.lhs.choices = [(id, id) for id in report_ids]
bt_diff_form.rhs.choices = bt_diff_form.lhs.choices
forward['bt_diff_form'] = bt_diff_form
return render_template("problems/item.html", **forward)
def train(config, bert_config, train_path, dev_path, rel2id, id2rel,
tokenizer):
if os.path.exists(config.output_dir) is False:
os.makedirs(config.output_dir, exist_ok=True)
if os.path.exists('./data/train_file.pkl'):
train_data = pickle.load(open("./data/train_file.pkl", mode='rb'))
else:
train_data = data.load_data(train_path, tokenizer, rel2id, num_rels)
pickle.dump(train_data, open("./data/train_file.pkl", mode='wb'))
dev_data = json.load(open(dev_path))
for sent in dev_data:
data.to_tuple(sent)
data_manager = data.SPO(train_data)
train_sampler = RandomSampler(data_manager)
train_data_loader = DataLoader(data_manager,
sampler=train_sampler,
batch_size=config.batch_size,
drop_last=True)
num_train_steps = int(
len(data_manager) / config.batch_size) * config.max_epoch
if config.bert_pretrained_model is not None:
logger.info('load bert weight')
Bert_model = BertModel.from_pretrained(config.bert_pretrained_model,
config=bert_config)
else:
logger.info('random initialize bert model')
Bert_model = BertModel(config=bert_config).init_weights()
Bert_model.to(device)
submodel = sub_model(config).to(device)
objmodel = obj_model(config).to(device)
loss_fuc = nn.BCELoss(reduction='none')
params = list(Bert_model.parameters()) + list(
submodel.parameters()) + list(objmodel.parameters())
optimizer = AdamW(params, lr=config.lr)
# Train!
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(data_manager))
logger.info(" Num Epochs = %d", config.max_epoch)
logger.info(" Total train batch size = %d", config.batch_size)
logger.info(" Total optimization steps = %d", num_train_steps)
logger.info(" Logging steps = %d", config.print_freq)
logger.info(" Save steps = %d", config.save_freq)
global_step = 0
Bert_model.train()
submodel.train()
objmodel.train()
for _ in range(config.max_epoch):
optimizer.zero_grad()
epoch_itorator = tqdm(train_data_loader, disable=None)
for step, batch in enumerate(epoch_itorator):
batch = tuple(t.to(device) for t in batch)
input_ids, segment_ids, input_masks, sub_positions, sub_heads, sub_tails, obj_heads, obj_tails = batch
bert_output = Bert_model(input_ids, input_masks, segment_ids)[0]
pred_sub_heads, pred_sub_tails = submodel(
bert_output) # [batch_size, seq_len, 1]
pred_obj_heads, pred_obj_tails = objmodel(bert_output,
sub_positions)
# 计算loss
mask = input_masks.view(-1)
# loss1
sub_heads = sub_heads.unsqueeze(-1) # [batch_szie, seq_len, 1]
sub_tails = sub_tails.unsqueeze(-1)
loss1_head = loss_fuc(pred_sub_heads, sub_heads).view(-1)
loss1_head = torch.sum(loss1_head * mask) / torch.sum(mask)
loss1_tail = loss_fuc(pred_sub_tails, sub_tails).view(-1)
loss1_tail = torch.sum(loss1_tail * mask) / torch.sum(mask)
loss1 = loss1_head + loss1_tail
# loss2
loss2_head = loss_fuc(pred_obj_heads,
obj_heads).view(-1, obj_heads.shape[-1])
loss2_head = torch.sum(
loss2_head * mask.unsqueeze(-1)) / torch.sum(mask)
loss2_tail = loss_fuc(pred_obj_tails,
obj_tails).view(-1, obj_tails.shape[-1])
loss2_tail = torch.sum(
loss2_tail * mask.unsqueeze(-1)) / torch.sum(mask)
loss2 = loss2_head + loss2_tail
# optimize
loss = loss1 + loss2
loss.backward()
optimizer.step()
optimizer.zero_grad()
global_step += 1
if (global_step + 1) % config.print_freq == 0:
logger.info(
"epoch : {} step: {} #### loss1: {} loss2: {}".format(
_, global_step + 1,
loss1.cpu().item(),
loss2.cpu().item()))
if (global_step + 1) % config.eval_freq == 0:
logger.info("***** Running evaluating *****")
with torch.no_grad():
Bert_model.eval()
submodel.eval()
objmodel.eval()
P, R, F1 = utils.metric(Bert_model, submodel, objmodel,
dev_data, id2rel, tokenizer)
logger.info(f'precision:{P}\nrecall:{R}\nF1:{F1}')
Bert_model.train()
submodel.train()
objmodel.train()
if (global_step + 1) % config.save_freq == 0:
# Save a trained model
model_name = "pytorch_model_%d" % (global_step + 1)
output_model_file = os.path.join(config.output_dir, model_name)
state = {
'bert_state_dict': Bert_model.state_dict(),
'subject_state_dict': submodel.state_dict(),
'object_state_dict': objmodel.state_dict(),
}
torch.save(state, output_model_file)
model_name = "pytorch_model_last"
output_model_file = os.path.join(config.output_dir, model_name)
state = {
'bert_state_dict': Bert_model.state_dict(),
'subject_state_dict': submodel.state_dict(),
'object_state_dict': objmodel.state_dict(),
}
torch.save(state, output_model_file)
def item(report_id):
result = (db.session.query(Report,
OpSysComponent).join(OpSysComponent).filter(
Report.id == report_id).first())
if result is None:
abort(404)
report, component = result
releases = (db.session.query(
ReportOpSysRelease, ReportOpSysRelease.count).filter(
ReportOpSysRelease.report_id == report_id).order_by(
desc(ReportOpSysRelease.count)).all())
arches = (db.session.query(
ReportArch,
ReportArch.count).filter(ReportArch.report_id == report_id).order_by(
desc(ReportArch.count)).all())
modes = (db.session.query(
ReportSelinuxMode, ReportSelinuxMode.count).filter(
ReportSelinuxMode.report_id == report_id).order_by(
desc(ReportSelinuxMode.count)).all())
history_select = lambda table: (db.session.query(table).filter(
table.report_id == report_id).all())
daily_history = history_select(ReportHistoryDaily)
weekly_history = history_select(ReportHistoryWeekly)
monthly_history = history_select(ReportHistoryMonthly)
packages = load_packages(db, report_id)
# creates a package_counts list with this structure:
# [(package name, count, [(package version, count in the version)])]
names = defaultdict(lambda: {"count": 0, "versions": defaultdict(int)})
for pkg in packages:
names[pkg.iname]["name"] = pkg.iname
names[pkg.iname]["count"] += pkg.count
names[pkg.iname]["versions"]["{0}:{1}-{2}".format(
pkg.iepoch, pkg.iversion, pkg.irelease)] += pkg.count
package_counts = []
for pkg in sorted(names.values(), key=itemgetter("count"), reverse=True):
package_counts.append((pkg["name"], pkg["count"],
sorted(pkg["versions"].items(),
key=itemgetter(1),
reverse=True)))
try:
backtrace = report.backtraces[0].frames
except:
backtrace = []
fid = 0
for frame in backtrace:
fid += 1
frame.nice_order = fid
contact_emails = []
is_maintainer = app.config["EVERYONE_IS_MAINTAINER"]
if not is_maintainer and g.user is not None:
if user_is_maintainer(db, g.user.username, component.id):
is_maintainer = True
if is_maintainer:
contact_emails = [
email_address for (email_address, ) in (db.session.query(
ContactEmail.email_address).join(ReportContactEmail).filter(
ReportContactEmail.report == report))
]
forward = dict(report=report,
component=component,
releases=metric(releases),
arches=metric(arches),
modes=metric(modes),
daily_history=daily_history,
weekly_history=weekly_history,
monthly_history=monthly_history,
crashed_packages=packages,
package_counts=package_counts,
backtrace=backtrace,
contact_emails=contact_emails)
if request_wants_json():
return jsonify(forward)
forward["is_maintainer"] = is_maintainer
forward["extfafs"] = get_external_faf_instances(db)
return render_template("reports/item.html", **forward)
