def run_eval(data_dir, checkpoints_dir, log_dir):
# load and update labels
df = pd.read_csv(data_dir / "train.csv")
df = df[["file_name", "category_id"]]
classes_map = utils.load_json(data_dir / "classes_map.json")
# Update class labels
new_labels = [classes_map[str(i)] for i in df["category_id"]]
df["new_labels"] = new_labels
ds = datatset.HerbDataSet(df, data_dir, 256, label="new_labels")
loader = DataLoader(ds, batch_size=16, shuffle=True)
# Metrics
acc = metrics.Accuracy()
f1 = metrics.F1()
writer = SummaryWriter(log_dir / "validation")
manager = metrics.MetricManager([acc, f1], writer)
# Loss
criterion = nn.CrossEntropyLoss()
# Model
m = model.get_model(models.resnet50(pretrained=False),
p1=0,
p2=0,
device="gpu") # no drop out, trying to over fit
Eval(manager, criterion, loader, m, checkpoints_dir).run()
def input():
contracts = ''
psc_codes = db.session.query(consolidated_data2.product_or_service_code_description).order_by(consolidated_data2.product_or_service_code_description).distinct()
set_asides = db.session.query(consolidated_data2.type_of_set_aside_code).order_by(consolidated_data2.type_of_set_aside_code).distinct()
# Precision, Recall, F1
# Return contracts if on post request
if request.method == "POST":
queryString = request.form.get('set_aside')
psc_filter = request.form['psc_code']
AccuracyScore = metrics.Accuracy(queryString)
PrecisionScore = metrics.Precision(queryString)
RecallScore = metrics.Recall(queryString)
F1Score = metrics.F1(queryString)
if psc_filter == "All":
contracts = consolidated_data2.query.filter_by(type_of_set_aside_code=queryString).order_by(consolidated_data2.base_and_exercised_options_value.desc()).limit(10)
else:
contracts = consolidated_data2.query.filter_by(type_of_set_aside_code=queryString, product_or_service_code_description=psc_filter).order_by(consolidated_data2.base_and_exercised_options_value.desc()).limit(10)
# Log the set aside, psc code, and the query to the server.
print(queryString + " : " + psc_filter)
print("return value: "+ str(contracts))
return render_template("input.html", accuracy="90.27", psc_codes=psc_codes, contracts=contracts, set_asides=set_asides,
AccuracyScore=AccuracyScore, PrecisionScore=PrecisionScore, RecallScore=RecallScore,
F1Score=F1Score, set_aside=queryString, psc_filter=psc_filter)
# If not a post request, just load the input page.
else:
return render_template("input.html", psc_codes=psc_codes, contracts='', set_asides=set_asides,
AccuracyScore=0, PrecisionScore=0, RecallScore=0, F1Score=0, set_aside="None", psc_filter="All")
def default_classical_scorings(task="predict"):
if task == 'predict':
scorings = (mm.Accuracy(tensor=False),
mm.BalancedAccuracy(tensor=False),
mm.F1Score(average='macro', tensor=False),
mm.Precision(average='macro', tensor=False),
mm.Recall(average='macro', tensor=False),
mm.ROCAUC(average='macro', tensor=False))
else:
scorings = (mm.CIndex(tensor=False, hazard=True), )
return scorings
def run_eval(data, checkpoints_dir, log_dir):
# Data
images_path = data / "images"
labels_path = data / "labels"
ds = dataset.HeartDataSet(images_path, labels_path)
loader = DataLoader(ds, batch_size=8, shuffle=True)
# Metrics
acc = metrics.Accuracy()
miou = metrics.Miou()
writer = SummaryWriter(log_dir / "validation")
manager = metrics.MetricManager([acc, miou], writer)
# Loss
criterion = nn.CrossEntropyLoss()
# criterion = nn.CrossEntropyLoss(weight=torch.tensor([1e-3,1,1,1], device="cuda"))
# Model
unet = model.ResNetUNet(4)
Eval(manager, criterion, loader, unet, checkpoints_dir).run()
def main(args):
""" Main benchmark routine. Parses the arguments, loads models and data,
runs the evaluation loop and produces the output.
"""
# Compose the model list
modellist = []
if args['model']:
modellist.append(args['model'])
benchmark = None
benchmark = load_benchmark(args['benchmark'])
modellist.extend(benchmark['models'])
# Run the model evaluation
is_silent = (args['output'] == 'html')
ev = evaluator.Evaluator(
modellist,
benchmark['data.test'],
benchmark['data.train'],
silent=is_silent)
with silence_stdout(is_silent):
res_df = ev.evaluate()
# Run the metric visualizer
html_viz = metrics.HTMLVisualizer([
metrics.Accuracy(),
metrics.SubjectBoxes()
])
html = html_viz.to_html(res_df)
if args['output'] == 'browser':
metrics.load_in_default_browser('\n'.join(html).encode('utf8'))
elif args['output'] == 'html':
print(' '.join(html))
def train(dataset, model_dir, writer):
dataloaders = dataset.get_dataloaders()
# we now set GPU training parameters
# if the given index is not available then we use index 0
# also when using multi gpu we should specify index 0
if dataset.config.gpu_index + 1 > torch.cuda.device_count(
) or dataset.config.multi_gpu:
dataset.config.gpu_index = 0
logging.info('Using GPU cuda:{}, script PID {}'.format(
dataset.config.gpu_index, os.getpid()))
if dataset.config.multi_gpu:
logging.info('Training on multi-GPU mode with {} devices'.format(
torch.cuda.device_count()))
device = torch.device('cuda:{}'.format(dataset.config.gpu_index))
if dataset.config.model == 'unet':
model = unet(input_size=dataset.config.num_feats,
num_classes=dataset.config.num_classes,
kernel_size=dataset.config.kernel_size).to(device)
else:
model = deeplab(backbone=dataset.config.backbone,
input_size=dataset.config.num_feats,
num_classes=dataset.config.num_classes,
kernel_size=dataset.config.kernel_size,
sigma=dataset.config.sigma).to(device)
# if use multi_gou then convert the model to DataParallel
if dataset.config.multi_gpu:
model = nn.DataParallel(model)
# create optimizer, loss function, and lr scheduler
optimizer = torch.optim.Adam(model.parameters(),
lr=dataset.config.lr,
weight_decay=1e-4)
criterion = nn.CrossEntropyLoss()
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer,
mode='min',
factor=dataset.config.lr_decay,
patience=dataset.config.lr_patience,
verbose=True)
logging.info('Config {}'.format(dataset.config))
logging.info(
'TB logs and checkpoint will be saved in {}'.format(model_dir))
phases = ['train', 'test']
# create metric trackers: we track lass, class accuracy, and overall accuracy
trackers = {
x: {
'loss':
metrics.LossMean(),
'acc':
metrics.Accuracy(),
'iou':
None,
'cm':
metrics.ConfusionMatrix(
num_classes=int(dataset.config.num_classes))
}
for x in phases
}
# create initial best state object
best_state = {
'model': model.state_dict(),
'optimizer': optimizer.state_dict(),
'scheduler': scheduler.state_dict() if scheduler else None,
'train_loss': float('inf'),
'test_loss': float('inf'),
'train_acc': 0.0,
'test_acc': 0.0,
'train_mIoU': 0.0,
'test_mIoU': 0.0,
'convergence_epoch': 0,
'num_epochs_since_best_acc': 0
}
# now we train!
for epoch in range(dataset.config.max_epochs):
for phase in phases:
if phase == 'train':
model.train()
else:
model.eval()
# reset metrics
trackers[phase]['loss'].reset()
trackers[phase]['cm'].reset()
for step_number, inputs in enumerate(
tqdm(dataloaders[phase],
desc='[{}/{}] {} '.format(epoch + 1,
dataset.config.max_epochs,
phase))):
data = inputs[0].to(device, dtype=torch.float).permute(0, 2, 1)
coords = inputs[1].to(device,
dtype=torch.float).permute(0, 2, 1)
label = inputs[2].to(device, dtype=torch.long)
# compute gradients on train only
with torch.set_grad_enabled(phase == 'train'):
out = model(data, coords)
loss = criterion(out, label)
if phase == 'train':
optimizer.zero_grad()
loss.backward()
optimizer.step()
# now we update metrics
trackers[phase]['loss'].update(average_loss=loss,
batch_size=data.size(0))
trackers[phase]['cm'].update(y_true=label, y_logits=out)
logging.info('Computing accuracy...')
# compare with my metrics
epoch_loss = trackers[phase]['loss'].result()
epoch_overall_acc = trackers[phase]['cm'].result(metric='accuracy')
epoch_iou = trackers[phase]['cm'].result(metric='iou')
epoch_miou = epoch_iou.mean()
logging.info(
'--------------------------------------------------------------------------------'
)
logging.info(
'[{}/{}] {} Loss: {:.2e}. Overall Acc: {:.4f}. mIoU {:.4f}'.
format(epoch + 1, dataset.config.max_epochs, phase, epoch_loss,
epoch_overall_acc, epoch_miou))
iou_per_class_str = ' '.join(
['{:.4f}'.format(s) for s in epoch_iou])
logging.info('IoU per class: {}'.format(iou_per_class_str))
logging.info(
'--------------------------------------------------------------------------------'
)
# we update our learning rate scheduler if loss does not improve
if phase == 'test' and scheduler:
scheduler.step(epoch_loss)
writer.add_scalar('params/lr', optimizer.param_groups[0]['lr'],
epoch + 1)
writer.add_scalar('loss/epoch_{}'.format(phase), epoch_loss,
epoch + 1)
writer.add_scalar('miou/epoch_{}'.format(phase), epoch_miou,
epoch + 1)
writer.add_scalar('acc_all/epoch_{}'.format(phase),
epoch_overall_acc, epoch + 1)
# after each epoch we update best state values as needed
# first we save our state when we get better test accuracy
if best_state['test_mIoU'] > trackers['test']['cm'].result(
metric='iou').mean():
best_state['num_epochs_since_best_acc'] += 1
else:
logging.info('Got a new best model with mIoU {:.4f}'.format(
trackers['test']['cm'].result(metric='iou').mean()))
best_state = {
'model': model.state_dict(),
'optimizer': optimizer.state_dict(),
'scheduler': scheduler.state_dict() if scheduler else None,
'train_loss': trackers['train']['loss'].result(),
'test_loss': trackers['test']['loss'].result(),
'train_acc': trackers['train']['cm'].result(metric='accuracy'),
'test_acc': trackers['test']['cm'].result(metric='accuracy'),
'train_mIoU':
trackers['train']['cm'].result(metric='iou').mean(),
'test_mIoU':
trackers['test']['cm'].result(metric='iou').mean(),
'convergence_epoch': epoch + 1,
'num_epochs_since_best_acc': 0
}
file_name = os.path.join(model_dir, 'best_state.pth')
torch.save(best_state, file_name)
logging.info('saved checkpoint in {}'.format(file_name))
# we check for early stopping when we have trained a min number of epochs
if epoch >= dataset.config.min_epochs and best_state[
'num_epochs_since_best_acc'] >= dataset.config.early_stopping:
logging.info('Accuracy did not improve for {} iterations!'.format(
dataset.config.early_stopping))
logging.info('[Early stopping]')
break
utl.dump_best_model_metrics_to_tensorboard(writer, phases, best_state)
logging.info('************************** DONE **************************')
def do_train():
device = args.device
set_seed(args.seed)
label_map = load_dict(args.tag_path)
id2label = {val: key for key, val in label_map.items()}
print("Loading the ernie model and tokenizer...")
model = ErnieForSequenceClassification(num_classes=len(label_map))
model.to(torch.device(device))
tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0")
print("============start train==========")
train_ds = DuEventExtraction(args.train_data, args.tag_path)
dev_ds = DuEventExtraction(args.dev_data, args.tag_path)
# test_ds = DuEventExtraction(args.test_data, args.tag_path)
trans_func = partial(convert_example,
tokenizer=tokenizer,
label_map=label_map,
max_seq_len=args.max_seq_len)
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.vocab[tokenizer.pad_token]),
Pad(axis=0, pad_val=tokenizer.vocab[tokenizer.pad_token]),
Stack(dtype="int64") # label
): fn(list(map(trans_func, samples)))
train_loader = DataLoader(dataset=train_ds,
batch_size=args.batch_size,
shuffle=True,
collate_fn=batchify_fn)
dev_loader = DataLoader(dataset=dev_ds,
batch_size=args.batch_size,
collate_fn=batchify_fn)
# test_loader = DataLoader(
# dataset=test_ds,
# batch_size=args.batch_size,
# collate_fn=batchify_fn)
num_training_steps = len(train_loader) * args.num_epoch
metric = metrics.Accuracy()
criterion = nn.CrossEntropyLoss()
optimizer = optim.AdamW(lr=args.learning_rate, params=model.parameters())
best_performerence = 0.0
model.train()
for epoch in range(args.num_epoch):
for step, (input_ids, token_type_ids,
labels) in enumerate(train_loader):
input_ids = to_var(input_ids, device=device).long()
token_type_ids = to_var(token_type_ids, device=device).long()
labels = to_var(labels, device=device).squeeze()
logits = model(input_ids, token_type_ids)
# print("labels shape: ", labels.shape, "logits shape: ", logits.shape)
loss = criterion(logits, labels)
probs = torch.softmax(logits, dim=1)
if device == "cuda":
probs = probs.detach().cpu()
labels = labels.detach().cpu()
# calculate accuracy
correct = metric.compute(probs, labels)
metric.update(correct)
acc = metric.accumulate()
optimizer.zero_grad()
loss.backward()
optimizer.step()
if device == "cuda":
loss_item = loss.detach().cpu().numpy().item()
else:
loss_item = loss.detach().numpy().item()
if step > 0 and step % args.skip_step == 0:
print(f'train epoch: {epoch} - step: {step} (total: {num_training_steps}) ' \
f'- loss: {loss_item:.6f} acc {acc:.5f}')
if step > 0 and step % args.valid_step == 0:
loss_dev, acc_dev = evaluate(model, criterion, metric,
dev_loader, device)
print(f'dev step: {step} - loss: {loss_dev:.6f} accuracy: {acc_dev:.5f}, ' \
f'current best {best_performerence:.5f}')
if acc_dev > best_performerence:
best_performerence = acc_dev
print(f'==============================================save best model ' \
f'best performerence {best_performerence:5f}')
if not os.path.exists(args.checkpoints):
os.mkdir(args.checkpoints)
torch.save(model.state_dict(),
'{}/best.pt'.format(args.checkpoints))
step += 1
# save the final model
torch.save(model.state_dict(), '{}/final.pt'.format(args.checkpoints))
def main():
warnings.filterwarnings('ignore')
# config
parser = argparse.ArgumentParser()
parser.add_argument('-s',
'--save',
default='./save',
help='保存的文件夹路径，如果有重名，会在其后加-来区别')
parser.add_argument('-is',
'--image_size',
default=224,
type=int,
help='patch会被resize到多大，默认时224 x 224')
parser.add_argument('-vts',
'--valid_test_size',
default=(0.1, 0.1),
type=float,
nargs=2,
help='训练集和测试集的大小，默认时0.1, 0.1')
parser.add_argument('-bs',
'--batch_size',
default=32,
type=int,
help='batch size，默认时32')
parser.add_argument('-nw',
'--num_workers',
default=12,
type=int,
help='多进程数目，默认时12')
parser.add_argument('-lr',
'--learning_rate',
default=0.0001,
type=float,
help='学习率大小，默认时0.0001')
parser.add_argument('-e',
'--epoch',
default=10,
type=int,
help='epoch 数量，默认是10')
parser.add_argument('--reduction',
default='mean',
help='聚合同一bag的instances时的聚合方式，默认时mean')
parser.add_argument('--multipler',
default=2.0,
type=float,
help="为了平衡pos和neg，在weight再乘以一个大于1的数，默认是2.0")
args = parser.parse_args()
save = args.save
image_size = (args.image_size, args.image_size)
valid_size, test_size = args.valid_test_size
batch_size = args.batch_size
num_workers = args.num_workers
lr = args.learning_rate
epoch = args.epoch
reduction = args.reduction
multipler = args.multipler
# ----- 读取数据 -----
neg_dir = './DATA/TCT/negative'
pos_dir = './DATA/TCT/positive'
dat = MilData.from2dir(neg_dir, pos_dir)
train_transfer = transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.RandomVerticalFlip(),
transforms.Resize(image_size),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
test_transfer = transforms.Compose([
transforms.Resize(image_size),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
train_dat, valid_dat, test_dat = dat.split_by_bag(
test_size,
valid_size,
train_transfer=train_transfer,
valid_transfer=test_transfer,
test_transfer=test_transfer)
dataloaders = {
'train':
data.DataLoader(train_dat,
batch_size=batch_size,
num_workers=num_workers,
shuffle=True),
'valid':
data.DataLoader(
valid_dat,
batch_size=batch_size,
num_workers=num_workers,
),
'test':
data.DataLoader(
test_dat,
batch_size=batch_size,
num_workers=num_workers,
)
}
# ----- 构建网络和优化器 -----
net = NormalCnn()
criterion = nn.BCELoss(reduction='none')
optimizer = optim.Adam(net.parameters(), lr=lr)
scorings = [
mm.Loss(),
mm.Recall(reduction=reduction),
mm.ROCAUC(reduction=reduction),
mm.BalancedAccuracy(reduction=reduction),
mm.F1Score(reduction=reduction),
mm.Precision(reduction=reduction),
mm.Accuracy(reduction=reduction)
]
# ----- 训练网络 -----
try:
net, hist, weighter = train(net,
criterion,
optimizer,
dataloaders,
epoch=epoch,
metrics=scorings,
weighter_multipler=multipler)
test_hist = evaluate(net, dataloaders['test'], criterion, scorings)
except Exception as e:
import ipdb
ipdb.set_trace() # XXX BREAKPOINT
# 保存结果
dirname = check_update_dirname(save)
torch.save(net.state_dict(), os.path.join(dirname, 'model.pth'))
torch.save(weighter, os.path.join(dirname, 'weigher.pth'))
pd.DataFrame(hist).to_csv(os.path.join(dirname, 'train.csv'))
with open(os.path.join(dirname, 'config.json'), 'w') as f:
json.dump(args.__dict__, f)
with open(os.path.join(dirname, 'test.json'), 'w') as f:
json.dump(test_hist, f)
def train(config, model_dir, writer):
"""
Function train and evaluate a part segmentation model for
the Shapenet dataset. The training parameters are specified
in the config file (for more details see config/config.py).
:param config: Dictionary with configuration paramters
:param model_dir: Checkpoint save directory
:param writer: Tensorboard SummaryWritter object
"""
phases = ['train', 'test']
datasets, dataloaders = ds.get_modelnet40_dataloaders(
root_dir=args.root_dir,
phases=phases,
batch_size=config['batch_size'],
augment=config['augment'])
# add number of classes to config
config['num_classes'] = 40
# we now set GPU training parameters
# if the given index is not available then we use index 0
# also when using multi gpu we should specify index 0
if config['gpu_index'] + 1 > torch.cuda.device_count(
) or config['multi_gpu']:
config['gpu_index'] = 0
logging.info('Using GPU cuda:{}, script PID {}'.format(
config['gpu_index'], os.getpid()))
if config['multi_gpu']:
logging.info('Training on multi-GPU mode with {} devices'.format(
torch.cuda.device_count()))
device = torch.device('cuda:{}'.format(config['gpu_index']))
# we load the model defined in the config file
model = res.resnet101(in_channels=config['in_channels'],
num_classes=config['num_classes'],
kernel_size=config['kernel_size']).to(device)
# if use multi_gpu then convert the model to DataParallel
if config['multi_gpu']:
model = nn.DataParallel(model)
# create optimizer, loss function, and lr scheduler
optimizer = torch.optim.Adam(model.parameters(),
lr=config['lr'],
weight_decay=1e-4)
criterion = nn.CrossEntropyLoss()
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer,
mode='min',
factor=config['lr_decay'],
patience=config['lr_patience'],
verbose=True)
logging.info('Config {}'.format(config))
logging.info(
'TB logs and checkpoint will be saved in {}'.format(model_dir))
utl.dump_config_details_to_tensorboard(writer, config)
# create metric trackers: we track lass, class accuracy, and overall accuracy
trackers = {
x: {
'loss': metrics.LossMean(),
'acc': metrics.Accuracy(),
'iou': None,
'cm':
metrics.ConfusionMatrix(num_classes=int(config['num_classes']))
}
for x in phases
}
# create initial best state object
best_state = {
'config': config,
'model': model.state_dict(),
'optimizer': optimizer.state_dict(),
'scheduler': scheduler.state_dict() if scheduler else None,
'train_loss': float('inf'),
'test_loss': float('inf'),
'train_acc': 0.0,
'test_acc': 0.0,
'train_class_acc': 0.0,
'test_class_acc': 0.0,
'convergence_epoch': 0,
'num_epochs_since_best_acc': 0
}
# now we train!
for epoch in range(config['max_epochs']):
for phase in phases:
if phase == 'train':
model.train()
else:
model.eval()
# reset metrics
trackers[phase]['loss'].reset()
trackers[phase]['cm'].reset()
for step_number, (data, label) in enumerate(
tqdm(dataloaders[phase],
desc='[{}/{}] {} '.format(epoch + 1,
config['max_epochs'],
phase))):
data = data.to(device, dtype=torch.float).permute(0, 2, 1)
label = label.to(device, dtype=torch.long).squeeze()
# compute gradients on train only
with torch.set_grad_enabled(phase == 'train'):
out = model(data)
loss = criterion(out, label)
if phase == 'train':
optimizer.zero_grad()
loss.backward()
optimizer.step()
# now we update metrics
trackers[phase]['loss'].update(average_loss=loss,
batch_size=data.size(0))
trackers[phase]['cm'].update(y_true=label, y_logits=out)
# logging.info('Computing accuracy...')
# compare with my metrics
epoch_loss = trackers[phase]['loss'].result()
epoch_overall_acc = trackers[phase]['cm'].result(metric='accuracy')
epoch_class_acc = trackers[phase]['cm'].result(
metric='class_accuracy').mean()
# we update our learning rate scheduler if loss does not improve
if phase == 'test' and scheduler:
scheduler.step(epoch_loss)
writer.add_scalar('params/lr', optimizer.param_groups[0]['lr'],
epoch + 1)
# log current results and dump in Tensorboard
logging.info(
'[{}/{}] {} Loss: {:.2e}. Overall Acc: {:.4f}. Class Acc {:.4f}'
.format(epoch + 1, config['max_epochs'], phase, epoch_loss,
epoch_overall_acc, epoch_class_acc))
writer.add_scalar('loss/epoch_{}'.format(phase), epoch_loss,
epoch + 1)
writer.add_scalar('acc_class/epoch_{}'.format(phase),
epoch_class_acc, epoch + 1)
writer.add_scalar('acc_all/epoch_{}'.format(phase),
epoch_overall_acc, epoch + 1)
# after each epoch we update best state values as needed
# first we save our state when we get better test accuracy
if best_state['test_acc'] > trackers['test']['cm'].result(
metric='accuracy'):
best_state['num_epochs_since_best_acc'] += 1
else:
logging.info('Got a new best model with accuracy {:.4f}'.format(
trackers['test']['cm'].result(metric='accuracy')))
best_state = {
'config':
config,
'model':
model.state_dict(),
'optimizer':
optimizer.state_dict(),
'scheduler':
scheduler.state_dict() if scheduler else None,
'train_loss':
trackers['train']['loss'].result(),
'test_loss':
trackers['test']['loss'].result(),
'train_acc':
trackers['train']['cm'].result(metric='accuracy'),
'test_acc':
trackers['test']['cm'].result(metric='accuracy'),
'train_class_acc':
trackers['train']['cm'].result(metric='class_accuracy').mean(),
'test_class_acc':
trackers['test']['cm'].result(metric='class_accuracy').mean(),
'convergence_epoch':
epoch + 1,
'num_epochs_since_best_acc':
0
}
file_name = os.path.join(model_dir, 'best_state.pth')
torch.save(best_state, file_name)
logging.info('saved checkpoint in {}'.format(file_name))
# we check for early stopping when we have trained a min number of epochs
if epoch >= config['min_epochs'] and best_state[
'num_epochs_since_best_acc'] >= config['early_stopping']:
logging.info('Accuracy did not improve for {} iterations!'.format(
config['early_stopping']))
logging.info('[Early stopping]')
break
utl.dump_best_model_metrics_to_tensorboard(writer, phases, best_state)
logging.info('************************** DONE **************************')
def auc(modelpath, idxlist, datasetpath, for_all=False):
model = torch.load(modelpath).cuda().eval()
dataset = dataset_class.Loading_dataset(idxlist, datasetpath, False, False)
my_dataset_loader = torch.utils.data.DataLoader(dataset=dataset,
batch_size=1,
shuffle=False,
num_workers=1)
eth = 0.5
print("testing..")
precisions = [0.0]
recalls = [1.0]
while (eth < 0.95):
idfix = 0
ious = 0
kappas = 0
accs = 0
tps = 0
fps = 0
precs = 0
recs = 0
for sample in my_dataset_loader:
pts_tensor = sample['pts'].float().permute(0, 3, 1, 2)
normals_tensor = sample['normals'].float().permute(0, 3, 1, 2)
colors_tensor = sample['colors'].float().permute(0, 3, 1, 2)
target_tensor = sample['gt']
input_tensor = torch.cat([pts_tensor, normals_tensor], 1).cuda()
# input_tensor = torch.cat([pts_tensor,normals_tensor,colors_tensor],1).cuda()
# input_tensor = colors_tensor.cuda()
# input_tensor = pts_tensor.cuda()
target_tensor = target_tensor.to(device="cuda", dtype=torch.long)
with torch.no_grad():
out = model(input_tensor)
softmax_mx = out.detach().permute(0, 2, 3, 1).cpu().numpy()[0]
softmax_mx[softmax_mx[:, :, 1] < eth] = 0.0
softmax_mx[softmax_mx[:, :, 1] != 0.0] = 1.0
pts = pts_tensor.detach().permute(0, 2, 3, 1).numpy()[0]
pts2 = pts.reshape((pts.shape[0] * pts.shape[1], 3))
idxs = np.any(pts2 != [0.0, 0.0, 0.0], axis=-1)
pred = softmax_mx[:, :, 1]
gta = target_tensor.cpu().numpy()[0]
pred = pred.reshape((pred.shape[0] * pred.shape[1]))
gta = gta.reshape((gta.shape[0] * gta.shape[1]))
pred = pred[idxs]
gta = gta[idxs]
union = np.logical_or(pred, gta)
intersection = np.logical_and(pred, gta)
iou = intersection.sum() / union.sum()
ious = ious + iou
kappa = metrics.Kappa_cohen(pred, gta)
acc = metrics.Accuracy(pred, gta)
something = metrics.precision_recall(pred, gta)
tps = something[2] / (something[2] + something[5]) + tps
fps = something[3] / (something[4] + something[3]) + fps
precs = precs + something[0]
recs = recs + something[1]
kappas = kappas + kappa
accs = accs + acc
idfix = idfix + 1
precisions.append(precs / idfix)
recalls.append(recs / idfix)
print(accs / idfix, kappas / idfix, ious / idfix)
eth = eth + 0.05
precisions.append(1.0)
recalls.append(0.0)
precisions = np.array(precisions)
recalls = np.array(recalls)
print(precisions, recalls)
print(m.auc(recalls, precisions))
print("TPS", tps / idfix)
print("FPS", fps / idfix)
print("MIoU", ious / idfix)
print("kappas", kappas / idfix)
print("accuracy", accs / idfix)
print("precision", precs / idfix)
print("recall", recs / idfix)
def train(model,
criterion,
optimizer,
dataloaders,
scheduler=NoneScheduler(None),
epoch=100,
device=torch.device('cuda:0'),
l2=0.0,
metrics=(mm.Loss(), mm.Accuracy()),
standard_metric_index=1,
clip_grad=False):
best_model_wts = copy.deepcopy(model.state_dict())
best_metric = 0.0
best_metric_name = metrics[standard_metric_index].__class__.__name__ + \
'_valid'
history = {
m.__class__.__name__ + p: []
for p in ['_train', '_valid'] for m in metrics
}
for e in range(epoch):
for phase in ['train', 'valid']:
if phase == 'train':
scheduler.step()
model.train()
iterator = pb.progressbar(dataloaders[phase], prefix='Train: ')
else:
model.eval()
iterator = dataloaders[phase]
for m in metrics:
m.reset()
for batch_x, batch_y in iterator:
batch_x = batch_x.to(device)
batch_y = batch_y.to(device)
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
logit = model(batch_x)
loss = criterion(logit, batch_y)
# 只给weight加l2正则化
if l2 > 0.0:
for p_n, p_v in model.named_parameters():
if p_n == 'weight':
loss += l2 * p_v.norm()
if phase == 'train':
loss.backward()
if clip_grad:
nn.utils.clip_grad_norm_(model.parameters(),
max_norm=1)
optimizer.step()
with torch.no_grad():
for m in metrics:
if isinstance(m, mm.Loss):
m.add(loss.cpu().item(), batch_x.size(0))
else:
m.add(logit.squeeze(), batch_y)
for m in metrics:
history[m.__class__.__name__ + '_' + phase].append(m.value())
print("Epoch: %d, Phase:%s, " % (e, phase) + ", ".join([
'%s: %.4f' % (m.__class__.__name__,
history[m.__class__.__name__ + '_' + phase][-1])
for m in metrics
]))
if phase == 'valid':
epoch_metric = history[best_metric_name][-1]
if epoch_metric > best_metric:
best_metric = epoch_metric
best_model_wts = copy.deepcopy(model.state_dict())
print("Best metric: %.4f" % best_metric)
model.load_state_dict(best_model_wts)
return model, history
def main():
# ----- 根据data来读取不同的数据和不同的loss、metrics -----
if config.args.data == 'brca':
rna = RnaData.predicted_data(config.brca_cli, config.brca_rna,
{'PAM50Call_RNAseq': 'pam50'})
rna.transform(tf.LabelMapper(config.brca_label_mapper))
out_shape = len(config.brca_label_mapper)
criterion = nn.CrossEntropyLoss()
scorings = (mm.Loss(), mm.Accuracy(), mm.BalancedAccuracy(),
mm.F1Score(average='macro'), mm.Precision(average='macro'),
mm.Recall(average='macro'), mm.ROCAUC(average='macro'))
elif config.args.data == 'survival':
if os.path.exists('./DATA/temp_pan.pth'):
rna = RnaData.load('./DATA/temp_pan.pth')
else:
rna = RnaData.survival_data(config.pan_cli, config.pan_rna,
'_OS_IND', '_OS')
out_shape = 1
if config.args.loss_type == 'cox':
criterion = NegativeLogLikelihood()
elif config.args.loss_type == 'svm':
criterion = SvmLoss(rank_ratio=config.args.svm_rankratio)
scorings = (mm.Loss(), mm.CIndex())
rna.transform(tf.ZeroFilterCol(0.8))
rna.transform(tf.MeanFilterCol(1))
rna.transform(tf.StdFilterCol(0.5))
norm = tf.Normalization()
rna.transform(norm)
# ----- 构建网络和优化器 -----
inpt_shape = rna.X.shape[1]
if config.args.net_type == 'mlp':
net = MLP(inpt_shape, out_shape, config.args.hidden_num,
config.args.block_num).cuda()
elif config.args.net_type == 'atten':
net = SelfAttentionNet(inpt_shape, out_shape, config.args.hidden_num,
config.args.bottle_num, config.args.block_num,
config.args.no_res, config.act,
config.args.no_head, config.args.no_bottle,
config.args.no_atten,
config.args.dropout_rate).cuda()
elif config.args.net_type == 'resnet':
net = ResidualNet(inpt_shape, out_shape, config.args.hidden_num,
config.args.bottle_num,
config.args.block_num).cuda()
# ----- 训练网络，cross validation -----
split_iterator = rna.split_cv(config.args.test_size,
config.args.cross_valid)
train_hists = []
test_hists = []
for split_index, (train_rna, test_rna) in enumerate(split_iterator):
print('##### save: %s, split: %d #####' %
(config.args.save, split_index))
# 从train中再分出一部分用作验证集，决定停止
train_rna, valid_rna = train_rna.split(0.1)
dats = {
'train': train_rna.to_torchdat(),
'valid': valid_rna.to_torchdat(),
}
dataloaders = {
k: data.DataLoader(v, batch_size=config.args.batch_size)
for k, v in dats.items()
}
test_dataloader = data.DataLoader(test_rna.to_torchdat(),
batch_size=config.args.batch_size)
# 网络训练前都进行一次参数重置，避免之前的训练的影响
net.reset_parameters()
# train
optimizer = optim.Adamax(net.parameters(),
lr=config.args.learning_rate)
lrs = config.lrs(optimizer)
net, hist = train(
net,
criterion,
optimizer,
dataloaders,
epoch=config.args.epoch,
metrics=scorings,
l2=config.args.l2,
standard_metric_index=config.args.standard_metric_index,
scheduler=lrs)
# test
test_res = evaluate(net, criterion, test_dataloader, metrics=scorings)
# 将多次训练的结果保存到一个df中
hist = pd.DataFrame(hist)
hist['split_index'] = split_index
train_hists.append(hist)
# 保存多次test的结果
test_res['split_index'] = split_index
test_hists.append(test_res)
# 每个split训练的模型保存为一个文件
torch.save(net.state_dict(),
os.path.join(config.save_dir, 'model%d.pth' % split_index))
# 保存train的结果
train_hists = pd.concat(train_hists)
train_hists.to_csv(os.path.join(config.save_dir, 'train.csv'))
# 保存test的结果
test_hists = pd.DataFrame(test_hists)
test_hists.to_csv(os.path.join(config.save_dir, 'test.csv'))