def infer(test_queue, model, criterion):
model.eval()
objs = utils.AverageMeter()
test_true = []
test_pred = []
with torch.no_grad():
for i, (data, label) in enumerate(test_queue):
data, label = data.to(DEVICE), label.to(DEVICE).squeeze()
data = data.permute(0, 2, 1).unsqueeze(3)
out, out_aux = model(data)
pred = out.max(dim=1)[1]
test_true.append(label.cpu().numpy())
test_pred.append(pred.detach().cpu().numpy())
loss = criterion(out, label.squeeze())
n = label.size(0)
objs.update(loss.item(), n)
test_true = np.concatenate(test_true)
test_pred = np.concatenate(test_pred)
overall_acc = metrics.accuracy_score(test_true, test_pred)
class_acc = metrics.balanced_accuracy_score(test_true, test_pred)
return overall_acc, class_acc, objs.avg
def train_step(train_queue, model, criterion, optimizer, args):
objs = utils.AverageMeter()
train_true = []
train_pred = []
for step, (data, label) in enumerate(tqdm(train_queue)):
model.train()
data, label = data.to(DEVICE), label.to(DEVICE).squeeze()
data = data.permute(0, 2, 1).unsqueeze(3)
n = data.size(0)
optimizer.zero_grad()
out, out_aux = model(data)
loss = criterion(out, label)
if args.auxiliary:
loss_aux = criterion(out_aux, label)
loss += args.auxiliary_weight * loss_aux
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip)
optimizer.step()
pred = out.max(dim=1)[1]
train_true.append(label.cpu().numpy())
train_pred.append(pred.detach().cpu().numpy())
objs.update(loss.item(), n)
train_true = np.concatenate(train_true)
train_pred = np.concatenate(train_pred)
overall_acc = metrics.accuracy_score(train_true, train_pred)
class_acc = metrics.balanced_accuracy_score(train_true, train_pred)
return overall_acc, class_acc, objs.avg
def train_step(train_queue, model, criterion, optimizer):
objs = utils.AverageMeter()
micro_f1 = 0.
count = 0.
for step, input in enumerate(train_queue):
model.train()
input = input.to(DEVICE)
target = input.y
n = input.x.size(0)
optimizer.zero_grad()
logits, logits_aux = model(input)
loss = criterion(logits, target)
if args.auxiliary:
loss_aux = criterion(logits_aux, target)
loss += args.auxiliary_weight * loss_aux
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip)
optimizer.step()
micro_f1 += utils.mF1(logits, target) * n
count += n
objs.update(loss.item(), n)
micro_f1 = float(micro_f1) / count
return micro_f1, objs.avg
def infer(valid_queue, model, criterion):
objs = utils.AverageMeter()
model.eval()
count = 0.
micro_f1 = 0.
with torch.no_grad():
for step, input in enumerate(valid_queue):
input = input.to(DEVICE)
target = input.y
logits, _ = model(input)
loss = criterion(logits, target)
n = target.size(0)
micro_f1 += utils.mF1(logits, target) * n
count += n
objs.update(loss.item(), n)
micro_f1 = float(micro_f1) / count
return micro_f1, objs.avg
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
if args.random_seed:
args.seed = np.random.randint(0, 1000, 1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
# dataset modelnet
pre_transform, transform = T.NormalizeScale(), T.SamplePoints(
args.num_points)
train_dataset = GeoData.ModelNet(os.path.join(args.data, 'modelnet10'),
'10', True, transform, pre_transform)
train_queue = DenseDataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.batch_size // 2)
test_dataset = GeoData.ModelNet(os.path.join(args.data, 'modelnet10'),
'10', False, transform, pre_transform)
valid_queue = DenseDataLoader(test_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.batch_size // 2)
n_classes = train_queue.dataset.num_classes
criterion = torch.nn.CrossEntropyLoss().cuda()
model = Network(args.init_channels,
n_classes,
args.num_cells,
criterion,
args.n_steps,
in_channels=args.in_channels,
emb_dims=args.emb_dims,
dropout=args.dropout,
k=args.k).cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
num_edges = model._steps * 2
post_train = 5
# import pdb;pdb.set_trace()
args.epochs = args.warmup_dec_epoch + args.decision_freq * (
num_edges - 1) + post_train + 1
logging.info("total epochs: %d", args.epochs)
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
normal_selected_idxs = torch.tensor(len(model.alphas_normal) * [-1],
requires_grad=False,
dtype=torch.int).cuda()
normal_candidate_flags = torch.tensor(len(model.alphas_normal) * [True],
requires_grad=False,
dtype=torch.bool).cuda()
logging.info('normal_selected_idxs: {}'.format(normal_selected_idxs))
logging.info('normal_candidate_flags: {}'.format(normal_candidate_flags))
model.normal_selected_idxs = normal_selected_idxs
model.normal_candidate_flags = normal_candidate_flags
print(F.softmax(torch.stack(model.alphas_normal, dim=0), dim=-1).detach())
count = 0
normal_probs_history = []
train_losses, valid_losses = utils.AverageMeter(), utils.AverageMeter()
for epoch in range(args.epochs):
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
# training
# import pdb;pdb.set_trace()
att = model.show_att()
beta = model.show_beta()
train_acc, train_losses = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
train_losses)
valid_overall_acc, valid_class_acc, valid_losses = infer(
valid_queue, model, criterion, valid_losses)
logging.info(
'train_acc %f\tvalid_overall_acc %f \t valid_class_acc %f',
train_acc, valid_overall_acc, valid_class_acc)
logging.info('beta %s', beta.cpu().detach().numpy())
logging.info('att %s', att.cpu().detach().numpy())
# make edge decisions
saved_memory_normal, model.normal_selected_idxs, \
model.normal_candidate_flags = edge_decision('normal',
model.alphas_normal,
model.normal_selected_idxs,
model.normal_candidate_flags,
normal_probs_history,
epoch,
model,
args)
if saved_memory_normal:
del train_queue, valid_queue
torch.cuda.empty_cache()
count += 1
new_batch_size = args.batch_size + args.batch_increase * count
logging.info("new_batch_size = {}".format(new_batch_size))
train_queue = DenseDataLoader(train_dataset,
batch_size=new_batch_size,
shuffle=True,
num_workers=args.batch_size // 2)
valid_queue = DenseDataLoader(test_dataset,
batch_size=new_batch_size,
shuffle=False,
num_workers=args.batch_size // 2)
# post validation
if args.post_val:
post_valid_overall_acc, post_valid_class_acc, valid_losses = infer(
valid_queue, model, criterion, valid_losses)
logging.info('post_valid_overall_acc %f',
post_valid_overall_acc)
writer.add_scalar('stats/train_acc', train_acc, epoch)
writer.add_scalar('stats/valid_overall_acc', valid_overall_acc, epoch)
writer.add_scalar('stats/valid_class_acc', valid_class_acc, epoch)
utils.save(model, os.path.join(args.save, 'weights.pt'))
scheduler.step()
logging.info("#" * 30 + " Done " + "#" * 30)
logging.info('genotype = %s', model.get_genotype())
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
if args.random_seed:
args.seed = np.random.randint(0, 1000, 1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
# dataset ppi
train_dataset = GeoData.PPI(os.path.join(args.data, 'ppi'), split='train')
train_queue = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True)
valid_dataset = GeoData.PPI(os.path.join(args.data, 'ppi'), split='val')
valid_queue = DataLoader(valid_dataset, batch_size=args.batch_size, shuffle=False)
n_classes = train_queue.dataset.num_classes
criterion = torch.nn.BCEWithLogitsLoss().cuda()
model = Network(args.init_channels, n_classes, args.num_cells, criterion,
args.n_steps, in_channels=args.in_channels).cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
num_edges = model._steps * 2
post_train = 5
args.epochs = args.warmup_dec_epoch + args.decision_freq * (num_edges - 1) + post_train + 1
logging.info("total epochs: %d", args.epochs)
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
normal_selected_idxs = torch.tensor(len(model.alphas_normal) * [-1], requires_grad=False, dtype=torch.int).cuda()
normal_candidate_flags = torch.tensor(len(model.alphas_normal) * [True], requires_grad=False, dtype=torch.bool).cuda()
logging.info('normal_selected_idxs: {}'.format(normal_selected_idxs))
logging.info('normal_candidate_flags: {}'.format(normal_candidate_flags))
model.normal_selected_idxs = normal_selected_idxs
model.normal_candidate_flags = normal_candidate_flags
print(F.softmax(torch.stack(model.alphas_normal, dim=0), dim=-1).detach())
count = 0
normal_probs_history = []
train_losses, valid_losses = utils.AverageMeter(), utils.AverageMeter()
for epoch in range(args.epochs):
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
# training
train_acc, train_losses = train(train_queue, valid_queue, model, architect, criterion, optimizer, lr, train_losses)
valid_acc, valid_losses = infer(valid_queue, model, criterion, valid_losses)
logging.info('train_acc %f\tvalid_acc %f', train_acc, valid_acc)
# make edge decisions
saved_memory_normal, model.normal_selected_idxs, \
model.normal_candidate_flags = edge_decision('normal',
model.alphas_normal,
model.normal_selected_idxs,
model.normal_candidate_flags,
normal_probs_history,
epoch,
model,
args)
if saved_memory_normal:
del train_queue, valid_queue
torch.cuda.empty_cache()
count += 1
new_batch_size = args.batch_size + args.batch_increase * count
logging.info("new_batch_size = {}".format(new_batch_size))
train_queue = DataLoader(train_dataset, batch_size=new_batch_size, shuffle=True)
valid_queue = DataLoader(valid_dataset, batch_size=new_batch_size, shuffle=False)
if args.post_val:
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('post valid_acc %f', valid_acc)
writer.add_scalar('stats/train_acc', train_acc, epoch)
writer.add_scalar('stats/valid_acc', valid_acc, epoch)
utils.save(model, os.path.join(args.save, 'weights.pt'))
scheduler.step()
logging.info("#" * 30 + " Done " + "#" * 30)
logging.info('genotype = %s', model.get_genotype())