def __init__(self, options):
super(WSODModel, self).__init__()
self.arch = options['base_arch']
if self.arch == 'densenet_cifar':
self.pretrained_model = densenet_cifar()
elif self.arch == 'densenet121':
pretrained_model = tv_models.densenet121(pretrained=False,growth_rate=12)
self.features = pretrained_model.features
self.classifier = nn.Linear(pretrained_model.classifier.in_features,options['num_classes'])
def run():
# args
args = cli()
print("run with args: {}".format(args))
# logs
writer = SummaryWriter(args.logs)
# Store args
with open(os.path.join(args.logs, "args.txt"), "wt") as f:
f.write(str(args) + "\n")
# model
args.device = None
if not args.disable_cuda and torch.cuda.is_available():
args.device = torch.device('cuda')
else:
args.device = torch.device('cpu')
if args.model == "densenet":
net = densenet_cifar()
net = net.to(device=args.device)
elif args.model == "denseprc":
net = denseprc_cifar_dipan(G=args.G, CMP=args.CMP, reduction=1.0)
net = net.to(device=args.device)
else:
raise NotImplementedError("{} is not available".format(args.model))
# data
trainset = torchvision.datasets.CIFAR10(
root="./data",
train=True,
download=True,
transform=transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010))
]))
testset = torchvision.datasets.CIFAR10(root="./data",
train=False,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
(0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010))
]))
trainloader = torch.utils.data.DataLoader(trainset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
testloader = torch.utils.data.DataLoader(testset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
classes = copy.deepcopy(trainset.classes)
# optimizer + loss
optimizer = torch.optim.SGD(params=net.parameters(),
lr=args.init_lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
xent = nn.CrossEntropyLoss()
scheduler = torch.optim.lr_scheduler.MultiStepLR(
optimizer, milestones=args.lr_schedule, gamma=0.1)
# train+test
train(net, optimizer, scheduler, xent, trainloader, writer, args)
test(net, xent, testloader, writer, args)
writer.close()
def main():
# set save path
save_path = args.save_path
if not os.path.exists(save_path):
os.mkdir(save_path)
# set gpu
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
print("valid size : {0} batch size : {1}".format(args.valid_size, args.batch_size))
# make data set
train_loader, valid_loader, test_loader,feature_train_loader = data.data_loader_make()
print(len(train_loader.dataset),len(valid_loader.dataset),len(test_loader.dataset))
num_class = args.num_class
if args.model == 'dense':
if (args.depth - 4) % 3:
raise Exception('Invalid depth')
block_config = [(args.depth - 4) // 6 for _ in range(3)]
growth_rate = 12
# layer , numclass #[18, 34, 50, 101, 110, 152]
if args.model == 'res':
#total number of layers if 6n + 2. if n is 5 then the depth of network is 32.
network = resnet.ResNet(18,num_classes=num_class).cuda()
elif args.model == 'dense':
#print(num_class)
#network = densenet.DenseNet(growth_rate=growth_rate,block_config=block_config, num_classes=num_class).cuda()
# bottleneck = False , reduction = 0.0 / reduction=0.5, bottleneck=True,
network = densenet.densenet_cifar().cuda()
#criterion = nn.CrossEntropyLoss(reduce=False)
criterion = nn.CrossEntropyLoss()
if args.opt == 'sgd': #,weight_decay=0.0001 # 0.001,5e-4
optimizer = optim.SGD(network.parameters(), lr=args.lr, momentum=args.momentum, nesterov=True) # , nesterov=True,weight_decay=0.001
elif args.opt == 'rms':
optimizer = optim.RMSprop(network.parameters(), lr=args.lr)
elif args.opt == 'adam':
optimizer = optim.Adam(network.parameters(), lr=args.lr)
if args.scheduler == True:
#scheduler = lr_scheduler.StepLR(optimizer,step_size=args.decay, gamma=args.gamma)
scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=[90, 120], gamma=0.1)
#scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[0.5 * args.epoch, 0.75 * args.epoch], gamma=0.1)
print("Model's state_dict:")
for param_tensor in network.state_dict():
print(param_tensor, "\t", network.state_dict()[param_tensor].size())
print('start learning')
train_loss_li = []
train_acc_li = []
valid_loss_li = []
valid_acc_li = []
test_loss_li = []
test_acc_li = []
for epoch in range(0, args.epoch):
if args.scheduler == True:
scheduler.step()
train_loss, train_acc, train_feature ,train_y= train(train_loader,network,criterion,optimizer,epoch+1)
valid_loss, valid_acc, valid_softmax, valid_correct, valid_feature ,valid_y= test(valid_loader,network,criterion,epoch+1,'valid')
test_loss, test_acc, test_softmax, test_correct, test_feature, test_y = test(test_loader,network,criterion,epoch+1,'test')
# save files
if args.epoch == epoch + 1:
torch.save(network.state_dict(), '{0}_{1}_{2}.pth'.format(save_path,'resnet110', epoch+1))
train_loss_li.append(train_loss)
valid_loss_li.append(valid_loss)
test_loss_li.append(test_loss)
train_acc_li.append(train_acc)
valid_acc_li.append(valid_acc)
test_acc_li.append(test_acc)
# get_feature
train_loss, train_acc, train_softmax, train_correct, train_feature,train_y = test(feature_train_loader, network, criterion,'finish','train')
test_loss, test_acc, test_softmax, test_correct, test_feature, test_y = test(test_loader, network, criterion, 'finish','test')
utlis.save_data('train_feature', train_feature, save_path)
utlis.save_data('train_y', train_y, save_path)
utlis.save_data('test_feature', test_feature, save_path)s
utlis.save_data('test_y', test_y, save_path)
# get softmax
for i in range(len(test_softmax)):
test_softmax[i] = test_softmax[i].item()
print('save logs')
utlis.save_data('test_correct', test_correct, save_path)
utlis.save_data('test_softmax', test_softmax, save_path)
utlis.save_loss_acc('train', train_loss_li, train_acc_li, save_path)
utlis.save_loss_acc('valid', valid_loss_li, valid_acc_li, save_path)
utlis.save_loss_acc('test', test_loss_li, test_acc_li, save_path)
print('draw curve')
utlis.draw_curve('train_acc.csv', 'valid_acc.csv', 'acc', save_path)
utlis.draw_curve('train_loss.csv', 'valid_loss.csv', 'loss', save_path)
utlis.draw_test_curve('test_acc.csv', 'test_loss.csv', save_path)
def main():
save_path = args.save_path
if not os.path.exists(save_path):
os.mkdir(save_path)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
print("valid size : {0} batch size : {1}".format(args.valid_size,
args.batch_size))
train_loader, valid_loader, test_loader, train_feature_loader = data.data_set(
args.valid_size, args.batch_size, save_path, args.data)
if args.data == 'cifar100':
num_class = 100
elif args.data == 'cifar10':
num_class = 10
if args.model == 'dense':
if (args.depth - 4) % 3:
raise Exception('Invalid depth')
block_config = [(args.depth - 4) // 6 for _ in range(3)]
growth_rate = 12
# layer , numclass #[18, 34, 50, 101, 110, 152]
if args.model == 'res':
network = resnet.ResNet(args.layer, num_class).cuda()
#network = resnet.resnet18().cuda()
elif args.model == 'dense':
#print(num_class)
#network = densenet.DenseNet(growth_rate=growth_rate,block_config=block_config, num_classes=num_class).cuda()
# bottleneck = False , reduction = 0.0 / reduction=0.5, bottleneck=True,
network = densenet.densenet_cifar().cuda()
#criterion = nn.CrossEntropyLoss()
criterion = nn.CrossEntropyLoss(reduce=False)
if args.opt == 'sgd': # 0.001,5e-4
optimizer = optim.SGD(network.parameters(),
lr=args.lr,
momentum=args.momentum,
nesterov=True,
weight_decay=0.001)
elif args.opt == 'rms':
optimizer = optim.RMSprop(network.parameters(), lr=args.lr)
elif args.opt == 'adam':
optimizer = optim.Adam(network.parameters(), lr=args.lr)
if args.scheduler == True:
scheduler = lr_scheduler.MultiStepLR(optimizer,
milestones=[160, 180],
gamma=0.1)
#scheduler = lr_scheduler.StepLR(optimizer,step_size=args.decay, gamma=args.gamma)
#scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[0.5 * args.epoch, 0.75 * args.epoch], gamma=0.1)
print("Model's state_dict:")
for param_tensor in network.state_dict():
print(param_tensor, "\t", network.state_dict()[param_tensor].size())
train_loss_li = []
train_acc_li = []
valid_loss_li = []
valid_acc_li = []
test_loss_li = []
test_acc_li = []
for epoch in range(0, args.epoch):
if args.scheduler == True:
scheduler.step()
train_loss, train_acc, train_loss_idv = train(train_loader, network,
criterion, optimizer,
epoch + 1)
valid_loss, valid_acc, valid_softmax, valid_correct, valid_y, valid_loss_idv = test(
valid_loader, network, criterion, epoch + 1, 'valid')
test_loss, test_acc, test_softmax, test_correct, test_y, test_loss_idv = test(
test_loader, network, criterion, epoch + 1, 'test')
feature_loss, feature_acc, feature_softmax, feature_correct, feature_y, feature_loss_idv = test(
train_feature_loader, network, criterion, epoch + 1, 'feature')
# save files
train_loss_li.append(train_loss)
valid_loss_li.append(valid_loss)
test_loss_li.append(test_loss)
train_acc_li.append(train_acc)
valid_acc_li.append(valid_acc)
test_acc_li.append(test_acc)
if epoch + 1 == args.epoch:
torch.save(
network.state_dict(),
save_path + '{0}_{1}.pth'.format('resnet110', epoch + 1))
utlis.save_data('train_loss_idv', train_loss_idv, save_path)
utlis.save_data('feature_loss_idv', feature_loss_idv, save_path)
for i in range(len(test_softmax)):
test_softmax[i] = test_softmax[i].item()
print(train_loss, train_acc, valid_loss, valid_acc, test_loss, test_acc)
print('save logs')
utlis.save_data('test_correct', test_correct, save_path)
utlis.save_data('test_softmax', test_softmax, save_path)
utlis.save_loss_acc('train', train_loss_li, train_acc_li, save_path)
utlis.save_loss_acc('valid', valid_loss_li, valid_acc_li, save_path)
utlis.save_loss_acc('test', test_loss_li, test_acc_li, save_path)
print('draw curve')
utlis.draw_curve('train_acc.csv', 'valid_acc.csv', 'acc', save_path)
utlis.draw_curve('train_loss.csv', 'valid_loss.csv', 'loss', save_path)
utlis.draw_test_curve('test_acc.csv', 'test_loss.csv', save_path)
MAX_STEPS = 80000
initial_lr = 0.1
saved_ckpt_path = './checkpoint/'
saved_summary_train_path = './summary/train/'
saved_summary_test_path = './summary/test/'
with tf.name_scope('input'):
x = tf.placeholder(dtype=tf.float32,
shape=[None, HEIGHT, WIDTH, CHANNELS],
name='x_input')
y = tf.placeholder(dtype=tf.int32, shape=[None], name='label')
keep_prob = tf.placeholder(dtype=tf.float32, name='keep_prob')
y_onehot = tf.one_hot(y, CLASSES, dtype=tf.float32)
logits = DenseNet.densenet_cifar(x, keep_prob, True)
with tf.name_scope("loss"):
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_onehot,
logits=logits,
name='loss'))
tf.summary.scalar('loss', loss)
with tf.name_scope('learning_rate'):
lr = tf.Variable(initial_lr, dtype=tf.float32)
tf.summary.scalar('learning_rate', lr)
#optimizer = tf.train.AdamOptimizer(lr).minimize(loss)
optimizer = tf.train.MomentumOptimizer(learning_rate=lr,
momentum=0.9).minimize(loss)