def sensing_decision(self, sensing_result, number_cpes):
"""
Algorithm sensing decision.
@param sensing_result
@param number_cpes
"""
#print "simple cycle"
#feedback cycle control
self._fb_cycle += 1
#initialize sum of scores for each decision (0,1)
sum_result0 = 0.0
sum_result1 = 0.0
#keeps the greatest reward for each decision
greatest_reward0 = 0.0
greatest_reward1 = 0.0
#sum of all scores for each decision
for decision, reward in zip(sensing_result, self._reward):
#sum of scores for "0" decision
if decision == IDLE:
sum_result0 += reward
if reward > greatest_reward0:
greatest_reward0 = reward
#sum of scores for "1" decision
elif decision == OCCUPIED:
sum_result1 += reward
if reward > greatest_reward1:
greatest_reward1 = reward
#divide the sum of scores by the total number of CPEs
score_r1 = sum_result1 / number_cpes
score_r0 = sum_result0 / number_cpes
#verifies which sum of scores is higher, score1 or score0
if score_r0 > score_r1:
sensing_decision = IDLE
elif score_r0 < score_r1:
sensing_decision = OCCUPIED
#if both scores are equal, then verifies the decision made by the cpe with the greatest individual score
elif score_r0 == score_r1:
if greatest_reward0 >= greatest_reward1:
sensing_decision = IDLE
elif greatest_reward0 < greatest_reward1:
sensing_decision = OCCUPIED
#verifies if is feedback cycle
#if (self._fb_cycle % (feedback_control-1) == 0):
Logger.append('sdc', 'decision', sensing_decision)
if sensing_decision == OCCUPIED:
self._total_occ += 1
else:
self._total_idle += 1
self.feedback(sensing_result, sensing_decision, increase_rate, decrease_rate)
def decision(self, energy):
"""
@param energy
"""
energy = np.sum(energy) / energy.size
self.cycle_counter += 1
if self.cycle_counter_max == self.cycle_counter:
self.cycle_counter = 0
sp = self.do_action(self.s, self.a)
rw = self.get_reward(energy, sp)
ap = self.e_greedy_selection(sp)
self.update_q_table(self.s, self.a, rw, sp, ap)
self.s = sp
self.a = ap
#self.epsilon *= 0.999
Logger.append('bayes_learning', 'hypothesis', 1.0 if energy > self.s else 0.0)
Logger.append('bayes_learning', 'feedback', self._feedback)
Logger.append('bayes_learning', 'state', self.s)
Logger.append('bayes_learning', 'reward', rw)
Logger.append('bayes_learning', 'action', self.a)
return 1 if (energy > sp) else 0, energy
def work(self, input_items, output_items):
for idx in range(len(input_items[0])):
decf, vf = self.algo1.decision(input_items[0][idx])
if decf == 0:
decf, vf = self.algo2.decision(input_items[1][idx])
Logger.append('hier', 'decision', self._xx[Logger._ch_status][decf])
return len(input_items[0])
def _tick(self):
"""
Called automatically each one second.
"""
for _d in (self._bps, self._pkts):
_d['cur'] = _d['counting']
_d['counting'] = 0
Logger.append(self._name, 'bps', self._bps['cur'] ) #pylint: disable=E1101
Logger.append(self._name, 'pkts', self._pkts['cur']) #pylint: disable=E1101
def get_pkt_accumulated(self, clear=False):
"""
@param clear
"""
tmp = self._pkts['accumulated']
Logger.append(self._name, 'pkt_accumulated', self._pkts['accumulated']) # pylint: disable=E1101
if clear:
self._pkts['accumulated'] = 0
return tmp
def decision(self, data_in):
""""
Implementation of base class abstract method.
@param data_in Mag squared of samples.
@return Tuple (status, energy)
"""
energy = np.sum(data_in) / data_in.size
dec = 1 if self.threshold < energy else 0
Logger.append('energy_decision', 'energy', energy)
Logger.append('energy_decision', 'decision', self._xx[Logger._ch_status][dec])
return dec, energy
def decision(self, data_in):
""""
Implementation of base class abstract method.
@param data_in Mag squared of samples.
@return Tuple (status, energy)
"""
_sum = self._algorithm.calculate_cyclo(data_in.tolist())
_sum = _sum / len(data_in)
dec = 1 if self.threshold < _sum else 0
Logger.append('cyclo_decision', 'decision', self._xx[Logger._ch_status][dec])
return dec, _sum
def decision(self, data_in):
"""
Called from a signal processing block to made a decision.
@param data_in Mag squared of samples.
@return Tuple (status, correlation).
"""
max_corr = -1.0
for wave in self._waveforms:
max_corr = max(abs(self.correlate(wave, data_in)), max_corr)
dec = 1 if self.threshold < max_corr else 0
Logger.append('waveform_decision', 'decision', self._xx[Logger._ch_status][dec])
return dec, max_corr
def decision(self, data_l, data_m):
"""
Function called from a signal processing block.
@param data_l Learner decision regarding channel occupancy.
@param data_m Manager decision regarding channel occupancy.
"""
self.strategy.wait()
self._iteraction += 1
final_dec = data_l
if self._valid_feedback:
final_dec = data_m
self._time += 19.3
self._count += 1
Logger.set('feedback_algorithm', 'total_feedback', self._count)
Logger.append('feedback_algorithm', 'activation', int(data_m))
# set feedback in our learning algorithm
self.learner.feedback = data_m
# Increase feedback interval if both algorithms are correct
if data_l == data_m:
self.strategy.increase_time()
# else decrease time
else:
self.strategy.decrease_time()
else:
Logger.append('feedback_algorithm', 'activation', -1)
self._time += 0.2
self._valid_feedback = False
if self.strategy.feedback():
self._manager.enable(True)
self._valid_feedback = True
Logger.append('feedback_algorithm', 'time', self._time)
Logger.append('feedback_algorithm', 'count', self._count)
Logger.append('bayes_decision', 'hypothesis', final_dec)
def work(self, input_items, output_items):
"""
@param input_items
@param output_items
"""
for idx in range(len(input_items[0])):
self._iteraction += 1
ed_dec = input_items[0][idx][0]
wf = input_items[1][idx]
final_dec = ed_dec
if ed_dec == 0:
final_dec = 1
final_dec = self._algorithm.decision(wf)[0]
Logger.set('feedback_algorithm', 'total_feedback', self._count)
Logger.append('bayes_decision', 'hypothesis', final_dec)
return len(input_items[0])
def __init__(self,
name="SNREstimator",
algorithm=SVR,
alpha=0.001):
"""
CTOR
@param name
@param algorithm
@param alpha
"""
self._estimator = digital.probe_mpsk_snr_est_c(algorithm, 10000, alpha)
UHDGenericArch.__init__(self,
name=name,
input_signature=self._estimator.input_signature(),
output_signature=self._estimator.output_signature())
Logger.register(name, ['snr', ])
self.register_scheduling(lambda: Logger.append(name, 'snr', self.get_snr()), delay_sec=0.2) #pylint: disable=E1101
def decision(self, signal):
"""
Method called for a USRP block (probably bayesian_detector).
@param energy Signal Energy
@param signal
"""
#hypothesis
# update counters
if self._feedback == 0 or self._feedback == 1:
self.update_global_counter()
th = self._th
r = self._r[th]
while True:
# update bayes
self.update_th_counter(signal)
# update risk
bayes_r, self._th = self.get_min_risk()
# invalidate threshold. Wait until a new is provided
if th == self._th or r == bayes_r :
self.feedback = -1
break
th = self._th
r = self._r[th]
# Clear black list of threshold
self._black_list = []
#print "d:", self.bayesian_hypothesis(self._th, signal), " f:", self.feedback, " t:", math.e ** self._th, " e:", sum(signal)/50
bayes_hyp = self.bayesian_hypothesis(self._th, signal)
# Save all data
dec = 1 if bayes_hyp > 0 else 0
Logger.append('bayes_decision', 'decision', self._xx[Logger._ch_status][dec])
Logger.append('bayes_decision', 'threshold', self._th)
Logger.append('bayes_decision', 'risk', self._r[self._th])
return dec, 0.0
def transmitter_loop(tb, channel_list, channel, options):
"""
US LOOP
@param tb
@param channel_list
@param channel
@param options
"""
# Connect to slave device
import xmlrpclib
proxy = xmlrpclib.ServerProxy("http://%s:8000/" % options.slave_addr)
start_t = time.time()
proxy.client_started()
proxy.set_channel(channel)
Logger.register('transmitter', ['channel', 'status', 'pkt'])
class TNamespace():
"""
"""
pass
# Sensing -> TX loop
t_namespace = TNamespace()
t_namespace.pkt_s = 0
t_namespace.status = 0
while time.time() < (start_t + options.duration):
can_transmit = True
if not options.tx_only:
# Sense
decision, t_namespace.status = tb.rx.sense_channel(channel_list[channel], options.sensing_duration)
# Update
print t_namespace.status
if t_namespace.status > 0.000005: # GMSK threahold
#if t_namespace.status > 0.000000005 :
print str(channel_list[channel]) + ' is occupied'
t_now = time.clock()
## Q-NOISE AQUI.
channel = (channel + 1) % len(channel_list)
####
can_transmit = False
# Change channel
proxy.set_channel(channel)
tb.tx.center_freq = channel_list[channel]
tb.rx.center_freq = channel_list[channel]
# Transmit
if can_transmit:
payload = 0
if options.pkt_size > 1:
bytelist = [1] * (options.pkt_size/4)
payload = pack('%sH' % len(bytelist), *bytelist)
else:
bytelist = ['a', ]
payload = pack('%sc' % 1, *bytelist)
# thred sending packets
def send_thread():
while t_namespace.pkt_sending:
tb.tx.send_pkt(payload)
t_namespace.pkt_s += 1
#t_namespace.count += 1
# init thread
th = Thread(target=send_thread)
t_namespace.pkt_sending = True
th.start()
# wait for options.sending_duration
time.sleep(options.sending_duration)
# stop sending
t_namespace.pkt_sending = False
th.join()
Logger.append('transmitter', 'channel', channel)
Logger.append('transmitter', 'status', t_namespace.status)
Logger.append('transmitter', 'pkt', t_namespace.pkt_s)
proxy.close_app()
def train_model(model, criterion, optimizer, scheduler):
since = time.time()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
best_model_wts = model.state_dict()
best_prec1 = 0
train_loader = dataloders['train']
train_batch_num = batch_nums['train']
start_epoch = args.start_epoch
stop = 0
log_file = os.path.join(args.checkpoint, 'log.txt')
title = 'cifar-10-' + args.arch
if args.resume:
print('==> Resuming from checkpoint...')
print(args.resume)
assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_prec1 = checkpoint['best_prec1']
start_epoch = checkpoint['epoch']+1
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(log_file, title=title, resume=True)
else:
if os.path.exists(log_file) and not args.overwrite:
logprint.info('%s exists. skip', log_file)
return 0
logger = Logger(log_file, title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
for epoch in range(start_epoch, args.epochs):
scheduler.step()
model.train() # Set model to training mode
print('Epoch [{} | {}] LR: {}'.format(epoch, args.epochs - 1, scheduler.get_lr()[0]))
print('-' * 10)
progbar = Progbar(train_batch_num)
# Iterate over data.
for batch_index, data in enumerate(train_loader):
# get the inputs
inputs, labels = data
# wrap them in Variable
if use_gpu:
inputs = Variable(inputs.cuda())
labels = Variable(labels.cuda(async=True))
else:
inputs, labels = Variable(inputs), Variable(labels)
# forward
outputs = model(inputs)
loss = criterion(outputs, labels)
prec1, prec5 = accuracy(outputs.data, labels.data, topk=(1, 5))
losses.update(loss.data[0], inputs.size(0))
top1.update(prec1[0], inputs.size(0))
top5.update(prec5[0], inputs.size(0))
# backward + optimize
optimizer.zero_grad()
loss.backward()
optimizer.step()
# statistics
logger.append([scheduler.get_lr()[0], losses.avg, None, top1.avg, None])
progbar.add(1, values=[("p1", top1.avg), ("p5", top5.avg), ("loss", losses.avg)])
# end of an epoch
print()
print('Train Loss epoch {epoch} {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch=epoch, loss=losses, top1=top1, top5=top5))
# evaluate on validation set
prec1 = validate(dataloders['val'], model, criterion, logger)
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
stop += 1
print(' * Best Prec@1 {top1:.3f}'.format(top1=best_prec1))
save_checkpoint({
'epoch': epoch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer' : optimizer.state_dict(),
}, is_best, filename='checkpoint_epoch{epoch}.pth.tar'.format(epoch=epoch), checkpoint=args.checkpoint)
# deep copy the model
if is_best:
best_model_wts = model.state_dict()
print ('better model obtained at epoch {epoch}'.format(epoch=epoch))
stop = 0
if(stop >= 20):
print("\nEarly stop happend at {}\n".format(epoch))
break
print()
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_prec1))
# load best model weights
model.load_state_dict(best_model_wts)
return model
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.Resize((256, 256)),
transforms.ToTensor(),
#transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
transforms.Normalize((0.5071, 0.4867, 0.4408),
(0.2675, 0.2565, 0.2761)),
])
transform_test = transforms.Compose([
transforms.Resize((256, 256)),
transforms.ToTensor(),
#transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
transforms.Normalize((0.5071, 0.4867, 0.4408),
(0.2675, 0.2565, 0.2761)),
])
"""
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
else:
dataloader = datasets.CIFAR100
num_classes = 100
"""
bird_dataset = datasets.ImageFolder(root="./", transform=data_transform)
###################################################
"""
n_train = int(len(bird_dataset) * 0.6)
n_val = len(bird_dataset) *0.2
n_test = len(bird_dataset) - n_train - n_val
trainset, valset, testset = torch.utils.data.random_split(
bird_dataset,
[n_train, n_val, n_test]
)
"""
n_train = int(len(bird_dataset) * 0.6)
n_val = len(bird_dataset) - n_val
trainset, valset, testset = torch.utils.data.random_split(
bird_dataset, [n_train, n_val])
###################################################
#trainset = dataloader(root='./data', train=True, download=True, transform=transform_train)
#trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers)
trainloader = torch.utils.data.DataLoader(trainset,
batch_size,
shuffle=True)
#testset = dataloader(root='./data', train=False, download=False, transform=transform_test)
#testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers)
test_loader = torch.utils.data.DataLoader(testset, batch_size)
# Model
print("==> creating model '{}'".format(args.arch))
model = Net() #モデルの呼び出し、今回は簡易版のため畳み込み3層、convlstm cell１層、全結合から構成される
model = torch.nn.DataParallel(model).to(device=device)
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Resume
title = 'cifar-10-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(trainloader, model, criterion, optimizer,
epoch, use_cuda)
test_loss, test_acc = test(testloader, model, criterion, epoch,
use_cuda)
# append logger file
logger.append(
[state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data loading code
args.data = '/Users/io/workspace'
traindir = os.path.join(args.data, 'val')
valdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(traindir, transforms.Compose([
transforms.RandomSizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])),
batch_size=args.train_batch, shuffle=True,
num_workers=args.workers, pin_memory=False)
val_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(valdir, transforms.Compose([
transforms.Scale(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.test_batch, shuffle=False,
num_workers=args.workers, pin_memory=False)
# create model
args.pretrained = True
if args.pretrained:
print("=> using pre-trained model '{}'".format(args.arch))
model = res.__dict__[args.arch](pretrained=True)
else:
print("=> creating model '{}'".format(args.arch))
model = res.__dict__[args.arch]()
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0))
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
# Resume
title = 'ImageNet-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
args.evaluate = True
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(val_loader, model, criterion, start_epoch, use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, use_cuda)
test_loss, test_acc = test(val_loader, model, criterion, epoch, use_cuda)
# append logger file
logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def main_worker(gpu, ngpus_per_node, args):
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
args.gpu = gpu
if args.gpu is not None:
print("Use GPU: {} for training".format(args.gpu))
if args.distributed:
if args.dist_url == "env://" and args.rank == -1:
args.rank = int(os.environ["RANK"])
if args.multiprocessing_distributed:
# For multiprocessing distributed training, rank needs to be the
# global rank among all the processes
args.rank = args.rank * ngpus_per_node + gpu
torch.distributed.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size,
rank=args.rank)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4), # with p = 1
transforms.RandomHorizontalFlip(), # with p = 0.5
transforms.ToTensor(), # it must be this guy that makes it CHW again
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
else:
dataloader = datasets.CIFAR100
num_classes = 100
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('densenet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.endswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
)
elif args.arch.startswith('resnet50'):
model = models.__dict__[args.arch](num_classes=num_classes,
include_top=True,
dropout_rate=0,
layer=args.layer)
elif args.arch.startswith('d1_resnet50'):
model = models.__dict__[args.arch](num_classes=num_classes,
include_top=True,
dropout_rate=0,
layer=args.layer,
is_shuff=args.shuff)
elif args.arch.endswith('vgg16'):
model = models.__dict__[args.arch](num_classes=num_classes,
include_top=True,
dropout_rate=0,
layer=args.layer)
elif args.arch.endswith('vgg16_sa'):
model = models.__dict__[args.arch](num_classes=num_classes,
include_top=True,
dropout_rate=0,
layer=args.layer)
elif args.arch.endswith('vgg16_1d'):
model = models.__dict__[args.arch](num_classes=num_classes,
include_top=True,
dropout_rate=0,
layer=args.layer,
is_shuff=args.shuff)
else:
model = models.__dict__[args.arch](num_classes=num_classes)
if args.distributed:
# For multiprocessing distributed, DistributedDataParallel constructor
# should always set the single device scope, otherwise,
# DistributedDataParallel will use all available devices.
if args.gpu is not None:
torch.cuda.set_device(args.gpu)
model.cuda(args.gpu)
# When using a single GPU per process and per
# DistributedDataParallel, we need to divide the batch size
# ourselves based on the total number of GPUs we have
args.train_batch = int(args.train_batch / ngpus_per_node)
args.test_batch = int(args.workers / ngpus_per_node)
args.workers = int(args.workers / ngpus_per_node)
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.gpu])
else:
model.cuda()
# DistributedDataParallel will divide and allocate batch_size to all
# available GPUs if device_ids are not set
model = torch.nn.parallel.DistributedDataParallel(model)
elif args.gpu is not None:
torch.cuda.set_device(args.gpu)
model = model.cuda(args.gpu)
else:
# DataParallel will divide and allocate batch_size to all available GPUs
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
# for name, param in model.named_parameters():
# print(name)
# for name in model.named_modules():
# print(name)
# for param in model.parameters():
# print(param)
trainset = dataloader(root='/data/users/yuefan/fanyue/dconv/data',
train=True,
download=True,
transform=transform_train)
if args.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(
trainset)
else:
train_sampler = None
trainloader = data.DataLoader(trainset,
batch_size=args.train_batch,
shuffle=(train_sampler is None),
num_workers=args.workers,
pin_memory=True,
sampler=train_sampler)
testset = dataloader(root='/data/users/yuefan/fanyue/dconv/data',
train=False,
download=False,
transform=transform_test)
testloader = data.DataLoader(testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=args.weight_decay)
# Resume
title = 'cifar-10-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(trainloader, model, criterion, optimizer,
epoch, args)
test_loss, test_acc = test(testloader, model, criterion, args)
# append logger file
logger.append(
[state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
if not args.multiprocessing_distributed or (
args.multiprocessing_distributed
and args.rank % ngpus_per_node == 0):
save_checkpoint(
{
'epoch': epoch + 1,
'acc': test_acc,
'state_dict': model.state_dict(),
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def main():
# Use CUDA
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
cudnn.benchmark = True
# Random seed
if args.manualSeed is None:
args.manualSeed = random.randint(1, 10000)
random.seed(args.manualSeed)
torch.manual_seed(args.manualSeed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(args.manualSeed)
# allow parallel training
def to_parallel(model):
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
return torch.nn.DataParallel(model)
return model
# make output dir
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Load dataset
trainloader, valloader, testloader, num_classes = get_loaders(
dataset=args.dataset,
download=False,
train_batch=args.train_batch,
test_batch=args.test_batch,
n_workers=args.workers,
data_dir='./data')
# Init model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnet') or args.arch.startswith('preresnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
block_name=args.block_name,
)
else:
model = models.__dict__[args.arch](num_classes=num_classes)
print(" Total params: %.2fM" %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
to_parallel(model).to(device)
# Set model Lipschitz hooker
print("==> set Lipschitz hooker ")
hooker = LipHooker(args.arch, args.checkpoint, device=device)
hooker.hook(model)
# Set criterion and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Set learning rate scheduler
print("==> creating scheduler '{}'".format(args.scheduler))
if args.scheduler.startswith('constant'):
scheduler = schedulers.__dict__[args.scheduler](optimizer=optimizer,
dpath=args.checkpoint)
elif args.scheduler.startswith('step'):
scheduler = schedulers.__dict__[args.scheduler](
optimizer=optimizer,
milestones=args.schedule,
gamma=args.gamma,
dpath=args.checkpoint)
elif args.scheduler.startswith('cosine'):
scheduler = schedulers.__dict__[args.scheduler](
optimizer=optimizer,
milestones=args.schedule,
epochs=args.epochs,
dpath=args.checkpoint)
elif args.scheduler.startswith('adacosine'):
scheduler = schedulers.__dict__[args.scheduler](optimizer=optimizer,
epochs=args.epochs,
dpath=args.checkpoint)
else:
raise KeyError(args.scheduler)
# Set info logger
title = args.dataset + '-' + args.arch
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Epoch', 'Time Elapsed', 'Learning Rate', 'Train Loss', 'Valid Loss',
'Test Loss', 'Train Acc.', 'Valid Acc.', 'Test Acc.'
])
# ---------- grow -----------
# Set model architecture tracker for grow
modelArch = None
if args.grow:
modelArch = ModelArch(args.arch,
model,
args.epochs,
args.depth,
max_depth=args.max_depth,
dpath=args.checkpoint,
dataset=args.dataset)
# Set trigger for grow
if args.grow and args.mode == 'adapt':
trigger = TolTrigger(tolerance=args.threshold,
window=args.window,
reserve=args.reserve,
epochs=args.epochs,
modelArch=modelArch)
# Training start
print("==> training start - epochs: %i" % args.epochs)
time_start = time.time()
best_val_acc = 0 # best test accuracy
best_epoch = 0
for epoch in range(args.epochs):
train_loss, train_acc = train(trainloader,
model,
criterion,
optimizer,
debug_batch_size=args.debug_batch_size,
device=device)
val_loss, val_acc = test(valloader, model, criterion, device=device)
test_loss, test_acc = test(testloader, model, criterion)
# print('\nEpoch: [%d | %d] LR: %f Train-Loss: %.4f Val-Loss: %.4f Train-Acc: %.4f Val-Acc: %.4f' % (epoch + 1, args.epochs, scheduler.lr_(), train_loss, val_loss, train_acc, val_acc))
logger.append([
epoch, (time.time() - time_start) / 60.,
scheduler.lr_(), train_loss, val_loss, test_loss, train_acc,
val_acc, test_acc
])
# save model
is_best = val_acc > best_val_acc
if is_best:
best_epoch = epoch + 1
best_val_acc = max(val_acc, best_val_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': val_acc,
'best_val_acc': best_val_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
# update grower and scheduler
if args.grow:
modelArch.update(epoch, is_best, model)
errs = None
if args.hooker:
errs = hooker.output(epoch)
scheduler.step_(epoch, errs)
# grow
if args.grow:
if args.mode == 'fixed': # grow specified at fixed epochs
if epoch + 1 in args.grow_epoch:
modelArch.grow(1) # dummy grow
model = models.__dict__[args.arch](
num_classes=num_classes,
block_name=args.block_name,
archs=modelArch.arch)
to_parallel(model).to(device)
model.load_state_dict(
modelArch.state_dict.state_dict, strict=False
) # True) # False due to buffer added during training to calculate lipschitz
if args.scheduler == 'cosine' and not args.schedule:
# cases that learning rate is continued
optimizer = optim.SGD(model.parameters(),
lr=scheduler.lr_(),
momentum=args.momentum,
weight_decay=args.weight_decay)
else:
# cases that learning rate is reset, aka lr restart
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
'''
probably have to copy the entire momentum history for each weight
but here just initialize the optimizer again
'''
# if multi epoch cosine or cosine_restart
scheduler.update(optimizer, epoch=epoch)
if args.hooker:
hooker.hook(model)
modelArch.record(epoch, model)
elif args.mode == 'adapt': # grow happens automatically
assert args.hooker, 'For adaptive training, model hooker must be provided to monitor some stats'
trigger.feed(errs)
err_indices = trigger.trigger(
epoch, modelArch.get_num_blocks_all_layer(
)) # propose candidate blocks to be growed
if err_indices:
err_indices = modelArch.grow(
err_indices
) # try duplicate it to see if any layer exceeds upper limit
if err_indices:
model = models.__dict__[args.arch](
num_classes=num_classes,
block_name=args.block_name,
archs=modelArch.arch)
to_parallel(model).to(device)
model.load_state_dict(
modelArch.state_dict.state_dict, strict=False
) # not strict matching due to Lipschitz buffer
if args.scheduler == 'cosine':
# cases that learning rate is continued
optimizer = optim.SGD(
model.parameters(),
lr=scheduler.lr_(),
momentum=args.momentum,
weight_decay=args.weight_decay)
else:
# cases that learning rate is reset, aka lr restart
optimizer = optim.SGD(
model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.hooker:
hooker.hook(model)
trigger.update(
err_indices) # reset history errors in trigger
scheduler.update(
optimizer,
epoch=epoch) # update optimizer in lr scheduler
modelArch.record(
epoch,
model) # keep the current architecture for record
else:
raise KeyError('Grow mode %s not supported!' % args.mode)
# print growing stats
if args.grow:
print('\nGrow epochs: ', modelArch.grow_epochs[1:], end=', ')
print('Num parameters: ', modelArch.num_parameters, end=', ')
print('PPE: %.2f' % modelArch._get_ppe())
# evaluate best model
print('Best val acc: %.4f at %i' %
(best_val_acc, best_epoch)) # this is the validation acc
best_checkpoint = torch.load(
os.path.join(args.checkpoint, 'model_best.pth.tar'))
if args.grow:
best_model = models.__dict__[args.arch](num_classes=num_classes,
block_name=args.block_name,
archs=modelArch.best_arch)
else:
best_model = models.__dict__[args.arch](num_classes=num_classes,
depth=args.depth,
block_name=args.block_name)
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
best_model = torch.nn.DataParallel(best_model)
best_model.to(device) # --
best_model.load_state_dict(best_checkpoint['state_dict'], strict=False)
test_loss, test_acc = test(testloader, best_model, criterion)
if args.grow:
print('Best arch: %s' % modelArch.__str__(best=True), end=', ')
print('Best Test Loss: %.4f, Best Test Acc: %.4f' %
(test_loss, test_acc))
# evaluate final model
test_loss, test_acc = test(testloader, model, criterion)
if args.grow:
print('Final arch: %s' % modelArch, end=', ')
print('Final Test Loss: %.4f, Final Test Acc: %.4f' %
(test_loss, test_acc))
print('Wall time: %.3f mins' % ((time.time() - time_start) / 60))
# round off
scheduler.close()
if args.hooker:
hooker.close()
if args.grow:
modelArch.close()
if args.mode == 'adapt':
trigger.close()
logger.close()
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
#### Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.Resize([120, 120]),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
transform_train_aug = transforms.Compose([
transforms.Resize([120, 120]),
transforms.RandomHorizontalFlip(),
#transforms.RandomCrop([60,60]),
transforms.ColorJitter(brightness=0.5,
contrast=0.5,
saturation=0.5,
hue=0.5),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.Resize([120, 120]),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
#### path ####
path = '/home/tzh/Documents/working_jinyu/pytorch-classification-master/Office31_Dataset_Yurong/Single-Domain-Adaptation/A-2-D/TRAIN'
path_test = '/home/tzh/Documents/working_jinyu/pytorch-classification-master/Office31_Dataset_Yurong/Single-Domain-Adaptation/A-2-D/TEST'
#path = 'binary/par_cell'
#path_test = 'binary/par_cell_test_ex'
trainset = my_dataset(transform_train, path)
train_set_aug = my_dataset(transform_train_aug, path)
testset = my_dataset(transform_test, path_test)
num_classes = 31
trainloader = data.DataLoader(trainset,
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers,
drop_last=True)
trainloader_aug = data.DataLoader(train_set_aug,
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers,
drop_last=True)
testloader = data.DataLoader(testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
drop_last=True)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('densenet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.endswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
block_name=args.block_name,
)
else:
model = models.__dict__[args.arch](num_classes=num_classes)
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion = nn.CrossEntropyLoss()
#### optimizer ####
#optimizer = NGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
optimizer = optim.Adam(model.parameters(), lr=args.lr)
# Resume
title = 'cifar-10-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# fine-tune normalziation
#train_loss, train_acc = test(trainloader, model, criterion, optimizer, epoch, use_cuda)
#print("~~~~~~~~~~~~~~~~~~~~~~~FINISHED AD NORM~~~~~~~~~~~~~~~~~~~~~~~~~~~")
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(trainloader, trainloader_aug, model,
criterion, optimizer, epoch, use_cuda)
test_loss, test_acc = test(testloader, model, criterion, epoch,
use_cuda)
# append logger file
logger.append(
[state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def main_worker(gpu, ngpus_per_node, args):
global best_acc
mem = os.popen(
'"nvidia-smi" --query-gpu=memory.total,memory.used --format=csv,nounits,noheader'
).read().split('\n')
total = mem[0].split(',')[0]
total = int(total)
max_mem = int(total * 0.5)
# x = torch.rand((256, 1024, max_mem)).cuda()
# del x
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
args.gpu = gpu
if args.gpu is not None:
print("Use GPU: {} for training".format(args.gpu))
if args.distributed:
if args.dist_url == "env://" and args.rank == -1:
args.rank = int(os.environ["RANK"])
if args.multiprocessing_distributed:
# For multiprocessing distributed training, rank needs to be the
# global rank among all the processes
args.rank = args.rank * ngpus_per_node + gpu
torch.distributed.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size,
rank=args.rank)
num_classes = 1000
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.endswith('resnet50'):
model = Resnet50(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('resnet50_shuffle'):
model = Resnet50_Shuffle(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('resnet152_1x1'):
model = Resnet152_1x1(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('resnet152_1x1lap'):
model = Resnet152_1x1LAP(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('resnet152_1x1lmp'):
model = Resnet152_1x1LMP(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('resnet50_1x1lap'):
model = Resnet50_1x1LAP(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('resnet50_1x1lmp'):
model = Resnet50_1x1LMP(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('resnet152_shuffle'):
model = Resnet152_Shuffle(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('resnet152_truncated'):
model = Resnet152_truncated(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('resnet50_truncated'):
model = Resnet50_truncated(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('vgg16_shuffle'):
model = VGG16_Shuffle(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('vgg16_rand'):
model = VGG16_Rand(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('vgg16_1d'):
model = VGG16_1d(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('vgg16_1x1lmp'):
model = VGG16_1x1LMP(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('vgg16_1x1lap'):
model = VGG16_1x1LAP(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('d1_resnet50'):
model = Resnet50_1d(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('resnet50_1x1'):
model = Resnet50_1x1(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('d1_resnet152'):
model = Resnet152_1d(args.drop, num_classes, True, args.layer)
elif args.arch.endswith('mobilenetv1_1x1lmp'):
model = MobileNetV1_1x1LMP(1 - 0.999, num_classes, True, args.layer)
elif args.arch.endswith('mobilenetv1_1x1lap'):
model = MobileNetV1_1x1LAP(1 - 0.999, num_classes, True, args.layer)
elif args.arch.endswith('mobilenetv2_1x1lmp'):
model = MobileNetV2_1x1LMP(num_classes, args.layer)
elif args.arch.endswith('mobilenetv2_1x1lap'):
model = MobileNetV2_1x1LAP(num_classes, args.layer)
else:
raise Exception('arch can only be vgg16 or resnet50!')
if args.distributed:
# For multiprocessing distributed, DistributedDataParallel constructor
# should always set the single device scope, otherwise,
# DistributedDataParallel will use all available devices.
if args.gpu is not None:
torch.cuda.set_device(args.gpu)
model.cuda(args.gpu)
# When using a single GPU per process and per
# DistributedDataParallel, we need to divide the batch size
# ourselves based on the total number of GPUs we have
args.train_batch = int(args.train_batch / ngpus_per_node)
args.test_batch = int(args.test_batch / ngpus_per_node)
args.workers = int(args.workers / ngpus_per_node)
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.gpu])
else:
warnings.warn(
'NOT DISTRIBUTED!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
model.cuda()
# DistributedDataParallel will divide and allocate batch_size to all
# available GPUs if device_ids are not set
model = torch.nn.parallel.DistributedDataParallel(model)
elif args.gpu is not None:
warnings.warn(
'NOT DISTRIBUTED!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
torch.cuda.set_device(args.gpu)
model = model.cuda(args.gpu)
else:
warnings.warn(
'NOT DISTRIBUTED!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
# DataParallel will divide and allocate batch_size to all available GPUs
model = torch.nn.DataParallel(model).cuda()
criterion = nn.CrossEntropyLoss().cuda(args.gpu)
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=args.weight_decay)
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
# for name, param in model.named_parameters():
# print(name)
# for name in model.named_modules():
# print(name)
# for param in model.parameters():
# print(param)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])
transform_test = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])
if args.dataset == 'xian':
print('ImageNet from Xian is used!')
traindir = '/BS/xian/work/data/imageNet1K/train/'
valdir = '/BS/database11/ILSVRC2012/val/'
else:
traindir = os.path.join(args.dataset, 'train')
valdir = os.path.join(args.dataset, 'val')
trainset = datasets.ImageFolder(traindir, transform_train)
if args.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(
trainset)
else:
train_sampler = None
trainloader = data.DataLoader(trainset,
batch_size=args.train_batch,
shuffle=(train_sampler is None),
num_workers=args.workers,
pin_memory=True,
sampler=train_sampler)
testset = datasets.ImageFolder(valdir, transform_test)
testloader = data.DataLoader(testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
# Resume
title = 'imagenet-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, args)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(trainloader, model, criterion, optimizer,
epoch, args)
test_loss, test_acc = test(testloader, model, criterion, args)
# append logger file
logger.append(
[state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
if not args.multiprocessing_distributed or (
args.multiprocessing_distributed
and args.rank % ngpus_per_node == 0):
save_checkpoint(
{
'epoch': epoch + 1,
'acc': test_acc,
'state_dict': model.state_dict(),
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
else:
dataloader = datasets.CIFAR100
num_classes = 100
trainset = dataloader(root='./data', train=True, download=True, transform=transform_train)
trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers)
testset = dataloader(root='./data', train=False, download=False, transform=transform_test)
testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('densenet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
block_name=args.block_name,
)
elif args.arch.startswith('preresnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
block_name=args.block_name,
)
elif args.arch.startswith('horesnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
eta=args.eta,
block_name=args.block_name,
feature_vec=args.feature_vec
)
elif args.arch.startswith('hopreresnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
eta=args.eta,
block_name=args.block_name,
feature_vec=args.feature_vec
)
elif args.arch.startswith('nagpreresnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
eta=args.eta,
block_name=args.block_name,
feature_vec=args.feature_vec
)
elif args.arch.startswith('mompreresnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
eta=args.eta,
block_name=args.block_name,
feature_vec=args.feature_vec
)
elif args.arch.startswith('v2_preresnet'):
model = models.__dict__[args.arch](
block_name='basicblock',
num_blocks=[2,2,2,2],
num_classes=num_classes
)
else:
print('Model is specified wrongly - Use standard model')
model = models.__dict__[args.arch](num_classes=num_classes)
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0))
criterion = nn.CrossEntropyLoss()
if args.optimizer.lower() == 'sgd':
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
# elif args.optimizer.lower() == 'adam':
# optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay)
elif args.optimizer.lower() == 'radam':
optimizer = RAdam(model.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay)
elif args.optimizer.lower() == 'adamw':
optimizer = AdamW(model.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay, warmup=args.warmup)
elif args.optimizer.lower() == 'srsgd':
iter_count = 1
optimizer = SGD_Adaptive(model.parameters(), lr=args.lr, weight_decay=args.weight_decay, iter_count=iter_count, restarting_iter=40)
# Resume
title = 'cifar-10-' + args.arch
# if args.resume:
# # Load checkpoint.
# print('==> Resuming from checkpoint..')
# assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
# args.checkpoint = os.path.dirname(args.resume)
# checkpoint = torch.load(args.resume)
# best_acc = checkpoint['best_acc']
# start_epoch = checkpoint['epoch']
# model.load_state_dict(checkpoint['state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer'])
# logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
# else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
if args.optimizer.lower() == 'srsgd':
if epoch == 80:
optimizer = SGD_Adaptive(model.parameters(), lr=args.lr * 0.1, weight_decay=args.weight_decay, iter_count=iter_count, restarting_iter=80)
elif epoch == 120:
optimizer = SGD_Adaptive(model.parameters(), lr=args.lr * 0.01, weight_decay=args.weight_decay, iter_count=iter_count, restarting_iter=200)
elif epoch == 160:
optimizer = SGD_Adaptive(model.parameters(), lr=args.lr * 0.001, weight_decay=args.weight_decay, iter_count=iter_count, restarting_iter=500)
elif epoch == 200:
optimizer = SGD_Adaptive(model.parameters(), lr=args.lr * 0.0001, weight_decay=args.weight_decay, iter_count=iter_count, restarting_iter=1000)
else:
adjust_learning_rate(optimizer, epoch)
logger.file.write('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr']))
if args.optimizer.lower() == 'srsgd':
train_loss, train_acc, iter_count = train(trainloader, model, criterion, optimizer, epoch, use_cuda, logger)
else:
train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda, logger)
test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda, logger)
# append logger file
logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc])
writer.add_scalars('train_loss', {args.model_name: train_loss}, epoch)
writer.add_scalars('test_loss', {args.model_name: test_loss}, epoch)
writer.add_scalars('train_acc', {args.model_name: train_acc}, epoch)
writer.add_scalars('test_acc', {args.model_name: test_acc}, epoch)
# writer.add_scalars('loss_tracking/train_loss', {args.model_name: train_loss}, epoch)
# writer.add_scalars('loss_tracking/test_loss', {args.model_name: test_loss}, epoch)
# writer.add_scalars('loss_tracking/train_acc', {args.model_name: train_acc}, epoch)
# writer.add_scalars('loss_tracking/test_acc', {args.model_name: test_acc}, epoch)
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer' : optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
logger.file.write('Best acc:%f'%best_acc)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def main():
args = parse_args()
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
torch.manual_seed(args.seed)
if torch.cuda.is_available():
device = 'cuda'
torch.cuda.manual_seed(args.seed)
else:
device = 'cpu'
print(f"==> Using device: {device}")
if args.checkpoint is None:
time_stamp = str(datetime.datetime.now().strftime('-%Y%m%d%H%M%S'))
args.checkpoint = args.model + time_stamp
args.checkpoint = 'checkpoints/' + args.checkpoint
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
save_args(args)
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title="ModelNet" + args.model)
logger.set_names([
"Epoch-Num", 'Learning-Rate', 'Train-Loss', 'Train-acc-B',
'Train-acc', 'Valid-Loss', 'Valid-acc-B', 'Valid-acc'
])
print('==> Preparing data..')
train_loader = DataLoader(ModelNet40(partition='train',
num_points=args.num_points),
num_workers=8,
batch_size=args.batch_size,
shuffle=True,
drop_last=True)
test_loader = DataLoader(ModelNet40(partition='test',
num_points=args.num_points),
num_workers=8,
batch_size=args.batch_size,
shuffle=True,
drop_last=False)
# Model
print('==> Building model..')
net = models.__dict__[args.model]()
criterion = nn.CrossEntropyLoss().to(device)
net = net.to(device)
# criterion = criterion.to(device)
if device == 'cuda':
net = torch.nn.DataParallel(net)
cudnn.benchmark = True
optimizer = torch.optim.SGD(net.parameters(),
lr=args.learning_rate,
momentum=0.9,
weight_decay=args.weight_decay)
scheduler = CosineAnnealingLR(optimizer,
args.epoch,
eta_min=args.learning_rate / 100)
best_test_acc = 0. # best test accuracy
best_train_acc = 0.
best_test_acc_avg = 0.
best_train_acc_avg = 0.
best_test_loss = float("inf")
best_train_loss = float("inf")
start_epoch = 0 # start from epoch 0 or last checkpoint epoch
for epoch in range(start_epoch, args.epoch):
print('Epoch(%d/%s) Learning Rate %s:' %
(epoch + 1, args.epoch, optimizer.param_groups[0]['lr']))
train_out = train(net, train_loader, optimizer, criterion,
device) # {"loss", "acc", "acc_avg", "time"}
test_out = validate(net, test_loader, criterion, device)
scheduler.step()
if test_out["acc"] > best_test_acc:
best_test_acc = test_out["acc"]
is_best = True
else:
is_best = False
best_test_acc = test_out["acc"] if (
test_out["acc"] > best_test_acc) else best_test_acc
best_train_acc = train_out["acc"] if (
train_out["acc"] > best_train_acc) else best_train_acc
best_test_acc_avg = test_out["acc_avg"] if (
test_out["acc_avg"] > best_test_acc_avg) else best_test_acc_avg
best_train_acc_avg = train_out["acc_avg"] if (
train_out["acc_avg"] > best_train_acc_avg) else best_train_acc_avg
best_test_loss = test_out["loss"] if (
test_out["loss"] < best_test_loss) else best_test_loss
best_train_loss = train_out["loss"] if (
train_out["loss"] < best_train_loss) else best_train_loss
save_model(net,
epoch,
path=args.checkpoint,
acc=test_out["acc"],
is_best=is_best)
logger.append([
epoch, optimizer.param_groups[0]['lr'], train_out["loss"],
train_out["acc_avg"], train_out["acc"], test_out["loss"],
test_out["acc_avg"], test_out["acc"]
])
print(
f"Training loss:{train_out['loss']} acc_avg:{train_out['acc_avg']} acc:{train_out['acc']} time:{train_out['time']}s)"
)
print(
f"Testing loss:{test_out['loss']} acc_avg:{test_out['acc_avg']} acc:{test_out['acc']}% time:{test_out['time']}s) \n\n"
)
logger.close()
print(f"++++++++" * 2 + "Final results" + "++++++++" * 2)
print(
f"++ Last Train time: {train_out['time']} | Last Test time: {test_out['time']} ++"
)
print(
f"++ Best Train loss: {best_train_loss} | Best Test loss: {best_test_loss} ++"
)
print(
f"++ Best Train acc_B: {best_train_acc_avg} | Best Test acc_B: {best_test_acc_avg} ++"
)
print(
f"++ Best Train acc: {best_train_acc} | Best Test acc: {best_test_acc} ++"
)
print(f"++++++++" * 5)
def execute_sensing_decision(hit_rate, num_steps):
"""
Execution of the sensing_decision module.
@param hit_rate A list with the hit ratio for every cpe. Example: hit_rate = [90, 78, 32] - the first cpe has a
hit ratio of 90%, the second has a hit ratio of 78% and the third has a hit ratio of 32%.
@param num_steps Number of executions of the sensing_result
"""
feedback_control = 5
increase_rate = 0.1
decrease_rate = 0.1
num_cpes = len(hit_rate)
list_str_cpes = []
for i in range(num_cpes):
list_str_cpes.append("cpe" + str(i+1))
# save strings in the Logger
Logger._enable = True
Logger.register("reward", list_str_cpes)
sdc = SDController(num_cpes, feedback_control, increase_rate, decrease_rate)
# each element of this array is also an array. the array of the index 0 (ie, array_hit[0]) corresponds to the hit_
# rate[0] and so on.
cpe_array = [0] * num_steps
array_hit = []
# num_cpes is the length of array_hit
for i in range(num_cpes):
# need to append as a list because otherwise if we modify some subarray, ALL arrays are modified too.
array_hit.append(list(cpe_array))
# list of lists, where the random indexes will be.
list_indexes = []
for i in range(num_cpes):
list_indexes.append(list([]))
# set some random positions of the arrays to one.
for i in range(num_cpes):
while len(list_indexes[i]) < (num_steps - (num_steps * hit_rate[i]/100)):
rand = random.randint(0, num_steps-1)
if rand not in list_indexes[i]:
list_indexes[i].append(rand)
array_hit[i][rand] = 1
for step in range(num_steps):
sensing_result = []
for cpe in range(num_cpes):
sensing_result.append(array_hit[cpe][step])
Logger.append("reward", list_str_cpes[cpe], sdc._reward[cpe])
sdc.sensing_decision(sensing_result, num_cpes)
Logger.dump('./dump', '/cpes', 0)
print "\n\n REWARD\n\n"
for cpe in range(num_cpes):
print "reward cpe %i: " %(cpe+1) + str(sdc._reward[cpe])
print "TOTAL HIT RATE: ", float(sdc._total_idle/float(num_steps))
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data loading code
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(traindir, transforms.Compose([
transforms.RandomSizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])),
batch_size=args.train_batch, shuffle=True,
num_workers=args.workers, pin_memory=True)
val_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(valdir, transforms.Compose([
transforms.Scale(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.test_batch, shuffle=False,
num_workers=args.workers, pin_memory=True)
# create model
if args.pretrained:
print("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=True)
elif args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
baseWidth=args.base_width,
cardinality=args.cardinality,
)
else:
print("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch]()
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0))
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
# Resume
title = 'ImageNet-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(val_loader, model, criterion, start_epoch, use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, use_cuda)
test_loss, test_acc = test(val_loader, model, criterion, epoch, use_cuda)
# append logger file
logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer' : optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def receiver_loop(tb, channel_list, channel, options):
"""
UP LOOP
@param tb
@param channel_list
@param channel
@param options
"""
import xmlrpclib
from SimpleXMLRPCServer import SimpleXMLRPCServer
class MyNamespace:
"""
"""
pass
g_namespace = MyNamespace()
g_namespace.tb = tb
g_namespace.options = options
g_namespace.server_run = True
class StoppableXMLRPCServer(SimpleXMLRPCServer):
"""Override of TIME_WAIT"""
allow_reuse_address = True
def __init__(self, options):
SimpleXMLRPCServer.__init__(self, options)
self.stop = False
def serve_forever(self):
while not self.stop:
self.handle_request()
print 'exiting server'
def shutdown(self):
self.stop = True
return 0
server = StoppableXMLRPCServer((options.slave_addr, 8000))
g_namespace.th = Thread(target=server.serve_forever )
# Flag que indica quando a execucao deve parar
# Flag that indicates when the execution must stop.
g_namespace.run = False
g_namespace.interferer_channel = 0
def set_channel(channel):
"""
RPC for changing the channel.
@param channel
"""
print "Received command to handoff to channel ", channel
if not g_namespace.options.tx_only:
g_namespace.tb.tx.center_freq = channel_list[channel]
g_namespace.tb.rx.center_freq = channel_list[channel]
g_namespace.interferer_channel = channel
return 1
def close_app():
"""
Closes the app.
"""
print "Received command to close"
g_namespace.run = False
return 1
def client_started():
"""
Notifies that the execution has started.
"""
g_namespace.run = True
return 1
Logger.register('receiver', ['channel', 'pkt', 'start_time'])
Logger.set('receiver', 'start_time', time.time())
# Registra funcoes no servidor XML e inicia thread do servidor
# Registers functions in the XML server and starts the server thread.
server.register_function(set_channel, 'set_channel')
server.register_function(close_app, 'close_app')
server.register_function(client_started, 'client_started')
g_namespace.th.start()
print "Receiver listening for commands in port 8000"
print "\t... Waiting for client_started call"
while g_namespace.run == False:
1;
print "\t...client connected"
global t_rcv, t_cor
channel = 0
# Enquanto nao recebeu a notificacao de parada, continua a execucao
# While the stop notify is not received, continues the execution.
while g_namespace.run:
print " ... r: ", t_rcv, ", c: ", t_cor
time.sleep(1)
if not options.tx_only:
Logger.append('receiver', 'channel', channel, time.time())
Logger.append('receiver', 'pkt', t_rcv)
print "Shutting down Server"
server.shutdown()
print "\t ... Server exited"
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
else:
dataloader = datasets.CIFAR100
num_classes = 100
trainset = dataloader(root='./data', train=True, download=True, transform=transform_train)
trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers)
testset = dataloader(root='./data', train=False, download=False, transform=transform_test)
testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers)
# Model
print("==> creating model")
model = ResNet(
num_classes=num_classes,
depth=args.depth,
norm_type=args.norm,
basicblock=args.basicblock,
)
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(model)
print(' Total params: %.4fM' % (sum(p.numel() for p in model.parameters())/1000000.0))
criterion = nn.CrossEntropyLoss()
optimizer = set_optimizer(model, args)
# Resume
title = '{}-ResNet-{}-{}'.format(args.dataset, args.depth, args.norm)
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss',
'Train Acc.', 'Valid Acc.', 'Train Acc.5', 'Valid Acc.5'])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr']))
train_loss, train_acc, train_acc5 = train(trainloader, model, criterion, optimizer, epoch, use_cuda)
test_loss, test_acc, test_acc5 = test(testloader, model, criterion, epoch, use_cuda)
# append logger file
logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc, train_acc5, test_acc5])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer' : optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
logger.close()
print('Best acc:')
print(best_acc)
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
else:
dataloader = datasets.CIFAR100
num_classes = 100
trainset = dataloader(root=args.data_root, train=True, download=True, transform=transform_train)
trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers)
testset = dataloader(root=args.data_root, train=False, download=False, transform=transform_test)
testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('densenet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.endswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
)
else:
model = models.__dict__[args.arch](num_classes=num_classes)
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0))
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
# Resume
title = 'cifar-10-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda)
test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda)
# append logger file
logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer' : optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
dataloader = datasets.CIFAR10 # CIFAR10 data set is loaded and has 10 classes
num_classes = 10
trainset = dataloader(root='./data',
train=True,
download=True,
transform=transform_train)
trainloader = data.DataLoader(trainset,
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers)
testset = dataloader(root='./data',
train=False,
download=False,
transform=transform_test)
testloader = data.DataLoader(testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('densenet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.endswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
)
else:
model = models.__dict__[args.arch](
num_classes=num_classes) # here is where we use the vgg19_bn
model = torch.nn.DataParallel(model).cuda() # Multi GPU call
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Resume
title = 'cifar-10-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(trainloader, model, criterion, optimizer,
epoch, use_cuda)
test_loss, test_acc = test(testloader, model, criterion, epoch,
use_cuda)
# append logger file
logger.append(
[state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss','Train Acc.', 'Valid Acc.', 'Train Acc.5',
'Valid Acc.5'])
# Train and validate
for epoch in range(start_epoch, args.epochs):
state['lr'] = adjust_learning_rate(state['lr'], optimizer, epoch, args.gamma, args.schedule)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr']))
train_loss, train_acc, train_acc5 = train_one_epoch(trainloader, model, criterion, optimizer, use_cuda=use_cuda)
test_loss, test_acc, test_acc5 = test(testloader, model, criterion, use_cuda=use_cuda)
# append logger file
logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc, train_acc5, test_acc5])
# save model ap
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
if do_save_checkpoint:
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer' : optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
logger.close()
print('Best acc:', best_acc)
def main():
global best_prec1, args
args.gpu = 0
args.world_size = 1
if args.distributed:
args.gpu = args.local_rank % torch.cuda.device_count()
torch.cuda.set_device(args.gpu)
torch.distributed.init_process_group(backend='nccl',
init_method='env://')
args.world_size = torch.distributed.get_world_size()
args.total_batch_size = args.world_size * args.batch_size
if not os.path.isdir(args.checkpoint) and args.local_rank == 0:
mkdir_p(args.checkpoint)
if args.fp16:
assert torch.backends.cudnn.enabled, "fp16 mode requires cudnn backend to be enabled."
if args.static_loss_scale != 1.0:
if not args.fp16:
print("Warning: if --fp16 is not used, static_loss_scale will be ignored.")
# create model
if args.pretrained:
print("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=True)
else:
print("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch]()
model = model.cuda()
if args.fp16:
model = network_to_half(model)
if args.distributed:
# shared param/delay all reduce turns off bucketing in DDP, for lower latency runs this can improve perf
# for the older version of APEX please use shared_param, for newer one it is delay_allreduce
model = DDP(model, delay_allreduce=True)
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(), args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.fp16:
optimizer = FP16_Optimizer(optimizer,
static_loss_scale=args.static_loss_scale,
dynamic_loss_scale=args.dynamic_loss_scale,
verbose=False)
# optionally resume from a checkpoint
title = 'ImageNet-' + args.arch
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume, map_location=lambda storage, loc: storage.cuda(args.gpu))
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})"
.format(args.resume, checkpoint['epoch']))
if args.local_rank == 0:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
else:
if args.local_rank == 0:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.', 'Valid Top5.'])
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'val')
if(args.arch == "inception_v3"):
crop_size = 299
val_size = 320 # I chose this value arbitrarily, we can adjust.
else:
crop_size = 224
val_size = 256
pipe = HybridTrainPipe(batch_size=args.batch_size, num_threads=args.workers, device_id=args.local_rank, data_dir=traindir, crop=crop_size, dali_cpu=args.dali_cpu)
pipe.build()
train_loader = DALIClassificationIterator(pipe, size=int(pipe.epoch_size("Reader") / args.world_size))
pipe = HybridValPipe(batch_size=args.batch_size, num_threads=args.workers, device_id=args.local_rank, data_dir=valdir, crop=crop_size, size=val_size)
pipe.build()
val_loader = DALIClassificationIterator(pipe, size=int(pipe.epoch_size("Reader") / args.world_size))
if args.evaluate:
validate(val_loader, model, criterion)
return
total_time = AverageMeter()
for epoch in range(args.start_epoch, args.epochs):
# train for one epoch
adjust_learning_rate(optimizer, epoch,args)
if args.local_rank == 0:
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, optimizer.param_groups[0]['lr']))
[train_loss, train_acc, avg_train_time] = train(train_loader, model, criterion, optimizer, epoch)
total_time.update(avg_train_time)
# evaluate on validation set
[test_loss, prec1, prec5] = validate(val_loader, model, criterion)
# remember best prec@1 and save checkpoint
if args.local_rank == 0:
# append logger file
logger.append([optimizer.param_groups[0]['lr'], train_loss, test_loss, train_acc, prec1, prec5])
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint({
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
}, is_best,checkpoint=args.checkpoint)
if epoch == args.epochs - 1:
print('##Top-1 {0}\n'
'##Top-5 {1}\n'
'##Perf {2}'.format(prec1, prec5, args.total_batch_size / total_time.avg))
# reset DALI iterators
train_loader.reset()
val_loader.reset()
if args.local_rank == 0:
logger.close()
def main():
global best_acc
# Data
print('==> Preparing data..')
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
])
# data_file = './data'
# data_file = 'C:/Users/dongxian/data/CIFAR'
data_file = '/data/wudongxian/dataset/CIFAR/'
if args.dataset == 'cifar10':
data_name = 'CIFAR10'
num_classes = 10
else:
data_name = 'CIFAR100'
num_classes = 100
trainset = getattr(torchvision.datasets,
data_name)(root=data_file,
train=True,
download=True,
transform=transform_train)
trainloader = torch.utils.data.DataLoader(trainset,
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers)
testset = getattr(torchvision.datasets,
data_name)(root=data_file,
train=False,
download=True,
transform=transform_test)
testloader = torch.utils.data.DataLoader(testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
# Model
print('==> Building model..')
net = models.__dict__[args.arch](num_classes=num_classes)
title = 'noisy_cifar-' + args.arch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Adv Train Loss', 'Nat Train Loss', 'Nat Valid Loss',
'Adv Train Acc', 'Nat Train Loss', 'Nat Valid Acc'
])
if torch.cuda.is_available():
net = net.to(device)
net = torch.nn.DataParallel(net)
print('Using', torch.cuda.device_count(), 'GPUs.')
cudnn.benchmark = True
print('Using CUDA..')
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
schedule_lr = optim.lr_scheduler.MultiStepLR(optimizer,
args.schedule,
gamma=args.gamma)
adversary = PGDAttack(model=net,
epsilon=0.031,
num_steps=10,
step_size=0.007)
for epoch in range(args.start_epoch, args.epochs):
schedule_lr.step(epoch)
state['lr'] = optimizer.state_dict()['param_groups'][0]['lr']
if args.mode == 'nat':
adv_train_loss, adv_train_acc = train(trainloader, net, criterion,
optimizer, epoch)
else:
adv_train_loss, adv_train_acc = train_adv(trainloader,
net,
criterion,
optimizer,
epoch,
adversary=adversary)
nat_train_loss, nat_train_acc = test(trainloader, net, criterion,
epoch)
nat_test_loss, nat_test_acc = test(testloader, net, criterion, epoch)
logger.append([
state['lr'], adv_train_loss, nat_train_loss, nat_test_loss,
adv_train_acc, nat_train_acc, nat_test_acc
])
# what is the best?
is_best = adv_train_acc > best_acc
best_acc = max(adv_train_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': net.state_dict(),
'acc': adv_train_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
if (epoch + 1) % 10 == 0 and epoch > 1:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': net.state_dict(),
'acc': adv_train_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint,
filename='checkpoint_{:0>3d}.pth.tar'.format(epoch + 1))
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# load data
print('==> Preparing dataset %s' % args.dataset)
features, landmarks, labels = pickle_2_img_and_landmark(args.dataset_path)
num_classes = 6
# Model
print("==> creating model '{}'".format(args.arch))
model = ResNetAndDGCNN(20, num_classes=num_classes)
# model = torch.nn.DataParallel(model).cuda()
model = model.cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
# print(' resnet params: %.2fM' % (sum(p.numel() for p in model.resnet.parameters())/1000000.0))
# print(' stgcn params: %.2fM' % (sum(p.numel() for p in model.st_gcn.parameters())/1000000.0))
criterion = nn.CrossEntropyLoss()
# 分层优化
# resnet_para = [model.conv1.parameters(), model.layer1.parameters(), model.layer2.parameters(), model.layer3.parameters(), model.layer4.parameters()]
# optimizer = optim.SGD([
# {'params': model.gcn11.parameters()},
# {'params': model.gcn12.parameters()},
# {'params': model.gcn21.parameters()},
# {'params': model.gcn22.parameters()},
# {'params': model.gcn31.parameters()},
# {'params': model.gcn32.parameters()},
# {'params': model.fc.parameters()},
# {'params': model.conv1.parameters(), 'lr': 0.005, 'weight_decay': 1e-3},
# {'params': model.bn1.parameters(), 'lr': 0.005, 'weight_decay': 1e-3},
# {'params': model.layer1.parameters(), 'lr': 0.005, 'weight_decay': 1e-3},
# {'params': model.layer2.parameters(), 'lr': 0.005, 'weight_decay': 1e-3},
# {'params': model.layer3.parameters(), 'lr': 0.005, 'weight_decay': 1e-3},
# {'params': model.layer4.parameters(), 'lr': 0.005, 'weight_decay': 1e-3},
# ], lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
# Resume
title = 'ckp-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log_stat.log'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log_stat.log'),
title=title)
logger.set_names([
'fold_num', 'Learning Rate', 'Train Loss', 'Valid Loss',
'Train Acc.', 'Valid Acc.'
])
# logging
logging.basicConfig(level=logging.DEBUG,
filename=os.path.join(args.checkpoint, 'log_info.log'),
filemode='a+',
format="%(asctime)-15s %(levelname)-8s %(message)s")
# log configuration
logging.info('-' * 10 + 'configuration' + '*' * 10)
for arg in vars(args):
logging.info((arg, str(getattr(args, arg))))
acc_fold = []
reset_lr = state['lr']
for f_num in range(args.folds):
state['lr'] = reset_lr
model.reset_all_weights()
# optimizer = optim.SGD([
# {'params': model.gcn11.parameters()},
# {'params': model.gcn12.parameters()},
# {'params': model.gcn21.parameters()},
# {'params': model.gcn22.parameters()},
# {'params': model.gcn31.parameters()},
# {'params': model.gcn32.parameters()},
# {'params': model.fc.parameters()},
# {'params': model.conv1.parameters(), 'lr': 0.005, 'weight_decay': 1e-3},
# {'params': model.bn1.parameters(), 'lr': 0.005, 'weight_decay': 1e-3},
# {'params': model.layer1.parameters(), 'lr': 0.005, 'weight_decay': 1e-3},
# {'params': model.layer2.parameters(), 'lr': 0.005, 'weight_decay': 1e-3},
# {'params': model.layer3.parameters(), 'lr': 0.005, 'weight_decay': 1e-3},
# {'params': model.layer4.parameters(), 'lr': 0.005, 'weight_decay': 1e-3},
# ], lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
print(args.lr)
# save each fold's acc and reset configuration
average_acc = 0
best_acc = 0
# 10-fold cross validation
train_x, train_lm, train_y = [], [], []
test_x, test_lm, test_y = [], [], []
for id_fold in range(args.folds):
if id_fold == f_num:
test_x = features[id_fold]
test_lm = landmarks[id_fold]
test_y = labels[id_fold]
else:
train_x = train_x + features[id_fold]
train_lm = train_lm + landmarks[id_fold]
train_y = train_y + labels[id_fold]
# convert array to tensor
train_x = torch.tensor(train_x,
dtype=torch.float) / 255.0 #(b_s, 128, 128)
train_x = train_x.unsqueeze(1) #(b_s, 1, 128, 128)
train_lm = np.stack(train_lm)
# 只要坐标信息， 不需要归一化
train_lm = (train_lm - np.mean(train_lm, axis=0)) / np.std(train_lm,
axis=0)
train_lm = torch.tensor(train_lm)
# train_lm = train_lm.unsqueeze(2)
test_x = torch.tensor(test_x, dtype=torch.float) / 255.0
test_x = test_x.unsqueeze(1)
# 只要坐标信息， 不需要归一化
test_lm = (test_lm - np.mean(test_lm, axis=0)) / np.std(test_lm,
axis=0)
test_lm = torch.tensor(test_lm)
# test_lm = test_lm.unsqueeze(2)
train_y, test_y = torch.tensor(train_y), torch.tensor(test_y)
train_dataset = torch.utils.data.TensorDataset(train_x, train_lm,
train_y)
train_iter = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=args.train_batch,
shuffle=True)
test_dataset = torch.utils.data.TensorDataset(test_x, test_lm, test_y)
test_iter = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=args.test_batch,
shuffle=False)
# test for fold order
print(len(test_dataset))
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(train_x + test_x, train_y + test_y,
model, criterion, start_epoch, use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
continue
# show plt
# plt.show(block=False)
# Train and val
for epoch in range(start_epoch, args.epochs):
# 在特定的epoch 调整学习率
adjust_learning_rate(optimizer, epoch)
# print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr']))
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, optimizer.param_groups[0]['lr']))
train_loss, train_acc = train(train_iter, model, criterion,
optimizer, epoch, use_cuda)
test_loss, test_acc = test(test_iter, model, criterion, epoch,
use_cuda)
# append logger file
logger.append([
f_num, state['lr'], train_loss, test_loss, train_acc, test_acc
])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
f_num,
checkpoint=args.checkpoint)
# compute average acc
acc_fold.append(best_acc)
average_acc = sum(acc_fold) / len(acc_fold)
logging.info('fold: %d, best_acc: %.2f, average_acc: %.2f' %
(f_num, best_acc, average_acc))
logger.close()
# logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
logging.info('acc_fold' + str(acc_fold))
print('average acc:')
print(average_acc)
def main():
global best_acc
global best_acc_lr_0
global best_acc_lr_1
global best_acc_lr_2
global best_acc_lr_3
global global_writer
global global_num_classes
global global_record
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
# transforms.RandomCrop(32, padding=4),
# transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
global_num_classes = 10 # add
else:
dataloader = datasets.CIFAR100
num_classes = 100
global_num_classes = 100 # add
trainset = dataloader(root='./data', train=True, download=True, transform=transform_train)
trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers)
testset = dataloader(root='./data', train=False, download=False, transform=transform_test)
testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('densenet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.endswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
)
else:
model = models.__dict__[args.arch](num_classes=num_classes)
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0))
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
# Resume
title = 'cifar-%d-' % (global_num_classes) + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
if os.path.exists(os.path.join(args.checkpoint, 'log.txt')):
os.system('rm ' + os.path.join(args.checkpoint, 'log.txt'))
print('exist log.txt and rm log.txt')
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.', 'Best Acc.'])
if os.path.exists(os.path.join(args.checkpoint, 'info.txt')):
os.system('rm ' + os.path.join(args.checkpoint, 'info.txt'))
print('exist info.txt and rm info.txt')
global_writer = open(os.path.join(args.checkpoint, 'info.txt'), 'a', 0)
print('random seed = %d' % args.manualSeed, file = global_writer)
if os.path.exists(os.path.join(args.checkpoint, 'record.txt')):
os.system('rm ' + os.path.join(args.checkpoint, 'record.txt'))
print('exist record.txt and rm record.txt')
global_record = open(os.path.join(args.checkpoint, 'record.txt'), 'a', 0)
print('random seed = %d' % args.manualSeed, file = global_record)
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr']))
train_loss, train_acc, angleW, theta = train(trainloader, model, criterion, optimizer, epoch, use_cuda, args.lam)
test_loss, test_acc, confusion_matrix, total_wrong = test(testloader, model, criterion, epoch, use_cuda)
# append logger file
logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc, best_acc])
print('epoch: {}'.format(epoch), file = global_record)
print('test_acc: {}'.format(test_acc), file = global_record)
print('angleW:', file = global_record)
print('{} {}'.format(angleW.size(0), angleW.size(1)), file = global_record)
for val in angleW:
for w in val:
global_record.write('{} '.format(w))
global_record.write('\n')
print('theta:', file = global_record)
print('{} {}'.format(theta.size(0), theta.size(1)), file = global_record)
for val in theta:
for w in val:
global_record.write('{} '.format(w))
global_record.write('\n')
print('confusion_matrix:', file = global_record)
print('{} {}'.format(confusion_matrix.size(0), confusion_matrix.size(1)), file = global_record)
for val in confusion_matrix:
for w in val:
global_record.write('{} '.format(int(w)))
global_record.write('\n')
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
print('best_acc: {}'.format(best_acc), file = global_record)
print('-'*100, file = global_record)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer' : optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint,
filename='checkpoint.pth.tar',
bestfilename='model_best.pth.tar')
print(state['lr'])
if isequal(state['lr'], 0.1):
print('lr==0.1')
is_best_lr_0 = test_acc > best_acc_lr_0
best_acc_lr_0 = max(test_acc, best_acc_lr_0)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc_lr_0,
'optimizer' : optimizer.state_dict(),
},
is_best_lr_0,
checkpoint=args.checkpoint,
filename='checkpoint_lr_0.pth.tar',
bestfilename='model_best_lr_0.pth.tar')
if isequal(state['lr'], 0.01):
print('lr==0.01')
is_best_lr_1 = test_acc > best_acc_lr_1
best_acc_lr_1 = max(test_acc, best_acc_lr_1)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc_lr_1,
'optimizer' : optimizer.state_dict(),
},
is_best_lr_1,
checkpoint=args.checkpoint,
filename='checkpoint_lr_1.pth.tar',
bestfilename='model_best_lr_1.pth.tar')
if isequal(state['lr'], 0.001):
print('lr==0.001')
is_best_lr_2 = test_acc > best_acc_lr_2
best_acc_lr_2 = max(test_acc, best_acc_lr_2)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc_lr_2,
'optimizer' : optimizer.state_dict(),
},
is_best_lr_2,
checkpoint=args.checkpoint,
filename='checkpoint_lr_2.pth.tar',
bestfilename='model_best_lr_2.pth.tar')
if isequal(state['lr'], 0.0001):
print('lr==0.0001')
is_best_lr_3 = test_acc > best_acc_lr_3
best_acc_lr_3 = max(test_acc, best_acc_lr_3)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc_lr_3,
'optimizer' : optimizer.state_dict(),
},
is_best_lr_3,
checkpoint=args.checkpoint,
filename='checkpoint_lr_3.pth.tar',
bestfilename='model_best_lr_3.pth.tar')
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
# add--------------------
print ('Best acc:\n lr=0.1 acc={:.4f}\n lr=0.01 acc={:.4f}\n lr=0.001 acc={:.4f}\n'.format(best_acc_lr_0, best_acc_lr_1, best_acc_lr_2))
print('Best acc: {:.4f}\n'.format(best_acc))
print('Best acc:', file = global_writer)
print(best_acc, file = global_writer)
def main():
global args, best_prec1
args = parser.parse_args()
args.distributed = args.world_size > 1
if args.distributed:
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size)
# Use CUDA
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
use_cuda = torch.cuda.is_available()
# Random seed
if args.manual_seed is None:
args.manual_seed = random.randint(1, 10000)
random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
if use_cuda:
torch.cuda.manual_seed_all(args.manual_seed)
# create model
if args.pretrained:
print("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=True,
num_attributes=len(
args.selected_attrs))
elif args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
baseWidth=args.base_width,
cardinality=args.cardinality,
num_attributes=len(args.all_attrs),
)
elif args.arch.startswith('shufflenet'):
model = models.__dict__[args.arch](groups=args.groups,
num_attributes=len(
args.selected_attrs))
else:
print("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=True,
num_attributes=len(
args.selected_attrs))
# if not args.distributed:
# if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
# model.features = torch.nn.DataParallel(model.features)
# model.cuda()
# else:
# model = torch.nn.DataParallel(model).cuda()
# else:
# model.cuda()
# model = torch.nn.parallel.DistributedDataParallel(model)
# define loss function (criterion) and optimizer
# optionally resume from a checkpoint
title = 'CelebAHQ-' + args.arch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
logger = None
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
args.checkpoint = os.path.dirname(args.resume)
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
criterion = nn.CrossEntropyLoss().cuda()
model = torch.nn.DataParallel(model).cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.resume:
if os.path.isfile(args.resume):
print("=> loading optimizer '{}'".format(args.resume))
checkpoint = torch.load(args.resume, map_location='cuda')
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded optimizer '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
args.checkpoint = os.path.dirname(args.resume)
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
print("=> no optimizer checkpoint found at '{}'".format(
args.resume))
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
cudnn.benchmark = True
# Data loading code
normalize = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
train_dataset = CelebAHQExtra(
args.data, 'celebahq_extra', 'train',
transforms.Compose([
transforms.Resize((256, 256)),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]), args.selected_attrs, None)
if args.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_dataset)
else:
train_sampler = None
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.train_batch,
shuffle=(train_sampler is None),
num_workers=args.workers,
pin_memory=True,
sampler=train_sampler)
val_loader = torch.utils.data.DataLoader(CelebAHQExtra(
args.data, 'celebahq_extra', 'test',
transforms.Compose(
[transforms.Resize((256, 256)),
transforms.ToTensor(), normalize]), args.selected_attrs, None),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
test_loader = torch.utils.data.DataLoader(CelebAHQExtra(
args.data, 'celebahq_extra', 'test',
transforms.Compose(
[transforms.Resize((256, 256)),
transforms.ToTensor(), normalize]), args.selected_attrs, None),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
if args.evaluate:
validate(test_loader, model, criterion)
return
# visualization
writer = SummaryWriter(os.path.join(args.checkpoint, 'logs'))
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
lr = adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, lr))
# train for one epoch
train_loss, train_acc = train(train_loader, model, criterion,
optimizer, epoch)
# evaluate on validation set
val_loss, prec1 = validate(val_loader, model, criterion)
# append logger file
logger.append([lr, train_loss, val_loss, train_acc, prec1])
# tensorboardX
writer.add_scalar('learning rate', lr, epoch + 1)
writer.add_scalars('loss', {
'train loss': train_loss,
'validation loss': val_loss
}, epoch + 1)
writer.add_scalars('accuracy', {
'train accuracy': train_acc,
'validation accuracy': prec1
}, epoch + 1)
# for name, param in model.named_parameters():
# writer.add_histogram(name, param.clone().cpu().data.numpy(), epoch + 1)
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.module.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.pdf'))
writer.close()
print('Best accuracy:')
print(best_prec1)
def main():
global best_acc
# Data
print('==> Preparing data..')
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
])
# data_file = './data'
# data_file = 'C:/Users/dongxian/data/CIFAR'
data_file = '/data/wudongxian/dataset/CIFAR/'
if args.dataset == 'cifar10':
data_name = 'CIFAR10'
num_classes = 10
else:
data_name = 'CIFAR100'
num_classes = 100
trainset = getattr(torchvision.datasets, data_name)(root=data_file, train=True,
download=True, transform=transform_train)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers)
testset = getattr(torchvision.datasets, data_name)(root=data_file, train=False, download=True, transform=transform_test)
testloader = torch.utils.data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers)
title = 'adv-train-' + args.arch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
logger = Logger(os.path.join(args.checkpoint, 'log_attack_results.txt'), title=title)
logger.set_names(['Adv Train Loss', 'Adv Valid Loss',
'Adv Train Acc', 'Adv Valid Acc'])
# create Model
print('==> Building model..')
net = models.__dict__[args.arch](num_classes=num_classes)
if torch.cuda.is_available():
net = net.to(device)
net = torch.nn.DataParallel(net)
print('Using', torch.cuda.device_count(), 'GPUs.')
cudnn.benchmark = True
print('Using CUDA..')
criterion = nn.CrossEntropyLoss()
saved_models = [v for v in os.listdir(args.checkpoint) if '0.pth.tar' in v]
saved_models = sorted(saved_models)
save_text_list('{}/models_names.txt'.format(args.checkpoint), saved_models)
for saved_model in saved_models:
# resume Model
state_dict = torch.load('{}/{}'.format(args.checkpoint, saved_model))
net.load_state_dict(state_dict['state_dict'])
if args.attack_type == 'ref':
adversary = PGDAttack(model=net, epsilon=0.031, num_steps=20, step_size=0.003)
elif args.attack_type == 'ours':
adversary = OurPGDAttack(model=net, epsilon=0.031, num_steps=20, step_size=0.003, B=args.B)
adv_train_loss, adv_train_acc = attack_over_test(trainloader, net, criterion, adversary)
adv_test_loss, adv_test_acc = attack_over_test(testloader, net, criterion, adversary)
logger.append([adv_train_loss, adv_test_loss, adv_train_acc, adv_test_acc])
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
def main():
global best_acc, state
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if args.attack_iter == 0:
attacker = NoOpAttacker()
else:
attacker = PGDAttacker(args.attack_iter, args.attack_epsilon, args.attack_step_size, prob_start_from_clean=0.2 if not args.evaluate else 0.0)
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data loading code
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'val')
# the mean and variant don't have too much influence
# (pic - 0.5) / 0.5 just make it easier for attacker.
# normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
# std=[0.229, 0.224, 0.225])
normalize = transforms.Normalize(mean=[0.5, 0.5, 0.5],
std=[0.5, 0.5, 0.5])
transform_train = transforms.Compose([
transforms.RandomSizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])
if args.fastaug:
transform_train.transforms.insert(0, FastAugmentation())
if args.lighting:
__imagenet_pca = {
'eigval': np.array([0.2175, 0.0188, 0.0045]),
'eigvec': np.array([
[-0.5675, 0.7192, 0.4009],
[-0.5808, -0.0045, -0.8140],
[-0.5836, -0.6948, 0.4203],
])
}
transform_train = transforms.Compose([
transforms.RandomSizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
Lighting(0.1, __imagenet_pca['eigval'], __imagenet_pca['eigvec']),
normalize
])
train_dataset = datasets.ImageFolder(traindir, transform_train)
train_loader = torch.utils.data.DataLoader((train_dataset),
batch_size=args.train_batch, shuffle=True,
num_workers=args.workers, pin_memory=True)
val_transforms = [
transforms.ToTensor(),
normalize,
]
if not args.already224:
val_transforms = [transforms.Scale(256), transforms.CenterCrop(224)] + val_transforms
val_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(valdir, transforms.Compose(val_transforms)),
batch_size=args.test_batch, shuffle=False,
num_workers=args.workers, pin_memory=True)
# create model
print("=> creating model '{}'".format(args.arch))
if args.mixbn:
norm_layer = MixBatchNorm2d
else:
norm_layer = None
model = net.__dict__[args.arch](num_classes=args.num_classes, norm_layer=norm_layer)
model.set_attacker(attacker)
model.set_mixbn(args.mixbn)
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0))
# define loss function (criterion) and optimizer
if args.smoothing == 0:
criterion = nn.CrossEntropyLoss(reduction='none').cuda()
# implement a cross_entropy with label smoothing.
# First, perform a log_softmax; then fill the selected classes with 1-smoothing
# At last, use kl_div, which means:
# KL(p||q) = -\int p(x)ln q(x) dx - (-\int p(x)ln p(x) dx)
# kl_div is different from Crossentropy with a const number (\int p(x) ln p(x))
else:
criterion = partial(label_smoothing_cross_entropy, classes=args.num_classes, dim=-1)
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=args.nesterov)
# Resume
title = 'ImageNet-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
for param_group in optimizer.param_groups:
param_group['lr'] = state['lr']
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
if args.load:
checkpoint = torch.load(args.load)
if args.mixbn:
to_merge = {}
for key in checkpoint['state_dict']:
if 'bn' in key:
tmp = key.split("bn")
aux_key = tmp[0] + 'bn' + tmp[1][0] + '.aux_bn' + tmp[1][1:]
to_merge[aux_key] = checkpoint['state_dict'][key]
elif 'downsample.1' in key:
tmp = key.split("downsample.1")
aux_key = tmp[0] + 'downsample.1.aux_bn' + tmp[1]
to_merge[aux_key] = checkpoint['state_dict'][key]
checkpoint['state_dict'].update(to_merge)
model.load_state_dict(checkpoint['state_dict'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(val_loader, model, criterion, start_epoch, use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
writer = SummaryWriter(log_dir=args.checkpoint)
warmup_scheduler = WarmUpLR(optimizer, len(train_loader) * args.warm, start_lr=args.warm_lr) if args.warm > 0 else None
for epoch in range(start_epoch, args.epochs):
if epoch >= args.warm and args.lr_schedule == 'step':
adjust_learning_rate(optimizer, epoch, args)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, optimizer.param_groups[-1]['lr']))
train_func = partial(train, train_loader=train_loader, model=model, criterion=criterion,
optimizer=optimizer, epoch=epoch, use_cuda=use_cuda,
warmup_scheduler=warmup_scheduler, mixbn=args.mixbn,
writer=writer, attacker=attacker)
if args.mixbn:
model.apply(to_mix_status)
train_loss, train_acc, loss_main, loss_aux, top1_main, top1_aux = train_func()
else:
train_loss, train_acc = train_func()
writer.add_scalar('Train/loss', train_loss, epoch)
writer.add_scalar('Train/acc', train_acc, epoch)
if args.mixbn:
writer.add_scalar('Train/loss_main', loss_main, epoch)
writer.add_scalar('Train/loss_aux', loss_aux, epoch)
writer.add_scalar('Train/acc_main', top1_main, epoch)
writer.add_scalar('Train/acc_aux', top1_aux, epoch)
model.apply(to_clean_status)
test_loss, test_acc = test(val_loader, model, criterion, epoch, use_cuda)
writer.add_scalar('Test/loss', test_loss, epoch)
writer.add_scalar('Test/acc', test_acc, epoch)
# append logger file
logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer' : optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
print('Best acc:')
print(best_acc)
writer.close()
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
def main():
global args, best_prec1
args = parser.parse_args()
if args.seed is not None:
random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.deterministic = True
warnings.warn('You have chosen to seed training. '
'This will turn on the CUDNN deterministic setting, '
'which can slow down your training considerably! '
'You may see unexpected behavior when restarting '
'from checkpoints.')
args.distributed = args.world_size > 1
if args.distributed:
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size)
# create model
print("=> creating model '{}'".format(args.arch))
if args.arch.split('_')[0] == 'mobilenetv3':
model = models.__dict__[args.arch](width_mult=args.width_mult,
dropout=args.dropout,
mode=args.mode,
fc_squeeze=args.fc_squeeze)
elif args.arch.split('_')[0] == 'mobilenetv2':
model = models.__dict__[args.arch](width_mult=args.width_mult,
dropout=args.dropout,
fc_squeeze=args.fc_squeeze)
else:
model = models.__dict__[args.arch](dropout=args.dropout)
if not args.distributed:
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
else:
model.cuda()
model = torch.nn.parallel.DistributedDataParallel(model)
# define loss function (criterion) and optimizer
if args.label_smoothing > 0:
# using Label Smoothing
criterion = LabelSmoothingLoss(smoothing=args.label_smoothing)
else:
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optionally resume from a checkpoint
title = 'ImageNet-' + args.arch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
arch = open(os.path.join(args.checkpoint, 'arch.txt'), 'w')
print(model, file=arch)
arch.close()
if os.path.exists(os.path.join(args.checkpoint, 'checkpoint.pth.tar')):
args.resume = os.path.join(args.checkpoint, 'checkpoint.pth.tar')
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
args.checkpoint = os.path.dirname(args.resume)
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Epoch', 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.', 'Valid Acc5.'
])
print_options(args.checkpoint, args)
cudnn.benchmark = True
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_transforms = transforms.Compose([
transforms.RandomResizedCrop(args.input_size), #224
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])
train_dataset = ImageNet(traindir, train_transforms)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
train_loader_len = len(train_loader)
val_loader = torch.utils.data.DataLoader(
ImageNet(
valdir,
transforms.Compose([
transforms.Resize(256), #256
transforms.CenterCrop(args.input_size), #224
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
val_loader_len = len(val_loader)
if args.evaluate:
from collections import OrderedDict
if os.path.isfile(args.weight):
print("=> loading pretrained weight '{}'".format(args.weight))
source_state = torch.load(args.weight)['state_dict']
target_state = OrderedDict()
for k, v in source_state.items():
if k[:7] != 'module.':
k = 'module.' + k
target_state[k] = v
model.load_state_dict(target_state)
else:
print("=> no weight found at '{}'".format(args.weight))
validate(val_loader, val_loader_len, model, criterion)
return
# visualization
writer = SummaryWriter(os.path.join(args.checkpoint, 'logs'))
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
print('\nEpoch: [%d | %d]' % (epoch + 1, args.epochs))
# train for one epoch
train_loss, train_acc = train(train_loader, train_loader_len, model,
criterion, optimizer, epoch)
# evaluate on validation set
val_loss, prec1, prec5 = validate(val_loader, val_loader_len, model,
criterion)
lr = optimizer.param_groups[0]['lr']
# append logger file
logger.append(
[epoch + 1, lr, train_loss, val_loss, train_acc, prec1, prec5])
# tensorboardX
writer.add_scalar('learning rate', lr, epoch + 1)
writer.add_scalars('loss', {
'train loss': train_loss,
'validation loss': val_loss
}, epoch + 1)
writer.add_scalars('accuracy', {
'train accuracy': train_acc,
'validation accuracy': prec1
}, epoch + 1)
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
writer.close()
print('Best accuracy:')
print(best_prec1)
time.sleep(360000)
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
os.makedirs(args.save_dir, exist_ok=True)
# ######################################### Dataset ################################################
train_transforms = transforms.Compose([
# transforms.RandomResizedCrop(224),
transforms.RandomResizedCrop(224, scale=(0.1, 1.0), ratio=(0.8, 1.25)), # according to official open LTH
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
val_transforms = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
# #################### train / valid dataset ####################
train_dir = os.path.join(args.img_dir, 'train')
valid_dir = os.path.join(args.img_dir, 'val')
trainset = datasets.ImageFolder(root=train_dir, transform=train_transforms)
devset = datasets.ImageFolder(root=valid_dir, transform=val_transforms)
print('Total images in train, ', len(trainset))
print('Total images in valid, ', len(devset))
# #################### data loader ####################
trainloader = data.DataLoader(trainset, batch_size=args.train_batch,
shuffle=True, num_workers=args.workers)
devloader = data.DataLoader(devset, batch_size=args.test_batch,
shuffle=False, num_workers=args.workers)
# ######################################### Model ##################################################
print("==> creating model ResNet={}".format(args.depth))
if args.depth == 18:
model = resnet18(pretrained=False)
model_ref = resnet18(pretrained=False)
teacher_model = resnet18(pretrained=False)
elif args.depth == 34:
model = resnet34(pretrained=False)
model_ref = resnet34(pretrained=False)
teacher_model = resnet34(pretrained=False)
elif args.depth == 50:
model = resnet50(pretrained=False)
model_ref = resnet50(pretrained=False)
teacher_model = resnet50(pretrained=False)
elif args.depth == 101:
model = resnet101(pretrained=False)
model_ref = resnet101(pretrained=False)
teacher_model = resnet101(pretrained=False)
elif args.depth == 152:
model = resnet152(pretrained=False)
model_ref = resnet152(pretrained=False)
teacher_model = resnet152(pretrained=False)
else:
model = resnet50(pretrained=False) # default Res-50
model_ref = resnet50(pretrained=False) # default Res-50
teacher_model = resnet50(pretrained=False) # default Res-50
model.cuda(device_ids[0]) # model to train (student model)
model_ref.cuda(device_ids[0]) # pruned model
teacher_model.cuda(device_ids[0]) # teacher model, the last epoch of unpruned training model
# ############################### Optimizer and Loss ###############################
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=args.lr,
momentum=args.momentum, weight_decay=args.weight_decay)
# ************* USE APEX *************
if USE_APEX:
print('Use APEX !!! Initialize Model with APEX')
model, optimizer = apex.amp.initialize(model, optimizer, loss_scale='dynamic', verbosity=0)
# ****************** multi-GPU ******************
model = nn.DataParallel(model, device_ids=device_ids)
model_ref = nn.DataParallel(model_ref, device_ids=device_ids)
teacher_model = nn.DataParallel(teacher_model, device_ids=device_ids)
cudnn.benchmark = True
print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0))
# ############################### Resume ###############################
# load pruned model (model_ref), use it to mute some weights of model
title = 'ImageNet'
if args.resume:
# Load checkpoint.
print('==> Getting reference model from checkpoint..')
assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
checkpoint = torch.load(args.resume, map_location=lambda storage, loc: storage)
best_acc = checkpoint['best_acc']
start_epoch = args.start_epoch
model_ref.load_state_dict(checkpoint['state_dict'])
logger = Logger(os.path.join(args.save_dir, 'log_scratch.txt'), title=title)
logger.set_names(['EPOCH', 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
# set some weights to zero, according to model_ref ---------------------------------
# ############## load Lottery Ticket (initialization parameters of un pruned model) ##############
if args.model:
print('==> Loading init model (Lottery Ticket) from %s' % args.model)
checkpoint = torch.load(args.model, map_location=lambda storage, loc: storage)
model.load_state_dict(checkpoint['state_dict'])
if 'init' in args.model:
start_epoch = 0
else:
start_epoch = checkpoint['epoch']
print('Start Epoch ', start_epoch)
for m, m_ref in zip(model.modules(), model_ref.modules()):
if isinstance(m, nn.Conv2d):
weight_copy = m_ref.weight.data.abs().clone()
mask = weight_copy.gt(0).float().cuda()
m.weight.data.mul_(mask)
# ############## load parameters of teacher model ##############
print('==> Loading teacher model (un-pruned) from %s' % args.teacher)
checkpoint = torch.load(args.teacher, map_location=lambda storage, loc: storage)
teacher_model.load_state_dict(checkpoint['state_dict'])
teacher_model.eval()
# ############################### Train and val ###############################
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr']))
num_parameters = get_conv_zero_param(model)
print('Zero parameters: {}'.format(num_parameters))
num_parameters = sum([param.nelement() for param in model.parameters()])
print('Parameters: {}'.format(num_parameters))
# train model
train_loss, train_acc = train(trainloader, model, teacher_model, optimizer, epoch, use_cuda)
# ######## acc on validation data each epoch ########
dev_loss, dev_acc = test(devloader, model, criterion, epoch, use_cuda)
# append logger file
logger.append([ epoch, state['lr'], train_loss, dev_loss, train_acc, dev_acc])
# save model after one epoch
# Note: save all models after one epoch, to help find the best rewind
is_best = dev_acc > best_acc
best_acc = max(dev_acc, best_acc)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': dev_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
}, is_best, checkpoint=args.save_dir, filename=str(epoch + 1)+'_checkpoint.pth.tar')
print('Best val acc:')
print(best_acc)
logger.close()
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data loading code
valdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
print('no DA')
transform_train = transforms.Compose([
transforms.ToTensor(),
normalize,
])
if args.img_size == 32:
train_loader = torch.utils.data.DataLoader(
ImageNet_Train('/BS/database11/ILSVRC2012_imgsize32/train',
transform_train),
# datasets.ImageFolder(traindir, transform_train),
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers)
transform_test = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.Resize(32),
transforms.ToTensor(),
normalize,
])
elif args.img_size == 64:
train_loader = torch.utils.data.DataLoader(
ImageNet_Train_64('/BS/database11/ILSVRC2012_imgsize64/',
transform_train),
# datasets.ImageFolder(traindir, transform_train),
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers)
transform_test = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.Resize(64),
transforms.ToTensor(),
normalize,
])
else:
raise Exception('img size can only be 32 or 64')
val_loader = torch.utils.data.DataLoader(datasets.ImageFolder(
valdir, transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
# create model
if args.pretrained:
print("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=True)
elif args.arch.startswith('d1_resnet50'):
model = models.__dict__[args.arch](
num_classes=1000,
include_top=True,
dropout_rate=0,
layer=args.layer,
is_shuff=False # TODO: check
)
elif args.arch.endswith('vgg16_1d'):
model = models.__dict__[args.arch](num_classes=1000,
include_top=True,
dropout_rate=0,
layer=args.layer,
is_shuff=False)
else:
raise Exception(
'you should only choose vgg16_1d or d1_resnet50 as the model')
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = False
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Resume
title = 'ImageNet-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(val_loader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(train_loader, model, criterion,
optimizer, epoch, use_cuda)
test_loss, test_acc = test(val_loader, model, criterion, epoch,
use_cuda)
# append logger file
logger.append(
[state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
2],
avg_cost[index, 3], avg_cost[index, 4], avg_cost[index, 5],
avg_cost[index, 6], avg_cost[index, 7], avg_cost[index, 8],
avg_cost[index, 9], avg_cost[index, 10], avg_cost[index, 11],
avg_cost[index, 12], avg_cost[index, 13], avg_cost[index, 14],
avg_cost[index, 15], avg_cost[index, 16], avg_cost[index, 17],
avg_cost[index, 18], avg_cost[index, 19], avg_cost[index, 20],
avg_cost[index, 21], avg_cost[index, 22], avg_cost[index, 23]))
logger.append([
index, avg_cost[index, 0], avg_cost[index, 1], avg_cost[index, 2],
avg_cost[index, 3], avg_cost[index, 4], avg_cost[index, 5],
avg_cost[index, 6], avg_cost[index, 7], avg_cost[index, 8],
avg_cost[index, 9], avg_cost[index, 10], avg_cost[index, 11],
avg_cost[index, 12], avg_cost[index, 13], avg_cost[index,
14], avg_cost[index,
15],
avg_cost[index, 16], avg_cost[index, 17], avg_cost[index,
18], avg_cost[index,
19],
avg_cost[index, 20], avg_cost[index, 21], avg_cost[index,
22], avg_cost[index,
23],
dist_loss_save[0].avg, dist_loss_save[1].avg, dist_loss_save[2].avg
])
if isbest:
best_loss = loss_index
print_index = index
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_loss': best_loss,
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
os.makedirs(args.save_dir, exist_ok=True)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
else:
dataloader = datasets.CIFAR100
num_classes = 100
trainset = dataloader(root='./data',
train=True,
download=True,
transform=transform_train)
trainloader = data.DataLoader(trainset,
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers)
testset = dataloader(root='./data',
train=False,
download=False,
transform=transform_test)
testloader = data.DataLoader(testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
model_ref = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('densenet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
model_ref = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
model_ref = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.endswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
)
model_ref = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
)
else:
model = models.__dict__[args.arch](num_classes=num_classes)
model_ref = models.__dict__[args.arch](num_classes=num_classes)
model.cuda()
model_ref.cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay) # default is 0.001
# Resume
title = 'cifar-10-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Getting reference model from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
# args.save_dir = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = args.start_epoch
model_ref.load_state_dict(checkpoint['state_dict'])
logger = Logger(os.path.join(args.save_dir, 'log_scratch.txt'),
title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'
])
# set some weights to zero, according to model_ref ---------------------------------
for m, m_ref in zip(model.modules(), model_ref.modules()):
if isinstance(m, nn.Conv2d):
weight_copy = m_ref.weight.data.abs().clone()
mask = weight_copy.gt(0).float().cuda()
n = mask.sum() / float(m.in_channels)
m.weight.data.normal_(0, math.sqrt(2. / n))
m.weight.data.mul_(mask)
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
num_parameters = get_conv_zero_param(model)
print('Zero parameters: {}'.format(num_parameters))
num_parameters = sum(
[param.nelement() for param in model.parameters()])
print('Parameters: {}'.format(num_parameters))
train_loss, train_acc = train(trainloader, model, criterion, optimizer,
epoch, use_cuda)
test_loss, test_acc = test(testloader, model, criterion, epoch,
use_cuda)
# append logger file
logger.append(
[state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.save_dir)
logger.close()
print('Best acc:')
print(best_acc)
val_acc_log = AverageMeter()
for i, (images, labels) in enumerate(train_loader):
# Convert torch tensor to Variable
images = Variable(images.cuda())
labels = Variable(labels.cuda())
# Forward + Backward + Optimize
optimizer.zero_grad() # zero the gradient buffer
outputs = net(images)
train_loss = criterion(outputs, labels)
if i == 0:
print(labels) # check dataLoader randomness
if epoch == 0:
# loss of the 1st mini-batch in the 1st epoch before backgrop, verify randomness of weight initialization
train_init_loss = train_loss
logger.append([0, train_init_loss, 0, 0, 0])
train_loss.backward()
optimizer.step()
prec1, prec5 = accuracy(outputs.data, labels.data, topk=(1, 5))
train_loss_log.update(train_loss.data[0], images.size(0))
train_acc_log.update(prec1[0], images.size(0))
if (i + 1) % 100 == 0:
print('Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Acc: %.8f' %
(epoch + 1, num_epochs, i + 1, len(train_dataset) //
batch_size, train_loss_log.avg, train_acc_log.avg))
# Test the Model
net.eval()
correct = 0
loss = 0
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.out):
mkdir_p(args.out)
# Data
print(
f'==> Preparing {"asymmetric" if args.asym else "symmetric"} nosiy cifar10'
)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=8),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
transform_val = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
trainset, valset = dataset.get_cifar10('./data',
args,
train=True,
download=True,
transform_train=transform_train,
transform_val=transform_val)
trainloader = data.DataLoader(trainset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4)
valloader = data.DataLoader(valset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4)
# Model
print("==> creating preact_resnet")
model = models.PreActResNet34()
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
val_criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Resume
title = 'noisy-cifar-10'
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.out = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.out, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.out, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
# Train and val
for epoch in range(start_epoch, args.epochs):
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(trainloader, model, optimizer, epoch,
use_cuda)
val_loss, val_acc = validate(valloader, model, val_criterion, epoch,
use_cuda)
# append logger file
logger.append([state['lr'], train_loss, val_loss, train_acc, val_acc])
# save model
is_best = val_acc > best_acc
best_acc = max(val_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': val_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
}, is_best)
logger.close()
logger.plot()
savefig(os.path.join(args.out, 'log.eps'))
print('Best acc:')
print(best_acc)
def main():
global best_acc
if not os.path.isdir(args.out):
mkdir_p(args.out)
# data
num_classes = 3
train_set = CxrDataset(STANFORD_CXR_BASE, "data/Stanford_train_small.csv")
val_set = CxrDataset(STANFORD_CXR_BASE, "data/Stanford_valid.csv")
trainloader = data.DataLoader(train_set,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
drop_last=True)
val_loader = data.DataLoader(val_set,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers)
# model
print("==> creating model")
def create_model(num_classes, ema=False):
model = DenseNet121(num_classes)
model = torch.nn.DataParallel(model).cuda()
if ema:
for param in model.parameters():
param.detach_()
return model
model = create_model(num_classes=num_classes, ema=False)
cudnn.benchmark = True
print('Ttoal params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=args.lr)
start_epoch = 0
# resume
title = 'supervised-Stanford'
if args.resume:
# load checkpoints
print('==> Resuming from checkpoints..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.out = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.out, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.out, 'log.txt'), title=title)
logger.set_names(['Train Loss', 'Valid Loss', 'Valid Acc.'])
writer = SummaryWriter(args.out)
step = 0
test_accs = []
# train and val
for epoch in range(start_epoch, args.epochs):
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_iteration, train_loss = train(trainloader, num_classes, model,
optimizer, criterion, epoch,
use_cuda)
train_acc_iteration, _, train_acc = validate(trainloader,
model,
num_classes,
criterion,
epoch,
use_cuda,
mode='Train Stats')
val_iteration, val_loss, val_acc = validate(val_loader,
model,
num_classes,
criterion,
epoch,
use_cuda,
mode='Valid Stats')
#step = args.val_iteration * (epoch+1)
writer.add_scalar('loss/train_loss', train_loss, (epoch + 1))
writer.add_scalar('loss/valid_loss', val_loss, (epoch + 1))
writer.add_scalar('accuracy/train_acc', train_acc, (epoch + 1))
writer.add_scalar('accuracy/val_acc', val_acc, (epoch + 1))
# append logger file
logger.append([train_loss, val_loss, val_acc])
# save model
is_best = val_acc > best_acc
best_acc = max(val_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': val_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict()
}, is_best)
logger.close()
writer.close()
print('Best acc:')
print(best_acc)
print('Mean acc:')
print(np.mean(val_accs[-20:]))
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data loading code
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
data_aug_scale = (0.08, 1.0) if args.modelsize == 'large' else (0.2, 1.0)
train_loader = torch.utils.data.DataLoader(datasets.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(224, scale=data_aug_scale),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])),
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(datasets.ImageFolder(
valdir,
transforms.Compose([
transforms.Scale(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.test_batch,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
# create model
if args.pretrained:
print("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=True)
elif 'resnext' in args.arch:
model = models.__dict__[args.arch](
baseWidth=args.base_width,
cardinality=args.cardinality,
)
else:
print("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch]()
flops, params = get_model_complexity_info(model, (224, 224),
as_strings=False,
print_per_layer_stat=False)
print('Flops: %.3f' % (flops / 1e9))
print('Params: %.2fM' % (params / 1e6))
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Resume
title = 'ImageNet-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..', args.resume)
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
# model may have more keys
t = model.state_dict()
c = checkpoint['state_dict']
flag = True
for k in t:
if k not in c:
print('not in loading dict! fill it', k, t[k])
c[k] = t[k]
flag = False
model.load_state_dict(c)
if flag:
print('optimizer load old state')
optimizer.load_state_dict(checkpoint['optimizer'])
else:
print('new optimizer !')
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(val_loader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(train_loader, model, criterion,
optimizer, epoch, use_cuda)
test_loss, test_acc = test(val_loader, model, criterion, epoch,
use_cuda)
# append logger file
logger.append(
[state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
print('Best acc:')
print(best_acc)
def select_links_and_channels():
"""
"""
_idx = random.randint(0, len(globs.arch) - 1)
tups = globs.arch[_idx]
links = []
links_to_tx_rx = {}
for tup in tups:
dev_tx = tup[0]
dev_rx = tup[1]
globs.tx_links[dev_tx] = [dev_rx, 'tx', -1]
globs.tx_links[dev_rx] = [dev_tx, 'rx', -1]
l = globs.links[dev_tx][dev_rx]
links.append(l)
links_to_tx_rx[l] = (dev_tx, dev_rx)
t_now = time.time()
links_to_ch = globs.decision_func(links, globs.use_channels)
Logger.append('bs', 'decision_time', time.time() - t_now)
for link, ch in links_to_ch.iteritems():
dev_tx, dev_rx = links_to_tx_rx[link]
# increment counter os used channels
globs.p_channels_count[ch] += 1
globs.allocated_channels.append(ch)
# tx links was {key: [other_side, operation]}
# tx links and now is {key: [other_side, operation, channel]}
globs.tx_links[dev_tx] = [dev_rx, "tx", ch]
globs.tx_links[dev_rx] = [dev_tx, "rx", ch]
#links = copy.deepcopy(globs.tx_links)
#used = []
#for dev1, _tup in links.iteritems():
# if globs.use_channels and dev1 not in used:
# dev2 = _tup[0]
# operation = _tup[1]
# used.extend( [dev1, dev2])
# ## LOGIC TO SELECT CHANNELS
# channel = globs.decision_func()
# # channel can be either a single element OR
# # a dictionary: {link_x: frequency, link_y: frequency y, ...}
# if isinstance(channel, dict):
# print "##### Channel list returned is a dict"
# channel = channel[globs.links[dev1][dev2]]
# ####
# # increment counter os used channels
# globs.p_channels_count[channel] += 1
# globs.allocated_channels.append( channel )
# # tx links was {key: [other_side, operation]}
# # tx links and now is {key: [other_side, operation, channel]}
# globs.tx_links[dev1].append(channel)
# globs.tx_links[dev2].append(channel)
Logger.append('bs', 'links', globs.tx_links)
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
else:
dataloader = datasets.CIFAR100
num_classes = 100
trainset = dataloader(root='./data',
train=True,
download=True,
transform=transform_train)
trainloader = data.DataLoader(trainset,
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers)
testset = dataloader(root='./data',
train=False,
download=False,
transform=transform_test)
testloader = data.DataLoader(testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('densenet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.endswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
)
else:
model = models.__dict__[args.arch](num_classes=num_classes)
# model = torch.nn.DataParallel(model).cuda()
# model = model.cuda(torch.device('cuda:1'))
model = model.cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
torch.save(model, 'tempolary.pth')
new_model = torch.load('tempolary.pth')
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Resume
title = 'cifar-10-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
if DEBUG:
# print(model)
show_low_rank(model, input_size=[32, 32])
print(' Start decomposition:')
look_up_table = get_look_up_table(model)
#print(model)
thresholds = [1e-2] #+0.01*x for x in range(20)]
T = np.array(thresholds)
cr = np.zeros(T.shape)
acc = np.zeros(T.shape)
model_path = 'net.pth'
torch.save(model, model_path)
for i, t in enumerate(thresholds):
test_model = torch.load(model_path)
cr[i] = show_low_rank(test_model,
input_size=[32, 32],
criterion=ValueThreshold(t))
test_model = f_decouple(test_model,
look_up_table,
criterion=ValueThreshold(t),
dataloader=testloader,
train=False)
#print(model)
print(' Done! test decoupled model')
test_loss, test_acc = test(testloader, test_model, criterion,
start_epoch, use_cuda)
print(' Test Loss : %.8f, Test Acc: %.2f' %
(test_loss, test_acc))
acc[i] = test_acc
if args.retrain:
# retrain model
'''
for m in test_model.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
'''
print(' Retrain decoupled model')
finetune_epoch = 4
#args.schedule = [int(finetune_epoch/2), finetune_epoch]
best_acc = 0.0
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
global state
init_lr = args.lr
state['lr'] = init_lr
for epoch in range(finetune_epoch):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, finetune_epoch, state['lr']))
train_loss, train_acc = train(trainloader, test_model,
criterion, optimizer, epoch,
use_cuda)
test_loss, test_acc = test(testloader, test_model,
criterion, epoch, use_cuda)
best_acc = max(test_acc, best_acc)
# append logger file
# logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc])
#logger.close()
#logger.plot()
#savefig(os.path.join(args.checkpoint, 'log.eps'))
acc[i] = best_acc
torch.save(OrderedDict([('acc', acc), ('cr', cr)]),
'result-vh+retrain.pth')
print(cr)
print(acc)
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(trainloader, model, criterion, optimizer,
epoch, use_cuda)
test_loss, test_acc = test(testloader, model, criterion, epoch,
use_cuda)
# append logger file
logger.append(
[state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def cognitive_radio_loop(options, radio, channel_list):
"""
Program loop here.
@param options
@param radio A RadioDevice instance.
@param channel_list List of Channel objects.
"""
# Export my server
command = callback_radio(radio)
my_rpc = RPCExporter(addr=("143.54.83.30", 8000 + options.my_id))
my_rpc.register_function('command', command)
my_rpc.start()
# Wait broker start
while not command.run:
1
####
#Import OTHER RADIOS RPCS
####
rpc_arr = []
for i in [0, 1, 2, 3]:
rpc_cli = RPCImporter(addr="http://%s:%d" % (HOSTS_IP[i], 8000 + i))
rpc_cli.register_function('command')
rpc_arr.append(rpc_cli)
# Import PassiveRadio RPC calls
bs_rpc = RPCImporter(addr="http://%s:9000" % (options.broker_ip))
bs_rpc.register_function('command')
# Register parameters for transmission
Logger.register('radio', ['tx_pkts', 'rx_pkts', 'rx2_pkts', 'channel', 'operation', 'receiver', 'starving',
'total_tx', 'total_rx', 'total_starving'])
# loop
pkt_len = options.pkt_len
payload = struct.pack('%sB' % pkt_len, *[options.my_id] * pkt_len)
print '##### Entering Transmitter loop'
c_starving = 0
while command.run:
"""
######## FUNCOES:
---- BW do canal
---- Channel's bandwidth.
radio.get_bandwidth()
---- Num. simbolos na modulacao
--- Number of symbols in the modulation.
radio.{tx,rx}.symbols()
---- B/S da modulacao
---- B/S of the modulation
radio.{tx,rx}.bits_per_symbol()
---- Pkt/s NO ULTIMO SEGUNDO
---- Pkt/s in the last second.
radio.{tx,rx}.counter.get_pkts()
---- b/s NO ULTIMO SEGUNDO.
---- b/s in the last second.
radio.{tx,rx}.counter.get_bps()
---- Pacotes acumulados desde a ultima chamada.
---- Accumulated packages since the last call.
radio.{tx,rx}.counter.get_pkt_accumulated(clear = False)
---- Troca de canal. channel eh um objeto Channel
---- Channel changing. channel is a Channel object.
radio.set_channel(channel)
#################
"""
# sense
while not command.sense and command.run:
1
if not command.run:
break
sense_data = []
radio.set_gain(0)
for channel in channel_list:
decision, energy = radio.ss.sense_channel(channel, 0.1)
sense_data.append((decision, float(energy), channel.get_channel()))
bs_rpc.command([options.my_id, 'sense_data', sense_data])
# CHOOSE RECEIVER
#while not command.request_transmission:
# 1
# Select a receiver randomly
#opt = [0, 1, 2, 3]
#opt.remove(options.my_id)
#receiver = random.choice(opt)
#print '##### DEVICE %d requesting TX to %d' % (options.my_id, receiver)
#bs_rpc.command([options.my_id, 'request_tx', receiver])
#radio.set_gain(radios_gain[radio.id])
# REQUEST CHANNEL
while not command.request_channel:
1
receiver, operation, channel_idx = bs_rpc.command([options.my_id, 'request_channel', 9999])
# Configure radio. Go to distant frequency if 'idle'
if channel_idx > -1 or operation == OPERATIONS['idle']:
if channel_idx > -1:
radio.set_channel(channel_list[channel_idx])
else:
radio.set_channel(CHANNEL_IDLE)
while not command.transmission:
1
# tx pkts is the number of packets transmitted
# globs.instant_rx is the number of packets received.
# Is global cause it is hard to be synchronized with the TX dev when WE are the RX
# The easiest way is to let the TX dev control when get this value. And we do it in the RPC server method
tx_pkts = globs.instant_rx = 0
if operation == OPERATIONS['tx']:
# change rx freq to another freq or we will receiver our own packets
radio.rx.radio.set_channel(CHANNEL_IDLE)
print "##### ... %d - TRANSMITTING - CHANNEL %d - TO: %d" % (options.my_id, channel_idx, receiver)
tx_pkts = PktSender.flood_by_time(duration=options.sending_duration,
tx_pkt_arch=radio.tx,
payload=payload)
globs.instant_rx = rpc_arr[receiver].command((options.my_id, 'get_accumulated', True))
print "##### ... Transmitted %d/%d pkts in Channel %d" % (tx_pkts, globs.instant_rx, channel_idx)
globs.total_tx += tx_pkts
c_starving = 0
elif operation == OPERATIONS['rx']:
print "##### ... %d - RECEIVING - CHANNEL %d" % (options.my_id, channel_idx)
receiver = 10
globs.wait_for_get = True
while globs.wait_for_get:
time.sleep(0.2)
tx_pkts = 0
globs.total_rx += globs.instant_rx
c_starving += 1
globs.total_starving += 1
elif operation == OPERATIONS['idle']:
print '##### ... %d is IDLE' % options.my_id
receiver = -1
time.sleep(options.sending_duration)
c_starving += 1
globs.total_starving += 1
bs_rpc.command([options.my_id, 'transmission_res', (tx_pkts, globs.instant_rx)])
_idle = 0 if operation != OPERATIONS['idle'] else -1
Logger.append('radio', 'tx_pkts', tx_pkts if _idle > -1 else _idle)
Logger.append('radio', 'rx_pkts', globs.instant_rx if operation == OPERATIONS['tx'] else _idle)
Logger.append('radio', 'rx2_pkts', globs.instant_rx if operation == OPERATIONS['rx'] else _idle)
Logger.append('radio', 'channel', channel_idx)
Logger.append('radio', 'operation', operation)
Logger.append('radio', 'receiver', receiver)
Logger.append('radio', 'starving', c_starving)
Logger.set('radio', 'total_tx', globs.total_tx)
Logger.set('radio', 'total_rx', globs.total_rx)
Logger.set('radio', 'total_starving', globs.total_starving)
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Dataset preprocessing
title = 'GTSRB'
# Create Datasets
transform_train_poisoned = transforms.Compose([
transforms.Resize((32, 32)),
TriggerAppending(trigger=args.trigger, alpha=args.alpha),
transforms.ToTensor(),
])
transform_train_benign = transforms.Compose([
transforms.Resize((32, 32)),
transforms.ToTensor(),
])
transform_test_poisoned = transforms.Compose([
transforms.Resize((32, 32)),
TriggerAppending(trigger=args.trigger, alpha=args.alpha),
transforms.ToTensor(),
])
transform_test_benign = transforms.Compose([
transforms.Resize((32, 32)),
transforms.ToTensor(),
])
print('==> Loading the dataset')
# Create Datasets
num_training = len(
dataset.GTSRB(root_dir='./data',
train=True,
transform=transform_train_benign))
num_poisoned = int(num_training * args.poison_rate)
idx = list(np.arange(num_training))
random.shuffle(idx)
poisoned_idx = idx[:num_poisoned]
benign_idx = idx[num_poisoned:]
poisoned_trainset = dataset.GTSRB_subset(
root_dir='./data',
train=True,
transform=transform_train_poisoned,
List=poisoned_idx,
y_target=args.y_target)
benign_trainset = dataset.GTSRB_subset(root_dir='./data',
train=True,
transform=transform_train_benign,
List=benign_idx,
y_target=None)
poisoned_testset = dataset.GTSRB(root_dir='./data',
train=False,
transform=transform_test_poisoned,
y_target=args.y_target)
benign_testset = dataset.GTSRB(root_dir='./data',
train=False,
transform=transform_test_benign,
y_target=None)
poisoned_trainloader = torch.utils.data.DataLoader(
poisoned_trainset,
batch_size=int(args.train_batch * args.poison_rate),
shuffle=True,
num_workers=args.workers)
benign_trainloader = torch.utils.data.DataLoader(
benign_trainset,
batch_size=int(args.train_batch * (1 - args.poison_rate) * 0.9),
shuffle=True,
num_workers=args.workers
) # *0.9 to prevent the iterations of benign data is less than that of poisoned data
poisoned_testloader = torch.utils.data.DataLoader(
poisoned_testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
benign_testloader = torch.utils.data.DataLoader(benign_testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
print(
"Num of training samples %i (Num of poisoned samples %i, Num of benign samples %i), Num of testing samples %i"
% (num_training, num_poisoned, num_training - num_poisoned,
len(benign_testset)))
# Model
model = ResNet18()
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print('Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Resume
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Benign Valid Loss',
'Poisoned Valid Loss', 'Train ACC.', 'Benign Valid ACC.',
'Poisoned Valid ACC.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(args, model, poisoned_trainloader,
benign_trainloader, criterion, optimizer,
epoch, use_cuda)
test_loss_benign, test_acc_benign = test(benign_testloader, model,
criterion, epoch, use_cuda)
test_loss_poisoned, test_acc_poisoned = test(poisoned_testloader,
model, criterion, epoch,
use_cuda)
# append logger file
logger.append([
state['lr'], train_loss, test_loss_benign, test_loss_poisoned,
train_acc, test_acc_benign, test_acc_poisoned
])
# save model
is_best = test_acc_benign > best_acc
best_acc = max(test_acc_benign, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc_benign,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def server_loop(options):
"""
Receiver radios in this loop
@param options
"""
# Export my 'command' function to be accessible via RPC
my_rpc = RPCExporter(addr=("143.54.83.30", 9000))
command = server_callback(globs.channel_list)
my_rpc.register_function('command', command)
my_rpc.start()
interferer_rpc = RPCImporter(addr="http://143.54.83.30:9001")
interferer_rpc.register_function('command')
####
#Import OTHER RADIOS RPCS
####
for i in DEVICES:
rpc_cli = RPCImporter(addr="http://143.54.83.30:%d" % (8000 + i))
rpc_cli.register_function('command')
globs.rpc_arr.append(rpc_cli)
def command_sense(val):
"""
@param val
"""
[x.command([BS_ID, 'sense', val]) for x in globs.rpc_arr]
def command_request_transmission(val):
"""
@param val
"""
[x.command([BS_ID, 'request_transmission', val]) for x in globs.rpc_arr]
def command_request_channel(val):
"""
@param val
"""
[x.command([BS_ID, 'request_channel', val]) for x in globs.rpc_arr]
def command_transmission(val):
"""
@param val
"""
[x.command([BS_ID, 'transmission', val]) for x in globs.rpc_arr]
# Start execution on all RC nodes
interferer_rpc.command([BS_ID, 'run', True])
[x.command([BS_ID, 'run', True]) for x in globs.rpc_arr]
t_fin = time.time() + options.test_duration
while t_fin > time.time():
t_it = time.time()
print "!!!!! SENSING PHASE"
command_sense(True)
while not globs.sense_done:
1
globs.sense_done = False
command_sense(False)
print "!!!!! TRANSMISSION PHASE"
command_transmission(True)
globs.p_fin_transmission_time = time.time() + options.sending_duration
while not globs.transmission_done:
1
globs.p_fin_transmission_time = None
## moment to evaluate links
# HERE
####
# clear all used data
globs.transmission_done = False
globs.clear()
globs.p_total_iterations += 1
command_transmission(False)
Logger.append('bs', 'it_dur', time.time() - t_it)
# Stop execution on all CR nodes
interferer_rpc.command([BS_ID, 'run', False])
[x.command([BS_ID, 'run', False]) for x in globs.rpc_arr]
def transmitter_loop(tb, channel_list, channel, options):
"""
US LOOP
@param tb
@param channel_list
@param channel
@param options
"""
# Q-Noise+ Parameters
lookback = 3
beta = 0.1
eps = 0.1
history_weight = [0.2, 0.35, 0.45]
noise_weight = 0.8
alpha = 0.2 # alpha + noise+weith = 1
epochs = 100
#Q-Noise+
learner = qnoise.Learner(channel_list, lookback, alpha, beta, eps, history_weight, noise_weight)
# Connect to slave device
import xmlrpclib
proxy = xmlrpclib.ServerProxy("http://%s:8000/" % options.slave_addr)
start_t = time.time()
proxy.client_started()
proxy.set_channel(channel)
Logger.register('transmitter', ['channel', 'status', 'pkt'])
class TNamespace():
"""
"""
pass
# Sensing -> TX loop
t_namespace = TNamespace()
t_namespace.pkt_s = 0
t_namespace.status = 0
t_namespace.iteration = 0
t_namespace.ss_result = {}
for ch in channel_list:
t_namespace.ss_result[ch.channel] = [0, 0, 0]
ch_qvalue = 0
while time.time() < (start_t + options.duration):
can_transmit = True
t_namespace.pkt_r = t_namespace.pkt_s = 0
# Sense
print '### START SENSING CHANNEL'
t_namespace.status = tb.rx.sense_channel(channel_list[channel], options.sensing_duration)
# Update
#print t_namespace.status
if t_namespace.status > 0.000005: # GMSK threahold
#if t_namespace.status > 0.000000005 :
print str(channel_list[channel]) + ' is occupied'
t_now = time.clock()
can_transmit = True
# Change channel
#proxy.set_channel( channel )
# Transmit
if can_transmit:
payload = 0
if options.pkt_size > 1:
bytelist = [1] * (options.pkt_size/4)
payload = pack('%sH' % len(bytelist), *bytelist)
else:
bytelist = ['a', ]
payload = pack('%sc' % 1, *bytelist)
# thread sending packets
def send_thread():
t_namespace.pkt_s = 0
#while t_namespace.pkt_sending:
while t_namespace.pkt_s < 200:
tb.tx.send_pkt(payload)
t_namespace.pkt_s += 1
#print 't: ', t_namespace.pkt_s
time.sleep(0.03)
#t_namespace.count += 1
# init thread
th = Thread(target=send_thread)
proxy.start_rcv()
t_namespace.pkt_sending = True
th.start()
# wait for options.sending_duration
#time.sleep( options.sending_duration )
# stop sending
t_namespace.pkt_sending = False
th.join()
t_namespace.pkt_r = proxy.get_rcv()
print 'pkt_s = ', t_namespace.pkt_s
print 'pkt_r = ', t_namespace.pkt_r
ch_qvalue = learner.evaluate(channel_list[channel].channel, t_namespace.pkt_s, t_namespace.pkt_r,
t_namespace.status)
t_namespace.ss_result[channel+1] = [t_namespace.status, 0 if can_transmit else 1, ch_qvalue]
channel = learner.choose_next_channel(channel_list[channel].channel) - 1 # -1 cause we use the index in the array.
print "Using channel ", channel_list[channel]
proxy.set_channel(channel)
tb.tx.radio.center_freq = channel_list[channel]
Logger.append('transmitter', 'channel', channel)
Logger.append('transmitter', 'status', t_namespace.status)
Logger.append('transmitter', 'pkt', t_namespace.pkt_s)
dump_qlearner(t_namespace.iteration, t_namespace.ss_result, noise_weight)
t_namespace.iteration += 1
proxy.close_app()
