def sensing_decision(self, sensing_result, number_cpes): """ Algorithm sensing decision. @param sensing_result @param number_cpes """ #print "simple cycle" #feedback cycle control self._fb_cycle += 1 #initialize sum of scores for each decision (0,1) sum_result0 = 0.0 sum_result1 = 0.0 #keeps the greatest reward for each decision greatest_reward0 = 0.0 greatest_reward1 = 0.0 #sum of all scores for each decision for decision, reward in zip(sensing_result, self._reward): #sum of scores for "0" decision if decision == IDLE: sum_result0 += reward if reward > greatest_reward0: greatest_reward0 = reward #sum of scores for "1" decision elif decision == OCCUPIED: sum_result1 += reward if reward > greatest_reward1: greatest_reward1 = reward #divide the sum of scores by the total number of CPEs score_r1 = sum_result1 / number_cpes score_r0 = sum_result0 / number_cpes #verifies which sum of scores is higher, score1 or score0 if score_r0 > score_r1: sensing_decision = IDLE elif score_r0 < score_r1: sensing_decision = OCCUPIED #if both scores are equal, then verifies the decision made by the cpe with the greatest individual score elif score_r0 == score_r1: if greatest_reward0 >= greatest_reward1: sensing_decision = IDLE elif greatest_reward0 < greatest_reward1: sensing_decision = OCCUPIED #verifies if is feedback cycle #if (self._fb_cycle % (feedback_control-1) == 0): Logger.append('sdc', 'decision', sensing_decision) if sensing_decision == OCCUPIED: self._total_occ += 1 else: self._total_idle += 1 self.feedback(sensing_result, sensing_decision, increase_rate, decrease_rate)
def decision(self, energy): """ @param energy """ energy = np.sum(energy) / energy.size self.cycle_counter += 1 if self.cycle_counter_max == self.cycle_counter: self.cycle_counter = 0 sp = self.do_action(self.s, self.a) rw = self.get_reward(energy, sp) ap = self.e_greedy_selection(sp) self.update_q_table(self.s, self.a, rw, sp, ap) self.s = sp self.a = ap #self.epsilon *= 0.999 Logger.append('bayes_learning', 'hypothesis', 1.0 if energy > self.s else 0.0) Logger.append('bayes_learning', 'feedback', self._feedback) Logger.append('bayes_learning', 'state', self.s) Logger.append('bayes_learning', 'reward', rw) Logger.append('bayes_learning', 'action', self.a) return 1 if (energy > sp) else 0, energy
def work(self, input_items, output_items): for idx in range(len(input_items[0])): decf, vf = self.algo1.decision(input_items[0][idx]) if decf == 0: decf, vf = self.algo2.decision(input_items[1][idx]) Logger.append('hier', 'decision', self._xx[Logger._ch_status][decf]) return len(input_items[0])
def _tick(self): """ Called automatically each one second. """ for _d in (self._bps, self._pkts): _d['cur'] = _d['counting'] _d['counting'] = 0 Logger.append(self._name, 'bps', self._bps['cur'] ) #pylint: disable=E1101 Logger.append(self._name, 'pkts', self._pkts['cur']) #pylint: disable=E1101
def get_pkt_accumulated(self, clear=False): """ @param clear """ tmp = self._pkts['accumulated'] Logger.append(self._name, 'pkt_accumulated', self._pkts['accumulated']) # pylint: disable=E1101 if clear: self._pkts['accumulated'] = 0 return tmp
def decision(self, data_in): """" Implementation of base class abstract method. @param data_in Mag squared of samples. @return Tuple (status, energy) """ energy = np.sum(data_in) / data_in.size dec = 1 if self.threshold < energy else 0 Logger.append('energy_decision', 'energy', energy) Logger.append('energy_decision', 'decision', self._xx[Logger._ch_status][dec]) return dec, energy
def decision(self, data_in): """" Implementation of base class abstract method. @param data_in Mag squared of samples. @return Tuple (status, energy) """ _sum = self._algorithm.calculate_cyclo(data_in.tolist()) _sum = _sum / len(data_in) dec = 1 if self.threshold < _sum else 0 Logger.append('cyclo_decision', 'decision', self._xx[Logger._ch_status][dec]) return dec, _sum
def decision(self, data_in): """ Called from a signal processing block to made a decision. @param data_in Mag squared of samples. @return Tuple (status, correlation). """ max_corr = -1.0 for wave in self._waveforms: max_corr = max(abs(self.correlate(wave, data_in)), max_corr) dec = 1 if self.threshold < max_corr else 0 Logger.append('waveform_decision', 'decision', self._xx[Logger._ch_status][dec]) return dec, max_corr
def decision(self, data_l, data_m): """ Function called from a signal processing block. @param data_l Learner decision regarding channel occupancy. @param data_m Manager decision regarding channel occupancy. """ self.strategy.wait() self._iteraction += 1 final_dec = data_l if self._valid_feedback: final_dec = data_m self._time += 19.3 self._count += 1 Logger.set('feedback_algorithm', 'total_feedback', self._count) Logger.append('feedback_algorithm', 'activation', int(data_m)) # set feedback in our learning algorithm self.learner.feedback = data_m # Increase feedback interval if both algorithms are correct if data_l == data_m: self.strategy.increase_time() # else decrease time else: self.strategy.decrease_time() else: Logger.append('feedback_algorithm', 'activation', -1) self._time += 0.2 self._valid_feedback = False if self.strategy.feedback(): self._manager.enable(True) self._valid_feedback = True Logger.append('feedback_algorithm', 'time', self._time) Logger.append('feedback_algorithm', 'count', self._count) Logger.append('bayes_decision', 'hypothesis', final_dec)
def work(self, input_items, output_items): """ @param input_items @param output_items """ for idx in range(len(input_items[0])): self._iteraction += 1 ed_dec = input_items[0][idx][0] wf = input_items[1][idx] final_dec = ed_dec if ed_dec == 0: final_dec = 1 final_dec = self._algorithm.decision(wf)[0] Logger.set('feedback_algorithm', 'total_feedback', self._count) Logger.append('bayes_decision', 'hypothesis', final_dec) return len(input_items[0])
def __init__(self, name="SNREstimator", algorithm=SVR, alpha=0.001): """ CTOR @param name @param algorithm @param alpha """ self._estimator = digital.probe_mpsk_snr_est_c(algorithm, 10000, alpha) UHDGenericArch.__init__(self, name=name, input_signature=self._estimator.input_signature(), output_signature=self._estimator.output_signature()) Logger.register(name, ['snr', ]) self.register_scheduling(lambda: Logger.append(name, 'snr', self.get_snr()), delay_sec=0.2) #pylint: disable=E1101
def decision(self, signal): """ Method called for a USRP block (probably bayesian_detector). @param energy Signal Energy @param signal """ #hypothesis # update counters if self._feedback == 0 or self._feedback == 1: self.update_global_counter() th = self._th r = self._r[th] while True: # update bayes self.update_th_counter(signal) # update risk bayes_r, self._th = self.get_min_risk() # invalidate threshold. Wait until a new is provided if th == self._th or r == bayes_r : self.feedback = -1 break th = self._th r = self._r[th] # Clear black list of threshold self._black_list = [] #print "d:", self.bayesian_hypothesis(self._th, signal), " f:", self.feedback, " t:", math.e ** self._th, " e:", sum(signal)/50 bayes_hyp = self.bayesian_hypothesis(self._th, signal) # Save all data dec = 1 if bayes_hyp > 0 else 0 Logger.append('bayes_decision', 'decision', self._xx[Logger._ch_status][dec]) Logger.append('bayes_decision', 'threshold', self._th) Logger.append('bayes_decision', 'risk', self._r[self._th]) return dec, 0.0
def transmitter_loop(tb, channel_list, channel, options): """ US LOOP @param tb @param channel_list @param channel @param options """ # Connect to slave device import xmlrpclib proxy = xmlrpclib.ServerProxy("http://%s:8000/" % options.slave_addr) start_t = time.time() proxy.client_started() proxy.set_channel(channel) Logger.register('transmitter', ['channel', 'status', 'pkt']) class TNamespace(): """ """ pass # Sensing -> TX loop t_namespace = TNamespace() t_namespace.pkt_s = 0 t_namespace.status = 0 while time.time() < (start_t + options.duration): can_transmit = True if not options.tx_only: # Sense decision, t_namespace.status = tb.rx.sense_channel(channel_list[channel], options.sensing_duration) # Update print t_namespace.status if t_namespace.status > 0.000005: # GMSK threahold #if t_namespace.status > 0.000000005 : print str(channel_list[channel]) + ' is occupied' t_now = time.clock() ## Q-NOISE AQUI. channel = (channel + 1) % len(channel_list) #### can_transmit = False # Change channel proxy.set_channel(channel) tb.tx.center_freq = channel_list[channel] tb.rx.center_freq = channel_list[channel] # Transmit if can_transmit: payload = 0 if options.pkt_size > 1: bytelist = [1] * (options.pkt_size/4) payload = pack('%sH' % len(bytelist), *bytelist) else: bytelist = ['a', ] payload = pack('%sc' % 1, *bytelist) # thred sending packets def send_thread(): while t_namespace.pkt_sending: tb.tx.send_pkt(payload) t_namespace.pkt_s += 1 #t_namespace.count += 1 # init thread th = Thread(target=send_thread) t_namespace.pkt_sending = True th.start() # wait for options.sending_duration time.sleep(options.sending_duration) # stop sending t_namespace.pkt_sending = False th.join() Logger.append('transmitter', 'channel', channel) Logger.append('transmitter', 'status', t_namespace.status) Logger.append('transmitter', 'pkt', t_namespace.pkt_s) proxy.close_app()
def train_model(model, criterion, optimizer, scheduler): since = time.time() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() best_model_wts = model.state_dict() best_prec1 = 0 train_loader = dataloders['train'] train_batch_num = batch_nums['train'] start_epoch = args.start_epoch stop = 0 log_file = os.path.join(args.checkpoint, 'log.txt') title = 'cifar-10-' + args.arch if args.resume: print('==> Resuming from checkpoint...') print(args.resume) assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_prec1 = checkpoint['best_prec1'] start_epoch = checkpoint['epoch']+1 model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(log_file, title=title, resume=True) else: if os.path.exists(log_file) and not args.overwrite: logprint.info('%s exists. skip', log_file) return 0 logger = Logger(log_file, title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) for epoch in range(start_epoch, args.epochs): scheduler.step() model.train() # Set model to training mode print('Epoch [{} | {}] LR: {}'.format(epoch, args.epochs - 1, scheduler.get_lr()[0])) print('-' * 10) progbar = Progbar(train_batch_num) # Iterate over data. for batch_index, data in enumerate(train_loader): # get the inputs inputs, labels = data # wrap them in Variable if use_gpu: inputs = Variable(inputs.cuda()) labels = Variable(labels.cuda(async=True)) else: inputs, labels = Variable(inputs), Variable(labels) # forward outputs = model(inputs) loss = criterion(outputs, labels) prec1, prec5 = accuracy(outputs.data, labels.data, topk=(1, 5)) losses.update(loss.data[0], inputs.size(0)) top1.update(prec1[0], inputs.size(0)) top5.update(prec5[0], inputs.size(0)) # backward + optimize optimizer.zero_grad() loss.backward() optimizer.step() # statistics logger.append([scheduler.get_lr()[0], losses.avg, None, top1.avg, None]) progbar.add(1, values=[("p1", top1.avg), ("p5", top5.avg), ("loss", losses.avg)]) # end of an epoch print() print('Train Loss epoch {epoch} {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( epoch=epoch, loss=losses, top1=top1, top5=top5)) # evaluate on validation set prec1 = validate(dataloders['val'], model, criterion, logger) is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) stop += 1 print(' * Best Prec@1 {top1:.3f}'.format(top1=best_prec1)) save_checkpoint({ 'epoch': epoch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer' : optimizer.state_dict(), }, is_best, filename='checkpoint_epoch{epoch}.pth.tar'.format(epoch=epoch), checkpoint=args.checkpoint) # deep copy the model if is_best: best_model_wts = model.state_dict() print ('better model obtained at epoch {epoch}'.format(epoch=epoch)) stop = 0 if(stop >= 20): print("\nEarly stop happend at {}\n".format(epoch)) break print() time_elapsed = time.time() - since print('Training complete in {:.0f}m {:.0f}s'.format( time_elapsed // 60, time_elapsed % 60)) print('Best val Acc: {:4f}'.format(best_prec1)) # load best model weights model.load_state_dict(best_model_wts) return model
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.Resize((256, 256)), transforms.ToTensor(), #transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)), ]) transform_test = transforms.Compose([ transforms.Resize((256, 256)), transforms.ToTensor(), #transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)), ]) """ if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 else: dataloader = datasets.CIFAR100 num_classes = 100 """ bird_dataset = datasets.ImageFolder(root="./", transform=data_transform) ################################################### """ n_train = int(len(bird_dataset) * 0.6) n_val = len(bird_dataset) *0.2 n_test = len(bird_dataset) - n_train - n_val trainset, valset, testset = torch.utils.data.random_split( bird_dataset, [n_train, n_val, n_test] ) """ n_train = int(len(bird_dataset) * 0.6) n_val = len(bird_dataset) - n_val trainset, valset, testset = torch.utils.data.random_split( bird_dataset, [n_train, n_val]) ################################################### #trainset = dataloader(root='./data', train=True, download=True, transform=transform_train) #trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) trainloader = torch.utils.data.DataLoader(trainset, batch_size, shuffle=True) #testset = dataloader(root='./data', train=False, download=False, transform=transform_test) #testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) test_loader = torch.utils.data.DataLoader(testset, batch_size) # Model print("==> creating model '{}'".format(args.arch)) model = Net() #モデルの呼び出し、今回は簡易版のため畳み込み3層、convlstm cell1層、全結合から構成される model = torch.nn.DataParallel(model).to(device=device) cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'cifar-10-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda) # append logger file logger.append( [state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data loading code args.data = '/Users/io/workspace' traindir = os.path.join(args.data, 'val') valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_loader = torch.utils.data.DataLoader( datasets.ImageFolder(traindir, transforms.Compose([ transforms.RandomSizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])), batch_size=args.train_batch, shuffle=True, num_workers=args.workers, pin_memory=False) val_loader = torch.utils.data.DataLoader( datasets.ImageFolder(valdir, transforms.Compose([ transforms.Scale(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=False) # create model args.pretrained = True if args.pretrained: print("=> using pre-trained model '{}'".format(args.arch)) model = res.__dict__[args.arch](pretrained=True) else: print("=> creating model '{}'".format(args.arch)) model = res.__dict__[args.arch]() if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'ImageNet-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) args.evaluate = True if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(val_loader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(val_loader, model, criterion, epoch, use_cuda) # append logger file logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def main_worker(gpu, ngpus_per_node, args): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) args.gpu = gpu if args.gpu is not None: print("Use GPU: {} for training".format(args.gpu)) if args.distributed: if args.dist_url == "env://" and args.rank == -1: args.rank = int(os.environ["RANK"]) if args.multiprocessing_distributed: # For multiprocessing distributed training, rank needs to be the # global rank among all the processes args.rank = args.rank * ngpus_per_node + gpu torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank) # Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), # with p = 1 transforms.RandomHorizontalFlip(), # with p = 0.5 transforms.ToTensor(), # it must be this guy that makes it CHW again transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 else: dataloader = datasets.CIFAR100 num_classes = 100 # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, ) elif args.arch.startswith('resnet50'): model = models.__dict__[args.arch](num_classes=num_classes, include_top=True, dropout_rate=0, layer=args.layer) elif args.arch.startswith('d1_resnet50'): model = models.__dict__[args.arch](num_classes=num_classes, include_top=True, dropout_rate=0, layer=args.layer, is_shuff=args.shuff) elif args.arch.endswith('vgg16'): model = models.__dict__[args.arch](num_classes=num_classes, include_top=True, dropout_rate=0, layer=args.layer) elif args.arch.endswith('vgg16_sa'): model = models.__dict__[args.arch](num_classes=num_classes, include_top=True, dropout_rate=0, layer=args.layer) elif args.arch.endswith('vgg16_1d'): model = models.__dict__[args.arch](num_classes=num_classes, include_top=True, dropout_rate=0, layer=args.layer, is_shuff=args.shuff) else: model = models.__dict__[args.arch](num_classes=num_classes) if args.distributed: # For multiprocessing distributed, DistributedDataParallel constructor # should always set the single device scope, otherwise, # DistributedDataParallel will use all available devices. if args.gpu is not None: torch.cuda.set_device(args.gpu) model.cuda(args.gpu) # When using a single GPU per process and per # DistributedDataParallel, we need to divide the batch size # ourselves based on the total number of GPUs we have args.train_batch = int(args.train_batch / ngpus_per_node) args.test_batch = int(args.workers / ngpus_per_node) args.workers = int(args.workers / ngpus_per_node) model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[args.gpu]) else: model.cuda() # DistributedDataParallel will divide and allocate batch_size to all # available GPUs if device_ids are not set model = torch.nn.parallel.DistributedDataParallel(model) elif args.gpu is not None: torch.cuda.set_device(args.gpu) model = model.cuda(args.gpu) else: # DataParallel will divide and allocate batch_size to all available GPUs if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) # for name, param in model.named_parameters(): # print(name) # for name in model.named_modules(): # print(name) # for param in model.parameters(): # print(param) trainset = dataloader(root='/data/users/yuefan/fanyue/dconv/data', train=True, download=True, transform=transform_train) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler( trainset) else: train_sampler = None trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True, sampler=train_sampler) testset = dataloader(root='/data/users/yuefan/fanyue/dconv/data', train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=True) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=args.weight_decay) # Resume title = 'cifar-10-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): if args.distributed: train_sampler.set_epoch(epoch) adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, args) test_loss, test_acc = test(testloader, model, criterion, args) # append logger file logger.append( [state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) if not args.multiprocessing_distributed or ( args.multiprocessing_distributed and args.rank % ngpus_per_node == 0): save_checkpoint( { 'epoch': epoch + 1, 'acc': test_acc, 'state_dict': model.state_dict(), 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def main(): # Use CUDA os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") cudnn.benchmark = True # Random seed if args.manualSeed is None: args.manualSeed = random.randint(1, 10000) random.seed(args.manualSeed) torch.manual_seed(args.manualSeed) if torch.cuda.is_available(): torch.cuda.manual_seed_all(args.manualSeed) # allow parallel training def to_parallel(model): if torch.cuda.device_count() > 1: print("Let's use", torch.cuda.device_count(), "GPUs!") return torch.nn.DataParallel(model) return model # make output dir if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Load dataset trainloader, valloader, testloader, num_classes = get_loaders( dataset=args.dataset, download=False, train_batch=args.train_batch, test_batch=args.test_batch, n_workers=args.workers, data_dir='./data') # Init model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnet') or args.arch.startswith('preresnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, block_name=args.block_name, ) else: model = models.__dict__[args.arch](num_classes=num_classes) print(" Total params: %.2fM" % (sum(p.numel() for p in model.parameters()) / 1000000.0)) to_parallel(model).to(device) # Set model Lipschitz hooker print("==> set Lipschitz hooker ") hooker = LipHooker(args.arch, args.checkpoint, device=device) hooker.hook(model) # Set criterion and optimizer criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Set learning rate scheduler print("==> creating scheduler '{}'".format(args.scheduler)) if args.scheduler.startswith('constant'): scheduler = schedulers.__dict__[args.scheduler](optimizer=optimizer, dpath=args.checkpoint) elif args.scheduler.startswith('step'): scheduler = schedulers.__dict__[args.scheduler]( optimizer=optimizer, milestones=args.schedule, gamma=args.gamma, dpath=args.checkpoint) elif args.scheduler.startswith('cosine'): scheduler = schedulers.__dict__[args.scheduler]( optimizer=optimizer, milestones=args.schedule, epochs=args.epochs, dpath=args.checkpoint) elif args.scheduler.startswith('adacosine'): scheduler = schedulers.__dict__[args.scheduler](optimizer=optimizer, epochs=args.epochs, dpath=args.checkpoint) else: raise KeyError(args.scheduler) # Set info logger title = args.dataset + '-' + args.arch logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Epoch', 'Time Elapsed', 'Learning Rate', 'Train Loss', 'Valid Loss', 'Test Loss', 'Train Acc.', 'Valid Acc.', 'Test Acc.' ]) # ---------- grow ----------- # Set model architecture tracker for grow modelArch = None if args.grow: modelArch = ModelArch(args.arch, model, args.epochs, args.depth, max_depth=args.max_depth, dpath=args.checkpoint, dataset=args.dataset) # Set trigger for grow if args.grow and args.mode == 'adapt': trigger = TolTrigger(tolerance=args.threshold, window=args.window, reserve=args.reserve, epochs=args.epochs, modelArch=modelArch) # Training start print("==> training start - epochs: %i" % args.epochs) time_start = time.time() best_val_acc = 0 # best test accuracy best_epoch = 0 for epoch in range(args.epochs): train_loss, train_acc = train(trainloader, model, criterion, optimizer, debug_batch_size=args.debug_batch_size, device=device) val_loss, val_acc = test(valloader, model, criterion, device=device) test_loss, test_acc = test(testloader, model, criterion) # print('\nEpoch: [%d | %d] LR: %f Train-Loss: %.4f Val-Loss: %.4f Train-Acc: %.4f Val-Acc: %.4f' % (epoch + 1, args.epochs, scheduler.lr_(), train_loss, val_loss, train_acc, val_acc)) logger.append([ epoch, (time.time() - time_start) / 60., scheduler.lr_(), train_loss, val_loss, test_loss, train_acc, val_acc, test_acc ]) # save model is_best = val_acc > best_val_acc if is_best: best_epoch = epoch + 1 best_val_acc = max(val_acc, best_val_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': val_acc, 'best_val_acc': best_val_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) # update grower and scheduler if args.grow: modelArch.update(epoch, is_best, model) errs = None if args.hooker: errs = hooker.output(epoch) scheduler.step_(epoch, errs) # grow if args.grow: if args.mode == 'fixed': # grow specified at fixed epochs if epoch + 1 in args.grow_epoch: modelArch.grow(1) # dummy grow model = models.__dict__[args.arch]( num_classes=num_classes, block_name=args.block_name, archs=modelArch.arch) to_parallel(model).to(device) model.load_state_dict( modelArch.state_dict.state_dict, strict=False ) # True) # False due to buffer added during training to calculate lipschitz if args.scheduler == 'cosine' and not args.schedule: # cases that learning rate is continued optimizer = optim.SGD(model.parameters(), lr=scheduler.lr_(), momentum=args.momentum, weight_decay=args.weight_decay) else: # cases that learning rate is reset, aka lr restart optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) ''' probably have to copy the entire momentum history for each weight but here just initialize the optimizer again ''' # if multi epoch cosine or cosine_restart scheduler.update(optimizer, epoch=epoch) if args.hooker: hooker.hook(model) modelArch.record(epoch, model) elif args.mode == 'adapt': # grow happens automatically assert args.hooker, 'For adaptive training, model hooker must be provided to monitor some stats' trigger.feed(errs) err_indices = trigger.trigger( epoch, modelArch.get_num_blocks_all_layer( )) # propose candidate blocks to be growed if err_indices: err_indices = modelArch.grow( err_indices ) # try duplicate it to see if any layer exceeds upper limit if err_indices: model = models.__dict__[args.arch]( num_classes=num_classes, block_name=args.block_name, archs=modelArch.arch) to_parallel(model).to(device) model.load_state_dict( modelArch.state_dict.state_dict, strict=False ) # not strict matching due to Lipschitz buffer if args.scheduler == 'cosine': # cases that learning rate is continued optimizer = optim.SGD( model.parameters(), lr=scheduler.lr_(), momentum=args.momentum, weight_decay=args.weight_decay) else: # cases that learning rate is reset, aka lr restart optimizer = optim.SGD( model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) if args.hooker: hooker.hook(model) trigger.update( err_indices) # reset history errors in trigger scheduler.update( optimizer, epoch=epoch) # update optimizer in lr scheduler modelArch.record( epoch, model) # keep the current architecture for record else: raise KeyError('Grow mode %s not supported!' % args.mode) # print growing stats if args.grow: print('\nGrow epochs: ', modelArch.grow_epochs[1:], end=', ') print('Num parameters: ', modelArch.num_parameters, end=', ') print('PPE: %.2f' % modelArch._get_ppe()) # evaluate best model print('Best val acc: %.4f at %i' % (best_val_acc, best_epoch)) # this is the validation acc best_checkpoint = torch.load( os.path.join(args.checkpoint, 'model_best.pth.tar')) if args.grow: best_model = models.__dict__[args.arch](num_classes=num_classes, block_name=args.block_name, archs=modelArch.best_arch) else: best_model = models.__dict__[args.arch](num_classes=num_classes, depth=args.depth, block_name=args.block_name) if torch.cuda.device_count() > 1: print("Let's use", torch.cuda.device_count(), "GPUs!") best_model = torch.nn.DataParallel(best_model) best_model.to(device) # -- best_model.load_state_dict(best_checkpoint['state_dict'], strict=False) test_loss, test_acc = test(testloader, best_model, criterion) if args.grow: print('Best arch: %s' % modelArch.__str__(best=True), end=', ') print('Best Test Loss: %.4f, Best Test Acc: %.4f' % (test_loss, test_acc)) # evaluate final model test_loss, test_acc = test(testloader, model, criterion) if args.grow: print('Final arch: %s' % modelArch, end=', ') print('Final Test Loss: %.4f, Final Test Acc: %.4f' % (test_loss, test_acc)) print('Wall time: %.3f mins' % ((time.time() - time_start) / 60)) # round off scheduler.close() if args.hooker: hooker.close() if args.grow: modelArch.close() if args.mode == 'adapt': trigger.close() logger.close()
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) #### Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.Resize([120, 120]), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_train_aug = transforms.Compose([ transforms.Resize([120, 120]), transforms.RandomHorizontalFlip(), #transforms.RandomCrop([60,60]), transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.5), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.Resize([120, 120]), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) #### path #### path = '/home/tzh/Documents/working_jinyu/pytorch-classification-master/Office31_Dataset_Yurong/Single-Domain-Adaptation/A-2-D/TRAIN' path_test = '/home/tzh/Documents/working_jinyu/pytorch-classification-master/Office31_Dataset_Yurong/Single-Domain-Adaptation/A-2-D/TEST' #path = 'binary/par_cell' #path_test = 'binary/par_cell_test_ex' trainset = my_dataset(transform_train, path) train_set_aug = my_dataset(transform_train_aug, path) testset = my_dataset(transform_test, path_test) num_classes = 31 trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers, drop_last=True) trainloader_aug = data.DataLoader(train_set_aug, batch_size=args.train_batch, shuffle=True, num_workers=args.workers, drop_last=True) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers, drop_last=True) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, block_name=args.block_name, ) else: model = models.__dict__[args.arch](num_classes=num_classes) model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) criterion = nn.CrossEntropyLoss() #### optimizer #### #optimizer = NGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) optimizer = optim.Adam(model.parameters(), lr=args.lr) # Resume title = 'cifar-10-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # fine-tune normalziation #train_loss, train_acc = test(trainloader, model, criterion, optimizer, epoch, use_cuda) #print("~~~~~~~~~~~~~~~~~~~~~~~FINISHED AD NORM~~~~~~~~~~~~~~~~~~~~~~~~~~~") # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(trainloader, trainloader_aug, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda) # append logger file logger.append( [state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def main_worker(gpu, ngpus_per_node, args): global best_acc mem = os.popen( '"nvidia-smi" --query-gpu=memory.total,memory.used --format=csv,nounits,noheader' ).read().split('\n') total = mem[0].split(',')[0] total = int(total) max_mem = int(total * 0.5) # x = torch.rand((256, 1024, max_mem)).cuda() # del x start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) args.gpu = gpu if args.gpu is not None: print("Use GPU: {} for training".format(args.gpu)) if args.distributed: if args.dist_url == "env://" and args.rank == -1: args.rank = int(os.environ["RANK"]) if args.multiprocessing_distributed: # For multiprocessing distributed training, rank needs to be the # global rank among all the processes args.rank = args.rank * ngpus_per_node + gpu torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank) num_classes = 1000 # Model print("==> creating model '{}'".format(args.arch)) if args.arch.endswith('resnet50'): model = Resnet50(args.drop, num_classes, True, args.layer) elif args.arch.endswith('resnet50_shuffle'): model = Resnet50_Shuffle(args.drop, num_classes, True, args.layer) elif args.arch.endswith('resnet152_1x1'): model = Resnet152_1x1(args.drop, num_classes, True, args.layer) elif args.arch.endswith('resnet152_1x1lap'): model = Resnet152_1x1LAP(args.drop, num_classes, True, args.layer) elif args.arch.endswith('resnet152_1x1lmp'): model = Resnet152_1x1LMP(args.drop, num_classes, True, args.layer) elif args.arch.endswith('resnet50_1x1lap'): model = Resnet50_1x1LAP(args.drop, num_classes, True, args.layer) elif args.arch.endswith('resnet50_1x1lmp'): model = Resnet50_1x1LMP(args.drop, num_classes, True, args.layer) elif args.arch.endswith('resnet152_shuffle'): model = Resnet152_Shuffle(args.drop, num_classes, True, args.layer) elif args.arch.endswith('resnet152_truncated'): model = Resnet152_truncated(args.drop, num_classes, True, args.layer) elif args.arch.endswith('resnet50_truncated'): model = Resnet50_truncated(args.drop, num_classes, True, args.layer) elif args.arch.endswith('vgg16_shuffle'): model = VGG16_Shuffle(args.drop, num_classes, True, args.layer) elif args.arch.endswith('vgg16_rand'): model = VGG16_Rand(args.drop, num_classes, True, args.layer) elif args.arch.endswith('vgg16_1d'): model = VGG16_1d(args.drop, num_classes, True, args.layer) elif args.arch.endswith('vgg16_1x1lmp'): model = VGG16_1x1LMP(args.drop, num_classes, True, args.layer) elif args.arch.endswith('vgg16_1x1lap'): model = VGG16_1x1LAP(args.drop, num_classes, True, args.layer) elif args.arch.endswith('d1_resnet50'): model = Resnet50_1d(args.drop, num_classes, True, args.layer) elif args.arch.endswith('resnet50_1x1'): model = Resnet50_1x1(args.drop, num_classes, True, args.layer) elif args.arch.endswith('d1_resnet152'): model = Resnet152_1d(args.drop, num_classes, True, args.layer) elif args.arch.endswith('mobilenetv1_1x1lmp'): model = MobileNetV1_1x1LMP(1 - 0.999, num_classes, True, args.layer) elif args.arch.endswith('mobilenetv1_1x1lap'): model = MobileNetV1_1x1LAP(1 - 0.999, num_classes, True, args.layer) elif args.arch.endswith('mobilenetv2_1x1lmp'): model = MobileNetV2_1x1LMP(num_classes, args.layer) elif args.arch.endswith('mobilenetv2_1x1lap'): model = MobileNetV2_1x1LAP(num_classes, args.layer) else: raise Exception('arch can only be vgg16 or resnet50!') if args.distributed: # For multiprocessing distributed, DistributedDataParallel constructor # should always set the single device scope, otherwise, # DistributedDataParallel will use all available devices. if args.gpu is not None: torch.cuda.set_device(args.gpu) model.cuda(args.gpu) # When using a single GPU per process and per # DistributedDataParallel, we need to divide the batch size # ourselves based on the total number of GPUs we have args.train_batch = int(args.train_batch / ngpus_per_node) args.test_batch = int(args.test_batch / ngpus_per_node) args.workers = int(args.workers / ngpus_per_node) model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[args.gpu]) else: warnings.warn( 'NOT DISTRIBUTED!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') model.cuda() # DistributedDataParallel will divide and allocate batch_size to all # available GPUs if device_ids are not set model = torch.nn.parallel.DistributedDataParallel(model) elif args.gpu is not None: warnings.warn( 'NOT DISTRIBUTED!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') torch.cuda.set_device(args.gpu) model = model.cuda(args.gpu) else: warnings.warn( 'NOT DISTRIBUTED!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') # DataParallel will divide and allocate batch_size to all available GPUs model = torch.nn.DataParallel(model).cuda() criterion = nn.CrossEntropyLoss().cuda(args.gpu) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=args.weight_decay) cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) # for name, param in model.named_parameters(): # print(name) # for name in model.named_modules(): # print(name) # for param in model.parameters(): # print(param) # Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), ]) transform_test = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), ]) if args.dataset == 'xian': print('ImageNet from Xian is used!') traindir = '/BS/xian/work/data/imageNet1K/train/' valdir = '/BS/database11/ILSVRC2012/val/' else: traindir = os.path.join(args.dataset, 'train') valdir = os.path.join(args.dataset, 'val') trainset = datasets.ImageFolder(traindir, transform_train) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler( trainset) else: train_sampler = None trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True, sampler=train_sampler) testset = datasets.ImageFolder(valdir, transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=True) # Resume title = 'imagenet-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, args) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): if args.distributed: train_sampler.set_epoch(epoch) adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, args) test_loss, test_acc = test(testloader, model, criterion, args) # append logger file logger.append( [state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) if not args.multiprocessing_distributed or ( args.multiprocessing_distributed and args.rank % ngpus_per_node == 0): save_checkpoint( { 'epoch': epoch + 1, 'acc': test_acc, 'state_dict': model.state_dict(), 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 else: dataloader = datasets.CIFAR100 num_classes = 100 trainset = dataloader(root='./data', train=True, download=True, transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = dataloader(root='./data', train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, block_name=args.block_name, ) elif args.arch.startswith('preresnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, block_name=args.block_name, ) elif args.arch.startswith('horesnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, eta=args.eta, block_name=args.block_name, feature_vec=args.feature_vec ) elif args.arch.startswith('hopreresnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, eta=args.eta, block_name=args.block_name, feature_vec=args.feature_vec ) elif args.arch.startswith('nagpreresnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, eta=args.eta, block_name=args.block_name, feature_vec=args.feature_vec ) elif args.arch.startswith('mompreresnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, eta=args.eta, block_name=args.block_name, feature_vec=args.feature_vec ) elif args.arch.startswith('v2_preresnet'): model = models.__dict__[args.arch]( block_name='basicblock', num_blocks=[2,2,2,2], num_classes=num_classes ) else: print('Model is specified wrongly - Use standard model') model = models.__dict__[args.arch](num_classes=num_classes) model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0)) criterion = nn.CrossEntropyLoss() if args.optimizer.lower() == 'sgd': optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # elif args.optimizer.lower() == 'adam': # optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay) elif args.optimizer.lower() == 'radam': optimizer = RAdam(model.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay) elif args.optimizer.lower() == 'adamw': optimizer = AdamW(model.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay, warmup=args.warmup) elif args.optimizer.lower() == 'srsgd': iter_count = 1 optimizer = SGD_Adaptive(model.parameters(), lr=args.lr, weight_decay=args.weight_decay, iter_count=iter_count, restarting_iter=40) # Resume title = 'cifar-10-' + args.arch # if args.resume: # # Load checkpoint. # print('==> Resuming from checkpoint..') # assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' # args.checkpoint = os.path.dirname(args.resume) # checkpoint = torch.load(args.resume) # best_acc = checkpoint['best_acc'] # start_epoch = checkpoint['epoch'] # model.load_state_dict(checkpoint['state_dict']) # optimizer.load_state_dict(checkpoint['optimizer']) # logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) # else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): if args.optimizer.lower() == 'srsgd': if epoch == 80: optimizer = SGD_Adaptive(model.parameters(), lr=args.lr * 0.1, weight_decay=args.weight_decay, iter_count=iter_count, restarting_iter=80) elif epoch == 120: optimizer = SGD_Adaptive(model.parameters(), lr=args.lr * 0.01, weight_decay=args.weight_decay, iter_count=iter_count, restarting_iter=200) elif epoch == 160: optimizer = SGD_Adaptive(model.parameters(), lr=args.lr * 0.001, weight_decay=args.weight_decay, iter_count=iter_count, restarting_iter=500) elif epoch == 200: optimizer = SGD_Adaptive(model.parameters(), lr=args.lr * 0.0001, weight_decay=args.weight_decay, iter_count=iter_count, restarting_iter=1000) else: adjust_learning_rate(optimizer, epoch) logger.file.write('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) if args.optimizer.lower() == 'srsgd': train_loss, train_acc, iter_count = train(trainloader, model, criterion, optimizer, epoch, use_cuda, logger) else: train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda, logger) test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda, logger) # append logger file logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc]) writer.add_scalars('train_loss', {args.model_name: train_loss}, epoch) writer.add_scalars('test_loss', {args.model_name: test_loss}, epoch) writer.add_scalars('train_acc', {args.model_name: train_acc}, epoch) writer.add_scalars('test_acc', {args.model_name: test_acc}, epoch) # writer.add_scalars('loss_tracking/train_loss', {args.model_name: train_loss}, epoch) # writer.add_scalars('loss_tracking/test_loss', {args.model_name: test_loss}, epoch) # writer.add_scalars('loss_tracking/train_acc', {args.model_name: train_acc}, epoch) # writer.add_scalars('loss_tracking/test_acc', {args.model_name: test_acc}, epoch) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.file.write('Best acc:%f'%best_acc) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def main(): args = parse_args() os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" torch.manual_seed(args.seed) if torch.cuda.is_available(): device = 'cuda' torch.cuda.manual_seed(args.seed) else: device = 'cpu' print(f"==> Using device: {device}") if args.checkpoint is None: time_stamp = str(datetime.datetime.now().strftime('-%Y%m%d%H%M%S')) args.checkpoint = args.model + time_stamp args.checkpoint = 'checkpoints/' + args.checkpoint if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) save_args(args) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title="ModelNet" + args.model) logger.set_names([ "Epoch-Num", 'Learning-Rate', 'Train-Loss', 'Train-acc-B', 'Train-acc', 'Valid-Loss', 'Valid-acc-B', 'Valid-acc' ]) print('==> Preparing data..') train_loader = DataLoader(ModelNet40(partition='train', num_points=args.num_points), num_workers=8, batch_size=args.batch_size, shuffle=True, drop_last=True) test_loader = DataLoader(ModelNet40(partition='test', num_points=args.num_points), num_workers=8, batch_size=args.batch_size, shuffle=True, drop_last=False) # Model print('==> Building model..') net = models.__dict__[args.model]() criterion = nn.CrossEntropyLoss().to(device) net = net.to(device) # criterion = criterion.to(device) if device == 'cuda': net = torch.nn.DataParallel(net) cudnn.benchmark = True optimizer = torch.optim.SGD(net.parameters(), lr=args.learning_rate, momentum=0.9, weight_decay=args.weight_decay) scheduler = CosineAnnealingLR(optimizer, args.epoch, eta_min=args.learning_rate / 100) best_test_acc = 0. # best test accuracy best_train_acc = 0. best_test_acc_avg = 0. best_train_acc_avg = 0. best_test_loss = float("inf") best_train_loss = float("inf") start_epoch = 0 # start from epoch 0 or last checkpoint epoch for epoch in range(start_epoch, args.epoch): print('Epoch(%d/%s) Learning Rate %s:' % (epoch + 1, args.epoch, optimizer.param_groups[0]['lr'])) train_out = train(net, train_loader, optimizer, criterion, device) # {"loss", "acc", "acc_avg", "time"} test_out = validate(net, test_loader, criterion, device) scheduler.step() if test_out["acc"] > best_test_acc: best_test_acc = test_out["acc"] is_best = True else: is_best = False best_test_acc = test_out["acc"] if ( test_out["acc"] > best_test_acc) else best_test_acc best_train_acc = train_out["acc"] if ( train_out["acc"] > best_train_acc) else best_train_acc best_test_acc_avg = test_out["acc_avg"] if ( test_out["acc_avg"] > best_test_acc_avg) else best_test_acc_avg best_train_acc_avg = train_out["acc_avg"] if ( train_out["acc_avg"] > best_train_acc_avg) else best_train_acc_avg best_test_loss = test_out["loss"] if ( test_out["loss"] < best_test_loss) else best_test_loss best_train_loss = train_out["loss"] if ( train_out["loss"] < best_train_loss) else best_train_loss save_model(net, epoch, path=args.checkpoint, acc=test_out["acc"], is_best=is_best) logger.append([ epoch, optimizer.param_groups[0]['lr'], train_out["loss"], train_out["acc_avg"], train_out["acc"], test_out["loss"], test_out["acc_avg"], test_out["acc"] ]) print( f"Training loss:{train_out['loss']} acc_avg:{train_out['acc_avg']} acc:{train_out['acc']} time:{train_out['time']}s)" ) print( f"Testing loss:{test_out['loss']} acc_avg:{test_out['acc_avg']} acc:{test_out['acc']}% time:{test_out['time']}s) \n\n" ) logger.close() print(f"++++++++" * 2 + "Final results" + "++++++++" * 2) print( f"++ Last Train time: {train_out['time']} | Last Test time: {test_out['time']} ++" ) print( f"++ Best Train loss: {best_train_loss} | Best Test loss: {best_test_loss} ++" ) print( f"++ Best Train acc_B: {best_train_acc_avg} | Best Test acc_B: {best_test_acc_avg} ++" ) print( f"++ Best Train acc: {best_train_acc} | Best Test acc: {best_test_acc} ++" ) print(f"++++++++" * 5)
def execute_sensing_decision(hit_rate, num_steps): """ Execution of the sensing_decision module. @param hit_rate A list with the hit ratio for every cpe. Example: hit_rate = [90, 78, 32] - the first cpe has a hit ratio of 90%, the second has a hit ratio of 78% and the third has a hit ratio of 32%. @param num_steps Number of executions of the sensing_result """ feedback_control = 5 increase_rate = 0.1 decrease_rate = 0.1 num_cpes = len(hit_rate) list_str_cpes = [] for i in range(num_cpes): list_str_cpes.append("cpe" + str(i+1)) # save strings in the Logger Logger._enable = True Logger.register("reward", list_str_cpes) sdc = SDController(num_cpes, feedback_control, increase_rate, decrease_rate) # each element of this array is also an array. the array of the index 0 (ie, array_hit[0]) corresponds to the hit_ # rate[0] and so on. cpe_array = [0] * num_steps array_hit = [] # num_cpes is the length of array_hit for i in range(num_cpes): # need to append as a list because otherwise if we modify some subarray, ALL arrays are modified too. array_hit.append(list(cpe_array)) # list of lists, where the random indexes will be. list_indexes = [] for i in range(num_cpes): list_indexes.append(list([])) # set some random positions of the arrays to one. for i in range(num_cpes): while len(list_indexes[i]) < (num_steps - (num_steps * hit_rate[i]/100)): rand = random.randint(0, num_steps-1) if rand not in list_indexes[i]: list_indexes[i].append(rand) array_hit[i][rand] = 1 for step in range(num_steps): sensing_result = [] for cpe in range(num_cpes): sensing_result.append(array_hit[cpe][step]) Logger.append("reward", list_str_cpes[cpe], sdc._reward[cpe]) sdc.sensing_decision(sensing_result, num_cpes) Logger.dump('./dump', '/cpes', 0) print "\n\n REWARD\n\n" for cpe in range(num_cpes): print "reward cpe %i: " %(cpe+1) + str(sdc._reward[cpe]) print "TOTAL HIT RATE: ", float(sdc._total_idle/float(num_steps))
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data loading code traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_loader = torch.utils.data.DataLoader( datasets.ImageFolder(traindir, transforms.Compose([ transforms.RandomSizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])), batch_size=args.train_batch, shuffle=True, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader( datasets.ImageFolder(valdir, transforms.Compose([ transforms.Scale(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=True) # create model if args.pretrained: print("=> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch](pretrained=True) elif args.arch.startswith('resnext'): model = models.__dict__[args.arch]( baseWidth=args.base_width, cardinality=args.cardinality, ) else: print("=> creating model '{}'".format(args.arch)) model = models.__dict__[args.arch]() if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0)) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'ImageNet-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(val_loader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(val_loader, model, criterion, epoch, use_cuda) # append logger file logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def receiver_loop(tb, channel_list, channel, options): """ UP LOOP @param tb @param channel_list @param channel @param options """ import xmlrpclib from SimpleXMLRPCServer import SimpleXMLRPCServer class MyNamespace: """ """ pass g_namespace = MyNamespace() g_namespace.tb = tb g_namespace.options = options g_namespace.server_run = True class StoppableXMLRPCServer(SimpleXMLRPCServer): """Override of TIME_WAIT""" allow_reuse_address = True def __init__(self, options): SimpleXMLRPCServer.__init__(self, options) self.stop = False def serve_forever(self): while not self.stop: self.handle_request() print 'exiting server' def shutdown(self): self.stop = True return 0 server = StoppableXMLRPCServer((options.slave_addr, 8000)) g_namespace.th = Thread(target=server.serve_forever ) # Flag que indica quando a execucao deve parar # Flag that indicates when the execution must stop. g_namespace.run = False g_namespace.interferer_channel = 0 def set_channel(channel): """ RPC for changing the channel. @param channel """ print "Received command to handoff to channel ", channel if not g_namespace.options.tx_only: g_namespace.tb.tx.center_freq = channel_list[channel] g_namespace.tb.rx.center_freq = channel_list[channel] g_namespace.interferer_channel = channel return 1 def close_app(): """ Closes the app. """ print "Received command to close" g_namespace.run = False return 1 def client_started(): """ Notifies that the execution has started. """ g_namespace.run = True return 1 Logger.register('receiver', ['channel', 'pkt', 'start_time']) Logger.set('receiver', 'start_time', time.time()) # Registra funcoes no servidor XML e inicia thread do servidor # Registers functions in the XML server and starts the server thread. server.register_function(set_channel, 'set_channel') server.register_function(close_app, 'close_app') server.register_function(client_started, 'client_started') g_namespace.th.start() print "Receiver listening for commands in port 8000" print "\t... Waiting for client_started call" while g_namespace.run == False: 1; print "\t...client connected" global t_rcv, t_cor channel = 0 # Enquanto nao recebeu a notificacao de parada, continua a execucao # While the stop notify is not received, continues the execution. while g_namespace.run: print " ... r: ", t_rcv, ", c: ", t_cor time.sleep(1) if not options.tx_only: Logger.append('receiver', 'channel', channel, time.time()) Logger.append('receiver', 'pkt', t_rcv) print "Shutting down Server" server.shutdown() print "\t ... Server exited"
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 else: dataloader = datasets.CIFAR100 num_classes = 100 trainset = dataloader(root='./data', train=True, download=True, transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = dataloader(root='./data', train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model") model = ResNet( num_classes=num_classes, depth=args.depth, norm_type=args.norm, basicblock=args.basicblock, ) model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(model) print(' Total params: %.4fM' % (sum(p.numel() for p in model.parameters())/1000000.0)) criterion = nn.CrossEntropyLoss() optimizer = set_optimizer(model, args) # Resume title = '{}-ResNet-{}-{}'.format(args.dataset, args.depth, args.norm) if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.', 'Train Acc.5', 'Valid Acc.5']) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc, train_acc5 = train(trainloader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc, test_acc5 = test(testloader, model, criterion, epoch, use_cuda) # append logger file logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc, train_acc5, test_acc5]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() print('Best acc:') print(best_acc)
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 else: dataloader = datasets.CIFAR100 num_classes = 100 trainset = dataloader(root=args.data_root, train=True, download=True, transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = dataloader(root=args.data_root, train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, ) else: model = models.__dict__[args.arch](num_classes=num_classes) model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0)) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'cifar-10-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda) # append logger file logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) dataloader = datasets.CIFAR10 # CIFAR10 data set is loaded and has 10 classes num_classes = 10 trainset = dataloader(root='./data', train=True, download=True, transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = dataloader(root='./data', train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, ) else: model = models.__dict__[args.arch]( num_classes=num_classes) # here is where we use the vgg19_bn model = torch.nn.DataParallel(model).cuda() # Multi GPU call cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'cifar-10-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda) # append logger file logger.append( [state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss','Train Acc.', 'Valid Acc.', 'Train Acc.5', 'Valid Acc.5']) # Train and validate for epoch in range(start_epoch, args.epochs): state['lr'] = adjust_learning_rate(state['lr'], optimizer, epoch, args.gamma, args.schedule) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc, train_acc5 = train_one_epoch(trainloader, model, criterion, optimizer, use_cuda=use_cuda) test_loss, test_acc, test_acc5 = test(testloader, model, criterion, use_cuda=use_cuda) # append logger file logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc, train_acc5, test_acc5]) # save model ap is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) if do_save_checkpoint: save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() print('Best acc:', best_acc)
def main(): global best_prec1, args args.gpu = 0 args.world_size = 1 if args.distributed: args.gpu = args.local_rank % torch.cuda.device_count() torch.cuda.set_device(args.gpu) torch.distributed.init_process_group(backend='nccl', init_method='env://') args.world_size = torch.distributed.get_world_size() args.total_batch_size = args.world_size * args.batch_size if not os.path.isdir(args.checkpoint) and args.local_rank == 0: mkdir_p(args.checkpoint) if args.fp16: assert torch.backends.cudnn.enabled, "fp16 mode requires cudnn backend to be enabled." if args.static_loss_scale != 1.0: if not args.fp16: print("Warning: if --fp16 is not used, static_loss_scale will be ignored.") # create model if args.pretrained: print("=> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch](pretrained=True) else: print("=> creating model '{}'".format(args.arch)) model = models.__dict__[args.arch]() model = model.cuda() if args.fp16: model = network_to_half(model) if args.distributed: # shared param/delay all reduce turns off bucketing in DDP, for lower latency runs this can improve perf # for the older version of APEX please use shared_param, for newer one it is delay_allreduce model = DDP(model, delay_allreduce=True) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) if args.fp16: optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.static_loss_scale, dynamic_loss_scale=args.dynamic_loss_scale, verbose=False) # optionally resume from a checkpoint title = 'ImageNet-' + args.arch if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume, map_location=lambda storage, loc: storage.cuda(args.gpu)) args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})" .format(args.resume, checkpoint['epoch'])) if args.local_rank == 0: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: print("=> no checkpoint found at '{}'".format(args.resume)) else: if args.local_rank == 0: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.', 'Valid Top5.']) traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') if(args.arch == "inception_v3"): crop_size = 299 val_size = 320 # I chose this value arbitrarily, we can adjust. else: crop_size = 224 val_size = 256 pipe = HybridTrainPipe(batch_size=args.batch_size, num_threads=args.workers, device_id=args.local_rank, data_dir=traindir, crop=crop_size, dali_cpu=args.dali_cpu) pipe.build() train_loader = DALIClassificationIterator(pipe, size=int(pipe.epoch_size("Reader") / args.world_size)) pipe = HybridValPipe(batch_size=args.batch_size, num_threads=args.workers, device_id=args.local_rank, data_dir=valdir, crop=crop_size, size=val_size) pipe.build() val_loader = DALIClassificationIterator(pipe, size=int(pipe.epoch_size("Reader") / args.world_size)) if args.evaluate: validate(val_loader, model, criterion) return total_time = AverageMeter() for epoch in range(args.start_epoch, args.epochs): # train for one epoch adjust_learning_rate(optimizer, epoch,args) if args.local_rank == 0: print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, optimizer.param_groups[0]['lr'])) [train_loss, train_acc, avg_train_time] = train(train_loader, model, criterion, optimizer, epoch) total_time.update(avg_train_time) # evaluate on validation set [test_loss, prec1, prec5] = validate(val_loader, model, criterion) # remember best prec@1 and save checkpoint if args.local_rank == 0: # append logger file logger.append([optimizer.param_groups[0]['lr'], train_loss, test_loss, train_acc, prec1, prec5]) is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) save_checkpoint({ 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best,checkpoint=args.checkpoint) if epoch == args.epochs - 1: print('##Top-1 {0}\n' '##Top-5 {1}\n' '##Perf {2}'.format(prec1, prec5, args.total_batch_size / total_time.avg)) # reset DALI iterators train_loader.reset() val_loader.reset() if args.local_rank == 0: logger.close()
def main(): global best_acc # Data print('==> Preparing data..') transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), ]) transform_test = transforms.Compose([ transforms.ToTensor(), ]) # data_file = './data' # data_file = 'C:/Users/dongxian/data/CIFAR' data_file = '/data/wudongxian/dataset/CIFAR/' if args.dataset == 'cifar10': data_name = 'CIFAR10' num_classes = 10 else: data_name = 'CIFAR100' num_classes = 100 trainset = getattr(torchvision.datasets, data_name)(root=data_file, train=True, download=True, transform=transform_train) trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = getattr(torchvision.datasets, data_name)(root=data_file, train=False, download=True, transform=transform_test) testloader = torch.utils.data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print('==> Building model..') net = models.__dict__[args.arch](num_classes=num_classes) title = 'noisy_cifar-' + args.arch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Adv Train Loss', 'Nat Train Loss', 'Nat Valid Loss', 'Adv Train Acc', 'Nat Train Loss', 'Nat Valid Acc' ]) if torch.cuda.is_available(): net = net.to(device) net = torch.nn.DataParallel(net) print('Using', torch.cuda.device_count(), 'GPUs.') cudnn.benchmark = True print('Using CUDA..') criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) schedule_lr = optim.lr_scheduler.MultiStepLR(optimizer, args.schedule, gamma=args.gamma) adversary = PGDAttack(model=net, epsilon=0.031, num_steps=10, step_size=0.007) for epoch in range(args.start_epoch, args.epochs): schedule_lr.step(epoch) state['lr'] = optimizer.state_dict()['param_groups'][0]['lr'] if args.mode == 'nat': adv_train_loss, adv_train_acc = train(trainloader, net, criterion, optimizer, epoch) else: adv_train_loss, adv_train_acc = train_adv(trainloader, net, criterion, optimizer, epoch, adversary=adversary) nat_train_loss, nat_train_acc = test(trainloader, net, criterion, epoch) nat_test_loss, nat_test_acc = test(testloader, net, criterion, epoch) logger.append([ state['lr'], adv_train_loss, nat_train_loss, nat_test_loss, adv_train_acc, nat_train_acc, nat_test_acc ]) # what is the best? is_best = adv_train_acc > best_acc best_acc = max(adv_train_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': net.state_dict(), 'acc': adv_train_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) if (epoch + 1) % 10 == 0 and epoch > 1: save_checkpoint( { 'epoch': epoch + 1, 'state_dict': net.state_dict(), 'acc': adv_train_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint, filename='checkpoint_{:0>3d}.pth.tar'.format(epoch + 1)) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps'))
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # load data print('==> Preparing dataset %s' % args.dataset) features, landmarks, labels = pickle_2_img_and_landmark(args.dataset_path) num_classes = 6 # Model print("==> creating model '{}'".format(args.arch)) model = ResNetAndDGCNN(20, num_classes=num_classes) # model = torch.nn.DataParallel(model).cuda() model = model.cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) # print(' resnet params: %.2fM' % (sum(p.numel() for p in model.resnet.parameters())/1000000.0)) # print(' stgcn params: %.2fM' % (sum(p.numel() for p in model.st_gcn.parameters())/1000000.0)) criterion = nn.CrossEntropyLoss() # 分层优化 # resnet_para = [model.conv1.parameters(), model.layer1.parameters(), model.layer2.parameters(), model.layer3.parameters(), model.layer4.parameters()] # optimizer = optim.SGD([ # {'params': model.gcn11.parameters()}, # {'params': model.gcn12.parameters()}, # {'params': model.gcn21.parameters()}, # {'params': model.gcn22.parameters()}, # {'params': model.gcn31.parameters()}, # {'params': model.gcn32.parameters()}, # {'params': model.fc.parameters()}, # {'params': model.conv1.parameters(), 'lr': 0.005, 'weight_decay': 1e-3}, # {'params': model.bn1.parameters(), 'lr': 0.005, 'weight_decay': 1e-3}, # {'params': model.layer1.parameters(), 'lr': 0.005, 'weight_decay': 1e-3}, # {'params': model.layer2.parameters(), 'lr': 0.005, 'weight_decay': 1e-3}, # {'params': model.layer3.parameters(), 'lr': 0.005, 'weight_decay': 1e-3}, # {'params': model.layer4.parameters(), 'lr': 0.005, 'weight_decay': 1e-3}, # ], lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) # Resume title = 'ckp-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log_stat.log'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log_stat.log'), title=title) logger.set_names([ 'fold_num', 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) # logging logging.basicConfig(level=logging.DEBUG, filename=os.path.join(args.checkpoint, 'log_info.log'), filemode='a+', format="%(asctime)-15s %(levelname)-8s %(message)s") # log configuration logging.info('-' * 10 + 'configuration' + '*' * 10) for arg in vars(args): logging.info((arg, str(getattr(args, arg)))) acc_fold = [] reset_lr = state['lr'] for f_num in range(args.folds): state['lr'] = reset_lr model.reset_all_weights() # optimizer = optim.SGD([ # {'params': model.gcn11.parameters()}, # {'params': model.gcn12.parameters()}, # {'params': model.gcn21.parameters()}, # {'params': model.gcn22.parameters()}, # {'params': model.gcn31.parameters()}, # {'params': model.gcn32.parameters()}, # {'params': model.fc.parameters()}, # {'params': model.conv1.parameters(), 'lr': 0.005, 'weight_decay': 1e-3}, # {'params': model.bn1.parameters(), 'lr': 0.005, 'weight_decay': 1e-3}, # {'params': model.layer1.parameters(), 'lr': 0.005, 'weight_decay': 1e-3}, # {'params': model.layer2.parameters(), 'lr': 0.005, 'weight_decay': 1e-3}, # {'params': model.layer3.parameters(), 'lr': 0.005, 'weight_decay': 1e-3}, # {'params': model.layer4.parameters(), 'lr': 0.005, 'weight_decay': 1e-3}, # ], lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) print(args.lr) # save each fold's acc and reset configuration average_acc = 0 best_acc = 0 # 10-fold cross validation train_x, train_lm, train_y = [], [], [] test_x, test_lm, test_y = [], [], [] for id_fold in range(args.folds): if id_fold == f_num: test_x = features[id_fold] test_lm = landmarks[id_fold] test_y = labels[id_fold] else: train_x = train_x + features[id_fold] train_lm = train_lm + landmarks[id_fold] train_y = train_y + labels[id_fold] # convert array to tensor train_x = torch.tensor(train_x, dtype=torch.float) / 255.0 #(b_s, 128, 128) train_x = train_x.unsqueeze(1) #(b_s, 1, 128, 128) train_lm = np.stack(train_lm) # 只要坐标信息, 不需要归一化 train_lm = (train_lm - np.mean(train_lm, axis=0)) / np.std(train_lm, axis=0) train_lm = torch.tensor(train_lm) # train_lm = train_lm.unsqueeze(2) test_x = torch.tensor(test_x, dtype=torch.float) / 255.0 test_x = test_x.unsqueeze(1) # 只要坐标信息, 不需要归一化 test_lm = (test_lm - np.mean(test_lm, axis=0)) / np.std(test_lm, axis=0) test_lm = torch.tensor(test_lm) # test_lm = test_lm.unsqueeze(2) train_y, test_y = torch.tensor(train_y), torch.tensor(test_y) train_dataset = torch.utils.data.TensorDataset(train_x, train_lm, train_y) train_iter = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=args.train_batch, shuffle=True) test_dataset = torch.utils.data.TensorDataset(test_x, test_lm, test_y) test_iter = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=args.test_batch, shuffle=False) # test for fold order print(len(test_dataset)) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(train_x + test_x, train_y + test_y, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) continue # show plt # plt.show(block=False) # Train and val for epoch in range(start_epoch, args.epochs): # 在特定的epoch 调整学习率 adjust_learning_rate(optimizer, epoch) # print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, optimizer.param_groups[0]['lr'])) train_loss, train_acc = train(train_iter, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(test_iter, model, criterion, epoch, use_cuda) # append logger file logger.append([ f_num, state['lr'], train_loss, test_loss, train_acc, test_acc ]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, f_num, checkpoint=args.checkpoint) # compute average acc acc_fold.append(best_acc) average_acc = sum(acc_fold) / len(acc_fold) logging.info('fold: %d, best_acc: %.2f, average_acc: %.2f' % (f_num, best_acc, average_acc)) logger.close() # logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) logging.info('acc_fold' + str(acc_fold)) print('average acc:') print(average_acc)
def main(): global best_acc global best_acc_lr_0 global best_acc_lr_1 global best_acc_lr_2 global best_acc_lr_3 global global_writer global global_num_classes global global_record start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ # transforms.RandomCrop(32, padding=4), # transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 global_num_classes = 10 # add else: dataloader = datasets.CIFAR100 num_classes = 100 global_num_classes = 100 # add trainset = dataloader(root='./data', train=True, download=True, transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = dataloader(root='./data', train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, ) else: model = models.__dict__[args.arch](num_classes=num_classes) model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0)) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'cifar-%d-' % (global_num_classes) + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: if os.path.exists(os.path.join(args.checkpoint, 'log.txt')): os.system('rm ' + os.path.join(args.checkpoint, 'log.txt')) print('exist log.txt and rm log.txt') logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.', 'Best Acc.']) if os.path.exists(os.path.join(args.checkpoint, 'info.txt')): os.system('rm ' + os.path.join(args.checkpoint, 'info.txt')) print('exist info.txt and rm info.txt') global_writer = open(os.path.join(args.checkpoint, 'info.txt'), 'a', 0) print('random seed = %d' % args.manualSeed, file = global_writer) if os.path.exists(os.path.join(args.checkpoint, 'record.txt')): os.system('rm ' + os.path.join(args.checkpoint, 'record.txt')) print('exist record.txt and rm record.txt') global_record = open(os.path.join(args.checkpoint, 'record.txt'), 'a', 0) print('random seed = %d' % args.manualSeed, file = global_record) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc, angleW, theta = train(trainloader, model, criterion, optimizer, epoch, use_cuda, args.lam) test_loss, test_acc, confusion_matrix, total_wrong = test(testloader, model, criterion, epoch, use_cuda) # append logger file logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc, best_acc]) print('epoch: {}'.format(epoch), file = global_record) print('test_acc: {}'.format(test_acc), file = global_record) print('angleW:', file = global_record) print('{} {}'.format(angleW.size(0), angleW.size(1)), file = global_record) for val in angleW: for w in val: global_record.write('{} '.format(w)) global_record.write('\n') print('theta:', file = global_record) print('{} {}'.format(theta.size(0), theta.size(1)), file = global_record) for val in theta: for w in val: global_record.write('{} '.format(w)) global_record.write('\n') print('confusion_matrix:', file = global_record) print('{} {}'.format(confusion_matrix.size(0), confusion_matrix.size(1)), file = global_record) for val in confusion_matrix: for w in val: global_record.write('{} '.format(int(w))) global_record.write('\n') # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) print('best_acc: {}'.format(best_acc), file = global_record) print('-'*100, file = global_record) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint, filename='checkpoint.pth.tar', bestfilename='model_best.pth.tar') print(state['lr']) if isequal(state['lr'], 0.1): print('lr==0.1') is_best_lr_0 = test_acc > best_acc_lr_0 best_acc_lr_0 = max(test_acc, best_acc_lr_0) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc_lr_0, 'optimizer' : optimizer.state_dict(), }, is_best_lr_0, checkpoint=args.checkpoint, filename='checkpoint_lr_0.pth.tar', bestfilename='model_best_lr_0.pth.tar') if isequal(state['lr'], 0.01): print('lr==0.01') is_best_lr_1 = test_acc > best_acc_lr_1 best_acc_lr_1 = max(test_acc, best_acc_lr_1) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc_lr_1, 'optimizer' : optimizer.state_dict(), }, is_best_lr_1, checkpoint=args.checkpoint, filename='checkpoint_lr_1.pth.tar', bestfilename='model_best_lr_1.pth.tar') if isequal(state['lr'], 0.001): print('lr==0.001') is_best_lr_2 = test_acc > best_acc_lr_2 best_acc_lr_2 = max(test_acc, best_acc_lr_2) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc_lr_2, 'optimizer' : optimizer.state_dict(), }, is_best_lr_2, checkpoint=args.checkpoint, filename='checkpoint_lr_2.pth.tar', bestfilename='model_best_lr_2.pth.tar') if isequal(state['lr'], 0.0001): print('lr==0.0001') is_best_lr_3 = test_acc > best_acc_lr_3 best_acc_lr_3 = max(test_acc, best_acc_lr_3) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc_lr_3, 'optimizer' : optimizer.state_dict(), }, is_best_lr_3, checkpoint=args.checkpoint, filename='checkpoint_lr_3.pth.tar', bestfilename='model_best_lr_3.pth.tar') logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) # add-------------------- print ('Best acc:\n lr=0.1 acc={:.4f}\n lr=0.01 acc={:.4f}\n lr=0.001 acc={:.4f}\n'.format(best_acc_lr_0, best_acc_lr_1, best_acc_lr_2)) print('Best acc: {:.4f}\n'.format(best_acc)) print('Best acc:', file = global_writer) print(best_acc, file = global_writer)
def main(): global args, best_prec1 args = parser.parse_args() args.distributed = args.world_size > 1 if args.distributed: dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size) # Use CUDA os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id use_cuda = torch.cuda.is_available() # Random seed if args.manual_seed is None: args.manual_seed = random.randint(1, 10000) random.seed(args.manual_seed) torch.manual_seed(args.manual_seed) if use_cuda: torch.cuda.manual_seed_all(args.manual_seed) # create model if args.pretrained: print("=> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch](pretrained=True, num_attributes=len( args.selected_attrs)) elif args.arch.startswith('resnext'): model = models.__dict__[args.arch]( baseWidth=args.base_width, cardinality=args.cardinality, num_attributes=len(args.all_attrs), ) elif args.arch.startswith('shufflenet'): model = models.__dict__[args.arch](groups=args.groups, num_attributes=len( args.selected_attrs)) else: print("=> creating model '{}'".format(args.arch)) model = models.__dict__[args.arch](pretrained=True, num_attributes=len( args.selected_attrs)) # if not args.distributed: # if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): # model.features = torch.nn.DataParallel(model.features) # model.cuda() # else: # model = torch.nn.DataParallel(model).cuda() # else: # model.cuda() # model = torch.nn.parallel.DistributedDataParallel(model) # define loss function (criterion) and optimizer # optionally resume from a checkpoint title = 'CelebAHQ-' + args.arch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) logger = None if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) args.checkpoint = os.path.dirname(args.resume) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: print("=> no checkpoint found at '{}'".format(args.resume)) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) criterion = nn.CrossEntropyLoss().cuda() model = torch.nn.DataParallel(model).cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) if args.resume: if os.path.isfile(args.resume): print("=> loading optimizer '{}'".format(args.resume)) checkpoint = torch.load(args.resume, map_location='cuda') optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded optimizer '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) args.checkpoint = os.path.dirname(args.resume) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: print("=> no optimizer checkpoint found at '{}'".format( args.resume)) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) cudnn.benchmark = True # Data loading code normalize = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) train_dataset = CelebAHQExtra( args.data, 'celebahq_extra', 'train', transforms.Compose([ transforms.Resize((256, 256)), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ]), args.selected_attrs, None) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler( train_dataset) else: train_sampler = None train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.train_batch, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True, sampler=train_sampler) val_loader = torch.utils.data.DataLoader(CelebAHQExtra( args.data, 'celebahq_extra', 'test', transforms.Compose( [transforms.Resize((256, 256)), transforms.ToTensor(), normalize]), args.selected_attrs, None), batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=True) test_loader = torch.utils.data.DataLoader(CelebAHQExtra( args.data, 'celebahq_extra', 'test', transforms.Compose( [transforms.Resize((256, 256)), transforms.ToTensor(), normalize]), args.selected_attrs, None), batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=True) if args.evaluate: validate(test_loader, model, criterion) return # visualization writer = SummaryWriter(os.path.join(args.checkpoint, 'logs')) for epoch in range(args.start_epoch, args.epochs): if args.distributed: train_sampler.set_epoch(epoch) lr = adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, lr)) # train for one epoch train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch) # evaluate on validation set val_loss, prec1 = validate(val_loader, model, criterion) # append logger file logger.append([lr, train_loss, val_loss, train_acc, prec1]) # tensorboardX writer.add_scalar('learning rate', lr, epoch + 1) writer.add_scalars('loss', { 'train loss': train_loss, 'validation loss': val_loss }, epoch + 1) writer.add_scalars('accuracy', { 'train accuracy': train_acc, 'validation accuracy': prec1 }, epoch + 1) # for name, param in model.named_parameters(): # writer.add_histogram(name, param.clone().cpu().data.numpy(), epoch + 1) is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.module.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.pdf')) writer.close() print('Best accuracy:') print(best_prec1)
def main(): global best_acc # Data print('==> Preparing data..') transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), ]) transform_test = transforms.Compose([ transforms.ToTensor(), ]) # data_file = './data' # data_file = 'C:/Users/dongxian/data/CIFAR' data_file = '/data/wudongxian/dataset/CIFAR/' if args.dataset == 'cifar10': data_name = 'CIFAR10' num_classes = 10 else: data_name = 'CIFAR100' num_classes = 100 trainset = getattr(torchvision.datasets, data_name)(root=data_file, train=True, download=True, transform=transform_train) trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = getattr(torchvision.datasets, data_name)(root=data_file, train=False, download=True, transform=transform_test) testloader = torch.utils.data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) title = 'adv-train-' + args.arch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) logger = Logger(os.path.join(args.checkpoint, 'log_attack_results.txt'), title=title) logger.set_names(['Adv Train Loss', 'Adv Valid Loss', 'Adv Train Acc', 'Adv Valid Acc']) # create Model print('==> Building model..') net = models.__dict__[args.arch](num_classes=num_classes) if torch.cuda.is_available(): net = net.to(device) net = torch.nn.DataParallel(net) print('Using', torch.cuda.device_count(), 'GPUs.') cudnn.benchmark = True print('Using CUDA..') criterion = nn.CrossEntropyLoss() saved_models = [v for v in os.listdir(args.checkpoint) if '0.pth.tar' in v] saved_models = sorted(saved_models) save_text_list('{}/models_names.txt'.format(args.checkpoint), saved_models) for saved_model in saved_models: # resume Model state_dict = torch.load('{}/{}'.format(args.checkpoint, saved_model)) net.load_state_dict(state_dict['state_dict']) if args.attack_type == 'ref': adversary = PGDAttack(model=net, epsilon=0.031, num_steps=20, step_size=0.003) elif args.attack_type == 'ours': adversary = OurPGDAttack(model=net, epsilon=0.031, num_steps=20, step_size=0.003, B=args.B) adv_train_loss, adv_train_acc = attack_over_test(trainloader, net, criterion, adversary) adv_test_loss, adv_test_acc = attack_over_test(testloader, net, criterion, adversary) logger.append([adv_train_loss, adv_test_loss, adv_train_acc, adv_test_acc]) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps'))
def main(): global best_acc, state start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if args.attack_iter == 0: attacker = NoOpAttacker() else: attacker = PGDAttacker(args.attack_iter, args.attack_epsilon, args.attack_step_size, prob_start_from_clean=0.2 if not args.evaluate else 0.0) if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data loading code traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') # the mean and variant don't have too much influence # (pic - 0.5) / 0.5 just make it easier for attacker. # normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], # std=[0.229, 0.224, 0.225]) normalize = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) transform_train = transforms.Compose([ transforms.RandomSizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ]) if args.fastaug: transform_train.transforms.insert(0, FastAugmentation()) if args.lighting: __imagenet_pca = { 'eigval': np.array([0.2175, 0.0188, 0.0045]), 'eigvec': np.array([ [-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.8140], [-0.5836, -0.6948, 0.4203], ]) } transform_train = transforms.Compose([ transforms.RandomSizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), Lighting(0.1, __imagenet_pca['eigval'], __imagenet_pca['eigvec']), normalize ]) train_dataset = datasets.ImageFolder(traindir, transform_train) train_loader = torch.utils.data.DataLoader((train_dataset), batch_size=args.train_batch, shuffle=True, num_workers=args.workers, pin_memory=True) val_transforms = [ transforms.ToTensor(), normalize, ] if not args.already224: val_transforms = [transforms.Scale(256), transforms.CenterCrop(224)] + val_transforms val_loader = torch.utils.data.DataLoader( datasets.ImageFolder(valdir, transforms.Compose(val_transforms)), batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=True) # create model print("=> creating model '{}'".format(args.arch)) if args.mixbn: norm_layer = MixBatchNorm2d else: norm_layer = None model = net.__dict__[args.arch](num_classes=args.num_classes, norm_layer=norm_layer) model.set_attacker(attacker) model.set_mixbn(args.mixbn) model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0)) # define loss function (criterion) and optimizer if args.smoothing == 0: criterion = nn.CrossEntropyLoss(reduction='none').cuda() # implement a cross_entropy with label smoothing. # First, perform a log_softmax; then fill the selected classes with 1-smoothing # At last, use kl_div, which means: # KL(p||q) = -\int p(x)ln q(x) dx - (-\int p(x)ln p(x) dx) # kl_div is different from Crossentropy with a const number (\int p(x) ln p(x)) else: criterion = partial(label_smoothing_cross_entropy, classes=args.num_classes, dim=-1) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=args.nesterov) # Resume title = 'ImageNet-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) for param_group in optimizer.param_groups: param_group['lr'] = state['lr'] logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: if args.load: checkpoint = torch.load(args.load) if args.mixbn: to_merge = {} for key in checkpoint['state_dict']: if 'bn' in key: tmp = key.split("bn") aux_key = tmp[0] + 'bn' + tmp[1][0] + '.aux_bn' + tmp[1][1:] to_merge[aux_key] = checkpoint['state_dict'][key] elif 'downsample.1' in key: tmp = key.split("downsample.1") aux_key = tmp[0] + 'downsample.1.aux_bn' + tmp[1] to_merge[aux_key] = checkpoint['state_dict'][key] checkpoint['state_dict'].update(to_merge) model.load_state_dict(checkpoint['state_dict']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(val_loader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val writer = SummaryWriter(log_dir=args.checkpoint) warmup_scheduler = WarmUpLR(optimizer, len(train_loader) * args.warm, start_lr=args.warm_lr) if args.warm > 0 else None for epoch in range(start_epoch, args.epochs): if epoch >= args.warm and args.lr_schedule == 'step': adjust_learning_rate(optimizer, epoch, args) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, optimizer.param_groups[-1]['lr'])) train_func = partial(train, train_loader=train_loader, model=model, criterion=criterion, optimizer=optimizer, epoch=epoch, use_cuda=use_cuda, warmup_scheduler=warmup_scheduler, mixbn=args.mixbn, writer=writer, attacker=attacker) if args.mixbn: model.apply(to_mix_status) train_loss, train_acc, loss_main, loss_aux, top1_main, top1_aux = train_func() else: train_loss, train_acc = train_func() writer.add_scalar('Train/loss', train_loss, epoch) writer.add_scalar('Train/acc', train_acc, epoch) if args.mixbn: writer.add_scalar('Train/loss_main', loss_main, epoch) writer.add_scalar('Train/loss_aux', loss_aux, epoch) writer.add_scalar('Train/acc_main', top1_main, epoch) writer.add_scalar('Train/acc_aux', top1_aux, epoch) model.apply(to_clean_status) test_loss, test_acc = test(val_loader, model, criterion, epoch, use_cuda) writer.add_scalar('Test/loss', test_loss, epoch) writer.add_scalar('Test/acc', test_acc, epoch) # append logger file logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) print('Best acc:') print(best_acc) writer.close() logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps'))
def main(): global args, best_prec1 args = parser.parse_args() if args.seed is not None: random.seed(args.seed) torch.manual_seed(args.seed) cudnn.deterministic = True warnings.warn('You have chosen to seed training. ' 'This will turn on the CUDNN deterministic setting, ' 'which can slow down your training considerably! ' 'You may see unexpected behavior when restarting ' 'from checkpoints.') args.distributed = args.world_size > 1 if args.distributed: dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size) # create model print("=> creating model '{}'".format(args.arch)) if args.arch.split('_')[0] == 'mobilenetv3': model = models.__dict__[args.arch](width_mult=args.width_mult, dropout=args.dropout, mode=args.mode, fc_squeeze=args.fc_squeeze) elif args.arch.split('_')[0] == 'mobilenetv2': model = models.__dict__[args.arch](width_mult=args.width_mult, dropout=args.dropout, fc_squeeze=args.fc_squeeze) else: model = models.__dict__[args.arch](dropout=args.dropout) if not args.distributed: if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() else: model.cuda() model = torch.nn.parallel.DistributedDataParallel(model) # define loss function (criterion) and optimizer if args.label_smoothing > 0: # using Label Smoothing criterion = LabelSmoothingLoss(smoothing=args.label_smoothing) else: criterion = nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optionally resume from a checkpoint title = 'ImageNet-' + args.arch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) arch = open(os.path.join(args.checkpoint, 'arch.txt'), 'w') print(model, file=arch) arch.close() if os.path.exists(os.path.join(args.checkpoint, 'checkpoint.pth.tar')): args.resume = os.path.join(args.checkpoint, 'checkpoint.pth.tar') if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) args.checkpoint = os.path.dirname(args.resume) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: print("=> no checkpoint found at '{}'".format(args.resume)) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Epoch', 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.', 'Valid Acc5.' ]) print_options(args.checkpoint, args) cudnn.benchmark = True traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_transforms = transforms.Compose([ transforms.RandomResizedCrop(args.input_size), #224 transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ]) train_dataset = ImageNet(traindir, train_transforms) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) train_loader_len = len(train_loader) val_loader = torch.utils.data.DataLoader( ImageNet( valdir, transforms.Compose([ transforms.Resize(256), #256 transforms.CenterCrop(args.input_size), #224 transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) val_loader_len = len(val_loader) if args.evaluate: from collections import OrderedDict if os.path.isfile(args.weight): print("=> loading pretrained weight '{}'".format(args.weight)) source_state = torch.load(args.weight)['state_dict'] target_state = OrderedDict() for k, v in source_state.items(): if k[:7] != 'module.': k = 'module.' + k target_state[k] = v model.load_state_dict(target_state) else: print("=> no weight found at '{}'".format(args.weight)) validate(val_loader, val_loader_len, model, criterion) return # visualization writer = SummaryWriter(os.path.join(args.checkpoint, 'logs')) for epoch in range(args.start_epoch, args.epochs): if args.distributed: train_sampler.set_epoch(epoch) print('\nEpoch: [%d | %d]' % (epoch + 1, args.epochs)) # train for one epoch train_loss, train_acc = train(train_loader, train_loader_len, model, criterion, optimizer, epoch) # evaluate on validation set val_loss, prec1, prec5 = validate(val_loader, val_loader_len, model, criterion) lr = optimizer.param_groups[0]['lr'] # append logger file logger.append( [epoch + 1, lr, train_loss, val_loss, train_acc, prec1, prec5]) # tensorboardX writer.add_scalar('learning rate', lr, epoch + 1) writer.add_scalars('loss', { 'train loss': train_loss, 'validation loss': val_loss }, epoch + 1) writer.add_scalars('accuracy', { 'train accuracy': train_acc, 'validation accuracy': prec1 }, epoch + 1) is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() writer.close() print('Best accuracy:') print(best_prec1) time.sleep(360000)
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch os.makedirs(args.save_dir, exist_ok=True) # ######################################### Dataset ################################################ train_transforms = transforms.Compose([ # transforms.RandomResizedCrop(224), transforms.RandomResizedCrop(224, scale=(0.1, 1.0), ratio=(0.8, 1.25)), # according to official open LTH transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) val_transforms = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) # #################### train / valid dataset #################### train_dir = os.path.join(args.img_dir, 'train') valid_dir = os.path.join(args.img_dir, 'val') trainset = datasets.ImageFolder(root=train_dir, transform=train_transforms) devset = datasets.ImageFolder(root=valid_dir, transform=val_transforms) print('Total images in train, ', len(trainset)) print('Total images in valid, ', len(devset)) # #################### data loader #################### trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) devloader = data.DataLoader(devset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # ######################################### Model ################################################## print("==> creating model ResNet={}".format(args.depth)) if args.depth == 18: model = resnet18(pretrained=False) model_ref = resnet18(pretrained=False) teacher_model = resnet18(pretrained=False) elif args.depth == 34: model = resnet34(pretrained=False) model_ref = resnet34(pretrained=False) teacher_model = resnet34(pretrained=False) elif args.depth == 50: model = resnet50(pretrained=False) model_ref = resnet50(pretrained=False) teacher_model = resnet50(pretrained=False) elif args.depth == 101: model = resnet101(pretrained=False) model_ref = resnet101(pretrained=False) teacher_model = resnet101(pretrained=False) elif args.depth == 152: model = resnet152(pretrained=False) model_ref = resnet152(pretrained=False) teacher_model = resnet152(pretrained=False) else: model = resnet50(pretrained=False) # default Res-50 model_ref = resnet50(pretrained=False) # default Res-50 teacher_model = resnet50(pretrained=False) # default Res-50 model.cuda(device_ids[0]) # model to train (student model) model_ref.cuda(device_ids[0]) # pruned model teacher_model.cuda(device_ids[0]) # teacher model, the last epoch of unpruned training model # ############################### Optimizer and Loss ############################### criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # ************* USE APEX ************* if USE_APEX: print('Use APEX !!! Initialize Model with APEX') model, optimizer = apex.amp.initialize(model, optimizer, loss_scale='dynamic', verbosity=0) # ****************** multi-GPU ****************** model = nn.DataParallel(model, device_ids=device_ids) model_ref = nn.DataParallel(model_ref, device_ids=device_ids) teacher_model = nn.DataParallel(teacher_model, device_ids=device_ids) cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) # ############################### Resume ############################### # load pruned model (model_ref), use it to mute some weights of model title = 'ImageNet' if args.resume: # Load checkpoint. print('==> Getting reference model from checkpoint..') assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' checkpoint = torch.load(args.resume, map_location=lambda storage, loc: storage) best_acc = checkpoint['best_acc'] start_epoch = args.start_epoch model_ref.load_state_dict(checkpoint['state_dict']) logger = Logger(os.path.join(args.save_dir, 'log_scratch.txt'), title=title) logger.set_names(['EPOCH', 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) # set some weights to zero, according to model_ref --------------------------------- # ############## load Lottery Ticket (initialization parameters of un pruned model) ############## if args.model: print('==> Loading init model (Lottery Ticket) from %s' % args.model) checkpoint = torch.load(args.model, map_location=lambda storage, loc: storage) model.load_state_dict(checkpoint['state_dict']) if 'init' in args.model: start_epoch = 0 else: start_epoch = checkpoint['epoch'] print('Start Epoch ', start_epoch) for m, m_ref in zip(model.modules(), model_ref.modules()): if isinstance(m, nn.Conv2d): weight_copy = m_ref.weight.data.abs().clone() mask = weight_copy.gt(0).float().cuda() m.weight.data.mul_(mask) # ############## load parameters of teacher model ############## print('==> Loading teacher model (un-pruned) from %s' % args.teacher) checkpoint = torch.load(args.teacher, map_location=lambda storage, loc: storage) teacher_model.load_state_dict(checkpoint['state_dict']) teacher_model.eval() # ############################### Train and val ############################### for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) num_parameters = get_conv_zero_param(model) print('Zero parameters: {}'.format(num_parameters)) num_parameters = sum([param.nelement() for param in model.parameters()]) print('Parameters: {}'.format(num_parameters)) # train model train_loss, train_acc = train(trainloader, model, teacher_model, optimizer, epoch, use_cuda) # ######## acc on validation data each epoch ######## dev_loss, dev_acc = test(devloader, model, criterion, epoch, use_cuda) # append logger file logger.append([ epoch, state['lr'], train_loss, dev_loss, train_acc, dev_acc]) # save model after one epoch # Note: save all models after one epoch, to help find the best rewind is_best = dev_acc > best_acc best_acc = max(dev_acc, best_acc) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': dev_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.save_dir, filename=str(epoch + 1)+'_checkpoint.pth.tar') print('Best val acc:') print(best_acc) logger.close()
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data loading code valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) print('no DA') transform_train = transforms.Compose([ transforms.ToTensor(), normalize, ]) if args.img_size == 32: train_loader = torch.utils.data.DataLoader( ImageNet_Train('/BS/database11/ILSVRC2012_imgsize32/train', transform_train), # datasets.ImageFolder(traindir, transform_train), batch_size=args.train_batch, shuffle=True, num_workers=args.workers) transform_test = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.Resize(32), transforms.ToTensor(), normalize, ]) elif args.img_size == 64: train_loader = torch.utils.data.DataLoader( ImageNet_Train_64('/BS/database11/ILSVRC2012_imgsize64/', transform_train), # datasets.ImageFolder(traindir, transform_train), batch_size=args.train_batch, shuffle=True, num_workers=args.workers) transform_test = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.Resize(64), transforms.ToTensor(), normalize, ]) else: raise Exception('img size can only be 32 or 64') val_loader = torch.utils.data.DataLoader(datasets.ImageFolder( valdir, transform_test), batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # create model if args.pretrained: print("=> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch](pretrained=True) elif args.arch.startswith('d1_resnet50'): model = models.__dict__[args.arch]( num_classes=1000, include_top=True, dropout_rate=0, layer=args.layer, is_shuff=False # TODO: check ) elif args.arch.endswith('vgg16_1d'): model = models.__dict__[args.arch](num_classes=1000, include_top=True, dropout_rate=0, layer=args.layer, is_shuff=False) else: raise Exception( 'you should only choose vgg16_1d or d1_resnet50 as the model') model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = False print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'ImageNet-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(val_loader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(val_loader, model, criterion, epoch, use_cuda) # append logger file logger.append( [state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
2], avg_cost[index, 3], avg_cost[index, 4], avg_cost[index, 5], avg_cost[index, 6], avg_cost[index, 7], avg_cost[index, 8], avg_cost[index, 9], avg_cost[index, 10], avg_cost[index, 11], avg_cost[index, 12], avg_cost[index, 13], avg_cost[index, 14], avg_cost[index, 15], avg_cost[index, 16], avg_cost[index, 17], avg_cost[index, 18], avg_cost[index, 19], avg_cost[index, 20], avg_cost[index, 21], avg_cost[index, 22], avg_cost[index, 23])) logger.append([ index, avg_cost[index, 0], avg_cost[index, 1], avg_cost[index, 2], avg_cost[index, 3], avg_cost[index, 4], avg_cost[index, 5], avg_cost[index, 6], avg_cost[index, 7], avg_cost[index, 8], avg_cost[index, 9], avg_cost[index, 10], avg_cost[index, 11], avg_cost[index, 12], avg_cost[index, 13], avg_cost[index, 14], avg_cost[index, 15], avg_cost[index, 16], avg_cost[index, 17], avg_cost[index, 18], avg_cost[index, 19], avg_cost[index, 20], avg_cost[index, 21], avg_cost[index, 22], avg_cost[index, 23], dist_loss_save[0].avg, dist_loss_save[1].avg, dist_loss_save[2].avg ]) if isbest: best_loss = loss_index print_index = index save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_loss': best_loss,
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch os.makedirs(args.save_dir, exist_ok=True) # Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 else: dataloader = datasets.CIFAR100 num_classes = 100 trainset = dataloader(root='./data', train=True, download=True, transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = dataloader(root='./data', train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) model_ref = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) model_ref = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) model_ref = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, ) model_ref = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, ) else: model = models.__dict__[args.arch](num_classes=num_classes) model_ref = models.__dict__[args.arch](num_classes=num_classes) model.cuda() model_ref.cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # default is 0.001 # Resume title = 'cifar-10-' + args.arch if args.resume: # Load checkpoint. print('==> Getting reference model from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' # args.save_dir = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = args.start_epoch model_ref.load_state_dict(checkpoint['state_dict']) logger = Logger(os.path.join(args.save_dir, 'log_scratch.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) # set some weights to zero, according to model_ref --------------------------------- for m, m_ref in zip(model.modules(), model_ref.modules()): if isinstance(m, nn.Conv2d): weight_copy = m_ref.weight.data.abs().clone() mask = weight_copy.gt(0).float().cuda() n = mask.sum() / float(m.in_channels) m.weight.data.normal_(0, math.sqrt(2. / n)) m.weight.data.mul_(mask) # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) num_parameters = get_conv_zero_param(model) print('Zero parameters: {}'.format(num_parameters)) num_parameters = sum( [param.nelement() for param in model.parameters()]) print('Parameters: {}'.format(num_parameters)) train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda) # append logger file logger.append( [state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.save_dir) logger.close() print('Best acc:') print(best_acc)
val_acc_log = AverageMeter() for i, (images, labels) in enumerate(train_loader): # Convert torch tensor to Variable images = Variable(images.cuda()) labels = Variable(labels.cuda()) # Forward + Backward + Optimize optimizer.zero_grad() # zero the gradient buffer outputs = net(images) train_loss = criterion(outputs, labels) if i == 0: print(labels) # check dataLoader randomness if epoch == 0: # loss of the 1st mini-batch in the 1st epoch before backgrop, verify randomness of weight initialization train_init_loss = train_loss logger.append([0, train_init_loss, 0, 0, 0]) train_loss.backward() optimizer.step() prec1, prec5 = accuracy(outputs.data, labels.data, topk=(1, 5)) train_loss_log.update(train_loss.data[0], images.size(0)) train_acc_log.update(prec1[0], images.size(0)) if (i + 1) % 100 == 0: print('Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Acc: %.8f' % (epoch + 1, num_epochs, i + 1, len(train_dataset) // batch_size, train_loss_log.avg, train_acc_log.avg)) # Test the Model net.eval() correct = 0 loss = 0
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.out): mkdir_p(args.out) # Data print( f'==> Preparing {"asymmetric" if args.asym else "symmetric"} nosiy cifar10' ) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=8), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_val = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) trainset, valset = dataset.get_cifar10('./data', args, train=True, download=True, transform_train=transform_train, transform_val=transform_val) trainloader = data.DataLoader(trainset, batch_size=args.batch_size, shuffle=True, num_workers=4) valloader = data.DataLoader(valset, batch_size=args.batch_size, shuffle=False, num_workers=4) # Model print("==> creating preact_resnet") model = models.PreActResNet34() model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) val_criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'noisy-cifar-10' if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.out = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.out, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.out, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) # Train and val for epoch in range(start_epoch, args.epochs): print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(trainloader, model, optimizer, epoch, use_cuda) val_loss, val_acc = validate(valloader, model, val_criterion, epoch, use_cuda) # append logger file logger.append([state['lr'], train_loss, val_loss, train_acc, val_acc]) # save model is_best = val_acc > best_acc best_acc = max(val_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': val_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best) logger.close() logger.plot() savefig(os.path.join(args.out, 'log.eps')) print('Best acc:') print(best_acc)
def main(): global best_acc if not os.path.isdir(args.out): mkdir_p(args.out) # data num_classes = 3 train_set = CxrDataset(STANFORD_CXR_BASE, "data/Stanford_train_small.csv") val_set = CxrDataset(STANFORD_CXR_BASE, "data/Stanford_valid.csv") trainloader = data.DataLoader(train_set, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, drop_last=True) val_loader = data.DataLoader(val_set, batch_size=args.batch_size, shuffle=False, num_workers=args.workers) # model print("==> creating model") def create_model(num_classes, ema=False): model = DenseNet121(num_classes) model = torch.nn.DataParallel(model).cuda() if ema: for param in model.parameters(): param.detach_() return model model = create_model(num_classes=num_classes, ema=False) cudnn.benchmark = True print('Ttoal params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=args.lr) start_epoch = 0 # resume title = 'supervised-Stanford' if args.resume: # load checkpoints print('==> Resuming from checkpoints..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.out = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.out, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.out, 'log.txt'), title=title) logger.set_names(['Train Loss', 'Valid Loss', 'Valid Acc.']) writer = SummaryWriter(args.out) step = 0 test_accs = [] # train and val for epoch in range(start_epoch, args.epochs): print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_iteration, train_loss = train(trainloader, num_classes, model, optimizer, criterion, epoch, use_cuda) train_acc_iteration, _, train_acc = validate(trainloader, model, num_classes, criterion, epoch, use_cuda, mode='Train Stats') val_iteration, val_loss, val_acc = validate(val_loader, model, num_classes, criterion, epoch, use_cuda, mode='Valid Stats') #step = args.val_iteration * (epoch+1) writer.add_scalar('loss/train_loss', train_loss, (epoch + 1)) writer.add_scalar('loss/valid_loss', val_loss, (epoch + 1)) writer.add_scalar('accuracy/train_acc', train_acc, (epoch + 1)) writer.add_scalar('accuracy/val_acc', val_acc, (epoch + 1)) # append logger file logger.append([train_loss, val_loss, val_acc]) # save model is_best = val_acc > best_acc best_acc = max(val_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': val_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict() }, is_best) logger.close() writer.close() print('Best acc:') print(best_acc) print('Mean acc:') print(np.mean(val_accs[-20:]))
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data loading code traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) data_aug_scale = (0.08, 1.0) if args.modelsize == 'large' else (0.2, 1.0) train_loader = torch.utils.data.DataLoader(datasets.ImageFolder( traindir, transforms.Compose([ transforms.RandomResizedCrop(224, scale=data_aug_scale), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])), batch_size=args.train_batch, shuffle=True, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(datasets.ImageFolder( valdir, transforms.Compose([ transforms.Scale(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.test_batch, shuffle=True, num_workers=args.workers, pin_memory=True) # create model if args.pretrained: print("=> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch](pretrained=True) elif 'resnext' in args.arch: model = models.__dict__[args.arch]( baseWidth=args.base_width, cardinality=args.cardinality, ) else: print("=> creating model '{}'".format(args.arch)) model = models.__dict__[args.arch]() flops, params = get_model_complexity_info(model, (224, 224), as_strings=False, print_per_layer_stat=False) print('Flops: %.3f' % (flops / 1e9)) print('Params: %.2fM' % (params / 1e6)) if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'ImageNet-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..', args.resume) assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] # model may have more keys t = model.state_dict() c = checkpoint['state_dict'] flag = True for k in t: if k not in c: print('not in loading dict! fill it', k, t[k]) c[k] = t[k] flag = False model.load_state_dict(c) if flag: print('optimizer load old state') optimizer.load_state_dict(checkpoint['optimizer']) else: print('new optimizer !') logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(val_loader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(val_loader, model, criterion, epoch, use_cuda) # append logger file logger.append( [state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() print('Best acc:') print(best_acc)
def select_links_and_channels(): """ """ _idx = random.randint(0, len(globs.arch) - 1) tups = globs.arch[_idx] links = [] links_to_tx_rx = {} for tup in tups: dev_tx = tup[0] dev_rx = tup[1] globs.tx_links[dev_tx] = [dev_rx, 'tx', -1] globs.tx_links[dev_rx] = [dev_tx, 'rx', -1] l = globs.links[dev_tx][dev_rx] links.append(l) links_to_tx_rx[l] = (dev_tx, dev_rx) t_now = time.time() links_to_ch = globs.decision_func(links, globs.use_channels) Logger.append('bs', 'decision_time', time.time() - t_now) for link, ch in links_to_ch.iteritems(): dev_tx, dev_rx = links_to_tx_rx[link] # increment counter os used channels globs.p_channels_count[ch] += 1 globs.allocated_channels.append(ch) # tx links was {key: [other_side, operation]} # tx links and now is {key: [other_side, operation, channel]} globs.tx_links[dev_tx] = [dev_rx, "tx", ch] globs.tx_links[dev_rx] = [dev_tx, "rx", ch] #links = copy.deepcopy(globs.tx_links) #used = [] #for dev1, _tup in links.iteritems(): # if globs.use_channels and dev1 not in used: # dev2 = _tup[0] # operation = _tup[1] # used.extend( [dev1, dev2]) # ## LOGIC TO SELECT CHANNELS # channel = globs.decision_func() # # channel can be either a single element OR # # a dictionary: {link_x: frequency, link_y: frequency y, ...} # if isinstance(channel, dict): # print "##### Channel list returned is a dict" # channel = channel[globs.links[dev1][dev2]] # #### # # increment counter os used channels # globs.p_channels_count[channel] += 1 # globs.allocated_channels.append( channel ) # # tx links was {key: [other_side, operation]} # # tx links and now is {key: [other_side, operation, channel]} # globs.tx_links[dev1].append(channel) # globs.tx_links[dev2].append(channel) Logger.append('bs', 'links', globs.tx_links)
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 else: dataloader = datasets.CIFAR100 num_classes = 100 trainset = dataloader(root='./data', train=True, download=True, transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = dataloader(root='./data', train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, ) else: model = models.__dict__[args.arch](num_classes=num_classes) # model = torch.nn.DataParallel(model).cuda() # model = model.cuda(torch.device('cuda:1')) model = model.cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) torch.save(model, 'tempolary.pth') new_model = torch.load('tempolary.pth') criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'cifar-10-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) if DEBUG: # print(model) show_low_rank(model, input_size=[32, 32]) print(' Start decomposition:') look_up_table = get_look_up_table(model) #print(model) thresholds = [1e-2] #+0.01*x for x in range(20)] T = np.array(thresholds) cr = np.zeros(T.shape) acc = np.zeros(T.shape) model_path = 'net.pth' torch.save(model, model_path) for i, t in enumerate(thresholds): test_model = torch.load(model_path) cr[i] = show_low_rank(test_model, input_size=[32, 32], criterion=ValueThreshold(t)) test_model = f_decouple(test_model, look_up_table, criterion=ValueThreshold(t), dataloader=testloader, train=False) #print(model) print(' Done! test decoupled model') test_loss, test_acc = test(testloader, test_model, criterion, start_epoch, use_cuda) print(' Test Loss : %.8f, Test Acc: %.2f' % (test_loss, test_acc)) acc[i] = test_acc if args.retrain: # retrain model ''' for m in test_model.modules(): if isinstance(m, nn.Conv2d): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels m.weight.data.normal_(0, math.sqrt(2. / n)) elif isinstance(m, nn.BatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_() ''' print(' Retrain decoupled model') finetune_epoch = 4 #args.schedule = [int(finetune_epoch/2), finetune_epoch] best_acc = 0.0 optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) global state init_lr = args.lr state['lr'] = init_lr for epoch in range(finetune_epoch): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, finetune_epoch, state['lr'])) train_loss, train_acc = train(trainloader, test_model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(testloader, test_model, criterion, epoch, use_cuda) best_acc = max(test_acc, best_acc) # append logger file # logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc]) #logger.close() #logger.plot() #savefig(os.path.join(args.checkpoint, 'log.eps')) acc[i] = best_acc torch.save(OrderedDict([('acc', acc), ('cr', cr)]), 'result-vh+retrain.pth') print(cr) print(acc) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda) # append logger file logger.append( [state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def cognitive_radio_loop(options, radio, channel_list): """ Program loop here. @param options @param radio A RadioDevice instance. @param channel_list List of Channel objects. """ # Export my server command = callback_radio(radio) my_rpc = RPCExporter(addr=("143.54.83.30", 8000 + options.my_id)) my_rpc.register_function('command', command) my_rpc.start() # Wait broker start while not command.run: 1 #### #Import OTHER RADIOS RPCS #### rpc_arr = [] for i in [0, 1, 2, 3]: rpc_cli = RPCImporter(addr="http://%s:%d" % (HOSTS_IP[i], 8000 + i)) rpc_cli.register_function('command') rpc_arr.append(rpc_cli) # Import PassiveRadio RPC calls bs_rpc = RPCImporter(addr="http://%s:9000" % (options.broker_ip)) bs_rpc.register_function('command') # Register parameters for transmission Logger.register('radio', ['tx_pkts', 'rx_pkts', 'rx2_pkts', 'channel', 'operation', 'receiver', 'starving', 'total_tx', 'total_rx', 'total_starving']) # loop pkt_len = options.pkt_len payload = struct.pack('%sB' % pkt_len, *[options.my_id] * pkt_len) print '##### Entering Transmitter loop' c_starving = 0 while command.run: """ ######## FUNCOES: ---- BW do canal ---- Channel's bandwidth. radio.get_bandwidth() ---- Num. simbolos na modulacao --- Number of symbols in the modulation. radio.{tx,rx}.symbols() ---- B/S da modulacao ---- B/S of the modulation radio.{tx,rx}.bits_per_symbol() ---- Pkt/s NO ULTIMO SEGUNDO ---- Pkt/s in the last second. radio.{tx,rx}.counter.get_pkts() ---- b/s NO ULTIMO SEGUNDO. ---- b/s in the last second. radio.{tx,rx}.counter.get_bps() ---- Pacotes acumulados desde a ultima chamada. ---- Accumulated packages since the last call. radio.{tx,rx}.counter.get_pkt_accumulated(clear = False) ---- Troca de canal. channel eh um objeto Channel ---- Channel changing. channel is a Channel object. radio.set_channel(channel) ################# """ # sense while not command.sense and command.run: 1 if not command.run: break sense_data = [] radio.set_gain(0) for channel in channel_list: decision, energy = radio.ss.sense_channel(channel, 0.1) sense_data.append((decision, float(energy), channel.get_channel())) bs_rpc.command([options.my_id, 'sense_data', sense_data]) # CHOOSE RECEIVER #while not command.request_transmission: # 1 # Select a receiver randomly #opt = [0, 1, 2, 3] #opt.remove(options.my_id) #receiver = random.choice(opt) #print '##### DEVICE %d requesting TX to %d' % (options.my_id, receiver) #bs_rpc.command([options.my_id, 'request_tx', receiver]) #radio.set_gain(radios_gain[radio.id]) # REQUEST CHANNEL while not command.request_channel: 1 receiver, operation, channel_idx = bs_rpc.command([options.my_id, 'request_channel', 9999]) # Configure radio. Go to distant frequency if 'idle' if channel_idx > -1 or operation == OPERATIONS['idle']: if channel_idx > -1: radio.set_channel(channel_list[channel_idx]) else: radio.set_channel(CHANNEL_IDLE) while not command.transmission: 1 # tx pkts is the number of packets transmitted # globs.instant_rx is the number of packets received. # Is global cause it is hard to be synchronized with the TX dev when WE are the RX # The easiest way is to let the TX dev control when get this value. And we do it in the RPC server method tx_pkts = globs.instant_rx = 0 if operation == OPERATIONS['tx']: # change rx freq to another freq or we will receiver our own packets radio.rx.radio.set_channel(CHANNEL_IDLE) print "##### ... %d - TRANSMITTING - CHANNEL %d - TO: %d" % (options.my_id, channel_idx, receiver) tx_pkts = PktSender.flood_by_time(duration=options.sending_duration, tx_pkt_arch=radio.tx, payload=payload) globs.instant_rx = rpc_arr[receiver].command((options.my_id, 'get_accumulated', True)) print "##### ... Transmitted %d/%d pkts in Channel %d" % (tx_pkts, globs.instant_rx, channel_idx) globs.total_tx += tx_pkts c_starving = 0 elif operation == OPERATIONS['rx']: print "##### ... %d - RECEIVING - CHANNEL %d" % (options.my_id, channel_idx) receiver = 10 globs.wait_for_get = True while globs.wait_for_get: time.sleep(0.2) tx_pkts = 0 globs.total_rx += globs.instant_rx c_starving += 1 globs.total_starving += 1 elif operation == OPERATIONS['idle']: print '##### ... %d is IDLE' % options.my_id receiver = -1 time.sleep(options.sending_duration) c_starving += 1 globs.total_starving += 1 bs_rpc.command([options.my_id, 'transmission_res', (tx_pkts, globs.instant_rx)]) _idle = 0 if operation != OPERATIONS['idle'] else -1 Logger.append('radio', 'tx_pkts', tx_pkts if _idle > -1 else _idle) Logger.append('radio', 'rx_pkts', globs.instant_rx if operation == OPERATIONS['tx'] else _idle) Logger.append('radio', 'rx2_pkts', globs.instant_rx if operation == OPERATIONS['rx'] else _idle) Logger.append('radio', 'channel', channel_idx) Logger.append('radio', 'operation', operation) Logger.append('radio', 'receiver', receiver) Logger.append('radio', 'starving', c_starving) Logger.set('radio', 'total_tx', globs.total_tx) Logger.set('radio', 'total_rx', globs.total_rx) Logger.set('radio', 'total_starving', globs.total_starving)
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Dataset preprocessing title = 'GTSRB' # Create Datasets transform_train_poisoned = transforms.Compose([ transforms.Resize((32, 32)), TriggerAppending(trigger=args.trigger, alpha=args.alpha), transforms.ToTensor(), ]) transform_train_benign = transforms.Compose([ transforms.Resize((32, 32)), transforms.ToTensor(), ]) transform_test_poisoned = transforms.Compose([ transforms.Resize((32, 32)), TriggerAppending(trigger=args.trigger, alpha=args.alpha), transforms.ToTensor(), ]) transform_test_benign = transforms.Compose([ transforms.Resize((32, 32)), transforms.ToTensor(), ]) print('==> Loading the dataset') # Create Datasets num_training = len( dataset.GTSRB(root_dir='./data', train=True, transform=transform_train_benign)) num_poisoned = int(num_training * args.poison_rate) idx = list(np.arange(num_training)) random.shuffle(idx) poisoned_idx = idx[:num_poisoned] benign_idx = idx[num_poisoned:] poisoned_trainset = dataset.GTSRB_subset( root_dir='./data', train=True, transform=transform_train_poisoned, List=poisoned_idx, y_target=args.y_target) benign_trainset = dataset.GTSRB_subset(root_dir='./data', train=True, transform=transform_train_benign, List=benign_idx, y_target=None) poisoned_testset = dataset.GTSRB(root_dir='./data', train=False, transform=transform_test_poisoned, y_target=args.y_target) benign_testset = dataset.GTSRB(root_dir='./data', train=False, transform=transform_test_benign, y_target=None) poisoned_trainloader = torch.utils.data.DataLoader( poisoned_trainset, batch_size=int(args.train_batch * args.poison_rate), shuffle=True, num_workers=args.workers) benign_trainloader = torch.utils.data.DataLoader( benign_trainset, batch_size=int(args.train_batch * (1 - args.poison_rate) * 0.9), shuffle=True, num_workers=args.workers ) # *0.9 to prevent the iterations of benign data is less than that of poisoned data poisoned_testloader = torch.utils.data.DataLoader( poisoned_testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) benign_testloader = torch.utils.data.DataLoader(benign_testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) print( "Num of training samples %i (Num of poisoned samples %i, Num of benign samples %i), Num of testing samples %i" % (num_training, num_poisoned, num_training - num_poisoned, len(benign_testset))) # Model model = ResNet18() model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print('Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Benign Valid Loss', 'Poisoned Valid Loss', 'Train ACC.', 'Benign Valid ACC.', 'Poisoned Valid ACC.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(args, model, poisoned_trainloader, benign_trainloader, criterion, optimizer, epoch, use_cuda) test_loss_benign, test_acc_benign = test(benign_testloader, model, criterion, epoch, use_cuda) test_loss_poisoned, test_acc_poisoned = test(poisoned_testloader, model, criterion, epoch, use_cuda) # append logger file logger.append([ state['lr'], train_loss, test_loss_benign, test_loss_poisoned, train_acc, test_acc_benign, test_acc_poisoned ]) # save model is_best = test_acc_benign > best_acc best_acc = max(test_acc_benign, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc_benign, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def server_loop(options): """ Receiver radios in this loop @param options """ # Export my 'command' function to be accessible via RPC my_rpc = RPCExporter(addr=("143.54.83.30", 9000)) command = server_callback(globs.channel_list) my_rpc.register_function('command', command) my_rpc.start() interferer_rpc = RPCImporter(addr="http://143.54.83.30:9001") interferer_rpc.register_function('command') #### #Import OTHER RADIOS RPCS #### for i in DEVICES: rpc_cli = RPCImporter(addr="http://143.54.83.30:%d" % (8000 + i)) rpc_cli.register_function('command') globs.rpc_arr.append(rpc_cli) def command_sense(val): """ @param val """ [x.command([BS_ID, 'sense', val]) for x in globs.rpc_arr] def command_request_transmission(val): """ @param val """ [x.command([BS_ID, 'request_transmission', val]) for x in globs.rpc_arr] def command_request_channel(val): """ @param val """ [x.command([BS_ID, 'request_channel', val]) for x in globs.rpc_arr] def command_transmission(val): """ @param val """ [x.command([BS_ID, 'transmission', val]) for x in globs.rpc_arr] # Start execution on all RC nodes interferer_rpc.command([BS_ID, 'run', True]) [x.command([BS_ID, 'run', True]) for x in globs.rpc_arr] t_fin = time.time() + options.test_duration while t_fin > time.time(): t_it = time.time() print "!!!!! SENSING PHASE" command_sense(True) while not globs.sense_done: 1 globs.sense_done = False command_sense(False) print "!!!!! TRANSMISSION PHASE" command_transmission(True) globs.p_fin_transmission_time = time.time() + options.sending_duration while not globs.transmission_done: 1 globs.p_fin_transmission_time = None ## moment to evaluate links # HERE #### # clear all used data globs.transmission_done = False globs.clear() globs.p_total_iterations += 1 command_transmission(False) Logger.append('bs', 'it_dur', time.time() - t_it) # Stop execution on all CR nodes interferer_rpc.command([BS_ID, 'run', False]) [x.command([BS_ID, 'run', False]) for x in globs.rpc_arr]
def transmitter_loop(tb, channel_list, channel, options): """ US LOOP @param tb @param channel_list @param channel @param options """ # Q-Noise+ Parameters lookback = 3 beta = 0.1 eps = 0.1 history_weight = [0.2, 0.35, 0.45] noise_weight = 0.8 alpha = 0.2 # alpha + noise+weith = 1 epochs = 100 #Q-Noise+ learner = qnoise.Learner(channel_list, lookback, alpha, beta, eps, history_weight, noise_weight) # Connect to slave device import xmlrpclib proxy = xmlrpclib.ServerProxy("http://%s:8000/" % options.slave_addr) start_t = time.time() proxy.client_started() proxy.set_channel(channel) Logger.register('transmitter', ['channel', 'status', 'pkt']) class TNamespace(): """ """ pass # Sensing -> TX loop t_namespace = TNamespace() t_namespace.pkt_s = 0 t_namespace.status = 0 t_namespace.iteration = 0 t_namespace.ss_result = {} for ch in channel_list: t_namespace.ss_result[ch.channel] = [0, 0, 0] ch_qvalue = 0 while time.time() < (start_t + options.duration): can_transmit = True t_namespace.pkt_r = t_namespace.pkt_s = 0 # Sense print '### START SENSING CHANNEL' t_namespace.status = tb.rx.sense_channel(channel_list[channel], options.sensing_duration) # Update #print t_namespace.status if t_namespace.status > 0.000005: # GMSK threahold #if t_namespace.status > 0.000000005 : print str(channel_list[channel]) + ' is occupied' t_now = time.clock() can_transmit = True # Change channel #proxy.set_channel( channel ) # Transmit if can_transmit: payload = 0 if options.pkt_size > 1: bytelist = [1] * (options.pkt_size/4) payload = pack('%sH' % len(bytelist), *bytelist) else: bytelist = ['a', ] payload = pack('%sc' % 1, *bytelist) # thread sending packets def send_thread(): t_namespace.pkt_s = 0 #while t_namespace.pkt_sending: while t_namespace.pkt_s < 200: tb.tx.send_pkt(payload) t_namespace.pkt_s += 1 #print 't: ', t_namespace.pkt_s time.sleep(0.03) #t_namespace.count += 1 # init thread th = Thread(target=send_thread) proxy.start_rcv() t_namespace.pkt_sending = True th.start() # wait for options.sending_duration #time.sleep( options.sending_duration ) # stop sending t_namespace.pkt_sending = False th.join() t_namespace.pkt_r = proxy.get_rcv() print 'pkt_s = ', t_namespace.pkt_s print 'pkt_r = ', t_namespace.pkt_r ch_qvalue = learner.evaluate(channel_list[channel].channel, t_namespace.pkt_s, t_namespace.pkt_r, t_namespace.status) t_namespace.ss_result[channel+1] = [t_namespace.status, 0 if can_transmit else 1, ch_qvalue] channel = learner.choose_next_channel(channel_list[channel].channel) - 1 # -1 cause we use the index in the array. print "Using channel ", channel_list[channel] proxy.set_channel(channel) tb.tx.radio.center_freq = channel_list[channel] Logger.append('transmitter', 'channel', channel) Logger.append('transmitter', 'status', t_namespace.status) Logger.append('transmitter', 'pkt', t_namespace.pkt_s) dump_qlearner(t_namespace.iteration, t_namespace.ss_result, noise_weight) t_namespace.iteration += 1 proxy.close_app()