def main(args): print("Loading data") train_dataset = datasets.ImageFolder(args.data + "/train", transform=train_transforms) val_dataset = datasets.ImageFolder(args.data + "/val", transform=val_transforms) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=1) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=1) print("Data loaded") model = Network(args) if args.cuda: model = model.cuda() optimizer = optim.Adam(model.parameters(), lr=args.lr) print("Starting training") train(args, model, optimizer, train_loader, val_loader)
def get(self): allowed_users = AllowedUser.query().order(AllowedUser.email) networks = Network.query().order(Network.name) template_values = { 'users': allowed_users, 'networks': networks } path = 'admin.html' self.response.out.write(template.render(path, template_values))
def parse_network(environment, data): if data.has_key('name'): name = data['name'] elif data.has_key('network'): name = data['network'] else: raise ConfigError, "Unnamed network" network = Network(name) if data.has_key('dhcp_server'): network.dhcp_server = data['dhcp_server'] for existing_network in environment.networks: if existing_network.name == network.name: raise ConfigError, "Network with given name already exists: %s" % network.name environment.networks.append(network) return network
def __init__(self, rng): Network.__init__(self, n_hidden_layer = n_hidden_layer, BN = BN) print " Fully connected layer 1:" self.layer.append(ReLU_layer(rng = rng, n_inputs = n_inputs, n_units = n_units, BN = BN, BN_epsilon=BN_epsilon, dropout=dropout_input, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test)) for k in range(n_hidden_layer-1): print " Fully connected layer "+ str(k) +":" self.layer.append(ReLU_layer(rng = rng, n_inputs = n_units, n_units = n_units, BN = BN, BN_epsilon=BN_epsilon, dropout=dropout_hidden, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test)) print " L2 SVM layer:" self.layer.append(linear_layer(rng = rng, n_inputs = n_units, n_units = n_classes, BN = BN, BN_epsilon=BN_epsilon, dropout=dropout_hidden, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test))
def __init__(self,name): Tk.__init__(self, None) self.title('Equilibrium Propagation') self.net = Network(name=name,hyperparameters={"batch_size":1}) self.hidden_sizes = self.net.hyperparameters["hidden_sizes"] self.n_layers = len(self.hidden_sizes) + 2 self.canvas = Canvas(self, width=600, height=(self.n_layers+1)*100) self.canvas.pack(side=BOTTOM) # INDEX OF TEST EXAMPLE (IN THE TEST SET) Label(self, text="image").pack(side=LEFT) self.index = StringVar() self.index.set("0") Entry(self, textvariable=self.index, width=5).pack(side=LEFT) self.update_canvas(first_time=True) Thread(target = self.run).start()
def main(): if not torch.cuda.is_available(): logging.info('No GPU device available') sys.exit(1) np.random.seed(args.seed) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info("args = %s", args) logging.info("unparsed_args = %s", unparsed) num_gpus = torch.cuda.device_count() genotype = eval("genotypes.%s" % args.arch) print('---------Genotype---------') logging.info(genotype) print('--------------------------') model = Network(args.init_channels, CLASSES, args.layers, args.auxiliary, genotype) if num_gpus > 1: model = nn.DataParallel(model) model = model.cuda() else: model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() criterion_smooth = CrossEntropyLabelSmooth(CLASSES, args.label_smooth) criterion_smooth = criterion_smooth.cuda() optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) data_dir = os.path.join(args.tmp_data_dir, 'imagenet') traindir = os.path.join(data_dir, 'train') validdir = os.path.join(data_dir, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_data = dset.ImageFolder( traindir, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.2), transforms.ToTensor(), normalize, ])) valid_data = dset.ImageFolder( validdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])) train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=args.workers) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=args.workers) # scheduler = torch.optim.lr_scheduler.StepLR(optimizer, args.decay_period, gamma=args.gamma) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) best_acc_top1 = 0 best_acc_top5 = 0 lr = args.learning_rate for epoch in range(args.epochs): if args.lr_scheduler == 'cosine': scheduler.step() current_lr = scheduler.get_lr()[0] elif args.lr_scheduler == 'linear': current_lr = adjust_lr(optimizer, epoch) else: print('Wrong lr type, exit') sys.exit(1) logging.info('Epoch: %d lr %e', epoch, current_lr) if epoch < 5 and args.batch_size > 256: for param_group in optimizer.param_groups: param_group['lr'] = lr * (epoch + 1) / 5.0 logging.info('Warming-up Epoch: %d, LR: %e', epoch, lr * (epoch + 1) / 5.0) if num_gpus > 1: model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs else: model.drop_path_prob = args.drop_path_prob * epoch / args.epochs epoch_start = time.time() train_acc, train_obj = train(train_queue, model, criterion_smooth, optimizer) logging.info('Train_acc: %f', train_acc) valid_acc_top1, valid_acc_top5, valid_obj = infer( valid_queue, model, criterion) logging.info('Valid_acc_top1: %f', valid_acc_top1) logging.info('Valid_acc_top5: %f', valid_acc_top5) epoch_duration = time.time() - epoch_start logging.info('Epoch time: %ds.', epoch_duration) is_best = False if valid_acc_top5 > best_acc_top5: best_acc_top5 = valid_acc_top5 if valid_acc_top1 > best_acc_top1: best_acc_top1 = valid_acc_top1 is_best = True utils.save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_acc_top1': best_acc_top1, 'optimizer': optimizer.state_dict(), }, is_best, args.save)
from scipy.stats import spearmanr from matplotlib.colors import colorConverter from model import Network from dataset import Ising from utils import evaluate, format_mse MODEL_PATH = "./cfg/net.pt" DATA_PATH = "../Ising/data/" CUDA = torch.cuda.is_available() BATCH_SIZE = 1 test_set = Ising(DATA_PATH, train=False) test_loader = DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=True) net = Network() net.load_state_dict(torch.load(MODEL_PATH)) if CUDA: net.to(torch.device('cuda')) net.eval() losses, targets, predictions = evaluate(net, test_loader, CUDA=CUDA) r = spearmanr(targets, predictions).correlation loss_color = colorConverter.to_rgba('mediumseagreen', alpha=.5) markerOptions = dict(markerfacecolor='none', markeredgewidth=1.5) fig, ax = plt.subplots(figsize=(10, 6))
num_points=args.num_points, data_dir=args.data), num_workers=8, batch_size=args.batch_size, shuffle=False, drop_last=False) n_classes = 40 logging.info('n_classes: %d', n_classes) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, n_classes, args.num_cells, args.auxiliary, genotype, in_channels=args.in_channels, emb_dims=args.emb_dims, dropout=args.dropout, k=args.k) model = model.cuda() # model = nn.DataParallel(model) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = torch.nn.CrossEntropyLoss().cuda() if args.use_sgd: print("Use SGD") optimizer = torch.optim.SGD(model.parameters(), lr=args.learning_rate * 100, momentum=args.momentum,
def __init__(self): self.net = Network() self.sess = tf.Session() self.sess.run(tf.global_variables_initializer()) self.restore()
def compute(self, x, budget, config, **kwargs): """ Get model with hyperparameters from config generated by get_configspace() """ config = get_config_dictionary(x, config) print("config", config) if (len(config.keys()) < len(x)): return 100 if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) gpu = 'cuda:0' np.random.seed(self.seed) torch.cuda.set_device(gpu) cudnn.benchmark = True torch.manual_seed(self.seed) cudnn.enabled = True torch.cuda.manual_seed(self.seed) logging.info('gpu device = %s' % gpu) logging.info("config = %s", config) genotype = eval("genotypes.%s" % 'PCDARTS') model = Network(self.init_channels, self.n_classes, config['n_conv_layers'], genotype) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() if config['optimizer'] == 'sgd': optimizer = torch.optim.SGD(model.parameters(), lr=config['initial_lr'], momentum=0.9, weight_decay=config['weight_decay'], nesterov=True) else: optimizer = get('opti_dict')[config['optimizer']]( model.parameters(), lr=config['initial_lr'], weight_decay=config['weight_decay']) if config['lr_scheduler'] == 'Cosine': lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, int(budget)) elif config['lr_scheduler'] == 'Exponential': lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1) indices = list(range(int(self.split * len(self.train_dataset)))) valid_indices = list( range(int(self.split * len(self.train_dataset)), len(self.train_dataset))) print("Training size=", len(indices)) training_sampler = SubsetRandomSampler(indices) valid_sampler = SubsetRandomSampler(valid_indices) train_queue = torch.utils.data.DataLoader(dataset=self.train_dataset, batch_size=self.batch_size, sampler=training_sampler) valid_queue = torch.utils.data.DataLoader(dataset=self.train_dataset, batch_size=self.batch_size, sampler=valid_sampler) for epoch in range(int(budget)): lr_scheduler.step() logging.info('epoch %d lr %e', epoch, lr_scheduler.get_lr()[0]) model.drop_path_prob = config['drop_path_prob'] * epoch / int( budget) train_acc, train_obj = train(train_queue, model, criterion, optimizer, grad_clip=config['grad_clip_value']) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) return valid_obj # Hyperband always minimizes, so we want to minimise the error, error = 1-acc
def __init__(self, session, action_dim, state_dim, batch_size=64, nsteps=1, trace_length=10, hidden_size=10, initial_epsilon=0.5, final_epsilon=0.01, replay_size=50000, gamma=0.9, replace_target_freq=10, doubleDQN=False, duelingDQN=False, DRQN=False, store_ckpt=True, ckpt_interval=100, ckpt_dir='./model'): """ Init agent. :param session: tensorflow session. :param action_dim: action dimension. :param state_dim: state dimension. :param batch_size: batch size. :param nsteps: nsteps λ :param trace_length: LSTM trace length. :param hidden_size: Fully connected layer to LSTM layer size. :param initial_epsilon: decay of the epsilon for random selection. :param final_epsilon: decay of the epsilon for random selection. :param replay_size: replay buffer size. :param gamma: decay rate. :param replace_target_freq: target net soft update interval. :param doubleDQN: use double dqn or not. :param duelingDQN: use dueling dqn or not. :param store_ckpt: store checkpoint or not. :param ckpt_interval: the interval for storing the checkpoint. :param ckpt_dir: the directory of the checkpoint files. """ self.session = session self.INITIAL_EPSILON = initial_epsilon self.FINAL_EPSILON = final_epsilon self.REPLAY_SIZE = replay_size self.DOUBLE_DQN = doubleDQN self.DUELING_DQN = duelingDQN self.DRQN = DRQN self.GAMMA = gamma self.REPLACE_TARGET_FREQ = replace_target_freq self.BATCH_SIZE = batch_size self.hidden_size = hidden_size self.trace_length = trace_length self.action_dim = action_dim self.state_dim = state_dim self.nsteps = nsteps self.nstep_buffer = [] # init experience replay self.replay_buffer = deque() # init some parameters self.time_step = 0 self.epsilon = self.INITIAL_EPSILON self.store_ckpt = store_ckpt self.ckpt_dir = ckpt_dir self.ckpt_interval = ckpt_interval self.main_net = Network(self.state_dim, self.action_dim, self.DUELING_DQN, self.DRQN, hidden_size, MAIN_SCOPE) self.target_net = Network(self.state_dim, self.action_dim, self.DUELING_DQN, self.DRQN, hidden_size, TARGET_SCOPE) self.global_step = tf.Variable(0, name="global_step", trainable=False) self.state_in = (np.zeros([1, self.hidden_size]), np.zeros([1, self.hidden_size])) # saver after defining variables self.saver = tf.train.Saver() self.create_training_method() self.soft_update() self.init_checkpoint() self.session.run([ tf.global_variables_initializer(), tf.local_variables_initializer() ]) # self.session = tf_debug.LocalCLIDebugWrapperSession(self.session) writer = tf.summary.FileWriter("./model/", self.session.graph) writer.close() pass
def main(): partition = pickle.load(open(data_directory + glove6b_data_file, 'rb')) # print(np.array(partition['train_data']).shape, np.array(partition['train_label']).shape, np.array(partition['validation_data']).shape, np.array(partition['validation_label']).shape) # print(partition['train_label'][4], partition['train_label'][1], partition['train_label'][0], partition['train_label'][3], np.array(partition['train_data']).shape) # print(np.array(partition['train_data'])[5].shape) train_params = { 'batch_size': batch_size, 'shuffle': False, 'num_workers': 4 } validation_params = { 'batch_size': validation_size, 'shuffle': False, 'num_workers': 4 } partition['train_data'], train_lengths = preprocess_data( partition['train_data']) partition['validation_data'], validation_lengths = preprocess_data( partition['validation_data']) print(partition['train_data'].size(), len(train_lengths), train_lengths[-1]) training_set = Dataset(partition['train_data'], partition['train_label'], train_lengths) training_generator = data.DataLoader(training_set, **train_params) validation_set = Dataset(partition['validation_data'], partition['validation_label'], validation_lengths) validation_generator = data.DataLoader(validation_set, **validation_params) randoms = pickle.load( open(glove_directory + glove6b_random_data_file, 'rb')) print(randoms.shape) loss_function = F.cross_entropy # loss_function = F.cross_entropy model = Network().double().to(cuda_device) optim = torch.optim.Adam(model.parameters(), weight_decay=0.005) train_acc = np.zeros(num_epochs) train_loss = np.zeros(num_epochs) validation_loss = np.zeros(num_epochs) for epoch in range(num_epochs): i = 0 epoch_loss = [] total_test_loss = 0.0 model.zero_grad() for x, y, img, lengths in training_generator: # print('image[0] in train:', img[0:6]) # print('INPUT[0] in train:', x[0]) # print(x.shape, y.shape, img.shape, lengths.shape) # print(x[list(lengths).index(max(list(lengths)))]) neg_samples = torch.zeros((x.shape[0], neg_sample_count, 300)) neg_samples_idx = [ random.sample(range(lengths[i]), neg_sample_count) for i in range(x.shape[0]) ] for k, list_index in enumerate(neg_samples_idx): for l, index in enumerate(list_index): neg_samples[k][l] = x[k][index] # print(neg_samples.size()) # 32 * 5 * 300 x = x.transpose(0, 1) t = random.sample(list(randoms), 26 * x.shape[1]) t = torch.Tensor(t) t = t.view(x.shape[1], 26, -1) # print(t.size()) # 32 * 26 * 300 # print(y.shape) # 32 * 1 * 300 # 26 * 1 * 300 # 5 * 1 * 300 # 32 * 1 * 300 # out: 32 * 32 * 300 result = torch.cat( (torch.cat((y.unsqueeze(1).double(), t.double()), dim=1), neg_samples.double()), dim=1) # print(result.size()) # 32 * 32 * 300 # print(torch.eq(y[0].float(), result[0][0])) # check # x = torch.stack(random.sample(range(lengths[0]), 10)) # print(x.size()) x, img, lengths, result = x.double().to(cuda_device), img.double( ).to(cuda_device), lengths.double().to( cuda_device), result.double().to(cuda_device) y_pred = model(x, img, lengths, result) # print(y.shape, y_pred.shape) # print('y_pred:', y_pred, y_pred.size(), y_pred[0].sum()) loss = loss_function( y_pred, torch.zeros((x.shape[1])).long().to(cuda_device)) epoch_loss.append(loss.item()) optim.zero_grad() loss.backward() optim.step() # print('grad:', model.linear2.weight.grad) print('[%d, %d] Train loss = %.5f' % (epoch + 1, i + batch_size, loss.item())) i += batch_size train_loss[epoch] = np.array(epoch_loss).mean() print('[%d] Train loss = %.5f' % (epoch + 1, np.array(epoch_loss).mean())) epoch_validation_loss = [] for x, y, img, lengths in validation_generator: neg_samples = torch.zeros((x.shape[0], neg_sample_count, 300)) neg_samples_idx = [ random.sample(range(lengths[i]), neg_sample_count) for i in range(x.shape[0]) ] for k, list_index in enumerate(neg_samples_idx): for l, index in enumerate(list_index): neg_samples[k][l] = x[k][index] x = x.transpose(0, 1) t = random.sample(list(randoms), 26 * x.shape[1]) t = torch.Tensor(t) t = t.view(x.shape[1], 26, -1) result = torch.cat( (torch.cat((y.unsqueeze(1).double(), t.double()), dim=1), neg_samples.double()), dim=1) x, img, lengths, result = x.double().to(cuda_device), img.double( ).to(cuda_device), lengths.double().to( cuda_device), result.double().to(cuda_device) y_pred = model(x, img, lengths, result) loss = loss_function( y_pred, torch.zeros((x.shape[1])).long().to(cuda_device)) epoch_validation_loss.append(loss.item()) print('[%d] Validation loss = %.5f' % (epoch + 1, np.array(epoch_validation_loss).mean())) validation_loss[epoch] = np.array(epoch_validation_loss).mean() # print(torch.cat((y, y), dim=1).size()) plt.plot([i + 1 for i in range(num_epochs)], [t for t in train_loss], marker='o') plt.plot([i + 1 for i in range(num_epochs)], [t for t in validation_loss], marker='o') plt.title('Changes in loss function') plt.xlabel('Epoch') plt.ylabel('Cross Entropy Loss') #plt.figure() plt.savefig(result_directory + 'loss-9.jpg') plt.show() pickle.dump({ 'train': train_loss, 'validation': validation_loss }, open(result_directory + '9.pkl', 'wb'))
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) #print(genotype) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) #stat(model, (1, 16, 16)) model = model.cuda() # input_size=(1,2,128) ''' Input = torch.randn(1, 1, 2, 128) Input = Input.type(torch.cuda.FloatTensor) macs, params = profile(model, inputs=(Input,), custom_ops={Network: Network.forward}) macs, params = clever_format([macs, params], "%.3f") print(macs, params) summary(model,(1,16,16)) ''' logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) ''' train_transform, valid_transform = utils._data_transforms_cifar10(args) if args.set=='cifar100': train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform) else: train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) #train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) #valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) ''' x = loadmat("/data/wx/PC-DARTS/data/datashujudoppler=100.mat") x = x.get('train_data') x = np.reshape(x, [-1, 1, 2, 128]) data_num = 22000 y1 = np.zeros([data_num, 1]) y2 = np.ones([data_num, 1]) y3 = np.ones([data_num, 1]) * 2 y4 = np.ones([data_num, 1]) * 3 y5 = np.ones([data_num, 1]) * 4 y6 = np.ones([data_num, 1]) * 5 y7 = np.ones([data_num, 1]) * 6 y8 = np.ones([data_num, 1]) * 7 y = np.vstack((y1, y2, y3, y4, y5, y6, y7, y8)) y = np.array(y) X_train, X_val, Y_train, Y_val = train_test_split(x, y, test_size=0.3, random_state=30) X_train = torch.from_numpy(X_train) Y_train = torch.from_numpy(Y_train) X_train = X_train.type(torch.FloatTensor) Y_train = Y_train.type(torch.LongTensor) Y_train = Y_train.type(torch.LongTensor) # Y_train=np.reshape(Y_train,(16800,4)) Y_train = Y_train.squeeze() print(Y_train.type) print(Y_train) print(X_train.shape, Y_train.shape) train_Queue = torch.utils.data.TensorDataset(X_train, Y_train) print(train_Queue) X_val = torch.from_numpy(X_val) Y_val = torch.from_numpy(Y_val) X_val = X_val.type(torch.FloatTensor) Y_val = Y_val.type(torch.LongTensor) # Y_train = one_hot_embedding(Y_train, 4) Y_val = Y_val.type(torch.LongTensor) Y_val = Y_val.squeeze() print(Y_val.type, Y_val) valid_Queue = torch.utils.data.TensorDataset(X_val, Y_val) train_queue = torch.utils.data.DataLoader(train_Queue, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader(valid_Queue, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) best_acc = 0.0 writer = SummaryWriter('./logs/balei') ''' Input = torch.randn(1, 1, 2, 128) Input=Input.type(torch.cuda.FloatTensor) macs, params = profile(model, inputs=(Input,),custom_ops={Network:Network.forward}) macs, params = clever_format([macs, params], "%.3f") print(macs, params) ''' for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) writer.add_scalar('Train/acc', train_acc, epoch) TAcc.write(str(train_acc) + ",") writer.add_scalar('Train/loss', train_obj, epoch) Tloss.write(str(train_obj) + ",") valid_acc, valid_obj = infer(valid_queue, model, criterion) if valid_acc > best_acc: best_acc = valid_acc utils.save(model, os.path.join(args.save, 'weights_best_acc.pt')) logging.info('valid_acc %f, best_acc %f', valid_acc, best_acc) writer.add_scalar('Valid/acc', valid_acc, epoch) VAcc.write(str(valid_acc) + ",") writer.add_scalar('Valid/loss', valid_obj, epoch) Vloss.write(str(valid_obj) + ",") utils.save(model, os.path.join(args.save, 'weights.pt'))
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) if args.is_cifar100: model = Network(args.init_channels, CIFAR100_CLASSES, args.layers, args.auxiliary, genotype) else: model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) if args.is_cifar100: train_transform, valid_transform = utils._data_transforms_cifar100( args) else: train_transform, valid_transform = utils._data_transforms_cifar10(args) if args.is_cifar100: train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform) else: train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) start_epoch = 0 if args.resume: checkpoint = torch.load(args.resume) start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) scheduler.load_state_dict(checkpoint['scheduler']) for epoch in range(start_epoch, args.epochs): logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) scheduler.step() logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) utils.save(model, os.path.join(args.save, 'weights.pt')) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'scheduler': scheduler.state_dict(), 'optimizer': optimizer.state_dict() })
def main(): # gpu device check if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) # init np.random.seed(args.seed) # set random seed torch.cuda.set_device(args.gpu) # set training device cudnn.benchmark = True # set cudnn benchmark torch.manual_seed(args.seed) # set torch random seed cudnn.enabled = True # set cudnn torch.cuda.manual_seed(args.seed) # set torch cuda seed logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) # search space genotype = eval("genotypes.%s" % args.arch) # TODO convert to tree model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) train_transform, valid_transform = utils._data_transforms_cifar10(args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) utils.save(model, os.path.join(args.save, 'weights.pt'))
def main(): np.random.seed(args.seed) random.seed(args.seed) if torch.cuda.is_available(): device = torch.device('cuda:{}'.format(args.gpu)) cudnn.benchmark = False torch.manual_seed(args.seed) cudnn.enabled = True cudnn.deterministic = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) else: device = torch.device('cpu') logging.info('No gpu device available') torch.manual_seed(args.seed) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.to(device) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) total_params = sum(x.data.nelement() for x in model.parameters()) logging.info('Model total parameters: {}'.format(total_params)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) train_transform, valid_transform = utils._data_transforms_cifar10( args.cutout, args.cutout_length) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=0) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batch_size // 2, shuffle=False, pin_memory=True, num_workers=0) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) for epoch in range(args.epochs): logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer, args.gpu) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion, args.gpu) logging.info('valid_acc %f', valid_acc) scheduler.step() utils.save(model, os.path.join(args.save, 'weights.pt'))
text = f.read() # We need turn our data into numerical tokens # Neural networks can only learn from numerical data chars = tuple(set(text)) # obtaining all of the unique characters being used in the text chars = tuple(set(text)) with open('weights/rnn_30_epochs.net', 'rb') as f: if torch.cuda.is_available(): checkpoint = torch.load(f) else: checkpoint = torch.load(f, map_location=torch.device('cpu')) net = Network(checkpoint['tokens'], n_hidden=checkpoint['n_hidden'], n_layers=checkpoint['n_layers']) net.load_state_dict(checkpoint['state_dict']) def predict(net, char, h=None, top_k=None): x = np.array([[net.char2int[char]]]) x = one_hot_encode(x, len(net.chars)) inputs = torch.from_numpy(x) if torch.cuda.is_available(): inputs = inputs.cuda() h = tuple([each.data for each in h]) out, h = net(inputs, h) p = F.softmax(out, dim=1).data if torch.cuda.is_available():
def main(): if not torch.cuda.is_available(): logging.info('No GPU device available') sys.exit(1) np.random.seed(args.seed) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info("args = %s", args) logging.info("unparsed args = %s", unparsed) num_gpus = torch.cuda.device_count() genotype = eval("genotypes.%s" % args.arch) print('---------Genotype---------') logging.info(genotype) print('--------------------------') model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = torch.nn.DataParallel(model) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD( model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay ) if args.cifar100: train_transform, valid_transform = utils._data_transforms_cifar100(args) else: train_transform, valid_transform = utils._data_transforms_cifar10(args) if args.cifar100: train_data = dset.CIFAR100(root=args.tmp_data_dir, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR100(root=args.tmp_data_dir, train=False, download=True, transform=valid_transform) else: train_data = dset.CIFAR10(root=args.tmp_data_dir, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.tmp_data_dir, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=args.workers) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=args.workers) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs)) best_acc = 0.0 for epoch in range(args.epochs): scheduler.step() logging.info('Epoch: %d lr %e', epoch, scheduler.get_lr()[0]) model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs model.drop_path_prob = args.drop_path_prob * epoch / args.epochs start_time = time.time() train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('Train_acc: %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) if valid_acc > best_acc: best_acc = valid_acc logging.info('Valid_acc: %f', valid_acc) end_time = time.time() duration = end_time - start_time print('Epoch time: %ds.' % duration ) utils.save(model.module, os.path.join(args.save, 'weights.pt'))
def post(self): email = self.request.get('email') network_name = self.request.get('network') network_key = Network.query(Network.name == network_name).get().key AllowedUser(email=email, network=network_key).put() self.redirect('/admin/user')
class GUI(Tk): def __init__(self,name): Tk.__init__(self, None) self.title('Equilibrium Propagation') self.net = Network(name=name,hyperparameters={"batch_size":1}) self.hidden_sizes = self.net.hyperparameters["hidden_sizes"] self.n_layers = len(self.hidden_sizes) + 2 self.canvas = Canvas(self, width=600, height=(self.n_layers+1)*100) self.canvas.pack(side=BOTTOM) # INDEX OF TEST EXAMPLE (IN THE TEST SET) Label(self, text="image").pack(side=LEFT) self.index = StringVar() self.index.set("0") Entry(self, textvariable=self.index, width=5).pack(side=LEFT) self.update_canvas(first_time=True) Thread(target = self.run).start() def update_canvas(self, first_time = False): units = [(28,28)] +[(10,n/10) for n in self.hidden_sizes]+[(1,10)] pixels = [(140,140)]+ [(n/2,50) for n in self.hidden_sizes]+[(250,25)] arrays = [256*layer.eval().reshape(dimensions) for layer,dimensions in zip(self.net.layers,units)] images = [Image.fromarray(array).resize(dimensions) for array,dimensions in zip(arrays,pixels)] self.imgTks = [ImageTk.PhotoImage(image) for image in images] [energy, cost, _] = self.net.measure() if first_time: self.img_canvas = [self.canvas.create_image(400, (self.n_layers-k)*100, image=imgTk) for k,imgTk in enumerate(self.imgTks)] self.energy_canvas = self.canvas.create_text( 20, 100, anchor=W, font="Purisa", text="Energy = %.1f" % (energy)) self.cost_canvas = self.canvas.create_text( 20, 200, anchor=W, font="Purisa", text="Cost = %.4f" % (cost)) else: for img_canvas, imgTk in zip(self.img_canvas,self.imgTks): self.canvas.itemconfig(img_canvas, image=imgTk) self.canvas.itemconfig(self.energy_canvas, text="Energy = %.1f" % (energy)) self.canvas.itemconfig(self.cost_canvas, text="Cost = %.4f" % (cost)) def run(self): while True: index = self.index.get() # index of the test example in the test set if index.isdigit(): index = int(index) index = (hash(index) % 10000) + 60000 self.net.change_mini_batch_index(index) self.net.free_phase(n_iterations=1, epsilon=np.float32(.1)) self.update_canvas() # FREQUENCY OF UPDATES (IN SECONDS) time.sleep(.1)
"alphas": [np.float32(.4), np.float32(.1), np.float32(.01)] } # HYPERPARAMETERS FOR A NETWORK WITH 3 HIDDEN LAYERS net3 = "net3", { "hidden_sizes": [500, 500, 500], "n_epochs": 500, "batch_size": 20, "n_it_neg": 500, "n_it_pos": 8, "epsilon": np.float32(.5), "beta": np.float32(1.), "alphas": [np.float32(.128), np.float32(.032), np.float32(.008), np.float32(.002)] } if __name__ == "__main__": # TRAIN A NETWORK WITH 1 HIDDEN LAYER train_net(Network(*net1))
def __init__(self, rng): Network.__init__(self, n_hidden_layer = 8, BN = BN) print " C3 layer:" self.layer.append(ReLU_conv_layer( rng, filter_shape=(128, 3, 3, 3), pool_shape=(1,1), pool_stride=(1,1), BN = BN, BN_epsilon=BN_epsilon, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test )) print " C3 P2 layers:" self.layer.append(ReLU_conv_layer( rng, filter_shape=(128, 128, 3, 3), pool_shape=(2,2), pool_stride=(2,2), BN = BN, BN_epsilon=BN_epsilon, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test )) print " C2 layer:" self.layer.append(ReLU_conv_layer( rng, filter_shape=(256, 128, 2, 2), pool_shape=(1,1), pool_stride=(1,1), BN = BN, BN_epsilon=BN_epsilon, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test )) print " C2 P2 layers:" self.layer.append(ReLU_conv_layer( rng, filter_shape=(256, 256, 2, 2), pool_shape=(2,2), pool_stride=(2,2), BN = BN, BN_epsilon=BN_epsilon, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test )) print " C2 layer:" self.layer.append(ReLU_conv_layer( rng, filter_shape=(512, 256, 2, 2), pool_shape=(1,1), pool_stride=(1,1), BN = BN, BN_epsilon=BN_epsilon, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test )) print " C2 P2 layers:" self.layer.append(ReLU_conv_layer( rng, filter_shape=(512, 512, 2, 2), pool_shape=(2,2), pool_stride=(2,2), BN = BN, BN_epsilon=BN_epsilon, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test )) print " C2 layer:" self.layer.append(ReLU_conv_layer( rng, filter_shape=(1024, 512, 2, 2), pool_shape=(1,1), pool_stride=(1,1), BN = BN, BN_epsilon=BN_epsilon, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test )) print " FC layer:" self.layer.append(ReLU_layer( rng = rng, n_inputs = 1024, n_units = 1024, BN = BN, BN_epsilon=BN_epsilon, dropout=dropout_hidden, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test )) print " L2 SVM layer:" self.layer.append(linear_layer( rng = rng, n_inputs= 1024, n_units = 10, BN = BN, BN_epsilon=BN_epsilon, dropout = dropout_hidden, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test ))
from darts.links.model import NetworkCIFAR from darts.genotypes import DARTS from darts.operations import * import sys sys.path.append('../cnn') from model import NetworkCIFAR as Network from chainer.datasets import get_cifar10 c_model = NetworkCIFAR(DARTS) chainer.serializers.load_npz('model.npz', c_model) chainer.global_config.train = False t_model = Network(36, 10, 20, True, DARTS) t_model.drop_path_prob = 0.2 t_model.load_state_dict(torch.load('cifar10_model.pt')) t_model.eval() train, val = get_cifar10() img, _ = train[0] x = img[None] t_x = torch.autograd.Variable(torch.FloatTensor(x)) c_y = c_model(x) t_y = t_model(t_x) np.testing.assert_almost_equal(c_y[0].data, t_y[0].detach().data.numpy(), decimal=5) #####################
def main(): np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True cudnn.enabled = True torch.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) if not args.arch: geno_s = pathlib.Path(geno_path).read_text() else: geno_s = "genotypes.%s" % args.arch genotype = eval(geno_s) model = Network(args.init_ch, 10, args.layers, args.auxiliary, genotype).cuda() logging.info(f"seed = {args.seed}") logging.info(f"geno_s = {geno_s}") logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD( model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.wd ) train_transform, valid_transform = utils._data_transforms_cifar10(args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batchsz, shuffle=True, pin_memory=True, num_workers=0 if 'pydevd' in sys.modules else 2) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batchsz, shuffle=False, pin_memory=True, num_workers=0 if 'pydevd' in sys.modules else 2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs)) lines = [f'epoch\ttrain_acc\tval_acc'] for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc: %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc: %f', valid_acc) lines.append(f'{epoch}\t{train_acc}\t{valid_acc}') timebudget.report() utils.save(model, os.path.join(args.save, 'trained.pt')) print('saved to: trained.pt') pathlib.Path(os.path.join(args.exp_path, 'eval.tsv')).write_text('\n'.join(lines))
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) if args.parallel: # multi gpu num_gpus = torch.cuda.device_count() logging.info('num of gpu devices = %d' % num_gpus) else: # single gpu torch.cuda.set_device(args.gpu) logging.info('gpu device = %d' % args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CLASSES, args.layers, args.auxiliary, genotype) if args.parallel: model = nn.DataParallel(model).cuda() else: model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() criterion_smooth = CrossEntropyLabelSmooth(CLASSES, args.label_smooth) criterion_smooth = criterion_smooth.cuda() optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) traindir = os.path.join(args.data, 'train') validdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_data = dset.ImageFolder( traindir, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.2), transforms.ToTensor(), normalize, ])) valid_data = dset.ImageFolder( validdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])) train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=4) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=4) if args.lr_scheduler == 'step': # DARTS code scheduler = torch.optim.lr_scheduler.StepLR(optimizer, args.decay_period, gamma=args.gamma) elif args.lr_scheduler == 'cosine' or args.lr_scheduler == 'linear': # PCDARTS code scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) else: raise ValueError("Wrong learning rate scheduler") # ---- resume ---- # start_epoch = 0 best_acc_top1 = 0.0 best_acc_top5 = 0.0 best_acc_epoch = 0 if args.resume: # in multi-gpu??? if os.path.isfile(args.resume): logging.info("=> loading checkpoint {}".format(args.resume)) device = torch.device("cuda") checkpoint = torch.load(args.resume, map_location=device) start_epoch = checkpoint['epoch'] best_acc_top1 = checkpoint['best_acc_top1'] best_acc_top5 = checkpoint['best_acc_top5'] best_acc_epoch = checkpoint['best_acc_epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logging.info( "=> loaded checkpoint {} (trained until epoch {})".format( args.resume, start_epoch - 1)) else: raise ValueError("Wrong args.resume") else: logging.info("=> training from scratch") for epoch in range(start_epoch, args.epochs): scheduler.step() if args.lr_scheduler == 'cosine' or args.lr_scheduler == 'step': scheduler.step() current_lr = scheduler.get_lr()[0] elif args.lr_scheduler == 'linear': current_lr = adjust_lr(optimizer, epoch) if epoch < 5 and args.batch_size > 256: for param_group in optimizer.param_groups: param_group['lr'] = args.learning_rate * (epoch + 1) / 5.0 logging.info('Warming-up epoch: %d, LR: %e', epoch, lr * (epoch + 1) / 5.0) else: logging.info('epoch %d lr %e', epoch, current_lr) if args.parallel: model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs else: model.drop_path_prob = args.drop_path_prob * epoch / args.epochs epoch_start = time.time() train_acc, train_obj = train(train_queue, model, criterion_smooth, optimizer) logging.info('train_acc %f', train_acc) valid_acc_top1, valid_acc_top5, valid_obj = infer( valid_queue, model, criterion) is_best = (valid_acc_top1 > best_acc_top1) if is_best: best_acc_top1 = valid_acc_top1 best_acc_top5 = valid_acc_top5 best_acc_epoch = epoch + 1 utils.save(model, os.path.join(args.save, 'best_weights.pt')) logging.info('valid_acc %f %f, best_acc %f %f (at epoch %d)', valid_acc_top1, valid_acc_top5, best_acc_top1, best_acc_top5, best_acc_epoch) logging.info('epoch time %d sec.', time.time() - epoch_start) utils.save_checkpoint( { 'epoch': epoch + 1, 'best_acc_top1': best_acc_top1, 'best_acc_top5': best_acc_top5, 'best_acc_epoch': best_acc_epoch, 'state_dict': model.state_dict(), 'best_acc_top1': best_acc_top1, 'optimizer': optimizer.state_dict(), }, is_best, args.save) utils.save(model, os.path.join(args.save, 'weights.pt'))
import torch import numpy as np import os from model import Network def show_compression_rate(model): state_dict = model.state_dict() cnt_compressed = 0 cnt_original = 0 for x in state_dict: weight = state_dict[x] print(np.sum(weight.numpy() != 0)) cnt_compressed += np.sum(weight.numpy() != 0) cnt_original += weight.numel() print('After Compression: %d' % cnt_compressed) print('Before Compression: %d' % cnt_original) print('Compression Rate: %.4f' % (float(cnt_original) / cnt_compressed)) if __name__ == '__main__': model = Network() #model.load_state_dict(torch.load('checkpoints/model_layer_1.pkl')) model.load_state_dict(torch.load('model.pkl')) show_compression_rate(model)
from config import * from model import Network import numpy as np import wdbedding try: import xml.etree.cElementTree as ET except ImportError: import xml.etree.ElementTree as ET if __name__ == '__main__': # network init network = Network("training") # embedding model init word_vec_model = wdbedding.load_word2vec_model(EN) print("word2vec model_loaded.") # about the file input_file_path = '../data/task2_input_en.xml' xmltree = ET.parse(input_file_path) xmlroot = xmltree.getroot() # iterate the xml tree for review in xmlroot: txt = review.text if txt[-1] == '\n': txt = txt[:-1] print(txt) test_data = [] test_data.append(wdbedding.embedding(word_vec_model, txt, EN)) test_data = np.reshape(test_data,
def main(args): place = fluid.CUDAPlace(fluid.dygraph.parallel.Env().dev_id) \ if args.use_data_parallel else fluid.CUDAPlace(0) with fluid.dygraph.guard(place): genotype = eval("genotypes.%s" % args.arch) model = Network( C=args.init_channels, num_classes=args.class_num, layers=args.layers, auxiliary=args.auxiliary, genotype=genotype) logger.info("param size = {:.6f}MB".format( count_parameters_in_MB(model.parameters()))) device_num = fluid.dygraph.parallel.Env().nranks step_per_epoch = int(args.trainset_num / (args.batch_size * device_num)) learning_rate = fluid.dygraph.CosineDecay(args.learning_rate, step_per_epoch, args.epochs) clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=args.grad_clip) optimizer = fluid.optimizer.MomentumOptimizer( learning_rate, momentum=args.momentum, regularization=fluid.regularizer.L2Decay(args.weight_decay), parameter_list=model.parameters(), grad_clip=clip) if args.use_data_parallel: strategy = fluid.dygraph.parallel.prepare_context() model = fluid.dygraph.parallel.DataParallel(model, strategy) train_loader = fluid.io.DataLoader.from_generator( capacity=64, use_double_buffer=True, iterable=True, return_list=True, use_multiprocess=args.use_multiprocess) valid_loader = fluid.io.DataLoader.from_generator( capacity=64, use_double_buffer=True, iterable=True, return_list=True, use_multiprocess=args.use_multiprocess) train_reader = reader.train_valid( batch_size=args.batch_size, is_train=True, is_shuffle=True, args=args) valid_reader = reader.train_valid( batch_size=args.batch_size, is_train=False, is_shuffle=False, args=args) if args.use_data_parallel: train_reader = fluid.contrib.reader.distributed_batch_reader( train_reader) train_loader.set_batch_generator(train_reader, places=place) valid_loader.set_batch_generator(valid_reader, places=place) save_parameters = (not args.use_data_parallel) or ( args.use_data_parallel and fluid.dygraph.parallel.Env().local_rank == 0) best_acc = 0 for epoch in range(args.epochs): drop_path_prob = args.drop_path_prob * epoch / args.epochs logger.info('Epoch {}, lr {:.6f}'.format( epoch, optimizer.current_step_lr())) train_top1 = train(model, train_loader, optimizer, epoch, drop_path_prob, args) logger.info("Epoch {}, train_acc {:.6f}".format(epoch, train_top1)) valid_top1 = valid(model, valid_loader, epoch, args) if valid_top1 > best_acc: best_acc = valid_top1 if save_parameters: fluid.save_dygraph(model.state_dict(), args.model_save_dir + "/best_model") logger.info("Epoch {}, valid_acc {:.6f}, best_valid_acc {:.6f}". format(epoch, valid_top1, best_acc))
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) """ train_transform, valid_transform = utils._data_transforms_cifar10(args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) """ data_dir = '../data/kmnist/' data_augmentations = transforms.ToTensor() # Load the Data here train_dataset = K49(data_dir, True, data_augmentations) #test_dataset = K49(data_dir, False, data_augmentations) num_train = len(train_dataset) indices = list(range(num_train)) split = int(np.floor(args.train_portion * num_train)) train_queue = torch.utils.data.DataLoader( dataset=train_dataset, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]), pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( dataset=train_dataset, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( indices[split:num_train]), pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) utils.save(model, os.path.join(args.save, 'weights.pt'))
import torch from torchsummary import summary from model import Network device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model = Network() #print(model) model = model.to(device) #list(model.children()) summary(model, input_size=(24, 16, 16))
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) dataset = Dataset(args.dataset) train_examples = torch.from_numpy(dataset.get_train().astype('int64')) #TODO: does below need reintroducing somewhere? # device = 'cuda' # model.to(device) CLASSES = dataset.get_shape()[0] criterion = nn.CrossEntropyLoss(reduction='mean') #criterion = CrossEntropyLabelSmooth(CLASSES, args.label_smooth) criterion = criterion.cuda() regularizer = { 'N2': N2(args.reg), 'N3': N3(args.reg), }[args.regularizer] genotype = eval("genotypes.%s" % args.arch) logging.info('genotype = %s', genotype) model = Network(args.channels, CLASSES, args.layers, criterion, regularizer, genotype, args.interleaved, dataset.get_shape(), args.emb_dim, args.init) model = model.cuda() optimizer = { 'Adagrad': lambda: optim.Adagrad(model.parameters(), lr=args.learning_rate), 'Adam': lambda: optim.Adam(model.parameters(), lr=args.learning_rate, weight_decay = args.weight_decay, betas=(args.decay1, args.decay2)), 'SGD': lambda: optim.SGD(model.parameters(), lr=args.learning_rate) }[args.optimizer]() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) #optimizer = torch.optim.SGD( # model.parameters(), # args.learning_rate, #momentum=args.momentum, #weight_decay=args.weight_decay # ) train_queue = torch.utils.data.DataLoader( train_examples, batch_size=args.batch_size, shuffle = True, #sampler=torch.utils.data.sampler.RandomSampler(), pin_memory=True, num_workers=2) #TODO do we want the learning rate min here? scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs), eta_min=args.learning_rate_min) best_acc = 0 patience = 0 curve = {'valid': [], 'test': []} for epoch in range(args.epochs): scheduler.step() lr = scheduler.get_lr()[0] logging.info('epoch %d lr %e', epoch, lr) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_epoch(train_examples, train_queue, model, optimizer, regularizer, args.batch_size) if (epoch + 1) % args.report_freq == 0: valid, test = [ avg_both(*dataset.eval(model, split, -1 if split != 'train' else 50000)) for split in ['valid', 'test'] ] curve['valid'].append(valid) curve['test'].append(test) #curve['train'].append(train) #print("\t TRAIN: ", train) print("\t VALID : ", valid) print("\t TEST: ", test) is_best = False if valid['MRR'] > best_acc: best_acc = valid['MRR'] is_best = True patience = 0 else: patience +=1 utils.save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_acc_top1': best_acc, 'optimizer' : optimizer.state_dict(), }, is_best, args.save) if patience >= 5: print('early stopping...') break #utils.save(model, os.path.join(args.save, 'weights.pt')) results = dataset.eval(model, 'test', -1) print("\n\nTEST : ", results) with open(os.path.join(args.save, 'curve.pkl'), 'wb') as f: pickle.dump(curve, f, pickle.HIGHEST_PROTOCOL)
def main(): os.environ["CUDA_VISIBLE_DEVICES"] = args.gpus if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %s' % args.gpus) logging.info("args = %s", args) num_gpus = torch.cuda.device_count() genotype = eval("genotypes.%s" % args.arch) print('---------Genotype---------') logging.info(genotype) print('--------------------------') model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) if num_gpus > 1: model = nn.DataParallel(model) model = model.cuda() else: model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) train_transform, valid_transform = utils._data_transforms_cifar10(args) if args.set == 'cifar100': train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform) else: train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) #train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) #valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=args.workers) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=args.workers) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) best_acc = 0.0 for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) if num_gpus > 1: model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs else: model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) if valid_acc > best_acc: best_acc = valid_acc logging.info('valid_acc %f, best_acc %f', valid_acc, best_acc) utils.save(model, os.path.join(args.save, 'weights.pt'))
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() total = 0 for m in model.modules(): if isinstance(m, nn.BatchNorm2d): total += m.weight.data.shape[0] bn = torch.zeros(total) index = 0 for m in model.modules(): if isinstance(m, nn.BatchNorm2d): size = m.weight.data.shape[0] bn[index:(index + size)] = m.weight.data.abs().clone() index += size y, i = torch.sort(bn) thre_index = int(total * 0.5) thre = y[thre_index] pruned = 0 cfg = [] cfg_mask = [] for k, m in enumerate(model.modules()): if isinstance(m, nn.BatchNorm2d): weight_copy = m.weight.data.clone() mask = weight_copy.abs().gt(thre).float().cuda() pruned = pruned + mask.shape[0] - torch.sum(mask) m.weight.data.mul_(mask) m.bias.data.mul_(mask) cfg.append(int(torch.sum(mask))) cfg_mask.append(mask.clone()) print( 'layer index: {:d} \t total channel: {:d} \t remaining channel: {:d}' .format(k, mask.shape[0], int(torch.sum(mask)))) elif isinstance(m, nn.MaxPool2d): cfg.append('M') pruned_ratio = pruned / total print('Pre-processing Successful!') utils.load(model, args.model_path) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() snr = [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20] sum = 0 TIM = 0 utils.load(model, args.model_path) for i in snr: data_num = 1000 test_data = loadmat("/data/wx/PC-DARTS/data/testdoppler=100/snr=" + str(i) + ".mat") x = test_data.get("train_data") print(x.shape) x = np.reshape(x, [-1, 1, 2, 128]) y1 = np.zeros([data_num, 1]) y2 = np.ones([data_num, 1]) y3 = np.ones([data_num, 1]) * 2 y4 = np.ones([data_num, 1]) * 3 y5 = np.ones([data_num, 1]) * 4 y6 = np.ones([data_num, 1]) * 5 y7 = np.ones([data_num, 1]) * 6 y8 = np.ones([data_num, 1]) * 7 y = np.vstack((y1, y2, y3, y4, y5, y6, y7, y8)) y = np.array(y) X_test = torch.from_numpy(x) Y_test = torch.from_numpy(y) X_test = X_test.type(torch.FloatTensor) Y_test = Y_test.type(torch.LongTensor) Y_test = Y_test.squeeze() test_Queue = torch.utils.data.TensorDataset(X_test, Y_test) test_queue = torch.utils.data.DataLoader(test_Queue, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) model.drop_path_prob = args.drop_path_prob if i == 0: Input = torch.randn(1, 1, 2, 128) Input = Input.type(torch.cuda.FloatTensor) macs, params = profile(model, inputs=(Input, )) macs, params = clever_format([macs, params], "%.3f") print("flops params") print(macs, params) summary(model, input_size=(1, 2, 128)) time1 = time.time() test_acc, test_obj, target, loggg = infer(test_queue, model, criterion) time2 = time.time() - time1 logging.info('第 %d snr test_acc %f', i, test_acc) sum += test_acc logging.info("第 %d snr time: %f", i, time2) TIM += time2 #print(target) #print(target.shape) #print(loggg.shape) target = target.cpu().detach().numpy() loggg = loggg.cpu().detach().numpy() cm = confusion_matrix(target, loggg) #print(cm) #plot_confusion_matrix(cm ,mods, title=" Confusion Matrix ( SNR=%d dB)" % (i)) ''' if i>=10: address_jpeg = '/data/wx/PC-DARTS/wx/picture/' + '100' + 'hz-CSS-snr=' + str(i) + '.pdf' plt.savefig(address_jpeg) plt.close('all') ''' #plt.show() ACC = sum / 11 TT = TIM / 11 print("average acc : ", ACC) print("average time : ", TT)
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) if is_wandb_used: wandb.init( project="automl-gradient-based-nas", name="ImageNet:" + str(args.arch), config=args, entity="automl" ) np.random.seed(args.seed) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CLASSES, args.layers, args.auxiliary, genotype) if args.parallel: model = nn.DataParallel(model).cuda() else: model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() criterion_smooth = CrossEntropyLabelSmooth(CLASSES, args.label_smooth) criterion_smooth = criterion_smooth.cuda() optimizer = torch.optim.SGD( model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay ) traindir = os.path.join(args.data, 'train') validdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_data = dset.ImageFolder( traindir, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ColorJitter( brightness=0.4, contrast=0.4, saturation=0.4, hue=0.2), transforms.ToTensor(), normalize, ])) valid_data = dset.ImageFolder( validdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=4) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=4) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, args.decay_period, gamma=args.gamma) best_acc_top1 = 0 for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion_smooth, optimizer) logging.info('train_acc %f', train_acc) valid_acc_top1, valid_acc_top5, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc_top1 %f', valid_acc_top1) logging.info('valid_acc_top5 %f', valid_acc_top5) is_best = False if valid_acc_top1 > best_acc_top1: best_acc_top1 = valid_acc_top1 is_best = True utils.save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_acc_top1': best_acc_top1, 'optimizer': optimizer.state_dict(), }, is_best, args.save)
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) in_channels, num_classes, dataset_in_torch, stride_for_aux = utils.dataset_fields( args, train=False) # new genotype = eval("genotypes.%s" % args.arch) #eval("genotypes.%s" % args.arch) model = Network(args.init_channels, in_channels, stride_for_aux, num_classes, args.layers, args.auxiliary, genotype, args.SE) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) train_data, valid_data = utils.dataset_split_and_transform( dataset_in_torch, args, train=False) # new train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) utils.save(model, os.path.join(args.save, 'weights.pt'))
transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ])) val_sampler = torch.utils.data.distributed.DistributedSampler( val_dataset, num_replicas=hvd.size(), rank=hvd.rank()) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.val_batch_size, sampler=val_sampler, **kwargs) # Set up standard ResNet-50 model. genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CLASSES, args.layers, args.auxiliary, genotype) # model = models.resnet50() if args.cuda: # Move model to GPU. model.cuda() #model load # model_path = "./model_imagenet_20190112/checkpoint-60.pth" # model.load_state_dict(torch.load(model_path)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() criterion_smooth = CrossEntropyLabelSmooth(CLASSES, args.label_smooth) criterion_smooth = criterion_smooth.cuda()
def __init__(self, rng): Network.__init__(self, n_hidden_layer=8, BN=BN) print " C3 layer:" self.layer.append( ReLU_conv_layer(rng, filter_shape=(128, 3, 3, 3), pool_shape=(1, 1), pool_stride=(1, 1), BN=BN, BN_epsilon=BN_epsilon, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test)) print " C3 P2 layers:" self.layer.append( ReLU_conv_layer(rng, filter_shape=(128, 128, 3, 3), pool_shape=(2, 2), pool_stride=(2, 2), BN=BN, BN_epsilon=BN_epsilon, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test)) print " C2 layer:" self.layer.append( ReLU_conv_layer(rng, filter_shape=(256, 128, 2, 2), pool_shape=(1, 1), pool_stride=(1, 1), BN=BN, BN_epsilon=BN_epsilon, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test)) print " C2 P2 layers:" self.layer.append( ReLU_conv_layer(rng, filter_shape=(256, 256, 2, 2), pool_shape=(2, 2), pool_stride=(2, 2), BN=BN, BN_epsilon=BN_epsilon, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test)) print " C2 layer:" self.layer.append( ReLU_conv_layer(rng, filter_shape=(512, 256, 2, 2), pool_shape=(1, 1), pool_stride=(1, 1), BN=BN, BN_epsilon=BN_epsilon, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test)) print " C2 P2 layers:" self.layer.append( ReLU_conv_layer(rng, filter_shape=(512, 512, 2, 2), pool_shape=(2, 2), pool_stride=(2, 2), BN=BN, BN_epsilon=BN_epsilon, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test)) print " C2 layer:" self.layer.append( ReLU_conv_layer(rng, filter_shape=(1024, 512, 2, 2), pool_shape=(1, 1), pool_stride=(1, 1), BN=BN, BN_epsilon=BN_epsilon, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test)) print " FC layer:" self.layer.append( ReLU_layer(rng=rng, n_inputs=1024, n_units=1024, BN=BN, BN_epsilon=BN_epsilon, dropout=dropout_hidden, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test)) print " L2 SVM layer:" self.layer.append( linear_layer(rng=rng, n_inputs=1024, n_units=10, BN=BN, BN_epsilon=BN_epsilon, dropout=dropout_hidden, binary_training=binary_training, stochastic_training=stochastic_training, binary_test=binary_test, stochastic_test=stochastic_test))
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) random.seed(args.seed) np.random.seed( args.data_seed) # cutout and load_corrupted_data use np.random torch.cuda.set_device(args.gpu) cudnn.benchmark = False torch.manual_seed(args.seed) cudnn.enabled = True cudnn.deterministic = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) if args.arch == 'resnet': model = ResNet18(CIFAR_CLASSES).cuda() args.auxiliary = False elif args.arch == 'resnet50': model = ResNet50(CIFAR_CLASSES).cuda() args.auxiliary = False elif args.arch == 'resnet34': model = ResNet34(CIFAR_CLASSES).cuda() args.auxiliary = False else: genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) train_transform, test_transform = utils._data_transforms_cifar10(args) # Load dataset if args.dataset == 'cifar10': noisy_train_data = CIFAR10(root=args.data, train=True, gold=False, gold_fraction=0.0, corruption_prob=args.corruption_prob, corruption_type=args.corruption_type, transform=train_transform, download=True, seed=args.data_seed) gold_train_data = CIFAR10(root=args.data, train=True, gold=True, gold_fraction=1.0, corruption_prob=args.corruption_prob, corruption_type=args.corruption_type, transform=train_transform, download=True, seed=args.data_seed) test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) elif args.dataset == 'cifar100': noisy_train_data = CIFAR100(root=args.data, train=True, gold=False, gold_fraction=0.0, corruption_prob=args.corruption_prob, corruption_type=args.corruption_type, transform=train_transform, download=True, seed=args.data_seed) gold_train_data = CIFAR100(root=args.data, train=True, gold=True, gold_fraction=1.0, corruption_prob=args.corruption_prob, corruption_type=args.corruption_type, transform=train_transform, download=True, seed=args.data_seed) test_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=test_transform) num_train = len(gold_train_data) indices = list(range(num_train)) split = int(np.floor(args.train_portion * num_train)) if args.gold_fraction == 1.0: train_data = gold_train_data else: train_data = noisy_train_data train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]), pin_memory=True, num_workers=0) if args.clean_valid: valid_data = gold_train_data else: valid_data = noisy_train_data valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:]), pin_memory=True, num_workers=0) test_queue = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) if args.loss_func == 'cce': criterion = nn.CrossEntropyLoss().cuda() elif args.loss_func == 'rll': criterion = utils.RobustLogLoss(alpha=args.alpha).cuda() elif args.loss_func == 'forward_gold': corruption_matrix = train_data.corruption_matrix criterion = utils.ForwardGoldLoss(corruption_matrix=corruption_matrix) else: assert False, "Invalid loss function '{}' given. Must be in {'cce', 'rll'}".format( args.loss_func) for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer_valid(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) test_acc, test_obj = infer(test_queue, model, criterion) logging.info('test_acc %f', test_acc) utils.save(model, os.path.join(args.save, 'weights.pt'))