def main(args):
print("Loading data")
train_dataset = datasets.ImageFolder(args.data + "/train",
transform=train_transforms)
val_dataset = datasets.ImageFolder(args.data + "/val",
transform=val_transforms)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=1)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=1)
print("Data loaded")
model = Network(args)
if args.cuda:
model = model.cuda()
optimizer = optim.Adam(model.parameters(),
lr=args.lr)
print("Starting training")
train(args, model, optimizer, train_loader, val_loader)
def get(self):
allowed_users = AllowedUser.query().order(AllowedUser.email)
networks = Network.query().order(Network.name)
template_values = {
'users': allowed_users,
'networks': networks
}
path = 'admin.html'
self.response.out.write(template.render(path, template_values))
def parse_network(environment, data):
if data.has_key('name'):
name = data['name']
elif data.has_key('network'):
name = data['network']
else:
raise ConfigError, "Unnamed network"
network = Network(name)
if data.has_key('dhcp_server'):
network.dhcp_server = data['dhcp_server']
for existing_network in environment.networks:
if existing_network.name == network.name:
raise ConfigError, "Network with given name already exists: %s" % network.name
environment.networks.append(network)
return network
def __init__(self, rng):
Network.__init__(self, n_hidden_layer = n_hidden_layer, BN = BN)
print " Fully connected layer 1:"
self.layer.append(ReLU_layer(rng = rng, n_inputs = n_inputs, n_units = n_units,
BN = BN, BN_epsilon=BN_epsilon, dropout=dropout_input,
binary_training=binary_training, stochastic_training=stochastic_training,
binary_test=binary_test, stochastic_test=stochastic_test))
for k in range(n_hidden_layer-1):
print " Fully connected layer "+ str(k) +":"
self.layer.append(ReLU_layer(rng = rng, n_inputs = n_units, n_units = n_units,
BN = BN, BN_epsilon=BN_epsilon, dropout=dropout_hidden,
binary_training=binary_training, stochastic_training=stochastic_training,
binary_test=binary_test, stochastic_test=stochastic_test))
print " L2 SVM layer:"
self.layer.append(linear_layer(rng = rng, n_inputs = n_units, n_units = n_classes,
BN = BN, BN_epsilon=BN_epsilon, dropout=dropout_hidden,
binary_training=binary_training, stochastic_training=stochastic_training,
binary_test=binary_test, stochastic_test=stochastic_test))
def __init__(self,name):
Tk.__init__(self, None)
self.title('Equilibrium Propagation')
self.net = Network(name=name,hyperparameters={"batch_size":1})
self.hidden_sizes = self.net.hyperparameters["hidden_sizes"]
self.n_layers = len(self.hidden_sizes) + 2
self.canvas = Canvas(self, width=600, height=(self.n_layers+1)*100)
self.canvas.pack(side=BOTTOM)
# INDEX OF TEST EXAMPLE (IN THE TEST SET)
Label(self, text="image").pack(side=LEFT)
self.index = StringVar()
self.index.set("0")
Entry(self, textvariable=self.index, width=5).pack(side=LEFT)
self.update_canvas(first_time=True)
Thread(target = self.run).start()
def main():
if not torch.cuda.is_available():
logging.info('No GPU device available')
sys.exit(1)
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info("args = %s", args)
logging.info("unparsed_args = %s", unparsed)
num_gpus = torch.cuda.device_count()
genotype = eval("genotypes.%s" % args.arch)
print('---------Genotype---------')
logging.info(genotype)
print('--------------------------')
model = Network(args.init_channels, CLASSES, args.layers, args.auxiliary,
genotype)
if num_gpus > 1:
model = nn.DataParallel(model)
model = model.cuda()
else:
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
criterion_smooth = CrossEntropyLabelSmooth(CLASSES, args.label_smooth)
criterion_smooth = criterion_smooth.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
data_dir = os.path.join(args.tmp_data_dir, 'imagenet')
traindir = os.path.join(data_dir, 'train')
validdir = os.path.join(data_dir, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_data = dset.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ColorJitter(brightness=0.4,
contrast=0.4,
saturation=0.4,
hue=0.2),
transforms.ToTensor(),
normalize,
]))
valid_data = dset.ImageFolder(
validdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
]))
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=args.workers)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=args.workers)
# scheduler = torch.optim.lr_scheduler.StepLR(optimizer, args.decay_period, gamma=args.gamma)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
best_acc_top1 = 0
best_acc_top5 = 0
lr = args.learning_rate
for epoch in range(args.epochs):
if args.lr_scheduler == 'cosine':
scheduler.step()
current_lr = scheduler.get_lr()[0]
elif args.lr_scheduler == 'linear':
current_lr = adjust_lr(optimizer, epoch)
else:
print('Wrong lr type, exit')
sys.exit(1)
logging.info('Epoch: %d lr %e', epoch, current_lr)
if epoch < 5 and args.batch_size > 256:
for param_group in optimizer.param_groups:
param_group['lr'] = lr * (epoch + 1) / 5.0
logging.info('Warming-up Epoch: %d, LR: %e', epoch,
lr * (epoch + 1) / 5.0)
if num_gpus > 1:
model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs
else:
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
epoch_start = time.time()
train_acc, train_obj = train(train_queue, model, criterion_smooth,
optimizer)
logging.info('Train_acc: %f', train_acc)
valid_acc_top1, valid_acc_top5, valid_obj = infer(
valid_queue, model, criterion)
logging.info('Valid_acc_top1: %f', valid_acc_top1)
logging.info('Valid_acc_top5: %f', valid_acc_top5)
epoch_duration = time.time() - epoch_start
logging.info('Epoch time: %ds.', epoch_duration)
is_best = False
if valid_acc_top5 > best_acc_top5:
best_acc_top5 = valid_acc_top5
if valid_acc_top1 > best_acc_top1:
best_acc_top1 = valid_acc_top1
is_best = True
utils.save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_acc_top1': best_acc_top1,
'optimizer': optimizer.state_dict(),
}, is_best, args.save)
from scipy.stats import spearmanr
from matplotlib.colors import colorConverter
from model import Network
from dataset import Ising
from utils import evaluate, format_mse
MODEL_PATH = "./cfg/net.pt"
DATA_PATH = "../Ising/data/"
CUDA = torch.cuda.is_available()
BATCH_SIZE = 1
test_set = Ising(DATA_PATH, train=False)
test_loader = DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=True)
net = Network()
net.load_state_dict(torch.load(MODEL_PATH))
if CUDA:
net.to(torch.device('cuda'))
net.eval()
losses, targets, predictions = evaluate(net, test_loader, CUDA=CUDA)
r = spearmanr(targets, predictions).correlation
loss_color = colorConverter.to_rgba('mediumseagreen', alpha=.5)
markerOptions = dict(markerfacecolor='none', markeredgewidth=1.5)
fig, ax = plt.subplots(figsize=(10, 6))
num_points=args.num_points,
data_dir=args.data),
num_workers=8,
batch_size=args.batch_size,
shuffle=False,
drop_last=False)
n_classes = 40
logging.info('n_classes: %d', n_classes)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels,
n_classes,
args.num_cells,
args.auxiliary,
genotype,
in_channels=args.in_channels,
emb_dims=args.emb_dims,
dropout=args.dropout,
k=args.k)
model = model.cuda()
# model = nn.DataParallel(model)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = torch.nn.CrossEntropyLoss().cuda()
if args.use_sgd:
print("Use SGD")
optimizer = torch.optim.SGD(model.parameters(),
lr=args.learning_rate * 100,
momentum=args.momentum,
def __init__(self):
self.net = Network()
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
self.restore()
def compute(self, x, budget, config, **kwargs):
"""
Get model with hyperparameters from config generated by get_configspace()
"""
config = get_config_dictionary(x, config)
print("config", config)
if (len(config.keys()) < len(x)):
return 100
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
gpu = 'cuda:0'
np.random.seed(self.seed)
torch.cuda.set_device(gpu)
cudnn.benchmark = True
torch.manual_seed(self.seed)
cudnn.enabled = True
torch.cuda.manual_seed(self.seed)
logging.info('gpu device = %s' % gpu)
logging.info("config = %s", config)
genotype = eval("genotypes.%s" % 'PCDARTS')
model = Network(self.init_channels, self.n_classes,
config['n_conv_layers'], genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
if config['optimizer'] == 'sgd':
optimizer = torch.optim.SGD(model.parameters(),
lr=config['initial_lr'],
momentum=0.9,
weight_decay=config['weight_decay'],
nesterov=True)
else:
optimizer = get('opti_dict')[config['optimizer']](
model.parameters(),
lr=config['initial_lr'],
weight_decay=config['weight_decay'])
if config['lr_scheduler'] == 'Cosine':
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, int(budget))
elif config['lr_scheduler'] == 'Exponential':
lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer,
gamma=0.1)
indices = list(range(int(self.split * len(self.train_dataset))))
valid_indices = list(
range(int(self.split * len(self.train_dataset)),
len(self.train_dataset)))
print("Training size=", len(indices))
training_sampler = SubsetRandomSampler(indices)
valid_sampler = SubsetRandomSampler(valid_indices)
train_queue = torch.utils.data.DataLoader(dataset=self.train_dataset,
batch_size=self.batch_size,
sampler=training_sampler)
valid_queue = torch.utils.data.DataLoader(dataset=self.train_dataset,
batch_size=self.batch_size,
sampler=valid_sampler)
for epoch in range(int(budget)):
lr_scheduler.step()
logging.info('epoch %d lr %e', epoch, lr_scheduler.get_lr()[0])
model.drop_path_prob = config['drop_path_prob'] * epoch / int(
budget)
train_acc, train_obj = train(train_queue,
model,
criterion,
optimizer,
grad_clip=config['grad_clip_value'])
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
return valid_obj # Hyperband always minimizes, so we want to minimise the error, error = 1-acc
def __init__(self,
session,
action_dim,
state_dim,
batch_size=64,
nsteps=1,
trace_length=10,
hidden_size=10,
initial_epsilon=0.5,
final_epsilon=0.01,
replay_size=50000,
gamma=0.9,
replace_target_freq=10,
doubleDQN=False,
duelingDQN=False,
DRQN=False,
store_ckpt=True,
ckpt_interval=100,
ckpt_dir='./model'):
"""
Init agent.
:param session: tensorflow session.
:param action_dim: action dimension.
:param state_dim: state dimension.
:param batch_size: batch size.
:param nsteps: nsteps λ
:param trace_length: LSTM trace length.
:param hidden_size: Fully connected layer to LSTM layer size.
:param initial_epsilon: decay of the epsilon for random selection.
:param final_epsilon: decay of the epsilon for random selection.
:param replay_size: replay buffer size.
:param gamma: decay rate.
:param replace_target_freq: target net soft update interval.
:param doubleDQN: use double dqn or not.
:param duelingDQN: use dueling dqn or not.
:param store_ckpt: store checkpoint or not.
:param ckpt_interval: the interval for storing the checkpoint.
:param ckpt_dir: the directory of the checkpoint files.
"""
self.session = session
self.INITIAL_EPSILON = initial_epsilon
self.FINAL_EPSILON = final_epsilon
self.REPLAY_SIZE = replay_size
self.DOUBLE_DQN = doubleDQN
self.DUELING_DQN = duelingDQN
self.DRQN = DRQN
self.GAMMA = gamma
self.REPLACE_TARGET_FREQ = replace_target_freq
self.BATCH_SIZE = batch_size
self.hidden_size = hidden_size
self.trace_length = trace_length
self.action_dim = action_dim
self.state_dim = state_dim
self.nsteps = nsteps
self.nstep_buffer = []
# init experience replay
self.replay_buffer = deque()
# init some parameters
self.time_step = 0
self.epsilon = self.INITIAL_EPSILON
self.store_ckpt = store_ckpt
self.ckpt_dir = ckpt_dir
self.ckpt_interval = ckpt_interval
self.main_net = Network(self.state_dim, self.action_dim,
self.DUELING_DQN, self.DRQN, hidden_size,
MAIN_SCOPE)
self.target_net = Network(self.state_dim, self.action_dim,
self.DUELING_DQN, self.DRQN, hidden_size,
TARGET_SCOPE)
self.global_step = tf.Variable(0, name="global_step", trainable=False)
self.state_in = (np.zeros([1, self.hidden_size]),
np.zeros([1, self.hidden_size]))
# saver after defining variables
self.saver = tf.train.Saver()
self.create_training_method()
self.soft_update()
self.init_checkpoint()
self.session.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
])
# self.session = tf_debug.LocalCLIDebugWrapperSession(self.session)
writer = tf.summary.FileWriter("./model/", self.session.graph)
writer.close()
pass
def main():
partition = pickle.load(open(data_directory + glove6b_data_file, 'rb'))
# print(np.array(partition['train_data']).shape, np.array(partition['train_label']).shape, np.array(partition['validation_data']).shape, np.array(partition['validation_label']).shape)
# print(partition['train_label'][4], partition['train_label'][1], partition['train_label'][0], partition['train_label'][3], np.array(partition['train_data']).shape)
# print(np.array(partition['train_data'])[5].shape)
train_params = {
'batch_size': batch_size,
'shuffle': False,
'num_workers': 4
}
validation_params = {
'batch_size': validation_size,
'shuffle': False,
'num_workers': 4
}
partition['train_data'], train_lengths = preprocess_data(
partition['train_data'])
partition['validation_data'], validation_lengths = preprocess_data(
partition['validation_data'])
print(partition['train_data'].size(), len(train_lengths),
train_lengths[-1])
training_set = Dataset(partition['train_data'], partition['train_label'],
train_lengths)
training_generator = data.DataLoader(training_set, **train_params)
validation_set = Dataset(partition['validation_data'],
partition['validation_label'], validation_lengths)
validation_generator = data.DataLoader(validation_set, **validation_params)
randoms = pickle.load(
open(glove_directory + glove6b_random_data_file, 'rb'))
print(randoms.shape)
loss_function = F.cross_entropy
# loss_function = F.cross_entropy
model = Network().double().to(cuda_device)
optim = torch.optim.Adam(model.parameters(), weight_decay=0.005)
train_acc = np.zeros(num_epochs)
train_loss = np.zeros(num_epochs)
validation_loss = np.zeros(num_epochs)
for epoch in range(num_epochs):
i = 0
epoch_loss = []
total_test_loss = 0.0
model.zero_grad()
for x, y, img, lengths in training_generator:
# print('image[0] in train:', img[0:6])
# print('INPUT[0] in train:', x[0])
# print(x.shape, y.shape, img.shape, lengths.shape)
# print(x[list(lengths).index(max(list(lengths)))])
neg_samples = torch.zeros((x.shape[0], neg_sample_count, 300))
neg_samples_idx = [
random.sample(range(lengths[i]), neg_sample_count)
for i in range(x.shape[0])
]
for k, list_index in enumerate(neg_samples_idx):
for l, index in enumerate(list_index):
neg_samples[k][l] = x[k][index]
# print(neg_samples.size()) # 32 * 5 * 300
x = x.transpose(0, 1)
t = random.sample(list(randoms), 26 * x.shape[1])
t = torch.Tensor(t)
t = t.view(x.shape[1], 26, -1)
# print(t.size()) # 32 * 26 * 300
# print(y.shape) # 32 * 1 * 300
# 26 * 1 * 300
# 5 * 1 * 300
# 32 * 1 * 300
# out: 32 * 32 * 300
result = torch.cat(
(torch.cat((y.unsqueeze(1).double(), t.double()),
dim=1), neg_samples.double()),
dim=1)
# print(result.size()) # 32 * 32 * 300
# print(torch.eq(y[0].float(), result[0][0])) # check
# x = torch.stack(random.sample(range(lengths[0]), 10))
# print(x.size())
x, img, lengths, result = x.double().to(cuda_device), img.double(
).to(cuda_device), lengths.double().to(
cuda_device), result.double().to(cuda_device)
y_pred = model(x, img, lengths, result)
# print(y.shape, y_pred.shape)
# print('y_pred:', y_pred, y_pred.size(), y_pred[0].sum())
loss = loss_function(
y_pred,
torch.zeros((x.shape[1])).long().to(cuda_device))
epoch_loss.append(loss.item())
optim.zero_grad()
loss.backward()
optim.step()
# print('grad:', model.linear2.weight.grad)
print('[%d, %d] Train loss = %.5f' %
(epoch + 1, i + batch_size, loss.item()))
i += batch_size
train_loss[epoch] = np.array(epoch_loss).mean()
print('[%d] Train loss = %.5f' %
(epoch + 1, np.array(epoch_loss).mean()))
epoch_validation_loss = []
for x, y, img, lengths in validation_generator:
neg_samples = torch.zeros((x.shape[0], neg_sample_count, 300))
neg_samples_idx = [
random.sample(range(lengths[i]), neg_sample_count)
for i in range(x.shape[0])
]
for k, list_index in enumerate(neg_samples_idx):
for l, index in enumerate(list_index):
neg_samples[k][l] = x[k][index]
x = x.transpose(0, 1)
t = random.sample(list(randoms), 26 * x.shape[1])
t = torch.Tensor(t)
t = t.view(x.shape[1], 26, -1)
result = torch.cat(
(torch.cat((y.unsqueeze(1).double(), t.double()),
dim=1), neg_samples.double()),
dim=1)
x, img, lengths, result = x.double().to(cuda_device), img.double(
).to(cuda_device), lengths.double().to(
cuda_device), result.double().to(cuda_device)
y_pred = model(x, img, lengths, result)
loss = loss_function(
y_pred,
torch.zeros((x.shape[1])).long().to(cuda_device))
epoch_validation_loss.append(loss.item())
print('[%d] Validation loss = %.5f' %
(epoch + 1, np.array(epoch_validation_loss).mean()))
validation_loss[epoch] = np.array(epoch_validation_loss).mean()
# print(torch.cat((y, y), dim=1).size())
plt.plot([i + 1 for i in range(num_epochs)], [t for t in train_loss],
marker='o')
plt.plot([i + 1 for i in range(num_epochs)], [t for t in validation_loss],
marker='o')
plt.title('Changes in loss function')
plt.xlabel('Epoch')
plt.ylabel('Cross Entropy Loss')
#plt.figure()
plt.savefig(result_directory + 'loss-9.jpg')
plt.show()
pickle.dump({
'train': train_loss,
'validation': validation_loss
}, open(result_directory + '9.pkl', 'wb'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
#print(genotype)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
#stat(model, (1, 16, 16))
model = model.cuda()
# input_size=(1,2,128)
'''
Input = torch.randn(1, 1, 2, 128)
Input = Input.type(torch.cuda.FloatTensor)
macs, params = profile(model, inputs=(Input,), custom_ops={Network: Network.forward})
macs, params = clever_format([macs, params], "%.3f")
print(macs, params)
summary(model,(1,16,16))
'''
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
'''
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.set=='cifar100':
train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform)
else:
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
#train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
#valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
'''
x = loadmat("/data/wx/PC-DARTS/data/datashujudoppler=100.mat")
x = x.get('train_data')
x = np.reshape(x, [-1, 1, 2, 128])
data_num = 22000
y1 = np.zeros([data_num, 1])
y2 = np.ones([data_num, 1])
y3 = np.ones([data_num, 1]) * 2
y4 = np.ones([data_num, 1]) * 3
y5 = np.ones([data_num, 1]) * 4
y6 = np.ones([data_num, 1]) * 5
y7 = np.ones([data_num, 1]) * 6
y8 = np.ones([data_num, 1]) * 7
y = np.vstack((y1, y2, y3, y4, y5, y6, y7, y8))
y = np.array(y)
X_train, X_val, Y_train, Y_val = train_test_split(x,
y,
test_size=0.3,
random_state=30)
X_train = torch.from_numpy(X_train)
Y_train = torch.from_numpy(Y_train)
X_train = X_train.type(torch.FloatTensor)
Y_train = Y_train.type(torch.LongTensor)
Y_train = Y_train.type(torch.LongTensor)
# Y_train=np.reshape(Y_train,(16800,4))
Y_train = Y_train.squeeze()
print(Y_train.type)
print(Y_train)
print(X_train.shape, Y_train.shape)
train_Queue = torch.utils.data.TensorDataset(X_train, Y_train)
print(train_Queue)
X_val = torch.from_numpy(X_val)
Y_val = torch.from_numpy(Y_val)
X_val = X_val.type(torch.FloatTensor)
Y_val = Y_val.type(torch.LongTensor)
# Y_train = one_hot_embedding(Y_train, 4)
Y_val = Y_val.type(torch.LongTensor)
Y_val = Y_val.squeeze()
print(Y_val.type, Y_val)
valid_Queue = torch.utils.data.TensorDataset(X_val, Y_val)
train_queue = torch.utils.data.DataLoader(train_Queue,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_Queue,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
best_acc = 0.0
writer = SummaryWriter('./logs/balei')
'''
Input = torch.randn(1, 1, 2, 128)
Input=Input.type(torch.cuda.FloatTensor)
macs, params = profile(model, inputs=(Input,),custom_ops={Network:Network.forward})
macs, params = clever_format([macs, params], "%.3f")
print(macs, params)
'''
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
writer.add_scalar('Train/acc', train_acc, epoch)
TAcc.write(str(train_acc) + ",")
writer.add_scalar('Train/loss', train_obj, epoch)
Tloss.write(str(train_obj) + ",")
valid_acc, valid_obj = infer(valid_queue, model, criterion)
if valid_acc > best_acc:
best_acc = valid_acc
utils.save(model, os.path.join(args.save, 'weights_best_acc.pt'))
logging.info('valid_acc %f, best_acc %f', valid_acc, best_acc)
writer.add_scalar('Valid/acc', valid_acc, epoch)
VAcc.write(str(valid_acc) + ",")
writer.add_scalar('Valid/loss', valid_obj, epoch)
Vloss.write(str(valid_obj) + ",")
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
if args.is_cifar100:
model = Network(args.init_channels, CIFAR100_CLASSES, args.layers,
args.auxiliary, genotype)
else:
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.is_cifar100:
train_transform, valid_transform = utils._data_transforms_cifar100(
args)
else:
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.is_cifar100:
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR100(root=args.data,
train=False,
download=True,
transform=valid_transform)
else:
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
start_epoch = 0
if args.resume:
checkpoint = torch.load(args.resume)
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
scheduler.load_state_dict(checkpoint['scheduler'])
for epoch in range(start_epoch, args.epochs):
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
scheduler.step()
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'scheduler': scheduler.state_dict(),
'optimizer': optimizer.state_dict()
})
def main():
# gpu device check
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
# init
np.random.seed(args.seed) # set random seed
torch.cuda.set_device(args.gpu) # set training device
cudnn.benchmark = True # set cudnn benchmark
torch.manual_seed(args.seed) # set torch random seed
cudnn.enabled = True # set cudnn
torch.cuda.manual_seed(args.seed) # set torch cuda seed
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
# search space
genotype = eval("genotypes.%s" % args.arch) # TODO convert to tree
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
np.random.seed(args.seed)
random.seed(args.seed)
if torch.cuda.is_available():
device = torch.device('cuda:{}'.format(args.gpu))
cudnn.benchmark = False
torch.manual_seed(args.seed)
cudnn.enabled = True
cudnn.deterministic = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
else:
device = torch.device('cpu')
logging.info('No gpu device available')
torch.manual_seed(args.seed)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.to(device)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
total_params = sum(x.data.nelement() for x in model.parameters())
logging.info('Model total parameters: {}'.format(total_params))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(
args.cutout, args.cutout_length)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=0)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size // 2,
shuffle=False,
pin_memory=True,
num_workers=0)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
for epoch in range(args.epochs):
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer,
args.gpu)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion, args.gpu)
logging.info('valid_acc %f', valid_acc)
scheduler.step()
utils.save(model, os.path.join(args.save, 'weights.pt'))
text = f.read()
# We need turn our data into numerical tokens
# Neural networks can only learn from numerical data
chars = tuple(set(text))
# obtaining all of the unique characters being used in the text
chars = tuple(set(text))
with open('weights/rnn_30_epochs.net', 'rb') as f:
if torch.cuda.is_available():
checkpoint = torch.load(f)
else:
checkpoint = torch.load(f, map_location=torch.device('cpu'))
net = Network(checkpoint['tokens'],
n_hidden=checkpoint['n_hidden'],
n_layers=checkpoint['n_layers'])
net.load_state_dict(checkpoint['state_dict'])
def predict(net, char, h=None, top_k=None):
x = np.array([[net.char2int[char]]])
x = one_hot_encode(x, len(net.chars))
inputs = torch.from_numpy(x)
if torch.cuda.is_available():
inputs = inputs.cuda()
h = tuple([each.data for each in h])
out, h = net(inputs, h)
p = F.softmax(out, dim=1).data
if torch.cuda.is_available():
def main():
if not torch.cuda.is_available():
logging.info('No GPU device available')
sys.exit(1)
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info("args = %s", args)
logging.info("unparsed args = %s", unparsed)
num_gpus = torch.cuda.device_count()
genotype = eval("genotypes.%s" % args.arch)
print('---------Genotype---------')
logging.info(genotype)
print('--------------------------')
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype)
model = torch.nn.DataParallel(model)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
if args.cifar100:
train_transform, valid_transform = utils._data_transforms_cifar100(args)
else:
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.cifar100:
train_data = dset.CIFAR100(root=args.tmp_data_dir, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR100(root=args.tmp_data_dir, train=False, download=True, transform=valid_transform)
else:
train_data = dset.CIFAR10(root=args.tmp_data_dir, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.tmp_data_dir, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=args.workers)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=args.workers)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
best_acc = 0.0
for epoch in range(args.epochs):
scheduler.step()
logging.info('Epoch: %d lr %e', epoch, scheduler.get_lr()[0])
model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
start_time = time.time()
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('Train_acc: %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
if valid_acc > best_acc:
best_acc = valid_acc
logging.info('Valid_acc: %f', valid_acc)
end_time = time.time()
duration = end_time - start_time
print('Epoch time: %ds.' % duration )
utils.save(model.module, os.path.join(args.save, 'weights.pt'))
def post(self):
email = self.request.get('email')
network_name = self.request.get('network')
network_key = Network.query(Network.name == network_name).get().key
AllowedUser(email=email, network=network_key).put()
self.redirect('/admin/user')
class GUI(Tk):
def __init__(self,name):
Tk.__init__(self, None)
self.title('Equilibrium Propagation')
self.net = Network(name=name,hyperparameters={"batch_size":1})
self.hidden_sizes = self.net.hyperparameters["hidden_sizes"]
self.n_layers = len(self.hidden_sizes) + 2
self.canvas = Canvas(self, width=600, height=(self.n_layers+1)*100)
self.canvas.pack(side=BOTTOM)
# INDEX OF TEST EXAMPLE (IN THE TEST SET)
Label(self, text="image").pack(side=LEFT)
self.index = StringVar()
self.index.set("0")
Entry(self, textvariable=self.index, width=5).pack(side=LEFT)
self.update_canvas(first_time=True)
Thread(target = self.run).start()
def update_canvas(self, first_time = False):
units = [(28,28)] +[(10,n/10) for n in self.hidden_sizes]+[(1,10)]
pixels = [(140,140)]+ [(n/2,50) for n in self.hidden_sizes]+[(250,25)]
arrays = [256*layer.eval().reshape(dimensions) for layer,dimensions in zip(self.net.layers,units)]
images = [Image.fromarray(array).resize(dimensions) for array,dimensions in zip(arrays,pixels)]
self.imgTks = [ImageTk.PhotoImage(image) for image in images]
[energy, cost, _] = self.net.measure()
if first_time:
self.img_canvas = [self.canvas.create_image(400, (self.n_layers-k)*100, image=imgTk) for k,imgTk in enumerate(self.imgTks)]
self.energy_canvas = self.canvas.create_text( 20, 100, anchor=W, font="Purisa", text="Energy = %.1f" % (energy))
self.cost_canvas = self.canvas.create_text( 20, 200, anchor=W, font="Purisa", text="Cost = %.4f" % (cost))
else:
for img_canvas, imgTk in zip(self.img_canvas,self.imgTks):
self.canvas.itemconfig(img_canvas, image=imgTk)
self.canvas.itemconfig(self.energy_canvas, text="Energy = %.1f" % (energy))
self.canvas.itemconfig(self.cost_canvas, text="Cost = %.4f" % (cost))
def run(self):
while True:
index = self.index.get() # index of the test example in the test set
if index.isdigit():
index = int(index)
index = (hash(index) % 10000) + 60000
self.net.change_mini_batch_index(index)
self.net.free_phase(n_iterations=1, epsilon=np.float32(.1))
self.update_canvas()
# FREQUENCY OF UPDATES (IN SECONDS)
time.sleep(.1)
"alphas": [np.float32(.4), np.float32(.1),
np.float32(.01)]
}
# HYPERPARAMETERS FOR A NETWORK WITH 3 HIDDEN LAYERS
net3 = "net3", {
"hidden_sizes": [500, 500, 500],
"n_epochs":
500,
"batch_size":
20,
"n_it_neg":
500,
"n_it_pos":
8,
"epsilon":
np.float32(.5),
"beta":
np.float32(1.),
"alphas":
[np.float32(.128),
np.float32(.032),
np.float32(.008),
np.float32(.002)]
}
if __name__ == "__main__":
# TRAIN A NETWORK WITH 1 HIDDEN LAYER
train_net(Network(*net1))
def __init__(self, rng):
Network.__init__(self, n_hidden_layer = 8, BN = BN)
print " C3 layer:"
self.layer.append(ReLU_conv_layer(
rng,
filter_shape=(128, 3, 3, 3),
pool_shape=(1,1),
pool_stride=(1,1),
BN = BN,
BN_epsilon=BN_epsilon,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test
))
print " C3 P2 layers:"
self.layer.append(ReLU_conv_layer(
rng,
filter_shape=(128, 128, 3, 3),
pool_shape=(2,2),
pool_stride=(2,2),
BN = BN,
BN_epsilon=BN_epsilon,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test
))
print " C2 layer:"
self.layer.append(ReLU_conv_layer(
rng,
filter_shape=(256, 128, 2, 2),
pool_shape=(1,1),
pool_stride=(1,1),
BN = BN,
BN_epsilon=BN_epsilon,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test
))
print " C2 P2 layers:"
self.layer.append(ReLU_conv_layer(
rng,
filter_shape=(256, 256, 2, 2),
pool_shape=(2,2),
pool_stride=(2,2),
BN = BN,
BN_epsilon=BN_epsilon,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test
))
print " C2 layer:"
self.layer.append(ReLU_conv_layer(
rng,
filter_shape=(512, 256, 2, 2),
pool_shape=(1,1),
pool_stride=(1,1),
BN = BN,
BN_epsilon=BN_epsilon,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test
))
print " C2 P2 layers:"
self.layer.append(ReLU_conv_layer(
rng,
filter_shape=(512, 512, 2, 2),
pool_shape=(2,2),
pool_stride=(2,2),
BN = BN,
BN_epsilon=BN_epsilon,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test
))
print " C2 layer:"
self.layer.append(ReLU_conv_layer(
rng,
filter_shape=(1024, 512, 2, 2),
pool_shape=(1,1),
pool_stride=(1,1),
BN = BN,
BN_epsilon=BN_epsilon,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test
))
print " FC layer:"
self.layer.append(ReLU_layer(
rng = rng,
n_inputs = 1024,
n_units = 1024,
BN = BN,
BN_epsilon=BN_epsilon,
dropout=dropout_hidden,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test
))
print " L2 SVM layer:"
self.layer.append(linear_layer(
rng = rng,
n_inputs= 1024,
n_units = 10,
BN = BN,
BN_epsilon=BN_epsilon,
dropout = dropout_hidden,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test
))
from darts.links.model import NetworkCIFAR
from darts.genotypes import DARTS
from darts.operations import *
import sys
sys.path.append('../cnn')
from model import NetworkCIFAR as Network
from chainer.datasets import get_cifar10
c_model = NetworkCIFAR(DARTS)
chainer.serializers.load_npz('model.npz', c_model)
chainer.global_config.train = False
t_model = Network(36, 10, 20, True, DARTS)
t_model.drop_path_prob = 0.2
t_model.load_state_dict(torch.load('cifar10_model.pt'))
t_model.eval()
train, val = get_cifar10()
img, _ = train[0]
x = img[None]
t_x = torch.autograd.Variable(torch.FloatTensor(x))
c_y = c_model(x)
t_y = t_model(t_x)
np.testing.assert_almost_equal(c_y[0].data,
t_y[0].detach().data.numpy(),
decimal=5)
#####################
def main():
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
cudnn.enabled = True
torch.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
if not args.arch:
geno_s = pathlib.Path(geno_path).read_text()
else:
geno_s = "genotypes.%s" % args.arch
genotype = eval(geno_s)
model = Network(args.init_ch, 10, args.layers, args.auxiliary, genotype).cuda()
logging.info(f"seed = {args.seed}")
logging.info(f"geno_s = {geno_s}")
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.wd
)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batchsz, shuffle=True, pin_memory=True, num_workers=0 if 'pydevd' in sys.modules else 2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batchsz, shuffle=False, pin_memory=True, num_workers=0 if 'pydevd' in sys.modules else 2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
lines = [f'epoch\ttrain_acc\tval_acc']
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc: %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc: %f', valid_acc)
lines.append(f'{epoch}\t{train_acc}\t{valid_acc}')
timebudget.report()
utils.save(model, os.path.join(args.save, 'trained.pt'))
print('saved to: trained.pt')
pathlib.Path(os.path.join(args.exp_path, 'eval.tsv')).write_text('\n'.join(lines))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
if args.parallel: # multi gpu
num_gpus = torch.cuda.device_count()
logging.info('num of gpu devices = %d' % num_gpus)
else: # single gpu
torch.cuda.set_device(args.gpu)
logging.info('gpu device = %d' % args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CLASSES, args.layers, args.auxiliary,
genotype)
if args.parallel:
model = nn.DataParallel(model).cuda()
else:
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
criterion_smooth = CrossEntropyLabelSmooth(CLASSES, args.label_smooth)
criterion_smooth = criterion_smooth.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
traindir = os.path.join(args.data, 'train')
validdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_data = dset.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ColorJitter(brightness=0.4,
contrast=0.4,
saturation=0.4,
hue=0.2),
transforms.ToTensor(),
normalize,
]))
valid_data = dset.ImageFolder(
validdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
]))
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=4)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=4)
if args.lr_scheduler == 'step':
# DARTS code
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
args.decay_period,
gamma=args.gamma)
elif args.lr_scheduler == 'cosine' or args.lr_scheduler == 'linear':
# PCDARTS code
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
else:
raise ValueError("Wrong learning rate scheduler")
# ---- resume ---- #
start_epoch = 0
best_acc_top1 = 0.0
best_acc_top5 = 0.0
best_acc_epoch = 0
if args.resume:
# in multi-gpu???
if os.path.isfile(args.resume):
logging.info("=> loading checkpoint {}".format(args.resume))
device = torch.device("cuda")
checkpoint = torch.load(args.resume, map_location=device)
start_epoch = checkpoint['epoch']
best_acc_top1 = checkpoint['best_acc_top1']
best_acc_top5 = checkpoint['best_acc_top5']
best_acc_epoch = checkpoint['best_acc_epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logging.info(
"=> loaded checkpoint {} (trained until epoch {})".format(
args.resume, start_epoch - 1))
else:
raise ValueError("Wrong args.resume")
else:
logging.info("=> training from scratch")
for epoch in range(start_epoch, args.epochs):
scheduler.step()
if args.lr_scheduler == 'cosine' or args.lr_scheduler == 'step':
scheduler.step()
current_lr = scheduler.get_lr()[0]
elif args.lr_scheduler == 'linear':
current_lr = adjust_lr(optimizer, epoch)
if epoch < 5 and args.batch_size > 256:
for param_group in optimizer.param_groups:
param_group['lr'] = args.learning_rate * (epoch + 1) / 5.0
logging.info('Warming-up epoch: %d, LR: %e', epoch,
lr * (epoch + 1) / 5.0)
else:
logging.info('epoch %d lr %e', epoch, current_lr)
if args.parallel:
model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs
else:
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
epoch_start = time.time()
train_acc, train_obj = train(train_queue, model, criterion_smooth,
optimizer)
logging.info('train_acc %f', train_acc)
valid_acc_top1, valid_acc_top5, valid_obj = infer(
valid_queue, model, criterion)
is_best = (valid_acc_top1 > best_acc_top1)
if is_best:
best_acc_top1 = valid_acc_top1
best_acc_top5 = valid_acc_top5
best_acc_epoch = epoch + 1
utils.save(model, os.path.join(args.save, 'best_weights.pt'))
logging.info('valid_acc %f %f, best_acc %f %f (at epoch %d)',
valid_acc_top1, valid_acc_top5, best_acc_top1,
best_acc_top5, best_acc_epoch)
logging.info('epoch time %d sec.', time.time() - epoch_start)
utils.save_checkpoint(
{
'epoch': epoch + 1,
'best_acc_top1': best_acc_top1,
'best_acc_top5': best_acc_top5,
'best_acc_epoch': best_acc_epoch,
'state_dict': model.state_dict(),
'best_acc_top1': best_acc_top1,
'optimizer': optimizer.state_dict(),
}, is_best, args.save)
utils.save(model, os.path.join(args.save, 'weights.pt'))
import torch
import numpy as np
import os
from model import Network
def show_compression_rate(model):
state_dict = model.state_dict()
cnt_compressed = 0
cnt_original = 0
for x in state_dict:
weight = state_dict[x]
print(np.sum(weight.numpy() != 0))
cnt_compressed += np.sum(weight.numpy() != 0)
cnt_original += weight.numel()
print('After Compression: %d' % cnt_compressed)
print('Before Compression: %d' % cnt_original)
print('Compression Rate: %.4f' % (float(cnt_original) / cnt_compressed))
if __name__ == '__main__':
model = Network()
#model.load_state_dict(torch.load('checkpoints/model_layer_1.pkl'))
model.load_state_dict(torch.load('model.pkl'))
show_compression_rate(model)
from config import *
from model import Network
import numpy as np
import wdbedding
try:
import xml.etree.cElementTree as ET
except ImportError:
import xml.etree.ElementTree as ET
if __name__ == '__main__':
# network init
network = Network("training")
# embedding model init
word_vec_model = wdbedding.load_word2vec_model(EN)
print("word2vec model_loaded.")
# about the file
input_file_path = '../data/task2_input_en.xml'
xmltree = ET.parse(input_file_path)
xmlroot = xmltree.getroot()
# iterate the xml tree
for review in xmlroot:
txt = review.text
if txt[-1] == '\n':
txt = txt[:-1]
print(txt)
test_data = []
test_data.append(wdbedding.embedding(word_vec_model, txt, EN))
test_data = np.reshape(test_data,
def main(args):
place = fluid.CUDAPlace(fluid.dygraph.parallel.Env().dev_id) \
if args.use_data_parallel else fluid.CUDAPlace(0)
with fluid.dygraph.guard(place):
genotype = eval("genotypes.%s" % args.arch)
model = Network(
C=args.init_channels,
num_classes=args.class_num,
layers=args.layers,
auxiliary=args.auxiliary,
genotype=genotype)
logger.info("param size = {:.6f}MB".format(
count_parameters_in_MB(model.parameters())))
device_num = fluid.dygraph.parallel.Env().nranks
step_per_epoch = int(args.trainset_num /
(args.batch_size * device_num))
learning_rate = fluid.dygraph.CosineDecay(args.learning_rate,
step_per_epoch, args.epochs)
clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=args.grad_clip)
optimizer = fluid.optimizer.MomentumOptimizer(
learning_rate,
momentum=args.momentum,
regularization=fluid.regularizer.L2Decay(args.weight_decay),
parameter_list=model.parameters(),
grad_clip=clip)
if args.use_data_parallel:
strategy = fluid.dygraph.parallel.prepare_context()
model = fluid.dygraph.parallel.DataParallel(model, strategy)
train_loader = fluid.io.DataLoader.from_generator(
capacity=64,
use_double_buffer=True,
iterable=True,
return_list=True,
use_multiprocess=args.use_multiprocess)
valid_loader = fluid.io.DataLoader.from_generator(
capacity=64,
use_double_buffer=True,
iterable=True,
return_list=True,
use_multiprocess=args.use_multiprocess)
train_reader = reader.train_valid(
batch_size=args.batch_size,
is_train=True,
is_shuffle=True,
args=args)
valid_reader = reader.train_valid(
batch_size=args.batch_size,
is_train=False,
is_shuffle=False,
args=args)
if args.use_data_parallel:
train_reader = fluid.contrib.reader.distributed_batch_reader(
train_reader)
train_loader.set_batch_generator(train_reader, places=place)
valid_loader.set_batch_generator(valid_reader, places=place)
save_parameters = (not args.use_data_parallel) or (
args.use_data_parallel and
fluid.dygraph.parallel.Env().local_rank == 0)
best_acc = 0
for epoch in range(args.epochs):
drop_path_prob = args.drop_path_prob * epoch / args.epochs
logger.info('Epoch {}, lr {:.6f}'.format(
epoch, optimizer.current_step_lr()))
train_top1 = train(model, train_loader, optimizer, epoch,
drop_path_prob, args)
logger.info("Epoch {}, train_acc {:.6f}".format(epoch, train_top1))
valid_top1 = valid(model, valid_loader, epoch, args)
if valid_top1 > best_acc:
best_acc = valid_top1
if save_parameters:
fluid.save_dygraph(model.state_dict(),
args.model_save_dir + "/best_model")
logger.info("Epoch {}, valid_acc {:.6f}, best_valid_acc {:.6f}".
format(epoch, valid_top1, best_acc))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
"""
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
"""
data_dir = '../data/kmnist/'
data_augmentations = transforms.ToTensor()
# Load the Data here
train_dataset = K49(data_dir, True, data_augmentations)
#test_dataset = K49(data_dir, False, data_augmentations)
num_train = len(train_dataset)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
dataset=train_dataset,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(
dataset=train_dataset,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
import torch
from torchsummary import summary
from model import Network
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = Network()
#print(model)
model = model.to(device)
#list(model.children())
summary(model, input_size=(24, 16, 16))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
dataset = Dataset(args.dataset)
train_examples = torch.from_numpy(dataset.get_train().astype('int64'))
#TODO: does below need reintroducing somewhere?
# device = 'cuda'
# model.to(device)
CLASSES = dataset.get_shape()[0]
criterion = nn.CrossEntropyLoss(reduction='mean')
#criterion = CrossEntropyLabelSmooth(CLASSES, args.label_smooth)
criterion = criterion.cuda()
regularizer = {
'N2': N2(args.reg),
'N3': N3(args.reg),
}[args.regularizer]
genotype = eval("genotypes.%s" % args.arch)
logging.info('genotype = %s', genotype)
model = Network(args.channels,
CLASSES, args.layers, criterion, regularizer, genotype, args.interleaved,
dataset.get_shape(), args.emb_dim, args.init)
model = model.cuda()
optimizer = {
'Adagrad': lambda: optim.Adagrad(model.parameters(), lr=args.learning_rate),
'Adam': lambda: optim.Adam(model.parameters(), lr=args.learning_rate, weight_decay = args.weight_decay, betas=(args.decay1, args.decay2)),
'SGD': lambda: optim.SGD(model.parameters(), lr=args.learning_rate)
}[args.optimizer]()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
#optimizer = torch.optim.SGD(
# model.parameters(),
# args.learning_rate,
#momentum=args.momentum,
#weight_decay=args.weight_decay
# )
train_queue = torch.utils.data.DataLoader(
train_examples, batch_size=args.batch_size,
shuffle = True,
#sampler=torch.utils.data.sampler.RandomSampler(),
pin_memory=True, num_workers=2)
#TODO do we want the learning rate min here?
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
best_acc = 0
patience = 0
curve = {'valid': [], 'test': []}
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_epoch(train_examples, train_queue, model, optimizer,
regularizer, args.batch_size)
if (epoch + 1) % args.report_freq == 0:
valid, test = [
avg_both(*dataset.eval(model, split, -1 if split != 'train' else 50000))
for split in ['valid', 'test']
]
curve['valid'].append(valid)
curve['test'].append(test)
#curve['train'].append(train)
#print("\t TRAIN: ", train)
print("\t VALID : ", valid)
print("\t TEST: ", test)
is_best = False
if valid['MRR'] > best_acc:
best_acc = valid['MRR']
is_best = True
patience = 0
else:
patience +=1
utils.save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_acc_top1': best_acc,
'optimizer' : optimizer.state_dict(),
}, is_best, args.save)
if patience >= 5:
print('early stopping...')
break
#utils.save(model, os.path.join(args.save, 'weights.pt'))
results = dataset.eval(model, 'test', -1)
print("\n\nTEST : ", results)
with open(os.path.join(args.save, 'curve.pkl'), 'wb') as f:
pickle.dump(curve, f, pickle.HIGHEST_PROTOCOL)
def main():
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpus
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %s' % args.gpus)
logging.info("args = %s", args)
num_gpus = torch.cuda.device_count()
genotype = eval("genotypes.%s" % args.arch)
print('---------Genotype---------')
logging.info(genotype)
print('--------------------------')
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
if num_gpus > 1:
model = nn.DataParallel(model)
model = model.cuda()
else:
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.set == 'cifar100':
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR100(root=args.data,
train=False,
download=True,
transform=valid_transform)
else:
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
#train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
#valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=args.workers)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=args.workers)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
best_acc = 0.0
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
if num_gpus > 1:
model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs
else:
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
if valid_acc > best_acc:
best_acc = valid_acc
logging.info('valid_acc %f, best_acc %f', valid_acc, best_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
total = 0
for m in model.modules():
if isinstance(m, nn.BatchNorm2d):
total += m.weight.data.shape[0]
bn = torch.zeros(total)
index = 0
for m in model.modules():
if isinstance(m, nn.BatchNorm2d):
size = m.weight.data.shape[0]
bn[index:(index + size)] = m.weight.data.abs().clone()
index += size
y, i = torch.sort(bn)
thre_index = int(total * 0.5)
thre = y[thre_index]
pruned = 0
cfg = []
cfg_mask = []
for k, m in enumerate(model.modules()):
if isinstance(m, nn.BatchNorm2d):
weight_copy = m.weight.data.clone()
mask = weight_copy.abs().gt(thre).float().cuda()
pruned = pruned + mask.shape[0] - torch.sum(mask)
m.weight.data.mul_(mask)
m.bias.data.mul_(mask)
cfg.append(int(torch.sum(mask)))
cfg_mask.append(mask.clone())
print(
'layer index: {:d} \t total channel: {:d} \t remaining channel: {:d}'
.format(k, mask.shape[0], int(torch.sum(mask))))
elif isinstance(m, nn.MaxPool2d):
cfg.append('M')
pruned_ratio = pruned / total
print('Pre-processing Successful!')
utils.load(model, args.model_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
snr = [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20]
sum = 0
TIM = 0
utils.load(model, args.model_path)
for i in snr:
data_num = 1000
test_data = loadmat("/data/wx/PC-DARTS/data/testdoppler=100/snr=" +
str(i) + ".mat")
x = test_data.get("train_data")
print(x.shape)
x = np.reshape(x, [-1, 1, 2, 128])
y1 = np.zeros([data_num, 1])
y2 = np.ones([data_num, 1])
y3 = np.ones([data_num, 1]) * 2
y4 = np.ones([data_num, 1]) * 3
y5 = np.ones([data_num, 1]) * 4
y6 = np.ones([data_num, 1]) * 5
y7 = np.ones([data_num, 1]) * 6
y8 = np.ones([data_num, 1]) * 7
y = np.vstack((y1, y2, y3, y4, y5, y6, y7, y8))
y = np.array(y)
X_test = torch.from_numpy(x)
Y_test = torch.from_numpy(y)
X_test = X_test.type(torch.FloatTensor)
Y_test = Y_test.type(torch.LongTensor)
Y_test = Y_test.squeeze()
test_Queue = torch.utils.data.TensorDataset(X_test, Y_test)
test_queue = torch.utils.data.DataLoader(test_Queue,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
model.drop_path_prob = args.drop_path_prob
if i == 0:
Input = torch.randn(1, 1, 2, 128)
Input = Input.type(torch.cuda.FloatTensor)
macs, params = profile(model, inputs=(Input, ))
macs, params = clever_format([macs, params], "%.3f")
print("flops params")
print(macs, params)
summary(model, input_size=(1, 2, 128))
time1 = time.time()
test_acc, test_obj, target, loggg = infer(test_queue, model, criterion)
time2 = time.time() - time1
logging.info('第 %d snr test_acc %f', i, test_acc)
sum += test_acc
logging.info("第 %d snr time: %f", i, time2)
TIM += time2
#print(target)
#print(target.shape)
#print(loggg.shape)
target = target.cpu().detach().numpy()
loggg = loggg.cpu().detach().numpy()
cm = confusion_matrix(target, loggg)
#print(cm)
#plot_confusion_matrix(cm ,mods, title=" Confusion Matrix ( SNR=%d dB)" % (i))
'''
if i>=10:
address_jpeg = '/data/wx/PC-DARTS/wx/picture/' + '100' + 'hz-CSS-snr=' + str(i) + '.pdf'
plt.savefig(address_jpeg)
plt.close('all') '''
#plt.show()
ACC = sum / 11
TT = TIM / 11
print("average acc : ", ACC)
print("average time : ", TT)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
if is_wandb_used:
wandb.init(
project="automl-gradient-based-nas",
name="ImageNet:" + str(args.arch),
config=args,
entity="automl"
)
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CLASSES, args.layers, args.auxiliary, genotype)
if args.parallel:
model = nn.DataParallel(model).cuda()
else:
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
criterion_smooth = CrossEntropyLabelSmooth(CLASSES, args.label_smooth)
criterion_smooth = criterion_smooth.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
traindir = os.path.join(args.data, 'train')
validdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
train_data = dset.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ColorJitter(
brightness=0.4,
contrast=0.4,
saturation=0.4,
hue=0.2),
transforms.ToTensor(),
normalize,
]))
valid_data = dset.ImageFolder(
validdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
]))
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=4)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=4)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, args.decay_period, gamma=args.gamma)
best_acc_top1 = 0
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion_smooth, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc_top1, valid_acc_top5, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc_top1 %f', valid_acc_top1)
logging.info('valid_acc_top5 %f', valid_acc_top5)
is_best = False
if valid_acc_top1 > best_acc_top1:
best_acc_top1 = valid_acc_top1
is_best = True
utils.save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_acc_top1': best_acc_top1,
'optimizer': optimizer.state_dict(),
}, is_best, args.save)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
in_channels, num_classes, dataset_in_torch, stride_for_aux = utils.dataset_fields(
args, train=False) # new
genotype = eval("genotypes.%s" %
args.arch) #eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, in_channels, stride_for_aux,
num_classes, args.layers, args.auxiliary, genotype,
args.SE)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_data, valid_data = utils.dataset_split_and_transform(
dataset_in_torch, args, train=False) # new
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]))
val_sampler = torch.utils.data.distributed.DistributedSampler(
val_dataset, num_replicas=hvd.size(), rank=hvd.rank())
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.val_batch_size,
sampler=val_sampler,
**kwargs)
# Set up standard ResNet-50 model.
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CLASSES, args.layers, args.auxiliary,
genotype)
# model = models.resnet50()
if args.cuda:
# Move model to GPU.
model.cuda()
#model load
# model_path = "./model_imagenet_20190112/checkpoint-60.pth"
# model.load_state_dict(torch.load(model_path))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
criterion_smooth = CrossEntropyLabelSmooth(CLASSES, args.label_smooth)
criterion_smooth = criterion_smooth.cuda()
def __init__(self, rng):
Network.__init__(self, n_hidden_layer=8, BN=BN)
print " C3 layer:"
self.layer.append(
ReLU_conv_layer(rng,
filter_shape=(128, 3, 3, 3),
pool_shape=(1, 1),
pool_stride=(1, 1),
BN=BN,
BN_epsilon=BN_epsilon,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test))
print " C3 P2 layers:"
self.layer.append(
ReLU_conv_layer(rng,
filter_shape=(128, 128, 3, 3),
pool_shape=(2, 2),
pool_stride=(2, 2),
BN=BN,
BN_epsilon=BN_epsilon,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test))
print " C2 layer:"
self.layer.append(
ReLU_conv_layer(rng,
filter_shape=(256, 128, 2, 2),
pool_shape=(1, 1),
pool_stride=(1, 1),
BN=BN,
BN_epsilon=BN_epsilon,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test))
print " C2 P2 layers:"
self.layer.append(
ReLU_conv_layer(rng,
filter_shape=(256, 256, 2, 2),
pool_shape=(2, 2),
pool_stride=(2, 2),
BN=BN,
BN_epsilon=BN_epsilon,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test))
print " C2 layer:"
self.layer.append(
ReLU_conv_layer(rng,
filter_shape=(512, 256, 2, 2),
pool_shape=(1, 1),
pool_stride=(1, 1),
BN=BN,
BN_epsilon=BN_epsilon,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test))
print " C2 P2 layers:"
self.layer.append(
ReLU_conv_layer(rng,
filter_shape=(512, 512, 2, 2),
pool_shape=(2, 2),
pool_stride=(2, 2),
BN=BN,
BN_epsilon=BN_epsilon,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test))
print " C2 layer:"
self.layer.append(
ReLU_conv_layer(rng,
filter_shape=(1024, 512, 2, 2),
pool_shape=(1, 1),
pool_stride=(1, 1),
BN=BN,
BN_epsilon=BN_epsilon,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test))
print " FC layer:"
self.layer.append(
ReLU_layer(rng=rng,
n_inputs=1024,
n_units=1024,
BN=BN,
BN_epsilon=BN_epsilon,
dropout=dropout_hidden,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test))
print " L2 SVM layer:"
self.layer.append(
linear_layer(rng=rng,
n_inputs=1024,
n_units=10,
BN=BN,
BN_epsilon=BN_epsilon,
dropout=dropout_hidden,
binary_training=binary_training,
stochastic_training=stochastic_training,
binary_test=binary_test,
stochastic_test=stochastic_test))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
random.seed(args.seed)
np.random.seed(
args.data_seed) # cutout and load_corrupted_data use np.random
torch.cuda.set_device(args.gpu)
cudnn.benchmark = False
torch.manual_seed(args.seed)
cudnn.enabled = True
cudnn.deterministic = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
if args.arch == 'resnet':
model = ResNet18(CIFAR_CLASSES).cuda()
args.auxiliary = False
elif args.arch == 'resnet50':
model = ResNet50(CIFAR_CLASSES).cuda()
args.auxiliary = False
elif args.arch == 'resnet34':
model = ResNet34(CIFAR_CLASSES).cuda()
args.auxiliary = False
else:
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, test_transform = utils._data_transforms_cifar10(args)
# Load dataset
if args.dataset == 'cifar10':
noisy_train_data = CIFAR10(root=args.data,
train=True,
gold=False,
gold_fraction=0.0,
corruption_prob=args.corruption_prob,
corruption_type=args.corruption_type,
transform=train_transform,
download=True,
seed=args.data_seed)
gold_train_data = CIFAR10(root=args.data,
train=True,
gold=True,
gold_fraction=1.0,
corruption_prob=args.corruption_prob,
corruption_type=args.corruption_type,
transform=train_transform,
download=True,
seed=args.data_seed)
test_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=test_transform)
elif args.dataset == 'cifar100':
noisy_train_data = CIFAR100(root=args.data,
train=True,
gold=False,
gold_fraction=0.0,
corruption_prob=args.corruption_prob,
corruption_type=args.corruption_type,
transform=train_transform,
download=True,
seed=args.data_seed)
gold_train_data = CIFAR100(root=args.data,
train=True,
gold=True,
gold_fraction=1.0,
corruption_prob=args.corruption_prob,
corruption_type=args.corruption_type,
transform=train_transform,
download=True,
seed=args.data_seed)
test_data = dset.CIFAR100(root=args.data,
train=False,
download=True,
transform=test_transform)
num_train = len(gold_train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
if args.gold_fraction == 1.0:
train_data = gold_train_data
else:
train_data = noisy_train_data
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=0)
if args.clean_valid:
valid_data = gold_train_data
else:
valid_data = noisy_train_data
valid_queue = torch.utils.data.DataLoader(
valid_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:]),
pin_memory=True,
num_workers=0)
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
if args.loss_func == 'cce':
criterion = nn.CrossEntropyLoss().cuda()
elif args.loss_func == 'rll':
criterion = utils.RobustLogLoss(alpha=args.alpha).cuda()
elif args.loss_func == 'forward_gold':
corruption_matrix = train_data.corruption_matrix
criterion = utils.ForwardGoldLoss(corruption_matrix=corruption_matrix)
else:
assert False, "Invalid loss function '{}' given. Must be in {'cce', 'rll'}".format(
args.loss_func)
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer_valid(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
