def train_post_prototype(assignment):
from config import get_prototype_config
from model_torch import Post_Prototype_RCNN_L2
from stream import Post_Data
config = get_prototype_config()
## 1. data prepare
trainData = Post_Data(is_train=True, **config)
testData = Post_Data(is_train=False, **config)
### 2. model & train
nn = Post_Prototype_RCNN_L2(assignment, **config)
LR = config['LR']
opt_ = torch.optim.SGD(nn.parameters(), lr=LR)
best_auc = 0
trainFeature_all, _ = trainData.get_all()
testFeature, testLabel = testData.get_all()
for i in range(config['train_iter']):
nn.generate_prototype(trainFeature_all)
feat, label = trainData.next()
loss, _ = nn(feat, label)
opt_.zero_grad()
loss.backward()
opt_.step()
loss_value = loss.data[0]
if i % 1 == 0:
score = post_evaluate(testFeature, testLabel, nn, **config)
best_auc = score if score > best_auc else best_auc
print('iter {}, auc:{} (best:{})'.format(i,
str(score)[:5],
str(best_auc)[:5]),
end=' ')
if i > 0:
print('{} sec'.format(str(time() - t1)[:4]))
t1 = time()
def test_conv_in_computational_graph(self):
nn = NN(3)
for param in nn.parameters():
assert param.requires_grad
afl = AffineCouplingLayer(3)
for param in afl.parameters():
assert param.requires_grad
def parameter_stats(nn: nn.Module, print_indivual_params=True):
"""Prints various statistics about the paramters of a neural neural network"""
p = torch.nn.utils.parameters_to_vector(nn.parameters()).detach().numpy()
print(
f"max/min = {max(p):.1f}/{min(p):.1f}, mean={np.mean(p):.1f}+/-{np.std(p):.2f}"
)
if print_indivual_params:
for name, p in nn.named_parameters():
p = p.detach().numpy()
print(
f"{name:40s}max/min = {np.max(p):.1f}/{np.min(p):.1f}, mean={np.mean(p):.1f}+/-{np.std(p):.2f}"
)
def train_weighted_post_prototype(assignment):
from config import get_prototype_config
from model_torch import Weighted_Post_Prototype_RCNN_L2
from stream import Weighted_Post_Data
config = get_prototype_config()
every_iter = config['every_iter']
## 1. data prepare
trainData = Weighted_Post_Data(is_train=True, **config)
testData = Weighted_Post_Data(is_train=False, **config)
### 2. model & train
nn = Weighted_Post_Prototype_RCNN_L2(assignment, **config)
LR = config['LR']
opt_ = torch.optim.SGD(nn.parameters(), lr=LR)
best_auc = 0
trainFeature_all, _, weight_all = trainData.get_all()
testFeature, testLabel, _ = testData.get_all()
for i in range(config['train_iter']):
nn.generate_prototype(trainFeature_all, weight_all)
feat, label, weight = trainData.next()
loss, _ = nn(feat, label, weight)
opt_.zero_grad()
loss.backward()
opt_.step()
#loss_value = loss.data[0]
if i % 1 == 0:
score = post_evaluate(testFeature, testLabel, nn, **config)
best_auc = score if score > best_auc else best_auc
if i % every_iter == every_iter - 1 and i > 0:
print('iter {}, auc:{} (best:{})'.format(
i,
str(score)[:5],
str(best_auc)[:5]),
end=' \n')
# print('{} sec'.format(str(time() - t1)[:4]))
#t1 = time()
trainData.restart()
reweight = []
for i in range(trainData.batch_number):
feat, label, weight = trainData.next()
reweight.extend(nn.measure_similarity(feat))
#print(reweight)
reweight = normalize_weight(reweight, config['upper'], config['lower'])
#print(reweight)
return reweight
def train(self, grad_mod):
'''
Computes an epoch of the full training step
'''
for model in self.mrf.nnformulas:
model.train()
self.optimizer.zero_grad()
preds = self.forward()
# print(preds[1])
loss = criterion[0](preds[0], y_true)
loss.backward()
if (grad_mod):
gradient = self.grad(preds)
for fidx, nn in enumerate(self.mrf.nnformulas):
for par in nn.parameters():
par.grad *= gradient[fidx]
self.optimizer.step()
return loss
x = self.pool(F.relu(self.conv2(x)))
#x = F.relu(self.conv2(x))
x = F.relu(self.conv3(x))
x = F.relu(self.conv4(x))
x = x.view(x.size(0), -1)
x = F.relu(self.fc(self.do(x)))
#x = F.relu(self.fc(x))
x = self.out(x)
return x
if __name__ == '__main__':
#Test code
nn = net(2, manifold=geoopt.manifolds.PoincareBall())
#nn = net(2,manifold=geoopt.manifolds.Euclidean())
#nn = net(2,manifold=geoopt.manifolds.Stiefel())
x = torch.randn(128, 1024, 2)
y = nn(x)
opt = geoopt.optim.RiemannianAdam(nn.parameters(), lr=1, stabilize=1)
opt.zero_grad()
opt.step()
print(nn)
#print(y)
def test(models, epoch, f=None):
global num_tests
num_tests += 1
class MStat:
def __init__(self, model):
model.eval()
self.model = model
self.correct = 0
class Stat:
def __init__(self, d, dnm):
self.domain = d
self.name = dnm
self.width = 0
self.max_eps = None
self.safe = 0
self.proved = 0
self.time = 0
self.domains = [
Stat(h.parseValues(d, goals), h.catStrs(d))
for d in args.test_domain
]
model_stats = [MStat(m) for m in models]
dict_map = dict(np.load("./dataset/AG/dict_map.npy").item())
lines = open("./dataset/en.key1").readlines()
adjacent_keys = [[] for i in range(len(dict_map))]
for line in lines:
tmp = line.strip().split()
ret = set(tmp[1:]).intersection(dict_map.keys())
ids = []
for x in ret:
ids.append(dict_map[x])
adjacent_keys[dict_map[tmp[0]]].extend(ids)
num_its = 0
saved_data_target = []
for data, target in test_loader:
if num_its >= args.test_size:
break
if num_tests == 1:
saved_data_target += list(zip(list(data), list(target)))
num_its += data.size()[0]
if num_its % 100 == 0:
print(num_its, model_stats[0].domains[0].safe * 100.0 / num_its)
if args.test_swap_delta > 0:
length = data.size()[1]
data = data.repeat(1, length)
for i in data:
for j in range(length - 1):
for _ in range(args.test_swap_delta):
t = np.random.randint(0, length)
while len(adjacent_keys[int(i[t])]) == 0:
t = np.random.randint(0, length)
cid = int(i[t])
i[j * length + t] = adjacent_keys[cid][0]
target = (target.view(-1, 1).repeat(1, length)).view(-1)
data = data.view(-1, length)
if h.use_cuda:
data, target = data.cuda().to_dtype(), target.cuda()
for m in model_stats:
with torch.no_grad():
pred = m.model(data).vanillaTensorPart().max(1, keepdim=True)[
1] # get the index of the max log-probability
m.correct += pred.eq(target.data.view_as(pred)).sum()
for stat in m.domains:
timer = Timer(shouldPrint=False)
with timer:
def calcData(data, target):
box = stat.domain.box(data,
w=m.model.w,
model=m.model,
untargeted=True,
target=target).to_dtype()
with torch.no_grad():
bs = m.model(box)
org = m.model(data).vanillaTensorPart().max(
1, keepdim=True)[1]
stat.width += bs.diameter().sum().item(
) # sum up batch loss
stat.proved += bs.isSafe(org).sum().item()
stat.safe += bs.isSafe(target).sum().item()
# stat.max_eps += 0 # TODO: calculate max_eps
if m.model.net.neuronCount(
) < 5000 or stat.domain in SYMETRIC_DOMAINS:
calcData(data, target)
else:
if args.test_swap_delta > 0:
length = data.size()[1]
pre_stat = copy.deepcopy(stat)
for i, (d, t) in enumerate(zip(data, target)):
calcData(d.unsqueeze(0), t.unsqueeze(0))
if (i + 1) % length == 0:
d_proved = (stat.proved -
pre_stat.proved) // length
d_safe = (stat.safe -
pre_stat.safe) // length
d_width = (stat.width -
pre_stat.width) / length
stat.proved = pre_stat.proved + d_proved
stat.safe = pre_stat.safe + d_safe
stat.width = pre_stat.width + d_width
pre_stat = copy.deepcopy(stat)
else:
for d, t in zip(data, target):
calcData(d.unsqueeze(0), t.unsqueeze(0))
stat.time += timer.getUnitTime()
l = num_its # len(test_loader.dataset)
for m in model_stats:
if args.lr_multistep:
m.model.lrschedule.step()
pr_corr = float(m.correct) / float(l)
if args.use_schedule:
m.model.lrschedule.step(1 - pr_corr)
h.printBoth(
('Test: {:12} trained with {:' + str(largest_domain) +
'} - Avg sec/ex {:1.12f}, Accuracy: {}/{} ({:3.1f}%)').format(
m.model.name, m.model.ty.name, m.model.speed, m.correct, l,
100. * pr_corr),
f=f)
model_stat_rec = ""
for stat in m.domains:
pr_safe = stat.safe / l
pr_proved = stat.proved / l
pr_corr_given_proved = pr_safe / pr_proved if pr_proved > 0 else 0.0
h.printBoth((
"\t{:" + str(largest_test_domain) +
"} - Width: {:<36.16f} Pr[Proved]={:<1.3f} Pr[Corr and Proved]={:<1.3f} Pr[Corr|Proved]={:<1.3f} {}Time = {:<7.5f}"
).format(
stat.name, stat.width / l, pr_proved, pr_safe,
pr_corr_given_proved,
"AvgMaxEps: {:1.10f} ".format(stat.max_eps / l)
if stat.max_eps is not None else "", stat.time),
f=f)
model_stat_rec += "{}_{:1.3f}_{:1.3f}_{:1.3f}__".format(
stat.name, pr_proved, pr_safe, pr_corr_given_proved)
prepedname = m.model.ty.name.replace(" ", "_").replace(
",", "").replace("(", "_").replace(")", "_").replace("=", "_")
net_file = os.path.join(
out_dir, m.model.name + "__" + prepedname + "_checkpoint_" +
str(epoch) + "_with_{:1.3f}".format(pr_corr))
h.printBoth("\tSaving netfile: {}\n".format(net_file + ".pynet"), f=f)
if (num_tests % args.save_freq == 1 or args.save_freq
== 1) and not args.dont_write and (num_tests > 1
or args.write_first):
print("Actually Saving")
torch.save(m.model.net, net_file + ".pynet")
if args.save_dot_net:
with h.mopen(args.dont_write, net_file + ".net", "w") as f2:
m.model.net.printNet(f2)
f2.close()
if args.onyx:
nn = copy.deepcopy(m.model.net)
nn.remove_norm()
torch.onnx.export(
nn,
h.zeros([1] + list(input_dims)),
net_file + ".onyx",
verbose=False,
input_names=["actual_input"] + [
"param" + str(i)
for i in range(len(list(nn.parameters())))
],
output_names=["output"])
if num_tests == 1 and not args.dont_write:
img_dir = os.path.join(out_dir, "images")
if not os.path.exists(img_dir):
os.makedirs(img_dir)
for img_num, (img, target) in zip(
range(args.number_save_images),
saved_data_target[:args.number_save_images]):
sz = ""
for s in img.size():
sz += str(s) + "x"
sz = sz[:-1]
img_file = os.path.join(
img_dir, args.dataset + "_" + sz + "_" + str(img_num))
if img_num == 0:
print("Saving image to: ", img_file + ".img")
with open(img_file + ".img", "w") as imgfile:
flatimg = img.view(h.product(img.size()))
for t in flatimg.cpu():
print(decimal.Decimal(float(t)).__format__("f"),
file=imgfile)
with open(img_file + ".class", "w") as imgfile:
print(int(target.item()), file=imgfile)
def train(self):
self.bnn.train(self.X, self.y)
for nn in self.bnn.nns:
for p in nn.parameters():
p.requires_grad = False
# In[9]:
n_actions = env.action_space.n
nn = reinforce(n_actions)
# In[12]:
num_episodes = 500
episode_durations = []
episode_mean = []
memory = ReplayMemory()
optimizer = optim.RMSprop(nn.parameters(), lr=8 * math.exp(-3))
# print(num_episodes)
for i in range(num_episodes):
# print(i)
state = torch.FloatTensor([env.reset()])
for t in count():
# print(state)
action = select_action(state)
next_state, reward, done, _ = env.step(action.item())
reward = torch.tensor([reward], device=device)
# Observe new state
if not done:
next_state = torch.FloatTensor([next_state])
else:
next_state = None
# self.bn = torch.nn.BatchNorm1d(nn_config[0])
self.fc = nn.ModuleList()
for x in range(0, len(af_config) - 1):
if af_config[x] is not None:
self.fc.append(af_config[x])
self.fc.append(nn.Linear(nn_arch[x], nn_arch[x + 1]))
if init_w[x] is not None:
self.fc[-1].weight.data.uniform_(-init_w[x], init_w[x])
else:
s = self.fc[-1].weight.data.size()[0]
s = 1. / np.sqrt(s)
self.fc[-1].weight.data.uniform_(-s, s)
if af_config[len(af_config) - 1] is not None:
self.fc.append(af_config[len(af_config) - 1])
def forward(self, x):
for f in range(0, len(self.fc)):
x = self.fc[f](x)
return x
if __name__ == '__main__':
nc = NN_CONFIG(nn_arch=[10, 100, 10], af_config=[None, nn.modules.ReLU(), nn.modules.ReLU()], init_w=[None, None],
lr=0.1, optim=None, load_path=None)
nn = Net(nc.nn_arch, nc.af_config, nc.init_w)
nn.to(torch.cuda.current_device())
print(next(nn.parameters()).device)
def train(self):
self.bnn.train(self.X,
(self.y - self.y.mean(dim=0)) / self.y.std(dim=0))
for nn in self.bnn.nns:
for p in nn.parameters():
p.requires_grad = False
# setup training set
trainset = HLWDataset('hlw/split/train.txt', opt.imagesize, training=True)
trainset_loader = torch.utils.data.DataLoader(trainset, shuffle=True, num_workers=6, batch_size=opt.batchsize)
# setup ng dsac estimator
loss = Loss(opt.imagesize)
ngdsac = NGDSAC(opt.hypotheses, opt.inlierthreshold, opt.inlierbeta, opt.inlieralpha, loss, opt.invalidloss)
# setup network
nn = Model(opt.capacity)
nn.train()
nn = nn.cuda()
# optimizer and lr schedule (schedule offset handled further below)
optimizer = optim.Adam(nn.parameters(), lr=opt.learningrate)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=opt.schedulestep, gamma=0.5)
# keep track of training progress
train_log = open('log_'+opt.session+'.txt', 'w', 1)
iteration = 0
epochs = int(opt.iterations / len(trainset)) # number of epochs from number of target iterations
for epoch in range(epochs):
print('=== Epoch: ', epoch, '========================================')
for inputs, labels, xStart, xEnd, imh, idx in trainset_loader:
start_time = time.time()
def test(models, epoch, f = None):
global num_tests
num_tests += 1
class MStat:
def __init__(self, model):
model.eval()
self.model = model
self.correct = 0
class Stat:
def __init__(self, d, dnm):
self.domain = d
self.name = dnm
self.width = 0
self.max_eps = 0
self.safe = 0
self.proved = 0
self.time = 0
self.domains = [ Stat(h.parseValues(domains,d), h.catStrs(d)) for d in args.test_domain ]
model_stats = [ MStat(m) for m in models ]
num_its = 0
saved_data_target = []
for data, target in test_loader:
if num_its >= args.test_size:
break
if num_tests == 1:
saved_data_target += list(zip(list(data), list(target)))
num_its += data.size()[0]
if h.use_cuda:
data, target = data.cuda(), target.cuda()
for m in model_stats:
with torch.no_grad():
pred = m.model(data).data.max(1, keepdim=True)[1] # get the index of the max log-probability
m.correct += pred.eq(target.data.view_as(pred)).sum()
for stat in m.domains:
timer = Timer(shouldPrint = False)
with timer:
def calcData(data, target):
box = stat.domain.box(data, m.model.w, model=m.model, untargeted = True, target=target)
with torch.no_grad():
bs = m.model(box)
org = m.model(data).max(1,keepdim=True)[1]
stat.width += bs.diameter().sum().item() # sum up batch loss
stat.proved += bs.isSafe(org).sum().item()
stat.safe += bs.isSafe(target).sum().item()
stat.max_eps += 0 # TODO: calculate max_eps
if m.model.net.neuronCount() < 5000 or stat.domain in SYMETRIC_DOMAINS:
calcData(data, target)
else:
for d,t in zip(data, target):
calcData(d.unsqueeze(0),t.unsqueeze(0))
stat.time += timer.getUnitTime()
l = num_its # len(test_loader.dataset)
for m in model_stats:
pr_corr = float(m.correct) / float(l)
if args.use_schedule:
m.model.lrschedule.step(1 - pr_corr)
h.printBoth(('Test: {:12} trained with {:'+ str(largest_domain) +'} - Avg sec/ex {:1.12f}, Accuracy: {}/{} ({:3.1f}%)').format(
m.model.name, m.model.ty.name,
m.model.speed,
m.correct, l, 100. * pr_corr), f = f)
model_stat_rec = ""
for stat in m.domains:
pr_safe = stat.safe / l
pr_proved = stat.proved / l
pr_corr_given_proved = pr_safe / pr_proved if pr_proved > 0 else 0.0
h.printBoth(("\t{:" + str(largest_test_domain)+"} - Width: {:<36.16f} Pr[Proved]={:<1.3f} Pr[Corr and Proved]={:<1.3f} Pr[Corr|Proved]={:<1.3f} AvgMaxEps: {:1.10f} Time = {:<7.5f}").format(
stat.name,
stat.width / l,
pr_proved,
pr_safe, pr_corr_given_proved,
stat.max_eps / l,
stat.time), f = f)
model_stat_rec += "{}_{:1.3f}_{:1.3f}_{:1.3f}__".format(stat.name, pr_proved, pr_safe, pr_corr_given_proved)
prepedname = m.model.ty.name.replace(" ", "_").replace(",", "").replace("(", "_").replace(")", "_").replace("=", "_")
net_file = os.path.join(out_dir, m.model.name +"__" +prepedname + "_checkpoint_"+str(epoch)+"_with_{:1.3f}".format(pr_corr))
h.printBoth("\tSaving netfile: {}\n".format(net_file + ".net"), f = f)
if num_tests % args.save_freq == 1 or args.save_freq == 1 and not args.dont_write:
torch.save(m.model.net, net_file + ".pynet")
with h.mopen(args.dont_write, net_file + ".net", "w") as f2:
m.model.net.printNet(f2)
f2.close()
if args.onyx:
nn = copy.deepcopy(m.model.net)
nn.remove_norm()
torch.onnx.export(nn, h.zeros([1] + list(input_dims)), net_file + ".onyx",
verbose=False, input_names=["actual_input"] + ["param"+str(i) for i in range(len(list(nn.parameters())))], output_names=["output"])
if num_tests == 1 and not args.dont_write:
img_dir = os.path.join(out_dir, "images")
if not os.path.exists(img_dir):
os.makedirs(img_dir)
for img_num,(img,target) in zip(range(args.number_save_images), saved_data_target[:args.number_save_images]):
sz = ""
for s in img.size():
sz += str(s) + "x"
sz = sz[:-1]
img_file = os.path.join(img_dir, args.dataset + "_" + sz + "_"+ str(img_num))
if img_num == 0:
print("Saving image to: ", img_file + ".img")
with open(img_file + ".img", "w") as imgfile:
flatimg = img.view(h.product(img.size()))
for t in flatimg.cpu():
print(decimal.Decimal(float(t)).__format__("f"), file=imgfile)
with open(img_file + ".class" , "w") as imgfile:
print(int(target.item()), file=imgfile)
x = torch.randn(1000, 30) y = torch.randn(1000, 10) class Network(nn.Module): def __init__(self): super().__init__() self.w1 = nn.Linear(30, 50) self.w2 = nn.Linear(50, 10) def forward(self, x): h = torch.relu(self.w1(x)) o = self.w2(h) return o nn = Network() op = torch.optim.Adam(nn.parameters(), lr=0.05) t1 = time.time() for i in range(2000): o = nn(x) l = torch.mean((o - y) ** 2) print(i, float(l)) l.backward() op.step() t2 = time.time() print(t2 - t1)
def main(tr_conf):
import torch.nn as nn
seed = 12345
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
tr = transforms.Compose([transforms.ToTensor()])
train_data = MovingObjects("train", tr, seed)
train_loader = DataLoader(train_data,
num_workers=tr_conf['b_s'],
batch_size=tr_conf['b_s'],
shuffle=True,
pin_memory=True)
param_list = []
in_chs = tr_conf['input_channels']
flow_l3 = nn.DataParallel(
_Glow(in_channels=4 * in_chs, mid_channels=512,
num_steps=24)).to(device)
flow_l2 = nn.DataParallel(
_Glow(in_channels=8 * in_chs, mid_channels=512,
num_steps=24)).to(device)
flow_l1 = nn.DataParallel(
_Glow(in_channels=16 * in_chs, mid_channels=512,
num_steps=24)).to(device)
nntheta3 = nn.DataParallel(
NNTheta(encoder_ch_in=4 * in_chs,
encoder_mode=tr_conf['encoder_mode'],
h_ch_in=2 * in_chs,
num_blocks=tr_conf['enc_depth'])).to(device) # z1:2x32x32
nntheta2 = nn.DataParallel(
NNTheta(encoder_ch_in=8 * in_chs,
encoder_mode=tr_conf['encoder_mode'],
h_ch_in=4 * in_chs,
num_blocks=tr_conf['enc_depth'])).to(device) # z2:4x16x16
nntheta1 = nn.DataParallel(
NNTheta(encoder_ch_in=16 * in_chs,
encoder_mode=tr_conf['encoder_mode'],
num_blocks=tr_conf['enc_depth'])).to(device)
model_path = '/b_test/azimi/results/VideoFlow/SMovement/exp12_2/sacred/snapshots/55.pth'
if tr_conf['resume']:
print('model loading ...')
flow_l3.load_state_dict(torch.load(model_path)['glow_l3'])
flow_l2.load_state_dict(torch.load(model_path)['glow_l2'])
flow_l1.load_state_dict(torch.load(model_path)['glow_l1'])
nntheta3.load_state_dict(torch.load(model_path)['nn_theta_l3'])
nntheta2.load_state_dict(torch.load(model_path)['nn_theta_l2'])
nntheta1.load_state_dict(torch.load(model_path)['nn_theta_l1'])
print("****LOAD THE OPTIMIZER")
glow = Glow(l3=flow_l3, l2=flow_l2, l1=flow_l1)
nn_theta = NN_Theta(l3=nntheta3, l2=nntheta2, l1=nntheta1)
for f_level in glow:
param_list += list(f_level.parameters())
for nn in nn_theta:
param_list += list(nn.parameters())
loss_fn = NLLLossVF()
optimizer = torch.optim.Adam(param_list, lr=tr_conf['lr'])
optimizer.load_state_dict(torch.load(model_path)['optimizer'])
optimizer.zero_grad()
# scheduler_step = sched.StepLR(optimizer, step_size=1, gamma=0.99)
# linear_decay = sched.LambdaLR(optimizer, lambda s: 1. - s / 150000. )
# linear_decay.step(global_step)
# scheduler = sched.LambdaLR(optimizer, lambda s: min(1., s / 10000))
# optimizer.load_state_dict(torch.load(model_path)['optimizer'])
for epoch in range(tr_conf['starting_epoch'], tr_conf['n_epoch']):
print("the learning rate for epoch {} is {}".format(
epoch, get_lr(optimizer)))
train_smovement(train_loader, glow, nn_theta, loss_fn, optimizer, None,
epoch)
def train():
if FLAGS.dnn_hidden_units:
dnn_hidden_units = FLAGS.dnn_hidden_units.split(",")
dnn_hidden_units = [
int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units
]
else:
dnn_hidden_units = []
# Get negative slope parameter for LeakyReLU
neg_slope = FLAGS.neg_slope
# use GPU if available
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("Device :", device)
# load data and sample the first batch
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir)
x_np, y_np = cifar10['train'].next_batch(FLAGS.batch_size)
x_np = x_np.reshape(FLAGS.batch_size, -1) # batchsize * pixels per image
# initialize MLP
nn = MLP(x_np.shape[1], dnn_hidden_units, 10, neg_slope).to(device)
crossEntropy = torch.nn.CrossEntropyLoss()
# initialize optimizer
optimizer = torch.optim.SGD(nn.parameters(), lr=FLAGS.learning_rate)
# initialization for plotting and metrics
test_accuracies = []
training_losses = []
training_accuracies = []
test_losses = []
# extract test data
x_test, y_test_np = cifar10['test'].images, cifar10['test'].labels
x_test = x_test.reshape(x_test.shape[0], -1)
x_test = torch.from_numpy(x_test).to(device)
y_test = torch.from_numpy(y_test_np).to(device)
# perform the forward step, backward step and updating of weights max_steps number of times,
for step in range(FLAGS.max_steps):
if (step + 1) % 100 == 0:
print(step + 1, "/", FLAGS.max_steps, "\n")
optimizer.zero_grad()
x = (torch.autograd.Variable(torch.from_numpy(x_np),
requires_grad=1)).to(device)
y = (torch.autograd.Variable(torch.from_numpy(y_np),
requires_grad=1)).to(device)
pred = nn(x).to(device)
train_acc = accuracy(pred, y)
# compute cross entropy loss
labels = torch.max(y, 1)[1]
loss = crossEntropy(pred, labels)
training_accuracies.append(train_acc)
training_losses.append(loss)
# evaluation on test set
if step % FLAGS.eval_freq == 0:
test_accuracies, test_losses = eval_on_test(
nn, crossEntropy, x_test, y_test, test_accuracies, test_losses)
# get a next batch
x_np, y_np = cifar10['train'].next_batch(FLAGS.batch_size)
x_np = x_np.reshape(FLAGS.batch_size,
-1) # batchsize * pixels per image
loss.backward()
optimizer.step()
# compute loss and accuracy on the test set a final time
test_accuracies, test_losses = eval_on_test(nn, crossEntropy, x_test,
y_test, test_accuracies,
test_losses)
print("Maximum accuracy :", max(test_accuracies),
"after %d steps\n" % (np.argmax(test_accuracies) * FLAGS.eval_freq))