loss.backward()
torch.nn.utils.clip_grad_norm_(net.parameters(), 3)
optimizer.step()
with torch.no_grad():
z = (z[:, 1, :, :] > z[:, 0, :, :]).float()
for j in range(batchsize):
cm = torch.zeros(2, 2).cuda()
for a, b in [(0, 0), (0, 1), (1, 0), (1, 1)]:
cm[a][b] = torch.sum(
(z[j] == a).float() * (y[j] == b).float() * D[j])
stats[batchchoise[j]] += cm
if i % 100 == 99 or i < 10:
print(i, "/50000", printloss / 100)
printloss = torch.zeros(1).cuda()
if i % 1000 == 999:
torch.save(net, "build/model.pth")
cm = torch.sum(stats, dim=0)
perf = dataloader.perf(cm)
print("perf", perf)
if perf[0] > 92:
print("training stops after reaching high training accuracy")
os._exit(0)
else:
stats = torch.zeros((len(miniworld.cities), 2, 2)).cuda()
print("training stops after reaching time limit")
os._exit(0)
z = largeforward(net, x.unsqueeze(0))
z = globalresize(z)
z = (z[0, 1, :, :] > z[0, 0, :, :]).float()
for a, b in [(0, 0), (0, 1), (1, 0), (1, 1)]:
cm[k][a][b] += torch.sum(
(z == a).float() * (y == b).float() * D)
if False:
nextI = len(os.listdir("build"))
debug = dataloader.torchTOpil(globalresize(x))
debug = PIL.Image.fromarray(numpy.uint8(debug))
debug.save("build/" + str(nextI) + "_x.png")
debug = (2.0 * y - 1) * D * 127 + 127
debug = debug.cpu().numpy()
debug = PIL.Image.fromarray(numpy.uint8(debug))
debug.save("build/" + str(nextI) + "_y.png")
debug = z.cpu().numpy() * 255
debug = PIL.Image.fromarray(numpy.uint8(debug))
debug.save("build/" + str(nextI) + "_z.png")
print("perf=", dataloader.perf(cm[k]))
numpy.savetxt("build/tmp.txt", dataloader.perf(cm).cpu().numpy())
print("-------- results ----------")
for k, city in enumerate(miniworld.cities):
print(city, dataloader.perf(cm[k]))
cm = torch.sum(cm, dim=0)
print("miniworld", dataloader.perf(cm))
x, y = x.cuda(), y.cuda().float()
h, w = y.shape[0], y.shape[1]
D = dataloader.distancetransform(y)
globalresize = torch.nn.AdaptiveAvgPool2d((h, w))
power2resize = torch.nn.AdaptiveAvgPool2d(
((h // 64) * 64, (w // 64) * 64))
x = power2resize(x)
z = largeforward(net, x.unsqueeze(0))
z = globalresize(z)
z = (z[0, 1, :, :] > z[0, 0, :, :]).float()
for a, b in [(0, 0), (0, 1), (1, 0), (1, 1)]:
cm[a][b] += torch.sum((z == a).float() * (y == b).float() * D)
if True:
nextI = len(os.listdir("build"))
debug = dataloader.torchTOpil(globalresize(x))
debug = PIL.Image.fromarray(numpy.uint8(debug))
debug.save("build/" + str(nextI) + "_x.png")
debug = (2.0 * y - 1) * D * 127 + 127
debug = debug.cpu().numpy()
debug = PIL.Image.fromarray(numpy.uint8(debug))
debug.save("build/" + str(nextI) + "_y.png")
debug = z.cpu().numpy() * 255
debug = PIL.Image.fromarray(numpy.uint8(debug))
debug.save("build/" + str(nextI) + "_z.png")
print("perf=", dataloader.perf(cm))
print("load data")
miniworld = dataloader.MiniWorld(flag="train")
miniworld.start()
print("train")
optimizer = torch.optim.Adam(net.parameters(), lr=0.0001)
if whereIam in ["ldtis706z", "wdtim719z"]:
batchsize = 16
else:
batchsize = 32
nbbatchs = 100000
printloss = torch.zeros(1).cuda()
stats = torch.zeros((len(miniworld.cities), 2, 2)).cuda()
for i in range(nbbatchs):
if i % 500 > 70:
perf = dataloader.perf(stats)
perf = [(perf[j][0], j) for j in range(perf.shape[0])]
perf = sorted(perf)
priority = numpy.ones(len(perf))
for j in range(stats.shape[0] // 2):
priority[perf[j][1]] = 100
x, y, batchchoise = miniworld.getbatch(batchsize, priority=priority)
else:
x, y, batchchoise = miniworld.getbatch(batchsize)
x, y = x.cuda(), y.cuda()
z = net(x)
D = dataloader.distancetransform(y)
cm[a][b] = torch.sum(
(z[j] == a).float() * (y[j] == b).float() * D[j])
stats[batchchoise[j]] += cm
if i < 10:
print(i, "/", nbbatchs, printloss)
if i < 1000 and i % 100 == 99:
print(i, "/", nbbatchs, printloss / 100)
printloss = torch.zeros(1).cuda()
if i >= 1000 and i % 300 == 299:
print(i, "/", nbbatchs, printloss / 300)
printloss = torch.zeros(1).cuda()
if i % 1000 == 999:
torch.save(net, "build/model.pth")
perf = dataloader.perf(torch.sum(stats, dim=0))
print(i, "perf", perf)
if perf[0] > 92:
print("training stops after reaching high training accuracy")
os._exit(0)
else:
stats = torch.zeros((len(miniworld.cities), 2, 2)).cuda()
if i > nbbatchs * 0.1:
loss = loss * 0.5
if i > nbbatchs * 0.2:
loss = loss * 0.5
if i > nbbatchs * 0.5:
loss = loss * 0.5
if i > nbbatchs * 0.8:
loss = loss * 0.5
z = (z[:, 1, :, :] > z[:, 0, :, :]).float()
for j in range(batchsize):
cm = torch.zeros(2, 2).cuda()
for a, b in [(0, 0), (0, 1), (1, 0), (1, 1)]:
cm[a][b] = torch.sum(
(z[j] == a).float() * (y[j] == b).float() * D[j])
stats[batchchoise[j]] += cm
if i % 100 == 99:
print(i, "/200000", printloss / 100)
printloss = torch.zeros(1).cuda()
if i % 1000 == 999:
torch.save(net, "build/model.pth")
cm = torch.sum(stats, dim=0)
perf = dataloader.perf(cm)
print("perf", perf)
if perf[0] > 92:
print("training stops after reaching high training accuracy")
os._exit(0)
else:
if len(sys.argv) == 2 and sys.argv[1] == "penalizemin":
perfs = [
dataloader.perf(stats[j])[0] for j in range(stats.shape[0])
]
tmp = perfs[:]
sorted(tmp)
threshold = tmp[stats.shape[0] // 3 + 1]
worse = set([
j for j in range(stats.shape[0]) if perfs[j] <= threshold
])