def neural_queue(element_type, queue_type):
enqueue_img = EnqueueNet(queue_type, element_type, arch=MLPNet)
dequeue_img = DequeueNet(queue_type, element_type, MLPNet)
empty_queue = ConstantNet(queue_type)
neural_ref = ModuleDict({"enqueue": enqueue_img,
"dequeue": dequeue_img,
"empty": empty_queue})
cuda(neural_ref)
return neural_ref
def neural_queue(element_type, queue_type):
enqueue_img = EnqueueNet(queue_type, element_type)
dequeue_img = DequeueNet(queue_type, element_type)
empty_queue = ConstantNet(queue_type)
neural_ref = ModuleDict({
"enqueue": enqueue_img,
"dequeue": dequeue_img,
"empty": empty_queue
})
cuda(neural_ref)
return neural_ref
def test_reverse_sketch():
"Test Reverse Sketch"
batch_size = 128
tl = mnistloader(batch_size)
items_iter = iter(tl)
matrix_stack = Type("Stack", (1, 28, 28), dtype="float32")
mnist_type = Type("mnist_type", (1, 28, 28), dtype="float32")
nstack = neural_stack(mnist_type, matrix_stack)
refstack = ref_stack()
rev_sketch = ReverseSketch(Type, nstack, refstack)
rev_items_iter = iter(tl)
cuda(rev_sketch)
optimizer = optim.Adam(rev_sketch.parameters(), lr=0.0001)
def plot_items(i, log, writer, **kwargs):
writer.add_image('fwd/1', log['forward'][0][0][0], i)
writer.add_image('fwd/2', log['forward'][0][1][0], i)
writer.add_image('fwd/3', log['forward'][0][2][0], i)
writer.add_image('rev/1', log['reverse'][0][0][0], i)
writer.add_image('rev/2', log['reverse'][0][1][0], i)
writer.add_image('rev/3', log['reverse'][0][2][0], i)
writer.add_image('out/1', log['out'][0][0][0], i)
writer.add_image('out/2', log['out'][0][1][0], i)
writer.add_image('out/3', log['out'][0][2][0], i)
def loss_gen():
nonlocal items_iter, rev_items_iter
# Refresh hte iterators if they run out
try:
items = iterget(items_iter, 3, transform=train_data)
rev_items = iterget(rev_items_iter, 3, transform=train_data)
except StopIteration:
print("End of Epoch")
items_iter = iter(tl)
rev_items_iter = iter(tl)
items = iterget(items_iter, 3, transform=train_data)
rev_items = iterget(rev_items_iter, 3, transform=train_data)
out_items = rev_sketch(items)
rev_items.reverse()
log_append("forward", items)
log_append("reverse", rev_items)
log_append("out", out_items)
losses = [nn.MSELoss()(out_items[i], rev_items[i]) for i in range(3)]
loss = sum(losses)
return loss
train(loss_gen, optimizer, [every_n(plot_items, 100)])
def benchmark_copy_sketch(batch_size, stack_len, seq_len, arch, log_dir, lr,
arch_opt, **kwargs):
stack_len = stack_len
seq_len = seq_len # From paper: between 1 and 20
BernSeq = bern_seq(seq_len)
MatrixStack = matrix_stack(1, seq_len, seq_len)
arch_opt['combine_inputs'] = lambda xs: stretch_cat(
xs, MatrixStack.size, 2)
arch_opt['activation'] = F.sigmoid
nstack = ModuleDict({
'push':
PushNet(MatrixStack, BernSeq, arch=ConvNet, arch_opt=arch_opt),
'pop':
PopNet(MatrixStack, BernSeq, arch=ConvNet, arch_opt=arch_opt),
'empty':
ConstantNet(MatrixStack)
})
refstack = ref_stack()
copy_sketch = CopySketch(BernSeq, nstack, refstack, seq_len)
cuda(copy_sketch)
bern_iter = BernSeq.iter(batch_size)
def loss_gen():
# Should copy the sequence, therefore the output should
items = take(bern_iter, seq_len)
rev_items = items.copy()
rev_items.reverse()
outputs = copy_sketch(items)
log("outputs", outputs)
log("items", items)
log("rev_items", rev_items)
# import pdb; pdb.set_trace()
return vec_dist(outputs, rev_items, dist=nn.BCELoss())
optimizer = optim.Adam(copy_sketch.parameters(), lr)
train(
loss_gen,
optimizer,
cont=converged(1000),
callbacks=[
print_loss(100),
every_n(plot_sketch, 500),
# common.plot_empty,
# common.plot_observes,
save_checkpoint(1000, copy_sketch)
],
log_dir=log_dir)
def onehot(i, onehot_len, batch_size):
# Dummy input that HAS to be 2D for the scatter (you can use view(-1,1) if needed)
y = torch.LongTensor(batch_size, 1).fill_(i)
# One hot encoding buffer that you create out of the loop and just keep reusing
y_onehot = torch.FloatTensor(batch_size, onehot_len)
# In your for loop
y_onehot.zero_()
return Variable(cuda(y_onehot.scatter_(1, y, 1)), requires_grad=False)
def benchmark_copy_sketch():
opt = handle_args()
string_len = 8
class SeqStack(Type):
size = (string_len, 1)
class BernSeq(Type):
size = (string_len, 1)
def bern_eq(*shape):
return cuda(torch.bernoulli(torch.ones(*shape).fill_(0.5)))
seq_sampler = infinite_samples(bern_eq, opt.batch_size, (string_len, 1),
True)
nqueue = neural_queue(SeqStack, BernSeq)
refqueue = ref_queue()
seq_len = 3 # From paper: between 1 and 20
copy_sketch = CopySketch(Type, nqueue, refqueue, seq_len)
cuda(copy_sketch)
def loss_gen():
items = iterget(seq_sampler, seq_len)
target_items = copy(items)
outputs = copy_sketch(items)
log_append("outputs", outputs)
log_append("items", items)
import pdb
pdb.set_trace()
losses = [
nn.BCELoss()(outputs[i], target_items[i]) for i in range(seq_len)
]
loss = sum(losses)
print("LOSS", loss)
return loss
every = 100
print_loss_gen = print_loss(every)
next(print_loss_gen)
optimizer = optim.Adam(copy_sketch.parameters(), opt.lr)
print(opt)
train(loss_gen,
optimizer,
cont=partial(max_iters, maxiters=1000000),
callbacks=[every_n(plot_items, 100), print_loss_gen])
def benchmark_copy_sketch():
opt = handle_args()
string_len = 8
class SeqStack(Type):
size = (string_len, 1)
class BernSeq(Type):
size = (string_len, 1)
def bern_eq(*shape):
return cuda(torch.bernoulli(torch.ones(*shape).fill_(0.5)))
seq_sampler = infinite_samples(bern_eq, opt.batch_size, (string_len, 1), True)
nqueue = neural_queue(SeqStack, BernSeq)
refqueue = ref_queue()
seq_len = 3 # From paper: between 1 and 20
copy_sketch = CopySketch(Type, nqueue, refqueue, seq_len)
cuda(copy_sketch)
def loss_gen():
items = iterget(seq_sampler, seq_len)
target_items = copy(items)
outputs = copy_sketch(items)
log_append("outputs", outputs)
log_append("items", items)
import pdb; pdb.set_trace()
losses = [nn.BCELoss()(outputs[i], target_items[i]) for i in range(seq_len)]
loss = sum(losses)
print("LOSS", loss)
return loss
every = 100
print_loss_gen = print_loss(every)
next(print_loss_gen)
optimizer = optim.Adam(copy_sketch.parameters(), opt.lr)
print(opt)
train(loss_gen, optimizer, cont=partial(max_iters, maxiters=1000000),
callbacks=[every_n(plot_items, 100), print_loss_gen])
def onehot1d(enum, length=None): "Encode an Enum as a one hot vector" EnumOneHot1D = compound_encoding(enum.__class__.__bases__[0], OneHot1D) length = EnumOneHot1D.typesize[0] if length is None else length return EnumOneHot1D(Variable(cuda(onehot(enum.value, length, 1))))
def sample(*shape): return Variable(cuda(torch.bernoulli(torch.ones(*shape).fill_(0.5))))
def choice1f(self, outlen): proj = Variable(cuda(torch.rand(self.choice_len, outlen))) return torch.matmul(self.choice1, proj)
def bern_eq(*shape): return cuda(torch.bernoulli(torch.ones(*shape).fill_(0.5)))
def choice1f(self, outlen):
proj = Variable(cuda(torch.rand(self.choice_len, outlen)))
return torch.matmul(self.choice1, proj)
def bern_eq(*shape):
return cuda(torch.bernoulli(torch.ones(*shape).fill_(0.5)))
def onehot1d(enum, length=None):
"Encode an Enum as a one hot vector"
EnumOneHot1D = compound_encoding(enum.__class__.__bases__[0], OneHot1D)
length = EnumOneHot1D.typesize[0] if length is None else length
return EnumOneHot1D(Variable(cuda(onehot(enum.value, length, 1))))