def noise_vs_val_acc(model, x, y, vx, vy):
with tf.Session() as sess:
conv = Conv(model=model)
for update, use_dropout in [('adam', False), ('langevin', False),
('adam', True)]:
title = str(update)
if use_dropout:
title += '+dropout'
print('\n\nTraining with optimizer update: ' + title)
for p in [0, 0.3, 0.5, 0.8, 0.9, 1.0]:
print('\n\nTraining with label noise p =', p, '\n')
noisy_y, frac_correct = random_label_flip(y, p=p)
print('Fraction of labels correct:', frac_correct, '\n\n')
val_t = conv.train(sess,
x,
noisy_y,
vx,
vy,
reset=True,
update=update,
use_dropout=use_dropout)
itr_t, v_t = map(list, zip(*val_t))
plt.plot(itr_t, v_t, label='p = ' + str(p))
plt.title('Update: ' + title)
plt.ylabel('Val acc.')
plt.ylim((0, 1))
plt.xlabel('Iterations')
lgd = plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
plt.savefig(osp.join(output_dir,
'noise_vs_val_acc_' + title + '.png'),
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
plt.clf()
def __init__(self, input_dim, embed_dim, conv_layers, num_labels,
batch_size):
"""
Linear chain CRF as in Assignment 2
"""
super(CRF, self).__init__()
self.input_dim = input_dim
self.embed_dim = embed_dim
self.conv_layers = conv_layers
self.num_labels = num_labels
self.batch_size = batch_size
self.use_cuda = torch.cuda.is_available()
"""
Initialize trainable parameters of CRF here
"""
self.conv_layer1 = Conv(5)
self.conv_layer2 = Conv(3)
self.params = nn.Parameter(
torch.zeros(num_labels * embed_dim + num_labels**2))
# self.w = params.narrow(0,0,num_labels * embed_dim).view(num_labels, embed_dim)
# self.T = params.narrow(0,num_labels * embed_dim, num_labels**2).view(num_labels,num_labels)
### Use GPU if available
if self.use_cuda:
[m.cuda() for m in self.modules()]
def __init__(self, num_speakers):
super(Discriminator, self).__init__()
self.discriminator_conv1 = Conv(64, 64, kernel_size=4, stride=2)
self.discriminator_conv2 = Conv(64, 64, kernel_size=4, stride=2)
self.discriminator_conv3 = Conv(64, 64, kernel_size=2)
self.fc1 = torch.nn.Linear(64 * 20, 64)
self.fc2 = torch.nn.Linear(64, num_speakers)
self.discriminator_softmax = torch.nn.LogSoftmax(dim=1)
def main():
"""
main func
"""
text, x, y, char2idx, idx2char = getty()
T = 100
config = {
'dim_hidden': 300,
'l': T,
'clip': 5,
'mu': 0.9,
'step_size': 0.001
}
#np.random.seed(42)
r = RNN(config)
r.accept([27])
ttb = r.sample('f', char2idx, idx2char)
r.fit(x[:T], y[:T], 100, char2idx, idx2char)
tta = r.sample('f', char2idx, idx2char)
print(ttb)
print(tta)
print(text[:T])
return
(data, label) = cifar()
N = 10000
data = np.array(data, dtype=float)[:N, ]
label = np.array(label)[:N, ]
data = normalize(data)
config = {
'input_shape': [3, 32, 32],
'mu': 0.9,
'step_size': 0.000001,
'step_decay': 0.95
}
nn = Net(config)
conv1 = Conv([3, 3], 6)
relu1 = Relu()
conv2 = Conv([3, 3], 32)
relu2 = Relu()
pool = MaxPool()
fc = FC([10])
nn.add(conv1)
nn.add(relu1)
nn.add(pool)
nn.add(fc)
print(nn)
nn.fit(data, label, 200)
def one_vs_all_exp(model, x, y, vx, vy):
with tf.Session() as sess:
conv = Conv(model=model)
classlist = list(set(y))
nclass = len(classlist)
sc_list = [4] #[0,1,2,3,4]
# single out a class in train and val sets: make labels in {0,1}
gc_list = list(set(classlist) - set(sc_list))
gy = y.copy()
for sc in sc_list:
gy[y == sc] = 0
for gc in gc_list:
gy[y == gc] = 1
gvy = vy.copy()
for sc in sc_list:
gvy[vy == sc] = 0
for gc in gc_list:
gvy[vy == gc] = 1
for update, use_dropout in [('adam', False), ('langevin', False),
('adam', True)]:
title = str(update)
if use_dropout:
title += '+dropout'
print('\n\nTraining with optimizer update: ' + title)
for p in [
0, 0.2, 0.3, 0.4, 0.45, 0.5, 0.55, 0.6, 0.7, 0.8, 0.9, 1.0
]:
print('\n\nTraining with label noise p =', p, '\n')
# generate noise transition matrix
noisy_y, frac_correct = random_label_flip(gy, p=p)
print('Fraction of labels correct:', frac_correct, '\n\n')
val_t = conv.train(sess,
x,
noisy_y,
vx,
gvy,
reset=True,
update=update,
use_dropout=use_dropout)
itr_t, v_t = map(list, zip(*val_t))
plt.plot(itr_t, v_t, label='p = ' + str(p))
plt.title('Update: ' + title)
plt.ylabel('Val acc.')
plt.ylim((0, 1))
plt.xlabel('Iterations')
lgd = plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
plt.savefig(osp.join(output_dir,
'one_vs_all_val_acc_' + title + '.png'),
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
plt.clf()
def init_params(self):
"""
Initialize trainable parameters of CRF here
"""
self.conv = Conv(self.conv_layers[0][-1],
self.out_channels,
padding=self.padding,
stride=self.stride)
self.W = torch.randn(self.num_labels,
self.cout_numel,
requires_grad=True)
self.T = torch.randn(self.num_labels,
self.num_labels,
requires_grad=True)
def test(self):
#output from custom implementation
X = torch.Tensor([[1, 1, 1, 0, 0], [0, 1, 1, 1, 0], [0, 0, 1, 1, 1],
[0, 0, 1, 1, 0], [0, 1, 1, 0, 0]])
X = X.reshape(
(1, 1, 5, 5)) # (batch_size x channel_size x height x weight)
conv = Conv()
K = torch.Tensor([[[[1, 0, 1], [0, 1, 0], [1, 0, 1]]]])
conv.init_params(K, 3, stride=1, padding=1)
output_custom = conv.forward(X)
print(output_custom)
#ouput from pytorch's implementation
output_pytorch = F.conv2d(X, K, padding=1, stride=1)
print(output_pytorch)
self.assertEqual(output_custom, output_pytorch)
def test_backward(self):
l = Conv([2, 2], 2)
l.accept([1, 3, 3])
l.w = np.array([[1.0, 0], [0, 0], [0, 0], [0, 1]])
l.b = np.array([[0.0, 0.1]])
l.fx = np.array([[[1, 2, 3, 4], [11, 12, 13, 14], [21, 22, 23, 24],
[31, 32, 33, 34]]])
dy = np.array([
[1, 2, 3, 4, -4, -3, -2, -1],
])
dx = l.backward(dy)
dx_true = np.array([[1, 2, 0, 3, 0, -3, 0, -2, -1]])
w_true = np.array([[210, -110], [220, -120], [230, -130], [240, -140]])
b_true = np.array([10, -10])
self.assertTrue(np.allclose(dx, dx_true))
self.assertTrue(np.allclose(l.dw, w_true))
self.assertTrue(np.allclose(l.db, b_true))
def test_init(self):
l = Conv([2, 2], 2)
self.assertEqual(l.hk, 2)
self.assertEqual(l.wk, 2)
self.assertEqual(l.dout, 2)
self.assertEqual(l.pad[0], 0)
self.assertEqual(l.pad[1], 0)
self.assertEqual(l.stride[0], 1)
self.assertEqual(l.stride[1], 1)
self.assertEqual(l.type, 'Convolution')
def best_sgld_var_exp(model, x, y, vx, vy):
with tf.Session() as sess:
conv = Conv(model=model)
best_sig_vs_p = []
for p in np.arange(0, 1, 0.1):
best_sig = 1e-4
best_val = -1
for ep in [-4, -3, -2, -1]:
sig = 10**ep
print('\n\np =', p, 'sig =', sig, '\n')
noisy_y, frac_correct = random_label_flip(y, p=p)
val_t = conv.train(sess,
x,
noisy_y,
vx,
vy,
reset=True,
update='langevin',
use_dropout=False,
hparams={
'eps_t': sig,
'niter': 4000
})
vacc = np.max([v[1] for v in val_t])
if vacc > best_val:
best_sig = sig
best_val = vacc
best_sig_vs_p.append((p, best_sig, best_val))
# display table
print('p\tbest_sig\tbest_val')
for p, best_sig, best_val in best_sig_vs_p:
print(p, '\t', best_sig, '\t', best_val)
# plt variation
p, best_sig, best_val = zip(*best_sig_vs_p)
plt.plot(p, best_sig)
plt.ylabel('Best sigma')
plt.xlabel('Label noise level (p)')
plt.savefig(osp.join(output_dir, 'sigma_vs_p.png'),
bbox_inches='tight')
plt.clf()
def __init__(self, en_residual_channels, en_dilation_channel, n_layers,
max_dilation):
super(ResidualBlock, self).__init__()
self.n_layers = n_layers
self.en_dilate_layers = torch.nn.ModuleList()
self.en_residual_layers = torch.nn.ModuleList()
loop_factor = math.floor(math.log2(max_dilation)) + 1
for i in range(self.n_layers):
dilation = 2**(i % loop_factor)
self.en_dilate_layers.append(
Conv(en_residual_channels,
en_dilation_channel,
kernel_size=2,
dilation=dilation,
w_init_gain='tanh',
is_causal=True))
self.en_residual_layers.append(
Conv(en_dilation_channel, en_residual_channels, kernel_size=1))
def build_model(self):
self.conv_layers = []
self.linear_layers = []
self.layers = []
# 1x28x28 -> 6x24x24
self.conv_layers += [Conv(1, 6, 5, self.activation)]
# 6x24x24 -> 6x12x12
self.conv_layers += [MaxPool_2()]
# 6x12x12 -> 16x8x8
self.conv_layers += [Conv(6, 16, 5, self.activation)]
# 16x8x8 -> 16x4x4
self.conv_layers += [MaxPool_2()]
# 256 -> 120
self.linear_layers += [Linear(16 * 4 * 4, 120, self.activation)]
# 120 -> 84
self.linear_layers += [Linear(120, 84, self.activation)]
# 84 -> 10
self.linear_layers += [Softmax(84, self.no_of_classes)]
self.layers = self.conv_layers + self.linear_layers
def test_forward(self):
l = Conv([2, 2], 2)
l.accept([1, 3, 3])
l.w = np.array([[1, 0], [0, 0], [0, 0], [0, 1]])
l.b = np.array([0.0, 0.1])
x = np.array([[[[1, 2, 3], [4, 5, 6], [7, 8, 9]]],
[[[-1, -2, -3], [-4, -5, -6], [-7, -8, -9]]]])
y = l.forward(x)
y_true = np.array([[1, 2, 4, 5, 5.1, 6.1, 8.1, 9.1],
[-1, -2, -4, -5, -4.9, -5.9, -7.9, -8.9]])
self.assertTrue(np.allclose(y, y_true))
def __init__(self):
super(Encoder, self).__init__()
n_in_channels = 256
n_residual_channels = 64
max_dilation = 128
n_layers = 16
self.embed = torch.nn.Embedding(n_in_channels, n_residual_channels)
self.en_residual = ResidualBlock(n_residual_channels,
n_residual_channels, n_layers,
max_dilation)
self.conv1x1 = Conv(n_residual_channels,
n_residual_channels,
kernel_size=1,
w_init_gain='relu')
self.avg_pooling_layer = torch.nn.AvgPool1d(kernel_size=800,
stride=200)
def __init__(self, n_speakers, n_in_channels, n_layers, max_dilation,
n_residual_channels, n_skip_channels, n_out_channels,
n_cond_channels, upsamp_window, upsamp_stride):
super(Decoder, self).__init__()
self.upsample = torch.nn.ConvTranspose1d(n_cond_channels,
n_cond_channels,
upsamp_window, upsamp_stride)
self.n_layers = n_layers
self.max_dilation = max_dilation
self.n_residual_channels = n_residual_channels
self.n_out_channels = n_out_channels
self.cond_layers = Conv(n_cond_channels,
2 * n_residual_channels * n_layers,
w_init_gain='tanh')
self.dilate_layers = torch.nn.ModuleList()
self.res_layers = torch.nn.ModuleList()
self.skip_layers = torch.nn.ModuleList()
self.embed = torch.nn.Embedding(n_in_channels, n_residual_channels)
self.conv_out = Conv(n_skip_channels,
n_out_channels,
bias=False,
w_init_gain='relu')
self.conv_end = Conv(n_out_channels,
n_out_channels,
bias=False,
w_init_gain='linear')
loop_factor = math.floor(math.log2(max_dilation)) + 1
for i in range(n_layers):
dilation = 2**(i % loop_factor)
# Kernel size is 2 in nv-wavenet
in_layer = Conv(n_residual_channels,
2 * n_residual_channels,
kernel_size=2,
dilation=dilation,
w_init_gain='tanh',
is_causal=True)
self.dilate_layers.append(in_layer)
# last one is not necessary
if i < n_layers - 1:
res_layer = Conv(n_residual_channels,
n_residual_channels,
w_init_gain='linear')
self.res_layers.append(res_layer)
skip_layer = Conv(n_residual_channels,
n_skip_channels,
w_init_gain='relu')
self.skip_layers.append(skip_layer)
embed_dim = 64
num_labels = 26
cuda = torch.cuda.is_available()
print("cuda : ")
print(cuda)
# Instantiate the CRF model
crf = CRF(input_dim, embed_dim, conv_shapes, num_labels, batch_size)
# Setup the optimizer
# opt = optim.LBFGS(crf.parameters())
opt = optim.Adam(crf.parameters())
from conv import Conv
convLayer = Conv()
convLayer.init_params(kernel_size=5)
##################################################
# Begin training
##################################################
step = 0
# Fetch dataset
dataset = get_dataset()
# split = int(0.5 * len(dataset.data)) # train-test split
split = int(0.01 * len(dataset.data)) # train-test split
# train_data, test_data = dataset.data[:split], dataset.data[split:]
# train_target, test_target = dataset.target[:split], dataset.target[split:]
train_data = dataset.data[:split]
test_data = train_data
from softmax import Softmax
import cv2
'''
input_img = np.array([[0,0,0,0,0,0], [0,0,50,0,29,0],
[0,0,80,31,2,0], [0,33,90,0,75,0],
[0,0,9,0,95,0], [0,0,0,0,0,0]
])
'''
train_images = mnist.train_images()[:1000]
train_labels = mnist.train_labels()[:1000]
test_images = mnist.test_images()[:1000] # test image size: 28 x 28
test_labels = mnist.test_labels()[:1000]
conv = Conv(8) # 28 x 28 -> 26 x 26 x 8
pool = MaxPool() # 26 x 26 x 8 -> 13 x 13 x 8
softmax = Softmax(13*13*8,10) # 10 nodes for 10 digits 0 -> 9
def forward(images,labels):
# transform image from [0->255] to [-0.5->0.5]
out = conv.forward((images / 255)-0.5)
out = pool.forward(out)
out = softmax.forward(out)
loss = -np.log(out[labels])
if np.argmax(out) == labels:
acc = 1
else:
acc = 0
return out,loss,acc
def parse(l, ch_out, stride, channel_wise=True):
shape = l.get_shape().as_list()
l_ori = l
parse1 = tf.nn.sigmoid(Conv('parse1', l, 1, 1, strides=1))
l1_left = l * parse1
l = l - l1_left
parse2 = tf.nn.sigmoid(Conv('parse2', l, 1, 1, strides=1))
l2_left = l * parse2
l = l - l2_left
parse3 = tf.nn.sigmoid(Conv('parse3', l, 1, 1, strides=1))
l3_left = l * parse3
l3_right = l - l3_left
l1_left = tf.keras.backend.resize_images(
Conv2D('l1_left',
AvgPooling('pool',
l1_left,
pool_size=8,
strides=8,
padding='VALID'),
1 * ch_out // 4,
3 if shape[1] // 8 > 2 else 1,
strides=1), 8 // stride, 8 // stride, 'channels_last')
l2_left = tf.keras.backend.resize_images(
Conv2D('l2_left',
AvgPooling('pool',
l2_left,
pool_size=4,
strides=4,
padding='VALID'),
1 * ch_out // 4,
3 if shape[1] // 4 > 2 else 1,
strides=1), 4 // stride, 4 // stride, 'channels_last')
l3_left = tf.keras.backend.resize_images(
Conv2D('l3_left',
AvgPooling('pool',
l3_left,
pool_size=2,
strides=2,
padding='VALID'),
1 * ch_out // 4,
3 if shape[1] // 2 > 2 else 1,
strides=1), 2 // stride, 2 // stride, 'channels_last')
l3_right = Conv2D('l3_right',
l3_right,
1 * ch_out // 4,
3 if shape[1] > 2 else 1,
strides=stride)
l_ori = Conv2D('l_ori',
l_ori,
ch_out // 4,
3,
strides=stride,
activation=BNReLU)
l = tf.concat([
tf.nn.sigmoid(BatchNorm('bn1', l1_left)) * l_ori,
tf.nn.sigmoid(BatchNorm('bn2', l2_left)) * l_ori,
tf.nn.sigmoid(BatchNorm('bn3', l3_left)) * l_ori,
tf.nn.sigmoid(BatchNorm('bn4', l3_right)) * l_ori
], -1)
return l
import torch
import torch.nn.functional as F
from conv import Conv
def get_torch_conv(image, kernel, padding=0):
return F.conv2d(image, kernel, padding=padding)
image = torch.tensor([[1,1,1,0,0],[0,1,1,1,0],[0,0,1,1,1],[0,0,1,1,0],[0,1,1,0,0]], dtype=torch.float)
image = torch.unsqueeze(image, 0)
image = torch.unsqueeze(image, 0)
kernel = torch.tensor([[1, 0, 1],[0, 1, 0],[1, 0, 1]], dtype=torch.float)
conv = Conv(3, kernel_tensor=kernel)
print(conv(image))
# check with PyTorch implementation
kernel = torch.unsqueeze(kernel, 0)
kernel = torch.unsqueeze(kernel, 0)
print(get_torch_conv(image, kernel, 1))
def conv(in_planes: int, out_planes: int, kernel_size=1, stride: int = 1, groups: int = 1, dilation: int = 1) -> nn.Conv2d:
"""1*1 convolution without padding"""
return Conv(in_planes, out_planes, kernel_size, stride=stride, bias=False)
o_x = yield dut.o_x
o_p = yield dut.o_p
eof = yield dut.o_eof
if o_valid:
print("Output:", o_ready, eof, (o_y, o_x), o_p)
if o_ready:
i_p += 1
m = Module()
#k = Array([0,0,0,
# 0,1,0,
# 0,0,0])
k = Array([1, 2, 1, 2, 4, 2, 1, 2, 1])
#k = Array([0,0,0,0,0,
# 0,0,0,0,0,
# 0,0,1,0,0,
# 0,0,0,0,0,
# 0,0,0,0,0])
dut = Conv(k, kw=kw, sh=0, w=w, h=h)
m.submodules.dut = dut
sim = Simulator(m)
sim.add_clock(1e-6)
sim.add_sync_process(process)
with sim.write_vcd("conv.vcd", "conv.gtkw"):
sim.run()
def sgld_noise_level_vs_val_acc(model, x, y, vx, vy):
with tf.Session() as sess:
conv = Conv(model=model)
use_dropout = False
p = 0.8
noisy_y, frac_correct = random_label_flip(y, p=p)
print('\n\nUsing label noise p = ' + str(p))
print('Fraction of labels correct:', frac_correct, '\n\n')
for lr in [1e-3, 5e-4, 1e-4, 1e-3]:
title = str(lr)
for sgld_noise_level in [5e-1, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5]:
print('\n\nlr =', lr, 'SGLD noise =', sgld_noise_level, '\n')
val_t = conv.train(sess,
x,
noisy_y,
vx,
vy,
reset=True,
update='langevin',
use_dropout=False,
hparams={
'lr': lr,
'eps_t': sgld_noise_level
})
itr_t, v_t = map(list, zip(*val_t))
plt.plot(itr_t,
v_t,
label='sgld noise level: ' + str(sgld_noise_level))
# plot plain adam with this learning rate
print('\n\nlr =', lr, 'Update = Adam\n')
val_t = conv.train(sess,
x,
noisy_y,
vx,
vy,
reset=True,
update='adam',
use_dropout=False,
hparams={'lr': lr})
itr_t, v_t = map(list, zip(*val_t))
plt.plot(itr_t, v_t, linestyle='--', label='adam')
# plot adam+dropout with this learning rate
print('\n\nlr =', lr, 'Update = Adam+Dropout\n')
val_t = conv.train(sess,
x,
noisy_y,
vx,
vy,
reset=True,
update='adam',
use_dropout=True,
hparams={'lr': lr})
itr_t, v_t = map(list, zip(*val_t))
plt.plot(itr_t, v_t, linestyle=':', label='adam+dropout')
# format plot
plt.title('Learning rate: ' + title)
plt.ylabel('Val acc.')
plt.ylim((0, 1))
plt.xlabel('Iterations')
lgd = plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
plt.savefig(osp.join(
output_dir,
'sgld_noise_level_vs_val_acc_lr_' + title + '.png'),
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
plt.clf()
def elaborate(self, platform):
m = Module()
p_s = Signal(13)
p_th = Signal(6)
def connect(c1, c2, ch):
m.d.comb += [
c1.i_p.eq(ch),
c2.i_p.eq(c1.o_p),
c1.i_valid.eq(self.i_valid),
c2.i_valid.eq(c1.o_valid),
c1.i_ready.eq(c2.o_ready),
c2.i_ready.eq(self.i_ready),
c1.i_reset.eq(self.i_reset),
c2.i_reset.eq(self.i_reset)
]
def select(c_r, c_g, c_b):
m.d.comb += [self.o.r.eq(c_r), self.o.g.eq(c_g), self.o.b.eq(c_b)]
# Identity
k_ident = Array([0, 0, 0, 0, 1, 0, 0, 0, 0])
sh_ident = 0
# Guassian blur
k_blur = Array([1, 2, 1, 2, 4, 2, 1, 2, 1])
sh_blur = 4
# Box blur
k_box = Array([1, 1, 1, 1, 1, 1, 1, 1, 1])
sh_box = 3
# Emboss
k_emboss = Array([-2, -1, 0, -1, 1, 1, 0, 1, 2])
sh_emboss = 0
# Create the convolution modules
m.submodules.extend_r = extend_r = Extend(n=1,
w=self.res_x,
h=self.res_y)
m.submodules.extend_g = extend_g = Extend(n=1,
w=self.res_x,
h=self.res_y)
m.submodules.extend_b = extend_b = Extend(n=1,
w=self.res_x,
h=self.res_y)
m.submodules.ident_r = ident_r = Conv(k_ident,
w=self.res_x + 2,
h=self.res_y + 2,
dw=5,
sh=sh_ident)
m.submodules.ident_g = ident_g = Conv(k_ident,
w=self.res_x + 2,
h=self.res_y + 2,
dw=6,
sh=sh_ident)
m.submodules.ident_b = ident_b = Conv(k_ident,
w=self.res_x + 2,
h=self.res_y + 2,
dw=5,
sh=sh_ident)
connect(extend_r, ident_r, self.i.r)
connect(extend_g, ident_g, self.i.g)
connect(extend_b, ident_b, self.i.b)
m.submodules.blur_r = blur_r = Conv(k_blur,
w=self.res_x,
h=self.res_y,
dw=5,
sh=sh_blur)
m.submodules.blur_g = blur_g = Conv(k_blur,
w=self.res_x,
h=self.res_y,
dw=6,
sh=sh_blur)
m.submodules.blur_b = blur_b = Conv(k_blur,
w=self.res_x,
h=self.res_y,
dw=5,
sh=sh_blur)
connect(extend_r, blur_r, self.i.r)
connect(extend_g, blur_g, self.i.g)
connect(extend_b, blur_b, self.i.b)
m.submodules.box_r = box_r = Conv(k_box,
w=self.res_x,
h=self.res_y,
dw=5,
sh=sh_box)
m.submodules.box_g = box_g = Conv(k_box,
w=self.res_x,
h=self.res_y,
dw=6,
sh=sh_box)
m.submodules.box_b = box_b = Conv(k_box,
w=self.res_x,
h=self.res_y,
dw=5,
sh=sh_box)
connect(extend_r, box_r, self.i.r)
connect(extend_g, box_g, self.i.g)
connect(extend_b, box_b, self.i.b)
m.submodules.emboss_r = emboss_r = Conv(k_emboss,
w=self.res_x,
h=self.res_y,
dw=5,
sh=sh_emboss)
m.submodules.emboss_g = emboss_g = Conv(k_emboss,
w=self.res_x,
h=self.res_y,
dw=6,
sh=sh_emboss)
m.submodules.emboss_b = emboss_b = Conv(k_emboss,
w=self.res_x,
h=self.res_y,
dw=5,
sh=sh_emboss)
connect(extend_r, emboss_r, self.i.r)
connect(extend_g, emboss_g, self.i.g)
connect(extend_b, emboss_b, self.i.b)
# Any channel can be used for these outputs
m.d.comb += [
self.o_ready.eq(extend_r.o_ready),
self.o_valid.eq(ident_r.o_valid),
self.o_x.eq(ident_r.o_x),
self.o_y.eq(ident_r.o_y),
self.o_eof.eq(ident_r.o_eof)
]
# Select the required convolution
with m.Switch(self.i_sel):
with m.Case(1):
select(blur_r.o_p, blur_g.o_p, blur_b.o_p)
with m.Case(2):
select(box_r.o_p, box_g.o_p, box_b.o_p)
with m.Case(3):
select(emboss_r.o_p, emboss_g.o_p, emboss_b.o_p)
with m.Default():
select(ident_r.o_p, ident_g.o_p, ident_b.o_p)
return m
def structured_noise_exp(model, x, y, vx, vy):
classlist = list(set(y))
nclass = len(classlist)
# manually set T
def gen_noise_mat(label_noise_rate=0.5):
"""T = np.eye(10)
# T from Patrini et al, 2017
T[2,2] = 1 - label_noise_rate
T[2,7] = label_noise_rate
T[3,3] = 1 - label_noise_rate
T[3,8] = label_noise_rate
T[5,5] = 1 - label_noise_rate
T[5,6] = label_noise_rate
T[6,6] = 1 - label_noise_rate
T[6,5] = label_noise_rate
T[7,7] = 1 - label_noise_rate
T[7,1] = label_noise_rate
"""
"""
# T to simulate uniform random label flip
T *= (1-label_noise_rate)
T[T==0] = label_noise_rate/float(nclass-1)
"""
T = np.eye(100)
k = 0
for i, j in list(
zip(np.random.permutation(100), np.random.permutation(100))):
if i == j:
continue
T[i, i] = 1 - args.label_noise_rate
T[i, j] = args.label_noise_rate
k += 1
if k == args.struct_noise:
break
return T
with tf.Session() as sess:
conv = Conv(model=model)
noise_levels = [0.7, 0.8]
noise_levels_label = ','.join(list(map(str, noise_levels)))
title = 'Noise level(s): ' + noise_levels_label
for noise_level in noise_levels:
for update, use_dropout in [('adam', False), ('langevin', False)
]: #,('adam',True)]:
for use_dither in [False, True][:1]:
label = 'p = ' + str(noise_level) + ',' + str(update)
if use_dropout:
label += '+dropout'
if use_dither:
label += '+dither'
print('\n\nTraining with optimizer update: ' +
str(update) + ' dither: ' + str(use_dither))
T = gen_noise_mat(label_noise_rate=noise_level)
noisy_y, frac_correct = pytorch_structured_label_flip(y,
p=T)
print('Fraction of labels correct:', frac_correct, '\n\n')
print('Noise level:', noise_level, '\n\n')
print('Noise matrix:\n', T, '\n\n')
if use_dither:
D = np.eye(10) + 0.05 * np.random.random(
size=T.shape) # dither matrix
D = D / (D.sum(axis=1)[:, np.newaxis])
print('Dither matrix:\n', D, '\n\n')
noisy_y, frac_correct = structured_label_flip(
y, p=D) # apply dither
print('Effective Noise matrix:\n', np.matmul(T, D),
'\n\n')
val_t = conv.train(sess,
x,
noisy_y,
vx,
vy,
reset=True,
update=update,
use_dropout=use_dropout)
itr_t, v_t = map(list, zip(*val_t))
plt.plot(itr_t, v_t, label=str(label))
plt.title(title)
plt.ylabel('Val acc.')
plt.ylim((0, 1))
plt.xlabel('Iterations')
lgd = plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
plt.savefig(osp.join(
output_dir, 'structured_exp_noise-' + noise_levels_label + '.png'),
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
plt.clf()
def test_repr(self):
l = Conv([2, 2], 2)
l.accept([2, 3, 3])
def foo(mod, op, d):
if (op[0] == "linear"):
xx = Linear(d)
# rnncell, lstmcell, grucell
elif (mod[0] in ["LSTMCell", "GRUCell"]) and (op[0] == "forward"):
xx = RNNCell(d)
elif op[0] in [
"conv1d",
"conv2d",
]:
xx = Conv(d)
elif (op[0] in Pointwise.ops):
xx = Pointwise(d)
elif (op[0] in Convert.ops):
xx = Convert(d)
elif op[0] in ["__matmul__", "matmul"]:
xx = Matmul(d)
elif op[0] == "embedding":
xx = Embedding(d)
#reduction
elif op[0] == "sum":
xx = Sum(d)
elif op[0] == "mean":
xx = Mean(d)
elif op[0] == "norm":
xx = Norm(d)
elif op[0] == "dropout":
xx = Dropout(d)
#Index, Slice, Join, Mutate
elif (op[0] == "cat"):
xx = Cat(d)
elif (op[0] == "reshape"):
xx = Reshape(d)
elif (op[0] == "masked_scatter_"):
xx = MaskedScatter(d)
elif (op[0] == "gather"):
xx = Gather(d)
elif (op[0] == "nonzero"):
xx = Nonzero(d)
elif (op[0] == "index_select"):
xx = IndexSelect(d)
elif (op[0] == "masked_select"):
xx = MaskedSelect(d)
#blas
elif op[0] in ["addmm", "addmm_"]:
xx = Addmm(d)
elif op[0] == "mm":
xx = Mm(d)
elif op[0] == "bmm":
xx = Bmm(d)
#softmax
elif op[0] == "softmax":
xx = Softmax(d)
elif op[0] == "log_softmax":
xx = LogSoftmax(d)
#loss
elif op[0] == "mse_loss":
xx = MSELoss(d)
#optimizers
elif op[0] == "adam":
xx = Adam(d)
#normalization
elif op[0] == "batch_norm":
xx = BatchNorm(d)
#random
elif op[0] == "randperm":
xx = RandPerm(d)
#misc
elif op[0] == "copy_":
xx = Copy(d)
elif op[0] == "clone":
xx = Clone(d)
elif op[0] == "contiguous":
xx = Contiguous(d)
elif op[0] == "any":
xx = Any(d)
elif (op[0] in Activation.ops):
xx = Activation(d)
elif op[0] == "to":
xx = Convert(d)
else:
xx = Foo(d)
return xx
class CRF(nn.Module):
def __init__(self,
input_dim,
embed_dim,
conv_layers,
num_labels,
batch_size,
m=14):
"""
Linear chain CRF as in Assignment 2
"""
super(CRF, self).__init__()
# crf param
self.input_dim = input_dim
self.embed_dim = embed_dim
self.num_labels = num_labels
self.batch_size = batch_size
self.use_cuda = torch.cuda.is_available()
self.m = m
# conv layer params
self.out_channels = 1 # output channel of conv layer
self.conv_layers = conv_layers
self.stride = (1, 1)
self.padding = True
self.cout_shape = self.get_cout_dim() # output shape of conv layer
self.cout_numel = self.cout_shape[0] * self.cout_shape[1]
self.init_params()
### Use GPU if available
if self.use_cuda:
[m.cuda() for m in self.modules()]
def init_params(self):
"""
Initialize trainable parameters of CRF here
"""
self.conv = Conv(self.conv_layers[0][-1],
self.out_channels,
padding=self.padding,
stride=self.stride)
self.W = torch.randn(self.num_labels,
self.cout_numel,
requires_grad=True)
self.T = torch.randn(self.num_labels,
self.num_labels,
requires_grad=True)
def get_cout_dim(self):
if self.padding:
return (int(np.ceil(self.input_dim[0] / self.stride[0])),
int(np.ceil(int(self.input_dim[1] / self.stride[1]))))
return None
# X: (batch_size, 14, 16, 8) dimensional tensor
# iterates over all words in a batch, and decodes them one by one
def forward(self, X):
"""
Implement the objective of CRF here.
The input (features) to the CRF module should be convolution features.
"""
decods = torch.zeros(self.batch_size, self.m, 1, dtype=torch.int)
for i in range(self.batch_size):
# Reshape the word to (14,1,16,8)
word = X[i].reshape(self.m, 1, self.input_dim[0],
self.input_dim[1])
# conv operation performed for one word independently to every letter
features = self.get_conv_features(word)
# now decode the sequence using conv features
decods[i] = self.dp_infer(features)
return decods
# input: x: (m, d), m is # of letters a word has, d is the feature dimension of letter image
# input: w: (26, d), letter weight vector
# input: T: (26, 26), letter-letter transition matrix
# output: letter_indices: (m, 1), letter labels of a word
# decode a sequence of letters for one word
def dp_infer(self, x):
w = self.W
T = self.T
m = self.m
pos_letter_value_table = torch.zeros((m, 26), dtype=torch.float64)
pos_best_prevletter_table = torch.zeros((m, 26), dtype=torch.int)
# for the position 1 (1st letter), special handling
# because only w and x dot product is covered and transition is not considered.
for i in range(26):
# print(w)
# print(x)
pos_letter_value_table[0, i] = torch.dot(w[i, :], x[0, :])
# pos_best_prevletter_table first row is all zero as there is no previous letter for the first letter
# start from 2nd position
for pos in range(1, m):
# go over all possible letters
for letter_ind in range(self.num_labels):
# get the previous letter scores
prev_letter_scores = pos_letter_value_table[pos - 1, :].clone()
# we need to calculate scores of combining the current letter and all previous letters
# no need to calculate the dot product because dot product only covers current letter and position
# which means it is independent of all previous letters
for prev_letter_ind in range(self.num_labels):
prev_letter_scores[prev_letter_ind] += T[prev_letter_ind,
letter_ind]
# find out which previous letter achieved the largest score by now
best_letter_ind = torch.argmax(prev_letter_scores)
# update the score of current positive with current letter
pos_letter_value_table[pos, letter_ind] = prev_letter_scores[
best_letter_ind] + torch.dot(w[letter_ind, :], x[pos, :])
# save the best previous letter for following tracking to generate most possible word
pos_best_prevletter_table[pos, letter_ind] = best_letter_ind
letter_indicies = torch.zeros((m, 1), dtype=torch.int)
letter_indicies[m - 1,
0] = torch.argmax(pos_letter_value_table[m - 1, :])
max_obj_val = pos_letter_value_table[m - 1, letter_indicies[m - 1, 0]]
# print(max_obj_val)
for pos in range(m - 2, -1, -1):
letter_indicies[pos, 0] = pos_best_prevletter_table[
pos + 1, letter_indicies[pos + 1, 0]]
return letter_indicies
def loss(self, X, labels):
"""
Compute the negative conditional log-likelihood of a labelling given a sequence.
"""
features = self.get_conv_features(X)
loss = blah
return loss
def backward(self):
"""
Return the gradient of the CRF layer
:return:
"""
gradient = blah
return gradient
# performs conv operation to every (16,8) image in the word. m = 14 (default) - word length
# returns flattened vector of new conv features
def get_conv_features(self, word):
"""
Generate convolution features for a given word
"""
cout = self.conv.forward(word)
cout = cout.reshape(cout.shape[0], self.cout_numel)
return cout
def elaborate(self, platform):
m = Module()
def connect(c1, c2, ch):
m.d.comb += [
c1.i_p.eq(ch),
c2.i_p.eq(c1.o_p),
c1.i_valid.eq(self.i_valid),
c2.i_valid.eq(c1.o_valid),
c1.i_ready.eq(c2.o_ready),
c2.i_ready.eq(self.i_ready),
c1.i_reset.eq(self.i_reset),
c2.i_reset.eq(self.i_reset)
]
def select(r, g, b):
m.d.comb += [
self.o.r.eq(r.o_p),
self.o.g.eq(g.o_p),
self.o.b.eq(b.o_p),
]
p_s = Signal(13)
p_m = Signal(5)
m.d.comb += [
p_s.eq((self.i.r * 38) + (self.i.g * 75) + (self.i.b * 15)),
p_m.eq(p_s[7:])
]
# Identity
k_ident = Array([0,0,0,
0,1,0,
0,0,0])
sh_ident = 0
# Guassian blur
k_blur = Array([1, 2 , 1,
2, 4, 2,
1, 2, 1])
sh_blur = 4
k_blur5 = Array([1, 4, 6, 4, 1,
4, 16, 24, 16, 4,
6, 24, 36, 24, 6,
4, 16, 24, 16, 4,
1, 4, 6, 4, 1])
sh_blur5 = 8
# Box blur
k_box = Array([1, 1, 1,
1, 1, 1,
1, 1, 1])
sh_box = 3
# Sharpen
k_sharp = Array([ 0,-1, 0,
-1, 5,-1,
0,-1, 0])
sh_sharp=0
# Emboss
k_emboss = Array([-2, -1, 0,
-1, 1, 1,
0 , 1, 2])
sh_emboss = 0
# Edge detection
k_edge = Array([-1,-1,-1,
-1, 8,-1,
-1,-1,-1])
sh_edge = 0
# Create the convolution modules
m.submodules.extend_r = extend_r = Extend(n=1, w=self.res_x, h=self.res_y)
m.submodules.extend_g = extend_g = Extend(n=1, w=self.res_x, h=self.res_y)
m.submodules.extend_b = extend_b = Extend(n=1, w=self.res_x, h=self.res_y)
m.submodules.blur_r = blur_r = Conv(k_blur, w=self.res_x, h=self.res_y, dw=5,sh=sh_blur)
m.submodules.blur_g = blur_g = Conv(k_blur, w=self.res_x, h=self.res_y, dw=6,sh=sh_blur)
m.submodules.blur_b = blur_b = Conv(k_blur, w=self.res_x, h=self.res_y, dw=5,sh=sh_blur)
connect(extend_r, blur_r, self.i.r)
connect(extend_g, blur_g, self.i.g)
connect(extend_b, blur_b, self.i.b)
m.submodules.ident_r = ident_r = Conv(k_ident, w=self.res_x + 2, h=self.res_y + 2, dw=5, sh=sh_ident)
m.submodules.ident_g = ident_g = Conv(k_ident, w=self.res_x + 2, h=self.res_y + 2, dw=6, sh=sh_ident)
m.submodules.ident_b = ident_b = Conv(k_ident, w=self.res_x + 2, h=self.res_y + 2, dw=5, sh=sh_ident)
connect(extend_r, ident_r, self.i.r)
connect(extend_g, ident_g, self.i.g)
connect(extend_b, ident_b, self.i.b)
m.submodules.blur5_r = blur5_r = Conv(k_blur5, kw=5, w=self.res_x, h=self.res_y, dw=5,sh=sh_blur5)
m.submodules.blur5_g = blur5_g = Conv(k_blur5, kw=5, w=self.res_x, h=self.res_y, dw=6,sh=sh_blur5)
m.submodules.blur5_b = blur5_b = Conv(k_blur5, kw=5, w=self.res_x, h=self.res_y, dw=5,sh=sh_blur5)
connect(extend_r, blur5_r, self.i.r)
connect(extend_g, blur5_g, self.i.g)
connect(extend_b, blur5_b, self.i.b)
m.submodules.box_r = box_r = Conv(k_box, w=self.res_x, h=self.res_y, dw=5,sh=sh_box)
m.submodules.box_g = box_g = Conv(k_box, w=self.res_x, h=self.res_y, dw=6,sh=sh_box)
m.submodules.box_b = box_b = Conv(k_box, w=self.res_x, h=self.res_y, dw=5,sh=sh_box)
connect(extend_r, box_r, self.i.r)
connect(extend_g, box_g, self.i.g)
connect(extend_b, box_b, self.i.b)
m.submodules.emboss_r = emboss_r = Conv(k_emboss, w=self.res_x, h=self.res_y, dw=5,sh=sh_emboss, same=1)
m.submodules.emboss_g = emboss_g = Conv(k_emboss, w=self.res_x, h=self.res_y, dw=6,sh=sh_emboss, same=1)
m.submodules.emboss_b = emboss_b = Conv(k_emboss, w=self.res_x, h=self.res_y, dw=5,sh=sh_emboss, same=1)
connect(extend_r, emboss_r, self.i.r)
connect(extend_g, emboss_g, self.i.g)
connect(extend_b, emboss_b, self.i.b)
m.submodules.sharp_r = sharp_r = Conv(k_sharp, w=self.res_x, h=self.res_y, dw=5,sh=sh_sharp, same=1)
m.submodules.sharp_g = sharp_g = Conv(k_sharp, w=self.res_x, h=self.res_y, dw=6,sh=sh_sharp, same=1)
m.submodules.sharp_b = sharp_b = Conv(k_sharp, w=self.res_x, h=self.res_y, dw=5,sh=sh_sharp, same=1)
connect(extend_r, sharp_r, self.i.r)
connect(extend_g, sharp_g, self.i.g)
connect(extend_b, sharp_b, self.i.b)
m.submodules.extend_m = extend_m = Extend(n=1, w=self.res_x, h=self.res_y)
m.submodules.edge_m = edge_m = Conv(k_edge, w=self.res_x, h=self.res_y, dw=5,sh=sh_edge)
m.submodules.edge_g = edge_g = Conv(k_edge, w=self.res_x, h=self.res_y, dw=6,sh=sh_edge)
connect(extend_m, edge_m, p_m)
connect(extend_m, edge_g, p_m >> 1)
# We are ready when extend is ready
m.d.comb += [
self.o_ready.eq(extend_r.o_ready),
# Any channel can be used for these outputs
self.o_valid.eq(blur_r.o_valid),
self.o_x.eq(blur_r.o_x),
self.o_y.eq(blur_r.o_y),
self.o_eof.eq(blur_r.o_eof)
]
# Select the required convolution
with m.Switch(self.i_sel):
with m.Case(1):
select(blur_r, blur_g, blur_b)
with m.Case(2):
select(box_r, box_g, box_b)
with m.Case(3):
select(blur5_r, blur5_g, blur5_b)
with m.Case(4):
select(emboss_r, emboss_g, emboss_b)
with m.Case(5):
select(sharp_r, sharp_g, sharp_b)
with m.Case(6):
select(edge_m, edge_g, edge_m)
with m.Default():
select(ident_r, ident_g, ident_b)
return m