def train_network(model, num_epochs = 100, minibatch_size=256,
dropout_rate = 0.5, eps_w = 0.01, eps_b = 0.01, mom = 0.9, wd = 0.0005):
num_layers = model.num_layers
num_weights = model.num_weights
last = time.time()
num_samples = minibatch_size
minibatch_size = minibatch_size / num_gpu
for i in xrange(num_epochs):
print "---------------------Epoch #", i
weightsgrad = [[None] * num_weights for z in range(num_gpu)]
biasgrad = [[None] * num_weights for z in range(num_gpu)]
for j in xrange(1, 1024):
count = j % num_gpu
owl.set_device(gpu_array[count])
data = owl.randn([227, 227, 3, minibatch_size], 0, 1)
label = owl.randn([1, minibatch_size], 0, 1)
out = train_one_mb(model, data, label, weightsgrad[count], biasgrad[count], dropout_rate)
out.start_eval()
if count == 0:
# Update
for k in range(num_weights):
for l in range(1, num_gpu):
weightsgrad[0][k] = weightsgrad[0][k] + weightsgrad[l][k]
biasgrad[0][k] = biasgrad[0][k] + biasgrad[l][k]
model.weightsdelta[k] = mom * model.weightsdelta[k] - eps_w / num_samples * (weightsgrad[0][k] + wd * model.weights[k])
model.biasdelta[k] = mom * model.biasdelta[k] - eps_b / num_samples * (biasgrad[0][k] + wd * model.bias[k])
model.weights[k] += model.weightsdelta[k]
model.bias[k] += model.biasdelta[k]
if j % (lazy * num_gpu) == 0:
print_training_accuracy(out, label, minibatch_size)
print "time: %s" % (time.time() - last)
last = time.time()
def init_random(self):
self.weights = [
owl.randn([self.filtersizes[0], self.filtersizes[0], 1, self.filters[0]], 0.0, 0.1),
owl.randn([self.filtersizes[1], self.filtersizes[1], self.filters[0], self.filters[1]], 0.0, 0.1),
owl.randn([128, self.convolution_output_size], 0.0, 0.1),
owl.randn([10, 128], 0.0, 0.1)
];
self.weightdelta = [
owl.zeros([self.filtersizes[0], self.filtersizes[0], 1, self.filters[0]]),
owl.zeros([self.filtersizes[1], self.filtersizes[1], self.filters[0], self.filters[1]]),
owl.zeros([128, self.convolution_output_size]),
owl.zeros([10, 128])
];
self.bias = [
owl.zeros([self.filters[0]]),
owl.zeros([self.filters[1]]),
owl.zeros([128, 1]),
owl.zeros([10, 1])
];
self.biasdelta = [
owl.zeros([self.filters[0]]),
owl.zeros([self.filters[1]]),
owl.zeros([128, 1]),
owl.zeros([10, 1])
];
def train_network(model, num_epochs = 100, minibatch_size=256,
dropout_rate = 0.5, eps_w = 0.01, eps_b = 0.01, mom = 0.9, wd = 0.0005):
num_layers = model.num_layers
num_weights = model.num_weights
last = time.time()
minibatch_size = minibatch_size # / num_gpu
for i in xrange(num_epochs):
print "---------------------Epoch #", i
weightsgrad = [[None] * num_weights for z in range(num_gpu)]
biasgrad = [[None] * num_weights for z in range(num_gpu)]
for j in xrange(1, 1024):
count = j % num_gpu
owl.set_device(gpu_array[count])
data = owl.randn([227, 227, 3, minibatch_size], 0, 1)
label = owl.randn([1, minibatch_size], 0, 1)
out = train_one_mb(model, data, label, weightsgrad[count], biasgrad[count], dropout_rate)
for k in weightsgrad[count]:
k.start_eval()
for k in biasgrad[count]:
k.start_eval()
if count == 0:
for k in range(0, num_gpu):
for l in weightsgrad[k]:
l.wait_for_eval()
for l in biasgrad[k]:
l.wait_for_eval()
print "time: %s" % (time.time() - last)
last = time.time()
def train_network(model,
num_epochs=100,
minibatch_size=256,
dropout_rate=0.5,
eps_w=0.01,
eps_b=0.01,
mom=0.9,
wd=0.0005):
gpu0 = owl.create_gpu_device(0)
gpu1 = owl.create_gpu_device(1)
num_layers = 20
num_weights = 8
count = 0
last = time.time()
# dp = ImageNetDataProvider(mean_file='/home/minjie/data/imagenet/imagenet_mean.binaryproto',
# train_db='/home/minjie/data/imagenet/ilsvrc12_train_lmdb',
# val_db='/home/minjie/data/imagenet/ilsvrc12_val_lmdb',
# test_db='/home/minjie/data/imagenet/ilsvrc12_test_lmdb')
for i in xrange(num_epochs):
print "---------------------Epoch #", i
for j in xrange(300):
count = count + 1
data = owl.randn([227, 227, 3, minibatch_size], 0, 1)
label = owl.randn([1, minibatch_size], 0, 1)
weightsgrad = [None] * num_weights
biasgrad = [None] * num_weights
num_samples = minibatch_size
'''
thisimg = samples[0, :]
print thisimg
imgdata = np.transpose(thisimg.reshape([3, 227*227])).reshape([227, 227, 3])
print imgdata
img = Image.fromarray(imgdata.astype(np.uint8))
img.save('testimg.jpg', format='JPEG')
exit(0)
'''
owl.set_device(gpu0)
out = train_one_mb(model, data, label, weightsgrad, biasgrad,
dropout_rate)
for k in range(num_weights):
model.weightsdelta[
k] = mom * model.weightsdelta[k] - eps_w / num_samples * (
weightsgrad[k] + wd * model.weights[k])
model.biasdelta[
k] = mom * model.biasdelta[k] - eps_b / num_samples * (
biasgrad[k] + wd * model.bias[k])
model.weights[k] += model.weightsdelta[k]
model.weights[k].start_eval()
model.bias[k] += model.biasdelta[k]
model.bias[k].start_eval()
if count % 3 == 0:
print_training_accuracy(out, label, data.shape[-1])
print "time: %s" % (time.time() - last)
last = time.time()
def __init__(self, input_size, hidden_size, output_size):
self.Layers = [input_size, hidden_size, output_size]
# Recurrent weights: take x_t, h_{t-1}, and bias unit
# and produce the 3 gates and the input to cell signal
self.ig_weight_data = owl.randn([self.Layers[0], self.Layers[1]], 0.0,
0.1)
self.fg_weight_data = owl.randn([self.Layers[0], self.Layers[1]], 0.0,
0.1)
self.og_weight_data = owl.randn([self.Layers[0], self.Layers[1]], 0.0,
0.1)
self.ff_weight_data = owl.randn([self.Layers[0], self.Layers[1]], 0.0,
0.1)
self.ig_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0,
0.1)
self.fg_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0,
0.1)
self.og_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0,
0.1)
self.ff_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0,
0.1)
self.ig_weight_bias = owl.zeros([self.Layers[1], 1])
self.fg_weight_bias = owl.zeros([self.Layers[1], 1])
self.og_weight_bias = owl.zeros([self.Layers[1], 1])
self.ff_weight_bias = owl.zeros([self.Layers[1], 1])
# Decoder weights (e.g. mapping to vocabulary)
self.decoder_weights = owl.randn([self.Layers[1], self.Layers[2]], 0.0,
0.1) # decoder
self.decoder_bias = owl.zeros([output_size, 1])
def train_network(model, num_epochs = 100, minibatch_size=256,
dropout_rate = 0.5, eps_w = 0.01, eps_b = 0.01, mom = 0.9, wd = 0.0005):
gpu0 = owl.create_gpu_device(0)
gpu1 = owl.create_gpu_device(1)
num_layers = 20
num_weights = 8
count = 0
last = time.time()
# dp = ImageNetDataProvider(mean_file='/home/minjie/data/imagenet/imagenet_mean.binaryproto',
# train_db='/home/minjie/data/imagenet/ilsvrc12_train_lmdb',
# val_db='/home/minjie/data/imagenet/ilsvrc12_val_lmdb',
# test_db='/home/minjie/data/imagenet/ilsvrc12_test_lmdb')
for i in xrange(num_epochs):
print "---------------------Epoch #", i
for j in xrange(300):
count = count + 1
data = owl.randn([227, 227, 3, minibatch_size], 0, 1)
label = owl.randn([1, minibatch_size], 0, 1)
weightsgrad = [None] * num_weights
biasgrad = [None] * num_weights
num_samples = minibatch_size
'''
thisimg = samples[0, :]
print thisimg
imgdata = np.transpose(thisimg.reshape([3, 227*227])).reshape([227, 227, 3])
print imgdata
img = Image.fromarray(imgdata.astype(np.uint8))
img.save('testimg.jpg', format='JPEG')
exit(0)
'''
owl.set_device(gpu0)
out = train_one_mb(model, data, label, weightsgrad, biasgrad, dropout_rate)
for k in range(num_weights):
model.weightsdelta[k] = mom * model.weightsdelta[k] - eps_w / num_samples * (weightsgrad[k] + wd * model.weights[k])
model.biasdelta[k] = mom * model.biasdelta[k] - eps_b / num_samples * (biasgrad[k] + wd * model.bias[k])
model.weights[k] += model.weightsdelta[k]
model.weights[k].start_eval()
model.bias[k] += model.biasdelta[k]
model.bias[k].start_eval()
if count % 3 == 0:
print_training_accuracy(out, label, data.shape[-1])
print "time: %s" % (time.time() - last)
last = time.time()
def __init__(self, data_file='mnist_all.mat', num_epochs=100, mb_size=256, eps_w=0.01, eps_b=0.01):
self.cpu = owl.create_cpu_device()
self.gpu = owl.create_gpu_device(0)
self.data_file = data_file
self.num_epochs=num_epochs
self.mb_size=mb_size
self.eps_w=eps_w
self.eps_b=eps_b
# init weight
l1 = 784; l2 = 256; l3 = 10
self.l1 = l1; self.l2 = l2; self.l3 = l3
self.w1 = owl.randn([l2, l1], 0.0, math.sqrt(4.0 / (l1 + l2)))
self.w2 = owl.randn([l3, l2], 0.0, math.sqrt(4.0 / (l2 + l3)))
self.b1 = owl.zeros([l2, 1])
self.b2 = owl.zeros([l3, 1])
def init_random(self):
self.weights = [
owl.randn([5, 5, 1, 16], 0.0, 0.1),
owl.randn([5, 5, 16, 32], 0.0, 0.1),
owl.randn([10, 512], 0.0, 0.1)
];
self.weightdelta = [
owl.zeros([5, 5, 1, 16]),
owl.zeros([5, 5, 16, 32]),
owl.zeros([10, 512])
];
self.bias = self.biasdelta = [
owl.zeros([16]),
owl.zeros([32]),
owl.zeros([10, 1])
];
def init_random(self):
self.weights = [
owl.randn([5, 5, 1, 16], 0.0, 0.1),
owl.randn([5, 5, 16, 32], 0.0, 0.1),
owl.randn([10, 512], 0.0, 0.1)
]
self.weightdelta = [
owl.zeros([5, 5, 1, 16]),
owl.zeros([5, 5, 16, 32]),
owl.zeros([10, 512])
]
self.bias = [owl.zeros([16]),
owl.zeros([32]),
owl.zeros([10, 1])]
self.biasdelta = [
owl.zeros([16]),
owl.zeros([32]),
owl.zeros([10, 1])
]
def init_random(self):
self.weights = [
owl.randn([11, 11, 3, 96], 0.0, 0.01),
owl.randn([5, 5, 96, 256], 0.0, 0.01),
owl.randn([3, 3, 256, 384], 0.0, 0.01),
owl.randn([3, 3, 384, 384], 0.0, 0.01),
owl.randn([3, 3, 384, 256], 0.0, 0.01),
owl.randn([4096, 9216], 0.0, 0.01),
owl.randn([4096, 4096], 0.0, 0.01),
owl.randn([1000, 4096], 0.0, 0.01)
];
self.weightsdelta = [
owl.zeros([11, 11, 3, 96]),
owl.zeros([5, 5, 96, 256]),
owl.zeros([3, 3, 256, 384]),
owl.zeros([3, 3, 384, 384]),
owl.zeros([3, 3, 384, 256]),
owl.zeros([4096, 9216]),
owl.zeros([4096, 4096]),
owl.zeros([1000, 4096])
];
self.bias = [
owl.zeros([96]),
owl.zeros([256]),
owl.zeros([384]),
owl.zeros([384]),
owl.zeros([256]),
owl.zeros([4096, 1]),
owl.zeros([4096, 1]),
owl.zeros([1000, 1])
];
self.biasdelta = [
owl.zeros([96]),
owl.zeros([256]),
owl.zeros([384]),
owl.zeros([384]),
owl.zeros([256]),
owl.zeros([4096, 1]),
owl.zeros([4096, 1]),
owl.zeros([1000, 1])
];
def __init__(self, vocab_size, input_size, hidden_size):
output_size = vocab_size
self.Layers = [input_size, hidden_size, output_size]
print 'Model size:', self.Layers
# Recurrent weights: take x_t, h_{t-1}, and bias unit
# and produce the 3 gates and the input to cell signal
# self.WIFOG = owl.randn([self.Layers[0] + self.Layers[1], self.Layers[1] * 4], 0.0, 0.1)
# self.BIFOG = owl.zeros([self.Layers[1] * 4, 1])
self.ig_weight_data = owl.randn([self.Layers[1], self.Layers[0]], 0.0, 0.1)
self.fg_weight_data = owl.randn([self.Layers[1], self.Layers[0]], 0.0, 0.1)
self.og_weight_data = owl.randn([self.Layers[1], self.Layers[0]], 0.0, 0.1)
self.ff_weight_data = owl.randn([self.Layers[1], self.Layers[0]], 0.0, 0.1)
self.ig_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1)
self.fg_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1)
self.og_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1)
self.ff_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1)
self.ig_weight_cell = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1)
self.fg_weight_cell = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1)
self.og_weight_cell = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1)
self.ff_weight_cell = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1)
self.ig_weight_bias = owl.zeros([self.Layers[1], 1])
self.fg_weight_bias = owl.zeros([self.Layers[1], 1])
self.og_weight_bias = owl.zeros([self.Layers[1], 1])
self.ff_weight_bias = owl.zeros([self.Layers[1], 1])
# Decoder weights (e.g. mapping to vocabulary)
self.decoder_weights = owl.randn([self.Layers[2], self.Layers[1]], 0.0, 0.1) # decoder
self.decoder_bias = owl.zeros([output_size, 1])
self.emb_weight = [None] * vocab_size
for i in range(vocab_size):
self.emb_weight[i] = owl.randn([input_size, 1], 0.0, 0.1)
import owl import numpy as np import demo_common as dc x1 = owl.randn([784, 128], 0.0, 0.1) x2 = owl.randn([784, 128], 0.0, 0.1) w = owl.randn([512, 784], 0.0, 0.1) b = owl.zeros([512, 1]) y1 = w * x1 + b y2 = w * x2 + b gw = y1 * x1.trans() + y2 * x2.trans() print gw.to_numpy()
import owl import owl.elewise as ele import numpy as np import demo_common x = owl.randn([784, 256], 0.0, 0.01) w = owl.randn([512, 784], 0.0, 0.01) b = owl.zeros([512, 1]) y = ele.relu(w * x + b) print y.to_numpy() e = owl.randn([512, 256], 0.0, 0.01) ey = ele.relu_back(e, y) ex = w.trans() * ey print ex.to_numpy()
data = data / np.var(data, 0)
# training parameters
epsilon = 0.01
momentum = 0.9
num_epochs = 40
batch_size = 64
num_batches = data.shape[0]//batch_size
# model parameters
num_vis = data.shape[1]
num_hid = 1024
# initialize weights
weights = 0.1 * owl.randn([num_vis, num_hid],0,1)
bias_v = owl.zeros([1,num_vis])
bias_h = owl.zeros([1,num_hid])
# initialize weight updates
d_weights = owl.zeros((num_vis,num_hid ))
d_bias_v = owl.zeros([1,num_vis])
d_bias_h = owl.zeros([1,num_hid])
start_time = time.time()
for epoch in range(num_epochs):
print("Epoch %i" % (epoch + 1))
err = []
for batch in range(num_batches):
np_set = data[batch*batch_size:(batch + 1)*batch_size,:]
import owl import owl.conv as co import numpy as np import demo_common x = owl.randn([227, 227, 3, 256], 0.0, 1) w = owl.randn([11, 11, 3, 96], 0.0, 0.1) b = owl.zeros([96]) conv = co.Convolver(pad_h=0, pad_w=0, stride_v=4, stride_h=4) y = conv.ff(x, w, b) print y.to_numpy() print y.shape ex = conv.bp(y, w) print ex.to_numpy() print ex.shape
def init_random(self):
self.weights = [
owl.randn([3, 3, 3, 64], 0.0, 0.01),
owl.randn([3, 3, 64, 64], 0.0, 0.01),
owl.randn([1, 1, 1, 1], 0.0, 0.01),
owl.randn([3, 3, 64, 128], 0.0, 0.01),
owl.randn([3, 3, 128, 128], 0.0, 0.01),
owl.randn([1, 1, 1, 1], 0.0, 0.01),
owl.randn([3, 3, 128, 256], 0.0, 0.01),
owl.randn([3, 3, 256, 256], 0.0, 0.01),
owl.randn([3, 3, 256, 256], 0.0, 0.01),
owl.randn([1, 1, 1, 1], 0.0, 0.01),
owl.randn([3, 3, 256, 512], 0.0, 0.01),
owl.randn([3, 3, 512, 512], 0.0, 0.01),
owl.randn([3, 3, 512, 512], 0.0, 0.01),
owl.randn([1, 1, 1, 1], 0.0, 0.01),
owl.randn([3, 3, 512, 512], 0.0, 0.01),
owl.randn([3, 3, 512, 512], 0.0, 0.01),
owl.randn([3, 3, 512, 512], 0.0, 0.01),
owl.randn([1, 1, 1, 1], 0.0, 0.01),
owl.randn([4096, 25088], 0.0, 0.005),
owl.randn([4096, 4096], 0.0, 0.005),
owl.randn([1000, 4096], 0.0, 0.01)
]
self.weightsdelta = [
owl.zeros([3, 3, 3, 64]),
owl.zeros([3, 3, 64, 64]),
owl.zeros([1, 1, 1, 1]),
owl.zeros([3, 3, 64, 128]),
owl.zeros([3, 3, 128, 128]),
owl.zeros([1, 1, 1, 1]),
owl.zeros([3, 3, 128, 256]),
owl.zeros([3, 3, 256, 256]),
owl.zeros([3, 3, 256, 256]),
owl.zeros([1, 1, 1, 1]),
owl.zeros([3, 3, 256, 512]),
owl.zeros([3, 3, 512, 512]),
owl.zeros([3, 3, 512, 512]),
owl.zeros([1, 1, 1, 1]),
owl.zeros([3, 3, 512, 512]),
owl.zeros([3, 3, 512, 512]),
owl.zeros([3, 3, 512, 512]),
owl.zeros([1, 1, 1, 1]),
owl.zeros([4096, 25088]),
owl.zeros([4096, 4096]),
owl.zeros([1000, 4096])
]
self.bias = [
owl.zeros([64]),
owl.zeros([64]),
owl.zeros([64]),
owl.zeros([128]),
owl.zeros([128]),
owl.zeros([128]),
owl.zeros([256]),
owl.zeros([256]),
owl.zeros([256]),
owl.zeros([256]),
owl.zeros([512]),
owl.zeros([512]),
owl.zeros([512]),
owl.zeros([512]),
owl.zeros([512]),
owl.zeros([512]),
owl.zeros([512]),
owl.zeros([512]),
owl.zeros([4096, 1]),
owl.zeros([4096, 1]),
owl.zeros([1000, 1])
]
self.biasdelta = [
owl.zeros([64]),
owl.zeros([64]),
owl.zeros([64]),
owl.zeros([128]),
owl.zeros([128]),
owl.zeros([128]),
owl.zeros([256]),
owl.zeros([256]),
owl.zeros([256]),
owl.zeros([256]),
owl.zeros([512]),
owl.zeros([512]),
owl.zeros([512]),
owl.zeros([512]),
owl.zeros([512]),
owl.zeros([512]),
owl.zeros([512]),
owl.zeros([512]),
owl.zeros([4096, 1]),
owl.zeros([4096, 1]),
owl.zeros([1000, 1])
]
def init_random(self):
last_channel = self.input_channel
last_scale = self.input_size
last_dim = last_scale * last_scale * last_channel
for i in range(self.num_weights):
if self.ff_infos[i]['ff_type'] == 'conv':
kernelsize = self.ff_infos[i]['convolution_param'].kernel_size
out_channel = self.ff_infos[i]['convolution_param'].num_output
stride = self.ff_infos[i]['convolution_param'].stride
pad = self.ff_infos[i]['convolution_param'].pad
print 'conv %d %d %d %d %d %d %d %d' % (i, kernelsize, out_channel, stride, pad, last_channel, last_scale, last_dim)
owl.randn([kernelsize, kernelsize, last_channel, out_channel], 0.0, self.ff_infos[i]['convolution_param'].weight_filler.std)
#weight
if self.ff_infos[i]['convolution_param'].weight_filler.type == "gaussian":
self.weights.append(owl.randn([kernelsize, kernelsize, last_channel, out_channel], 0.0, self.ff_infos[i]['convolution_param'].weight_filler.std))
elif self.ff_infos[i]['convolution_param'].weight_filler.type == "constant":
self.weights.append(owl.zeros([kernelsize, kernelsize, last_channel, out_channel]) + self.ff_infos[i]['convolution_param'].weight_filler.value)
else:
assert False
self.weightsdelta.append(owl.zeros([kernelsize, kernelsize, last_channel, out_channel]))
#bias
if self.ff_infos[i]['convolution_param'].bias_filler.type == "gaussian":
self.bias.append(owl.randn([out_channel], 0.0, self.ff_infos[i]['convolution_param'].bias_filler.std))
elif self.ff_infos[i]['convolution_param'].bias_filler.type == "constant":
self.bias.append(owl.zeros([out_channel]) + self.ff_infos[i]['convolution_param'].bias_filler.value)
else:
assert False
self.biasdelta.append(owl.zeros([out_channel]))
last_channel = out_channel
last_scale = (last_scale + pad * 2 - kernelsize) / stride + 1
last_dim = last_scale * last_scale * last_channel
elif self.ff_infos[i]['ff_type'] == 'pooling':
kernelsize = self.ff_infos[i]['pooling_param'].kernel_size
stride = self.ff_infos[i]['pooling_param'].stride
pad = self.ff_infos[i]['pooling_param'].pad
print 'pool %d %d %d %d %d %d %d' % (i, kernelsize, stride, pad, last_channel, last_scale, last_dim)
self.weights.append(owl.zeros([1]))
self.weightsdelta.append(owl.zeros([1]))
self.bias.append(owl.zeros([1]))
self.biasdelta.append(owl.zeros([1]))
last_channel = out_channel
last_scale = (last_scale + pad * 2 - kernelsize) / stride + 1
last_dim = last_scale * last_scale * last_channel
elif self.ff_infos[i]['ff_type'] == 'fully':
out_channel = self.ff_infos[i]['fully_param'].num_output
print 'fully %d %d %d' % (i, last_dim, out_channel)
#weight
if self.ff_infos[i]['fully_param'].weight_filler.type == "gaussian":
self.weights.append(owl.randn([out_channel, last_dim], 0.0, self.ff_infos[i]['fully_param'].weight_filler.std))
elif self.ff_infos[i]['fully_param'].weight_filler.type == "constant":
self.weights.append(owl.zeros([out_channel, last_dim]) + self.ff_infos[i]['fully_param'].weight_filler.value)
else:
assert False
self.weightsdelta.append(owl.zeros([out_channel, last_dim]))
#bias
if self.ff_infos[i]['fully_param'].bias_filler.type == "gaussian":
self.bias.append(owl.randn([out_channel, 1], 0.0, self.ff_infos[i]['fully_param'].weight_filler.std))
elif self.ff_infos[i]['fully_param'].bias_filler.type == "constant":
self.bias.append(owl.zeros([out_channel, 1]) + self.ff_infos[i]['fully_param'].weight_filler.value)
else:
assert False
self.biasdelta.append(owl.zeros([out_channel, 1]))
last_dim = out_channel
last_channel = out_channel
def __init__(self, vocab_size, input_size, hidden_size):
output_size = vocab_size
self.Layers = [input_size, hidden_size, output_size]
print 'Model size:', self.Layers
# Recurrent weights: take x_t, h_{t-1}, and bias unit
# and produce the 3 gates and the input to cell signal
# self.WIFOG = owl.randn([self.Layers[0] + self.Layers[1], self.Layers[1] * 4], 0.0, 0.1)
# self.BIFOG = owl.zeros([self.Layers[1] * 4, 1])
self.ig_weight_data = owl.randn([self.Layers[1], self.Layers[0]], 0.0,
0.1)
self.fg_weight_data = owl.randn([self.Layers[1], self.Layers[0]], 0.0,
0.1)
self.og_weight_data = owl.randn([self.Layers[1], self.Layers[0]], 0.0,
0.1)
self.ff_weight_data = owl.randn([self.Layers[1], self.Layers[0]], 0.0,
0.1)
self.ig_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0,
0.1)
self.fg_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0,
0.1)
self.og_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0,
0.1)
self.ff_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0,
0.1)
self.ig_weight_cell = owl.randn([self.Layers[1], self.Layers[1]], 0.0,
0.1)
self.fg_weight_cell = owl.randn([self.Layers[1], self.Layers[1]], 0.0,
0.1)
self.og_weight_cell = owl.randn([self.Layers[1], self.Layers[1]], 0.0,
0.1)
self.ff_weight_cell = owl.randn([self.Layers[1], self.Layers[1]], 0.0,
0.1)
self.ig_weight_bias = owl.zeros([self.Layers[1], 1])
self.fg_weight_bias = owl.zeros([self.Layers[1], 1])
self.og_weight_bias = owl.zeros([self.Layers[1], 1])
self.ff_weight_bias = owl.zeros([self.Layers[1], 1])
# Decoder weights (e.g. mapping to vocabulary)
self.decoder_weights = owl.randn([self.Layers[2], self.Layers[1]], 0.0,
0.1) # decoder
self.decoder_bias = owl.zeros([output_size, 1])
self.emb_weight = [None] * vocab_size
for i in range(vocab_size):
self.emb_weight[i] = owl.randn([input_size, 1], 0.0, 0.1)
import owl cpu = owl.create_cpu_device() owl.set_device(cpu) x = owl.randn([1000,5000],0,64) y = x.histogram(8) print y.to_numpy()
