def create_net(self, dropout=0.5):
self.embed = layer.Dense('embed',
self.embed_size,
input_sample_shape=(self.vocab_size,
))
self.embed.to_device(self.dev)
self.lstm = layer.LSTM(
name='lstm',
hidden_size=self.hidden_size,
num_stacks=self.num_stack_layers,
dropout=dropout,
input_sample_shape=(
self.embed_size,
))
self.lstm.to_device(self.dev)
self.dense = layer.Dense(
'dense',
2, #output shape
input_sample_shape=(
self.hidden_size,
))
self.dense.to_device(self.dev)
self.sft = layer.Softmax('softmax',
input_sample_shape=(
2,
))
self.sft.to_device(self.dev)
self.loss = loss.SoftmaxCrossEntropy()
def create_net(use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
ConvBnReLU(net, 'conv1_1', 64, (3, 32, 32))
net.add(layer.Dropout('drop1', 0.3))
ConvBnReLU(net, 'conv1_2', 64)
net.add(layer.MaxPooling2D('pool1', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv2_1', 128)
net.add(layer.Dropout('drop2_1', 0.4))
ConvBnReLU(net, 'conv2_2', 128)
net.add(layer.MaxPooling2D('pool2', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv3_1', 256)
net.add(layer.Dropout('drop3_1', 0.4))
ConvBnReLU(net, 'conv3_2', 256)
net.add(layer.Dropout('drop3_2', 0.4))
ConvBnReLU(net, 'conv3_3', 256)
net.add(layer.MaxPooling2D('pool3', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv4_1', 512)
net.add(layer.Dropout('drop4_1', 0.4))
ConvBnReLU(net, 'conv4_2', 512)
net.add(layer.Dropout('drop4_2', 0.4))
ConvBnReLU(net, 'conv4_3', 512)
net.add(layer.MaxPooling2D('pool4', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv5_1', 512)
net.add(layer.Dropout('drop5_1', 0.4))
ConvBnReLU(net, 'conv5_2', 512)
net.add(layer.Dropout('drop5_2', 0.4))
ConvBnReLU(net, 'conv5_3', 512)
net.add(layer.MaxPooling2D('pool5', 2, 2, border_mode='valid'))
net.add(layer.Flatten('flat'))
net.add(layer.Dropout('drop_flat', 0.5))
net.add(layer.Dense('ip1', 512))
net.add(layer.BatchNormalization('batchnorm_ip1'))
net.add(layer.Activation('relu_ip1'))
net.add(layer.Dropout('drop_ip2', 0.5))
net.add(layer.Dense('ip2', 10))
print 'Start intialization............'
for (p, name) in zip(net.param_values(), net.param_names()):
print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
initializer.gaussian(p, 0, 3 * 3 * p.shape[0])
else:
p.gaussian(0, 0.02)
else:
p.set_value(0)
print name, p.l1()
return net
def create_net(input_shape, use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
net.add(
layer.Conv2D('conv1',
nb_kernels=32,
kernel=7,
stride=3,
pad=1,
input_sample_shape=input_shape))
net.add(layer.Activation('relu1'))
net.add(layer.MaxPooling2D('pool1', 2, 2, border_mode='valid'))
net.add(layer.Conv2D('conv2', nb_kernels=64, kernel=5, stride=3))
net.add(layer.Activation('relu2'))
net.add(layer.MaxPooling2D('pool2', 2, 2, border_mode='valid'))
net.add(layer.Conv2D('conv3', nb_kernels=128, kernel=3, stride=1, pad=2))
net.add(layer.Activation('relu3'))
net.add(layer.MaxPooling2D('pool3', 2, 2, border_mode='valid'))
net.add(layer.Conv2D('conv4', nb_kernels=256, kernel=3, stride=1))
net.add(layer.Activation('relu4'))
net.add(layer.MaxPooling2D('pool4', 2, 2, border_mode='valid'))
net.add(layer.Flatten('flat'))
net.add(layer.Dense('ip5', 256))
net.add(layer.Activation('relu5'))
net.add(layer.Dense('ip6', 16))
net.add(layer.Activation('relu6'))
net.add(layer.Dense('ip7', 2))
print 'Parameter intialization............'
for (p, name) in zip(net.param_values(), net.param_names()):
print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
initializer.gaussian(p, 0, p.size())
else:
p.gaussian(0, 0.02)
else:
p.set_value(0)
print name, p.l1()
return net
def create_net(depth, nb_classes, dense=0, use_cpu=True):
if use_cpu:
layer.engine = 'singacpp'
net = densenet_base(depth)
# this part was not included in the pytorch model
if dense > 0:
net.add(layer.Dense('hidden-dense', dense))
net.add(layer.Activation('act-dense'))
net.add(layer.Dropout('dropout'))
net.add(layer.Dense('sigmoid', nb_classes))
return net
def create_net(input_shape, use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
ConvBnReLUPool(net, 'conv1', 32, input_shape)
ConvBnReLUPool(net, 'conv2', 64)
ConvBnReLUPool(net, 'conv3', 128)
ConvBnReLUPool(net, 'conv4', 128)
ConvBnReLUPool(net, 'conv5', 256)
ConvBnReLUPool(net, 'conv6', 256)
ConvBnReLUPool(net, 'conv7', 512)
ConvBnReLUPool(net, 'conv8', 512)
net.add(layer.Flatten('flat'))
net.add(layer.Dense('ip1', 256))
net.add(layer.BatchNormalization('bn1'))
net.add(layer.Activation('relu1'))
net.add(layer.Dropout('dropout1', 0.2))
net.add(layer.Dense('ip2', 16))
net.add(layer.BatchNormalization('bn2'))
net.add(layer.Activation('relu2'))
net.add(layer.Dropout('dropout2', 0.2))
net.add(layer.Dense('ip3', 2))
print 'Parameter intialization............'
for (p, name) in zip(net.param_values(), net.param_names()):
print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
initializer.gaussian(p, 0, p.size())
else:
p.gaussian(0, 0.02)
else:
p.set_value(0)
print name, p.l1()
return net
def test_save_load(self):
ffn = net.FeedForwardNet(loss.SoftmaxCrossEntropy())
ffn.add(layer.Conv2D('conv', 4, 3, input_sample_shape=(3, 12, 12)))
ffn.add(layer.Flatten('flat'))
# ffn.add(layer.BatchNorm('bn'))
ffn.add(layer.Dense('dense', num_output=4))
for pname, pval in zip(ffn.param_names(), ffn.param_values()):
pval.set_value(0.1)
ffn.save('test_snaphost')
ffn.save('test_pickle', use_pickle=True)
ffn.load('test_snaphost')
ffn.load('test_pickle', use_pickle=True)
def build_net(self):
if self.use_cpu:
layer.engine = 'singacpp'
else:
layer.engine = 'cudnn'
self.net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(),
metric.Accuracy())
self.net.add(
Reshape('reshape1', (self.vocab_size, ),
input_sample_shape=(self.maxlen, self.vocab_size)))
self.net.add(layer.Dense('embed',
self.embed_size)) # output: (embed_size, )
self.net.add(layer.Dropout('dropout'))
self.net.add(Reshape('reshape2', (1, self.maxlen, self.embed_size)))
self.net.add(
layer.Conv2D('conv',
self.filters, (self.kernel_size, self.embed_size),
border_mode='valid')) # output: (filter, embed_size)
if self.use_cpu == False:
self.net.add(layer.BatchNormalization('batchNorm'))
self.net.add(layer.Activation('activ')) # output: (filter, embed_size)
self.net.add(layer.MaxPooling2D('max', stride=self.pool_size))
self.net.add(layer.Flatten('flatten'))
self.net.add(layer.Dense('dense', 2))
def create_net(use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
net.add(
layer.Conv2D("conv1", 16, 3, 1, pad=1, input_sample_shape=(3, 32, 32)))
net.add(layer.BatchNormalization("bn1"))
net.add(layer.Activation("relu1"))
Block(net, "2a", 16, 1)
Block(net, "2b", 16, 1)
Block(net, "2c", 16, 1)
Block(net, "3a", 32, 2)
Block(net, "3b", 32, 1)
Block(net, "3c", 32, 1)
Block(net, "4a", 64, 2)
Block(net, "4b", 64, 1)
Block(net, "4c", 64, 1)
net.add(layer.AvgPooling2D("pool4", 8, 8, border_mode='valid'))
net.add(layer.Flatten('flat'))
net.add(layer.Dense('ip5', 10))
print 'Start intialization............'
for (p, name) in zip(net.param_values(), net.param_names()):
# print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
# initializer.gaussian(p, 0, math.sqrt(2.0/p.shape[1]))
initializer.gaussian(p, 0, 9.0 * p.shape[0])
else:
initializer.uniform(p, p.shape[0], p.shape[1])
else:
p.set_value(0)
# print name, p.l1()
return net
def test_train_one_batch(self):
ffn = net.FeedForwardNet(loss.SoftmaxCrossEntropy())
ffn.add(layer.Conv2D('conv', 4, 3, input_sample_shape=(3, 12, 12)))
ffn.add(layer.Flatten('flat'))
ffn.add(layer.Dense('dense', num_output=4))
for pname, pval in zip(ffn.param_names(), ffn.param_values()):
pval.set_value(0.1)
x = tensor.Tensor((4, 3, 12, 12))
x.gaussian(0, 0.01)
y = np.asarray([[1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0]],
dtype=np.int32)
y = tensor.from_numpy(y)
o = ffn.forward(True, x)
ffn.loss.forward(True, o, y)
g = ffn.loss.backward()
for pname, pvalue, pgrad in ffn.backward(g):
self.assertEqual(len(pvalue), len(pgrad))
for p, g in zip(pvalue, pgrad):
self.assertEqual(p.size(), g.size())
def create_net(in_shape, hyperpara, use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
height, width, kernel_y, kernel_x, stride_y, stride_x = hyperpara[0], hyperpara[1], hyperpara[2], hyperpara[3], hyperpara[4], hyperpara[5]
print ("kernel_x: ", kernel_x)
print ("stride_x: ", stride_x)
net = myffnet.ProbFeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
net.add(layer.Conv2D('conv1', 100, kernel=(kernel_y, kernel_x), stride=(stride_y, stride_x), pad=(0, 0),
input_sample_shape=(int(in_shape[0]), int(in_shape[1]), int(in_shape[2]))))
net.add(layer.Activation('relu1'))
net.add(layer.MaxPooling2D('pool1', 2, 1, pad=0))
net.add(layer.Flatten('flat'))
net.add(layer.Dense('dense', 2))
for (pname, pvalue) in zip(net.param_names(), net.param_values()):
if len(pvalue.shape) > 1:
initializer.gaussian(pvalue, pvalue.shape[0], pvalue.shape[1])
else:
pvalue.set_value(0)
print (pname, pvalue.l1())
return net
decoderw=param['decoder_w']
densew,denseb=param['dense_w'],param['dense_b']
hiddensize=param['hidden_size']
numstacks=param['num_stacks']
drop_out=param['dropout']
vocab_size=7000
cuda = device.create_cuda_gpu_on(1)
encoder = layer.LSTM(name='lstm1', hidden_size=hiddensize, num_stacks=numstacks, dropout=drop_out, input_sample_shape=(vocab_size,))
decoder = layer.LSTM(name='lstm2', hidden_size=hiddensize, num_stacks=numstacks, dropout=drop_out, input_sample_shape=(vocab_size,))
encoder.to_device(cuda)
decoder.to_device(cuda)
encoder_w = encoder.param_values()[0]
encoder_w.uniform(-0.08, 0.08)
decoder.param_values()[0].copy_from_numpy(decoderw, offset=0)
dense = layer.Dense('dense', vocab_size, input_sample_shape=(hiddensize,))
dense.to_device(cuda)
dense.param_values()[0].copy_from_numpy(densew,offset=0)
dense.param_values()[1].copy_from_numpy(denseb,offset=0)
metadata,idx_q,idx_a=load_data()
idx2w=metadata['idx2w']
batchq=idx_q[555:556]
batcha=idx_a[555:556]
inputs=convert(batchq,1,20,vocab_size,cuda)
inputs.append(tensor.Tensor())
inputs.append(tensor.Tensor())
outputs = encoder.forward(model_pb2.kEval, inputs)[-2:]
start = np.zeros((1,1,7000),dtype=np.float32)
def create_net(use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
W0_specs = {'init': 'gaussian', 'mean': 0, 'std': 0.0001}
W1_specs = {'init': 'gaussian', 'mean': 0, 'std': 0.01}
W2_specs = {'init': 'gaussian', 'mean': 0, 'std': 0.01, 'decay_mult': 250}
b_specs = {'init': 'constant', 'value': 0, 'lr_mult': 2, 'decay_mult': 0}
net.add(
layer.Conv2D('conv1',
32,
5,
1,
W_specs=W0_specs.copy(),
b_specs=b_specs.copy(),
pad=2,
input_sample_shape=(
3,
32,
32,
)))
net.add(layer.MaxPooling2D('pool1', 3, 2, pad=1))
net.add(layer.Activation('relu1'))
net.add(layer.LRN(name='lrn1', size=3, alpha=5e-5))
net.add(
layer.Conv2D('conv2',
32,
5,
1,
W_specs=W1_specs.copy(),
b_specs=b_specs.copy(),
pad=2))
net.add(layer.Activation('relu2'))
net.add(layer.AvgPooling2D('pool2', 3, 2, pad=1))
net.add(layer.LRN('lrn2', size=3, alpha=5e-5))
net.add(
layer.Conv2D('conv3',
64,
5,
1,
W_specs=W1_specs.copy(),
b_specs=b_specs.copy(),
pad=2))
net.add(layer.Activation('relu3'))
net.add(layer.AvgPooling2D('pool3', 3, 2, pad=1))
net.add(layer.Flatten('flat'))
net.add(
layer.Dense('dense',
10,
W_specs=W2_specs.copy(),
b_specs=b_specs.copy()))
for (p, specs) in zip(net.param_values(), net.param_specs()):
filler = specs.filler
if filler.type == 'gaussian':
p.gaussian(filler.mean, filler.std)
else:
p.set_value(0)
print specs.name, filler.type, p.l1()
return net
def __init__(self,
dev,
rows=28,
cols=28,
channels=1,
noise_size=100,
hidden_size=128,
batch=128,
interval=1000,
learning_rate=0.001,
epochs=1000000,
d_steps=3,
g_steps=1,
dataset_filepath='mnist.pkl.gz',
file_dir='lsgan_images/'):
self.dev = dev
self.rows = rows
self.cols = cols
self.channels = channels
self.feature_size = self.rows * self.cols * self.channels
self.noise_size = noise_size
self.hidden_size = hidden_size
self.batch = batch
self.batch_size = self.batch // 2
self.interval = interval
self.learning_rate = learning_rate
self.epochs = epochs
self.d_steps = d_steps
self.g_steps = g_steps
self.dataset_filepath = dataset_filepath
self.file_dir = file_dir
self.g_w0_specs = {
'init': 'xavier',
}
self.g_b0_specs = {
'init': 'constant',
'value': 0,
}
self.g_w1_specs = {
'init': 'xavier',
}
self.g_b1_specs = {
'init': 'constant',
'value': 0,
}
self.gen_net = ffnet.FeedForwardNet(loss.SquaredError(), )
self.gen_net_fc_0 = layer.Dense(name='g_fc_0',
num_output=self.hidden_size,
use_bias=True,
W_specs=self.g_w0_specs,
b_specs=self.g_b0_specs,
input_sample_shape=(self.noise_size, ))
self.gen_net_relu_0 = layer.Activation(
name='g_relu_0',
mode='relu',
input_sample_shape=(self.hidden_size, ))
self.gen_net_fc_1 = layer.Dense(
name='g_fc_1',
num_output=self.feature_size,
use_bias=True,
W_specs=self.g_w1_specs,
b_specs=self.g_b1_specs,
input_sample_shape=(self.hidden_size, ))
self.gen_net_sigmoid_1 = layer.Activation(
name='g_relu_1',
mode='sigmoid',
input_sample_shape=(self.feature_size, ))
self.gen_net.add(self.gen_net_fc_0)
self.gen_net.add(self.gen_net_relu_0)
self.gen_net.add(self.gen_net_fc_1)
self.gen_net.add(self.gen_net_sigmoid_1)
for (p, specs) in zip(self.gen_net.param_values(),
self.gen_net.param_specs()):
filler = specs.filler
if filler.type == 'gaussian':
p.gaussian(filler.mean, filler.std)
elif filler.type == 'xavier':
initializer.xavier(p)
else:
p.set_value(0)
print(specs.name, filler.type, p.l1())
self.gen_net.to_device(self.dev)
self.d_w0_specs = {
'init': 'xavier',
}
self.d_b0_specs = {
'init': 'constant',
'value': 0,
}
self.d_w1_specs = {
'init': 'xavier',
}
self.d_b1_specs = {
'init': 'constant',
'value': 0,
}
self.dis_net = ffnet.FeedForwardNet(loss.SquaredError(), )
self.dis_net_fc_0 = layer.Dense(
name='d_fc_0',
num_output=self.hidden_size,
use_bias=True,
W_specs=self.d_w0_specs,
b_specs=self.d_b0_specs,
input_sample_shape=(self.feature_size, ))
self.dis_net_relu_0 = layer.Activation(
name='d_relu_0',
mode='relu',
input_sample_shape=(self.hidden_size, ))
self.dis_net_fc_1 = layer.Dense(
name='d_fc_1',
num_output=1,
use_bias=True,
W_specs=self.d_w1_specs,
b_specs=self.d_b1_specs,
input_sample_shape=(self.hidden_size, ))
self.dis_net.add(self.dis_net_fc_0)
self.dis_net.add(self.dis_net_relu_0)
self.dis_net.add(self.dis_net_fc_1)
for (p, specs) in zip(self.dis_net.param_values(),
self.dis_net.param_specs()):
filler = specs.filler
if filler.type == 'gaussian':
p.gaussian(filler.mean, filler.std)
elif filler.type == 'xavier':
initializer.xavier(p)
else:
p.set_value(0)
print(specs.name, filler.type, p.l1())
self.dis_net.to_device(self.dev)
self.combined_net = ffnet.FeedForwardNet(loss.SquaredError(), )
for l in self.gen_net.layers:
self.combined_net.add(l)
for l in self.dis_net.layers:
self.combined_net.add(l)
self.combined_net.to_device(self.dev)
def train(data,
max_epoch,
hidden_size=100,
seq_length=100,
batch_size=16,
num_stacks=1,
dropout=0.5,
model_path='model'):
# SGD with L2 gradient normalization
opt = optimizer.RMSProp(constraint=optimizer.L2Constraint(5))
cuda = device.create_cuda_gpu()
rnn = layer.LSTM(name='lstm',
hidden_size=hidden_size,
num_stacks=num_stacks,
dropout=dropout,
input_sample_shape=(data.vocab_size, ))
rnn.to_device(cuda)
print 'created rnn'
rnn_w = rnn.param_values()[0]
rnn_w.uniform(-0.08, 0.08) # init all rnn parameters
print 'rnn weight l1 = %f' % (rnn_w.l1())
dense = layer.Dense('dense',
data.vocab_size,
input_sample_shape=(hidden_size, ))
dense.to_device(cuda)
dense_w = dense.param_values()[0]
dense_b = dense.param_values()[1]
print 'dense w ', dense_w.shape
print 'dense b ', dense_b.shape
initializer.uniform(dense_w, dense_w.shape[0], 0)
print 'dense weight l1 = %f' % (dense_w.l1())
dense_b.set_value(0)
print 'dense b l1 = %f' % (dense_b.l1())
g_dense_w = tensor.Tensor(dense_w.shape, cuda)
g_dense_b = tensor.Tensor(dense_b.shape, cuda)
lossfun = loss.SoftmaxCrossEntropy()
for epoch in range(max_epoch):
train_loss = 0
for b in range(data.num_train_batch):
batch = data.train_dat[b * batch_size:(b + 1) * batch_size]
inputs, labels = convert(batch, batch_size, seq_length,
data.vocab_size, cuda)
inputs.append(tensor.Tensor())
inputs.append(tensor.Tensor())
outputs = rnn.forward(model_pb2.kTrain, inputs)[0:-2]
grads = []
batch_loss = 0
g_dense_w.set_value(0.0)
g_dense_b.set_value(0.0)
for output, label in zip(outputs, labels):
act = dense.forward(model_pb2.kTrain, output)
lvalue = lossfun.forward(model_pb2.kTrain, act, label)
batch_loss += lvalue.l1()
grad = lossfun.backward()
grad /= batch_size
grad, gwb = dense.backward(model_pb2.kTrain, grad)
grads.append(grad)
g_dense_w += gwb[0]
g_dense_b += gwb[1]
# print output.l1(), act.l1()
utils.update_progress(
b * 1.0 / data.num_train_batch,
'training loss = %f' % (batch_loss / seq_length))
train_loss += batch_loss
grads.append(tensor.Tensor())
grads.append(tensor.Tensor())
g_rnn_w = rnn.backward(model_pb2.kTrain, grads)[1][0]
dense_w, dense_b = dense.param_values()
opt.apply_with_lr(epoch, get_lr(epoch), g_rnn_w, rnn_w, 'rnnw')
opt.apply_with_lr(epoch, get_lr(epoch), g_dense_w, dense_w,
'dense_w')
opt.apply_with_lr(epoch, get_lr(epoch), g_dense_b, dense_b,
'dense_b')
print '\nEpoch %d, train loss is %f' % \
(epoch, train_loss / data.num_train_batch / seq_length)
eval_loss = 0
for b in range(data.num_test_batch):
batch = data.val_dat[b * batch_size:(b + 1) * batch_size]
inputs, labels = convert(batch, batch_size, seq_length,
data.vocab_size, cuda)
inputs.append(tensor.Tensor())
inputs.append(tensor.Tensor())
outputs = rnn.forward(model_pb2.kEval, inputs)[0:-2]
for output, label in zip(outputs, labels):
output = dense.forward(model_pb2.kEval, output)
eval_loss += lossfun.forward(model_pb2.kEval, output,
label).l1()
print 'Epoch %d, evaluation loss is %f' % \
(epoch, eval_loss / data.num_test_batch / seq_length)
if (epoch + 1) % 30 == 0:
# checkpoint the file model
with open('%s_%d.bin' % (model_path, epoch), 'wb') as fd:
print 'saving model to %s' % model_path
d = {}
for name, w in zip(['rnn_w', 'dense_w', 'dense_b'],
[rnn_w, dense_w, dense_b]):
w.to_host()
d[name] = tensor.to_numpy(w)
w.to_device(cuda)
d['idx_to_char'] = data.idx_to_char
d['char_to_idx'] = data.char_to_idx
d['hidden_size'] = hidden_size
d['num_stacks'] = num_stacks
d['dropout'] = dropout
pickle.dump(d, fd)
def sample(model_path, nsamples=100, seed_text='', do_sample=True):
with open(model_path, 'rb') as fd:
d = pickle.load(fd)
rnn_w = tensor.from_numpy(d['rnn_w'])
idx_to_char = d['idx_to_char']
char_to_idx = d['char_to_idx']
vocab_size = len(idx_to_char)
dense_w = tensor.from_numpy(d['dense_w'])
dense_b = tensor.from_numpy(d['dense_b'])
hidden_size = d['hidden_size']
num_stacks = d['num_stacks']
dropout = d['dropout']
cuda = device.create_cuda_gpu()
rnn = layer.LSTM(name='lstm',
hidden_size=hidden_size,
num_stacks=num_stacks,
dropout=dropout,
input_sample_shape=(len(idx_to_char), ))
rnn.to_device(cuda)
rnn.param_values()[0].copy_data(rnn_w)
dense = layer.Dense('dense',
vocab_size,
input_sample_shape=(hidden_size, ))
dense.to_device(cuda)
dense.param_values()[0].copy_data(dense_w)
dense.param_values()[1].copy_data(dense_b)
hx = tensor.Tensor((num_stacks, 1, hidden_size), cuda)
cx = tensor.Tensor((num_stacks, 1, hidden_size), cuda)
hx.set_value(0.0)
cx.set_value(0.0)
if len(seed_text) > 0:
for c in seed_text:
x = np.zeros((1, vocab_size), dtype=np.float32)
x[0, char_to_idx[c]] = 1
tx = tensor.from_numpy(x)
tx.to_device(cuda)
inputs = [tx, hx, cx]
outputs = rnn.forward(False, inputs)
y = dense.forward(False, outputs[0])
y = tensor.softmax(y)
hx = outputs[1]
cx = outputs[2]
sys.stdout.write(seed_text)
else:
y = tensor.Tensor((1, vocab_size), cuda)
y.set_value(1.0 / vocab_size)
for i in range(nsamples):
y.to_host()
prob = tensor.to_numpy(y)[0]
if do_sample:
cur = np.random.choice(vocab_size, 1, p=prob)[0]
else:
cur = np.argmax(prob)
sys.stdout.write(idx_to_char[cur])
x = np.zeros((1, vocab_size), dtype=np.float32)
x[0, cur] = 1
tx = tensor.from_numpy(x)
tx.to_device(cuda)
inputs = [tx, hx, cx]
outputs = rnn.forward(False, inputs)
y = dense.forward(False, outputs[0])
y = tensor.softmax(y)
hx = outputs[1]
cx = outputs[2]
print('')
def test_dense(self):
dense = layer.Dense('ip', 32, input_sample_shape=(64, ))
out_sample_shape = dense.get_output_sample_shape()
self.check_shape(out_sample_shape, (32, ))
def create_net(input_shape, use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
ConvBnReLU(net, 'conv1_1', 64, input_shape)
#net.add(layer.Dropout('drop1', 0.3))
net.add(layer.MaxPooling2D('pool0', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv1_2', 128)
net.add(layer.MaxPooling2D('pool1', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv2_1', 128)
net.add(layer.Dropout('drop2_1', 0.4))
ConvBnReLU(net, 'conv2_2', 128)
net.add(layer.MaxPooling2D('pool2', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv3_1', 256)
net.add(layer.Dropout('drop3_1', 0.4))
ConvBnReLU(net, 'conv3_2', 256)
net.add(layer.Dropout('drop3_2', 0.4))
ConvBnReLU(net, 'conv3_3', 256)
net.add(layer.MaxPooling2D('pool3', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv4_1', 256)
net.add(layer.Dropout('drop4_1', 0.4))
ConvBnReLU(net, 'conv4_2', 256)
net.add(layer.Dropout('drop4_2', 0.4))
ConvBnReLU(net, 'conv4_3', 256)
net.add(layer.MaxPooling2D('pool4', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv5_1', 512)
net.add(layer.Dropout('drop5_1', 0.4))
ConvBnReLU(net, 'conv5_2', 512)
net.add(layer.Dropout('drop5_2', 0.4))
ConvBnReLU(net, 'conv5_3', 512)
net.add(layer.MaxPooling2D('pool5', 2, 2, border_mode='valid'))
#ConvBnReLU(net, 'conv6_1', 512)
#net.add(layer.Dropout('drop6_1', 0.4))
#ConvBnReLU(net, 'conv6_2', 512)
#net.add(layer.Dropout('drop6_2', 0.4))
#ConvBnReLU(net, 'conv6_3', 512)
#net.add(layer.MaxPooling2D('pool6', 2, 2, border_mode='valid'))
#ConvBnReLU(net, 'conv7_1', 512)
#net.add(layer.Dropout('drop7_1', 0.4))
#ConvBnReLU(net, 'conv7_2', 512)
#net.add(layer.Dropout('drop7_2', 0.4))
#ConvBnReLU(net, 'conv7_3', 512)
#net.add(layer.MaxPooling2D('pool7', 2, 2, border_mode='valid'))
net.add(layer.Flatten('flat'))
net.add(layer.Dense('ip1', 256))
net.add(layer.BatchNormalization('bn1'))
net.add(layer.Activation('relu1'))
net.add(layer.Dropout('dropout1', 0.2))
net.add(layer.Dense('ip2', 16))
net.add(layer.BatchNormalization('bn2'))
net.add(layer.Activation('relu2'))
net.add(layer.Dropout('dropout2', 0.2))
net.add(layer.Dense('ip3', 2))
print 'Parameter intialization............'
for (p, name) in zip(net.param_values(), net.param_names()):
print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
initializer.gaussian(p, 0, p.size())
else:
p.gaussian(0, 0.02)
else:
p.set_value(0)
print name, p.l1()
return net
