def test_mult_inputs(self):
ffn = net.FeedForwardNet(loss.SoftmaxCrossEntropy())
s1 = ffn.add(layer.Activation('relu1', input_sample_shape=(2, )), [])
s2 = ffn.add(layer.Activation('relu2', input_sample_shape=(2, )), [])
ffn.add(layer.Merge('merge', input_sample_shape=(2, )), [s1, s2])
x1 = tensor.Tensor((2, 2))
x1.set_value(1.1)
x2 = tensor.Tensor((2, 2))
x2.set_value(0.9)
out = ffn.forward(False, {'relu1': x1, 'relu2': x2})
out = tensor.to_numpy(out)
self.assertAlmostEqual(np.average(out), 2)
def test_single_input_output(self):
ffn = net.FeedForwardNet(loss.SoftmaxCrossEntropy())
ffn.add(layer.Activation('relu1', input_sample_shape=(2,)))
ffn.add(layer.Activation('relu2'))
x = np.array([[-1, 1], [1, 1], [-1, -2]], dtype=np.float32)
x = tensor.from_numpy(x)
y = tensor.Tensor((3,))
y.set_value(0)
out, _ = ffn.evaluate(x, y)
self.assertAlmostEqual(out * 3,
- math.log(1.0/(1+math.exp(1))) - math.log(0.5) -math.log(0.5),
5);
def create_net(input_shape, use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
ConvBnReLUPool(net, 'conv1', 32, input_shape)
ConvBnReLUPool(net, 'conv2', 64)
ConvBnReLUPool(net, 'conv3', 128)
ConvBnReLUPool(net, 'conv4', 128)
ConvBnReLUPool(net, 'conv5', 256)
ConvBnReLUPool(net, 'conv6', 256)
ConvBnReLUPool(net, 'conv7', 512)
ConvBnReLUPool(net, 'conv8', 512)
net.add(layer.Flatten('flat'))
net.add(layer.Dense('ip1', 256))
net.add(layer.BatchNormalization('bn1'))
net.add(layer.Activation('relu1'))
net.add(layer.Dropout('dropout1', 0.2))
net.add(layer.Dense('ip2', 16))
net.add(layer.BatchNormalization('bn2'))
net.add(layer.Activation('relu2'))
net.add(layer.Dropout('dropout2', 0.2))
net.add(layer.Dense('ip3', 2))
print 'Parameter intialization............'
for (p, name) in zip(net.param_values(), net.param_names()):
print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
initializer.gaussian(p, 0, p.size())
else:
p.gaussian(0, 0.02)
else:
p.set_value(0)
print name, p.l1()
return net
def create_net(use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
ConvBnReLU(net, 'conv1_1', 64, (3, 32, 32))
net.add(layer.Dropout('drop1', 0.3))
ConvBnReLU(net, 'conv1_2', 64)
net.add(layer.MaxPooling2D('pool1', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv2_1', 128)
net.add(layer.Dropout('drop2_1', 0.4))
ConvBnReLU(net, 'conv2_2', 128)
net.add(layer.MaxPooling2D('pool2', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv3_1', 256)
net.add(layer.Dropout('drop3_1', 0.4))
ConvBnReLU(net, 'conv3_2', 256)
net.add(layer.Dropout('drop3_2', 0.4))
ConvBnReLU(net, 'conv3_3', 256)
net.add(layer.MaxPooling2D('pool3', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv4_1', 512)
net.add(layer.Dropout('drop4_1', 0.4))
ConvBnReLU(net, 'conv4_2', 512)
net.add(layer.Dropout('drop4_2', 0.4))
ConvBnReLU(net, 'conv4_3', 512)
net.add(layer.MaxPooling2D('pool4', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv5_1', 512)
net.add(layer.Dropout('drop5_1', 0.4))
ConvBnReLU(net, 'conv5_2', 512)
net.add(layer.Dropout('drop5_2', 0.4))
ConvBnReLU(net, 'conv5_3', 512)
net.add(layer.MaxPooling2D('pool5', 2, 2, border_mode='valid'))
net.add(layer.Flatten('flat'))
net.add(layer.Dropout('drop_flat', 0.5))
net.add(layer.Dense('ip1', 512))
net.add(layer.BatchNormalization('batchnorm_ip1'))
net.add(layer.Activation('relu_ip1'))
net.add(layer.Dropout('drop_ip2', 0.5))
net.add(layer.Dense('ip2', 10))
print 'Start intialization............'
for (p, name) in zip(net.param_values(), net.param_names()):
print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
initializer.gaussian(p, 0, 3 * 3 * p.shape[0])
else:
p.gaussian(0, 0.02)
else:
p.set_value(0)
print name, p.l1()
return net
def ConvBnReLU(net, name, nb_filers, sample_shape=None):
net.add(
layer.Conv2D(name + '_1',
nb_filers,
3,
1,
pad=1,
input_sample_shape=sample_shape))
net.add(layer.BatchNormalization(name + '_2'))
net.add(layer.Activation(name + '_3'))
def ConvBnReLUPool(net, name, nb_filers, sample_shape=None):
net.add(
layer.Conv2D(name + '_conv',
nb_filers,
3,
1,
pad=1,
input_sample_shape=sample_shape))
net.add(layer.BatchNormalization(name + '_bn'))
net.add(layer.Activation(name + '_relu'))
net.add(layer.MaxPooling2D(name + '_pool', 2, 2, border_mode='valid'))
def create_net(depth, nb_classes, dense=0, use_cpu=True):
if use_cpu:
layer.engine = 'singacpp'
net = densenet_base(depth)
# this part was not included in the pytorch model
if dense > 0:
net.add(layer.Dense('hidden-dense', dense))
net.add(layer.Activation('act-dense'))
net.add(layer.Dropout('dropout'))
net.add(layer.Dense('sigmoid', nb_classes))
return net
def Block(net, name, nb_filters, stride):
split = net.add(layer.Split(name + "-split", 2))
if stride > 1:
net.add(layer.Conv2D(name + "-br2-conv", nb_filters, 1, stride, pad=0),
split)
br2bn = net.add(layer.BatchNormalization(name + "-br2-bn"))
net.add(layer.Conv2D(name + "-br1-conv1", nb_filters, 3, stride, pad=1),
split)
net.add(layer.BatchNormalization(name + "-br1-bn1"))
net.add(layer.Activation(name + "-br1-relu"))
net.add(layer.Conv2D(name + "-br1-conv2", nb_filters, 3, 1, pad=1))
br1bn2 = net.add(layer.BatchNormalization(name + "-br1-bn2"))
if stride > 1:
net.add(layer.Merge(name + "-merge"), [br1bn2, br2bn])
else:
net.add(layer.Merge(name + "-merge"), [br1bn2, split])
def create_net(input_shape, use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
net.add(
layer.Conv2D('conv1',
nb_kernels=32,
kernel=7,
stride=3,
pad=1,
input_sample_shape=input_shape))
net.add(layer.Activation('relu1'))
net.add(layer.MaxPooling2D('pool1', 2, 2, border_mode='valid'))
net.add(layer.Conv2D('conv2', nb_kernels=64, kernel=5, stride=3))
net.add(layer.Activation('relu2'))
net.add(layer.MaxPooling2D('pool2', 2, 2, border_mode='valid'))
net.add(layer.Conv2D('conv3', nb_kernels=128, kernel=3, stride=1, pad=2))
net.add(layer.Activation('relu3'))
net.add(layer.MaxPooling2D('pool3', 2, 2, border_mode='valid'))
net.add(layer.Conv2D('conv4', nb_kernels=256, kernel=3, stride=1))
net.add(layer.Activation('relu4'))
net.add(layer.MaxPooling2D('pool4', 2, 2, border_mode='valid'))
net.add(layer.Flatten('flat'))
net.add(layer.Dense('ip5', 256))
net.add(layer.Activation('relu5'))
net.add(layer.Dense('ip6', 16))
net.add(layer.Activation('relu6'))
net.add(layer.Dense('ip7', 2))
print 'Parameter intialization............'
for (p, name) in zip(net.param_values(), net.param_names()):
print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
initializer.gaussian(p, 0, p.size())
else:
p.gaussian(0, 0.02)
else:
p.set_value(0)
print name, p.l1()
return net
def create_net(use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
net.add(
layer.Conv2D("conv1", 16, 3, 1, pad=1, input_sample_shape=(3, 32, 32)))
net.add(layer.BatchNormalization("bn1"))
net.add(layer.Activation("relu1"))
Block(net, "2a", 16, 1)
Block(net, "2b", 16, 1)
Block(net, "2c", 16, 1)
Block(net, "3a", 32, 2)
Block(net, "3b", 32, 1)
Block(net, "3c", 32, 1)
Block(net, "4a", 64, 2)
Block(net, "4b", 64, 1)
Block(net, "4c", 64, 1)
net.add(layer.AvgPooling2D("pool4", 8, 8, border_mode='valid'))
net.add(layer.Flatten('flat'))
net.add(layer.Dense('ip5', 10))
print 'Start intialization............'
for (p, name) in zip(net.param_values(), net.param_names()):
# print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
# initializer.gaussian(p, 0, math.sqrt(2.0/p.shape[1]))
initializer.gaussian(p, 0, 9.0 * p.shape[0])
else:
initializer.uniform(p, p.shape[0], p.shape[1])
else:
p.set_value(0)
# print name, p.l1()
return net
def create_net(in_shape, hyperpara, use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
height, width, kernel_y, kernel_x, stride_y, stride_x = hyperpara[0], hyperpara[1], hyperpara[2], hyperpara[3], hyperpara[4], hyperpara[5]
print ("kernel_x: ", kernel_x)
print ("stride_x: ", stride_x)
net = myffnet.ProbFeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
net.add(layer.Conv2D('conv1', 100, kernel=(kernel_y, kernel_x), stride=(stride_y, stride_x), pad=(0, 0),
input_sample_shape=(int(in_shape[0]), int(in_shape[1]), int(in_shape[2]))))
net.add(layer.Activation('relu1'))
net.add(layer.MaxPooling2D('pool1', 2, 1, pad=0))
net.add(layer.Flatten('flat'))
net.add(layer.Dense('dense', 2))
for (pname, pvalue) in zip(net.param_names(), net.param_values()):
if len(pvalue.shape) > 1:
initializer.gaussian(pvalue, pvalue.shape[0], pvalue.shape[1])
else:
pvalue.set_value(0)
print (pname, pvalue.l1())
return net
def build_net(self):
if self.use_cpu:
layer.engine = 'singacpp'
else:
layer.engine = 'cudnn'
self.net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(),
metric.Accuracy())
self.net.add(
Reshape('reshape1', (self.vocab_size, ),
input_sample_shape=(self.maxlen, self.vocab_size)))
self.net.add(layer.Dense('embed',
self.embed_size)) # output: (embed_size, )
self.net.add(layer.Dropout('dropout'))
self.net.add(Reshape('reshape2', (1, self.maxlen, self.embed_size)))
self.net.add(
layer.Conv2D('conv',
self.filters, (self.kernel_size, self.embed_size),
border_mode='valid')) # output: (filter, embed_size)
if self.use_cpu == False:
self.net.add(layer.BatchNormalization('batchNorm'))
self.net.add(layer.Activation('activ')) # output: (filter, embed_size)
self.net.add(layer.MaxPooling2D('max', stride=self.pool_size))
self.net.add(layer.Flatten('flatten'))
self.net.add(layer.Dense('dense', 2))
def create_net(use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
W0_specs = {'init': 'gaussian', 'mean': 0, 'std': 0.0001}
W1_specs = {'init': 'gaussian', 'mean': 0, 'std': 0.01}
W2_specs = {'init': 'gaussian', 'mean': 0, 'std': 0.01, 'decay_mult': 250}
b_specs = {'init': 'constant', 'value': 0, 'lr_mult': 2, 'decay_mult': 0}
net.add(
layer.Conv2D('conv1',
32,
5,
1,
W_specs=W0_specs.copy(),
b_specs=b_specs.copy(),
pad=2,
input_sample_shape=(
3,
32,
32,
)))
net.add(layer.MaxPooling2D('pool1', 3, 2, pad=1))
net.add(layer.Activation('relu1'))
net.add(layer.LRN(name='lrn1', size=3, alpha=5e-5))
net.add(
layer.Conv2D('conv2',
32,
5,
1,
W_specs=W1_specs.copy(),
b_specs=b_specs.copy(),
pad=2))
net.add(layer.Activation('relu2'))
net.add(layer.AvgPooling2D('pool2', 3, 2, pad=1))
net.add(layer.LRN('lrn2', size=3, alpha=5e-5))
net.add(
layer.Conv2D('conv3',
64,
5,
1,
W_specs=W1_specs.copy(),
b_specs=b_specs.copy(),
pad=2))
net.add(layer.Activation('relu3'))
net.add(layer.AvgPooling2D('pool3', 3, 2, pad=1))
net.add(layer.Flatten('flat'))
net.add(
layer.Dense('dense',
10,
W_specs=W2_specs.copy(),
b_specs=b_specs.copy()))
for (p, specs) in zip(net.param_values(), net.param_specs()):
filler = specs.filler
if filler.type == 'gaussian':
p.gaussian(filler.mean, filler.std)
else:
p.set_value(0)
print specs.name, filler.type, p.l1()
return net
def __init__(self,
dev,
rows=28,
cols=28,
channels=1,
noise_size=100,
hidden_size=128,
batch=128,
interval=1000,
learning_rate=0.001,
epochs=1000000,
d_steps=3,
g_steps=1,
dataset_filepath='mnist.pkl.gz',
file_dir='lsgan_images/'):
self.dev = dev
self.rows = rows
self.cols = cols
self.channels = channels
self.feature_size = self.rows * self.cols * self.channels
self.noise_size = noise_size
self.hidden_size = hidden_size
self.batch = batch
self.batch_size = self.batch // 2
self.interval = interval
self.learning_rate = learning_rate
self.epochs = epochs
self.d_steps = d_steps
self.g_steps = g_steps
self.dataset_filepath = dataset_filepath
self.file_dir = file_dir
self.g_w0_specs = {
'init': 'xavier',
}
self.g_b0_specs = {
'init': 'constant',
'value': 0,
}
self.g_w1_specs = {
'init': 'xavier',
}
self.g_b1_specs = {
'init': 'constant',
'value': 0,
}
self.gen_net = ffnet.FeedForwardNet(loss.SquaredError(), )
self.gen_net_fc_0 = layer.Dense(name='g_fc_0',
num_output=self.hidden_size,
use_bias=True,
W_specs=self.g_w0_specs,
b_specs=self.g_b0_specs,
input_sample_shape=(self.noise_size, ))
self.gen_net_relu_0 = layer.Activation(
name='g_relu_0',
mode='relu',
input_sample_shape=(self.hidden_size, ))
self.gen_net_fc_1 = layer.Dense(
name='g_fc_1',
num_output=self.feature_size,
use_bias=True,
W_specs=self.g_w1_specs,
b_specs=self.g_b1_specs,
input_sample_shape=(self.hidden_size, ))
self.gen_net_sigmoid_1 = layer.Activation(
name='g_relu_1',
mode='sigmoid',
input_sample_shape=(self.feature_size, ))
self.gen_net.add(self.gen_net_fc_0)
self.gen_net.add(self.gen_net_relu_0)
self.gen_net.add(self.gen_net_fc_1)
self.gen_net.add(self.gen_net_sigmoid_1)
for (p, specs) in zip(self.gen_net.param_values(),
self.gen_net.param_specs()):
filler = specs.filler
if filler.type == 'gaussian':
p.gaussian(filler.mean, filler.std)
elif filler.type == 'xavier':
initializer.xavier(p)
else:
p.set_value(0)
print(specs.name, filler.type, p.l1())
self.gen_net.to_device(self.dev)
self.d_w0_specs = {
'init': 'xavier',
}
self.d_b0_specs = {
'init': 'constant',
'value': 0,
}
self.d_w1_specs = {
'init': 'xavier',
}
self.d_b1_specs = {
'init': 'constant',
'value': 0,
}
self.dis_net = ffnet.FeedForwardNet(loss.SquaredError(), )
self.dis_net_fc_0 = layer.Dense(
name='d_fc_0',
num_output=self.hidden_size,
use_bias=True,
W_specs=self.d_w0_specs,
b_specs=self.d_b0_specs,
input_sample_shape=(self.feature_size, ))
self.dis_net_relu_0 = layer.Activation(
name='d_relu_0',
mode='relu',
input_sample_shape=(self.hidden_size, ))
self.dis_net_fc_1 = layer.Dense(
name='d_fc_1',
num_output=1,
use_bias=True,
W_specs=self.d_w1_specs,
b_specs=self.d_b1_specs,
input_sample_shape=(self.hidden_size, ))
self.dis_net.add(self.dis_net_fc_0)
self.dis_net.add(self.dis_net_relu_0)
self.dis_net.add(self.dis_net_fc_1)
for (p, specs) in zip(self.dis_net.param_values(),
self.dis_net.param_specs()):
filler = specs.filler
if filler.type == 'gaussian':
p.gaussian(filler.mean, filler.std)
elif filler.type == 'xavier':
initializer.xavier(p)
else:
p.set_value(0)
print(specs.name, filler.type, p.l1())
self.dis_net.to_device(self.dev)
self.combined_net = ffnet.FeedForwardNet(loss.SquaredError(), )
for l in self.gen_net.layers:
self.combined_net.add(l)
for l in self.dis_net.layers:
self.combined_net.add(l)
self.combined_net.to_device(self.dev)
def test_activation(self):
input_sample_shape = (64, 1, 12)
act = layer.Activation('act', input_sample_shape=input_sample_shape)
out_sample_shape = act.get_output_sample_shape()
self.check_shape(out_sample_shape, input_sample_shape)
def create_net(input_shape, use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
ConvBnReLU(net, 'conv1_1', 64, input_shape)
#net.add(layer.Dropout('drop1', 0.3))
net.add(layer.MaxPooling2D('pool0', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv1_2', 128)
net.add(layer.MaxPooling2D('pool1', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv2_1', 128)
net.add(layer.Dropout('drop2_1', 0.4))
ConvBnReLU(net, 'conv2_2', 128)
net.add(layer.MaxPooling2D('pool2', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv3_1', 256)
net.add(layer.Dropout('drop3_1', 0.4))
ConvBnReLU(net, 'conv3_2', 256)
net.add(layer.Dropout('drop3_2', 0.4))
ConvBnReLU(net, 'conv3_3', 256)
net.add(layer.MaxPooling2D('pool3', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv4_1', 256)
net.add(layer.Dropout('drop4_1', 0.4))
ConvBnReLU(net, 'conv4_2', 256)
net.add(layer.Dropout('drop4_2', 0.4))
ConvBnReLU(net, 'conv4_3', 256)
net.add(layer.MaxPooling2D('pool4', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv5_1', 512)
net.add(layer.Dropout('drop5_1', 0.4))
ConvBnReLU(net, 'conv5_2', 512)
net.add(layer.Dropout('drop5_2', 0.4))
ConvBnReLU(net, 'conv5_3', 512)
net.add(layer.MaxPooling2D('pool5', 2, 2, border_mode='valid'))
#ConvBnReLU(net, 'conv6_1', 512)
#net.add(layer.Dropout('drop6_1', 0.4))
#ConvBnReLU(net, 'conv6_2', 512)
#net.add(layer.Dropout('drop6_2', 0.4))
#ConvBnReLU(net, 'conv6_3', 512)
#net.add(layer.MaxPooling2D('pool6', 2, 2, border_mode='valid'))
#ConvBnReLU(net, 'conv7_1', 512)
#net.add(layer.Dropout('drop7_1', 0.4))
#ConvBnReLU(net, 'conv7_2', 512)
#net.add(layer.Dropout('drop7_2', 0.4))
#ConvBnReLU(net, 'conv7_3', 512)
#net.add(layer.MaxPooling2D('pool7', 2, 2, border_mode='valid'))
net.add(layer.Flatten('flat'))
net.add(layer.Dense('ip1', 256))
net.add(layer.BatchNormalization('bn1'))
net.add(layer.Activation('relu1'))
net.add(layer.Dropout('dropout1', 0.2))
net.add(layer.Dense('ip2', 16))
net.add(layer.BatchNormalization('bn2'))
net.add(layer.Activation('relu2'))
net.add(layer.Dropout('dropout2', 0.2))
net.add(layer.Dense('ip3', 2))
print 'Parameter intialization............'
for (p, name) in zip(net.param_values(), net.param_names()):
print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
initializer.gaussian(p, 0, p.size())
else:
p.gaussian(0, 0.02)
else:
p.set_value(0)
print name, p.l1()
return net
