def train(data_dir, net, num_epoch=20, batch_size=250):
print 'Start intialization............'
cuda = device.create_cuda_gpu()
net.to_device(cuda)
opt = optimizer.SGD(momentum=0.9,weight_decay=0.04)
for (p, specs) in zip(net.param_values(), net.param_specs()):
filler = specs.filler
if filler.type == 'gaussian':
initializer.gaussian(p, filler.mean, filler.std)
else:
p.set_value(0)
opt.register(p, specs)
print specs.name, filler.type, p.l1()
print 'Loading data ..................'
train_x, train_y = load_dataset(data_dir,1)
test_x, test_y = load_dataset(data_dir,2)
tx = tensor.Tensor((batch_size,3), cuda)
ty = tensor.Tensor((batch_size,),cuda, core_pb2.kInt)
#ta = tensor.Tensor((batch_size,3), cuda)
#tb = tensor.Tensor((batch_size,),cuda, core_pb2.kInt)
num_train_batch = train_x.shape[0]/batch_size
num_test_batch = test_x.shape[0]/batch_size
idx = np.arange(train_x.shape[0], dtype=np.int32)
id = np.arange(test_x.shape[0],dtype=np.int32)
for epoch in range(num_epoch):
np.random.shuffle(idx)
loss, acc = 0.000,0.000
print 'Epoch %d' % epoch
for b in range(num_train_batch):
x = train_x[idx[b * batch_size:(b+1)* batch_size]]
y = train_y[idx[b * batch_size:(b+1)* batch_size]]
tx.copy_from_numpy(x)
ty.copy_from_numpy(y)
grads, (l, a) = net.train(tx, ty)
loss += l
acc += a
for (s, p, g) in zip(net.param_specs(), net.param_values(), grads):
opt.apply_with_lr(epoch, get_lr(epoch), g, p, str(s.name))
# update progress bar
utils.update_progress(b * 1.0 / num_train_batch,
'training loss = %f, accuracy = %f' % (l, a))
info = '\ntraining loss = %f, training accuracy = %f' \
% (loss/num_train_batch, acc/num_train_batch)
print info
loss,acc=0.000,0.000
np.random.shuffle(id)
for b in range(num_test_batch):
x = test_x[b * batch_size:(b+1) * batch_size]
y = test_y[b * batch_size:(b+1) * batch_size]
tx.copy_from_numpy(x)
ty.copy_from_numpy(y)
l, a = net.evaluate(tx, ty)
loss += l
acc += a
print 'test loss = %f, test accuracy = %f' \
% (loss / num_test_batch, acc / num_test_batch)
net.save('model.bin') # save model params into checkpoint file
def init_params(net, weight_path=None, is_train=False):
'''Init parameters randomly or from checkpoint file.
Args:
net, a constructed neural net
weight_path, checkpoint file path
is_train, if false, then a checkpoint file must be presented
'''
assert is_train is True or weight_path is not None, \
'must provide a checkpoint file for serving'
if weight_path is None:
for pname, pval in zip(net.param_names(), net.param_values()):
if 'conv' in pname and len(pval.shape) > 1:
initializer.gaussian(pval, 0, pval.shape[1])
elif 'dense' in pname:
if len(pval.shape) > 1:
initializer.gaussian(pval, 0, pval.shape[0])
else:
pval.set_value(0)
# init params from batch norm layer
elif 'mean' in pname or 'beta' in pname:
pval.set_value(0)
elif 'var' in pname:
pval.set_value(1)
elif 'gamma' in pname:
initializer.uniform(pval, 0, 1)
else:
net.load(weight_path, use_pickle=True)
def init_params(net, weight_path=None, is_train=False):
'''Init parameters randomly or from checkpoint file.
Args:
net, a constructed neural net
weight_path, checkpoint file path
is_train, if false, then a checkpoint file must be presented
'''
assert is_train is True or weight_path is not None, \
'must provide a checkpoint file for serving'
if weight_path is None:
for pname, pval in zip(net.param_names(), net.param_values()):
if 'conv' in pname and len(pval.shape) > 1:
initializer.gaussian(pval, 0, pval.shape[1])
elif 'dense' in pname:
if len(pval.shape) > 1:
initializer.gaussian(pval, 0, pval.shape[0])
else:
pval.set_value(0)
# init params from batch norm layer
elif 'mean' in pname or 'beta' in pname:
pval.set_value(0)
elif 'var' in pname:
pval.set_value(1)
elif 'gamma' in pname:
initializer.uniform(pval, 0, 1)
else:
net.load(weight_path, use_pickle=True)
def create_net(use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
ConvBnReLU(net, 'conv1_1', 64, (3, 32, 32))
net.add(layer.Dropout('drop1', 0.3))
ConvBnReLU(net, 'conv1_2', 64)
net.add(layer.MaxPooling2D('pool1', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv2_1', 128)
net.add(layer.Dropout('drop2_1', 0.4))
ConvBnReLU(net, 'conv2_2', 128)
net.add(layer.MaxPooling2D('pool2', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv3_1', 256)
net.add(layer.Dropout('drop3_1', 0.4))
ConvBnReLU(net, 'conv3_2', 256)
net.add(layer.Dropout('drop3_2', 0.4))
ConvBnReLU(net, 'conv3_3', 256)
net.add(layer.MaxPooling2D('pool3', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv4_1', 512)
net.add(layer.Dropout('drop4_1', 0.4))
ConvBnReLU(net, 'conv4_2', 512)
net.add(layer.Dropout('drop4_2', 0.4))
ConvBnReLU(net, 'conv4_3', 512)
net.add(layer.MaxPooling2D('pool4', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv5_1', 512)
net.add(layer.Dropout('drop5_1', 0.4))
ConvBnReLU(net, 'conv5_2', 512)
net.add(layer.Dropout('drop5_2', 0.4))
ConvBnReLU(net, 'conv5_3', 512)
net.add(layer.MaxPooling2D('pool5', 2, 2, border_mode='valid'))
net.add(layer.Flatten('flat'))
net.add(layer.Dropout('drop_flat', 0.5))
net.add(layer.Dense('ip1', 512))
net.add(layer.BatchNormalization('batchnorm_ip1'))
net.add(layer.Activation('relu_ip1'))
net.add(layer.Dropout('drop_ip2', 0.5))
net.add(layer.Dense('ip2', 10))
print('Start intialization............')
for (p, name) in zip(net.param_values(), net.param_names()):
print(name, p.shape)
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
initializer.gaussian(p, 0, 3 * 3 * p.shape[0])
else:
p.gaussian(0, 0.02)
else:
p.set_value(0)
print(name, p.l1())
return net
def create_net(use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
ConvBnReLU(net, 'conv1_1', 64, (3, 32, 32))
net.add(layer.Dropout('drop1', 0.3))
ConvBnReLU(net, 'conv1_2', 64)
net.add(layer.MaxPooling2D('pool1', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv2_1', 128)
net.add(layer.Dropout('drop2_1', 0.4))
ConvBnReLU(net, 'conv2_2', 128)
net.add(layer.MaxPooling2D('pool2', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv3_1', 256)
net.add(layer.Dropout('drop3_1', 0.4))
ConvBnReLU(net, 'conv3_2', 256)
net.add(layer.Dropout('drop3_2', 0.4))
ConvBnReLU(net, 'conv3_3', 256)
net.add(layer.MaxPooling2D('pool3', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv4_1', 512)
net.add(layer.Dropout('drop4_1', 0.4))
ConvBnReLU(net, 'conv4_2', 512)
net.add(layer.Dropout('drop4_2', 0.4))
ConvBnReLU(net, 'conv4_3', 512)
net.add(layer.MaxPooling2D('pool4', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv5_1', 512)
net.add(layer.Dropout('drop5_1', 0.4))
ConvBnReLU(net, 'conv5_2', 512)
net.add(layer.Dropout('drop5_2', 0.4))
ConvBnReLU(net, 'conv5_3', 512)
net.add(layer.MaxPooling2D('pool5', 2, 2, border_mode='valid'))
net.add(layer.Flatten('flat'))
net.add(layer.Dropout('drop_flat', 0.5))
net.add(layer.Dense('ip1', 512))
net.add(layer.BatchNormalization('batchnorm_ip1'))
net.add(layer.Activation('relu_ip1'))
net.add(layer.Dropout('drop_ip2', 0.5))
net.add(layer.Dense('ip2', 10))
print 'Start intialization............'
for (p, name) in zip(net.param_values(), net.param_names()):
print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
initializer.gaussian(p, 0, 3 * 3 * p.shape[0])
else:
p.gaussian(0, 0.02)
else:
p.set_value(0)
print name, p.l1()
return net
def create_net(input_shape, use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
net.add(
layer.Conv2D('conv1',
nb_kernels=32,
kernel=7,
stride=3,
pad=1,
input_sample_shape=input_shape))
net.add(layer.Activation('relu1'))
net.add(layer.MaxPooling2D('pool1', 2, 2, border_mode='valid'))
net.add(layer.Conv2D('conv2', nb_kernels=64, kernel=5, stride=3))
net.add(layer.Activation('relu2'))
net.add(layer.MaxPooling2D('pool2', 2, 2, border_mode='valid'))
net.add(layer.Conv2D('conv3', nb_kernels=128, kernel=3, stride=1, pad=2))
net.add(layer.Activation('relu3'))
net.add(layer.MaxPooling2D('pool3', 2, 2, border_mode='valid'))
net.add(layer.Conv2D('conv4', nb_kernels=256, kernel=3, stride=1))
net.add(layer.Activation('relu4'))
net.add(layer.MaxPooling2D('pool4', 2, 2, border_mode='valid'))
net.add(layer.Flatten('flat'))
net.add(layer.Dense('ip5', 256))
net.add(layer.Activation('relu5'))
net.add(layer.Dense('ip6', 16))
net.add(layer.Activation('relu6'))
net.add(layer.Dense('ip7', 2))
print 'Parameter intialization............'
for (p, name) in zip(net.param_values(), net.param_names()):
print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
initializer.gaussian(p, 0, p.size())
else:
p.gaussian(0, 0.02)
else:
p.set_value(0)
print name, p.l1()
return net
def create_net(input_shape, use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
ConvBnReLUPool(net, 'conv1', 32, input_shape)
ConvBnReLUPool(net, 'conv2', 64)
ConvBnReLUPool(net, 'conv3', 128)
ConvBnReLUPool(net, 'conv4', 128)
ConvBnReLUPool(net, 'conv5', 256)
ConvBnReLUPool(net, 'conv6', 256)
ConvBnReLUPool(net, 'conv7', 512)
ConvBnReLUPool(net, 'conv8', 512)
net.add(layer.Flatten('flat'))
net.add(layer.Dense('ip1', 256))
net.add(layer.BatchNormalization('bn1'))
net.add(layer.Activation('relu1'))
net.add(layer.Dropout('dropout1', 0.2))
net.add(layer.Dense('ip2', 16))
net.add(layer.BatchNormalization('bn2'))
net.add(layer.Activation('relu2'))
net.add(layer.Dropout('dropout2', 0.2))
net.add(layer.Dense('ip3', 2))
print 'Parameter intialization............'
for (p, name) in zip(net.param_values(), net.param_names()):
print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
initializer.gaussian(p, 0, p.size())
else:
p.gaussian(0, 0.02)
else:
p.set_value(0)
print name, p.l1()
return net
def create_net(use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
net.add(
layer.Conv2D("conv1", 16, 3, 1, pad=1, input_sample_shape=(3, 32, 32)))
net.add(layer.BatchNormalization("bn1"))
net.add(layer.Activation("relu1"))
Block(net, "2a", 16, 1)
Block(net, "2b", 16, 1)
Block(net, "2c", 16, 1)
Block(net, "3a", 32, 2)
Block(net, "3b", 32, 1)
Block(net, "3c", 32, 1)
Block(net, "4a", 64, 2)
Block(net, "4b", 64, 1)
Block(net, "4c", 64, 1)
net.add(layer.AvgPooling2D("pool4", 8, 8, border_mode='valid'))
net.add(layer.Flatten('flat'))
net.add(layer.Dense('ip5', 10))
print 'Start intialization............'
for (p, name) in zip(net.param_values(), net.param_names()):
# print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
# initializer.gaussian(p, 0, math.sqrt(2.0/p.shape[1]))
initializer.gaussian(p, 0, 9.0 * p.shape[0])
else:
initializer.uniform(p, p.shape[0], p.shape[1])
else:
p.set_value(0)
# print name, p.l1()
return net
def init_params(net, weight_path=None):
if weight_path is None:
for pname, pval in zip(net.param_names(), net.param_values()):
print(pname, pval.shape)
if 'conv' in pname and len(pval.shape) > 1:
initializer.gaussian(pval, 0, pval.shape[1])
elif 'dense' in pname:
if len(pval.shape) > 1:
initializer.gaussian(pval, 0, pval.shape[0])
else:
pval.set_value(0)
# init params from batch norm layer
elif 'mean' in pname or 'beta' in pname:
pval.set_value(0)
elif 'var' in pname or 'gamma' in pname:
pval.set_value(1)
else:
net.load(weight_path, use_pickle='pickle' in weight_path)
def create_net(use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
net.add(layer.Conv2D("conv1", 16, 3, 1, pad=1, input_sample_shape=(3, 32, 32)))
net.add(layer.BatchNormalization("bn1"))
net.add(layer.Activation("relu1"))
Block(net, "2a", 16, 1)
Block(net, "2b", 16, 1)
Block(net, "2c", 16, 1)
Block(net, "3a", 32, 2)
Block(net, "3b", 32, 1)
Block(net, "3c", 32, 1)
Block(net, "4a", 64, 2)
Block(net, "4b", 64, 1)
Block(net, "4c", 64, 1)
net.add(layer.AvgPooling2D("pool4", 8, 8, border_mode='valid'))
net.add(layer.Flatten('flat'))
net.add(layer.Dense('ip5', 10))
print('Start intialization............')
for (p, name) in zip(net.param_values(), net.param_names()):
# print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
# initializer.gaussian(p, 0, math.sqrt(2.0/p.shape[1]))
initializer.gaussian(p, 0, 9.0 * p.shape[0])
else:
initializer.uniform(p, p.shape[0], p.shape[1])
else:
p.set_value(0)
# print name, p.l1()
return net
def init_params(net, weight_path=None):
if weight_path == None:
for pname, pval in zip(net.param_names(), net.param_values()):
print(pname, pval.shape)
if 'conv' in pname and len(pval.shape) > 1:
initializer.gaussian(pval, 0, pval.shape[1])
elif 'dense' in pname:
if len(pval.shape) > 1:
initializer.gaussian(pval, 0, pval.shape[0])
else:
pval.set_value(0)
# init params from batch norm layer
elif 'mean' in pname or 'beta' in pname:
pval.set_value(0)
elif 'var' in pname:
pval.set_value(1)
elif 'gamma' in pname:
initializer.uniform(pval, 0, 1)
else:
net.load(weight_path, use_pickle = 'pickle' in weight_path)
def create_net(in_shape, hyperpara, use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
height, width, kernel_y, kernel_x, stride_y, stride_x = hyperpara[0], hyperpara[1], hyperpara[2], hyperpara[3], hyperpara[4], hyperpara[5]
print ("kernel_x: ", kernel_x)
print ("stride_x: ", stride_x)
net = myffnet.ProbFeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
net.add(layer.Conv2D('conv1', 100, kernel=(kernel_y, kernel_x), stride=(stride_y, stride_x), pad=(0, 0),
input_sample_shape=(int(in_shape[0]), int(in_shape[1]), int(in_shape[2]))))
net.add(layer.Activation('relu1'))
net.add(layer.MaxPooling2D('pool1', 2, 1, pad=0))
net.add(layer.Flatten('flat'))
net.add(layer.Dense('dense', 2))
for (pname, pvalue) in zip(net.param_names(), net.param_values()):
if len(pvalue.shape) > 1:
initializer.gaussian(pvalue, pvalue.shape[0], pvalue.shape[1])
else:
pvalue.set_value(0)
print (pname, pvalue.l1())
return net
def create_net(input_shape, use_cpu=False):
if use_cpu:
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
ConvBnReLU(net, 'conv1_1', 64, input_shape)
#net.add(layer.Dropout('drop1', 0.3))
net.add(layer.MaxPooling2D('pool0', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv1_2', 128)
net.add(layer.MaxPooling2D('pool1', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv2_1', 128)
net.add(layer.Dropout('drop2_1', 0.4))
ConvBnReLU(net, 'conv2_2', 128)
net.add(layer.MaxPooling2D('pool2', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv3_1', 256)
net.add(layer.Dropout('drop3_1', 0.4))
ConvBnReLU(net, 'conv3_2', 256)
net.add(layer.Dropout('drop3_2', 0.4))
ConvBnReLU(net, 'conv3_3', 256)
net.add(layer.MaxPooling2D('pool3', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv4_1', 256)
net.add(layer.Dropout('drop4_1', 0.4))
ConvBnReLU(net, 'conv4_2', 256)
net.add(layer.Dropout('drop4_2', 0.4))
ConvBnReLU(net, 'conv4_3', 256)
net.add(layer.MaxPooling2D('pool4', 2, 2, border_mode='valid'))
ConvBnReLU(net, 'conv5_1', 512)
net.add(layer.Dropout('drop5_1', 0.4))
ConvBnReLU(net, 'conv5_2', 512)
net.add(layer.Dropout('drop5_2', 0.4))
ConvBnReLU(net, 'conv5_3', 512)
net.add(layer.MaxPooling2D('pool5', 2, 2, border_mode='valid'))
#ConvBnReLU(net, 'conv6_1', 512)
#net.add(layer.Dropout('drop6_1', 0.4))
#ConvBnReLU(net, 'conv6_2', 512)
#net.add(layer.Dropout('drop6_2', 0.4))
#ConvBnReLU(net, 'conv6_3', 512)
#net.add(layer.MaxPooling2D('pool6', 2, 2, border_mode='valid'))
#ConvBnReLU(net, 'conv7_1', 512)
#net.add(layer.Dropout('drop7_1', 0.4))
#ConvBnReLU(net, 'conv7_2', 512)
#net.add(layer.Dropout('drop7_2', 0.4))
#ConvBnReLU(net, 'conv7_3', 512)
#net.add(layer.MaxPooling2D('pool7', 2, 2, border_mode='valid'))
net.add(layer.Flatten('flat'))
net.add(layer.Dense('ip1', 256))
net.add(layer.BatchNormalization('bn1'))
net.add(layer.Activation('relu1'))
net.add(layer.Dropout('dropout1', 0.2))
net.add(layer.Dense('ip2', 16))
net.add(layer.BatchNormalization('bn2'))
net.add(layer.Activation('relu2'))
net.add(layer.Dropout('dropout2', 0.2))
net.add(layer.Dense('ip3', 2))
print 'Parameter intialization............'
for (p, name) in zip(net.param_values(), net.param_names()):
print name, p.shape
if 'mean' in name or 'beta' in name:
p.set_value(0.0)
elif 'var' in name:
p.set_value(1.0)
elif 'gamma' in name:
initializer.uniform(p, 0, 1)
elif len(p.shape) > 1:
if 'conv' in name:
initializer.gaussian(p, 0, p.size())
else:
p.gaussian(0, 0.02)
else:
p.set_value(0)
print name, p.l1()
return net
