def layers(self):
init_uni = Uniform(low=-0.1, high=0.1)
bn = False
return [
DOG((5.0, 4.0, 3.0, 1.6), 1.8),
Conv((5, 5, 16),
init=init_uni,
activation=Rectlin(),
batch_norm=bn),
Pooling((2, 2)),
Conv((5, 5, 32),
init=init_uni,
activation=Rectlin(),
batch_norm=bn),
Pooling((2, 2)),
Affine(nout=500,
init=init_uni,
activation=Rectlin(),
batch_norm=bn),
Affine(nout=self.noutputs,
init=init_uni,
bias=Constant(0),
activation=Softmax() if self.use_softmax else Logistic(
shortcut=True))
]
def _createLayers(self, num_actions):
init_xavier_conv = Xavier(local=True)
init_xavier_affine = Xavier(local=False)
layers = []
layers.append(
Conv((8, 8, 32),
strides=4,
init=init_xavier_conv,
activation=Rectlin(),
batch_norm=self.batch_norm))
layers.append(
Conv((4, 4, 64),
strides=2,
init=init_xavier_conv,
activation=Rectlin(),
batch_norm=self.batch_norm))
layers.append(
Conv((2, 2, 128),
strides=1,
init=init_xavier_conv,
activation=Rectlin(),
batch_norm=self.batch_norm))
layers.append(
Affine(nout=256,
init=init_xavier_affine,
activation=Rectlin(),
batch_norm=self.batch_norm))
layers.append(Affine(nout=num_actions, init=init_xavier_affine))
return layers
def create_network():
init = Kaiming()
padding = dict(pad_d=1, pad_h=1, pad_w=1)
strides = dict(str_d=2, str_h=2, str_w=2)
dilation = dict(dil_d=2, dil_h=2, dil_w=2)
common = dict(init=init, batch_norm=True, activation=Rectlin())
layers = [
Conv((9, 9, 9, 16),
padding=padding,
strides=strides,
init=init,
activation=Rectlin()),
Conv((5, 5, 5, 32), dilation=dilation, **common),
Conv((3, 3, 3, 64), dilation=dilation, **common),
Pooling((2, 2, 2), padding=padding, strides=strides),
Conv((2, 2, 2, 128), **common),
Conv((2, 2, 2, 128), **common),
Conv((2, 2, 2, 128), **common),
Conv((2, 2, 2, 256), **common),
Conv((2, 2, 2, 1024), **common),
Conv((2, 2, 2, 4096), **common),
Conv((2, 2, 2, 2048), **common),
Conv((2, 2, 2, 1024), **common),
Dropout(),
Affine(2,
init=Kaiming(local=False),
batch_norm=True,
activation=Softmax())
]
return Model(layers=layers)
def _createLayers(self, num_actions):
# create network
init_norm = Gaussian(loc=0.0, scale=0.01)
layers = []
# The first hidden layer convolves 32 filters of 8x8 with stride 4 with the input image and applies a rectifier nonlinearity.
layers.append(
Conv((8, 8, 32),
strides=4,
init=init_norm,
activation=Rectlin(),
batch_norm=self.batch_norm))
# The second hidden layer convolves 64 filters of 4x4 with stride 2, again followed by a rectifier nonlinearity.
layers.append(
Conv((4, 4, 64),
strides=2,
init=init_norm,
activation=Rectlin(),
batch_norm=self.batch_norm))
# This is followed by a third convolutional layer that convolves 64 filters of 3x3 with stride 1 followed by a rectifier.
layers.append(
Conv((3, 3, 64),
strides=1,
init=init_norm,
activation=Rectlin(),
batch_norm=self.batch_norm))
# The final hidden layer is fully-connected and consists of 512 rectifier units.
layers.append(
Affine(nout=512,
init=init_norm,
activation=Rectlin(),
batch_norm=self.batch_norm))
# The output layer is a fully-connected linear layer with a single output for each valid action.
layers.append(Affine(nout=num_actions, init=init_norm))
return layers
def _createLayers(self, num_actions):
# create network
init_norm = Gaussian(loc=0.0, scale=0.01)
layers = []
if self.screen_dim == (84, 84):
# The first hidden layer convolves 32 filters of 8x8 with stride 4 with the input image and applies a rectifier nonlinearity.
logger.info("Using 8x8 filter for first Conv")
layers.append(
Conv((8, 8, 32),
strides=4,
init=init_norm,
activation=Rectlin()))
elif self.screen_dim == (52, 40):
# The first hidden layer convolves 32 filters of 5x4 with stride 2 with the input image and applies a rectifier nonlinearity.
logger.info("Using 5x4 filter for first Conv")
layers.append(
Conv((5, 4, 32),
strides=2,
init=init_norm,
activation=Rectlin()))
else:
raise NotImplementedError("Unsupported screen dim.")
# The second hidden layer convolves 64 filters of 4x4 with stride 2, again followed by a rectifier nonlinearity.
layers.append(
Conv((4, 4, 64), strides=2, init=init_norm, activation=Rectlin()))
# This is followed by a third convolutional layer that convolves 64 filters of 3x3 with stride 1 followed by a rectifier.
layers.append(
Conv((3, 3, 64), strides=1, init=init_norm, activation=Rectlin()))
# The final hidden layer is fully-connected and consists of 512 rectifier units.
layers.append(Affine(nout=512, init=init_norm, activation=Rectlin()))
# The output layer is a fully-connected linear layer with a single output for each valid action.
layers.append(Affine(nout=num_actions, init=init_norm))
return layers
def run(train, test):
init = Gaussian(scale=0.01)
layers = [
Conv((3, 3, 128),
init=init,
activation=Rectlin(),
strides=dict(str_h=1, str_w=2)),
Conv((3, 3, 256), init=init, batch_norm=True, activation=Rectlin()),
Pooling(2, strides=2),
Conv((2, 2, 512), init=init, batch_norm=True, activation=Rectlin()),
DeepBiRNN(256,
init=init,
activation=Rectlin(),
reset_cells=True,
depth=3),
RecurrentLast(),
Affine(32, init=init, batch_norm=True, activation=Rectlin()),
Affine(nout=common['nclasses'], init=init, activation=Softmax())
]
model = Model(layers=layers)
opt = Adadelta()
metric = Misclassification()
callbacks = Callbacks(model,
eval_set=test,
metric=metric,
**args.callback_args)
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
model.fit(train,
optimizer=opt,
num_epochs=args.epochs,
cost=cost,
callbacks=callbacks)
return model
def create_frcn_model(frcn_fine_tune=False):
b1 = BranchNode(name="b1")
imagenet_layers = add_vgg_layers()
HW = (7, 7)
frcn_layers = [
RoiPooling(layers=imagenet_layers, HW=HW,
bprop_enabled=frcn_fine_tune),
Affine(nout=4096,
init=Gaussian(scale=0.005),
bias=Constant(.1),
activation=Rectlin()),
Dropout(keep=0.5),
Affine(nout=4096,
init=Gaussian(scale=0.005),
bias=Constant(.1),
activation=Rectlin()),
Dropout(keep=0.5), b1,
Affine(nout=21,
init=Gaussian(scale=0.01),
bias=Constant(0),
activation=Softmax())
]
bb_layers = [
b1,
Affine(nout=84,
init=Gaussian(scale=0.001),
bias=Constant(0),
activation=Identity())
]
return Model(layers=Tree([frcn_layers, bb_layers]))
def _createLayers(self, num_actions):
# create network
init_xavier_conv = Xavier(local=True)
init_xavier_affine = Xavier(local=False)
# init_uniform_conv = Uniform(low=-.01, high=.01)
# init_uniform_affine = Uniform(low=-.01, high=.01)
layers = []
# The first hidden layer convolves 32 filters of 8x8 with stride 4 with the input image and applies a rectifier nonlinearity.
# layers.append(Conv((8, 8, 32), strides=4, init=init_xavier_conv, activation=Rectlin(), batch_norm=self.batch_norm))
layers.append(
Conv((5, 5, 32),
strides=2,
init=init_xavier_conv,
activation=Rectlin(),
batch_norm=self.batch_norm))
# The second hidden layer convolves 64 filters of 4x4 with stride 2, again followed by a rectifier nonlinearity.
# layers.append(Conv((4, 4, 64), strides=2, init=init_xavier_conv, activation=Rectlin(), batch_norm=self.batch_norm))
layers.append(
Conv((5, 5, 32),
strides=2,
init=init_xavier_conv,
activation=Rectlin(),
batch_norm=self.batch_norm))
# This is followed by a third convolutional layer that convolves 64 filters of 3x3 with stride 1 followed by a rectifier.
# layers.append(Conv((3, 3, 64), strides=1, init=init_xavier_conv, activation=Rectlin(), batch_norm=self.batch_norm))
# The final hidden layer is fully-connected and consists of 512 rectifier units.
layers.append(
Affine(nout=512,
init=init_xavier_affine,
activation=Rectlin(),
batch_norm=self.batch_norm))
# The output layer is a fully-connected linear layer with a single output for each valid action.
layers.append(Affine(nout=num_actions, init=init_xavier_affine))
return layers
def constuct_network():
"""
Constructs the layers of our RCNN architecture. It is similar to AlexNet but simplified to only a
few convolutional layers and 3 LSTM layers.
"""
layers = [
Conv((11, 11, 64),
init=Gaussian(scale=0.01),
bias=Constant(0),
activation=Rectlin(),
padding=3,
strides=4),
Pooling(3, strides=2),
Conv((7, 7, 128),
init=Gaussian(scale=0.01),
bias=Constant(1),
activation=Rectlin(),
padding=2),
Pooling(3, strides=2),
Conv((5, 5, 256),
init=Gaussian(scale=0.03),
bias=Constant(0),
activation=Rectlin(),
padding=1),
Conv((3, 3, 256),
init=Gaussian(scale=0.03),
bias=Constant(1),
activation=Rectlin(),
padding=1),
Pooling(3, strides=2),
Affine(nout=4096,
init=Gaussian(scale=0.01),
bias=Constant(1),
activation=Rectlin()),
DropoutBinary(keep=0.5),
LSTM(512,
init=Gaussian(scale=0.03),
activation=Rectlin(),
gate_activation=Tanh()),
LSTM(512,
init=Gaussian(scale=0.03),
activation=Rectlin(),
gate_activation=Tanh()),
LSTM(512,
init=Gaussian(scale=0.03),
activation=Rectlin(),
gate_activation=Tanh()),
Affine(nout=4096,
init=Gaussian(scale=0.01),
bias=Constant(1),
activation=Rectlin()),
DropoutBinary(keep=0.5),
Affine(nout=101,
init=Gaussian(scale=0.01),
bias=Constant(-7),
activation=Softmax())
]
return Model(layers=layers)
def layers(self):
return [
Conv((7, 7, 96),
init=Gaussian(scale=0.0001),
bias=Constant(0),
activation=Rectlin(),
padding=3,
strides=1),
LRN(31, ascale=0.001, bpower=0.75),
Pooling(3, strides=2, padding=1),
Conv((5, 5, 256),
init=Gaussian(scale=0.01),
bias=Constant(0),
activation=Rectlin(),
padding=2,
strides=1),
LRN(31, ascale=0.001, bpower=0.75),
Pooling(3, strides=2, padding=1),
Conv((3, 3, 384),
init=Gaussian(scale=0.01),
bias=Constant(0),
activation=Rectlin(),
padding=1,
strides=1),
Conv((3, 3, 384),
init=Gaussian(scale=0.01),
bias=Constant(0),
activation=Rectlin(),
padding=1,
strides=1),
Conv((3, 3, 256),
init=Gaussian(scale=0.01),
bias=Constant(0),
activation=Rectlin(),
padding=1,
strides=1),
Pooling(3, strides=2, padding=1),
Affine(nout=4096,
init=Gaussian(scale=0.01),
bias=Constant(0),
activation=Identity()),
Dropout(keep=0.5),
Affine(nout=4096,
init=Gaussian(scale=0.01),
bias=Constant(0),
activation=Identity()),
Dropout(keep=0.5),
Affine(nout=self.noutputs,
init=Gaussian(scale=0.01),
bias=Constant(0),
activation=Softmax() if self.use_softmax else Logistic(
shortcut=True))
]
def test_multi_optimizer(backend_default):
opt_gdm = GradientDescentMomentum(learning_rate=0.001,
momentum_coef=0.9,
wdecay=0.005)
opt_ada = Adadelta()
opt_adam = Adam()
opt_rms = RMSProp()
opt_rms_1 = RMSProp(gradient_clip_value=5)
init_one = Gaussian(scale=0.01)
l1 = Conv((11, 11, 64),
strides=4,
padding=3,
init=init_one,
bias=Constant(0),
activation=Rectlin())
l2 = Affine(nout=4096,
init=init_one,
bias=Constant(1),
activation=Rectlin())
l3 = LSTM(output_size=1000,
init=init_one,
activation=Logistic(),
gate_activation=Tanh())
l4 = GRU(output_size=100,
init=init_one,
activation=Logistic(),
gate_activation=Tanh())
layers = [l1, l2, l3, l4]
layer_list = []
for layer in layers:
if isinstance(layer, list):
layer_list.extend(layer)
else:
layer_list.append(layer)
opt = MultiOptimizer({
'default': opt_gdm,
'Bias': opt_ada,
'Convolution': opt_adam,
'Linear': opt_rms,
'LSTM': opt_rms_1,
'GRU': opt_rms_1
})
map_list = opt._map_optimizers(layer_list)
assert map_list[opt_adam][0].__class__.__name__ == 'Convolution'
assert map_list[opt_ada][0].__class__.__name__ == 'Bias'
assert map_list[opt_rms][0].__class__.__name__ == 'Linear'
assert map_list[opt_gdm][0].__class__.__name__ == 'Activation'
assert map_list[opt_rms_1][0].__class__.__name__ == 'LSTM'
assert map_list[opt_rms_1][1].__class__.__name__ == 'GRU'
def constructCNN(self):
layers = []
if self.network_type == "idsia":
layers.append(
Conv((3, 3, 100),
strides=1,
init=Kaiming(),
bias=Constant(0.0),
activation=Rectlin(),
name="Conv1"))
layers.append(Pooling(2, op="max", strides=2, name="neon_pool1"))
layers.append(
Conv((4, 4, 150),
strides=1,
init=Kaiming(),
bias=Constant(0.0),
activation=Rectlin(),
name="Conv2"))
layers.append(Pooling(2, op="max", strides=2, name="neon_pool2"))
layers.append(
Conv((3, 3, 250),
strides=1,
init=Kaiming(),
bias=Constant(0.0),
activation=Rectlin(),
name="Conv3"))
layers.append(Pooling(2, op="max", strides=2, name="neon_pool3"))
layers.append(
Affine(nout=200,
init=Kaiming(local=False),
bias=Constant(0.0),
activation=Rectlin(),
name="neon_fc1"))
layers.append(
Affine(nout=self.class_num,
init=Kaiming(local=False),
bias=Constant(0.0),
activation=Softmax(),
name="neon_fc2"))
elif self.network_type == "resnet-56":
layers = resnet(9, self.class_num,
int(self.resize_size[0] / 4)) # 6*9 + 2 = 56
elif self.network_type == "resnet-32":
layers = resnet(5, self.class_num,
int(self.resize_size[0] / 4)) # 6*5 + 2 = 32
elif self.network_type == "resnet-20":
layers = resnet(3, self.class_num,
int(self.resize_size[0] / 4)) # 6*3 + 2 = 20
return layers
def conv_params(fsize, nfm, strides=1, relu=True, batch_norm=True):
return dict(fshape=(fsize, fsize, nfm),
strides=strides,
activation=(Rectlin() if relu else None),
padding=(fsize // 2),
batch_norm=batch_norm,
init=Kaiming(local=True))
def main(args):
# load up the mnist data set
dataset = MNIST(path=args.data_dir)
# initialize model object
mlp = Model(layers=[
Affine(nout=100,
init=Gaussian(loc=0.0, scale=0.01),
activation=Rectlin()),
Affine(nout=10,
init=Gaussian(loc=0.0, scale=0.01),
activation=Logistic(shortcut=True))
])
# setup optimizer
optimizer = GradientDescentMomentum(0.1,
momentum_coef=0.9,
stochastic_round=args.rounding)
# configure callbacks
callbacks = Callbacks(mlp,
eval_set=dataset.valid_iter,
**args.callback_args)
# run fit
# setup cost function as CrossEntropy
mlp.fit(dataset.train_iter,
optimizer=optimizer,
num_epochs=args.epochs,
cost=GeneralizedCost(costfunc=CrossEntropyBinary()),
callbacks=callbacks)
error_rate = mlp.eval(dataset.valid_iter, metric=Misclassification())
neon_logger.display('Classification accuracy = %.4f' % (1 - error_rate))
def build_model(self):
# setup weight initialization function
init_norm = Gaussian(loc=0.0, scale=0.01)
# setup model layers
layers = [
Affine(nout=100,
init=init_norm,
bias=Uniform(),
activation=Rectlin()),
Affine(nout=10,
init=init_norm,
bias=Uniform(),
activation=Logistic(shortcut=True))
]
# setup cost function as CrossEntropy
self.cost = GeneralizedCost(costfunc=CrossEntropyBinary())
# setup optimizer
self.optimizer = GradientDescentMomentum(
0.1, momentum_coef=0.9, stochastic_round=self.args.rounding)
# initialize model object
self.model = ModelDist(layers=layers)
def __init__(self, depth=9):
self.depth = depth
depth = 9
train = (3, 32, 32)
nfms = [2**(stage + 4) for stage in sorted(list(range(3)) * depth)]
strides = [
1 if cur == prev else 2 for cur, prev in zip(nfms[1:], nfms[:-1])
]
# Now construct the network
layers = [Conv(**self.conv_params(3, 16))]
layers.append(self.module_s1(nfms[0], True))
for nfm, stride in zip(nfms[1:], strides):
res_module = self.module_s1(
nfm) if stride == 1 else self.module_s2(nfm)
layers.append(res_module)
layers.append(BatchNorm())
layers.append(Activation(Rectlin()))
layers.append(Pooling('all', op='avg'))
layers.append(
Affine(10,
init=Kaiming(local=False),
batch_norm=True,
activation=Softmax()))
self.layers = layers
model = Model(layers=layers)
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
model.initialize(train, cost=cost)
self.model = model
def create_network(stage_depth):
# Structure of the deep residual part of the network:
# stage_depth modules of 2 convolutional layers each at feature map depths of 16, 32, 64
nfms = [2**(stage + 4) for stage in sorted(list(range(3)) * stage_depth)]
strides = [
1 if cur == prev else 2 for cur, prev in zip(nfms[1:], nfms[:-1])
]
# Now construct the network
layers = [Conv(**conv_params(3, 16))]
layers.append(module_s1(nfms[0], True))
for nfm, stride in zip(nfms[1:], strides):
res_module = module_s1(nfm) if stride == 1 else module_s2(nfm)
layers.append(res_module)
layers.append(BatchNorm())
layers.append(Activation(Rectlin()))
layers.append(Pooling('all', op='avg'))
layers.append(
Affine(10,
init=Kaiming(local=False),
batch_norm=True,
activation=Softmax()))
return Model(layers=layers), GeneralizedCost(costfunc=CrossEntropyMulti())
def build(self):
# setup model layers
layers = [Affine(nout=100, init=self.init, bias=self.init, activation=Rectlin()),
Affine(nout=2, init=self.init, bias=self.init, activation=Softmax())]
# initialize model object
self.model = Model(layers=layers)
def conv_params(fsize, nfm, stride=1, relu=True):
return dict(fshape=(fsize, fsize, nfm),
strides=stride,
padding=(1 if fsize > 1 else 0),
activation=(Rectlin() if relu else None),
init=init1,
batch_norm=True)
def projection_skip(nfm, stride=1):
mainpath = [Conv(**conv_params(3, nfm, stride=stride)),
Conv(**conv_params(3, nfm, relu=False))]
sidepath = [SkipNode() if stride == 1 else Conv(**conv_params(1, nfm, stride, relu=False))]
module = [MergeSum([mainpath, sidepath]),
Activation(Rectlin())]
return module
def test_model_get_outputs_rnn(backend_default, data):
data_path = load_ptb_test(path=data)
data_set = Text(time_steps=50, path=data_path)
# weight initialization
init = Constant(0.08)
# model initialization
layers = [
Recurrent(150, init, activation=Logistic()),
Affine(len(data_set.vocab), init, bias=init, activation=Rectlin())
]
model = Model(layers=layers)
output = model.get_outputs(data_set)
assert output.shape == (data_set.ndata, data_set.seq_length,
data_set.nclass)
# since the init are all constant and model is un-trained:
# along the feature dim, the values should be all the same
assert np.allclose(output[0, 0], output[0, 0, 0], rtol=0, atol=1e-5)
assert np.allclose(output[0, 1], output[0, 1, 0], rtol=0, atol=1e-5)
# along the time dim, the values should be increasing:
assert np.alltrue(output[0, 2] > output[0, 1])
assert np.alltrue(output[0, 1] > output[0, 0])
def __init__(self,
overlapping_classes=None,
exclusive_classes=None,
analytics_input=True,
network_type='conv_net',
num_words=60,
width=100,
lookup_size=0,
lookup_dim=0,
optimizer=Adam()):
assert (overlapping_classes is not None) or (exclusive_classes
is not None)
self.width = width
self.num_words = num_words
self.overlapping_classes = overlapping_classes
self.exclusive_classes = exclusive_classes
self.analytics_input = analytics_input
self.recurrent = network_type == 'lstm'
self.lookup_size = lookup_size
self.lookup_dim = lookup_dim
init = GlorotUniform()
activation = Rectlin(slope=1E-05)
gate = Logistic()
input_layers = self.input_layers(analytics_input, init, activation,
gate)
if self.overlapping_classes is None:
output_layers = [
Affine(len(self.exclusive_classes), init, activation=Softmax())
]
elif self.exclusive_classes is None:
output_layers = [
Affine(len(self.overlapping_classes),
init,
activation=Logistic())
]
else:
output_branch = BranchNode(name='exclusive_overlapping')
output_layers = Tree([[
SkipNode(), output_branch,
Affine(len(self.exclusive_classes), init, activation=Softmax())
],
[
output_branch,
Affine(len(self.overlapping_classes),
init,
activation=Logistic())
]])
layers = [
input_layers,
# this is where inputs meet, and where we may want to add depth or
# additional functionality
Dropout(keep=0.8),
output_layers
]
super(ClassifierNetwork, self).__init__(layers, optimizer=optimizer)
def _createLayers(self):
# create network
init_norm = Gaussian(loc=0.0, scale=0.01)
layers = []
for i in xrange(self.num_layers):
layers.append(Affine(nout=self.hidden_nodes, init=init_norm, activation=Rectlin()))
layers.append(Affine(nout=self.num_actions, init = init_norm))
return layers
def test_multi_optimizer(backend_default_mkl):
"""
A test for MultiOptimizer.
"""
opt_gdm = GradientDescentMomentum(
learning_rate=0.001, momentum_coef=0.9, wdecay=0.005)
opt_ada = Adadelta()
opt_adam = Adam()
opt_rms = RMSProp()
opt_rms_1 = RMSProp(gradient_clip_value=5)
init_one = Gaussian(scale=0.01)
l1 = Conv((11, 11, 64), strides=4, padding=3,
init=init_one, bias=Constant(0), activation=Rectlin())
l2 = Affine(nout=4096, init=init_one,
bias=Constant(1), activation=Rectlin())
l3 = LSTM(output_size=1000, init=init_one, activation=Logistic(), gate_activation=Tanh())
l4 = GRU(output_size=100, init=init_one, activation=Logistic(), gate_activation=Tanh())
layers = [l1, l2, l3, l4]
layer_list = []
for layer in layers:
if isinstance(layer, list):
layer_list.extend(layer)
else:
layer_list.append(layer)
for l in layer_list:
l.configure(in_obj=(16, 28, 28))
l.allocate()
# separate layer_list into two, the last two recurrent layers and the rest
layer_list1, layer_list2 = layer_list[:-2], layer_list[-2:]
opt = MultiOptimizer({'default': opt_gdm,
'Bias': opt_ada,
'Convolution': opt_adam,
'Convolution_bias': opt_adam,
'Linear': opt_rms,
'LSTM': opt_rms_1,
'GRU': opt_rms_1})
layers_to_optimize1 = [l for l in layer_list1 if isinstance(l, ParameterLayer)]
layers_to_optimize2 = [l for l in layer_list2 if isinstance(l, ParameterLayer)]
opt.optimize(layers_to_optimize1, 0)
assert opt.map_list[opt_adam][0].__class__.__name__ is 'Convolution_bias'
assert opt.map_list[opt_rms][0].__class__.__name__ == 'Linear'
opt.optimize(layers_to_optimize2, 0)
assert opt.map_list[opt_rms_1][0].__class__.__name__ == 'LSTM'
assert opt.map_list[opt_rms_1][1].__class__.__name__ == 'GRU'
def create_network():
# weight initialization
g1 = Gaussian(scale=0.01)
g5 = Gaussian(scale=0.005)
c0 = Constant(0)
c1 = Constant(1)
# model initialization
padding = {'pad_d': 1, 'pad_h': 1, 'pad_w': 1}
strides = {'str_d': 2, 'str_h': 2, 'str_w': 2}
layers = [
Conv((3, 3, 3, 64), padding=padding, init=g1, bias=c0, activation=Rectlin()),
Pooling((1, 2, 2), strides={'str_d': 1, 'str_h': 2, 'str_w': 2}),
Conv((3, 3, 3, 128), padding=padding, init=g1, bias=c1, activation=Rectlin()),
Pooling((2, 2, 2), strides=strides),
Conv((3, 3, 3, 256), padding=padding, init=g1, bias=c1, activation=Rectlin()),
Pooling((2, 2, 2), strides=strides),
Conv((3, 3, 3, 256), padding=padding, init=g1, bias=c1, activation=Rectlin()),
Pooling((2, 2, 2), strides=strides),
Conv((3, 3, 3, 256), padding=padding, init=g1, bias=c1, activation=Rectlin()),
Pooling((2, 2, 2), strides=strides),
Affine(nout=2048, init=g5, bias=c1, activation=Rectlin()),
Dropout(keep=0.5),
Affine(nout=2048, init=g5, bias=c1, activation=Rectlin()),
Dropout(keep=0.5),
Affine(nout=101, init=g1, bias=c0, activation=Softmax())
]
return Model(layers=layers)
def test_leaky_rectlin_derivative_mixed(backend_default):
slope = 0.2
inputs = np.array([[4, 0], [-2, 9]])
outputs = np.array([[1, 0], [slope, 1]])
compare_tensors(Rectlin(slope=slope),
inputs,
outputs,
deriv=True,
tol=1e-7)
def test_leaky_rectlin_derivative_negatives(backend_default):
slope = 0.2
inputs = np.array([[-1, -3], [-2, -4]])
outputs = np.array([[0, 0], [0, 0]]) + slope
compare_tensors(Rectlin(slope=slope),
inputs,
outputs,
deriv=True,
tol=1e-7)
def _createLayers(self, num_actions): # create network init_norm = Gaussian(loc=0.0, scale=0.01) layers = [] # The final hidden layer is fully-connected and consists of 512 rectifier units. layers.append(Affine(nout=64, init=init_norm, bias=init_norm, activation=Rectlin())) # The output layer is a fully-connected linear layer with a single output for each valid action. layers.append(Affine(nout=num_actions, init=init_norm, bias=init_norm)) return layers
def run(args, train, test):
init_uni = Uniform(low=-0.1, high=0.1)
opt_gdm = GradientDescentMomentum(learning_rate=0.01,
momentum_coef=0.9,
stochastic_round=args.rounding)
layers = [Conv((5, 5, 16), init=init_uni, activation=Rectlin(), batch_norm=True),
Pooling((2, 2)),
Conv((5, 5, 32), init=init_uni, activation=Rectlin(), batch_norm=True),
Pooling((2, 2)),
Affine(nout=500, init=init_uni, activation=Rectlin(), batch_norm=True),
Affine(nout=10, init=init_uni, activation=Softmax())]
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
mlp = Model(layers=layers)
callbacks = Callbacks(mlp, eval_set=test, **args.callback_args)
mlp.fit(train, optimizer=opt_gdm, num_epochs=args.epochs, cost=cost, callbacks=callbacks)
err = mlp.eval(test, metric=Misclassification())*100
print('Misclassification error = %.2f%%' % err)
return err
def module_factory(nfm, stride=1):
projection = None if stride == 1 else IdentityInit()
module = [
Conv(**conv_params(3, nfm, stride=stride)),
Conv(**conv_params(3, nfm, relu=False))
]
module = module if args.network == 'plain' else [
ResidualModule(module, projection)
]
module.append(Activation(Rectlin()))
return module
