def __init__(self, inputs, dataset, stage_depth,
batch_norm=False, activation=False, preprocess=False):
nfms = [2**(stage + 4) for stage in sorted(list(range(3)) * stage_depth)]
strides = [1 if cur == prev else 2 for cur, prev in zip(nfms[1:], nfms[:-1])]
layers = []
if preprocess and dataset == 'cifar10':
layers = Preprocess(functor=cifar_mean_subtract)
layers.append(Convolution(**conv_params(3, 16, batch_norm=batch_norm)))
layers.append(f_module(nfms[0], first=True, batch_norm=batch_norm))
for nfm, stride in zip(nfms[1:], strides):
layers.append(f_module(nfm, strides=stride, batch_norm=batch_norm))
if batch_norm:
layers.append(BatchNorm())
if activation:
layers.append(Activation(Rectlin()))
layers.append(Pool2D(8, strides=2, op='avg'))
if dataset == 'cifar10':
ax.Y.length = 10
layers.append(Affine(axes=ax.Y, weight_init=KaimingInit(),
batch_norm=batch_norm, activation=Softmax()))
elif dataset == 'i1k':
ax.Y.length = 1000
layers.append(Affine(axes=ax.Y, weight_init=KaimingInit(),
batch_norm=batch_norm, activation=Softmax()))
else:
raise ValueError("Incorrect dataset provided")
super(mini_residual_network, self).__init__(layers=layers)
def __init__(self,
inputs,
stage_depth,
batch_norm=True,
activation=True,
preprocess=True):
nfms = [
2**(stage + 4) for stage in sorted(list(range(3)) * stage_depth)
]
strides = [
1 if cur == prev else 2 for cur, prev in zip(nfms[1:], nfms[:-1])
]
layers = []
if preprocess:
layers = Preprocess(functor=cifar_mean_subtract)
parallel_axis = inputs['image'].axes.batch_axes()
with ng.metadata(device_id=('1', '2'), parallel=parallel_axis[0]):
layers.append(
Convolution(**conv_params(3, 16, batch_norm=batch_norm)))
layers.append(f_module(nfms[0], first=True))
for nfm, stride in zip(nfms[1:], strides):
layers.append(f_module(nfm, strides=stride))
if batch_norm:
layers.append(BatchNorm())
if activation:
layers.append(Activation(Rectlin()))
layers.append(Pool2D(8, strides=2, op='avg'))
layers.append(
Affine(axes=ax.Y,
weight_init=KaimingInit(),
batch_norm=batch_norm,
activation=Softmax()))
self.layers = layers
def __init__(self, branch_units, activation=Rectlin(),
bias_init=UniformInit(low=-0.08, high=0.08),
filter_init=XavierInit()):
(p1, p2, p3, p4) = branch_units
self.branch_1 = Convolution((1, 1, p1[0]), activation=activation,
bias_init=bias_init,
filter_init=filter_init)
self.branch_2 = [Convolution((1, 1, p2[0]), activation=activation,
bias_init=bias_init,
filter_init=filter_init),
Convolution((3, 3, p2[1]), activation=activation,
bias_init=bias_init,
filter_init=filter_init, padding=1)]
self.branch_3 = [Convolution((1, 1, p3[0]), activation=activation,
bias_init=bias_init,
filter_init=filter_init),
Convolution((5, 5, p3[1]), activation=activation,
bias_init=bias_init,
filter_init=filter_init, padding=2)]
self.branch_4 = [Pool2D(fshape=3, padding=1, strides=1, op="max"),
Convolution((1, 1, p3[0]), activation=activation,
bias_init=bias_init,
filter_init=filter_init)]
def __init__(self, stage_depth):
nfms = [2**(stage + 4) for stage in sorted(list(range(3)) * stage_depth)]
print(nfms)
strides = [1 if cur == prev else 2 for cur, prev in zip(nfms[1:], nfms[:-1])]
layers = [Preprocess(functor=cifar_mean_subtract),
Convolution(**conv_params(3, 16)),
f_module(nfms[0], first=True)]
for nfm, stride in zip(nfms[1:], strides):
layers.append(f_module(nfm, strides=stride))
layers.append(BatchNorm())
layers.append(Activation(Rectlin()))
layers.append(Pool2D(8, op='avg'))
layers.append(Affine(axes=ax.Y,
weight_init=KaimingInit(),
activation=Softmax()))
super(residual_network, self).__init__(layers=layers)
def cifar_mean_subtract(x):
bgr_mean = ng.persistent_tensor(axes=x.axes[0],
initial_value=np.array([[104., 119.,
127.]]))
y = ng.expand_dims((x - bgr_mean) / 255., ax.D, 1)
return y
init_uni = UniformInit(-0.1, 0.1)
seq1 = Sequential([
Preprocess(functor=cifar_mean_subtract),
Convolution((5, 5, 16), filter_init=init_uni, activation=Rectlin()),
Pool2D(2, strides=2),
Convolution((5, 5, 32), filter_init=init_uni, activation=Rectlin()),
Pool2D(2, strides=2),
Affine(nout=500, weight_init=init_uni, activation=Rectlin()),
Affine(axes=ax.Y, weight_init=init_uni, activation=Softmax())
])
######################
# Input specification
ax.C.length, ax.H.length, ax.W.length = train_set.shapes['image']
ax.D.length = 1
ax.N.length = args.batch_size
ax.Y.length = 10
# placeholders with descriptive names
inputs = dict(image=ng.placeholder([ax.C, ax.H, ax.W, ax.N]),
'label': {'data': y_train,
'axes': ('batch',)}}
train_set = ArrayIterator(train_data,
batch_size=args.batch_size,
total_iterations=args.num_iterations)
inputs = train_set.make_placeholders(include_iteration=True)
ax.Y.length = 1000 # number of outputs of last layer.
# weight initialization
init = UniformInit(low=-0.08, high=0.08)
# Setup model
seq1 = Sequential([Convolution((11, 11, 64), filter_init=GaussianInit(var=0.01),
bias_init=init,
activation=Rectlin(), padding=3, strides=4),
Pool2D(3, strides=2),
Convolution((5, 5, 192), filter_init=GaussianInit(var=0.01),
bias_init=init,
activation=Rectlin(), padding=2),
Pool2D(3, strides=2),
Convolution((3, 3, 384), filter_init=GaussianInit(var=0.03),
bias_init=init,
activation=Rectlin(), padding=1),
Convolution((3, 3, 256), filter_init=GaussianInit(var=0.03),
bias_init=init,
activation=Rectlin(), padding=1),
Convolution((3, 3, 256), filter_init=GaussianInit(var=0.03),
bias_init=init,
activation=Rectlin(), padding=1),
Pool2D(3, strides=2),
Affine(nout=4096, weight_init=GaussianInit(var=0.01),
branch_2_output = self.branch_2[0](in_obj)
branch_2_output = self.branch_2[1](branch_2_output)
branch_3_output = self.branch_3[0](in_obj)
branch_3_output = self.branch_3[1](branch_3_output)
branch_4_output = self.branch_4[0](in_obj)
branch_4_output = self.branch_4[1](branch_4_output)
outputs = [branch_1_output, branch_2_output, branch_3_output, branch_4_output]
# This does the equivalent of neon's merge-broadcast
return ng.concat_along_axis(outputs, branch_1_output.axes.channel_axis())
seq1 = Sequential([Convolution((7, 7, 64), padding=3, strides=2,
activation=Rectlin(), bias_init=bias_init,
filter_init=XavierInit()),
Pool2D(fshape=3, padding=1, strides=2, op='max'),
Convolution((1, 1, 64), activation=Rectlin(),
bias_init=bias_init, filter_init=XavierInit()),
Convolution((3, 3, 192), activation=Rectlin(),
bias_init=bias_init, filter_init=XavierInit(),
padding=1),
Pool2D(fshape=3, padding=1, strides=2, op='max'),
Inception([(64,), (96, 128), (16, 32), (32,)]),
Inception([(128,), (128, 192), (32, 96), (64,)]),
Pool2D(fshape=3, padding=1, strides=2, op='max'),
Inception([(192,), (96, 208), (16, 48), (64,)]),
Inception([(160,), (112, 224), (24, 64), (64,)]),
Inception([(128,), (128, 256), (24, 64), (64,)]),
Inception([(112,), (144, 288), (32, 64), (64,)]),
Inception([(256,), (160, 320), (32, 128), (128,)]),
Pool2D(fshape=3, padding=1, strides=2, op='max'),