def create_network():
init = Kaiming()
padding = dict(pad_d=1, pad_h=1, pad_w=1)
strides = dict(str_d=2, str_h=2, str_w=2)
dilation = dict(dil_d=2, dil_h=2, dil_w=2)
common = dict(init=init, batch_norm=True, activation=Rectlin())
layers = [
Conv((9, 9, 9, 16),
padding=padding,
strides=strides,
init=init,
activation=Rectlin()),
Conv((5, 5, 5, 32), dilation=dilation, **common),
Conv((3, 3, 3, 64), dilation=dilation, **common),
Pooling((2, 2, 2), padding=padding, strides=strides),
Conv((2, 2, 2, 128), **common),
Conv((2, 2, 2, 128), **common),
Conv((2, 2, 2, 128), **common),
Conv((2, 2, 2, 256), **common),
Conv((2, 2, 2, 1024), **common),
Conv((2, 2, 2, 4096), **common),
Conv((2, 2, 2, 2048), **common),
Conv((2, 2, 2, 1024), **common),
Dropout(),
Affine(2,
init=Kaiming(local=False),
batch_norm=True,
activation=Softmax())
]
return Model(layers=layers)
def constructCNN(self):
layers = []
if self.network_type == "idsia":
layers.append(
Conv((3, 3, 100),
strides=1,
init=Kaiming(),
bias=Constant(0.0),
activation=Rectlin(),
name="Conv1"))
layers.append(Pooling(2, op="max", strides=2, name="neon_pool1"))
layers.append(
Conv((4, 4, 150),
strides=1,
init=Kaiming(),
bias=Constant(0.0),
activation=Rectlin(),
name="Conv2"))
layers.append(Pooling(2, op="max", strides=2, name="neon_pool2"))
layers.append(
Conv((3, 3, 250),
strides=1,
init=Kaiming(),
bias=Constant(0.0),
activation=Rectlin(),
name="Conv3"))
layers.append(Pooling(2, op="max", strides=2, name="neon_pool3"))
layers.append(
Affine(nout=200,
init=Kaiming(local=False),
bias=Constant(0.0),
activation=Rectlin(),
name="neon_fc1"))
layers.append(
Affine(nout=self.class_num,
init=Kaiming(local=False),
bias=Constant(0.0),
activation=Softmax(),
name="neon_fc2"))
elif self.network_type == "resnet-56":
layers = resnet(9, self.class_num,
int(self.resize_size[0] / 4)) # 6*9 + 2 = 56
elif self.network_type == "resnet-32":
layers = resnet(5, self.class_num,
int(self.resize_size[0] / 4)) # 6*5 + 2 = 32
elif self.network_type == "resnet-20":
layers = resnet(3, self.class_num,
int(self.resize_size[0] / 4)) # 6*3 + 2 = 20
return layers
def conv_params(fsize, nfm, strides=1, relu=True, batch_norm=True):
return dict(fshape=(fsize, fsize, nfm),
strides=strides,
activation=(Rectlin() if relu else None),
padding=(fsize // 2),
batch_norm=batch_norm,
init=Kaiming(local=True))
def create_network(stage_depth):
# Structure of the deep residual part of the network:
# stage_depth modules of 2 convolutional layers each at feature map depths of 16, 32, 64
nfms = [2**(stage + 4) for stage in sorted(list(range(3)) * stage_depth)]
strides = [
1 if cur == prev else 2 for cur, prev in zip(nfms[1:], nfms[:-1])
]
# Now construct the network
layers = [Conv(**conv_params(3, 16))]
layers.append(module_s1(nfms[0], True))
for nfm, stride in zip(nfms[1:], strides):
res_module = module_s1(nfm) if stride == 1 else module_s2(nfm)
layers.append(res_module)
layers.append(BatchNorm())
layers.append(Activation(Rectlin()))
layers.append(Pooling('all', op='avg'))
layers.append(
Affine(10,
init=Kaiming(local=False),
batch_norm=True,
activation=Softmax()))
return Model(layers=layers), GeneralizedCost(costfunc=CrossEntropyMulti())
def __init__(self, depth=9):
self.depth = depth
depth = 9
train = (3, 32, 32)
nfms = [2**(stage + 4) for stage in sorted(list(range(3)) * depth)]
strides = [
1 if cur == prev else 2 for cur, prev in zip(nfms[1:], nfms[:-1])
]
# Now construct the network
layers = [Conv(**self.conv_params(3, 16))]
layers.append(self.module_s1(nfms[0], True))
for nfm, stride in zip(nfms[1:], strides):
res_module = self.module_s1(
nfm) if stride == 1 else self.module_s2(nfm)
layers.append(res_module)
layers.append(BatchNorm())
layers.append(Activation(Rectlin()))
layers.append(Pooling('all', op='avg'))
layers.append(
Affine(10,
init=Kaiming(local=False),
batch_norm=True,
activation=Softmax()))
self.layers = layers
model = Model(layers=layers)
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
model.initialize(train, cost=cost)
self.model = model
def create_network(stage_depth):
if stage_depth in (18, 18):
stages = (2, 2, 2, 2)
elif stage_depth in (34, 50):
stages = (3, 4, 6, 3)
elif stage_depth in (68, 101):
stages = (3, 4, 23, 3)
elif stage_depth in (102, 152):
stages = (3, 8, 36, 3)
else:
raise ValueError('Invalid stage_depth value'.format(stage_depth))
bottleneck = False
if stage_depth in (50, 101, 152):
bottleneck = True
layers = [Conv(**conv_params(7, 64, strides=2)), Pooling(3, strides=2)]
# Structure of the deep residual part of the network:
# stage_depth modules of 2 convolutional layers each at feature map depths
# of 64, 128, 256, 512
nfms = list(
itt.chain.from_iterable(
[itt.repeat(2**(x + 6), r) for x, r in enumerate(stages)]))
strides = [-1] + [
1 if cur == prev else 2 for cur, prev in zip(nfms[1:], nfms[:-1])
]
for nfm, stride in zip(nfms, strides):
layers.append(module_factory(nfm, bottleneck, stride))
layers.append(Pooling('all', op='avg'))
layers.append(
Affine(nout=1000, init=Kaiming(local=False), activation=Softmax()))
return Model(layers=layers), GeneralizedCost(costfunc=CrossEntropyMulti())
def gen_model(num_channels, height, width):
assert NervanaObject.be is not None, 'need to generate a backend before using this function'
init_uni = Kaiming()
# we have 1 issue, they have bias layers we don't allow batchnorm and biases
conv_common = dict(padding=1, init=init_uni, activation=Rectlin(), batch_norm=True)
# set up the layers
layers = []
# need to store a ref to the pooling layers to pass
# to the upsampling layers to get the argmax indicies
# for upsampling, this stack holds the pooling layer refs
pool_layers = []
# first loop generates the encoder layers
nchan = [64, 128, 256, 512, 512]
for ind in range(len(nchan)):
nchanu = nchan[ind]
lrng = 2 if ind <= 1 else 3
for lind in range(lrng):
nm = 'conv%d_%d' % (ind+1, lind+1)
layers.append(Conv((3, 3, nchanu), strides=1, name=nm, **conv_common))
layers.append(Pooling(2, strides=2, name='conv%d_pool' % ind))
pool_layers.append(layers[-1])
if ind >= 2:
layers.append(Dropout(keep=0.5, name='drop%d' % (ind+1)))
# this loop generates the decoder layers
for ind in range(len(nchan)-1,-1,-1):
nchanu = nchan[ind]
lrng = 2 if ind <= 1 else 3
# upsampling layers need a ref to the corresponding pooling layer
# to access the argmx indices for upsampling
layers.append(Upsampling(2, pool_layers.pop(), strides=2, padding=0,
name='conv%d_unpool' % ind))
for lind in range(lrng):
nm = 'deconv%d_%d' % (ind+1, lind+1)
if ind < 4 and lind == lrng-1:
nchanu = nchan[ind]/2
layers.append(Conv((3, 3, nchanu), strides=1, name=nm, **conv_common))
if ind == 0:
break
if ind >= 2:
layers.append(Dropout(keep=0.5, name='drop%d' % (ind+1)))
# last conv layer outputs 12 channels, 1 for each output class
# with a pixelwise softmax over the channels
act_last = PixelwiseSoftmax(num_channels, height, width, name="PixelwiseSoftmax")
conv_last = dict(padding=1, init=init_uni, activation=act_last, batch_norm=False)
layers.append(Conv((3, 3, num_channels), strides=1, name='deconv_out', **conv_last))
return layers
def layers(self):
bn = True
return [
Conv((7, 7, 96), init=Kaiming(), activation=Explin(), batch_norm=bn, padding=3, strides=1)\
if self.bn_first_layer else\
Conv((7, 7, 96), init=Kaiming(), bias=Constant(0), activation=Explin(), padding=3, strides=1),
Pooling(3, strides=2, padding=1),
Conv((7, 7, 128), init=Kaiming(), activation=Explin(), batch_norm=bn, padding=3, strides=1),
Pooling(3, strides=2, padding=1),
Conv((5, 5, 256), init=Kaiming(), activation=Explin(), batch_norm=bn, padding=2, strides=1),
Pooling(3, strides=2, padding=1),
Conv((3, 3, 384), init=Kaiming(), activation=Explin(), batch_norm=bn, padding=1, strides=1),
Conv((3, 3, 384), init=Kaiming(), activation=Explin(), batch_norm=bn, padding=1, strides=1),
Conv((3, 3, 384), init=Kaiming(), activation=Explin(), batch_norm=bn, padding=1, strides=1),
Pooling(3, strides=2, padding=1, op='avg'),
Affine(nout=self.noutputs, init=Kaiming(), activation=Explin(), batch_norm=bn),
Affine(nout=self.noutputs, init=Kaiming(), activation=Explin(), batch_norm=bn),
Affine(nout=self.noutputs, init=Kaiming(), bias=Constant(0),
activation=Softmax() if self.use_softmax else Logistic(shortcut=True))
]
def input_layers(self, analytics_input, init, activation, gate):
"""
return the input layers. we currently support convolutional and LSTM
:return:
"""
if self.recurrent:
if analytics_input:
# support analytics + content
input_layers = MergeMultistream([[
LSTM(300,
init,
init_inner=Kaiming(),
activation=activation,
gate_activation=gate,
reset_cells=True),
RecurrentSum()
], [Affine(30, init, activation=activation)]], 'stack')
else:
# content only
input_layers = [
LSTM(300,
init,
init_inner=Kaiming(),
activation=activation,
gate_activation=gate,
reset_cells=True),
RecurrentSum()
]
else:
if analytics_input:
# support analytics + content
input_layers = MergeMultistream([
self.conv_net(activation),
[Affine(30, init, activation=Logistic())]
], 'stack')
else:
# content only
input_layers = self.conv_net(activation)
return input_layers
def conv_net(self, activation, init=Kaiming(), version=-1):
width = max([self.width, self.lookup_dim])
if version == -1:
if self.lookup_size:
pre_layers = [
LookupTable(vocab_size=self.lookup_size,
embedding_dim=width,
init=GlorotUniform()),
Reshape((1, self.num_words, width)),
]
first_width = width
else:
pre_layers = [
Conv((1, width, width),
padding=0,
init=init,
activation=activation)
]
first_width = 1
return pre_layers + \
[
MergeBroadcast(
[
[
Conv((3, first_width, 15), padding={'pad_h': 1, 'pad_w': 0}, init=init,
activation=activation)
],
[
Conv((5, first_width, 15), padding={'pad_h': 2, 'pad_w': 0}, init=init,
activation=activation)
],
[
Conv((7, first_width, 15), padding={'pad_h': 3, 'pad_w': 0}, init=init,
activation=activation)
],
],
merge='depth'
),
NoisyDropout(keep=0.5, noise_pct=1.0, noise_std=0.001),
Conv((5, 1, 15), strides={'str_h': 2 if self.num_words > 59 else 1,
'str_w': 1}, padding=0, init=init,
activation=activation),
NoisyDropout(keep=0.9, noise_pct=1.0, noise_std=0.00001),
Conv((3, 1, 9), strides={'str_h': 2, 'str_w': 1}, padding=0, init=init,
activation=activation),
NoisyDropout(keep=0.9, noise_pct=1.0, noise_std=0.00001),
Conv((9, 1, 9), strides={'str_h': 2, 'str_w': 1}, padding=0, init=init,
activation=activation)
]
def conv_params(fsize,
nfm,
padding='SAME',
strides=1,
activation=Rectlin(),
batch_norm=True):
fsize = fsize if isinstance(fsize, tuple) else (fsize, fsize)
fshape = fsize + (nfm, )
padding = {
'pad_h': (fsize[0] // 2 if padding == 'SAME' else 0),
'pad_w': (fsize[1] // 2 if padding == 'SAME' else 0),
'pad_d': 0
}
strides = {'str_h': strides, 'str_w': strides, 'str_d': 1}
return dict(fshape=fshape,
strides=strides,
activation=activation,
padding=padding,
batch_norm=batch_norm,
init=Kaiming(local=True))
def resnet(depth, num_classes, s):
# Structure of the deep residual part of the network:
# args.depth modules of 2 convolutional layers each at feature map depths of 16, 32, 64
nfms = [2**(stage + 4) for stage in sorted(list(range(3)) * depth)]
strides = [1] + [
1 if cur == prev else 2 for cur, prev in zip(nfms[1:], nfms[:-1])
]
# Now construct the network
layers = [Conv(**conv_params(3, 16))]
for nfm, stride in zip(nfms, strides):
layers.append(module_factory(nfm, stride))
layers.append(Pooling(s, op='avg'))
layers.append(
Affine(nout=num_classes,
init=Kaiming(local=False),
batch_norm=True,
activation=Softmax()))
return layers
nfms = [2**(stage + 4) for stage in sorted(range(3) * args.depth)]
strides = [1 if cur == prev else 2 for cur, prev in zip(nfms[1:], nfms[:-1])]
# Now construct the network
layers = [Conv(**conv_params(3, 16))]
layers.append(module_s1(nfms[0], True))
for nfm, stride in zip(nfms[1:], strides):
res_module = module_s1(nfm) if stride == 1 else module_s2(nfm)
layers.append(res_module)
layers.append(BatchNorm())
layers.append(Activation(Rectlin()))
layers.append(Pooling('all', op='avg'))
layers.append(
Affine(10,
init=Kaiming(local=False),
batch_norm=True,
activation=Softmax()))
model = Model(layers=layers)
opt = GradientDescentMomentum(0.1,
0.9,
wdecay=0.0001,
schedule=Schedule([82, 124], 0.1))
# configure callbacks
valmetric = Misclassification()
callbacks = Callbacks(model,
eval_set=test,
metric=valmetric,
**args.callback_args)
def create_network():
layers = [
DataTransform(transform=Normalizer(divisor=128.)),
Conv((11, 11, 96),
init=Kaiming(),
activation=Rectlin(),
strides=4,
padding=1),
Conv((1, 1, 96), init=Kaiming(), activation=Rectlin(), strides=1),
Conv((3, 3, 96),
init=Kaiming(),
activation=Rectlin(),
strides=2,
padding=1), # 54->2,
Conv((5, 5, 256), init=Kaiming(), activation=Rectlin(),
strides=1), # 27->2,
Conv((1, 1, 256), init=Kaiming(), activation=Rectlin(), strides=1),
Conv((3, 3, 256),
init=Kaiming(),
activation=Rectlin(),
strides=2,
padding=1), # 23->1,
Conv((3, 3, 384),
init=Kaiming(),
activation=Rectlin(),
strides=1,
padding=1),
Conv((1, 1, 384), init=Kaiming(), activation=Rectlin(), strides=1),
Conv((3, 3, 384),
init=Kaiming(),
activation=Rectlin(),
strides=2,
padding=1), # 12->,
Dropout(keep=0.5),
Conv((3, 3, 1024),
init=Kaiming(),
activation=Rectlin(),
strides=1,
padding=1),
Conv((1, 1, 1024), init=Kaiming(), activation=Rectlin(), strides=1),
Conv((1, 1, 1000), init=Kaiming(), activation=Rectlin(), strides=1),
Pooling(6, op='avg'),
Activation(Softmax())
]
return Model(layers=layers), GeneralizedCost(costfunc=CrossEntropyMulti())
def layers(self):
bn = True
return [
# input 128
Conv((7, 7, 96),
init=Kaiming(),
bias=Constant(0),
activation=Explin(),
padding=3,
strides=1),
Pooling(3, strides=2, padding=1),
# 64
Conv((7, 7, 128),
init=Kaiming(),
activation=Explin(),
batch_norm=bn,
padding=3,
strides=1),
Pooling(3, strides=2, padding=1),
# 32
Conv((5, 5, 256),
init=Kaiming(),
activation=Explin(),
batch_norm=bn,
padding=2,
strides=1),
Pooling(3, strides=2, padding=1),
# 16
Conv((3, 3, 384),
init=Kaiming(),
activation=Explin(),
batch_norm=bn,
padding=1,
strides=1),
Conv((3, 3, 384),
init=Kaiming(),
activation=Explin(),
batch_norm=bn,
padding=1,
strides=1),
Conv((3, 3, 384),
init=Kaiming(),
activation=Explin(),
batch_norm=bn,
padding=1,
strides=1),
Pooling(3, strides=2, padding=1),
# 8
Conv((3, 3, 8192),
init=Kaiming(),
activation=Explin(),
batch_norm=bn,
padding=1,
strides=1),
Pooling('all', op='avg'),
Affine(nout=self.noutputs,
init=Kaiming(),
bias=Constant(0),
activation=Softmax() if self.use_softmax else Logistic(
shortcut=True))
]
def layers(self):
bn = True
return [
# input 128
Conv((7, 7, 64),
init=Kaiming(),
bias=Constant(0),
activation=Explin(),
padding=3,
strides=1),
Pooling(3, strides=2, padding=1),
# 64
Conv((3, 3, 96),
init=Kaiming(),
activation=Explin(),
batch_norm=bn,
padding=1,
strides=1),
Conv((3, 3, 96),
init=Kaiming(),
activation=Explin(),
batch_norm=bn,
padding=1,
strides=1),
Pooling(3, strides=2, padding=1),
# 32
Conv((3, 3, 192),
init=Kaiming(),
activation=Explin(),
batch_norm=bn,
padding=1,
strides=1),
Conv((3, 3, 192),
init=Kaiming(),
activation=Explin(),
batch_norm=bn,
padding=1,
strides=1),
Pooling(3, strides=2, padding=1),
# 16
Conv((3, 3, 384),
init=Kaiming(),
activation=Explin(),
batch_norm=bn,
padding=1,
strides=1),
Conv((3, 3, 384),
init=Kaiming(),
activation=Explin(),
batch_norm=bn,
padding=1,
strides=1),
Conv((3, 3, 384),
init=Kaiming(),
activation=Explin(),
batch_norm=bn,
padding=1,
strides=1),
# this 4th deep layer may have been in for vgg3pool64all run? can not fit for 6fold so commented
#Conv((3, 3, 384), init=Kaiming(), activation=Explin(), batch_norm=bn, padding=1, strides=1),
Pooling(3, strides=2, padding=1),
# 8
Conv((3, 3, 6144),
init=Kaiming(),
activation=Explin(),
batch_norm=bn,
padding=1,
strides=1),
Pooling('all', op='avg'),
Affine(nout=self.noutputs,
init=Kaiming(),
bias=Constant(0),
activation=Softmax() if self.use_softmax else Logistic(
shortcut=True))
]
def module_factory(nfm, stride=1):
mainpath = [Conv(**conv_params(3, nfm, stride=stride)),
Conv(**conv_params(3, nfm, relu=False))]
sidepath = [SkipNode() if stride == 1 else Conv(**id_params(nfm))]
module = [MergeSum([mainpath, sidepath]),
Activation(Rectlin())]
return module
# Structure of the deep residual part of the network:
# args.depth modules of 2 convolutional layers each at feature map depths of 16, 32, 64
nfms = [2**(stage + 4) for stage in sorted(range(3) * args.depth)]
strides = [1] + [1 if cur == prev else 2 for cur, prev in zip(nfms[1:], nfms[:-1])]
# Now construct the network
layers = [Conv(**conv_params(3, 16))]
for nfm, stride in zip(nfms, strides):
layers.append(module_factory(nfm, stride))
layers.append(Pooling(8, op='avg'))
layers.append(Affine(nout=10, init=Kaiming(local=False), batch_norm=True, activation=Softmax()))
model = Model(layers=layers)
opt = GradientDescentMomentum(0.1, 0.9, wdecay=0.0001,
schedule=Schedule([90, 123], 0.1))
# configure callbacks
callbacks = Callbacks(model, eval_set=test, metric=Misclassification(), **args.callback_args)
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
model.fit(train, optimizer=opt, num_epochs=args.epochs, cost=cost, callbacks=callbacks)
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
#be.enable_winograd = 4 # default to winograd 4 for fast autotune
SUBSET = args.subset
train_set = HDF5IteratorOneHot('/mnt/data/medical/luna16/luna16_roi_except_subset{}_augmented.h5'.format(SUBSET), \
flip_enable=True, rot90_enable=True, crop_enable=False, border_size=5)
valid_set = HDF5IteratorOneHot('/mnt/data/medical/luna16/luna16_roi_subset{}_augmented.h5'.format(SUBSET), \
flip_enable=False, rot90_enable=False, crop_enable=False, border_size=5)
print('Using subset{}'.format(SUBSET))
init_uni = Kaiming()
relu = Rectlin()
bn = True
convp1 = dict(init=init_uni, batch_norm=bn, activation=relu, padding=1)
layers = [
Conv((5, 5, 24), **convp1),
Pooling(2, op='max'),
Conv((3, 3, 32), **convp1),
Pooling(2, op='max'),
Conv((3, 3, 48), **convp1),
Pooling('all', op='avg'),
Affine(512, init=init_uni, batch_norm=True, activation=relu),
Affine(2, init=init_uni, activation=Softmax())
]
# layers.append(Affine(nout=100, init=Kaiming(local=False), batch_norm=True, activation=Softmax()))
scales = [112, 128, 160, 240]
for scale in scales:
print scale
layers = []
layers += [Conv(**conv_params(7, 32, 2))]
for nfm, stride in zip(nfms, strides):
layers.append(module_factory(nfm, stride))
layers.append(Pooling(7, op='avg'))
layers.append(
Conv(fshape=(1, 1, 100), init=Kaiming(local=True), batch_norm=True))
layers.append(Pooling(fshape='all', op='avg'))
layers.append(Activation(Softmax()))
model = Model(layers=layers)
test = ImageLoader(set_name='validation',
shuffle=False,
do_transforms=False,
inner_size=scale,
scale_range=scale,
repo_dir=args.data_dir)
model.load_params("/home/users/hunter/bigfeat_dropout.pkl")
softmaxes = model.get_outputs(test)
from neon.util.persist import save_obj