def __init__(self, layers):
self._network = []
for layer in layers:
layer_type = layer.pop("type")
if layer_type == "data":
# this is a data layer
new_layer = DataLayer(**layer)
elif layer_type == "conv":
new_layer = ConvLayer(**layer)
elif layer_type == "pool":
new_layer = PoolLayer(**layer)
elif layer_type == "dense":
new_layer = DenseLayer(**layer)
elif layer_type == "relu":
new_layer = ReLULayer()
elif layer_type == "loss":
new_layer = LossLayer(**layer)
else:
raise NotImplementedError(
"Layer type: {0} not found".format(layer_type))
self._network.append(new_layer)
self.initialize()
def __init__(self, config):
ModelBase.__init__(self)
self.config = config
self.verbose = self.config['verbose']
self.name = 'alexnet'
batch_size = config['batch_size']
flag_datalayer = config['use_data_layer']
lib_conv = config['lib_conv']
n_softmax_out = config['n_softmax_out']
# ##################### BUILD NETWORK ##########################
# allocate symbolic variables for the data
# 'rand' is a random array used for random cropping/mirroring of data
x = T.ftensor4('x')
y = T.lvector('y')
rand = T.fvector('rand')
lr = T.scalar('lr')
if self.verbose: print 'AlexNet 2/16'
self.layers = []
params = []
weight_types = []
if flag_datalayer:
data_layer = DataLayer(input=x,
image_shape=(3, 256, 256, batch_size),
cropsize=227,
rand=rand,
mirror=True,
flag_rand=config['rand_crop'])
layer1_input = data_layer.output
else:
layer1_input = x
convpool_layer1 = ConvPoolLayer(input=layer1_input,
image_shape=(3, 227, 227, batch_size),
filter_shape=(3, 11, 11, 96),
convstride=4,
padsize=0,
group=1,
poolsize=3,
poolstride=2,
bias_init=0.0,
lrn=True,
lib_conv=lib_conv,
verbose=self.verbose)
self.layers.append(convpool_layer1)
params += convpool_layer1.params
weight_types += convpool_layer1.weight_type
convpool_layer2 = ConvPoolLayer(input=convpool_layer1.output,
image_shape=(96, 27, 27, batch_size),
filter_shape=(96, 5, 5, 256),
convstride=1,
padsize=2,
group=2,
poolsize=3,
poolstride=2,
bias_init=0.1,
lrn=True,
lib_conv=lib_conv,
verbose=self.verbose)
self.layers.append(convpool_layer2)
params += convpool_layer2.params
weight_types += convpool_layer2.weight_type
convpool_layer3 = ConvPoolLayer(input=convpool_layer2.output,
image_shape=(256, 13, 13, batch_size),
filter_shape=(256, 3, 3, 384),
convstride=1,
padsize=1,
group=1,
poolsize=1,
poolstride=0,
bias_init=0.0,
lrn=False,
lib_conv=lib_conv,
verbose=self.verbose)
self.layers.append(convpool_layer3)
params += convpool_layer3.params
weight_types += convpool_layer3.weight_type
convpool_layer4 = ConvPoolLayer(input=convpool_layer3.output,
image_shape=(384, 13, 13, batch_size),
filter_shape=(384, 3, 3, 384),
convstride=1,
padsize=1,
group=2,
poolsize=1,
poolstride=0,
bias_init=0.1,
lrn=False,
lib_conv=lib_conv,
verbose=self.verbose)
self.layers.append(convpool_layer4)
params += convpool_layer4.params
weight_types += convpool_layer4.weight_type
convpool_layer5 = ConvPoolLayer(input=convpool_layer4.output,
image_shape=(384, 13, 13, batch_size),
filter_shape=(384, 3, 3, 256),
convstride=1,
padsize=1,
group=2,
poolsize=3,
poolstride=2,
bias_init=0.0,
lrn=False,
lib_conv=lib_conv,
verbose=self.verbose)
self.layers.append(convpool_layer5)
params += convpool_layer5.params
weight_types += convpool_layer5.weight_type
fc_layer6_input = T.flatten(
convpool_layer5.output.dimshuffle(3, 0, 1, 2), 2)
fc_layer6 = FCLayer(input=fc_layer6_input,
n_in=9216,
n_out=4096,
verbose=self.verbose)
self.layers.append(fc_layer6)
params += fc_layer6.params
weight_types += fc_layer6.weight_type
dropout_layer6 = DropoutLayer(fc_layer6.output,
n_in=4096,
n_out=4096,
verbose=self.verbose)
fc_layer7 = FCLayer(input=dropout_layer6.output,
n_in=4096,
n_out=4096,
verbose=self.verbose)
self.layers.append(fc_layer7)
params += fc_layer7.params
weight_types += fc_layer7.weight_type
dropout_layer7 = DropoutLayer(fc_layer7.output,
n_in=4096,
n_out=4096,
verbose=self.verbose)
softmax_layer8 = SoftmaxLayer(input=dropout_layer7.output,
n_in=4096,
n_out=n_softmax_out,
verbose=self.verbose)
self.layers.append(softmax_layer8)
params += softmax_layer8.params
weight_types += softmax_layer8.weight_type
# #################### NETWORK BUILT #######################
self.p_y_given_x = softmax_layer8.p_y_given_x
self.y_pred = softmax_layer8.y_pred
self.output = self.p_y_given_x
self.cost = softmax_layer8.negative_log_likelihood(y)
self.error = softmax_layer8.errors(y)
if n_softmax_out < 5:
self.error_top_5 = softmax_layer8.errors_top_x(y, n_softmax_out)
else:
self.error_top_5 = softmax_layer8.errors_top_x(y, 5)
self.params = params
# inputs
self.x = x
self.y = y
self.rand = rand
self.lr = lr
self.shared_x = theano.shared(
np.zeros(
(3, config['input_width'], config['input_height'],
config['file_batch_size']), # for loading large batch
dtype=theano.config.floatX),
borrow=True)
self.shared_y = theano.shared(np.zeros((config['file_batch_size'], ),
dtype=int),
borrow=True)
self.shared_lr = theano.shared(np.float32(config['learning_rate']))
# training related
self.base_lr = np.float32(config['learning_rate'])
self.step_idx = 0
self.mu = config['momentum'] # def: 0.9 # momentum
self.eta = config['weight_decay'] #0.0002 # weight decay
self.weight_types = weight_types
self.batch_size = batch_size
self.grads = T.grad(self.cost, self.params)
subb_ind = T.iscalar('subb') # sub batch index
#print self.shared_x[:,:,:,subb_ind*self.batch_size:(subb_ind+1)*self.batch_size].shape.eval()
self.subb_ind = subb_ind
self.shared_x_slice = self.shared_x[:, :, :, subb_ind *
self.batch_size:(subb_ind + 1) *
self.batch_size]
self.shared_y_slice = self.shared_y[subb_ind *
self.batch_size:(subb_ind + 1) *
self.batch_size]
def __init__(self, config):
self.config = config
batch_size = config['batch_size']
flag_datalayer = config['use_data_layer']
lib_conv = config['lib_conv']
# ##################### BUILD NETWORK ##########################
# allocate symbolic variables for the data
# 'rand' is a random array used for random cropping/mirroring of data
x = T.ftensor4('x')
y = T.ivector('y')
rand = T.fvector('rand')
print '... building the model'
self.layers = []
params = []
weight_types = []
if flag_datalayer:
data_layer = DataLayer(input=x, image_shape=(3, 256, 256,
batch_size),
cropsize=227, rand=rand, mirror=True,
flag_rand=config['rand_crop'])
layer1_input = data_layer.output
else:
layer1_input = x
convpool_layer1 = ConvPoolLayer(input=layer1_input,
image_shape=(3, 227, 227, batch_size),
filter_shape=(3, 11, 11, 96),
convstride=4, padsize=0, group=1,
poolsize=3, poolstride=2,
bias_init=0.0, lrn=True,
lib_conv=lib_conv,
)
self.layers.append(convpool_layer1)
params += convpool_layer1.params
weight_types += convpool_layer1.weight_type
convpool_layer2 = ConvPoolLayer(input=convpool_layer1.output,
image_shape=(96, 27, 27, batch_size),
filter_shape=(96, 5, 5, 256),
convstride=1, padsize=2, group=2,
poolsize=3, poolstride=2,
bias_init=0.1, lrn=True,
lib_conv=lib_conv,
)
self.layers.append(convpool_layer2)
params += convpool_layer2.params
weight_types += convpool_layer2.weight_type
convpool_layer3 = ConvPoolLayer(input=convpool_layer2.output,
image_shape=(256, 13, 13, batch_size),
filter_shape=(256, 3, 3, 384),
convstride=1, padsize=1, group=1,
poolsize=1, poolstride=0,
bias_init=0.0, lrn=False,
lib_conv=lib_conv,
)
self.layers.append(convpool_layer3)
params += convpool_layer3.params
weight_types += convpool_layer3.weight_type
convpool_layer4 = ConvPoolLayer(input=convpool_layer3.output,
image_shape=(384, 13, 13, batch_size),
filter_shape=(384, 3, 3, 384),
convstride=1, padsize=1, group=2,
poolsize=1, poolstride=0,
bias_init=0.1, lrn=False,
lib_conv=lib_conv,
)
self.layers.append(convpool_layer4)
params += convpool_layer4.params
weight_types += convpool_layer4.weight_type
convpool_layer5 = ConvPoolLayer(input=convpool_layer4.output,
image_shape=(384, 13, 13, batch_size),
filter_shape=(384, 3, 3, 256),
convstride=1, padsize=1, group=2,
poolsize=3, poolstride=2,
bias_init=0.0, lrn=False,
lib_conv=lib_conv,
)
self.layers.append(convpool_layer5)
params += convpool_layer5.params
weight_types += convpool_layer5.weight_type
fc_layer6_input = T.flatten(
convpool_layer5.output.dimshuffle(3, 0, 1, 2), 2)
fc_layer6 = FCLayer(input=fc_layer6_input, n_in=9216, n_out=4096)
self.layers.append(fc_layer6)
params += fc_layer6.params
weight_types += fc_layer6.weight_type
dropout_layer6 = DropoutLayer(fc_layer6.output, n_in=4096, n_out=4096)
fc_layer7 = FCLayer(input=dropout_layer6.output, n_in=4096, n_out=4096)
self.layers.append(fc_layer7)
params += fc_layer7.params
weight_types += fc_layer7.weight_type
dropout_layer7 = DropoutLayer(fc_layer7.output, n_in=4096, n_out=4096)
softmax_layer8 = SoftmaxLayer(
input=dropout_layer7.output, n_in=4096, n_out=1000)
self.layers.append(softmax_layer8)
params += softmax_layer8.params
weight_types += softmax_layer8.weight_type
# #################### NETWORK BUILT #######################
self.cost = softmax_layer8.negative_log_likelihood(y)
self.errors = softmax_layer8.errors(y)
self.errors_top_5 = softmax_layer8.errors_top_x(y, 5)
self.params = params
self.x = x
self.y = y
self.rand = rand
self.weight_types = weight_types
self.batch_size = batch_size
def __init__(self, config):
self.config = config
batch_size = config['batch_size']
batch_size = config['batch_size']
flag_datalayer = config['use_data_layer']
lib_conv = config['lib_conv']
layers = []
params = []
weight_types = []
# ##################### BUILD NETWORK ##########################
# allocate symbolic variables for the data
# 'rand' is a random array used for random cropping/mirroring of data
x1 = T.ftensor4('x1')
x2 = T.ftensor4('x2')
y = T.lvector('y') # The ground truth to be compared with will go here
rand1 = T.fvector('rand1')
rand2 = T.fvector('rand2')
print '... building the model'
if flag_datalayer:
data_layerA = DataLayer(input=x1,
image_shape=(3, 256, 256, batch_size),
cropsize=227,
rand=rand,
mirror=True,
flag_rand=config['rand_crop'])
layer1A_input = data_layerA.output
else:
layer1A_input = x1
if flag_datalayer:
data_layerB = DataLayer(input=x2,
image_shape=(3, 256, 256, batch_size),
cropsize=227,
rand=rand,
mirror=True,
flag_rand=config['rand_crop'])
layer1B_input = data_layerB.output
else:
layer1B_input = x2
fc_layer2_input = T.concatenate(
(T.flatten(layer1A_input.dimshuffle(3, 0, 1, 2),
2), T.flatten(layer1B_input.dimshuffle(3, 0, 1, 2), 2)),
axis=1)
fc_layer2 = FCLayer(input=fc_layer2_input, n_in=154587 * 2, n_out=4096)
layers.append(fc_layer2)
params += fc_layer2.params
weight_types += fc_layer2.weight_type
dropout_layer2 = DropoutLayer(fc_layer2.output, n_in=4096, n_out=4096)
fc_layer3 = FCLayer(input=dropout_layer2.output, n_in=4096, n_out=4096)
layers.append(fc_layer3)
params += fc_layer3.params
weight_types += fc_layer3.weight_type
dropout_layer3 = DropoutLayer(fc_layer3.output, n_in=4096, n_out=4096)
# Final softmax layer
softmax_layer3 = SoftmaxLayer(
input=dropout_layer3.output, n_in=4096,
n_out=2) # Only a single binary output is required!
layers.append(softmax_layer3)
params += softmax_layer3.params
weight_types += softmax_layer3.weight_type
# #################### NETWORK BUILT #######################
self.cost = softmax_layer3.negative_log_likelihood(y)
self.errors = softmax_layer3.errors(y)
self.errors_top_5 = softmax_layer3.errors_top_x(y, 5)
self.x1 = x1
self.x2 = x2
self.y = y
self.rand1 = rand1
self.rand2 = rand2
self.layers = layers
self.params = params
self.weight_types = weight_types
self.batch_size = batch_size
def image_repr(self, x, rand, config):
batch_size = config['batch_size']
flag_datalayer = config['use_data_layer']
lib_conv = config['lib_conv']
layers = []
params = []
weight_types = []
if flag_datalayer:
data_layer = DataLayer(input=x,
image_shape=(3, 256, 256, batch_size),
cropsize=227,
rand=rand,
mirror=True,
flag_rand=config['rand_crop'])
layer1_input = data_layer.output
else:
layer1_input = x
convpool_layer1 = ConvPoolLayer(
input=layer1_input,
image_shape=(3, 227, 227, batch_size),
filter_shape=(3, 11, 11, 96),
convstride=4,
padsize=0,
group=1,
poolsize=3,
poolstride=2,
bias_init=0.0,
lrn=True,
lib_conv=lib_conv,
)
layers.append(convpool_layer1)
params += convpool_layer1.params
weight_types += convpool_layer1.weight_type
convpool_layer2 = ConvPoolLayer(
input=convpool_layer1.output,
image_shape=(96, 27, 27, batch_size),
filter_shape=(96, 5, 5, 256),
convstride=1,
padsize=2,
group=2,
poolsize=3,
poolstride=2,
bias_init=0.1,
lrn=True,
lib_conv=lib_conv,
)
layers.append(convpool_layer2)
params += convpool_layer2.params
weight_types += convpool_layer2.weight_type
convpool_layer3 = ConvPoolLayer(
input=convpool_layer2.output,
image_shape=(256, 13, 13, batch_size),
filter_shape=(256, 3, 3, 384),
convstride=1,
padsize=1,
group=1,
poolsize=1,
poolstride=0,
bias_init=0.0,
lrn=False,
lib_conv=lib_conv,
)
layers.append(convpool_layer3)
params += convpool_layer3.params
weight_types += convpool_layer3.weight_type
convpool_layer4 = ConvPoolLayer(
input=convpool_layer3.output,
image_shape=(384, 13, 13, batch_size),
filter_shape=(384, 3, 3, 384),
convstride=1,
padsize=1,
group=2,
poolsize=1,
poolstride=0,
bias_init=0.1,
lrn=False,
lib_conv=lib_conv,
)
layers.append(convpool_layer4)
params += convpool_layer4.params
weight_types += convpool_layer4.weight_type
convpool_layer5 = ConvPoolLayer(
input=convpool_layer4.output,
image_shape=(384, 13, 13, batch_size),
filter_shape=(384, 3, 3, 256),
convstride=1,
padsize=1,
group=2,
poolsize=3,
poolstride=2,
bias_init=0.0,
lrn=False,
lib_conv=lib_conv,
)
layers.append(convpool_layer5)
params += convpool_layer5.params
weight_types += convpool_layer5.weight_type
fc_layer6_input = T.flatten(
convpool_layer5.output.dimshuffle(3, 0, 1, 2), 2)
fc_layer6 = MaxoutLayer(input=fc_layer6_input, n_in=9216, n_out=4096)
layers.append(fc_layer6)
params += fc_layer6.params
weight_types += fc_layer6.weight_type
dropout_layer6 = DropoutLayer(fc_layer6.output, n_in=4096, n_out=4096)
fc_layer7 = MaxoutLayer(input=dropout_layer6.output,
n_in=4096,
n_out=4096)
layers.append(fc_layer7)
params += fc_layer7.params
weight_types += fc_layer7.weight_type
#dropout_layer7 = DropoutLayer(fc_layer7.output, n_in=4096, n_out=4096)
# Rename weight types so that weights can be shared
new_weight_types = []
counter_W = 0
counter_b = 0
for w in weight_types:
if w == 'W':
new_weight_types.append('W' + str(counter_W))
counter_W += 1
elif w == 'b':
new_weight_types.append('b' + str(counter_b))
counter_b += 1
weight_types = new_weight_types
return fc_layer7, layers, params, weight_types