def test_layer(self):
self.append_layer(Layers.ZeroPaddingLayer(3))
self.append_layer(Layers.ConvLayer(64, 7, 2, use_bias_in=False))
self.append_layer(Layers.BatchNormalizationLayer())
self.append_layer(Layers.ActivationLayer('relu'))
self.append_layer(Layers.ZeroPaddingLayer(1))
self.append_layer(
Layers.PoolLayer(3, 2, type_in='max', concatenate=True))
self.dense_block(6)
self.transition_block(128)
self.dense_block(12)
self.transition_block(256)
self.dense_block(32)
self.transition_block(640)
self.dense_block(32)
self.append_layer(Layers.BatchNormalizationLayer())
self.append_layer(Layers.PoolLayer(2, 2, pool_type_in='globalaverage'))
self.append_layer(Layers.ActivationLayer('sigmoid'))
def ConstructTreeConvolution(nodes, numFea, numCon, numDis, numOut,\
Wleft, Wright, Bconstruct,\
Wcomb_ae, Wcomb_orig, \
Wconv_root, Wconv_left, Wconv_right, Wconv_sib, Bconv,\
Wdis, Woutput, Bdis, Boutput,\
poolCutoff
):
# nodes
# numFea: # of the word/symbol feature size
# numCon: # of the convolution size
# Wleft: left weights of continous binary tree autoencoder
# Wright: right weights of continous binary tree autoencoder
# Bconstruct: the biase for the autoencoder
# Wcomb_ae, Wcomb_orig: the weights for the combination of
# autoencoder and the original vector
# (no biase for this sate)
# Wconv_root, Wconv_left, Wconv_right, Bconv: the weights for covolution
# Bconv: Biases for covolution
numNodes = len(nodes)
layers = [None] * numNodes
# construct layers for each node
# layers = |---leaf---|---non_leaf---|
numLeaf = 0
for idx in xrange(numNodes):
node = nodes[idx]
if len(node.children) == 0:
numLeaf += 1
layers[idx] = Lay.layer('vec_'+str(idx)+'_' + node.word,\
range( node.bidx, node.bidx + numFea),\
numFea
)
layers[idx].act = 'embedding'
# auto encoding
# layers = |---leaf---|---non_leaf(autoencoded)---| (numNodes)
# |---non_leaf(original)---| ( numNonLeaf)
numNonLeaf = numNodes - numLeaf
layers.extend([None] * (2 * numNonLeaf))
for idx in xrange(numLeaf, numNodes):
node = nodes[idx]
layers[idx + numNonLeaf] = layers[idx]
tmplayer = Lay.layer('ae_'+str(idx)+'_'+node.word,\
Bconstruct, numFea)
tmplayer.act = 'autoencoding'
layers[idx] = tmplayer
# add reconstruction connections
for idx in xrange(0, numNodes):
node = nodes[idx]
if node.parent == None:
continue
tmplayer = layers[idx]
parent = layers[node.parent]
if node.leftRate != 0:
leftcon = Con.connection(tmplayer, parent,\
numFea, numFea, Wleft, Wcoef = node.leftRate * node.leafNum/nodes[node.parent].leafNum)
if node.rightRate != 0:
rightcon = Con.connection(tmplayer, parent,\
numFea, numFea, Wright, Wcoef = node.rightRate * node.leafNum/nodes[node.parent].leafNum)
# combinition of the constructed and original value
# layers = |---leaf---|---non_leaf(combinition)---| (numNodes)
# |---non_leaf(original)---|---non_leaf(ae)---| (2 * numNonLeaf)
for idx in xrange(numLeaf, numNodes):
aelayer = layers[idx]
origlayer = layers[idx + numNonLeaf]
layers[idx + numNonLeaf * 2] = aelayer
comlayer = Lay.layer('comb_' + str(idx) + '_' + nodes[idx].word, None,
numFea)
comlayer.act = 'combination'
layers[idx] = comlayer
# connecton auto encoded vector and original vector
con_ae = Con.connection(aelayer, comlayer, numFea, numFea, Wcomb_ae)
con_orig = Con.connection(origlayer, comlayer, numFea, numFea,
Wcomb_orig)
# CONVOLVE!!! and POOOOL!!!
# layers = |---leaf---|---non_leaf(combition)---| => (numNodes)
# |---non_leaf(original)---|---non_leaf(ae)---| => (2 * numNonLeaf)
# |------------convolution----------|
queue = [(numNodes - 1, None)]
poolTop = Lay.PoolLayer('poolTop', numCon)
poolLeft = Lay.PoolLayer('poolLeft', numCon)
poolRight = Lay.PoolLayer('poolRight', numCon)
layerCnt = 0
rootChildrenNum = len(nodes[-1].children) - 1
while True:
curLen = len(queue)
#layerCnt.append( curLen )
if curLen == 0:
break
nextQueue = []
for (nodeidx, info) in queue:
curLayer = layers[nodeidx]
curNode = nodes[nodeidx]
conLayer = Lay.layer('Convolve_' + curLayer.name, \
Bconv, numCon)
conLayer.act = 'convolution'
layers.append(conLayer)
# add root connection
rootCon = Con.connection(curLayer, conLayer, numFea, numCon,
Wconv_root)
# add sibling connections
for sib in curNode.siblings:
sibNode = nodes[sib]
sibLayer = layers[sib]
sib_childrenNum = len(sibNode.children)
if sib_childrenNum == 0:
sib_childrenNum = 1
sib_Weight = 1.0 * sib_childrenNum / len(curNode.siblings)
sibCon = Con.connection(sibLayer, conLayer, \
numFea, numCon, Wconv_sib, sib_Weight)
childNum = len(curNode.children)
#print curLayer.name, info
# pooling
if layerCnt < poolCutoff:
poolCon = Con.PoolConnection(conLayer, poolTop)
else: # TODO if layerCnt >= poolCutoff
if info == 'l' or info == 'lr':
poolCon = Con.PoolConnection(conLayer, poolLeft)
if info == 'r' or info == 'lr':
poolCon = Con.PoolConnection(conLayer, poolRight)
# for each child of the current node
for child in curNode.children:
childNode = nodes[child]
childLayer = layers[child]
if layerCnt != 0 and info != 'u':
childinfo = info
else:
rootChildrenNum = len(curNode.children) - 1
if rootChildrenNum == 0:
childinfo = 'u'
elif childNode.pos <= rootChildrenNum / 2.0:
childinfo = 'l'
else: # childNode.pos > rootChildrenNum/2.0:
childinfo = 'r'
#else:
# childinfo = 'lr'
nextQueue.append((child, childinfo)) # add to child
if childNum == 1:
leftWeight = .5
rightWeight = .5
else:
rightWeight = childNode.pos / (childNum - 1.0)
leftWeight = 1 - rightWeight
if leftWeight != 0:
leftCon = Con.connection(childLayer, conLayer,\
numFea, numCon, Wconv_left, leftWeight)
if rightWeight != 0:
rightCon = Con.connection(childLayer, conLayer,\
numFea, numCon, Wconv_right, rightWeight)
# end of each child of the current node
queue = nextQueue
layerCnt += 1
# end of current layer
layers.append(poolTop)
layers.append(poolLeft)
layers.append(poolRight)
# reorder
# layers = |---leaf---|---non_leaf(ae)---| => (numNodes)
# |---non_leaf(original)---|---non_leaf(comb)---| => (2 * numNonLeaf)
# |------------convolution----------|
for idx in xrange(numLeaf, numLeaf + numNonLeaf):
tmp = layers[idx]
layers[idx] = layers[idx + 2 * numNonLeaf]
layers[idx + 2 * numNonLeaf] = tmp
# discrimitive layer
# layers = |---leaf---|---non_leaf(ae)---| => (numNodes)
# |---non_leaf(original)---|---non_leaf(comb)---| => (2 * numNonLeaf)
# |------------convolution----------|
# |---3 pools---|
## POOOOOOOOOOOL
# # pool all
# poolLayer = Lay.PoolLayer('pool', numCon)
# for idx in xrange( numNodes + 2* numNonLeaf, len(layers) ):
# poolCon = Con.PoolConnection(layers[idx], poolLayer)
# poolLayer.connectDown.append(poolCon)
# layers[idx].connectUp.append(poolCon)
# layers.append(poolLayer)
# discriminative layer
# layers = |---leaf---|---non_leaf(ae)---| => (numNodes)
# |---non_leaf(original)---|---non_leaf(comb)---| => (2 * numNonLeaf)
# |------------convolution----------|
# |---3 pools---|
# |---discriminative layer-----|
# |--output--|
numPool = 3
lenlayer = len(layers)
conbegin = lenlayer - numPool
discriminative = Lay.layer('discriminative', Bdis, numDis)
discriminative.act = 'hidden'
output = Lay.layer('outputlayer', Boutput, numOut)
output.act = 'softmax'
#One Weight Size
ows = numDis * numCon
for idx in xrange(numPool):
poollayer = layers[idx + conbegin]
con = Con.connection(poollayer, discriminative, numCon, numDis,
Wdis[idx * ows:(idx * ows + ows)])
outcon = Con.connection(discriminative, output, numDis, numOut, Woutput)
if numOut > 1:
output._activate = Activation.softmax
output._activatePrime = None
layers.append(discriminative)
layers.append(output)
# add successive connections
numlayers = len(layers)
for idx in xrange(numlayers):
if idx > 0:
layers[idx].successiveLower = layers[idx - 1]
if idx < numlayers - 1:
layers[idx].successiveUpper = layers[idx + 1]
return layers
def transition_block(self, conv_in):
self.append_layer(Layers.BatchNormalizationLayer())
self.append_layer(Layers.ActivationLayer('relu'))
self.append_layer(Layers.ConvLayer(conv_in, 1, 1, use_bias_in=False))
self.append_layer(
Layers.PoolLayer(2, 2, type_in='average', concatenate=True))
def __init__(self, input, input_shape, rng=None, Ws=None, bs=None):
assert(
(rng is None and Ws is None and bs is None) or
(rng is not None and Ws is None and bs is None) or
(rng is None and Ws is not None and bs is not None)
)
if Ws is None and bs is None:
Ws = [None] * 5
bs = [None] * 5
self.trainable_layers = []
############################## conv1 ##############################
self.layer_conv1 = Layers.ConvLayer(
input=input,
input_shape=input_shape,
filter_shape=(96, 3, 11, 11),
stride=(4, 4),
border_mode='valid',
activation=T.nnet.relu,
rng=rng,
W=Ws[0],
b=bs[0]
)
conv1_output_shape = (None, 96, 55, 55) # (227 - 11) / 4 + 1 = 55
self.trainable_layers.append(self.layer_conv1)
self.layer_pool1 = Layers.PoolLayer(
input=self.layer_conv1.output,
kernel_size=(3, 3),
stride=(2, 2)
)
pool1_output_shape = (None, 96, 27, 27) # (55 - 3) / 2 + 1 = 27
self.layer_norm1 = normalization.LocalResponseNormalization2DLayer(
incoming=pool1_output_shape, # Actual input is param to .get_output_for()
# alpha=0.0001,
alpha=0.0001/5, # Caffe documentation uses alpha/n as coeff of summation
k=1, # Not specified; Caffe says default is 1, but AlexNet paper says 2
beta=0.75,
n=5
)
self.layer_norm1.output = self.layer_norm1.get_output_for(self.layer_pool1.output)
self.layer_norm1.output_group1 = self.layer_norm1.output[:, :48, :, :]
self.layer_norm1.output_group2 = self.layer_norm1.output[:, 48:, :, :]
norm1_group1_output_shape = (None, 48, 27, 27)
norm1_group2_output_shape = (None, 48, 27, 27)
############################## conv2 ##############################
self.layer_conv2_group1 = Layers.ConvLayer(
input=self.layer_norm1.output_group1,
input_shape=norm1_group1_output_shape,
filter_shape=(128, 48, 5, 5),
stride=(1, 1),
border_mode=(2, 2), # pad by 2 pixels to each edge, so height and width += 4
activation=T.nnet.relu,
rng=rng,
W=Ws[1][:128] if Ws[1] is not None else None,
b=bs[1][:128] if bs[1] is not None else None
)
conv2_group1_output_shape = (None, 128, 27, 27) # (27 + 4 - 5) / 1 + 1 = 27
self.trainable_layers.append(self.layer_conv2_group1)
self.layer_conv2_group2 = Layers.ConvLayer(
input=self.layer_norm1.output_group2,
input_shape=norm1_group2_output_shape,
filter_shape=(128, 48, 5, 5),
stride=(1, 1),
border_mode=(2, 2), # pad by 2 pixels to each edge, so height and width += 4
activation=T.nnet.relu,
rng=rng,
W=Ws[1][128:] if Ws[1] is not None else None,
b=bs[1][128:] if bs[1] is not None else None
)
conv2_group2_output_shape = (None, 128, 27, 27) # (27 + 4 - 5) / 1 + 1 = 27
self.trainable_layers.append(self.layer_conv2_group2)
self.layer_pool2_group1 = Layers.PoolLayer(
input=self.layer_conv2_group1.output,
kernel_size=(3, 3),
stride=(2, 2)
)
pool2_group1_output_shape = (None, 128, 13, 13) # (27 - 3) / 2 + 1 = 13
self.layer_pool2_group2 = Layers.PoolLayer(
input=self.layer_conv2_group2.output,
kernel_size=(3, 3),
stride=(2, 2)
)
pool2_group2_output_shape = (None, 128, 13, 13) # (27 - 3) / 2 + 1 = 13
self.layer_norm2_group1 = normalization.LocalResponseNormalization2DLayer(
incoming=pool2_group1_output_shape, # Actual input is param to .get_output_for()
# alpha=0.0001,
alpha=0.0001/5, # Caffe documentation uses alpha/n as coeff of summation
k=1, # Not specified; Caffe says default is 1
beta=0.75,
n=5
)
self.layer_norm2_group1.output = self.layer_norm2_group1.get_output_for(self.layer_pool2_group1.output)
norm2_group1_output_shape = pool2_group1_output_shape
self.layer_norm2_group2 = normalization.LocalResponseNormalization2DLayer(
incoming=pool2_group2_output_shape, # Actual input is param to .get_output_for()
# alpha=0.0001,
alpha=0.0001/5, # Caffe documentation uses alpha/n as coeff of summation
k=1, # Not specified; Caffe says default is 1
beta=0.75,
n=5
)
self.layer_norm2_group2.output = self.layer_norm2_group2.get_output_for(self.layer_pool2_group2.output)
norm2_group2_output_shape = pool2_group2_output_shape
############################## conv3 ##############################
norm2_grouped_output = T.concatenate((self.layer_norm2_group1.output, self.layer_norm2_group2.output), axis=1)
norm2_grouped_output_shape = (None, 256, 13, 13)
self.layer_conv3 = Layers.ConvLayer(
input=norm2_grouped_output,
input_shape=norm2_grouped_output_shape,
filter_shape=(384, 256, 3, 3),
stride=(1, 1),
border_mode=(1, 1),
activation=T.nnet.relu,
rng=rng,
W=Ws[2],
b=bs[2]
)
conv3_output_shape = (None, 384, 13, 13) # (13 + 2 - 3) / 1 + 1 = 13
self.trainable_layers.append(self.layer_conv3)
self.layer_conv3.output_group1 = self.layer_conv3.output[:, :192, :, :]
self.layer_conv3.output_group2 = self.layer_conv3.output[:, 192:, :, :]
conv3_group1_output_shape = (None, 192, 13, 13)
conv3_group2_output_shape = (None, 192, 13, 13)
############################## conv4 ##############################
self.layer_conv4_group1 = Layers.ConvLayer(
input=self.layer_conv3.output_group1,
input_shape=conv3_group1_output_shape,
filter_shape=(192, 192, 3, 3),
stride=(1, 1),
border_mode=(1, 1),
activation=T.nnet.relu,
rng=rng,
W=Ws[3][:192] if Ws[3] is not None else None,
b=bs[3][:192] if bs[3] is not None else None
)
conv4_group1_output_shape = (None, 192, 13, 13) # (13 + 2 - 3) / 1 + 1 = 13
self.trainable_layers.append(self.layer_conv4_group1)
self.layer_conv4_group2 = Layers.ConvLayer(
input=self.layer_conv3.output_group2,
input_shape=conv3_group2_output_shape,
filter_shape=(192, 192, 3, 3),
stride=(1, 1),
border_mode=(1, 1),
activation=T.nnet.relu,
rng=rng,
W=Ws[3][192:] if Ws[3] is not None else None,
b=bs[3][192:] if bs[3] is not None else None
)
conv4_group2_output_shape = (None, 192, 13, 13) # (13 + 2 - 3) / 1 + 1 = 13
self.trainable_layers.append(self.layer_conv4_group2)
############################## conv5 ##############################
self.layer_conv5_group1 = Layers.ConvLayer(
input=self.layer_conv4_group1.output,
input_shape=conv4_group1_output_shape,
filter_shape=(128, 192, 3, 3),
stride=(1, 1),
border_mode=(1, 1),
activation=T.nnet.relu,
rng=rng,
W=Ws[4][:128] if Ws[4] is not None else None,
b=bs[4][:128] if bs[4] is not None else None
)
conv5_group1_output_shape = (None, 128, 13, 13) # (13 + 2 - 3) / 1 + 1 = 13
self.trainable_layers.append(self.layer_conv5_group1)
self.layer_pool5_group1 = Layers.PoolLayer(
input=self.layer_conv5_group1.output,
kernel_size=(3, 3),
stride=(2, 2)
)
pool5_group1_output_shape = (None, 128, 6, 6) # (13 - 3) / 2 + 1 = 6
self.layer_conv5_group2 = Layers.ConvLayer(
input=self.layer_conv4_group2.output,
input_shape=conv4_group2_output_shape,
filter_shape=(128, 192, 3, 3),
stride=(1, 1),
border_mode=(1, 1),
activation=T.nnet.relu,
rng=rng,
W=Ws[4][128:] if Ws[4] is not None else None,
b=bs[4][128:] if bs[4] is not None else None
)
conv5_group2_output_shape = (None, 128, 13, 13) # (13 + 2 - 3) / 1 + 1 = 13
self.trainable_layers.append(self.layer_conv5_group2)
self.layer_pool5_group2 = Layers.PoolLayer(
input=self.layer_conv5_group2.output,
kernel_size=(3, 3),
stride=(2, 2)
)
pool5_group2_output_shape = (None, 128, 6, 6) # (13 - 3) / 2 + 1 = 6
pool5_grouped_output = T.concatenate((self.layer_pool5_group1.output, self.layer_pool5_group2.output), axis=1)
self.output = pool5_grouped_output
self.trainable_params = [param for layer in self.trainable_layers for param in layer.params]
